1#include "cache.h"
2#include "refs.h"
3#include "pkt-line.h"
4#include "commit.h"
5#include "tag.h"
6#include "exec_cmd.h"
7#include "pack.h"
8#include "sideband.h"
9#include "fetch-pack.h"
10#include "remote.h"
11#include "run-command.h"
12static int transfer_unpack_limit = -1;
14static int fetch_unpack_limit = -1;
15static int unpack_limit = 100;
16static struct fetch_pack_args args = {
17 /* .uploadpack = */ "git-upload-pack",
18};
19static const char fetch_pack_usage[] =
21"git fetch-pack [--all] [--quiet|-q] [--keep|-k] [--thin] [--include-tag] [--upload-pack=<git-upload-pack>] [--depth=<n>] [--no-progress] [-v] [<host>:]<directory> [<refs>...]";
22#define COMPLETE (1U << 0)
24#define COMMON (1U << 1)
25#define COMMON_REF (1U << 2)
26#define SEEN (1U << 3)
27#define POPPED (1U << 4)
28static int marked;
30/*
32 * After sending this many "have"s if we do not get any new ACK , we
33 * give up traversing our history.
34 */
35#define MAX_IN_VAIN 256
36static struct commit_list *rev_list;
38static int non_common_revs, multi_ack, use_sideband;
39static void rev_list_push(struct commit *commit, int mark)
41{
42 if (!(commit->object.flags & mark)) {
43 commit->object.flags |= mark;
44 if (!(commit->object.parsed))
46 if (parse_commit(commit))
47 return;
48 insert_by_date(commit, &rev_list);
50 if (!(commit->object.flags & COMMON))
52 non_common_revs++;
53 }
54}
55static int rev_list_insert_ref(const char *path, const unsigned char *sha1, int flag, void *cb_data)
57{
58 struct object *o = deref_tag(parse_object(sha1), path, 0);
59 if (o && o->type == OBJ_COMMIT)
61 rev_list_push((struct commit *)o, SEEN);
62 return 0;
64}
65static int clear_marks(const char *path, const unsigned char *sha1, int flag, void *cb_data)
67{
68 struct object *o = deref_tag(parse_object(sha1), path, 0);
69 if (o && o->type == OBJ_COMMIT)
71 clear_commit_marks((struct commit *)o,
72 COMMON | COMMON_REF | SEEN | POPPED);
73 return 0;
74}
75/*
77 This function marks a rev and its ancestors as common.
78 In some cases, it is desirable to mark only the ancestors (for example
79 when only the server does not yet know that they are common).
80*/
81static void mark_common(struct commit *commit,
83 int ancestors_only, int dont_parse)
84{
85 if (commit != NULL && !(commit->object.flags & COMMON)) {
86 struct object *o = (struct object *)commit;
87 if (!ancestors_only)
89 o->flags |= COMMON;
90 if (!(o->flags & SEEN))
92 rev_list_push(commit, SEEN);
93 else {
94 struct commit_list *parents;
95 if (!ancestors_only && !(o->flags & POPPED))
97 non_common_revs--;
98 if (!o->parsed && !dont_parse)
99 if (parse_commit(commit))
100 return;
101 for (parents = commit->parents;
103 parents;
104 parents = parents->next)
105 mark_common(parents->item, 0, dont_parse);
106 }
107 }
108}
109/*
111 Get the next rev to send, ignoring the common.
112*/
113static const unsigned char* get_rev(void)
115{
116 struct commit *commit = NULL;
117 while (commit == NULL) {
119 unsigned int mark;
120 struct commit_list *parents;
121 if (rev_list == NULL || non_common_revs == 0)
123 return NULL;
124 commit = rev_list->item;
126 if (!commit->object.parsed)
127 parse_commit(commit);
128 parents = commit->parents;
129 commit->object.flags |= POPPED;
131 if (!(commit->object.flags & COMMON))
132 non_common_revs--;
133 if (commit->object.flags & COMMON) {
135 /* do not send "have", and ignore ancestors */
136 commit = NULL;
137 mark = COMMON | SEEN;
138 } else if (commit->object.flags & COMMON_REF)
139 /* send "have", and ignore ancestors */
140 mark = COMMON | SEEN;
141 else
142 /* send "have", also for its ancestors */
143 mark = SEEN;
144 while (parents) {
146 if (!(parents->item->object.flags & SEEN))
147 rev_list_push(parents->item, mark);
148 if (mark & COMMON)
149 mark_common(parents->item, 1, 0);
150 parents = parents->next;
151 }
152 rev_list = rev_list->next;
154 }
155 return commit->object.sha1;
157}
158static int find_common(int fd[2], unsigned char *result_sha1,
160 struct ref *refs)
161{
162 int fetching;
163 int count = 0, flushes = 0, retval;
164 const unsigned char *sha1;
165 unsigned in_vain = 0;
166 int got_continue = 0;
167 if (marked)
169 for_each_ref(clear_marks, NULL);
170 marked = 1;
171 for_each_ref(rev_list_insert_ref, NULL);
173 fetching = 0;
175 for ( ; refs ; refs = refs->next) {
176 unsigned char *remote = refs->old_sha1;
177 struct object *o;
178 /*
180 * If that object is complete (i.e. it is an ancestor of a
181 * local ref), we tell them we have it but do not have to
182 * tell them about its ancestors, which they already know
183 * about.
184 *
185 * We use lookup_object here because we are only
186 * interested in the case we *know* the object is
187 * reachable and we have already scanned it.
188 */
189 if (((o = lookup_object(remote)) != NULL) &&
190 (o->flags & COMPLETE)) {
191 continue;
192 }
193 if (!fetching)
195 packet_write(fd[1], "want %s%s%s%s%s%s%s%s\n",
196 sha1_to_hex(remote),
197 (multi_ack ? " multi_ack" : ""),
198 (use_sideband == 2 ? " side-band-64k" : ""),
199 (use_sideband == 1 ? " side-band" : ""),
200 (args.use_thin_pack ? " thin-pack" : ""),
201 (args.no_progress ? " no-progress" : ""),
202 (args.include_tag ? " include-tag" : ""),
203 " ofs-delta");
204 else
205 packet_write(fd[1], "want %s\n", sha1_to_hex(remote));
206 fetching++;
207 }
208 if (is_repository_shallow())
209 write_shallow_commits(fd[1], 1);
210 if (args.depth > 0)
211 packet_write(fd[1], "deepen %d", args.depth);
212 packet_flush(fd[1]);
213 if (!fetching)
214 return 1;
215 if (args.depth > 0) {
217 char line[1024];
218 unsigned char sha1[20];
219 while (packet_read_line(fd[0], line, sizeof(line))) {
221 if (!prefixcmp(line, "shallow ")) {
222 if (get_sha1_hex(line + 8, sha1))
223 die("invalid shallow line: %s", line);
224 register_shallow(sha1);
225 continue;
226 }
227 if (!prefixcmp(line, "unshallow ")) {
228 if (get_sha1_hex(line + 10, sha1))
229 die("invalid unshallow line: %s", line);
230 if (!lookup_object(sha1))
231 die("object not found: %s", line);
232 /* make sure that it is parsed as shallow */
233 if (!parse_object(sha1))
234 die("error in object: %s", line);
235 if (unregister_shallow(sha1))
236 die("no shallow found: %s", line);
237 continue;
238 }
239 die("expected shallow/unshallow, got %s", line);
240 }
241 }
242 flushes = 0;
244 retval = -1;
245 while ((sha1 = get_rev())) {
246 packet_write(fd[1], "have %s\n", sha1_to_hex(sha1));
247 if (args.verbose)
248 fprintf(stderr, "have %s\n", sha1_to_hex(sha1));
249 in_vain++;
250 if (!(31 & ++count)) {
251 int ack;
252 packet_flush(fd[1]);
254 flushes++;
255 /*
257 * We keep one window "ahead" of the other side, and
258 * will wait for an ACK only on the next one
259 */
260 if (count == 32)
261 continue;
262 do {
264 ack = get_ack(fd[0], result_sha1);
265 if (args.verbose && ack)
266 fprintf(stderr, "got ack %d %s\n", ack,
267 sha1_to_hex(result_sha1));
268 if (ack == 1) {
269 flushes = 0;
270 multi_ack = 0;
271 retval = 0;
272 goto done;
273 } else if (ack == 2) {
274 struct commit *commit =
275 lookup_commit(result_sha1);
276 mark_common(commit, 0, 1);
277 retval = 0;
278 in_vain = 0;
279 got_continue = 1;
280 }
281 } while (ack);
282 flushes--;
283 if (got_continue && MAX_IN_VAIN < in_vain) {
284 if (args.verbose)
285 fprintf(stderr, "giving up\n");
286 break; /* give up */
287 }
288 }
289 }
290done:
291 packet_write(fd[1], "done\n");
292 if (args.verbose)
293 fprintf(stderr, "done\n");
294 if (retval != 0) {
295 multi_ack = 0;
296 flushes++;
297 }
298 while (flushes || multi_ack) {
299 int ack = get_ack(fd[0], result_sha1);
300 if (ack) {
301 if (args.verbose)
302 fprintf(stderr, "got ack (%d) %s\n", ack,
303 sha1_to_hex(result_sha1));
304 if (ack == 1)
305 return 0;
306 multi_ack = 1;
307 continue;
308 }
309 flushes--;
310 }
311 /* it is no error to fetch into a completely empty repo */
312 return count ? retval : 0;
313}
314static struct commit_list *complete;
316static int mark_complete(const char *path, const unsigned char *sha1, int flag, void *cb_data)
318{
319 struct object *o = parse_object(sha1);
320 while (o && o->type == OBJ_TAG) {
322 struct tag *t = (struct tag *) o;
323 if (!t->tagged)
324 break; /* broken repository */
325 o->flags |= COMPLETE;
326 o = parse_object(t->tagged->sha1);
327 }
328 if (o && o->type == OBJ_COMMIT) {
329 struct commit *commit = (struct commit *)o;
330 commit->object.flags |= COMPLETE;
331 insert_by_date(commit, &complete);
332 }
333 return 0;
334}
335static void mark_recent_complete_commits(unsigned long cutoff)
337{
338 while (complete && cutoff <= complete->item->date) {
339 if (args.verbose)
340 fprintf(stderr, "Marking %s as complete\n",
341 sha1_to_hex(complete->item->object.sha1));
342 pop_most_recent_commit(&complete, COMPLETE);
343 }
344}
345static void filter_refs(struct ref **refs, int nr_match, char **match)
347{
348 struct ref **return_refs;
349 struct ref *newlist = NULL;
350 struct ref **newtail = &newlist;
351 struct ref *ref, *next;
352 struct ref *fastarray[32];
353 if (nr_match && !args.fetch_all) {
355 if (ARRAY_SIZE(fastarray) < nr_match)
356 return_refs = xcalloc(nr_match, sizeof(struct ref *));
357 else {
358 return_refs = fastarray;
359 memset(return_refs, 0, sizeof(struct ref *) * nr_match);
360 }
361 }
362 else
363 return_refs = NULL;
364 for (ref = *refs; ref; ref = next) {
366 next = ref->next;
367 if (!memcmp(ref->name, "refs/", 5) &&
368 check_ref_format(ref->name + 5))
369 ; /* trash */
370 else if (args.fetch_all &&
371 (!args.depth || prefixcmp(ref->name, "refs/tags/") )) {
372 *newtail = ref;
373 ref->next = NULL;
374 newtail = &ref->next;
375 continue;
376 }
377 else {
378 int order = path_match(ref->name, nr_match, match);
379 if (order) {
380 return_refs[order-1] = ref;
381 continue; /* we will link it later */
382 }
383 }
384 free(ref);
385 }
386 if (!args.fetch_all) {
388 int i;
389 for (i = 0; i < nr_match; i++) {
390 ref = return_refs[i];
391 if (ref) {
392 *newtail = ref;
393 ref->next = NULL;
394 newtail = &ref->next;
395 }
396 }
397 if (return_refs != fastarray)
398 free(return_refs);
399 }
400 *refs = newlist;
401}
402static int everything_local(struct ref **refs, int nr_match, char **match)
404{
405 struct ref *ref;
406 int retval;
407 unsigned long cutoff = 0;
408 save_commit_buffer = 0;
410 for (ref = *refs; ref; ref = ref->next) {
412 struct object *o;
413 o = parse_object(ref->old_sha1);
415 if (!o)
416 continue;
417 /* We already have it -- which may mean that we were
419 * in sync with the other side at some time after
420 * that (it is OK if we guess wrong here).
421 */
422 if (o->type == OBJ_COMMIT) {
423 struct commit *commit = (struct commit *)o;
424 if (!cutoff || cutoff < commit->date)
425 cutoff = commit->date;
426 }
427 }
428 if (!args.depth) {
430 for_each_ref(mark_complete, NULL);
431 if (cutoff)
432 mark_recent_complete_commits(cutoff);
433 }
434 /*
436 * Mark all complete remote refs as common refs.
437 * Don't mark them common yet; the server has to be told so first.
438 */
439 for (ref = *refs; ref; ref = ref->next) {
440 struct object *o = deref_tag(lookup_object(ref->old_sha1),
441 NULL, 0);
442 if (!o || o->type != OBJ_COMMIT || !(o->flags & COMPLETE))
444 continue;
445 if (!(o->flags & SEEN)) {
447 rev_list_push((struct commit *)o, COMMON_REF | SEEN);
448 mark_common((struct commit *)o, 1, 1);
450 }
451 }
452 filter_refs(refs, nr_match, match);
454 for (retval = 1, ref = *refs; ref ; ref = ref->next) {
456 const unsigned char *remote = ref->old_sha1;
457 unsigned char local[20];
458 struct object *o;
459 o = lookup_object(remote);
461 if (!o || !(o->flags & COMPLETE)) {
462 retval = 0;
463 if (!args.verbose)
464 continue;
465 fprintf(stderr,
466 "want %s (%s)\n", sha1_to_hex(remote),
467 ref->name);
468 continue;
469 }
470 hashcpy(ref->new_sha1, local);
472 if (!args.verbose)
473 continue;
474 fprintf(stderr,
475 "already have %s (%s)\n", sha1_to_hex(remote),
476 ref->name);
477 }
478 return retval;
479}
480static int sideband_demux(int fd, void *data)
482{
483 int *xd = data;
484 return recv_sideband("fetch-pack", xd[0], fd);
486}
487static int get_pack(int xd[2], char **pack_lockfile)
489{
490 struct async demux;
491 const char *argv[20];
492 char keep_arg[256];
493 char hdr_arg[256];
494 const char **av;
495 int do_keep = args.keep_pack;
496 struct child_process cmd;
497 memset(&demux, 0, sizeof(demux));
499 if (use_sideband) {
500 /* xd[] is talking with upload-pack; subprocess reads from
501 * xd[0], spits out band#2 to stderr, and feeds us band#1
502 * through demux->out.
503 */
504 demux.proc = sideband_demux;
505 demux.data = xd;
506 if (start_async(&demux))
507 die("fetch-pack: unable to fork off sideband"
508 " demultiplexer");
509 }
510 else
511 demux.out = xd[0];
512 memset(&cmd, 0, sizeof(cmd));
514 cmd.argv = argv;
515 av = argv;
516 *hdr_arg = 0;
517 if (!args.keep_pack && unpack_limit) {
518 struct pack_header header;
519 if (read_pack_header(demux.out, &header))
521 die("protocol error: bad pack header");
522 snprintf(hdr_arg, sizeof(hdr_arg),
523 "--pack_header=%"PRIu32",%"PRIu32,
524 ntohl(header.hdr_version), ntohl(header.hdr_entries));
525 if (ntohl(header.hdr_entries) < unpack_limit)
526 do_keep = 0;
527 else
528 do_keep = 1;
529 }
530 if (do_keep) {
532 if (pack_lockfile)
533 cmd.out = -1;
534 *av++ = "index-pack";
535 *av++ = "--stdin";
536 if (!args.quiet && !args.no_progress)
537 *av++ = "-v";
538 if (args.use_thin_pack)
539 *av++ = "--fix-thin";
540 if (args.lock_pack || unpack_limit) {
541 int s = sprintf(keep_arg,
542 "--keep=fetch-pack %"PRIuMAX " on ", (uintmax_t) getpid());
543 if (gethostname(keep_arg + s, sizeof(keep_arg) - s))
544 strcpy(keep_arg + s, "localhost");
545 *av++ = keep_arg;
546 }
547 }
548 else {
549 *av++ = "unpack-objects";
550 if (args.quiet)
551 *av++ = "-q";
552 }
553 if (*hdr_arg)
554 *av++ = hdr_arg;
555 *av++ = NULL;
556 cmd.in = demux.out;
558 cmd.git_cmd = 1;
559 if (start_command(&cmd))
560 die("fetch-pack: unable to fork off %s", argv[0]);
561 if (do_keep && pack_lockfile) {
562 *pack_lockfile = index_pack_lockfile(cmd.out);
563 close(cmd.out);
564 }
565 if (finish_command(&cmd))
567 die("%s failed", argv[0]);
568 if (use_sideband && finish_async(&demux))
569 die("error in sideband demultiplexer");
570 return 0;
571}
572static struct ref *do_fetch_pack(int fd[2],
574 const struct ref *orig_ref,
575 int nr_match,
576 char **match,
577 char **pack_lockfile)
578{
579 struct ref *ref = copy_ref_list(orig_ref);
580 unsigned char sha1[20];
581 if (is_repository_shallow() && !server_supports("shallow"))
583 die("Server does not support shallow clients");
584 if (server_supports("multi_ack")) {
585 if (args.verbose)
586 fprintf(stderr, "Server supports multi_ack\n");
587 multi_ack = 1;
588 }
589 if (server_supports("side-band-64k")) {
590 if (args.verbose)
591 fprintf(stderr, "Server supports side-band-64k\n");
592 use_sideband = 2;
593 }
594 else if (server_supports("side-band")) {
595 if (args.verbose)
596 fprintf(stderr, "Server supports side-band\n");
597 use_sideband = 1;
598 }
599 if (everything_local(&ref, nr_match, match)) {
600 packet_flush(fd[1]);
601 goto all_done;
602 }
603 if (find_common(fd, sha1, ref) < 0)
604 if (!args.keep_pack)
605 /* When cloning, it is not unusual to have
606 * no common commit.
607 */
608 warning("no common commits");
609 if (get_pack(fd, pack_lockfile))
611 die("git fetch-pack: fetch failed.");
612 all_done:
614 return ref;
615}
616static int remove_duplicates(int nr_heads, char **heads)
618{
619 int src, dst;
620 for (src = dst = 0; src < nr_heads; src++) {
622 /* If heads[src] is different from any of
623 * heads[0..dst], push it in.
624 */
625 int i;
626 for (i = 0; i < dst; i++) {
627 if (!strcmp(heads[i], heads[src]))
628 break;
629 }
630 if (i < dst)
631 continue;
632 if (src != dst)
633 heads[dst] = heads[src];
634 dst++;
635 }
636 return dst;
637}
638static int fetch_pack_config(const char *var, const char *value, void *cb)
640{
641 if (strcmp(var, "fetch.unpacklimit") == 0) {
642 fetch_unpack_limit = git_config_int(var, value);
643 return 0;
644 }
645 if (strcmp(var, "transfer.unpacklimit") == 0) {
647 transfer_unpack_limit = git_config_int(var, value);
648 return 0;
649 }
650 return git_default_config(var, value, cb);
652}
653static struct lock_file lock;
655static void fetch_pack_setup(void)
657{
658 static int did_setup;
659 if (did_setup)
660 return;
661 git_config(fetch_pack_config, NULL);
662 if (0 <= transfer_unpack_limit)
663 unpack_limit = transfer_unpack_limit;
664 else if (0 <= fetch_unpack_limit)
665 unpack_limit = fetch_unpack_limit;
666 did_setup = 1;
667}
668int cmd_fetch_pack(int argc, const char **argv, const char *prefix)
670{
671 int i, ret, nr_heads;
672 struct ref *ref = NULL;
673 char *dest = NULL, **heads;
674 int fd[2];
675 struct child_process *conn;
676 nr_heads = 0;
678 heads = NULL;
679 for (i = 1; i < argc; i++) {
680 const char *arg = argv[i];
681 if (*arg == '-') {
683 if (!prefixcmp(arg, "--upload-pack=")) {
684 args.uploadpack = arg + 14;
685 continue;
686 }
687 if (!prefixcmp(arg, "--exec=")) {
688 args.uploadpack = arg + 7;
689 continue;
690 }
691 if (!strcmp("--quiet", arg) || !strcmp("-q", arg)) {
692 args.quiet = 1;
693 continue;
694 }
695 if (!strcmp("--keep", arg) || !strcmp("-k", arg)) {
696 args.lock_pack = args.keep_pack;
697 args.keep_pack = 1;
698 continue;
699 }
700 if (!strcmp("--thin", arg)) {
701 args.use_thin_pack = 1;
702 continue;
703 }
704 if (!strcmp("--include-tag", arg)) {
705 args.include_tag = 1;
706 continue;
707 }
708 if (!strcmp("--all", arg)) {
709 args.fetch_all = 1;
710 continue;
711 }
712 if (!strcmp("-v", arg)) {
713 args.verbose = 1;
714 continue;
715 }
716 if (!prefixcmp(arg, "--depth=")) {
717 args.depth = strtol(arg + 8, NULL, 0);
718 continue;
719 }
720 if (!strcmp("--no-progress", arg)) {
721 args.no_progress = 1;
722 continue;
723 }
724 usage(fetch_pack_usage);
725 }
726 dest = (char *)arg;
727 heads = (char **)(argv + i + 1);
728 nr_heads = argc - i - 1;
729 break;
730 }
731 if (!dest)
732 usage(fetch_pack_usage);
733 conn = git_connect(fd, (char *)dest, args.uploadpack,
735 args.verbose ? CONNECT_VERBOSE : 0);
736 if (conn) {
737 get_remote_heads(fd[0], &ref, 0, NULL, 0, NULL);
738 ref = fetch_pack(&args, fd, conn, ref, dest, nr_heads, heads, NULL);
740 close(fd[0]);
741 close(fd[1]);
742 if (finish_connect(conn))
743 ref = NULL;
744 } else {
745 ref = NULL;
746 }
747 ret = !ref;
748 if (!ret && nr_heads) {
750 /* If the heads to pull were given, we should have
751 * consumed all of them by matching the remote.
752 * Otherwise, 'git fetch remote no-such-ref' would
753 * silently succeed without issuing an error.
754 */
755 for (i = 0; i < nr_heads; i++)
756 if (heads[i] && heads[i][0]) {
757 error("no such remote ref %s", heads[i]);
758 ret = 1;
759 }
760 }
761 while (ref) {
762 printf("%s %s\n",
763 sha1_to_hex(ref->old_sha1), ref->name);
764 ref = ref->next;
765 }
766 return ret;
768}
769struct ref *fetch_pack(struct fetch_pack_args *my_args,
771 int fd[], struct child_process *conn,
772 const struct ref *ref,
773 const char *dest,
774 int nr_heads,
775 char **heads,
776 char **pack_lockfile)
777{
778 struct stat st;
779 struct ref *ref_cpy;
780 fetch_pack_setup();
782 if (&args != my_args)
783 memcpy(&args, my_args, sizeof(args));
784 if (args.depth > 0) {
785 if (stat(git_path("shallow"), &st))
786 st.st_mtime = 0;
787 }
788 if (heads && nr_heads)
790 nr_heads = remove_duplicates(nr_heads, heads);
791 if (!ref) {
792 packet_flush(fd[1]);
793 die("no matching remote head");
794 }
795 ref_cpy = do_fetch_pack(fd, ref, nr_heads, heads, pack_lockfile);
796 if (args.depth > 0) {
798 struct cache_time mtime;
799 char *shallow = git_path("shallow");
800 int fd;
801 mtime.sec = st.st_mtime;
803 mtime.nsec = ST_MTIME_NSEC(st);
804 if (stat(shallow, &st)) {
805 if (mtime.sec)
806 die("shallow file was removed during fetch");
807 } else if (st.st_mtime != mtime.sec
808#ifdef USE_NSEC
809 || ST_MTIME_NSEC(st) != mtime.nsec
810#endif
811 )
812 die("shallow file was changed during fetch");
813 fd = hold_lock_file_for_update(&lock, shallow,
815 LOCK_DIE_ON_ERROR);
816 if (!write_shallow_commits(fd, 0)) {
817 unlink(shallow);
818 rollback_lock_file(&lock);
819 } else {
820 commit_lock_file(&lock);
821 }
822 }
823 reprepare_packed_git();
825 return ref_cpy;
826}