1#include "cache.h"
2#include "refs.h"
3#include "pkt-line.h"
4#include "commit.h"
5#include "tag.h"
6#include "exec_cmd.h"
7#include "pack.h"
8#include "sideband.h"
9#include "fetch-pack.h"
10#include "remote.h"
11#include "run-command.h"
12static int transfer_unpack_limit = -1;
14static int fetch_unpack_limit = -1;
15static int unpack_limit = 100;
16static struct fetch_pack_args args = {
17 /* .uploadpack = */ "git-upload-pack",
18};
19static const char fetch_pack_usage[] =
21"git fetch-pack [--all] [--quiet|-q] [--keep|-k] [--thin] [--include-tag] [--upload-pack=<git-upload-pack>] [--depth=<n>] [--no-progress] [-v] [<host>:]<directory> [<refs>...]";
22#define COMPLETE (1U << 0)
24#define COMMON (1U << 1)
25#define COMMON_REF (1U << 2)
26#define SEEN (1U << 3)
27#define POPPED (1U << 4)
28static int marked;
30/*
32 * After sending this many "have"s if we do not get any new ACK , we
33 * give up traversing our history.
34 */
35#define MAX_IN_VAIN 256
36static struct commit_list *rev_list;
38static int non_common_revs, multi_ack, use_sideband;
39static void rev_list_push(struct commit *commit, int mark)
41{
42 if (!(commit->object.flags & mark)) {
43 commit->object.flags |= mark;
44 if (!(commit->object.parsed))
46 if (parse_commit(commit))
47 return;
48 insert_by_date(commit, &rev_list);
50 if (!(commit->object.flags & COMMON))
52 non_common_revs++;
53 }
54}
55static int rev_list_insert_ref(const char *path, const unsigned char *sha1, int flag, void *cb_data)
57{
58 struct object *o = deref_tag(parse_object(sha1), path, 0);
59 if (o && o->type == OBJ_COMMIT)
61 rev_list_push((struct commit *)o, SEEN);
62 return 0;
64}
65static int clear_marks(const char *path, const unsigned char *sha1, int flag, void *cb_data)
67{
68 struct object *o = deref_tag(parse_object(sha1), path, 0);
69 if (o && o->type == OBJ_COMMIT)
71 clear_commit_marks((struct commit *)o,
72 COMMON | COMMON_REF | SEEN | POPPED);
73 return 0;
74}
75/*
77 This function marks a rev and its ancestors as common.
78 In some cases, it is desirable to mark only the ancestors (for example
79 when only the server does not yet know that they are common).
80*/
81static void mark_common(struct commit *commit,
83 int ancestors_only, int dont_parse)
84{
85 if (commit != NULL && !(commit->object.flags & COMMON)) {
86 struct object *o = (struct object *)commit;
87 if (!ancestors_only)
89 o->flags |= COMMON;
90 if (!(o->flags & SEEN))
92 rev_list_push(commit, SEEN);
93 else {
94 struct commit_list *parents;
95 if (!ancestors_only && !(o->flags & POPPED))
97 non_common_revs--;
98 if (!o->parsed && !dont_parse)
99 if (parse_commit(commit))
100 return;
101 for (parents = commit->parents;
103 parents;
104 parents = parents->next)
105 mark_common(parents->item, 0, dont_parse);
106 }
107 }
108}
109/*
111 Get the next rev to send, ignoring the common.
112*/
113static const unsigned char* get_rev(void)
115{
116 struct commit *commit = NULL;
117 while (commit == NULL) {
119 unsigned int mark;
120 struct commit_list *parents;
121 if (rev_list == NULL || non_common_revs == 0)
123 return NULL;
124 commit = rev_list->item;
126 if (!commit->object.parsed)
127 parse_commit(commit);
128 parents = commit->parents;
129 commit->object.flags |= POPPED;
131 if (!(commit->object.flags & COMMON))
132 non_common_revs--;
133 if (commit->object.flags & COMMON) {
135 /* do not send "have", and ignore ancestors */
136 commit = NULL;
137 mark = COMMON | SEEN;
138 } else if (commit->object.flags & COMMON_REF)
139 /* send "have", and ignore ancestors */
140 mark = COMMON | SEEN;
141 else
142 /* send "have", also for its ancestors */
143 mark = SEEN;
144 while (parents) {
146 if (!(parents->item->object.flags & SEEN))
147 rev_list_push(parents->item, mark);
148 if (mark & COMMON)
149 mark_common(parents->item, 1, 0);
150 parents = parents->next;
151 }
152 rev_list = rev_list->next;
154 }
155 return commit->object.sha1;
157}
158static int find_common(int fd[2], unsigned char *result_sha1,
160 struct ref *refs)
161{
162 int fetching;
163 int count = 0, flushes = 0, retval;
164 const unsigned char *sha1;
165 unsigned in_vain = 0;
166 int got_continue = 0;
167 if (marked)
169 for_each_ref(clear_marks, NULL);
170 marked = 1;
171 for_each_ref(rev_list_insert_ref, NULL);
173 fetching = 0;
175 for ( ; refs ; refs = refs->next) {
176 unsigned char *remote = refs->old_sha1;
177 struct object *o;
178 /*
180 * If that object is complete (i.e. it is an ancestor of a
181 * local ref), we tell them we have it but do not have to
182 * tell them about its ancestors, which they already know
183 * about.
184 *
185 * We use lookup_object here because we are only
186 * interested in the case we *know* the object is
187 * reachable and we have already scanned it.
188 */
189 if (((o = lookup_object(remote)) != NULL) &&
190 (o->flags & COMPLETE)) {
191 continue;
192 }
193 if (!fetching)
195 packet_write(fd[1], "want %s%s%s%s%s%s%s%s\n",
196 sha1_to_hex(remote),
197 (multi_ack ? " multi_ack" : ""),
198 (use_sideband == 2 ? " side-band-64k" : ""),
199 (use_sideband == 1 ? " side-band" : ""),
200 (args.use_thin_pack ? " thin-pack" : ""),
201 (args.no_progress ? " no-progress" : ""),
202 (args.include_tag ? " include-tag" : ""),
203 " ofs-delta");
204 else
205 packet_write(fd[1], "want %s\n", sha1_to_hex(remote));
206 fetching++;
207 }
208 if (is_repository_shallow())
209 write_shallow_commits(fd[1], 1);
210 if (args.depth > 0)
211 packet_write(fd[1], "deepen %d", args.depth);
212 packet_flush(fd[1]);
213 if (!fetching)
214 return 1;
215 if (args.depth > 0) {
217 char line[1024];
218 unsigned char sha1[20];
219 int len;
220 while ((len = packet_read_line(fd[0], line, sizeof(line)))) {
222 if (!prefixcmp(line, "shallow ")) {
223 if (get_sha1_hex(line + 8, sha1))
224 die("invalid shallow line: %s", line);
225 register_shallow(sha1);
226 continue;
227 }
228 if (!prefixcmp(line, "unshallow ")) {
229 if (get_sha1_hex(line + 10, sha1))
230 die("invalid unshallow line: %s", line);
231 if (!lookup_object(sha1))
232 die("object not found: %s", line);
233 /* make sure that it is parsed as shallow */
234 if (!parse_object(sha1))
235 die("error in object: %s", line);
236 if (unregister_shallow(sha1))
237 die("no shallow found: %s", line);
238 continue;
239 }
240 die("expected shallow/unshallow, got %s", line);
241 }
242 }
243 flushes = 0;
245 retval = -1;
246 while ((sha1 = get_rev())) {
247 packet_write(fd[1], "have %s\n", sha1_to_hex(sha1));
248 if (args.verbose)
249 fprintf(stderr, "have %s\n", sha1_to_hex(sha1));
250 in_vain++;
251 if (!(31 & ++count)) {
252 int ack;
253 packet_flush(fd[1]);
255 flushes++;
256 /*
258 * We keep one window "ahead" of the other side, and
259 * will wait for an ACK only on the next one
260 */
261 if (count == 32)
262 continue;
263 do {
265 ack = get_ack(fd[0], result_sha1);
266 if (args.verbose && ack)
267 fprintf(stderr, "got ack %d %s\n", ack,
268 sha1_to_hex(result_sha1));
269 if (ack == 1) {
270 flushes = 0;
271 multi_ack = 0;
272 retval = 0;
273 goto done;
274 } else if (ack == 2) {
275 struct commit *commit =
276 lookup_commit(result_sha1);
277 mark_common(commit, 0, 1);
278 retval = 0;
279 in_vain = 0;
280 got_continue = 1;
281 }
282 } while (ack);
283 flushes--;
284 if (got_continue && MAX_IN_VAIN < in_vain) {
285 if (args.verbose)
286 fprintf(stderr, "giving up\n");
287 break; /* give up */
288 }
289 }
290 }
291done:
292 packet_write(fd[1], "done\n");
293 if (args.verbose)
294 fprintf(stderr, "done\n");
295 if (retval != 0) {
296 multi_ack = 0;
297 flushes++;
298 }
299 while (flushes || multi_ack) {
300 int ack = get_ack(fd[0], result_sha1);
301 if (ack) {
302 if (args.verbose)
303 fprintf(stderr, "got ack (%d) %s\n", ack,
304 sha1_to_hex(result_sha1));
305 if (ack == 1)
306 return 0;
307 multi_ack = 1;
308 continue;
309 }
310 flushes--;
311 }
312 /* it is no error to fetch into a completely empty repo */
313 return count ? retval : 0;
314}
315static struct commit_list *complete;
317static int mark_complete(const char *path, const unsigned char *sha1, int flag, void *cb_data)
319{
320 struct object *o = parse_object(sha1);
321 while (o && o->type == OBJ_TAG) {
323 struct tag *t = (struct tag *) o;
324 if (!t->tagged)
325 break; /* broken repository */
326 o->flags |= COMPLETE;
327 o = parse_object(t->tagged->sha1);
328 }
329 if (o && o->type == OBJ_COMMIT) {
330 struct commit *commit = (struct commit *)o;
331 commit->object.flags |= COMPLETE;
332 insert_by_date(commit, &complete);
333 }
334 return 0;
335}
336static void mark_recent_complete_commits(unsigned long cutoff)
338{
339 while (complete && cutoff <= complete->item->date) {
340 if (args.verbose)
341 fprintf(stderr, "Marking %s as complete\n",
342 sha1_to_hex(complete->item->object.sha1));
343 pop_most_recent_commit(&complete, COMPLETE);
344 }
345}
346static void filter_refs(struct ref **refs, int nr_match, char **match)
348{
349 struct ref **return_refs;
350 struct ref *newlist = NULL;
351 struct ref **newtail = &newlist;
352 struct ref *ref, *next;
353 struct ref *fastarray[32];
354 if (nr_match && !args.fetch_all) {
356 if (ARRAY_SIZE(fastarray) < nr_match)
357 return_refs = xcalloc(nr_match, sizeof(struct ref *));
358 else {
359 return_refs = fastarray;
360 memset(return_refs, 0, sizeof(struct ref *) * nr_match);
361 }
362 }
363 else
364 return_refs = NULL;
365 for (ref = *refs; ref; ref = next) {
367 next = ref->next;
368 if (!memcmp(ref->name, "refs/", 5) &&
369 check_ref_format(ref->name + 5))
370 ; /* trash */
371 else if (args.fetch_all &&
372 (!args.depth || prefixcmp(ref->name, "refs/tags/") )) {
373 *newtail = ref;
374 ref->next = NULL;
375 newtail = &ref->next;
376 continue;
377 }
378 else {
379 int order = path_match(ref->name, nr_match, match);
380 if (order) {
381 return_refs[order-1] = ref;
382 continue; /* we will link it later */
383 }
384 }
385 free(ref);
386 }
387 if (!args.fetch_all) {
389 int i;
390 for (i = 0; i < nr_match; i++) {
391 ref = return_refs[i];
392 if (ref) {
393 *newtail = ref;
394 ref->next = NULL;
395 newtail = &ref->next;
396 }
397 }
398 if (return_refs != fastarray)
399 free(return_refs);
400 }
401 *refs = newlist;
402}
403static int everything_local(struct ref **refs, int nr_match, char **match)
405{
406 struct ref *ref;
407 int retval;
408 unsigned long cutoff = 0;
409 save_commit_buffer = 0;
411 for (ref = *refs; ref; ref = ref->next) {
413 struct object *o;
414 o = parse_object(ref->old_sha1);
416 if (!o)
417 continue;
418 /* We already have it -- which may mean that we were
420 * in sync with the other side at some time after
421 * that (it is OK if we guess wrong here).
422 */
423 if (o->type == OBJ_COMMIT) {
424 struct commit *commit = (struct commit *)o;
425 if (!cutoff || cutoff < commit->date)
426 cutoff = commit->date;
427 }
428 }
429 if (!args.depth) {
431 for_each_ref(mark_complete, NULL);
432 if (cutoff)
433 mark_recent_complete_commits(cutoff);
434 }
435 /*
437 * Mark all complete remote refs as common refs.
438 * Don't mark them common yet; the server has to be told so first.
439 */
440 for (ref = *refs; ref; ref = ref->next) {
441 struct object *o = deref_tag(lookup_object(ref->old_sha1),
442 NULL, 0);
443 if (!o || o->type != OBJ_COMMIT || !(o->flags & COMPLETE))
445 continue;
446 if (!(o->flags & SEEN)) {
448 rev_list_push((struct commit *)o, COMMON_REF | SEEN);
449 mark_common((struct commit *)o, 1, 1);
451 }
452 }
453 filter_refs(refs, nr_match, match);
455 for (retval = 1, ref = *refs; ref ; ref = ref->next) {
457 const unsigned char *remote = ref->old_sha1;
458 unsigned char local[20];
459 struct object *o;
460 o = lookup_object(remote);
462 if (!o || !(o->flags & COMPLETE)) {
463 retval = 0;
464 if (!args.verbose)
465 continue;
466 fprintf(stderr,
467 "want %s (%s)\n", sha1_to_hex(remote),
468 ref->name);
469 continue;
470 }
471 hashcpy(ref->new_sha1, local);
473 if (!args.verbose)
474 continue;
475 fprintf(stderr,
476 "already have %s (%s)\n", sha1_to_hex(remote),
477 ref->name);
478 }
479 return retval;
480}
481static int sideband_demux(int fd, void *data)
483{
484 int *xd = data;
485 return recv_sideband("fetch-pack", xd[0], fd, 2);
487}
488static int get_pack(int xd[2], char **pack_lockfile)
490{
491 struct async demux;
492 const char *argv[20];
493 char keep_arg[256];
494 char hdr_arg[256];
495 const char **av;
496 int do_keep = args.keep_pack;
497 struct child_process cmd;
498 memset(&demux, 0, sizeof(demux));
500 if (use_sideband) {
501 /* xd[] is talking with upload-pack; subprocess reads from
502 * xd[0], spits out band#2 to stderr, and feeds us band#1
503 * through demux->out.
504 */
505 demux.proc = sideband_demux;
506 demux.data = xd;
507 if (start_async(&demux))
508 die("fetch-pack: unable to fork off sideband"
509 " demultiplexer");
510 }
511 else
512 demux.out = xd[0];
513 memset(&cmd, 0, sizeof(cmd));
515 cmd.argv = argv;
516 av = argv;
517 *hdr_arg = 0;
518 if (!args.keep_pack && unpack_limit) {
519 struct pack_header header;
520 if (read_pack_header(demux.out, &header))
522 die("protocol error: bad pack header");
523 snprintf(hdr_arg, sizeof(hdr_arg),
524 "--pack_header=%"PRIu32",%"PRIu32,
525 ntohl(header.hdr_version), ntohl(header.hdr_entries));
526 if (ntohl(header.hdr_entries) < unpack_limit)
527 do_keep = 0;
528 else
529 do_keep = 1;
530 }
531 if (do_keep) {
533 if (pack_lockfile)
534 cmd.out = -1;
535 *av++ = "index-pack";
536 *av++ = "--stdin";
537 if (!args.quiet && !args.no_progress)
538 *av++ = "-v";
539 if (args.use_thin_pack)
540 *av++ = "--fix-thin";
541 if (args.lock_pack || unpack_limit) {
542 int s = sprintf(keep_arg,
543 "--keep=fetch-pack %d on ", getpid());
544 if (gethostname(keep_arg + s, sizeof(keep_arg) - s))
545 strcpy(keep_arg + s, "localhost");
546 *av++ = keep_arg;
547 }
548 }
549 else {
550 *av++ = "unpack-objects";
551 if (args.quiet)
552 *av++ = "-q";
553 }
554 if (*hdr_arg)
555 *av++ = hdr_arg;
556 *av++ = NULL;
557 cmd.in = demux.out;
559 cmd.git_cmd = 1;
560 if (start_command(&cmd))
561 die("fetch-pack: unable to fork off %s", argv[0]);
562 if (do_keep && pack_lockfile) {
563 *pack_lockfile = index_pack_lockfile(cmd.out);
564 close(cmd.out);
565 }
566 if (finish_command(&cmd))
568 die("%s failed", argv[0]);
569 if (use_sideband && finish_async(&demux))
570 die("error in sideband demultiplexer");
571 return 0;
572}
573static struct ref *do_fetch_pack(int fd[2],
575 const struct ref *orig_ref,
576 int nr_match,
577 char **match,
578 char **pack_lockfile)
579{
580 struct ref *ref = copy_ref_list(orig_ref);
581 unsigned char sha1[20];
582 if (is_repository_shallow() && !server_supports("shallow"))
584 die("Server does not support shallow clients");
585 if (server_supports("multi_ack")) {
586 if (args.verbose)
587 fprintf(stderr, "Server supports multi_ack\n");
588 multi_ack = 1;
589 }
590 if (server_supports("side-band-64k")) {
591 if (args.verbose)
592 fprintf(stderr, "Server supports side-band-64k\n");
593 use_sideband = 2;
594 }
595 else if (server_supports("side-band")) {
596 if (args.verbose)
597 fprintf(stderr, "Server supports side-band\n");
598 use_sideband = 1;
599 }
600 if (everything_local(&ref, nr_match, match)) {
601 packet_flush(fd[1]);
602 goto all_done;
603 }
604 if (find_common(fd, sha1, ref) < 0)
605 if (!args.keep_pack)
606 /* When cloning, it is not unusual to have
607 * no common commit.
608 */
609 fprintf(stderr, "warning: no common commits\n");
610 if (get_pack(fd, pack_lockfile))
612 die("git fetch-pack: fetch failed.");
613 all_done:
615 return ref;
616}
617static int remove_duplicates(int nr_heads, char **heads)
619{
620 int src, dst;
621 for (src = dst = 0; src < nr_heads; src++) {
623 /* If heads[src] is different from any of
624 * heads[0..dst], push it in.
625 */
626 int i;
627 for (i = 0; i < dst; i++) {
628 if (!strcmp(heads[i], heads[src]))
629 break;
630 }
631 if (i < dst)
632 continue;
633 if (src != dst)
634 heads[dst] = heads[src];
635 dst++;
636 }
637 return dst;
638}
639static int fetch_pack_config(const char *var, const char *value, void *cb)
641{
642 if (strcmp(var, "fetch.unpacklimit") == 0) {
643 fetch_unpack_limit = git_config_int(var, value);
644 return 0;
645 }
646 if (strcmp(var, "transfer.unpacklimit") == 0) {
648 transfer_unpack_limit = git_config_int(var, value);
649 return 0;
650 }
651 return git_default_config(var, value, cb);
653}
654static struct lock_file lock;
656static void fetch_pack_setup(void)
658{
659 static int did_setup;
660 if (did_setup)
661 return;
662 git_config(fetch_pack_config, NULL);
663 if (0 <= transfer_unpack_limit)
664 unpack_limit = transfer_unpack_limit;
665 else if (0 <= fetch_unpack_limit)
666 unpack_limit = fetch_unpack_limit;
667 did_setup = 1;
668}
669int cmd_fetch_pack(int argc, const char **argv, const char *prefix)
671{
672 int i, ret, nr_heads;
673 struct ref *ref = NULL;
674 char *dest = NULL, **heads;
675 int fd[2];
676 struct child_process *conn;
677 nr_heads = 0;
679 heads = NULL;
680 for (i = 1; i < argc; i++) {
681 const char *arg = argv[i];
682 if (*arg == '-') {
684 if (!prefixcmp(arg, "--upload-pack=")) {
685 args.uploadpack = arg + 14;
686 continue;
687 }
688 if (!prefixcmp(arg, "--exec=")) {
689 args.uploadpack = arg + 7;
690 continue;
691 }
692 if (!strcmp("--quiet", arg) || !strcmp("-q", arg)) {
693 args.quiet = 1;
694 continue;
695 }
696 if (!strcmp("--keep", arg) || !strcmp("-k", arg)) {
697 args.lock_pack = args.keep_pack;
698 args.keep_pack = 1;
699 continue;
700 }
701 if (!strcmp("--thin", arg)) {
702 args.use_thin_pack = 1;
703 continue;
704 }
705 if (!strcmp("--include-tag", arg)) {
706 args.include_tag = 1;
707 continue;
708 }
709 if (!strcmp("--all", arg)) {
710 args.fetch_all = 1;
711 continue;
712 }
713 if (!strcmp("-v", arg)) {
714 args.verbose = 1;
715 continue;
716 }
717 if (!prefixcmp(arg, "--depth=")) {
718 args.depth = strtol(arg + 8, NULL, 0);
719 continue;
720 }
721 if (!strcmp("--no-progress", arg)) {
722 args.no_progress = 1;
723 continue;
724 }
725 usage(fetch_pack_usage);
726 }
727 dest = (char *)arg;
728 heads = (char **)(argv + i + 1);
729 nr_heads = argc - i - 1;
730 break;
731 }
732 if (!dest)
733 usage(fetch_pack_usage);
734 conn = git_connect(fd, (char *)dest, args.uploadpack,
736 args.verbose ? CONNECT_VERBOSE : 0);
737 if (conn) {
738 get_remote_heads(fd[0], &ref, 0, NULL, 0);
739 ref = fetch_pack(&args, fd, conn, ref, dest, nr_heads, heads, NULL);
741 close(fd[0]);
742 close(fd[1]);
743 if (finish_connect(conn))
744 ref = NULL;
745 } else {
746 ref = NULL;
747 }
748 ret = !ref;
749 if (!ret && nr_heads) {
751 /* If the heads to pull were given, we should have
752 * consumed all of them by matching the remote.
753 * Otherwise, 'git fetch remote no-such-ref' would
754 * silently succeed without issuing an error.
755 */
756 for (i = 0; i < nr_heads; i++)
757 if (heads[i] && heads[i][0]) {
758 error("no such remote ref %s", heads[i]);
759 ret = 1;
760 }
761 }
762 while (ref) {
763 printf("%s %s\n",
764 sha1_to_hex(ref->old_sha1), ref->name);
765 ref = ref->next;
766 }
767 return ret;
769}
770struct ref *fetch_pack(struct fetch_pack_args *my_args,
772 int fd[], struct child_process *conn,
773 const struct ref *ref,
774 const char *dest,
775 int nr_heads,
776 char **heads,
777 char **pack_lockfile)
778{
779 struct stat st;
780 struct ref *ref_cpy;
781 fetch_pack_setup();
783 if (&args != my_args)
784 memcpy(&args, my_args, sizeof(args));
785 if (args.depth > 0) {
786 if (stat(git_path("shallow"), &st))
787 st.st_mtime = 0;
788 }
789 if (heads && nr_heads)
791 nr_heads = remove_duplicates(nr_heads, heads);
792 if (!ref) {
793 packet_flush(fd[1]);
794 die("no matching remote head");
795 }
796 ref_cpy = do_fetch_pack(fd, ref, nr_heads, heads, pack_lockfile);
797 if (args.depth > 0) {
799 struct cache_time mtime;
800 char *shallow = git_path("shallow");
801 int fd;
802 mtime.sec = st.st_mtime;
804#ifdef USE_NSEC
805 mtime.usec = st.st_mtim.usec;
806#endif
807 if (stat(shallow, &st)) {
808 if (mtime.sec)
809 die("shallow file was removed during fetch");
810 } else if (st.st_mtime != mtime.sec
811#ifdef USE_NSEC
812 || st.st_mtim.usec != mtime.usec
813#endif
814 )
815 die("shallow file was changed during fetch");
816 fd = hold_lock_file_for_update(&lock, shallow,
818 LOCK_DIE_ON_ERROR);
819 if (!write_shallow_commits(fd, 0)) {
820 unlink(shallow);
821 rollback_lock_file(&lock);
822 } else {
823 commit_lock_file(&lock);
824 }
825 }
826 reprepare_packed_git();
828 return ref_cpy;
829}