1#include "cache.h"
2#include "refs.h"
3#include "pkt-line.h"
4#include "commit.h"
5#include "tag.h"
6#include "exec_cmd.h"
7#include "pack.h"
8#include "sideband.h"
9#include "fetch-pack.h"
10#include "remote.h"
11#include "run-command.h"
12static int transfer_unpack_limit = -1;
14static int fetch_unpack_limit = -1;
15static int unpack_limit = 100;
16static struct fetch_pack_args args = {
17 /* .uploadpack = */ "git-upload-pack",
18};
19static const char fetch_pack_usage[] =
21"git-fetch-pack [--all] [--quiet|-q] [--keep|-k] [--thin] [--include-tag] [--upload-pack=<git-upload-pack>] [--depth=<n>] [--no-progress] [-v] [<host>:]<directory> [<refs>...]";
22#define COMPLETE (1U << 0)
24#define COMMON (1U << 1)
25#define COMMON_REF (1U << 2)
26#define SEEN (1U << 3)
27#define POPPED (1U << 4)
28static int marked;
30/*
32 * After sending this many "have"s if we do not get any new ACK , we
33 * give up traversing our history.
34 */
35#define MAX_IN_VAIN 256
36static struct commit_list *rev_list;
38static int non_common_revs, multi_ack, use_sideband;
39static void rev_list_push(struct commit *commit, int mark)
41{
42 if (!(commit->object.flags & mark)) {
43 commit->object.flags |= mark;
44 if (!(commit->object.parsed))
46 if (parse_commit(commit))
47 return;
48 insert_by_date(commit, &rev_list);
50 if (!(commit->object.flags & COMMON))
52 non_common_revs++;
53 }
54}
55static int rev_list_insert_ref(const char *path, const unsigned char *sha1, int flag, void *cb_data)
57{
58 struct object *o = deref_tag(parse_object(sha1), path, 0);
59 if (o && o->type == OBJ_COMMIT)
61 rev_list_push((struct commit *)o, SEEN);
62 return 0;
64}
65static int clear_marks(const char *path, const unsigned char *sha1, int flag, void *cb_data)
67{
68 struct object *o = deref_tag(parse_object(sha1), path, 0);
69 if (o && o->type == OBJ_COMMIT)
71 clear_commit_marks((struct commit *)o,
72 COMMON | COMMON_REF | SEEN | POPPED);
73 return 0;
74}
75/*
77 This function marks a rev and its ancestors as common.
78 In some cases, it is desirable to mark only the ancestors (for example
79 when only the server does not yet know that they are common).
80*/
81static void mark_common(struct commit *commit,
83 int ancestors_only, int dont_parse)
84{
85 if (commit != NULL && !(commit->object.flags & COMMON)) {
86 struct object *o = (struct object *)commit;
87 if (!ancestors_only)
89 o->flags |= COMMON;
90 if (!(o->flags & SEEN))
92 rev_list_push(commit, SEEN);
93 else {
94 struct commit_list *parents;
95 if (!ancestors_only && !(o->flags & POPPED))
97 non_common_revs--;
98 if (!o->parsed && !dont_parse)
99 if (parse_commit(commit))
100 return;
101 for (parents = commit->parents;
103 parents;
104 parents = parents->next)
105 mark_common(parents->item, 0, dont_parse);
106 }
107 }
108}
109/*
111 Get the next rev to send, ignoring the common.
112*/
113static const unsigned char* get_rev(void)
115{
116 struct commit *commit = NULL;
117 while (commit == NULL) {
119 unsigned int mark;
120 struct commit_list *parents = NULL;
121 if (rev_list == NULL || non_common_revs == 0)
123 return NULL;
124 commit = rev_list->item;
126 if (!(commit->object.parsed))
127 if (!parse_commit(commit))
128 parents = commit->parents;
129 commit->object.flags |= POPPED;
131 if (!(commit->object.flags & COMMON))
132 non_common_revs--;
133 if (commit->object.flags & COMMON) {
135 /* do not send "have", and ignore ancestors */
136 commit = NULL;
137 mark = COMMON | SEEN;
138 } else if (commit->object.flags & COMMON_REF)
139 /* send "have", and ignore ancestors */
140 mark = COMMON | SEEN;
141 else
142 /* send "have", also for its ancestors */
143 mark = SEEN;
144 while (parents) {
146 if (!(parents->item->object.flags & SEEN))
147 rev_list_push(parents->item, mark);
148 if (mark & COMMON)
149 mark_common(parents->item, 1, 0);
150 parents = parents->next;
151 }
152 rev_list = rev_list->next;
154 }
155 return commit->object.sha1;
157}
158static int find_common(int fd[2], unsigned char *result_sha1,
160 struct ref *refs)
161{
162 int fetching;
163 int count = 0, flushes = 0, retval;
164 const unsigned char *sha1;
165 unsigned in_vain = 0;
166 int got_continue = 0;
167 if (marked)
169 for_each_ref(clear_marks, NULL);
170 marked = 1;
171 for_each_ref(rev_list_insert_ref, NULL);
173 fetching = 0;
175 for ( ; refs ; refs = refs->next) {
176 unsigned char *remote = refs->old_sha1;
177 struct object *o;
178 /*
180 * If that object is complete (i.e. it is an ancestor of a
181 * local ref), we tell them we have it but do not have to
182 * tell them about its ancestors, which they already know
183 * about.
184 *
185 * We use lookup_object here because we are only
186 * interested in the case we *know* the object is
187 * reachable and we have already scanned it.
188 */
189 if (((o = lookup_object(remote)) != NULL) &&
190 (o->flags & COMPLETE)) {
191 continue;
192 }
193 if (!fetching)
195 packet_write(fd[1], "want %s%s%s%s%s%s%s%s\n",
196 sha1_to_hex(remote),
197 (multi_ack ? " multi_ack" : ""),
198 (use_sideband == 2 ? " side-band-64k" : ""),
199 (use_sideband == 1 ? " side-band" : ""),
200 (args.use_thin_pack ? " thin-pack" : ""),
201 (args.no_progress ? " no-progress" : ""),
202 (args.include_tag ? " include-tag" : ""),
203 " ofs-delta");
204 else
205 packet_write(fd[1], "want %s\n", sha1_to_hex(remote));
206 fetching++;
207 }
208 if (is_repository_shallow())
209 write_shallow_commits(fd[1], 1);
210 if (args.depth > 0)
211 packet_write(fd[1], "deepen %d", args.depth);
212 packet_flush(fd[1]);
213 if (!fetching)
214 return 1;
215 if (args.depth > 0) {
217 char line[1024];
218 unsigned char sha1[20];
219 int len;
220 while ((len = packet_read_line(fd[0], line, sizeof(line)))) {
222 if (!prefixcmp(line, "shallow ")) {
223 if (get_sha1_hex(line + 8, sha1))
224 die("invalid shallow line: %s", line);
225 register_shallow(sha1);
226 continue;
227 }
228 if (!prefixcmp(line, "unshallow ")) {
229 if (get_sha1_hex(line + 10, sha1))
230 die("invalid unshallow line: %s", line);
231 if (!lookup_object(sha1))
232 die("object not found: %s", line);
233 /* make sure that it is parsed as shallow */
234 if (!parse_object(sha1))
235 die("error in object: %s", line);
236 if (unregister_shallow(sha1))
237 die("no shallow found: %s", line);
238 continue;
239 }
240 die("expected shallow/unshallow, got %s", line);
241 }
242 }
243 flushes = 0;
245 retval = -1;
246 while ((sha1 = get_rev())) {
247 packet_write(fd[1], "have %s\n", sha1_to_hex(sha1));
248 if (args.verbose)
249 fprintf(stderr, "have %s\n", sha1_to_hex(sha1));
250 in_vain++;
251 if (!(31 & ++count)) {
252 int ack;
253 packet_flush(fd[1]);
255 flushes++;
256 /*
258 * We keep one window "ahead" of the other side, and
259 * will wait for an ACK only on the next one
260 */
261 if (count == 32)
262 continue;
263 do {
265 ack = get_ack(fd[0], result_sha1);
266 if (args.verbose && ack)
267 fprintf(stderr, "got ack %d %s\n", ack,
268 sha1_to_hex(result_sha1));
269 if (ack == 1) {
270 flushes = 0;
271 multi_ack = 0;
272 retval = 0;
273 goto done;
274 } else if (ack == 2) {
275 struct commit *commit =
276 lookup_commit(result_sha1);
277 mark_common(commit, 0, 1);
278 retval = 0;
279 in_vain = 0;
280 got_continue = 1;
281 }
282 } while (ack);
283 flushes--;
284 if (got_continue && MAX_IN_VAIN < in_vain) {
285 if (args.verbose)
286 fprintf(stderr, "giving up\n");
287 break; /* give up */
288 }
289 }
290 }
291done:
292 packet_write(fd[1], "done\n");
293 if (args.verbose)
294 fprintf(stderr, "done\n");
295 if (retval != 0) {
296 multi_ack = 0;
297 flushes++;
298 }
299 while (flushes || multi_ack) {
300 int ack = get_ack(fd[0], result_sha1);
301 if (ack) {
302 if (args.verbose)
303 fprintf(stderr, "got ack (%d) %s\n", ack,
304 sha1_to_hex(result_sha1));
305 if (ack == 1)
306 return 0;
307 multi_ack = 1;
308 continue;
309 }
310 flushes--;
311 }
312 return retval;
313}
314static struct commit_list *complete;
316static int mark_complete(const char *path, const unsigned char *sha1, int flag, void *cb_data)
318{
319 struct object *o = parse_object(sha1);
320 while (o && o->type == OBJ_TAG) {
322 struct tag *t = (struct tag *) o;
323 if (!t->tagged)
324 break; /* broken repository */
325 o->flags |= COMPLETE;
326 o = parse_object(t->tagged->sha1);
327 }
328 if (o && o->type == OBJ_COMMIT) {
329 struct commit *commit = (struct commit *)o;
330 commit->object.flags |= COMPLETE;
331 insert_by_date(commit, &complete);
332 }
333 return 0;
334}
335static void mark_recent_complete_commits(unsigned long cutoff)
337{
338 while (complete && cutoff <= complete->item->date) {
339 if (args.verbose)
340 fprintf(stderr, "Marking %s as complete\n",
341 sha1_to_hex(complete->item->object.sha1));
342 pop_most_recent_commit(&complete, COMPLETE);
343 }
344}
345static void filter_refs(struct ref **refs, int nr_match, char **match)
347{
348 struct ref **return_refs;
349 struct ref *newlist = NULL;
350 struct ref **newtail = &newlist;
351 struct ref *ref, *next;
352 struct ref *fastarray[32];
353 if (nr_match && !args.fetch_all) {
355 if (ARRAY_SIZE(fastarray) < nr_match)
356 return_refs = xcalloc(nr_match, sizeof(struct ref *));
357 else {
358 return_refs = fastarray;
359 memset(return_refs, 0, sizeof(struct ref *) * nr_match);
360 }
361 }
362 else
363 return_refs = NULL;
364 for (ref = *refs; ref; ref = next) {
366 next = ref->next;
367 if (!memcmp(ref->name, "refs/", 5) &&
368 check_ref_format(ref->name + 5))
369 ; /* trash */
370 else if (args.fetch_all &&
371 (!args.depth || prefixcmp(ref->name, "refs/tags/") )) {
372 *newtail = ref;
373 ref->next = NULL;
374 newtail = &ref->next;
375 continue;
376 }
377 else {
378 int order = path_match(ref->name, nr_match, match);
379 if (order) {
380 return_refs[order-1] = ref;
381 continue; /* we will link it later */
382 }
383 }
384 free(ref);
385 }
386 if (!args.fetch_all) {
388 int i;
389 for (i = 0; i < nr_match; i++) {
390 ref = return_refs[i];
391 if (ref) {
392 *newtail = ref;
393 ref->next = NULL;
394 newtail = &ref->next;
395 }
396 }
397 if (return_refs != fastarray)
398 free(return_refs);
399 }
400 *refs = newlist;
401}
402static int everything_local(struct ref **refs, int nr_match, char **match)
404{
405 struct ref *ref;
406 int retval;
407 unsigned long cutoff = 0;
408 save_commit_buffer = 0;
410 for (ref = *refs; ref; ref = ref->next) {
412 struct object *o;
413 o = parse_object(ref->old_sha1);
415 if (!o)
416 continue;
417 /* We already have it -- which may mean that we were
419 * in sync with the other side at some time after
420 * that (it is OK if we guess wrong here).
421 */
422 if (o->type == OBJ_COMMIT) {
423 struct commit *commit = (struct commit *)o;
424 if (!cutoff || cutoff < commit->date)
425 cutoff = commit->date;
426 }
427 }
428 if (!args.depth) {
430 for_each_ref(mark_complete, NULL);
431 if (cutoff)
432 mark_recent_complete_commits(cutoff);
433 }
434 /*
436 * Mark all complete remote refs as common refs.
437 * Don't mark them common yet; the server has to be told so first.
438 */
439 for (ref = *refs; ref; ref = ref->next) {
440 struct object *o = deref_tag(lookup_object(ref->old_sha1),
441 NULL, 0);
442 if (!o || o->type != OBJ_COMMIT || !(o->flags & COMPLETE))
444 continue;
445 if (!(o->flags & SEEN)) {
447 rev_list_push((struct commit *)o, COMMON_REF | SEEN);
448 mark_common((struct commit *)o, 1, 1);
450 }
451 }
452 filter_refs(refs, nr_match, match);
454 for (retval = 1, ref = *refs; ref ; ref = ref->next) {
456 const unsigned char *remote = ref->old_sha1;
457 unsigned char local[20];
458 struct object *o;
459 o = lookup_object(remote);
461 if (!o || !(o->flags & COMPLETE)) {
462 retval = 0;
463 if (!args.verbose)
464 continue;
465 fprintf(stderr,
466 "want %s (%s)\n", sha1_to_hex(remote),
467 ref->name);
468 continue;
469 }
470 hashcpy(ref->new_sha1, local);
472 if (!args.verbose)
473 continue;
474 fprintf(stderr,
475 "already have %s (%s)\n", sha1_to_hex(remote),
476 ref->name);
477 }
478 return retval;
479}
480static int sideband_demux(int fd, void *data)
482{
483 int *xd = data;
484 return recv_sideband("fetch-pack", xd[0], fd, 2);
486}
487static int get_pack(int xd[2], char **pack_lockfile)
489{
490 struct async demux;
491 const char *argv[20];
492 char keep_arg[256];
493 char hdr_arg[256];
494 const char **av;
495 int do_keep = args.keep_pack;
496 struct child_process cmd;
497 memset(&demux, 0, sizeof(demux));
499 if (use_sideband) {
500 /* xd[] is talking with upload-pack; subprocess reads from
501 * xd[0], spits out band#2 to stderr, and feeds us band#1
502 * through demux->out.
503 */
504 demux.proc = sideband_demux;
505 demux.data = xd;
506 if (start_async(&demux))
507 die("fetch-pack: unable to fork off sideband"
508 " demultiplexer");
509 }
510 else
511 demux.out = xd[0];
512 memset(&cmd, 0, sizeof(cmd));
514 cmd.argv = argv;
515 av = argv;
516 *hdr_arg = 0;
517 if (!args.keep_pack && unpack_limit) {
518 struct pack_header header;
519 if (read_pack_header(demux.out, &header))
521 die("protocol error: bad pack header");
522 snprintf(hdr_arg, sizeof(hdr_arg), "--pack_header=%u,%u",
523 ntohl(header.hdr_version), ntohl(header.hdr_entries));
524 if (ntohl(header.hdr_entries) < unpack_limit)
525 do_keep = 0;
526 else
527 do_keep = 1;
528 }
529 if (do_keep) {
531 if (pack_lockfile)
532 cmd.out = -1;
533 *av++ = "index-pack";
534 *av++ = "--stdin";
535 if (!args.quiet && !args.no_progress)
536 *av++ = "-v";
537 if (args.use_thin_pack)
538 *av++ = "--fix-thin";
539 if (args.lock_pack || unpack_limit) {
540 int s = sprintf(keep_arg,
541 "--keep=fetch-pack %d on ", getpid());
542 if (gethostname(keep_arg + s, sizeof(keep_arg) - s))
543 strcpy(keep_arg + s, "localhost");
544 *av++ = keep_arg;
545 }
546 }
547 else {
548 *av++ = "unpack-objects";
549 if (args.quiet)
550 *av++ = "-q";
551 }
552 if (*hdr_arg)
553 *av++ = hdr_arg;
554 *av++ = NULL;
555 cmd.in = demux.out;
557 cmd.git_cmd = 1;
558 if (start_command(&cmd))
559 die("fetch-pack: unable to fork off %s", argv[0]);
560 if (do_keep && pack_lockfile) {
561 *pack_lockfile = index_pack_lockfile(cmd.out);
562 close(cmd.out);
563 }
564 if (finish_command(&cmd))
566 die("%s failed", argv[0]);
567 if (use_sideband && finish_async(&demux))
568 die("error in sideband demultiplexer");
569 return 0;
570}
571static struct ref *do_fetch_pack(int fd[2],
573 const struct ref *orig_ref,
574 int nr_match,
575 char **match,
576 char **pack_lockfile)
577{
578 struct ref *ref = copy_ref_list(orig_ref);
579 unsigned char sha1[20];
580 if (is_repository_shallow() && !server_supports("shallow"))
582 die("Server does not support shallow clients");
583 if (server_supports("multi_ack")) {
584 if (args.verbose)
585 fprintf(stderr, "Server supports multi_ack\n");
586 multi_ack = 1;
587 }
588 if (server_supports("side-band-64k")) {
589 if (args.verbose)
590 fprintf(stderr, "Server supports side-band-64k\n");
591 use_sideband = 2;
592 }
593 else if (server_supports("side-band")) {
594 if (args.verbose)
595 fprintf(stderr, "Server supports side-band\n");
596 use_sideband = 1;
597 }
598 if (everything_local(&ref, nr_match, match)) {
599 packet_flush(fd[1]);
600 goto all_done;
601 }
602 if (find_common(fd, sha1, ref) < 0)
603 if (!args.keep_pack)
604 /* When cloning, it is not unusual to have
605 * no common commit.
606 */
607 fprintf(stderr, "warning: no common commits\n");
608 if (get_pack(fd, pack_lockfile))
610 die("git-fetch-pack: fetch failed.");
611 all_done:
613 return ref;
614}
615static int remove_duplicates(int nr_heads, char **heads)
617{
618 int src, dst;
619 for (src = dst = 0; src < nr_heads; src++) {
621 /* If heads[src] is different from any of
622 * heads[0..dst], push it in.
623 */
624 int i;
625 for (i = 0; i < dst; i++) {
626 if (!strcmp(heads[i], heads[src]))
627 break;
628 }
629 if (i < dst)
630 continue;
631 if (src != dst)
632 heads[dst] = heads[src];
633 dst++;
634 }
635 return dst;
636}
637static int fetch_pack_config(const char *var, const char *value)
639{
640 if (strcmp(var, "fetch.unpacklimit") == 0) {
641 fetch_unpack_limit = git_config_int(var, value);
642 return 0;
643 }
644 if (strcmp(var, "transfer.unpacklimit") == 0) {
646 transfer_unpack_limit = git_config_int(var, value);
647 return 0;
648 }
649 return git_default_config(var, value);
651}
652static struct lock_file lock;
654static void fetch_pack_setup(void)
656{
657 static int did_setup;
658 if (did_setup)
659 return;
660 git_config(fetch_pack_config);
661 if (0 <= transfer_unpack_limit)
662 unpack_limit = transfer_unpack_limit;
663 else if (0 <= fetch_unpack_limit)
664 unpack_limit = fetch_unpack_limit;
665 did_setup = 1;
666}
667int cmd_fetch_pack(int argc, const char **argv, const char *prefix)
669{
670 int i, ret, nr_heads;
671 struct ref *ref = NULL;
672 char *dest = NULL, **heads;
673 int fd[2];
674 struct child_process *conn;
675 nr_heads = 0;
677 heads = NULL;
678 for (i = 1; i < argc; i++) {
679 const char *arg = argv[i];
680 if (*arg == '-') {
682 if (!prefixcmp(arg, "--upload-pack=")) {
683 args.uploadpack = arg + 14;
684 continue;
685 }
686 if (!prefixcmp(arg, "--exec=")) {
687 args.uploadpack = arg + 7;
688 continue;
689 }
690 if (!strcmp("--quiet", arg) || !strcmp("-q", arg)) {
691 args.quiet = 1;
692 continue;
693 }
694 if (!strcmp("--keep", arg) || !strcmp("-k", arg)) {
695 args.lock_pack = args.keep_pack;
696 args.keep_pack = 1;
697 continue;
698 }
699 if (!strcmp("--thin", arg)) {
700 args.use_thin_pack = 1;
701 continue;
702 }
703 if (!strcmp("--include-tag", arg)) {
704 args.include_tag = 1;
705 continue;
706 }
707 if (!strcmp("--all", arg)) {
708 args.fetch_all = 1;
709 continue;
710 }
711 if (!strcmp("-v", arg)) {
712 args.verbose = 1;
713 continue;
714 }
715 if (!prefixcmp(arg, "--depth=")) {
716 args.depth = strtol(arg + 8, NULL, 0);
717 continue;
718 }
719 if (!strcmp("--no-progress", arg)) {
720 args.no_progress = 1;
721 continue;
722 }
723 usage(fetch_pack_usage);
724 }
725 dest = (char *)arg;
726 heads = (char **)(argv + i + 1);
727 nr_heads = argc - i - 1;
728 break;
729 }
730 if (!dest)
731 usage(fetch_pack_usage);
732 conn = git_connect(fd, (char *)dest, args.uploadpack,
734 args.verbose ? CONNECT_VERBOSE : 0);
735 if (conn) {
736 get_remote_heads(fd[0], &ref, 0, NULL, 0);
737 ref = fetch_pack(&args, fd, conn, ref, dest, nr_heads, heads, NULL);
739 close(fd[0]);
740 close(fd[1]);
741 if (finish_connect(conn))
742 ref = NULL;
743 } else {
744 ref = NULL;
745 }
746 ret = !ref;
747 if (!ret && nr_heads) {
749 /* If the heads to pull were given, we should have
750 * consumed all of them by matching the remote.
751 * Otherwise, 'git-fetch remote no-such-ref' would
752 * silently succeed without issuing an error.
753 */
754 for (i = 0; i < nr_heads; i++)
755 if (heads[i] && heads[i][0]) {
756 error("no such remote ref %s", heads[i]);
757 ret = 1;
758 }
759 }
760 while (ref) {
761 printf("%s %s\n",
762 sha1_to_hex(ref->old_sha1), ref->name);
763 ref = ref->next;
764 }
765 return ret;
767}
768struct ref *fetch_pack(struct fetch_pack_args *my_args,
770 int fd[], struct child_process *conn,
771 const struct ref *ref,
772 const char *dest,
773 int nr_heads,
774 char **heads,
775 char **pack_lockfile)
776{
777 struct stat st;
778 struct ref *ref_cpy;
779 fetch_pack_setup();
781 memcpy(&args, my_args, sizeof(args));
782 if (args.depth > 0) {
783 if (stat(git_path("shallow"), &st))
784 st.st_mtime = 0;
785 }
786 if (heads && nr_heads)
788 nr_heads = remove_duplicates(nr_heads, heads);
789 if (!ref) {
790 packet_flush(fd[1]);
791 die("no matching remote head");
792 }
793 ref_cpy = do_fetch_pack(fd, ref, nr_heads, heads, pack_lockfile);
794 if (args.depth > 0) {
796 struct cache_time mtime;
797 char *shallow = git_path("shallow");
798 int fd;
799 mtime.sec = st.st_mtime;
801#ifdef USE_NSEC
802 mtime.usec = st.st_mtim.usec;
803#endif
804 if (stat(shallow, &st)) {
805 if (mtime.sec)
806 die("shallow file was removed during fetch");
807 } else if (st.st_mtime != mtime.sec
808#ifdef USE_NSEC
809 || st.st_mtim.usec != mtime.usec
810#endif
811 )
812 die("shallow file was changed during fetch");
813 fd = hold_lock_file_for_update(&lock, shallow, 1);
815 if (!write_shallow_commits(fd, 0)) {
816 unlink(shallow);
817 rollback_lock_file(&lock);
818 } else {
819 commit_lock_file(&lock);
820 }
821 }
822 return ref_cpy;
824}