1#include "cache.h"
2#include "refs.h"
3#include "pkt-line.h"
4#include "commit.h"
5#include "tag.h"
6#include "exec_cmd.h"
7#include "pack.h"
8#include "sideband.h"
9#include "fetch-pack.h"
10#include "remote.h"
11#include "run-command.h"
12static int transfer_unpack_limit = -1;
14static int fetch_unpack_limit = -1;
15static int unpack_limit = 100;
16static struct fetch_pack_args args = {
17 /* .uploadpack = */ "git-upload-pack",
18};
19static const char fetch_pack_usage[] =
21"git-fetch-pack [--all] [--quiet|-q] [--keep|-k] [--thin] [--include-tag] [--upload-pack=<git-upload-pack>] [--depth=<n>] [--no-progress] [-v] [<host>:]<directory> [<refs>...]";
22#define COMPLETE (1U << 0)
24#define COMMON (1U << 1)
25#define COMMON_REF (1U << 2)
26#define SEEN (1U << 3)
27#define POPPED (1U << 4)
28/*
30 * After sending this many "have"s if we do not get any new ACK , we
31 * give up traversing our history.
32 */
33#define MAX_IN_VAIN 256
34static struct commit_list *rev_list;
36static int non_common_revs, multi_ack, use_sideband;
37static void rev_list_push(struct commit *commit, int mark)
39{
40 if (!(commit->object.flags & mark)) {
41 commit->object.flags |= mark;
42 if (!(commit->object.parsed))
44 if (parse_commit(commit))
45 return;
46 insert_by_date(commit, &rev_list);
48 if (!(commit->object.flags & COMMON))
50 non_common_revs++;
51 }
52}
53static int rev_list_insert_ref(const char *path, const unsigned char *sha1, int flag, void *cb_data)
55{
56 struct object *o = deref_tag(parse_object(sha1), path, 0);
57 if (o && o->type == OBJ_COMMIT)
59 rev_list_push((struct commit *)o, SEEN);
60 return 0;
62}
63/*
65 This function marks a rev and its ancestors as common.
66 In some cases, it is desirable to mark only the ancestors (for example
67 when only the server does not yet know that they are common).
68*/
69static void mark_common(struct commit *commit,
71 int ancestors_only, int dont_parse)
72{
73 if (commit != NULL && !(commit->object.flags & COMMON)) {
74 struct object *o = (struct object *)commit;
75 if (!ancestors_only)
77 o->flags |= COMMON;
78 if (!(o->flags & SEEN))
80 rev_list_push(commit, SEEN);
81 else {
82 struct commit_list *parents;
83 if (!ancestors_only && !(o->flags & POPPED))
85 non_common_revs--;
86 if (!o->parsed && !dont_parse)
87 if (parse_commit(commit))
88 return;
89 for (parents = commit->parents;
91 parents;
92 parents = parents->next)
93 mark_common(parents->item, 0, dont_parse);
94 }
95 }
96}
97/*
99 Get the next rev to send, ignoring the common.
100*/
101static const unsigned char* get_rev(void)
103{
104 struct commit *commit = NULL;
105 while (commit == NULL) {
107 unsigned int mark;
108 struct commit_list *parents = NULL;
109 if (rev_list == NULL || non_common_revs == 0)
111 return NULL;
112 commit = rev_list->item;
114 if (!(commit->object.parsed))
115 if (!parse_commit(commit))
116 parents = commit->parents;
117 commit->object.flags |= POPPED;
119 if (!(commit->object.flags & COMMON))
120 non_common_revs--;
121 if (commit->object.flags & COMMON) {
123 /* do not send "have", and ignore ancestors */
124 commit = NULL;
125 mark = COMMON | SEEN;
126 } else if (commit->object.flags & COMMON_REF)
127 /* send "have", and ignore ancestors */
128 mark = COMMON | SEEN;
129 else
130 /* send "have", also for its ancestors */
131 mark = SEEN;
132 while (parents) {
134 if (!(parents->item->object.flags & SEEN))
135 rev_list_push(parents->item, mark);
136 if (mark & COMMON)
137 mark_common(parents->item, 1, 0);
138 parents = parents->next;
139 }
140 rev_list = rev_list->next;
142 }
143 return commit->object.sha1;
145}
146static int find_common(int fd[2], unsigned char *result_sha1,
148 struct ref *refs)
149{
150 int fetching;
151 int count = 0, flushes = 0, retval;
152 const unsigned char *sha1;
153 unsigned in_vain = 0;
154 int got_continue = 0;
155 for_each_ref(rev_list_insert_ref, NULL);
157 fetching = 0;
159 for ( ; refs ; refs = refs->next) {
160 unsigned char *remote = refs->old_sha1;
161 struct object *o;
162 /*
164 * If that object is complete (i.e. it is an ancestor of a
165 * local ref), we tell them we have it but do not have to
166 * tell them about its ancestors, which they already know
167 * about.
168 *
169 * We use lookup_object here because we are only
170 * interested in the case we *know* the object is
171 * reachable and we have already scanned it.
172 */
173 if (((o = lookup_object(remote)) != NULL) &&
174 (o->flags & COMPLETE)) {
175 continue;
176 }
177 if (!fetching)
179 packet_write(fd[1], "want %s%s%s%s%s%s%s%s\n",
180 sha1_to_hex(remote),
181 (multi_ack ? " multi_ack" : ""),
182 (use_sideband == 2 ? " side-band-64k" : ""),
183 (use_sideband == 1 ? " side-band" : ""),
184 (args.use_thin_pack ? " thin-pack" : ""),
185 (args.no_progress ? " no-progress" : ""),
186 (args.include_tag ? " include-tag" : ""),
187 " ofs-delta");
188 else
189 packet_write(fd[1], "want %s\n", sha1_to_hex(remote));
190 fetching++;
191 }
192 if (is_repository_shallow())
193 write_shallow_commits(fd[1], 1);
194 if (args.depth > 0)
195 packet_write(fd[1], "deepen %d", args.depth);
196 packet_flush(fd[1]);
197 if (!fetching)
198 return 1;
199 if (args.depth > 0) {
201 char line[1024];
202 unsigned char sha1[20];
203 int len;
204 while ((len = packet_read_line(fd[0], line, sizeof(line)))) {
206 if (!prefixcmp(line, "shallow ")) {
207 if (get_sha1_hex(line + 8, sha1))
208 die("invalid shallow line: %s", line);
209 register_shallow(sha1);
210 continue;
211 }
212 if (!prefixcmp(line, "unshallow ")) {
213 if (get_sha1_hex(line + 10, sha1))
214 die("invalid unshallow line: %s", line);
215 if (!lookup_object(sha1))
216 die("object not found: %s", line);
217 /* make sure that it is parsed as shallow */
218 if (!parse_object(sha1))
219 die("error in object: %s", line);
220 if (unregister_shallow(sha1))
221 die("no shallow found: %s", line);
222 continue;
223 }
224 die("expected shallow/unshallow, got %s", line);
225 }
226 }
227 flushes = 0;
229 retval = -1;
230 while ((sha1 = get_rev())) {
231 packet_write(fd[1], "have %s\n", sha1_to_hex(sha1));
232 if (args.verbose)
233 fprintf(stderr, "have %s\n", sha1_to_hex(sha1));
234 in_vain++;
235 if (!(31 & ++count)) {
236 int ack;
237 packet_flush(fd[1]);
239 flushes++;
240 /*
242 * We keep one window "ahead" of the other side, and
243 * will wait for an ACK only on the next one
244 */
245 if (count == 32)
246 continue;
247 do {
249 ack = get_ack(fd[0], result_sha1);
250 if (args.verbose && ack)
251 fprintf(stderr, "got ack %d %s\n", ack,
252 sha1_to_hex(result_sha1));
253 if (ack == 1) {
254 flushes = 0;
255 multi_ack = 0;
256 retval = 0;
257 goto done;
258 } else if (ack == 2) {
259 struct commit *commit =
260 lookup_commit(result_sha1);
261 mark_common(commit, 0, 1);
262 retval = 0;
263 in_vain = 0;
264 got_continue = 1;
265 }
266 } while (ack);
267 flushes--;
268 if (got_continue && MAX_IN_VAIN < in_vain) {
269 if (args.verbose)
270 fprintf(stderr, "giving up\n");
271 break; /* give up */
272 }
273 }
274 }
275done:
276 packet_write(fd[1], "done\n");
277 if (args.verbose)
278 fprintf(stderr, "done\n");
279 if (retval != 0) {
280 multi_ack = 0;
281 flushes++;
282 }
283 while (flushes || multi_ack) {
284 int ack = get_ack(fd[0], result_sha1);
285 if (ack) {
286 if (args.verbose)
287 fprintf(stderr, "got ack (%d) %s\n", ack,
288 sha1_to_hex(result_sha1));
289 if (ack == 1)
290 return 0;
291 multi_ack = 1;
292 continue;
293 }
294 flushes--;
295 }
296 return retval;
297}
298static struct commit_list *complete;
300static int mark_complete(const char *path, const unsigned char *sha1, int flag, void *cb_data)
302{
303 struct object *o = parse_object(sha1);
304 while (o && o->type == OBJ_TAG) {
306 struct tag *t = (struct tag *) o;
307 if (!t->tagged)
308 break; /* broken repository */
309 o->flags |= COMPLETE;
310 o = parse_object(t->tagged->sha1);
311 }
312 if (o && o->type == OBJ_COMMIT) {
313 struct commit *commit = (struct commit *)o;
314 commit->object.flags |= COMPLETE;
315 insert_by_date(commit, &complete);
316 }
317 return 0;
318}
319static void mark_recent_complete_commits(unsigned long cutoff)
321{
322 while (complete && cutoff <= complete->item->date) {
323 if (args.verbose)
324 fprintf(stderr, "Marking %s as complete\n",
325 sha1_to_hex(complete->item->object.sha1));
326 pop_most_recent_commit(&complete, COMPLETE);
327 }
328}
329static void filter_refs(struct ref **refs, int nr_match, char **match)
331{
332 struct ref **return_refs;
333 struct ref *newlist = NULL;
334 struct ref **newtail = &newlist;
335 struct ref *ref, *next;
336 struct ref *fastarray[32];
337 if (nr_match && !args.fetch_all) {
339 if (ARRAY_SIZE(fastarray) < nr_match)
340 return_refs = xcalloc(nr_match, sizeof(struct ref *));
341 else {
342 return_refs = fastarray;
343 memset(return_refs, 0, sizeof(struct ref *) * nr_match);
344 }
345 }
346 else
347 return_refs = NULL;
348 for (ref = *refs; ref; ref = next) {
350 next = ref->next;
351 if (!memcmp(ref->name, "refs/", 5) &&
352 check_ref_format(ref->name + 5))
353 ; /* trash */
354 else if (args.fetch_all &&
355 (!args.depth || prefixcmp(ref->name, "refs/tags/") )) {
356 *newtail = ref;
357 ref->next = NULL;
358 newtail = &ref->next;
359 continue;
360 }
361 else {
362 int order = path_match(ref->name, nr_match, match);
363 if (order) {
364 return_refs[order-1] = ref;
365 continue; /* we will link it later */
366 }
367 }
368 free(ref);
369 }
370 if (!args.fetch_all) {
372 int i;
373 for (i = 0; i < nr_match; i++) {
374 ref = return_refs[i];
375 if (ref) {
376 *newtail = ref;
377 ref->next = NULL;
378 newtail = &ref->next;
379 }
380 }
381 if (return_refs != fastarray)
382 free(return_refs);
383 }
384 *refs = newlist;
385}
386static int everything_local(struct ref **refs, int nr_match, char **match)
388{
389 struct ref *ref;
390 int retval;
391 unsigned long cutoff = 0;
392 save_commit_buffer = 0;
394 for (ref = *refs; ref; ref = ref->next) {
396 struct object *o;
397 o = parse_object(ref->old_sha1);
399 if (!o)
400 continue;
401 /* We already have it -- which may mean that we were
403 * in sync with the other side at some time after
404 * that (it is OK if we guess wrong here).
405 */
406 if (o->type == OBJ_COMMIT) {
407 struct commit *commit = (struct commit *)o;
408 if (!cutoff || cutoff < commit->date)
409 cutoff = commit->date;
410 }
411 }
412 if (!args.depth) {
414 for_each_ref(mark_complete, NULL);
415 if (cutoff)
416 mark_recent_complete_commits(cutoff);
417 }
418 /*
420 * Mark all complete remote refs as common refs.
421 * Don't mark them common yet; the server has to be told so first.
422 */
423 for (ref = *refs; ref; ref = ref->next) {
424 struct object *o = deref_tag(lookup_object(ref->old_sha1),
425 NULL, 0);
426 if (!o || o->type != OBJ_COMMIT || !(o->flags & COMPLETE))
428 continue;
429 if (!(o->flags & SEEN)) {
431 rev_list_push((struct commit *)o, COMMON_REF | SEEN);
432 mark_common((struct commit *)o, 1, 1);
434 }
435 }
436 filter_refs(refs, nr_match, match);
438 for (retval = 1, ref = *refs; ref ; ref = ref->next) {
440 const unsigned char *remote = ref->old_sha1;
441 unsigned char local[20];
442 struct object *o;
443 o = lookup_object(remote);
445 if (!o || !(o->flags & COMPLETE)) {
446 retval = 0;
447 if (!args.verbose)
448 continue;
449 fprintf(stderr,
450 "want %s (%s)\n", sha1_to_hex(remote),
451 ref->name);
452 continue;
453 }
454 hashcpy(ref->new_sha1, local);
456 if (!args.verbose)
457 continue;
458 fprintf(stderr,
459 "already have %s (%s)\n", sha1_to_hex(remote),
460 ref->name);
461 }
462 return retval;
463}
464static int sideband_demux(int fd, void *data)
466{
467 int *xd = data;
468 return recv_sideband("fetch-pack", xd[0], fd, 2);
470}
471static int get_pack(int xd[2], char **pack_lockfile)
473{
474 struct async demux;
475 const char *argv[20];
476 char keep_arg[256];
477 char hdr_arg[256];
478 const char **av;
479 int do_keep = args.keep_pack;
480 struct child_process cmd;
481 memset(&demux, 0, sizeof(demux));
483 if (use_sideband) {
484 /* xd[] is talking with upload-pack; subprocess reads from
485 * xd[0], spits out band#2 to stderr, and feeds us band#1
486 * through demux->out.
487 */
488 demux.proc = sideband_demux;
489 demux.data = xd;
490 if (start_async(&demux))
491 die("fetch-pack: unable to fork off sideband"
492 " demultiplexer");
493 }
494 else
495 demux.out = xd[0];
496 memset(&cmd, 0, sizeof(cmd));
498 cmd.argv = argv;
499 av = argv;
500 *hdr_arg = 0;
501 if (!args.keep_pack && unpack_limit) {
502 struct pack_header header;
503 if (read_pack_header(demux.out, &header))
505 die("protocol error: bad pack header");
506 snprintf(hdr_arg, sizeof(hdr_arg), "--pack_header=%u,%u",
507 ntohl(header.hdr_version), ntohl(header.hdr_entries));
508 if (ntohl(header.hdr_entries) < unpack_limit)
509 do_keep = 0;
510 else
511 do_keep = 1;
512 }
513 if (do_keep) {
515 if (pack_lockfile)
516 cmd.out = -1;
517 *av++ = "index-pack";
518 *av++ = "--stdin";
519 if (!args.quiet && !args.no_progress)
520 *av++ = "-v";
521 if (args.use_thin_pack)
522 *av++ = "--fix-thin";
523 if (args.lock_pack || unpack_limit) {
524 int s = sprintf(keep_arg,
525 "--keep=fetch-pack %d on ", getpid());
526 if (gethostname(keep_arg + s, sizeof(keep_arg) - s))
527 strcpy(keep_arg + s, "localhost");
528 *av++ = keep_arg;
529 }
530 }
531 else {
532 *av++ = "unpack-objects";
533 if (args.quiet)
534 *av++ = "-q";
535 }
536 if (*hdr_arg)
537 *av++ = hdr_arg;
538 *av++ = NULL;
539 cmd.in = demux.out;
541 cmd.git_cmd = 1;
542 if (start_command(&cmd))
543 die("fetch-pack: unable to fork off %s", argv[0]);
544 if (do_keep && pack_lockfile) {
545 *pack_lockfile = index_pack_lockfile(cmd.out);
546 close(cmd.out);
547 }
548 if (finish_command(&cmd))
550 die("%s failed", argv[0]);
551 if (use_sideband && finish_async(&demux))
552 die("error in sideband demultiplexer");
553 return 0;
554}
555static struct ref *do_fetch_pack(int fd[2],
557 const struct ref *orig_ref,
558 int nr_match,
559 char **match,
560 char **pack_lockfile)
561{
562 struct ref *ref = copy_ref_list(orig_ref);
563 unsigned char sha1[20];
564 if (is_repository_shallow() && !server_supports("shallow"))
566 die("Server does not support shallow clients");
567 if (server_supports("multi_ack")) {
568 if (args.verbose)
569 fprintf(stderr, "Server supports multi_ack\n");
570 multi_ack = 1;
571 }
572 if (server_supports("side-band-64k")) {
573 if (args.verbose)
574 fprintf(stderr, "Server supports side-band-64k\n");
575 use_sideband = 2;
576 }
577 else if (server_supports("side-band")) {
578 if (args.verbose)
579 fprintf(stderr, "Server supports side-band\n");
580 use_sideband = 1;
581 }
582 if (everything_local(&ref, nr_match, match)) {
583 packet_flush(fd[1]);
584 goto all_done;
585 }
586 if (find_common(fd, sha1, ref) < 0)
587 if (!args.keep_pack)
588 /* When cloning, it is not unusual to have
589 * no common commit.
590 */
591 fprintf(stderr, "warning: no common commits\n");
592 if (get_pack(fd, pack_lockfile))
594 die("git-fetch-pack: fetch failed.");
595 all_done:
597 return ref;
598}
599static int remove_duplicates(int nr_heads, char **heads)
601{
602 int src, dst;
603 for (src = dst = 0; src < nr_heads; src++) {
605 /* If heads[src] is different from any of
606 * heads[0..dst], push it in.
607 */
608 int i;
609 for (i = 0; i < dst; i++) {
610 if (!strcmp(heads[i], heads[src]))
611 break;
612 }
613 if (i < dst)
614 continue;
615 if (src != dst)
616 heads[dst] = heads[src];
617 dst++;
618 }
619 return dst;
620}
621static int fetch_pack_config(const char *var, const char *value)
623{
624 if (strcmp(var, "fetch.unpacklimit") == 0) {
625 fetch_unpack_limit = git_config_int(var, value);
626 return 0;
627 }
628 if (strcmp(var, "transfer.unpacklimit") == 0) {
630 transfer_unpack_limit = git_config_int(var, value);
631 return 0;
632 }
633 return git_default_config(var, value);
635}
636static struct lock_file lock;
638static void fetch_pack_setup(void)
640{
641 static int did_setup;
642 if (did_setup)
643 return;
644 git_config(fetch_pack_config);
645 if (0 <= transfer_unpack_limit)
646 unpack_limit = transfer_unpack_limit;
647 else if (0 <= fetch_unpack_limit)
648 unpack_limit = fetch_unpack_limit;
649 did_setup = 1;
650}
651int cmd_fetch_pack(int argc, const char **argv, const char *prefix)
653{
654 int i, ret, nr_heads;
655 struct ref *ref = NULL;
656 char *dest = NULL, **heads;
657 int fd[2];
658 struct child_process *conn;
659 nr_heads = 0;
661 heads = NULL;
662 for (i = 1; i < argc; i++) {
663 const char *arg = argv[i];
664 if (*arg == '-') {
666 if (!prefixcmp(arg, "--upload-pack=")) {
667 args.uploadpack = arg + 14;
668 continue;
669 }
670 if (!prefixcmp(arg, "--exec=")) {
671 args.uploadpack = arg + 7;
672 continue;
673 }
674 if (!strcmp("--quiet", arg) || !strcmp("-q", arg)) {
675 args.quiet = 1;
676 continue;
677 }
678 if (!strcmp("--keep", arg) || !strcmp("-k", arg)) {
679 args.lock_pack = args.keep_pack;
680 args.keep_pack = 1;
681 continue;
682 }
683 if (!strcmp("--thin", arg)) {
684 args.use_thin_pack = 1;
685 continue;
686 }
687 if (!strcmp("--include-tag", arg)) {
688 args.include_tag = 1;
689 continue;
690 }
691 if (!strcmp("--all", arg)) {
692 args.fetch_all = 1;
693 continue;
694 }
695 if (!strcmp("-v", arg)) {
696 args.verbose = 1;
697 continue;
698 }
699 if (!prefixcmp(arg, "--depth=")) {
700 args.depth = strtol(arg + 8, NULL, 0);
701 continue;
702 }
703 if (!strcmp("--no-progress", arg)) {
704 args.no_progress = 1;
705 continue;
706 }
707 usage(fetch_pack_usage);
708 }
709 dest = (char *)arg;
710 heads = (char **)(argv + i + 1);
711 nr_heads = argc - i - 1;
712 break;
713 }
714 if (!dest)
715 usage(fetch_pack_usage);
716 conn = git_connect(fd, (char *)dest, args.uploadpack,
718 args.verbose ? CONNECT_VERBOSE : 0);
719 if (conn) {
720 get_remote_heads(fd[0], &ref, 0, NULL, 0);
721 ref = fetch_pack(&args, fd, conn, ref, dest, nr_heads, heads, NULL);
723 close(fd[0]);
724 close(fd[1]);
725 if (finish_connect(conn))
726 ref = NULL;
727 } else {
728 ref = NULL;
729 }
730 ret = !ref;
731 if (!ret && nr_heads) {
733 /* If the heads to pull were given, we should have
734 * consumed all of them by matching the remote.
735 * Otherwise, 'git-fetch remote no-such-ref' would
736 * silently succeed without issuing an error.
737 */
738 for (i = 0; i < nr_heads; i++)
739 if (heads[i] && heads[i][0]) {
740 error("no such remote ref %s", heads[i]);
741 ret = 1;
742 }
743 }
744 while (ref) {
745 printf("%s %s\n",
746 sha1_to_hex(ref->old_sha1), ref->name);
747 ref = ref->next;
748 }
749 return ret;
751}
752struct ref *fetch_pack(struct fetch_pack_args *my_args,
754 int fd[], struct child_process *conn,
755 const struct ref *ref,
756 const char *dest,
757 int nr_heads,
758 char **heads,
759 char **pack_lockfile)
760{
761 struct stat st;
762 struct ref *ref_cpy;
763 fetch_pack_setup();
765 memcpy(&args, my_args, sizeof(args));
766 if (args.depth > 0) {
767 if (stat(git_path("shallow"), &st))
768 st.st_mtime = 0;
769 }
770 if (heads && nr_heads)
772 nr_heads = remove_duplicates(nr_heads, heads);
773 if (!ref) {
774 packet_flush(fd[1]);
775 die("no matching remote head");
776 }
777 ref_cpy = do_fetch_pack(fd, ref, nr_heads, heads, pack_lockfile);
778 if (args.depth > 0) {
780 struct cache_time mtime;
781 char *shallow = git_path("shallow");
782 int fd;
783 mtime.sec = st.st_mtime;
785#ifdef USE_NSEC
786 mtime.usec = st.st_mtim.usec;
787#endif
788 if (stat(shallow, &st)) {
789 if (mtime.sec)
790 die("shallow file was removed during fetch");
791 } else if (st.st_mtime != mtime.sec
792#ifdef USE_NSEC
793 || st.st_mtim.usec != mtime.usec
794#endif
795 )
796 die("shallow file was changed during fetch");
797 fd = hold_lock_file_for_update(&lock, shallow, 1);
799 if (!write_shallow_commits(fd, 0)) {
800 unlink(shallow);
801 rollback_lock_file(&lock);
802 } else {
803 commit_lock_file(&lock);
804 }
805 }
806 return ref_cpy;
808}