1#include "cache.h"
2#include "refs.h"
3#include "pkt-line.h"
4#include "commit.h"
5#include "tag.h"
6#include "exec_cmd.h"
7#include "pack.h"
8#include "sideband.h"
9#include "fetch-pack.h"
10#include "remote.h"
11#include "run-command.h"
12
13static int transfer_unpack_limit = -1;
14static int fetch_unpack_limit = -1;
15static int unpack_limit = 100;
16static struct fetch_pack_args args = {
17 /* .uploadpack = */ "git-upload-pack",
18};
19
20static const char fetch_pack_usage[] =
21"git-fetch-pack [--all] [--quiet|-q] [--keep|-k] [--thin] [--upload-pack=<git-upload-pack>] [--depth=<n>] [--no-progress] [-v] [<host>:]<directory> [<refs>...]";
22
23#define COMPLETE (1U << 0)
24#define COMMON (1U << 1)
25#define COMMON_REF (1U << 2)
26#define SEEN (1U << 3)
27#define POPPED (1U << 4)
28
29/*
30 * After sending this many "have"s if we do not get any new ACK , we
31 * give up traversing our history.
32 */
33#define MAX_IN_VAIN 256
34
35static struct commit_list *rev_list;
36static int non_common_revs, multi_ack, use_sideband;
37
38static void rev_list_push(struct commit *commit, int mark)
39{
40 if (!(commit->object.flags & mark)) {
41 commit->object.flags |= mark;
42
43 if (!(commit->object.parsed))
44 if (parse_commit(commit))
45 return;
46
47 insert_by_date(commit, &rev_list);
48
49 if (!(commit->object.flags & COMMON))
50 non_common_revs++;
51 }
52}
53
54static int rev_list_insert_ref(const char *path, const unsigned char *sha1, int flag, void *cb_data)
55{
56 struct object *o = deref_tag(parse_object(sha1), path, 0);
57
58 if (o && o->type == OBJ_COMMIT)
59 rev_list_push((struct commit *)o, SEEN);
60
61 return 0;
62}
63
64/*
65 This function marks a rev and its ancestors as common.
66 In some cases, it is desirable to mark only the ancestors (for example
67 when only the server does not yet know that they are common).
68*/
69
70static void mark_common(struct commit *commit,
71 int ancestors_only, int dont_parse)
72{
73 if (commit != NULL && !(commit->object.flags & COMMON)) {
74 struct object *o = (struct object *)commit;
75
76 if (!ancestors_only)
77 o->flags |= COMMON;
78
79 if (!(o->flags & SEEN))
80 rev_list_push(commit, SEEN);
81 else {
82 struct commit_list *parents;
83
84 if (!ancestors_only && !(o->flags & POPPED))
85 non_common_revs--;
86 if (!o->parsed && !dont_parse)
87 if (parse_commit(commit))
88 return;
89
90 for (parents = commit->parents;
91 parents;
92 parents = parents->next)
93 mark_common(parents->item, 0, dont_parse);
94 }
95 }
96}
97
98/*
99 Get the next rev to send, ignoring the common.
100*/
101
102static const unsigned char* get_rev(void)
103{
104 struct commit *commit = NULL;
105
106 while (commit == NULL) {
107 unsigned int mark;
108 struct commit_list *parents = NULL;
109
110 if (rev_list == NULL || non_common_revs == 0)
111 return NULL;
112
113 commit = rev_list->item;
114 if (!(commit->object.parsed))
115 if (!parse_commit(commit))
116 parents = commit->parents;
117
118 commit->object.flags |= POPPED;
119 if (!(commit->object.flags & COMMON))
120 non_common_revs--;
121
122 if (commit->object.flags & COMMON) {
123 /* do not send "have", and ignore ancestors */
124 commit = NULL;
125 mark = COMMON | SEEN;
126 } else if (commit->object.flags & COMMON_REF)
127 /* send "have", and ignore ancestors */
128 mark = COMMON | SEEN;
129 else
130 /* send "have", also for its ancestors */
131 mark = SEEN;
132
133 while (parents) {
134 if (!(parents->item->object.flags & SEEN))
135 rev_list_push(parents->item, mark);
136 if (mark & COMMON)
137 mark_common(parents->item, 1, 0);
138 parents = parents->next;
139 }
140
141 rev_list = rev_list->next;
142 }
143
144 return commit->object.sha1;
145}
146
147static int find_common(int fd[2], unsigned char *result_sha1,
148 struct ref *refs)
149{
150 int fetching;
151 int count = 0, flushes = 0, retval;
152 const unsigned char *sha1;
153 unsigned in_vain = 0;
154 int got_continue = 0;
155
156 for_each_ref(rev_list_insert_ref, NULL);
157
158 fetching = 0;
159 for ( ; refs ; refs = refs->next) {
160 unsigned char *remote = refs->old_sha1;
161 struct object *o;
162
163 /*
164 * If that object is complete (i.e. it is an ancestor of a
165 * local ref), we tell them we have it but do not have to
166 * tell them about its ancestors, which they already know
167 * about.
168 *
169 * We use lookup_object here because we are only
170 * interested in the case we *know* the object is
171 * reachable and we have already scanned it.
172 */
173 if (((o = lookup_object(remote)) != NULL) &&
174 (o->flags & COMPLETE)) {
175 continue;
176 }
177
178 if (!fetching)
179 packet_write(fd[1], "want %s%s%s%s%s%s%s\n",
180 sha1_to_hex(remote),
181 (multi_ack ? " multi_ack" : ""),
182 (use_sideband == 2 ? " side-band-64k" : ""),
183 (use_sideband == 1 ? " side-band" : ""),
184 (args.use_thin_pack ? " thin-pack" : ""),
185 (args.no_progress ? " no-progress" : ""),
186 " ofs-delta");
187 else
188 packet_write(fd[1], "want %s\n", sha1_to_hex(remote));
189 fetching++;
190 }
191 if (is_repository_shallow())
192 write_shallow_commits(fd[1], 1);
193 if (args.depth > 0)
194 packet_write(fd[1], "deepen %d", args.depth);
195 packet_flush(fd[1]);
196 if (!fetching)
197 return 1;
198
199 if (args.depth > 0) {
200 char line[1024];
201 unsigned char sha1[20];
202 int len;
203
204 while ((len = packet_read_line(fd[0], line, sizeof(line)))) {
205 if (!prefixcmp(line, "shallow ")) {
206 if (get_sha1_hex(line + 8, sha1))
207 die("invalid shallow line: %s", line);
208 register_shallow(sha1);
209 continue;
210 }
211 if (!prefixcmp(line, "unshallow ")) {
212 if (get_sha1_hex(line + 10, sha1))
213 die("invalid unshallow line: %s", line);
214 if (!lookup_object(sha1))
215 die("object not found: %s", line);
216 /* make sure that it is parsed as shallow */
217 if (!parse_object(sha1))
218 die("error in object: %s", line);
219 if (unregister_shallow(sha1))
220 die("no shallow found: %s", line);
221 continue;
222 }
223 die("expected shallow/unshallow, got %s", line);
224 }
225 }
226
227 flushes = 0;
228 retval = -1;
229 while ((sha1 = get_rev())) {
230 packet_write(fd[1], "have %s\n", sha1_to_hex(sha1));
231 if (args.verbose)
232 fprintf(stderr, "have %s\n", sha1_to_hex(sha1));
233 in_vain++;
234 if (!(31 & ++count)) {
235 int ack;
236
237 packet_flush(fd[1]);
238 flushes++;
239
240 /*
241 * We keep one window "ahead" of the other side, and
242 * will wait for an ACK only on the next one
243 */
244 if (count == 32)
245 continue;
246
247 do {
248 ack = get_ack(fd[0], result_sha1);
249 if (args.verbose && ack)
250 fprintf(stderr, "got ack %d %s\n", ack,
251 sha1_to_hex(result_sha1));
252 if (ack == 1) {
253 flushes = 0;
254 multi_ack = 0;
255 retval = 0;
256 goto done;
257 } else if (ack == 2) {
258 struct commit *commit =
259 lookup_commit(result_sha1);
260 mark_common(commit, 0, 1);
261 retval = 0;
262 in_vain = 0;
263 got_continue = 1;
264 }
265 } while (ack);
266 flushes--;
267 if (got_continue && MAX_IN_VAIN < in_vain) {
268 if (args.verbose)
269 fprintf(stderr, "giving up\n");
270 break; /* give up */
271 }
272 }
273 }
274done:
275 packet_write(fd[1], "done\n");
276 if (args.verbose)
277 fprintf(stderr, "done\n");
278 if (retval != 0) {
279 multi_ack = 0;
280 flushes++;
281 }
282 while (flushes || multi_ack) {
283 int ack = get_ack(fd[0], result_sha1);
284 if (ack) {
285 if (args.verbose)
286 fprintf(stderr, "got ack (%d) %s\n", ack,
287 sha1_to_hex(result_sha1));
288 if (ack == 1)
289 return 0;
290 multi_ack = 1;
291 continue;
292 }
293 flushes--;
294 }
295 return retval;
296}
297
298static struct commit_list *complete;
299
300static int mark_complete(const char *path, const unsigned char *sha1, int flag, void *cb_data)
301{
302 struct object *o = parse_object(sha1);
303
304 while (o && o->type == OBJ_TAG) {
305 struct tag *t = (struct tag *) o;
306 if (!t->tagged)
307 break; /* broken repository */
308 o->flags |= COMPLETE;
309 o = parse_object(t->tagged->sha1);
310 }
311 if (o && o->type == OBJ_COMMIT) {
312 struct commit *commit = (struct commit *)o;
313 commit->object.flags |= COMPLETE;
314 insert_by_date(commit, &complete);
315 }
316 return 0;
317}
318
319static void mark_recent_complete_commits(unsigned long cutoff)
320{
321 while (complete && cutoff <= complete->item->date) {
322 if (args.verbose)
323 fprintf(stderr, "Marking %s as complete\n",
324 sha1_to_hex(complete->item->object.sha1));
325 pop_most_recent_commit(&complete, COMPLETE);
326 }
327}
328
329static void filter_refs(struct ref **refs, int nr_match, char **match)
330{
331 struct ref **return_refs;
332 struct ref *newlist = NULL;
333 struct ref **newtail = &newlist;
334 struct ref *ref, *next;
335 struct ref *fastarray[32];
336
337 if (nr_match && !args.fetch_all) {
338 if (ARRAY_SIZE(fastarray) < nr_match)
339 return_refs = xcalloc(nr_match, sizeof(struct ref *));
340 else {
341 return_refs = fastarray;
342 memset(return_refs, 0, sizeof(struct ref *) * nr_match);
343 }
344 }
345 else
346 return_refs = NULL;
347
348 for (ref = *refs; ref; ref = next) {
349 next = ref->next;
350 if (!memcmp(ref->name, "refs/", 5) &&
351 check_ref_format(ref->name + 5))
352 ; /* trash */
353 else if (args.fetch_all &&
354 (!args.depth || prefixcmp(ref->name, "refs/tags/") )) {
355 *newtail = ref;
356 ref->next = NULL;
357 newtail = &ref->next;
358 continue;
359 }
360 else {
361 int order = path_match(ref->name, nr_match, match);
362 if (order) {
363 return_refs[order-1] = ref;
364 continue; /* we will link it later */
365 }
366 }
367 free(ref);
368 }
369
370 if (!args.fetch_all) {
371 int i;
372 for (i = 0; i < nr_match; i++) {
373 ref = return_refs[i];
374 if (ref) {
375 *newtail = ref;
376 ref->next = NULL;
377 newtail = &ref->next;
378 }
379 }
380 if (return_refs != fastarray)
381 free(return_refs);
382 }
383 *refs = newlist;
384}
385
386static int everything_local(struct ref **refs, int nr_match, char **match)
387{
388 struct ref *ref;
389 int retval;
390 unsigned long cutoff = 0;
391
392 save_commit_buffer = 0;
393
394 for (ref = *refs; ref; ref = ref->next) {
395 struct object *o;
396
397 o = parse_object(ref->old_sha1);
398 if (!o)
399 continue;
400
401 /* We already have it -- which may mean that we were
402 * in sync with the other side at some time after
403 * that (it is OK if we guess wrong here).
404 */
405 if (o->type == OBJ_COMMIT) {
406 struct commit *commit = (struct commit *)o;
407 if (!cutoff || cutoff < commit->date)
408 cutoff = commit->date;
409 }
410 }
411
412 if (!args.depth) {
413 for_each_ref(mark_complete, NULL);
414 if (cutoff)
415 mark_recent_complete_commits(cutoff);
416 }
417
418 /*
419 * Mark all complete remote refs as common refs.
420 * Don't mark them common yet; the server has to be told so first.
421 */
422 for (ref = *refs; ref; ref = ref->next) {
423 struct object *o = deref_tag(lookup_object(ref->old_sha1),
424 NULL, 0);
425
426 if (!o || o->type != OBJ_COMMIT || !(o->flags & COMPLETE))
427 continue;
428
429 if (!(o->flags & SEEN)) {
430 rev_list_push((struct commit *)o, COMMON_REF | SEEN);
431
432 mark_common((struct commit *)o, 1, 1);
433 }
434 }
435
436 filter_refs(refs, nr_match, match);
437
438 for (retval = 1, ref = *refs; ref ; ref = ref->next) {
439 const unsigned char *remote = ref->old_sha1;
440 unsigned char local[20];
441 struct object *o;
442
443 o = lookup_object(remote);
444 if (!o || !(o->flags & COMPLETE)) {
445 retval = 0;
446 if (!args.verbose)
447 continue;
448 fprintf(stderr,
449 "want %s (%s)\n", sha1_to_hex(remote),
450 ref->name);
451 continue;
452 }
453
454 hashcpy(ref->new_sha1, local);
455 if (!args.verbose)
456 continue;
457 fprintf(stderr,
458 "already have %s (%s)\n", sha1_to_hex(remote),
459 ref->name);
460 }
461 return retval;
462}
463
464static int sideband_demux(int fd, void *data)
465{
466 int *xd = data;
467
468 return recv_sideband("fetch-pack", xd[0], fd, 2);
469}
470
471static int get_pack(int xd[2], char **pack_lockfile)
472{
473 struct async demux;
474 const char *argv[20];
475 char keep_arg[256];
476 char hdr_arg[256];
477 const char **av;
478 int do_keep = args.keep_pack;
479 struct child_process cmd;
480
481 memset(&demux, 0, sizeof(demux));
482 if (use_sideband) {
483 /* xd[] is talking with upload-pack; subprocess reads from
484 * xd[0], spits out band#2 to stderr, and feeds us band#1
485 * through demux->out.
486 */
487 demux.proc = sideband_demux;
488 demux.data = xd;
489 if (start_async(&demux))
490 die("fetch-pack: unable to fork off sideband"
491 " demultiplexer");
492 }
493 else
494 demux.out = xd[0];
495
496 memset(&cmd, 0, sizeof(cmd));
497 cmd.argv = argv;
498 av = argv;
499 *hdr_arg = 0;
500 if (!args.keep_pack && unpack_limit) {
501 struct pack_header header;
502
503 if (read_pack_header(demux.out, &header))
504 die("protocol error: bad pack header");
505 snprintf(hdr_arg, sizeof(hdr_arg), "--pack_header=%u,%u",
506 ntohl(header.hdr_version), ntohl(header.hdr_entries));
507 if (ntohl(header.hdr_entries) < unpack_limit)
508 do_keep = 0;
509 else
510 do_keep = 1;
511 }
512
513 if (do_keep) {
514 if (pack_lockfile)
515 cmd.out = -1;
516 *av++ = "index-pack";
517 *av++ = "--stdin";
518 if (!args.quiet && !args.no_progress)
519 *av++ = "-v";
520 if (args.use_thin_pack)
521 *av++ = "--fix-thin";
522 if (args.lock_pack || unpack_limit) {
523 int s = sprintf(keep_arg,
524 "--keep=fetch-pack %d on ", getpid());
525 if (gethostname(keep_arg + s, sizeof(keep_arg) - s))
526 strcpy(keep_arg + s, "localhost");
527 *av++ = keep_arg;
528 }
529 }
530 else {
531 *av++ = "unpack-objects";
532 if (args.quiet)
533 *av++ = "-q";
534 }
535 if (*hdr_arg)
536 *av++ = hdr_arg;
537 *av++ = NULL;
538
539 cmd.in = demux.out;
540 cmd.git_cmd = 1;
541 if (start_command(&cmd))
542 die("fetch-pack: unable to fork off %s", argv[0]);
543 if (do_keep && pack_lockfile) {
544 *pack_lockfile = index_pack_lockfile(cmd.out);
545 close(cmd.out);
546 }
547
548 if (finish_command(&cmd))
549 die("%s failed", argv[0]);
550 if (use_sideband && finish_async(&demux))
551 die("error in sideband demultiplexer");
552 return 0;
553}
554
555static struct ref *do_fetch_pack(int fd[2],
556 const struct ref *orig_ref,
557 int nr_match,
558 char **match,
559 char **pack_lockfile)
560{
561 struct ref *ref = copy_ref_list(orig_ref);
562 unsigned char sha1[20];
563
564 if (is_repository_shallow() && !server_supports("shallow"))
565 die("Server does not support shallow clients");
566 if (server_supports("multi_ack")) {
567 if (args.verbose)
568 fprintf(stderr, "Server supports multi_ack\n");
569 multi_ack = 1;
570 }
571 if (server_supports("side-band-64k")) {
572 if (args.verbose)
573 fprintf(stderr, "Server supports side-band-64k\n");
574 use_sideband = 2;
575 }
576 else if (server_supports("side-band")) {
577 if (args.verbose)
578 fprintf(stderr, "Server supports side-band\n");
579 use_sideband = 1;
580 }
581 if (everything_local(&ref, nr_match, match)) {
582 packet_flush(fd[1]);
583 goto all_done;
584 }
585 if (find_common(fd, sha1, ref) < 0)
586 if (!args.keep_pack)
587 /* When cloning, it is not unusual to have
588 * no common commit.
589 */
590 fprintf(stderr, "warning: no common commits\n");
591
592 if (get_pack(fd, pack_lockfile))
593 die("git-fetch-pack: fetch failed.");
594
595 all_done:
596 return ref;
597}
598
599static int remove_duplicates(int nr_heads, char **heads)
600{
601 int src, dst;
602
603 for (src = dst = 0; src < nr_heads; src++) {
604 /* If heads[src] is different from any of
605 * heads[0..dst], push it in.
606 */
607 int i;
608 for (i = 0; i < dst; i++) {
609 if (!strcmp(heads[i], heads[src]))
610 break;
611 }
612 if (i < dst)
613 continue;
614 if (src != dst)
615 heads[dst] = heads[src];
616 dst++;
617 }
618 return dst;
619}
620
621static int fetch_pack_config(const char *var, const char *value)
622{
623 if (strcmp(var, "fetch.unpacklimit") == 0) {
624 fetch_unpack_limit = git_config_int(var, value);
625 return 0;
626 }
627
628 if (strcmp(var, "transfer.unpacklimit") == 0) {
629 transfer_unpack_limit = git_config_int(var, value);
630 return 0;
631 }
632
633 return git_default_config(var, value);
634}
635
636static struct lock_file lock;
637
638static void fetch_pack_setup(void)
639{
640 static int did_setup;
641 if (did_setup)
642 return;
643 git_config(fetch_pack_config);
644 if (0 <= transfer_unpack_limit)
645 unpack_limit = transfer_unpack_limit;
646 else if (0 <= fetch_unpack_limit)
647 unpack_limit = fetch_unpack_limit;
648 did_setup = 1;
649}
650
651int cmd_fetch_pack(int argc, const char **argv, const char *prefix)
652{
653 int i, ret, nr_heads;
654 struct ref *ref = NULL;
655 char *dest = NULL, **heads;
656 int fd[2];
657 struct child_process *conn;
658
659 nr_heads = 0;
660 heads = NULL;
661 for (i = 1; i < argc; i++) {
662 const char *arg = argv[i];
663
664 if (*arg == '-') {
665 if (!prefixcmp(arg, "--upload-pack=")) {
666 args.uploadpack = arg + 14;
667 continue;
668 }
669 if (!prefixcmp(arg, "--exec=")) {
670 args.uploadpack = arg + 7;
671 continue;
672 }
673 if (!strcmp("--quiet", arg) || !strcmp("-q", arg)) {
674 args.quiet = 1;
675 continue;
676 }
677 if (!strcmp("--keep", arg) || !strcmp("-k", arg)) {
678 args.lock_pack = args.keep_pack;
679 args.keep_pack = 1;
680 continue;
681 }
682 if (!strcmp("--thin", arg)) {
683 args.use_thin_pack = 1;
684 continue;
685 }
686 if (!strcmp("--all", arg)) {
687 args.fetch_all = 1;
688 continue;
689 }
690 if (!strcmp("-v", arg)) {
691 args.verbose = 1;
692 continue;
693 }
694 if (!prefixcmp(arg, "--depth=")) {
695 args.depth = strtol(arg + 8, NULL, 0);
696 continue;
697 }
698 if (!strcmp("--no-progress", arg)) {
699 args.no_progress = 1;
700 continue;
701 }
702 usage(fetch_pack_usage);
703 }
704 dest = (char *)arg;
705 heads = (char **)(argv + i + 1);
706 nr_heads = argc - i - 1;
707 break;
708 }
709 if (!dest)
710 usage(fetch_pack_usage);
711
712 conn = git_connect(fd, (char *)dest, args.uploadpack,
713 args.verbose ? CONNECT_VERBOSE : 0);
714 if (conn) {
715 get_remote_heads(fd[0], &ref, 0, NULL, 0);
716
717 ref = fetch_pack(&args, fd, conn, ref, dest, nr_heads, heads, NULL);
718 close(fd[0]);
719 close(fd[1]);
720 if (finish_connect(conn))
721 ref = NULL;
722 } else {
723 ref = NULL;
724 }
725 ret = !ref;
726
727 if (!ret && nr_heads) {
728 /* If the heads to pull were given, we should have
729 * consumed all of them by matching the remote.
730 * Otherwise, 'git-fetch remote no-such-ref' would
731 * silently succeed without issuing an error.
732 */
733 for (i = 0; i < nr_heads; i++)
734 if (heads[i] && heads[i][0]) {
735 error("no such remote ref %s", heads[i]);
736 ret = 1;
737 }
738 }
739 while (ref) {
740 printf("%s %s\n",
741 sha1_to_hex(ref->old_sha1), ref->name);
742 ref = ref->next;
743 }
744
745 return ret;
746}
747
748struct ref *fetch_pack(struct fetch_pack_args *my_args,
749 int fd[], struct child_process *conn,
750 const struct ref *ref,
751 const char *dest,
752 int nr_heads,
753 char **heads,
754 char **pack_lockfile)
755{
756 struct stat st;
757 struct ref *ref_cpy;
758
759 fetch_pack_setup();
760 memcpy(&args, my_args, sizeof(args));
761 if (args.depth > 0) {
762 if (stat(git_path("shallow"), &st))
763 st.st_mtime = 0;
764 }
765
766 if (heads && nr_heads)
767 nr_heads = remove_duplicates(nr_heads, heads);
768 if (!ref) {
769 packet_flush(fd[1]);
770 die("no matching remote head");
771 }
772 ref_cpy = do_fetch_pack(fd, ref, nr_heads, heads, pack_lockfile);
773
774 if (args.depth > 0) {
775 struct cache_time mtime;
776 char *shallow = git_path("shallow");
777 int fd;
778
779 mtime.sec = st.st_mtime;
780#ifdef USE_NSEC
781 mtime.usec = st.st_mtim.usec;
782#endif
783 if (stat(shallow, &st)) {
784 if (mtime.sec)
785 die("shallow file was removed during fetch");
786 } else if (st.st_mtime != mtime.sec
787#ifdef USE_NSEC
788 || st.st_mtim.usec != mtime.usec
789#endif
790 )
791 die("shallow file was changed during fetch");
792
793 fd = hold_lock_file_for_update(&lock, shallow, 1);
794 if (!write_shallow_commits(fd, 0)) {
795 unlink(shallow);
796 rollback_lock_file(&lock);
797 } else {
798 commit_lock_file(&lock);
799 }
800 }
801
802 return ref_cpy;
803}