1#include "cache.h"
2#include "refs.h"
3#include "pkt-line.h"
4#include "commit.h"
5#include "tag.h"
6#include "exec_cmd.h"
7#include "pack.h"
8#include "sideband.h"
9
10static int keep_pack;
11static int unpack_limit = 100;
12static int quiet;
13static int verbose;
14static int fetch_all;
15static int depth;
16static const char fetch_pack_usage[] =
17"git-fetch-pack [--all] [--quiet|-q] [--keep|-k] [--thin] [--upload-pack=<git-upload-pack>] [--depth=<n>] [-v] [<host>:]<directory> [<refs>...]";
18static const char *uploadpack = "git-upload-pack";
19
20#define COMPLETE (1U << 0)
21#define COMMON (1U << 1)
22#define COMMON_REF (1U << 2)
23#define SEEN (1U << 3)
24#define POPPED (1U << 4)
25
26/*
27 * After sending this many "have"s if we do not get any new ACK , we
28 * give up traversing our history.
29 */
30#define MAX_IN_VAIN 256
31
32static struct commit_list *rev_list;
33static int non_common_revs, multi_ack, use_thin_pack, use_sideband;
34
35static void rev_list_push(struct commit *commit, int mark)
36{
37 if (!(commit->object.flags & mark)) {
38 commit->object.flags |= mark;
39
40 if (!(commit->object.parsed))
41 parse_commit(commit);
42
43 insert_by_date(commit, &rev_list);
44
45 if (!(commit->object.flags & COMMON))
46 non_common_revs++;
47 }
48}
49
50static int rev_list_insert_ref(const char *path, const unsigned char *sha1, int flag, void *cb_data)
51{
52 struct object *o = deref_tag(parse_object(sha1), path, 0);
53
54 if (o && o->type == OBJ_COMMIT)
55 rev_list_push((struct commit *)o, SEEN);
56
57 return 0;
58}
59
60/*
61 This function marks a rev and its ancestors as common.
62 In some cases, it is desirable to mark only the ancestors (for example
63 when only the server does not yet know that they are common).
64*/
65
66static void mark_common(struct commit *commit,
67 int ancestors_only, int dont_parse)
68{
69 if (commit != NULL && !(commit->object.flags & COMMON)) {
70 struct object *o = (struct object *)commit;
71
72 if (!ancestors_only)
73 o->flags |= COMMON;
74
75 if (!(o->flags & SEEN))
76 rev_list_push(commit, SEEN);
77 else {
78 struct commit_list *parents;
79
80 if (!ancestors_only && !(o->flags & POPPED))
81 non_common_revs--;
82 if (!o->parsed && !dont_parse)
83 parse_commit(commit);
84
85 for (parents = commit->parents;
86 parents;
87 parents = parents->next)
88 mark_common(parents->item, 0, dont_parse);
89 }
90 }
91}
92
93/*
94 Get the next rev to send, ignoring the common.
95*/
96
97static const unsigned char* get_rev(void)
98{
99 struct commit *commit = NULL;
100
101 while (commit == NULL) {
102 unsigned int mark;
103 struct commit_list* parents;
104
105 if (rev_list == NULL || non_common_revs == 0)
106 return NULL;
107
108 commit = rev_list->item;
109 if (!(commit->object.parsed))
110 parse_commit(commit);
111 commit->object.flags |= POPPED;
112 if (!(commit->object.flags & COMMON))
113 non_common_revs--;
114
115 parents = commit->parents;
116
117 if (commit->object.flags & COMMON) {
118 /* do not send "have", and ignore ancestors */
119 commit = NULL;
120 mark = COMMON | SEEN;
121 } else if (commit->object.flags & COMMON_REF)
122 /* send "have", and ignore ancestors */
123 mark = COMMON | SEEN;
124 else
125 /* send "have", also for its ancestors */
126 mark = SEEN;
127
128 while (parents) {
129 if (!(parents->item->object.flags & SEEN))
130 rev_list_push(parents->item, mark);
131 if (mark & COMMON)
132 mark_common(parents->item, 1, 0);
133 parents = parents->next;
134 }
135
136 rev_list = rev_list->next;
137 }
138
139 return commit->object.sha1;
140}
141
142static int find_common(int fd[2], unsigned char *result_sha1,
143 struct ref *refs)
144{
145 int fetching;
146 int count = 0, flushes = 0, retval;
147 const unsigned char *sha1;
148 unsigned in_vain = 0;
149 int got_continue = 0;
150
151 for_each_ref(rev_list_insert_ref, NULL);
152
153 fetching = 0;
154 for ( ; refs ; refs = refs->next) {
155 unsigned char *remote = refs->old_sha1;
156 struct object *o;
157
158 /*
159 * If that object is complete (i.e. it is an ancestor of a
160 * local ref), we tell them we have it but do not have to
161 * tell them about its ancestors, which they already know
162 * about.
163 *
164 * We use lookup_object here because we are only
165 * interested in the case we *know* the object is
166 * reachable and we have already scanned it.
167 */
168 if (((o = lookup_object(remote)) != NULL) &&
169 (o->flags & COMPLETE)) {
170 continue;
171 }
172
173 if (!fetching)
174 packet_write(fd[1], "want %s%s%s%s%s%s\n",
175 sha1_to_hex(remote),
176 (multi_ack ? " multi_ack" : ""),
177 (use_sideband == 2 ? " side-band-64k" : ""),
178 (use_sideband == 1 ? " side-band" : ""),
179 (use_thin_pack ? " thin-pack" : ""),
180 " ofs-delta");
181 else
182 packet_write(fd[1], "want %s\n", sha1_to_hex(remote));
183 fetching++;
184 }
185 if (is_repository_shallow())
186 write_shallow_commits(fd[1], 1);
187 if (depth > 0)
188 packet_write(fd[1], "deepen %d", depth);
189 packet_flush(fd[1]);
190 if (!fetching)
191 return 1;
192
193 if (depth > 0) {
194 char line[1024];
195 unsigned char sha1[20];
196 int len;
197
198 while ((len = packet_read_line(fd[0], line, sizeof(line)))) {
199 if (!strncmp("shallow ", line, 8)) {
200 if (get_sha1_hex(line + 8, sha1))
201 die("invalid shallow line: %s", line);
202 register_shallow(sha1);
203 continue;
204 }
205 if (!strncmp("unshallow ", line, 10)) {
206 if (get_sha1_hex(line + 10, sha1))
207 die("invalid unshallow line: %s", line);
208 if (!lookup_object(sha1))
209 die("object not found: %s", line);
210 /* make sure that it is parsed as shallow */
211 parse_object(sha1);
212 if (unregister_shallow(sha1))
213 die("no shallow found: %s", line);
214 continue;
215 }
216 die("expected shallow/unshallow, got %s", line);
217 }
218 }
219
220 flushes = 0;
221 retval = -1;
222 while ((sha1 = get_rev())) {
223 packet_write(fd[1], "have %s\n", sha1_to_hex(sha1));
224 if (verbose)
225 fprintf(stderr, "have %s\n", sha1_to_hex(sha1));
226 in_vain++;
227 if (!(31 & ++count)) {
228 int ack;
229
230 packet_flush(fd[1]);
231 flushes++;
232
233 /*
234 * We keep one window "ahead" of the other side, and
235 * will wait for an ACK only on the next one
236 */
237 if (count == 32)
238 continue;
239
240 do {
241 ack = get_ack(fd[0], result_sha1);
242 if (verbose && ack)
243 fprintf(stderr, "got ack %d %s\n", ack,
244 sha1_to_hex(result_sha1));
245 if (ack == 1) {
246 flushes = 0;
247 multi_ack = 0;
248 retval = 0;
249 goto done;
250 } else if (ack == 2) {
251 struct commit *commit =
252 lookup_commit(result_sha1);
253 mark_common(commit, 0, 1);
254 retval = 0;
255 in_vain = 0;
256 got_continue = 1;
257 }
258 } while (ack);
259 flushes--;
260 if (got_continue && MAX_IN_VAIN < in_vain) {
261 if (verbose)
262 fprintf(stderr, "giving up\n");
263 break; /* give up */
264 }
265 }
266 }
267done:
268 packet_write(fd[1], "done\n");
269 if (verbose)
270 fprintf(stderr, "done\n");
271 if (retval != 0) {
272 multi_ack = 0;
273 flushes++;
274 }
275 while (flushes || multi_ack) {
276 int ack = get_ack(fd[0], result_sha1);
277 if (ack) {
278 if (verbose)
279 fprintf(stderr, "got ack (%d) %s\n", ack,
280 sha1_to_hex(result_sha1));
281 if (ack == 1)
282 return 0;
283 multi_ack = 1;
284 continue;
285 }
286 flushes--;
287 }
288 return retval;
289}
290
291static struct commit_list *complete;
292
293static int mark_complete(const char *path, const unsigned char *sha1, int flag, void *cb_data)
294{
295 struct object *o = parse_object(sha1);
296
297 while (o && o->type == OBJ_TAG) {
298 struct tag *t = (struct tag *) o;
299 if (!t->tagged)
300 break; /* broken repository */
301 o->flags |= COMPLETE;
302 o = parse_object(t->tagged->sha1);
303 }
304 if (o && o->type == OBJ_COMMIT) {
305 struct commit *commit = (struct commit *)o;
306 commit->object.flags |= COMPLETE;
307 insert_by_date(commit, &complete);
308 }
309 return 0;
310}
311
312static void mark_recent_complete_commits(unsigned long cutoff)
313{
314 while (complete && cutoff <= complete->item->date) {
315 if (verbose)
316 fprintf(stderr, "Marking %s as complete\n",
317 sha1_to_hex(complete->item->object.sha1));
318 pop_most_recent_commit(&complete, COMPLETE);
319 }
320}
321
322static void filter_refs(struct ref **refs, int nr_match, char **match)
323{
324 struct ref **return_refs;
325 struct ref *newlist = NULL;
326 struct ref **newtail = &newlist;
327 struct ref *ref, *next;
328 struct ref *fastarray[32];
329
330 if (nr_match && !fetch_all) {
331 if (ARRAY_SIZE(fastarray) < nr_match)
332 return_refs = xcalloc(nr_match, sizeof(struct ref *));
333 else {
334 return_refs = fastarray;
335 memset(return_refs, 0, sizeof(struct ref *) * nr_match);
336 }
337 }
338 else
339 return_refs = NULL;
340
341 for (ref = *refs; ref; ref = next) {
342 next = ref->next;
343 if (!memcmp(ref->name, "refs/", 5) &&
344 check_ref_format(ref->name + 5))
345 ; /* trash */
346 else if (fetch_all &&
347 (!depth || strncmp(ref->name, "refs/tags/", 10) )) {
348 *newtail = ref;
349 ref->next = NULL;
350 newtail = &ref->next;
351 continue;
352 }
353 else {
354 int order = path_match(ref->name, nr_match, match);
355 if (order) {
356 return_refs[order-1] = ref;
357 continue; /* we will link it later */
358 }
359 }
360 free(ref);
361 }
362
363 if (!fetch_all) {
364 int i;
365 for (i = 0; i < nr_match; i++) {
366 ref = return_refs[i];
367 if (ref) {
368 *newtail = ref;
369 ref->next = NULL;
370 newtail = &ref->next;
371 }
372 }
373 if (return_refs != fastarray)
374 free(return_refs);
375 }
376 *refs = newlist;
377}
378
379static int everything_local(struct ref **refs, int nr_match, char **match)
380{
381 struct ref *ref;
382 int retval;
383 unsigned long cutoff = 0;
384
385 track_object_refs = 0;
386 save_commit_buffer = 0;
387
388 for (ref = *refs; ref; ref = ref->next) {
389 struct object *o;
390
391 o = parse_object(ref->old_sha1);
392 if (!o)
393 continue;
394
395 /* We already have it -- which may mean that we were
396 * in sync with the other side at some time after
397 * that (it is OK if we guess wrong here).
398 */
399 if (o->type == OBJ_COMMIT) {
400 struct commit *commit = (struct commit *)o;
401 if (!cutoff || cutoff < commit->date)
402 cutoff = commit->date;
403 }
404 }
405
406 if (!depth) {
407 for_each_ref(mark_complete, NULL);
408 if (cutoff)
409 mark_recent_complete_commits(cutoff);
410 }
411
412 /*
413 * Mark all complete remote refs as common refs.
414 * Don't mark them common yet; the server has to be told so first.
415 */
416 for (ref = *refs; ref; ref = ref->next) {
417 struct object *o = deref_tag(lookup_object(ref->old_sha1),
418 NULL, 0);
419
420 if (!o || o->type != OBJ_COMMIT || !(o->flags & COMPLETE))
421 continue;
422
423 if (!(o->flags & SEEN)) {
424 rev_list_push((struct commit *)o, COMMON_REF | SEEN);
425
426 mark_common((struct commit *)o, 1, 1);
427 }
428 }
429
430 filter_refs(refs, nr_match, match);
431
432 for (retval = 1, ref = *refs; ref ; ref = ref->next) {
433 const unsigned char *remote = ref->old_sha1;
434 unsigned char local[20];
435 struct object *o;
436
437 o = lookup_object(remote);
438 if (!o || !(o->flags & COMPLETE)) {
439 retval = 0;
440 if (!verbose)
441 continue;
442 fprintf(stderr,
443 "want %s (%s)\n", sha1_to_hex(remote),
444 ref->name);
445 continue;
446 }
447
448 hashcpy(ref->new_sha1, local);
449 if (!verbose)
450 continue;
451 fprintf(stderr,
452 "already have %s (%s)\n", sha1_to_hex(remote),
453 ref->name);
454 }
455 return retval;
456}
457
458static pid_t setup_sideband(int fd[2], int xd[2])
459{
460 pid_t side_pid;
461
462 if (!use_sideband) {
463 fd[0] = xd[0];
464 fd[1] = xd[1];
465 return 0;
466 }
467 /* xd[] is talking with upload-pack; subprocess reads from
468 * xd[0], spits out band#2 to stderr, and feeds us band#1
469 * through our fd[0].
470 */
471 if (pipe(fd) < 0)
472 die("fetch-pack: unable to set up pipe");
473 side_pid = fork();
474 if (side_pid < 0)
475 die("fetch-pack: unable to fork off sideband demultiplexer");
476 if (!side_pid) {
477 /* subprocess */
478 close(fd[0]);
479 if (xd[0] != xd[1])
480 close(xd[1]);
481 if (recv_sideband("fetch-pack", xd[0], fd[1], 2))
482 exit(1);
483 exit(0);
484 }
485 close(xd[0]);
486 close(fd[1]);
487 fd[1] = xd[1];
488 return side_pid;
489}
490
491static int get_pack(int xd[2])
492{
493 int status;
494 pid_t pid, side_pid;
495 int fd[2];
496 const char *argv[20];
497 char keep_arg[256];
498 char hdr_arg[256];
499 const char **av;
500 int do_keep = keep_pack;
501
502 side_pid = setup_sideband(fd, xd);
503
504 av = argv;
505 *hdr_arg = 0;
506 if (unpack_limit) {
507 struct pack_header header;
508
509 if (read_pack_header(fd[0], &header))
510 die("protocol error: bad pack header");
511 snprintf(hdr_arg, sizeof(hdr_arg), "--pack_header=%u,%u",
512 ntohl(header.hdr_version), ntohl(header.hdr_entries));
513 if (ntohl(header.hdr_entries) < unpack_limit)
514 do_keep = 0;
515 else
516 do_keep = 1;
517 }
518
519 if (do_keep) {
520 *av++ = "index-pack";
521 *av++ = "--stdin";
522 if (!quiet)
523 *av++ = "-v";
524 if (use_thin_pack)
525 *av++ = "--fix-thin";
526 if (keep_pack > 1 || unpack_limit) {
527 int s = sprintf(keep_arg,
528 "--keep=fetch-pack %d on ", getpid());
529 if (gethostname(keep_arg + s, sizeof(keep_arg) - s))
530 strcpy(keep_arg + s, "localhost");
531 *av++ = keep_arg;
532 }
533 }
534 else {
535 *av++ = "unpack-objects";
536 if (quiet)
537 *av++ = "-q";
538 }
539 if (*hdr_arg)
540 *av++ = hdr_arg;
541 *av++ = NULL;
542
543 pid = fork();
544 if (pid < 0)
545 die("fetch-pack: unable to fork off %s", argv[0]);
546 if (!pid) {
547 dup2(fd[0], 0);
548 close(fd[0]);
549 close(fd[1]);
550 execv_git_cmd(argv);
551 die("%s exec failed", argv[0]);
552 }
553 close(fd[0]);
554 close(fd[1]);
555 while (waitpid(pid, &status, 0) < 0) {
556 if (errno != EINTR)
557 die("waiting for %s: %s", argv[0], strerror(errno));
558 }
559 if (WIFEXITED(status)) {
560 int code = WEXITSTATUS(status);
561 if (code)
562 die("%s died with error code %d", argv[0], code);
563 return 0;
564 }
565 if (WIFSIGNALED(status)) {
566 int sig = WTERMSIG(status);
567 die("%s died of signal %d", argv[0], sig);
568 }
569 die("%s died of unnatural causes %d", argv[0], status);
570}
571
572static int fetch_pack(int fd[2], int nr_match, char **match)
573{
574 struct ref *ref;
575 unsigned char sha1[20];
576
577 get_remote_heads(fd[0], &ref, 0, NULL, 0);
578 if (is_repository_shallow() && !server_supports("shallow"))
579 die("Server does not support shallow clients");
580 if (server_supports("multi_ack")) {
581 if (verbose)
582 fprintf(stderr, "Server supports multi_ack\n");
583 multi_ack = 1;
584 }
585 if (server_supports("side-band-64k")) {
586 if (verbose)
587 fprintf(stderr, "Server supports side-band-64k\n");
588 use_sideband = 2;
589 }
590 else if (server_supports("side-band")) {
591 if (verbose)
592 fprintf(stderr, "Server supports side-band\n");
593 use_sideband = 1;
594 }
595 if (!ref) {
596 packet_flush(fd[1]);
597 die("no matching remote head");
598 }
599 if (everything_local(&ref, nr_match, match)) {
600 packet_flush(fd[1]);
601 goto all_done;
602 }
603 if (find_common(fd, sha1, ref) < 0)
604 if (keep_pack != 1)
605 /* When cloning, it is not unusual to have
606 * no common commit.
607 */
608 fprintf(stderr, "warning: no common commits\n");
609
610 if (get_pack(fd))
611 die("git-fetch-pack: fetch failed.");
612
613 all_done:
614 while (ref) {
615 printf("%s %s\n",
616 sha1_to_hex(ref->old_sha1), ref->name);
617 ref = ref->next;
618 }
619 return 0;
620}
621
622static int remove_duplicates(int nr_heads, char **heads)
623{
624 int src, dst;
625
626 for (src = dst = 0; src < nr_heads; src++) {
627 /* If heads[src] is different from any of
628 * heads[0..dst], push it in.
629 */
630 int i;
631 for (i = 0; i < dst; i++) {
632 if (!strcmp(heads[i], heads[src]))
633 break;
634 }
635 if (i < dst)
636 continue;
637 if (src != dst)
638 heads[dst] = heads[src];
639 dst++;
640 }
641 heads[dst] = 0;
642 return dst;
643}
644
645static int fetch_pack_config(const char *var, const char *value)
646{
647 if (strcmp(var, "fetch.unpacklimit") == 0) {
648 unpack_limit = git_config_int(var, value);
649 return 0;
650 }
651
652 return git_default_config(var, value);
653}
654
655static struct lock_file lock;
656
657int main(int argc, char **argv)
658{
659 int i, ret, nr_heads;
660 char *dest = NULL, **heads;
661 int fd[2];
662 pid_t pid;
663 struct stat st;
664
665 setup_git_directory();
666 setup_ident();
667 git_config(fetch_pack_config);
668
669 nr_heads = 0;
670 heads = NULL;
671 for (i = 1; i < argc; i++) {
672 char *arg = argv[i];
673
674 if (*arg == '-') {
675 if (!strncmp("--upload-pack=", arg, 14)) {
676 uploadpack = arg + 14;
677 continue;
678 }
679 if (!strncmp("--exec=", arg, 7)) {
680 uploadpack = arg + 7;
681 continue;
682 }
683 if (!strcmp("--quiet", arg) || !strcmp("-q", arg)) {
684 quiet = 1;
685 continue;
686 }
687 if (!strcmp("--keep", arg) || !strcmp("-k", arg)) {
688 keep_pack++;
689 unpack_limit = 0;
690 continue;
691 }
692 if (!strcmp("--thin", arg)) {
693 use_thin_pack = 1;
694 continue;
695 }
696 if (!strcmp("--all", arg)) {
697 fetch_all = 1;
698 continue;
699 }
700 if (!strcmp("-v", arg)) {
701 verbose = 1;
702 continue;
703 }
704 if (!strncmp("--depth=", arg, 8)) {
705 depth = strtol(arg + 8, NULL, 0);
706 if (stat(git_path("shallow"), &st))
707 st.st_mtime = 0;
708 continue;
709 }
710 usage(fetch_pack_usage);
711 }
712 dest = arg;
713 heads = argv + i + 1;
714 nr_heads = argc - i - 1;
715 break;
716 }
717 if (!dest)
718 usage(fetch_pack_usage);
719 pid = git_connect(fd, dest, uploadpack);
720 if (pid < 0)
721 return 1;
722 if (heads && nr_heads)
723 nr_heads = remove_duplicates(nr_heads, heads);
724 ret = fetch_pack(fd, nr_heads, heads);
725 close(fd[0]);
726 close(fd[1]);
727 ret |= finish_connect(pid);
728
729 if (!ret && nr_heads) {
730 /* If the heads to pull were given, we should have
731 * consumed all of them by matching the remote.
732 * Otherwise, 'git-fetch remote no-such-ref' would
733 * silently succeed without issuing an error.
734 */
735 for (i = 0; i < nr_heads; i++)
736 if (heads[i] && heads[i][0]) {
737 error("no such remote ref %s", heads[i]);
738 ret = 1;
739 }
740 }
741
742 if (!ret && depth > 0) {
743 struct cache_time mtime;
744 char *shallow = git_path("shallow");
745 int fd;
746
747 mtime.sec = st.st_mtime;
748#ifdef USE_NSEC
749 mtime.usec = st.st_mtim.usec;
750#endif
751 if (stat(shallow, &st)) {
752 if (mtime.sec)
753 die("shallow file was removed during fetch");
754 } else if (st.st_mtime != mtime.sec
755#ifdef USE_NSEC
756 || st.st_mtim.usec != mtime.usec
757#endif
758 )
759 die("shallow file was changed during fetch");
760
761 fd = hold_lock_file_for_update(&lock, shallow, 1);
762 if (!write_shallow_commits(fd, 0)) {
763 unlink(shallow);
764 rollback_lock_file(&lock);
765 } else {
766 close(fd);
767 commit_lock_file(&lock);
768 }
769 }
770
771 return !!ret;
772}