1#include "cache.h"
2#include "refs.h"
3#include "pkt-line.h"
4#include "commit.h"
5#include "tag.h"
6#include "exec_cmd.h"
7#include "pack.h"
8#include "sideband.h"
9static int keep_pack;
11static int transfer_unpack_limit = -1;
12static int fetch_unpack_limit = -1;
13static int unpack_limit = 100;
14static int quiet;
15static int verbose;
16static int fetch_all;
17static int depth;
18static const char fetch_pack_usage[] =
19"git-fetch-pack [--all] [--quiet|-q] [--keep|-k] [--thin] [--upload-pack=<git-upload-pack>] [--depth=<n>] [-v] [<host>:]<directory> [<refs>...]";
20static const char *uploadpack = "git-upload-pack";
21#define COMPLETE (1U << 0)
23#define COMMON (1U << 1)
24#define COMMON_REF (1U << 2)
25#define SEEN (1U << 3)
26#define POPPED (1U << 4)
27/*
29 * After sending this many "have"s if we do not get any new ACK , we
30 * give up traversing our history.
31 */
32#define MAX_IN_VAIN 256
33static struct commit_list *rev_list;
35static int non_common_revs, multi_ack, use_thin_pack, use_sideband;
36static void rev_list_push(struct commit *commit, int mark)
38{
39 if (!(commit->object.flags & mark)) {
40 commit->object.flags |= mark;
41 if (!(commit->object.parsed))
43 parse_commit(commit);
44 insert_by_date(commit, &rev_list);
46 if (!(commit->object.flags & COMMON))
48 non_common_revs++;
49 }
50}
51static int rev_list_insert_ref(const char *path, const unsigned char *sha1, int flag, void *cb_data)
53{
54 struct object *o = deref_tag(parse_object(sha1), path, 0);
55 if (o && o->type == OBJ_COMMIT)
57 rev_list_push((struct commit *)o, SEEN);
58 return 0;
60}
61/*
63 This function marks a rev and its ancestors as common.
64 In some cases, it is desirable to mark only the ancestors (for example
65 when only the server does not yet know that they are common).
66*/
67static void mark_common(struct commit *commit,
69 int ancestors_only, int dont_parse)
70{
71 if (commit != NULL && !(commit->object.flags & COMMON)) {
72 struct object *o = (struct object *)commit;
73 if (!ancestors_only)
75 o->flags |= COMMON;
76 if (!(o->flags & SEEN))
78 rev_list_push(commit, SEEN);
79 else {
80 struct commit_list *parents;
81 if (!ancestors_only && !(o->flags & POPPED))
83 non_common_revs--;
84 if (!o->parsed && !dont_parse)
85 parse_commit(commit);
86 for (parents = commit->parents;
88 parents;
89 parents = parents->next)
90 mark_common(parents->item, 0, dont_parse);
91 }
92 }
93}
94/*
96 Get the next rev to send, ignoring the common.
97*/
98static const unsigned char* get_rev(void)
100{
101 struct commit *commit = NULL;
102 while (commit == NULL) {
104 unsigned int mark;
105 struct commit_list* parents;
106 if (rev_list == NULL || non_common_revs == 0)
108 return NULL;
109 commit = rev_list->item;
111 if (!(commit->object.parsed))
112 parse_commit(commit);
113 commit->object.flags |= POPPED;
114 if (!(commit->object.flags & COMMON))
115 non_common_revs--;
116
117 parents = commit->parents;
118 if (commit->object.flags & COMMON) {
120 /* do not send "have", and ignore ancestors */
121 commit = NULL;
122 mark = COMMON | SEEN;
123 } else if (commit->object.flags & COMMON_REF)
124 /* send "have", and ignore ancestors */
125 mark = COMMON | SEEN;
126 else
127 /* send "have", also for its ancestors */
128 mark = SEEN;
129 while (parents) {
131 if (!(parents->item->object.flags & SEEN))
132 rev_list_push(parents->item, mark);
133 if (mark & COMMON)
134 mark_common(parents->item, 1, 0);
135 parents = parents->next;
136 }
137 rev_list = rev_list->next;
139 }
140 return commit->object.sha1;
142}
143static int find_common(int fd[2], unsigned char *result_sha1,
145 struct ref *refs)
146{
147 int fetching;
148 int count = 0, flushes = 0, retval;
149 const unsigned char *sha1;
150 unsigned in_vain = 0;
151 int got_continue = 0;
152 for_each_ref(rev_list_insert_ref, NULL);
154 fetching = 0;
156 for ( ; refs ; refs = refs->next) {
157 unsigned char *remote = refs->old_sha1;
158 struct object *o;
159 /*
161 * If that object is complete (i.e. it is an ancestor of a
162 * local ref), we tell them we have it but do not have to
163 * tell them about its ancestors, which they already know
164 * about.
165 *
166 * We use lookup_object here because we are only
167 * interested in the case we *know* the object is
168 * reachable and we have already scanned it.
169 */
170 if (((o = lookup_object(remote)) != NULL) &&
171 (o->flags & COMPLETE)) {
172 continue;
173 }
174 if (!fetching)
176 packet_write(fd[1], "want %s%s%s%s%s%s\n",
177 sha1_to_hex(remote),
178 (multi_ack ? " multi_ack" : ""),
179 (use_sideband == 2 ? " side-band-64k" : ""),
180 (use_sideband == 1 ? " side-band" : ""),
181 (use_thin_pack ? " thin-pack" : ""),
182 " ofs-delta");
183 else
184 packet_write(fd[1], "want %s\n", sha1_to_hex(remote));
185 fetching++;
186 }
187 if (is_repository_shallow())
188 write_shallow_commits(fd[1], 1);
189 if (depth > 0)
190 packet_write(fd[1], "deepen %d", depth);
191 packet_flush(fd[1]);
192 if (!fetching)
193 return 1;
194 if (depth > 0) {
196 char line[1024];
197 unsigned char sha1[20];
198 int len;
199 while ((len = packet_read_line(fd[0], line, sizeof(line)))) {
201 if (!strncmp("shallow ", line, 8)) {
202 if (get_sha1_hex(line + 8, sha1))
203 die("invalid shallow line: %s", line);
204 register_shallow(sha1);
205 continue;
206 }
207 if (!strncmp("unshallow ", line, 10)) {
208 if (get_sha1_hex(line + 10, sha1))
209 die("invalid unshallow line: %s", line);
210 if (!lookup_object(sha1))
211 die("object not found: %s", line);
212 /* make sure that it is parsed as shallow */
213 parse_object(sha1);
214 if (unregister_shallow(sha1))
215 die("no shallow found: %s", line);
216 continue;
217 }
218 die("expected shallow/unshallow, got %s", line);
219 }
220 }
221 flushes = 0;
223 retval = -1;
224 while ((sha1 = get_rev())) {
225 packet_write(fd[1], "have %s\n", sha1_to_hex(sha1));
226 if (verbose)
227 fprintf(stderr, "have %s\n", sha1_to_hex(sha1));
228 in_vain++;
229 if (!(31 & ++count)) {
230 int ack;
231 packet_flush(fd[1]);
233 flushes++;
234 /*
236 * We keep one window "ahead" of the other side, and
237 * will wait for an ACK only on the next one
238 */
239 if (count == 32)
240 continue;
241 do {
243 ack = get_ack(fd[0], result_sha1);
244 if (verbose && ack)
245 fprintf(stderr, "got ack %d %s\n", ack,
246 sha1_to_hex(result_sha1));
247 if (ack == 1) {
248 flushes = 0;
249 multi_ack = 0;
250 retval = 0;
251 goto done;
252 } else if (ack == 2) {
253 struct commit *commit =
254 lookup_commit(result_sha1);
255 mark_common(commit, 0, 1);
256 retval = 0;
257 in_vain = 0;
258 got_continue = 1;
259 }
260 } while (ack);
261 flushes--;
262 if (got_continue && MAX_IN_VAIN < in_vain) {
263 if (verbose)
264 fprintf(stderr, "giving up\n");
265 break; /* give up */
266 }
267 }
268 }
269done:
270 packet_write(fd[1], "done\n");
271 if (verbose)
272 fprintf(stderr, "done\n");
273 if (retval != 0) {
274 multi_ack = 0;
275 flushes++;
276 }
277 while (flushes || multi_ack) {
278 int ack = get_ack(fd[0], result_sha1);
279 if (ack) {
280 if (verbose)
281 fprintf(stderr, "got ack (%d) %s\n", ack,
282 sha1_to_hex(result_sha1));
283 if (ack == 1)
284 return 0;
285 multi_ack = 1;
286 continue;
287 }
288 flushes--;
289 }
290 return retval;
291}
292static struct commit_list *complete;
294static int mark_complete(const char *path, const unsigned char *sha1, int flag, void *cb_data)
296{
297 struct object *o = parse_object(sha1);
298 while (o && o->type == OBJ_TAG) {
300 struct tag *t = (struct tag *) o;
301 if (!t->tagged)
302 break; /* broken repository */
303 o->flags |= COMPLETE;
304 o = parse_object(t->tagged->sha1);
305 }
306 if (o && o->type == OBJ_COMMIT) {
307 struct commit *commit = (struct commit *)o;
308 commit->object.flags |= COMPLETE;
309 insert_by_date(commit, &complete);
310 }
311 return 0;
312}
313static void mark_recent_complete_commits(unsigned long cutoff)
315{
316 while (complete && cutoff <= complete->item->date) {
317 if (verbose)
318 fprintf(stderr, "Marking %s as complete\n",
319 sha1_to_hex(complete->item->object.sha1));
320 pop_most_recent_commit(&complete, COMPLETE);
321 }
322}
323static void filter_refs(struct ref **refs, int nr_match, char **match)
325{
326 struct ref **return_refs;
327 struct ref *newlist = NULL;
328 struct ref **newtail = &newlist;
329 struct ref *ref, *next;
330 struct ref *fastarray[32];
331 if (nr_match && !fetch_all) {
333 if (ARRAY_SIZE(fastarray) < nr_match)
334 return_refs = xcalloc(nr_match, sizeof(struct ref *));
335 else {
336 return_refs = fastarray;
337 memset(return_refs, 0, sizeof(struct ref *) * nr_match);
338 }
339 }
340 else
341 return_refs = NULL;
342 for (ref = *refs; ref; ref = next) {
344 next = ref->next;
345 if (!memcmp(ref->name, "refs/", 5) &&
346 check_ref_format(ref->name + 5))
347 ; /* trash */
348 else if (fetch_all &&
349 (!depth || strncmp(ref->name, "refs/tags/", 10) )) {
350 *newtail = ref;
351 ref->next = NULL;
352 newtail = &ref->next;
353 continue;
354 }
355 else {
356 int order = path_match(ref->name, nr_match, match);
357 if (order) {
358 return_refs[order-1] = ref;
359 continue; /* we will link it later */
360 }
361 }
362 free(ref);
363 }
364 if (!fetch_all) {
366 int i;
367 for (i = 0; i < nr_match; i++) {
368 ref = return_refs[i];
369 if (ref) {
370 *newtail = ref;
371 ref->next = NULL;
372 newtail = &ref->next;
373 }
374 }
375 if (return_refs != fastarray)
376 free(return_refs);
377 }
378 *refs = newlist;
379}
380static int everything_local(struct ref **refs, int nr_match, char **match)
382{
383 struct ref *ref;
384 int retval;
385 unsigned long cutoff = 0;
386 track_object_refs = 0;
388 save_commit_buffer = 0;
389 for (ref = *refs; ref; ref = ref->next) {
391 struct object *o;
392 o = parse_object(ref->old_sha1);
394 if (!o)
395 continue;
396 /* We already have it -- which may mean that we were
398 * in sync with the other side at some time after
399 * that (it is OK if we guess wrong here).
400 */
401 if (o->type == OBJ_COMMIT) {
402 struct commit *commit = (struct commit *)o;
403 if (!cutoff || cutoff < commit->date)
404 cutoff = commit->date;
405 }
406 }
407 if (!depth) {
409 for_each_ref(mark_complete, NULL);
410 if (cutoff)
411 mark_recent_complete_commits(cutoff);
412 }
413 /*
415 * Mark all complete remote refs as common refs.
416 * Don't mark them common yet; the server has to be told so first.
417 */
418 for (ref = *refs; ref; ref = ref->next) {
419 struct object *o = deref_tag(lookup_object(ref->old_sha1),
420 NULL, 0);
421 if (!o || o->type != OBJ_COMMIT || !(o->flags & COMPLETE))
423 continue;
424 if (!(o->flags & SEEN)) {
426 rev_list_push((struct commit *)o, COMMON_REF | SEEN);
427 mark_common((struct commit *)o, 1, 1);
429 }
430 }
431 filter_refs(refs, nr_match, match);
433 for (retval = 1, ref = *refs; ref ; ref = ref->next) {
435 const unsigned char *remote = ref->old_sha1;
436 unsigned char local[20];
437 struct object *o;
438 o = lookup_object(remote);
440 if (!o || !(o->flags & COMPLETE)) {
441 retval = 0;
442 if (!verbose)
443 continue;
444 fprintf(stderr,
445 "want %s (%s)\n", sha1_to_hex(remote),
446 ref->name);
447 continue;
448 }
449 hashcpy(ref->new_sha1, local);
451 if (!verbose)
452 continue;
453 fprintf(stderr,
454 "already have %s (%s)\n", sha1_to_hex(remote),
455 ref->name);
456 }
457 return retval;
458}
459static pid_t setup_sideband(int fd[2], int xd[2])
461{
462 pid_t side_pid;
463 if (!use_sideband) {
465 fd[0] = xd[0];
466 fd[1] = xd[1];
467 return 0;
468 }
469 /* xd[] is talking with upload-pack; subprocess reads from
470 * xd[0], spits out band#2 to stderr, and feeds us band#1
471 * through our fd[0].
472 */
473 if (pipe(fd) < 0)
474 die("fetch-pack: unable to set up pipe");
475 side_pid = fork();
476 if (side_pid < 0)
477 die("fetch-pack: unable to fork off sideband demultiplexer");
478 if (!side_pid) {
479 /* subprocess */
480 close(fd[0]);
481 if (xd[0] != xd[1])
482 close(xd[1]);
483 if (recv_sideband("fetch-pack", xd[0], fd[1], 2))
484 exit(1);
485 exit(0);
486 }
487 close(xd[0]);
488 close(fd[1]);
489 fd[1] = xd[1];
490 return side_pid;
491}
492static int get_pack(int xd[2])
494{
495 int status;
496 pid_t pid, side_pid;
497 int fd[2];
498 const char *argv[20];
499 char keep_arg[256];
500 char hdr_arg[256];
501 const char **av;
502 int do_keep = keep_pack;
503 side_pid = setup_sideband(fd, xd);
505 av = argv;
507 *hdr_arg = 0;
508 if (unpack_limit) {
509 struct pack_header header;
510 if (read_pack_header(fd[0], &header))
512 die("protocol error: bad pack header");
513 snprintf(hdr_arg, sizeof(hdr_arg), "--pack_header=%u,%u",
514 ntohl(header.hdr_version), ntohl(header.hdr_entries));
515 if (ntohl(header.hdr_entries) < unpack_limit)
516 do_keep = 0;
517 else
518 do_keep = 1;
519 }
520 if (do_keep) {
522 *av++ = "index-pack";
523 *av++ = "--stdin";
524 if (!quiet)
525 *av++ = "-v";
526 if (use_thin_pack)
527 *av++ = "--fix-thin";
528 if (keep_pack > 1 || unpack_limit) {
529 int s = sprintf(keep_arg,
530 "--keep=fetch-pack %d on ", getpid());
531 if (gethostname(keep_arg + s, sizeof(keep_arg) - s))
532 strcpy(keep_arg + s, "localhost");
533 *av++ = keep_arg;
534 }
535 }
536 else {
537 *av++ = "unpack-objects";
538 if (quiet)
539 *av++ = "-q";
540 }
541 if (*hdr_arg)
542 *av++ = hdr_arg;
543 *av++ = NULL;
544 pid = fork();
546 if (pid < 0)
547 die("fetch-pack: unable to fork off %s", argv[0]);
548 if (!pid) {
549 dup2(fd[0], 0);
550 close(fd[0]);
551 close(fd[1]);
552 execv_git_cmd(argv);
553 die("%s exec failed", argv[0]);
554 }
555 close(fd[0]);
556 close(fd[1]);
557 while (waitpid(pid, &status, 0) < 0) {
558 if (errno != EINTR)
559 die("waiting for %s: %s", argv[0], strerror(errno));
560 }
561 if (WIFEXITED(status)) {
562 int code = WEXITSTATUS(status);
563 if (code)
564 die("%s died with error code %d", argv[0], code);
565 return 0;
566 }
567 if (WIFSIGNALED(status)) {
568 int sig = WTERMSIG(status);
569 die("%s died of signal %d", argv[0], sig);
570 }
571 die("%s died of unnatural causes %d", argv[0], status);
572}
573static int fetch_pack(int fd[2], int nr_match, char **match)
575{
576 struct ref *ref;
577 unsigned char sha1[20];
578 get_remote_heads(fd[0], &ref, 0, NULL, 0);
580 if (is_repository_shallow() && !server_supports("shallow"))
581 die("Server does not support shallow clients");
582 if (server_supports("multi_ack")) {
583 if (verbose)
584 fprintf(stderr, "Server supports multi_ack\n");
585 multi_ack = 1;
586 }
587 if (server_supports("side-band-64k")) {
588 if (verbose)
589 fprintf(stderr, "Server supports side-band-64k\n");
590 use_sideband = 2;
591 }
592 else if (server_supports("side-band")) {
593 if (verbose)
594 fprintf(stderr, "Server supports side-band\n");
595 use_sideband = 1;
596 }
597 if (!ref) {
598 packet_flush(fd[1]);
599 die("no matching remote head");
600 }
601 if (everything_local(&ref, nr_match, match)) {
602 packet_flush(fd[1]);
603 goto all_done;
604 }
605 if (find_common(fd, sha1, ref) < 0)
606 if (keep_pack != 1)
607 /* When cloning, it is not unusual to have
608 * no common commit.
609 */
610 fprintf(stderr, "warning: no common commits\n");
611 if (get_pack(fd))
613 die("git-fetch-pack: fetch failed.");
614 all_done:
616 while (ref) {
617 printf("%s %s\n",
618 sha1_to_hex(ref->old_sha1), ref->name);
619 ref = ref->next;
620 }
621 return 0;
622}
623static int remove_duplicates(int nr_heads, char **heads)
625{
626 int src, dst;
627 for (src = dst = 0; src < nr_heads; src++) {
629 /* If heads[src] is different from any of
630 * heads[0..dst], push it in.
631 */
632 int i;
633 for (i = 0; i < dst; i++) {
634 if (!strcmp(heads[i], heads[src]))
635 break;
636 }
637 if (i < dst)
638 continue;
639 if (src != dst)
640 heads[dst] = heads[src];
641 dst++;
642 }
643 heads[dst] = 0;
644 return dst;
645}
646static int fetch_pack_config(const char *var, const char *value)
648{
649 if (strcmp(var, "fetch.unpacklimit") == 0) {
650 fetch_unpack_limit = git_config_int(var, value);
651 return 0;
652 }
653 if (strcmp(var, "transfer.unpacklimit") == 0) {
655 transfer_unpack_limit = git_config_int(var, value);
656 return 0;
657 }
658 return git_default_config(var, value);
660}
661static struct lock_file lock;
663int main(int argc, char **argv)
665{
666 int i, ret, nr_heads;
667 char *dest = NULL, **heads;
668 int fd[2];
669 pid_t pid;
670 struct stat st;
671 setup_git_directory();
673 git_config(fetch_pack_config);
674 if (0 <= transfer_unpack_limit)
676 unpack_limit = transfer_unpack_limit;
677 else if (0 <= fetch_unpack_limit)
678 unpack_limit = fetch_unpack_limit;
679 nr_heads = 0;
681 heads = NULL;
682 for (i = 1; i < argc; i++) {
683 char *arg = argv[i];
684 if (*arg == '-') {
686 if (!strncmp("--upload-pack=", arg, 14)) {
687 uploadpack = arg + 14;
688 continue;
689 }
690 if (!strncmp("--exec=", arg, 7)) {
691 uploadpack = arg + 7;
692 continue;
693 }
694 if (!strcmp("--quiet", arg) || !strcmp("-q", arg)) {
695 quiet = 1;
696 continue;
697 }
698 if (!strcmp("--keep", arg) || !strcmp("-k", arg)) {
699 keep_pack++;
700 unpack_limit = 0;
701 continue;
702 }
703 if (!strcmp("--thin", arg)) {
704 use_thin_pack = 1;
705 continue;
706 }
707 if (!strcmp("--all", arg)) {
708 fetch_all = 1;
709 continue;
710 }
711 if (!strcmp("-v", arg)) {
712 verbose = 1;
713 continue;
714 }
715 if (!strncmp("--depth=", arg, 8)) {
716 depth = strtol(arg + 8, NULL, 0);
717 if (stat(git_path("shallow"), &st))
718 st.st_mtime = 0;
719 continue;
720 }
721 usage(fetch_pack_usage);
722 }
723 dest = arg;
724 heads = argv + i + 1;
725 nr_heads = argc - i - 1;
726 break;
727 }
728 if (!dest)
729 usage(fetch_pack_usage);
730 pid = git_connect(fd, dest, uploadpack);
731 if (pid < 0)
732 return 1;
733 if (heads && nr_heads)
734 nr_heads = remove_duplicates(nr_heads, heads);
735 ret = fetch_pack(fd, nr_heads, heads);
736 close(fd[0]);
737 close(fd[1]);
738 ret |= finish_connect(pid);
739 if (!ret && nr_heads) {
741 /* If the heads to pull were given, we should have
742 * consumed all of them by matching the remote.
743 * Otherwise, 'git-fetch remote no-such-ref' would
744 * silently succeed without issuing an error.
745 */
746 for (i = 0; i < nr_heads; i++)
747 if (heads[i] && heads[i][0]) {
748 error("no such remote ref %s", heads[i]);
749 ret = 1;
750 }
751 }
752 if (!ret && depth > 0) {
754 struct cache_time mtime;
755 char *shallow = git_path("shallow");
756 int fd;
757 mtime.sec = st.st_mtime;
759#ifdef USE_NSEC
760 mtime.usec = st.st_mtim.usec;
761#endif
762 if (stat(shallow, &st)) {
763 if (mtime.sec)
764 die("shallow file was removed during fetch");
765 } else if (st.st_mtime != mtime.sec
766#ifdef USE_NSEC
767 || st.st_mtim.usec != mtime.usec
768#endif
769 )
770 die("shallow file was changed during fetch");
771 fd = hold_lock_file_for_update(&lock, shallow, 1);
773 if (!write_shallow_commits(fd, 0)) {
774 unlink(shallow);
775 rollback_lock_file(&lock);
776 } else {
777 close(fd);
778 commit_lock_file(&lock);
779 }
780 }
781 return !!ret;
783}