builtin / fetch-pack.con commit Merge branch 'jc/maint-smart-http-race-upload-pack' into maint (c43a1be)
   1#include "builtin.h"
   2#include "refs.h"
   3#include "pkt-line.h"
   4#include "commit.h"
   5#include "tag.h"
   6#include "exec_cmd.h"
   7#include "pack.h"
   8#include "sideband.h"
   9#include "fetch-pack.h"
  10#include "remote.h"
  11#include "run-command.h"
  12#include "transport.h"
  13
  14static int transfer_unpack_limit = -1;
  15static int fetch_unpack_limit = -1;
  16static int unpack_limit = 100;
  17static int prefer_ofs_delta = 1;
  18static int no_done = 0;
  19static struct fetch_pack_args args = {
  20        /* .uploadpack = */ "git-upload-pack",
  21};
  22
  23static const char fetch_pack_usage[] =
  24"git fetch-pack [--all] [--quiet|-q] [--keep|-k] [--thin] [--include-tag] [--upload-pack=<git-upload-pack>] [--depth=<n>] [--no-progress] [-v] [<host>:]<directory> [<refs>...]";
  25
  26#define COMPLETE        (1U << 0)
  27#define COMMON          (1U << 1)
  28#define COMMON_REF      (1U << 2)
  29#define SEEN            (1U << 3)
  30#define POPPED          (1U << 4)
  31
  32static int marked;
  33
  34/*
  35 * After sending this many "have"s if we do not get any new ACK , we
  36 * give up traversing our history.
  37 */
  38#define MAX_IN_VAIN 256
  39
  40static struct commit_list *rev_list;
  41static int non_common_revs, multi_ack, use_sideband;
  42
  43static void rev_list_push(struct commit *commit, int mark)
  44{
  45        if (!(commit->object.flags & mark)) {
  46                commit->object.flags |= mark;
  47
  48                if (!(commit->object.parsed))
  49                        if (parse_commit(commit))
  50                                return;
  51
  52                commit_list_insert_by_date(commit, &rev_list);
  53
  54                if (!(commit->object.flags & COMMON))
  55                        non_common_revs++;
  56        }
  57}
  58
  59static int rev_list_insert_ref(const char *path, const unsigned char *sha1, int flag, void *cb_data)
  60{
  61        struct object *o = deref_tag(parse_object(sha1), path, 0);
  62
  63        if (o && o->type == OBJ_COMMIT)
  64                rev_list_push((struct commit *)o, SEEN);
  65
  66        return 0;
  67}
  68
  69static int clear_marks(const char *path, const unsigned char *sha1, int flag, void *cb_data)
  70{
  71        struct object *o = deref_tag(parse_object(sha1), path, 0);
  72
  73        if (o && o->type == OBJ_COMMIT)
  74                clear_commit_marks((struct commit *)o,
  75                                   COMMON | COMMON_REF | SEEN | POPPED);
  76        return 0;
  77}
  78
  79/*
  80   This function marks a rev and its ancestors as common.
  81   In some cases, it is desirable to mark only the ancestors (for example
  82   when only the server does not yet know that they are common).
  83*/
  84
  85static void mark_common(struct commit *commit,
  86                int ancestors_only, int dont_parse)
  87{
  88        if (commit != NULL && !(commit->object.flags & COMMON)) {
  89                struct object *o = (struct object *)commit;
  90
  91                if (!ancestors_only)
  92                        o->flags |= COMMON;
  93
  94                if (!(o->flags & SEEN))
  95                        rev_list_push(commit, SEEN);
  96                else {
  97                        struct commit_list *parents;
  98
  99                        if (!ancestors_only && !(o->flags & POPPED))
 100                                non_common_revs--;
 101                        if (!o->parsed && !dont_parse)
 102                                if (parse_commit(commit))
 103                                        return;
 104
 105                        for (parents = commit->parents;
 106                                        parents;
 107                                        parents = parents->next)
 108                                mark_common(parents->item, 0, dont_parse);
 109                }
 110        }
 111}
 112
 113/*
 114  Get the next rev to send, ignoring the common.
 115*/
 116
 117static const unsigned char *get_rev(void)
 118{
 119        struct commit *commit = NULL;
 120
 121        while (commit == NULL) {
 122                unsigned int mark;
 123                struct commit_list *parents;
 124
 125                if (rev_list == NULL || non_common_revs == 0)
 126                        return NULL;
 127
 128                commit = rev_list->item;
 129                if (!commit->object.parsed)
 130                        parse_commit(commit);
 131                parents = commit->parents;
 132
 133                commit->object.flags |= POPPED;
 134                if (!(commit->object.flags & COMMON))
 135                        non_common_revs--;
 136
 137                if (commit->object.flags & COMMON) {
 138                        /* do not send "have", and ignore ancestors */
 139                        commit = NULL;
 140                        mark = COMMON | SEEN;
 141                } else if (commit->object.flags & COMMON_REF)
 142                        /* send "have", and ignore ancestors */
 143                        mark = COMMON | SEEN;
 144                else
 145                        /* send "have", also for its ancestors */
 146                        mark = SEEN;
 147
 148                while (parents) {
 149                        if (!(parents->item->object.flags & SEEN))
 150                                rev_list_push(parents->item, mark);
 151                        if (mark & COMMON)
 152                                mark_common(parents->item, 1, 0);
 153                        parents = parents->next;
 154                }
 155
 156                rev_list = rev_list->next;
 157        }
 158
 159        return commit->object.sha1;
 160}
 161
 162enum ack_type {
 163        NAK = 0,
 164        ACK,
 165        ACK_continue,
 166        ACK_common,
 167        ACK_ready
 168};
 169
 170static void consume_shallow_list(int fd)
 171{
 172        if (args.stateless_rpc && args.depth > 0) {
 173                /* If we sent a depth we will get back "duplicate"
 174                 * shallow and unshallow commands every time there
 175                 * is a block of have lines exchanged.
 176                 */
 177                char line[1000];
 178                while (packet_read_line(fd, line, sizeof(line))) {
 179                        if (!prefixcmp(line, "shallow "))
 180                                continue;
 181                        if (!prefixcmp(line, "unshallow "))
 182                                continue;
 183                        die("git fetch-pack: expected shallow list");
 184                }
 185        }
 186}
 187
 188static enum ack_type get_ack(int fd, unsigned char *result_sha1)
 189{
 190        static char line[1000];
 191        int len = packet_read_line(fd, line, sizeof(line));
 192
 193        if (!len)
 194                die("git fetch-pack: expected ACK/NAK, got EOF");
 195        if (line[len-1] == '\n')
 196                line[--len] = 0;
 197        if (!strcmp(line, "NAK"))
 198                return NAK;
 199        if (!prefixcmp(line, "ACK ")) {
 200                if (!get_sha1_hex(line+4, result_sha1)) {
 201                        if (strstr(line+45, "continue"))
 202                                return ACK_continue;
 203                        if (strstr(line+45, "common"))
 204                                return ACK_common;
 205                        if (strstr(line+45, "ready"))
 206                                return ACK_ready;
 207                        return ACK;
 208                }
 209        }
 210        die("git fetch_pack: expected ACK/NAK, got '%s'", line);
 211}
 212
 213static void send_request(int fd, struct strbuf *buf)
 214{
 215        if (args.stateless_rpc) {
 216                send_sideband(fd, -1, buf->buf, buf->len, LARGE_PACKET_MAX);
 217                packet_flush(fd);
 218        } else
 219                safe_write(fd, buf->buf, buf->len);
 220}
 221
 222static void insert_one_alternate_ref(const struct ref *ref, void *unused)
 223{
 224        rev_list_insert_ref(NULL, ref->old_sha1, 0, NULL);
 225}
 226
 227static void insert_alternate_refs(void)
 228{
 229        for_each_alternate_ref(insert_one_alternate_ref, NULL);
 230}
 231
 232#define INITIAL_FLUSH 16
 233#define PIPESAFE_FLUSH 32
 234#define LARGE_FLUSH 1024
 235
 236static int next_flush(int count)
 237{
 238        int flush_limit = args.stateless_rpc ? LARGE_FLUSH : PIPESAFE_FLUSH;
 239
 240        if (count < flush_limit)
 241                count <<= 1;
 242        else
 243                count += flush_limit;
 244        return count;
 245}
 246
 247static int find_common(int fd[2], unsigned char *result_sha1,
 248                       struct ref *refs)
 249{
 250        int fetching;
 251        int count = 0, flushes = 0, flush_at = INITIAL_FLUSH, retval;
 252        const unsigned char *sha1;
 253        unsigned in_vain = 0;
 254        int got_continue = 0;
 255        int got_ready = 0;
 256        struct strbuf req_buf = STRBUF_INIT;
 257        size_t state_len = 0;
 258
 259        if (args.stateless_rpc && multi_ack == 1)
 260                die("--stateless-rpc requires multi_ack_detailed");
 261        if (marked)
 262                for_each_ref(clear_marks, NULL);
 263        marked = 1;
 264
 265        for_each_ref(rev_list_insert_ref, NULL);
 266        insert_alternate_refs();
 267
 268        fetching = 0;
 269        for ( ; refs ; refs = refs->next) {
 270                unsigned char *remote = refs->old_sha1;
 271                const char *remote_hex;
 272                struct object *o;
 273
 274                /*
 275                 * If that object is complete (i.e. it is an ancestor of a
 276                 * local ref), we tell them we have it but do not have to
 277                 * tell them about its ancestors, which they already know
 278                 * about.
 279                 *
 280                 * We use lookup_object here because we are only
 281                 * interested in the case we *know* the object is
 282                 * reachable and we have already scanned it.
 283                 */
 284                if (((o = lookup_object(remote)) != NULL) &&
 285                                (o->flags & COMPLETE)) {
 286                        continue;
 287                }
 288
 289                remote_hex = sha1_to_hex(remote);
 290                if (!fetching) {
 291                        struct strbuf c = STRBUF_INIT;
 292                        if (multi_ack == 2)     strbuf_addstr(&c, " multi_ack_detailed");
 293                        if (multi_ack == 1)     strbuf_addstr(&c, " multi_ack");
 294                        if (no_done)            strbuf_addstr(&c, " no-done");
 295                        if (use_sideband == 2)  strbuf_addstr(&c, " side-band-64k");
 296                        if (use_sideband == 1)  strbuf_addstr(&c, " side-band");
 297                        if (args.use_thin_pack) strbuf_addstr(&c, " thin-pack");
 298                        if (args.no_progress)   strbuf_addstr(&c, " no-progress");
 299                        if (args.include_tag)   strbuf_addstr(&c, " include-tag");
 300                        if (prefer_ofs_delta)   strbuf_addstr(&c, " ofs-delta");
 301                        packet_buf_write(&req_buf, "want %s%s\n", remote_hex, c.buf);
 302                        strbuf_release(&c);
 303                } else
 304                        packet_buf_write(&req_buf, "want %s\n", remote_hex);
 305                fetching++;
 306        }
 307
 308        if (!fetching) {
 309                strbuf_release(&req_buf);
 310                packet_flush(fd[1]);
 311                return 1;
 312        }
 313
 314        if (is_repository_shallow())
 315                write_shallow_commits(&req_buf, 1);
 316        if (args.depth > 0)
 317                packet_buf_write(&req_buf, "deepen %d", args.depth);
 318        packet_buf_flush(&req_buf);
 319        state_len = req_buf.len;
 320
 321        if (args.depth > 0) {
 322                char line[1024];
 323                unsigned char sha1[20];
 324
 325                send_request(fd[1], &req_buf);
 326                while (packet_read_line(fd[0], line, sizeof(line))) {
 327                        if (!prefixcmp(line, "shallow ")) {
 328                                if (get_sha1_hex(line + 8, sha1))
 329                                        die("invalid shallow line: %s", line);
 330                                register_shallow(sha1);
 331                                continue;
 332                        }
 333                        if (!prefixcmp(line, "unshallow ")) {
 334                                if (get_sha1_hex(line + 10, sha1))
 335                                        die("invalid unshallow line: %s", line);
 336                                if (!lookup_object(sha1))
 337                                        die("object not found: %s", line);
 338                                /* make sure that it is parsed as shallow */
 339                                if (!parse_object(sha1))
 340                                        die("error in object: %s", line);
 341                                if (unregister_shallow(sha1))
 342                                        die("no shallow found: %s", line);
 343                                continue;
 344                        }
 345                        die("expected shallow/unshallow, got %s", line);
 346                }
 347        } else if (!args.stateless_rpc)
 348                send_request(fd[1], &req_buf);
 349
 350        if (!args.stateless_rpc) {
 351                /* If we aren't using the stateless-rpc interface
 352                 * we don't need to retain the headers.
 353                 */
 354                strbuf_setlen(&req_buf, 0);
 355                state_len = 0;
 356        }
 357
 358        flushes = 0;
 359        retval = -1;
 360        while ((sha1 = get_rev())) {
 361                packet_buf_write(&req_buf, "have %s\n", sha1_to_hex(sha1));
 362                if (args.verbose)
 363                        fprintf(stderr, "have %s\n", sha1_to_hex(sha1));
 364                in_vain++;
 365                if (flush_at <= ++count) {
 366                        int ack;
 367
 368                        packet_buf_flush(&req_buf);
 369                        send_request(fd[1], &req_buf);
 370                        strbuf_setlen(&req_buf, state_len);
 371                        flushes++;
 372                        flush_at = next_flush(count);
 373
 374                        /*
 375                         * We keep one window "ahead" of the other side, and
 376                         * will wait for an ACK only on the next one
 377                         */
 378                        if (!args.stateless_rpc && count == INITIAL_FLUSH)
 379                                continue;
 380
 381                        consume_shallow_list(fd[0]);
 382                        do {
 383                                ack = get_ack(fd[0], result_sha1);
 384                                if (args.verbose && ack)
 385                                        fprintf(stderr, "got ack %d %s\n", ack,
 386                                                        sha1_to_hex(result_sha1));
 387                                switch (ack) {
 388                                case ACK:
 389                                        flushes = 0;
 390                                        multi_ack = 0;
 391                                        retval = 0;
 392                                        goto done;
 393                                case ACK_common:
 394                                case ACK_ready:
 395                                case ACK_continue: {
 396                                        struct commit *commit =
 397                                                lookup_commit(result_sha1);
 398                                        if (!commit)
 399                                                die("invalid commit %s", sha1_to_hex(result_sha1));
 400                                        if (args.stateless_rpc
 401                                         && ack == ACK_common
 402                                         && !(commit->object.flags & COMMON)) {
 403                                                /* We need to replay the have for this object
 404                                                 * on the next RPC request so the peer knows
 405                                                 * it is in common with us.
 406                                                 */
 407                                                const char *hex = sha1_to_hex(result_sha1);
 408                                                packet_buf_write(&req_buf, "have %s\n", hex);
 409                                                state_len = req_buf.len;
 410                                        }
 411                                        mark_common(commit, 0, 1);
 412                                        retval = 0;
 413                                        in_vain = 0;
 414                                        got_continue = 1;
 415                                        if (ack == ACK_ready) {
 416                                                rev_list = NULL;
 417                                                got_ready = 1;
 418                                        }
 419                                        break;
 420                                        }
 421                                }
 422                        } while (ack);
 423                        flushes--;
 424                        if (got_continue && MAX_IN_VAIN < in_vain) {
 425                                if (args.verbose)
 426                                        fprintf(stderr, "giving up\n");
 427                                break; /* give up */
 428                        }
 429                }
 430        }
 431done:
 432        if (!got_ready || !no_done) {
 433                packet_buf_write(&req_buf, "done\n");
 434                send_request(fd[1], &req_buf);
 435        }
 436        if (args.verbose)
 437                fprintf(stderr, "done\n");
 438        if (retval != 0) {
 439                multi_ack = 0;
 440                flushes++;
 441        }
 442        strbuf_release(&req_buf);
 443
 444        consume_shallow_list(fd[0]);
 445        while (flushes || multi_ack) {
 446                int ack = get_ack(fd[0], result_sha1);
 447                if (ack) {
 448                        if (args.verbose)
 449                                fprintf(stderr, "got ack (%d) %s\n", ack,
 450                                        sha1_to_hex(result_sha1));
 451                        if (ack == ACK)
 452                                return 0;
 453                        multi_ack = 1;
 454                        continue;
 455                }
 456                flushes--;
 457        }
 458        /* it is no error to fetch into a completely empty repo */
 459        return count ? retval : 0;
 460}
 461
 462static struct commit_list *complete;
 463
 464static int mark_complete(const char *path, const unsigned char *sha1, int flag, void *cb_data)
 465{
 466        struct object *o = parse_object(sha1);
 467
 468        while (o && o->type == OBJ_TAG) {
 469                struct tag *t = (struct tag *) o;
 470                if (!t->tagged)
 471                        break; /* broken repository */
 472                o->flags |= COMPLETE;
 473                o = parse_object(t->tagged->sha1);
 474        }
 475        if (o && o->type == OBJ_COMMIT) {
 476                struct commit *commit = (struct commit *)o;
 477                if (!(commit->object.flags & COMPLETE)) {
 478                        commit->object.flags |= COMPLETE;
 479                        commit_list_insert_by_date(commit, &complete);
 480                }
 481        }
 482        return 0;
 483}
 484
 485static void mark_recent_complete_commits(unsigned long cutoff)
 486{
 487        while (complete && cutoff <= complete->item->date) {
 488                if (args.verbose)
 489                        fprintf(stderr, "Marking %s as complete\n",
 490                                sha1_to_hex(complete->item->object.sha1));
 491                pop_most_recent_commit(&complete, COMPLETE);
 492        }
 493}
 494
 495static void filter_refs(struct ref **refs, int nr_match, char **match)
 496{
 497        struct ref **return_refs;
 498        struct ref *newlist = NULL;
 499        struct ref **newtail = &newlist;
 500        struct ref *ref, *next;
 501        struct ref *fastarray[32];
 502
 503        if (nr_match && !args.fetch_all) {
 504                if (ARRAY_SIZE(fastarray) < nr_match)
 505                        return_refs = xcalloc(nr_match, sizeof(struct ref *));
 506                else {
 507                        return_refs = fastarray;
 508                        memset(return_refs, 0, sizeof(struct ref *) * nr_match);
 509                }
 510        }
 511        else
 512                return_refs = NULL;
 513
 514        for (ref = *refs; ref; ref = next) {
 515                next = ref->next;
 516                if (!memcmp(ref->name, "refs/", 5) &&
 517                    check_ref_format(ref->name + 5))
 518                        ; /* trash */
 519                else if (args.fetch_all &&
 520                         (!args.depth || prefixcmp(ref->name, "refs/tags/") )) {
 521                        *newtail = ref;
 522                        ref->next = NULL;
 523                        newtail = &ref->next;
 524                        continue;
 525                }
 526                else {
 527                        int order = path_match(ref->name, nr_match, match);
 528                        if (order) {
 529                                return_refs[order-1] = ref;
 530                                continue; /* we will link it later */
 531                        }
 532                }
 533                free(ref);
 534        }
 535
 536        if (!args.fetch_all) {
 537                int i;
 538                for (i = 0; i < nr_match; i++) {
 539                        ref = return_refs[i];
 540                        if (ref) {
 541                                *newtail = ref;
 542                                ref->next = NULL;
 543                                newtail = &ref->next;
 544                        }
 545                }
 546                if (return_refs != fastarray)
 547                        free(return_refs);
 548        }
 549        *refs = newlist;
 550}
 551
 552static int everything_local(struct ref **refs, int nr_match, char **match)
 553{
 554        struct ref *ref;
 555        int retval;
 556        unsigned long cutoff = 0;
 557
 558        save_commit_buffer = 0;
 559
 560        for (ref = *refs; ref; ref = ref->next) {
 561                struct object *o;
 562
 563                o = parse_object(ref->old_sha1);
 564                if (!o)
 565                        continue;
 566
 567                /* We already have it -- which may mean that we were
 568                 * in sync with the other side at some time after
 569                 * that (it is OK if we guess wrong here).
 570                 */
 571                if (o->type == OBJ_COMMIT) {
 572                        struct commit *commit = (struct commit *)o;
 573                        if (!cutoff || cutoff < commit->date)
 574                                cutoff = commit->date;
 575                }
 576        }
 577
 578        if (!args.depth) {
 579                for_each_ref(mark_complete, NULL);
 580                if (cutoff)
 581                        mark_recent_complete_commits(cutoff);
 582        }
 583
 584        /*
 585         * Mark all complete remote refs as common refs.
 586         * Don't mark them common yet; the server has to be told so first.
 587         */
 588        for (ref = *refs; ref; ref = ref->next) {
 589                struct object *o = deref_tag(lookup_object(ref->old_sha1),
 590                                             NULL, 0);
 591
 592                if (!o || o->type != OBJ_COMMIT || !(o->flags & COMPLETE))
 593                        continue;
 594
 595                if (!(o->flags & SEEN)) {
 596                        rev_list_push((struct commit *)o, COMMON_REF | SEEN);
 597
 598                        mark_common((struct commit *)o, 1, 1);
 599                }
 600        }
 601
 602        filter_refs(refs, nr_match, match);
 603
 604        for (retval = 1, ref = *refs; ref ; ref = ref->next) {
 605                const unsigned char *remote = ref->old_sha1;
 606                unsigned char local[20];
 607                struct object *o;
 608
 609                o = lookup_object(remote);
 610                if (!o || !(o->flags & COMPLETE)) {
 611                        retval = 0;
 612                        if (!args.verbose)
 613                                continue;
 614                        fprintf(stderr,
 615                                "want %s (%s)\n", sha1_to_hex(remote),
 616                                ref->name);
 617                        continue;
 618                }
 619
 620                hashcpy(ref->new_sha1, local);
 621                if (!args.verbose)
 622                        continue;
 623                fprintf(stderr,
 624                        "already have %s (%s)\n", sha1_to_hex(remote),
 625                        ref->name);
 626        }
 627        return retval;
 628}
 629
 630static int sideband_demux(int in, int out, void *data)
 631{
 632        int *xd = data;
 633
 634        int ret = recv_sideband("fetch-pack", xd[0], out);
 635        close(out);
 636        return ret;
 637}
 638
 639static int get_pack(int xd[2], char **pack_lockfile)
 640{
 641        struct async demux;
 642        const char *argv[20];
 643        char keep_arg[256];
 644        char hdr_arg[256];
 645        const char **av;
 646        int do_keep = args.keep_pack;
 647        struct child_process cmd;
 648
 649        memset(&demux, 0, sizeof(demux));
 650        if (use_sideband) {
 651                /* xd[] is talking with upload-pack; subprocess reads from
 652                 * xd[0], spits out band#2 to stderr, and feeds us band#1
 653                 * through demux->out.
 654                 */
 655                demux.proc = sideband_demux;
 656                demux.data = xd;
 657                demux.out = -1;
 658                if (start_async(&demux))
 659                        die("fetch-pack: unable to fork off sideband"
 660                            " demultiplexer");
 661        }
 662        else
 663                demux.out = xd[0];
 664
 665        memset(&cmd, 0, sizeof(cmd));
 666        cmd.argv = argv;
 667        av = argv;
 668        *hdr_arg = 0;
 669        if (!args.keep_pack && unpack_limit) {
 670                struct pack_header header;
 671
 672                if (read_pack_header(demux.out, &header))
 673                        die("protocol error: bad pack header");
 674                snprintf(hdr_arg, sizeof(hdr_arg),
 675                         "--pack_header=%"PRIu32",%"PRIu32,
 676                         ntohl(header.hdr_version), ntohl(header.hdr_entries));
 677                if (ntohl(header.hdr_entries) < unpack_limit)
 678                        do_keep = 0;
 679                else
 680                        do_keep = 1;
 681        }
 682
 683        if (do_keep) {
 684                if (pack_lockfile)
 685                        cmd.out = -1;
 686                *av++ = "index-pack";
 687                *av++ = "--stdin";
 688                if (!args.quiet && !args.no_progress)
 689                        *av++ = "-v";
 690                if (args.use_thin_pack)
 691                        *av++ = "--fix-thin";
 692                if (args.lock_pack || unpack_limit) {
 693                        int s = sprintf(keep_arg,
 694                                        "--keep=fetch-pack %"PRIuMAX " on ", (uintmax_t) getpid());
 695                        if (gethostname(keep_arg + s, sizeof(keep_arg) - s))
 696                                strcpy(keep_arg + s, "localhost");
 697                        *av++ = keep_arg;
 698                }
 699        }
 700        else {
 701                *av++ = "unpack-objects";
 702                if (args.quiet)
 703                        *av++ = "-q";
 704        }
 705        if (*hdr_arg)
 706                *av++ = hdr_arg;
 707        *av++ = NULL;
 708
 709        cmd.in = demux.out;
 710        cmd.git_cmd = 1;
 711        if (start_command(&cmd))
 712                die("fetch-pack: unable to fork off %s", argv[0]);
 713        if (do_keep && pack_lockfile) {
 714                *pack_lockfile = index_pack_lockfile(cmd.out);
 715                close(cmd.out);
 716        }
 717
 718        if (finish_command(&cmd))
 719                die("%s failed", argv[0]);
 720        if (use_sideband && finish_async(&demux))
 721                die("error in sideband demultiplexer");
 722        return 0;
 723}
 724
 725static struct ref *do_fetch_pack(int fd[2],
 726                const struct ref *orig_ref,
 727                int nr_match,
 728                char **match,
 729                char **pack_lockfile)
 730{
 731        struct ref *ref = copy_ref_list(orig_ref);
 732        unsigned char sha1[20];
 733
 734        if (is_repository_shallow() && !server_supports("shallow"))
 735                die("Server does not support shallow clients");
 736        if (server_supports("multi_ack_detailed")) {
 737                if (args.verbose)
 738                        fprintf(stderr, "Server supports multi_ack_detailed\n");
 739                multi_ack = 2;
 740                if (server_supports("no-done")) {
 741                        if (args.verbose)
 742                                fprintf(stderr, "Server supports no-done\n");
 743                        if (args.stateless_rpc)
 744                                no_done = 1;
 745                }
 746        }
 747        else if (server_supports("multi_ack")) {
 748                if (args.verbose)
 749                        fprintf(stderr, "Server supports multi_ack\n");
 750                multi_ack = 1;
 751        }
 752        if (server_supports("side-band-64k")) {
 753                if (args.verbose)
 754                        fprintf(stderr, "Server supports side-band-64k\n");
 755                use_sideband = 2;
 756        }
 757        else if (server_supports("side-band")) {
 758                if (args.verbose)
 759                        fprintf(stderr, "Server supports side-band\n");
 760                use_sideband = 1;
 761        }
 762        if (server_supports("ofs-delta")) {
 763                if (args.verbose)
 764                        fprintf(stderr, "Server supports ofs-delta\n");
 765        } else
 766                prefer_ofs_delta = 0;
 767        if (everything_local(&ref, nr_match, match)) {
 768                packet_flush(fd[1]);
 769                goto all_done;
 770        }
 771        if (find_common(fd, sha1, ref) < 0)
 772                if (!args.keep_pack)
 773                        /* When cloning, it is not unusual to have
 774                         * no common commit.
 775                         */
 776                        warning("no common commits");
 777
 778        if (args.stateless_rpc)
 779                packet_flush(fd[1]);
 780        if (get_pack(fd, pack_lockfile))
 781                die("git fetch-pack: fetch failed.");
 782
 783 all_done:
 784        return ref;
 785}
 786
 787static int remove_duplicates(int nr_heads, char **heads)
 788{
 789        int src, dst;
 790
 791        for (src = dst = 0; src < nr_heads; src++) {
 792                /* If heads[src] is different from any of
 793                 * heads[0..dst], push it in.
 794                 */
 795                int i;
 796                for (i = 0; i < dst; i++) {
 797                        if (!strcmp(heads[i], heads[src]))
 798                                break;
 799                }
 800                if (i < dst)
 801                        continue;
 802                if (src != dst)
 803                        heads[dst] = heads[src];
 804                dst++;
 805        }
 806        return dst;
 807}
 808
 809static int fetch_pack_config(const char *var, const char *value, void *cb)
 810{
 811        if (strcmp(var, "fetch.unpacklimit") == 0) {
 812                fetch_unpack_limit = git_config_int(var, value);
 813                return 0;
 814        }
 815
 816        if (strcmp(var, "transfer.unpacklimit") == 0) {
 817                transfer_unpack_limit = git_config_int(var, value);
 818                return 0;
 819        }
 820
 821        if (strcmp(var, "repack.usedeltabaseoffset") == 0) {
 822                prefer_ofs_delta = git_config_bool(var, value);
 823                return 0;
 824        }
 825
 826        return git_default_config(var, value, cb);
 827}
 828
 829static struct lock_file lock;
 830
 831static void fetch_pack_setup(void)
 832{
 833        static int did_setup;
 834        if (did_setup)
 835                return;
 836        git_config(fetch_pack_config, NULL);
 837        if (0 <= transfer_unpack_limit)
 838                unpack_limit = transfer_unpack_limit;
 839        else if (0 <= fetch_unpack_limit)
 840                unpack_limit = fetch_unpack_limit;
 841        did_setup = 1;
 842}
 843
 844int cmd_fetch_pack(int argc, const char **argv, const char *prefix)
 845{
 846        int i, ret, nr_heads;
 847        struct ref *ref = NULL;
 848        char *dest = NULL, **heads;
 849        int fd[2];
 850        char *pack_lockfile = NULL;
 851        char **pack_lockfile_ptr = NULL;
 852        struct child_process *conn;
 853
 854        packet_trace_identity("fetch-pack");
 855
 856        nr_heads = 0;
 857        heads = NULL;
 858        for (i = 1; i < argc; i++) {
 859                const char *arg = argv[i];
 860
 861                if (*arg == '-') {
 862                        if (!prefixcmp(arg, "--upload-pack=")) {
 863                                args.uploadpack = arg + 14;
 864                                continue;
 865                        }
 866                        if (!prefixcmp(arg, "--exec=")) {
 867                                args.uploadpack = arg + 7;
 868                                continue;
 869                        }
 870                        if (!strcmp("--quiet", arg) || !strcmp("-q", arg)) {
 871                                args.quiet = 1;
 872                                continue;
 873                        }
 874                        if (!strcmp("--keep", arg) || !strcmp("-k", arg)) {
 875                                args.lock_pack = args.keep_pack;
 876                                args.keep_pack = 1;
 877                                continue;
 878                        }
 879                        if (!strcmp("--thin", arg)) {
 880                                args.use_thin_pack = 1;
 881                                continue;
 882                        }
 883                        if (!strcmp("--include-tag", arg)) {
 884                                args.include_tag = 1;
 885                                continue;
 886                        }
 887                        if (!strcmp("--all", arg)) {
 888                                args.fetch_all = 1;
 889                                continue;
 890                        }
 891                        if (!strcmp("-v", arg)) {
 892                                args.verbose = 1;
 893                                continue;
 894                        }
 895                        if (!prefixcmp(arg, "--depth=")) {
 896                                args.depth = strtol(arg + 8, NULL, 0);
 897                                continue;
 898                        }
 899                        if (!strcmp("--no-progress", arg)) {
 900                                args.no_progress = 1;
 901                                continue;
 902                        }
 903                        if (!strcmp("--stateless-rpc", arg)) {
 904                                args.stateless_rpc = 1;
 905                                continue;
 906                        }
 907                        if (!strcmp("--lock-pack", arg)) {
 908                                args.lock_pack = 1;
 909                                pack_lockfile_ptr = &pack_lockfile;
 910                                continue;
 911                        }
 912                        usage(fetch_pack_usage);
 913                }
 914                dest = (char *)arg;
 915                heads = (char **)(argv + i + 1);
 916                nr_heads = argc - i - 1;
 917                break;
 918        }
 919        if (!dest)
 920                usage(fetch_pack_usage);
 921
 922        if (args.stateless_rpc) {
 923                conn = NULL;
 924                fd[0] = 0;
 925                fd[1] = 1;
 926        } else {
 927                conn = git_connect(fd, (char *)dest, args.uploadpack,
 928                                   args.verbose ? CONNECT_VERBOSE : 0);
 929        }
 930
 931        get_remote_heads(fd[0], &ref, 0, NULL, 0, NULL);
 932
 933        ref = fetch_pack(&args, fd, conn, ref, dest,
 934                nr_heads, heads, pack_lockfile_ptr);
 935        if (pack_lockfile) {
 936                printf("lock %s\n", pack_lockfile);
 937                fflush(stdout);
 938        }
 939        close(fd[0]);
 940        close(fd[1]);
 941        if (finish_connect(conn))
 942                ref = NULL;
 943        ret = !ref;
 944
 945        if (!ret && nr_heads) {
 946                /* If the heads to pull were given, we should have
 947                 * consumed all of them by matching the remote.
 948                 * Otherwise, 'git fetch remote no-such-ref' would
 949                 * silently succeed without issuing an error.
 950                 */
 951                for (i = 0; i < nr_heads; i++)
 952                        if (heads[i] && heads[i][0]) {
 953                                error("no such remote ref %s", heads[i]);
 954                                ret = 1;
 955                        }
 956        }
 957        while (ref) {
 958                printf("%s %s\n",
 959                       sha1_to_hex(ref->old_sha1), ref->name);
 960                ref = ref->next;
 961        }
 962
 963        return ret;
 964}
 965
 966struct ref *fetch_pack(struct fetch_pack_args *my_args,
 967                       int fd[], struct child_process *conn,
 968                       const struct ref *ref,
 969                const char *dest,
 970                int nr_heads,
 971                char **heads,
 972                char **pack_lockfile)
 973{
 974        struct stat st;
 975        struct ref *ref_cpy;
 976
 977        fetch_pack_setup();
 978        if (&args != my_args)
 979                memcpy(&args, my_args, sizeof(args));
 980        if (args.depth > 0) {
 981                if (stat(git_path("shallow"), &st))
 982                        st.st_mtime = 0;
 983        }
 984
 985        if (heads && nr_heads)
 986                nr_heads = remove_duplicates(nr_heads, heads);
 987        if (!ref) {
 988                packet_flush(fd[1]);
 989                die("no matching remote head");
 990        }
 991        ref_cpy = do_fetch_pack(fd, ref, nr_heads, heads, pack_lockfile);
 992
 993        if (args.depth > 0) {
 994                struct cache_time mtime;
 995                struct strbuf sb = STRBUF_INIT;
 996                char *shallow = git_path("shallow");
 997                int fd;
 998
 999                mtime.sec = st.st_mtime;
1000                mtime.nsec = ST_MTIME_NSEC(st);
1001                if (stat(shallow, &st)) {
1002                        if (mtime.sec)
1003                                die("shallow file was removed during fetch");
1004                } else if (st.st_mtime != mtime.sec
1005#ifdef USE_NSEC
1006                                || ST_MTIME_NSEC(st) != mtime.nsec
1007#endif
1008                          )
1009                        die("shallow file was changed during fetch");
1010
1011                fd = hold_lock_file_for_update(&lock, shallow,
1012                                               LOCK_DIE_ON_ERROR);
1013                if (!write_shallow_commits(&sb, 0)
1014                 || write_in_full(fd, sb.buf, sb.len) != sb.len) {
1015                        unlink_or_warn(shallow);
1016                        rollback_lock_file(&lock);
1017                } else {
1018                        commit_lock_file(&lock);
1019                }
1020                strbuf_release(&sb);
1021        }
1022
1023        reprepare_packed_git();
1024        return ref_cpy;
1025}