fetch-pack.con commit Merge branch 'jk/fast-import-empty-ls' (7e39472)
   1#include "cache.h"
   2#include "refs.h"
   3#include "pkt-line.h"
   4#include "commit.h"
   5#include "tag.h"
   6#include "exec_cmd.h"
   7#include "pack.h"
   8#include "sideband.h"
   9#include "fetch-pack.h"
  10#include "remote.h"
  11#include "run-command.h"
  12#include "transport.h"
  13#include "version.h"
  14#include "prio-queue.h"
  15
  16static int transfer_unpack_limit = -1;
  17static int fetch_unpack_limit = -1;
  18static int unpack_limit = 100;
  19static int prefer_ofs_delta = 1;
  20static int no_done;
  21static int fetch_fsck_objects = -1;
  22static int transfer_fsck_objects = -1;
  23static int agent_supported;
  24static struct lock_file shallow_lock;
  25static const char *alternate_shallow_file;
  26
  27#define COMPLETE        (1U << 0)
  28#define COMMON          (1U << 1)
  29#define COMMON_REF      (1U << 2)
  30#define SEEN            (1U << 3)
  31#define POPPED          (1U << 4)
  32
  33static int marked;
  34
  35/*
  36 * After sending this many "have"s if we do not get any new ACK , we
  37 * give up traversing our history.
  38 */
  39#define MAX_IN_VAIN 256
  40
  41static struct prio_queue rev_list = { compare_commits_by_commit_date };
  42static int non_common_revs, multi_ack, use_sideband, allow_tip_sha1_in_want;
  43
  44static void rev_list_push(struct commit *commit, int mark)
  45{
  46        if (!(commit->object.flags & mark)) {
  47                commit->object.flags |= mark;
  48
  49                if (!(commit->object.parsed))
  50                        if (parse_commit(commit))
  51                                return;
  52
  53                prio_queue_put(&rev_list, commit);
  54
  55                if (!(commit->object.flags & COMMON))
  56                        non_common_revs++;
  57        }
  58}
  59
  60static int rev_list_insert_ref(const char *refname, const unsigned char *sha1, int flag, void *cb_data)
  61{
  62        struct object *o = deref_tag(parse_object(sha1), refname, 0);
  63
  64        if (o && o->type == OBJ_COMMIT)
  65                rev_list_push((struct commit *)o, SEEN);
  66
  67        return 0;
  68}
  69
  70static int clear_marks(const char *refname, const unsigned char *sha1, int flag, void *cb_data)
  71{
  72        struct object *o = deref_tag(parse_object(sha1), refname, 0);
  73
  74        if (o && o->type == OBJ_COMMIT)
  75                clear_commit_marks((struct commit *)o,
  76                                   COMMON | COMMON_REF | SEEN | POPPED);
  77        return 0;
  78}
  79
  80/*
  81   This function marks a rev and its ancestors as common.
  82   In some cases, it is desirable to mark only the ancestors (for example
  83   when only the server does not yet know that they are common).
  84*/
  85
  86static void mark_common(struct commit *commit,
  87                int ancestors_only, int dont_parse)
  88{
  89        if (commit != NULL && !(commit->object.flags & COMMON)) {
  90                struct object *o = (struct object *)commit;
  91
  92                if (!ancestors_only)
  93                        o->flags |= COMMON;
  94
  95                if (!(o->flags & SEEN))
  96                        rev_list_push(commit, SEEN);
  97                else {
  98                        struct commit_list *parents;
  99
 100                        if (!ancestors_only && !(o->flags & POPPED))
 101                                non_common_revs--;
 102                        if (!o->parsed && !dont_parse)
 103                                if (parse_commit(commit))
 104                                        return;
 105
 106                        for (parents = commit->parents;
 107                                        parents;
 108                                        parents = parents->next)
 109                                mark_common(parents->item, 0, dont_parse);
 110                }
 111        }
 112}
 113
 114/*
 115  Get the next rev to send, ignoring the common.
 116*/
 117
 118static const unsigned char *get_rev(void)
 119{
 120        struct commit *commit = NULL;
 121
 122        while (commit == NULL) {
 123                unsigned int mark;
 124                struct commit_list *parents;
 125
 126                if (rev_list.nr == 0 || non_common_revs == 0)
 127                        return NULL;
 128
 129                commit = prio_queue_get(&rev_list);
 130                if (!commit->object.parsed)
 131                        parse_commit(commit);
 132                parents = commit->parents;
 133
 134                commit->object.flags |= POPPED;
 135                if (!(commit->object.flags & COMMON))
 136                        non_common_revs--;
 137
 138                if (commit->object.flags & COMMON) {
 139                        /* do not send "have", and ignore ancestors */
 140                        commit = NULL;
 141                        mark = COMMON | SEEN;
 142                } else if (commit->object.flags & COMMON_REF)
 143                        /* send "have", and ignore ancestors */
 144                        mark = COMMON | SEEN;
 145                else
 146                        /* send "have", also for its ancestors */
 147                        mark = SEEN;
 148
 149                while (parents) {
 150                        if (!(parents->item->object.flags & SEEN))
 151                                rev_list_push(parents->item, mark);
 152                        if (mark & COMMON)
 153                                mark_common(parents->item, 1, 0);
 154                        parents = parents->next;
 155                }
 156        }
 157
 158        return commit->object.sha1;
 159}
 160
 161enum ack_type {
 162        NAK = 0,
 163        ACK,
 164        ACK_continue,
 165        ACK_common,
 166        ACK_ready
 167};
 168
 169static void consume_shallow_list(struct fetch_pack_args *args, int fd)
 170{
 171        if (args->stateless_rpc && args->depth > 0) {
 172                /* If we sent a depth we will get back "duplicate"
 173                 * shallow and unshallow commands every time there
 174                 * is a block of have lines exchanged.
 175                 */
 176                char *line;
 177                while ((line = packet_read_line(fd, NULL))) {
 178                        if (!prefixcmp(line, "shallow "))
 179                                continue;
 180                        if (!prefixcmp(line, "unshallow "))
 181                                continue;
 182                        die("git fetch-pack: expected shallow list");
 183                }
 184        }
 185}
 186
 187struct write_shallow_data {
 188        struct strbuf *out;
 189        int use_pack_protocol;
 190        int count;
 191};
 192
 193static int write_one_shallow(const struct commit_graft *graft, void *cb_data)
 194{
 195        struct write_shallow_data *data = cb_data;
 196        const char *hex = sha1_to_hex(graft->sha1);
 197        data->count++;
 198        if (data->use_pack_protocol)
 199                packet_buf_write(data->out, "shallow %s", hex);
 200        else {
 201                strbuf_addstr(data->out, hex);
 202                strbuf_addch(data->out, '\n');
 203        }
 204        return 0;
 205}
 206
 207static int write_shallow_commits(struct strbuf *out, int use_pack_protocol)
 208{
 209        struct write_shallow_data data;
 210        data.out = out;
 211        data.use_pack_protocol = use_pack_protocol;
 212        data.count = 0;
 213        for_each_commit_graft(write_one_shallow, &data);
 214        return data.count;
 215}
 216
 217static enum ack_type get_ack(int fd, unsigned char *result_sha1)
 218{
 219        int len;
 220        char *line = packet_read_line(fd, &len);
 221
 222        if (!len)
 223                die("git fetch-pack: expected ACK/NAK, got EOF");
 224        if (!strcmp(line, "NAK"))
 225                return NAK;
 226        if (!prefixcmp(line, "ACK ")) {
 227                if (!get_sha1_hex(line+4, result_sha1)) {
 228                        if (len < 45)
 229                                return ACK;
 230                        if (strstr(line+45, "continue"))
 231                                return ACK_continue;
 232                        if (strstr(line+45, "common"))
 233                                return ACK_common;
 234                        if (strstr(line+45, "ready"))
 235                                return ACK_ready;
 236                        return ACK;
 237                }
 238        }
 239        die("git fetch_pack: expected ACK/NAK, got '%s'", line);
 240}
 241
 242static void send_request(struct fetch_pack_args *args,
 243                         int fd, struct strbuf *buf)
 244{
 245        if (args->stateless_rpc) {
 246                send_sideband(fd, -1, buf->buf, buf->len, LARGE_PACKET_MAX);
 247                packet_flush(fd);
 248        } else
 249                write_or_die(fd, buf->buf, buf->len);
 250}
 251
 252static void insert_one_alternate_ref(const struct ref *ref, void *unused)
 253{
 254        rev_list_insert_ref(NULL, ref->old_sha1, 0, NULL);
 255}
 256
 257#define INITIAL_FLUSH 16
 258#define PIPESAFE_FLUSH 32
 259#define LARGE_FLUSH 1024
 260
 261static int next_flush(struct fetch_pack_args *args, int count)
 262{
 263        int flush_limit = args->stateless_rpc ? LARGE_FLUSH : PIPESAFE_FLUSH;
 264
 265        if (count < flush_limit)
 266                count <<= 1;
 267        else
 268                count += flush_limit;
 269        return count;
 270}
 271
 272static int find_common(struct fetch_pack_args *args,
 273                       int fd[2], unsigned char *result_sha1,
 274                       struct ref *refs)
 275{
 276        int fetching;
 277        int count = 0, flushes = 0, flush_at = INITIAL_FLUSH, retval;
 278        const unsigned char *sha1;
 279        unsigned in_vain = 0;
 280        int got_continue = 0;
 281        int got_ready = 0;
 282        struct strbuf req_buf = STRBUF_INIT;
 283        size_t state_len = 0;
 284
 285        if (args->stateless_rpc && multi_ack == 1)
 286                die("--stateless-rpc requires multi_ack_detailed");
 287        if (marked)
 288                for_each_ref(clear_marks, NULL);
 289        marked = 1;
 290
 291        for_each_ref(rev_list_insert_ref, NULL);
 292        for_each_alternate_ref(insert_one_alternate_ref, NULL);
 293
 294        fetching = 0;
 295        for ( ; refs ; refs = refs->next) {
 296                unsigned char *remote = refs->old_sha1;
 297                const char *remote_hex;
 298                struct object *o;
 299
 300                /*
 301                 * If that object is complete (i.e. it is an ancestor of a
 302                 * local ref), we tell them we have it but do not have to
 303                 * tell them about its ancestors, which they already know
 304                 * about.
 305                 *
 306                 * We use lookup_object here because we are only
 307                 * interested in the case we *know* the object is
 308                 * reachable and we have already scanned it.
 309                 */
 310                if (((o = lookup_object(remote)) != NULL) &&
 311                                (o->flags & COMPLETE)) {
 312                        continue;
 313                }
 314
 315                remote_hex = sha1_to_hex(remote);
 316                if (!fetching) {
 317                        struct strbuf c = STRBUF_INIT;
 318                        if (multi_ack == 2)     strbuf_addstr(&c, " multi_ack_detailed");
 319                        if (multi_ack == 1)     strbuf_addstr(&c, " multi_ack");
 320                        if (no_done)            strbuf_addstr(&c, " no-done");
 321                        if (use_sideband == 2)  strbuf_addstr(&c, " side-band-64k");
 322                        if (use_sideband == 1)  strbuf_addstr(&c, " side-band");
 323                        if (args->use_thin_pack) strbuf_addstr(&c, " thin-pack");
 324                        if (args->no_progress)   strbuf_addstr(&c, " no-progress");
 325                        if (args->include_tag)   strbuf_addstr(&c, " include-tag");
 326                        if (prefer_ofs_delta)   strbuf_addstr(&c, " ofs-delta");
 327                        if (agent_supported)    strbuf_addf(&c, " agent=%s",
 328                                                            git_user_agent_sanitized());
 329                        packet_buf_write(&req_buf, "want %s%s\n", remote_hex, c.buf);
 330                        strbuf_release(&c);
 331                } else
 332                        packet_buf_write(&req_buf, "want %s\n", remote_hex);
 333                fetching++;
 334        }
 335
 336        if (!fetching) {
 337                strbuf_release(&req_buf);
 338                packet_flush(fd[1]);
 339                return 1;
 340        }
 341
 342        if (is_repository_shallow())
 343                write_shallow_commits(&req_buf, 1);
 344        if (args->depth > 0)
 345                packet_buf_write(&req_buf, "deepen %d", args->depth);
 346        packet_buf_flush(&req_buf);
 347        state_len = req_buf.len;
 348
 349        if (args->depth > 0) {
 350                char *line;
 351                unsigned char sha1[20];
 352
 353                send_request(args, fd[1], &req_buf);
 354                while ((line = packet_read_line(fd[0], NULL))) {
 355                        if (!prefixcmp(line, "shallow ")) {
 356                                if (get_sha1_hex(line + 8, sha1))
 357                                        die("invalid shallow line: %s", line);
 358                                register_shallow(sha1);
 359                                continue;
 360                        }
 361                        if (!prefixcmp(line, "unshallow ")) {
 362                                if (get_sha1_hex(line + 10, sha1))
 363                                        die("invalid unshallow line: %s", line);
 364                                if (!lookup_object(sha1))
 365                                        die("object not found: %s", line);
 366                                /* make sure that it is parsed as shallow */
 367                                if (!parse_object(sha1))
 368                                        die("error in object: %s", line);
 369                                if (unregister_shallow(sha1))
 370                                        die("no shallow found: %s", line);
 371                                continue;
 372                        }
 373                        die("expected shallow/unshallow, got %s", line);
 374                }
 375        } else if (!args->stateless_rpc)
 376                send_request(args, fd[1], &req_buf);
 377
 378        if (!args->stateless_rpc) {
 379                /* If we aren't using the stateless-rpc interface
 380                 * we don't need to retain the headers.
 381                 */
 382                strbuf_setlen(&req_buf, 0);
 383                state_len = 0;
 384        }
 385
 386        flushes = 0;
 387        retval = -1;
 388        while ((sha1 = get_rev())) {
 389                packet_buf_write(&req_buf, "have %s\n", sha1_to_hex(sha1));
 390                if (args->verbose)
 391                        fprintf(stderr, "have %s\n", sha1_to_hex(sha1));
 392                in_vain++;
 393                if (flush_at <= ++count) {
 394                        int ack;
 395
 396                        packet_buf_flush(&req_buf);
 397                        send_request(args, fd[1], &req_buf);
 398                        strbuf_setlen(&req_buf, state_len);
 399                        flushes++;
 400                        flush_at = next_flush(args, count);
 401
 402                        /*
 403                         * We keep one window "ahead" of the other side, and
 404                         * will wait for an ACK only on the next one
 405                         */
 406                        if (!args->stateless_rpc && count == INITIAL_FLUSH)
 407                                continue;
 408
 409                        consume_shallow_list(args, fd[0]);
 410                        do {
 411                                ack = get_ack(fd[0], result_sha1);
 412                                if (args->verbose && ack)
 413                                        fprintf(stderr, "got ack %d %s\n", ack,
 414                                                        sha1_to_hex(result_sha1));
 415                                switch (ack) {
 416                                case ACK:
 417                                        flushes = 0;
 418                                        multi_ack = 0;
 419                                        retval = 0;
 420                                        goto done;
 421                                case ACK_common:
 422                                case ACK_ready:
 423                                case ACK_continue: {
 424                                        struct commit *commit =
 425                                                lookup_commit(result_sha1);
 426                                        if (!commit)
 427                                                die("invalid commit %s", sha1_to_hex(result_sha1));
 428                                        if (args->stateless_rpc
 429                                         && ack == ACK_common
 430                                         && !(commit->object.flags & COMMON)) {
 431                                                /* We need to replay the have for this object
 432                                                 * on the next RPC request so the peer knows
 433                                                 * it is in common with us.
 434                                                 */
 435                                                const char *hex = sha1_to_hex(result_sha1);
 436                                                packet_buf_write(&req_buf, "have %s\n", hex);
 437                                                state_len = req_buf.len;
 438                                        }
 439                                        mark_common(commit, 0, 1);
 440                                        retval = 0;
 441                                        in_vain = 0;
 442                                        got_continue = 1;
 443                                        if (ack == ACK_ready) {
 444                                                clear_prio_queue(&rev_list);
 445                                                got_ready = 1;
 446                                        }
 447                                        break;
 448                                        }
 449                                }
 450                        } while (ack);
 451                        flushes--;
 452                        if (got_continue && MAX_IN_VAIN < in_vain) {
 453                                if (args->verbose)
 454                                        fprintf(stderr, "giving up\n");
 455                                break; /* give up */
 456                        }
 457                }
 458        }
 459done:
 460        if (!got_ready || !no_done) {
 461                packet_buf_write(&req_buf, "done\n");
 462                send_request(args, fd[1], &req_buf);
 463        }
 464        if (args->verbose)
 465                fprintf(stderr, "done\n");
 466        if (retval != 0) {
 467                multi_ack = 0;
 468                flushes++;
 469        }
 470        strbuf_release(&req_buf);
 471
 472        consume_shallow_list(args, fd[0]);
 473        while (flushes || multi_ack) {
 474                int ack = get_ack(fd[0], result_sha1);
 475                if (ack) {
 476                        if (args->verbose)
 477                                fprintf(stderr, "got ack (%d) %s\n", ack,
 478                                        sha1_to_hex(result_sha1));
 479                        if (ack == ACK)
 480                                return 0;
 481                        multi_ack = 1;
 482                        continue;
 483                }
 484                flushes--;
 485        }
 486        /* it is no error to fetch into a completely empty repo */
 487        return count ? retval : 0;
 488}
 489
 490static struct commit_list *complete;
 491
 492static int mark_complete(const char *refname, const unsigned char *sha1, int flag, void *cb_data)
 493{
 494        struct object *o = parse_object(sha1);
 495
 496        while (o && o->type == OBJ_TAG) {
 497                struct tag *t = (struct tag *) o;
 498                if (!t->tagged)
 499                        break; /* broken repository */
 500                o->flags |= COMPLETE;
 501                o = parse_object(t->tagged->sha1);
 502        }
 503        if (o && o->type == OBJ_COMMIT) {
 504                struct commit *commit = (struct commit *)o;
 505                if (!(commit->object.flags & COMPLETE)) {
 506                        commit->object.flags |= COMPLETE;
 507                        commit_list_insert(commit, &complete);
 508                }
 509        }
 510        return 0;
 511}
 512
 513static void mark_recent_complete_commits(struct fetch_pack_args *args,
 514                                         unsigned long cutoff)
 515{
 516        while (complete && cutoff <= complete->item->date) {
 517                if (args->verbose)
 518                        fprintf(stderr, "Marking %s as complete\n",
 519                                sha1_to_hex(complete->item->object.sha1));
 520                pop_most_recent_commit(&complete, COMPLETE);
 521        }
 522}
 523
 524static void filter_refs(struct fetch_pack_args *args,
 525                        struct ref **refs,
 526                        struct ref **sought, int nr_sought)
 527{
 528        struct ref *newlist = NULL;
 529        struct ref **newtail = &newlist;
 530        struct ref *ref, *next;
 531        int i;
 532
 533        i = 0;
 534        for (ref = *refs; ref; ref = next) {
 535                int keep = 0;
 536                next = ref->next;
 537
 538                if (!memcmp(ref->name, "refs/", 5) &&
 539                    check_refname_format(ref->name + 5, 0))
 540                        ; /* trash */
 541                else {
 542                        while (i < nr_sought) {
 543                                int cmp = strcmp(ref->name, sought[i]->name);
 544                                if (cmp < 0)
 545                                        break; /* definitely do not have it */
 546                                else if (cmp == 0) {
 547                                        keep = 1; /* definitely have it */
 548                                        sought[i]->matched = 1;
 549                                }
 550                                i++;
 551                        }
 552                }
 553
 554                if (!keep && args->fetch_all &&
 555                    (!args->depth || prefixcmp(ref->name, "refs/tags/")))
 556                        keep = 1;
 557
 558                if (keep) {
 559                        *newtail = ref;
 560                        ref->next = NULL;
 561                        newtail = &ref->next;
 562                } else {
 563                        free(ref);
 564                }
 565        }
 566
 567        /* Append unmatched requests to the list */
 568        if (allow_tip_sha1_in_want) {
 569                for (i = 0; i < nr_sought; i++) {
 570                        ref = sought[i];
 571                        if (ref->matched)
 572                                continue;
 573                        if (get_sha1_hex(ref->name, ref->old_sha1))
 574                                continue;
 575
 576                        ref->matched = 1;
 577                        *newtail = ref;
 578                        ref->next = NULL;
 579                        newtail = &ref->next;
 580                }
 581        }
 582        *refs = newlist;
 583}
 584
 585static void mark_alternate_complete(const struct ref *ref, void *unused)
 586{
 587        mark_complete(NULL, ref->old_sha1, 0, NULL);
 588}
 589
 590static int everything_local(struct fetch_pack_args *args,
 591                            struct ref **refs,
 592                            struct ref **sought, int nr_sought)
 593{
 594        struct ref *ref;
 595        int retval;
 596        unsigned long cutoff = 0;
 597
 598        save_commit_buffer = 0;
 599
 600        for (ref = *refs; ref; ref = ref->next) {
 601                struct object *o;
 602
 603                if (!has_sha1_file(ref->old_sha1))
 604                        continue;
 605
 606                o = parse_object(ref->old_sha1);
 607                if (!o)
 608                        continue;
 609
 610                /* We already have it -- which may mean that we were
 611                 * in sync with the other side at some time after
 612                 * that (it is OK if we guess wrong here).
 613                 */
 614                if (o->type == OBJ_COMMIT) {
 615                        struct commit *commit = (struct commit *)o;
 616                        if (!cutoff || cutoff < commit->date)
 617                                cutoff = commit->date;
 618                }
 619        }
 620
 621        if (!args->depth) {
 622                for_each_ref(mark_complete, NULL);
 623                for_each_alternate_ref(mark_alternate_complete, NULL);
 624                commit_list_sort_by_date(&complete);
 625                if (cutoff)
 626                        mark_recent_complete_commits(args, cutoff);
 627        }
 628
 629        /*
 630         * Mark all complete remote refs as common refs.
 631         * Don't mark them common yet; the server has to be told so first.
 632         */
 633        for (ref = *refs; ref; ref = ref->next) {
 634                struct object *o = deref_tag(lookup_object(ref->old_sha1),
 635                                             NULL, 0);
 636
 637                if (!o || o->type != OBJ_COMMIT || !(o->flags & COMPLETE))
 638                        continue;
 639
 640                if (!(o->flags & SEEN)) {
 641                        rev_list_push((struct commit *)o, COMMON_REF | SEEN);
 642
 643                        mark_common((struct commit *)o, 1, 1);
 644                }
 645        }
 646
 647        filter_refs(args, refs, sought, nr_sought);
 648
 649        for (retval = 1, ref = *refs; ref ; ref = ref->next) {
 650                const unsigned char *remote = ref->old_sha1;
 651                unsigned char local[20];
 652                struct object *o;
 653
 654                o = lookup_object(remote);
 655                if (!o || !(o->flags & COMPLETE)) {
 656                        retval = 0;
 657                        if (!args->verbose)
 658                                continue;
 659                        fprintf(stderr,
 660                                "want %s (%s)\n", sha1_to_hex(remote),
 661                                ref->name);
 662                        continue;
 663                }
 664
 665                hashcpy(ref->new_sha1, local);
 666                if (!args->verbose)
 667                        continue;
 668                fprintf(stderr,
 669                        "already have %s (%s)\n", sha1_to_hex(remote),
 670                        ref->name);
 671        }
 672        return retval;
 673}
 674
 675static int sideband_demux(int in, int out, void *data)
 676{
 677        int *xd = data;
 678
 679        int ret = recv_sideband("fetch-pack", xd[0], out);
 680        close(out);
 681        return ret;
 682}
 683
 684static int get_pack(struct fetch_pack_args *args,
 685                    int xd[2], char **pack_lockfile)
 686{
 687        struct async demux;
 688        const char *argv[22];
 689        char keep_arg[256];
 690        char hdr_arg[256];
 691        const char **av;
 692        int do_keep = args->keep_pack;
 693        struct child_process cmd;
 694        int ret;
 695
 696        memset(&demux, 0, sizeof(demux));
 697        if (use_sideband) {
 698                /* xd[] is talking with upload-pack; subprocess reads from
 699                 * xd[0], spits out band#2 to stderr, and feeds us band#1
 700                 * through demux->out.
 701                 */
 702                demux.proc = sideband_demux;
 703                demux.data = xd;
 704                demux.out = -1;
 705                if (start_async(&demux))
 706                        die("fetch-pack: unable to fork off sideband"
 707                            " demultiplexer");
 708        }
 709        else
 710                demux.out = xd[0];
 711
 712        memset(&cmd, 0, sizeof(cmd));
 713        cmd.argv = argv;
 714        av = argv;
 715        *hdr_arg = 0;
 716        if (!args->keep_pack && unpack_limit) {
 717                struct pack_header header;
 718
 719                if (read_pack_header(demux.out, &header))
 720                        die("protocol error: bad pack header");
 721                snprintf(hdr_arg, sizeof(hdr_arg),
 722                         "--pack_header=%"PRIu32",%"PRIu32,
 723                         ntohl(header.hdr_version), ntohl(header.hdr_entries));
 724                if (ntohl(header.hdr_entries) < unpack_limit)
 725                        do_keep = 0;
 726                else
 727                        do_keep = 1;
 728        }
 729
 730        if (alternate_shallow_file) {
 731                *av++ = "--shallow-file";
 732                *av++ = alternate_shallow_file;
 733        }
 734
 735        if (do_keep) {
 736                if (pack_lockfile)
 737                        cmd.out = -1;
 738                *av++ = "index-pack";
 739                *av++ = "--stdin";
 740                if (!args->quiet && !args->no_progress)
 741                        *av++ = "-v";
 742                if (args->use_thin_pack)
 743                        *av++ = "--fix-thin";
 744                if (args->lock_pack || unpack_limit) {
 745                        int s = sprintf(keep_arg,
 746                                        "--keep=fetch-pack %"PRIuMAX " on ", (uintmax_t) getpid());
 747                        if (gethostname(keep_arg + s, sizeof(keep_arg) - s))
 748                                strcpy(keep_arg + s, "localhost");
 749                        *av++ = keep_arg;
 750                }
 751                if (args->check_self_contained_and_connected)
 752                        *av++ = "--check-self-contained-and-connected";
 753        }
 754        else {
 755                *av++ = "unpack-objects";
 756                if (args->quiet || args->no_progress)
 757                        *av++ = "-q";
 758                args->check_self_contained_and_connected = 0;
 759        }
 760        if (*hdr_arg)
 761                *av++ = hdr_arg;
 762        if (fetch_fsck_objects >= 0
 763            ? fetch_fsck_objects
 764            : transfer_fsck_objects >= 0
 765            ? transfer_fsck_objects
 766            : 0)
 767                *av++ = "--strict";
 768        *av++ = NULL;
 769
 770        cmd.in = demux.out;
 771        cmd.git_cmd = 1;
 772        if (start_command(&cmd))
 773                die("fetch-pack: unable to fork off %s", argv[0]);
 774        if (do_keep && pack_lockfile) {
 775                *pack_lockfile = index_pack_lockfile(cmd.out);
 776                close(cmd.out);
 777        }
 778
 779        ret = finish_command(&cmd);
 780        if (!ret || (args->check_self_contained_and_connected && ret == 1))
 781                args->self_contained_and_connected =
 782                        args->check_self_contained_and_connected &&
 783                        ret == 0;
 784        else
 785                die("%s failed", argv[0]);
 786        if (use_sideband && finish_async(&demux))
 787                die("error in sideband demultiplexer");
 788        return 0;
 789}
 790
 791static int cmp_ref_by_name(const void *a_, const void *b_)
 792{
 793        const struct ref *a = *((const struct ref **)a_);
 794        const struct ref *b = *((const struct ref **)b_);
 795        return strcmp(a->name, b->name);
 796}
 797
 798static void setup_alternate_shallow(void)
 799{
 800        struct strbuf sb = STRBUF_INIT;
 801        int fd;
 802
 803        check_shallow_file_for_update();
 804        fd = hold_lock_file_for_update(&shallow_lock, git_path("shallow"),
 805                                       LOCK_DIE_ON_ERROR);
 806        if (write_shallow_commits(&sb, 0)) {
 807                if (write_in_full(fd, sb.buf, sb.len) != sb.len)
 808                        die_errno("failed to write to %s", shallow_lock.filename);
 809                alternate_shallow_file = shallow_lock.filename;
 810        } else
 811                /*
 812                 * is_repository_shallow() sees empty string as "no
 813                 * shallow file".
 814                 */
 815                alternate_shallow_file = "";
 816        strbuf_release(&sb);
 817}
 818
 819static struct ref *do_fetch_pack(struct fetch_pack_args *args,
 820                                 int fd[2],
 821                                 const struct ref *orig_ref,
 822                                 struct ref **sought, int nr_sought,
 823                                 char **pack_lockfile)
 824{
 825        struct ref *ref = copy_ref_list(orig_ref);
 826        unsigned char sha1[20];
 827        const char *agent_feature;
 828        int agent_len;
 829
 830        sort_ref_list(&ref, ref_compare_name);
 831        qsort(sought, nr_sought, sizeof(*sought), cmp_ref_by_name);
 832
 833        if (is_repository_shallow() && !server_supports("shallow"))
 834                die("Server does not support shallow clients");
 835        if (server_supports("multi_ack_detailed")) {
 836                if (args->verbose)
 837                        fprintf(stderr, "Server supports multi_ack_detailed\n");
 838                multi_ack = 2;
 839                if (server_supports("no-done")) {
 840                        if (args->verbose)
 841                                fprintf(stderr, "Server supports no-done\n");
 842                        if (args->stateless_rpc)
 843                                no_done = 1;
 844                }
 845        }
 846        else if (server_supports("multi_ack")) {
 847                if (args->verbose)
 848                        fprintf(stderr, "Server supports multi_ack\n");
 849                multi_ack = 1;
 850        }
 851        if (server_supports("side-band-64k")) {
 852                if (args->verbose)
 853                        fprintf(stderr, "Server supports side-band-64k\n");
 854                use_sideband = 2;
 855        }
 856        else if (server_supports("side-band")) {
 857                if (args->verbose)
 858                        fprintf(stderr, "Server supports side-band\n");
 859                use_sideband = 1;
 860        }
 861        if (server_supports("allow-tip-sha1-in-want")) {
 862                if (args->verbose)
 863                        fprintf(stderr, "Server supports allow-tip-sha1-in-want\n");
 864                allow_tip_sha1_in_want = 1;
 865        }
 866        if (!server_supports("thin-pack"))
 867                args->use_thin_pack = 0;
 868        if (!server_supports("no-progress"))
 869                args->no_progress = 0;
 870        if (!server_supports("include-tag"))
 871                args->include_tag = 0;
 872        if (server_supports("ofs-delta")) {
 873                if (args->verbose)
 874                        fprintf(stderr, "Server supports ofs-delta\n");
 875        } else
 876                prefer_ofs_delta = 0;
 877
 878        if ((agent_feature = server_feature_value("agent", &agent_len))) {
 879                agent_supported = 1;
 880                if (args->verbose && agent_len)
 881                        fprintf(stderr, "Server version is %.*s\n",
 882                                agent_len, agent_feature);
 883        }
 884
 885        if (everything_local(args, &ref, sought, nr_sought)) {
 886                packet_flush(fd[1]);
 887                goto all_done;
 888        }
 889        if (find_common(args, fd, sha1, ref) < 0)
 890                if (!args->keep_pack)
 891                        /* When cloning, it is not unusual to have
 892                         * no common commit.
 893                         */
 894                        warning("no common commits");
 895
 896        if (args->stateless_rpc)
 897                packet_flush(fd[1]);
 898        if (args->depth > 0)
 899                setup_alternate_shallow();
 900        else
 901                alternate_shallow_file = NULL;
 902        if (get_pack(args, fd, pack_lockfile))
 903                die("git fetch-pack: fetch failed.");
 904
 905 all_done:
 906        return ref;
 907}
 908
 909static int fetch_pack_config(const char *var, const char *value, void *cb)
 910{
 911        if (strcmp(var, "fetch.unpacklimit") == 0) {
 912                fetch_unpack_limit = git_config_int(var, value);
 913                return 0;
 914        }
 915
 916        if (strcmp(var, "transfer.unpacklimit") == 0) {
 917                transfer_unpack_limit = git_config_int(var, value);
 918                return 0;
 919        }
 920
 921        if (strcmp(var, "repack.usedeltabaseoffset") == 0) {
 922                prefer_ofs_delta = git_config_bool(var, value);
 923                return 0;
 924        }
 925
 926        if (!strcmp(var, "fetch.fsckobjects")) {
 927                fetch_fsck_objects = git_config_bool(var, value);
 928                return 0;
 929        }
 930
 931        if (!strcmp(var, "transfer.fsckobjects")) {
 932                transfer_fsck_objects = git_config_bool(var, value);
 933                return 0;
 934        }
 935
 936        return git_default_config(var, value, cb);
 937}
 938
 939static void fetch_pack_setup(void)
 940{
 941        static int did_setup;
 942        if (did_setup)
 943                return;
 944        git_config(fetch_pack_config, NULL);
 945        if (0 <= transfer_unpack_limit)
 946                unpack_limit = transfer_unpack_limit;
 947        else if (0 <= fetch_unpack_limit)
 948                unpack_limit = fetch_unpack_limit;
 949        did_setup = 1;
 950}
 951
 952static int remove_duplicates_in_refs(struct ref **ref, int nr)
 953{
 954        struct string_list names = STRING_LIST_INIT_NODUP;
 955        int src, dst;
 956
 957        for (src = dst = 0; src < nr; src++) {
 958                struct string_list_item *item;
 959                item = string_list_insert(&names, ref[src]->name);
 960                if (item->util)
 961                        continue; /* already have it */
 962                item->util = ref[src];
 963                if (src != dst)
 964                        ref[dst] = ref[src];
 965                dst++;
 966        }
 967        for (src = dst; src < nr; src++)
 968                ref[src] = NULL;
 969        string_list_clear(&names, 0);
 970        return dst;
 971}
 972
 973struct ref *fetch_pack(struct fetch_pack_args *args,
 974                       int fd[], struct child_process *conn,
 975                       const struct ref *ref,
 976                       const char *dest,
 977                       struct ref **sought, int nr_sought,
 978                       char **pack_lockfile)
 979{
 980        struct ref *ref_cpy;
 981
 982        fetch_pack_setup();
 983        if (nr_sought)
 984                nr_sought = remove_duplicates_in_refs(sought, nr_sought);
 985
 986        if (!ref) {
 987                packet_flush(fd[1]);
 988                die("no matching remote head");
 989        }
 990        ref_cpy = do_fetch_pack(args, fd, ref, sought, nr_sought, pack_lockfile);
 991
 992        if (args->depth > 0 && alternate_shallow_file) {
 993                if (*alternate_shallow_file == '\0') { /* --unshallow */
 994                        unlink_or_warn(git_path("shallow"));
 995                        rollback_lock_file(&shallow_lock);
 996                } else
 997                        commit_lock_file(&shallow_lock);
 998        }
 999
1000        reprepare_packed_git();
1001        return ref_cpy;
1002}