fetch-pack.con commit Merge branch 'nd/warn-more-for-devs' (e998e7a)
   1#include "cache.h"
   2#include "repository.h"
   3#include "config.h"
   4#include "lockfile.h"
   5#include "refs.h"
   6#include "pkt-line.h"
   7#include "commit.h"
   8#include "tag.h"
   9#include "exec-cmd.h"
  10#include "pack.h"
  11#include "sideband.h"
  12#include "fetch-pack.h"
  13#include "remote.h"
  14#include "run-command.h"
  15#include "connect.h"
  16#include "transport.h"
  17#include "version.h"
  18#include "prio-queue.h"
  19#include "sha1-array.h"
  20#include "oidset.h"
  21#include "packfile.h"
  22
  23static int transfer_unpack_limit = -1;
  24static int fetch_unpack_limit = -1;
  25static int unpack_limit = 100;
  26static int prefer_ofs_delta = 1;
  27static int no_done;
  28static int deepen_since_ok;
  29static int deepen_not_ok;
  30static int fetch_fsck_objects = -1;
  31static int transfer_fsck_objects = -1;
  32static int agent_supported;
  33static int server_supports_filtering;
  34static struct lock_file shallow_lock;
  35static const char *alternate_shallow_file;
  36
  37/* Remember to update object flag allocation in object.h */
  38#define COMPLETE        (1U << 0)
  39#define COMMON          (1U << 1)
  40#define COMMON_REF      (1U << 2)
  41#define SEEN            (1U << 3)
  42#define POPPED          (1U << 4)
  43#define ALTERNATE       (1U << 5)
  44
  45static int marked;
  46
  47/*
  48 * After sending this many "have"s if we do not get any new ACK , we
  49 * give up traversing our history.
  50 */
  51#define MAX_IN_VAIN 256
  52
  53static struct prio_queue rev_list = { compare_commits_by_commit_date };
  54static int non_common_revs, multi_ack, use_sideband;
  55/* Allow specifying sha1 if it is a ref tip. */
  56#define ALLOW_TIP_SHA1  01
  57/* Allow request of a sha1 if it is reachable from a ref (possibly hidden ref). */
  58#define ALLOW_REACHABLE_SHA1    02
  59static unsigned int allow_unadvertised_object_request;
  60
  61__attribute__((format (printf, 2, 3)))
  62static inline void print_verbose(const struct fetch_pack_args *args,
  63                                 const char *fmt, ...)
  64{
  65        va_list params;
  66
  67        if (!args->verbose)
  68                return;
  69
  70        va_start(params, fmt);
  71        vfprintf(stderr, fmt, params);
  72        va_end(params);
  73        fputc('\n', stderr);
  74}
  75
  76struct alternate_object_cache {
  77        struct object **items;
  78        size_t nr, alloc;
  79};
  80
  81static void cache_one_alternate(const char *refname,
  82                                const struct object_id *oid,
  83                                void *vcache)
  84{
  85        struct alternate_object_cache *cache = vcache;
  86        struct object *obj = parse_object(oid);
  87
  88        if (!obj || (obj->flags & ALTERNATE))
  89                return;
  90
  91        obj->flags |= ALTERNATE;
  92        ALLOC_GROW(cache->items, cache->nr + 1, cache->alloc);
  93        cache->items[cache->nr++] = obj;
  94}
  95
  96static void for_each_cached_alternate(void (*cb)(struct object *))
  97{
  98        static int initialized;
  99        static struct alternate_object_cache cache;
 100        size_t i;
 101
 102        if (!initialized) {
 103                for_each_alternate_ref(cache_one_alternate, &cache);
 104                initialized = 1;
 105        }
 106
 107        for (i = 0; i < cache.nr; i++)
 108                cb(cache.items[i]);
 109}
 110
 111static void rev_list_push(struct commit *commit, int mark)
 112{
 113        if (!(commit->object.flags & mark)) {
 114                commit->object.flags |= mark;
 115
 116                if (parse_commit(commit))
 117                        return;
 118
 119                prio_queue_put(&rev_list, commit);
 120
 121                if (!(commit->object.flags & COMMON))
 122                        non_common_revs++;
 123        }
 124}
 125
 126static int rev_list_insert_ref(const char *refname, const struct object_id *oid)
 127{
 128        struct object *o = deref_tag(parse_object(oid), refname, 0);
 129
 130        if (o && o->type == OBJ_COMMIT)
 131                rev_list_push((struct commit *)o, SEEN);
 132
 133        return 0;
 134}
 135
 136static int rev_list_insert_ref_oid(const char *refname, const struct object_id *oid,
 137                                   int flag, void *cb_data)
 138{
 139        return rev_list_insert_ref(refname, oid);
 140}
 141
 142static int clear_marks(const char *refname, const struct object_id *oid,
 143                       int flag, void *cb_data)
 144{
 145        struct object *o = deref_tag(parse_object(oid), refname, 0);
 146
 147        if (o && o->type == OBJ_COMMIT)
 148                clear_commit_marks((struct commit *)o,
 149                                   COMMON | COMMON_REF | SEEN | POPPED);
 150        return 0;
 151}
 152
 153/*
 154   This function marks a rev and its ancestors as common.
 155   In some cases, it is desirable to mark only the ancestors (for example
 156   when only the server does not yet know that they are common).
 157*/
 158
 159static void mark_common(struct commit *commit,
 160                int ancestors_only, int dont_parse)
 161{
 162        if (commit != NULL && !(commit->object.flags & COMMON)) {
 163                struct object *o = (struct object *)commit;
 164
 165                if (!ancestors_only)
 166                        o->flags |= COMMON;
 167
 168                if (!(o->flags & SEEN))
 169                        rev_list_push(commit, SEEN);
 170                else {
 171                        struct commit_list *parents;
 172
 173                        if (!ancestors_only && !(o->flags & POPPED))
 174                                non_common_revs--;
 175                        if (!o->parsed && !dont_parse)
 176                                if (parse_commit(commit))
 177                                        return;
 178
 179                        for (parents = commit->parents;
 180                                        parents;
 181                                        parents = parents->next)
 182                                mark_common(parents->item, 0, dont_parse);
 183                }
 184        }
 185}
 186
 187/*
 188  Get the next rev to send, ignoring the common.
 189*/
 190
 191static const struct object_id *get_rev(void)
 192{
 193        struct commit *commit = NULL;
 194
 195        while (commit == NULL) {
 196                unsigned int mark;
 197                struct commit_list *parents;
 198
 199                if (rev_list.nr == 0 || non_common_revs == 0)
 200                        return NULL;
 201
 202                commit = prio_queue_get(&rev_list);
 203                parse_commit(commit);
 204                parents = commit->parents;
 205
 206                commit->object.flags |= POPPED;
 207                if (!(commit->object.flags & COMMON))
 208                        non_common_revs--;
 209
 210                if (commit->object.flags & COMMON) {
 211                        /* do not send "have", and ignore ancestors */
 212                        commit = NULL;
 213                        mark = COMMON | SEEN;
 214                } else if (commit->object.flags & COMMON_REF)
 215                        /* send "have", and ignore ancestors */
 216                        mark = COMMON | SEEN;
 217                else
 218                        /* send "have", also for its ancestors */
 219                        mark = SEEN;
 220
 221                while (parents) {
 222                        if (!(parents->item->object.flags & SEEN))
 223                                rev_list_push(parents->item, mark);
 224                        if (mark & COMMON)
 225                                mark_common(parents->item, 1, 0);
 226                        parents = parents->next;
 227                }
 228        }
 229
 230        return &commit->object.oid;
 231}
 232
 233enum ack_type {
 234        NAK = 0,
 235        ACK,
 236        ACK_continue,
 237        ACK_common,
 238        ACK_ready
 239};
 240
 241static void consume_shallow_list(struct fetch_pack_args *args, int fd)
 242{
 243        if (args->stateless_rpc && args->deepen) {
 244                /* If we sent a depth we will get back "duplicate"
 245                 * shallow and unshallow commands every time there
 246                 * is a block of have lines exchanged.
 247                 */
 248                char *line;
 249                while ((line = packet_read_line(fd, NULL))) {
 250                        if (starts_with(line, "shallow "))
 251                                continue;
 252                        if (starts_with(line, "unshallow "))
 253                                continue;
 254                        die(_("git fetch-pack: expected shallow list"));
 255                }
 256        }
 257}
 258
 259static enum ack_type get_ack(int fd, struct object_id *result_oid)
 260{
 261        int len;
 262        char *line = packet_read_line(fd, &len);
 263        const char *arg;
 264
 265        if (!line)
 266                die(_("git fetch-pack: expected ACK/NAK, got a flush packet"));
 267        if (!strcmp(line, "NAK"))
 268                return NAK;
 269        if (skip_prefix(line, "ACK ", &arg)) {
 270                if (!get_oid_hex(arg, result_oid)) {
 271                        arg += 40;
 272                        len -= arg - line;
 273                        if (len < 1)
 274                                return ACK;
 275                        if (strstr(arg, "continue"))
 276                                return ACK_continue;
 277                        if (strstr(arg, "common"))
 278                                return ACK_common;
 279                        if (strstr(arg, "ready"))
 280                                return ACK_ready;
 281                        return ACK;
 282                }
 283        }
 284        if (skip_prefix(line, "ERR ", &arg))
 285                die(_("remote error: %s"), arg);
 286        die(_("git fetch-pack: expected ACK/NAK, got '%s'"), line);
 287}
 288
 289static void send_request(struct fetch_pack_args *args,
 290                         int fd, struct strbuf *buf)
 291{
 292        if (args->stateless_rpc) {
 293                send_sideband(fd, -1, buf->buf, buf->len, LARGE_PACKET_MAX);
 294                packet_flush(fd);
 295        } else
 296                write_or_die(fd, buf->buf, buf->len);
 297}
 298
 299static void insert_one_alternate_object(struct object *obj)
 300{
 301        rev_list_insert_ref(NULL, &obj->oid);
 302}
 303
 304#define INITIAL_FLUSH 16
 305#define PIPESAFE_FLUSH 32
 306#define LARGE_FLUSH 16384
 307
 308static int next_flush(int stateless_rpc, int count)
 309{
 310        if (stateless_rpc) {
 311                if (count < LARGE_FLUSH)
 312                        count <<= 1;
 313                else
 314                        count = count * 11 / 10;
 315        } else {
 316                if (count < PIPESAFE_FLUSH)
 317                        count <<= 1;
 318                else
 319                        count += PIPESAFE_FLUSH;
 320        }
 321        return count;
 322}
 323
 324static int find_common(struct fetch_pack_args *args,
 325                       int fd[2], struct object_id *result_oid,
 326                       struct ref *refs)
 327{
 328        int fetching;
 329        int count = 0, flushes = 0, flush_at = INITIAL_FLUSH, retval;
 330        const struct object_id *oid;
 331        unsigned in_vain = 0;
 332        int got_continue = 0;
 333        int got_ready = 0;
 334        struct strbuf req_buf = STRBUF_INIT;
 335        size_t state_len = 0;
 336
 337        if (args->stateless_rpc && multi_ack == 1)
 338                die(_("--stateless-rpc requires multi_ack_detailed"));
 339        if (marked)
 340                for_each_ref(clear_marks, NULL);
 341        marked = 1;
 342
 343        for_each_ref(rev_list_insert_ref_oid, NULL);
 344        for_each_cached_alternate(insert_one_alternate_object);
 345
 346        fetching = 0;
 347        for ( ; refs ; refs = refs->next) {
 348                struct object_id *remote = &refs->old_oid;
 349                const char *remote_hex;
 350                struct object *o;
 351
 352                /*
 353                 * If that object is complete (i.e. it is an ancestor of a
 354                 * local ref), we tell them we have it but do not have to
 355                 * tell them about its ancestors, which they already know
 356                 * about.
 357                 *
 358                 * We use lookup_object here because we are only
 359                 * interested in the case we *know* the object is
 360                 * reachable and we have already scanned it.
 361                 */
 362                if (((o = lookup_object(remote->hash)) != NULL) &&
 363                                (o->flags & COMPLETE)) {
 364                        continue;
 365                }
 366
 367                remote_hex = oid_to_hex(remote);
 368                if (!fetching) {
 369                        struct strbuf c = STRBUF_INIT;
 370                        if (multi_ack == 2)     strbuf_addstr(&c, " multi_ack_detailed");
 371                        if (multi_ack == 1)     strbuf_addstr(&c, " multi_ack");
 372                        if (no_done)            strbuf_addstr(&c, " no-done");
 373                        if (use_sideband == 2)  strbuf_addstr(&c, " side-band-64k");
 374                        if (use_sideband == 1)  strbuf_addstr(&c, " side-band");
 375                        if (args->deepen_relative) strbuf_addstr(&c, " deepen-relative");
 376                        if (args->use_thin_pack) strbuf_addstr(&c, " thin-pack");
 377                        if (args->no_progress)   strbuf_addstr(&c, " no-progress");
 378                        if (args->include_tag)   strbuf_addstr(&c, " include-tag");
 379                        if (prefer_ofs_delta)   strbuf_addstr(&c, " ofs-delta");
 380                        if (deepen_since_ok)    strbuf_addstr(&c, " deepen-since");
 381                        if (deepen_not_ok)      strbuf_addstr(&c, " deepen-not");
 382                        if (agent_supported)    strbuf_addf(&c, " agent=%s",
 383                                                            git_user_agent_sanitized());
 384                        if (args->filter_options.choice)
 385                                strbuf_addstr(&c, " filter");
 386                        packet_buf_write(&req_buf, "want %s%s\n", remote_hex, c.buf);
 387                        strbuf_release(&c);
 388                } else
 389                        packet_buf_write(&req_buf, "want %s\n", remote_hex);
 390                fetching++;
 391        }
 392
 393        if (!fetching) {
 394                strbuf_release(&req_buf);
 395                packet_flush(fd[1]);
 396                return 1;
 397        }
 398
 399        if (is_repository_shallow())
 400                write_shallow_commits(&req_buf, 1, NULL);
 401        if (args->depth > 0)
 402                packet_buf_write(&req_buf, "deepen %d", args->depth);
 403        if (args->deepen_since) {
 404                timestamp_t max_age = approxidate(args->deepen_since);
 405                packet_buf_write(&req_buf, "deepen-since %"PRItime, max_age);
 406        }
 407        if (args->deepen_not) {
 408                int i;
 409                for (i = 0; i < args->deepen_not->nr; i++) {
 410                        struct string_list_item *s = args->deepen_not->items + i;
 411                        packet_buf_write(&req_buf, "deepen-not %s", s->string);
 412                }
 413        }
 414        if (server_supports_filtering && args->filter_options.choice)
 415                packet_buf_write(&req_buf, "filter %s",
 416                                 args->filter_options.filter_spec);
 417        packet_buf_flush(&req_buf);
 418        state_len = req_buf.len;
 419
 420        if (args->deepen) {
 421                char *line;
 422                const char *arg;
 423                struct object_id oid;
 424
 425                send_request(args, fd[1], &req_buf);
 426                while ((line = packet_read_line(fd[0], NULL))) {
 427                        if (skip_prefix(line, "shallow ", &arg)) {
 428                                if (get_oid_hex(arg, &oid))
 429                                        die(_("invalid shallow line: %s"), line);
 430                                register_shallow(&oid);
 431                                continue;
 432                        }
 433                        if (skip_prefix(line, "unshallow ", &arg)) {
 434                                if (get_oid_hex(arg, &oid))
 435                                        die(_("invalid unshallow line: %s"), line);
 436                                if (!lookup_object(oid.hash))
 437                                        die(_("object not found: %s"), line);
 438                                /* make sure that it is parsed as shallow */
 439                                if (!parse_object(&oid))
 440                                        die(_("error in object: %s"), line);
 441                                if (unregister_shallow(&oid))
 442                                        die(_("no shallow found: %s"), line);
 443                                continue;
 444                        }
 445                        die(_("expected shallow/unshallow, got %s"), line);
 446                }
 447        } else if (!args->stateless_rpc)
 448                send_request(args, fd[1], &req_buf);
 449
 450        if (!args->stateless_rpc) {
 451                /* If we aren't using the stateless-rpc interface
 452                 * we don't need to retain the headers.
 453                 */
 454                strbuf_setlen(&req_buf, 0);
 455                state_len = 0;
 456        }
 457
 458        flushes = 0;
 459        retval = -1;
 460        if (args->no_dependents)
 461                goto done;
 462        while ((oid = get_rev())) {
 463                packet_buf_write(&req_buf, "have %s\n", oid_to_hex(oid));
 464                print_verbose(args, "have %s", oid_to_hex(oid));
 465                in_vain++;
 466                if (flush_at <= ++count) {
 467                        int ack;
 468
 469                        packet_buf_flush(&req_buf);
 470                        send_request(args, fd[1], &req_buf);
 471                        strbuf_setlen(&req_buf, state_len);
 472                        flushes++;
 473                        flush_at = next_flush(args->stateless_rpc, count);
 474
 475                        /*
 476                         * We keep one window "ahead" of the other side, and
 477                         * will wait for an ACK only on the next one
 478                         */
 479                        if (!args->stateless_rpc && count == INITIAL_FLUSH)
 480                                continue;
 481
 482                        consume_shallow_list(args, fd[0]);
 483                        do {
 484                                ack = get_ack(fd[0], result_oid);
 485                                if (ack)
 486                                        print_verbose(args, _("got %s %d %s"), "ack",
 487                                                      ack, oid_to_hex(result_oid));
 488                                switch (ack) {
 489                                case ACK:
 490                                        flushes = 0;
 491                                        multi_ack = 0;
 492                                        retval = 0;
 493                                        goto done;
 494                                case ACK_common:
 495                                case ACK_ready:
 496                                case ACK_continue: {
 497                                        struct commit *commit =
 498                                                lookup_commit(result_oid);
 499                                        if (!commit)
 500                                                die(_("invalid commit %s"), oid_to_hex(result_oid));
 501                                        if (args->stateless_rpc
 502                                         && ack == ACK_common
 503                                         && !(commit->object.flags & COMMON)) {
 504                                                /* We need to replay the have for this object
 505                                                 * on the next RPC request so the peer knows
 506                                                 * it is in common with us.
 507                                                 */
 508                                                const char *hex = oid_to_hex(result_oid);
 509                                                packet_buf_write(&req_buf, "have %s\n", hex);
 510                                                state_len = req_buf.len;
 511                                                /*
 512                                                 * Reset in_vain because an ack
 513                                                 * for this commit has not been
 514                                                 * seen.
 515                                                 */
 516                                                in_vain = 0;
 517                                        } else if (!args->stateless_rpc
 518                                                   || ack != ACK_common)
 519                                                in_vain = 0;
 520                                        mark_common(commit, 0, 1);
 521                                        retval = 0;
 522                                        got_continue = 1;
 523                                        if (ack == ACK_ready) {
 524                                                clear_prio_queue(&rev_list);
 525                                                got_ready = 1;
 526                                        }
 527                                        break;
 528                                        }
 529                                }
 530                        } while (ack);
 531                        flushes--;
 532                        if (got_continue && MAX_IN_VAIN < in_vain) {
 533                                print_verbose(args, _("giving up"));
 534                                break; /* give up */
 535                        }
 536                }
 537        }
 538done:
 539        if (!got_ready || !no_done) {
 540                packet_buf_write(&req_buf, "done\n");
 541                send_request(args, fd[1], &req_buf);
 542        }
 543        print_verbose(args, _("done"));
 544        if (retval != 0) {
 545                multi_ack = 0;
 546                flushes++;
 547        }
 548        strbuf_release(&req_buf);
 549
 550        if (!got_ready || !no_done)
 551                consume_shallow_list(args, fd[0]);
 552        while (flushes || multi_ack) {
 553                int ack = get_ack(fd[0], result_oid);
 554                if (ack) {
 555                        print_verbose(args, _("got %s (%d) %s"), "ack",
 556                                      ack, oid_to_hex(result_oid));
 557                        if (ack == ACK)
 558                                return 0;
 559                        multi_ack = 1;
 560                        continue;
 561                }
 562                flushes--;
 563        }
 564        /* it is no error to fetch into a completely empty repo */
 565        return count ? retval : 0;
 566}
 567
 568static struct commit_list *complete;
 569
 570static int mark_complete(const struct object_id *oid)
 571{
 572        struct object *o = parse_object(oid);
 573
 574        while (o && o->type == OBJ_TAG) {
 575                struct tag *t = (struct tag *) o;
 576                if (!t->tagged)
 577                        break; /* broken repository */
 578                o->flags |= COMPLETE;
 579                o = parse_object(&t->tagged->oid);
 580        }
 581        if (o && o->type == OBJ_COMMIT) {
 582                struct commit *commit = (struct commit *)o;
 583                if (!(commit->object.flags & COMPLETE)) {
 584                        commit->object.flags |= COMPLETE;
 585                        commit_list_insert(commit, &complete);
 586                }
 587        }
 588        return 0;
 589}
 590
 591static int mark_complete_oid(const char *refname, const struct object_id *oid,
 592                             int flag, void *cb_data)
 593{
 594        return mark_complete(oid);
 595}
 596
 597static void mark_recent_complete_commits(struct fetch_pack_args *args,
 598                                         timestamp_t cutoff)
 599{
 600        while (complete && cutoff <= complete->item->date) {
 601                print_verbose(args, _("Marking %s as complete"),
 602                              oid_to_hex(&complete->item->object.oid));
 603                pop_most_recent_commit(&complete, COMPLETE);
 604        }
 605}
 606
 607static void add_refs_to_oidset(struct oidset *oids, struct ref *refs)
 608{
 609        for (; refs; refs = refs->next)
 610                oidset_insert(oids, &refs->old_oid);
 611}
 612
 613static int tip_oids_contain(struct oidset *tip_oids,
 614                            struct ref *unmatched, struct ref *newlist,
 615                            const struct object_id *id)
 616{
 617        /*
 618         * Note that this only looks at the ref lists the first time it's
 619         * called. This works out in filter_refs() because even though it may
 620         * add to "newlist" between calls, the additions will always be for
 621         * oids that are already in the set.
 622         */
 623        if (!tip_oids->map.map.tablesize) {
 624                add_refs_to_oidset(tip_oids, unmatched);
 625                add_refs_to_oidset(tip_oids, newlist);
 626        }
 627        return oidset_contains(tip_oids, id);
 628}
 629
 630static void filter_refs(struct fetch_pack_args *args,
 631                        struct ref **refs,
 632                        struct ref **sought, int nr_sought)
 633{
 634        struct ref *newlist = NULL;
 635        struct ref **newtail = &newlist;
 636        struct ref *unmatched = NULL;
 637        struct ref *ref, *next;
 638        struct oidset tip_oids = OIDSET_INIT;
 639        int i;
 640
 641        i = 0;
 642        for (ref = *refs; ref; ref = next) {
 643                int keep = 0;
 644                next = ref->next;
 645
 646                if (starts_with(ref->name, "refs/") &&
 647                    check_refname_format(ref->name, 0))
 648                        ; /* trash */
 649                else {
 650                        while (i < nr_sought) {
 651                                int cmp = strcmp(ref->name, sought[i]->name);
 652                                if (cmp < 0)
 653                                        break; /* definitely do not have it */
 654                                else if (cmp == 0) {
 655                                        keep = 1; /* definitely have it */
 656                                        sought[i]->match_status = REF_MATCHED;
 657                                }
 658                                i++;
 659                        }
 660                }
 661
 662                if (!keep && args->fetch_all &&
 663                    (!args->deepen || !starts_with(ref->name, "refs/tags/")))
 664                        keep = 1;
 665
 666                if (keep) {
 667                        *newtail = ref;
 668                        ref->next = NULL;
 669                        newtail = &ref->next;
 670                } else {
 671                        ref->next = unmatched;
 672                        unmatched = ref;
 673                }
 674        }
 675
 676        /* Append unmatched requests to the list */
 677        for (i = 0; i < nr_sought; i++) {
 678                struct object_id oid;
 679                const char *p;
 680
 681                ref = sought[i];
 682                if (ref->match_status != REF_NOT_MATCHED)
 683                        continue;
 684                if (parse_oid_hex(ref->name, &oid, &p) ||
 685                    *p != '\0' ||
 686                    oidcmp(&oid, &ref->old_oid))
 687                        continue;
 688
 689                if ((allow_unadvertised_object_request &
 690                     (ALLOW_TIP_SHA1 | ALLOW_REACHABLE_SHA1)) ||
 691                    tip_oids_contain(&tip_oids, unmatched, newlist,
 692                                     &ref->old_oid)) {
 693                        ref->match_status = REF_MATCHED;
 694                        *newtail = copy_ref(ref);
 695                        newtail = &(*newtail)->next;
 696                } else {
 697                        ref->match_status = REF_UNADVERTISED_NOT_ALLOWED;
 698                }
 699        }
 700
 701        oidset_clear(&tip_oids);
 702        for (ref = unmatched; ref; ref = next) {
 703                next = ref->next;
 704                free(ref);
 705        }
 706
 707        *refs = newlist;
 708}
 709
 710static void mark_alternate_complete(struct object *obj)
 711{
 712        mark_complete(&obj->oid);
 713}
 714
 715struct loose_object_iter {
 716        struct oidset *loose_object_set;
 717        struct ref *refs;
 718};
 719
 720/*
 721 *  If the number of refs is not larger than the number of loose objects,
 722 *  this function stops inserting.
 723 */
 724static int add_loose_objects_to_set(const struct object_id *oid,
 725                                    const char *path,
 726                                    void *data)
 727{
 728        struct loose_object_iter *iter = data;
 729        oidset_insert(iter->loose_object_set, oid);
 730        if (iter->refs == NULL)
 731                return 1;
 732
 733        iter->refs = iter->refs->next;
 734        return 0;
 735}
 736
 737static int everything_local(struct fetch_pack_args *args,
 738                            struct ref **refs,
 739                            struct ref **sought, int nr_sought)
 740{
 741        struct ref *ref;
 742        int retval;
 743        int old_save_commit_buffer = save_commit_buffer;
 744        timestamp_t cutoff = 0;
 745        struct oidset loose_oid_set = OIDSET_INIT;
 746        int use_oidset = 0;
 747        struct loose_object_iter iter = {&loose_oid_set, *refs};
 748
 749        /* Enumerate all loose objects or know refs are not so many. */
 750        use_oidset = !for_each_loose_object(add_loose_objects_to_set,
 751                                            &iter, 0);
 752
 753        save_commit_buffer = 0;
 754
 755        for (ref = *refs; ref; ref = ref->next) {
 756                struct object *o;
 757                unsigned int flags = OBJECT_INFO_QUICK;
 758
 759                if (use_oidset &&
 760                    !oidset_contains(&loose_oid_set, &ref->old_oid)) {
 761                        /*
 762                         * I know this does not exist in the loose form,
 763                         * so check if it exists in a non-loose form.
 764                         */
 765                        flags |= OBJECT_INFO_IGNORE_LOOSE;
 766                }
 767
 768                if (!has_object_file_with_flags(&ref->old_oid, flags))
 769                        continue;
 770                o = parse_object(&ref->old_oid);
 771                if (!o)
 772                        continue;
 773
 774                /* We already have it -- which may mean that we were
 775                 * in sync with the other side at some time after
 776                 * that (it is OK if we guess wrong here).
 777                 */
 778                if (o->type == OBJ_COMMIT) {
 779                        struct commit *commit = (struct commit *)o;
 780                        if (!cutoff || cutoff < commit->date)
 781                                cutoff = commit->date;
 782                }
 783        }
 784
 785        oidset_clear(&loose_oid_set);
 786
 787        if (!args->no_dependents) {
 788                if (!args->deepen) {
 789                        for_each_ref(mark_complete_oid, NULL);
 790                        for_each_cached_alternate(mark_alternate_complete);
 791                        commit_list_sort_by_date(&complete);
 792                        if (cutoff)
 793                                mark_recent_complete_commits(args, cutoff);
 794                }
 795
 796                /*
 797                 * Mark all complete remote refs as common refs.
 798                 * Don't mark them common yet; the server has to be told so first.
 799                 */
 800                for (ref = *refs; ref; ref = ref->next) {
 801                        struct object *o = deref_tag(lookup_object(ref->old_oid.hash),
 802                                                     NULL, 0);
 803
 804                        if (!o || o->type != OBJ_COMMIT || !(o->flags & COMPLETE))
 805                                continue;
 806
 807                        if (!(o->flags & SEEN)) {
 808                                rev_list_push((struct commit *)o, COMMON_REF | SEEN);
 809
 810                                mark_common((struct commit *)o, 1, 1);
 811                        }
 812                }
 813        }
 814
 815        filter_refs(args, refs, sought, nr_sought);
 816
 817        for (retval = 1, ref = *refs; ref ; ref = ref->next) {
 818                const struct object_id *remote = &ref->old_oid;
 819                struct object *o;
 820
 821                o = lookup_object(remote->hash);
 822                if (!o || !(o->flags & COMPLETE)) {
 823                        retval = 0;
 824                        print_verbose(args, "want %s (%s)", oid_to_hex(remote),
 825                                      ref->name);
 826                        continue;
 827                }
 828                print_verbose(args, _("already have %s (%s)"), oid_to_hex(remote),
 829                              ref->name);
 830        }
 831
 832        save_commit_buffer = old_save_commit_buffer;
 833
 834        return retval;
 835}
 836
 837static int sideband_demux(int in, int out, void *data)
 838{
 839        int *xd = data;
 840        int ret;
 841
 842        ret = recv_sideband("fetch-pack", xd[0], out);
 843        close(out);
 844        return ret;
 845}
 846
 847static int get_pack(struct fetch_pack_args *args,
 848                    int xd[2], char **pack_lockfile)
 849{
 850        struct async demux;
 851        int do_keep = args->keep_pack;
 852        const char *cmd_name;
 853        struct pack_header header;
 854        int pass_header = 0;
 855        struct child_process cmd = CHILD_PROCESS_INIT;
 856        int ret;
 857
 858        memset(&demux, 0, sizeof(demux));
 859        if (use_sideband) {
 860                /* xd[] is talking with upload-pack; subprocess reads from
 861                 * xd[0], spits out band#2 to stderr, and feeds us band#1
 862                 * through demux->out.
 863                 */
 864                demux.proc = sideband_demux;
 865                demux.data = xd;
 866                demux.out = -1;
 867                demux.isolate_sigpipe = 1;
 868                if (start_async(&demux))
 869                        die(_("fetch-pack: unable to fork off sideband demultiplexer"));
 870        }
 871        else
 872                demux.out = xd[0];
 873
 874        if (!args->keep_pack && unpack_limit) {
 875
 876                if (read_pack_header(demux.out, &header))
 877                        die(_("protocol error: bad pack header"));
 878                pass_header = 1;
 879                if (ntohl(header.hdr_entries) < unpack_limit)
 880                        do_keep = 0;
 881                else
 882                        do_keep = 1;
 883        }
 884
 885        if (alternate_shallow_file) {
 886                argv_array_push(&cmd.args, "--shallow-file");
 887                argv_array_push(&cmd.args, alternate_shallow_file);
 888        }
 889
 890        if (do_keep || args->from_promisor) {
 891                if (pack_lockfile)
 892                        cmd.out = -1;
 893                cmd_name = "index-pack";
 894                argv_array_push(&cmd.args, cmd_name);
 895                argv_array_push(&cmd.args, "--stdin");
 896                if (!args->quiet && !args->no_progress)
 897                        argv_array_push(&cmd.args, "-v");
 898                if (args->use_thin_pack)
 899                        argv_array_push(&cmd.args, "--fix-thin");
 900                if (do_keep && (args->lock_pack || unpack_limit)) {
 901                        char hostname[HOST_NAME_MAX + 1];
 902                        if (xgethostname(hostname, sizeof(hostname)))
 903                                xsnprintf(hostname, sizeof(hostname), "localhost");
 904                        argv_array_pushf(&cmd.args,
 905                                        "--keep=fetch-pack %"PRIuMAX " on %s",
 906                                        (uintmax_t)getpid(), hostname);
 907                }
 908                if (args->check_self_contained_and_connected)
 909                        argv_array_push(&cmd.args, "--check-self-contained-and-connected");
 910                if (args->from_promisor)
 911                        argv_array_push(&cmd.args, "--promisor");
 912        }
 913        else {
 914                cmd_name = "unpack-objects";
 915                argv_array_push(&cmd.args, cmd_name);
 916                if (args->quiet || args->no_progress)
 917                        argv_array_push(&cmd.args, "-q");
 918                args->check_self_contained_and_connected = 0;
 919        }
 920
 921        if (pass_header)
 922                argv_array_pushf(&cmd.args, "--pack_header=%"PRIu32",%"PRIu32,
 923                                 ntohl(header.hdr_version),
 924                                 ntohl(header.hdr_entries));
 925        if (fetch_fsck_objects >= 0
 926            ? fetch_fsck_objects
 927            : transfer_fsck_objects >= 0
 928            ? transfer_fsck_objects
 929            : 0) {
 930                if (args->from_promisor)
 931                        /*
 932                         * We cannot use --strict in index-pack because it
 933                         * checks both broken objects and links, but we only
 934                         * want to check for broken objects.
 935                         */
 936                        argv_array_push(&cmd.args, "--fsck-objects");
 937                else
 938                        argv_array_push(&cmd.args, "--strict");
 939        }
 940
 941        cmd.in = demux.out;
 942        cmd.git_cmd = 1;
 943        if (start_command(&cmd))
 944                die(_("fetch-pack: unable to fork off %s"), cmd_name);
 945        if (do_keep && pack_lockfile) {
 946                *pack_lockfile = index_pack_lockfile(cmd.out);
 947                close(cmd.out);
 948        }
 949
 950        if (!use_sideband)
 951                /* Closed by start_command() */
 952                xd[0] = -1;
 953
 954        ret = finish_command(&cmd);
 955        if (!ret || (args->check_self_contained_and_connected && ret == 1))
 956                args->self_contained_and_connected =
 957                        args->check_self_contained_and_connected &&
 958                        ret == 0;
 959        else
 960                die(_("%s failed"), cmd_name);
 961        if (use_sideband && finish_async(&demux))
 962                die(_("error in sideband demultiplexer"));
 963        return 0;
 964}
 965
 966static int cmp_ref_by_name(const void *a_, const void *b_)
 967{
 968        const struct ref *a = *((const struct ref **)a_);
 969        const struct ref *b = *((const struct ref **)b_);
 970        return strcmp(a->name, b->name);
 971}
 972
 973static struct ref *do_fetch_pack(struct fetch_pack_args *args,
 974                                 int fd[2],
 975                                 const struct ref *orig_ref,
 976                                 struct ref **sought, int nr_sought,
 977                                 struct shallow_info *si,
 978                                 char **pack_lockfile)
 979{
 980        struct ref *ref = copy_ref_list(orig_ref);
 981        struct object_id oid;
 982        const char *agent_feature;
 983        int agent_len;
 984
 985        sort_ref_list(&ref, ref_compare_name);
 986        QSORT(sought, nr_sought, cmp_ref_by_name);
 987
 988        if ((args->depth > 0 || is_repository_shallow()) && !server_supports("shallow"))
 989                die(_("Server does not support shallow clients"));
 990        if (args->depth > 0 || args->deepen_since || args->deepen_not)
 991                args->deepen = 1;
 992        if (server_supports("multi_ack_detailed")) {
 993                print_verbose(args, _("Server supports multi_ack_detailed"));
 994                multi_ack = 2;
 995                if (server_supports("no-done")) {
 996                        print_verbose(args, _("Server supports no-done"));
 997                        if (args->stateless_rpc)
 998                                no_done = 1;
 999                }
1000        }
1001        else if (server_supports("multi_ack")) {
1002                print_verbose(args, _("Server supports multi_ack"));
1003                multi_ack = 1;
1004        }
1005        if (server_supports("side-band-64k")) {
1006                print_verbose(args, _("Server supports side-band-64k"));
1007                use_sideband = 2;
1008        }
1009        else if (server_supports("side-band")) {
1010                print_verbose(args, _("Server supports side-band"));
1011                use_sideband = 1;
1012        }
1013        if (server_supports("allow-tip-sha1-in-want")) {
1014                print_verbose(args, _("Server supports allow-tip-sha1-in-want"));
1015                allow_unadvertised_object_request |= ALLOW_TIP_SHA1;
1016        }
1017        if (server_supports("allow-reachable-sha1-in-want")) {
1018                print_verbose(args, _("Server supports allow-reachable-sha1-in-want"));
1019                allow_unadvertised_object_request |= ALLOW_REACHABLE_SHA1;
1020        }
1021        if (!server_supports("thin-pack"))
1022                args->use_thin_pack = 0;
1023        if (!server_supports("no-progress"))
1024                args->no_progress = 0;
1025        if (!server_supports("include-tag"))
1026                args->include_tag = 0;
1027        if (server_supports("ofs-delta"))
1028                print_verbose(args, _("Server supports ofs-delta"));
1029        else
1030                prefer_ofs_delta = 0;
1031
1032        if (server_supports("filter")) {
1033                server_supports_filtering = 1;
1034                print_verbose(args, _("Server supports filter"));
1035        } else if (args->filter_options.choice) {
1036                warning("filtering not recognized by server, ignoring");
1037        }
1038
1039        if ((agent_feature = server_feature_value("agent", &agent_len))) {
1040                agent_supported = 1;
1041                if (agent_len)
1042                        print_verbose(args, _("Server version is %.*s"),
1043                                      agent_len, agent_feature);
1044        }
1045        if (server_supports("deepen-since"))
1046                deepen_since_ok = 1;
1047        else if (args->deepen_since)
1048                die(_("Server does not support --shallow-since"));
1049        if (server_supports("deepen-not"))
1050                deepen_not_ok = 1;
1051        else if (args->deepen_not)
1052                die(_("Server does not support --shallow-exclude"));
1053        if (!server_supports("deepen-relative") && args->deepen_relative)
1054                die(_("Server does not support --deepen"));
1055
1056        if (everything_local(args, &ref, sought, nr_sought)) {
1057                packet_flush(fd[1]);
1058                goto all_done;
1059        }
1060        if (find_common(args, fd, &oid, ref) < 0)
1061                if (!args->keep_pack)
1062                        /* When cloning, it is not unusual to have
1063                         * no common commit.
1064                         */
1065                        warning(_("no common commits"));
1066
1067        if (args->stateless_rpc)
1068                packet_flush(fd[1]);
1069        if (args->deepen)
1070                setup_alternate_shallow(&shallow_lock, &alternate_shallow_file,
1071                                        NULL);
1072        else if (si->nr_ours || si->nr_theirs)
1073                alternate_shallow_file = setup_temporary_shallow(si->shallow);
1074        else
1075                alternate_shallow_file = NULL;
1076        if (get_pack(args, fd, pack_lockfile))
1077                die(_("git fetch-pack: fetch failed."));
1078
1079 all_done:
1080        return ref;
1081}
1082
1083static void add_shallow_requests(struct strbuf *req_buf,
1084                                 const struct fetch_pack_args *args)
1085{
1086        if (is_repository_shallow())
1087                write_shallow_commits(req_buf, 1, NULL);
1088        if (args->depth > 0)
1089                packet_buf_write(req_buf, "deepen %d", args->depth);
1090        if (args->deepen_since) {
1091                timestamp_t max_age = approxidate(args->deepen_since);
1092                packet_buf_write(req_buf, "deepen-since %"PRItime, max_age);
1093        }
1094        if (args->deepen_not) {
1095                int i;
1096                for (i = 0; i < args->deepen_not->nr; i++) {
1097                        struct string_list_item *s = args->deepen_not->items + i;
1098                        packet_buf_write(req_buf, "deepen-not %s", s->string);
1099                }
1100        }
1101}
1102
1103static void add_wants(const struct ref *wants, struct strbuf *req_buf)
1104{
1105        for ( ; wants ; wants = wants->next) {
1106                const struct object_id *remote = &wants->old_oid;
1107                const char *remote_hex;
1108                struct object *o;
1109
1110                /*
1111                 * If that object is complete (i.e. it is an ancestor of a
1112                 * local ref), we tell them we have it but do not have to
1113                 * tell them about its ancestors, which they already know
1114                 * about.
1115                 *
1116                 * We use lookup_object here because we are only
1117                 * interested in the case we *know* the object is
1118                 * reachable and we have already scanned it.
1119                 */
1120                if (((o = lookup_object(remote->hash)) != NULL) &&
1121                    (o->flags & COMPLETE)) {
1122                        continue;
1123                }
1124
1125                remote_hex = oid_to_hex(remote);
1126                packet_buf_write(req_buf, "want %s\n", remote_hex);
1127        }
1128}
1129
1130static void add_common(struct strbuf *req_buf, struct oidset *common)
1131{
1132        struct oidset_iter iter;
1133        const struct object_id *oid;
1134        oidset_iter_init(common, &iter);
1135
1136        while ((oid = oidset_iter_next(&iter))) {
1137                packet_buf_write(req_buf, "have %s\n", oid_to_hex(oid));
1138        }
1139}
1140
1141static int add_haves(struct strbuf *req_buf, int *haves_to_send, int *in_vain)
1142{
1143        int ret = 0;
1144        int haves_added = 0;
1145        const struct object_id *oid;
1146
1147        while ((oid = get_rev())) {
1148                packet_buf_write(req_buf, "have %s\n", oid_to_hex(oid));
1149                if (++haves_added >= *haves_to_send)
1150                        break;
1151        }
1152
1153        *in_vain += haves_added;
1154        if (!haves_added || *in_vain >= MAX_IN_VAIN) {
1155                /* Send Done */
1156                packet_buf_write(req_buf, "done\n");
1157                ret = 1;
1158        }
1159
1160        /* Increase haves to send on next round */
1161        *haves_to_send = next_flush(1, *haves_to_send);
1162
1163        return ret;
1164}
1165
1166static int send_fetch_request(int fd_out, const struct fetch_pack_args *args,
1167                              const struct ref *wants, struct oidset *common,
1168                              int *haves_to_send, int *in_vain)
1169{
1170        int ret = 0;
1171        struct strbuf req_buf = STRBUF_INIT;
1172
1173        if (server_supports_v2("fetch", 1))
1174                packet_buf_write(&req_buf, "command=fetch");
1175        if (server_supports_v2("agent", 0))
1176                packet_buf_write(&req_buf, "agent=%s", git_user_agent_sanitized());
1177
1178        packet_buf_delim(&req_buf);
1179        if (args->use_thin_pack)
1180                packet_buf_write(&req_buf, "thin-pack");
1181        if (args->no_progress)
1182                packet_buf_write(&req_buf, "no-progress");
1183        if (args->include_tag)
1184                packet_buf_write(&req_buf, "include-tag");
1185        if (prefer_ofs_delta)
1186                packet_buf_write(&req_buf, "ofs-delta");
1187
1188        /* Add shallow-info and deepen request */
1189        if (server_supports_feature("fetch", "shallow", 0))
1190                add_shallow_requests(&req_buf, args);
1191        else if (is_repository_shallow() || args->deepen)
1192                die(_("Server does not support shallow requests"));
1193
1194        /* add wants */
1195        add_wants(wants, &req_buf);
1196
1197        /* Add all of the common commits we've found in previous rounds */
1198        add_common(&req_buf, common);
1199
1200        /* Add initial haves */
1201        ret = add_haves(&req_buf, haves_to_send, in_vain);
1202
1203        /* Send request */
1204        packet_buf_flush(&req_buf);
1205        write_or_die(fd_out, req_buf.buf, req_buf.len);
1206
1207        strbuf_release(&req_buf);
1208        return ret;
1209}
1210
1211/*
1212 * Processes a section header in a server's response and checks if it matches
1213 * `section`.  If the value of `peek` is 1, the header line will be peeked (and
1214 * not consumed); if 0, the line will be consumed and the function will die if
1215 * the section header doesn't match what was expected.
1216 */
1217static int process_section_header(struct packet_reader *reader,
1218                                  const char *section, int peek)
1219{
1220        int ret;
1221
1222        if (packet_reader_peek(reader) != PACKET_READ_NORMAL)
1223                die("error reading section header '%s'", section);
1224
1225        ret = !strcmp(reader->line, section);
1226
1227        if (!peek) {
1228                if (!ret)
1229                        die("expected '%s', received '%s'",
1230                            section, reader->line);
1231                packet_reader_read(reader);
1232        }
1233
1234        return ret;
1235}
1236
1237static int process_acks(struct packet_reader *reader, struct oidset *common)
1238{
1239        /* received */
1240        int received_ready = 0;
1241        int received_ack = 0;
1242
1243        process_section_header(reader, "acknowledgments", 0);
1244        while (packet_reader_read(reader) == PACKET_READ_NORMAL) {
1245                const char *arg;
1246
1247                if (!strcmp(reader->line, "NAK"))
1248                        continue;
1249
1250                if (skip_prefix(reader->line, "ACK ", &arg)) {
1251                        struct object_id oid;
1252                        if (!get_oid_hex(arg, &oid)) {
1253                                struct commit *commit;
1254                                oidset_insert(common, &oid);
1255                                commit = lookup_commit(&oid);
1256                                mark_common(commit, 0, 1);
1257                        }
1258                        continue;
1259                }
1260
1261                if (!strcmp(reader->line, "ready")) {
1262                        clear_prio_queue(&rev_list);
1263                        received_ready = 1;
1264                        continue;
1265                }
1266
1267                die("unexpected acknowledgment line: '%s'", reader->line);
1268        }
1269
1270        if (reader->status != PACKET_READ_FLUSH &&
1271            reader->status != PACKET_READ_DELIM)
1272                die("error processing acks: %d", reader->status);
1273
1274        /* return 0 if no common, 1 if there are common, or 2 if ready */
1275        return received_ready ? 2 : (received_ack ? 1 : 0);
1276}
1277
1278static void receive_shallow_info(struct fetch_pack_args *args,
1279                                 struct packet_reader *reader)
1280{
1281        process_section_header(reader, "shallow-info", 0);
1282        while (packet_reader_read(reader) == PACKET_READ_NORMAL) {
1283                const char *arg;
1284                struct object_id oid;
1285
1286                if (skip_prefix(reader->line, "shallow ", &arg)) {
1287                        if (get_oid_hex(arg, &oid))
1288                                die(_("invalid shallow line: %s"), reader->line);
1289                        register_shallow(&oid);
1290                        continue;
1291                }
1292                if (skip_prefix(reader->line, "unshallow ", &arg)) {
1293                        if (get_oid_hex(arg, &oid))
1294                                die(_("invalid unshallow line: %s"), reader->line);
1295                        if (!lookup_object(oid.hash))
1296                                die(_("object not found: %s"), reader->line);
1297                        /* make sure that it is parsed as shallow */
1298                        if (!parse_object(&oid))
1299                                die(_("error in object: %s"), reader->line);
1300                        if (unregister_shallow(&oid))
1301                                die(_("no shallow found: %s"), reader->line);
1302                        continue;
1303                }
1304                die(_("expected shallow/unshallow, got %s"), reader->line);
1305        }
1306
1307        if (reader->status != PACKET_READ_FLUSH &&
1308            reader->status != PACKET_READ_DELIM)
1309                die("error processing shallow info: %d", reader->status);
1310
1311        setup_alternate_shallow(&shallow_lock, &alternate_shallow_file, NULL);
1312        args->deepen = 1;
1313}
1314
1315enum fetch_state {
1316        FETCH_CHECK_LOCAL = 0,
1317        FETCH_SEND_REQUEST,
1318        FETCH_PROCESS_ACKS,
1319        FETCH_GET_PACK,
1320        FETCH_DONE,
1321};
1322
1323static struct ref *do_fetch_pack_v2(struct fetch_pack_args *args,
1324                                    int fd[2],
1325                                    const struct ref *orig_ref,
1326                                    struct ref **sought, int nr_sought,
1327                                    char **pack_lockfile)
1328{
1329        struct ref *ref = copy_ref_list(orig_ref);
1330        enum fetch_state state = FETCH_CHECK_LOCAL;
1331        struct oidset common = OIDSET_INIT;
1332        struct packet_reader reader;
1333        int in_vain = 0;
1334        int haves_to_send = INITIAL_FLUSH;
1335        packet_reader_init(&reader, fd[0], NULL, 0,
1336                           PACKET_READ_CHOMP_NEWLINE);
1337
1338        while (state != FETCH_DONE) {
1339                switch (state) {
1340                case FETCH_CHECK_LOCAL:
1341                        sort_ref_list(&ref, ref_compare_name);
1342                        QSORT(sought, nr_sought, cmp_ref_by_name);
1343
1344                        /* v2 supports these by default */
1345                        allow_unadvertised_object_request |= ALLOW_REACHABLE_SHA1;
1346                        use_sideband = 2;
1347                        if (args->depth > 0 || args->deepen_since || args->deepen_not)
1348                                args->deepen = 1;
1349
1350                        if (marked)
1351                                for_each_ref(clear_marks, NULL);
1352                        marked = 1;
1353
1354                        for_each_ref(rev_list_insert_ref_oid, NULL);
1355                        for_each_cached_alternate(insert_one_alternate_object);
1356
1357                        /* Filter 'ref' by 'sought' and those that aren't local */
1358                        if (everything_local(args, &ref, sought, nr_sought))
1359                                state = FETCH_DONE;
1360                        else
1361                                state = FETCH_SEND_REQUEST;
1362                        break;
1363                case FETCH_SEND_REQUEST:
1364                        if (send_fetch_request(fd[1], args, ref, &common,
1365                                               &haves_to_send, &in_vain))
1366                                state = FETCH_GET_PACK;
1367                        else
1368                                state = FETCH_PROCESS_ACKS;
1369                        break;
1370                case FETCH_PROCESS_ACKS:
1371                        /* Process ACKs/NAKs */
1372                        switch (process_acks(&reader, &common)) {
1373                        case 2:
1374                                state = FETCH_GET_PACK;
1375                                break;
1376                        case 1:
1377                                in_vain = 0;
1378                                /* fallthrough */
1379                        default:
1380                                state = FETCH_SEND_REQUEST;
1381                                break;
1382                        }
1383                        break;
1384                case FETCH_GET_PACK:
1385                        /* Check for shallow-info section */
1386                        if (process_section_header(&reader, "shallow-info", 1))
1387                                receive_shallow_info(args, &reader);
1388
1389                        /* get the pack */
1390                        process_section_header(&reader, "packfile", 0);
1391                        if (get_pack(args, fd, pack_lockfile))
1392                                die(_("git fetch-pack: fetch failed."));
1393
1394                        state = FETCH_DONE;
1395                        break;
1396                case FETCH_DONE:
1397                        continue;
1398                }
1399        }
1400
1401        oidset_clear(&common);
1402        return ref;
1403}
1404
1405static void fetch_pack_config(void)
1406{
1407        git_config_get_int("fetch.unpacklimit", &fetch_unpack_limit);
1408        git_config_get_int("transfer.unpacklimit", &transfer_unpack_limit);
1409        git_config_get_bool("repack.usedeltabaseoffset", &prefer_ofs_delta);
1410        git_config_get_bool("fetch.fsckobjects", &fetch_fsck_objects);
1411        git_config_get_bool("transfer.fsckobjects", &transfer_fsck_objects);
1412
1413        git_config(git_default_config, NULL);
1414}
1415
1416static void fetch_pack_setup(void)
1417{
1418        static int did_setup;
1419        if (did_setup)
1420                return;
1421        fetch_pack_config();
1422        if (0 <= transfer_unpack_limit)
1423                unpack_limit = transfer_unpack_limit;
1424        else if (0 <= fetch_unpack_limit)
1425                unpack_limit = fetch_unpack_limit;
1426        did_setup = 1;
1427}
1428
1429static int remove_duplicates_in_refs(struct ref **ref, int nr)
1430{
1431        struct string_list names = STRING_LIST_INIT_NODUP;
1432        int src, dst;
1433
1434        for (src = dst = 0; src < nr; src++) {
1435                struct string_list_item *item;
1436                item = string_list_insert(&names, ref[src]->name);
1437                if (item->util)
1438                        continue; /* already have it */
1439                item->util = ref[src];
1440                if (src != dst)
1441                        ref[dst] = ref[src];
1442                dst++;
1443        }
1444        for (src = dst; src < nr; src++)
1445                ref[src] = NULL;
1446        string_list_clear(&names, 0);
1447        return dst;
1448}
1449
1450static void update_shallow(struct fetch_pack_args *args,
1451                           struct ref **sought, int nr_sought,
1452                           struct shallow_info *si)
1453{
1454        struct oid_array ref = OID_ARRAY_INIT;
1455        int *status;
1456        int i;
1457
1458        if (args->deepen && alternate_shallow_file) {
1459                if (*alternate_shallow_file == '\0') { /* --unshallow */
1460                        unlink_or_warn(git_path_shallow());
1461                        rollback_lock_file(&shallow_lock);
1462                } else
1463                        commit_lock_file(&shallow_lock);
1464                return;
1465        }
1466
1467        if (!si->shallow || !si->shallow->nr)
1468                return;
1469
1470        if (args->cloning) {
1471                /*
1472                 * remote is shallow, but this is a clone, there are
1473                 * no objects in repo to worry about. Accept any
1474                 * shallow points that exist in the pack (iow in repo
1475                 * after get_pack() and reprepare_packed_git())
1476                 */
1477                struct oid_array extra = OID_ARRAY_INIT;
1478                struct object_id *oid = si->shallow->oid;
1479                for (i = 0; i < si->shallow->nr; i++)
1480                        if (has_object_file(&oid[i]))
1481                                oid_array_append(&extra, &oid[i]);
1482                if (extra.nr) {
1483                        setup_alternate_shallow(&shallow_lock,
1484                                                &alternate_shallow_file,
1485                                                &extra);
1486                        commit_lock_file(&shallow_lock);
1487                }
1488                oid_array_clear(&extra);
1489                return;
1490        }
1491
1492        if (!si->nr_ours && !si->nr_theirs)
1493                return;
1494
1495        remove_nonexistent_theirs_shallow(si);
1496        if (!si->nr_ours && !si->nr_theirs)
1497                return;
1498        for (i = 0; i < nr_sought; i++)
1499                oid_array_append(&ref, &sought[i]->old_oid);
1500        si->ref = &ref;
1501
1502        if (args->update_shallow) {
1503                /*
1504                 * remote is also shallow, .git/shallow may be updated
1505                 * so all refs can be accepted. Make sure we only add
1506                 * shallow roots that are actually reachable from new
1507                 * refs.
1508                 */
1509                struct oid_array extra = OID_ARRAY_INIT;
1510                struct object_id *oid = si->shallow->oid;
1511                assign_shallow_commits_to_refs(si, NULL, NULL);
1512                if (!si->nr_ours && !si->nr_theirs) {
1513                        oid_array_clear(&ref);
1514                        return;
1515                }
1516                for (i = 0; i < si->nr_ours; i++)
1517                        oid_array_append(&extra, &oid[si->ours[i]]);
1518                for (i = 0; i < si->nr_theirs; i++)
1519                        oid_array_append(&extra, &oid[si->theirs[i]]);
1520                setup_alternate_shallow(&shallow_lock,
1521                                        &alternate_shallow_file,
1522                                        &extra);
1523                commit_lock_file(&shallow_lock);
1524                oid_array_clear(&extra);
1525                oid_array_clear(&ref);
1526                return;
1527        }
1528
1529        /*
1530         * remote is also shallow, check what ref is safe to update
1531         * without updating .git/shallow
1532         */
1533        status = xcalloc(nr_sought, sizeof(*status));
1534        assign_shallow_commits_to_refs(si, NULL, status);
1535        if (si->nr_ours || si->nr_theirs) {
1536                for (i = 0; i < nr_sought; i++)
1537                        if (status[i])
1538                                sought[i]->status = REF_STATUS_REJECT_SHALLOW;
1539        }
1540        free(status);
1541        oid_array_clear(&ref);
1542}
1543
1544struct ref *fetch_pack(struct fetch_pack_args *args,
1545                       int fd[], struct child_process *conn,
1546                       const struct ref *ref,
1547                       const char *dest,
1548                       struct ref **sought, int nr_sought,
1549                       struct oid_array *shallow,
1550                       char **pack_lockfile,
1551                       enum protocol_version version)
1552{
1553        struct ref *ref_cpy;
1554        struct shallow_info si;
1555
1556        fetch_pack_setup();
1557        if (nr_sought)
1558                nr_sought = remove_duplicates_in_refs(sought, nr_sought);
1559
1560        if (!ref) {
1561                packet_flush(fd[1]);
1562                die(_("no matching remote head"));
1563        }
1564        prepare_shallow_info(&si, shallow);
1565        if (version == protocol_v2)
1566                ref_cpy = do_fetch_pack_v2(args, fd, ref, sought, nr_sought,
1567                                           pack_lockfile);
1568        else
1569                ref_cpy = do_fetch_pack(args, fd, ref, sought, nr_sought,
1570                                        &si, pack_lockfile);
1571        reprepare_packed_git(the_repository);
1572        update_shallow(args, sought, nr_sought, &si);
1573        clear_shallow_info(&si);
1574        return ref_cpy;
1575}
1576
1577int report_unmatched_refs(struct ref **sought, int nr_sought)
1578{
1579        int i, ret = 0;
1580
1581        for (i = 0; i < nr_sought; i++) {
1582                if (!sought[i])
1583                        continue;
1584                switch (sought[i]->match_status) {
1585                case REF_MATCHED:
1586                        continue;
1587                case REF_NOT_MATCHED:
1588                        error(_("no such remote ref %s"), sought[i]->name);
1589                        break;
1590                case REF_UNADVERTISED_NOT_ALLOWED:
1591                        error(_("Server does not allow request for unadvertised object %s"),
1592                              sought[i]->name);
1593                        break;
1594                }
1595                ret = 1;
1596        }
1597        return ret;
1598}