fetch-pack.con commit Merge branch 'en/rebase-i-microfixes' (b345b77)
   1#include "cache.h"
   2#include "repository.h"
   3#include "config.h"
   4#include "lockfile.h"
   5#include "refs.h"
   6#include "pkt-line.h"
   7#include "commit.h"
   8#include "tag.h"
   9#include "exec-cmd.h"
  10#include "pack.h"
  11#include "sideband.h"
  12#include "fetch-pack.h"
  13#include "remote.h"
  14#include "run-command.h"
  15#include "connect.h"
  16#include "transport.h"
  17#include "version.h"
  18#include "prio-queue.h"
  19#include "sha1-array.h"
  20#include "oidset.h"
  21#include "packfile.h"
  22#include "object-store.h"
  23
  24static int transfer_unpack_limit = -1;
  25static int fetch_unpack_limit = -1;
  26static int unpack_limit = 100;
  27static int prefer_ofs_delta = 1;
  28static int no_done;
  29static int deepen_since_ok;
  30static int deepen_not_ok;
  31static int fetch_fsck_objects = -1;
  32static int transfer_fsck_objects = -1;
  33static int agent_supported;
  34static int server_supports_filtering;
  35static struct lock_file shallow_lock;
  36static const char *alternate_shallow_file;
  37
  38/* Remember to update object flag allocation in object.h */
  39#define COMPLETE        (1U << 0)
  40#define COMMON          (1U << 1)
  41#define COMMON_REF      (1U << 2)
  42#define SEEN            (1U << 3)
  43#define POPPED          (1U << 4)
  44#define ALTERNATE       (1U << 5)
  45
  46static int marked;
  47
  48/*
  49 * After sending this many "have"s if we do not get any new ACK , we
  50 * give up traversing our history.
  51 */
  52#define MAX_IN_VAIN 256
  53
  54static struct prio_queue rev_list = { compare_commits_by_commit_date };
  55static int non_common_revs, multi_ack, use_sideband;
  56/* Allow specifying sha1 if it is a ref tip. */
  57#define ALLOW_TIP_SHA1  01
  58/* Allow request of a sha1 if it is reachable from a ref (possibly hidden ref). */
  59#define ALLOW_REACHABLE_SHA1    02
  60static unsigned int allow_unadvertised_object_request;
  61
  62__attribute__((format (printf, 2, 3)))
  63static inline void print_verbose(const struct fetch_pack_args *args,
  64                                 const char *fmt, ...)
  65{
  66        va_list params;
  67
  68        if (!args->verbose)
  69                return;
  70
  71        va_start(params, fmt);
  72        vfprintf(stderr, fmt, params);
  73        va_end(params);
  74        fputc('\n', stderr);
  75}
  76
  77struct alternate_object_cache {
  78        struct object **items;
  79        size_t nr, alloc;
  80};
  81
  82static void cache_one_alternate(const char *refname,
  83                                const struct object_id *oid,
  84                                void *vcache)
  85{
  86        struct alternate_object_cache *cache = vcache;
  87        struct object *obj = parse_object(oid);
  88
  89        if (!obj || (obj->flags & ALTERNATE))
  90                return;
  91
  92        obj->flags |= ALTERNATE;
  93        ALLOC_GROW(cache->items, cache->nr + 1, cache->alloc);
  94        cache->items[cache->nr++] = obj;
  95}
  96
  97static void for_each_cached_alternate(void (*cb)(struct object *))
  98{
  99        static int initialized;
 100        static struct alternate_object_cache cache;
 101        size_t i;
 102
 103        if (!initialized) {
 104                for_each_alternate_ref(cache_one_alternate, &cache);
 105                initialized = 1;
 106        }
 107
 108        for (i = 0; i < cache.nr; i++)
 109                cb(cache.items[i]);
 110}
 111
 112static void rev_list_push(struct commit *commit, int mark)
 113{
 114        if (!(commit->object.flags & mark)) {
 115                commit->object.flags |= mark;
 116
 117                if (parse_commit(commit))
 118                        return;
 119
 120                prio_queue_put(&rev_list, commit);
 121
 122                if (!(commit->object.flags & COMMON))
 123                        non_common_revs++;
 124        }
 125}
 126
 127static int rev_list_insert_ref(const char *refname, const struct object_id *oid)
 128{
 129        struct object *o = deref_tag(parse_object(oid), refname, 0);
 130
 131        if (o && o->type == OBJ_COMMIT)
 132                rev_list_push((struct commit *)o, SEEN);
 133
 134        return 0;
 135}
 136
 137static int rev_list_insert_ref_oid(const char *refname, const struct object_id *oid,
 138                                   int flag, void *cb_data)
 139{
 140        return rev_list_insert_ref(refname, oid);
 141}
 142
 143static int clear_marks(const char *refname, const struct object_id *oid,
 144                       int flag, void *cb_data)
 145{
 146        struct object *o = deref_tag(parse_object(oid), refname, 0);
 147
 148        if (o && o->type == OBJ_COMMIT)
 149                clear_commit_marks((struct commit *)o,
 150                                   COMMON | COMMON_REF | SEEN | POPPED);
 151        return 0;
 152}
 153
 154/*
 155   This function marks a rev and its ancestors as common.
 156   In some cases, it is desirable to mark only the ancestors (for example
 157   when only the server does not yet know that they are common).
 158*/
 159
 160static void mark_common(struct commit *commit,
 161                int ancestors_only, int dont_parse)
 162{
 163        if (commit != NULL && !(commit->object.flags & COMMON)) {
 164                struct object *o = (struct object *)commit;
 165
 166                if (!ancestors_only)
 167                        o->flags |= COMMON;
 168
 169                if (!(o->flags & SEEN))
 170                        rev_list_push(commit, SEEN);
 171                else {
 172                        struct commit_list *parents;
 173
 174                        if (!ancestors_only && !(o->flags & POPPED))
 175                                non_common_revs--;
 176                        if (!o->parsed && !dont_parse)
 177                                if (parse_commit(commit))
 178                                        return;
 179
 180                        for (parents = commit->parents;
 181                                        parents;
 182                                        parents = parents->next)
 183                                mark_common(parents->item, 0, dont_parse);
 184                }
 185        }
 186}
 187
 188/*
 189  Get the next rev to send, ignoring the common.
 190*/
 191
 192static const struct object_id *get_rev(void)
 193{
 194        struct commit *commit = NULL;
 195
 196        while (commit == NULL) {
 197                unsigned int mark;
 198                struct commit_list *parents;
 199
 200                if (rev_list.nr == 0 || non_common_revs == 0)
 201                        return NULL;
 202
 203                commit = prio_queue_get(&rev_list);
 204                parse_commit(commit);
 205                parents = commit->parents;
 206
 207                commit->object.flags |= POPPED;
 208                if (!(commit->object.flags & COMMON))
 209                        non_common_revs--;
 210
 211                if (commit->object.flags & COMMON) {
 212                        /* do not send "have", and ignore ancestors */
 213                        commit = NULL;
 214                        mark = COMMON | SEEN;
 215                } else if (commit->object.flags & COMMON_REF)
 216                        /* send "have", and ignore ancestors */
 217                        mark = COMMON | SEEN;
 218                else
 219                        /* send "have", also for its ancestors */
 220                        mark = SEEN;
 221
 222                while (parents) {
 223                        if (!(parents->item->object.flags & SEEN))
 224                                rev_list_push(parents->item, mark);
 225                        if (mark & COMMON)
 226                                mark_common(parents->item, 1, 0);
 227                        parents = parents->next;
 228                }
 229        }
 230
 231        return &commit->object.oid;
 232}
 233
 234enum ack_type {
 235        NAK = 0,
 236        ACK,
 237        ACK_continue,
 238        ACK_common,
 239        ACK_ready
 240};
 241
 242static void consume_shallow_list(struct fetch_pack_args *args, int fd)
 243{
 244        if (args->stateless_rpc && args->deepen) {
 245                /* If we sent a depth we will get back "duplicate"
 246                 * shallow and unshallow commands every time there
 247                 * is a block of have lines exchanged.
 248                 */
 249                char *line;
 250                while ((line = packet_read_line(fd, NULL))) {
 251                        if (starts_with(line, "shallow "))
 252                                continue;
 253                        if (starts_with(line, "unshallow "))
 254                                continue;
 255                        die(_("git fetch-pack: expected shallow list"));
 256                }
 257        }
 258}
 259
 260static enum ack_type get_ack(int fd, struct object_id *result_oid)
 261{
 262        int len;
 263        char *line = packet_read_line(fd, &len);
 264        const char *arg;
 265
 266        if (!line)
 267                die(_("git fetch-pack: expected ACK/NAK, got a flush packet"));
 268        if (!strcmp(line, "NAK"))
 269                return NAK;
 270        if (skip_prefix(line, "ACK ", &arg)) {
 271                if (!get_oid_hex(arg, result_oid)) {
 272                        arg += 40;
 273                        len -= arg - line;
 274                        if (len < 1)
 275                                return ACK;
 276                        if (strstr(arg, "continue"))
 277                                return ACK_continue;
 278                        if (strstr(arg, "common"))
 279                                return ACK_common;
 280                        if (strstr(arg, "ready"))
 281                                return ACK_ready;
 282                        return ACK;
 283                }
 284        }
 285        if (skip_prefix(line, "ERR ", &arg))
 286                die(_("remote error: %s"), arg);
 287        die(_("git fetch-pack: expected ACK/NAK, got '%s'"), line);
 288}
 289
 290static void send_request(struct fetch_pack_args *args,
 291                         int fd, struct strbuf *buf)
 292{
 293        if (args->stateless_rpc) {
 294                send_sideband(fd, -1, buf->buf, buf->len, LARGE_PACKET_MAX);
 295                packet_flush(fd);
 296        } else
 297                write_or_die(fd, buf->buf, buf->len);
 298}
 299
 300static void insert_one_alternate_object(struct object *obj)
 301{
 302        rev_list_insert_ref(NULL, &obj->oid);
 303}
 304
 305#define INITIAL_FLUSH 16
 306#define PIPESAFE_FLUSH 32
 307#define LARGE_FLUSH 16384
 308
 309static int next_flush(int stateless_rpc, int count)
 310{
 311        if (stateless_rpc) {
 312                if (count < LARGE_FLUSH)
 313                        count <<= 1;
 314                else
 315                        count = count * 11 / 10;
 316        } else {
 317                if (count < PIPESAFE_FLUSH)
 318                        count <<= 1;
 319                else
 320                        count += PIPESAFE_FLUSH;
 321        }
 322        return count;
 323}
 324
 325static int find_common(struct fetch_pack_args *args,
 326                       int fd[2], struct object_id *result_oid,
 327                       struct ref *refs)
 328{
 329        int fetching;
 330        int count = 0, flushes = 0, flush_at = INITIAL_FLUSH, retval;
 331        const struct object_id *oid;
 332        unsigned in_vain = 0;
 333        int got_continue = 0;
 334        int got_ready = 0;
 335        struct strbuf req_buf = STRBUF_INIT;
 336        size_t state_len = 0;
 337
 338        if (args->stateless_rpc && multi_ack == 1)
 339                die(_("--stateless-rpc requires multi_ack_detailed"));
 340        if (marked)
 341                for_each_ref(clear_marks, NULL);
 342        marked = 1;
 343
 344        for_each_ref(rev_list_insert_ref_oid, NULL);
 345        for_each_cached_alternate(insert_one_alternate_object);
 346
 347        fetching = 0;
 348        for ( ; refs ; refs = refs->next) {
 349                struct object_id *remote = &refs->old_oid;
 350                const char *remote_hex;
 351                struct object *o;
 352
 353                /*
 354                 * If that object is complete (i.e. it is an ancestor of a
 355                 * local ref), we tell them we have it but do not have to
 356                 * tell them about its ancestors, which they already know
 357                 * about.
 358                 *
 359                 * We use lookup_object here because we are only
 360                 * interested in the case we *know* the object is
 361                 * reachable and we have already scanned it.
 362                 */
 363                if (((o = lookup_object(remote->hash)) != NULL) &&
 364                                (o->flags & COMPLETE)) {
 365                        continue;
 366                }
 367
 368                remote_hex = oid_to_hex(remote);
 369                if (!fetching) {
 370                        struct strbuf c = STRBUF_INIT;
 371                        if (multi_ack == 2)     strbuf_addstr(&c, " multi_ack_detailed");
 372                        if (multi_ack == 1)     strbuf_addstr(&c, " multi_ack");
 373                        if (no_done)            strbuf_addstr(&c, " no-done");
 374                        if (use_sideband == 2)  strbuf_addstr(&c, " side-band-64k");
 375                        if (use_sideband == 1)  strbuf_addstr(&c, " side-band");
 376                        if (args->deepen_relative) strbuf_addstr(&c, " deepen-relative");
 377                        if (args->use_thin_pack) strbuf_addstr(&c, " thin-pack");
 378                        if (args->no_progress)   strbuf_addstr(&c, " no-progress");
 379                        if (args->include_tag)   strbuf_addstr(&c, " include-tag");
 380                        if (prefer_ofs_delta)   strbuf_addstr(&c, " ofs-delta");
 381                        if (deepen_since_ok)    strbuf_addstr(&c, " deepen-since");
 382                        if (deepen_not_ok)      strbuf_addstr(&c, " deepen-not");
 383                        if (agent_supported)    strbuf_addf(&c, " agent=%s",
 384                                                            git_user_agent_sanitized());
 385                        if (args->filter_options.choice)
 386                                strbuf_addstr(&c, " filter");
 387                        packet_buf_write(&req_buf, "want %s%s\n", remote_hex, c.buf);
 388                        strbuf_release(&c);
 389                } else
 390                        packet_buf_write(&req_buf, "want %s\n", remote_hex);
 391                fetching++;
 392        }
 393
 394        if (!fetching) {
 395                strbuf_release(&req_buf);
 396                packet_flush(fd[1]);
 397                return 1;
 398        }
 399
 400        if (is_repository_shallow(the_repository))
 401                write_shallow_commits(&req_buf, 1, NULL);
 402        if (args->depth > 0)
 403                packet_buf_write(&req_buf, "deepen %d", args->depth);
 404        if (args->deepen_since) {
 405                timestamp_t max_age = approxidate(args->deepen_since);
 406                packet_buf_write(&req_buf, "deepen-since %"PRItime, max_age);
 407        }
 408        if (args->deepen_not) {
 409                int i;
 410                for (i = 0; i < args->deepen_not->nr; i++) {
 411                        struct string_list_item *s = args->deepen_not->items + i;
 412                        packet_buf_write(&req_buf, "deepen-not %s", s->string);
 413                }
 414        }
 415        if (server_supports_filtering && args->filter_options.choice)
 416                packet_buf_write(&req_buf, "filter %s",
 417                                 args->filter_options.filter_spec);
 418        packet_buf_flush(&req_buf);
 419        state_len = req_buf.len;
 420
 421        if (args->deepen) {
 422                char *line;
 423                const char *arg;
 424                struct object_id oid;
 425
 426                send_request(args, fd[1], &req_buf);
 427                while ((line = packet_read_line(fd[0], NULL))) {
 428                        if (skip_prefix(line, "shallow ", &arg)) {
 429                                if (get_oid_hex(arg, &oid))
 430                                        die(_("invalid shallow line: %s"), line);
 431                                register_shallow(the_repository, &oid);
 432                                continue;
 433                        }
 434                        if (skip_prefix(line, "unshallow ", &arg)) {
 435                                if (get_oid_hex(arg, &oid))
 436                                        die(_("invalid unshallow line: %s"), line);
 437                                if (!lookup_object(oid.hash))
 438                                        die(_("object not found: %s"), line);
 439                                /* make sure that it is parsed as shallow */
 440                                if (!parse_object(&oid))
 441                                        die(_("error in object: %s"), line);
 442                                if (unregister_shallow(&oid))
 443                                        die(_("no shallow found: %s"), line);
 444                                continue;
 445                        }
 446                        die(_("expected shallow/unshallow, got %s"), line);
 447                }
 448        } else if (!args->stateless_rpc)
 449                send_request(args, fd[1], &req_buf);
 450
 451        if (!args->stateless_rpc) {
 452                /* If we aren't using the stateless-rpc interface
 453                 * we don't need to retain the headers.
 454                 */
 455                strbuf_setlen(&req_buf, 0);
 456                state_len = 0;
 457        }
 458
 459        flushes = 0;
 460        retval = -1;
 461        if (args->no_dependents)
 462                goto done;
 463        while ((oid = get_rev())) {
 464                packet_buf_write(&req_buf, "have %s\n", oid_to_hex(oid));
 465                print_verbose(args, "have %s", oid_to_hex(oid));
 466                in_vain++;
 467                if (flush_at <= ++count) {
 468                        int ack;
 469
 470                        packet_buf_flush(&req_buf);
 471                        send_request(args, fd[1], &req_buf);
 472                        strbuf_setlen(&req_buf, state_len);
 473                        flushes++;
 474                        flush_at = next_flush(args->stateless_rpc, count);
 475
 476                        /*
 477                         * We keep one window "ahead" of the other side, and
 478                         * will wait for an ACK only on the next one
 479                         */
 480                        if (!args->stateless_rpc && count == INITIAL_FLUSH)
 481                                continue;
 482
 483                        consume_shallow_list(args, fd[0]);
 484                        do {
 485                                ack = get_ack(fd[0], result_oid);
 486                                if (ack)
 487                                        print_verbose(args, _("got %s %d %s"), "ack",
 488                                                      ack, oid_to_hex(result_oid));
 489                                switch (ack) {
 490                                case ACK:
 491                                        flushes = 0;
 492                                        multi_ack = 0;
 493                                        retval = 0;
 494                                        goto done;
 495                                case ACK_common:
 496                                case ACK_ready:
 497                                case ACK_continue: {
 498                                        struct commit *commit =
 499                                                lookup_commit(result_oid);
 500                                        if (!commit)
 501                                                die(_("invalid commit %s"), oid_to_hex(result_oid));
 502                                        if (args->stateless_rpc
 503                                         && ack == ACK_common
 504                                         && !(commit->object.flags & COMMON)) {
 505                                                /* We need to replay the have for this object
 506                                                 * on the next RPC request so the peer knows
 507                                                 * it is in common with us.
 508                                                 */
 509                                                const char *hex = oid_to_hex(result_oid);
 510                                                packet_buf_write(&req_buf, "have %s\n", hex);
 511                                                state_len = req_buf.len;
 512                                                /*
 513                                                 * Reset in_vain because an ack
 514                                                 * for this commit has not been
 515                                                 * seen.
 516                                                 */
 517                                                in_vain = 0;
 518                                        } else if (!args->stateless_rpc
 519                                                   || ack != ACK_common)
 520                                                in_vain = 0;
 521                                        mark_common(commit, 0, 1);
 522                                        retval = 0;
 523                                        got_continue = 1;
 524                                        if (ack == ACK_ready) {
 525                                                clear_prio_queue(&rev_list);
 526                                                got_ready = 1;
 527                                        }
 528                                        break;
 529                                        }
 530                                }
 531                        } while (ack);
 532                        flushes--;
 533                        if (got_continue && MAX_IN_VAIN < in_vain) {
 534                                print_verbose(args, _("giving up"));
 535                                break; /* give up */
 536                        }
 537                }
 538        }
 539done:
 540        if (!got_ready || !no_done) {
 541                packet_buf_write(&req_buf, "done\n");
 542                send_request(args, fd[1], &req_buf);
 543        }
 544        print_verbose(args, _("done"));
 545        if (retval != 0) {
 546                multi_ack = 0;
 547                flushes++;
 548        }
 549        strbuf_release(&req_buf);
 550
 551        if (!got_ready || !no_done)
 552                consume_shallow_list(args, fd[0]);
 553        while (flushes || multi_ack) {
 554                int ack = get_ack(fd[0], result_oid);
 555                if (ack) {
 556                        print_verbose(args, _("got %s (%d) %s"), "ack",
 557                                      ack, oid_to_hex(result_oid));
 558                        if (ack == ACK)
 559                                return 0;
 560                        multi_ack = 1;
 561                        continue;
 562                }
 563                flushes--;
 564        }
 565        /* it is no error to fetch into a completely empty repo */
 566        return count ? retval : 0;
 567}
 568
 569static struct commit_list *complete;
 570
 571static int mark_complete(const struct object_id *oid)
 572{
 573        struct object *o = parse_object(oid);
 574
 575        while (o && o->type == OBJ_TAG) {
 576                struct tag *t = (struct tag *) o;
 577                if (!t->tagged)
 578                        break; /* broken repository */
 579                o->flags |= COMPLETE;
 580                o = parse_object(&t->tagged->oid);
 581        }
 582        if (o && o->type == OBJ_COMMIT) {
 583                struct commit *commit = (struct commit *)o;
 584                if (!(commit->object.flags & COMPLETE)) {
 585                        commit->object.flags |= COMPLETE;
 586                        commit_list_insert(commit, &complete);
 587                }
 588        }
 589        return 0;
 590}
 591
 592static int mark_complete_oid(const char *refname, const struct object_id *oid,
 593                             int flag, void *cb_data)
 594{
 595        return mark_complete(oid);
 596}
 597
 598static void mark_recent_complete_commits(struct fetch_pack_args *args,
 599                                         timestamp_t cutoff)
 600{
 601        while (complete && cutoff <= complete->item->date) {
 602                print_verbose(args, _("Marking %s as complete"),
 603                              oid_to_hex(&complete->item->object.oid));
 604                pop_most_recent_commit(&complete, COMPLETE);
 605        }
 606}
 607
 608static void add_refs_to_oidset(struct oidset *oids, struct ref *refs)
 609{
 610        for (; refs; refs = refs->next)
 611                oidset_insert(oids, &refs->old_oid);
 612}
 613
 614static int tip_oids_contain(struct oidset *tip_oids,
 615                            struct ref *unmatched, struct ref *newlist,
 616                            const struct object_id *id)
 617{
 618        /*
 619         * Note that this only looks at the ref lists the first time it's
 620         * called. This works out in filter_refs() because even though it may
 621         * add to "newlist" between calls, the additions will always be for
 622         * oids that are already in the set.
 623         */
 624        if (!tip_oids->map.map.tablesize) {
 625                add_refs_to_oidset(tip_oids, unmatched);
 626                add_refs_to_oidset(tip_oids, newlist);
 627        }
 628        return oidset_contains(tip_oids, id);
 629}
 630
 631static void filter_refs(struct fetch_pack_args *args,
 632                        struct ref **refs,
 633                        struct ref **sought, int nr_sought)
 634{
 635        struct ref *newlist = NULL;
 636        struct ref **newtail = &newlist;
 637        struct ref *unmatched = NULL;
 638        struct ref *ref, *next;
 639        struct oidset tip_oids = OIDSET_INIT;
 640        int i;
 641
 642        i = 0;
 643        for (ref = *refs; ref; ref = next) {
 644                int keep = 0;
 645                next = ref->next;
 646
 647                if (starts_with(ref->name, "refs/") &&
 648                    check_refname_format(ref->name, 0))
 649                        ; /* trash */
 650                else {
 651                        while (i < nr_sought) {
 652                                int cmp = strcmp(ref->name, sought[i]->name);
 653                                if (cmp < 0)
 654                                        break; /* definitely do not have it */
 655                                else if (cmp == 0) {
 656                                        keep = 1; /* definitely have it */
 657                                        sought[i]->match_status = REF_MATCHED;
 658                                }
 659                                i++;
 660                        }
 661
 662                        if (!keep && args->fetch_all &&
 663                            (!args->deepen || !starts_with(ref->name, "refs/tags/")))
 664                                keep = 1;
 665                }
 666
 667                if (keep) {
 668                        *newtail = ref;
 669                        ref->next = NULL;
 670                        newtail = &ref->next;
 671                } else {
 672                        ref->next = unmatched;
 673                        unmatched = ref;
 674                }
 675        }
 676
 677        /* Append unmatched requests to the list */
 678        for (i = 0; i < nr_sought; i++) {
 679                struct object_id oid;
 680                const char *p;
 681
 682                ref = sought[i];
 683                if (ref->match_status != REF_NOT_MATCHED)
 684                        continue;
 685                if (parse_oid_hex(ref->name, &oid, &p) ||
 686                    *p != '\0' ||
 687                    oidcmp(&oid, &ref->old_oid))
 688                        continue;
 689
 690                if ((allow_unadvertised_object_request &
 691                     (ALLOW_TIP_SHA1 | ALLOW_REACHABLE_SHA1)) ||
 692                    tip_oids_contain(&tip_oids, unmatched, newlist,
 693                                     &ref->old_oid)) {
 694                        ref->match_status = REF_MATCHED;
 695                        *newtail = copy_ref(ref);
 696                        newtail = &(*newtail)->next;
 697                } else {
 698                        ref->match_status = REF_UNADVERTISED_NOT_ALLOWED;
 699                }
 700        }
 701
 702        oidset_clear(&tip_oids);
 703        for (ref = unmatched; ref; ref = next) {
 704                next = ref->next;
 705                free(ref);
 706        }
 707
 708        *refs = newlist;
 709}
 710
 711static void mark_alternate_complete(struct object *obj)
 712{
 713        mark_complete(&obj->oid);
 714}
 715
 716struct loose_object_iter {
 717        struct oidset *loose_object_set;
 718        struct ref *refs;
 719};
 720
 721/*
 722 *  If the number of refs is not larger than the number of loose objects,
 723 *  this function stops inserting.
 724 */
 725static int add_loose_objects_to_set(const struct object_id *oid,
 726                                    const char *path,
 727                                    void *data)
 728{
 729        struct loose_object_iter *iter = data;
 730        oidset_insert(iter->loose_object_set, oid);
 731        if (iter->refs == NULL)
 732                return 1;
 733
 734        iter->refs = iter->refs->next;
 735        return 0;
 736}
 737
 738static int everything_local(struct fetch_pack_args *args,
 739                            struct ref **refs,
 740                            struct ref **sought, int nr_sought)
 741{
 742        struct ref *ref;
 743        int retval;
 744        int old_save_commit_buffer = save_commit_buffer;
 745        timestamp_t cutoff = 0;
 746        struct oidset loose_oid_set = OIDSET_INIT;
 747        int use_oidset = 0;
 748        struct loose_object_iter iter = {&loose_oid_set, *refs};
 749
 750        /* Enumerate all loose objects or know refs are not so many. */
 751        use_oidset = !for_each_loose_object(add_loose_objects_to_set,
 752                                            &iter, 0);
 753
 754        save_commit_buffer = 0;
 755
 756        for (ref = *refs; ref; ref = ref->next) {
 757                struct object *o;
 758                unsigned int flags = OBJECT_INFO_QUICK;
 759
 760                if (use_oidset &&
 761                    !oidset_contains(&loose_oid_set, &ref->old_oid)) {
 762                        /*
 763                         * I know this does not exist in the loose form,
 764                         * so check if it exists in a non-loose form.
 765                         */
 766                        flags |= OBJECT_INFO_IGNORE_LOOSE;
 767                }
 768
 769                if (!has_object_file_with_flags(&ref->old_oid, flags))
 770                        continue;
 771                o = parse_object(&ref->old_oid);
 772                if (!o)
 773                        continue;
 774
 775                /* We already have it -- which may mean that we were
 776                 * in sync with the other side at some time after
 777                 * that (it is OK if we guess wrong here).
 778                 */
 779                if (o->type == OBJ_COMMIT) {
 780                        struct commit *commit = (struct commit *)o;
 781                        if (!cutoff || cutoff < commit->date)
 782                                cutoff = commit->date;
 783                }
 784        }
 785
 786        oidset_clear(&loose_oid_set);
 787
 788        if (!args->no_dependents) {
 789                if (!args->deepen) {
 790                        for_each_ref(mark_complete_oid, NULL);
 791                        for_each_cached_alternate(mark_alternate_complete);
 792                        commit_list_sort_by_date(&complete);
 793                        if (cutoff)
 794                                mark_recent_complete_commits(args, cutoff);
 795                }
 796
 797                /*
 798                 * Mark all complete remote refs as common refs.
 799                 * Don't mark them common yet; the server has to be told so first.
 800                 */
 801                for (ref = *refs; ref; ref = ref->next) {
 802                        struct object *o = deref_tag(lookup_object(ref->old_oid.hash),
 803                                                     NULL, 0);
 804
 805                        if (!o || o->type != OBJ_COMMIT || !(o->flags & COMPLETE))
 806                                continue;
 807
 808                        if (!(o->flags & SEEN)) {
 809                                rev_list_push((struct commit *)o, COMMON_REF | SEEN);
 810
 811                                mark_common((struct commit *)o, 1, 1);
 812                        }
 813                }
 814        }
 815
 816        filter_refs(args, refs, sought, nr_sought);
 817
 818        for (retval = 1, ref = *refs; ref ; ref = ref->next) {
 819                const struct object_id *remote = &ref->old_oid;
 820                struct object *o;
 821
 822                o = lookup_object(remote->hash);
 823                if (!o || !(o->flags & COMPLETE)) {
 824                        retval = 0;
 825                        print_verbose(args, "want %s (%s)", oid_to_hex(remote),
 826                                      ref->name);
 827                        continue;
 828                }
 829                print_verbose(args, _("already have %s (%s)"), oid_to_hex(remote),
 830                              ref->name);
 831        }
 832
 833        save_commit_buffer = old_save_commit_buffer;
 834
 835        return retval;
 836}
 837
 838static int sideband_demux(int in, int out, void *data)
 839{
 840        int *xd = data;
 841        int ret;
 842
 843        ret = recv_sideband("fetch-pack", xd[0], out);
 844        close(out);
 845        return ret;
 846}
 847
 848static int get_pack(struct fetch_pack_args *args,
 849                    int xd[2], char **pack_lockfile)
 850{
 851        struct async demux;
 852        int do_keep = args->keep_pack;
 853        const char *cmd_name;
 854        struct pack_header header;
 855        int pass_header = 0;
 856        struct child_process cmd = CHILD_PROCESS_INIT;
 857        int ret;
 858
 859        memset(&demux, 0, sizeof(demux));
 860        if (use_sideband) {
 861                /* xd[] is talking with upload-pack; subprocess reads from
 862                 * xd[0], spits out band#2 to stderr, and feeds us band#1
 863                 * through demux->out.
 864                 */
 865                demux.proc = sideband_demux;
 866                demux.data = xd;
 867                demux.out = -1;
 868                demux.isolate_sigpipe = 1;
 869                if (start_async(&demux))
 870                        die(_("fetch-pack: unable to fork off sideband demultiplexer"));
 871        }
 872        else
 873                demux.out = xd[0];
 874
 875        if (!args->keep_pack && unpack_limit) {
 876
 877                if (read_pack_header(demux.out, &header))
 878                        die(_("protocol error: bad pack header"));
 879                pass_header = 1;
 880                if (ntohl(header.hdr_entries) < unpack_limit)
 881                        do_keep = 0;
 882                else
 883                        do_keep = 1;
 884        }
 885
 886        if (alternate_shallow_file) {
 887                argv_array_push(&cmd.args, "--shallow-file");
 888                argv_array_push(&cmd.args, alternate_shallow_file);
 889        }
 890
 891        if (do_keep || args->from_promisor) {
 892                if (pack_lockfile)
 893                        cmd.out = -1;
 894                cmd_name = "index-pack";
 895                argv_array_push(&cmd.args, cmd_name);
 896                argv_array_push(&cmd.args, "--stdin");
 897                if (!args->quiet && !args->no_progress)
 898                        argv_array_push(&cmd.args, "-v");
 899                if (args->use_thin_pack)
 900                        argv_array_push(&cmd.args, "--fix-thin");
 901                if (do_keep && (args->lock_pack || unpack_limit)) {
 902                        char hostname[HOST_NAME_MAX + 1];
 903                        if (xgethostname(hostname, sizeof(hostname)))
 904                                xsnprintf(hostname, sizeof(hostname), "localhost");
 905                        argv_array_pushf(&cmd.args,
 906                                        "--keep=fetch-pack %"PRIuMAX " on %s",
 907                                        (uintmax_t)getpid(), hostname);
 908                }
 909                if (args->check_self_contained_and_connected)
 910                        argv_array_push(&cmd.args, "--check-self-contained-and-connected");
 911                if (args->from_promisor)
 912                        argv_array_push(&cmd.args, "--promisor");
 913        }
 914        else {
 915                cmd_name = "unpack-objects";
 916                argv_array_push(&cmd.args, cmd_name);
 917                if (args->quiet || args->no_progress)
 918                        argv_array_push(&cmd.args, "-q");
 919                args->check_self_contained_and_connected = 0;
 920        }
 921
 922        if (pass_header)
 923                argv_array_pushf(&cmd.args, "--pack_header=%"PRIu32",%"PRIu32,
 924                                 ntohl(header.hdr_version),
 925                                 ntohl(header.hdr_entries));
 926        if (fetch_fsck_objects >= 0
 927            ? fetch_fsck_objects
 928            : transfer_fsck_objects >= 0
 929            ? transfer_fsck_objects
 930            : 0) {
 931                if (args->from_promisor)
 932                        /*
 933                         * We cannot use --strict in index-pack because it
 934                         * checks both broken objects and links, but we only
 935                         * want to check for broken objects.
 936                         */
 937                        argv_array_push(&cmd.args, "--fsck-objects");
 938                else
 939                        argv_array_push(&cmd.args, "--strict");
 940        }
 941
 942        cmd.in = demux.out;
 943        cmd.git_cmd = 1;
 944        if (start_command(&cmd))
 945                die(_("fetch-pack: unable to fork off %s"), cmd_name);
 946        if (do_keep && pack_lockfile) {
 947                *pack_lockfile = index_pack_lockfile(cmd.out);
 948                close(cmd.out);
 949        }
 950
 951        if (!use_sideband)
 952                /* Closed by start_command() */
 953                xd[0] = -1;
 954
 955        ret = finish_command(&cmd);
 956        if (!ret || (args->check_self_contained_and_connected && ret == 1))
 957                args->self_contained_and_connected =
 958                        args->check_self_contained_and_connected &&
 959                        ret == 0;
 960        else
 961                die(_("%s failed"), cmd_name);
 962        if (use_sideband && finish_async(&demux))
 963                die(_("error in sideband demultiplexer"));
 964        return 0;
 965}
 966
 967static int cmp_ref_by_name(const void *a_, const void *b_)
 968{
 969        const struct ref *a = *((const struct ref **)a_);
 970        const struct ref *b = *((const struct ref **)b_);
 971        return strcmp(a->name, b->name);
 972}
 973
 974static struct ref *do_fetch_pack(struct fetch_pack_args *args,
 975                                 int fd[2],
 976                                 const struct ref *orig_ref,
 977                                 struct ref **sought, int nr_sought,
 978                                 struct shallow_info *si,
 979                                 char **pack_lockfile)
 980{
 981        struct ref *ref = copy_ref_list(orig_ref);
 982        struct object_id oid;
 983        const char *agent_feature;
 984        int agent_len;
 985
 986        sort_ref_list(&ref, ref_compare_name);
 987        QSORT(sought, nr_sought, cmp_ref_by_name);
 988
 989        if ((args->depth > 0 || is_repository_shallow(the_repository)) && !server_supports("shallow"))
 990                die(_("Server does not support shallow clients"));
 991        if (args->depth > 0 || args->deepen_since || args->deepen_not)
 992                args->deepen = 1;
 993        if (server_supports("multi_ack_detailed")) {
 994                print_verbose(args, _("Server supports multi_ack_detailed"));
 995                multi_ack = 2;
 996                if (server_supports("no-done")) {
 997                        print_verbose(args, _("Server supports no-done"));
 998                        if (args->stateless_rpc)
 999                                no_done = 1;
1000                }
1001        }
1002        else if (server_supports("multi_ack")) {
1003                print_verbose(args, _("Server supports multi_ack"));
1004                multi_ack = 1;
1005        }
1006        if (server_supports("side-band-64k")) {
1007                print_verbose(args, _("Server supports side-band-64k"));
1008                use_sideband = 2;
1009        }
1010        else if (server_supports("side-band")) {
1011                print_verbose(args, _("Server supports side-band"));
1012                use_sideband = 1;
1013        }
1014        if (server_supports("allow-tip-sha1-in-want")) {
1015                print_verbose(args, _("Server supports allow-tip-sha1-in-want"));
1016                allow_unadvertised_object_request |= ALLOW_TIP_SHA1;
1017        }
1018        if (server_supports("allow-reachable-sha1-in-want")) {
1019                print_verbose(args, _("Server supports allow-reachable-sha1-in-want"));
1020                allow_unadvertised_object_request |= ALLOW_REACHABLE_SHA1;
1021        }
1022        if (!server_supports("thin-pack"))
1023                args->use_thin_pack = 0;
1024        if (!server_supports("no-progress"))
1025                args->no_progress = 0;
1026        if (!server_supports("include-tag"))
1027                args->include_tag = 0;
1028        if (server_supports("ofs-delta"))
1029                print_verbose(args, _("Server supports ofs-delta"));
1030        else
1031                prefer_ofs_delta = 0;
1032
1033        if (server_supports("filter")) {
1034                server_supports_filtering = 1;
1035                print_verbose(args, _("Server supports filter"));
1036        } else if (args->filter_options.choice) {
1037                warning("filtering not recognized by server, ignoring");
1038        }
1039
1040        if ((agent_feature = server_feature_value("agent", &agent_len))) {
1041                agent_supported = 1;
1042                if (agent_len)
1043                        print_verbose(args, _("Server version is %.*s"),
1044                                      agent_len, agent_feature);
1045        }
1046        if (server_supports("deepen-since"))
1047                deepen_since_ok = 1;
1048        else if (args->deepen_since)
1049                die(_("Server does not support --shallow-since"));
1050        if (server_supports("deepen-not"))
1051                deepen_not_ok = 1;
1052        else if (args->deepen_not)
1053                die(_("Server does not support --shallow-exclude"));
1054        if (!server_supports("deepen-relative") && args->deepen_relative)
1055                die(_("Server does not support --deepen"));
1056
1057        if (everything_local(args, &ref, sought, nr_sought)) {
1058                packet_flush(fd[1]);
1059                goto all_done;
1060        }
1061        if (find_common(args, fd, &oid, ref) < 0)
1062                if (!args->keep_pack)
1063                        /* When cloning, it is not unusual to have
1064                         * no common commit.
1065                         */
1066                        warning(_("no common commits"));
1067
1068        if (args->stateless_rpc)
1069                packet_flush(fd[1]);
1070        if (args->deepen)
1071                setup_alternate_shallow(&shallow_lock, &alternate_shallow_file,
1072                                        NULL);
1073        else if (si->nr_ours || si->nr_theirs)
1074                alternate_shallow_file = setup_temporary_shallow(si->shallow);
1075        else
1076                alternate_shallow_file = NULL;
1077        if (get_pack(args, fd, pack_lockfile))
1078                die(_("git fetch-pack: fetch failed."));
1079
1080 all_done:
1081        return ref;
1082}
1083
1084static void add_shallow_requests(struct strbuf *req_buf,
1085                                 const struct fetch_pack_args *args)
1086{
1087        if (is_repository_shallow(the_repository))
1088                write_shallow_commits(req_buf, 1, NULL);
1089        if (args->depth > 0)
1090                packet_buf_write(req_buf, "deepen %d", args->depth);
1091        if (args->deepen_since) {
1092                timestamp_t max_age = approxidate(args->deepen_since);
1093                packet_buf_write(req_buf, "deepen-since %"PRItime, max_age);
1094        }
1095        if (args->deepen_not) {
1096                int i;
1097                for (i = 0; i < args->deepen_not->nr; i++) {
1098                        struct string_list_item *s = args->deepen_not->items + i;
1099                        packet_buf_write(req_buf, "deepen-not %s", s->string);
1100                }
1101        }
1102}
1103
1104static void add_wants(const struct ref *wants, struct strbuf *req_buf)
1105{
1106        for ( ; wants ; wants = wants->next) {
1107                const struct object_id *remote = &wants->old_oid;
1108                const char *remote_hex;
1109                struct object *o;
1110
1111                /*
1112                 * If that object is complete (i.e. it is an ancestor of a
1113                 * local ref), we tell them we have it but do not have to
1114                 * tell them about its ancestors, which they already know
1115                 * about.
1116                 *
1117                 * We use lookup_object here because we are only
1118                 * interested in the case we *know* the object is
1119                 * reachable and we have already scanned it.
1120                 */
1121                if (((o = lookup_object(remote->hash)) != NULL) &&
1122                    (o->flags & COMPLETE)) {
1123                        continue;
1124                }
1125
1126                remote_hex = oid_to_hex(remote);
1127                packet_buf_write(req_buf, "want %s\n", remote_hex);
1128        }
1129}
1130
1131static void add_common(struct strbuf *req_buf, struct oidset *common)
1132{
1133        struct oidset_iter iter;
1134        const struct object_id *oid;
1135        oidset_iter_init(common, &iter);
1136
1137        while ((oid = oidset_iter_next(&iter))) {
1138                packet_buf_write(req_buf, "have %s\n", oid_to_hex(oid));
1139        }
1140}
1141
1142static int add_haves(struct strbuf *req_buf, int *haves_to_send, int *in_vain)
1143{
1144        int ret = 0;
1145        int haves_added = 0;
1146        const struct object_id *oid;
1147
1148        while ((oid = get_rev())) {
1149                packet_buf_write(req_buf, "have %s\n", oid_to_hex(oid));
1150                if (++haves_added >= *haves_to_send)
1151                        break;
1152        }
1153
1154        *in_vain += haves_added;
1155        if (!haves_added || *in_vain >= MAX_IN_VAIN) {
1156                /* Send Done */
1157                packet_buf_write(req_buf, "done\n");
1158                ret = 1;
1159        }
1160
1161        /* Increase haves to send on next round */
1162        *haves_to_send = next_flush(1, *haves_to_send);
1163
1164        return ret;
1165}
1166
1167static int send_fetch_request(int fd_out, const struct fetch_pack_args *args,
1168                              const struct ref *wants, struct oidset *common,
1169                              int *haves_to_send, int *in_vain)
1170{
1171        int ret = 0;
1172        struct strbuf req_buf = STRBUF_INIT;
1173
1174        if (server_supports_v2("fetch", 1))
1175                packet_buf_write(&req_buf, "command=fetch");
1176        if (server_supports_v2("agent", 0))
1177                packet_buf_write(&req_buf, "agent=%s", git_user_agent_sanitized());
1178        if (args->server_options && args->server_options->nr &&
1179            server_supports_v2("server-option", 1)) {
1180                int i;
1181                for (i = 0; i < args->server_options->nr; i++)
1182                        packet_write_fmt(fd_out, "server-option=%s",
1183                                         args->server_options->items[i].string);
1184        }
1185
1186        packet_buf_delim(&req_buf);
1187        if (args->use_thin_pack)
1188                packet_buf_write(&req_buf, "thin-pack");
1189        if (args->no_progress)
1190                packet_buf_write(&req_buf, "no-progress");
1191        if (args->include_tag)
1192                packet_buf_write(&req_buf, "include-tag");
1193        if (prefer_ofs_delta)
1194                packet_buf_write(&req_buf, "ofs-delta");
1195
1196        /* Add shallow-info and deepen request */
1197        if (server_supports_feature("fetch", "shallow", 0))
1198                add_shallow_requests(&req_buf, args);
1199        else if (is_repository_shallow(the_repository) || args->deepen)
1200                die(_("Server does not support shallow requests"));
1201
1202        /* Add filter */
1203        if (server_supports_feature("fetch", "filter", 0) &&
1204            args->filter_options.choice) {
1205                print_verbose(args, _("Server supports filter"));
1206                packet_buf_write(&req_buf, "filter %s",
1207                                 args->filter_options.filter_spec);
1208        } else if (args->filter_options.choice) {
1209                warning("filtering not recognized by server, ignoring");
1210        }
1211
1212        /* add wants */
1213        add_wants(wants, &req_buf);
1214
1215        if (args->no_dependents) {
1216                packet_buf_write(&req_buf, "done");
1217                ret = 1;
1218        } else {
1219                /* Add all of the common commits we've found in previous rounds */
1220                add_common(&req_buf, common);
1221
1222                /* Add initial haves */
1223                ret = add_haves(&req_buf, haves_to_send, in_vain);
1224        }
1225
1226        /* Send request */
1227        packet_buf_flush(&req_buf);
1228        write_or_die(fd_out, req_buf.buf, req_buf.len);
1229
1230        strbuf_release(&req_buf);
1231        return ret;
1232}
1233
1234/*
1235 * Processes a section header in a server's response and checks if it matches
1236 * `section`.  If the value of `peek` is 1, the header line will be peeked (and
1237 * not consumed); if 0, the line will be consumed and the function will die if
1238 * the section header doesn't match what was expected.
1239 */
1240static int process_section_header(struct packet_reader *reader,
1241                                  const char *section, int peek)
1242{
1243        int ret;
1244
1245        if (packet_reader_peek(reader) != PACKET_READ_NORMAL)
1246                die("error reading section header '%s'", section);
1247
1248        ret = !strcmp(reader->line, section);
1249
1250        if (!peek) {
1251                if (!ret)
1252                        die("expected '%s', received '%s'",
1253                            section, reader->line);
1254                packet_reader_read(reader);
1255        }
1256
1257        return ret;
1258}
1259
1260static int process_acks(struct packet_reader *reader, struct oidset *common)
1261{
1262        /* received */
1263        int received_ready = 0;
1264        int received_ack = 0;
1265
1266        process_section_header(reader, "acknowledgments", 0);
1267        while (packet_reader_read(reader) == PACKET_READ_NORMAL) {
1268                const char *arg;
1269
1270                if (!strcmp(reader->line, "NAK"))
1271                        continue;
1272
1273                if (skip_prefix(reader->line, "ACK ", &arg)) {
1274                        struct object_id oid;
1275                        if (!get_oid_hex(arg, &oid)) {
1276                                struct commit *commit;
1277                                oidset_insert(common, &oid);
1278                                commit = lookup_commit(&oid);
1279                                mark_common(commit, 0, 1);
1280                        }
1281                        continue;
1282                }
1283
1284                if (!strcmp(reader->line, "ready")) {
1285                        clear_prio_queue(&rev_list);
1286                        received_ready = 1;
1287                        continue;
1288                }
1289
1290                die("unexpected acknowledgment line: '%s'", reader->line);
1291        }
1292
1293        if (reader->status != PACKET_READ_FLUSH &&
1294            reader->status != PACKET_READ_DELIM)
1295                die("error processing acks: %d", reader->status);
1296
1297        /* return 0 if no common, 1 if there are common, or 2 if ready */
1298        return received_ready ? 2 : (received_ack ? 1 : 0);
1299}
1300
1301static void receive_shallow_info(struct fetch_pack_args *args,
1302                                 struct packet_reader *reader)
1303{
1304        process_section_header(reader, "shallow-info", 0);
1305        while (packet_reader_read(reader) == PACKET_READ_NORMAL) {
1306                const char *arg;
1307                struct object_id oid;
1308
1309                if (skip_prefix(reader->line, "shallow ", &arg)) {
1310                        if (get_oid_hex(arg, &oid))
1311                                die(_("invalid shallow line: %s"), reader->line);
1312                        register_shallow(the_repository, &oid);
1313                        continue;
1314                }
1315                if (skip_prefix(reader->line, "unshallow ", &arg)) {
1316                        if (get_oid_hex(arg, &oid))
1317                                die(_("invalid unshallow line: %s"), reader->line);
1318                        if (!lookup_object(oid.hash))
1319                                die(_("object not found: %s"), reader->line);
1320                        /* make sure that it is parsed as shallow */
1321                        if (!parse_object(&oid))
1322                                die(_("error in object: %s"), reader->line);
1323                        if (unregister_shallow(&oid))
1324                                die(_("no shallow found: %s"), reader->line);
1325                        continue;
1326                }
1327                die(_("expected shallow/unshallow, got %s"), reader->line);
1328        }
1329
1330        if (reader->status != PACKET_READ_FLUSH &&
1331            reader->status != PACKET_READ_DELIM)
1332                die("error processing shallow info: %d", reader->status);
1333
1334        setup_alternate_shallow(&shallow_lock, &alternate_shallow_file, NULL);
1335        args->deepen = 1;
1336}
1337
1338enum fetch_state {
1339        FETCH_CHECK_LOCAL = 0,
1340        FETCH_SEND_REQUEST,
1341        FETCH_PROCESS_ACKS,
1342        FETCH_GET_PACK,
1343        FETCH_DONE,
1344};
1345
1346static struct ref *do_fetch_pack_v2(struct fetch_pack_args *args,
1347                                    int fd[2],
1348                                    const struct ref *orig_ref,
1349                                    struct ref **sought, int nr_sought,
1350                                    char **pack_lockfile)
1351{
1352        struct ref *ref = copy_ref_list(orig_ref);
1353        enum fetch_state state = FETCH_CHECK_LOCAL;
1354        struct oidset common = OIDSET_INIT;
1355        struct packet_reader reader;
1356        int in_vain = 0;
1357        int haves_to_send = INITIAL_FLUSH;
1358        packet_reader_init(&reader, fd[0], NULL, 0,
1359                           PACKET_READ_CHOMP_NEWLINE);
1360
1361        while (state != FETCH_DONE) {
1362                switch (state) {
1363                case FETCH_CHECK_LOCAL:
1364                        sort_ref_list(&ref, ref_compare_name);
1365                        QSORT(sought, nr_sought, cmp_ref_by_name);
1366
1367                        /* v2 supports these by default */
1368                        allow_unadvertised_object_request |= ALLOW_REACHABLE_SHA1;
1369                        use_sideband = 2;
1370                        if (args->depth > 0 || args->deepen_since || args->deepen_not)
1371                                args->deepen = 1;
1372
1373                        if (marked)
1374                                for_each_ref(clear_marks, NULL);
1375                        marked = 1;
1376
1377                        for_each_ref(rev_list_insert_ref_oid, NULL);
1378                        for_each_cached_alternate(insert_one_alternate_object);
1379
1380                        /* Filter 'ref' by 'sought' and those that aren't local */
1381                        if (everything_local(args, &ref, sought, nr_sought))
1382                                state = FETCH_DONE;
1383                        else
1384                                state = FETCH_SEND_REQUEST;
1385                        break;
1386                case FETCH_SEND_REQUEST:
1387                        if (send_fetch_request(fd[1], args, ref, &common,
1388                                               &haves_to_send, &in_vain))
1389                                state = FETCH_GET_PACK;
1390                        else
1391                                state = FETCH_PROCESS_ACKS;
1392                        break;
1393                case FETCH_PROCESS_ACKS:
1394                        /* Process ACKs/NAKs */
1395                        switch (process_acks(&reader, &common)) {
1396                        case 2:
1397                                state = FETCH_GET_PACK;
1398                                break;
1399                        case 1:
1400                                in_vain = 0;
1401                                /* fallthrough */
1402                        default:
1403                                state = FETCH_SEND_REQUEST;
1404                                break;
1405                        }
1406                        break;
1407                case FETCH_GET_PACK:
1408                        /* Check for shallow-info section */
1409                        if (process_section_header(&reader, "shallow-info", 1))
1410                                receive_shallow_info(args, &reader);
1411
1412                        /* get the pack */
1413                        process_section_header(&reader, "packfile", 0);
1414                        if (get_pack(args, fd, pack_lockfile))
1415                                die(_("git fetch-pack: fetch failed."));
1416
1417                        state = FETCH_DONE;
1418                        break;
1419                case FETCH_DONE:
1420                        continue;
1421                }
1422        }
1423
1424        oidset_clear(&common);
1425        return ref;
1426}
1427
1428static void fetch_pack_config(void)
1429{
1430        git_config_get_int("fetch.unpacklimit", &fetch_unpack_limit);
1431        git_config_get_int("transfer.unpacklimit", &transfer_unpack_limit);
1432        git_config_get_bool("repack.usedeltabaseoffset", &prefer_ofs_delta);
1433        git_config_get_bool("fetch.fsckobjects", &fetch_fsck_objects);
1434        git_config_get_bool("transfer.fsckobjects", &transfer_fsck_objects);
1435
1436        git_config(git_default_config, NULL);
1437}
1438
1439static void fetch_pack_setup(void)
1440{
1441        static int did_setup;
1442        if (did_setup)
1443                return;
1444        fetch_pack_config();
1445        if (0 <= transfer_unpack_limit)
1446                unpack_limit = transfer_unpack_limit;
1447        else if (0 <= fetch_unpack_limit)
1448                unpack_limit = fetch_unpack_limit;
1449        did_setup = 1;
1450}
1451
1452static int remove_duplicates_in_refs(struct ref **ref, int nr)
1453{
1454        struct string_list names = STRING_LIST_INIT_NODUP;
1455        int src, dst;
1456
1457        for (src = dst = 0; src < nr; src++) {
1458                struct string_list_item *item;
1459                item = string_list_insert(&names, ref[src]->name);
1460                if (item->util)
1461                        continue; /* already have it */
1462                item->util = ref[src];
1463                if (src != dst)
1464                        ref[dst] = ref[src];
1465                dst++;
1466        }
1467        for (src = dst; src < nr; src++)
1468                ref[src] = NULL;
1469        string_list_clear(&names, 0);
1470        return dst;
1471}
1472
1473static void update_shallow(struct fetch_pack_args *args,
1474                           struct ref **sought, int nr_sought,
1475                           struct shallow_info *si)
1476{
1477        struct oid_array ref = OID_ARRAY_INIT;
1478        int *status;
1479        int i;
1480
1481        if (args->deepen && alternate_shallow_file) {
1482                if (*alternate_shallow_file == '\0') { /* --unshallow */
1483                        unlink_or_warn(git_path_shallow(the_repository));
1484                        rollback_lock_file(&shallow_lock);
1485                } else
1486                        commit_lock_file(&shallow_lock);
1487                return;
1488        }
1489
1490        if (!si->shallow || !si->shallow->nr)
1491                return;
1492
1493        if (args->cloning) {
1494                /*
1495                 * remote is shallow, but this is a clone, there are
1496                 * no objects in repo to worry about. Accept any
1497                 * shallow points that exist in the pack (iow in repo
1498                 * after get_pack() and reprepare_packed_git())
1499                 */
1500                struct oid_array extra = OID_ARRAY_INIT;
1501                struct object_id *oid = si->shallow->oid;
1502                for (i = 0; i < si->shallow->nr; i++)
1503                        if (has_object_file(&oid[i]))
1504                                oid_array_append(&extra, &oid[i]);
1505                if (extra.nr) {
1506                        setup_alternate_shallow(&shallow_lock,
1507                                                &alternate_shallow_file,
1508                                                &extra);
1509                        commit_lock_file(&shallow_lock);
1510                }
1511                oid_array_clear(&extra);
1512                return;
1513        }
1514
1515        if (!si->nr_ours && !si->nr_theirs)
1516                return;
1517
1518        remove_nonexistent_theirs_shallow(si);
1519        if (!si->nr_ours && !si->nr_theirs)
1520                return;
1521        for (i = 0; i < nr_sought; i++)
1522                oid_array_append(&ref, &sought[i]->old_oid);
1523        si->ref = &ref;
1524
1525        if (args->update_shallow) {
1526                /*
1527                 * remote is also shallow, .git/shallow may be updated
1528                 * so all refs can be accepted. Make sure we only add
1529                 * shallow roots that are actually reachable from new
1530                 * refs.
1531                 */
1532                struct oid_array extra = OID_ARRAY_INIT;
1533                struct object_id *oid = si->shallow->oid;
1534                assign_shallow_commits_to_refs(si, NULL, NULL);
1535                if (!si->nr_ours && !si->nr_theirs) {
1536                        oid_array_clear(&ref);
1537                        return;
1538                }
1539                for (i = 0; i < si->nr_ours; i++)
1540                        oid_array_append(&extra, &oid[si->ours[i]]);
1541                for (i = 0; i < si->nr_theirs; i++)
1542                        oid_array_append(&extra, &oid[si->theirs[i]]);
1543                setup_alternate_shallow(&shallow_lock,
1544                                        &alternate_shallow_file,
1545                                        &extra);
1546                commit_lock_file(&shallow_lock);
1547                oid_array_clear(&extra);
1548                oid_array_clear(&ref);
1549                return;
1550        }
1551
1552        /*
1553         * remote is also shallow, check what ref is safe to update
1554         * without updating .git/shallow
1555         */
1556        status = xcalloc(nr_sought, sizeof(*status));
1557        assign_shallow_commits_to_refs(si, NULL, status);
1558        if (si->nr_ours || si->nr_theirs) {
1559                for (i = 0; i < nr_sought; i++)
1560                        if (status[i])
1561                                sought[i]->status = REF_STATUS_REJECT_SHALLOW;
1562        }
1563        free(status);
1564        oid_array_clear(&ref);
1565}
1566
1567struct ref *fetch_pack(struct fetch_pack_args *args,
1568                       int fd[], struct child_process *conn,
1569                       const struct ref *ref,
1570                       const char *dest,
1571                       struct ref **sought, int nr_sought,
1572                       struct oid_array *shallow,
1573                       char **pack_lockfile,
1574                       enum protocol_version version)
1575{
1576        struct ref *ref_cpy;
1577        struct shallow_info si;
1578
1579        fetch_pack_setup();
1580        if (nr_sought)
1581                nr_sought = remove_duplicates_in_refs(sought, nr_sought);
1582
1583        if (!ref) {
1584                packet_flush(fd[1]);
1585                die(_("no matching remote head"));
1586        }
1587        prepare_shallow_info(&si, shallow);
1588        if (version == protocol_v2)
1589                ref_cpy = do_fetch_pack_v2(args, fd, ref, sought, nr_sought,
1590                                           pack_lockfile);
1591        else
1592                ref_cpy = do_fetch_pack(args, fd, ref, sought, nr_sought,
1593                                        &si, pack_lockfile);
1594        reprepare_packed_git(the_repository);
1595        update_shallow(args, sought, nr_sought, &si);
1596        clear_shallow_info(&si);
1597        return ref_cpy;
1598}
1599
1600int report_unmatched_refs(struct ref **sought, int nr_sought)
1601{
1602        int i, ret = 0;
1603
1604        for (i = 0; i < nr_sought; i++) {
1605                if (!sought[i])
1606                        continue;
1607                switch (sought[i]->match_status) {
1608                case REF_MATCHED:
1609                        continue;
1610                case REF_NOT_MATCHED:
1611                        error(_("no such remote ref %s"), sought[i]->name);
1612                        break;
1613                case REF_UNADVERTISED_NOT_ALLOWED:
1614                        error(_("Server does not allow request for unadvertised object %s"),
1615                              sought[i]->name);
1616                        break;
1617                }
1618                ret = 1;
1619        }
1620        return ret;
1621}