fetch-pack.con commit commit: allow lookup_commit_graft to handle arbitrary repositories (b9dbddf)
   1#include "cache.h"
   2#include "repository.h"
   3#include "config.h"
   4#include "lockfile.h"
   5#include "refs.h"
   6#include "pkt-line.h"
   7#include "commit.h"
   8#include "tag.h"
   9#include "exec_cmd.h"
  10#include "pack.h"
  11#include "sideband.h"
  12#include "fetch-pack.h"
  13#include "remote.h"
  14#include "run-command.h"
  15#include "connect.h"
  16#include "transport.h"
  17#include "version.h"
  18#include "prio-queue.h"
  19#include "sha1-array.h"
  20#include "oidset.h"
  21#include "packfile.h"
  22#include "object-store.h"
  23
  24static int transfer_unpack_limit = -1;
  25static int fetch_unpack_limit = -1;
  26static int unpack_limit = 100;
  27static int prefer_ofs_delta = 1;
  28static int no_done;
  29static int deepen_since_ok;
  30static int deepen_not_ok;
  31static int fetch_fsck_objects = -1;
  32static int transfer_fsck_objects = -1;
  33static int agent_supported;
  34static int server_supports_filtering;
  35static struct lock_file shallow_lock;
  36static const char *alternate_shallow_file;
  37
  38/* Remember to update object flag allocation in object.h */
  39#define COMPLETE        (1U << 0)
  40#define COMMON          (1U << 1)
  41#define COMMON_REF      (1U << 2)
  42#define SEEN            (1U << 3)
  43#define POPPED          (1U << 4)
  44#define ALTERNATE       (1U << 5)
  45
  46static int marked;
  47
  48/*
  49 * After sending this many "have"s if we do not get any new ACK , we
  50 * give up traversing our history.
  51 */
  52#define MAX_IN_VAIN 256
  53
  54static struct prio_queue rev_list = { compare_commits_by_commit_date };
  55static int non_common_revs, multi_ack, use_sideband;
  56/* Allow specifying sha1 if it is a ref tip. */
  57#define ALLOW_TIP_SHA1  01
  58/* Allow request of a sha1 if it is reachable from a ref (possibly hidden ref). */
  59#define ALLOW_REACHABLE_SHA1    02
  60static unsigned int allow_unadvertised_object_request;
  61
  62__attribute__((format (printf, 2, 3)))
  63static inline void print_verbose(const struct fetch_pack_args *args,
  64                                 const char *fmt, ...)
  65{
  66        va_list params;
  67
  68        if (!args->verbose)
  69                return;
  70
  71        va_start(params, fmt);
  72        vfprintf(stderr, fmt, params);
  73        va_end(params);
  74        fputc('\n', stderr);
  75}
  76
  77struct alternate_object_cache {
  78        struct object **items;
  79        size_t nr, alloc;
  80};
  81
  82static void cache_one_alternate(const char *refname,
  83                                const struct object_id *oid,
  84                                void *vcache)
  85{
  86        struct alternate_object_cache *cache = vcache;
  87        struct object *obj = parse_object(oid);
  88
  89        if (!obj || (obj->flags & ALTERNATE))
  90                return;
  91
  92        obj->flags |= ALTERNATE;
  93        ALLOC_GROW(cache->items, cache->nr + 1, cache->alloc);
  94        cache->items[cache->nr++] = obj;
  95}
  96
  97static void for_each_cached_alternate(void (*cb)(struct object *))
  98{
  99        static int initialized;
 100        static struct alternate_object_cache cache;
 101        size_t i;
 102
 103        if (!initialized) {
 104                for_each_alternate_ref(cache_one_alternate, &cache);
 105                initialized = 1;
 106        }
 107
 108        for (i = 0; i < cache.nr; i++)
 109                cb(cache.items[i]);
 110}
 111
 112static void rev_list_push(struct commit *commit, int mark)
 113{
 114        if (!(commit->object.flags & mark)) {
 115                commit->object.flags |= mark;
 116
 117                if (parse_commit(commit))
 118                        return;
 119
 120                prio_queue_put(&rev_list, commit);
 121
 122                if (!(commit->object.flags & COMMON))
 123                        non_common_revs++;
 124        }
 125}
 126
 127static int rev_list_insert_ref(const char *refname, const struct object_id *oid)
 128{
 129        struct object *o = deref_tag(parse_object(oid), refname, 0);
 130
 131        if (o && o->type == OBJ_COMMIT)
 132                rev_list_push((struct commit *)o, SEEN);
 133
 134        return 0;
 135}
 136
 137static int rev_list_insert_ref_oid(const char *refname, const struct object_id *oid,
 138                                   int flag, void *cb_data)
 139{
 140        return rev_list_insert_ref(refname, oid);
 141}
 142
 143static int clear_marks(const char *refname, const struct object_id *oid,
 144                       int flag, void *cb_data)
 145{
 146        struct object *o = deref_tag(parse_object(oid), refname, 0);
 147
 148        if (o && o->type == OBJ_COMMIT)
 149                clear_commit_marks((struct commit *)o,
 150                                   COMMON | COMMON_REF | SEEN | POPPED);
 151        return 0;
 152}
 153
 154/*
 155   This function marks a rev and its ancestors as common.
 156   In some cases, it is desirable to mark only the ancestors (for example
 157   when only the server does not yet know that they are common).
 158*/
 159
 160static void mark_common(struct commit *commit,
 161                int ancestors_only, int dont_parse)
 162{
 163        if (commit != NULL && !(commit->object.flags & COMMON)) {
 164                struct object *o = (struct object *)commit;
 165
 166                if (!ancestors_only)
 167                        o->flags |= COMMON;
 168
 169                if (!(o->flags & SEEN))
 170                        rev_list_push(commit, SEEN);
 171                else {
 172                        struct commit_list *parents;
 173
 174                        if (!ancestors_only && !(o->flags & POPPED))
 175                                non_common_revs--;
 176                        if (!o->parsed && !dont_parse)
 177                                if (parse_commit(commit))
 178                                        return;
 179
 180                        for (parents = commit->parents;
 181                                        parents;
 182                                        parents = parents->next)
 183                                mark_common(parents->item, 0, dont_parse);
 184                }
 185        }
 186}
 187
 188/*
 189  Get the next rev to send, ignoring the common.
 190*/
 191
 192static const struct object_id *get_rev(void)
 193{
 194        struct commit *commit = NULL;
 195
 196        while (commit == NULL) {
 197                unsigned int mark;
 198                struct commit_list *parents;
 199
 200                if (rev_list.nr == 0 || non_common_revs == 0)
 201                        return NULL;
 202
 203                commit = prio_queue_get(&rev_list);
 204                parse_commit(commit);
 205                parents = commit->parents;
 206
 207                commit->object.flags |= POPPED;
 208                if (!(commit->object.flags & COMMON))
 209                        non_common_revs--;
 210
 211                if (commit->object.flags & COMMON) {
 212                        /* do not send "have", and ignore ancestors */
 213                        commit = NULL;
 214                        mark = COMMON | SEEN;
 215                } else if (commit->object.flags & COMMON_REF)
 216                        /* send "have", and ignore ancestors */
 217                        mark = COMMON | SEEN;
 218                else
 219                        /* send "have", also for its ancestors */
 220                        mark = SEEN;
 221
 222                while (parents) {
 223                        if (!(parents->item->object.flags & SEEN))
 224                                rev_list_push(parents->item, mark);
 225                        if (mark & COMMON)
 226                                mark_common(parents->item, 1, 0);
 227                        parents = parents->next;
 228                }
 229        }
 230
 231        return &commit->object.oid;
 232}
 233
 234enum ack_type {
 235        NAK = 0,
 236        ACK,
 237        ACK_continue,
 238        ACK_common,
 239        ACK_ready
 240};
 241
 242static void consume_shallow_list(struct fetch_pack_args *args, int fd)
 243{
 244        if (args->stateless_rpc && args->deepen) {
 245                /* If we sent a depth we will get back "duplicate"
 246                 * shallow and unshallow commands every time there
 247                 * is a block of have lines exchanged.
 248                 */
 249                char *line;
 250                while ((line = packet_read_line(fd, NULL))) {
 251                        if (starts_with(line, "shallow "))
 252                                continue;
 253                        if (starts_with(line, "unshallow "))
 254                                continue;
 255                        die(_("git fetch-pack: expected shallow list"));
 256                }
 257        }
 258}
 259
 260static enum ack_type get_ack(int fd, struct object_id *result_oid)
 261{
 262        int len;
 263        char *line = packet_read_line(fd, &len);
 264        const char *arg;
 265
 266        if (!line)
 267                die(_("git fetch-pack: expected ACK/NAK, got a flush packet"));
 268        if (!strcmp(line, "NAK"))
 269                return NAK;
 270        if (skip_prefix(line, "ACK ", &arg)) {
 271                if (!get_oid_hex(arg, result_oid)) {
 272                        arg += 40;
 273                        len -= arg - line;
 274                        if (len < 1)
 275                                return ACK;
 276                        if (strstr(arg, "continue"))
 277                                return ACK_continue;
 278                        if (strstr(arg, "common"))
 279                                return ACK_common;
 280                        if (strstr(arg, "ready"))
 281                                return ACK_ready;
 282                        return ACK;
 283                }
 284        }
 285        if (skip_prefix(line, "ERR ", &arg))
 286                die(_("remote error: %s"), arg);
 287        die(_("git fetch-pack: expected ACK/NAK, got '%s'"), line);
 288}
 289
 290static void send_request(struct fetch_pack_args *args,
 291                         int fd, struct strbuf *buf)
 292{
 293        if (args->stateless_rpc) {
 294                send_sideband(fd, -1, buf->buf, buf->len, LARGE_PACKET_MAX);
 295                packet_flush(fd);
 296        } else
 297                write_or_die(fd, buf->buf, buf->len);
 298}
 299
 300static void insert_one_alternate_object(struct object *obj)
 301{
 302        rev_list_insert_ref(NULL, &obj->oid);
 303}
 304
 305#define INITIAL_FLUSH 16
 306#define PIPESAFE_FLUSH 32
 307#define LARGE_FLUSH 16384
 308
 309static int next_flush(struct fetch_pack_args *args, int count)
 310{
 311        if (args->stateless_rpc) {
 312                if (count < LARGE_FLUSH)
 313                        count <<= 1;
 314                else
 315                        count = count * 11 / 10;
 316        } else {
 317                if (count < PIPESAFE_FLUSH)
 318                        count <<= 1;
 319                else
 320                        count += PIPESAFE_FLUSH;
 321        }
 322        return count;
 323}
 324
 325static int find_common(struct fetch_pack_args *args,
 326                       int fd[2], struct object_id *result_oid,
 327                       struct ref *refs)
 328{
 329        int fetching;
 330        int count = 0, flushes = 0, flush_at = INITIAL_FLUSH, retval;
 331        const struct object_id *oid;
 332        unsigned in_vain = 0;
 333        int got_continue = 0;
 334        int got_ready = 0;
 335        struct strbuf req_buf = STRBUF_INIT;
 336        size_t state_len = 0;
 337
 338        if (args->stateless_rpc && multi_ack == 1)
 339                die(_("--stateless-rpc requires multi_ack_detailed"));
 340        if (marked)
 341                for_each_ref(clear_marks, NULL);
 342        marked = 1;
 343
 344        for_each_ref(rev_list_insert_ref_oid, NULL);
 345        for_each_cached_alternate(insert_one_alternate_object);
 346
 347        fetching = 0;
 348        for ( ; refs ; refs = refs->next) {
 349                struct object_id *remote = &refs->old_oid;
 350                const char *remote_hex;
 351                struct object *o;
 352
 353                /*
 354                 * If that object is complete (i.e. it is an ancestor of a
 355                 * local ref), we tell them we have it but do not have to
 356                 * tell them about its ancestors, which they already know
 357                 * about.
 358                 *
 359                 * We use lookup_object here because we are only
 360                 * interested in the case we *know* the object is
 361                 * reachable and we have already scanned it.
 362                 */
 363                if (((o = lookup_object(remote->hash)) != NULL) &&
 364                                (o->flags & COMPLETE)) {
 365                        continue;
 366                }
 367
 368                remote_hex = oid_to_hex(remote);
 369                if (!fetching) {
 370                        struct strbuf c = STRBUF_INIT;
 371                        if (multi_ack == 2)     strbuf_addstr(&c, " multi_ack_detailed");
 372                        if (multi_ack == 1)     strbuf_addstr(&c, " multi_ack");
 373                        if (no_done)            strbuf_addstr(&c, " no-done");
 374                        if (use_sideband == 2)  strbuf_addstr(&c, " side-band-64k");
 375                        if (use_sideband == 1)  strbuf_addstr(&c, " side-band");
 376                        if (args->deepen_relative) strbuf_addstr(&c, " deepen-relative");
 377                        if (args->use_thin_pack) strbuf_addstr(&c, " thin-pack");
 378                        if (args->no_progress)   strbuf_addstr(&c, " no-progress");
 379                        if (args->include_tag)   strbuf_addstr(&c, " include-tag");
 380                        if (prefer_ofs_delta)   strbuf_addstr(&c, " ofs-delta");
 381                        if (deepen_since_ok)    strbuf_addstr(&c, " deepen-since");
 382                        if (deepen_not_ok)      strbuf_addstr(&c, " deepen-not");
 383                        if (agent_supported)    strbuf_addf(&c, " agent=%s",
 384                                                            git_user_agent_sanitized());
 385                        if (args->filter_options.choice)
 386                                strbuf_addstr(&c, " filter");
 387                        packet_buf_write(&req_buf, "want %s%s\n", remote_hex, c.buf);
 388                        strbuf_release(&c);
 389                } else
 390                        packet_buf_write(&req_buf, "want %s\n", remote_hex);
 391                fetching++;
 392        }
 393
 394        if (!fetching) {
 395                strbuf_release(&req_buf);
 396                packet_flush(fd[1]);
 397                return 1;
 398        }
 399
 400        if (is_repository_shallow(the_repository))
 401                write_shallow_commits(&req_buf, 1, NULL);
 402        if (args->depth > 0)
 403                packet_buf_write(&req_buf, "deepen %d", args->depth);
 404        if (args->deepen_since) {
 405                timestamp_t max_age = approxidate(args->deepen_since);
 406                packet_buf_write(&req_buf, "deepen-since %"PRItime, max_age);
 407        }
 408        if (args->deepen_not) {
 409                int i;
 410                for (i = 0; i < args->deepen_not->nr; i++) {
 411                        struct string_list_item *s = args->deepen_not->items + i;
 412                        packet_buf_write(&req_buf, "deepen-not %s", s->string);
 413                }
 414        }
 415        if (server_supports_filtering && args->filter_options.choice)
 416                packet_buf_write(&req_buf, "filter %s",
 417                                 args->filter_options.filter_spec);
 418        packet_buf_flush(&req_buf);
 419        state_len = req_buf.len;
 420
 421        if (args->deepen) {
 422                char *line;
 423                const char *arg;
 424                struct object_id oid;
 425
 426                send_request(args, fd[1], &req_buf);
 427                while ((line = packet_read_line(fd[0], NULL))) {
 428                        if (skip_prefix(line, "shallow ", &arg)) {
 429                                if (get_oid_hex(arg, &oid))
 430                                        die(_("invalid shallow line: %s"), line);
 431                                register_shallow(the_repository, &oid);
 432                                continue;
 433                        }
 434                        if (skip_prefix(line, "unshallow ", &arg)) {
 435                                if (get_oid_hex(arg, &oid))
 436                                        die(_("invalid unshallow line: %s"), line);
 437                                if (!lookup_object(oid.hash))
 438                                        die(_("object not found: %s"), line);
 439                                /* make sure that it is parsed as shallow */
 440                                if (!parse_object(&oid))
 441                                        die(_("error in object: %s"), line);
 442                                if (unregister_shallow(&oid))
 443                                        die(_("no shallow found: %s"), line);
 444                                continue;
 445                        }
 446                        die(_("expected shallow/unshallow, got %s"), line);
 447                }
 448        } else if (!args->stateless_rpc)
 449                send_request(args, fd[1], &req_buf);
 450
 451        if (!args->stateless_rpc) {
 452                /* If we aren't using the stateless-rpc interface
 453                 * we don't need to retain the headers.
 454                 */
 455                strbuf_setlen(&req_buf, 0);
 456                state_len = 0;
 457        }
 458
 459        flushes = 0;
 460        retval = -1;
 461        if (args->no_dependents)
 462                goto done;
 463        while ((oid = get_rev())) {
 464                packet_buf_write(&req_buf, "have %s\n", oid_to_hex(oid));
 465                print_verbose(args, "have %s", oid_to_hex(oid));
 466                in_vain++;
 467                if (flush_at <= ++count) {
 468                        int ack;
 469
 470                        packet_buf_flush(&req_buf);
 471                        send_request(args, fd[1], &req_buf);
 472                        strbuf_setlen(&req_buf, state_len);
 473                        flushes++;
 474                        flush_at = next_flush(args, count);
 475
 476                        /*
 477                         * We keep one window "ahead" of the other side, and
 478                         * will wait for an ACK only on the next one
 479                         */
 480                        if (!args->stateless_rpc && count == INITIAL_FLUSH)
 481                                continue;
 482
 483                        consume_shallow_list(args, fd[0]);
 484                        do {
 485                                ack = get_ack(fd[0], result_oid);
 486                                if (ack)
 487                                        print_verbose(args, _("got %s %d %s"), "ack",
 488                                                      ack, oid_to_hex(result_oid));
 489                                switch (ack) {
 490                                case ACK:
 491                                        flushes = 0;
 492                                        multi_ack = 0;
 493                                        retval = 0;
 494                                        goto done;
 495                                case ACK_common:
 496                                case ACK_ready:
 497                                case ACK_continue: {
 498                                        struct commit *commit =
 499                                                lookup_commit(result_oid);
 500                                        if (!commit)
 501                                                die(_("invalid commit %s"), oid_to_hex(result_oid));
 502                                        if (args->stateless_rpc
 503                                         && ack == ACK_common
 504                                         && !(commit->object.flags & COMMON)) {
 505                                                /* We need to replay the have for this object
 506                                                 * on the next RPC request so the peer knows
 507                                                 * it is in common with us.
 508                                                 */
 509                                                const char *hex = oid_to_hex(result_oid);
 510                                                packet_buf_write(&req_buf, "have %s\n", hex);
 511                                                state_len = req_buf.len;
 512                                                /*
 513                                                 * Reset in_vain because an ack
 514                                                 * for this commit has not been
 515                                                 * seen.
 516                                                 */
 517                                                in_vain = 0;
 518                                        } else if (!args->stateless_rpc
 519                                                   || ack != ACK_common)
 520                                                in_vain = 0;
 521                                        mark_common(commit, 0, 1);
 522                                        retval = 0;
 523                                        got_continue = 1;
 524                                        if (ack == ACK_ready) {
 525                                                clear_prio_queue(&rev_list);
 526                                                got_ready = 1;
 527                                        }
 528                                        break;
 529                                        }
 530                                }
 531                        } while (ack);
 532                        flushes--;
 533                        if (got_continue && MAX_IN_VAIN < in_vain) {
 534                                print_verbose(args, _("giving up"));
 535                                break; /* give up */
 536                        }
 537                }
 538        }
 539done:
 540        if (!got_ready || !no_done) {
 541                packet_buf_write(&req_buf, "done\n");
 542                send_request(args, fd[1], &req_buf);
 543        }
 544        print_verbose(args, _("done"));
 545        if (retval != 0) {
 546                multi_ack = 0;
 547                flushes++;
 548        }
 549        strbuf_release(&req_buf);
 550
 551        if (!got_ready || !no_done)
 552                consume_shallow_list(args, fd[0]);
 553        while (flushes || multi_ack) {
 554                int ack = get_ack(fd[0], result_oid);
 555                if (ack) {
 556                        print_verbose(args, _("got %s (%d) %s"), "ack",
 557                                      ack, oid_to_hex(result_oid));
 558                        if (ack == ACK)
 559                                return 0;
 560                        multi_ack = 1;
 561                        continue;
 562                }
 563                flushes--;
 564        }
 565        /* it is no error to fetch into a completely empty repo */
 566        return count ? retval : 0;
 567}
 568
 569static struct commit_list *complete;
 570
 571static int mark_complete(const struct object_id *oid)
 572{
 573        struct object *o = parse_object(oid);
 574
 575        while (o && o->type == OBJ_TAG) {
 576                struct tag *t = (struct tag *) o;
 577                if (!t->tagged)
 578                        break; /* broken repository */
 579                o->flags |= COMPLETE;
 580                o = parse_object(&t->tagged->oid);
 581        }
 582        if (o && o->type == OBJ_COMMIT) {
 583                struct commit *commit = (struct commit *)o;
 584                if (!(commit->object.flags & COMPLETE)) {
 585                        commit->object.flags |= COMPLETE;
 586                        commit_list_insert(commit, &complete);
 587                }
 588        }
 589        return 0;
 590}
 591
 592static int mark_complete_oid(const char *refname, const struct object_id *oid,
 593                             int flag, void *cb_data)
 594{
 595        return mark_complete(oid);
 596}
 597
 598static void mark_recent_complete_commits(struct fetch_pack_args *args,
 599                                         timestamp_t cutoff)
 600{
 601        while (complete && cutoff <= complete->item->date) {
 602                print_verbose(args, _("Marking %s as complete"),
 603                              oid_to_hex(&complete->item->object.oid));
 604                pop_most_recent_commit(&complete, COMPLETE);
 605        }
 606}
 607
 608static void add_refs_to_oidset(struct oidset *oids, struct ref *refs)
 609{
 610        for (; refs; refs = refs->next)
 611                oidset_insert(oids, &refs->old_oid);
 612}
 613
 614static int tip_oids_contain(struct oidset *tip_oids,
 615                            struct ref *unmatched, struct ref *newlist,
 616                            const struct object_id *id)
 617{
 618        /*
 619         * Note that this only looks at the ref lists the first time it's
 620         * called. This works out in filter_refs() because even though it may
 621         * add to "newlist" between calls, the additions will always be for
 622         * oids that are already in the set.
 623         */
 624        if (!tip_oids->map.map.tablesize) {
 625                add_refs_to_oidset(tip_oids, unmatched);
 626                add_refs_to_oidset(tip_oids, newlist);
 627        }
 628        return oidset_contains(tip_oids, id);
 629}
 630
 631static void filter_refs(struct fetch_pack_args *args,
 632                        struct ref **refs,
 633                        struct ref **sought, int nr_sought)
 634{
 635        struct ref *newlist = NULL;
 636        struct ref **newtail = &newlist;
 637        struct ref *unmatched = NULL;
 638        struct ref *ref, *next;
 639        struct oidset tip_oids = OIDSET_INIT;
 640        int i;
 641
 642        i = 0;
 643        for (ref = *refs; ref; ref = next) {
 644                int keep = 0;
 645                next = ref->next;
 646
 647                if (starts_with(ref->name, "refs/") &&
 648                    check_refname_format(ref->name, 0))
 649                        ; /* trash */
 650                else {
 651                        while (i < nr_sought) {
 652                                int cmp = strcmp(ref->name, sought[i]->name);
 653                                if (cmp < 0)
 654                                        break; /* definitely do not have it */
 655                                else if (cmp == 0) {
 656                                        keep = 1; /* definitely have it */
 657                                        sought[i]->match_status = REF_MATCHED;
 658                                }
 659                                i++;
 660                        }
 661                }
 662
 663                if (!keep && args->fetch_all &&
 664                    (!args->deepen || !starts_with(ref->name, "refs/tags/")))
 665                        keep = 1;
 666
 667                if (keep) {
 668                        *newtail = ref;
 669                        ref->next = NULL;
 670                        newtail = &ref->next;
 671                } else {
 672                        ref->next = unmatched;
 673                        unmatched = ref;
 674                }
 675        }
 676
 677        /* Append unmatched requests to the list */
 678        for (i = 0; i < nr_sought; i++) {
 679                struct object_id oid;
 680                const char *p;
 681
 682                ref = sought[i];
 683                if (ref->match_status != REF_NOT_MATCHED)
 684                        continue;
 685                if (parse_oid_hex(ref->name, &oid, &p) ||
 686                    *p != '\0' ||
 687                    oidcmp(&oid, &ref->old_oid))
 688                        continue;
 689
 690                if ((allow_unadvertised_object_request &
 691                     (ALLOW_TIP_SHA1 | ALLOW_REACHABLE_SHA1)) ||
 692                    tip_oids_contain(&tip_oids, unmatched, newlist,
 693                                     &ref->old_oid)) {
 694                        ref->match_status = REF_MATCHED;
 695                        *newtail = copy_ref(ref);
 696                        newtail = &(*newtail)->next;
 697                } else {
 698                        ref->match_status = REF_UNADVERTISED_NOT_ALLOWED;
 699                }
 700        }
 701
 702        oidset_clear(&tip_oids);
 703        for (ref = unmatched; ref; ref = next) {
 704                next = ref->next;
 705                free(ref);
 706        }
 707
 708        *refs = newlist;
 709}
 710
 711static void mark_alternate_complete(struct object *obj)
 712{
 713        mark_complete(&obj->oid);
 714}
 715
 716struct loose_object_iter {
 717        struct oidset *loose_object_set;
 718        struct ref *refs;
 719};
 720
 721/*
 722 *  If the number of refs is not larger than the number of loose objects,
 723 *  this function stops inserting.
 724 */
 725static int add_loose_objects_to_set(const struct object_id *oid,
 726                                    const char *path,
 727                                    void *data)
 728{
 729        struct loose_object_iter *iter = data;
 730        oidset_insert(iter->loose_object_set, oid);
 731        if (iter->refs == NULL)
 732                return 1;
 733
 734        iter->refs = iter->refs->next;
 735        return 0;
 736}
 737
 738static int everything_local(struct fetch_pack_args *args,
 739                            struct ref **refs,
 740                            struct ref **sought, int nr_sought)
 741{
 742        struct ref *ref;
 743        int retval;
 744        int old_save_commit_buffer = save_commit_buffer;
 745        timestamp_t cutoff = 0;
 746        struct oidset loose_oid_set = OIDSET_INIT;
 747        int use_oidset = 0;
 748        struct loose_object_iter iter = {&loose_oid_set, *refs};
 749
 750        /* Enumerate all loose objects or know refs are not so many. */
 751        use_oidset = !for_each_loose_object(add_loose_objects_to_set,
 752                                            &iter, 0);
 753
 754        save_commit_buffer = 0;
 755
 756        for (ref = *refs; ref; ref = ref->next) {
 757                struct object *o;
 758                unsigned int flags = OBJECT_INFO_QUICK;
 759
 760                if (use_oidset &&
 761                    !oidset_contains(&loose_oid_set, &ref->old_oid)) {
 762                        /*
 763                         * I know this does not exist in the loose form,
 764                         * so check if it exists in a non-loose form.
 765                         */
 766                        flags |= OBJECT_INFO_IGNORE_LOOSE;
 767                }
 768
 769                if (!has_object_file_with_flags(&ref->old_oid, flags))
 770                        continue;
 771                o = parse_object(&ref->old_oid);
 772                if (!o)
 773                        continue;
 774
 775                /* We already have it -- which may mean that we were
 776                 * in sync with the other side at some time after
 777                 * that (it is OK if we guess wrong here).
 778                 */
 779                if (o->type == OBJ_COMMIT) {
 780                        struct commit *commit = (struct commit *)o;
 781                        if (!cutoff || cutoff < commit->date)
 782                                cutoff = commit->date;
 783                }
 784        }
 785
 786        oidset_clear(&loose_oid_set);
 787
 788        if (!args->no_dependents) {
 789                if (!args->deepen) {
 790                        for_each_ref(mark_complete_oid, NULL);
 791                        for_each_cached_alternate(mark_alternate_complete);
 792                        commit_list_sort_by_date(&complete);
 793                        if (cutoff)
 794                                mark_recent_complete_commits(args, cutoff);
 795                }
 796
 797                /*
 798                 * Mark all complete remote refs as common refs.
 799                 * Don't mark them common yet; the server has to be told so first.
 800                 */
 801                for (ref = *refs; ref; ref = ref->next) {
 802                        struct object *o = deref_tag(lookup_object(ref->old_oid.hash),
 803                                                     NULL, 0);
 804
 805                        if (!o || o->type != OBJ_COMMIT || !(o->flags & COMPLETE))
 806                                continue;
 807
 808                        if (!(o->flags & SEEN)) {
 809                                rev_list_push((struct commit *)o, COMMON_REF | SEEN);
 810
 811                                mark_common((struct commit *)o, 1, 1);
 812                        }
 813                }
 814        }
 815
 816        filter_refs(args, refs, sought, nr_sought);
 817
 818        for (retval = 1, ref = *refs; ref ; ref = ref->next) {
 819                const struct object_id *remote = &ref->old_oid;
 820                struct object *o;
 821
 822                o = lookup_object(remote->hash);
 823                if (!o || !(o->flags & COMPLETE)) {
 824                        retval = 0;
 825                        print_verbose(args, "want %s (%s)", oid_to_hex(remote),
 826                                      ref->name);
 827                        continue;
 828                }
 829                print_verbose(args, _("already have %s (%s)"), oid_to_hex(remote),
 830                              ref->name);
 831        }
 832
 833        save_commit_buffer = old_save_commit_buffer;
 834
 835        return retval;
 836}
 837
 838static int sideband_demux(int in, int out, void *data)
 839{
 840        int *xd = data;
 841        int ret;
 842
 843        ret = recv_sideband("fetch-pack", xd[0], out);
 844        close(out);
 845        return ret;
 846}
 847
 848static int get_pack(struct fetch_pack_args *args,
 849                    int xd[2], char **pack_lockfile)
 850{
 851        struct async demux;
 852        int do_keep = args->keep_pack;
 853        const char *cmd_name;
 854        struct pack_header header;
 855        int pass_header = 0;
 856        struct child_process cmd = CHILD_PROCESS_INIT;
 857        int ret;
 858
 859        memset(&demux, 0, sizeof(demux));
 860        if (use_sideband) {
 861                /* xd[] is talking with upload-pack; subprocess reads from
 862                 * xd[0], spits out band#2 to stderr, and feeds us band#1
 863                 * through demux->out.
 864                 */
 865                demux.proc = sideband_demux;
 866                demux.data = xd;
 867                demux.out = -1;
 868                demux.isolate_sigpipe = 1;
 869                if (start_async(&demux))
 870                        die(_("fetch-pack: unable to fork off sideband demultiplexer"));
 871        }
 872        else
 873                demux.out = xd[0];
 874
 875        if (!args->keep_pack && unpack_limit) {
 876
 877                if (read_pack_header(demux.out, &header))
 878                        die(_("protocol error: bad pack header"));
 879                pass_header = 1;
 880                if (ntohl(header.hdr_entries) < unpack_limit)
 881                        do_keep = 0;
 882                else
 883                        do_keep = 1;
 884        }
 885
 886        if (alternate_shallow_file) {
 887                argv_array_push(&cmd.args, "--shallow-file");
 888                argv_array_push(&cmd.args, alternate_shallow_file);
 889        }
 890
 891        if (do_keep || args->from_promisor) {
 892                if (pack_lockfile)
 893                        cmd.out = -1;
 894                cmd_name = "index-pack";
 895                argv_array_push(&cmd.args, cmd_name);
 896                argv_array_push(&cmd.args, "--stdin");
 897                if (!args->quiet && !args->no_progress)
 898                        argv_array_push(&cmd.args, "-v");
 899                if (args->use_thin_pack)
 900                        argv_array_push(&cmd.args, "--fix-thin");
 901                if (do_keep && (args->lock_pack || unpack_limit)) {
 902                        char hostname[HOST_NAME_MAX + 1];
 903                        if (xgethostname(hostname, sizeof(hostname)))
 904                                xsnprintf(hostname, sizeof(hostname), "localhost");
 905                        argv_array_pushf(&cmd.args,
 906                                        "--keep=fetch-pack %"PRIuMAX " on %s",
 907                                        (uintmax_t)getpid(), hostname);
 908                }
 909                if (args->check_self_contained_and_connected)
 910                        argv_array_push(&cmd.args, "--check-self-contained-and-connected");
 911                if (args->from_promisor)
 912                        argv_array_push(&cmd.args, "--promisor");
 913        }
 914        else {
 915                cmd_name = "unpack-objects";
 916                argv_array_push(&cmd.args, cmd_name);
 917                if (args->quiet || args->no_progress)
 918                        argv_array_push(&cmd.args, "-q");
 919                args->check_self_contained_and_connected = 0;
 920        }
 921
 922        if (pass_header)
 923                argv_array_pushf(&cmd.args, "--pack_header=%"PRIu32",%"PRIu32,
 924                                 ntohl(header.hdr_version),
 925                                 ntohl(header.hdr_entries));
 926        if (fetch_fsck_objects >= 0
 927            ? fetch_fsck_objects
 928            : transfer_fsck_objects >= 0
 929            ? transfer_fsck_objects
 930            : 0) {
 931                if (args->from_promisor)
 932                        /*
 933                         * We cannot use --strict in index-pack because it
 934                         * checks both broken objects and links, but we only
 935                         * want to check for broken objects.
 936                         */
 937                        argv_array_push(&cmd.args, "--fsck-objects");
 938                else
 939                        argv_array_push(&cmd.args, "--strict");
 940        }
 941
 942        cmd.in = demux.out;
 943        cmd.git_cmd = 1;
 944        if (start_command(&cmd))
 945                die(_("fetch-pack: unable to fork off %s"), cmd_name);
 946        if (do_keep && pack_lockfile) {
 947                *pack_lockfile = index_pack_lockfile(cmd.out);
 948                close(cmd.out);
 949        }
 950
 951        if (!use_sideband)
 952                /* Closed by start_command() */
 953                xd[0] = -1;
 954
 955        ret = finish_command(&cmd);
 956        if (!ret || (args->check_self_contained_and_connected && ret == 1))
 957                args->self_contained_and_connected =
 958                        args->check_self_contained_and_connected &&
 959                        ret == 0;
 960        else
 961                die(_("%s failed"), cmd_name);
 962        if (use_sideband && finish_async(&demux))
 963                die(_("error in sideband demultiplexer"));
 964        return 0;
 965}
 966
 967static int cmp_ref_by_name(const void *a_, const void *b_)
 968{
 969        const struct ref *a = *((const struct ref **)a_);
 970        const struct ref *b = *((const struct ref **)b_);
 971        return strcmp(a->name, b->name);
 972}
 973
 974static struct ref *do_fetch_pack(struct fetch_pack_args *args,
 975                                 int fd[2],
 976                                 const struct ref *orig_ref,
 977                                 struct ref **sought, int nr_sought,
 978                                 struct shallow_info *si,
 979                                 char **pack_lockfile)
 980{
 981        struct ref *ref = copy_ref_list(orig_ref);
 982        struct object_id oid;
 983        const char *agent_feature;
 984        int agent_len;
 985
 986        sort_ref_list(&ref, ref_compare_name);
 987        QSORT(sought, nr_sought, cmp_ref_by_name);
 988
 989        if ((args->depth > 0 || is_repository_shallow(the_repository)) && !server_supports("shallow"))
 990                die(_("Server does not support shallow clients"));
 991        if (args->depth > 0 || args->deepen_since || args->deepen_not)
 992                args->deepen = 1;
 993        if (server_supports("multi_ack_detailed")) {
 994                print_verbose(args, _("Server supports multi_ack_detailed"));
 995                multi_ack = 2;
 996                if (server_supports("no-done")) {
 997                        print_verbose(args, _("Server supports no-done"));
 998                        if (args->stateless_rpc)
 999                                no_done = 1;
1000                }
1001        }
1002        else if (server_supports("multi_ack")) {
1003                print_verbose(args, _("Server supports multi_ack"));
1004                multi_ack = 1;
1005        }
1006        if (server_supports("side-band-64k")) {
1007                print_verbose(args, _("Server supports side-band-64k"));
1008                use_sideband = 2;
1009        }
1010        else if (server_supports("side-band")) {
1011                print_verbose(args, _("Server supports side-band"));
1012                use_sideband = 1;
1013        }
1014        if (server_supports("allow-tip-sha1-in-want")) {
1015                print_verbose(args, _("Server supports allow-tip-sha1-in-want"));
1016                allow_unadvertised_object_request |= ALLOW_TIP_SHA1;
1017        }
1018        if (server_supports("allow-reachable-sha1-in-want")) {
1019                print_verbose(args, _("Server supports allow-reachable-sha1-in-want"));
1020                allow_unadvertised_object_request |= ALLOW_REACHABLE_SHA1;
1021        }
1022        if (!server_supports("thin-pack"))
1023                args->use_thin_pack = 0;
1024        if (!server_supports("no-progress"))
1025                args->no_progress = 0;
1026        if (!server_supports("include-tag"))
1027                args->include_tag = 0;
1028        if (server_supports("ofs-delta"))
1029                print_verbose(args, _("Server supports ofs-delta"));
1030        else
1031                prefer_ofs_delta = 0;
1032
1033        if (server_supports("filter")) {
1034                server_supports_filtering = 1;
1035                print_verbose(args, _("Server supports filter"));
1036        } else if (args->filter_options.choice) {
1037                warning("filtering not recognized by server, ignoring");
1038        }
1039
1040        if ((agent_feature = server_feature_value("agent", &agent_len))) {
1041                agent_supported = 1;
1042                if (agent_len)
1043                        print_verbose(args, _("Server version is %.*s"),
1044                                      agent_len, agent_feature);
1045        }
1046        if (server_supports("deepen-since"))
1047                deepen_since_ok = 1;
1048        else if (args->deepen_since)
1049                die(_("Server does not support --shallow-since"));
1050        if (server_supports("deepen-not"))
1051                deepen_not_ok = 1;
1052        else if (args->deepen_not)
1053                die(_("Server does not support --shallow-exclude"));
1054        if (!server_supports("deepen-relative") && args->deepen_relative)
1055                die(_("Server does not support --deepen"));
1056
1057        if (everything_local(args, &ref, sought, nr_sought)) {
1058                packet_flush(fd[1]);
1059                goto all_done;
1060        }
1061        if (find_common(args, fd, &oid, ref) < 0)
1062                if (!args->keep_pack)
1063                        /* When cloning, it is not unusual to have
1064                         * no common commit.
1065                         */
1066                        warning(_("no common commits"));
1067
1068        if (args->stateless_rpc)
1069                packet_flush(fd[1]);
1070        if (args->deepen)
1071                setup_alternate_shallow(&shallow_lock, &alternate_shallow_file,
1072                                        NULL);
1073        else if (si->nr_ours || si->nr_theirs)
1074                alternate_shallow_file = setup_temporary_shallow(si->shallow);
1075        else
1076                alternate_shallow_file = NULL;
1077        if (get_pack(args, fd, pack_lockfile))
1078                die(_("git fetch-pack: fetch failed."));
1079
1080 all_done:
1081        return ref;
1082}
1083
1084static void fetch_pack_config(void)
1085{
1086        git_config_get_int("fetch.unpacklimit", &fetch_unpack_limit);
1087        git_config_get_int("transfer.unpacklimit", &transfer_unpack_limit);
1088        git_config_get_bool("repack.usedeltabaseoffset", &prefer_ofs_delta);
1089        git_config_get_bool("fetch.fsckobjects", &fetch_fsck_objects);
1090        git_config_get_bool("transfer.fsckobjects", &transfer_fsck_objects);
1091
1092        git_config(git_default_config, NULL);
1093}
1094
1095static void fetch_pack_setup(void)
1096{
1097        static int did_setup;
1098        if (did_setup)
1099                return;
1100        fetch_pack_config();
1101        if (0 <= transfer_unpack_limit)
1102                unpack_limit = transfer_unpack_limit;
1103        else if (0 <= fetch_unpack_limit)
1104                unpack_limit = fetch_unpack_limit;
1105        did_setup = 1;
1106}
1107
1108static int remove_duplicates_in_refs(struct ref **ref, int nr)
1109{
1110        struct string_list names = STRING_LIST_INIT_NODUP;
1111        int src, dst;
1112
1113        for (src = dst = 0; src < nr; src++) {
1114                struct string_list_item *item;
1115                item = string_list_insert(&names, ref[src]->name);
1116                if (item->util)
1117                        continue; /* already have it */
1118                item->util = ref[src];
1119                if (src != dst)
1120                        ref[dst] = ref[src];
1121                dst++;
1122        }
1123        for (src = dst; src < nr; src++)
1124                ref[src] = NULL;
1125        string_list_clear(&names, 0);
1126        return dst;
1127}
1128
1129static void update_shallow(struct fetch_pack_args *args,
1130                           struct ref **sought, int nr_sought,
1131                           struct shallow_info *si)
1132{
1133        struct oid_array ref = OID_ARRAY_INIT;
1134        int *status;
1135        int i;
1136
1137        if (args->deepen && alternate_shallow_file) {
1138                if (*alternate_shallow_file == '\0') { /* --unshallow */
1139                        unlink_or_warn(git_path_shallow(the_repository));
1140                        rollback_lock_file(&shallow_lock);
1141                } else
1142                        commit_lock_file(&shallow_lock);
1143                return;
1144        }
1145
1146        if (!si->shallow || !si->shallow->nr)
1147                return;
1148
1149        if (args->cloning) {
1150                /*
1151                 * remote is shallow, but this is a clone, there are
1152                 * no objects in repo to worry about. Accept any
1153                 * shallow points that exist in the pack (iow in repo
1154                 * after get_pack() and reprepare_packed_git())
1155                 */
1156                struct oid_array extra = OID_ARRAY_INIT;
1157                struct object_id *oid = si->shallow->oid;
1158                for (i = 0; i < si->shallow->nr; i++)
1159                        if (has_object_file(&oid[i]))
1160                                oid_array_append(&extra, &oid[i]);
1161                if (extra.nr) {
1162                        setup_alternate_shallow(&shallow_lock,
1163                                                &alternate_shallow_file,
1164                                                &extra);
1165                        commit_lock_file(&shallow_lock);
1166                }
1167                oid_array_clear(&extra);
1168                return;
1169        }
1170
1171        if (!si->nr_ours && !si->nr_theirs)
1172                return;
1173
1174        remove_nonexistent_theirs_shallow(si);
1175        if (!si->nr_ours && !si->nr_theirs)
1176                return;
1177        for (i = 0; i < nr_sought; i++)
1178                oid_array_append(&ref, &sought[i]->old_oid);
1179        si->ref = &ref;
1180
1181        if (args->update_shallow) {
1182                /*
1183                 * remote is also shallow, .git/shallow may be updated
1184                 * so all refs can be accepted. Make sure we only add
1185                 * shallow roots that are actually reachable from new
1186                 * refs.
1187                 */
1188                struct oid_array extra = OID_ARRAY_INIT;
1189                struct object_id *oid = si->shallow->oid;
1190                assign_shallow_commits_to_refs(si, NULL, NULL);
1191                if (!si->nr_ours && !si->nr_theirs) {
1192                        oid_array_clear(&ref);
1193                        return;
1194                }
1195                for (i = 0; i < si->nr_ours; i++)
1196                        oid_array_append(&extra, &oid[si->ours[i]]);
1197                for (i = 0; i < si->nr_theirs; i++)
1198                        oid_array_append(&extra, &oid[si->theirs[i]]);
1199                setup_alternate_shallow(&shallow_lock,
1200                                        &alternate_shallow_file,
1201                                        &extra);
1202                commit_lock_file(&shallow_lock);
1203                oid_array_clear(&extra);
1204                oid_array_clear(&ref);
1205                return;
1206        }
1207
1208        /*
1209         * remote is also shallow, check what ref is safe to update
1210         * without updating .git/shallow
1211         */
1212        status = xcalloc(nr_sought, sizeof(*status));
1213        assign_shallow_commits_to_refs(si, NULL, status);
1214        if (si->nr_ours || si->nr_theirs) {
1215                for (i = 0; i < nr_sought; i++)
1216                        if (status[i])
1217                                sought[i]->status = REF_STATUS_REJECT_SHALLOW;
1218        }
1219        free(status);
1220        oid_array_clear(&ref);
1221}
1222
1223struct ref *fetch_pack(struct fetch_pack_args *args,
1224                       int fd[], struct child_process *conn,
1225                       const struct ref *ref,
1226                       const char *dest,
1227                       struct ref **sought, int nr_sought,
1228                       struct oid_array *shallow,
1229                       char **pack_lockfile)
1230{
1231        struct ref *ref_cpy;
1232        struct shallow_info si;
1233
1234        fetch_pack_setup();
1235        if (nr_sought)
1236                nr_sought = remove_duplicates_in_refs(sought, nr_sought);
1237
1238        if (!ref) {
1239                packet_flush(fd[1]);
1240                die(_("no matching remote head"));
1241        }
1242        prepare_shallow_info(&si, shallow);
1243        ref_cpy = do_fetch_pack(args, fd, ref, sought, nr_sought,
1244                                &si, pack_lockfile);
1245        reprepare_packed_git(the_repository);
1246        update_shallow(args, sought, nr_sought, &si);
1247        clear_shallow_info(&si);
1248        return ref_cpy;
1249}
1250
1251int report_unmatched_refs(struct ref **sought, int nr_sought)
1252{
1253        int i, ret = 0;
1254
1255        for (i = 0; i < nr_sought; i++) {
1256                if (!sought[i])
1257                        continue;
1258                switch (sought[i]->match_status) {
1259                case REF_MATCHED:
1260                        continue;
1261                case REF_NOT_MATCHED:
1262                        error(_("no such remote ref %s"), sought[i]->name);
1263                        break;
1264                case REF_UNADVERTISED_NOT_ALLOWED:
1265                        error(_("Server does not allow request for unadvertised object %s"),
1266                              sought[i]->name);
1267                        break;
1268                }
1269                ret = 1;
1270        }
1271        return ret;
1272}