fetch-pack.con commit Merge branch 'st/lib-gpg-kill-stray-agent' into maint (638eb4e)
   1#include "cache.h"
   2#include "config.h"
   3#include "lockfile.h"
   4#include "refs.h"
   5#include "pkt-line.h"
   6#include "commit.h"
   7#include "tag.h"
   8#include "exec_cmd.h"
   9#include "pack.h"
  10#include "sideband.h"
  11#include "fetch-pack.h"
  12#include "remote.h"
  13#include "run-command.h"
  14#include "connect.h"
  15#include "transport.h"
  16#include "version.h"
  17#include "prio-queue.h"
  18#include "sha1-array.h"
  19#include "oidset.h"
  20
  21static int transfer_unpack_limit = -1;
  22static int fetch_unpack_limit = -1;
  23static int unpack_limit = 100;
  24static int prefer_ofs_delta = 1;
  25static int no_done;
  26static int deepen_since_ok;
  27static int deepen_not_ok;
  28static int fetch_fsck_objects = -1;
  29static int transfer_fsck_objects = -1;
  30static int agent_supported;
  31static struct lock_file shallow_lock;
  32static const char *alternate_shallow_file;
  33
  34/* Remember to update object flag allocation in object.h */
  35#define COMPLETE        (1U << 0)
  36#define COMMON          (1U << 1)
  37#define COMMON_REF      (1U << 2)
  38#define SEEN            (1U << 3)
  39#define POPPED          (1U << 4)
  40#define ALTERNATE       (1U << 5)
  41
  42static int marked;
  43
  44/*
  45 * After sending this many "have"s if we do not get any new ACK , we
  46 * give up traversing our history.
  47 */
  48#define MAX_IN_VAIN 256
  49
  50static struct prio_queue rev_list = { compare_commits_by_commit_date };
  51static int non_common_revs, multi_ack, use_sideband;
  52/* Allow specifying sha1 if it is a ref tip. */
  53#define ALLOW_TIP_SHA1  01
  54/* Allow request of a sha1 if it is reachable from a ref (possibly hidden ref). */
  55#define ALLOW_REACHABLE_SHA1    02
  56static unsigned int allow_unadvertised_object_request;
  57
  58__attribute__((format (printf, 2, 3)))
  59static inline void print_verbose(const struct fetch_pack_args *args,
  60                                 const char *fmt, ...)
  61{
  62        va_list params;
  63
  64        if (!args->verbose)
  65                return;
  66
  67        va_start(params, fmt);
  68        vfprintf(stderr, fmt, params);
  69        va_end(params);
  70        fputc('\n', stderr);
  71}
  72
  73struct alternate_object_cache {
  74        struct object **items;
  75        size_t nr, alloc;
  76};
  77
  78static void cache_one_alternate(const char *refname,
  79                                const struct object_id *oid,
  80                                void *vcache)
  81{
  82        struct alternate_object_cache *cache = vcache;
  83        struct object *obj = parse_object(oid);
  84
  85        if (!obj || (obj->flags & ALTERNATE))
  86                return;
  87
  88        obj->flags |= ALTERNATE;
  89        ALLOC_GROW(cache->items, cache->nr + 1, cache->alloc);
  90        cache->items[cache->nr++] = obj;
  91}
  92
  93static void for_each_cached_alternate(void (*cb)(struct object *))
  94{
  95        static int initialized;
  96        static struct alternate_object_cache cache;
  97        size_t i;
  98
  99        if (!initialized) {
 100                for_each_alternate_ref(cache_one_alternate, &cache);
 101                initialized = 1;
 102        }
 103
 104        for (i = 0; i < cache.nr; i++)
 105                cb(cache.items[i]);
 106}
 107
 108static void rev_list_push(struct commit *commit, int mark)
 109{
 110        if (!(commit->object.flags & mark)) {
 111                commit->object.flags |= mark;
 112
 113                if (parse_commit(commit))
 114                        return;
 115
 116                prio_queue_put(&rev_list, commit);
 117
 118                if (!(commit->object.flags & COMMON))
 119                        non_common_revs++;
 120        }
 121}
 122
 123static int rev_list_insert_ref(const char *refname, const struct object_id *oid)
 124{
 125        struct object *o = deref_tag(parse_object(oid), refname, 0);
 126
 127        if (o && o->type == OBJ_COMMIT)
 128                rev_list_push((struct commit *)o, SEEN);
 129
 130        return 0;
 131}
 132
 133static int rev_list_insert_ref_oid(const char *refname, const struct object_id *oid,
 134                                   int flag, void *cb_data)
 135{
 136        return rev_list_insert_ref(refname, oid);
 137}
 138
 139static int clear_marks(const char *refname, const struct object_id *oid,
 140                       int flag, void *cb_data)
 141{
 142        struct object *o = deref_tag(parse_object(oid), refname, 0);
 143
 144        if (o && o->type == OBJ_COMMIT)
 145                clear_commit_marks((struct commit *)o,
 146                                   COMMON | COMMON_REF | SEEN | POPPED);
 147        return 0;
 148}
 149
 150/*
 151   This function marks a rev and its ancestors as common.
 152   In some cases, it is desirable to mark only the ancestors (for example
 153   when only the server does not yet know that they are common).
 154*/
 155
 156static void mark_common(struct commit *commit,
 157                int ancestors_only, int dont_parse)
 158{
 159        if (commit != NULL && !(commit->object.flags & COMMON)) {
 160                struct object *o = (struct object *)commit;
 161
 162                if (!ancestors_only)
 163                        o->flags |= COMMON;
 164
 165                if (!(o->flags & SEEN))
 166                        rev_list_push(commit, SEEN);
 167                else {
 168                        struct commit_list *parents;
 169
 170                        if (!ancestors_only && !(o->flags & POPPED))
 171                                non_common_revs--;
 172                        if (!o->parsed && !dont_parse)
 173                                if (parse_commit(commit))
 174                                        return;
 175
 176                        for (parents = commit->parents;
 177                                        parents;
 178                                        parents = parents->next)
 179                                mark_common(parents->item, 0, dont_parse);
 180                }
 181        }
 182}
 183
 184/*
 185  Get the next rev to send, ignoring the common.
 186*/
 187
 188static const struct object_id *get_rev(void)
 189{
 190        struct commit *commit = NULL;
 191
 192        while (commit == NULL) {
 193                unsigned int mark;
 194                struct commit_list *parents;
 195
 196                if (rev_list.nr == 0 || non_common_revs == 0)
 197                        return NULL;
 198
 199                commit = prio_queue_get(&rev_list);
 200                parse_commit(commit);
 201                parents = commit->parents;
 202
 203                commit->object.flags |= POPPED;
 204                if (!(commit->object.flags & COMMON))
 205                        non_common_revs--;
 206
 207                if (commit->object.flags & COMMON) {
 208                        /* do not send "have", and ignore ancestors */
 209                        commit = NULL;
 210                        mark = COMMON | SEEN;
 211                } else if (commit->object.flags & COMMON_REF)
 212                        /* send "have", and ignore ancestors */
 213                        mark = COMMON | SEEN;
 214                else
 215                        /* send "have", also for its ancestors */
 216                        mark = SEEN;
 217
 218                while (parents) {
 219                        if (!(parents->item->object.flags & SEEN))
 220                                rev_list_push(parents->item, mark);
 221                        if (mark & COMMON)
 222                                mark_common(parents->item, 1, 0);
 223                        parents = parents->next;
 224                }
 225        }
 226
 227        return &commit->object.oid;
 228}
 229
 230enum ack_type {
 231        NAK = 0,
 232        ACK,
 233        ACK_continue,
 234        ACK_common,
 235        ACK_ready
 236};
 237
 238static void consume_shallow_list(struct fetch_pack_args *args, int fd)
 239{
 240        if (args->stateless_rpc && args->deepen) {
 241                /* If we sent a depth we will get back "duplicate"
 242                 * shallow and unshallow commands every time there
 243                 * is a block of have lines exchanged.
 244                 */
 245                char *line;
 246                while ((line = packet_read_line(fd, NULL))) {
 247                        if (starts_with(line, "shallow "))
 248                                continue;
 249                        if (starts_with(line, "unshallow "))
 250                                continue;
 251                        die(_("git fetch-pack: expected shallow list"));
 252                }
 253        }
 254}
 255
 256static enum ack_type get_ack(int fd, struct object_id *result_oid)
 257{
 258        int len;
 259        char *line = packet_read_line(fd, &len);
 260        const char *arg;
 261
 262        if (!len)
 263                die(_("git fetch-pack: expected ACK/NAK, got EOF"));
 264        if (!strcmp(line, "NAK"))
 265                return NAK;
 266        if (skip_prefix(line, "ACK ", &arg)) {
 267                if (!get_oid_hex(arg, result_oid)) {
 268                        arg += 40;
 269                        len -= arg - line;
 270                        if (len < 1)
 271                                return ACK;
 272                        if (strstr(arg, "continue"))
 273                                return ACK_continue;
 274                        if (strstr(arg, "common"))
 275                                return ACK_common;
 276                        if (strstr(arg, "ready"))
 277                                return ACK_ready;
 278                        return ACK;
 279                }
 280        }
 281        if (skip_prefix(line, "ERR ", &arg))
 282                die(_("remote error: %s"), arg);
 283        die(_("git fetch-pack: expected ACK/NAK, got '%s'"), line);
 284}
 285
 286static void send_request(struct fetch_pack_args *args,
 287                         int fd, struct strbuf *buf)
 288{
 289        if (args->stateless_rpc) {
 290                send_sideband(fd, -1, buf->buf, buf->len, LARGE_PACKET_MAX);
 291                packet_flush(fd);
 292        } else
 293                write_or_die(fd, buf->buf, buf->len);
 294}
 295
 296static void insert_one_alternate_object(struct object *obj)
 297{
 298        rev_list_insert_ref(NULL, &obj->oid);
 299}
 300
 301#define INITIAL_FLUSH 16
 302#define PIPESAFE_FLUSH 32
 303#define LARGE_FLUSH 16384
 304
 305static int next_flush(struct fetch_pack_args *args, int count)
 306{
 307        if (args->stateless_rpc) {
 308                if (count < LARGE_FLUSH)
 309                        count <<= 1;
 310                else
 311                        count = count * 11 / 10;
 312        } else {
 313                if (count < PIPESAFE_FLUSH)
 314                        count <<= 1;
 315                else
 316                        count += PIPESAFE_FLUSH;
 317        }
 318        return count;
 319}
 320
 321static int find_common(struct fetch_pack_args *args,
 322                       int fd[2], struct object_id *result_oid,
 323                       struct ref *refs)
 324{
 325        int fetching;
 326        int count = 0, flushes = 0, flush_at = INITIAL_FLUSH, retval;
 327        const struct object_id *oid;
 328        unsigned in_vain = 0;
 329        int got_continue = 0;
 330        int got_ready = 0;
 331        struct strbuf req_buf = STRBUF_INIT;
 332        size_t state_len = 0;
 333
 334        if (args->stateless_rpc && multi_ack == 1)
 335                die(_("--stateless-rpc requires multi_ack_detailed"));
 336        if (marked)
 337                for_each_ref(clear_marks, NULL);
 338        marked = 1;
 339
 340        for_each_ref(rev_list_insert_ref_oid, NULL);
 341        for_each_cached_alternate(insert_one_alternate_object);
 342
 343        fetching = 0;
 344        for ( ; refs ; refs = refs->next) {
 345                struct object_id *remote = &refs->old_oid;
 346                const char *remote_hex;
 347                struct object *o;
 348
 349                /*
 350                 * If that object is complete (i.e. it is an ancestor of a
 351                 * local ref), we tell them we have it but do not have to
 352                 * tell them about its ancestors, which they already know
 353                 * about.
 354                 *
 355                 * We use lookup_object here because we are only
 356                 * interested in the case we *know* the object is
 357                 * reachable and we have already scanned it.
 358                 */
 359                if (((o = lookup_object(remote->hash)) != NULL) &&
 360                                (o->flags & COMPLETE)) {
 361                        continue;
 362                }
 363
 364                remote_hex = oid_to_hex(remote);
 365                if (!fetching) {
 366                        struct strbuf c = STRBUF_INIT;
 367                        if (multi_ack == 2)     strbuf_addstr(&c, " multi_ack_detailed");
 368                        if (multi_ack == 1)     strbuf_addstr(&c, " multi_ack");
 369                        if (no_done)            strbuf_addstr(&c, " no-done");
 370                        if (use_sideband == 2)  strbuf_addstr(&c, " side-band-64k");
 371                        if (use_sideband == 1)  strbuf_addstr(&c, " side-band");
 372                        if (args->deepen_relative) strbuf_addstr(&c, " deepen-relative");
 373                        if (args->use_thin_pack) strbuf_addstr(&c, " thin-pack");
 374                        if (args->no_progress)   strbuf_addstr(&c, " no-progress");
 375                        if (args->include_tag)   strbuf_addstr(&c, " include-tag");
 376                        if (prefer_ofs_delta)   strbuf_addstr(&c, " ofs-delta");
 377                        if (deepen_since_ok)    strbuf_addstr(&c, " deepen-since");
 378                        if (deepen_not_ok)      strbuf_addstr(&c, " deepen-not");
 379                        if (agent_supported)    strbuf_addf(&c, " agent=%s",
 380                                                            git_user_agent_sanitized());
 381                        packet_buf_write(&req_buf, "want %s%s\n", remote_hex, c.buf);
 382                        strbuf_release(&c);
 383                } else
 384                        packet_buf_write(&req_buf, "want %s\n", remote_hex);
 385                fetching++;
 386        }
 387
 388        if (!fetching) {
 389                strbuf_release(&req_buf);
 390                packet_flush(fd[1]);
 391                return 1;
 392        }
 393
 394        if (is_repository_shallow())
 395                write_shallow_commits(&req_buf, 1, NULL);
 396        if (args->depth > 0)
 397                packet_buf_write(&req_buf, "deepen %d", args->depth);
 398        if (args->deepen_since) {
 399                timestamp_t max_age = approxidate(args->deepen_since);
 400                packet_buf_write(&req_buf, "deepen-since %"PRItime, max_age);
 401        }
 402        if (args->deepen_not) {
 403                int i;
 404                for (i = 0; i < args->deepen_not->nr; i++) {
 405                        struct string_list_item *s = args->deepen_not->items + i;
 406                        packet_buf_write(&req_buf, "deepen-not %s", s->string);
 407                }
 408        }
 409        packet_buf_flush(&req_buf);
 410        state_len = req_buf.len;
 411
 412        if (args->deepen) {
 413                char *line;
 414                const char *arg;
 415                struct object_id oid;
 416
 417                send_request(args, fd[1], &req_buf);
 418                while ((line = packet_read_line(fd[0], NULL))) {
 419                        if (skip_prefix(line, "shallow ", &arg)) {
 420                                if (get_oid_hex(arg, &oid))
 421                                        die(_("invalid shallow line: %s"), line);
 422                                register_shallow(&oid);
 423                                continue;
 424                        }
 425                        if (skip_prefix(line, "unshallow ", &arg)) {
 426                                if (get_oid_hex(arg, &oid))
 427                                        die(_("invalid unshallow line: %s"), line);
 428                                if (!lookup_object(oid.hash))
 429                                        die(_("object not found: %s"), line);
 430                                /* make sure that it is parsed as shallow */
 431                                if (!parse_object(&oid))
 432                                        die(_("error in object: %s"), line);
 433                                if (unregister_shallow(&oid))
 434                                        die(_("no shallow found: %s"), line);
 435                                continue;
 436                        }
 437                        die(_("expected shallow/unshallow, got %s"), line);
 438                }
 439        } else if (!args->stateless_rpc)
 440                send_request(args, fd[1], &req_buf);
 441
 442        if (!args->stateless_rpc) {
 443                /* If we aren't using the stateless-rpc interface
 444                 * we don't need to retain the headers.
 445                 */
 446                strbuf_setlen(&req_buf, 0);
 447                state_len = 0;
 448        }
 449
 450        flushes = 0;
 451        retval = -1;
 452        while ((oid = get_rev())) {
 453                packet_buf_write(&req_buf, "have %s\n", oid_to_hex(oid));
 454                print_verbose(args, "have %s", oid_to_hex(oid));
 455                in_vain++;
 456                if (flush_at <= ++count) {
 457                        int ack;
 458
 459                        packet_buf_flush(&req_buf);
 460                        send_request(args, fd[1], &req_buf);
 461                        strbuf_setlen(&req_buf, state_len);
 462                        flushes++;
 463                        flush_at = next_flush(args, count);
 464
 465                        /*
 466                         * We keep one window "ahead" of the other side, and
 467                         * will wait for an ACK only on the next one
 468                         */
 469                        if (!args->stateless_rpc && count == INITIAL_FLUSH)
 470                                continue;
 471
 472                        consume_shallow_list(args, fd[0]);
 473                        do {
 474                                ack = get_ack(fd[0], result_oid);
 475                                if (ack)
 476                                        print_verbose(args, _("got %s %d %s"), "ack",
 477                                                      ack, oid_to_hex(result_oid));
 478                                switch (ack) {
 479                                case ACK:
 480                                        flushes = 0;
 481                                        multi_ack = 0;
 482                                        retval = 0;
 483                                        goto done;
 484                                case ACK_common:
 485                                case ACK_ready:
 486                                case ACK_continue: {
 487                                        struct commit *commit =
 488                                                lookup_commit(result_oid);
 489                                        if (!commit)
 490                                                die(_("invalid commit %s"), oid_to_hex(result_oid));
 491                                        if (args->stateless_rpc
 492                                         && ack == ACK_common
 493                                         && !(commit->object.flags & COMMON)) {
 494                                                /* We need to replay the have for this object
 495                                                 * on the next RPC request so the peer knows
 496                                                 * it is in common with us.
 497                                                 */
 498                                                const char *hex = oid_to_hex(result_oid);
 499                                                packet_buf_write(&req_buf, "have %s\n", hex);
 500                                                state_len = req_buf.len;
 501                                                /*
 502                                                 * Reset in_vain because an ack
 503                                                 * for this commit has not been
 504                                                 * seen.
 505                                                 */
 506                                                in_vain = 0;
 507                                        } else if (!args->stateless_rpc
 508                                                   || ack != ACK_common)
 509                                                in_vain = 0;
 510                                        mark_common(commit, 0, 1);
 511                                        retval = 0;
 512                                        got_continue = 1;
 513                                        if (ack == ACK_ready) {
 514                                                clear_prio_queue(&rev_list);
 515                                                got_ready = 1;
 516                                        }
 517                                        break;
 518                                        }
 519                                }
 520                        } while (ack);
 521                        flushes--;
 522                        if (got_continue && MAX_IN_VAIN < in_vain) {
 523                                print_verbose(args, _("giving up"));
 524                                break; /* give up */
 525                        }
 526                }
 527        }
 528done:
 529        if (!got_ready || !no_done) {
 530                packet_buf_write(&req_buf, "done\n");
 531                send_request(args, fd[1], &req_buf);
 532        }
 533        print_verbose(args, _("done"));
 534        if (retval != 0) {
 535                multi_ack = 0;
 536                flushes++;
 537        }
 538        strbuf_release(&req_buf);
 539
 540        if (!got_ready || !no_done)
 541                consume_shallow_list(args, fd[0]);
 542        while (flushes || multi_ack) {
 543                int ack = get_ack(fd[0], result_oid);
 544                if (ack) {
 545                        print_verbose(args, _("got %s (%d) %s"), "ack",
 546                                      ack, oid_to_hex(result_oid));
 547                        if (ack == ACK)
 548                                return 0;
 549                        multi_ack = 1;
 550                        continue;
 551                }
 552                flushes--;
 553        }
 554        /* it is no error to fetch into a completely empty repo */
 555        return count ? retval : 0;
 556}
 557
 558static struct commit_list *complete;
 559
 560static int mark_complete(const struct object_id *oid)
 561{
 562        struct object *o = parse_object(oid);
 563
 564        while (o && o->type == OBJ_TAG) {
 565                struct tag *t = (struct tag *) o;
 566                if (!t->tagged)
 567                        break; /* broken repository */
 568                o->flags |= COMPLETE;
 569                o = parse_object(&t->tagged->oid);
 570        }
 571        if (o && o->type == OBJ_COMMIT) {
 572                struct commit *commit = (struct commit *)o;
 573                if (!(commit->object.flags & COMPLETE)) {
 574                        commit->object.flags |= COMPLETE;
 575                        commit_list_insert(commit, &complete);
 576                }
 577        }
 578        return 0;
 579}
 580
 581static int mark_complete_oid(const char *refname, const struct object_id *oid,
 582                             int flag, void *cb_data)
 583{
 584        return mark_complete(oid);
 585}
 586
 587static void mark_recent_complete_commits(struct fetch_pack_args *args,
 588                                         timestamp_t cutoff)
 589{
 590        while (complete && cutoff <= complete->item->date) {
 591                print_verbose(args, _("Marking %s as complete"),
 592                              oid_to_hex(&complete->item->object.oid));
 593                pop_most_recent_commit(&complete, COMPLETE);
 594        }
 595}
 596
 597static void add_refs_to_oidset(struct oidset *oids, struct ref *refs)
 598{
 599        for (; refs; refs = refs->next)
 600                oidset_insert(oids, &refs->old_oid);
 601}
 602
 603static int tip_oids_contain(struct oidset *tip_oids,
 604                            struct ref *unmatched, struct ref *newlist,
 605                            const struct object_id *id)
 606{
 607        /*
 608         * Note that this only looks at the ref lists the first time it's
 609         * called. This works out in filter_refs() because even though it may
 610         * add to "newlist" between calls, the additions will always be for
 611         * oids that are already in the set.
 612         */
 613        if (!tip_oids->map.tablesize) {
 614                add_refs_to_oidset(tip_oids, unmatched);
 615                add_refs_to_oidset(tip_oids, newlist);
 616        }
 617        return oidset_contains(tip_oids, id);
 618}
 619
 620static void filter_refs(struct fetch_pack_args *args,
 621                        struct ref **refs,
 622                        struct ref **sought, int nr_sought)
 623{
 624        struct ref *newlist = NULL;
 625        struct ref **newtail = &newlist;
 626        struct ref *unmatched = NULL;
 627        struct ref *ref, *next;
 628        struct oidset tip_oids = OIDSET_INIT;
 629        int i;
 630
 631        i = 0;
 632        for (ref = *refs; ref; ref = next) {
 633                int keep = 0;
 634                next = ref->next;
 635
 636                if (starts_with(ref->name, "refs/") &&
 637                    check_refname_format(ref->name, 0))
 638                        ; /* trash */
 639                else {
 640                        while (i < nr_sought) {
 641                                int cmp = strcmp(ref->name, sought[i]->name);
 642                                if (cmp < 0)
 643                                        break; /* definitely do not have it */
 644                                else if (cmp == 0) {
 645                                        keep = 1; /* definitely have it */
 646                                        sought[i]->match_status = REF_MATCHED;
 647                                }
 648                                i++;
 649                        }
 650                }
 651
 652                if (!keep && args->fetch_all &&
 653                    (!args->deepen || !starts_with(ref->name, "refs/tags/")))
 654                        keep = 1;
 655
 656                if (keep) {
 657                        *newtail = ref;
 658                        ref->next = NULL;
 659                        newtail = &ref->next;
 660                } else {
 661                        ref->next = unmatched;
 662                        unmatched = ref;
 663                }
 664        }
 665
 666        /* Append unmatched requests to the list */
 667        for (i = 0; i < nr_sought; i++) {
 668                struct object_id oid;
 669                const char *p;
 670
 671                ref = sought[i];
 672                if (ref->match_status != REF_NOT_MATCHED)
 673                        continue;
 674                if (parse_oid_hex(ref->name, &oid, &p) ||
 675                    *p != '\0' ||
 676                    oidcmp(&oid, &ref->old_oid))
 677                        continue;
 678
 679                if ((allow_unadvertised_object_request &
 680                     (ALLOW_TIP_SHA1 | ALLOW_REACHABLE_SHA1)) ||
 681                    tip_oids_contain(&tip_oids, unmatched, newlist,
 682                                     &ref->old_oid)) {
 683                        ref->match_status = REF_MATCHED;
 684                        *newtail = copy_ref(ref);
 685                        newtail = &(*newtail)->next;
 686                } else {
 687                        ref->match_status = REF_UNADVERTISED_NOT_ALLOWED;
 688                }
 689        }
 690
 691        oidset_clear(&tip_oids);
 692        for (ref = unmatched; ref; ref = next) {
 693                next = ref->next;
 694                free(ref);
 695        }
 696
 697        *refs = newlist;
 698}
 699
 700static void mark_alternate_complete(struct object *obj)
 701{
 702        mark_complete(&obj->oid);
 703}
 704
 705static int everything_local(struct fetch_pack_args *args,
 706                            struct ref **refs,
 707                            struct ref **sought, int nr_sought)
 708{
 709        struct ref *ref;
 710        int retval;
 711        timestamp_t cutoff = 0;
 712
 713        save_commit_buffer = 0;
 714
 715        for (ref = *refs; ref; ref = ref->next) {
 716                struct object *o;
 717
 718                if (!has_object_file(&ref->old_oid))
 719                        continue;
 720
 721                o = parse_object(&ref->old_oid);
 722                if (!o)
 723                        continue;
 724
 725                /* We already have it -- which may mean that we were
 726                 * in sync with the other side at some time after
 727                 * that (it is OK if we guess wrong here).
 728                 */
 729                if (o->type == OBJ_COMMIT) {
 730                        struct commit *commit = (struct commit *)o;
 731                        if (!cutoff || cutoff < commit->date)
 732                                cutoff = commit->date;
 733                }
 734        }
 735
 736        if (!args->deepen) {
 737                for_each_ref(mark_complete_oid, NULL);
 738                for_each_cached_alternate(mark_alternate_complete);
 739                commit_list_sort_by_date(&complete);
 740                if (cutoff)
 741                        mark_recent_complete_commits(args, cutoff);
 742        }
 743
 744        /*
 745         * Mark all complete remote refs as common refs.
 746         * Don't mark them common yet; the server has to be told so first.
 747         */
 748        for (ref = *refs; ref; ref = ref->next) {
 749                struct object *o = deref_tag(lookup_object(ref->old_oid.hash),
 750                                             NULL, 0);
 751
 752                if (!o || o->type != OBJ_COMMIT || !(o->flags & COMPLETE))
 753                        continue;
 754
 755                if (!(o->flags & SEEN)) {
 756                        rev_list_push((struct commit *)o, COMMON_REF | SEEN);
 757
 758                        mark_common((struct commit *)o, 1, 1);
 759                }
 760        }
 761
 762        filter_refs(args, refs, sought, nr_sought);
 763
 764        for (retval = 1, ref = *refs; ref ; ref = ref->next) {
 765                const struct object_id *remote = &ref->old_oid;
 766                struct object *o;
 767
 768                o = lookup_object(remote->hash);
 769                if (!o || !(o->flags & COMPLETE)) {
 770                        retval = 0;
 771                        print_verbose(args, "want %s (%s)", oid_to_hex(remote),
 772                                      ref->name);
 773                        continue;
 774                }
 775                print_verbose(args, _("already have %s (%s)"), oid_to_hex(remote),
 776                              ref->name);
 777        }
 778        return retval;
 779}
 780
 781static int sideband_demux(int in, int out, void *data)
 782{
 783        int *xd = data;
 784        int ret;
 785
 786        ret = recv_sideband("fetch-pack", xd[0], out);
 787        close(out);
 788        return ret;
 789}
 790
 791static int get_pack(struct fetch_pack_args *args,
 792                    int xd[2], char **pack_lockfile)
 793{
 794        struct async demux;
 795        int do_keep = args->keep_pack;
 796        const char *cmd_name;
 797        struct pack_header header;
 798        int pass_header = 0;
 799        struct child_process cmd = CHILD_PROCESS_INIT;
 800        int ret;
 801
 802        memset(&demux, 0, sizeof(demux));
 803        if (use_sideband) {
 804                /* xd[] is talking with upload-pack; subprocess reads from
 805                 * xd[0], spits out band#2 to stderr, and feeds us band#1
 806                 * through demux->out.
 807                 */
 808                demux.proc = sideband_demux;
 809                demux.data = xd;
 810                demux.out = -1;
 811                demux.isolate_sigpipe = 1;
 812                if (start_async(&demux))
 813                        die(_("fetch-pack: unable to fork off sideband demultiplexer"));
 814        }
 815        else
 816                demux.out = xd[0];
 817
 818        if (!args->keep_pack && unpack_limit) {
 819
 820                if (read_pack_header(demux.out, &header))
 821                        die(_("protocol error: bad pack header"));
 822                pass_header = 1;
 823                if (ntohl(header.hdr_entries) < unpack_limit)
 824                        do_keep = 0;
 825                else
 826                        do_keep = 1;
 827        }
 828
 829        if (alternate_shallow_file) {
 830                argv_array_push(&cmd.args, "--shallow-file");
 831                argv_array_push(&cmd.args, alternate_shallow_file);
 832        }
 833
 834        if (do_keep) {
 835                if (pack_lockfile)
 836                        cmd.out = -1;
 837                cmd_name = "index-pack";
 838                argv_array_push(&cmd.args, cmd_name);
 839                argv_array_push(&cmd.args, "--stdin");
 840                if (!args->quiet && !args->no_progress)
 841                        argv_array_push(&cmd.args, "-v");
 842                if (args->use_thin_pack)
 843                        argv_array_push(&cmd.args, "--fix-thin");
 844                if (args->lock_pack || unpack_limit) {
 845                        char hostname[HOST_NAME_MAX + 1];
 846                        if (xgethostname(hostname, sizeof(hostname)))
 847                                xsnprintf(hostname, sizeof(hostname), "localhost");
 848                        argv_array_pushf(&cmd.args,
 849                                        "--keep=fetch-pack %"PRIuMAX " on %s",
 850                                        (uintmax_t)getpid(), hostname);
 851                }
 852                if (args->check_self_contained_and_connected)
 853                        argv_array_push(&cmd.args, "--check-self-contained-and-connected");
 854        }
 855        else {
 856                cmd_name = "unpack-objects";
 857                argv_array_push(&cmd.args, cmd_name);
 858                if (args->quiet || args->no_progress)
 859                        argv_array_push(&cmd.args, "-q");
 860                args->check_self_contained_and_connected = 0;
 861        }
 862
 863        if (pass_header)
 864                argv_array_pushf(&cmd.args, "--pack_header=%"PRIu32",%"PRIu32,
 865                                 ntohl(header.hdr_version),
 866                                 ntohl(header.hdr_entries));
 867        if (fetch_fsck_objects >= 0
 868            ? fetch_fsck_objects
 869            : transfer_fsck_objects >= 0
 870            ? transfer_fsck_objects
 871            : 0)
 872                argv_array_push(&cmd.args, "--strict");
 873
 874        cmd.in = demux.out;
 875        cmd.git_cmd = 1;
 876        if (start_command(&cmd))
 877                die(_("fetch-pack: unable to fork off %s"), cmd_name);
 878        if (do_keep && pack_lockfile) {
 879                *pack_lockfile = index_pack_lockfile(cmd.out);
 880                close(cmd.out);
 881        }
 882
 883        if (!use_sideband)
 884                /* Closed by start_command() */
 885                xd[0] = -1;
 886
 887        ret = finish_command(&cmd);
 888        if (!ret || (args->check_self_contained_and_connected && ret == 1))
 889                args->self_contained_and_connected =
 890                        args->check_self_contained_and_connected &&
 891                        ret == 0;
 892        else
 893                die(_("%s failed"), cmd_name);
 894        if (use_sideband && finish_async(&demux))
 895                die(_("error in sideband demultiplexer"));
 896        return 0;
 897}
 898
 899static int cmp_ref_by_name(const void *a_, const void *b_)
 900{
 901        const struct ref *a = *((const struct ref **)a_);
 902        const struct ref *b = *((const struct ref **)b_);
 903        return strcmp(a->name, b->name);
 904}
 905
 906static struct ref *do_fetch_pack(struct fetch_pack_args *args,
 907                                 int fd[2],
 908                                 const struct ref *orig_ref,
 909                                 struct ref **sought, int nr_sought,
 910                                 struct shallow_info *si,
 911                                 char **pack_lockfile)
 912{
 913        struct ref *ref = copy_ref_list(orig_ref);
 914        struct object_id oid;
 915        const char *agent_feature;
 916        int agent_len;
 917
 918        sort_ref_list(&ref, ref_compare_name);
 919        QSORT(sought, nr_sought, cmp_ref_by_name);
 920
 921        if ((args->depth > 0 || is_repository_shallow()) && !server_supports("shallow"))
 922                die(_("Server does not support shallow clients"));
 923        if (args->depth > 0 || args->deepen_since || args->deepen_not)
 924                args->deepen = 1;
 925        if (server_supports("multi_ack_detailed")) {
 926                print_verbose(args, _("Server supports multi_ack_detailed"));
 927                multi_ack = 2;
 928                if (server_supports("no-done")) {
 929                        print_verbose(args, _("Server supports no-done"));
 930                        if (args->stateless_rpc)
 931                                no_done = 1;
 932                }
 933        }
 934        else if (server_supports("multi_ack")) {
 935                print_verbose(args, _("Server supports multi_ack"));
 936                multi_ack = 1;
 937        }
 938        if (server_supports("side-band-64k")) {
 939                print_verbose(args, _("Server supports side-band-64k"));
 940                use_sideband = 2;
 941        }
 942        else if (server_supports("side-band")) {
 943                print_verbose(args, _("Server supports side-band"));
 944                use_sideband = 1;
 945        }
 946        if (server_supports("allow-tip-sha1-in-want")) {
 947                print_verbose(args, _("Server supports allow-tip-sha1-in-want"));
 948                allow_unadvertised_object_request |= ALLOW_TIP_SHA1;
 949        }
 950        if (server_supports("allow-reachable-sha1-in-want")) {
 951                print_verbose(args, _("Server supports allow-reachable-sha1-in-want"));
 952                allow_unadvertised_object_request |= ALLOW_REACHABLE_SHA1;
 953        }
 954        if (!server_supports("thin-pack"))
 955                args->use_thin_pack = 0;
 956        if (!server_supports("no-progress"))
 957                args->no_progress = 0;
 958        if (!server_supports("include-tag"))
 959                args->include_tag = 0;
 960        if (server_supports("ofs-delta"))
 961                print_verbose(args, _("Server supports ofs-delta"));
 962        else
 963                prefer_ofs_delta = 0;
 964
 965        if ((agent_feature = server_feature_value("agent", &agent_len))) {
 966                agent_supported = 1;
 967                if (agent_len)
 968                        print_verbose(args, _("Server version is %.*s"),
 969                                      agent_len, agent_feature);
 970        }
 971        if (server_supports("deepen-since"))
 972                deepen_since_ok = 1;
 973        else if (args->deepen_since)
 974                die(_("Server does not support --shallow-since"));
 975        if (server_supports("deepen-not"))
 976                deepen_not_ok = 1;
 977        else if (args->deepen_not)
 978                die(_("Server does not support --shallow-exclude"));
 979        if (!server_supports("deepen-relative") && args->deepen_relative)
 980                die(_("Server does not support --deepen"));
 981
 982        if (everything_local(args, &ref, sought, nr_sought)) {
 983                packet_flush(fd[1]);
 984                goto all_done;
 985        }
 986        if (find_common(args, fd, &oid, ref) < 0)
 987                if (!args->keep_pack)
 988                        /* When cloning, it is not unusual to have
 989                         * no common commit.
 990                         */
 991                        warning(_("no common commits"));
 992
 993        if (args->stateless_rpc)
 994                packet_flush(fd[1]);
 995        if (args->deepen)
 996                setup_alternate_shallow(&shallow_lock, &alternate_shallow_file,
 997                                        NULL);
 998        else if (si->nr_ours || si->nr_theirs)
 999                alternate_shallow_file = setup_temporary_shallow(si->shallow);
1000        else
1001                alternate_shallow_file = NULL;
1002        if (get_pack(args, fd, pack_lockfile))
1003                die(_("git fetch-pack: fetch failed."));
1004
1005 all_done:
1006        return ref;
1007}
1008
1009static void fetch_pack_config(void)
1010{
1011        git_config_get_int("fetch.unpacklimit", &fetch_unpack_limit);
1012        git_config_get_int("transfer.unpacklimit", &transfer_unpack_limit);
1013        git_config_get_bool("repack.usedeltabaseoffset", &prefer_ofs_delta);
1014        git_config_get_bool("fetch.fsckobjects", &fetch_fsck_objects);
1015        git_config_get_bool("transfer.fsckobjects", &transfer_fsck_objects);
1016
1017        git_config(git_default_config, NULL);
1018}
1019
1020static void fetch_pack_setup(void)
1021{
1022        static int did_setup;
1023        if (did_setup)
1024                return;
1025        fetch_pack_config();
1026        if (0 <= transfer_unpack_limit)
1027                unpack_limit = transfer_unpack_limit;
1028        else if (0 <= fetch_unpack_limit)
1029                unpack_limit = fetch_unpack_limit;
1030        did_setup = 1;
1031}
1032
1033static int remove_duplicates_in_refs(struct ref **ref, int nr)
1034{
1035        struct string_list names = STRING_LIST_INIT_NODUP;
1036        int src, dst;
1037
1038        for (src = dst = 0; src < nr; src++) {
1039                struct string_list_item *item;
1040                item = string_list_insert(&names, ref[src]->name);
1041                if (item->util)
1042                        continue; /* already have it */
1043                item->util = ref[src];
1044                if (src != dst)
1045                        ref[dst] = ref[src];
1046                dst++;
1047        }
1048        for (src = dst; src < nr; src++)
1049                ref[src] = NULL;
1050        string_list_clear(&names, 0);
1051        return dst;
1052}
1053
1054static void update_shallow(struct fetch_pack_args *args,
1055                           struct ref **sought, int nr_sought,
1056                           struct shallow_info *si)
1057{
1058        struct oid_array ref = OID_ARRAY_INIT;
1059        int *status;
1060        int i;
1061
1062        if (args->deepen && alternate_shallow_file) {
1063                if (*alternate_shallow_file == '\0') { /* --unshallow */
1064                        unlink_or_warn(git_path_shallow());
1065                        rollback_lock_file(&shallow_lock);
1066                } else
1067                        commit_lock_file(&shallow_lock);
1068                return;
1069        }
1070
1071        if (!si->shallow || !si->shallow->nr)
1072                return;
1073
1074        if (args->cloning) {
1075                /*
1076                 * remote is shallow, but this is a clone, there are
1077                 * no objects in repo to worry about. Accept any
1078                 * shallow points that exist in the pack (iow in repo
1079                 * after get_pack() and reprepare_packed_git())
1080                 */
1081                struct oid_array extra = OID_ARRAY_INIT;
1082                struct object_id *oid = si->shallow->oid;
1083                for (i = 0; i < si->shallow->nr; i++)
1084                        if (has_object_file(&oid[i]))
1085                                oid_array_append(&extra, &oid[i]);
1086                if (extra.nr) {
1087                        setup_alternate_shallow(&shallow_lock,
1088                                                &alternate_shallow_file,
1089                                                &extra);
1090                        commit_lock_file(&shallow_lock);
1091                }
1092                oid_array_clear(&extra);
1093                return;
1094        }
1095
1096        if (!si->nr_ours && !si->nr_theirs)
1097                return;
1098
1099        remove_nonexistent_theirs_shallow(si);
1100        if (!si->nr_ours && !si->nr_theirs)
1101                return;
1102        for (i = 0; i < nr_sought; i++)
1103                oid_array_append(&ref, &sought[i]->old_oid);
1104        si->ref = &ref;
1105
1106        if (args->update_shallow) {
1107                /*
1108                 * remote is also shallow, .git/shallow may be updated
1109                 * so all refs can be accepted. Make sure we only add
1110                 * shallow roots that are actually reachable from new
1111                 * refs.
1112                 */
1113                struct oid_array extra = OID_ARRAY_INIT;
1114                struct object_id *oid = si->shallow->oid;
1115                assign_shallow_commits_to_refs(si, NULL, NULL);
1116                if (!si->nr_ours && !si->nr_theirs) {
1117                        oid_array_clear(&ref);
1118                        return;
1119                }
1120                for (i = 0; i < si->nr_ours; i++)
1121                        oid_array_append(&extra, &oid[si->ours[i]]);
1122                for (i = 0; i < si->nr_theirs; i++)
1123                        oid_array_append(&extra, &oid[si->theirs[i]]);
1124                setup_alternate_shallow(&shallow_lock,
1125                                        &alternate_shallow_file,
1126                                        &extra);
1127                commit_lock_file(&shallow_lock);
1128                oid_array_clear(&extra);
1129                oid_array_clear(&ref);
1130                return;
1131        }
1132
1133        /*
1134         * remote is also shallow, check what ref is safe to update
1135         * without updating .git/shallow
1136         */
1137        status = xcalloc(nr_sought, sizeof(*status));
1138        assign_shallow_commits_to_refs(si, NULL, status);
1139        if (si->nr_ours || si->nr_theirs) {
1140                for (i = 0; i < nr_sought; i++)
1141                        if (status[i])
1142                                sought[i]->status = REF_STATUS_REJECT_SHALLOW;
1143        }
1144        free(status);
1145        oid_array_clear(&ref);
1146}
1147
1148struct ref *fetch_pack(struct fetch_pack_args *args,
1149                       int fd[], struct child_process *conn,
1150                       const struct ref *ref,
1151                       const char *dest,
1152                       struct ref **sought, int nr_sought,
1153                       struct oid_array *shallow,
1154                       char **pack_lockfile)
1155{
1156        struct ref *ref_cpy;
1157        struct shallow_info si;
1158
1159        fetch_pack_setup();
1160        if (nr_sought)
1161                nr_sought = remove_duplicates_in_refs(sought, nr_sought);
1162
1163        if (!ref) {
1164                packet_flush(fd[1]);
1165                die(_("no matching remote head"));
1166        }
1167        prepare_shallow_info(&si, shallow);
1168        ref_cpy = do_fetch_pack(args, fd, ref, sought, nr_sought,
1169                                &si, pack_lockfile);
1170        reprepare_packed_git();
1171        update_shallow(args, sought, nr_sought, &si);
1172        clear_shallow_info(&si);
1173        return ref_cpy;
1174}
1175
1176int report_unmatched_refs(struct ref **sought, int nr_sought)
1177{
1178        int i, ret = 0;
1179
1180        for (i = 0; i < nr_sought; i++) {
1181                if (!sought[i])
1182                        continue;
1183                switch (sought[i]->match_status) {
1184                case REF_MATCHED:
1185                        continue;
1186                case REF_NOT_MATCHED:
1187                        error(_("no such remote ref %s"), sought[i]->name);
1188                        break;
1189                case REF_UNADVERTISED_NOT_ALLOWED:
1190                        error(_("Server does not allow request for unadvertised object %s"),
1191                              sought[i]->name);
1192                        break;
1193                }
1194                ret = 1;
1195        }
1196        return ret;
1197}