fetch-pack.con commit diff-lib: allow ita entries treated as "not yet exist in index" (425a28e)
   1#include "cache.h"
   2#include "lockfile.h"
   3#include "refs.h"
   4#include "pkt-line.h"
   5#include "commit.h"
   6#include "tag.h"
   7#include "exec_cmd.h"
   8#include "pack.h"
   9#include "sideband.h"
  10#include "fetch-pack.h"
  11#include "remote.h"
  12#include "run-command.h"
  13#include "connect.h"
  14#include "transport.h"
  15#include "version.h"
  16#include "prio-queue.h"
  17#include "sha1-array.h"
  18
  19static int transfer_unpack_limit = -1;
  20static int fetch_unpack_limit = -1;
  21static int unpack_limit = 100;
  22static int prefer_ofs_delta = 1;
  23static int no_done;
  24static int fetch_fsck_objects = -1;
  25static int transfer_fsck_objects = -1;
  26static int agent_supported;
  27static struct lock_file shallow_lock;
  28static const char *alternate_shallow_file;
  29
  30/* Remember to update object flag allocation in object.h */
  31#define COMPLETE        (1U << 0)
  32#define COMMON          (1U << 1)
  33#define COMMON_REF      (1U << 2)
  34#define SEEN            (1U << 3)
  35#define POPPED          (1U << 4)
  36
  37static int marked;
  38
  39/*
  40 * After sending this many "have"s if we do not get any new ACK , we
  41 * give up traversing our history.
  42 */
  43#define MAX_IN_VAIN 256
  44
  45static struct prio_queue rev_list = { compare_commits_by_commit_date };
  46static int non_common_revs, multi_ack, use_sideband;
  47/* Allow specifying sha1 if it is a ref tip. */
  48#define ALLOW_TIP_SHA1  01
  49/* Allow request of a sha1 if it is reachable from a ref (possibly hidden ref). */
  50#define ALLOW_REACHABLE_SHA1    02
  51static unsigned int allow_unadvertised_object_request;
  52
  53static void rev_list_push(struct commit *commit, int mark)
  54{
  55        if (!(commit->object.flags & mark)) {
  56                commit->object.flags |= mark;
  57
  58                if (parse_commit(commit))
  59                        return;
  60
  61                prio_queue_put(&rev_list, commit);
  62
  63                if (!(commit->object.flags & COMMON))
  64                        non_common_revs++;
  65        }
  66}
  67
  68static int rev_list_insert_ref(const char *refname, const unsigned char *sha1)
  69{
  70        struct object *o = deref_tag(parse_object(sha1), refname, 0);
  71
  72        if (o && o->type == OBJ_COMMIT)
  73                rev_list_push((struct commit *)o, SEEN);
  74
  75        return 0;
  76}
  77
  78static int rev_list_insert_ref_oid(const char *refname, const struct object_id *oid,
  79                                   int flag, void *cb_data)
  80{
  81        return rev_list_insert_ref(refname, oid->hash);
  82}
  83
  84static int clear_marks(const char *refname, const struct object_id *oid,
  85                       int flag, void *cb_data)
  86{
  87        struct object *o = deref_tag(parse_object(oid->hash), refname, 0);
  88
  89        if (o && o->type == OBJ_COMMIT)
  90                clear_commit_marks((struct commit *)o,
  91                                   COMMON | COMMON_REF | SEEN | POPPED);
  92        return 0;
  93}
  94
  95/*
  96   This function marks a rev and its ancestors as common.
  97   In some cases, it is desirable to mark only the ancestors (for example
  98   when only the server does not yet know that they are common).
  99*/
 100
 101static void mark_common(struct commit *commit,
 102                int ancestors_only, int dont_parse)
 103{
 104        if (commit != NULL && !(commit->object.flags & COMMON)) {
 105                struct object *o = (struct object *)commit;
 106
 107                if (!ancestors_only)
 108                        o->flags |= COMMON;
 109
 110                if (!(o->flags & SEEN))
 111                        rev_list_push(commit, SEEN);
 112                else {
 113                        struct commit_list *parents;
 114
 115                        if (!ancestors_only && !(o->flags & POPPED))
 116                                non_common_revs--;
 117                        if (!o->parsed && !dont_parse)
 118                                if (parse_commit(commit))
 119                                        return;
 120
 121                        for (parents = commit->parents;
 122                                        parents;
 123                                        parents = parents->next)
 124                                mark_common(parents->item, 0, dont_parse);
 125                }
 126        }
 127}
 128
 129/*
 130  Get the next rev to send, ignoring the common.
 131*/
 132
 133static const unsigned char *get_rev(void)
 134{
 135        struct commit *commit = NULL;
 136
 137        while (commit == NULL) {
 138                unsigned int mark;
 139                struct commit_list *parents;
 140
 141                if (rev_list.nr == 0 || non_common_revs == 0)
 142                        return NULL;
 143
 144                commit = prio_queue_get(&rev_list);
 145                parse_commit(commit);
 146                parents = commit->parents;
 147
 148                commit->object.flags |= POPPED;
 149                if (!(commit->object.flags & COMMON))
 150                        non_common_revs--;
 151
 152                if (commit->object.flags & COMMON) {
 153                        /* do not send "have", and ignore ancestors */
 154                        commit = NULL;
 155                        mark = COMMON | SEEN;
 156                } else if (commit->object.flags & COMMON_REF)
 157                        /* send "have", and ignore ancestors */
 158                        mark = COMMON | SEEN;
 159                else
 160                        /* send "have", also for its ancestors */
 161                        mark = SEEN;
 162
 163                while (parents) {
 164                        if (!(parents->item->object.flags & SEEN))
 165                                rev_list_push(parents->item, mark);
 166                        if (mark & COMMON)
 167                                mark_common(parents->item, 1, 0);
 168                        parents = parents->next;
 169                }
 170        }
 171
 172        return commit->object.oid.hash;
 173}
 174
 175enum ack_type {
 176        NAK = 0,
 177        ACK,
 178        ACK_continue,
 179        ACK_common,
 180        ACK_ready
 181};
 182
 183static void consume_shallow_list(struct fetch_pack_args *args, int fd)
 184{
 185        if (args->stateless_rpc && args->depth > 0) {
 186                /* If we sent a depth we will get back "duplicate"
 187                 * shallow and unshallow commands every time there
 188                 * is a block of have lines exchanged.
 189                 */
 190                char *line;
 191                while ((line = packet_read_line(fd, NULL))) {
 192                        if (starts_with(line, "shallow "))
 193                                continue;
 194                        if (starts_with(line, "unshallow "))
 195                                continue;
 196                        die("git fetch-pack: expected shallow list");
 197                }
 198        }
 199}
 200
 201static enum ack_type get_ack(int fd, unsigned char *result_sha1)
 202{
 203        int len;
 204        char *line = packet_read_line(fd, &len);
 205        const char *arg;
 206
 207        if (!len)
 208                die("git fetch-pack: expected ACK/NAK, got EOF");
 209        if (!strcmp(line, "NAK"))
 210                return NAK;
 211        if (skip_prefix(line, "ACK ", &arg)) {
 212                if (!get_sha1_hex(arg, result_sha1)) {
 213                        arg += 40;
 214                        len -= arg - line;
 215                        if (len < 1)
 216                                return ACK;
 217                        if (strstr(arg, "continue"))
 218                                return ACK_continue;
 219                        if (strstr(arg, "common"))
 220                                return ACK_common;
 221                        if (strstr(arg, "ready"))
 222                                return ACK_ready;
 223                        return ACK;
 224                }
 225        }
 226        die("git fetch_pack: expected ACK/NAK, got '%s'", line);
 227}
 228
 229static void send_request(struct fetch_pack_args *args,
 230                         int fd, struct strbuf *buf)
 231{
 232        if (args->stateless_rpc) {
 233                send_sideband(fd, -1, buf->buf, buf->len, LARGE_PACKET_MAX);
 234                packet_flush(fd);
 235        } else
 236                write_or_die(fd, buf->buf, buf->len);
 237}
 238
 239static void insert_one_alternate_ref(const struct ref *ref, void *unused)
 240{
 241        rev_list_insert_ref(NULL, ref->old_oid.hash);
 242}
 243
 244#define INITIAL_FLUSH 16
 245#define PIPESAFE_FLUSH 32
 246#define LARGE_FLUSH 16384
 247
 248static int next_flush(struct fetch_pack_args *args, int count)
 249{
 250        if (args->stateless_rpc) {
 251                if (count < LARGE_FLUSH)
 252                        count <<= 1;
 253                else
 254                        count = count * 11 / 10;
 255        } else {
 256                if (count < PIPESAFE_FLUSH)
 257                        count <<= 1;
 258                else
 259                        count += PIPESAFE_FLUSH;
 260        }
 261        return count;
 262}
 263
 264static int find_common(struct fetch_pack_args *args,
 265                       int fd[2], unsigned char *result_sha1,
 266                       struct ref *refs)
 267{
 268        int fetching;
 269        int count = 0, flushes = 0, flush_at = INITIAL_FLUSH, retval;
 270        const unsigned char *sha1;
 271        unsigned in_vain = 0;
 272        int got_continue = 0;
 273        int got_ready = 0;
 274        struct strbuf req_buf = STRBUF_INIT;
 275        size_t state_len = 0;
 276
 277        if (args->stateless_rpc && multi_ack == 1)
 278                die("--stateless-rpc requires multi_ack_detailed");
 279        if (marked)
 280                for_each_ref(clear_marks, NULL);
 281        marked = 1;
 282
 283        for_each_ref(rev_list_insert_ref_oid, NULL);
 284        for_each_alternate_ref(insert_one_alternate_ref, NULL);
 285
 286        fetching = 0;
 287        for ( ; refs ; refs = refs->next) {
 288                unsigned char *remote = refs->old_oid.hash;
 289                const char *remote_hex;
 290                struct object *o;
 291
 292                /*
 293                 * If that object is complete (i.e. it is an ancestor of a
 294                 * local ref), we tell them we have it but do not have to
 295                 * tell them about its ancestors, which they already know
 296                 * about.
 297                 *
 298                 * We use lookup_object here because we are only
 299                 * interested in the case we *know* the object is
 300                 * reachable and we have already scanned it.
 301                 */
 302                if (((o = lookup_object(remote)) != NULL) &&
 303                                (o->flags & COMPLETE)) {
 304                        continue;
 305                }
 306
 307                remote_hex = sha1_to_hex(remote);
 308                if (!fetching) {
 309                        struct strbuf c = STRBUF_INIT;
 310                        if (multi_ack == 2)     strbuf_addstr(&c, " multi_ack_detailed");
 311                        if (multi_ack == 1)     strbuf_addstr(&c, " multi_ack");
 312                        if (no_done)            strbuf_addstr(&c, " no-done");
 313                        if (use_sideband == 2)  strbuf_addstr(&c, " side-band-64k");
 314                        if (use_sideband == 1)  strbuf_addstr(&c, " side-band");
 315                        if (args->use_thin_pack) strbuf_addstr(&c, " thin-pack");
 316                        if (args->no_progress)   strbuf_addstr(&c, " no-progress");
 317                        if (args->include_tag)   strbuf_addstr(&c, " include-tag");
 318                        if (prefer_ofs_delta)   strbuf_addstr(&c, " ofs-delta");
 319                        if (agent_supported)    strbuf_addf(&c, " agent=%s",
 320                                                            git_user_agent_sanitized());
 321                        packet_buf_write(&req_buf, "want %s%s\n", remote_hex, c.buf);
 322                        strbuf_release(&c);
 323                } else
 324                        packet_buf_write(&req_buf, "want %s\n", remote_hex);
 325                fetching++;
 326        }
 327
 328        if (!fetching) {
 329                strbuf_release(&req_buf);
 330                packet_flush(fd[1]);
 331                return 1;
 332        }
 333
 334        if (is_repository_shallow())
 335                write_shallow_commits(&req_buf, 1, NULL);
 336        if (args->depth > 0)
 337                packet_buf_write(&req_buf, "deepen %d", args->depth);
 338        packet_buf_flush(&req_buf);
 339        state_len = req_buf.len;
 340
 341        if (args->depth > 0) {
 342                char *line;
 343                const char *arg;
 344                unsigned char sha1[20];
 345
 346                send_request(args, fd[1], &req_buf);
 347                while ((line = packet_read_line(fd[0], NULL))) {
 348                        if (skip_prefix(line, "shallow ", &arg)) {
 349                                if (get_sha1_hex(arg, sha1))
 350                                        die("invalid shallow line: %s", line);
 351                                register_shallow(sha1);
 352                                continue;
 353                        }
 354                        if (skip_prefix(line, "unshallow ", &arg)) {
 355                                if (get_sha1_hex(arg, sha1))
 356                                        die("invalid unshallow line: %s", line);
 357                                if (!lookup_object(sha1))
 358                                        die("object not found: %s", line);
 359                                /* make sure that it is parsed as shallow */
 360                                if (!parse_object(sha1))
 361                                        die("error in object: %s", line);
 362                                if (unregister_shallow(sha1))
 363                                        die("no shallow found: %s", line);
 364                                continue;
 365                        }
 366                        die("expected shallow/unshallow, got %s", line);
 367                }
 368        } else if (!args->stateless_rpc)
 369                send_request(args, fd[1], &req_buf);
 370
 371        if (!args->stateless_rpc) {
 372                /* If we aren't using the stateless-rpc interface
 373                 * we don't need to retain the headers.
 374                 */
 375                strbuf_setlen(&req_buf, 0);
 376                state_len = 0;
 377        }
 378
 379        flushes = 0;
 380        retval = -1;
 381        while ((sha1 = get_rev())) {
 382                packet_buf_write(&req_buf, "have %s\n", sha1_to_hex(sha1));
 383                if (args->verbose)
 384                        fprintf(stderr, "have %s\n", sha1_to_hex(sha1));
 385                in_vain++;
 386                if (flush_at <= ++count) {
 387                        int ack;
 388
 389                        packet_buf_flush(&req_buf);
 390                        send_request(args, fd[1], &req_buf);
 391                        strbuf_setlen(&req_buf, state_len);
 392                        flushes++;
 393                        flush_at = next_flush(args, count);
 394
 395                        /*
 396                         * We keep one window "ahead" of the other side, and
 397                         * will wait for an ACK only on the next one
 398                         */
 399                        if (!args->stateless_rpc && count == INITIAL_FLUSH)
 400                                continue;
 401
 402                        consume_shallow_list(args, fd[0]);
 403                        do {
 404                                ack = get_ack(fd[0], result_sha1);
 405                                if (args->verbose && ack)
 406                                        fprintf(stderr, "got ack %d %s\n", ack,
 407                                                        sha1_to_hex(result_sha1));
 408                                switch (ack) {
 409                                case ACK:
 410                                        flushes = 0;
 411                                        multi_ack = 0;
 412                                        retval = 0;
 413                                        goto done;
 414                                case ACK_common:
 415                                case ACK_ready:
 416                                case ACK_continue: {
 417                                        struct commit *commit =
 418                                                lookup_commit(result_sha1);
 419                                        if (!commit)
 420                                                die("invalid commit %s", sha1_to_hex(result_sha1));
 421                                        if (args->stateless_rpc
 422                                         && ack == ACK_common
 423                                         && !(commit->object.flags & COMMON)) {
 424                                                /* We need to replay the have for this object
 425                                                 * on the next RPC request so the peer knows
 426                                                 * it is in common with us.
 427                                                 */
 428                                                const char *hex = sha1_to_hex(result_sha1);
 429                                                packet_buf_write(&req_buf, "have %s\n", hex);
 430                                                state_len = req_buf.len;
 431                                        }
 432                                        mark_common(commit, 0, 1);
 433                                        retval = 0;
 434                                        in_vain = 0;
 435                                        got_continue = 1;
 436                                        if (ack == ACK_ready) {
 437                                                clear_prio_queue(&rev_list);
 438                                                got_ready = 1;
 439                                        }
 440                                        break;
 441                                        }
 442                                }
 443                        } while (ack);
 444                        flushes--;
 445                        if (got_continue && MAX_IN_VAIN < in_vain) {
 446                                if (args->verbose)
 447                                        fprintf(stderr, "giving up\n");
 448                                break; /* give up */
 449                        }
 450                }
 451        }
 452done:
 453        if (!got_ready || !no_done) {
 454                packet_buf_write(&req_buf, "done\n");
 455                send_request(args, fd[1], &req_buf);
 456        }
 457        if (args->verbose)
 458                fprintf(stderr, "done\n");
 459        if (retval != 0) {
 460                multi_ack = 0;
 461                flushes++;
 462        }
 463        strbuf_release(&req_buf);
 464
 465        if (!got_ready || !no_done)
 466                consume_shallow_list(args, fd[0]);
 467        while (flushes || multi_ack) {
 468                int ack = get_ack(fd[0], result_sha1);
 469                if (ack) {
 470                        if (args->verbose)
 471                                fprintf(stderr, "got ack (%d) %s\n", ack,
 472                                        sha1_to_hex(result_sha1));
 473                        if (ack == ACK)
 474                                return 0;
 475                        multi_ack = 1;
 476                        continue;
 477                }
 478                flushes--;
 479        }
 480        /* it is no error to fetch into a completely empty repo */
 481        return count ? retval : 0;
 482}
 483
 484static struct commit_list *complete;
 485
 486static int mark_complete(const unsigned char *sha1)
 487{
 488        struct object *o = parse_object(sha1);
 489
 490        while (o && o->type == OBJ_TAG) {
 491                struct tag *t = (struct tag *) o;
 492                if (!t->tagged)
 493                        break; /* broken repository */
 494                o->flags |= COMPLETE;
 495                o = parse_object(t->tagged->oid.hash);
 496        }
 497        if (o && o->type == OBJ_COMMIT) {
 498                struct commit *commit = (struct commit *)o;
 499                if (!(commit->object.flags & COMPLETE)) {
 500                        commit->object.flags |= COMPLETE;
 501                        commit_list_insert(commit, &complete);
 502                }
 503        }
 504        return 0;
 505}
 506
 507static int mark_complete_oid(const char *refname, const struct object_id *oid,
 508                             int flag, void *cb_data)
 509{
 510        return mark_complete(oid->hash);
 511}
 512
 513static void mark_recent_complete_commits(struct fetch_pack_args *args,
 514                                         unsigned long cutoff)
 515{
 516        while (complete && cutoff <= complete->item->date) {
 517                if (args->verbose)
 518                        fprintf(stderr, "Marking %s as complete\n",
 519                                oid_to_hex(&complete->item->object.oid));
 520                pop_most_recent_commit(&complete, COMPLETE);
 521        }
 522}
 523
 524static void filter_refs(struct fetch_pack_args *args,
 525                        struct ref **refs,
 526                        struct ref **sought, int nr_sought)
 527{
 528        struct ref *newlist = NULL;
 529        struct ref **newtail = &newlist;
 530        struct ref *ref, *next;
 531        int i;
 532
 533        i = 0;
 534        for (ref = *refs; ref; ref = next) {
 535                int keep = 0;
 536                next = ref->next;
 537
 538                if (starts_with(ref->name, "refs/") &&
 539                    check_refname_format(ref->name, 0))
 540                        ; /* trash */
 541                else {
 542                        while (i < nr_sought) {
 543                                int cmp = strcmp(ref->name, sought[i]->name);
 544                                if (cmp < 0)
 545                                        break; /* definitely do not have it */
 546                                else if (cmp == 0) {
 547                                        keep = 1; /* definitely have it */
 548                                        sought[i]->matched = 1;
 549                                }
 550                                i++;
 551                        }
 552                }
 553
 554                if (!keep && args->fetch_all &&
 555                    (!args->depth || !starts_with(ref->name, "refs/tags/")))
 556                        keep = 1;
 557
 558                if (keep) {
 559                        *newtail = ref;
 560                        ref->next = NULL;
 561                        newtail = &ref->next;
 562                } else {
 563                        free(ref);
 564                }
 565        }
 566
 567        /* Append unmatched requests to the list */
 568        if ((allow_unadvertised_object_request &
 569            (ALLOW_TIP_SHA1 | ALLOW_REACHABLE_SHA1))) {
 570                for (i = 0; i < nr_sought; i++) {
 571                        unsigned char sha1[20];
 572
 573                        ref = sought[i];
 574                        if (ref->matched)
 575                                continue;
 576                        if (get_sha1_hex(ref->name, sha1) ||
 577                            ref->name[40] != '\0' ||
 578                            hashcmp(sha1, ref->old_oid.hash))
 579                                continue;
 580
 581                        ref->matched = 1;
 582                        *newtail = copy_ref(ref);
 583                        newtail = &(*newtail)->next;
 584                }
 585        }
 586        *refs = newlist;
 587}
 588
 589static void mark_alternate_complete(const struct ref *ref, void *unused)
 590{
 591        mark_complete(ref->old_oid.hash);
 592}
 593
 594static int everything_local(struct fetch_pack_args *args,
 595                            struct ref **refs,
 596                            struct ref **sought, int nr_sought)
 597{
 598        struct ref *ref;
 599        int retval;
 600        unsigned long cutoff = 0;
 601
 602        save_commit_buffer = 0;
 603
 604        for (ref = *refs; ref; ref = ref->next) {
 605                struct object *o;
 606
 607                if (!has_object_file(&ref->old_oid))
 608                        continue;
 609
 610                o = parse_object(ref->old_oid.hash);
 611                if (!o)
 612                        continue;
 613
 614                /* We already have it -- which may mean that we were
 615                 * in sync with the other side at some time after
 616                 * that (it is OK if we guess wrong here).
 617                 */
 618                if (o->type == OBJ_COMMIT) {
 619                        struct commit *commit = (struct commit *)o;
 620                        if (!cutoff || cutoff < commit->date)
 621                                cutoff = commit->date;
 622                }
 623        }
 624
 625        if (!args->depth) {
 626                for_each_ref(mark_complete_oid, NULL);
 627                for_each_alternate_ref(mark_alternate_complete, NULL);
 628                commit_list_sort_by_date(&complete);
 629                if (cutoff)
 630                        mark_recent_complete_commits(args, cutoff);
 631        }
 632
 633        /*
 634         * Mark all complete remote refs as common refs.
 635         * Don't mark them common yet; the server has to be told so first.
 636         */
 637        for (ref = *refs; ref; ref = ref->next) {
 638                struct object *o = deref_tag(lookup_object(ref->old_oid.hash),
 639                                             NULL, 0);
 640
 641                if (!o || o->type != OBJ_COMMIT || !(o->flags & COMPLETE))
 642                        continue;
 643
 644                if (!(o->flags & SEEN)) {
 645                        rev_list_push((struct commit *)o, COMMON_REF | SEEN);
 646
 647                        mark_common((struct commit *)o, 1, 1);
 648                }
 649        }
 650
 651        filter_refs(args, refs, sought, nr_sought);
 652
 653        for (retval = 1, ref = *refs; ref ; ref = ref->next) {
 654                const unsigned char *remote = ref->old_oid.hash;
 655                struct object *o;
 656
 657                o = lookup_object(remote);
 658                if (!o || !(o->flags & COMPLETE)) {
 659                        retval = 0;
 660                        if (!args->verbose)
 661                                continue;
 662                        fprintf(stderr,
 663                                "want %s (%s)\n", sha1_to_hex(remote),
 664                                ref->name);
 665                        continue;
 666                }
 667                if (!args->verbose)
 668                        continue;
 669                fprintf(stderr,
 670                        "already have %s (%s)\n", sha1_to_hex(remote),
 671                        ref->name);
 672        }
 673        return retval;
 674}
 675
 676static int sideband_demux(int in, int out, void *data)
 677{
 678        int *xd = data;
 679        int ret;
 680
 681        ret = recv_sideband("fetch-pack", xd[0], out);
 682        close(out);
 683        return ret;
 684}
 685
 686static int get_pack(struct fetch_pack_args *args,
 687                    int xd[2], char **pack_lockfile)
 688{
 689        struct async demux;
 690        int do_keep = args->keep_pack;
 691        const char *cmd_name;
 692        struct pack_header header;
 693        int pass_header = 0;
 694        struct child_process cmd = CHILD_PROCESS_INIT;
 695        int ret;
 696
 697        memset(&demux, 0, sizeof(demux));
 698        if (use_sideband) {
 699                /* xd[] is talking with upload-pack; subprocess reads from
 700                 * xd[0], spits out band#2 to stderr, and feeds us band#1
 701                 * through demux->out.
 702                 */
 703                demux.proc = sideband_demux;
 704                demux.data = xd;
 705                demux.out = -1;
 706                demux.isolate_sigpipe = 1;
 707                if (start_async(&demux))
 708                        die("fetch-pack: unable to fork off sideband"
 709                            " demultiplexer");
 710        }
 711        else
 712                demux.out = xd[0];
 713
 714        if (!args->keep_pack && unpack_limit) {
 715
 716                if (read_pack_header(demux.out, &header))
 717                        die("protocol error: bad pack header");
 718                pass_header = 1;
 719                if (ntohl(header.hdr_entries) < unpack_limit)
 720                        do_keep = 0;
 721                else
 722                        do_keep = 1;
 723        }
 724
 725        if (alternate_shallow_file) {
 726                argv_array_push(&cmd.args, "--shallow-file");
 727                argv_array_push(&cmd.args, alternate_shallow_file);
 728        }
 729
 730        if (do_keep) {
 731                if (pack_lockfile)
 732                        cmd.out = -1;
 733                cmd_name = "index-pack";
 734                argv_array_push(&cmd.args, cmd_name);
 735                argv_array_push(&cmd.args, "--stdin");
 736                if (!args->quiet && !args->no_progress)
 737                        argv_array_push(&cmd.args, "-v");
 738                if (args->use_thin_pack)
 739                        argv_array_push(&cmd.args, "--fix-thin");
 740                if (args->lock_pack || unpack_limit) {
 741                        char hostname[256];
 742                        if (gethostname(hostname, sizeof(hostname)))
 743                                xsnprintf(hostname, sizeof(hostname), "localhost");
 744                        argv_array_pushf(&cmd.args,
 745                                        "--keep=fetch-pack %"PRIuMAX " on %s",
 746                                        (uintmax_t)getpid(), hostname);
 747                }
 748                if (args->check_self_contained_and_connected)
 749                        argv_array_push(&cmd.args, "--check-self-contained-and-connected");
 750        }
 751        else {
 752                cmd_name = "unpack-objects";
 753                argv_array_push(&cmd.args, cmd_name);
 754                if (args->quiet || args->no_progress)
 755                        argv_array_push(&cmd.args, "-q");
 756                args->check_self_contained_and_connected = 0;
 757        }
 758
 759        if (pass_header)
 760                argv_array_pushf(&cmd.args, "--pack_header=%"PRIu32",%"PRIu32,
 761                                 ntohl(header.hdr_version),
 762                                 ntohl(header.hdr_entries));
 763        if (fetch_fsck_objects >= 0
 764            ? fetch_fsck_objects
 765            : transfer_fsck_objects >= 0
 766            ? transfer_fsck_objects
 767            : 0)
 768                argv_array_push(&cmd.args, "--strict");
 769
 770        cmd.in = demux.out;
 771        cmd.git_cmd = 1;
 772        if (start_command(&cmd))
 773                die("fetch-pack: unable to fork off %s", cmd_name);
 774        if (do_keep && pack_lockfile) {
 775                *pack_lockfile = index_pack_lockfile(cmd.out);
 776                close(cmd.out);
 777        }
 778
 779        if (!use_sideband)
 780                /* Closed by start_command() */
 781                xd[0] = -1;
 782
 783        ret = finish_command(&cmd);
 784        if (!ret || (args->check_self_contained_and_connected && ret == 1))
 785                args->self_contained_and_connected =
 786                        args->check_self_contained_and_connected &&
 787                        ret == 0;
 788        else
 789                die("%s failed", cmd_name);
 790        if (use_sideband && finish_async(&demux))
 791                die("error in sideband demultiplexer");
 792        return 0;
 793}
 794
 795static int cmp_ref_by_name(const void *a_, const void *b_)
 796{
 797        const struct ref *a = *((const struct ref **)a_);
 798        const struct ref *b = *((const struct ref **)b_);
 799        return strcmp(a->name, b->name);
 800}
 801
 802static struct ref *do_fetch_pack(struct fetch_pack_args *args,
 803                                 int fd[2],
 804                                 const struct ref *orig_ref,
 805                                 struct ref **sought, int nr_sought,
 806                                 struct shallow_info *si,
 807                                 char **pack_lockfile)
 808{
 809        struct ref *ref = copy_ref_list(orig_ref);
 810        unsigned char sha1[20];
 811        const char *agent_feature;
 812        int agent_len;
 813
 814        sort_ref_list(&ref, ref_compare_name);
 815        qsort(sought, nr_sought, sizeof(*sought), cmp_ref_by_name);
 816
 817        if ((args->depth > 0 || is_repository_shallow()) && !server_supports("shallow"))
 818                die("Server does not support shallow clients");
 819        if (server_supports("multi_ack_detailed")) {
 820                if (args->verbose)
 821                        fprintf(stderr, "Server supports multi_ack_detailed\n");
 822                multi_ack = 2;
 823                if (server_supports("no-done")) {
 824                        if (args->verbose)
 825                                fprintf(stderr, "Server supports no-done\n");
 826                        if (args->stateless_rpc)
 827                                no_done = 1;
 828                }
 829        }
 830        else if (server_supports("multi_ack")) {
 831                if (args->verbose)
 832                        fprintf(stderr, "Server supports multi_ack\n");
 833                multi_ack = 1;
 834        }
 835        if (server_supports("side-band-64k")) {
 836                if (args->verbose)
 837                        fprintf(stderr, "Server supports side-band-64k\n");
 838                use_sideband = 2;
 839        }
 840        else if (server_supports("side-band")) {
 841                if (args->verbose)
 842                        fprintf(stderr, "Server supports side-band\n");
 843                use_sideband = 1;
 844        }
 845        if (server_supports("allow-tip-sha1-in-want")) {
 846                if (args->verbose)
 847                        fprintf(stderr, "Server supports allow-tip-sha1-in-want\n");
 848                allow_unadvertised_object_request |= ALLOW_TIP_SHA1;
 849        }
 850        if (server_supports("allow-reachable-sha1-in-want")) {
 851                if (args->verbose)
 852                        fprintf(stderr, "Server supports allow-reachable-sha1-in-want\n");
 853                allow_unadvertised_object_request |= ALLOW_REACHABLE_SHA1;
 854        }
 855        if (!server_supports("thin-pack"))
 856                args->use_thin_pack = 0;
 857        if (!server_supports("no-progress"))
 858                args->no_progress = 0;
 859        if (!server_supports("include-tag"))
 860                args->include_tag = 0;
 861        if (server_supports("ofs-delta")) {
 862                if (args->verbose)
 863                        fprintf(stderr, "Server supports ofs-delta\n");
 864        } else
 865                prefer_ofs_delta = 0;
 866
 867        if ((agent_feature = server_feature_value("agent", &agent_len))) {
 868                agent_supported = 1;
 869                if (args->verbose && agent_len)
 870                        fprintf(stderr, "Server version is %.*s\n",
 871                                agent_len, agent_feature);
 872        }
 873
 874        if (everything_local(args, &ref, sought, nr_sought)) {
 875                packet_flush(fd[1]);
 876                goto all_done;
 877        }
 878        if (find_common(args, fd, sha1, ref) < 0)
 879                if (!args->keep_pack)
 880                        /* When cloning, it is not unusual to have
 881                         * no common commit.
 882                         */
 883                        warning("no common commits");
 884
 885        if (args->stateless_rpc)
 886                packet_flush(fd[1]);
 887        if (args->depth > 0)
 888                setup_alternate_shallow(&shallow_lock, &alternate_shallow_file,
 889                                        NULL);
 890        else if (si->nr_ours || si->nr_theirs)
 891                alternate_shallow_file = setup_temporary_shallow(si->shallow);
 892        else
 893                alternate_shallow_file = NULL;
 894        if (get_pack(args, fd, pack_lockfile))
 895                die("git fetch-pack: fetch failed.");
 896
 897 all_done:
 898        return ref;
 899}
 900
 901static void fetch_pack_config(void)
 902{
 903        git_config_get_int("fetch.unpacklimit", &fetch_unpack_limit);
 904        git_config_get_int("transfer.unpacklimit", &transfer_unpack_limit);
 905        git_config_get_bool("repack.usedeltabaseoffset", &prefer_ofs_delta);
 906        git_config_get_bool("fetch.fsckobjects", &fetch_fsck_objects);
 907        git_config_get_bool("transfer.fsckobjects", &transfer_fsck_objects);
 908
 909        git_config(git_default_config, NULL);
 910}
 911
 912static void fetch_pack_setup(void)
 913{
 914        static int did_setup;
 915        if (did_setup)
 916                return;
 917        fetch_pack_config();
 918        if (0 <= transfer_unpack_limit)
 919                unpack_limit = transfer_unpack_limit;
 920        else if (0 <= fetch_unpack_limit)
 921                unpack_limit = fetch_unpack_limit;
 922        did_setup = 1;
 923}
 924
 925static int remove_duplicates_in_refs(struct ref **ref, int nr)
 926{
 927        struct string_list names = STRING_LIST_INIT_NODUP;
 928        int src, dst;
 929
 930        for (src = dst = 0; src < nr; src++) {
 931                struct string_list_item *item;
 932                item = string_list_insert(&names, ref[src]->name);
 933                if (item->util)
 934                        continue; /* already have it */
 935                item->util = ref[src];
 936                if (src != dst)
 937                        ref[dst] = ref[src];
 938                dst++;
 939        }
 940        for (src = dst; src < nr; src++)
 941                ref[src] = NULL;
 942        string_list_clear(&names, 0);
 943        return dst;
 944}
 945
 946static void update_shallow(struct fetch_pack_args *args,
 947                           struct ref **sought, int nr_sought,
 948                           struct shallow_info *si)
 949{
 950        struct sha1_array ref = SHA1_ARRAY_INIT;
 951        int *status;
 952        int i;
 953
 954        if (args->depth > 0 && alternate_shallow_file) {
 955                if (*alternate_shallow_file == '\0') { /* --unshallow */
 956                        unlink_or_warn(git_path_shallow());
 957                        rollback_lock_file(&shallow_lock);
 958                } else
 959                        commit_lock_file(&shallow_lock);
 960                return;
 961        }
 962
 963        if (!si->shallow || !si->shallow->nr)
 964                return;
 965
 966        if (args->cloning) {
 967                /*
 968                 * remote is shallow, but this is a clone, there are
 969                 * no objects in repo to worry about. Accept any
 970                 * shallow points that exist in the pack (iow in repo
 971                 * after get_pack() and reprepare_packed_git())
 972                 */
 973                struct sha1_array extra = SHA1_ARRAY_INIT;
 974                unsigned char (*sha1)[20] = si->shallow->sha1;
 975                for (i = 0; i < si->shallow->nr; i++)
 976                        if (has_sha1_file(sha1[i]))
 977                                sha1_array_append(&extra, sha1[i]);
 978                if (extra.nr) {
 979                        setup_alternate_shallow(&shallow_lock,
 980                                                &alternate_shallow_file,
 981                                                &extra);
 982                        commit_lock_file(&shallow_lock);
 983                }
 984                sha1_array_clear(&extra);
 985                return;
 986        }
 987
 988        if (!si->nr_ours && !si->nr_theirs)
 989                return;
 990
 991        remove_nonexistent_theirs_shallow(si);
 992        if (!si->nr_ours && !si->nr_theirs)
 993                return;
 994        for (i = 0; i < nr_sought; i++)
 995                sha1_array_append(&ref, sought[i]->old_oid.hash);
 996        si->ref = &ref;
 997
 998        if (args->update_shallow) {
 999                /*
1000                 * remote is also shallow, .git/shallow may be updated
1001                 * so all refs can be accepted. Make sure we only add
1002                 * shallow roots that are actually reachable from new
1003                 * refs.
1004                 */
1005                struct sha1_array extra = SHA1_ARRAY_INIT;
1006                unsigned char (*sha1)[20] = si->shallow->sha1;
1007                assign_shallow_commits_to_refs(si, NULL, NULL);
1008                if (!si->nr_ours && !si->nr_theirs) {
1009                        sha1_array_clear(&ref);
1010                        return;
1011                }
1012                for (i = 0; i < si->nr_ours; i++)
1013                        sha1_array_append(&extra, sha1[si->ours[i]]);
1014                for (i = 0; i < si->nr_theirs; i++)
1015                        sha1_array_append(&extra, sha1[si->theirs[i]]);
1016                setup_alternate_shallow(&shallow_lock,
1017                                        &alternate_shallow_file,
1018                                        &extra);
1019                commit_lock_file(&shallow_lock);
1020                sha1_array_clear(&extra);
1021                sha1_array_clear(&ref);
1022                return;
1023        }
1024
1025        /*
1026         * remote is also shallow, check what ref is safe to update
1027         * without updating .git/shallow
1028         */
1029        status = xcalloc(nr_sought, sizeof(*status));
1030        assign_shallow_commits_to_refs(si, NULL, status);
1031        if (si->nr_ours || si->nr_theirs) {
1032                for (i = 0; i < nr_sought; i++)
1033                        if (status[i])
1034                                sought[i]->status = REF_STATUS_REJECT_SHALLOW;
1035        }
1036        free(status);
1037        sha1_array_clear(&ref);
1038}
1039
1040struct ref *fetch_pack(struct fetch_pack_args *args,
1041                       int fd[], struct child_process *conn,
1042                       const struct ref *ref,
1043                       const char *dest,
1044                       struct ref **sought, int nr_sought,
1045                       struct sha1_array *shallow,
1046                       char **pack_lockfile)
1047{
1048        struct ref *ref_cpy;
1049        struct shallow_info si;
1050
1051        fetch_pack_setup();
1052        if (nr_sought)
1053                nr_sought = remove_duplicates_in_refs(sought, nr_sought);
1054
1055        if (!ref) {
1056                packet_flush(fd[1]);
1057                die("no matching remote head");
1058        }
1059        prepare_shallow_info(&si, shallow);
1060        ref_cpy = do_fetch_pack(args, fd, ref, sought, nr_sought,
1061                                &si, pack_lockfile);
1062        reprepare_packed_git();
1063        update_shallow(args, sought, nr_sought, &si);
1064        clear_shallow_info(&si);
1065        return ref_cpy;
1066}