fetch-pack.con commit Merge branch 'jc/cvsserver' into maint-2.10 (31add46)
   1#include "cache.h"
   2#include "lockfile.h"
   3#include "refs.h"
   4#include "pkt-line.h"
   5#include "commit.h"
   6#include "tag.h"
   7#include "exec_cmd.h"
   8#include "pack.h"
   9#include "sideband.h"
  10#include "fetch-pack.h"
  11#include "remote.h"
  12#include "run-command.h"
  13#include "connect.h"
  14#include "transport.h"
  15#include "version.h"
  16#include "prio-queue.h"
  17#include "sha1-array.h"
  18
  19static int transfer_unpack_limit = -1;
  20static int fetch_unpack_limit = -1;
  21static int unpack_limit = 100;
  22static int prefer_ofs_delta = 1;
  23static int no_done;
  24static int fetch_fsck_objects = -1;
  25static int transfer_fsck_objects = -1;
  26static int agent_supported;
  27static struct lock_file shallow_lock;
  28static const char *alternate_shallow_file;
  29
  30/* Remember to update object flag allocation in object.h */
  31#define COMPLETE        (1U << 0)
  32#define COMMON          (1U << 1)
  33#define COMMON_REF      (1U << 2)
  34#define SEEN            (1U << 3)
  35#define POPPED          (1U << 4)
  36
  37static int marked;
  38
  39/*
  40 * After sending this many "have"s if we do not get any new ACK , we
  41 * give up traversing our history.
  42 */
  43#define MAX_IN_VAIN 256
  44
  45static struct prio_queue rev_list = { compare_commits_by_commit_date };
  46static int non_common_revs, multi_ack, use_sideband;
  47/* Allow specifying sha1 if it is a ref tip. */
  48#define ALLOW_TIP_SHA1  01
  49/* Allow request of a sha1 if it is reachable from a ref (possibly hidden ref). */
  50#define ALLOW_REACHABLE_SHA1    02
  51static unsigned int allow_unadvertised_object_request;
  52
  53static void rev_list_push(struct commit *commit, int mark)
  54{
  55        if (!(commit->object.flags & mark)) {
  56                commit->object.flags |= mark;
  57
  58                if (parse_commit(commit))
  59                        return;
  60
  61                prio_queue_put(&rev_list, commit);
  62
  63                if (!(commit->object.flags & COMMON))
  64                        non_common_revs++;
  65        }
  66}
  67
  68static int rev_list_insert_ref(const char *refname, const unsigned char *sha1)
  69{
  70        struct object *o = deref_tag(parse_object(sha1), refname, 0);
  71
  72        if (o && o->type == OBJ_COMMIT)
  73                rev_list_push((struct commit *)o, SEEN);
  74
  75        return 0;
  76}
  77
  78static int rev_list_insert_ref_oid(const char *refname, const struct object_id *oid,
  79                                   int flag, void *cb_data)
  80{
  81        return rev_list_insert_ref(refname, oid->hash);
  82}
  83
  84static int clear_marks(const char *refname, const struct object_id *oid,
  85                       int flag, void *cb_data)
  86{
  87        struct object *o = deref_tag(parse_object(oid->hash), refname, 0);
  88
  89        if (o && o->type == OBJ_COMMIT)
  90                clear_commit_marks((struct commit *)o,
  91                                   COMMON | COMMON_REF | SEEN | POPPED);
  92        return 0;
  93}
  94
  95/*
  96   This function marks a rev and its ancestors as common.
  97   In some cases, it is desirable to mark only the ancestors (for example
  98   when only the server does not yet know that they are common).
  99*/
 100
 101static void mark_common(struct commit *commit,
 102                int ancestors_only, int dont_parse)
 103{
 104        if (commit != NULL && !(commit->object.flags & COMMON)) {
 105                struct object *o = (struct object *)commit;
 106
 107                if (!ancestors_only)
 108                        o->flags |= COMMON;
 109
 110                if (!(o->flags & SEEN))
 111                        rev_list_push(commit, SEEN);
 112                else {
 113                        struct commit_list *parents;
 114
 115                        if (!ancestors_only && !(o->flags & POPPED))
 116                                non_common_revs--;
 117                        if (!o->parsed && !dont_parse)
 118                                if (parse_commit(commit))
 119                                        return;
 120
 121                        for (parents = commit->parents;
 122                                        parents;
 123                                        parents = parents->next)
 124                                mark_common(parents->item, 0, dont_parse);
 125                }
 126        }
 127}
 128
 129/*
 130  Get the next rev to send, ignoring the common.
 131*/
 132
 133static const unsigned char *get_rev(void)
 134{
 135        struct commit *commit = NULL;
 136
 137        while (commit == NULL) {
 138                unsigned int mark;
 139                struct commit_list *parents;
 140
 141                if (rev_list.nr == 0 || non_common_revs == 0)
 142                        return NULL;
 143
 144                commit = prio_queue_get(&rev_list);
 145                parse_commit(commit);
 146                parents = commit->parents;
 147
 148                commit->object.flags |= POPPED;
 149                if (!(commit->object.flags & COMMON))
 150                        non_common_revs--;
 151
 152                if (commit->object.flags & COMMON) {
 153                        /* do not send "have", and ignore ancestors */
 154                        commit = NULL;
 155                        mark = COMMON | SEEN;
 156                } else if (commit->object.flags & COMMON_REF)
 157                        /* send "have", and ignore ancestors */
 158                        mark = COMMON | SEEN;
 159                else
 160                        /* send "have", also for its ancestors */
 161                        mark = SEEN;
 162
 163                while (parents) {
 164                        if (!(parents->item->object.flags & SEEN))
 165                                rev_list_push(parents->item, mark);
 166                        if (mark & COMMON)
 167                                mark_common(parents->item, 1, 0);
 168                        parents = parents->next;
 169                }
 170        }
 171
 172        return commit->object.oid.hash;
 173}
 174
 175enum ack_type {
 176        NAK = 0,
 177        ACK,
 178        ACK_continue,
 179        ACK_common,
 180        ACK_ready
 181};
 182
 183static void consume_shallow_list(struct fetch_pack_args *args, int fd)
 184{
 185        if (args->stateless_rpc && args->depth > 0) {
 186                /* If we sent a depth we will get back "duplicate"
 187                 * shallow and unshallow commands every time there
 188                 * is a block of have lines exchanged.
 189                 */
 190                char *line;
 191                while ((line = packet_read_line(fd, NULL))) {
 192                        if (starts_with(line, "shallow "))
 193                                continue;
 194                        if (starts_with(line, "unshallow "))
 195                                continue;
 196                        die("git fetch-pack: expected shallow list");
 197                }
 198        }
 199}
 200
 201static enum ack_type get_ack(int fd, unsigned char *result_sha1)
 202{
 203        int len;
 204        char *line = packet_read_line(fd, &len);
 205        const char *arg;
 206
 207        if (!len)
 208                die("git fetch-pack: expected ACK/NAK, got EOF");
 209        if (!strcmp(line, "NAK"))
 210                return NAK;
 211        if (skip_prefix(line, "ACK ", &arg)) {
 212                if (!get_sha1_hex(arg, result_sha1)) {
 213                        arg += 40;
 214                        len -= arg - line;
 215                        if (len < 1)
 216                                return ACK;
 217                        if (strstr(arg, "continue"))
 218                                return ACK_continue;
 219                        if (strstr(arg, "common"))
 220                                return ACK_common;
 221                        if (strstr(arg, "ready"))
 222                                return ACK_ready;
 223                        return ACK;
 224                }
 225        }
 226        die("git fetch_pack: expected ACK/NAK, got '%s'", line);
 227}
 228
 229static void send_request(struct fetch_pack_args *args,
 230                         int fd, struct strbuf *buf)
 231{
 232        if (args->stateless_rpc) {
 233                send_sideband(fd, -1, buf->buf, buf->len, LARGE_PACKET_MAX);
 234                packet_flush(fd);
 235        } else
 236                write_or_die(fd, buf->buf, buf->len);
 237}
 238
 239static void insert_one_alternate_ref(const struct ref *ref, void *unused)
 240{
 241        rev_list_insert_ref(NULL, ref->old_oid.hash);
 242}
 243
 244#define INITIAL_FLUSH 16
 245#define PIPESAFE_FLUSH 32
 246#define LARGE_FLUSH 16384
 247
 248static int next_flush(struct fetch_pack_args *args, int count)
 249{
 250        if (args->stateless_rpc) {
 251                if (count < LARGE_FLUSH)
 252                        count <<= 1;
 253                else
 254                        count = count * 11 / 10;
 255        } else {
 256                if (count < PIPESAFE_FLUSH)
 257                        count <<= 1;
 258                else
 259                        count += PIPESAFE_FLUSH;
 260        }
 261        return count;
 262}
 263
 264static int find_common(struct fetch_pack_args *args,
 265                       int fd[2], unsigned char *result_sha1,
 266                       struct ref *refs)
 267{
 268        int fetching;
 269        int count = 0, flushes = 0, flush_at = INITIAL_FLUSH, retval;
 270        const unsigned char *sha1;
 271        unsigned in_vain = 0;
 272        int got_continue = 0;
 273        int got_ready = 0;
 274        struct strbuf req_buf = STRBUF_INIT;
 275        size_t state_len = 0;
 276
 277        if (args->stateless_rpc && multi_ack == 1)
 278                die("--stateless-rpc requires multi_ack_detailed");
 279        if (marked)
 280                for_each_ref(clear_marks, NULL);
 281        marked = 1;
 282
 283        for_each_ref(rev_list_insert_ref_oid, NULL);
 284        for_each_alternate_ref(insert_one_alternate_ref, NULL);
 285
 286        fetching = 0;
 287        for ( ; refs ; refs = refs->next) {
 288                unsigned char *remote = refs->old_oid.hash;
 289                const char *remote_hex;
 290                struct object *o;
 291
 292                /*
 293                 * If that object is complete (i.e. it is an ancestor of a
 294                 * local ref), we tell them we have it but do not have to
 295                 * tell them about its ancestors, which they already know
 296                 * about.
 297                 *
 298                 * We use lookup_object here because we are only
 299                 * interested in the case we *know* the object is
 300                 * reachable and we have already scanned it.
 301                 */
 302                if (((o = lookup_object(remote)) != NULL) &&
 303                                (o->flags & COMPLETE)) {
 304                        continue;
 305                }
 306
 307                remote_hex = sha1_to_hex(remote);
 308                if (!fetching) {
 309                        struct strbuf c = STRBUF_INIT;
 310                        if (multi_ack == 2)     strbuf_addstr(&c, " multi_ack_detailed");
 311                        if (multi_ack == 1)     strbuf_addstr(&c, " multi_ack");
 312                        if (no_done)            strbuf_addstr(&c, " no-done");
 313                        if (use_sideband == 2)  strbuf_addstr(&c, " side-band-64k");
 314                        if (use_sideband == 1)  strbuf_addstr(&c, " side-band");
 315                        if (args->use_thin_pack) strbuf_addstr(&c, " thin-pack");
 316                        if (args->no_progress)   strbuf_addstr(&c, " no-progress");
 317                        if (args->include_tag)   strbuf_addstr(&c, " include-tag");
 318                        if (prefer_ofs_delta)   strbuf_addstr(&c, " ofs-delta");
 319                        if (agent_supported)    strbuf_addf(&c, " agent=%s",
 320                                                            git_user_agent_sanitized());
 321                        packet_buf_write(&req_buf, "want %s%s\n", remote_hex, c.buf);
 322                        strbuf_release(&c);
 323                } else
 324                        packet_buf_write(&req_buf, "want %s\n", remote_hex);
 325                fetching++;
 326        }
 327
 328        if (!fetching) {
 329                strbuf_release(&req_buf);
 330                packet_flush(fd[1]);
 331                return 1;
 332        }
 333
 334        if (is_repository_shallow())
 335                write_shallow_commits(&req_buf, 1, NULL);
 336        if (args->depth > 0)
 337                packet_buf_write(&req_buf, "deepen %d", args->depth);
 338        packet_buf_flush(&req_buf);
 339        state_len = req_buf.len;
 340
 341        if (args->depth > 0) {
 342                char *line;
 343                const char *arg;
 344                unsigned char sha1[20];
 345
 346                send_request(args, fd[1], &req_buf);
 347                while ((line = packet_read_line(fd[0], NULL))) {
 348                        if (skip_prefix(line, "shallow ", &arg)) {
 349                                if (get_sha1_hex(arg, sha1))
 350                                        die("invalid shallow line: %s", line);
 351                                register_shallow(sha1);
 352                                continue;
 353                        }
 354                        if (skip_prefix(line, "unshallow ", &arg)) {
 355                                if (get_sha1_hex(arg, sha1))
 356                                        die("invalid unshallow line: %s", line);
 357                                if (!lookup_object(sha1))
 358                                        die("object not found: %s", line);
 359                                /* make sure that it is parsed as shallow */
 360                                if (!parse_object(sha1))
 361                                        die("error in object: %s", line);
 362                                if (unregister_shallow(sha1))
 363                                        die("no shallow found: %s", line);
 364                                continue;
 365                        }
 366                        die("expected shallow/unshallow, got %s", line);
 367                }
 368        } else if (!args->stateless_rpc)
 369                send_request(args, fd[1], &req_buf);
 370
 371        if (!args->stateless_rpc) {
 372                /* If we aren't using the stateless-rpc interface
 373                 * we don't need to retain the headers.
 374                 */
 375                strbuf_setlen(&req_buf, 0);
 376                state_len = 0;
 377        }
 378
 379        flushes = 0;
 380        retval = -1;
 381        while ((sha1 = get_rev())) {
 382                packet_buf_write(&req_buf, "have %s\n", sha1_to_hex(sha1));
 383                if (args->verbose)
 384                        fprintf(stderr, "have %s\n", sha1_to_hex(sha1));
 385                in_vain++;
 386                if (flush_at <= ++count) {
 387                        int ack;
 388
 389                        packet_buf_flush(&req_buf);
 390                        send_request(args, fd[1], &req_buf);
 391                        strbuf_setlen(&req_buf, state_len);
 392                        flushes++;
 393                        flush_at = next_flush(args, count);
 394
 395                        /*
 396                         * We keep one window "ahead" of the other side, and
 397                         * will wait for an ACK only on the next one
 398                         */
 399                        if (!args->stateless_rpc && count == INITIAL_FLUSH)
 400                                continue;
 401
 402                        consume_shallow_list(args, fd[0]);
 403                        do {
 404                                ack = get_ack(fd[0], result_sha1);
 405                                if (args->verbose && ack)
 406                                        fprintf(stderr, "got ack %d %s\n", ack,
 407                                                        sha1_to_hex(result_sha1));
 408                                switch (ack) {
 409                                case ACK:
 410                                        flushes = 0;
 411                                        multi_ack = 0;
 412                                        retval = 0;
 413                                        goto done;
 414                                case ACK_common:
 415                                case ACK_ready:
 416                                case ACK_continue: {
 417                                        struct commit *commit =
 418                                                lookup_commit(result_sha1);
 419                                        if (!commit)
 420                                                die("invalid commit %s", sha1_to_hex(result_sha1));
 421                                        if (args->stateless_rpc
 422                                         && ack == ACK_common
 423                                         && !(commit->object.flags & COMMON)) {
 424                                                /* We need to replay the have for this object
 425                                                 * on the next RPC request so the peer knows
 426                                                 * it is in common with us.
 427                                                 */
 428                                                const char *hex = sha1_to_hex(result_sha1);
 429                                                packet_buf_write(&req_buf, "have %s\n", hex);
 430                                                state_len = req_buf.len;
 431                                                /*
 432                                                 * Reset in_vain because an ack
 433                                                 * for this commit has not been
 434                                                 * seen.
 435                                                 */
 436                                                in_vain = 0;
 437                                        } else if (!args->stateless_rpc
 438                                                   || ack != ACK_common)
 439                                                in_vain = 0;
 440                                        mark_common(commit, 0, 1);
 441                                        retval = 0;
 442                                        got_continue = 1;
 443                                        if (ack == ACK_ready) {
 444                                                clear_prio_queue(&rev_list);
 445                                                got_ready = 1;
 446                                        }
 447                                        break;
 448                                        }
 449                                }
 450                        } while (ack);
 451                        flushes--;
 452                        if (got_continue && MAX_IN_VAIN < in_vain) {
 453                                if (args->verbose)
 454                                        fprintf(stderr, "giving up\n");
 455                                break; /* give up */
 456                        }
 457                }
 458        }
 459done:
 460        if (!got_ready || !no_done) {
 461                packet_buf_write(&req_buf, "done\n");
 462                send_request(args, fd[1], &req_buf);
 463        }
 464        if (args->verbose)
 465                fprintf(stderr, "done\n");
 466        if (retval != 0) {
 467                multi_ack = 0;
 468                flushes++;
 469        }
 470        strbuf_release(&req_buf);
 471
 472        if (!got_ready || !no_done)
 473                consume_shallow_list(args, fd[0]);
 474        while (flushes || multi_ack) {
 475                int ack = get_ack(fd[0], result_sha1);
 476                if (ack) {
 477                        if (args->verbose)
 478                                fprintf(stderr, "got ack (%d) %s\n", ack,
 479                                        sha1_to_hex(result_sha1));
 480                        if (ack == ACK)
 481                                return 0;
 482                        multi_ack = 1;
 483                        continue;
 484                }
 485                flushes--;
 486        }
 487        /* it is no error to fetch into a completely empty repo */
 488        return count ? retval : 0;
 489}
 490
 491static struct commit_list *complete;
 492
 493static int mark_complete(const unsigned char *sha1)
 494{
 495        struct object *o = parse_object(sha1);
 496
 497        while (o && o->type == OBJ_TAG) {
 498                struct tag *t = (struct tag *) o;
 499                if (!t->tagged)
 500                        break; /* broken repository */
 501                o->flags |= COMPLETE;
 502                o = parse_object(t->tagged->oid.hash);
 503        }
 504        if (o && o->type == OBJ_COMMIT) {
 505                struct commit *commit = (struct commit *)o;
 506                if (!(commit->object.flags & COMPLETE)) {
 507                        commit->object.flags |= COMPLETE;
 508                        commit_list_insert(commit, &complete);
 509                }
 510        }
 511        return 0;
 512}
 513
 514static int mark_complete_oid(const char *refname, const struct object_id *oid,
 515                             int flag, void *cb_data)
 516{
 517        return mark_complete(oid->hash);
 518}
 519
 520static void mark_recent_complete_commits(struct fetch_pack_args *args,
 521                                         unsigned long cutoff)
 522{
 523        while (complete && cutoff <= complete->item->date) {
 524                if (args->verbose)
 525                        fprintf(stderr, "Marking %s as complete\n",
 526                                oid_to_hex(&complete->item->object.oid));
 527                pop_most_recent_commit(&complete, COMPLETE);
 528        }
 529}
 530
 531static void filter_refs(struct fetch_pack_args *args,
 532                        struct ref **refs,
 533                        struct ref **sought, int nr_sought)
 534{
 535        struct ref *newlist = NULL;
 536        struct ref **newtail = &newlist;
 537        struct ref *ref, *next;
 538        int i;
 539
 540        i = 0;
 541        for (ref = *refs; ref; ref = next) {
 542                int keep = 0;
 543                next = ref->next;
 544
 545                if (starts_with(ref->name, "refs/") &&
 546                    check_refname_format(ref->name, 0))
 547                        ; /* trash */
 548                else {
 549                        while (i < nr_sought) {
 550                                int cmp = strcmp(ref->name, sought[i]->name);
 551                                if (cmp < 0)
 552                                        break; /* definitely do not have it */
 553                                else if (cmp == 0) {
 554                                        keep = 1; /* definitely have it */
 555                                        sought[i]->matched = 1;
 556                                }
 557                                i++;
 558                        }
 559                }
 560
 561                if (!keep && args->fetch_all &&
 562                    (!args->depth || !starts_with(ref->name, "refs/tags/")))
 563                        keep = 1;
 564
 565                if (keep) {
 566                        *newtail = ref;
 567                        ref->next = NULL;
 568                        newtail = &ref->next;
 569                } else {
 570                        free(ref);
 571                }
 572        }
 573
 574        /* Append unmatched requests to the list */
 575        if ((allow_unadvertised_object_request &
 576            (ALLOW_TIP_SHA1 | ALLOW_REACHABLE_SHA1))) {
 577                for (i = 0; i < nr_sought; i++) {
 578                        unsigned char sha1[20];
 579
 580                        ref = sought[i];
 581                        if (ref->matched)
 582                                continue;
 583                        if (get_sha1_hex(ref->name, sha1) ||
 584                            ref->name[40] != '\0' ||
 585                            hashcmp(sha1, ref->old_oid.hash))
 586                                continue;
 587
 588                        ref->matched = 1;
 589                        *newtail = copy_ref(ref);
 590                        newtail = &(*newtail)->next;
 591                }
 592        }
 593        *refs = newlist;
 594}
 595
 596static void mark_alternate_complete(const struct ref *ref, void *unused)
 597{
 598        mark_complete(ref->old_oid.hash);
 599}
 600
 601static int everything_local(struct fetch_pack_args *args,
 602                            struct ref **refs,
 603                            struct ref **sought, int nr_sought)
 604{
 605        struct ref *ref;
 606        int retval;
 607        unsigned long cutoff = 0;
 608
 609        save_commit_buffer = 0;
 610
 611        for (ref = *refs; ref; ref = ref->next) {
 612                struct object *o;
 613
 614                if (!has_object_file(&ref->old_oid))
 615                        continue;
 616
 617                o = parse_object(ref->old_oid.hash);
 618                if (!o)
 619                        continue;
 620
 621                /* We already have it -- which may mean that we were
 622                 * in sync with the other side at some time after
 623                 * that (it is OK if we guess wrong here).
 624                 */
 625                if (o->type == OBJ_COMMIT) {
 626                        struct commit *commit = (struct commit *)o;
 627                        if (!cutoff || cutoff < commit->date)
 628                                cutoff = commit->date;
 629                }
 630        }
 631
 632        if (!args->depth) {
 633                for_each_ref(mark_complete_oid, NULL);
 634                for_each_alternate_ref(mark_alternate_complete, NULL);
 635                commit_list_sort_by_date(&complete);
 636                if (cutoff)
 637                        mark_recent_complete_commits(args, cutoff);
 638        }
 639
 640        /*
 641         * Mark all complete remote refs as common refs.
 642         * Don't mark them common yet; the server has to be told so first.
 643         */
 644        for (ref = *refs; ref; ref = ref->next) {
 645                struct object *o = deref_tag(lookup_object(ref->old_oid.hash),
 646                                             NULL, 0);
 647
 648                if (!o || o->type != OBJ_COMMIT || !(o->flags & COMPLETE))
 649                        continue;
 650
 651                if (!(o->flags & SEEN)) {
 652                        rev_list_push((struct commit *)o, COMMON_REF | SEEN);
 653
 654                        mark_common((struct commit *)o, 1, 1);
 655                }
 656        }
 657
 658        filter_refs(args, refs, sought, nr_sought);
 659
 660        for (retval = 1, ref = *refs; ref ; ref = ref->next) {
 661                const unsigned char *remote = ref->old_oid.hash;
 662                struct object *o;
 663
 664                o = lookup_object(remote);
 665                if (!o || !(o->flags & COMPLETE)) {
 666                        retval = 0;
 667                        if (!args->verbose)
 668                                continue;
 669                        fprintf(stderr,
 670                                "want %s (%s)\n", sha1_to_hex(remote),
 671                                ref->name);
 672                        continue;
 673                }
 674                if (!args->verbose)
 675                        continue;
 676                fprintf(stderr,
 677                        "already have %s (%s)\n", sha1_to_hex(remote),
 678                        ref->name);
 679        }
 680        return retval;
 681}
 682
 683static int sideband_demux(int in, int out, void *data)
 684{
 685        int *xd = data;
 686        int ret;
 687
 688        ret = recv_sideband("fetch-pack", xd[0], out);
 689        close(out);
 690        return ret;
 691}
 692
 693static int get_pack(struct fetch_pack_args *args,
 694                    int xd[2], char **pack_lockfile)
 695{
 696        struct async demux;
 697        int do_keep = args->keep_pack;
 698        const char *cmd_name;
 699        struct pack_header header;
 700        int pass_header = 0;
 701        struct child_process cmd = CHILD_PROCESS_INIT;
 702        int ret;
 703
 704        memset(&demux, 0, sizeof(demux));
 705        if (use_sideband) {
 706                /* xd[] is talking with upload-pack; subprocess reads from
 707                 * xd[0], spits out band#2 to stderr, and feeds us band#1
 708                 * through demux->out.
 709                 */
 710                demux.proc = sideband_demux;
 711                demux.data = xd;
 712                demux.out = -1;
 713                demux.isolate_sigpipe = 1;
 714                if (start_async(&demux))
 715                        die("fetch-pack: unable to fork off sideband"
 716                            " demultiplexer");
 717        }
 718        else
 719                demux.out = xd[0];
 720
 721        if (!args->keep_pack && unpack_limit) {
 722
 723                if (read_pack_header(demux.out, &header))
 724                        die("protocol error: bad pack header");
 725                pass_header = 1;
 726                if (ntohl(header.hdr_entries) < unpack_limit)
 727                        do_keep = 0;
 728                else
 729                        do_keep = 1;
 730        }
 731
 732        if (alternate_shallow_file) {
 733                argv_array_push(&cmd.args, "--shallow-file");
 734                argv_array_push(&cmd.args, alternate_shallow_file);
 735        }
 736
 737        if (do_keep) {
 738                if (pack_lockfile)
 739                        cmd.out = -1;
 740                cmd_name = "index-pack";
 741                argv_array_push(&cmd.args, cmd_name);
 742                argv_array_push(&cmd.args, "--stdin");
 743                if (!args->quiet && !args->no_progress)
 744                        argv_array_push(&cmd.args, "-v");
 745                if (args->use_thin_pack)
 746                        argv_array_push(&cmd.args, "--fix-thin");
 747                if (args->lock_pack || unpack_limit) {
 748                        char hostname[256];
 749                        if (gethostname(hostname, sizeof(hostname)))
 750                                xsnprintf(hostname, sizeof(hostname), "localhost");
 751                        argv_array_pushf(&cmd.args,
 752                                        "--keep=fetch-pack %"PRIuMAX " on %s",
 753                                        (uintmax_t)getpid(), hostname);
 754                }
 755                if (args->check_self_contained_and_connected)
 756                        argv_array_push(&cmd.args, "--check-self-contained-and-connected");
 757        }
 758        else {
 759                cmd_name = "unpack-objects";
 760                argv_array_push(&cmd.args, cmd_name);
 761                if (args->quiet || args->no_progress)
 762                        argv_array_push(&cmd.args, "-q");
 763                args->check_self_contained_and_connected = 0;
 764        }
 765
 766        if (pass_header)
 767                argv_array_pushf(&cmd.args, "--pack_header=%"PRIu32",%"PRIu32,
 768                                 ntohl(header.hdr_version),
 769                                 ntohl(header.hdr_entries));
 770        if (fetch_fsck_objects >= 0
 771            ? fetch_fsck_objects
 772            : transfer_fsck_objects >= 0
 773            ? transfer_fsck_objects
 774            : 0)
 775                argv_array_push(&cmd.args, "--strict");
 776
 777        cmd.in = demux.out;
 778        cmd.git_cmd = 1;
 779        if (start_command(&cmd))
 780                die("fetch-pack: unable to fork off %s", cmd_name);
 781        if (do_keep && pack_lockfile) {
 782                *pack_lockfile = index_pack_lockfile(cmd.out);
 783                close(cmd.out);
 784        }
 785
 786        if (!use_sideband)
 787                /* Closed by start_command() */
 788                xd[0] = -1;
 789
 790        ret = finish_command(&cmd);
 791        if (!ret || (args->check_self_contained_and_connected && ret == 1))
 792                args->self_contained_and_connected =
 793                        args->check_self_contained_and_connected &&
 794                        ret == 0;
 795        else
 796                die("%s failed", cmd_name);
 797        if (use_sideband && finish_async(&demux))
 798                die("error in sideband demultiplexer");
 799        return 0;
 800}
 801
 802static int cmp_ref_by_name(const void *a_, const void *b_)
 803{
 804        const struct ref *a = *((const struct ref **)a_);
 805        const struct ref *b = *((const struct ref **)b_);
 806        return strcmp(a->name, b->name);
 807}
 808
 809static struct ref *do_fetch_pack(struct fetch_pack_args *args,
 810                                 int fd[2],
 811                                 const struct ref *orig_ref,
 812                                 struct ref **sought, int nr_sought,
 813                                 struct shallow_info *si,
 814                                 char **pack_lockfile)
 815{
 816        struct ref *ref = copy_ref_list(orig_ref);
 817        unsigned char sha1[20];
 818        const char *agent_feature;
 819        int agent_len;
 820
 821        sort_ref_list(&ref, ref_compare_name);
 822        qsort(sought, nr_sought, sizeof(*sought), cmp_ref_by_name);
 823
 824        if ((args->depth > 0 || is_repository_shallow()) && !server_supports("shallow"))
 825                die("Server does not support shallow clients");
 826        if (server_supports("multi_ack_detailed")) {
 827                if (args->verbose)
 828                        fprintf(stderr, "Server supports multi_ack_detailed\n");
 829                multi_ack = 2;
 830                if (server_supports("no-done")) {
 831                        if (args->verbose)
 832                                fprintf(stderr, "Server supports no-done\n");
 833                        if (args->stateless_rpc)
 834                                no_done = 1;
 835                }
 836        }
 837        else if (server_supports("multi_ack")) {
 838                if (args->verbose)
 839                        fprintf(stderr, "Server supports multi_ack\n");
 840                multi_ack = 1;
 841        }
 842        if (server_supports("side-band-64k")) {
 843                if (args->verbose)
 844                        fprintf(stderr, "Server supports side-band-64k\n");
 845                use_sideband = 2;
 846        }
 847        else if (server_supports("side-band")) {
 848                if (args->verbose)
 849                        fprintf(stderr, "Server supports side-band\n");
 850                use_sideband = 1;
 851        }
 852        if (server_supports("allow-tip-sha1-in-want")) {
 853                if (args->verbose)
 854                        fprintf(stderr, "Server supports allow-tip-sha1-in-want\n");
 855                allow_unadvertised_object_request |= ALLOW_TIP_SHA1;
 856        }
 857        if (server_supports("allow-reachable-sha1-in-want")) {
 858                if (args->verbose)
 859                        fprintf(stderr, "Server supports allow-reachable-sha1-in-want\n");
 860                allow_unadvertised_object_request |= ALLOW_REACHABLE_SHA1;
 861        }
 862        if (!server_supports("thin-pack"))
 863                args->use_thin_pack = 0;
 864        if (!server_supports("no-progress"))
 865                args->no_progress = 0;
 866        if (!server_supports("include-tag"))
 867                args->include_tag = 0;
 868        if (server_supports("ofs-delta")) {
 869                if (args->verbose)
 870                        fprintf(stderr, "Server supports ofs-delta\n");
 871        } else
 872                prefer_ofs_delta = 0;
 873
 874        if ((agent_feature = server_feature_value("agent", &agent_len))) {
 875                agent_supported = 1;
 876                if (args->verbose && agent_len)
 877                        fprintf(stderr, "Server version is %.*s\n",
 878                                agent_len, agent_feature);
 879        }
 880
 881        if (everything_local(args, &ref, sought, nr_sought)) {
 882                packet_flush(fd[1]);
 883                goto all_done;
 884        }
 885        if (find_common(args, fd, sha1, ref) < 0)
 886                if (!args->keep_pack)
 887                        /* When cloning, it is not unusual to have
 888                         * no common commit.
 889                         */
 890                        warning("no common commits");
 891
 892        if (args->stateless_rpc)
 893                packet_flush(fd[1]);
 894        if (args->depth > 0)
 895                setup_alternate_shallow(&shallow_lock, &alternate_shallow_file,
 896                                        NULL);
 897        else if (si->nr_ours || si->nr_theirs)
 898                alternate_shallow_file = setup_temporary_shallow(si->shallow);
 899        else
 900                alternate_shallow_file = NULL;
 901        if (get_pack(args, fd, pack_lockfile))
 902                die("git fetch-pack: fetch failed.");
 903
 904 all_done:
 905        return ref;
 906}
 907
 908static void fetch_pack_config(void)
 909{
 910        git_config_get_int("fetch.unpacklimit", &fetch_unpack_limit);
 911        git_config_get_int("transfer.unpacklimit", &transfer_unpack_limit);
 912        git_config_get_bool("repack.usedeltabaseoffset", &prefer_ofs_delta);
 913        git_config_get_bool("fetch.fsckobjects", &fetch_fsck_objects);
 914        git_config_get_bool("transfer.fsckobjects", &transfer_fsck_objects);
 915
 916        git_config(git_default_config, NULL);
 917}
 918
 919static void fetch_pack_setup(void)
 920{
 921        static int did_setup;
 922        if (did_setup)
 923                return;
 924        fetch_pack_config();
 925        if (0 <= transfer_unpack_limit)
 926                unpack_limit = transfer_unpack_limit;
 927        else if (0 <= fetch_unpack_limit)
 928                unpack_limit = fetch_unpack_limit;
 929        did_setup = 1;
 930}
 931
 932static int remove_duplicates_in_refs(struct ref **ref, int nr)
 933{
 934        struct string_list names = STRING_LIST_INIT_NODUP;
 935        int src, dst;
 936
 937        for (src = dst = 0; src < nr; src++) {
 938                struct string_list_item *item;
 939                item = string_list_insert(&names, ref[src]->name);
 940                if (item->util)
 941                        continue; /* already have it */
 942                item->util = ref[src];
 943                if (src != dst)
 944                        ref[dst] = ref[src];
 945                dst++;
 946        }
 947        for (src = dst; src < nr; src++)
 948                ref[src] = NULL;
 949        string_list_clear(&names, 0);
 950        return dst;
 951}
 952
 953static void update_shallow(struct fetch_pack_args *args,
 954                           struct ref **sought, int nr_sought,
 955                           struct shallow_info *si)
 956{
 957        struct sha1_array ref = SHA1_ARRAY_INIT;
 958        int *status;
 959        int i;
 960
 961        if (args->depth > 0 && alternate_shallow_file) {
 962                if (*alternate_shallow_file == '\0') { /* --unshallow */
 963                        unlink_or_warn(git_path_shallow());
 964                        rollback_lock_file(&shallow_lock);
 965                } else
 966                        commit_lock_file(&shallow_lock);
 967                return;
 968        }
 969
 970        if (!si->shallow || !si->shallow->nr)
 971                return;
 972
 973        if (args->cloning) {
 974                /*
 975                 * remote is shallow, but this is a clone, there are
 976                 * no objects in repo to worry about. Accept any
 977                 * shallow points that exist in the pack (iow in repo
 978                 * after get_pack() and reprepare_packed_git())
 979                 */
 980                struct sha1_array extra = SHA1_ARRAY_INIT;
 981                unsigned char (*sha1)[20] = si->shallow->sha1;
 982                for (i = 0; i < si->shallow->nr; i++)
 983                        if (has_sha1_file(sha1[i]))
 984                                sha1_array_append(&extra, sha1[i]);
 985                if (extra.nr) {
 986                        setup_alternate_shallow(&shallow_lock,
 987                                                &alternate_shallow_file,
 988                                                &extra);
 989                        commit_lock_file(&shallow_lock);
 990                }
 991                sha1_array_clear(&extra);
 992                return;
 993        }
 994
 995        if (!si->nr_ours && !si->nr_theirs)
 996                return;
 997
 998        remove_nonexistent_theirs_shallow(si);
 999        if (!si->nr_ours && !si->nr_theirs)
1000                return;
1001        for (i = 0; i < nr_sought; i++)
1002                sha1_array_append(&ref, sought[i]->old_oid.hash);
1003        si->ref = &ref;
1004
1005        if (args->update_shallow) {
1006                /*
1007                 * remote is also shallow, .git/shallow may be updated
1008                 * so all refs can be accepted. Make sure we only add
1009                 * shallow roots that are actually reachable from new
1010                 * refs.
1011                 */
1012                struct sha1_array extra = SHA1_ARRAY_INIT;
1013                unsigned char (*sha1)[20] = si->shallow->sha1;
1014                assign_shallow_commits_to_refs(si, NULL, NULL);
1015                if (!si->nr_ours && !si->nr_theirs) {
1016                        sha1_array_clear(&ref);
1017                        return;
1018                }
1019                for (i = 0; i < si->nr_ours; i++)
1020                        sha1_array_append(&extra, sha1[si->ours[i]]);
1021                for (i = 0; i < si->nr_theirs; i++)
1022                        sha1_array_append(&extra, sha1[si->theirs[i]]);
1023                setup_alternate_shallow(&shallow_lock,
1024                                        &alternate_shallow_file,
1025                                        &extra);
1026                commit_lock_file(&shallow_lock);
1027                sha1_array_clear(&extra);
1028                sha1_array_clear(&ref);
1029                return;
1030        }
1031
1032        /*
1033         * remote is also shallow, check what ref is safe to update
1034         * without updating .git/shallow
1035         */
1036        status = xcalloc(nr_sought, sizeof(*status));
1037        assign_shallow_commits_to_refs(si, NULL, status);
1038        if (si->nr_ours || si->nr_theirs) {
1039                for (i = 0; i < nr_sought; i++)
1040                        if (status[i])
1041                                sought[i]->status = REF_STATUS_REJECT_SHALLOW;
1042        }
1043        free(status);
1044        sha1_array_clear(&ref);
1045}
1046
1047struct ref *fetch_pack(struct fetch_pack_args *args,
1048                       int fd[], struct child_process *conn,
1049                       const struct ref *ref,
1050                       const char *dest,
1051                       struct ref **sought, int nr_sought,
1052                       struct sha1_array *shallow,
1053                       char **pack_lockfile)
1054{
1055        struct ref *ref_cpy;
1056        struct shallow_info si;
1057
1058        fetch_pack_setup();
1059        if (nr_sought)
1060                nr_sought = remove_duplicates_in_refs(sought, nr_sought);
1061
1062        if (!ref) {
1063                packet_flush(fd[1]);
1064                die("no matching remote head");
1065        }
1066        prepare_shallow_info(&si, shallow);
1067        ref_cpy = do_fetch_pack(args, fd, ref, sought, nr_sought,
1068                                &si, pack_lockfile);
1069        reprepare_packed_git();
1070        update_shallow(args, sought, nr_sought, &si);
1071        clear_shallow_info(&si);
1072        return ref_cpy;
1073}