fetch-pack.con commit git-svn: optimize --branch and --branch-all-ref (098749d)
   1#include "cache.h"
   2#include "refs.h"
   3#include "pkt-line.h"
   4#include "commit.h"
   5#include "tag.h"
   6
   7static int keep_pack;
   8static int quiet;
   9static int verbose;
  10static int fetch_all;
  11static const char fetch_pack_usage[] =
  12"git-fetch-pack [--all] [-q] [-v] [-k] [--thin] [--exec=upload-pack] [host:]directory <refs>...";
  13static const char *exec = "git-upload-pack";
  14
  15#define COMPLETE        (1U << 0)
  16#define COMMON          (1U << 1)
  17#define COMMON_REF      (1U << 2)
  18#define SEEN            (1U << 3)
  19#define POPPED          (1U << 4)
  20
  21static struct commit_list *rev_list = NULL;
  22static int non_common_revs = 0, multi_ack = 0, use_thin_pack = 0;
  23
  24static void rev_list_push(struct commit *commit, int mark)
  25{
  26        if (!(commit->object.flags & mark)) {
  27                commit->object.flags |= mark;
  28
  29                if (!(commit->object.parsed))
  30                        parse_commit(commit);
  31
  32                insert_by_date(commit, &rev_list);
  33
  34                if (!(commit->object.flags & COMMON))
  35                        non_common_revs++;
  36        }
  37}
  38
  39static int rev_list_insert_ref(const char *path, const unsigned char *sha1)
  40{
  41        struct object *o = deref_tag(parse_object(sha1), path, 0);
  42
  43        if (o && o->type == commit_type)
  44                rev_list_push((struct commit *)o, SEEN);
  45
  46        return 0;
  47}
  48
  49/*
  50   This function marks a rev and its ancestors as common.
  51   In some cases, it is desirable to mark only the ancestors (for example
  52   when only the server does not yet know that they are common).
  53*/
  54
  55static void mark_common(struct commit *commit,
  56                int ancestors_only, int dont_parse)
  57{
  58        if (commit != NULL && !(commit->object.flags & COMMON)) {
  59                struct object *o = (struct object *)commit;
  60
  61                if (!ancestors_only)
  62                        o->flags |= COMMON;
  63
  64                if (!(o->flags & SEEN))
  65                        rev_list_push(commit, SEEN);
  66                else {
  67                        struct commit_list *parents;
  68
  69                        if (!ancestors_only && !(o->flags & POPPED))
  70                                non_common_revs--;
  71                        if (!o->parsed && !dont_parse)
  72                                parse_commit(commit);
  73
  74                        for (parents = commit->parents;
  75                                        parents;
  76                                        parents = parents->next)
  77                                mark_common(parents->item, 0, dont_parse);
  78                }
  79        }
  80}
  81
  82/*
  83  Get the next rev to send, ignoring the common.
  84*/
  85
  86static const unsigned char* get_rev(void)
  87{
  88        struct commit *commit = NULL;
  89
  90        while (commit == NULL) {
  91                unsigned int mark;
  92                struct commit_list* parents;
  93
  94                if (rev_list == NULL || non_common_revs == 0)
  95                        return NULL;
  96
  97                commit = rev_list->item;
  98                if (!(commit->object.parsed))
  99                        parse_commit(commit);
 100                commit->object.flags |= POPPED;
 101                if (!(commit->object.flags & COMMON))
 102                        non_common_revs--;
 103        
 104                parents = commit->parents;
 105
 106                if (commit->object.flags & COMMON) {
 107                        /* do not send "have", and ignore ancestors */
 108                        commit = NULL;
 109                        mark = COMMON | SEEN;
 110                } else if (commit->object.flags & COMMON_REF)
 111                        /* send "have", and ignore ancestors */
 112                        mark = COMMON | SEEN;
 113                else
 114                        /* send "have", also for its ancestors */
 115                        mark = SEEN;
 116
 117                while (parents) {
 118                        if (!(parents->item->object.flags & SEEN))
 119                                rev_list_push(parents->item, mark);
 120                        if (mark & COMMON)
 121                                mark_common(parents->item, 1, 0);
 122                        parents = parents->next;
 123                }
 124
 125                rev_list = rev_list->next;
 126        }
 127
 128        return commit->object.sha1;
 129}
 130
 131static int find_common(int fd[2], unsigned char *result_sha1,
 132                       struct ref *refs)
 133{
 134        int fetching;
 135        int count = 0, flushes = 0, retval;
 136        const unsigned char *sha1;
 137
 138        for_each_ref(rev_list_insert_ref);
 139
 140        fetching = 0;
 141        for ( ; refs ; refs = refs->next) {
 142                unsigned char *remote = refs->old_sha1;
 143                struct object *o;
 144
 145                /*
 146                 * If that object is complete (i.e. it is an ancestor of a
 147                 * local ref), we tell them we have it but do not have to
 148                 * tell them about its ancestors, which they already know
 149                 * about.
 150                 *
 151                 * We use lookup_object here because we are only
 152                 * interested in the case we *know* the object is
 153                 * reachable and we have already scanned it.
 154                 */
 155                if (((o = lookup_object(remote)) != NULL) &&
 156                                (o->flags & COMPLETE)) {
 157                        continue;
 158                }
 159
 160                packet_write(fd[1], "want %s%s%s\n", sha1_to_hex(remote),
 161                             (multi_ack ? " multi_ack" : ""),
 162                             (use_thin_pack ? " thin-pack" : ""));
 163                fetching++;
 164        }
 165        packet_flush(fd[1]);
 166        if (!fetching)
 167                return 1;
 168
 169        flushes = 0;
 170        retval = -1;
 171        while ((sha1 = get_rev())) {
 172                packet_write(fd[1], "have %s\n", sha1_to_hex(sha1));
 173                if (verbose)
 174                        fprintf(stderr, "have %s\n", sha1_to_hex(sha1));
 175                if (!(31 & ++count)) {
 176                        int ack;
 177
 178                        packet_flush(fd[1]);
 179                        flushes++;
 180
 181                        /*
 182                         * We keep one window "ahead" of the other side, and
 183                         * will wait for an ACK only on the next one
 184                         */
 185                        if (count == 32)
 186                                continue;
 187
 188                        do {
 189                                ack = get_ack(fd[0], result_sha1);
 190                                if (verbose && ack)
 191                                        fprintf(stderr, "got ack %d %s\n", ack,
 192                                                        sha1_to_hex(result_sha1));
 193                                if (ack == 1) {
 194                                        flushes = 0;
 195                                        multi_ack = 0;
 196                                        retval = 0;
 197                                        goto done;
 198                                } else if (ack == 2) {
 199                                        struct commit *commit =
 200                                                lookup_commit(result_sha1);
 201                                        mark_common(commit, 0, 1);
 202                                        retval = 0;
 203                                }
 204                        } while (ack);
 205                        flushes--;
 206                }
 207        }
 208done:
 209        packet_write(fd[1], "done\n");
 210        if (verbose)
 211                fprintf(stderr, "done\n");
 212        if (retval != 0) {
 213                multi_ack = 0;
 214                flushes++;
 215        }
 216        while (flushes || multi_ack) {
 217                int ack = get_ack(fd[0], result_sha1);
 218                if (ack) {
 219                        if (verbose)
 220                                fprintf(stderr, "got ack (%d) %s\n", ack,
 221                                        sha1_to_hex(result_sha1));
 222                        if (ack == 1)
 223                                return 0;
 224                        multi_ack = 1;
 225                        continue;
 226                }
 227                flushes--;
 228        }
 229        return retval;
 230}
 231
 232static struct commit_list *complete = NULL;
 233
 234static int mark_complete(const char *path, const unsigned char *sha1)
 235{
 236        struct object *o = parse_object(sha1);
 237
 238        while (o && o->type == tag_type) {
 239                struct tag *t = (struct tag *) o;
 240                if (!t->tagged)
 241                        break; /* broken repository */
 242                o->flags |= COMPLETE;
 243                o = parse_object(t->tagged->sha1);
 244        }
 245        if (o && o->type == commit_type) {
 246                struct commit *commit = (struct commit *)o;
 247                commit->object.flags |= COMPLETE;
 248                insert_by_date(commit, &complete);
 249        }
 250        return 0;
 251}
 252
 253static void mark_recent_complete_commits(unsigned long cutoff)
 254{
 255        while (complete && cutoff <= complete->item->date) {
 256                if (verbose)
 257                        fprintf(stderr, "Marking %s as complete\n",
 258                                sha1_to_hex(complete->item->object.sha1));
 259                pop_most_recent_commit(&complete, COMPLETE);
 260        }
 261}
 262
 263static void filter_refs(struct ref **refs, int nr_match, char **match)
 264{
 265        struct ref **return_refs;
 266        struct ref *newlist = NULL;
 267        struct ref **newtail = &newlist;
 268        struct ref *ref, *next;
 269        struct ref *fastarray[32];
 270
 271        if (nr_match && !fetch_all) {
 272                if (ARRAY_SIZE(fastarray) < nr_match)
 273                        return_refs = xcalloc(nr_match, sizeof(struct ref *));
 274                else {
 275                        return_refs = fastarray;
 276                        memset(return_refs, 0, sizeof(struct ref *) * nr_match);
 277                }
 278        }
 279        else
 280                return_refs = NULL;
 281
 282        for (ref = *refs; ref; ref = next) {
 283                next = ref->next;
 284                if (!memcmp(ref->name, "refs/", 5) &&
 285                    check_ref_format(ref->name + 5))
 286                        ; /* trash */
 287                else if (fetch_all) {
 288                        *newtail = ref;
 289                        ref->next = NULL;
 290                        newtail = &ref->next;
 291                        continue;
 292                }
 293                else {
 294                        int order = path_match(ref->name, nr_match, match);
 295                        if (order) {
 296                                return_refs[order-1] = ref;
 297                                continue; /* we will link it later */
 298                        }
 299                }
 300                free(ref);
 301        }
 302
 303        if (!fetch_all) {
 304                int i;
 305                for (i = 0; i < nr_match; i++) {
 306                        ref = return_refs[i];
 307                        if (ref) {
 308                                *newtail = ref;
 309                                ref->next = NULL;
 310                                newtail = &ref->next;
 311                        }
 312                }
 313                if (return_refs != fastarray)
 314                        free(return_refs);
 315        }
 316        *refs = newlist;
 317}
 318
 319static int everything_local(struct ref **refs, int nr_match, char **match)
 320{
 321        struct ref *ref;
 322        int retval;
 323        unsigned long cutoff = 0;
 324
 325        track_object_refs = 0;
 326        save_commit_buffer = 0;
 327
 328        for (ref = *refs; ref; ref = ref->next) {
 329                struct object *o;
 330
 331                o = parse_object(ref->old_sha1);
 332                if (!o)
 333                        continue;
 334
 335                /* We already have it -- which may mean that we were
 336                 * in sync with the other side at some time after
 337                 * that (it is OK if we guess wrong here).
 338                 */
 339                if (o->type == commit_type) {
 340                        struct commit *commit = (struct commit *)o;
 341                        if (!cutoff || cutoff < commit->date)
 342                                cutoff = commit->date;
 343                }
 344        }
 345
 346        for_each_ref(mark_complete);
 347        if (cutoff)
 348                mark_recent_complete_commits(cutoff);
 349
 350        /*
 351         * Mark all complete remote refs as common refs.
 352         * Don't mark them common yet; the server has to be told so first.
 353         */
 354        for (ref = *refs; ref; ref = ref->next) {
 355                struct object *o = deref_tag(lookup_object(ref->old_sha1),
 356                                             NULL, 0);
 357
 358                if (!o || o->type != commit_type || !(o->flags & COMPLETE))
 359                        continue;
 360
 361                if (!(o->flags & SEEN)) {
 362                        rev_list_push((struct commit *)o, COMMON_REF | SEEN);
 363
 364                        mark_common((struct commit *)o, 1, 1);
 365                }
 366        }
 367
 368        filter_refs(refs, nr_match, match);
 369
 370        for (retval = 1, ref = *refs; ref ; ref = ref->next) {
 371                const unsigned char *remote = ref->old_sha1;
 372                unsigned char local[20];
 373                struct object *o;
 374
 375                o = lookup_object(remote);
 376                if (!o || !(o->flags & COMPLETE)) {
 377                        retval = 0;
 378                        if (!verbose)
 379                                continue;
 380                        fprintf(stderr,
 381                                "want %s (%s)\n", sha1_to_hex(remote),
 382                                ref->name);
 383                        continue;
 384                }
 385
 386                memcpy(ref->new_sha1, local, 20);
 387                if (!verbose)
 388                        continue;
 389                fprintf(stderr,
 390                        "already have %s (%s)\n", sha1_to_hex(remote),
 391                        ref->name);
 392        }
 393        return retval;
 394}
 395
 396static int fetch_pack(int fd[2], int nr_match, char **match)
 397{
 398        struct ref *ref;
 399        unsigned char sha1[20];
 400        int status;
 401
 402        get_remote_heads(fd[0], &ref, 0, NULL, 0);
 403        if (server_supports("multi_ack")) {
 404                if (verbose)
 405                        fprintf(stderr, "Server supports multi_ack\n");
 406                multi_ack = 1;
 407        }
 408        if (!ref) {
 409                packet_flush(fd[1]);
 410                die("no matching remote head");
 411        }
 412        if (everything_local(&ref, nr_match, match)) {
 413                packet_flush(fd[1]);
 414                goto all_done;
 415        }
 416        if (find_common(fd, sha1, ref) < 0)
 417                if (!keep_pack)
 418                        /* When cloning, it is not unusual to have
 419                         * no common commit.
 420                         */
 421                        fprintf(stderr, "warning: no common commits\n");
 422
 423        if (keep_pack)
 424                status = receive_keep_pack(fd, "git-fetch-pack", quiet);
 425        else
 426                status = receive_unpack_pack(fd, "git-fetch-pack", quiet);
 427
 428        if (status)
 429                die("git-fetch-pack: fetch failed.");
 430
 431 all_done:
 432        while (ref) {
 433                printf("%s %s\n",
 434                       sha1_to_hex(ref->old_sha1), ref->name);
 435                ref = ref->next;
 436        }
 437        return 0;
 438}
 439
 440int main(int argc, char **argv)
 441{
 442        int i, ret, nr_heads;
 443        char *dest = NULL, **heads;
 444        int fd[2];
 445        pid_t pid;
 446
 447        setup_git_directory();
 448
 449        nr_heads = 0;
 450        heads = NULL;
 451        for (i = 1; i < argc; i++) {
 452                char *arg = argv[i];
 453
 454                if (*arg == '-') {
 455                        if (!strncmp("--exec=", arg, 7)) {
 456                                exec = arg + 7;
 457                                continue;
 458                        }
 459                        if (!strcmp("--quiet", arg) || !strcmp("-q", arg)) {
 460                                quiet = 1;
 461                                continue;
 462                        }
 463                        if (!strcmp("--keep", arg) || !strcmp("-k", arg)) {
 464                                keep_pack = 1;
 465                                continue;
 466                        }
 467                        if (!strcmp("--thin", arg)) {
 468                                use_thin_pack = 1;
 469                                continue;
 470                        }
 471                        if (!strcmp("--all", arg)) {
 472                                fetch_all = 1;
 473                                continue;
 474                        }
 475                        if (!strcmp("-v", arg)) {
 476                                verbose = 1;
 477                                continue;
 478                        }
 479                        usage(fetch_pack_usage);
 480                }
 481                dest = arg;
 482                heads = argv + i + 1;
 483                nr_heads = argc - i - 1;
 484                break;
 485        }
 486        if (!dest)
 487                usage(fetch_pack_usage);
 488        if (keep_pack)
 489                use_thin_pack = 0;
 490        pid = git_connect(fd, dest, exec);
 491        if (pid < 0)
 492                return 1;
 493        ret = fetch_pack(fd, nr_heads, heads);
 494        close(fd[0]);
 495        close(fd[1]);
 496        finish_connect(pid);
 497
 498        if (!ret && nr_heads) {
 499                /* If the heads to pull were given, we should have
 500                 * consumed all of them by matching the remote.
 501                 * Otherwise, 'git-fetch remote no-such-ref' would
 502                 * silently succeed without issuing an error.
 503                 */
 504                for (i = 0; i < nr_heads; i++)
 505                        if (heads[i] && heads[i][0]) {
 506                                error("no such remote ref %s", heads[i]);
 507                                ret = 1;
 508                        }
 509        }
 510
 511        return ret;
 512}