4fb464c1ef553528021b981132919551517412ac
   1/*
   2(See Documentation/git-fast-import.txt for maintained documentation.)
   3Format of STDIN stream:
   4
   5  stream ::= cmd*;
   6
   7  cmd ::= new_blob
   8        | new_commit
   9        | new_tag
  10        | reset_branch
  11        | checkpoint
  12        | progress
  13        ;
  14
  15  new_blob ::= 'blob' lf
  16    mark?
  17    file_content;
  18  file_content ::= data;
  19
  20  new_commit ::= 'commit' sp ref_str lf
  21    mark?
  22    ('author' (sp name)? sp '<' email '>' sp when lf)?
  23    'committer' (sp name)? sp '<' email '>' sp when lf
  24    commit_msg
  25    ('from' sp commit-ish lf)?
  26    ('merge' sp commit-ish lf)*
  27    (file_change | ls)*
  28    lf?;
  29  commit_msg ::= data;
  30
  31  ls ::= 'ls' sp '"' quoted(path) '"' lf;
  32
  33  file_change ::= file_clr
  34    | file_del
  35    | file_rnm
  36    | file_cpy
  37    | file_obm
  38    | file_inm;
  39  file_clr ::= 'deleteall' lf;
  40  file_del ::= 'D' sp path_str lf;
  41  file_rnm ::= 'R' sp path_str sp path_str lf;
  42  file_cpy ::= 'C' sp path_str sp path_str lf;
  43  file_obm ::= 'M' sp mode sp (hexsha1 | idnum) sp path_str lf;
  44  file_inm ::= 'M' sp mode sp 'inline' sp path_str lf
  45    data;
  46  note_obm ::= 'N' sp (hexsha1 | idnum) sp commit-ish lf;
  47  note_inm ::= 'N' sp 'inline' sp commit-ish lf
  48    data;
  49
  50  new_tag ::= 'tag' sp tag_str lf
  51    'from' sp commit-ish lf
  52    ('tagger' (sp name)? sp '<' email '>' sp when lf)?
  53    tag_msg;
  54  tag_msg ::= data;
  55
  56  reset_branch ::= 'reset' sp ref_str lf
  57    ('from' sp commit-ish lf)?
  58    lf?;
  59
  60  checkpoint ::= 'checkpoint' lf
  61    lf?;
  62
  63  progress ::= 'progress' sp not_lf* lf
  64    lf?;
  65
  66     # note: the first idnum in a stream should be 1 and subsequent
  67     # idnums should not have gaps between values as this will cause
  68     # the stream parser to reserve space for the gapped values.  An
  69     # idnum can be updated in the future to a new object by issuing
  70     # a new mark directive with the old idnum.
  71     #
  72  mark ::= 'mark' sp idnum lf;
  73  data ::= (delimited_data | exact_data)
  74    lf?;
  75
  76    # note: delim may be any string but must not contain lf.
  77    # data_line may contain any data but must not be exactly
  78    # delim.
  79  delimited_data ::= 'data' sp '<<' delim lf
  80    (data_line lf)*
  81    delim lf;
  82
  83     # note: declen indicates the length of binary_data in bytes.
  84     # declen does not include the lf preceding the binary data.
  85     #
  86  exact_data ::= 'data' sp declen lf
  87    binary_data;
  88
  89     # note: quoted strings are C-style quoting supporting \c for
  90     # common escapes of 'c' (e..g \n, \t, \\, \") or \nnn where nnn
  91     # is the signed byte value in octal.  Note that the only
  92     # characters which must actually be escaped to protect the
  93     # stream formatting is: \, " and LF.  Otherwise these values
  94     # are UTF8.
  95     #
  96  commit-ish  ::= (ref_str | hexsha1 | sha1exp_str | idnum);
  97  ref_str     ::= ref;
  98  sha1exp_str ::= sha1exp;
  99  tag_str     ::= tag;
 100  path_str    ::= path    | '"' quoted(path)    '"' ;
 101  mode        ::= '100644' | '644'
 102                | '100755' | '755'
 103                | '120000'
 104                ;
 105
 106  declen ::= # unsigned 32 bit value, ascii base10 notation;
 107  bigint ::= # unsigned integer value, ascii base10 notation;
 108  binary_data ::= # file content, not interpreted;
 109
 110  when         ::= raw_when | rfc2822_when;
 111  raw_when     ::= ts sp tz;
 112  rfc2822_when ::= # Valid RFC 2822 date and time;
 113
 114  sp ::= # ASCII space character;
 115  lf ::= # ASCII newline (LF) character;
 116
 117     # note: a colon (':') must precede the numerical value assigned to
 118     # an idnum.  This is to distinguish it from a ref or tag name as
 119     # GIT does not permit ':' in ref or tag strings.
 120     #
 121  idnum   ::= ':' bigint;
 122  path    ::= # GIT style file path, e.g. "a/b/c";
 123  ref     ::= # GIT ref name, e.g. "refs/heads/MOZ_GECKO_EXPERIMENT";
 124  tag     ::= # GIT tag name, e.g. "FIREFOX_1_5";
 125  sha1exp ::= # Any valid GIT SHA1 expression;
 126  hexsha1 ::= # SHA1 in hexadecimal format;
 127
 128     # note: name and email are UTF8 strings, however name must not
 129     # contain '<' or lf and email must not contain any of the
 130     # following: '<', '>', lf.
 131     #
 132  name  ::= # valid GIT author/committer name;
 133  email ::= # valid GIT author/committer email;
 134  ts    ::= # time since the epoch in seconds, ascii base10 notation;
 135  tz    ::= # GIT style timezone;
 136
 137     # note: comments, get-mark, ls-tree, and cat-blob requests may
 138     # appear anywhere in the input, except within a data command. Any
 139     # form of the data command always escapes the related input from
 140     # comment processing.
 141     #
 142     # In case it is not clear, the '#' that starts the comment
 143     # must be the first character on that line (an lf
 144     # preceded it).
 145     #
 146
 147  get_mark ::= 'get-mark' sp idnum lf;
 148  cat_blob ::= 'cat-blob' sp (hexsha1 | idnum) lf;
 149  ls_tree  ::= 'ls' sp (hexsha1 | idnum) sp path_str lf;
 150
 151  comment ::= '#' not_lf* lf;
 152  not_lf  ::= # Any byte that is not ASCII newline (LF);
 153*/
 154
 155#include "builtin.h"
 156#include "cache.h"
 157#include "lockfile.h"
 158#include "object.h"
 159#include "blob.h"
 160#include "tree.h"
 161#include "commit.h"
 162#include "delta.h"
 163#include "pack.h"
 164#include "refs.h"
 165#include "csum-file.h"
 166#include "quote.h"
 167#include "exec_cmd.h"
 168#include "dir.h"
 169#include "run-command.h"
 170
 171#define PACK_ID_BITS 16
 172#define MAX_PACK_ID ((1<<PACK_ID_BITS)-1)
 173#define DEPTH_BITS 13
 174#define MAX_DEPTH ((1<<DEPTH_BITS)-1)
 175
 176/*
 177 * We abuse the setuid bit on directories to mean "do not delta".
 178 */
 179#define NO_DELTA S_ISUID
 180
 181struct object_entry {
 182        struct pack_idx_entry idx;
 183        struct object_entry *next;
 184        uint32_t type : TYPE_BITS,
 185                pack_id : PACK_ID_BITS,
 186                depth : DEPTH_BITS;
 187};
 188
 189struct object_entry_pool {
 190        struct object_entry_pool *next_pool;
 191        struct object_entry *next_free;
 192        struct object_entry *end;
 193        struct object_entry entries[FLEX_ARRAY]; /* more */
 194};
 195
 196struct mark_set {
 197        union {
 198                struct object_entry *marked[1024];
 199                struct mark_set *sets[1024];
 200        } data;
 201        unsigned int shift;
 202};
 203
 204struct last_object {
 205        struct strbuf data;
 206        off_t offset;
 207        unsigned int depth;
 208        unsigned no_swap : 1;
 209};
 210
 211struct mem_pool {
 212        struct mem_pool *next_pool;
 213        char *next_free;
 214        char *end;
 215        uintmax_t space[FLEX_ARRAY]; /* more */
 216};
 217
 218struct atom_str {
 219        struct atom_str *next_atom;
 220        unsigned short str_len;
 221        char str_dat[FLEX_ARRAY]; /* more */
 222};
 223
 224struct tree_content;
 225struct tree_entry {
 226        struct tree_content *tree;
 227        struct atom_str *name;
 228        struct tree_entry_ms {
 229                uint16_t mode;
 230                unsigned char sha1[20];
 231        } versions[2];
 232};
 233
 234struct tree_content {
 235        unsigned int entry_capacity; /* must match avail_tree_content */
 236        unsigned int entry_count;
 237        unsigned int delta_depth;
 238        struct tree_entry *entries[FLEX_ARRAY]; /* more */
 239};
 240
 241struct avail_tree_content {
 242        unsigned int entry_capacity; /* must match tree_content */
 243        struct avail_tree_content *next_avail;
 244};
 245
 246struct branch {
 247        struct branch *table_next_branch;
 248        struct branch *active_next_branch;
 249        const char *name;
 250        struct tree_entry branch_tree;
 251        uintmax_t last_commit;
 252        uintmax_t num_notes;
 253        unsigned active : 1;
 254        unsigned delete : 1;
 255        unsigned pack_id : PACK_ID_BITS;
 256        unsigned char sha1[20];
 257};
 258
 259struct tag {
 260        struct tag *next_tag;
 261        const char *name;
 262        unsigned int pack_id;
 263        unsigned char sha1[20];
 264};
 265
 266struct hash_list {
 267        struct hash_list *next;
 268        unsigned char sha1[20];
 269};
 270
 271typedef enum {
 272        WHENSPEC_RAW = 1,
 273        WHENSPEC_RFC2822,
 274        WHENSPEC_NOW
 275} whenspec_type;
 276
 277struct recent_command {
 278        struct recent_command *prev;
 279        struct recent_command *next;
 280        char *buf;
 281};
 282
 283/* Configured limits on output */
 284static unsigned long max_depth = 10;
 285static off_t max_packsize;
 286static int unpack_limit = 100;
 287static int force_update;
 288static int pack_compression_level = Z_DEFAULT_COMPRESSION;
 289static int pack_compression_seen;
 290
 291/* Stats and misc. counters */
 292static uintmax_t alloc_count;
 293static uintmax_t marks_set_count;
 294static uintmax_t object_count_by_type[1 << TYPE_BITS];
 295static uintmax_t duplicate_count_by_type[1 << TYPE_BITS];
 296static uintmax_t delta_count_by_type[1 << TYPE_BITS];
 297static uintmax_t delta_count_attempts_by_type[1 << TYPE_BITS];
 298static unsigned long object_count;
 299static unsigned long branch_count;
 300static unsigned long branch_load_count;
 301static int failure;
 302static FILE *pack_edges;
 303static unsigned int show_stats = 1;
 304static int global_argc;
 305static char **global_argv;
 306
 307/* Memory pools */
 308static size_t mem_pool_alloc = 2*1024*1024 - sizeof(struct mem_pool);
 309static size_t total_allocd;
 310static struct mem_pool *mem_pool;
 311
 312/* Atom management */
 313static unsigned int atom_table_sz = 4451;
 314static unsigned int atom_cnt;
 315static struct atom_str **atom_table;
 316
 317/* The .pack file being generated */
 318static struct pack_idx_option pack_idx_opts;
 319static unsigned int pack_id;
 320static struct sha1file *pack_file;
 321static struct packed_git *pack_data;
 322static struct packed_git **all_packs;
 323static off_t pack_size;
 324
 325/* Table of objects we've written. */
 326static unsigned int object_entry_alloc = 5000;
 327static struct object_entry_pool *blocks;
 328static struct object_entry *object_table[1 << 16];
 329static struct mark_set *marks;
 330static const char *export_marks_file;
 331static const char *import_marks_file;
 332static int import_marks_file_from_stream;
 333static int import_marks_file_ignore_missing;
 334static int relative_marks_paths;
 335
 336/* Our last blob */
 337static struct last_object last_blob = { STRBUF_INIT, 0, 0, 0 };
 338
 339/* Tree management */
 340static unsigned int tree_entry_alloc = 1000;
 341static void *avail_tree_entry;
 342static unsigned int avail_tree_table_sz = 100;
 343static struct avail_tree_content **avail_tree_table;
 344static struct strbuf old_tree = STRBUF_INIT;
 345static struct strbuf new_tree = STRBUF_INIT;
 346
 347/* Branch data */
 348static unsigned long max_active_branches = 5;
 349static unsigned long cur_active_branches;
 350static unsigned long branch_table_sz = 1039;
 351static struct branch **branch_table;
 352static struct branch *active_branches;
 353
 354/* Tag data */
 355static struct tag *first_tag;
 356static struct tag *last_tag;
 357
 358/* Input stream parsing */
 359static whenspec_type whenspec = WHENSPEC_RAW;
 360static struct strbuf command_buf = STRBUF_INIT;
 361static int unread_command_buf;
 362static struct recent_command cmd_hist = {&cmd_hist, &cmd_hist, NULL};
 363static struct recent_command *cmd_tail = &cmd_hist;
 364static struct recent_command *rc_free;
 365static unsigned int cmd_save = 100;
 366static uintmax_t next_mark;
 367static struct strbuf new_data = STRBUF_INIT;
 368static int seen_data_command;
 369static int require_explicit_termination;
 370
 371/* Signal handling */
 372static volatile sig_atomic_t checkpoint_requested;
 373
 374/* Where to write output of cat-blob commands */
 375static int cat_blob_fd = STDOUT_FILENO;
 376
 377static void parse_argv(void);
 378static void parse_get_mark(const char *p);
 379static void parse_cat_blob(const char *p);
 380static void parse_ls(const char *p, struct branch *b);
 381
 382static void write_branch_report(FILE *rpt, struct branch *b)
 383{
 384        fprintf(rpt, "%s:\n", b->name);
 385
 386        fprintf(rpt, "  status      :");
 387        if (b->active)
 388                fputs(" active", rpt);
 389        if (b->branch_tree.tree)
 390                fputs(" loaded", rpt);
 391        if (is_null_sha1(b->branch_tree.versions[1].sha1))
 392                fputs(" dirty", rpt);
 393        fputc('\n', rpt);
 394
 395        fprintf(rpt, "  tip commit  : %s\n", sha1_to_hex(b->sha1));
 396        fprintf(rpt, "  old tree    : %s\n", sha1_to_hex(b->branch_tree.versions[0].sha1));
 397        fprintf(rpt, "  cur tree    : %s\n", sha1_to_hex(b->branch_tree.versions[1].sha1));
 398        fprintf(rpt, "  commit clock: %" PRIuMAX "\n", b->last_commit);
 399
 400        fputs("  last pack   : ", rpt);
 401        if (b->pack_id < MAX_PACK_ID)
 402                fprintf(rpt, "%u", b->pack_id);
 403        fputc('\n', rpt);
 404
 405        fputc('\n', rpt);
 406}
 407
 408static void dump_marks_helper(FILE *, uintmax_t, struct mark_set *);
 409
 410static void write_crash_report(const char *err)
 411{
 412        char *loc = git_pathdup("fast_import_crash_%"PRIuMAX, (uintmax_t) getpid());
 413        FILE *rpt = fopen(loc, "w");
 414        struct branch *b;
 415        unsigned long lu;
 416        struct recent_command *rc;
 417
 418        if (!rpt) {
 419                error("can't write crash report %s: %s", loc, strerror(errno));
 420                free(loc);
 421                return;
 422        }
 423
 424        fprintf(stderr, "fast-import: dumping crash report to %s\n", loc);
 425
 426        fprintf(rpt, "fast-import crash report:\n");
 427        fprintf(rpt, "    fast-import process: %"PRIuMAX"\n", (uintmax_t) getpid());
 428        fprintf(rpt, "    parent process     : %"PRIuMAX"\n", (uintmax_t) getppid());
 429        fprintf(rpt, "    at %s\n", show_date(time(NULL), 0, DATE_MODE(ISO8601)));
 430        fputc('\n', rpt);
 431
 432        fputs("fatal: ", rpt);
 433        fputs(err, rpt);
 434        fputc('\n', rpt);
 435
 436        fputc('\n', rpt);
 437        fputs("Most Recent Commands Before Crash\n", rpt);
 438        fputs("---------------------------------\n", rpt);
 439        for (rc = cmd_hist.next; rc != &cmd_hist; rc = rc->next) {
 440                if (rc->next == &cmd_hist)
 441                        fputs("* ", rpt);
 442                else
 443                        fputs("  ", rpt);
 444                fputs(rc->buf, rpt);
 445                fputc('\n', rpt);
 446        }
 447
 448        fputc('\n', rpt);
 449        fputs("Active Branch LRU\n", rpt);
 450        fputs("-----------------\n", rpt);
 451        fprintf(rpt, "    active_branches = %lu cur, %lu max\n",
 452                cur_active_branches,
 453                max_active_branches);
 454        fputc('\n', rpt);
 455        fputs("  pos  clock name\n", rpt);
 456        fputs("  ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n", rpt);
 457        for (b = active_branches, lu = 0; b; b = b->active_next_branch)
 458                fprintf(rpt, "  %2lu) %6" PRIuMAX" %s\n",
 459                        ++lu, b->last_commit, b->name);
 460
 461        fputc('\n', rpt);
 462        fputs("Inactive Branches\n", rpt);
 463        fputs("-----------------\n", rpt);
 464        for (lu = 0; lu < branch_table_sz; lu++) {
 465                for (b = branch_table[lu]; b; b = b->table_next_branch)
 466                        write_branch_report(rpt, b);
 467        }
 468
 469        if (first_tag) {
 470                struct tag *tg;
 471                fputc('\n', rpt);
 472                fputs("Annotated Tags\n", rpt);
 473                fputs("--------------\n", rpt);
 474                for (tg = first_tag; tg; tg = tg->next_tag) {
 475                        fputs(sha1_to_hex(tg->sha1), rpt);
 476                        fputc(' ', rpt);
 477                        fputs(tg->name, rpt);
 478                        fputc('\n', rpt);
 479                }
 480        }
 481
 482        fputc('\n', rpt);
 483        fputs("Marks\n", rpt);
 484        fputs("-----\n", rpt);
 485        if (export_marks_file)
 486                fprintf(rpt, "  exported to %s\n", export_marks_file);
 487        else
 488                dump_marks_helper(rpt, 0, marks);
 489
 490        fputc('\n', rpt);
 491        fputs("-------------------\n", rpt);
 492        fputs("END OF CRASH REPORT\n", rpt);
 493        fclose(rpt);
 494        free(loc);
 495}
 496
 497static void end_packfile(void);
 498static void unkeep_all_packs(void);
 499static void dump_marks(void);
 500
 501static NORETURN void die_nicely(const char *err, va_list params)
 502{
 503        static int zombie;
 504        char message[2 * PATH_MAX];
 505
 506        vsnprintf(message, sizeof(message), err, params);
 507        fputs("fatal: ", stderr);
 508        fputs(message, stderr);
 509        fputc('\n', stderr);
 510
 511        if (!zombie) {
 512                zombie = 1;
 513                write_crash_report(message);
 514                end_packfile();
 515                unkeep_all_packs();
 516                dump_marks();
 517        }
 518        exit(128);
 519}
 520
 521#ifndef SIGUSR1 /* Windows, for example */
 522
 523static void set_checkpoint_signal(void)
 524{
 525}
 526
 527#else
 528
 529static void checkpoint_signal(int signo)
 530{
 531        checkpoint_requested = 1;
 532}
 533
 534static void set_checkpoint_signal(void)
 535{
 536        struct sigaction sa;
 537
 538        memset(&sa, 0, sizeof(sa));
 539        sa.sa_handler = checkpoint_signal;
 540        sigemptyset(&sa.sa_mask);
 541        sa.sa_flags = SA_RESTART;
 542        sigaction(SIGUSR1, &sa, NULL);
 543}
 544
 545#endif
 546
 547static void alloc_objects(unsigned int cnt)
 548{
 549        struct object_entry_pool *b;
 550
 551        b = xmalloc(sizeof(struct object_entry_pool)
 552                + cnt * sizeof(struct object_entry));
 553        b->next_pool = blocks;
 554        b->next_free = b->entries;
 555        b->end = b->entries + cnt;
 556        blocks = b;
 557        alloc_count += cnt;
 558}
 559
 560static struct object_entry *new_object(unsigned char *sha1)
 561{
 562        struct object_entry *e;
 563
 564        if (blocks->next_free == blocks->end)
 565                alloc_objects(object_entry_alloc);
 566
 567        e = blocks->next_free++;
 568        hashcpy(e->idx.sha1, sha1);
 569        return e;
 570}
 571
 572static struct object_entry *find_object(unsigned char *sha1)
 573{
 574        unsigned int h = sha1[0] << 8 | sha1[1];
 575        struct object_entry *e;
 576        for (e = object_table[h]; e; e = e->next)
 577                if (!hashcmp(sha1, e->idx.sha1))
 578                        return e;
 579        return NULL;
 580}
 581
 582static struct object_entry *insert_object(unsigned char *sha1)
 583{
 584        unsigned int h = sha1[0] << 8 | sha1[1];
 585        struct object_entry *e = object_table[h];
 586
 587        while (e) {
 588                if (!hashcmp(sha1, e->idx.sha1))
 589                        return e;
 590                e = e->next;
 591        }
 592
 593        e = new_object(sha1);
 594        e->next = object_table[h];
 595        e->idx.offset = 0;
 596        object_table[h] = e;
 597        return e;
 598}
 599
 600static unsigned int hc_str(const char *s, size_t len)
 601{
 602        unsigned int r = 0;
 603        while (len-- > 0)
 604                r = r * 31 + *s++;
 605        return r;
 606}
 607
 608static void *pool_alloc(size_t len)
 609{
 610        struct mem_pool *p;
 611        void *r;
 612
 613        /* round up to a 'uintmax_t' alignment */
 614        if (len & (sizeof(uintmax_t) - 1))
 615                len += sizeof(uintmax_t) - (len & (sizeof(uintmax_t) - 1));
 616
 617        for (p = mem_pool; p; p = p->next_pool)
 618                if ((p->end - p->next_free >= len))
 619                        break;
 620
 621        if (!p) {
 622                if (len >= (mem_pool_alloc/2)) {
 623                        total_allocd += len;
 624                        return xmalloc(len);
 625                }
 626                total_allocd += sizeof(struct mem_pool) + mem_pool_alloc;
 627                p = xmalloc(st_add(sizeof(struct mem_pool), mem_pool_alloc));
 628                p->next_pool = mem_pool;
 629                p->next_free = (char *) p->space;
 630                p->end = p->next_free + mem_pool_alloc;
 631                mem_pool = p;
 632        }
 633
 634        r = p->next_free;
 635        p->next_free += len;
 636        return r;
 637}
 638
 639static void *pool_calloc(size_t count, size_t size)
 640{
 641        size_t len = count * size;
 642        void *r = pool_alloc(len);
 643        memset(r, 0, len);
 644        return r;
 645}
 646
 647static char *pool_strdup(const char *s)
 648{
 649        size_t len = strlen(s) + 1;
 650        char *r = pool_alloc(len);
 651        memcpy(r, s, len);
 652        return r;
 653}
 654
 655static void insert_mark(uintmax_t idnum, struct object_entry *oe)
 656{
 657        struct mark_set *s = marks;
 658        while ((idnum >> s->shift) >= 1024) {
 659                s = pool_calloc(1, sizeof(struct mark_set));
 660                s->shift = marks->shift + 10;
 661                s->data.sets[0] = marks;
 662                marks = s;
 663        }
 664        while (s->shift) {
 665                uintmax_t i = idnum >> s->shift;
 666                idnum -= i << s->shift;
 667                if (!s->data.sets[i]) {
 668                        s->data.sets[i] = pool_calloc(1, sizeof(struct mark_set));
 669                        s->data.sets[i]->shift = s->shift - 10;
 670                }
 671                s = s->data.sets[i];
 672        }
 673        if (!s->data.marked[idnum])
 674                marks_set_count++;
 675        s->data.marked[idnum] = oe;
 676}
 677
 678static struct object_entry *find_mark(uintmax_t idnum)
 679{
 680        uintmax_t orig_idnum = idnum;
 681        struct mark_set *s = marks;
 682        struct object_entry *oe = NULL;
 683        if ((idnum >> s->shift) < 1024) {
 684                while (s && s->shift) {
 685                        uintmax_t i = idnum >> s->shift;
 686                        idnum -= i << s->shift;
 687                        s = s->data.sets[i];
 688                }
 689                if (s)
 690                        oe = s->data.marked[idnum];
 691        }
 692        if (!oe)
 693                die("mark :%" PRIuMAX " not declared", orig_idnum);
 694        return oe;
 695}
 696
 697static struct atom_str *to_atom(const char *s, unsigned short len)
 698{
 699        unsigned int hc = hc_str(s, len) % atom_table_sz;
 700        struct atom_str *c;
 701
 702        for (c = atom_table[hc]; c; c = c->next_atom)
 703                if (c->str_len == len && !strncmp(s, c->str_dat, len))
 704                        return c;
 705
 706        c = pool_alloc(sizeof(struct atom_str) + len + 1);
 707        c->str_len = len;
 708        memcpy(c->str_dat, s, len);
 709        c->str_dat[len] = 0;
 710        c->next_atom = atom_table[hc];
 711        atom_table[hc] = c;
 712        atom_cnt++;
 713        return c;
 714}
 715
 716static struct branch *lookup_branch(const char *name)
 717{
 718        unsigned int hc = hc_str(name, strlen(name)) % branch_table_sz;
 719        struct branch *b;
 720
 721        for (b = branch_table[hc]; b; b = b->table_next_branch)
 722                if (!strcmp(name, b->name))
 723                        return b;
 724        return NULL;
 725}
 726
 727static struct branch *new_branch(const char *name)
 728{
 729        unsigned int hc = hc_str(name, strlen(name)) % branch_table_sz;
 730        struct branch *b = lookup_branch(name);
 731
 732        if (b)
 733                die("Invalid attempt to create duplicate branch: %s", name);
 734        if (check_refname_format(name, REFNAME_ALLOW_ONELEVEL))
 735                die("Branch name doesn't conform to GIT standards: %s", name);
 736
 737        b = pool_calloc(1, sizeof(struct branch));
 738        b->name = pool_strdup(name);
 739        b->table_next_branch = branch_table[hc];
 740        b->branch_tree.versions[0].mode = S_IFDIR;
 741        b->branch_tree.versions[1].mode = S_IFDIR;
 742        b->num_notes = 0;
 743        b->active = 0;
 744        b->pack_id = MAX_PACK_ID;
 745        branch_table[hc] = b;
 746        branch_count++;
 747        return b;
 748}
 749
 750static unsigned int hc_entries(unsigned int cnt)
 751{
 752        cnt = cnt & 7 ? (cnt / 8) + 1 : cnt / 8;
 753        return cnt < avail_tree_table_sz ? cnt : avail_tree_table_sz - 1;
 754}
 755
 756static struct tree_content *new_tree_content(unsigned int cnt)
 757{
 758        struct avail_tree_content *f, *l = NULL;
 759        struct tree_content *t;
 760        unsigned int hc = hc_entries(cnt);
 761
 762        for (f = avail_tree_table[hc]; f; l = f, f = f->next_avail)
 763                if (f->entry_capacity >= cnt)
 764                        break;
 765
 766        if (f) {
 767                if (l)
 768                        l->next_avail = f->next_avail;
 769                else
 770                        avail_tree_table[hc] = f->next_avail;
 771        } else {
 772                cnt = cnt & 7 ? ((cnt / 8) + 1) * 8 : cnt;
 773                f = pool_alloc(sizeof(*t) + sizeof(t->entries[0]) * cnt);
 774                f->entry_capacity = cnt;
 775        }
 776
 777        t = (struct tree_content*)f;
 778        t->entry_count = 0;
 779        t->delta_depth = 0;
 780        return t;
 781}
 782
 783static void release_tree_entry(struct tree_entry *e);
 784static void release_tree_content(struct tree_content *t)
 785{
 786        struct avail_tree_content *f = (struct avail_tree_content*)t;
 787        unsigned int hc = hc_entries(f->entry_capacity);
 788        f->next_avail = avail_tree_table[hc];
 789        avail_tree_table[hc] = f;
 790}
 791
 792static void release_tree_content_recursive(struct tree_content *t)
 793{
 794        unsigned int i;
 795        for (i = 0; i < t->entry_count; i++)
 796                release_tree_entry(t->entries[i]);
 797        release_tree_content(t);
 798}
 799
 800static struct tree_content *grow_tree_content(
 801        struct tree_content *t,
 802        int amt)
 803{
 804        struct tree_content *r = new_tree_content(t->entry_count + amt);
 805        r->entry_count = t->entry_count;
 806        r->delta_depth = t->delta_depth;
 807        memcpy(r->entries,t->entries,t->entry_count*sizeof(t->entries[0]));
 808        release_tree_content(t);
 809        return r;
 810}
 811
 812static struct tree_entry *new_tree_entry(void)
 813{
 814        struct tree_entry *e;
 815
 816        if (!avail_tree_entry) {
 817                unsigned int n = tree_entry_alloc;
 818                total_allocd += n * sizeof(struct tree_entry);
 819                ALLOC_ARRAY(e, n);
 820                avail_tree_entry = e;
 821                while (n-- > 1) {
 822                        *((void**)e) = e + 1;
 823                        e++;
 824                }
 825                *((void**)e) = NULL;
 826        }
 827
 828        e = avail_tree_entry;
 829        avail_tree_entry = *((void**)e);
 830        return e;
 831}
 832
 833static void release_tree_entry(struct tree_entry *e)
 834{
 835        if (e->tree)
 836                release_tree_content_recursive(e->tree);
 837        *((void**)e) = avail_tree_entry;
 838        avail_tree_entry = e;
 839}
 840
 841static struct tree_content *dup_tree_content(struct tree_content *s)
 842{
 843        struct tree_content *d;
 844        struct tree_entry *a, *b;
 845        unsigned int i;
 846
 847        if (!s)
 848                return NULL;
 849        d = new_tree_content(s->entry_count);
 850        for (i = 0; i < s->entry_count; i++) {
 851                a = s->entries[i];
 852                b = new_tree_entry();
 853                memcpy(b, a, sizeof(*a));
 854                if (a->tree && is_null_sha1(b->versions[1].sha1))
 855                        b->tree = dup_tree_content(a->tree);
 856                else
 857                        b->tree = NULL;
 858                d->entries[i] = b;
 859        }
 860        d->entry_count = s->entry_count;
 861        d->delta_depth = s->delta_depth;
 862
 863        return d;
 864}
 865
 866static void start_packfile(void)
 867{
 868        static char tmp_file[PATH_MAX];
 869        struct packed_git *p;
 870        struct pack_header hdr;
 871        int pack_fd;
 872
 873        pack_fd = odb_mkstemp(tmp_file, sizeof(tmp_file),
 874                              "pack/tmp_pack_XXXXXX");
 875        FLEX_ALLOC_STR(p, pack_name, tmp_file);
 876        p->pack_fd = pack_fd;
 877        p->do_not_close = 1;
 878        pack_file = sha1fd(pack_fd, p->pack_name);
 879
 880        hdr.hdr_signature = htonl(PACK_SIGNATURE);
 881        hdr.hdr_version = htonl(2);
 882        hdr.hdr_entries = 0;
 883        sha1write(pack_file, &hdr, sizeof(hdr));
 884
 885        pack_data = p;
 886        pack_size = sizeof(hdr);
 887        object_count = 0;
 888
 889        REALLOC_ARRAY(all_packs, pack_id + 1);
 890        all_packs[pack_id] = p;
 891}
 892
 893static const char *create_index(void)
 894{
 895        const char *tmpfile;
 896        struct pack_idx_entry **idx, **c, **last;
 897        struct object_entry *e;
 898        struct object_entry_pool *o;
 899
 900        /* Build the table of object IDs. */
 901        ALLOC_ARRAY(idx, object_count);
 902        c = idx;
 903        for (o = blocks; o; o = o->next_pool)
 904                for (e = o->next_free; e-- != o->entries;)
 905                        if (pack_id == e->pack_id)
 906                                *c++ = &e->idx;
 907        last = idx + object_count;
 908        if (c != last)
 909                die("internal consistency error creating the index");
 910
 911        tmpfile = write_idx_file(NULL, idx, object_count, &pack_idx_opts, pack_data->sha1);
 912        free(idx);
 913        return tmpfile;
 914}
 915
 916static char *keep_pack(const char *curr_index_name)
 917{
 918        static char name[PATH_MAX];
 919        static const char *keep_msg = "fast-import";
 920        int keep_fd;
 921
 922        keep_fd = odb_pack_keep(name, sizeof(name), pack_data->sha1);
 923        if (keep_fd < 0)
 924                die_errno("cannot create keep file");
 925        write_or_die(keep_fd, keep_msg, strlen(keep_msg));
 926        if (close(keep_fd))
 927                die_errno("failed to write keep file");
 928
 929        snprintf(name, sizeof(name), "%s/pack/pack-%s.pack",
 930                 get_object_directory(), sha1_to_hex(pack_data->sha1));
 931        if (finalize_object_file(pack_data->pack_name, name))
 932                die("cannot store pack file");
 933
 934        snprintf(name, sizeof(name), "%s/pack/pack-%s.idx",
 935                 get_object_directory(), sha1_to_hex(pack_data->sha1));
 936        if (finalize_object_file(curr_index_name, name))
 937                die("cannot store index file");
 938        free((void *)curr_index_name);
 939        return name;
 940}
 941
 942static void unkeep_all_packs(void)
 943{
 944        static char name[PATH_MAX];
 945        int k;
 946
 947        for (k = 0; k < pack_id; k++) {
 948                struct packed_git *p = all_packs[k];
 949                snprintf(name, sizeof(name), "%s/pack/pack-%s.keep",
 950                         get_object_directory(), sha1_to_hex(p->sha1));
 951                unlink_or_warn(name);
 952        }
 953}
 954
 955static int loosen_small_pack(const struct packed_git *p)
 956{
 957        struct child_process unpack = CHILD_PROCESS_INIT;
 958
 959        if (lseek(p->pack_fd, 0, SEEK_SET) < 0)
 960                die_errno("Failed seeking to start of '%s'", p->pack_name);
 961
 962        unpack.in = p->pack_fd;
 963        unpack.git_cmd = 1;
 964        unpack.stdout_to_stderr = 1;
 965        argv_array_push(&unpack.args, "unpack-objects");
 966        if (!show_stats)
 967                argv_array_push(&unpack.args, "-q");
 968
 969        return run_command(&unpack);
 970}
 971
 972static void end_packfile(void)
 973{
 974        static int running;
 975
 976        if (running || !pack_data)
 977                return;
 978
 979        running = 1;
 980        clear_delta_base_cache();
 981        if (object_count) {
 982                struct packed_git *new_p;
 983                unsigned char cur_pack_sha1[20];
 984                char *idx_name;
 985                int i;
 986                struct branch *b;
 987                struct tag *t;
 988
 989                close_pack_windows(pack_data);
 990                sha1close(pack_file, cur_pack_sha1, 0);
 991                fixup_pack_header_footer(pack_data->pack_fd, pack_data->sha1,
 992                                    pack_data->pack_name, object_count,
 993                                    cur_pack_sha1, pack_size);
 994
 995                if (object_count <= unpack_limit) {
 996                        if (!loosen_small_pack(pack_data))
 997                                goto discard_pack;
 998                }
 999
1000                close(pack_data->pack_fd);
1001                idx_name = keep_pack(create_index());
1002
1003                /* Register the packfile with core git's machinery. */
1004                new_p = add_packed_git(idx_name, strlen(idx_name), 1);
1005                if (!new_p)
1006                        die("core git rejected index %s", idx_name);
1007                all_packs[pack_id] = new_p;
1008                install_packed_git(new_p);
1009
1010                /* Print the boundary */
1011                if (pack_edges) {
1012                        fprintf(pack_edges, "%s:", new_p->pack_name);
1013                        for (i = 0; i < branch_table_sz; i++) {
1014                                for (b = branch_table[i]; b; b = b->table_next_branch) {
1015                                        if (b->pack_id == pack_id)
1016                                                fprintf(pack_edges, " %s", sha1_to_hex(b->sha1));
1017                                }
1018                        }
1019                        for (t = first_tag; t; t = t->next_tag) {
1020                                if (t->pack_id == pack_id)
1021                                        fprintf(pack_edges, " %s", sha1_to_hex(t->sha1));
1022                        }
1023                        fputc('\n', pack_edges);
1024                        fflush(pack_edges);
1025                }
1026
1027                pack_id++;
1028        }
1029        else {
1030discard_pack:
1031                close(pack_data->pack_fd);
1032                unlink_or_warn(pack_data->pack_name);
1033        }
1034        free(pack_data);
1035        pack_data = NULL;
1036        running = 0;
1037
1038        /* We can't carry a delta across packfiles. */
1039        strbuf_release(&last_blob.data);
1040        last_blob.offset = 0;
1041        last_blob.depth = 0;
1042}
1043
1044static void cycle_packfile(void)
1045{
1046        end_packfile();
1047        start_packfile();
1048}
1049
1050static int store_object(
1051        enum object_type type,
1052        struct strbuf *dat,
1053        struct last_object *last,
1054        unsigned char *sha1out,
1055        uintmax_t mark)
1056{
1057        void *out, *delta;
1058        struct object_entry *e;
1059        unsigned char hdr[96];
1060        unsigned char sha1[20];
1061        unsigned long hdrlen, deltalen;
1062        git_SHA_CTX c;
1063        git_zstream s;
1064
1065        hdrlen = xsnprintf((char *)hdr, sizeof(hdr), "%s %lu",
1066                           typename(type), (unsigned long)dat->len) + 1;
1067        git_SHA1_Init(&c);
1068        git_SHA1_Update(&c, hdr, hdrlen);
1069        git_SHA1_Update(&c, dat->buf, dat->len);
1070        git_SHA1_Final(sha1, &c);
1071        if (sha1out)
1072                hashcpy(sha1out, sha1);
1073
1074        e = insert_object(sha1);
1075        if (mark)
1076                insert_mark(mark, e);
1077        if (e->idx.offset) {
1078                duplicate_count_by_type[type]++;
1079                return 1;
1080        } else if (find_sha1_pack(sha1, packed_git)) {
1081                e->type = type;
1082                e->pack_id = MAX_PACK_ID;
1083                e->idx.offset = 1; /* just not zero! */
1084                duplicate_count_by_type[type]++;
1085                return 1;
1086        }
1087
1088        if (last && last->data.buf && last->depth < max_depth && dat->len > 20) {
1089                delta_count_attempts_by_type[type]++;
1090                delta = diff_delta(last->data.buf, last->data.len,
1091                        dat->buf, dat->len,
1092                        &deltalen, dat->len - 20);
1093        } else
1094                delta = NULL;
1095
1096        git_deflate_init(&s, pack_compression_level);
1097        if (delta) {
1098                s.next_in = delta;
1099                s.avail_in = deltalen;
1100        } else {
1101                s.next_in = (void *)dat->buf;
1102                s.avail_in = dat->len;
1103        }
1104        s.avail_out = git_deflate_bound(&s, s.avail_in);
1105        s.next_out = out = xmalloc(s.avail_out);
1106        while (git_deflate(&s, Z_FINISH) == Z_OK)
1107                ; /* nothing */
1108        git_deflate_end(&s);
1109
1110        /* Determine if we should auto-checkpoint. */
1111        if ((max_packsize && (pack_size + 60 + s.total_out) > max_packsize)
1112                || (pack_size + 60 + s.total_out) < pack_size) {
1113
1114                /* This new object needs to *not* have the current pack_id. */
1115                e->pack_id = pack_id + 1;
1116                cycle_packfile();
1117
1118                /* We cannot carry a delta into the new pack. */
1119                if (delta) {
1120                        free(delta);
1121                        delta = NULL;
1122
1123                        git_deflate_init(&s, pack_compression_level);
1124                        s.next_in = (void *)dat->buf;
1125                        s.avail_in = dat->len;
1126                        s.avail_out = git_deflate_bound(&s, s.avail_in);
1127                        s.next_out = out = xrealloc(out, s.avail_out);
1128                        while (git_deflate(&s, Z_FINISH) == Z_OK)
1129                                ; /* nothing */
1130                        git_deflate_end(&s);
1131                }
1132        }
1133
1134        e->type = type;
1135        e->pack_id = pack_id;
1136        e->idx.offset = pack_size;
1137        object_count++;
1138        object_count_by_type[type]++;
1139
1140        crc32_begin(pack_file);
1141
1142        if (delta) {
1143                off_t ofs = e->idx.offset - last->offset;
1144                unsigned pos = sizeof(hdr) - 1;
1145
1146                delta_count_by_type[type]++;
1147                e->depth = last->depth + 1;
1148
1149                hdrlen = encode_in_pack_object_header(OBJ_OFS_DELTA, deltalen, hdr);
1150                sha1write(pack_file, hdr, hdrlen);
1151                pack_size += hdrlen;
1152
1153                hdr[pos] = ofs & 127;
1154                while (ofs >>= 7)
1155                        hdr[--pos] = 128 | (--ofs & 127);
1156                sha1write(pack_file, hdr + pos, sizeof(hdr) - pos);
1157                pack_size += sizeof(hdr) - pos;
1158        } else {
1159                e->depth = 0;
1160                hdrlen = encode_in_pack_object_header(type, dat->len, hdr);
1161                sha1write(pack_file, hdr, hdrlen);
1162                pack_size += hdrlen;
1163        }
1164
1165        sha1write(pack_file, out, s.total_out);
1166        pack_size += s.total_out;
1167
1168        e->idx.crc32 = crc32_end(pack_file);
1169
1170        free(out);
1171        free(delta);
1172        if (last) {
1173                if (last->no_swap) {
1174                        last->data = *dat;
1175                } else {
1176                        strbuf_swap(&last->data, dat);
1177                }
1178                last->offset = e->idx.offset;
1179                last->depth = e->depth;
1180        }
1181        return 0;
1182}
1183
1184static void truncate_pack(struct sha1file_checkpoint *checkpoint)
1185{
1186        if (sha1file_truncate(pack_file, checkpoint))
1187                die_errno("cannot truncate pack to skip duplicate");
1188        pack_size = checkpoint->offset;
1189}
1190
1191static void stream_blob(uintmax_t len, unsigned char *sha1out, uintmax_t mark)
1192{
1193        size_t in_sz = 64 * 1024, out_sz = 64 * 1024;
1194        unsigned char *in_buf = xmalloc(in_sz);
1195        unsigned char *out_buf = xmalloc(out_sz);
1196        struct object_entry *e;
1197        unsigned char sha1[20];
1198        unsigned long hdrlen;
1199        off_t offset;
1200        git_SHA_CTX c;
1201        git_zstream s;
1202        struct sha1file_checkpoint checkpoint;
1203        int status = Z_OK;
1204
1205        /* Determine if we should auto-checkpoint. */
1206        if ((max_packsize && (pack_size + 60 + len) > max_packsize)
1207                || (pack_size + 60 + len) < pack_size)
1208                cycle_packfile();
1209
1210        sha1file_checkpoint(pack_file, &checkpoint);
1211        offset = checkpoint.offset;
1212
1213        hdrlen = snprintf((char *)out_buf, out_sz, "blob %" PRIuMAX, len) + 1;
1214        if (out_sz <= hdrlen)
1215                die("impossibly large object header");
1216
1217        git_SHA1_Init(&c);
1218        git_SHA1_Update(&c, out_buf, hdrlen);
1219
1220        crc32_begin(pack_file);
1221
1222        git_deflate_init(&s, pack_compression_level);
1223
1224        hdrlen = encode_in_pack_object_header(OBJ_BLOB, len, out_buf);
1225        if (out_sz <= hdrlen)
1226                die("impossibly large object header");
1227
1228        s.next_out = out_buf + hdrlen;
1229        s.avail_out = out_sz - hdrlen;
1230
1231        while (status != Z_STREAM_END) {
1232                if (0 < len && !s.avail_in) {
1233                        size_t cnt = in_sz < len ? in_sz : (size_t)len;
1234                        size_t n = fread(in_buf, 1, cnt, stdin);
1235                        if (!n && feof(stdin))
1236                                die("EOF in data (%" PRIuMAX " bytes remaining)", len);
1237
1238                        git_SHA1_Update(&c, in_buf, n);
1239                        s.next_in = in_buf;
1240                        s.avail_in = n;
1241                        len -= n;
1242                }
1243
1244                status = git_deflate(&s, len ? 0 : Z_FINISH);
1245
1246                if (!s.avail_out || status == Z_STREAM_END) {
1247                        size_t n = s.next_out - out_buf;
1248                        sha1write(pack_file, out_buf, n);
1249                        pack_size += n;
1250                        s.next_out = out_buf;
1251                        s.avail_out = out_sz;
1252                }
1253
1254                switch (status) {
1255                case Z_OK:
1256                case Z_BUF_ERROR:
1257                case Z_STREAM_END:
1258                        continue;
1259                default:
1260                        die("unexpected deflate failure: %d", status);
1261                }
1262        }
1263        git_deflate_end(&s);
1264        git_SHA1_Final(sha1, &c);
1265
1266        if (sha1out)
1267                hashcpy(sha1out, sha1);
1268
1269        e = insert_object(sha1);
1270
1271        if (mark)
1272                insert_mark(mark, e);
1273
1274        if (e->idx.offset) {
1275                duplicate_count_by_type[OBJ_BLOB]++;
1276                truncate_pack(&checkpoint);
1277
1278        } else if (find_sha1_pack(sha1, packed_git)) {
1279                e->type = OBJ_BLOB;
1280                e->pack_id = MAX_PACK_ID;
1281                e->idx.offset = 1; /* just not zero! */
1282                duplicate_count_by_type[OBJ_BLOB]++;
1283                truncate_pack(&checkpoint);
1284
1285        } else {
1286                e->depth = 0;
1287                e->type = OBJ_BLOB;
1288                e->pack_id = pack_id;
1289                e->idx.offset = offset;
1290                e->idx.crc32 = crc32_end(pack_file);
1291                object_count++;
1292                object_count_by_type[OBJ_BLOB]++;
1293        }
1294
1295        free(in_buf);
1296        free(out_buf);
1297}
1298
1299/* All calls must be guarded by find_object() or find_mark() to
1300 * ensure the 'struct object_entry' passed was written by this
1301 * process instance.  We unpack the entry by the offset, avoiding
1302 * the need for the corresponding .idx file.  This unpacking rule
1303 * works because we only use OBJ_REF_DELTA within the packfiles
1304 * created by fast-import.
1305 *
1306 * oe must not be NULL.  Such an oe usually comes from giving
1307 * an unknown SHA-1 to find_object() or an undefined mark to
1308 * find_mark().  Callers must test for this condition and use
1309 * the standard read_sha1_file() when it happens.
1310 *
1311 * oe->pack_id must not be MAX_PACK_ID.  Such an oe is usually from
1312 * find_mark(), where the mark was reloaded from an existing marks
1313 * file and is referencing an object that this fast-import process
1314 * instance did not write out to a packfile.  Callers must test for
1315 * this condition and use read_sha1_file() instead.
1316 */
1317static void *gfi_unpack_entry(
1318        struct object_entry *oe,
1319        unsigned long *sizep)
1320{
1321        enum object_type type;
1322        struct packed_git *p = all_packs[oe->pack_id];
1323        if (p == pack_data && p->pack_size < (pack_size + 20)) {
1324                /* The object is stored in the packfile we are writing to
1325                 * and we have modified it since the last time we scanned
1326                 * back to read a previously written object.  If an old
1327                 * window covered [p->pack_size, p->pack_size + 20) its
1328                 * data is stale and is not valid.  Closing all windows
1329                 * and updating the packfile length ensures we can read
1330                 * the newly written data.
1331                 */
1332                close_pack_windows(p);
1333                sha1flush(pack_file);
1334
1335                /* We have to offer 20 bytes additional on the end of
1336                 * the packfile as the core unpacker code assumes the
1337                 * footer is present at the file end and must promise
1338                 * at least 20 bytes within any window it maps.  But
1339                 * we don't actually create the footer here.
1340                 */
1341                p->pack_size = pack_size + 20;
1342        }
1343        return unpack_entry(p, oe->idx.offset, &type, sizep);
1344}
1345
1346static const char *get_mode(const char *str, uint16_t *modep)
1347{
1348        unsigned char c;
1349        uint16_t mode = 0;
1350
1351        while ((c = *str++) != ' ') {
1352                if (c < '0' || c > '7')
1353                        return NULL;
1354                mode = (mode << 3) + (c - '0');
1355        }
1356        *modep = mode;
1357        return str;
1358}
1359
1360static void load_tree(struct tree_entry *root)
1361{
1362        unsigned char *sha1 = root->versions[1].sha1;
1363        struct object_entry *myoe;
1364        struct tree_content *t;
1365        unsigned long size;
1366        char *buf;
1367        const char *c;
1368
1369        root->tree = t = new_tree_content(8);
1370        if (is_null_sha1(sha1))
1371                return;
1372
1373        myoe = find_object(sha1);
1374        if (myoe && myoe->pack_id != MAX_PACK_ID) {
1375                if (myoe->type != OBJ_TREE)
1376                        die("Not a tree: %s", sha1_to_hex(sha1));
1377                t->delta_depth = myoe->depth;
1378                buf = gfi_unpack_entry(myoe, &size);
1379                if (!buf)
1380                        die("Can't load tree %s", sha1_to_hex(sha1));
1381        } else {
1382                enum object_type type;
1383                buf = read_sha1_file(sha1, &type, &size);
1384                if (!buf || type != OBJ_TREE)
1385                        die("Can't load tree %s", sha1_to_hex(sha1));
1386        }
1387
1388        c = buf;
1389        while (c != (buf + size)) {
1390                struct tree_entry *e = new_tree_entry();
1391
1392                if (t->entry_count == t->entry_capacity)
1393                        root->tree = t = grow_tree_content(t, t->entry_count);
1394                t->entries[t->entry_count++] = e;
1395
1396                e->tree = NULL;
1397                c = get_mode(c, &e->versions[1].mode);
1398                if (!c)
1399                        die("Corrupt mode in %s", sha1_to_hex(sha1));
1400                e->versions[0].mode = e->versions[1].mode;
1401                e->name = to_atom(c, strlen(c));
1402                c += e->name->str_len + 1;
1403                hashcpy(e->versions[0].sha1, (unsigned char *)c);
1404                hashcpy(e->versions[1].sha1, (unsigned char *)c);
1405                c += 20;
1406        }
1407        free(buf);
1408}
1409
1410static int tecmp0 (const void *_a, const void *_b)
1411{
1412        struct tree_entry *a = *((struct tree_entry**)_a);
1413        struct tree_entry *b = *((struct tree_entry**)_b);
1414        return base_name_compare(
1415                a->name->str_dat, a->name->str_len, a->versions[0].mode,
1416                b->name->str_dat, b->name->str_len, b->versions[0].mode);
1417}
1418
1419static int tecmp1 (const void *_a, const void *_b)
1420{
1421        struct tree_entry *a = *((struct tree_entry**)_a);
1422        struct tree_entry *b = *((struct tree_entry**)_b);
1423        return base_name_compare(
1424                a->name->str_dat, a->name->str_len, a->versions[1].mode,
1425                b->name->str_dat, b->name->str_len, b->versions[1].mode);
1426}
1427
1428static void mktree(struct tree_content *t, int v, struct strbuf *b)
1429{
1430        size_t maxlen = 0;
1431        unsigned int i;
1432
1433        if (!v)
1434                qsort(t->entries,t->entry_count,sizeof(t->entries[0]),tecmp0);
1435        else
1436                qsort(t->entries,t->entry_count,sizeof(t->entries[0]),tecmp1);
1437
1438        for (i = 0; i < t->entry_count; i++) {
1439                if (t->entries[i]->versions[v].mode)
1440                        maxlen += t->entries[i]->name->str_len + 34;
1441        }
1442
1443        strbuf_reset(b);
1444        strbuf_grow(b, maxlen);
1445        for (i = 0; i < t->entry_count; i++) {
1446                struct tree_entry *e = t->entries[i];
1447                if (!e->versions[v].mode)
1448                        continue;
1449                strbuf_addf(b, "%o %s%c",
1450                        (unsigned int)(e->versions[v].mode & ~NO_DELTA),
1451                        e->name->str_dat, '\0');
1452                strbuf_add(b, e->versions[v].sha1, 20);
1453        }
1454}
1455
1456static void store_tree(struct tree_entry *root)
1457{
1458        struct tree_content *t;
1459        unsigned int i, j, del;
1460        struct last_object lo = { STRBUF_INIT, 0, 0, /* no_swap */ 1 };
1461        struct object_entry *le = NULL;
1462
1463        if (!is_null_sha1(root->versions[1].sha1))
1464                return;
1465
1466        if (!root->tree)
1467                load_tree(root);
1468        t = root->tree;
1469
1470        for (i = 0; i < t->entry_count; i++) {
1471                if (t->entries[i]->tree)
1472                        store_tree(t->entries[i]);
1473        }
1474
1475        if (!(root->versions[0].mode & NO_DELTA))
1476                le = find_object(root->versions[0].sha1);
1477        if (S_ISDIR(root->versions[0].mode) && le && le->pack_id == pack_id) {
1478                mktree(t, 0, &old_tree);
1479                lo.data = old_tree;
1480                lo.offset = le->idx.offset;
1481                lo.depth = t->delta_depth;
1482        }
1483
1484        mktree(t, 1, &new_tree);
1485        store_object(OBJ_TREE, &new_tree, &lo, root->versions[1].sha1, 0);
1486
1487        t->delta_depth = lo.depth;
1488        for (i = 0, j = 0, del = 0; i < t->entry_count; i++) {
1489                struct tree_entry *e = t->entries[i];
1490                if (e->versions[1].mode) {
1491                        e->versions[0].mode = e->versions[1].mode;
1492                        hashcpy(e->versions[0].sha1, e->versions[1].sha1);
1493                        t->entries[j++] = e;
1494                } else {
1495                        release_tree_entry(e);
1496                        del++;
1497                }
1498        }
1499        t->entry_count -= del;
1500}
1501
1502static void tree_content_replace(
1503        struct tree_entry *root,
1504        const unsigned char *sha1,
1505        const uint16_t mode,
1506        struct tree_content *newtree)
1507{
1508        if (!S_ISDIR(mode))
1509                die("Root cannot be a non-directory");
1510        hashclr(root->versions[0].sha1);
1511        hashcpy(root->versions[1].sha1, sha1);
1512        if (root->tree)
1513                release_tree_content_recursive(root->tree);
1514        root->tree = newtree;
1515}
1516
1517static int tree_content_set(
1518        struct tree_entry *root,
1519        const char *p,
1520        const unsigned char *sha1,
1521        const uint16_t mode,
1522        struct tree_content *subtree)
1523{
1524        struct tree_content *t;
1525        const char *slash1;
1526        unsigned int i, n;
1527        struct tree_entry *e;
1528
1529        slash1 = strchrnul(p, '/');
1530        n = slash1 - p;
1531        if (!n)
1532                die("Empty path component found in input");
1533        if (!*slash1 && !S_ISDIR(mode) && subtree)
1534                die("Non-directories cannot have subtrees");
1535
1536        if (!root->tree)
1537                load_tree(root);
1538        t = root->tree;
1539        for (i = 0; i < t->entry_count; i++) {
1540                e = t->entries[i];
1541                if (e->name->str_len == n && !strncmp_icase(p, e->name->str_dat, n)) {
1542                        if (!*slash1) {
1543                                if (!S_ISDIR(mode)
1544                                                && e->versions[1].mode == mode
1545                                                && !hashcmp(e->versions[1].sha1, sha1))
1546                                        return 0;
1547                                e->versions[1].mode = mode;
1548                                hashcpy(e->versions[1].sha1, sha1);
1549                                if (e->tree)
1550                                        release_tree_content_recursive(e->tree);
1551                                e->tree = subtree;
1552
1553                                /*
1554                                 * We need to leave e->versions[0].sha1 alone
1555                                 * to avoid modifying the preimage tree used
1556                                 * when writing out the parent directory.
1557                                 * But after replacing the subdir with a
1558                                 * completely different one, it's not a good
1559                                 * delta base any more, and besides, we've
1560                                 * thrown away the tree entries needed to
1561                                 * make a delta against it.
1562                                 *
1563                                 * So let's just explicitly disable deltas
1564                                 * for the subtree.
1565                                 */
1566                                if (S_ISDIR(e->versions[0].mode))
1567                                        e->versions[0].mode |= NO_DELTA;
1568
1569                                hashclr(root->versions[1].sha1);
1570                                return 1;
1571                        }
1572                        if (!S_ISDIR(e->versions[1].mode)) {
1573                                e->tree = new_tree_content(8);
1574                                e->versions[1].mode = S_IFDIR;
1575                        }
1576                        if (!e->tree)
1577                                load_tree(e);
1578                        if (tree_content_set(e, slash1 + 1, sha1, mode, subtree)) {
1579                                hashclr(root->versions[1].sha1);
1580                                return 1;
1581                        }
1582                        return 0;
1583                }
1584        }
1585
1586        if (t->entry_count == t->entry_capacity)
1587                root->tree = t = grow_tree_content(t, t->entry_count);
1588        e = new_tree_entry();
1589        e->name = to_atom(p, n);
1590        e->versions[0].mode = 0;
1591        hashclr(e->versions[0].sha1);
1592        t->entries[t->entry_count++] = e;
1593        if (*slash1) {
1594                e->tree = new_tree_content(8);
1595                e->versions[1].mode = S_IFDIR;
1596                tree_content_set(e, slash1 + 1, sha1, mode, subtree);
1597        } else {
1598                e->tree = subtree;
1599                e->versions[1].mode = mode;
1600                hashcpy(e->versions[1].sha1, sha1);
1601        }
1602        hashclr(root->versions[1].sha1);
1603        return 1;
1604}
1605
1606static int tree_content_remove(
1607        struct tree_entry *root,
1608        const char *p,
1609        struct tree_entry *backup_leaf,
1610        int allow_root)
1611{
1612        struct tree_content *t;
1613        const char *slash1;
1614        unsigned int i, n;
1615        struct tree_entry *e;
1616
1617        slash1 = strchrnul(p, '/');
1618        n = slash1 - p;
1619
1620        if (!root->tree)
1621                load_tree(root);
1622
1623        if (!*p && allow_root) {
1624                e = root;
1625                goto del_entry;
1626        }
1627
1628        t = root->tree;
1629        for (i = 0; i < t->entry_count; i++) {
1630                e = t->entries[i];
1631                if (e->name->str_len == n && !strncmp_icase(p, e->name->str_dat, n)) {
1632                        if (*slash1 && !S_ISDIR(e->versions[1].mode))
1633                                /*
1634                                 * If p names a file in some subdirectory, and a
1635                                 * file or symlink matching the name of the
1636                                 * parent directory of p exists, then p cannot
1637                                 * exist and need not be deleted.
1638                                 */
1639                                return 1;
1640                        if (!*slash1 || !S_ISDIR(e->versions[1].mode))
1641                                goto del_entry;
1642                        if (!e->tree)
1643                                load_tree(e);
1644                        if (tree_content_remove(e, slash1 + 1, backup_leaf, 0)) {
1645                                for (n = 0; n < e->tree->entry_count; n++) {
1646                                        if (e->tree->entries[n]->versions[1].mode) {
1647                                                hashclr(root->versions[1].sha1);
1648                                                return 1;
1649                                        }
1650                                }
1651                                backup_leaf = NULL;
1652                                goto del_entry;
1653                        }
1654                        return 0;
1655                }
1656        }
1657        return 0;
1658
1659del_entry:
1660        if (backup_leaf)
1661                memcpy(backup_leaf, e, sizeof(*backup_leaf));
1662        else if (e->tree)
1663                release_tree_content_recursive(e->tree);
1664        e->tree = NULL;
1665        e->versions[1].mode = 0;
1666        hashclr(e->versions[1].sha1);
1667        hashclr(root->versions[1].sha1);
1668        return 1;
1669}
1670
1671static int tree_content_get(
1672        struct tree_entry *root,
1673        const char *p,
1674        struct tree_entry *leaf,
1675        int allow_root)
1676{
1677        struct tree_content *t;
1678        const char *slash1;
1679        unsigned int i, n;
1680        struct tree_entry *e;
1681
1682        slash1 = strchrnul(p, '/');
1683        n = slash1 - p;
1684        if (!n && !allow_root)
1685                die("Empty path component found in input");
1686
1687        if (!root->tree)
1688                load_tree(root);
1689
1690        if (!n) {
1691                e = root;
1692                goto found_entry;
1693        }
1694
1695        t = root->tree;
1696        for (i = 0; i < t->entry_count; i++) {
1697                e = t->entries[i];
1698                if (e->name->str_len == n && !strncmp_icase(p, e->name->str_dat, n)) {
1699                        if (!*slash1)
1700                                goto found_entry;
1701                        if (!S_ISDIR(e->versions[1].mode))
1702                                return 0;
1703                        if (!e->tree)
1704                                load_tree(e);
1705                        return tree_content_get(e, slash1 + 1, leaf, 0);
1706                }
1707        }
1708        return 0;
1709
1710found_entry:
1711        memcpy(leaf, e, sizeof(*leaf));
1712        if (e->tree && is_null_sha1(e->versions[1].sha1))
1713                leaf->tree = dup_tree_content(e->tree);
1714        else
1715                leaf->tree = NULL;
1716        return 1;
1717}
1718
1719static int update_branch(struct branch *b)
1720{
1721        static const char *msg = "fast-import";
1722        struct ref_transaction *transaction;
1723        unsigned char old_sha1[20];
1724        struct strbuf err = STRBUF_INIT;
1725
1726        if (is_null_sha1(b->sha1)) {
1727                if (b->delete)
1728                        delete_ref(b->name, NULL, 0);
1729                return 0;
1730        }
1731        if (read_ref(b->name, old_sha1))
1732                hashclr(old_sha1);
1733        if (!force_update && !is_null_sha1(old_sha1)) {
1734                struct commit *old_cmit, *new_cmit;
1735
1736                old_cmit = lookup_commit_reference_gently(old_sha1, 0);
1737                new_cmit = lookup_commit_reference_gently(b->sha1, 0);
1738                if (!old_cmit || !new_cmit)
1739                        return error("Branch %s is missing commits.", b->name);
1740
1741                if (!in_merge_bases(old_cmit, new_cmit)) {
1742                        warning("Not updating %s"
1743                                " (new tip %s does not contain %s)",
1744                                b->name, sha1_to_hex(b->sha1), sha1_to_hex(old_sha1));
1745                        return -1;
1746                }
1747        }
1748        transaction = ref_transaction_begin(&err);
1749        if (!transaction ||
1750            ref_transaction_update(transaction, b->name, b->sha1, old_sha1,
1751                                   0, msg, &err) ||
1752            ref_transaction_commit(transaction, &err)) {
1753                ref_transaction_free(transaction);
1754                error("%s", err.buf);
1755                strbuf_release(&err);
1756                return -1;
1757        }
1758        ref_transaction_free(transaction);
1759        strbuf_release(&err);
1760        return 0;
1761}
1762
1763static void dump_branches(void)
1764{
1765        unsigned int i;
1766        struct branch *b;
1767
1768        for (i = 0; i < branch_table_sz; i++) {
1769                for (b = branch_table[i]; b; b = b->table_next_branch)
1770                        failure |= update_branch(b);
1771        }
1772}
1773
1774static void dump_tags(void)
1775{
1776        static const char *msg = "fast-import";
1777        struct tag *t;
1778        struct strbuf ref_name = STRBUF_INIT;
1779        struct strbuf err = STRBUF_INIT;
1780        struct ref_transaction *transaction;
1781
1782        transaction = ref_transaction_begin(&err);
1783        if (!transaction) {
1784                failure |= error("%s", err.buf);
1785                goto cleanup;
1786        }
1787        for (t = first_tag; t; t = t->next_tag) {
1788                strbuf_reset(&ref_name);
1789                strbuf_addf(&ref_name, "refs/tags/%s", t->name);
1790
1791                if (ref_transaction_update(transaction, ref_name.buf,
1792                                           t->sha1, NULL, 0, msg, &err)) {
1793                        failure |= error("%s", err.buf);
1794                        goto cleanup;
1795                }
1796        }
1797        if (ref_transaction_commit(transaction, &err))
1798                failure |= error("%s", err.buf);
1799
1800 cleanup:
1801        ref_transaction_free(transaction);
1802        strbuf_release(&ref_name);
1803        strbuf_release(&err);
1804}
1805
1806static void dump_marks_helper(FILE *f,
1807        uintmax_t base,
1808        struct mark_set *m)
1809{
1810        uintmax_t k;
1811        if (m->shift) {
1812                for (k = 0; k < 1024; k++) {
1813                        if (m->data.sets[k])
1814                                dump_marks_helper(f, base + (k << m->shift),
1815                                        m->data.sets[k]);
1816                }
1817        } else {
1818                for (k = 0; k < 1024; k++) {
1819                        if (m->data.marked[k])
1820                                fprintf(f, ":%" PRIuMAX " %s\n", base + k,
1821                                        sha1_to_hex(m->data.marked[k]->idx.sha1));
1822                }
1823        }
1824}
1825
1826static void dump_marks(void)
1827{
1828        static struct lock_file mark_lock;
1829        FILE *f;
1830
1831        if (!export_marks_file)
1832                return;
1833
1834        if (hold_lock_file_for_update(&mark_lock, export_marks_file, 0) < 0) {
1835                failure |= error("Unable to write marks file %s: %s",
1836                        export_marks_file, strerror(errno));
1837                return;
1838        }
1839
1840        f = fdopen_lock_file(&mark_lock, "w");
1841        if (!f) {
1842                int saved_errno = errno;
1843                rollback_lock_file(&mark_lock);
1844                failure |= error("Unable to write marks file %s: %s",
1845                        export_marks_file, strerror(saved_errno));
1846                return;
1847        }
1848
1849        dump_marks_helper(f, 0, marks);
1850        if (commit_lock_file(&mark_lock)) {
1851                failure |= error("Unable to write file %s: %s",
1852                        export_marks_file, strerror(errno));
1853                return;
1854        }
1855}
1856
1857static void read_marks(void)
1858{
1859        char line[512];
1860        FILE *f = fopen(import_marks_file, "r");
1861        if (f)
1862                ;
1863        else if (import_marks_file_ignore_missing && errno == ENOENT)
1864                return; /* Marks file does not exist */
1865        else
1866                die_errno("cannot read '%s'", import_marks_file);
1867        while (fgets(line, sizeof(line), f)) {
1868                uintmax_t mark;
1869                char *end;
1870                unsigned char sha1[20];
1871                struct object_entry *e;
1872
1873                end = strchr(line, '\n');
1874                if (line[0] != ':' || !end)
1875                        die("corrupt mark line: %s", line);
1876                *end = 0;
1877                mark = strtoumax(line + 1, &end, 10);
1878                if (!mark || end == line + 1
1879                        || *end != ' ' || get_sha1_hex(end + 1, sha1))
1880                        die("corrupt mark line: %s", line);
1881                e = find_object(sha1);
1882                if (!e) {
1883                        enum object_type type = sha1_object_info(sha1, NULL);
1884                        if (type < 0)
1885                                die("object not found: %s", sha1_to_hex(sha1));
1886                        e = insert_object(sha1);
1887                        e->type = type;
1888                        e->pack_id = MAX_PACK_ID;
1889                        e->idx.offset = 1; /* just not zero! */
1890                }
1891                insert_mark(mark, e);
1892        }
1893        fclose(f);
1894}
1895
1896
1897static int read_next_command(void)
1898{
1899        static int stdin_eof = 0;
1900
1901        if (stdin_eof) {
1902                unread_command_buf = 0;
1903                return EOF;
1904        }
1905
1906        for (;;) {
1907                const char *p;
1908
1909                if (unread_command_buf) {
1910                        unread_command_buf = 0;
1911                } else {
1912                        struct recent_command *rc;
1913
1914                        strbuf_detach(&command_buf, NULL);
1915                        stdin_eof = strbuf_getline_lf(&command_buf, stdin);
1916                        if (stdin_eof)
1917                                return EOF;
1918
1919                        if (!seen_data_command
1920                                && !starts_with(command_buf.buf, "feature ")
1921                                && !starts_with(command_buf.buf, "option ")) {
1922                                parse_argv();
1923                        }
1924
1925                        rc = rc_free;
1926                        if (rc)
1927                                rc_free = rc->next;
1928                        else {
1929                                rc = cmd_hist.next;
1930                                cmd_hist.next = rc->next;
1931                                cmd_hist.next->prev = &cmd_hist;
1932                                free(rc->buf);
1933                        }
1934
1935                        rc->buf = command_buf.buf;
1936                        rc->prev = cmd_tail;
1937                        rc->next = cmd_hist.prev;
1938                        rc->prev->next = rc;
1939                        cmd_tail = rc;
1940                }
1941                if (skip_prefix(command_buf.buf, "get-mark ", &p)) {
1942                        parse_get_mark(p);
1943                        continue;
1944                }
1945                if (skip_prefix(command_buf.buf, "cat-blob ", &p)) {
1946                        parse_cat_blob(p);
1947                        continue;
1948                }
1949                if (command_buf.buf[0] == '#')
1950                        continue;
1951                return 0;
1952        }
1953}
1954
1955static void skip_optional_lf(void)
1956{
1957        int term_char = fgetc(stdin);
1958        if (term_char != '\n' && term_char != EOF)
1959                ungetc(term_char, stdin);
1960}
1961
1962static void parse_mark(void)
1963{
1964        const char *v;
1965        if (skip_prefix(command_buf.buf, "mark :", &v)) {
1966                next_mark = strtoumax(v, NULL, 10);
1967                read_next_command();
1968        }
1969        else
1970                next_mark = 0;
1971}
1972
1973static int parse_data(struct strbuf *sb, uintmax_t limit, uintmax_t *len_res)
1974{
1975        const char *data;
1976        strbuf_reset(sb);
1977
1978        if (!skip_prefix(command_buf.buf, "data ", &data))
1979                die("Expected 'data n' command, found: %s", command_buf.buf);
1980
1981        if (skip_prefix(data, "<<", &data)) {
1982                char *term = xstrdup(data);
1983                size_t term_len = command_buf.len - (data - command_buf.buf);
1984
1985                strbuf_detach(&command_buf, NULL);
1986                for (;;) {
1987                        if (strbuf_getline_lf(&command_buf, stdin) == EOF)
1988                                die("EOF in data (terminator '%s' not found)", term);
1989                        if (term_len == command_buf.len
1990                                && !strcmp(term, command_buf.buf))
1991                                break;
1992                        strbuf_addbuf(sb, &command_buf);
1993                        strbuf_addch(sb, '\n');
1994                }
1995                free(term);
1996        }
1997        else {
1998                uintmax_t len = strtoumax(data, NULL, 10);
1999                size_t n = 0, length = (size_t)len;
2000
2001                if (limit && limit < len) {
2002                        *len_res = len;
2003                        return 0;
2004                }
2005                if (length < len)
2006                        die("data is too large to use in this context");
2007
2008                while (n < length) {
2009                        size_t s = strbuf_fread(sb, length - n, stdin);
2010                        if (!s && feof(stdin))
2011                                die("EOF in data (%lu bytes remaining)",
2012                                        (unsigned long)(length - n));
2013                        n += s;
2014                }
2015        }
2016
2017        skip_optional_lf();
2018        return 1;
2019}
2020
2021static int validate_raw_date(const char *src, struct strbuf *result)
2022{
2023        const char *orig_src = src;
2024        char *endp;
2025        unsigned long num;
2026
2027        errno = 0;
2028
2029        num = strtoul(src, &endp, 10);
2030        /* NEEDSWORK: perhaps check for reasonable values? */
2031        if (errno || endp == src || *endp != ' ')
2032                return -1;
2033
2034        src = endp + 1;
2035        if (*src != '-' && *src != '+')
2036                return -1;
2037
2038        num = strtoul(src + 1, &endp, 10);
2039        if (errno || endp == src + 1 || *endp || 1400 < num)
2040                return -1;
2041
2042        strbuf_addstr(result, orig_src);
2043        return 0;
2044}
2045
2046static char *parse_ident(const char *buf)
2047{
2048        const char *ltgt;
2049        size_t name_len;
2050        struct strbuf ident = STRBUF_INIT;
2051
2052        /* ensure there is a space delimiter even if there is no name */
2053        if (*buf == '<')
2054                --buf;
2055
2056        ltgt = buf + strcspn(buf, "<>");
2057        if (*ltgt != '<')
2058                die("Missing < in ident string: %s", buf);
2059        if (ltgt != buf && ltgt[-1] != ' ')
2060                die("Missing space before < in ident string: %s", buf);
2061        ltgt = ltgt + 1 + strcspn(ltgt + 1, "<>");
2062        if (*ltgt != '>')
2063                die("Missing > in ident string: %s", buf);
2064        ltgt++;
2065        if (*ltgt != ' ')
2066                die("Missing space after > in ident string: %s", buf);
2067        ltgt++;
2068        name_len = ltgt - buf;
2069        strbuf_add(&ident, buf, name_len);
2070
2071        switch (whenspec) {
2072        case WHENSPEC_RAW:
2073                if (validate_raw_date(ltgt, &ident) < 0)
2074                        die("Invalid raw date \"%s\" in ident: %s", ltgt, buf);
2075                break;
2076        case WHENSPEC_RFC2822:
2077                if (parse_date(ltgt, &ident) < 0)
2078                        die("Invalid rfc2822 date \"%s\" in ident: %s", ltgt, buf);
2079                break;
2080        case WHENSPEC_NOW:
2081                if (strcmp("now", ltgt))
2082                        die("Date in ident must be 'now': %s", buf);
2083                datestamp(&ident);
2084                break;
2085        }
2086
2087        return strbuf_detach(&ident, NULL);
2088}
2089
2090static void parse_and_store_blob(
2091        struct last_object *last,
2092        unsigned char *sha1out,
2093        uintmax_t mark)
2094{
2095        static struct strbuf buf = STRBUF_INIT;
2096        uintmax_t len;
2097
2098        if (parse_data(&buf, big_file_threshold, &len))
2099                store_object(OBJ_BLOB, &buf, last, sha1out, mark);
2100        else {
2101                if (last) {
2102                        strbuf_release(&last->data);
2103                        last->offset = 0;
2104                        last->depth = 0;
2105                }
2106                stream_blob(len, sha1out, mark);
2107                skip_optional_lf();
2108        }
2109}
2110
2111static void parse_new_blob(void)
2112{
2113        read_next_command();
2114        parse_mark();
2115        parse_and_store_blob(&last_blob, NULL, next_mark);
2116}
2117
2118static void unload_one_branch(void)
2119{
2120        while (cur_active_branches
2121                && cur_active_branches >= max_active_branches) {
2122                uintmax_t min_commit = ULONG_MAX;
2123                struct branch *e, *l = NULL, *p = NULL;
2124
2125                for (e = active_branches; e; e = e->active_next_branch) {
2126                        if (e->last_commit < min_commit) {
2127                                p = l;
2128                                min_commit = e->last_commit;
2129                        }
2130                        l = e;
2131                }
2132
2133                if (p) {
2134                        e = p->active_next_branch;
2135                        p->active_next_branch = e->active_next_branch;
2136                } else {
2137                        e = active_branches;
2138                        active_branches = e->active_next_branch;
2139                }
2140                e->active = 0;
2141                e->active_next_branch = NULL;
2142                if (e->branch_tree.tree) {
2143                        release_tree_content_recursive(e->branch_tree.tree);
2144                        e->branch_tree.tree = NULL;
2145                }
2146                cur_active_branches--;
2147        }
2148}
2149
2150static void load_branch(struct branch *b)
2151{
2152        load_tree(&b->branch_tree);
2153        if (!b->active) {
2154                b->active = 1;
2155                b->active_next_branch = active_branches;
2156                active_branches = b;
2157                cur_active_branches++;
2158                branch_load_count++;
2159        }
2160}
2161
2162static unsigned char convert_num_notes_to_fanout(uintmax_t num_notes)
2163{
2164        unsigned char fanout = 0;
2165        while ((num_notes >>= 8))
2166                fanout++;
2167        return fanout;
2168}
2169
2170static void construct_path_with_fanout(const char *hex_sha1,
2171                unsigned char fanout, char *path)
2172{
2173        unsigned int i = 0, j = 0;
2174        if (fanout >= 20)
2175                die("Too large fanout (%u)", fanout);
2176        while (fanout) {
2177                path[i++] = hex_sha1[j++];
2178                path[i++] = hex_sha1[j++];
2179                path[i++] = '/';
2180                fanout--;
2181        }
2182        memcpy(path + i, hex_sha1 + j, 40 - j);
2183        path[i + 40 - j] = '\0';
2184}
2185
2186static uintmax_t do_change_note_fanout(
2187                struct tree_entry *orig_root, struct tree_entry *root,
2188                char *hex_sha1, unsigned int hex_sha1_len,
2189                char *fullpath, unsigned int fullpath_len,
2190                unsigned char fanout)
2191{
2192        struct tree_content *t = root->tree;
2193        struct tree_entry *e, leaf;
2194        unsigned int i, tmp_hex_sha1_len, tmp_fullpath_len;
2195        uintmax_t num_notes = 0;
2196        unsigned char sha1[20];
2197        char realpath[60];
2198
2199        for (i = 0; t && i < t->entry_count; i++) {
2200                e = t->entries[i];
2201                tmp_hex_sha1_len = hex_sha1_len + e->name->str_len;
2202                tmp_fullpath_len = fullpath_len;
2203
2204                /*
2205                 * We're interested in EITHER existing note entries (entries
2206                 * with exactly 40 hex chars in path, not including directory
2207                 * separators), OR directory entries that may contain note
2208                 * entries (with < 40 hex chars in path).
2209                 * Also, each path component in a note entry must be a multiple
2210                 * of 2 chars.
2211                 */
2212                if (!e->versions[1].mode ||
2213                    tmp_hex_sha1_len > 40 ||
2214                    e->name->str_len % 2)
2215                        continue;
2216
2217                /* This _may_ be a note entry, or a subdir containing notes */
2218                memcpy(hex_sha1 + hex_sha1_len, e->name->str_dat,
2219                       e->name->str_len);
2220                if (tmp_fullpath_len)
2221                        fullpath[tmp_fullpath_len++] = '/';
2222                memcpy(fullpath + tmp_fullpath_len, e->name->str_dat,
2223                       e->name->str_len);
2224                tmp_fullpath_len += e->name->str_len;
2225                fullpath[tmp_fullpath_len] = '\0';
2226
2227                if (tmp_hex_sha1_len == 40 && !get_sha1_hex(hex_sha1, sha1)) {
2228                        /* This is a note entry */
2229                        if (fanout == 0xff) {
2230                                /* Counting mode, no rename */
2231                                num_notes++;
2232                                continue;
2233                        }
2234                        construct_path_with_fanout(hex_sha1, fanout, realpath);
2235                        if (!strcmp(fullpath, realpath)) {
2236                                /* Note entry is in correct location */
2237                                num_notes++;
2238                                continue;
2239                        }
2240
2241                        /* Rename fullpath to realpath */
2242                        if (!tree_content_remove(orig_root, fullpath, &leaf, 0))
2243                                die("Failed to remove path %s", fullpath);
2244                        tree_content_set(orig_root, realpath,
2245                                leaf.versions[1].sha1,
2246                                leaf.versions[1].mode,
2247                                leaf.tree);
2248                } else if (S_ISDIR(e->versions[1].mode)) {
2249                        /* This is a subdir that may contain note entries */
2250                        if (!e->tree)
2251                                load_tree(e);
2252                        num_notes += do_change_note_fanout(orig_root, e,
2253                                hex_sha1, tmp_hex_sha1_len,
2254                                fullpath, tmp_fullpath_len, fanout);
2255                }
2256
2257                /* The above may have reallocated the current tree_content */
2258                t = root->tree;
2259        }
2260        return num_notes;
2261}
2262
2263static uintmax_t change_note_fanout(struct tree_entry *root,
2264                unsigned char fanout)
2265{
2266        char hex_sha1[40], path[60];
2267        return do_change_note_fanout(root, root, hex_sha1, 0, path, 0, fanout);
2268}
2269
2270/*
2271 * Given a pointer into a string, parse a mark reference:
2272 *
2273 *   idnum ::= ':' bigint;
2274 *
2275 * Return the first character after the value in *endptr.
2276 *
2277 * Complain if the following character is not what is expected,
2278 * either a space or end of the string.
2279 */
2280static uintmax_t parse_mark_ref(const char *p, char **endptr)
2281{
2282        uintmax_t mark;
2283
2284        assert(*p == ':');
2285        p++;
2286        mark = strtoumax(p, endptr, 10);
2287        if (*endptr == p)
2288                die("No value after ':' in mark: %s", command_buf.buf);
2289        return mark;
2290}
2291
2292/*
2293 * Parse the mark reference, and complain if this is not the end of
2294 * the string.
2295 */
2296static uintmax_t parse_mark_ref_eol(const char *p)
2297{
2298        char *end;
2299        uintmax_t mark;
2300
2301        mark = parse_mark_ref(p, &end);
2302        if (*end != '\0')
2303                die("Garbage after mark: %s", command_buf.buf);
2304        return mark;
2305}
2306
2307/*
2308 * Parse the mark reference, demanding a trailing space.  Return a
2309 * pointer to the space.
2310 */
2311static uintmax_t parse_mark_ref_space(const char **p)
2312{
2313        uintmax_t mark;
2314        char *end;
2315
2316        mark = parse_mark_ref(*p, &end);
2317        if (*end++ != ' ')
2318                die("Missing space after mark: %s", command_buf.buf);
2319        *p = end;
2320        return mark;
2321}
2322
2323static void file_change_m(const char *p, struct branch *b)
2324{
2325        static struct strbuf uq = STRBUF_INIT;
2326        const char *endp;
2327        struct object_entry *oe;
2328        unsigned char sha1[20];
2329        uint16_t mode, inline_data = 0;
2330
2331        p = get_mode(p, &mode);
2332        if (!p)
2333                die("Corrupt mode: %s", command_buf.buf);
2334        switch (mode) {
2335        case 0644:
2336        case 0755:
2337                mode |= S_IFREG;
2338        case S_IFREG | 0644:
2339        case S_IFREG | 0755:
2340        case S_IFLNK:
2341        case S_IFDIR:
2342        case S_IFGITLINK:
2343                /* ok */
2344                break;
2345        default:
2346                die("Corrupt mode: %s", command_buf.buf);
2347        }
2348
2349        if (*p == ':') {
2350                oe = find_mark(parse_mark_ref_space(&p));
2351                hashcpy(sha1, oe->idx.sha1);
2352        } else if (skip_prefix(p, "inline ", &p)) {
2353                inline_data = 1;
2354                oe = NULL; /* not used with inline_data, but makes gcc happy */
2355        } else {
2356                if (get_sha1_hex(p, sha1))
2357                        die("Invalid dataref: %s", command_buf.buf);
2358                oe = find_object(sha1);
2359                p += 40;
2360                if (*p++ != ' ')
2361                        die("Missing space after SHA1: %s", command_buf.buf);
2362        }
2363
2364        strbuf_reset(&uq);
2365        if (!unquote_c_style(&uq, p, &endp)) {
2366                if (*endp)
2367                        die("Garbage after path in: %s", command_buf.buf);
2368                p = uq.buf;
2369        }
2370
2371        /* Git does not track empty, non-toplevel directories. */
2372        if (S_ISDIR(mode) && !hashcmp(sha1, EMPTY_TREE_SHA1_BIN) && *p) {
2373                tree_content_remove(&b->branch_tree, p, NULL, 0);
2374                return;
2375        }
2376
2377        if (S_ISGITLINK(mode)) {
2378                if (inline_data)
2379                        die("Git links cannot be specified 'inline': %s",
2380                                command_buf.buf);
2381                else if (oe) {
2382                        if (oe->type != OBJ_COMMIT)
2383                                die("Not a commit (actually a %s): %s",
2384                                        typename(oe->type), command_buf.buf);
2385                }
2386                /*
2387                 * Accept the sha1 without checking; it expected to be in
2388                 * another repository.
2389                 */
2390        } else if (inline_data) {
2391                if (S_ISDIR(mode))
2392                        die("Directories cannot be specified 'inline': %s",
2393                                command_buf.buf);
2394                if (p != uq.buf) {
2395                        strbuf_addstr(&uq, p);
2396                        p = uq.buf;
2397                }
2398                read_next_command();
2399                parse_and_store_blob(&last_blob, sha1, 0);
2400        } else {
2401                enum object_type expected = S_ISDIR(mode) ?
2402                                                OBJ_TREE: OBJ_BLOB;
2403                enum object_type type = oe ? oe->type :
2404                                        sha1_object_info(sha1, NULL);
2405                if (type < 0)
2406                        die("%s not found: %s",
2407                                        S_ISDIR(mode) ?  "Tree" : "Blob",
2408                                        command_buf.buf);
2409                if (type != expected)
2410                        die("Not a %s (actually a %s): %s",
2411                                typename(expected), typename(type),
2412                                command_buf.buf);
2413        }
2414
2415        if (!*p) {
2416                tree_content_replace(&b->branch_tree, sha1, mode, NULL);
2417                return;
2418        }
2419        tree_content_set(&b->branch_tree, p, sha1, mode, NULL);
2420}
2421
2422static void file_change_d(const char *p, struct branch *b)
2423{
2424        static struct strbuf uq = STRBUF_INIT;
2425        const char *endp;
2426
2427        strbuf_reset(&uq);
2428        if (!unquote_c_style(&uq, p, &endp)) {
2429                if (*endp)
2430                        die("Garbage after path in: %s", command_buf.buf);
2431                p = uq.buf;
2432        }
2433        tree_content_remove(&b->branch_tree, p, NULL, 1);
2434}
2435
2436static void file_change_cr(const char *s, struct branch *b, int rename)
2437{
2438        const char *d;
2439        static struct strbuf s_uq = STRBUF_INIT;
2440        static struct strbuf d_uq = STRBUF_INIT;
2441        const char *endp;
2442        struct tree_entry leaf;
2443
2444        strbuf_reset(&s_uq);
2445        if (!unquote_c_style(&s_uq, s, &endp)) {
2446                if (*endp != ' ')
2447                        die("Missing space after source: %s", command_buf.buf);
2448        } else {
2449                endp = strchr(s, ' ');
2450                if (!endp)
2451                        die("Missing space after source: %s", command_buf.buf);
2452                strbuf_add(&s_uq, s, endp - s);
2453        }
2454        s = s_uq.buf;
2455
2456        endp++;
2457        if (!*endp)
2458                die("Missing dest: %s", command_buf.buf);
2459
2460        d = endp;
2461        strbuf_reset(&d_uq);
2462        if (!unquote_c_style(&d_uq, d, &endp)) {
2463                if (*endp)
2464                        die("Garbage after dest in: %s", command_buf.buf);
2465                d = d_uq.buf;
2466        }
2467
2468        memset(&leaf, 0, sizeof(leaf));
2469        if (rename)
2470                tree_content_remove(&b->branch_tree, s, &leaf, 1);
2471        else
2472                tree_content_get(&b->branch_tree, s, &leaf, 1);
2473        if (!leaf.versions[1].mode)
2474                die("Path %s not in branch", s);
2475        if (!*d) {      /* C "path/to/subdir" "" */
2476                tree_content_replace(&b->branch_tree,
2477                        leaf.versions[1].sha1,
2478                        leaf.versions[1].mode,
2479                        leaf.tree);
2480                return;
2481        }
2482        tree_content_set(&b->branch_tree, d,
2483                leaf.versions[1].sha1,
2484                leaf.versions[1].mode,
2485                leaf.tree);
2486}
2487
2488static void note_change_n(const char *p, struct branch *b, unsigned char *old_fanout)
2489{
2490        static struct strbuf uq = STRBUF_INIT;
2491        struct object_entry *oe;
2492        struct branch *s;
2493        unsigned char sha1[20], commit_sha1[20];
2494        char path[60];
2495        uint16_t inline_data = 0;
2496        unsigned char new_fanout;
2497
2498        /*
2499         * When loading a branch, we don't traverse its tree to count the real
2500         * number of notes (too expensive to do this for all non-note refs).
2501         * This means that recently loaded notes refs might incorrectly have
2502         * b->num_notes == 0, and consequently, old_fanout might be wrong.
2503         *
2504         * Fix this by traversing the tree and counting the number of notes
2505         * when b->num_notes == 0. If the notes tree is truly empty, the
2506         * calculation should not take long.
2507         */
2508        if (b->num_notes == 0 && *old_fanout == 0) {
2509                /* Invoke change_note_fanout() in "counting mode". */
2510                b->num_notes = change_note_fanout(&b->branch_tree, 0xff);
2511                *old_fanout = convert_num_notes_to_fanout(b->num_notes);
2512        }
2513
2514        /* Now parse the notemodify command. */
2515        /* <dataref> or 'inline' */
2516        if (*p == ':') {
2517                oe = find_mark(parse_mark_ref_space(&p));
2518                hashcpy(sha1, oe->idx.sha1);
2519        } else if (skip_prefix(p, "inline ", &p)) {
2520                inline_data = 1;
2521                oe = NULL; /* not used with inline_data, but makes gcc happy */
2522        } else {
2523                if (get_sha1_hex(p, sha1))
2524                        die("Invalid dataref: %s", command_buf.buf);
2525                oe = find_object(sha1);
2526                p += 40;
2527                if (*p++ != ' ')
2528                        die("Missing space after SHA1: %s", command_buf.buf);
2529        }
2530
2531        /* <commit-ish> */
2532        s = lookup_branch(p);
2533        if (s) {
2534                if (is_null_sha1(s->sha1))
2535                        die("Can't add a note on empty branch.");
2536                hashcpy(commit_sha1, s->sha1);
2537        } else if (*p == ':') {
2538                uintmax_t commit_mark = parse_mark_ref_eol(p);
2539                struct object_entry *commit_oe = find_mark(commit_mark);
2540                if (commit_oe->type != OBJ_COMMIT)
2541                        die("Mark :%" PRIuMAX " not a commit", commit_mark);
2542                hashcpy(commit_sha1, commit_oe->idx.sha1);
2543        } else if (!get_sha1(p, commit_sha1)) {
2544                unsigned long size;
2545                char *buf = read_object_with_reference(commit_sha1,
2546                        commit_type, &size, commit_sha1);
2547                if (!buf || size < 46)
2548                        die("Not a valid commit: %s", p);
2549                free(buf);
2550        } else
2551                die("Invalid ref name or SHA1 expression: %s", p);
2552
2553        if (inline_data) {
2554                if (p != uq.buf) {
2555                        strbuf_addstr(&uq, p);
2556                        p = uq.buf;
2557                }
2558                read_next_command();
2559                parse_and_store_blob(&last_blob, sha1, 0);
2560        } else if (oe) {
2561                if (oe->type != OBJ_BLOB)
2562                        die("Not a blob (actually a %s): %s",
2563                                typename(oe->type), command_buf.buf);
2564        } else if (!is_null_sha1(sha1)) {
2565                enum object_type type = sha1_object_info(sha1, NULL);
2566                if (type < 0)
2567                        die("Blob not found: %s", command_buf.buf);
2568                if (type != OBJ_BLOB)
2569                        die("Not a blob (actually a %s): %s",
2570                            typename(type), command_buf.buf);
2571        }
2572
2573        construct_path_with_fanout(sha1_to_hex(commit_sha1), *old_fanout, path);
2574        if (tree_content_remove(&b->branch_tree, path, NULL, 0))
2575                b->num_notes--;
2576
2577        if (is_null_sha1(sha1))
2578                return; /* nothing to insert */
2579
2580        b->num_notes++;
2581        new_fanout = convert_num_notes_to_fanout(b->num_notes);
2582        construct_path_with_fanout(sha1_to_hex(commit_sha1), new_fanout, path);
2583        tree_content_set(&b->branch_tree, path, sha1, S_IFREG | 0644, NULL);
2584}
2585
2586static void file_change_deleteall(struct branch *b)
2587{
2588        release_tree_content_recursive(b->branch_tree.tree);
2589        hashclr(b->branch_tree.versions[0].sha1);
2590        hashclr(b->branch_tree.versions[1].sha1);
2591        load_tree(&b->branch_tree);
2592        b->num_notes = 0;
2593}
2594
2595static void parse_from_commit(struct branch *b, char *buf, unsigned long size)
2596{
2597        if (!buf || size < 46)
2598                die("Not a valid commit: %s", sha1_to_hex(b->sha1));
2599        if (memcmp("tree ", buf, 5)
2600                || get_sha1_hex(buf + 5, b->branch_tree.versions[1].sha1))
2601                die("The commit %s is corrupt", sha1_to_hex(b->sha1));
2602        hashcpy(b->branch_tree.versions[0].sha1,
2603                b->branch_tree.versions[1].sha1);
2604}
2605
2606static void parse_from_existing(struct branch *b)
2607{
2608        if (is_null_sha1(b->sha1)) {
2609                hashclr(b->branch_tree.versions[0].sha1);
2610                hashclr(b->branch_tree.versions[1].sha1);
2611        } else {
2612                unsigned long size;
2613                char *buf;
2614
2615                buf = read_object_with_reference(b->sha1,
2616                        commit_type, &size, b->sha1);
2617                parse_from_commit(b, buf, size);
2618                free(buf);
2619        }
2620}
2621
2622static int parse_from(struct branch *b)
2623{
2624        const char *from;
2625        struct branch *s;
2626        unsigned char sha1[20];
2627
2628        if (!skip_prefix(command_buf.buf, "from ", &from))
2629                return 0;
2630
2631        hashcpy(sha1, b->branch_tree.versions[1].sha1);
2632
2633        s = lookup_branch(from);
2634        if (b == s)
2635                die("Can't create a branch from itself: %s", b->name);
2636        else if (s) {
2637                unsigned char *t = s->branch_tree.versions[1].sha1;
2638                hashcpy(b->sha1, s->sha1);
2639                hashcpy(b->branch_tree.versions[0].sha1, t);
2640                hashcpy(b->branch_tree.versions[1].sha1, t);
2641        } else if (*from == ':') {
2642                uintmax_t idnum = parse_mark_ref_eol(from);
2643                struct object_entry *oe = find_mark(idnum);
2644                if (oe->type != OBJ_COMMIT)
2645                        die("Mark :%" PRIuMAX " not a commit", idnum);
2646                if (hashcmp(b->sha1, oe->idx.sha1)) {
2647                        hashcpy(b->sha1, oe->idx.sha1);
2648                        if (oe->pack_id != MAX_PACK_ID) {
2649                                unsigned long size;
2650                                char *buf = gfi_unpack_entry(oe, &size);
2651                                parse_from_commit(b, buf, size);
2652                                free(buf);
2653                        } else
2654                                parse_from_existing(b);
2655                }
2656        } else if (!get_sha1(from, b->sha1)) {
2657                parse_from_existing(b);
2658                if (is_null_sha1(b->sha1))
2659                        b->delete = 1;
2660        }
2661        else
2662                die("Invalid ref name or SHA1 expression: %s", from);
2663
2664        if (b->branch_tree.tree && hashcmp(sha1, b->branch_tree.versions[1].sha1)) {
2665                release_tree_content_recursive(b->branch_tree.tree);
2666                b->branch_tree.tree = NULL;
2667        }
2668
2669        read_next_command();
2670        return 1;
2671}
2672
2673static struct hash_list *parse_merge(unsigned int *count)
2674{
2675        struct hash_list *list = NULL, **tail = &list, *n;
2676        const char *from;
2677        struct branch *s;
2678
2679        *count = 0;
2680        while (skip_prefix(command_buf.buf, "merge ", &from)) {
2681                n = xmalloc(sizeof(*n));
2682                s = lookup_branch(from);
2683                if (s)
2684                        hashcpy(n->sha1, s->sha1);
2685                else if (*from == ':') {
2686                        uintmax_t idnum = parse_mark_ref_eol(from);
2687                        struct object_entry *oe = find_mark(idnum);
2688                        if (oe->type != OBJ_COMMIT)
2689                                die("Mark :%" PRIuMAX " not a commit", idnum);
2690                        hashcpy(n->sha1, oe->idx.sha1);
2691                } else if (!get_sha1(from, n->sha1)) {
2692                        unsigned long size;
2693                        char *buf = read_object_with_reference(n->sha1,
2694                                commit_type, &size, n->sha1);
2695                        if (!buf || size < 46)
2696                                die("Not a valid commit: %s", from);
2697                        free(buf);
2698                } else
2699                        die("Invalid ref name or SHA1 expression: %s", from);
2700
2701                n->next = NULL;
2702                *tail = n;
2703                tail = &n->next;
2704
2705                (*count)++;
2706                read_next_command();
2707        }
2708        return list;
2709}
2710
2711static void parse_new_commit(const char *arg)
2712{
2713        static struct strbuf msg = STRBUF_INIT;
2714        struct branch *b;
2715        char *author = NULL;
2716        char *committer = NULL;
2717        struct hash_list *merge_list = NULL;
2718        unsigned int merge_count;
2719        unsigned char prev_fanout, new_fanout;
2720        const char *v;
2721
2722        b = lookup_branch(arg);
2723        if (!b)
2724                b = new_branch(arg);
2725
2726        read_next_command();
2727        parse_mark();
2728        if (skip_prefix(command_buf.buf, "author ", &v)) {
2729                author = parse_ident(v);
2730                read_next_command();
2731        }
2732        if (skip_prefix(command_buf.buf, "committer ", &v)) {
2733                committer = parse_ident(v);
2734                read_next_command();
2735        }
2736        if (!committer)
2737                die("Expected committer but didn't get one");
2738        parse_data(&msg, 0, NULL);
2739        read_next_command();
2740        parse_from(b);
2741        merge_list = parse_merge(&merge_count);
2742
2743        /* ensure the branch is active/loaded */
2744        if (!b->branch_tree.tree || !max_active_branches) {
2745                unload_one_branch();
2746                load_branch(b);
2747        }
2748
2749        prev_fanout = convert_num_notes_to_fanout(b->num_notes);
2750
2751        /* file_change* */
2752        while (command_buf.len > 0) {
2753                if (skip_prefix(command_buf.buf, "M ", &v))
2754                        file_change_m(v, b);
2755                else if (skip_prefix(command_buf.buf, "D ", &v))
2756                        file_change_d(v, b);
2757                else if (skip_prefix(command_buf.buf, "R ", &v))
2758                        file_change_cr(v, b, 1);
2759                else if (skip_prefix(command_buf.buf, "C ", &v))
2760                        file_change_cr(v, b, 0);
2761                else if (skip_prefix(command_buf.buf, "N ", &v))
2762                        note_change_n(v, b, &prev_fanout);
2763                else if (!strcmp("deleteall", command_buf.buf))
2764                        file_change_deleteall(b);
2765                else if (skip_prefix(command_buf.buf, "ls ", &v))
2766                        parse_ls(v, b);
2767                else {
2768                        unread_command_buf = 1;
2769                        break;
2770                }
2771                if (read_next_command() == EOF)
2772                        break;
2773        }
2774
2775        new_fanout = convert_num_notes_to_fanout(b->num_notes);
2776        if (new_fanout != prev_fanout)
2777                b->num_notes = change_note_fanout(&b->branch_tree, new_fanout);
2778
2779        /* build the tree and the commit */
2780        store_tree(&b->branch_tree);
2781        hashcpy(b->branch_tree.versions[0].sha1,
2782                b->branch_tree.versions[1].sha1);
2783
2784        strbuf_reset(&new_data);
2785        strbuf_addf(&new_data, "tree %s\n",
2786                sha1_to_hex(b->branch_tree.versions[1].sha1));
2787        if (!is_null_sha1(b->sha1))
2788                strbuf_addf(&new_data, "parent %s\n", sha1_to_hex(b->sha1));
2789        while (merge_list) {
2790                struct hash_list *next = merge_list->next;
2791                strbuf_addf(&new_data, "parent %s\n", sha1_to_hex(merge_list->sha1));
2792                free(merge_list);
2793                merge_list = next;
2794        }
2795        strbuf_addf(&new_data,
2796                "author %s\n"
2797                "committer %s\n"
2798                "\n",
2799                author ? author : committer, committer);
2800        strbuf_addbuf(&new_data, &msg);
2801        free(author);
2802        free(committer);
2803
2804        if (!store_object(OBJ_COMMIT, &new_data, NULL, b->sha1, next_mark))
2805                b->pack_id = pack_id;
2806        b->last_commit = object_count_by_type[OBJ_COMMIT];
2807}
2808
2809static void parse_new_tag(const char *arg)
2810{
2811        static struct strbuf msg = STRBUF_INIT;
2812        const char *from;
2813        char *tagger;
2814        struct branch *s;
2815        struct tag *t;
2816        uintmax_t from_mark = 0;
2817        unsigned char sha1[20];
2818        enum object_type type;
2819        const char *v;
2820
2821        t = pool_alloc(sizeof(struct tag));
2822        memset(t, 0, sizeof(struct tag));
2823        t->name = pool_strdup(arg);
2824        if (last_tag)
2825                last_tag->next_tag = t;
2826        else
2827                first_tag = t;
2828        last_tag = t;
2829        read_next_command();
2830
2831        /* from ... */
2832        if (!skip_prefix(command_buf.buf, "from ", &from))
2833                die("Expected from command, got %s", command_buf.buf);
2834        s = lookup_branch(from);
2835        if (s) {
2836                if (is_null_sha1(s->sha1))
2837                        die("Can't tag an empty branch.");
2838                hashcpy(sha1, s->sha1);
2839                type = OBJ_COMMIT;
2840        } else if (*from == ':') {
2841                struct object_entry *oe;
2842                from_mark = parse_mark_ref_eol(from);
2843                oe = find_mark(from_mark);
2844                type = oe->type;
2845                hashcpy(sha1, oe->idx.sha1);
2846        } else if (!get_sha1(from, sha1)) {
2847                struct object_entry *oe = find_object(sha1);
2848                if (!oe) {
2849                        type = sha1_object_info(sha1, NULL);
2850                        if (type < 0)
2851                                die("Not a valid object: %s", from);
2852                } else
2853                        type = oe->type;
2854        } else
2855                die("Invalid ref name or SHA1 expression: %s", from);
2856        read_next_command();
2857
2858        /* tagger ... */
2859        if (skip_prefix(command_buf.buf, "tagger ", &v)) {
2860                tagger = parse_ident(v);
2861                read_next_command();
2862        } else
2863                tagger = NULL;
2864
2865        /* tag payload/message */
2866        parse_data(&msg, 0, NULL);
2867
2868        /* build the tag object */
2869        strbuf_reset(&new_data);
2870
2871        strbuf_addf(&new_data,
2872                    "object %s\n"
2873                    "type %s\n"
2874                    "tag %s\n",
2875                    sha1_to_hex(sha1), typename(type), t->name);
2876        if (tagger)
2877                strbuf_addf(&new_data,
2878                            "tagger %s\n", tagger);
2879        strbuf_addch(&new_data, '\n');
2880        strbuf_addbuf(&new_data, &msg);
2881        free(tagger);
2882
2883        if (store_object(OBJ_TAG, &new_data, NULL, t->sha1, 0))
2884                t->pack_id = MAX_PACK_ID;
2885        else
2886                t->pack_id = pack_id;
2887}
2888
2889static void parse_reset_branch(const char *arg)
2890{
2891        struct branch *b;
2892
2893        b = lookup_branch(arg);
2894        if (b) {
2895                hashclr(b->sha1);
2896                hashclr(b->branch_tree.versions[0].sha1);
2897                hashclr(b->branch_tree.versions[1].sha1);
2898                if (b->branch_tree.tree) {
2899                        release_tree_content_recursive(b->branch_tree.tree);
2900                        b->branch_tree.tree = NULL;
2901                }
2902        }
2903        else
2904                b = new_branch(arg);
2905        read_next_command();
2906        parse_from(b);
2907        if (command_buf.len > 0)
2908                unread_command_buf = 1;
2909}
2910
2911static void cat_blob_write(const char *buf, unsigned long size)
2912{
2913        if (write_in_full(cat_blob_fd, buf, size) != size)
2914                die_errno("Write to frontend failed");
2915}
2916
2917static void cat_blob(struct object_entry *oe, unsigned char sha1[20])
2918{
2919        struct strbuf line = STRBUF_INIT;
2920        unsigned long size;
2921        enum object_type type = 0;
2922        char *buf;
2923
2924        if (!oe || oe->pack_id == MAX_PACK_ID) {
2925                buf = read_sha1_file(sha1, &type, &size);
2926        } else {
2927                type = oe->type;
2928                buf = gfi_unpack_entry(oe, &size);
2929        }
2930
2931        /*
2932         * Output based on batch_one_object() from cat-file.c.
2933         */
2934        if (type <= 0) {
2935                strbuf_reset(&line);
2936                strbuf_addf(&line, "%s missing\n", sha1_to_hex(sha1));
2937                cat_blob_write(line.buf, line.len);
2938                strbuf_release(&line);
2939                free(buf);
2940                return;
2941        }
2942        if (!buf)
2943                die("Can't read object %s", sha1_to_hex(sha1));
2944        if (type != OBJ_BLOB)
2945                die("Object %s is a %s but a blob was expected.",
2946                    sha1_to_hex(sha1), typename(type));
2947        strbuf_reset(&line);
2948        strbuf_addf(&line, "%s %s %lu\n", sha1_to_hex(sha1),
2949                                                typename(type), size);
2950        cat_blob_write(line.buf, line.len);
2951        strbuf_release(&line);
2952        cat_blob_write(buf, size);
2953        cat_blob_write("\n", 1);
2954        if (oe && oe->pack_id == pack_id) {
2955                last_blob.offset = oe->idx.offset;
2956                strbuf_attach(&last_blob.data, buf, size, size);
2957                last_blob.depth = oe->depth;
2958        } else
2959                free(buf);
2960}
2961
2962static void parse_get_mark(const char *p)
2963{
2964        struct object_entry *oe = oe;
2965        char output[42];
2966
2967        /* get-mark SP <object> LF */
2968        if (*p != ':')
2969                die("Not a mark: %s", p);
2970
2971        oe = find_mark(parse_mark_ref_eol(p));
2972        if (!oe)
2973                die("Unknown mark: %s", command_buf.buf);
2974
2975        snprintf(output, sizeof(output), "%s\n", sha1_to_hex(oe->idx.sha1));
2976        cat_blob_write(output, 41);
2977}
2978
2979static void parse_cat_blob(const char *p)
2980{
2981        struct object_entry *oe = oe;
2982        unsigned char sha1[20];
2983
2984        /* cat-blob SP <object> LF */
2985        if (*p == ':') {
2986                oe = find_mark(parse_mark_ref_eol(p));
2987                if (!oe)
2988                        die("Unknown mark: %s", command_buf.buf);
2989                hashcpy(sha1, oe->idx.sha1);
2990        } else {
2991                if (get_sha1_hex(p, sha1))
2992                        die("Invalid dataref: %s", command_buf.buf);
2993                if (p[40])
2994                        die("Garbage after SHA1: %s", command_buf.buf);
2995                oe = find_object(sha1);
2996        }
2997
2998        cat_blob(oe, sha1);
2999}
3000
3001static struct object_entry *dereference(struct object_entry *oe,
3002                                        unsigned char sha1[20])
3003{
3004        unsigned long size;
3005        char *buf = NULL;
3006        if (!oe) {
3007                enum object_type type = sha1_object_info(sha1, NULL);
3008                if (type < 0)
3009                        die("object not found: %s", sha1_to_hex(sha1));
3010                /* cache it! */
3011                oe = insert_object(sha1);
3012                oe->type = type;
3013                oe->pack_id = MAX_PACK_ID;
3014                oe->idx.offset = 1;
3015        }
3016        switch (oe->type) {
3017        case OBJ_TREE:  /* easy case. */
3018                return oe;
3019        case OBJ_COMMIT:
3020        case OBJ_TAG:
3021                break;
3022        default:
3023                die("Not a tree-ish: %s", command_buf.buf);
3024        }
3025
3026        if (oe->pack_id != MAX_PACK_ID) {       /* in a pack being written */
3027                buf = gfi_unpack_entry(oe, &size);
3028        } else {
3029                enum object_type unused;
3030                buf = read_sha1_file(sha1, &unused, &size);
3031        }
3032        if (!buf)
3033                die("Can't load object %s", sha1_to_hex(sha1));
3034
3035        /* Peel one layer. */
3036        switch (oe->type) {
3037        case OBJ_TAG:
3038                if (size < 40 + strlen("object ") ||
3039                    get_sha1_hex(buf + strlen("object "), sha1))
3040                        die("Invalid SHA1 in tag: %s", command_buf.buf);
3041                break;
3042        case OBJ_COMMIT:
3043                if (size < 40 + strlen("tree ") ||
3044                    get_sha1_hex(buf + strlen("tree "), sha1))
3045                        die("Invalid SHA1 in commit: %s", command_buf.buf);
3046        }
3047
3048        free(buf);
3049        return find_object(sha1);
3050}
3051
3052static struct object_entry *parse_treeish_dataref(const char **p)
3053{
3054        unsigned char sha1[20];
3055        struct object_entry *e;
3056
3057        if (**p == ':') {       /* <mark> */
3058                e = find_mark(parse_mark_ref_space(p));
3059                if (!e)
3060                        die("Unknown mark: %s", command_buf.buf);
3061                hashcpy(sha1, e->idx.sha1);
3062        } else {        /* <sha1> */
3063                if (get_sha1_hex(*p, sha1))
3064                        die("Invalid dataref: %s", command_buf.buf);
3065                e = find_object(sha1);
3066                *p += 40;
3067                if (*(*p)++ != ' ')
3068                        die("Missing space after tree-ish: %s", command_buf.buf);
3069        }
3070
3071        while (!e || e->type != OBJ_TREE)
3072                e = dereference(e, sha1);
3073        return e;
3074}
3075
3076static void print_ls(int mode, const unsigned char *sha1, const char *path)
3077{
3078        static struct strbuf line = STRBUF_INIT;
3079
3080        /* See show_tree(). */
3081        const char *type =
3082                S_ISGITLINK(mode) ? commit_type :
3083                S_ISDIR(mode) ? tree_type :
3084                blob_type;
3085
3086        if (!mode) {
3087                /* missing SP path LF */
3088                strbuf_reset(&line);
3089                strbuf_addstr(&line, "missing ");
3090                quote_c_style(path, &line, NULL, 0);
3091                strbuf_addch(&line, '\n');
3092        } else {
3093                /* mode SP type SP object_name TAB path LF */
3094                strbuf_reset(&line);
3095                strbuf_addf(&line, "%06o %s %s\t",
3096                                mode & ~NO_DELTA, type, sha1_to_hex(sha1));
3097                quote_c_style(path, &line, NULL, 0);
3098                strbuf_addch(&line, '\n');
3099        }
3100        cat_blob_write(line.buf, line.len);
3101}
3102
3103static void parse_ls(const char *p, struct branch *b)
3104{
3105        struct tree_entry *root = NULL;
3106        struct tree_entry leaf = {NULL};
3107
3108        /* ls SP (<tree-ish> SP)? <path> */
3109        if (*p == '"') {
3110                if (!b)
3111                        die("Not in a commit: %s", command_buf.buf);
3112                root = &b->branch_tree;
3113        } else {
3114                struct object_entry *e = parse_treeish_dataref(&p);
3115                root = new_tree_entry();
3116                hashcpy(root->versions[1].sha1, e->idx.sha1);
3117                if (!is_null_sha1(root->versions[1].sha1))
3118                        root->versions[1].mode = S_IFDIR;
3119                load_tree(root);
3120        }
3121        if (*p == '"') {
3122                static struct strbuf uq = STRBUF_INIT;
3123                const char *endp;
3124                strbuf_reset(&uq);
3125                if (unquote_c_style(&uq, p, &endp))
3126                        die("Invalid path: %s", command_buf.buf);
3127                if (*endp)
3128                        die("Garbage after path in: %s", command_buf.buf);
3129                p = uq.buf;
3130        }
3131        tree_content_get(root, p, &leaf, 1);
3132        /*
3133         * A directory in preparation would have a sha1 of zero
3134         * until it is saved.  Save, for simplicity.
3135         */
3136        if (S_ISDIR(leaf.versions[1].mode))
3137                store_tree(&leaf);
3138
3139        print_ls(leaf.versions[1].mode, leaf.versions[1].sha1, p);
3140        if (leaf.tree)
3141                release_tree_content_recursive(leaf.tree);
3142        if (!b || root != &b->branch_tree)
3143                release_tree_entry(root);
3144}
3145
3146static void checkpoint(void)
3147{
3148        checkpoint_requested = 0;
3149        if (object_count) {
3150                cycle_packfile();
3151                dump_branches();
3152                dump_tags();
3153                dump_marks();
3154        }
3155}
3156
3157static void parse_checkpoint(void)
3158{
3159        checkpoint_requested = 1;
3160        skip_optional_lf();
3161}
3162
3163static void parse_progress(void)
3164{
3165        fwrite(command_buf.buf, 1, command_buf.len, stdout);
3166        fputc('\n', stdout);
3167        fflush(stdout);
3168        skip_optional_lf();
3169}
3170
3171static char* make_fast_import_path(const char *path)
3172{
3173        if (!relative_marks_paths || is_absolute_path(path))
3174                return xstrdup(path);
3175        return xstrdup(git_path("info/fast-import/%s", path));
3176}
3177
3178static void option_import_marks(const char *marks,
3179                                        int from_stream, int ignore_missing)
3180{
3181        if (import_marks_file) {
3182                if (from_stream)
3183                        die("Only one import-marks command allowed per stream");
3184
3185                /* read previous mark file */
3186                if(!import_marks_file_from_stream)
3187                        read_marks();
3188        }
3189
3190        import_marks_file = make_fast_import_path(marks);
3191        safe_create_leading_directories_const(import_marks_file);
3192        import_marks_file_from_stream = from_stream;
3193        import_marks_file_ignore_missing = ignore_missing;
3194}
3195
3196static void option_date_format(const char *fmt)
3197{
3198        if (!strcmp(fmt, "raw"))
3199                whenspec = WHENSPEC_RAW;
3200        else if (!strcmp(fmt, "rfc2822"))
3201                whenspec = WHENSPEC_RFC2822;
3202        else if (!strcmp(fmt, "now"))
3203                whenspec = WHENSPEC_NOW;
3204        else
3205                die("unknown --date-format argument %s", fmt);
3206}
3207
3208static unsigned long ulong_arg(const char *option, const char *arg)
3209{
3210        char *endptr;
3211        unsigned long rv = strtoul(arg, &endptr, 0);
3212        if (strchr(arg, '-') || endptr == arg || *endptr)
3213                die("%s: argument must be a non-negative integer", option);
3214        return rv;
3215}
3216
3217static void option_depth(const char *depth)
3218{
3219        max_depth = ulong_arg("--depth", depth);
3220        if (max_depth > MAX_DEPTH)
3221                die("--depth cannot exceed %u", MAX_DEPTH);
3222}
3223
3224static void option_active_branches(const char *branches)
3225{
3226        max_active_branches = ulong_arg("--active-branches", branches);
3227}
3228
3229static void option_export_marks(const char *marks)
3230{
3231        export_marks_file = make_fast_import_path(marks);
3232        safe_create_leading_directories_const(export_marks_file);
3233}
3234
3235static void option_cat_blob_fd(const char *fd)
3236{
3237        unsigned long n = ulong_arg("--cat-blob-fd", fd);
3238        if (n > (unsigned long) INT_MAX)
3239                die("--cat-blob-fd cannot exceed %d", INT_MAX);
3240        cat_blob_fd = (int) n;
3241}
3242
3243static void option_export_pack_edges(const char *edges)
3244{
3245        if (pack_edges)
3246                fclose(pack_edges);
3247        pack_edges = fopen(edges, "a");
3248        if (!pack_edges)
3249                die_errno("Cannot open '%s'", edges);
3250}
3251
3252static int parse_one_option(const char *option)
3253{
3254        if (skip_prefix(option, "max-pack-size=", &option)) {
3255                unsigned long v;
3256                if (!git_parse_ulong(option, &v))
3257                        return 0;
3258                if (v < 8192) {
3259                        warning("max-pack-size is now in bytes, assuming --max-pack-size=%lum", v);
3260                        v *= 1024 * 1024;
3261                } else if (v < 1024 * 1024) {
3262                        warning("minimum max-pack-size is 1 MiB");
3263                        v = 1024 * 1024;
3264                }
3265                max_packsize = v;
3266        } else if (skip_prefix(option, "big-file-threshold=", &option)) {
3267                unsigned long v;
3268                if (!git_parse_ulong(option, &v))
3269                        return 0;
3270                big_file_threshold = v;
3271        } else if (skip_prefix(option, "depth=", &option)) {
3272                option_depth(option);
3273        } else if (skip_prefix(option, "active-branches=", &option)) {
3274                option_active_branches(option);
3275        } else if (skip_prefix(option, "export-pack-edges=", &option)) {
3276                option_export_pack_edges(option);
3277        } else if (starts_with(option, "quiet")) {
3278                show_stats = 0;
3279        } else if (starts_with(option, "stats")) {
3280                show_stats = 1;
3281        } else {
3282                return 0;
3283        }
3284
3285        return 1;
3286}
3287
3288static int parse_one_feature(const char *feature, int from_stream)
3289{
3290        const char *arg;
3291
3292        if (skip_prefix(feature, "date-format=", &arg)) {
3293                option_date_format(arg);
3294        } else if (skip_prefix(feature, "import-marks=", &arg)) {
3295                option_import_marks(arg, from_stream, 0);
3296        } else if (skip_prefix(feature, "import-marks-if-exists=", &arg)) {
3297                option_import_marks(arg, from_stream, 1);
3298        } else if (skip_prefix(feature, "export-marks=", &arg)) {
3299                option_export_marks(arg);
3300        } else if (!strcmp(feature, "get-mark")) {
3301                ; /* Don't die - this feature is supported */
3302        } else if (!strcmp(feature, "cat-blob")) {
3303                ; /* Don't die - this feature is supported */
3304        } else if (!strcmp(feature, "relative-marks")) {
3305                relative_marks_paths = 1;
3306        } else if (!strcmp(feature, "no-relative-marks")) {
3307                relative_marks_paths = 0;
3308        } else if (!strcmp(feature, "done")) {
3309                require_explicit_termination = 1;
3310        } else if (!strcmp(feature, "force")) {
3311                force_update = 1;
3312        } else if (!strcmp(feature, "notes") || !strcmp(feature, "ls")) {
3313                ; /* do nothing; we have the feature */
3314        } else {
3315                return 0;
3316        }
3317
3318        return 1;
3319}
3320
3321static void parse_feature(const char *feature)
3322{
3323        if (seen_data_command)
3324                die("Got feature command '%s' after data command", feature);
3325
3326        if (parse_one_feature(feature, 1))
3327                return;
3328
3329        die("This version of fast-import does not support feature %s.", feature);
3330}
3331
3332static void parse_option(const char *option)
3333{
3334        if (seen_data_command)
3335                die("Got option command '%s' after data command", option);
3336
3337        if (parse_one_option(option))
3338                return;
3339
3340        die("This version of fast-import does not support option: %s", option);
3341}
3342
3343static void git_pack_config(void)
3344{
3345        int indexversion_value;
3346        int limit;
3347        unsigned long packsizelimit_value;
3348
3349        if (!git_config_get_ulong("pack.depth", &max_depth)) {
3350                if (max_depth > MAX_DEPTH)
3351                        max_depth = MAX_DEPTH;
3352        }
3353        if (!git_config_get_int("pack.compression", &pack_compression_level)) {
3354                if (pack_compression_level == -1)
3355                        pack_compression_level = Z_DEFAULT_COMPRESSION;
3356                else if (pack_compression_level < 0 ||
3357                         pack_compression_level > Z_BEST_COMPRESSION)
3358                        git_die_config("pack.compression",
3359                                        "bad pack compression level %d", pack_compression_level);
3360                pack_compression_seen = 1;
3361        }
3362        if (!git_config_get_int("pack.indexversion", &indexversion_value)) {
3363                pack_idx_opts.version = indexversion_value;
3364                if (pack_idx_opts.version > 2)
3365                        git_die_config("pack.indexversion",
3366                                        "bad pack.indexversion=%"PRIu32, pack_idx_opts.version);
3367        }
3368        if (!git_config_get_ulong("pack.packsizelimit", &packsizelimit_value))
3369                max_packsize = packsizelimit_value;
3370
3371        if (!git_config_get_int("fastimport.unpacklimit", &limit))
3372                unpack_limit = limit;
3373        else if (!git_config_get_int("transfer.unpacklimit", &limit))
3374                unpack_limit = limit;
3375
3376        git_config(git_default_config, NULL);
3377}
3378
3379static const char fast_import_usage[] =
3380"git fast-import [--date-format=<f>] [--max-pack-size=<n>] [--big-file-threshold=<n>] [--depth=<n>] [--active-branches=<n>] [--export-marks=<marks.file>]";
3381
3382static void parse_argv(void)
3383{
3384        unsigned int i;
3385
3386        for (i = 1; i < global_argc; i++) {
3387                const char *a = global_argv[i];
3388
3389                if (*a != '-' || !strcmp(a, "--"))
3390                        break;
3391
3392                if (!skip_prefix(a, "--", &a))
3393                        die("unknown option %s", a);
3394
3395                if (parse_one_option(a))
3396                        continue;
3397
3398                if (parse_one_feature(a, 0))
3399                        continue;
3400
3401                if (skip_prefix(a, "cat-blob-fd=", &a)) {
3402                        option_cat_blob_fd(a);
3403                        continue;
3404                }
3405
3406                die("unknown option --%s", a);
3407        }
3408        if (i != global_argc)
3409                usage(fast_import_usage);
3410
3411        seen_data_command = 1;
3412        if (import_marks_file)
3413                read_marks();
3414}
3415
3416int main(int argc, char **argv)
3417{
3418        unsigned int i;
3419
3420        git_extract_argv0_path(argv[0]);
3421
3422        git_setup_gettext();
3423
3424        if (argc == 2 && !strcmp(argv[1], "-h"))
3425                usage(fast_import_usage);
3426
3427        setup_git_directory();
3428        reset_pack_idx_option(&pack_idx_opts);
3429        git_pack_config();
3430        if (!pack_compression_seen && core_compression_seen)
3431                pack_compression_level = core_compression_level;
3432
3433        alloc_objects(object_entry_alloc);
3434        strbuf_init(&command_buf, 0);
3435        atom_table = xcalloc(atom_table_sz, sizeof(struct atom_str*));
3436        branch_table = xcalloc(branch_table_sz, sizeof(struct branch*));
3437        avail_tree_table = xcalloc(avail_tree_table_sz, sizeof(struct avail_tree_content*));
3438        marks = pool_calloc(1, sizeof(struct mark_set));
3439
3440        global_argc = argc;
3441        global_argv = argv;
3442
3443        rc_free = pool_alloc(cmd_save * sizeof(*rc_free));
3444        for (i = 0; i < (cmd_save - 1); i++)
3445                rc_free[i].next = &rc_free[i + 1];
3446        rc_free[cmd_save - 1].next = NULL;
3447
3448        prepare_packed_git();
3449        start_packfile();
3450        set_die_routine(die_nicely);
3451        set_checkpoint_signal();
3452        while (read_next_command() != EOF) {
3453                const char *v;
3454                if (!strcmp("blob", command_buf.buf))
3455                        parse_new_blob();
3456                else if (skip_prefix(command_buf.buf, "ls ", &v))
3457                        parse_ls(v, NULL);
3458                else if (skip_prefix(command_buf.buf, "commit ", &v))
3459                        parse_new_commit(v);
3460                else if (skip_prefix(command_buf.buf, "tag ", &v))
3461                        parse_new_tag(v);
3462                else if (skip_prefix(command_buf.buf, "reset ", &v))
3463                        parse_reset_branch(v);
3464                else if (!strcmp("checkpoint", command_buf.buf))
3465                        parse_checkpoint();
3466                else if (!strcmp("done", command_buf.buf))
3467                        break;
3468                else if (starts_with(command_buf.buf, "progress "))
3469                        parse_progress();
3470                else if (skip_prefix(command_buf.buf, "feature ", &v))
3471                        parse_feature(v);
3472                else if (skip_prefix(command_buf.buf, "option git ", &v))
3473                        parse_option(v);
3474                else if (starts_with(command_buf.buf, "option "))
3475                        /* ignore non-git options*/;
3476                else
3477                        die("Unsupported command: %s", command_buf.buf);
3478
3479                if (checkpoint_requested)
3480                        checkpoint();
3481        }
3482
3483        /* argv hasn't been parsed yet, do so */
3484        if (!seen_data_command)
3485                parse_argv();
3486
3487        if (require_explicit_termination && feof(stdin))
3488                die("stream ends early");
3489
3490        end_packfile();
3491
3492        dump_branches();
3493        dump_tags();
3494        unkeep_all_packs();
3495        dump_marks();
3496
3497        if (pack_edges)
3498                fclose(pack_edges);
3499
3500        if (show_stats) {
3501                uintmax_t total_count = 0, duplicate_count = 0;
3502                for (i = 0; i < ARRAY_SIZE(object_count_by_type); i++)
3503                        total_count += object_count_by_type[i];
3504                for (i = 0; i < ARRAY_SIZE(duplicate_count_by_type); i++)
3505                        duplicate_count += duplicate_count_by_type[i];
3506
3507                fprintf(stderr, "%s statistics:\n", argv[0]);
3508                fprintf(stderr, "---------------------------------------------------------------------\n");
3509                fprintf(stderr, "Alloc'd objects: %10" PRIuMAX "\n", alloc_count);
3510                fprintf(stderr, "Total objects:   %10" PRIuMAX " (%10" PRIuMAX " duplicates                  )\n", total_count, duplicate_count);
3511                fprintf(stderr, "      blobs  :   %10" PRIuMAX " (%10" PRIuMAX " duplicates %10" PRIuMAX " deltas of %10" PRIuMAX" attempts)\n", object_count_by_type[OBJ_BLOB], duplicate_count_by_type[OBJ_BLOB], delta_count_by_type[OBJ_BLOB], delta_count_attempts_by_type[OBJ_BLOB]);
3512                fprintf(stderr, "      trees  :   %10" PRIuMAX " (%10" PRIuMAX " duplicates %10" PRIuMAX " deltas of %10" PRIuMAX" attempts)\n", object_count_by_type[OBJ_TREE], duplicate_count_by_type[OBJ_TREE], delta_count_by_type[OBJ_TREE], delta_count_attempts_by_type[OBJ_TREE]);
3513                fprintf(stderr, "      commits:   %10" PRIuMAX " (%10" PRIuMAX " duplicates %10" PRIuMAX " deltas of %10" PRIuMAX" attempts)\n", object_count_by_type[OBJ_COMMIT], duplicate_count_by_type[OBJ_COMMIT], delta_count_by_type[OBJ_COMMIT], delta_count_attempts_by_type[OBJ_COMMIT]);
3514                fprintf(stderr, "      tags   :   %10" PRIuMAX " (%10" PRIuMAX " duplicates %10" PRIuMAX " deltas of %10" PRIuMAX" attempts)\n", object_count_by_type[OBJ_TAG], duplicate_count_by_type[OBJ_TAG], delta_count_by_type[OBJ_TAG], delta_count_attempts_by_type[OBJ_TAG]);
3515                fprintf(stderr, "Total branches:  %10lu (%10lu loads     )\n", branch_count, branch_load_count);
3516                fprintf(stderr, "      marks:     %10" PRIuMAX " (%10" PRIuMAX " unique    )\n", (((uintmax_t)1) << marks->shift) * 1024, marks_set_count);
3517                fprintf(stderr, "      atoms:     %10u\n", atom_cnt);
3518                fprintf(stderr, "Memory total:    %10" PRIuMAX " KiB\n", (total_allocd + alloc_count*sizeof(struct object_entry))/1024);
3519                fprintf(stderr, "       pools:    %10lu KiB\n", (unsigned long)(total_allocd/1024));
3520                fprintf(stderr, "     objects:    %10" PRIuMAX " KiB\n", (alloc_count*sizeof(struct object_entry))/1024);
3521                fprintf(stderr, "---------------------------------------------------------------------\n");
3522                pack_report();
3523                fprintf(stderr, "---------------------------------------------------------------------\n");
3524                fprintf(stderr, "\n");
3525        }
3526
3527        return failure ? 1 : 0;
3528}