builtin / unpack-objects.con commit Merge branch 'tb/grep-column' (d036d66)
   1#include "builtin.h"
   2#include "cache.h"
   3#include "config.h"
   4#include "object-store.h"
   5#include "object.h"
   6#include "delta.h"
   7#include "pack.h"
   8#include "blob.h"
   9#include "commit.h"
  10#include "tag.h"
  11#include "tree.h"
  12#include "tree-walk.h"
  13#include "progress.h"
  14#include "decorate.h"
  15#include "fsck.h"
  16
  17static int dry_run, quiet, recover, has_errors, strict;
  18static const char unpack_usage[] = "git unpack-objects [-n] [-q] [-r] [--strict]";
  19
  20/* We always read in 4kB chunks. */
  21static unsigned char buffer[4096];
  22static unsigned int offset, len;
  23static off_t consumed_bytes;
  24static off_t max_input_size;
  25static git_hash_ctx ctx;
  26static struct fsck_options fsck_options = FSCK_OPTIONS_STRICT;
  27
  28/*
  29 * When running under --strict mode, objects whose reachability are
  30 * suspect are kept in core without getting written in the object
  31 * store.
  32 */
  33struct obj_buffer {
  34        char *buffer;
  35        unsigned long size;
  36};
  37
  38static struct decoration obj_decorate;
  39
  40static struct obj_buffer *lookup_object_buffer(struct object *base)
  41{
  42        return lookup_decoration(&obj_decorate, base);
  43}
  44
  45static void add_object_buffer(struct object *object, char *buffer, unsigned long size)
  46{
  47        struct obj_buffer *obj;
  48        obj = xcalloc(1, sizeof(struct obj_buffer));
  49        obj->buffer = buffer;
  50        obj->size = size;
  51        if (add_decoration(&obj_decorate, object, obj))
  52                die("object %s tried to add buffer twice!", oid_to_hex(&object->oid));
  53}
  54
  55/*
  56 * Make sure at least "min" bytes are available in the buffer, and
  57 * return the pointer to the buffer.
  58 */
  59static void *fill(int min)
  60{
  61        if (min <= len)
  62                return buffer + offset;
  63        if (min > sizeof(buffer))
  64                die("cannot fill %d bytes", min);
  65        if (offset) {
  66                the_hash_algo->update_fn(&ctx, buffer, offset);
  67                memmove(buffer, buffer + offset, len);
  68                offset = 0;
  69        }
  70        do {
  71                ssize_t ret = xread(0, buffer + len, sizeof(buffer) - len);
  72                if (ret <= 0) {
  73                        if (!ret)
  74                                die("early EOF");
  75                        die_errno("read error on input");
  76                }
  77                len += ret;
  78        } while (len < min);
  79        return buffer;
  80}
  81
  82static void use(int bytes)
  83{
  84        if (bytes > len)
  85                die("used more bytes than were available");
  86        len -= bytes;
  87        offset += bytes;
  88
  89        /* make sure off_t is sufficiently large not to wrap */
  90        if (signed_add_overflows(consumed_bytes, bytes))
  91                die("pack too large for current definition of off_t");
  92        consumed_bytes += bytes;
  93        if (max_input_size && consumed_bytes > max_input_size)
  94                die(_("pack exceeds maximum allowed size"));
  95}
  96
  97static void *get_data(unsigned long size)
  98{
  99        git_zstream stream;
 100        void *buf = xmallocz(size);
 101
 102        memset(&stream, 0, sizeof(stream));
 103
 104        stream.next_out = buf;
 105        stream.avail_out = size;
 106        stream.next_in = fill(1);
 107        stream.avail_in = len;
 108        git_inflate_init(&stream);
 109
 110        for (;;) {
 111                int ret = git_inflate(&stream, 0);
 112                use(len - stream.avail_in);
 113                if (stream.total_out == size && ret == Z_STREAM_END)
 114                        break;
 115                if (ret != Z_OK) {
 116                        error("inflate returned %d", ret);
 117                        FREE_AND_NULL(buf);
 118                        if (!recover)
 119                                exit(1);
 120                        has_errors = 1;
 121                        break;
 122                }
 123                stream.next_in = fill(1);
 124                stream.avail_in = len;
 125        }
 126        git_inflate_end(&stream);
 127        return buf;
 128}
 129
 130struct delta_info {
 131        struct object_id base_oid;
 132        unsigned nr;
 133        off_t base_offset;
 134        unsigned long size;
 135        void *delta;
 136        struct delta_info *next;
 137};
 138
 139static struct delta_info *delta_list;
 140
 141static void add_delta_to_list(unsigned nr, const struct object_id *base_oid,
 142                              off_t base_offset,
 143                              void *delta, unsigned long size)
 144{
 145        struct delta_info *info = xmalloc(sizeof(*info));
 146
 147        oidcpy(&info->base_oid, base_oid);
 148        info->base_offset = base_offset;
 149        info->size = size;
 150        info->delta = delta;
 151        info->nr = nr;
 152        info->next = delta_list;
 153        delta_list = info;
 154}
 155
 156struct obj_info {
 157        off_t offset;
 158        struct object_id oid;
 159        struct object *obj;
 160};
 161
 162/* Remember to update object flag allocation in object.h */
 163#define FLAG_OPEN (1u<<20)
 164#define FLAG_WRITTEN (1u<<21)
 165
 166static struct obj_info *obj_list;
 167static unsigned nr_objects;
 168
 169/*
 170 * Called only from check_object() after it verified this object
 171 * is Ok.
 172 */
 173static void write_cached_object(struct object *obj, struct obj_buffer *obj_buf)
 174{
 175        struct object_id oid;
 176
 177        if (write_object_file(obj_buf->buffer, obj_buf->size,
 178                              type_name(obj->type), &oid) < 0)
 179                die("failed to write object %s", oid_to_hex(&obj->oid));
 180        obj->flags |= FLAG_WRITTEN;
 181}
 182
 183/*
 184 * At the very end of the processing, write_rest() scans the objects
 185 * that have reachability requirements and calls this function.
 186 * Verify its reachability and validity recursively and write it out.
 187 */
 188static int check_object(struct object *obj, int type, void *data, struct fsck_options *options)
 189{
 190        struct obj_buffer *obj_buf;
 191
 192        if (!obj)
 193                return 1;
 194
 195        if (obj->flags & FLAG_WRITTEN)
 196                return 0;
 197
 198        if (type != OBJ_ANY && obj->type != type)
 199                die("object type mismatch");
 200
 201        if (!(obj->flags & FLAG_OPEN)) {
 202                unsigned long size;
 203                int type = oid_object_info(the_repository, &obj->oid, &size);
 204                if (type != obj->type || type <= 0)
 205                        die("object of unexpected type");
 206                obj->flags |= FLAG_WRITTEN;
 207                return 0;
 208        }
 209
 210        obj_buf = lookup_object_buffer(obj);
 211        if (!obj_buf)
 212                die("Whoops! Cannot find object '%s'", oid_to_hex(&obj->oid));
 213        if (fsck_object(obj, obj_buf->buffer, obj_buf->size, &fsck_options))
 214                die("fsck error in packed object");
 215        fsck_options.walk = check_object;
 216        if (fsck_walk(obj, NULL, &fsck_options))
 217                die("Error on reachable objects of %s", oid_to_hex(&obj->oid));
 218        write_cached_object(obj, obj_buf);
 219        return 0;
 220}
 221
 222static void write_rest(void)
 223{
 224        unsigned i;
 225        for (i = 0; i < nr_objects; i++) {
 226                if (obj_list[i].obj)
 227                        check_object(obj_list[i].obj, OBJ_ANY, NULL, NULL);
 228        }
 229}
 230
 231static void added_object(unsigned nr, enum object_type type,
 232                         void *data, unsigned long size);
 233
 234/*
 235 * Write out nr-th object from the list, now we know the contents
 236 * of it.  Under --strict, this buffers structured objects in-core,
 237 * to be checked at the end.
 238 */
 239static void write_object(unsigned nr, enum object_type type,
 240                         void *buf, unsigned long size)
 241{
 242        if (!strict) {
 243                if (write_object_file(buf, size, type_name(type),
 244                                      &obj_list[nr].oid) < 0)
 245                        die("failed to write object");
 246                added_object(nr, type, buf, size);
 247                free(buf);
 248                obj_list[nr].obj = NULL;
 249        } else if (type == OBJ_BLOB) {
 250                struct blob *blob;
 251                if (write_object_file(buf, size, type_name(type),
 252                                      &obj_list[nr].oid) < 0)
 253                        die("failed to write object");
 254                added_object(nr, type, buf, size);
 255                free(buf);
 256
 257                blob = lookup_blob(&obj_list[nr].oid);
 258                if (blob)
 259                        blob->object.flags |= FLAG_WRITTEN;
 260                else
 261                        die("invalid blob object");
 262                obj_list[nr].obj = NULL;
 263        } else {
 264                struct object *obj;
 265                int eaten;
 266                hash_object_file(buf, size, type_name(type), &obj_list[nr].oid);
 267                added_object(nr, type, buf, size);
 268                obj = parse_object_buffer(&obj_list[nr].oid, type, size, buf,
 269                                          &eaten);
 270                if (!obj)
 271                        die("invalid %s", type_name(type));
 272                add_object_buffer(obj, buf, size);
 273                obj->flags |= FLAG_OPEN;
 274                obj_list[nr].obj = obj;
 275        }
 276}
 277
 278static void resolve_delta(unsigned nr, enum object_type type,
 279                          void *base, unsigned long base_size,
 280                          void *delta, unsigned long delta_size)
 281{
 282        void *result;
 283        unsigned long result_size;
 284
 285        result = patch_delta(base, base_size,
 286                             delta, delta_size,
 287                             &result_size);
 288        if (!result)
 289                die("failed to apply delta");
 290        free(delta);
 291        write_object(nr, type, result, result_size);
 292}
 293
 294/*
 295 * We now know the contents of an object (which is nr-th in the pack);
 296 * resolve all the deltified objects that are based on it.
 297 */
 298static void added_object(unsigned nr, enum object_type type,
 299                         void *data, unsigned long size)
 300{
 301        struct delta_info **p = &delta_list;
 302        struct delta_info *info;
 303
 304        while ((info = *p) != NULL) {
 305                if (!oidcmp(&info->base_oid, &obj_list[nr].oid) ||
 306                    info->base_offset == obj_list[nr].offset) {
 307                        *p = info->next;
 308                        p = &delta_list;
 309                        resolve_delta(info->nr, type, data, size,
 310                                      info->delta, info->size);
 311                        free(info);
 312                        continue;
 313                }
 314                p = &info->next;
 315        }
 316}
 317
 318static void unpack_non_delta_entry(enum object_type type, unsigned long size,
 319                                   unsigned nr)
 320{
 321        void *buf = get_data(size);
 322
 323        if (!dry_run && buf)
 324                write_object(nr, type, buf, size);
 325        else
 326                free(buf);
 327}
 328
 329static int resolve_against_held(unsigned nr, const struct object_id *base,
 330                                void *delta_data, unsigned long delta_size)
 331{
 332        struct object *obj;
 333        struct obj_buffer *obj_buffer;
 334        obj = lookup_object(base->hash);
 335        if (!obj)
 336                return 0;
 337        obj_buffer = lookup_object_buffer(obj);
 338        if (!obj_buffer)
 339                return 0;
 340        resolve_delta(nr, obj->type, obj_buffer->buffer,
 341                      obj_buffer->size, delta_data, delta_size);
 342        return 1;
 343}
 344
 345static void unpack_delta_entry(enum object_type type, unsigned long delta_size,
 346                               unsigned nr)
 347{
 348        void *delta_data, *base;
 349        unsigned long base_size;
 350        struct object_id base_oid;
 351
 352        if (type == OBJ_REF_DELTA) {
 353                hashcpy(base_oid.hash, fill(the_hash_algo->rawsz));
 354                use(the_hash_algo->rawsz);
 355                delta_data = get_data(delta_size);
 356                if (dry_run || !delta_data) {
 357                        free(delta_data);
 358                        return;
 359                }
 360                if (has_object_file(&base_oid))
 361                        ; /* Ok we have this one */
 362                else if (resolve_against_held(nr, &base_oid,
 363                                              delta_data, delta_size))
 364                        return; /* we are done */
 365                else {
 366                        /* cannot resolve yet --- queue it */
 367                        oidclr(&obj_list[nr].oid);
 368                        add_delta_to_list(nr, &base_oid, 0, delta_data, delta_size);
 369                        return;
 370                }
 371        } else {
 372                unsigned base_found = 0;
 373                unsigned char *pack, c;
 374                off_t base_offset;
 375                unsigned lo, mid, hi;
 376
 377                pack = fill(1);
 378                c = *pack;
 379                use(1);
 380                base_offset = c & 127;
 381                while (c & 128) {
 382                        base_offset += 1;
 383                        if (!base_offset || MSB(base_offset, 7))
 384                                die("offset value overflow for delta base object");
 385                        pack = fill(1);
 386                        c = *pack;
 387                        use(1);
 388                        base_offset = (base_offset << 7) + (c & 127);
 389                }
 390                base_offset = obj_list[nr].offset - base_offset;
 391                if (base_offset <= 0 || base_offset >= obj_list[nr].offset)
 392                        die("offset value out of bound for delta base object");
 393
 394                delta_data = get_data(delta_size);
 395                if (dry_run || !delta_data) {
 396                        free(delta_data);
 397                        return;
 398                }
 399                lo = 0;
 400                hi = nr;
 401                while (lo < hi) {
 402                        mid = lo + (hi - lo) / 2;
 403                        if (base_offset < obj_list[mid].offset) {
 404                                hi = mid;
 405                        } else if (base_offset > obj_list[mid].offset) {
 406                                lo = mid + 1;
 407                        } else {
 408                                oidcpy(&base_oid, &obj_list[mid].oid);
 409                                base_found = !is_null_oid(&base_oid);
 410                                break;
 411                        }
 412                }
 413                if (!base_found) {
 414                        /*
 415                         * The delta base object is itself a delta that
 416                         * has not been resolved yet.
 417                         */
 418                        oidclr(&obj_list[nr].oid);
 419                        add_delta_to_list(nr, &null_oid, base_offset, delta_data, delta_size);
 420                        return;
 421                }
 422        }
 423
 424        if (resolve_against_held(nr, &base_oid, delta_data, delta_size))
 425                return;
 426
 427        base = read_object_file(&base_oid, &type, &base_size);
 428        if (!base) {
 429                error("failed to read delta-pack base object %s",
 430                      oid_to_hex(&base_oid));
 431                if (!recover)
 432                        exit(1);
 433                has_errors = 1;
 434                return;
 435        }
 436        resolve_delta(nr, type, base, base_size, delta_data, delta_size);
 437        free(base);
 438}
 439
 440static void unpack_one(unsigned nr)
 441{
 442        unsigned shift;
 443        unsigned char *pack;
 444        unsigned long size, c;
 445        enum object_type type;
 446
 447        obj_list[nr].offset = consumed_bytes;
 448
 449        pack = fill(1);
 450        c = *pack;
 451        use(1);
 452        type = (c >> 4) & 7;
 453        size = (c & 15);
 454        shift = 4;
 455        while (c & 0x80) {
 456                pack = fill(1);
 457                c = *pack;
 458                use(1);
 459                size += (c & 0x7f) << shift;
 460                shift += 7;
 461        }
 462
 463        switch (type) {
 464        case OBJ_COMMIT:
 465        case OBJ_TREE:
 466        case OBJ_BLOB:
 467        case OBJ_TAG:
 468                unpack_non_delta_entry(type, size, nr);
 469                return;
 470        case OBJ_REF_DELTA:
 471        case OBJ_OFS_DELTA:
 472                unpack_delta_entry(type, size, nr);
 473                return;
 474        default:
 475                error("bad object type %d", type);
 476                has_errors = 1;
 477                if (recover)
 478                        return;
 479                exit(1);
 480        }
 481}
 482
 483static void unpack_all(void)
 484{
 485        int i;
 486        struct progress *progress = NULL;
 487        struct pack_header *hdr = fill(sizeof(struct pack_header));
 488
 489        nr_objects = ntohl(hdr->hdr_entries);
 490
 491        if (ntohl(hdr->hdr_signature) != PACK_SIGNATURE)
 492                die("bad pack file");
 493        if (!pack_version_ok(hdr->hdr_version))
 494                die("unknown pack file version %"PRIu32,
 495                        ntohl(hdr->hdr_version));
 496        use(sizeof(struct pack_header));
 497
 498        if (!quiet)
 499                progress = start_progress(_("Unpacking objects"), nr_objects);
 500        obj_list = xcalloc(nr_objects, sizeof(*obj_list));
 501        for (i = 0; i < nr_objects; i++) {
 502                unpack_one(i);
 503                display_progress(progress, i + 1);
 504        }
 505        stop_progress(&progress);
 506
 507        if (delta_list)
 508                die("unresolved deltas left after unpacking");
 509}
 510
 511int cmd_unpack_objects(int argc, const char **argv, const char *prefix)
 512{
 513        int i;
 514        struct object_id oid;
 515
 516        check_replace_refs = 0;
 517
 518        git_config(git_default_config, NULL);
 519
 520        quiet = !isatty(2);
 521
 522        for (i = 1 ; i < argc; i++) {
 523                const char *arg = argv[i];
 524
 525                if (*arg == '-') {
 526                        if (!strcmp(arg, "-n")) {
 527                                dry_run = 1;
 528                                continue;
 529                        }
 530                        if (!strcmp(arg, "-q")) {
 531                                quiet = 1;
 532                                continue;
 533                        }
 534                        if (!strcmp(arg, "-r")) {
 535                                recover = 1;
 536                                continue;
 537                        }
 538                        if (!strcmp(arg, "--strict")) {
 539                                strict = 1;
 540                                continue;
 541                        }
 542                        if (skip_prefix(arg, "--strict=", &arg)) {
 543                                strict = 1;
 544                                fsck_set_msg_types(&fsck_options, arg);
 545                                continue;
 546                        }
 547                        if (starts_with(arg, "--pack_header=")) {
 548                                struct pack_header *hdr;
 549                                char *c;
 550
 551                                hdr = (struct pack_header *)buffer;
 552                                hdr->hdr_signature = htonl(PACK_SIGNATURE);
 553                                hdr->hdr_version = htonl(strtoul(arg + 14, &c, 10));
 554                                if (*c != ',')
 555                                        die("bad %s", arg);
 556                                hdr->hdr_entries = htonl(strtoul(c + 1, &c, 10));
 557                                if (*c)
 558                                        die("bad %s", arg);
 559                                len = sizeof(*hdr);
 560                                continue;
 561                        }
 562                        if (skip_prefix(arg, "--max-input-size=", &arg)) {
 563                                max_input_size = strtoumax(arg, NULL, 10);
 564                                continue;
 565                        }
 566                        usage(unpack_usage);
 567                }
 568
 569                /* We don't take any non-flag arguments now.. Maybe some day */
 570                usage(unpack_usage);
 571        }
 572        the_hash_algo->init_fn(&ctx);
 573        unpack_all();
 574        the_hash_algo->update_fn(&ctx, buffer, offset);
 575        the_hash_algo->final_fn(oid.hash, &ctx);
 576        if (strict) {
 577                write_rest();
 578                if (fsck_finish(&fsck_options))
 579                        die(_("fsck error in pack objects"));
 580        }
 581        if (hashcmp(fill(the_hash_algo->rawsz), oid.hash))
 582                die("final sha1 did not match");
 583        use(the_hash_algo->rawsz);
 584
 585        /* Write the last part of the buffer to stdout */
 586        while (len) {
 587                int ret = xwrite(1, buffer + offset, len);
 588                if (ret <= 0)
 589                        break;
 590                len -= ret;
 591                offset += ret;
 592        }
 593
 594        /* All done */
 595        return has_errors;
 596}