1#include "builtin.h" 2#include "cache.h" 3#include "repository.h" 4#include "config.h" 5#include "attr.h" 6#include "object.h" 7#include "blob.h" 8#include "commit.h" 9#include "tag.h" 10#include "tree.h" 11#include "delta.h" 12#include "pack.h" 13#include "pack-revindex.h" 14#include "csum-file.h" 15#include "tree-walk.h" 16#include "diff.h" 17#include "revision.h" 18#include "list-objects.h" 19#include "list-objects-filter.h" 20#include "list-objects-filter-options.h" 21#include "pack-objects.h" 22#include "progress.h" 23#include "refs.h" 24#include "streaming.h" 25#include "thread-utils.h" 26#include "pack-bitmap.h" 27#include "reachable.h" 28#include "sha1-array.h" 29#include "argv-array.h" 30#include "list.h" 31#include "packfile.h" 32#include "object-store.h" 33#include "dir.h" 34 35#define IN_PACK(obj) oe_in_pack(&to_pack, obj) 36#define SIZE(obj) oe_size(&to_pack, obj) 37#define SET_SIZE(obj,size) oe_set_size(&to_pack, obj, size) 38#define DELTA_SIZE(obj) oe_delta_size(&to_pack, obj) 39#define DELTA(obj) oe_delta(&to_pack, obj) 40#define DELTA_CHILD(obj) oe_delta_child(&to_pack, obj) 41#define DELTA_SIBLING(obj) oe_delta_sibling(&to_pack, obj) 42#define SET_DELTA(obj, val) oe_set_delta(&to_pack, obj, val) 43#define SET_DELTA_SIZE(obj, val) oe_set_delta_size(&to_pack, obj, val) 44#define SET_DELTA_CHILD(obj, val) oe_set_delta_child(&to_pack, obj, val) 45#define SET_DELTA_SIBLING(obj, val) oe_set_delta_sibling(&to_pack, obj, val) 46 47static const char *pack_usage[] = { 48 N_("git pack-objects --stdout [<options>...] [< <ref-list> | < <object-list>]"), 49 N_("git pack-objects [<options>...] <base-name> [< <ref-list> | < <object-list>]"), 50 NULL 51}; 52 53/* 54 * Objects we are going to pack are collected in the `to_pack` structure. 55 * It contains an array (dynamically expanded) of the object data, and a map 56 * that can resolve SHA1s to their position in the array. 57 */ 58static struct packing_data to_pack; 59 60static struct pack_idx_entry **written_list; 61static uint32_t nr_result, nr_written, nr_seen; 62 63static int non_empty; 64static int reuse_delta = 1, reuse_object = 1; 65static int keep_unreachable, unpack_unreachable, include_tag; 66static timestamp_t unpack_unreachable_expiration; 67static int pack_loose_unreachable; 68static int local; 69static int have_non_local_packs; 70static int incremental; 71static int ignore_packed_keep_on_disk; 72static int ignore_packed_keep_in_core; 73static int allow_ofs_delta; 74static struct pack_idx_option pack_idx_opts; 75static const char *base_name; 76static int progress = 1; 77static int window = 10; 78static unsigned long pack_size_limit; 79static int depth = 50; 80static int delta_search_threads; 81static int pack_to_stdout; 82static int num_preferred_base; 83static struct progress *progress_state; 84 85static struct packed_git *reuse_packfile; 86static uint32_t reuse_packfile_objects; 87static off_t reuse_packfile_offset; 88 89static int use_bitmap_index_default = 1; 90static int use_bitmap_index = -1; 91static int write_bitmap_index; 92static uint16_t write_bitmap_options; 93 94static int exclude_promisor_objects; 95 96static unsigned long delta_cache_size = 0; 97static unsigned long max_delta_cache_size = DEFAULT_DELTA_CACHE_SIZE; 98static unsigned long cache_max_small_delta_size = 1000; 99 100static unsigned long window_memory_limit = 0; 101 102static struct list_objects_filter_options filter_options; 103 104enum missing_action { 105 MA_ERROR = 0, /* fail if any missing objects are encountered */ 106 MA_ALLOW_ANY, /* silently allow ALL missing objects */ 107 MA_ALLOW_PROMISOR, /* silently allow all missing PROMISOR objects */ 108}; 109static enum missing_action arg_missing_action; 110static show_object_fn fn_show_object; 111 112/* 113 * stats 114 */ 115static uint32_t written, written_delta; 116static uint32_t reused, reused_delta; 117 118/* 119 * Indexed commits 120 */ 121static struct commit **indexed_commits; 122static unsigned int indexed_commits_nr; 123static unsigned int indexed_commits_alloc; 124 125static void index_commit_for_bitmap(struct commit *commit) 126{ 127 if (indexed_commits_nr >= indexed_commits_alloc) { 128 indexed_commits_alloc = (indexed_commits_alloc + 32) * 2; 129 REALLOC_ARRAY(indexed_commits, indexed_commits_alloc); 130 } 131 132 indexed_commits[indexed_commits_nr++] = commit; 133} 134 135static void *get_delta(struct object_entry *entry) 136{ 137 unsigned long size, base_size, delta_size; 138 void *buf, *base_buf, *delta_buf; 139 enum object_type type; 140 141 buf = read_object_file(&entry->idx.oid, &type, &size); 142 if (!buf) 143 die("unable to read %s", oid_to_hex(&entry->idx.oid)); 144 base_buf = read_object_file(&DELTA(entry)->idx.oid, &type, 145 &base_size); 146 if (!base_buf) 147 die("unable to read %s", 148 oid_to_hex(&DELTA(entry)->idx.oid)); 149 delta_buf = diff_delta(base_buf, base_size, 150 buf, size, &delta_size, 0); 151 if (!delta_buf || delta_size != DELTA_SIZE(entry)) 152 die("delta size changed"); 153 free(buf); 154 free(base_buf); 155 return delta_buf; 156} 157 158static unsigned long do_compress(void **pptr, unsigned long size) 159{ 160 git_zstream stream; 161 void *in, *out; 162 unsigned long maxsize; 163 164 git_deflate_init(&stream, pack_compression_level); 165 maxsize = git_deflate_bound(&stream, size); 166 167 in = *pptr; 168 out = xmalloc(maxsize); 169 *pptr = out; 170 171 stream.next_in = in; 172 stream.avail_in = size; 173 stream.next_out = out; 174 stream.avail_out = maxsize; 175 while (git_deflate(&stream, Z_FINISH) == Z_OK) 176 ; /* nothing */ 177 git_deflate_end(&stream); 178 179 free(in); 180 return stream.total_out; 181} 182 183static unsigned long write_large_blob_data(struct git_istream *st, struct hashfile *f, 184 const struct object_id *oid) 185{ 186 git_zstream stream; 187 unsigned char ibuf[1024 * 16]; 188 unsigned char obuf[1024 * 16]; 189 unsigned long olen = 0; 190 191 git_deflate_init(&stream, pack_compression_level); 192 193 for (;;) { 194 ssize_t readlen; 195 int zret = Z_OK; 196 readlen = read_istream(st, ibuf, sizeof(ibuf)); 197 if (readlen == -1) 198 die(_("unable to read %s"), oid_to_hex(oid)); 199 200 stream.next_in = ibuf; 201 stream.avail_in = readlen; 202 while ((stream.avail_in || readlen == 0) && 203 (zret == Z_OK || zret == Z_BUF_ERROR)) { 204 stream.next_out = obuf; 205 stream.avail_out = sizeof(obuf); 206 zret = git_deflate(&stream, readlen ? 0 : Z_FINISH); 207 hashwrite(f, obuf, stream.next_out - obuf); 208 olen += stream.next_out - obuf; 209 } 210 if (stream.avail_in) 211 die(_("deflate error (%d)"), zret); 212 if (readlen == 0) { 213 if (zret != Z_STREAM_END) 214 die(_("deflate error (%d)"), zret); 215 break; 216 } 217 } 218 git_deflate_end(&stream); 219 return olen; 220} 221 222/* 223 * we are going to reuse the existing object data as is. make 224 * sure it is not corrupt. 225 */ 226static int check_pack_inflate(struct packed_git *p, 227 struct pack_window **w_curs, 228 off_t offset, 229 off_t len, 230 unsigned long expect) 231{ 232 git_zstream stream; 233 unsigned char fakebuf[4096], *in; 234 int st; 235 236 memset(&stream, 0, sizeof(stream)); 237 git_inflate_init(&stream); 238 do { 239 in = use_pack(p, w_curs, offset, &stream.avail_in); 240 stream.next_in = in; 241 stream.next_out = fakebuf; 242 stream.avail_out = sizeof(fakebuf); 243 st = git_inflate(&stream, Z_FINISH); 244 offset += stream.next_in - in; 245 } while (st == Z_OK || st == Z_BUF_ERROR); 246 git_inflate_end(&stream); 247 return (st == Z_STREAM_END && 248 stream.total_out == expect && 249 stream.total_in == len) ? 0 : -1; 250} 251 252static void copy_pack_data(struct hashfile *f, 253 struct packed_git *p, 254 struct pack_window **w_curs, 255 off_t offset, 256 off_t len) 257{ 258 unsigned char *in; 259 unsigned long avail; 260 261 while (len) { 262 in = use_pack(p, w_curs, offset, &avail); 263 if (avail > len) 264 avail = (unsigned long)len; 265 hashwrite(f, in, avail); 266 offset += avail; 267 len -= avail; 268 } 269} 270 271/* Return 0 if we will bust the pack-size limit */ 272static unsigned long write_no_reuse_object(struct hashfile *f, struct object_entry *entry, 273 unsigned long limit, int usable_delta) 274{ 275 unsigned long size, datalen; 276 unsigned char header[MAX_PACK_OBJECT_HEADER], 277 dheader[MAX_PACK_OBJECT_HEADER]; 278 unsigned hdrlen; 279 enum object_type type; 280 void *buf; 281 struct git_istream *st = NULL; 282 283 if (!usable_delta) { 284 if (oe_type(entry) == OBJ_BLOB && 285 oe_size_greater_than(&to_pack, entry, big_file_threshold) && 286 (st = open_istream(&entry->idx.oid, &type, &size, NULL)) != NULL) 287 buf = NULL; 288 else { 289 buf = read_object_file(&entry->idx.oid, &type, &size); 290 if (!buf) 291 die(_("unable to read %s"), 292 oid_to_hex(&entry->idx.oid)); 293 } 294 /* 295 * make sure no cached delta data remains from a 296 * previous attempt before a pack split occurred. 297 */ 298 FREE_AND_NULL(entry->delta_data); 299 entry->z_delta_size = 0; 300 } else if (entry->delta_data) { 301 size = DELTA_SIZE(entry); 302 buf = entry->delta_data; 303 entry->delta_data = NULL; 304 type = (allow_ofs_delta && DELTA(entry)->idx.offset) ? 305 OBJ_OFS_DELTA : OBJ_REF_DELTA; 306 } else { 307 buf = get_delta(entry); 308 size = DELTA_SIZE(entry); 309 type = (allow_ofs_delta && DELTA(entry)->idx.offset) ? 310 OBJ_OFS_DELTA : OBJ_REF_DELTA; 311 } 312 313 if (st) /* large blob case, just assume we don't compress well */ 314 datalen = size; 315 else if (entry->z_delta_size) 316 datalen = entry->z_delta_size; 317 else 318 datalen = do_compress(&buf, size); 319 320 /* 321 * The object header is a byte of 'type' followed by zero or 322 * more bytes of length. 323 */ 324 hdrlen = encode_in_pack_object_header(header, sizeof(header), 325 type, size); 326 327 if (type == OBJ_OFS_DELTA) { 328 /* 329 * Deltas with relative base contain an additional 330 * encoding of the relative offset for the delta 331 * base from this object's position in the pack. 332 */ 333 off_t ofs = entry->idx.offset - DELTA(entry)->idx.offset; 334 unsigned pos = sizeof(dheader) - 1; 335 dheader[pos] = ofs & 127; 336 while (ofs >>= 7) 337 dheader[--pos] = 128 | (--ofs & 127); 338 if (limit && hdrlen + sizeof(dheader) - pos + datalen + 20 >= limit) { 339 if (st) 340 close_istream(st); 341 free(buf); 342 return 0; 343 } 344 hashwrite(f, header, hdrlen); 345 hashwrite(f, dheader + pos, sizeof(dheader) - pos); 346 hdrlen += sizeof(dheader) - pos; 347 } else if (type == OBJ_REF_DELTA) { 348 /* 349 * Deltas with a base reference contain 350 * an additional 20 bytes for the base sha1. 351 */ 352 if (limit && hdrlen + 20 + datalen + 20 >= limit) { 353 if (st) 354 close_istream(st); 355 free(buf); 356 return 0; 357 } 358 hashwrite(f, header, hdrlen); 359 hashwrite(f, DELTA(entry)->idx.oid.hash, 20); 360 hdrlen += 20; 361 } else { 362 if (limit && hdrlen + datalen + 20 >= limit) { 363 if (st) 364 close_istream(st); 365 free(buf); 366 return 0; 367 } 368 hashwrite(f, header, hdrlen); 369 } 370 if (st) { 371 datalen = write_large_blob_data(st, f, &entry->idx.oid); 372 close_istream(st); 373 } else { 374 hashwrite(f, buf, datalen); 375 free(buf); 376 } 377 378 return hdrlen + datalen; 379} 380 381/* Return 0 if we will bust the pack-size limit */ 382static off_t write_reuse_object(struct hashfile *f, struct object_entry *entry, 383 unsigned long limit, int usable_delta) 384{ 385 struct packed_git *p = IN_PACK(entry); 386 struct pack_window *w_curs = NULL; 387 struct revindex_entry *revidx; 388 off_t offset; 389 enum object_type type = oe_type(entry); 390 off_t datalen; 391 unsigned char header[MAX_PACK_OBJECT_HEADER], 392 dheader[MAX_PACK_OBJECT_HEADER]; 393 unsigned hdrlen; 394 unsigned long entry_size = SIZE(entry); 395 396 if (DELTA(entry)) 397 type = (allow_ofs_delta && DELTA(entry)->idx.offset) ? 398 OBJ_OFS_DELTA : OBJ_REF_DELTA; 399 hdrlen = encode_in_pack_object_header(header, sizeof(header), 400 type, entry_size); 401 402 offset = entry->in_pack_offset; 403 revidx = find_pack_revindex(p, offset); 404 datalen = revidx[1].offset - offset; 405 if (!pack_to_stdout && p->index_version > 1 && 406 check_pack_crc(p, &w_curs, offset, datalen, revidx->nr)) { 407 error("bad packed object CRC for %s", 408 oid_to_hex(&entry->idx.oid)); 409 unuse_pack(&w_curs); 410 return write_no_reuse_object(f, entry, limit, usable_delta); 411 } 412 413 offset += entry->in_pack_header_size; 414 datalen -= entry->in_pack_header_size; 415 416 if (!pack_to_stdout && p->index_version == 1 && 417 check_pack_inflate(p, &w_curs, offset, datalen, entry_size)) { 418 error("corrupt packed object for %s", 419 oid_to_hex(&entry->idx.oid)); 420 unuse_pack(&w_curs); 421 return write_no_reuse_object(f, entry, limit, usable_delta); 422 } 423 424 if (type == OBJ_OFS_DELTA) { 425 off_t ofs = entry->idx.offset - DELTA(entry)->idx.offset; 426 unsigned pos = sizeof(dheader) - 1; 427 dheader[pos] = ofs & 127; 428 while (ofs >>= 7) 429 dheader[--pos] = 128 | (--ofs & 127); 430 if (limit && hdrlen + sizeof(dheader) - pos + datalen + 20 >= limit) { 431 unuse_pack(&w_curs); 432 return 0; 433 } 434 hashwrite(f, header, hdrlen); 435 hashwrite(f, dheader + pos, sizeof(dheader) - pos); 436 hdrlen += sizeof(dheader) - pos; 437 reused_delta++; 438 } else if (type == OBJ_REF_DELTA) { 439 if (limit && hdrlen + 20 + datalen + 20 >= limit) { 440 unuse_pack(&w_curs); 441 return 0; 442 } 443 hashwrite(f, header, hdrlen); 444 hashwrite(f, DELTA(entry)->idx.oid.hash, 20); 445 hdrlen += 20; 446 reused_delta++; 447 } else { 448 if (limit && hdrlen + datalen + 20 >= limit) { 449 unuse_pack(&w_curs); 450 return 0; 451 } 452 hashwrite(f, header, hdrlen); 453 } 454 copy_pack_data(f, p, &w_curs, offset, datalen); 455 unuse_pack(&w_curs); 456 reused++; 457 return hdrlen + datalen; 458} 459 460/* Return 0 if we will bust the pack-size limit */ 461static off_t write_object(struct hashfile *f, 462 struct object_entry *entry, 463 off_t write_offset) 464{ 465 unsigned long limit; 466 off_t len; 467 int usable_delta, to_reuse; 468 469 if (!pack_to_stdout) 470 crc32_begin(f); 471 472 /* apply size limit if limited packsize and not first object */ 473 if (!pack_size_limit || !nr_written) 474 limit = 0; 475 else if (pack_size_limit <= write_offset) 476 /* 477 * the earlier object did not fit the limit; avoid 478 * mistaking this with unlimited (i.e. limit = 0). 479 */ 480 limit = 1; 481 else 482 limit = pack_size_limit - write_offset; 483 484 if (!DELTA(entry)) 485 usable_delta = 0; /* no delta */ 486 else if (!pack_size_limit) 487 usable_delta = 1; /* unlimited packfile */ 488 else if (DELTA(entry)->idx.offset == (off_t)-1) 489 usable_delta = 0; /* base was written to another pack */ 490 else if (DELTA(entry)->idx.offset) 491 usable_delta = 1; /* base already exists in this pack */ 492 else 493 usable_delta = 0; /* base could end up in another pack */ 494 495 if (!reuse_object) 496 to_reuse = 0; /* explicit */ 497 else if (!IN_PACK(entry)) 498 to_reuse = 0; /* can't reuse what we don't have */ 499 else if (oe_type(entry) == OBJ_REF_DELTA || 500 oe_type(entry) == OBJ_OFS_DELTA) 501 /* check_object() decided it for us ... */ 502 to_reuse = usable_delta; 503 /* ... but pack split may override that */ 504 else if (oe_type(entry) != entry->in_pack_type) 505 to_reuse = 0; /* pack has delta which is unusable */ 506 else if (DELTA(entry)) 507 to_reuse = 0; /* we want to pack afresh */ 508 else 509 to_reuse = 1; /* we have it in-pack undeltified, 510 * and we do not need to deltify it. 511 */ 512 513 if (!to_reuse) 514 len = write_no_reuse_object(f, entry, limit, usable_delta); 515 else 516 len = write_reuse_object(f, entry, limit, usable_delta); 517 if (!len) 518 return 0; 519 520 if (usable_delta) 521 written_delta++; 522 written++; 523 if (!pack_to_stdout) 524 entry->idx.crc32 = crc32_end(f); 525 return len; 526} 527 528enum write_one_status { 529 WRITE_ONE_SKIP = -1, /* already written */ 530 WRITE_ONE_BREAK = 0, /* writing this will bust the limit; not written */ 531 WRITE_ONE_WRITTEN = 1, /* normal */ 532 WRITE_ONE_RECURSIVE = 2 /* already scheduled to be written */ 533}; 534 535static enum write_one_status write_one(struct hashfile *f, 536 struct object_entry *e, 537 off_t *offset) 538{ 539 off_t size; 540 int recursing; 541 542 /* 543 * we set offset to 1 (which is an impossible value) to mark 544 * the fact that this object is involved in "write its base 545 * first before writing a deltified object" recursion. 546 */ 547 recursing = (e->idx.offset == 1); 548 if (recursing) { 549 warning("recursive delta detected for object %s", 550 oid_to_hex(&e->idx.oid)); 551 return WRITE_ONE_RECURSIVE; 552 } else if (e->idx.offset || e->preferred_base) { 553 /* offset is non zero if object is written already. */ 554 return WRITE_ONE_SKIP; 555 } 556 557 /* if we are deltified, write out base object first. */ 558 if (DELTA(e)) { 559 e->idx.offset = 1; /* now recurse */ 560 switch (write_one(f, DELTA(e), offset)) { 561 case WRITE_ONE_RECURSIVE: 562 /* we cannot depend on this one */ 563 SET_DELTA(e, NULL); 564 break; 565 default: 566 break; 567 case WRITE_ONE_BREAK: 568 e->idx.offset = recursing; 569 return WRITE_ONE_BREAK; 570 } 571 } 572 573 e->idx.offset = *offset; 574 size = write_object(f, e, *offset); 575 if (!size) { 576 e->idx.offset = recursing; 577 return WRITE_ONE_BREAK; 578 } 579 written_list[nr_written++] = &e->idx; 580 581 /* make sure off_t is sufficiently large not to wrap */ 582 if (signed_add_overflows(*offset, size)) 583 die("pack too large for current definition of off_t"); 584 *offset += size; 585 return WRITE_ONE_WRITTEN; 586} 587 588static int mark_tagged(const char *path, const struct object_id *oid, int flag, 589 void *cb_data) 590{ 591 struct object_id peeled; 592 struct object_entry *entry = packlist_find(&to_pack, oid->hash, NULL); 593 594 if (entry) 595 entry->tagged = 1; 596 if (!peel_ref(path, &peeled)) { 597 entry = packlist_find(&to_pack, peeled.hash, NULL); 598 if (entry) 599 entry->tagged = 1; 600 } 601 return 0; 602} 603 604static inline void add_to_write_order(struct object_entry **wo, 605 unsigned int *endp, 606 struct object_entry *e) 607{ 608 if (e->filled) 609 return; 610 wo[(*endp)++] = e; 611 e->filled = 1; 612} 613 614static void add_descendants_to_write_order(struct object_entry **wo, 615 unsigned int *endp, 616 struct object_entry *e) 617{ 618 int add_to_order = 1; 619 while (e) { 620 if (add_to_order) { 621 struct object_entry *s; 622 /* add this node... */ 623 add_to_write_order(wo, endp, e); 624 /* all its siblings... */ 625 for (s = DELTA_SIBLING(e); s; s = DELTA_SIBLING(s)) { 626 add_to_write_order(wo, endp, s); 627 } 628 } 629 /* drop down a level to add left subtree nodes if possible */ 630 if (DELTA_CHILD(e)) { 631 add_to_order = 1; 632 e = DELTA_CHILD(e); 633 } else { 634 add_to_order = 0; 635 /* our sibling might have some children, it is next */ 636 if (DELTA_SIBLING(e)) { 637 e = DELTA_SIBLING(e); 638 continue; 639 } 640 /* go back to our parent node */ 641 e = DELTA(e); 642 while (e && !DELTA_SIBLING(e)) { 643 /* we're on the right side of a subtree, keep 644 * going up until we can go right again */ 645 e = DELTA(e); 646 } 647 if (!e) { 648 /* done- we hit our original root node */ 649 return; 650 } 651 /* pass it off to sibling at this level */ 652 e = DELTA_SIBLING(e); 653 } 654 }; 655} 656 657static void add_family_to_write_order(struct object_entry **wo, 658 unsigned int *endp, 659 struct object_entry *e) 660{ 661 struct object_entry *root; 662 663 for (root = e; DELTA(root); root = DELTA(root)) 664 ; /* nothing */ 665 add_descendants_to_write_order(wo, endp, root); 666} 667 668static struct object_entry **compute_write_order(void) 669{ 670 unsigned int i, wo_end, last_untagged; 671 672 struct object_entry **wo; 673 struct object_entry *objects = to_pack.objects; 674 675 for (i = 0; i < to_pack.nr_objects; i++) { 676 objects[i].tagged = 0; 677 objects[i].filled = 0; 678 SET_DELTA_CHILD(&objects[i], NULL); 679 SET_DELTA_SIBLING(&objects[i], NULL); 680 } 681 682 /* 683 * Fully connect delta_child/delta_sibling network. 684 * Make sure delta_sibling is sorted in the original 685 * recency order. 686 */ 687 for (i = to_pack.nr_objects; i > 0;) { 688 struct object_entry *e = &objects[--i]; 689 if (!DELTA(e)) 690 continue; 691 /* Mark me as the first child */ 692 e->delta_sibling_idx = DELTA(e)->delta_child_idx; 693 SET_DELTA_CHILD(DELTA(e), e); 694 } 695 696 /* 697 * Mark objects that are at the tip of tags. 698 */ 699 for_each_tag_ref(mark_tagged, NULL); 700 701 /* 702 * Give the objects in the original recency order until 703 * we see a tagged tip. 704 */ 705 ALLOC_ARRAY(wo, to_pack.nr_objects); 706 for (i = wo_end = 0; i < to_pack.nr_objects; i++) { 707 if (objects[i].tagged) 708 break; 709 add_to_write_order(wo, &wo_end, &objects[i]); 710 } 711 last_untagged = i; 712 713 /* 714 * Then fill all the tagged tips. 715 */ 716 for (; i < to_pack.nr_objects; i++) { 717 if (objects[i].tagged) 718 add_to_write_order(wo, &wo_end, &objects[i]); 719 } 720 721 /* 722 * And then all remaining commits and tags. 723 */ 724 for (i = last_untagged; i < to_pack.nr_objects; i++) { 725 if (oe_type(&objects[i]) != OBJ_COMMIT && 726 oe_type(&objects[i]) != OBJ_TAG) 727 continue; 728 add_to_write_order(wo, &wo_end, &objects[i]); 729 } 730 731 /* 732 * And then all the trees. 733 */ 734 for (i = last_untagged; i < to_pack.nr_objects; i++) { 735 if (oe_type(&objects[i]) != OBJ_TREE) 736 continue; 737 add_to_write_order(wo, &wo_end, &objects[i]); 738 } 739 740 /* 741 * Finally all the rest in really tight order 742 */ 743 for (i = last_untagged; i < to_pack.nr_objects; i++) { 744 if (!objects[i].filled) 745 add_family_to_write_order(wo, &wo_end, &objects[i]); 746 } 747 748 if (wo_end != to_pack.nr_objects) 749 die("ordered %u objects, expected %"PRIu32, wo_end, to_pack.nr_objects); 750 751 return wo; 752} 753 754static off_t write_reused_pack(struct hashfile *f) 755{ 756 unsigned char buffer[8192]; 757 off_t to_write, total; 758 int fd; 759 760 if (!is_pack_valid(reuse_packfile)) 761 die("packfile is invalid: %s", reuse_packfile->pack_name); 762 763 fd = git_open(reuse_packfile->pack_name); 764 if (fd < 0) 765 die_errno("unable to open packfile for reuse: %s", 766 reuse_packfile->pack_name); 767 768 if (lseek(fd, sizeof(struct pack_header), SEEK_SET) == -1) 769 die_errno("unable to seek in reused packfile"); 770 771 if (reuse_packfile_offset < 0) 772 reuse_packfile_offset = reuse_packfile->pack_size - 20; 773 774 total = to_write = reuse_packfile_offset - sizeof(struct pack_header); 775 776 while (to_write) { 777 int read_pack = xread(fd, buffer, sizeof(buffer)); 778 779 if (read_pack <= 0) 780 die_errno("unable to read from reused packfile"); 781 782 if (read_pack > to_write) 783 read_pack = to_write; 784 785 hashwrite(f, buffer, read_pack); 786 to_write -= read_pack; 787 788 /* 789 * We don't know the actual number of objects written, 790 * only how many bytes written, how many bytes total, and 791 * how many objects total. So we can fake it by pretending all 792 * objects we are writing are the same size. This gives us a 793 * smooth progress meter, and at the end it matches the true 794 * answer. 795 */ 796 written = reuse_packfile_objects * 797 (((double)(total - to_write)) / total); 798 display_progress(progress_state, written); 799 } 800 801 close(fd); 802 written = reuse_packfile_objects; 803 display_progress(progress_state, written); 804 return reuse_packfile_offset - sizeof(struct pack_header); 805} 806 807static const char no_split_warning[] = N_( 808"disabling bitmap writing, packs are split due to pack.packSizeLimit" 809); 810 811static void write_pack_file(void) 812{ 813 uint32_t i = 0, j; 814 struct hashfile *f; 815 off_t offset; 816 uint32_t nr_remaining = nr_result; 817 time_t last_mtime = 0; 818 struct object_entry **write_order; 819 820 if (progress > pack_to_stdout) 821 progress_state = start_progress(_("Writing objects"), nr_result); 822 ALLOC_ARRAY(written_list, to_pack.nr_objects); 823 write_order = compute_write_order(); 824 825 do { 826 struct object_id oid; 827 char *pack_tmp_name = NULL; 828 829 if (pack_to_stdout) 830 f = hashfd_throughput(1, "<stdout>", progress_state); 831 else 832 f = create_tmp_packfile(&pack_tmp_name); 833 834 offset = write_pack_header(f, nr_remaining); 835 836 if (reuse_packfile) { 837 off_t packfile_size; 838 assert(pack_to_stdout); 839 840 packfile_size = write_reused_pack(f); 841 offset += packfile_size; 842 } 843 844 nr_written = 0; 845 for (; i < to_pack.nr_objects; i++) { 846 struct object_entry *e = write_order[i]; 847 if (write_one(f, e, &offset) == WRITE_ONE_BREAK) 848 break; 849 display_progress(progress_state, written); 850 } 851 852 /* 853 * Did we write the wrong # entries in the header? 854 * If so, rewrite it like in fast-import 855 */ 856 if (pack_to_stdout) { 857 finalize_hashfile(f, oid.hash, CSUM_HASH_IN_STREAM | CSUM_CLOSE); 858 } else if (nr_written == nr_remaining) { 859 finalize_hashfile(f, oid.hash, CSUM_HASH_IN_STREAM | CSUM_FSYNC | CSUM_CLOSE); 860 } else { 861 int fd = finalize_hashfile(f, oid.hash, 0); 862 fixup_pack_header_footer(fd, oid.hash, pack_tmp_name, 863 nr_written, oid.hash, offset); 864 close(fd); 865 if (write_bitmap_index) { 866 warning(_(no_split_warning)); 867 write_bitmap_index = 0; 868 } 869 } 870 871 if (!pack_to_stdout) { 872 struct stat st; 873 struct strbuf tmpname = STRBUF_INIT; 874 875 /* 876 * Packs are runtime accessed in their mtime 877 * order since newer packs are more likely to contain 878 * younger objects. So if we are creating multiple 879 * packs then we should modify the mtime of later ones 880 * to preserve this property. 881 */ 882 if (stat(pack_tmp_name, &st) < 0) { 883 warning_errno("failed to stat %s", pack_tmp_name); 884 } else if (!last_mtime) { 885 last_mtime = st.st_mtime; 886 } else { 887 struct utimbuf utb; 888 utb.actime = st.st_atime; 889 utb.modtime = --last_mtime; 890 if (utime(pack_tmp_name, &utb) < 0) 891 warning_errno("failed utime() on %s", pack_tmp_name); 892 } 893 894 strbuf_addf(&tmpname, "%s-", base_name); 895 896 if (write_bitmap_index) { 897 bitmap_writer_set_checksum(oid.hash); 898 bitmap_writer_build_type_index( 899 &to_pack, written_list, nr_written); 900 } 901 902 finish_tmp_packfile(&tmpname, pack_tmp_name, 903 written_list, nr_written, 904 &pack_idx_opts, oid.hash); 905 906 if (write_bitmap_index) { 907 strbuf_addf(&tmpname, "%s.bitmap", oid_to_hex(&oid)); 908 909 stop_progress(&progress_state); 910 911 bitmap_writer_show_progress(progress); 912 bitmap_writer_reuse_bitmaps(&to_pack); 913 bitmap_writer_select_commits(indexed_commits, indexed_commits_nr, -1); 914 bitmap_writer_build(&to_pack); 915 bitmap_writer_finish(written_list, nr_written, 916 tmpname.buf, write_bitmap_options); 917 write_bitmap_index = 0; 918 } 919 920 strbuf_release(&tmpname); 921 free(pack_tmp_name); 922 puts(oid_to_hex(&oid)); 923 } 924 925 /* mark written objects as written to previous pack */ 926 for (j = 0; j < nr_written; j++) { 927 written_list[j]->offset = (off_t)-1; 928 } 929 nr_remaining -= nr_written; 930 } while (nr_remaining && i < to_pack.nr_objects); 931 932 free(written_list); 933 free(write_order); 934 stop_progress(&progress_state); 935 if (written != nr_result) 936 die("wrote %"PRIu32" objects while expecting %"PRIu32, 937 written, nr_result); 938} 939 940static int no_try_delta(const char *path) 941{ 942 static struct attr_check *check; 943 944 if (!check) 945 check = attr_check_initl("delta", NULL); 946 if (git_check_attr(path, check)) 947 return 0; 948 if (ATTR_FALSE(check->items[0].value)) 949 return 1; 950 return 0; 951} 952 953/* 954 * When adding an object, check whether we have already added it 955 * to our packing list. If so, we can skip. However, if we are 956 * being asked to excludei t, but the previous mention was to include 957 * it, make sure to adjust its flags and tweak our numbers accordingly. 958 * 959 * As an optimization, we pass out the index position where we would have 960 * found the item, since that saves us from having to look it up again a 961 * few lines later when we want to add the new entry. 962 */ 963static int have_duplicate_entry(const struct object_id *oid, 964 int exclude, 965 uint32_t *index_pos) 966{ 967 struct object_entry *entry; 968 969 entry = packlist_find(&to_pack, oid->hash, index_pos); 970 if (!entry) 971 return 0; 972 973 if (exclude) { 974 if (!entry->preferred_base) 975 nr_result--; 976 entry->preferred_base = 1; 977 } 978 979 return 1; 980} 981 982static int want_found_object(int exclude, struct packed_git *p) 983{ 984 if (exclude) 985 return 1; 986 if (incremental) 987 return 0; 988 989 /* 990 * When asked to do --local (do not include an object that appears in a 991 * pack we borrow from elsewhere) or --honor-pack-keep (do not include 992 * an object that appears in a pack marked with .keep), finding a pack 993 * that matches the criteria is sufficient for us to decide to omit it. 994 * However, even if this pack does not satisfy the criteria, we need to 995 * make sure no copy of this object appears in _any_ pack that makes us 996 * to omit the object, so we need to check all the packs. 997 * 998 * We can however first check whether these options can possible matter; 999 * if they do not matter we know we want the object in generated pack.1000 * Otherwise, we signal "-1" at the end to tell the caller that we do1001 * not know either way, and it needs to check more packs.1002 */1003 if (!ignore_packed_keep_on_disk &&1004 !ignore_packed_keep_in_core &&1005 (!local || !have_non_local_packs))1006 return 1;10071008 if (local && !p->pack_local)1009 return 0;1010 if (p->pack_local &&1011 ((ignore_packed_keep_on_disk && p->pack_keep) ||1012 (ignore_packed_keep_in_core && p->pack_keep_in_core)))1013 return 0;10141015 /* we don't know yet; keep looking for more packs */1016 return -1;1017}10181019/*1020 * Check whether we want the object in the pack (e.g., we do not want1021 * objects found in non-local stores if the "--local" option was used).1022 *1023 * If the caller already knows an existing pack it wants to take the object1024 * from, that is passed in *found_pack and *found_offset; otherwise this1025 * function finds if there is any pack that has the object and returns the pack1026 * and its offset in these variables.1027 */1028static int want_object_in_pack(const struct object_id *oid,1029 int exclude,1030 struct packed_git **found_pack,1031 off_t *found_offset)1032{1033 int want;1034 struct list_head *pos;10351036 if (!exclude && local && has_loose_object_nonlocal(oid->hash))1037 return 0;10381039 /*1040 * If we already know the pack object lives in, start checks from that1041 * pack - in the usual case when neither --local was given nor .keep files1042 * are present we will determine the answer right now.1043 */1044 if (*found_pack) {1045 want = want_found_object(exclude, *found_pack);1046 if (want != -1)1047 return want;1048 }1049 list_for_each(pos, get_packed_git_mru(the_repository)) {1050 struct packed_git *p = list_entry(pos, struct packed_git, mru);1051 off_t offset;10521053 if (p == *found_pack)1054 offset = *found_offset;1055 else1056 offset = find_pack_entry_one(oid->hash, p);10571058 if (offset) {1059 if (!*found_pack) {1060 if (!is_pack_valid(p))1061 continue;1062 *found_offset = offset;1063 *found_pack = p;1064 }1065 want = want_found_object(exclude, p);1066 if (!exclude && want > 0)1067 list_move(&p->mru,1068 get_packed_git_mru(the_repository));1069 if (want != -1)1070 return want;1071 }1072 }10731074 return 1;1075}10761077static void create_object_entry(const struct object_id *oid,1078 enum object_type type,1079 uint32_t hash,1080 int exclude,1081 int no_try_delta,1082 uint32_t index_pos,1083 struct packed_git *found_pack,1084 off_t found_offset)1085{1086 struct object_entry *entry;10871088 entry = packlist_alloc(&to_pack, oid->hash, index_pos);1089 entry->hash = hash;1090 oe_set_type(entry, type);1091 if (exclude)1092 entry->preferred_base = 1;1093 else1094 nr_result++;1095 if (found_pack) {1096 oe_set_in_pack(&to_pack, entry, found_pack);1097 entry->in_pack_offset = found_offset;1098 }10991100 entry->no_try_delta = no_try_delta;1101}11021103static const char no_closure_warning[] = N_(1104"disabling bitmap writing, as some objects are not being packed"1105);11061107static int add_object_entry(const struct object_id *oid, enum object_type type,1108 const char *name, int exclude)1109{1110 struct packed_git *found_pack = NULL;1111 off_t found_offset = 0;1112 uint32_t index_pos;11131114 display_progress(progress_state, ++nr_seen);11151116 if (have_duplicate_entry(oid, exclude, &index_pos))1117 return 0;11181119 if (!want_object_in_pack(oid, exclude, &found_pack, &found_offset)) {1120 /* The pack is missing an object, so it will not have closure */1121 if (write_bitmap_index) {1122 warning(_(no_closure_warning));1123 write_bitmap_index = 0;1124 }1125 return 0;1126 }11271128 create_object_entry(oid, type, pack_name_hash(name),1129 exclude, name && no_try_delta(name),1130 index_pos, found_pack, found_offset);1131 return 1;1132}11331134static int add_object_entry_from_bitmap(const struct object_id *oid,1135 enum object_type type,1136 int flags, uint32_t name_hash,1137 struct packed_git *pack, off_t offset)1138{1139 uint32_t index_pos;11401141 display_progress(progress_state, ++nr_seen);11421143 if (have_duplicate_entry(oid, 0, &index_pos))1144 return 0;11451146 if (!want_object_in_pack(oid, 0, &pack, &offset))1147 return 0;11481149 create_object_entry(oid, type, name_hash, 0, 0, index_pos, pack, offset);1150 return 1;1151}11521153struct pbase_tree_cache {1154 struct object_id oid;1155 int ref;1156 int temporary;1157 void *tree_data;1158 unsigned long tree_size;1159};11601161static struct pbase_tree_cache *(pbase_tree_cache[256]);1162static int pbase_tree_cache_ix(const struct object_id *oid)1163{1164 return oid->hash[0] % ARRAY_SIZE(pbase_tree_cache);1165}1166static int pbase_tree_cache_ix_incr(int ix)1167{1168 return (ix+1) % ARRAY_SIZE(pbase_tree_cache);1169}11701171static struct pbase_tree {1172 struct pbase_tree *next;1173 /* This is a phony "cache" entry; we are not1174 * going to evict it or find it through _get()1175 * mechanism -- this is for the toplevel node that1176 * would almost always change with any commit.1177 */1178 struct pbase_tree_cache pcache;1179} *pbase_tree;11801181static struct pbase_tree_cache *pbase_tree_get(const struct object_id *oid)1182{1183 struct pbase_tree_cache *ent, *nent;1184 void *data;1185 unsigned long size;1186 enum object_type type;1187 int neigh;1188 int my_ix = pbase_tree_cache_ix(oid);1189 int available_ix = -1;11901191 /* pbase-tree-cache acts as a limited hashtable.1192 * your object will be found at your index or within a few1193 * slots after that slot if it is cached.1194 */1195 for (neigh = 0; neigh < 8; neigh++) {1196 ent = pbase_tree_cache[my_ix];1197 if (ent && !oidcmp(&ent->oid, oid)) {1198 ent->ref++;1199 return ent;1200 }1201 else if (((available_ix < 0) && (!ent || !ent->ref)) ||1202 ((0 <= available_ix) &&1203 (!ent && pbase_tree_cache[available_ix])))1204 available_ix = my_ix;1205 if (!ent)1206 break;1207 my_ix = pbase_tree_cache_ix_incr(my_ix);1208 }12091210 /* Did not find one. Either we got a bogus request or1211 * we need to read and perhaps cache.1212 */1213 data = read_object_file(oid, &type, &size);1214 if (!data)1215 return NULL;1216 if (type != OBJ_TREE) {1217 free(data);1218 return NULL;1219 }12201221 /* We need to either cache or return a throwaway copy */12221223 if (available_ix < 0)1224 ent = NULL;1225 else {1226 ent = pbase_tree_cache[available_ix];1227 my_ix = available_ix;1228 }12291230 if (!ent) {1231 nent = xmalloc(sizeof(*nent));1232 nent->temporary = (available_ix < 0);1233 }1234 else {1235 /* evict and reuse */1236 free(ent->tree_data);1237 nent = ent;1238 }1239 oidcpy(&nent->oid, oid);1240 nent->tree_data = data;1241 nent->tree_size = size;1242 nent->ref = 1;1243 if (!nent->temporary)1244 pbase_tree_cache[my_ix] = nent;1245 return nent;1246}12471248static void pbase_tree_put(struct pbase_tree_cache *cache)1249{1250 if (!cache->temporary) {1251 cache->ref--;1252 return;1253 }1254 free(cache->tree_data);1255 free(cache);1256}12571258static int name_cmp_len(const char *name)1259{1260 int i;1261 for (i = 0; name[i] && name[i] != '\n' && name[i] != '/'; i++)1262 ;1263 return i;1264}12651266static void add_pbase_object(struct tree_desc *tree,1267 const char *name,1268 int cmplen,1269 const char *fullname)1270{1271 struct name_entry entry;1272 int cmp;12731274 while (tree_entry(tree,&entry)) {1275 if (S_ISGITLINK(entry.mode))1276 continue;1277 cmp = tree_entry_len(&entry) != cmplen ? 1 :1278 memcmp(name, entry.path, cmplen);1279 if (cmp > 0)1280 continue;1281 if (cmp < 0)1282 return;1283 if (name[cmplen] != '/') {1284 add_object_entry(entry.oid,1285 object_type(entry.mode),1286 fullname, 1);1287 return;1288 }1289 if (S_ISDIR(entry.mode)) {1290 struct tree_desc sub;1291 struct pbase_tree_cache *tree;1292 const char *down = name+cmplen+1;1293 int downlen = name_cmp_len(down);12941295 tree = pbase_tree_get(entry.oid);1296 if (!tree)1297 return;1298 init_tree_desc(&sub, tree->tree_data, tree->tree_size);12991300 add_pbase_object(&sub, down, downlen, fullname);1301 pbase_tree_put(tree);1302 }1303 }1304}13051306static unsigned *done_pbase_paths;1307static int done_pbase_paths_num;1308static int done_pbase_paths_alloc;1309static int done_pbase_path_pos(unsigned hash)1310{1311 int lo = 0;1312 int hi = done_pbase_paths_num;1313 while (lo < hi) {1314 int mi = lo + (hi - lo) / 2;1315 if (done_pbase_paths[mi] == hash)1316 return mi;1317 if (done_pbase_paths[mi] < hash)1318 hi = mi;1319 else1320 lo = mi + 1;1321 }1322 return -lo-1;1323}13241325static int check_pbase_path(unsigned hash)1326{1327 int pos = done_pbase_path_pos(hash);1328 if (0 <= pos)1329 return 1;1330 pos = -pos - 1;1331 ALLOC_GROW(done_pbase_paths,1332 done_pbase_paths_num + 1,1333 done_pbase_paths_alloc);1334 done_pbase_paths_num++;1335 if (pos < done_pbase_paths_num)1336 MOVE_ARRAY(done_pbase_paths + pos + 1, done_pbase_paths + pos,1337 done_pbase_paths_num - pos - 1);1338 done_pbase_paths[pos] = hash;1339 return 0;1340}13411342static void add_preferred_base_object(const char *name)1343{1344 struct pbase_tree *it;1345 int cmplen;1346 unsigned hash = pack_name_hash(name);13471348 if (!num_preferred_base || check_pbase_path(hash))1349 return;13501351 cmplen = name_cmp_len(name);1352 for (it = pbase_tree; it; it = it->next) {1353 if (cmplen == 0) {1354 add_object_entry(&it->pcache.oid, OBJ_TREE, NULL, 1);1355 }1356 else {1357 struct tree_desc tree;1358 init_tree_desc(&tree, it->pcache.tree_data, it->pcache.tree_size);1359 add_pbase_object(&tree, name, cmplen, name);1360 }1361 }1362}13631364static void add_preferred_base(struct object_id *oid)1365{1366 struct pbase_tree *it;1367 void *data;1368 unsigned long size;1369 struct object_id tree_oid;13701371 if (window <= num_preferred_base++)1372 return;13731374 data = read_object_with_reference(oid, tree_type, &size, &tree_oid);1375 if (!data)1376 return;13771378 for (it = pbase_tree; it; it = it->next) {1379 if (!oidcmp(&it->pcache.oid, &tree_oid)) {1380 free(data);1381 return;1382 }1383 }13841385 it = xcalloc(1, sizeof(*it));1386 it->next = pbase_tree;1387 pbase_tree = it;13881389 oidcpy(&it->pcache.oid, &tree_oid);1390 it->pcache.tree_data = data;1391 it->pcache.tree_size = size;1392}13931394static void cleanup_preferred_base(void)1395{1396 struct pbase_tree *it;1397 unsigned i;13981399 it = pbase_tree;1400 pbase_tree = NULL;1401 while (it) {1402 struct pbase_tree *tmp = it;1403 it = tmp->next;1404 free(tmp->pcache.tree_data);1405 free(tmp);1406 }14071408 for (i = 0; i < ARRAY_SIZE(pbase_tree_cache); i++) {1409 if (!pbase_tree_cache[i])1410 continue;1411 free(pbase_tree_cache[i]->tree_data);1412 FREE_AND_NULL(pbase_tree_cache[i]);1413 }14141415 FREE_AND_NULL(done_pbase_paths);1416 done_pbase_paths_num = done_pbase_paths_alloc = 0;1417}14181419static void check_object(struct object_entry *entry)1420{1421 unsigned long canonical_size;14221423 if (IN_PACK(entry)) {1424 struct packed_git *p = IN_PACK(entry);1425 struct pack_window *w_curs = NULL;1426 const unsigned char *base_ref = NULL;1427 struct object_entry *base_entry;1428 unsigned long used, used_0;1429 unsigned long avail;1430 off_t ofs;1431 unsigned char *buf, c;1432 enum object_type type;1433 unsigned long in_pack_size;14341435 buf = use_pack(p, &w_curs, entry->in_pack_offset, &avail);14361437 /*1438 * We want in_pack_type even if we do not reuse delta1439 * since non-delta representations could still be reused.1440 */1441 used = unpack_object_header_buffer(buf, avail,1442 &type,1443 &in_pack_size);1444 if (used == 0)1445 goto give_up;14461447 if (type < 0)1448 BUG("invalid type %d", type);1449 entry->in_pack_type = type;14501451 /*1452 * Determine if this is a delta and if so whether we can1453 * reuse it or not. Otherwise let's find out as cheaply as1454 * possible what the actual type and size for this object is.1455 */1456 switch (entry->in_pack_type) {1457 default:1458 /* Not a delta hence we've already got all we need. */1459 oe_set_type(entry, entry->in_pack_type);1460 SET_SIZE(entry, in_pack_size);1461 entry->in_pack_header_size = used;1462 if (oe_type(entry) < OBJ_COMMIT || oe_type(entry) > OBJ_BLOB)1463 goto give_up;1464 unuse_pack(&w_curs);1465 return;1466 case OBJ_REF_DELTA:1467 if (reuse_delta && !entry->preferred_base)1468 base_ref = use_pack(p, &w_curs,1469 entry->in_pack_offset + used, NULL);1470 entry->in_pack_header_size = used + 20;1471 break;1472 case OBJ_OFS_DELTA:1473 buf = use_pack(p, &w_curs,1474 entry->in_pack_offset + used, NULL);1475 used_0 = 0;1476 c = buf[used_0++];1477 ofs = c & 127;1478 while (c & 128) {1479 ofs += 1;1480 if (!ofs || MSB(ofs, 7)) {1481 error("delta base offset overflow in pack for %s",1482 oid_to_hex(&entry->idx.oid));1483 goto give_up;1484 }1485 c = buf[used_0++];1486 ofs = (ofs << 7) + (c & 127);1487 }1488 ofs = entry->in_pack_offset - ofs;1489 if (ofs <= 0 || ofs >= entry->in_pack_offset) {1490 error("delta base offset out of bound for %s",1491 oid_to_hex(&entry->idx.oid));1492 goto give_up;1493 }1494 if (reuse_delta && !entry->preferred_base) {1495 struct revindex_entry *revidx;1496 revidx = find_pack_revindex(p, ofs);1497 if (!revidx)1498 goto give_up;1499 base_ref = nth_packed_object_sha1(p, revidx->nr);1500 }1501 entry->in_pack_header_size = used + used_0;1502 break;1503 }15041505 if (base_ref && (base_entry = packlist_find(&to_pack, base_ref, NULL))) {1506 /*1507 * If base_ref was set above that means we wish to1508 * reuse delta data, and we even found that base1509 * in the list of objects we want to pack. Goodie!1510 *1511 * Depth value does not matter - find_deltas() will1512 * never consider reused delta as the base object to1513 * deltify other objects against, in order to avoid1514 * circular deltas.1515 */1516 oe_set_type(entry, entry->in_pack_type);1517 SET_SIZE(entry, in_pack_size); /* delta size */1518 SET_DELTA(entry, base_entry);1519 SET_DELTA_SIZE(entry, in_pack_size);1520 entry->delta_sibling_idx = base_entry->delta_child_idx;1521 SET_DELTA_CHILD(base_entry, entry);1522 unuse_pack(&w_curs);1523 return;1524 }15251526 if (oe_type(entry)) {1527 off_t delta_pos;15281529 /*1530 * This must be a delta and we already know what the1531 * final object type is. Let's extract the actual1532 * object size from the delta header.1533 */1534 delta_pos = entry->in_pack_offset + entry->in_pack_header_size;1535 canonical_size = get_size_from_delta(p, &w_curs, delta_pos);1536 if (canonical_size == 0)1537 goto give_up;1538 SET_SIZE(entry, canonical_size);1539 unuse_pack(&w_curs);1540 return;1541 }15421543 /*1544 * No choice but to fall back to the recursive delta walk1545 * with sha1_object_info() to find about the object type1546 * at this point...1547 */1548 give_up:1549 unuse_pack(&w_curs);1550 }15511552 oe_set_type(entry,1553 oid_object_info(the_repository, &entry->idx.oid, &canonical_size));1554 if (entry->type_valid) {1555 SET_SIZE(entry, canonical_size);1556 } else {1557 /*1558 * Bad object type is checked in prepare_pack(). This is1559 * to permit a missing preferred base object to be ignored1560 * as a preferred base. Doing so can result in a larger1561 * pack file, but the transfer will still take place.1562 */1563 }1564}15651566static int pack_offset_sort(const void *_a, const void *_b)1567{1568 const struct object_entry *a = *(struct object_entry **)_a;1569 const struct object_entry *b = *(struct object_entry **)_b;1570 const struct packed_git *a_in_pack = IN_PACK(a);1571 const struct packed_git *b_in_pack = IN_PACK(b);15721573 /* avoid filesystem trashing with loose objects */1574 if (!a_in_pack && !b_in_pack)1575 return oidcmp(&a->idx.oid, &b->idx.oid);15761577 if (a_in_pack < b_in_pack)1578 return -1;1579 if (a_in_pack > b_in_pack)1580 return 1;1581 return a->in_pack_offset < b->in_pack_offset ? -1 :1582 (a->in_pack_offset > b->in_pack_offset);1583}15841585/*1586 * Drop an on-disk delta we were planning to reuse. Naively, this would1587 * just involve blanking out the "delta" field, but we have to deal1588 * with some extra book-keeping:1589 *1590 * 1. Removing ourselves from the delta_sibling linked list.1591 *1592 * 2. Updating our size/type to the non-delta representation. These were1593 * either not recorded initially (size) or overwritten with the delta type1594 * (type) when check_object() decided to reuse the delta.1595 *1596 * 3. Resetting our delta depth, as we are now a base object.1597 */1598static void drop_reused_delta(struct object_entry *entry)1599{1600 unsigned *idx = &to_pack.objects[entry->delta_idx - 1].delta_child_idx;1601 struct object_info oi = OBJECT_INFO_INIT;1602 enum object_type type;1603 unsigned long size;16041605 while (*idx) {1606 struct object_entry *oe = &to_pack.objects[*idx - 1];16071608 if (oe == entry)1609 *idx = oe->delta_sibling_idx;1610 else1611 idx = &oe->delta_sibling_idx;1612 }1613 SET_DELTA(entry, NULL);1614 entry->depth = 0;16151616 oi.sizep = &size;1617 oi.typep = &type;1618 if (packed_object_info(the_repository, IN_PACK(entry), entry->in_pack_offset, &oi) < 0) {1619 /*1620 * We failed to get the info from this pack for some reason;1621 * fall back to sha1_object_info, which may find another copy.1622 * And if that fails, the error will be recorded in oe_type(entry)1623 * and dealt with in prepare_pack().1624 */1625 oe_set_type(entry,1626 oid_object_info(the_repository, &entry->idx.oid, &size));1627 } else {1628 oe_set_type(entry, type);1629 }1630 SET_SIZE(entry, size);1631}16321633/*1634 * Follow the chain of deltas from this entry onward, throwing away any links1635 * that cause us to hit a cycle (as determined by the DFS state flags in1636 * the entries).1637 *1638 * We also detect too-long reused chains that would violate our --depth1639 * limit.1640 */1641static void break_delta_chains(struct object_entry *entry)1642{1643 /*1644 * The actual depth of each object we will write is stored as an int,1645 * as it cannot exceed our int "depth" limit. But before we break1646 * changes based no that limit, we may potentially go as deep as the1647 * number of objects, which is elsewhere bounded to a uint32_t.1648 */1649 uint32_t total_depth;1650 struct object_entry *cur, *next;16511652 for (cur = entry, total_depth = 0;1653 cur;1654 cur = DELTA(cur), total_depth++) {1655 if (cur->dfs_state == DFS_DONE) {1656 /*1657 * We've already seen this object and know it isn't1658 * part of a cycle. We do need to append its depth1659 * to our count.1660 */1661 total_depth += cur->depth;1662 break;1663 }16641665 /*1666 * We break cycles before looping, so an ACTIVE state (or any1667 * other cruft which made its way into the state variable)1668 * is a bug.1669 */1670 if (cur->dfs_state != DFS_NONE)1671 die("BUG: confusing delta dfs state in first pass: %d",1672 cur->dfs_state);16731674 /*1675 * Now we know this is the first time we've seen the object. If1676 * it's not a delta, we're done traversing, but we'll mark it1677 * done to save time on future traversals.1678 */1679 if (!DELTA(cur)) {1680 cur->dfs_state = DFS_DONE;1681 break;1682 }16831684 /*1685 * Mark ourselves as active and see if the next step causes1686 * us to cycle to another active object. It's important to do1687 * this _before_ we loop, because it impacts where we make the1688 * cut, and thus how our total_depth counter works.1689 * E.g., We may see a partial loop like:1690 *1691 * A -> B -> C -> D -> B1692 *1693 * Cutting B->C breaks the cycle. But now the depth of A is1694 * only 1, and our total_depth counter is at 3. The size of the1695 * error is always one less than the size of the cycle we1696 * broke. Commits C and D were "lost" from A's chain.1697 *1698 * If we instead cut D->B, then the depth of A is correct at 3.1699 * We keep all commits in the chain that we examined.1700 */1701 cur->dfs_state = DFS_ACTIVE;1702 if (DELTA(cur)->dfs_state == DFS_ACTIVE) {1703 drop_reused_delta(cur);1704 cur->dfs_state = DFS_DONE;1705 break;1706 }1707 }17081709 /*1710 * And now that we've gone all the way to the bottom of the chain, we1711 * need to clear the active flags and set the depth fields as1712 * appropriate. Unlike the loop above, which can quit when it drops a1713 * delta, we need to keep going to look for more depth cuts. So we need1714 * an extra "next" pointer to keep going after we reset cur->delta.1715 */1716 for (cur = entry; cur; cur = next) {1717 next = DELTA(cur);17181719 /*1720 * We should have a chain of zero or more ACTIVE states down to1721 * a final DONE. We can quit after the DONE, because either it1722 * has no bases, or we've already handled them in a previous1723 * call.1724 */1725 if (cur->dfs_state == DFS_DONE)1726 break;1727 else if (cur->dfs_state != DFS_ACTIVE)1728 die("BUG: confusing delta dfs state in second pass: %d",1729 cur->dfs_state);17301731 /*1732 * If the total_depth is more than depth, then we need to snip1733 * the chain into two or more smaller chains that don't exceed1734 * the maximum depth. Most of the resulting chains will contain1735 * (depth + 1) entries (i.e., depth deltas plus one base), and1736 * the last chain (i.e., the one containing entry) will contain1737 * whatever entries are left over, namely1738 * (total_depth % (depth + 1)) of them.1739 *1740 * Since we are iterating towards decreasing depth, we need to1741 * decrement total_depth as we go, and we need to write to the1742 * entry what its final depth will be after all of the1743 * snipping. Since we're snipping into chains of length (depth1744 * + 1) entries, the final depth of an entry will be its1745 * original depth modulo (depth + 1). Any time we encounter an1746 * entry whose final depth is supposed to be zero, we snip it1747 * from its delta base, thereby making it so.1748 */1749 cur->depth = (total_depth--) % (depth + 1);1750 if (!cur->depth)1751 drop_reused_delta(cur);17521753 cur->dfs_state = DFS_DONE;1754 }1755}17561757static void get_object_details(void)1758{1759 uint32_t i;1760 struct object_entry **sorted_by_offset;17611762 if (progress)1763 progress_state = start_progress(_("Counting objects"),1764 to_pack.nr_objects);17651766 sorted_by_offset = xcalloc(to_pack.nr_objects, sizeof(struct object_entry *));1767 for (i = 0; i < to_pack.nr_objects; i++)1768 sorted_by_offset[i] = to_pack.objects + i;1769 QSORT(sorted_by_offset, to_pack.nr_objects, pack_offset_sort);17701771 for (i = 0; i < to_pack.nr_objects; i++) {1772 struct object_entry *entry = sorted_by_offset[i];1773 check_object(entry);1774 if (entry->type_valid &&1775 oe_size_greater_than(&to_pack, entry, big_file_threshold))1776 entry->no_try_delta = 1;1777 display_progress(progress_state, i + 1);1778 }1779 stop_progress(&progress_state);17801781 /*1782 * This must happen in a second pass, since we rely on the delta1783 * information for the whole list being completed.1784 */1785 for (i = 0; i < to_pack.nr_objects; i++)1786 break_delta_chains(&to_pack.objects[i]);17871788 free(sorted_by_offset);1789}17901791/*1792 * We search for deltas in a list sorted by type, by filename hash, and then1793 * by size, so that we see progressively smaller and smaller files.1794 * That's because we prefer deltas to be from the bigger file1795 * to the smaller -- deletes are potentially cheaper, but perhaps1796 * more importantly, the bigger file is likely the more recent1797 * one. The deepest deltas are therefore the oldest objects which are1798 * less susceptible to be accessed often.1799 */1800static int type_size_sort(const void *_a, const void *_b)1801{1802 const struct object_entry *a = *(struct object_entry **)_a;1803 const struct object_entry *b = *(struct object_entry **)_b;1804 enum object_type a_type = oe_type(a);1805 enum object_type b_type = oe_type(b);1806 unsigned long a_size = SIZE(a);1807 unsigned long b_size = SIZE(b);18081809 if (a_type > b_type)1810 return -1;1811 if (a_type < b_type)1812 return 1;1813 if (a->hash > b->hash)1814 return -1;1815 if (a->hash < b->hash)1816 return 1;1817 if (a->preferred_base > b->preferred_base)1818 return -1;1819 if (a->preferred_base < b->preferred_base)1820 return 1;1821 if (a_size > b_size)1822 return -1;1823 if (a_size < b_size)1824 return 1;1825 return a < b ? -1 : (a > b); /* newest first */1826}18271828struct unpacked {1829 struct object_entry *entry;1830 void *data;1831 struct delta_index *index;1832 unsigned depth;1833};18341835static int delta_cacheable(unsigned long src_size, unsigned long trg_size,1836 unsigned long delta_size)1837{1838 if (max_delta_cache_size && delta_cache_size + delta_size > max_delta_cache_size)1839 return 0;18401841 if (delta_size < cache_max_small_delta_size)1842 return 1;18431844 /* cache delta, if objects are large enough compared to delta size */1845 if ((src_size >> 20) + (trg_size >> 21) > (delta_size >> 10))1846 return 1;18471848 return 0;1849}18501851#ifndef NO_PTHREADS18521853static pthread_mutex_t read_mutex;1854#define read_lock() pthread_mutex_lock(&read_mutex)1855#define read_unlock() pthread_mutex_unlock(&read_mutex)18561857static pthread_mutex_t cache_mutex;1858#define cache_lock() pthread_mutex_lock(&cache_mutex)1859#define cache_unlock() pthread_mutex_unlock(&cache_mutex)18601861static pthread_mutex_t progress_mutex;1862#define progress_lock() pthread_mutex_lock(&progress_mutex)1863#define progress_unlock() pthread_mutex_unlock(&progress_mutex)18641865#else18661867#define read_lock() (void)01868#define read_unlock() (void)01869#define cache_lock() (void)01870#define cache_unlock() (void)01871#define progress_lock() (void)01872#define progress_unlock() (void)018731874#endif18751876/*1877 * Return the size of the object without doing any delta1878 * reconstruction (so non-deltas are true object sizes, but deltas1879 * return the size of the delta data).1880 */1881unsigned long oe_get_size_slow(struct packing_data *pack,1882 const struct object_entry *e)1883{1884 struct packed_git *p;1885 struct pack_window *w_curs;1886 unsigned char *buf;1887 enum object_type type;1888 unsigned long used, avail, size;18891890 if (e->type_ != OBJ_OFS_DELTA && e->type_ != OBJ_REF_DELTA) {1891 read_lock();1892 if (oid_object_info(the_repository, &e->idx.oid, &size) < 0)1893 die(_("unable to get size of %s"),1894 oid_to_hex(&e->idx.oid));1895 read_unlock();1896 return size;1897 }18981899 p = oe_in_pack(pack, e);1900 if (!p)1901 BUG("when e->type is a delta, it must belong to a pack");19021903 read_lock();1904 w_curs = NULL;1905 buf = use_pack(p, &w_curs, e->in_pack_offset, &avail);1906 used = unpack_object_header_buffer(buf, avail, &type, &size);1907 if (used == 0)1908 die(_("unable to parse object header of %s"),1909 oid_to_hex(&e->idx.oid));19101911 unuse_pack(&w_curs);1912 read_unlock();1913 return size;1914}19151916static int try_delta(struct unpacked *trg, struct unpacked *src,1917 unsigned max_depth, unsigned long *mem_usage)1918{1919 struct object_entry *trg_entry = trg->entry;1920 struct object_entry *src_entry = src->entry;1921 unsigned long trg_size, src_size, delta_size, sizediff, max_size, sz;1922 unsigned ref_depth;1923 enum object_type type;1924 void *delta_buf;19251926 /* Don't bother doing diffs between different types */1927 if (oe_type(trg_entry) != oe_type(src_entry))1928 return -1;19291930 /*1931 * We do not bother to try a delta that we discarded on an1932 * earlier try, but only when reusing delta data. Note that1933 * src_entry that is marked as the preferred_base should always1934 * be considered, as even if we produce a suboptimal delta against1935 * it, we will still save the transfer cost, as we already know1936 * the other side has it and we won't send src_entry at all.1937 */1938 if (reuse_delta && IN_PACK(trg_entry) &&1939 IN_PACK(trg_entry) == IN_PACK(src_entry) &&1940 !src_entry->preferred_base &&1941 trg_entry->in_pack_type != OBJ_REF_DELTA &&1942 trg_entry->in_pack_type != OBJ_OFS_DELTA)1943 return 0;19441945 /* Let's not bust the allowed depth. */1946 if (src->depth >= max_depth)1947 return 0;19481949 /* Now some size filtering heuristics. */1950 trg_size = SIZE(trg_entry);1951 if (!DELTA(trg_entry)) {1952 max_size = trg_size/2 - 20;1953 ref_depth = 1;1954 } else {1955 max_size = DELTA_SIZE(trg_entry);1956 ref_depth = trg->depth;1957 }1958 max_size = (uint64_t)max_size * (max_depth - src->depth) /1959 (max_depth - ref_depth + 1);1960 if (max_size == 0)1961 return 0;1962 src_size = SIZE(src_entry);1963 sizediff = src_size < trg_size ? trg_size - src_size : 0;1964 if (sizediff >= max_size)1965 return 0;1966 if (trg_size < src_size / 32)1967 return 0;19681969 /* Load data if not already done */1970 if (!trg->data) {1971 read_lock();1972 trg->data = read_object_file(&trg_entry->idx.oid, &type, &sz);1973 read_unlock();1974 if (!trg->data)1975 die("object %s cannot be read",1976 oid_to_hex(&trg_entry->idx.oid));1977 if (sz != trg_size)1978 die("object %s inconsistent object length (%lu vs %lu)",1979 oid_to_hex(&trg_entry->idx.oid), sz,1980 trg_size);1981 *mem_usage += sz;1982 }1983 if (!src->data) {1984 read_lock();1985 src->data = read_object_file(&src_entry->idx.oid, &type, &sz);1986 read_unlock();1987 if (!src->data) {1988 if (src_entry->preferred_base) {1989 static int warned = 0;1990 if (!warned++)1991 warning("object %s cannot be read",1992 oid_to_hex(&src_entry->idx.oid));1993 /*1994 * Those objects are not included in the1995 * resulting pack. Be resilient and ignore1996 * them if they can't be read, in case the1997 * pack could be created nevertheless.1998 */1999 return 0;2000 }2001 die("object %s cannot be read",2002 oid_to_hex(&src_entry->idx.oid));2003 }2004 if (sz != src_size)2005 die("object %s inconsistent object length (%lu vs %lu)",2006 oid_to_hex(&src_entry->idx.oid), sz,2007 src_size);2008 *mem_usage += sz;2009 }2010 if (!src->index) {2011 src->index = create_delta_index(src->data, src_size);2012 if (!src->index) {2013 static int warned = 0;2014 if (!warned++)2015 warning("suboptimal pack - out of memory");2016 return 0;2017 }2018 *mem_usage += sizeof_delta_index(src->index);2019 }20202021 delta_buf = create_delta(src->index, trg->data, trg_size, &delta_size, max_size);2022 if (!delta_buf)2023 return 0;2024 if (delta_size >= (1U << OE_DELTA_SIZE_BITS)) {2025 free(delta_buf);2026 return 0;2027 }20282029 if (DELTA(trg_entry)) {2030 /* Prefer only shallower same-sized deltas. */2031 if (delta_size == DELTA_SIZE(trg_entry) &&2032 src->depth + 1 >= trg->depth) {2033 free(delta_buf);2034 return 0;2035 }2036 }20372038 /*2039 * Handle memory allocation outside of the cache2040 * accounting lock. Compiler will optimize the strangeness2041 * away when NO_PTHREADS is defined.2042 */2043 free(trg_entry->delta_data);2044 cache_lock();2045 if (trg_entry->delta_data) {2046 delta_cache_size -= DELTA_SIZE(trg_entry);2047 trg_entry->delta_data = NULL;2048 }2049 if (delta_cacheable(src_size, trg_size, delta_size)) {2050 delta_cache_size += delta_size;2051 cache_unlock();2052 trg_entry->delta_data = xrealloc(delta_buf, delta_size);2053 } else {2054 cache_unlock();2055 free(delta_buf);2056 }20572058 SET_DELTA(trg_entry, src_entry);2059 SET_DELTA_SIZE(trg_entry, delta_size);2060 trg->depth = src->depth + 1;20612062 return 1;2063}20642065static unsigned int check_delta_limit(struct object_entry *me, unsigned int n)2066{2067 struct object_entry *child = DELTA_CHILD(me);2068 unsigned int m = n;2069 while (child) {2070 unsigned int c = check_delta_limit(child, n + 1);2071 if (m < c)2072 m = c;2073 child = DELTA_SIBLING(child);2074 }2075 return m;2076}20772078static unsigned long free_unpacked(struct unpacked *n)2079{2080 unsigned long freed_mem = sizeof_delta_index(n->index);2081 free_delta_index(n->index);2082 n->index = NULL;2083 if (n->data) {2084 freed_mem += SIZE(n->entry);2085 FREE_AND_NULL(n->data);2086 }2087 n->entry = NULL;2088 n->depth = 0;2089 return freed_mem;2090}20912092static void find_deltas(struct object_entry **list, unsigned *list_size,2093 int window, int depth, unsigned *processed)2094{2095 uint32_t i, idx = 0, count = 0;2096 struct unpacked *array;2097 unsigned long mem_usage = 0;20982099 array = xcalloc(window, sizeof(struct unpacked));21002101 for (;;) {2102 struct object_entry *entry;2103 struct unpacked *n = array + idx;2104 int j, max_depth, best_base = -1;21052106 progress_lock();2107 if (!*list_size) {2108 progress_unlock();2109 break;2110 }2111 entry = *list++;2112 (*list_size)--;2113 if (!entry->preferred_base) {2114 (*processed)++;2115 display_progress(progress_state, *processed);2116 }2117 progress_unlock();21182119 mem_usage -= free_unpacked(n);2120 n->entry = entry;21212122 while (window_memory_limit &&2123 mem_usage > window_memory_limit &&2124 count > 1) {2125 uint32_t tail = (idx + window - count) % window;2126 mem_usage -= free_unpacked(array + tail);2127 count--;2128 }21292130 /* We do not compute delta to *create* objects we are not2131 * going to pack.2132 */2133 if (entry->preferred_base)2134 goto next;21352136 /*2137 * If the current object is at pack edge, take the depth the2138 * objects that depend on the current object into account2139 * otherwise they would become too deep.2140 */2141 max_depth = depth;2142 if (DELTA_CHILD(entry)) {2143 max_depth -= check_delta_limit(entry, 0);2144 if (max_depth <= 0)2145 goto next;2146 }21472148 j = window;2149 while (--j > 0) {2150 int ret;2151 uint32_t other_idx = idx + j;2152 struct unpacked *m;2153 if (other_idx >= window)2154 other_idx -= window;2155 m = array + other_idx;2156 if (!m->entry)2157 break;2158 ret = try_delta(n, m, max_depth, &mem_usage);2159 if (ret < 0)2160 break;2161 else if (ret > 0)2162 best_base = other_idx;2163 }21642165 /*2166 * If we decided to cache the delta data, then it is best2167 * to compress it right away. First because we have to do2168 * it anyway, and doing it here while we're threaded will2169 * save a lot of time in the non threaded write phase,2170 * as well as allow for caching more deltas within2171 * the same cache size limit.2172 * ...2173 * But only if not writing to stdout, since in that case2174 * the network is most likely throttling writes anyway,2175 * and therefore it is best to go to the write phase ASAP2176 * instead, as we can afford spending more time compressing2177 * between writes at that moment.2178 */2179 if (entry->delta_data && !pack_to_stdout) {2180 unsigned long size;21812182 size = do_compress(&entry->delta_data, DELTA_SIZE(entry));2183 if (size < (1U << OE_Z_DELTA_BITS)) {2184 entry->z_delta_size = size;2185 cache_lock();2186 delta_cache_size -= DELTA_SIZE(entry);2187 delta_cache_size += entry->z_delta_size;2188 cache_unlock();2189 } else {2190 FREE_AND_NULL(entry->delta_data);2191 entry->z_delta_size = 0;2192 }2193 }21942195 /* if we made n a delta, and if n is already at max2196 * depth, leaving it in the window is pointless. we2197 * should evict it first.2198 */2199 if (DELTA(entry) && max_depth <= n->depth)2200 continue;22012202 /*2203 * Move the best delta base up in the window, after the2204 * currently deltified object, to keep it longer. It will2205 * be the first base object to be attempted next.2206 */2207 if (DELTA(entry)) {2208 struct unpacked swap = array[best_base];2209 int dist = (window + idx - best_base) % window;2210 int dst = best_base;2211 while (dist--) {2212 int src = (dst + 1) % window;2213 array[dst] = array[src];2214 dst = src;2215 }2216 array[dst] = swap;2217 }22182219 next:2220 idx++;2221 if (count + 1 < window)2222 count++;2223 if (idx >= window)2224 idx = 0;2225 }22262227 for (i = 0; i < window; ++i) {2228 free_delta_index(array[i].index);2229 free(array[i].data);2230 }2231 free(array);2232}22332234#ifndef NO_PTHREADS22352236static void try_to_free_from_threads(size_t size)2237{2238 read_lock();2239 release_pack_memory(size);2240 read_unlock();2241}22422243static try_to_free_t old_try_to_free_routine;22442245/*2246 * The main thread waits on the condition that (at least) one of the workers2247 * has stopped working (which is indicated in the .working member of2248 * struct thread_params).2249 * When a work thread has completed its work, it sets .working to 0 and2250 * signals the main thread and waits on the condition that .data_ready2251 * becomes 1.2252 */22532254struct thread_params {2255 pthread_t thread;2256 struct object_entry **list;2257 unsigned list_size;2258 unsigned remaining;2259 int window;2260 int depth;2261 int working;2262 int data_ready;2263 pthread_mutex_t mutex;2264 pthread_cond_t cond;2265 unsigned *processed;2266};22672268static pthread_cond_t progress_cond;22692270/*2271 * Mutex and conditional variable can't be statically-initialized on Windows.2272 */2273static void init_threaded_search(void)2274{2275 init_recursive_mutex(&read_mutex);2276 pthread_mutex_init(&cache_mutex, NULL);2277 pthread_mutex_init(&progress_mutex, NULL);2278 pthread_cond_init(&progress_cond, NULL);2279 old_try_to_free_routine = set_try_to_free_routine(try_to_free_from_threads);2280}22812282static void cleanup_threaded_search(void)2283{2284 set_try_to_free_routine(old_try_to_free_routine);2285 pthread_cond_destroy(&progress_cond);2286 pthread_mutex_destroy(&read_mutex);2287 pthread_mutex_destroy(&cache_mutex);2288 pthread_mutex_destroy(&progress_mutex);2289}22902291static void *threaded_find_deltas(void *arg)2292{2293 struct thread_params *me = arg;22942295 progress_lock();2296 while (me->remaining) {2297 progress_unlock();22982299 find_deltas(me->list, &me->remaining,2300 me->window, me->depth, me->processed);23012302 progress_lock();2303 me->working = 0;2304 pthread_cond_signal(&progress_cond);2305 progress_unlock();23062307 /*2308 * We must not set ->data_ready before we wait on the2309 * condition because the main thread may have set it to 12310 * before we get here. In order to be sure that new2311 * work is available if we see 1 in ->data_ready, it2312 * was initialized to 0 before this thread was spawned2313 * and we reset it to 0 right away.2314 */2315 pthread_mutex_lock(&me->mutex);2316 while (!me->data_ready)2317 pthread_cond_wait(&me->cond, &me->mutex);2318 me->data_ready = 0;2319 pthread_mutex_unlock(&me->mutex);23202321 progress_lock();2322 }2323 progress_unlock();2324 /* leave ->working 1 so that this doesn't get more work assigned */2325 return NULL;2326}23272328static void ll_find_deltas(struct object_entry **list, unsigned list_size,2329 int window, int depth, unsigned *processed)2330{2331 struct thread_params *p;2332 int i, ret, active_threads = 0;23332334 init_threaded_search();23352336 if (delta_search_threads <= 1) {2337 find_deltas(list, &list_size, window, depth, processed);2338 cleanup_threaded_search();2339 return;2340 }2341 if (progress > pack_to_stdout)2342 fprintf(stderr, "Delta compression using up to %d threads.\n",2343 delta_search_threads);2344 p = xcalloc(delta_search_threads, sizeof(*p));23452346 /* Partition the work amongst work threads. */2347 for (i = 0; i < delta_search_threads; i++) {2348 unsigned sub_size = list_size / (delta_search_threads - i);23492350 /* don't use too small segments or no deltas will be found */2351 if (sub_size < 2*window && i+1 < delta_search_threads)2352 sub_size = 0;23532354 p[i].window = window;2355 p[i].depth = depth;2356 p[i].processed = processed;2357 p[i].working = 1;2358 p[i].data_ready = 0;23592360 /* try to split chunks on "path" boundaries */2361 while (sub_size && sub_size < list_size &&2362 list[sub_size]->hash &&2363 list[sub_size]->hash == list[sub_size-1]->hash)2364 sub_size++;23652366 p[i].list = list;2367 p[i].list_size = sub_size;2368 p[i].remaining = sub_size;23692370 list += sub_size;2371 list_size -= sub_size;2372 }23732374 /* Start work threads. */2375 for (i = 0; i < delta_search_threads; i++) {2376 if (!p[i].list_size)2377 continue;2378 pthread_mutex_init(&p[i].mutex, NULL);2379 pthread_cond_init(&p[i].cond, NULL);2380 ret = pthread_create(&p[i].thread, NULL,2381 threaded_find_deltas, &p[i]);2382 if (ret)2383 die("unable to create thread: %s", strerror(ret));2384 active_threads++;2385 }23862387 /*2388 * Now let's wait for work completion. Each time a thread is done2389 * with its work, we steal half of the remaining work from the2390 * thread with the largest number of unprocessed objects and give2391 * it to that newly idle thread. This ensure good load balancing2392 * until the remaining object list segments are simply too short2393 * to be worth splitting anymore.2394 */2395 while (active_threads) {2396 struct thread_params *target = NULL;2397 struct thread_params *victim = NULL;2398 unsigned sub_size = 0;23992400 progress_lock();2401 for (;;) {2402 for (i = 0; !target && i < delta_search_threads; i++)2403 if (!p[i].working)2404 target = &p[i];2405 if (target)2406 break;2407 pthread_cond_wait(&progress_cond, &progress_mutex);2408 }24092410 for (i = 0; i < delta_search_threads; i++)2411 if (p[i].remaining > 2*window &&2412 (!victim || victim->remaining < p[i].remaining))2413 victim = &p[i];2414 if (victim) {2415 sub_size = victim->remaining / 2;2416 list = victim->list + victim->list_size - sub_size;2417 while (sub_size && list[0]->hash &&2418 list[0]->hash == list[-1]->hash) {2419 list++;2420 sub_size--;2421 }2422 if (!sub_size) {2423 /*2424 * It is possible for some "paths" to have2425 * so many objects that no hash boundary2426 * might be found. Let's just steal the2427 * exact half in that case.2428 */2429 sub_size = victim->remaining / 2;2430 list -= sub_size;2431 }2432 target->list = list;2433 victim->list_size -= sub_size;2434 victim->remaining -= sub_size;2435 }2436 target->list_size = sub_size;2437 target->remaining = sub_size;2438 target->working = 1;2439 progress_unlock();24402441 pthread_mutex_lock(&target->mutex);2442 target->data_ready = 1;2443 pthread_cond_signal(&target->cond);2444 pthread_mutex_unlock(&target->mutex);24452446 if (!sub_size) {2447 pthread_join(target->thread, NULL);2448 pthread_cond_destroy(&target->cond);2449 pthread_mutex_destroy(&target->mutex);2450 active_threads--;2451 }2452 }2453 cleanup_threaded_search();2454 free(p);2455}24562457#else2458#define ll_find_deltas(l, s, w, d, p) find_deltas(l, &s, w, d, p)2459#endif24602461static void add_tag_chain(const struct object_id *oid)2462{2463 struct tag *tag;24642465 /*2466 * We catch duplicates already in add_object_entry(), but we'd2467 * prefer to do this extra check to avoid having to parse the2468 * tag at all if we already know that it's being packed (e.g., if2469 * it was included via bitmaps, we would not have parsed it2470 * previously).2471 */2472 if (packlist_find(&to_pack, oid->hash, NULL))2473 return;24742475 tag = lookup_tag(oid);2476 while (1) {2477 if (!tag || parse_tag(tag) || !tag->tagged)2478 die("unable to pack objects reachable from tag %s",2479 oid_to_hex(oid));24802481 add_object_entry(&tag->object.oid, OBJ_TAG, NULL, 0);24822483 if (tag->tagged->type != OBJ_TAG)2484 return;24852486 tag = (struct tag *)tag->tagged;2487 }2488}24892490static int add_ref_tag(const char *path, const struct object_id *oid, int flag, void *cb_data)2491{2492 struct object_id peeled;24932494 if (starts_with(path, "refs/tags/") && /* is a tag? */2495 !peel_ref(path, &peeled) && /* peelable? */2496 packlist_find(&to_pack, peeled.hash, NULL)) /* object packed? */2497 add_tag_chain(oid);2498 return 0;2499}25002501static void prepare_pack(int window, int depth)2502{2503 struct object_entry **delta_list;2504 uint32_t i, nr_deltas;2505 unsigned n;25062507 get_object_details();25082509 /*2510 * If we're locally repacking then we need to be doubly careful2511 * from now on in order to make sure no stealth corruption gets2512 * propagated to the new pack. Clients receiving streamed packs2513 * should validate everything they get anyway so no need to incur2514 * the additional cost here in that case.2515 */2516 if (!pack_to_stdout)2517 do_check_packed_object_crc = 1;25182519 if (!to_pack.nr_objects || !window || !depth)2520 return;25212522 ALLOC_ARRAY(delta_list, to_pack.nr_objects);2523 nr_deltas = n = 0;25242525 for (i = 0; i < to_pack.nr_objects; i++) {2526 struct object_entry *entry = to_pack.objects + i;25272528 if (DELTA(entry))2529 /* This happens if we decided to reuse existing2530 * delta from a pack. "reuse_delta &&" is implied.2531 */2532 continue;25332534 if (!entry->type_valid ||2535 oe_size_less_than(&to_pack, entry, 50))2536 continue;25372538 if (entry->no_try_delta)2539 continue;25402541 if (!entry->preferred_base) {2542 nr_deltas++;2543 if (oe_type(entry) < 0)2544 die("unable to get type of object %s",2545 oid_to_hex(&entry->idx.oid));2546 } else {2547 if (oe_type(entry) < 0) {2548 /*2549 * This object is not found, but we2550 * don't have to include it anyway.2551 */2552 continue;2553 }2554 }25552556 delta_list[n++] = entry;2557 }25582559 if (nr_deltas && n > 1) {2560 unsigned nr_done = 0;2561 if (progress)2562 progress_state = start_progress(_("Compressing objects"),2563 nr_deltas);2564 QSORT(delta_list, n, type_size_sort);2565 ll_find_deltas(delta_list, n, window+1, depth, &nr_done);2566 stop_progress(&progress_state);2567 if (nr_done != nr_deltas)2568 die("inconsistency with delta count");2569 }2570 free(delta_list);2571}25722573static int git_pack_config(const char *k, const char *v, void *cb)2574{2575 if (!strcmp(k, "pack.window")) {2576 window = git_config_int(k, v);2577 return 0;2578 }2579 if (!strcmp(k, "pack.windowmemory")) {2580 window_memory_limit = git_config_ulong(k, v);2581 return 0;2582 }2583 if (!strcmp(k, "pack.depth")) {2584 depth = git_config_int(k, v);2585 return 0;2586 }2587 if (!strcmp(k, "pack.deltacachesize")) {2588 max_delta_cache_size = git_config_int(k, v);2589 return 0;2590 }2591 if (!strcmp(k, "pack.deltacachelimit")) {2592 cache_max_small_delta_size = git_config_int(k, v);2593 return 0;2594 }2595 if (!strcmp(k, "pack.writebitmaphashcache")) {2596 if (git_config_bool(k, v))2597 write_bitmap_options |= BITMAP_OPT_HASH_CACHE;2598 else2599 write_bitmap_options &= ~BITMAP_OPT_HASH_CACHE;2600 }2601 if (!strcmp(k, "pack.usebitmaps")) {2602 use_bitmap_index_default = git_config_bool(k, v);2603 return 0;2604 }2605 if (!strcmp(k, "pack.threads")) {2606 delta_search_threads = git_config_int(k, v);2607 if (delta_search_threads < 0)2608 die("invalid number of threads specified (%d)",2609 delta_search_threads);2610#ifdef NO_PTHREADS2611 if (delta_search_threads != 1) {2612 warning("no threads support, ignoring %s", k);2613 delta_search_threads = 0;2614 }2615#endif2616 return 0;2617 }2618 if (!strcmp(k, "pack.indexversion")) {2619 pack_idx_opts.version = git_config_int(k, v);2620 if (pack_idx_opts.version > 2)2621 die("bad pack.indexversion=%"PRIu32,2622 pack_idx_opts.version);2623 return 0;2624 }2625 return git_default_config(k, v, cb);2626}26272628static void read_object_list_from_stdin(void)2629{2630 char line[GIT_MAX_HEXSZ + 1 + PATH_MAX + 2];2631 struct object_id oid;2632 const char *p;26332634 for (;;) {2635 if (!fgets(line, sizeof(line), stdin)) {2636 if (feof(stdin))2637 break;2638 if (!ferror(stdin))2639 die("fgets returned NULL, not EOF, not error!");2640 if (errno != EINTR)2641 die_errno("fgets");2642 clearerr(stdin);2643 continue;2644 }2645 if (line[0] == '-') {2646 if (get_oid_hex(line+1, &oid))2647 die("expected edge object ID, got garbage:\n %s",2648 line);2649 add_preferred_base(&oid);2650 continue;2651 }2652 if (parse_oid_hex(line, &oid, &p))2653 die("expected object ID, got garbage:\n %s", line);26542655 add_preferred_base_object(p + 1);2656 add_object_entry(&oid, OBJ_NONE, p + 1, 0);2657 }2658}26592660/* Remember to update object flag allocation in object.h */2661#define OBJECT_ADDED (1u<<20)26622663static void show_commit(struct commit *commit, void *data)2664{2665 add_object_entry(&commit->object.oid, OBJ_COMMIT, NULL, 0);2666 commit->object.flags |= OBJECT_ADDED;26672668 if (write_bitmap_index)2669 index_commit_for_bitmap(commit);2670}26712672static void show_object(struct object *obj, const char *name, void *data)2673{2674 add_preferred_base_object(name);2675 add_object_entry(&obj->oid, obj->type, name, 0);2676 obj->flags |= OBJECT_ADDED;2677}26782679static void show_object__ma_allow_any(struct object *obj, const char *name, void *data)2680{2681 assert(arg_missing_action == MA_ALLOW_ANY);26822683 /*2684 * Quietly ignore ALL missing objects. This avoids problems with2685 * staging them now and getting an odd error later.2686 */2687 if (!has_object_file(&obj->oid))2688 return;26892690 show_object(obj, name, data);2691}26922693static void show_object__ma_allow_promisor(struct object *obj, const char *name, void *data)2694{2695 assert(arg_missing_action == MA_ALLOW_PROMISOR);26962697 /*2698 * Quietly ignore EXPECTED missing objects. This avoids problems with2699 * staging them now and getting an odd error later.2700 */2701 if (!has_object_file(&obj->oid) && is_promisor_object(&obj->oid))2702 return;27032704 show_object(obj, name, data);2705}27062707static int option_parse_missing_action(const struct option *opt,2708 const char *arg, int unset)2709{2710 assert(arg);2711 assert(!unset);27122713 if (!strcmp(arg, "error")) {2714 arg_missing_action = MA_ERROR;2715 fn_show_object = show_object;2716 return 0;2717 }27182719 if (!strcmp(arg, "allow-any")) {2720 arg_missing_action = MA_ALLOW_ANY;2721 fetch_if_missing = 0;2722 fn_show_object = show_object__ma_allow_any;2723 return 0;2724 }27252726 if (!strcmp(arg, "allow-promisor")) {2727 arg_missing_action = MA_ALLOW_PROMISOR;2728 fetch_if_missing = 0;2729 fn_show_object = show_object__ma_allow_promisor;2730 return 0;2731 }27322733 die(_("invalid value for --missing"));2734 return 0;2735}27362737static void show_edge(struct commit *commit)2738{2739 add_preferred_base(&commit->object.oid);2740}27412742struct in_pack_object {2743 off_t offset;2744 struct object *object;2745};27462747struct in_pack {2748 unsigned int alloc;2749 unsigned int nr;2750 struct in_pack_object *array;2751};27522753static void mark_in_pack_object(struct object *object, struct packed_git *p, struct in_pack *in_pack)2754{2755 in_pack->array[in_pack->nr].offset = find_pack_entry_one(object->oid.hash, p);2756 in_pack->array[in_pack->nr].object = object;2757 in_pack->nr++;2758}27592760/*2761 * Compare the objects in the offset order, in order to emulate the2762 * "git rev-list --objects" output that produced the pack originally.2763 */2764static int ofscmp(const void *a_, const void *b_)2765{2766 struct in_pack_object *a = (struct in_pack_object *)a_;2767 struct in_pack_object *b = (struct in_pack_object *)b_;27682769 if (a->offset < b->offset)2770 return -1;2771 else if (a->offset > b->offset)2772 return 1;2773 else2774 return oidcmp(&a->object->oid, &b->object->oid);2775}27762777static void add_objects_in_unpacked_packs(struct rev_info *revs)2778{2779 struct packed_git *p;2780 struct in_pack in_pack;2781 uint32_t i;27822783 memset(&in_pack, 0, sizeof(in_pack));27842785 for (p = get_packed_git(the_repository); p; p = p->next) {2786 struct object_id oid;2787 struct object *o;27882789 if (!p->pack_local || p->pack_keep || p->pack_keep_in_core)2790 continue;2791 if (open_pack_index(p))2792 die("cannot open pack index");27932794 ALLOC_GROW(in_pack.array,2795 in_pack.nr + p->num_objects,2796 in_pack.alloc);27972798 for (i = 0; i < p->num_objects; i++) {2799 nth_packed_object_oid(&oid, p, i);2800 o = lookup_unknown_object(oid.hash);2801 if (!(o->flags & OBJECT_ADDED))2802 mark_in_pack_object(o, p, &in_pack);2803 o->flags |= OBJECT_ADDED;2804 }2805 }28062807 if (in_pack.nr) {2808 QSORT(in_pack.array, in_pack.nr, ofscmp);2809 for (i = 0; i < in_pack.nr; i++) {2810 struct object *o = in_pack.array[i].object;2811 add_object_entry(&o->oid, o->type, "", 0);2812 }2813 }2814 free(in_pack.array);2815}28162817static int add_loose_object(const struct object_id *oid, const char *path,2818 void *data)2819{2820 enum object_type type = oid_object_info(the_repository, oid, NULL);28212822 if (type < 0) {2823 warning("loose object at %s could not be examined", path);2824 return 0;2825 }28262827 add_object_entry(oid, type, "", 0);2828 return 0;2829}28302831/*2832 * We actually don't even have to worry about reachability here.2833 * add_object_entry will weed out duplicates, so we just add every2834 * loose object we find.2835 */2836static void add_unreachable_loose_objects(void)2837{2838 for_each_loose_file_in_objdir(get_object_directory(),2839 add_loose_object,2840 NULL, NULL, NULL);2841}28422843static int has_sha1_pack_kept_or_nonlocal(const struct object_id *oid)2844{2845 static struct packed_git *last_found = (void *)1;2846 struct packed_git *p;28472848 p = (last_found != (void *)1) ? last_found :2849 get_packed_git(the_repository);28502851 while (p) {2852 if ((!p->pack_local || p->pack_keep ||2853 p->pack_keep_in_core) &&2854 find_pack_entry_one(oid->hash, p)) {2855 last_found = p;2856 return 1;2857 }2858 if (p == last_found)2859 p = get_packed_git(the_repository);2860 else2861 p = p->next;2862 if (p == last_found)2863 p = p->next;2864 }2865 return 0;2866}28672868/*2869 * Store a list of sha1s that are should not be discarded2870 * because they are either written too recently, or are2871 * reachable from another object that was.2872 *2873 * This is filled by get_object_list.2874 */2875static struct oid_array recent_objects;28762877static int loosened_object_can_be_discarded(const struct object_id *oid,2878 timestamp_t mtime)2879{2880 if (!unpack_unreachable_expiration)2881 return 0;2882 if (mtime > unpack_unreachable_expiration)2883 return 0;2884 if (oid_array_lookup(&recent_objects, oid) >= 0)2885 return 0;2886 return 1;2887}28882889static void loosen_unused_packed_objects(struct rev_info *revs)2890{2891 struct packed_git *p;2892 uint32_t i;2893 struct object_id oid;28942895 for (p = get_packed_git(the_repository); p; p = p->next) {2896 if (!p->pack_local || p->pack_keep || p->pack_keep_in_core)2897 continue;28982899 if (open_pack_index(p))2900 die("cannot open pack index");29012902 for (i = 0; i < p->num_objects; i++) {2903 nth_packed_object_oid(&oid, p, i);2904 if (!packlist_find(&to_pack, oid.hash, NULL) &&2905 !has_sha1_pack_kept_or_nonlocal(&oid) &&2906 !loosened_object_can_be_discarded(&oid, p->mtime))2907 if (force_object_loose(&oid, p->mtime))2908 die("unable to force loose object");2909 }2910 }2911}29122913/*2914 * This tracks any options which pack-reuse code expects to be on, or which a2915 * reader of the pack might not understand, and which would therefore prevent2916 * blind reuse of what we have on disk.2917 */2918static int pack_options_allow_reuse(void)2919{2920 return pack_to_stdout &&2921 allow_ofs_delta &&2922 !ignore_packed_keep_on_disk &&2923 !ignore_packed_keep_in_core &&2924 (!local || !have_non_local_packs) &&2925 !incremental;2926}29272928static int get_object_list_from_bitmap(struct rev_info *revs)2929{2930 if (prepare_bitmap_walk(revs) < 0)2931 return -1;29322933 if (pack_options_allow_reuse() &&2934 !reuse_partial_packfile_from_bitmap(2935 &reuse_packfile,2936 &reuse_packfile_objects,2937 &reuse_packfile_offset)) {2938 assert(reuse_packfile_objects);2939 nr_result += reuse_packfile_objects;2940 display_progress(progress_state, nr_result);2941 }29422943 traverse_bitmap_commit_list(&add_object_entry_from_bitmap);2944 return 0;2945}29462947static void record_recent_object(struct object *obj,2948 const char *name,2949 void *data)2950{2951 oid_array_append(&recent_objects, &obj->oid);2952}29532954static void record_recent_commit(struct commit *commit, void *data)2955{2956 oid_array_append(&recent_objects, &commit->object.oid);2957}29582959static void get_object_list(int ac, const char **av)2960{2961 struct rev_info revs;2962 char line[1000];2963 int flags = 0;29642965 init_revisions(&revs, NULL);2966 save_commit_buffer = 0;2967 setup_revisions(ac, av, &revs, NULL);29682969 /* make sure shallows are read */2970 is_repository_shallow();29712972 while (fgets(line, sizeof(line), stdin) != NULL) {2973 int len = strlen(line);2974 if (len && line[len - 1] == '\n')2975 line[--len] = 0;2976 if (!len)2977 break;2978 if (*line == '-') {2979 if (!strcmp(line, "--not")) {2980 flags ^= UNINTERESTING;2981 write_bitmap_index = 0;2982 continue;2983 }2984 if (starts_with(line, "--shallow ")) {2985 struct object_id oid;2986 if (get_oid_hex(line + 10, &oid))2987 die("not an SHA-1 '%s'", line + 10);2988 register_shallow(&oid);2989 use_bitmap_index = 0;2990 continue;2991 }2992 die("not a rev '%s'", line);2993 }2994 if (handle_revision_arg(line, &revs, flags, REVARG_CANNOT_BE_FILENAME))2995 die("bad revision '%s'", line);2996 }29972998 if (use_bitmap_index && !get_object_list_from_bitmap(&revs))2999 return;30003001 if (prepare_revision_walk(&revs))3002 die("revision walk setup failed");3003 mark_edges_uninteresting(&revs, show_edge);30043005 if (!fn_show_object)3006 fn_show_object = show_object;3007 traverse_commit_list_filtered(&filter_options, &revs,3008 show_commit, fn_show_object, NULL,3009 NULL);30103011 if (unpack_unreachable_expiration) {3012 revs.ignore_missing_links = 1;3013 if (add_unseen_recent_objects_to_traversal(&revs,3014 unpack_unreachable_expiration))3015 die("unable to add recent objects");3016 if (prepare_revision_walk(&revs))3017 die("revision walk setup failed");3018 traverse_commit_list(&revs, record_recent_commit,3019 record_recent_object, NULL);3020 }30213022 if (keep_unreachable)3023 add_objects_in_unpacked_packs(&revs);3024 if (pack_loose_unreachable)3025 add_unreachable_loose_objects();3026 if (unpack_unreachable)3027 loosen_unused_packed_objects(&revs);30283029 oid_array_clear(&recent_objects);3030}30313032static void add_extra_kept_packs(const struct string_list *names)3033{3034 struct packed_git *p;30353036 if (!names->nr)3037 return;30383039 for (p = get_packed_git(the_repository); p; p = p->next) {3040 const char *name = basename(p->pack_name);3041 int i;30423043 if (!p->pack_local)3044 continue;30453046 for (i = 0; i < names->nr; i++)3047 if (!fspathcmp(name, names->items[i].string))3048 break;30493050 if (i < names->nr) {3051 p->pack_keep_in_core = 1;3052 ignore_packed_keep_in_core = 1;3053 continue;3054 }3055 }3056}30573058static int option_parse_index_version(const struct option *opt,3059 const char *arg, int unset)3060{3061 char *c;3062 const char *val = arg;3063 pack_idx_opts.version = strtoul(val, &c, 10);3064 if (pack_idx_opts.version > 2)3065 die(_("unsupported index version %s"), val);3066 if (*c == ',' && c[1])3067 pack_idx_opts.off32_limit = strtoul(c+1, &c, 0);3068 if (*c || pack_idx_opts.off32_limit & 0x80000000)3069 die(_("bad index version '%s'"), val);3070 return 0;3071}30723073static int option_parse_unpack_unreachable(const struct option *opt,3074 const char *arg, int unset)3075{3076 if (unset) {3077 unpack_unreachable = 0;3078 unpack_unreachable_expiration = 0;3079 }3080 else {3081 unpack_unreachable = 1;3082 if (arg)3083 unpack_unreachable_expiration = approxidate(arg);3084 }3085 return 0;3086}30873088int cmd_pack_objects(int argc, const char **argv, const char *prefix)3089{3090 int use_internal_rev_list = 0;3091 int thin = 0;3092 int shallow = 0;3093 int all_progress_implied = 0;3094 struct argv_array rp = ARGV_ARRAY_INIT;3095 int rev_list_unpacked = 0, rev_list_all = 0, rev_list_reflog = 0;3096 int rev_list_index = 0;3097 struct string_list keep_pack_list = STRING_LIST_INIT_NODUP;3098 struct option pack_objects_options[] = {3099 OPT_SET_INT('q', "quiet", &progress,3100 N_("do not show progress meter"), 0),3101 OPT_SET_INT(0, "progress", &progress,3102 N_("show progress meter"), 1),3103 OPT_SET_INT(0, "all-progress", &progress,3104 N_("show progress meter during object writing phase"), 2),3105 OPT_BOOL(0, "all-progress-implied",3106 &all_progress_implied,3107 N_("similar to --all-progress when progress meter is shown")),3108 { OPTION_CALLBACK, 0, "index-version", NULL, N_("version[,offset]"),3109 N_("write the pack index file in the specified idx format version"),3110 0, option_parse_index_version },3111 OPT_MAGNITUDE(0, "max-pack-size", &pack_size_limit,3112 N_("maximum size of each output pack file")),3113 OPT_BOOL(0, "local", &local,3114 N_("ignore borrowed objects from alternate object store")),3115 OPT_BOOL(0, "incremental", &incremental,3116 N_("ignore packed objects")),3117 OPT_INTEGER(0, "window", &window,3118 N_("limit pack window by objects")),3119 OPT_MAGNITUDE(0, "window-memory", &window_memory_limit,3120 N_("limit pack window by memory in addition to object limit")),3121 OPT_INTEGER(0, "depth", &depth,3122 N_("maximum length of delta chain allowed in the resulting pack")),3123 OPT_BOOL(0, "reuse-delta", &reuse_delta,3124 N_("reuse existing deltas")),3125 OPT_BOOL(0, "reuse-object", &reuse_object,3126 N_("reuse existing objects")),3127 OPT_BOOL(0, "delta-base-offset", &allow_ofs_delta,3128 N_("use OFS_DELTA objects")),3129 OPT_INTEGER(0, "threads", &delta_search_threads,3130 N_("use threads when searching for best delta matches")),3131 OPT_BOOL(0, "non-empty", &non_empty,3132 N_("do not create an empty pack output")),3133 OPT_BOOL(0, "revs", &use_internal_rev_list,3134 N_("read revision arguments from standard input")),3135 { OPTION_SET_INT, 0, "unpacked", &rev_list_unpacked, NULL,3136 N_("limit the objects to those that are not yet packed"),3137 PARSE_OPT_NOARG | PARSE_OPT_NONEG, NULL, 1 },3138 { OPTION_SET_INT, 0, "all", &rev_list_all, NULL,3139 N_("include objects reachable from any reference"),3140 PARSE_OPT_NOARG | PARSE_OPT_NONEG, NULL, 1 },3141 { OPTION_SET_INT, 0, "reflog", &rev_list_reflog, NULL,3142 N_("include objects referred by reflog entries"),3143 PARSE_OPT_NOARG | PARSE_OPT_NONEG, NULL, 1 },3144 { OPTION_SET_INT, 0, "indexed-objects", &rev_list_index, NULL,3145 N_("include objects referred to by the index"),3146 PARSE_OPT_NOARG | PARSE_OPT_NONEG, NULL, 1 },3147 OPT_BOOL(0, "stdout", &pack_to_stdout,3148 N_("output pack to stdout")),3149 OPT_BOOL(0, "include-tag", &include_tag,3150 N_("include tag objects that refer to objects to be packed")),3151 OPT_BOOL(0, "keep-unreachable", &keep_unreachable,3152 N_("keep unreachable objects")),3153 OPT_BOOL(0, "pack-loose-unreachable", &pack_loose_unreachable,3154 N_("pack loose unreachable objects")),3155 { OPTION_CALLBACK, 0, "unpack-unreachable", NULL, N_("time"),3156 N_("unpack unreachable objects newer than <time>"),3157 PARSE_OPT_OPTARG, option_parse_unpack_unreachable },3158 OPT_BOOL(0, "thin", &thin,3159 N_("create thin packs")),3160 OPT_BOOL(0, "shallow", &shallow,3161 N_("create packs suitable for shallow fetches")),3162 OPT_BOOL(0, "honor-pack-keep", &ignore_packed_keep_on_disk,3163 N_("ignore packs that have companion .keep file")),3164 OPT_STRING_LIST(0, "keep-pack", &keep_pack_list, N_("name"),3165 N_("ignore this pack")),3166 OPT_INTEGER(0, "compression", &pack_compression_level,3167 N_("pack compression level")),3168 OPT_SET_INT(0, "keep-true-parents", &grafts_replace_parents,3169 N_("do not hide commits by grafts"), 0),3170 OPT_BOOL(0, "use-bitmap-index", &use_bitmap_index,3171 N_("use a bitmap index if available to speed up counting objects")),3172 OPT_BOOL(0, "write-bitmap-index", &write_bitmap_index,3173 N_("write a bitmap index together with the pack index")),3174 OPT_PARSE_LIST_OBJECTS_FILTER(&filter_options),3175 { OPTION_CALLBACK, 0, "missing", NULL, N_("action"),3176 N_("handling for missing objects"), PARSE_OPT_NONEG,3177 option_parse_missing_action },3178 OPT_BOOL(0, "exclude-promisor-objects", &exclude_promisor_objects,3179 N_("do not pack objects in promisor packfiles")),3180 OPT_END(),3181 };31823183 if (DFS_NUM_STATES > (1 << OE_DFS_STATE_BITS))3184 BUG("too many dfs states, increase OE_DFS_STATE_BITS");31853186 check_replace_refs = 0;31873188 reset_pack_idx_option(&pack_idx_opts);3189 git_config(git_pack_config, NULL);31903191 progress = isatty(2);3192 argc = parse_options(argc, argv, prefix, pack_objects_options,3193 pack_usage, 0);31943195 if (argc) {3196 base_name = argv[0];3197 argc--;3198 }3199 if (pack_to_stdout != !base_name || argc)3200 usage_with_options(pack_usage, pack_objects_options);32013202 if (depth >= (1 << OE_DEPTH_BITS)) {3203 warning(_("delta chain depth %d is too deep, forcing %d"),3204 depth, (1 << OE_DEPTH_BITS) - 1);3205 depth = (1 << OE_DEPTH_BITS) - 1;3206 }3207 if (cache_max_small_delta_size >= (1U << OE_Z_DELTA_BITS)) {3208 warning(_("pack.deltaCacheLimit is too high, forcing %d"),3209 (1U << OE_Z_DELTA_BITS) - 1);3210 cache_max_small_delta_size = (1U << OE_Z_DELTA_BITS) - 1;3211 }32123213 argv_array_push(&rp, "pack-objects");3214 if (thin) {3215 use_internal_rev_list = 1;3216 argv_array_push(&rp, shallow3217 ? "--objects-edge-aggressive"3218 : "--objects-edge");3219 } else3220 argv_array_push(&rp, "--objects");32213222 if (rev_list_all) {3223 use_internal_rev_list = 1;3224 argv_array_push(&rp, "--all");3225 }3226 if (rev_list_reflog) {3227 use_internal_rev_list = 1;3228 argv_array_push(&rp, "--reflog");3229 }3230 if (rev_list_index) {3231 use_internal_rev_list = 1;3232 argv_array_push(&rp, "--indexed-objects");3233 }3234 if (rev_list_unpacked) {3235 use_internal_rev_list = 1;3236 argv_array_push(&rp, "--unpacked");3237 }32383239 if (exclude_promisor_objects) {3240 use_internal_rev_list = 1;3241 fetch_if_missing = 0;3242 argv_array_push(&rp, "--exclude-promisor-objects");3243 }3244 if (unpack_unreachable || keep_unreachable || pack_loose_unreachable)3245 use_internal_rev_list = 1;32463247 if (!reuse_object)3248 reuse_delta = 0;3249 if (pack_compression_level == -1)3250 pack_compression_level = Z_DEFAULT_COMPRESSION;3251 else if (pack_compression_level < 0 || pack_compression_level > Z_BEST_COMPRESSION)3252 die("bad pack compression level %d", pack_compression_level);32533254 if (!delta_search_threads) /* --threads=0 means autodetect */3255 delta_search_threads = online_cpus();32563257#ifdef NO_PTHREADS3258 if (delta_search_threads != 1)3259 warning("no threads support, ignoring --threads");3260#endif3261 if (!pack_to_stdout && !pack_size_limit)3262 pack_size_limit = pack_size_limit_cfg;3263 if (pack_to_stdout && pack_size_limit)3264 die("--max-pack-size cannot be used to build a pack for transfer.");3265 if (pack_size_limit && pack_size_limit < 1024*1024) {3266 warning("minimum pack size limit is 1 MiB");3267 pack_size_limit = 1024*1024;3268 }32693270 if (!pack_to_stdout && thin)3271 die("--thin cannot be used to build an indexable pack.");32723273 if (keep_unreachable && unpack_unreachable)3274 die("--keep-unreachable and --unpack-unreachable are incompatible.");3275 if (!rev_list_all || !rev_list_reflog || !rev_list_index)3276 unpack_unreachable_expiration = 0;32773278 if (filter_options.choice) {3279 if (!pack_to_stdout)3280 die("cannot use --filter without --stdout.");3281 use_bitmap_index = 0;3282 }32833284 /*3285 * "soft" reasons not to use bitmaps - for on-disk repack by default we want3286 *3287 * - to produce good pack (with bitmap index not-yet-packed objects are3288 * packed in suboptimal order).3289 *3290 * - to use more robust pack-generation codepath (avoiding possible3291 * bugs in bitmap code and possible bitmap index corruption).3292 */3293 if (!pack_to_stdout)3294 use_bitmap_index_default = 0;32953296 if (use_bitmap_index < 0)3297 use_bitmap_index = use_bitmap_index_default;32983299 /* "hard" reasons not to use bitmaps; these just won't work at all */3300 if (!use_internal_rev_list || (!pack_to_stdout && write_bitmap_index) || is_repository_shallow())3301 use_bitmap_index = 0;33023303 if (pack_to_stdout || !rev_list_all)3304 write_bitmap_index = 0;33053306 if (progress && all_progress_implied)3307 progress = 2;33083309 add_extra_kept_packs(&keep_pack_list);3310 if (ignore_packed_keep_on_disk) {3311 struct packed_git *p;3312 for (p = get_packed_git(the_repository); p; p = p->next)3313 if (p->pack_local && p->pack_keep)3314 break;3315 if (!p) /* no keep-able packs found */3316 ignore_packed_keep_on_disk = 0;3317 }3318 if (local) {3319 /*3320 * unlike ignore_packed_keep_on_disk above, we do not3321 * want to unset "local" based on looking at packs, as3322 * it also covers non-local objects3323 */3324 struct packed_git *p;3325 for (p = get_packed_git(the_repository); p; p = p->next) {3326 if (!p->pack_local) {3327 have_non_local_packs = 1;3328 break;3329 }3330 }3331 }33323333 prepare_packing_data(&to_pack);33343335 if (progress)3336 progress_state = start_progress(_("Enumerating objects"), 0);3337 if (!use_internal_rev_list)3338 read_object_list_from_stdin();3339 else {3340 get_object_list(rp.argc, rp.argv);3341 argv_array_clear(&rp);3342 }3343 cleanup_preferred_base();3344 if (include_tag && nr_result)3345 for_each_ref(add_ref_tag, NULL);3346 stop_progress(&progress_state);33473348 if (non_empty && !nr_result)3349 return 0;3350 if (nr_result)3351 prepare_pack(window, depth);3352 write_pack_file();3353 if (progress)3354 fprintf(stderr, "Total %"PRIu32" (delta %"PRIu32"),"3355 " reused %"PRIu32" (delta %"PRIu32")\n",3356 written, written_delta, reused, reused_delta);3357 return 0;3358}