1#include"builtin.h" 2#include"cache.h" 3#include"repository.h" 4#include"config.h" 5#include"attr.h" 6#include"object.h" 7#include"blob.h" 8#include"commit.h" 9#include"tag.h" 10#include"tree.h" 11#include"delta.h" 12#include"pack.h" 13#include"pack-revindex.h" 14#include"csum-file.h" 15#include"tree-walk.h" 16#include"diff.h" 17#include"revision.h" 18#include"list-objects.h" 19#include"list-objects-filter.h" 20#include"list-objects-filter-options.h" 21#include"pack-objects.h" 22#include"progress.h" 23#include"refs.h" 24#include"streaming.h" 25#include"thread-utils.h" 26#include"pack-bitmap.h" 27#include"delta-islands.h" 28#include"reachable.h" 29#include"sha1-array.h" 30#include"argv-array.h" 31#include"list.h" 32#include"packfile.h" 33#include"object-store.h" 34#include"dir.h" 35#include"midx.h" 36#include"trace2.h" 37 38#define IN_PACK(obj) oe_in_pack(&to_pack, obj) 39#define SIZE(obj) oe_size(&to_pack, obj) 40#define SET_SIZE(obj,size) oe_set_size(&to_pack, obj, size) 41#define DELTA_SIZE(obj) oe_delta_size(&to_pack, obj) 42#define DELTA(obj) oe_delta(&to_pack, obj) 43#define DELTA_CHILD(obj) oe_delta_child(&to_pack, obj) 44#define DELTA_SIBLING(obj) oe_delta_sibling(&to_pack, obj) 45#define SET_DELTA(obj, val) oe_set_delta(&to_pack, obj, val) 46#define SET_DELTA_EXT(obj, oid) oe_set_delta_ext(&to_pack, obj, oid) 47#define SET_DELTA_SIZE(obj, val) oe_set_delta_size(&to_pack, obj, val) 48#define SET_DELTA_CHILD(obj, val) oe_set_delta_child(&to_pack, obj, val) 49#define SET_DELTA_SIBLING(obj, val) oe_set_delta_sibling(&to_pack, obj, val) 50 51static const char*pack_usage[] = { 52N_("git pack-objects --stdout [<options>...] [< <ref-list> | < <object-list>]"), 53N_("git pack-objects [<options>...] <base-name> [< <ref-list> | < <object-list>]"), 54 NULL 55}; 56 57/* 58 * Objects we are going to pack are collected in the `to_pack` structure. 59 * It contains an array (dynamically expanded) of the object data, and a map 60 * that can resolve SHA1s to their position in the array. 61 */ 62static struct packing_data to_pack; 63 64static struct pack_idx_entry **written_list; 65static uint32_t nr_result, nr_written, nr_seen; 66static struct bitmap_index *bitmap_git; 67static uint32_t write_layer; 68 69static int non_empty; 70static int reuse_delta =1, reuse_object =1; 71static int keep_unreachable, unpack_unreachable, include_tag; 72static timestamp_t unpack_unreachable_expiration; 73static int pack_loose_unreachable; 74static int local; 75static int have_non_local_packs; 76static int incremental; 77static int ignore_packed_keep_on_disk; 78static int ignore_packed_keep_in_core; 79static int allow_ofs_delta; 80static struct pack_idx_option pack_idx_opts; 81static const char*base_name; 82static int progress =1; 83static int window =10; 84static unsigned long pack_size_limit; 85static int depth =50; 86static int delta_search_threads; 87static int pack_to_stdout; 88static int sparse; 89static int thin; 90static int num_preferred_base; 91static struct progress *progress_state; 92 93static struct packed_git *reuse_packfile; 94static uint32_t reuse_packfile_objects; 95static off_t reuse_packfile_offset; 96 97static int use_bitmap_index_default =1; 98static int use_bitmap_index = -1; 99static enum{ 100 WRITE_BITMAP_FALSE =0, 101 WRITE_BITMAP_QUIET, 102 WRITE_BITMAP_TRUE, 103} write_bitmap_index; 104static uint16_t write_bitmap_options = BITMAP_OPT_HASH_CACHE; 105 106static int exclude_promisor_objects; 107 108static int use_delta_islands; 109 110static unsigned long delta_cache_size =0; 111static unsigned long max_delta_cache_size = DEFAULT_DELTA_CACHE_SIZE; 112static unsigned long cache_max_small_delta_size =1000; 113 114static unsigned long window_memory_limit =0; 115 116static struct list_objects_filter_options filter_options; 117 118enum missing_action { 119 MA_ERROR =0,/* fail if any missing objects are encountered */ 120 MA_ALLOW_ANY,/* silently allow ALL missing objects */ 121 MA_ALLOW_PROMISOR,/* silently allow all missing PROMISOR objects */ 122}; 123static enum missing_action arg_missing_action; 124static show_object_fn fn_show_object; 125 126/* 127 * stats 128 */ 129static uint32_t written, written_delta; 130static uint32_t reused, reused_delta; 131 132/* 133 * Indexed commits 134 */ 135static struct commit **indexed_commits; 136static unsigned int indexed_commits_nr; 137static unsigned int indexed_commits_alloc; 138 139static voidindex_commit_for_bitmap(struct commit *commit) 140{ 141if(indexed_commits_nr >= indexed_commits_alloc) { 142 indexed_commits_alloc = (indexed_commits_alloc +32) *2; 143REALLOC_ARRAY(indexed_commits, indexed_commits_alloc); 144} 145 146 indexed_commits[indexed_commits_nr++] = commit; 147} 148 149static void*get_delta(struct object_entry *entry) 150{ 151unsigned long size, base_size, delta_size; 152void*buf, *base_buf, *delta_buf; 153enum object_type type; 154 155 buf =read_object_file(&entry->idx.oid, &type, &size); 156if(!buf) 157die(_("unable to read%s"),oid_to_hex(&entry->idx.oid)); 158 base_buf =read_object_file(&DELTA(entry)->idx.oid, &type, 159&base_size); 160if(!base_buf) 161die("unable to read%s", 162oid_to_hex(&DELTA(entry)->idx.oid)); 163 delta_buf =diff_delta(base_buf, base_size, 164 buf, size, &delta_size,0); 165/* 166 * We succesfully computed this delta once but dropped it for 167 * memory reasons. Something is very wrong if this time we 168 * recompute and create a different delta. 169 */ 170if(!delta_buf || delta_size !=DELTA_SIZE(entry)) 171BUG("delta size changed"); 172free(buf); 173free(base_buf); 174return delta_buf; 175} 176 177static unsigned longdo_compress(void**pptr,unsigned long size) 178{ 179 git_zstream stream; 180void*in, *out; 181unsigned long maxsize; 182 183git_deflate_init(&stream, pack_compression_level); 184 maxsize =git_deflate_bound(&stream, size); 185 186 in = *pptr; 187 out =xmalloc(maxsize); 188*pptr = out; 189 190 stream.next_in = in; 191 stream.avail_in = size; 192 stream.next_out = out; 193 stream.avail_out = maxsize; 194while(git_deflate(&stream, Z_FINISH) == Z_OK) 195;/* nothing */ 196git_deflate_end(&stream); 197 198free(in); 199return stream.total_out; 200} 201 202static unsigned longwrite_large_blob_data(struct git_istream *st,struct hashfile *f, 203const struct object_id *oid) 204{ 205 git_zstream stream; 206unsigned char ibuf[1024*16]; 207unsigned char obuf[1024*16]; 208unsigned long olen =0; 209 210git_deflate_init(&stream, pack_compression_level); 211 212for(;;) { 213 ssize_t readlen; 214int zret = Z_OK; 215 readlen =read_istream(st, ibuf,sizeof(ibuf)); 216if(readlen == -1) 217die(_("unable to read%s"),oid_to_hex(oid)); 218 219 stream.next_in = ibuf; 220 stream.avail_in = readlen; 221while((stream.avail_in || readlen ==0) && 222(zret == Z_OK || zret == Z_BUF_ERROR)) { 223 stream.next_out = obuf; 224 stream.avail_out =sizeof(obuf); 225 zret =git_deflate(&stream, readlen ?0: Z_FINISH); 226hashwrite(f, obuf, stream.next_out - obuf); 227 olen += stream.next_out - obuf; 228} 229if(stream.avail_in) 230die(_("deflate error (%d)"), zret); 231if(readlen ==0) { 232if(zret != Z_STREAM_END) 233die(_("deflate error (%d)"), zret); 234break; 235} 236} 237git_deflate_end(&stream); 238return olen; 239} 240 241/* 242 * we are going to reuse the existing object data as is. make 243 * sure it is not corrupt. 244 */ 245static intcheck_pack_inflate(struct packed_git *p, 246struct pack_window **w_curs, 247 off_t offset, 248 off_t len, 249unsigned long expect) 250{ 251 git_zstream stream; 252unsigned char fakebuf[4096], *in; 253int st; 254 255memset(&stream,0,sizeof(stream)); 256git_inflate_init(&stream); 257do{ 258 in =use_pack(p, w_curs, offset, &stream.avail_in); 259 stream.next_in = in; 260 stream.next_out = fakebuf; 261 stream.avail_out =sizeof(fakebuf); 262 st =git_inflate(&stream, Z_FINISH); 263 offset += stream.next_in - in; 264}while(st == Z_OK || st == Z_BUF_ERROR); 265git_inflate_end(&stream); 266return(st == Z_STREAM_END && 267 stream.total_out == expect && 268 stream.total_in == len) ?0: -1; 269} 270 271static voidcopy_pack_data(struct hashfile *f, 272struct packed_git *p, 273struct pack_window **w_curs, 274 off_t offset, 275 off_t len) 276{ 277unsigned char*in; 278unsigned long avail; 279 280while(len) { 281 in =use_pack(p, w_curs, offset, &avail); 282if(avail > len) 283 avail = (unsigned long)len; 284hashwrite(f, in, avail); 285 offset += avail; 286 len -= avail; 287} 288} 289 290/* Return 0 if we will bust the pack-size limit */ 291static unsigned longwrite_no_reuse_object(struct hashfile *f,struct object_entry *entry, 292unsigned long limit,int usable_delta) 293{ 294unsigned long size, datalen; 295unsigned char header[MAX_PACK_OBJECT_HEADER], 296 dheader[MAX_PACK_OBJECT_HEADER]; 297unsigned hdrlen; 298enum object_type type; 299void*buf; 300struct git_istream *st = NULL; 301const unsigned hashsz = the_hash_algo->rawsz; 302 303if(!usable_delta) { 304if(oe_type(entry) == OBJ_BLOB && 305oe_size_greater_than(&to_pack, entry, big_file_threshold) && 306(st =open_istream(&entry->idx.oid, &type, &size, NULL)) != NULL) 307 buf = NULL; 308else{ 309 buf =read_object_file(&entry->idx.oid, &type, &size); 310if(!buf) 311die(_("unable to read%s"), 312oid_to_hex(&entry->idx.oid)); 313} 314/* 315 * make sure no cached delta data remains from a 316 * previous attempt before a pack split occurred. 317 */ 318FREE_AND_NULL(entry->delta_data); 319 entry->z_delta_size =0; 320}else if(entry->delta_data) { 321 size =DELTA_SIZE(entry); 322 buf = entry->delta_data; 323 entry->delta_data = NULL; 324 type = (allow_ofs_delta &&DELTA(entry)->idx.offset) ? 325 OBJ_OFS_DELTA : OBJ_REF_DELTA; 326}else{ 327 buf =get_delta(entry); 328 size =DELTA_SIZE(entry); 329 type = (allow_ofs_delta &&DELTA(entry)->idx.offset) ? 330 OBJ_OFS_DELTA : OBJ_REF_DELTA; 331} 332 333if(st)/* large blob case, just assume we don't compress well */ 334 datalen = size; 335else if(entry->z_delta_size) 336 datalen = entry->z_delta_size; 337else 338 datalen =do_compress(&buf, size); 339 340/* 341 * The object header is a byte of 'type' followed by zero or 342 * more bytes of length. 343 */ 344 hdrlen =encode_in_pack_object_header(header,sizeof(header), 345 type, size); 346 347if(type == OBJ_OFS_DELTA) { 348/* 349 * Deltas with relative base contain an additional 350 * encoding of the relative offset for the delta 351 * base from this object's position in the pack. 352 */ 353 off_t ofs = entry->idx.offset -DELTA(entry)->idx.offset; 354unsigned pos =sizeof(dheader) -1; 355 dheader[pos] = ofs &127; 356while(ofs >>=7) 357 dheader[--pos] =128| (--ofs &127); 358if(limit && hdrlen +sizeof(dheader) - pos + datalen + hashsz >= limit) { 359if(st) 360close_istream(st); 361free(buf); 362return0; 363} 364hashwrite(f, header, hdrlen); 365hashwrite(f, dheader + pos,sizeof(dheader) - pos); 366 hdrlen +=sizeof(dheader) - pos; 367}else if(type == OBJ_REF_DELTA) { 368/* 369 * Deltas with a base reference contain 370 * additional bytes for the base object ID. 371 */ 372if(limit && hdrlen + hashsz + datalen + hashsz >= limit) { 373if(st) 374close_istream(st); 375free(buf); 376return0; 377} 378hashwrite(f, header, hdrlen); 379hashwrite(f,DELTA(entry)->idx.oid.hash, hashsz); 380 hdrlen += hashsz; 381}else{ 382if(limit && hdrlen + datalen + hashsz >= limit) { 383if(st) 384close_istream(st); 385free(buf); 386return0; 387} 388hashwrite(f, header, hdrlen); 389} 390if(st) { 391 datalen =write_large_blob_data(st, f, &entry->idx.oid); 392close_istream(st); 393}else{ 394hashwrite(f, buf, datalen); 395free(buf); 396} 397 398return hdrlen + datalen; 399} 400 401/* Return 0 if we will bust the pack-size limit */ 402static off_t write_reuse_object(struct hashfile *f,struct object_entry *entry, 403unsigned long limit,int usable_delta) 404{ 405struct packed_git *p =IN_PACK(entry); 406struct pack_window *w_curs = NULL; 407struct revindex_entry *revidx; 408 off_t offset; 409enum object_type type =oe_type(entry); 410 off_t datalen; 411unsigned char header[MAX_PACK_OBJECT_HEADER], 412 dheader[MAX_PACK_OBJECT_HEADER]; 413unsigned hdrlen; 414const unsigned hashsz = the_hash_algo->rawsz; 415unsigned long entry_size =SIZE(entry); 416 417if(DELTA(entry)) 418 type = (allow_ofs_delta &&DELTA(entry)->idx.offset) ? 419 OBJ_OFS_DELTA : OBJ_REF_DELTA; 420 hdrlen =encode_in_pack_object_header(header,sizeof(header), 421 type, entry_size); 422 423 offset = entry->in_pack_offset; 424 revidx =find_pack_revindex(p, offset); 425 datalen = revidx[1].offset - offset; 426if(!pack_to_stdout && p->index_version >1&& 427check_pack_crc(p, &w_curs, offset, datalen, revidx->nr)) { 428error(_("bad packed object CRC for%s"), 429oid_to_hex(&entry->idx.oid)); 430unuse_pack(&w_curs); 431returnwrite_no_reuse_object(f, entry, limit, usable_delta); 432} 433 434 offset += entry->in_pack_header_size; 435 datalen -= entry->in_pack_header_size; 436 437if(!pack_to_stdout && p->index_version ==1&& 438check_pack_inflate(p, &w_curs, offset, datalen, entry_size)) { 439error(_("corrupt packed object for%s"), 440oid_to_hex(&entry->idx.oid)); 441unuse_pack(&w_curs); 442returnwrite_no_reuse_object(f, entry, limit, usable_delta); 443} 444 445if(type == OBJ_OFS_DELTA) { 446 off_t ofs = entry->idx.offset -DELTA(entry)->idx.offset; 447unsigned pos =sizeof(dheader) -1; 448 dheader[pos] = ofs &127; 449while(ofs >>=7) 450 dheader[--pos] =128| (--ofs &127); 451if(limit && hdrlen +sizeof(dheader) - pos + datalen + hashsz >= limit) { 452unuse_pack(&w_curs); 453return0; 454} 455hashwrite(f, header, hdrlen); 456hashwrite(f, dheader + pos,sizeof(dheader) - pos); 457 hdrlen +=sizeof(dheader) - pos; 458 reused_delta++; 459}else if(type == OBJ_REF_DELTA) { 460if(limit && hdrlen + hashsz + datalen + hashsz >= limit) { 461unuse_pack(&w_curs); 462return0; 463} 464hashwrite(f, header, hdrlen); 465hashwrite(f,DELTA(entry)->idx.oid.hash, hashsz); 466 hdrlen += hashsz; 467 reused_delta++; 468}else{ 469if(limit && hdrlen + datalen + hashsz >= limit) { 470unuse_pack(&w_curs); 471return0; 472} 473hashwrite(f, header, hdrlen); 474} 475copy_pack_data(f, p, &w_curs, offset, datalen); 476unuse_pack(&w_curs); 477 reused++; 478return hdrlen + datalen; 479} 480 481/* Return 0 if we will bust the pack-size limit */ 482static off_t write_object(struct hashfile *f, 483struct object_entry *entry, 484 off_t write_offset) 485{ 486unsigned long limit; 487 off_t len; 488int usable_delta, to_reuse; 489 490if(!pack_to_stdout) 491crc32_begin(f); 492 493/* apply size limit if limited packsize and not first object */ 494if(!pack_size_limit || !nr_written) 495 limit =0; 496else if(pack_size_limit <= write_offset) 497/* 498 * the earlier object did not fit the limit; avoid 499 * mistaking this with unlimited (i.e. limit = 0). 500 */ 501 limit =1; 502else 503 limit = pack_size_limit - write_offset; 504 505if(!DELTA(entry)) 506 usable_delta =0;/* no delta */ 507else if(!pack_size_limit) 508 usable_delta =1;/* unlimited packfile */ 509else if(DELTA(entry)->idx.offset == (off_t)-1) 510 usable_delta =0;/* base was written to another pack */ 511else if(DELTA(entry)->idx.offset) 512 usable_delta =1;/* base already exists in this pack */ 513else 514 usable_delta =0;/* base could end up in another pack */ 515 516if(!reuse_object) 517 to_reuse =0;/* explicit */ 518else if(!IN_PACK(entry)) 519 to_reuse =0;/* can't reuse what we don't have */ 520else if(oe_type(entry) == OBJ_REF_DELTA || 521oe_type(entry) == OBJ_OFS_DELTA) 522/* check_object() decided it for us ... */ 523 to_reuse = usable_delta; 524/* ... but pack split may override that */ 525else if(oe_type(entry) != entry->in_pack_type) 526 to_reuse =0;/* pack has delta which is unusable */ 527else if(DELTA(entry)) 528 to_reuse =0;/* we want to pack afresh */ 529else 530 to_reuse =1;/* we have it in-pack undeltified, 531 * and we do not need to deltify it. 532 */ 533 534if(!to_reuse) 535 len =write_no_reuse_object(f, entry, limit, usable_delta); 536else 537 len =write_reuse_object(f, entry, limit, usable_delta); 538if(!len) 539return0; 540 541if(usable_delta) 542 written_delta++; 543 written++; 544if(!pack_to_stdout) 545 entry->idx.crc32 =crc32_end(f); 546return len; 547} 548 549enum write_one_status { 550 WRITE_ONE_SKIP = -1,/* already written */ 551 WRITE_ONE_BREAK =0,/* writing this will bust the limit; not written */ 552 WRITE_ONE_WRITTEN =1,/* normal */ 553 WRITE_ONE_RECURSIVE =2/* already scheduled to be written */ 554}; 555 556static enum write_one_status write_one(struct hashfile *f, 557struct object_entry *e, 558 off_t *offset) 559{ 560 off_t size; 561int recursing; 562 563/* 564 * we set offset to 1 (which is an impossible value) to mark 565 * the fact that this object is involved in "write its base 566 * first before writing a deltified object" recursion. 567 */ 568 recursing = (e->idx.offset ==1); 569if(recursing) { 570warning(_("recursive delta detected for object%s"), 571oid_to_hex(&e->idx.oid)); 572return WRITE_ONE_RECURSIVE; 573}else if(e->idx.offset || e->preferred_base) { 574/* offset is non zero if object is written already. */ 575return WRITE_ONE_SKIP; 576} 577 578/* if we are deltified, write out base object first. */ 579if(DELTA(e)) { 580 e->idx.offset =1;/* now recurse */ 581switch(write_one(f,DELTA(e), offset)) { 582case WRITE_ONE_RECURSIVE: 583/* we cannot depend on this one */ 584SET_DELTA(e, NULL); 585break; 586default: 587break; 588case WRITE_ONE_BREAK: 589 e->idx.offset = recursing; 590return WRITE_ONE_BREAK; 591} 592} 593 594 e->idx.offset = *offset; 595 size =write_object(f, e, *offset); 596if(!size) { 597 e->idx.offset = recursing; 598return WRITE_ONE_BREAK; 599} 600 written_list[nr_written++] = &e->idx; 601 602/* make sure off_t is sufficiently large not to wrap */ 603if(signed_add_overflows(*offset, size)) 604die(_("pack too large for current definition of off_t")); 605*offset += size; 606return WRITE_ONE_WRITTEN; 607} 608 609static intmark_tagged(const char*path,const struct object_id *oid,int flag, 610void*cb_data) 611{ 612struct object_id peeled; 613struct object_entry *entry =packlist_find(&to_pack, oid); 614 615if(entry) 616 entry->tagged =1; 617if(!peel_ref(path, &peeled)) { 618 entry =packlist_find(&to_pack, &peeled); 619if(entry) 620 entry->tagged =1; 621} 622return0; 623} 624 625staticinlinevoidadd_to_write_order(struct object_entry **wo, 626unsigned int*endp, 627struct object_entry *e) 628{ 629if(e->filled ||oe_layer(&to_pack, e) != write_layer) 630return; 631 wo[(*endp)++] = e; 632 e->filled =1; 633} 634 635static voidadd_descendants_to_write_order(struct object_entry **wo, 636unsigned int*endp, 637struct object_entry *e) 638{ 639int add_to_order =1; 640while(e) { 641if(add_to_order) { 642struct object_entry *s; 643/* add this node... */ 644add_to_write_order(wo, endp, e); 645/* all its siblings... */ 646for(s =DELTA_SIBLING(e); s; s =DELTA_SIBLING(s)) { 647add_to_write_order(wo, endp, s); 648} 649} 650/* drop down a level to add left subtree nodes if possible */ 651if(DELTA_CHILD(e)) { 652 add_to_order =1; 653 e =DELTA_CHILD(e); 654}else{ 655 add_to_order =0; 656/* our sibling might have some children, it is next */ 657if(DELTA_SIBLING(e)) { 658 e =DELTA_SIBLING(e); 659continue; 660} 661/* go back to our parent node */ 662 e =DELTA(e); 663while(e && !DELTA_SIBLING(e)) { 664/* we're on the right side of a subtree, keep 665 * going up until we can go right again */ 666 e =DELTA(e); 667} 668if(!e) { 669/* done- we hit our original root node */ 670return; 671} 672/* pass it off to sibling at this level */ 673 e =DELTA_SIBLING(e); 674} 675}; 676} 677 678static voidadd_family_to_write_order(struct object_entry **wo, 679unsigned int*endp, 680struct object_entry *e) 681{ 682struct object_entry *root; 683 684for(root = e;DELTA(root); root =DELTA(root)) 685;/* nothing */ 686add_descendants_to_write_order(wo, endp, root); 687} 688 689static voidcompute_layer_order(struct object_entry **wo,unsigned int*wo_end) 690{ 691unsigned int i, last_untagged; 692struct object_entry *objects = to_pack.objects; 693 694for(i =0; i < to_pack.nr_objects; i++) { 695if(objects[i].tagged) 696break; 697add_to_write_order(wo, wo_end, &objects[i]); 698} 699 last_untagged = i; 700 701/* 702 * Then fill all the tagged tips. 703 */ 704for(; i < to_pack.nr_objects; i++) { 705if(objects[i].tagged) 706add_to_write_order(wo, wo_end, &objects[i]); 707} 708 709/* 710 * And then all remaining commits and tags. 711 */ 712for(i = last_untagged; i < to_pack.nr_objects; i++) { 713if(oe_type(&objects[i]) != OBJ_COMMIT && 714oe_type(&objects[i]) != OBJ_TAG) 715continue; 716add_to_write_order(wo, wo_end, &objects[i]); 717} 718 719/* 720 * And then all the trees. 721 */ 722for(i = last_untagged; i < to_pack.nr_objects; i++) { 723if(oe_type(&objects[i]) != OBJ_TREE) 724continue; 725add_to_write_order(wo, wo_end, &objects[i]); 726} 727 728/* 729 * Finally all the rest in really tight order 730 */ 731for(i = last_untagged; i < to_pack.nr_objects; i++) { 732if(!objects[i].filled &&oe_layer(&to_pack, &objects[i]) == write_layer) 733add_family_to_write_order(wo, wo_end, &objects[i]); 734} 735} 736 737static struct object_entry **compute_write_order(void) 738{ 739uint32_t max_layers =1; 740unsigned int i, wo_end; 741 742struct object_entry **wo; 743struct object_entry *objects = to_pack.objects; 744 745for(i =0; i < to_pack.nr_objects; i++) { 746 objects[i].tagged =0; 747 objects[i].filled =0; 748SET_DELTA_CHILD(&objects[i], NULL); 749SET_DELTA_SIBLING(&objects[i], NULL); 750} 751 752/* 753 * Fully connect delta_child/delta_sibling network. 754 * Make sure delta_sibling is sorted in the original 755 * recency order. 756 */ 757for(i = to_pack.nr_objects; i >0;) { 758struct object_entry *e = &objects[--i]; 759if(!DELTA(e)) 760continue; 761/* Mark me as the first child */ 762 e->delta_sibling_idx =DELTA(e)->delta_child_idx; 763SET_DELTA_CHILD(DELTA(e), e); 764} 765 766/* 767 * Mark objects that are at the tip of tags. 768 */ 769for_each_tag_ref(mark_tagged, NULL); 770 771if(use_delta_islands) 772 max_layers =compute_pack_layers(&to_pack); 773 774ALLOC_ARRAY(wo, to_pack.nr_objects); 775 wo_end =0; 776 777for(; write_layer < max_layers; ++write_layer) 778compute_layer_order(wo, &wo_end); 779 780if(wo_end != to_pack.nr_objects) 781die(_("ordered%uobjects, expected %"PRIu32), 782 wo_end, to_pack.nr_objects); 783 784return wo; 785} 786 787static off_t write_reused_pack(struct hashfile *f) 788{ 789unsigned char buffer[8192]; 790 off_t to_write, total; 791int fd; 792 793if(!is_pack_valid(reuse_packfile)) 794die(_("packfile is invalid:%s"), reuse_packfile->pack_name); 795 796 fd =git_open(reuse_packfile->pack_name); 797if(fd <0) 798die_errno(_("unable to open packfile for reuse:%s"), 799 reuse_packfile->pack_name); 800 801if(lseek(fd,sizeof(struct pack_header), SEEK_SET) == -1) 802die_errno(_("unable to seek in reused packfile")); 803 804if(reuse_packfile_offset <0) 805 reuse_packfile_offset = reuse_packfile->pack_size - the_hash_algo->rawsz; 806 807 total = to_write = reuse_packfile_offset -sizeof(struct pack_header); 808 809while(to_write) { 810int read_pack =xread(fd, buffer,sizeof(buffer)); 811 812if(read_pack <=0) 813die_errno(_("unable to read from reused packfile")); 814 815if(read_pack > to_write) 816 read_pack = to_write; 817 818hashwrite(f, buffer, read_pack); 819 to_write -= read_pack; 820 821/* 822 * We don't know the actual number of objects written, 823 * only how many bytes written, how many bytes total, and 824 * how many objects total. So we can fake it by pretending all 825 * objects we are writing are the same size. This gives us a 826 * smooth progress meter, and at the end it matches the true 827 * answer. 828 */ 829 written = reuse_packfile_objects * 830(((double)(total - to_write)) / total); 831display_progress(progress_state, written); 832} 833 834close(fd); 835 written = reuse_packfile_objects; 836display_progress(progress_state, written); 837return reuse_packfile_offset -sizeof(struct pack_header); 838} 839 840static const char no_split_warning[] =N_( 841"disabling bitmap writing, packs are split due to pack.packSizeLimit" 842); 843 844static voidwrite_pack_file(void) 845{ 846uint32_t i =0, j; 847struct hashfile *f; 848 off_t offset; 849uint32_t nr_remaining = nr_result; 850time_t last_mtime =0; 851struct object_entry **write_order; 852 853if(progress > pack_to_stdout) 854 progress_state =start_progress(_("Writing objects"), nr_result); 855ALLOC_ARRAY(written_list, to_pack.nr_objects); 856 write_order =compute_write_order(); 857 858do{ 859struct object_id oid; 860char*pack_tmp_name = NULL; 861 862if(pack_to_stdout) 863 f =hashfd_throughput(1,"<stdout>", progress_state); 864else 865 f =create_tmp_packfile(&pack_tmp_name); 866 867 offset =write_pack_header(f, nr_remaining); 868 869if(reuse_packfile) { 870 off_t packfile_size; 871assert(pack_to_stdout); 872 873 packfile_size =write_reused_pack(f); 874 offset += packfile_size; 875} 876 877 nr_written =0; 878for(; i < to_pack.nr_objects; i++) { 879struct object_entry *e = write_order[i]; 880if(write_one(f, e, &offset) == WRITE_ONE_BREAK) 881break; 882display_progress(progress_state, written); 883} 884 885/* 886 * Did we write the wrong # entries in the header? 887 * If so, rewrite it like in fast-import 888 */ 889if(pack_to_stdout) { 890finalize_hashfile(f, oid.hash, CSUM_HASH_IN_STREAM | CSUM_CLOSE); 891}else if(nr_written == nr_remaining) { 892finalize_hashfile(f, oid.hash, CSUM_HASH_IN_STREAM | CSUM_FSYNC | CSUM_CLOSE); 893}else{ 894int fd =finalize_hashfile(f, oid.hash,0); 895fixup_pack_header_footer(fd, oid.hash, pack_tmp_name, 896 nr_written, oid.hash, offset); 897close(fd); 898if(write_bitmap_index) { 899if(write_bitmap_index != WRITE_BITMAP_QUIET) 900warning(_(no_split_warning)); 901 write_bitmap_index =0; 902} 903} 904 905if(!pack_to_stdout) { 906struct stat st; 907struct strbuf tmpname = STRBUF_INIT; 908 909/* 910 * Packs are runtime accessed in their mtime 911 * order since newer packs are more likely to contain 912 * younger objects. So if we are creating multiple 913 * packs then we should modify the mtime of later ones 914 * to preserve this property. 915 */ 916if(stat(pack_tmp_name, &st) <0) { 917warning_errno(_("failed to stat%s"), pack_tmp_name); 918}else if(!last_mtime) { 919 last_mtime = st.st_mtime; 920}else{ 921struct utimbuf utb; 922 utb.actime = st.st_atime; 923 utb.modtime = --last_mtime; 924if(utime(pack_tmp_name, &utb) <0) 925warning_errno(_("failed utime() on%s"), pack_tmp_name); 926} 927 928strbuf_addf(&tmpname,"%s-", base_name); 929 930if(write_bitmap_index) { 931bitmap_writer_set_checksum(oid.hash); 932bitmap_writer_build_type_index( 933&to_pack, written_list, nr_written); 934} 935 936finish_tmp_packfile(&tmpname, pack_tmp_name, 937 written_list, nr_written, 938&pack_idx_opts, oid.hash); 939 940if(write_bitmap_index) { 941strbuf_addf(&tmpname,"%s.bitmap",oid_to_hex(&oid)); 942 943stop_progress(&progress_state); 944 945bitmap_writer_show_progress(progress); 946bitmap_writer_reuse_bitmaps(&to_pack); 947bitmap_writer_select_commits(indexed_commits, indexed_commits_nr, -1); 948bitmap_writer_build(&to_pack); 949bitmap_writer_finish(written_list, nr_written, 950 tmpname.buf, write_bitmap_options); 951 write_bitmap_index =0; 952} 953 954strbuf_release(&tmpname); 955free(pack_tmp_name); 956puts(oid_to_hex(&oid)); 957} 958 959/* mark written objects as written to previous pack */ 960for(j =0; j < nr_written; j++) { 961 written_list[j]->offset = (off_t)-1; 962} 963 nr_remaining -= nr_written; 964}while(nr_remaining && i < to_pack.nr_objects); 965 966free(written_list); 967free(write_order); 968stop_progress(&progress_state); 969if(written != nr_result) 970die(_("wrote %"PRIu32" objects while expecting %"PRIu32), 971 written, nr_result); 972trace2_data_intmax("pack-objects", the_repository, 973"write_pack_file/wrote", nr_result); 974} 975 976static intno_try_delta(const char*path) 977{ 978static struct attr_check *check; 979 980if(!check) 981 check =attr_check_initl("delta", NULL); 982git_check_attr(the_repository->index, path, check); 983if(ATTR_FALSE(check->items[0].value)) 984return1; 985return0; 986} 987 988/* 989 * When adding an object, check whether we have already added it 990 * to our packing list. If so, we can skip. However, if we are 991 * being asked to excludei t, but the previous mention was to include 992 * it, make sure to adjust its flags and tweak our numbers accordingly. 993 * 994 * As an optimization, we pass out the index position where we would have 995 * found the item, since that saves us from having to look it up again a 996 * few lines later when we want to add the new entry. 997 */ 998static inthave_duplicate_entry(const struct object_id *oid, 999int exclude)1000{1001struct object_entry *entry;10021003 entry =packlist_find(&to_pack, oid);1004if(!entry)1005return0;10061007if(exclude) {1008if(!entry->preferred_base)1009 nr_result--;1010 entry->preferred_base =1;1011}10121013return1;1014}10151016static intwant_found_object(int exclude,struct packed_git *p)1017{1018if(exclude)1019return1;1020if(incremental)1021return0;10221023/*1024 * When asked to do --local (do not include an object that appears in a1025 * pack we borrow from elsewhere) or --honor-pack-keep (do not include1026 * an object that appears in a pack marked with .keep), finding a pack1027 * that matches the criteria is sufficient for us to decide to omit it.1028 * However, even if this pack does not satisfy the criteria, we need to1029 * make sure no copy of this object appears in _any_ pack that makes us1030 * to omit the object, so we need to check all the packs.1031 *1032 * We can however first check whether these options can possible matter;1033 * if they do not matter we know we want the object in generated pack.1034 * Otherwise, we signal "-1" at the end to tell the caller that we do1035 * not know either way, and it needs to check more packs.1036 */1037if(!ignore_packed_keep_on_disk &&1038!ignore_packed_keep_in_core &&1039(!local || !have_non_local_packs))1040return1;10411042if(local && !p->pack_local)1043return0;1044if(p->pack_local &&1045((ignore_packed_keep_on_disk && p->pack_keep) ||1046(ignore_packed_keep_in_core && p->pack_keep_in_core)))1047return0;10481049/* we don't know yet; keep looking for more packs */1050return-1;1051}10521053/*1054 * Check whether we want the object in the pack (e.g., we do not want1055 * objects found in non-local stores if the "--local" option was used).1056 *1057 * If the caller already knows an existing pack it wants to take the object1058 * from, that is passed in *found_pack and *found_offset; otherwise this1059 * function finds if there is any pack that has the object and returns the pack1060 * and its offset in these variables.1061 */1062static intwant_object_in_pack(const struct object_id *oid,1063int exclude,1064struct packed_git **found_pack,1065 off_t *found_offset)1066{1067int want;1068struct list_head *pos;1069struct multi_pack_index *m;10701071if(!exclude && local &&has_loose_object_nonlocal(oid))1072return0;10731074/*1075 * If we already know the pack object lives in, start checks from that1076 * pack - in the usual case when neither --local was given nor .keep files1077 * are present we will determine the answer right now.1078 */1079if(*found_pack) {1080 want =want_found_object(exclude, *found_pack);1081if(want != -1)1082return want;1083}10841085for(m =get_multi_pack_index(the_repository); m; m = m->next) {1086struct pack_entry e;1087if(fill_midx_entry(the_repository, oid, &e, m)) {1088struct packed_git *p = e.p;1089 off_t offset;10901091if(p == *found_pack)1092 offset = *found_offset;1093else1094 offset =find_pack_entry_one(oid->hash, p);10951096if(offset) {1097if(!*found_pack) {1098if(!is_pack_valid(p))1099continue;1100*found_offset = offset;1101*found_pack = p;1102}1103 want =want_found_object(exclude, p);1104if(want != -1)1105return want;1106}1107}1108}11091110list_for_each(pos,get_packed_git_mru(the_repository)) {1111struct packed_git *p =list_entry(pos,struct packed_git, mru);1112 off_t offset;11131114if(p == *found_pack)1115 offset = *found_offset;1116else1117 offset =find_pack_entry_one(oid->hash, p);11181119if(offset) {1120if(!*found_pack) {1121if(!is_pack_valid(p))1122continue;1123*found_offset = offset;1124*found_pack = p;1125}1126 want =want_found_object(exclude, p);1127if(!exclude && want >0)1128list_move(&p->mru,1129get_packed_git_mru(the_repository));1130if(want != -1)1131return want;1132}1133}11341135return1;1136}11371138static voidcreate_object_entry(const struct object_id *oid,1139enum object_type type,1140uint32_t hash,1141int exclude,1142int no_try_delta,1143struct packed_git *found_pack,1144 off_t found_offset)1145{1146struct object_entry *entry;11471148 entry =packlist_alloc(&to_pack, oid);1149 entry->hash = hash;1150oe_set_type(entry, type);1151if(exclude)1152 entry->preferred_base =1;1153else1154 nr_result++;1155if(found_pack) {1156oe_set_in_pack(&to_pack, entry, found_pack);1157 entry->in_pack_offset = found_offset;1158}11591160 entry->no_try_delta = no_try_delta;1161}11621163static const char no_closure_warning[] =N_(1164"disabling bitmap writing, as some objects are not being packed"1165);11661167static intadd_object_entry(const struct object_id *oid,enum object_type type,1168const char*name,int exclude)1169{1170struct packed_git *found_pack = NULL;1171 off_t found_offset =0;11721173display_progress(progress_state, ++nr_seen);11741175if(have_duplicate_entry(oid, exclude))1176return0;11771178if(!want_object_in_pack(oid, exclude, &found_pack, &found_offset)) {1179/* The pack is missing an object, so it will not have closure */1180if(write_bitmap_index) {1181if(write_bitmap_index != WRITE_BITMAP_QUIET)1182warning(_(no_closure_warning));1183 write_bitmap_index =0;1184}1185return0;1186}11871188create_object_entry(oid, type,pack_name_hash(name),1189 exclude, name &&no_try_delta(name),1190 found_pack, found_offset);1191return1;1192}11931194static intadd_object_entry_from_bitmap(const struct object_id *oid,1195enum object_type type,1196int flags,uint32_t name_hash,1197struct packed_git *pack, off_t offset)1198{1199display_progress(progress_state, ++nr_seen);12001201if(have_duplicate_entry(oid,0))1202return0;12031204if(!want_object_in_pack(oid,0, &pack, &offset))1205return0;12061207create_object_entry(oid, type, name_hash,0,0, pack, offset);1208return1;1209}12101211struct pbase_tree_cache {1212struct object_id oid;1213int ref;1214int temporary;1215void*tree_data;1216unsigned long tree_size;1217};12181219static struct pbase_tree_cache *(pbase_tree_cache[256]);1220static intpbase_tree_cache_ix(const struct object_id *oid)1221{1222return oid->hash[0] %ARRAY_SIZE(pbase_tree_cache);1223}1224static intpbase_tree_cache_ix_incr(int ix)1225{1226return(ix+1) %ARRAY_SIZE(pbase_tree_cache);1227}12281229static struct pbase_tree {1230struct pbase_tree *next;1231/* This is a phony "cache" entry; we are not1232 * going to evict it or find it through _get()1233 * mechanism -- this is for the toplevel node that1234 * would almost always change with any commit.1235 */1236struct pbase_tree_cache pcache;1237} *pbase_tree;12381239static struct pbase_tree_cache *pbase_tree_get(const struct object_id *oid)1240{1241struct pbase_tree_cache *ent, *nent;1242void*data;1243unsigned long size;1244enum object_type type;1245int neigh;1246int my_ix =pbase_tree_cache_ix(oid);1247int available_ix = -1;12481249/* pbase-tree-cache acts as a limited hashtable.1250 * your object will be found at your index or within a few1251 * slots after that slot if it is cached.1252 */1253for(neigh =0; neigh <8; neigh++) {1254 ent = pbase_tree_cache[my_ix];1255if(ent &&oideq(&ent->oid, oid)) {1256 ent->ref++;1257return ent;1258}1259else if(((available_ix <0) && (!ent || !ent->ref)) ||1260((0<= available_ix) &&1261(!ent && pbase_tree_cache[available_ix])))1262 available_ix = my_ix;1263if(!ent)1264break;1265 my_ix =pbase_tree_cache_ix_incr(my_ix);1266}12671268/* Did not find one. Either we got a bogus request or1269 * we need to read and perhaps cache.1270 */1271 data =read_object_file(oid, &type, &size);1272if(!data)1273return NULL;1274if(type != OBJ_TREE) {1275free(data);1276return NULL;1277}12781279/* We need to either cache or return a throwaway copy */12801281if(available_ix <0)1282 ent = NULL;1283else{1284 ent = pbase_tree_cache[available_ix];1285 my_ix = available_ix;1286}12871288if(!ent) {1289 nent =xmalloc(sizeof(*nent));1290 nent->temporary = (available_ix <0);1291}1292else{1293/* evict and reuse */1294free(ent->tree_data);1295 nent = ent;1296}1297oidcpy(&nent->oid, oid);1298 nent->tree_data = data;1299 nent->tree_size = size;1300 nent->ref =1;1301if(!nent->temporary)1302 pbase_tree_cache[my_ix] = nent;1303return nent;1304}13051306static voidpbase_tree_put(struct pbase_tree_cache *cache)1307{1308if(!cache->temporary) {1309 cache->ref--;1310return;1311}1312free(cache->tree_data);1313free(cache);1314}13151316static intname_cmp_len(const char*name)1317{1318int i;1319for(i =0; name[i] && name[i] !='\n'&& name[i] !='/'; i++)1320;1321return i;1322}13231324static voidadd_pbase_object(struct tree_desc *tree,1325const char*name,1326int cmplen,1327const char*fullname)1328{1329struct name_entry entry;1330int cmp;13311332while(tree_entry(tree,&entry)) {1333if(S_ISGITLINK(entry.mode))1334continue;1335 cmp =tree_entry_len(&entry) != cmplen ?1:1336memcmp(name, entry.path, cmplen);1337if(cmp >0)1338continue;1339if(cmp <0)1340return;1341if(name[cmplen] !='/') {1342add_object_entry(&entry.oid,1343object_type(entry.mode),1344 fullname,1);1345return;1346}1347if(S_ISDIR(entry.mode)) {1348struct tree_desc sub;1349struct pbase_tree_cache *tree;1350const char*down = name+cmplen+1;1351int downlen =name_cmp_len(down);13521353 tree =pbase_tree_get(&entry.oid);1354if(!tree)1355return;1356init_tree_desc(&sub, tree->tree_data, tree->tree_size);13571358add_pbase_object(&sub, down, downlen, fullname);1359pbase_tree_put(tree);1360}1361}1362}13631364static unsigned*done_pbase_paths;1365static int done_pbase_paths_num;1366static int done_pbase_paths_alloc;1367static intdone_pbase_path_pos(unsigned hash)1368{1369int lo =0;1370int hi = done_pbase_paths_num;1371while(lo < hi) {1372int mi = lo + (hi - lo) /2;1373if(done_pbase_paths[mi] == hash)1374return mi;1375if(done_pbase_paths[mi] < hash)1376 hi = mi;1377else1378 lo = mi +1;1379}1380return-lo-1;1381}13821383static intcheck_pbase_path(unsigned hash)1384{1385int pos =done_pbase_path_pos(hash);1386if(0<= pos)1387return1;1388 pos = -pos -1;1389ALLOC_GROW(done_pbase_paths,1390 done_pbase_paths_num +1,1391 done_pbase_paths_alloc);1392 done_pbase_paths_num++;1393if(pos < done_pbase_paths_num)1394MOVE_ARRAY(done_pbase_paths + pos +1, done_pbase_paths + pos,1395 done_pbase_paths_num - pos -1);1396 done_pbase_paths[pos] = hash;1397return0;1398}13991400static voidadd_preferred_base_object(const char*name)1401{1402struct pbase_tree *it;1403int cmplen;1404unsigned hash =pack_name_hash(name);14051406if(!num_preferred_base ||check_pbase_path(hash))1407return;14081409 cmplen =name_cmp_len(name);1410for(it = pbase_tree; it; it = it->next) {1411if(cmplen ==0) {1412add_object_entry(&it->pcache.oid, OBJ_TREE, NULL,1);1413}1414else{1415struct tree_desc tree;1416init_tree_desc(&tree, it->pcache.tree_data, it->pcache.tree_size);1417add_pbase_object(&tree, name, cmplen, name);1418}1419}1420}14211422static voidadd_preferred_base(struct object_id *oid)1423{1424struct pbase_tree *it;1425void*data;1426unsigned long size;1427struct object_id tree_oid;14281429if(window <= num_preferred_base++)1430return;14311432 data =read_object_with_reference(the_repository, oid,1433 tree_type, &size, &tree_oid);1434if(!data)1435return;14361437for(it = pbase_tree; it; it = it->next) {1438if(oideq(&it->pcache.oid, &tree_oid)) {1439free(data);1440return;1441}1442}14431444 it =xcalloc(1,sizeof(*it));1445 it->next = pbase_tree;1446 pbase_tree = it;14471448oidcpy(&it->pcache.oid, &tree_oid);1449 it->pcache.tree_data = data;1450 it->pcache.tree_size = size;1451}14521453static voidcleanup_preferred_base(void)1454{1455struct pbase_tree *it;1456unsigned i;14571458 it = pbase_tree;1459 pbase_tree = NULL;1460while(it) {1461struct pbase_tree *tmp = it;1462 it = tmp->next;1463free(tmp->pcache.tree_data);1464free(tmp);1465}14661467for(i =0; i <ARRAY_SIZE(pbase_tree_cache); i++) {1468if(!pbase_tree_cache[i])1469continue;1470free(pbase_tree_cache[i]->tree_data);1471FREE_AND_NULL(pbase_tree_cache[i]);1472}14731474FREE_AND_NULL(done_pbase_paths);1475 done_pbase_paths_num = done_pbase_paths_alloc =0;1476}14771478/*1479 * Return 1 iff the object specified by "delta" can be sent1480 * literally as a delta against the base in "base_sha1". If1481 * so, then *base_out will point to the entry in our packing1482 * list, or NULL if we must use the external-base list.1483 *1484 * Depth value does not matter - find_deltas() will1485 * never consider reused delta as the base object to1486 * deltify other objects against, in order to avoid1487 * circular deltas.1488 */1489static intcan_reuse_delta(const unsigned char*base_sha1,1490struct object_entry *delta,1491struct object_entry **base_out)1492{1493struct object_entry *base;1494struct object_id base_oid;14951496if(!base_sha1)1497return0;14981499oidread(&base_oid, base_sha1);15001501/*1502 * First see if we're already sending the base (or it's explicitly in1503 * our "excluded" list).1504 */1505 base =packlist_find(&to_pack, &base_oid);1506if(base) {1507if(!in_same_island(&delta->idx.oid, &base->idx.oid))1508return0;1509*base_out = base;1510return1;1511}15121513/*1514 * Otherwise, reachability bitmaps may tell us if the receiver has it,1515 * even if it was buried too deep in history to make it into the1516 * packing list.1517 */1518if(thin &&bitmap_has_oid_in_uninteresting(bitmap_git, &base_oid)) {1519if(use_delta_islands) {1520if(!in_same_island(&delta->idx.oid, &base_oid))1521return0;1522}1523*base_out = NULL;1524return1;1525}15261527return0;1528}15291530static voidcheck_object(struct object_entry *entry)1531{1532unsigned long canonical_size;15331534if(IN_PACK(entry)) {1535struct packed_git *p =IN_PACK(entry);1536struct pack_window *w_curs = NULL;1537const unsigned char*base_ref = NULL;1538struct object_entry *base_entry;1539unsigned long used, used_0;1540unsigned long avail;1541 off_t ofs;1542unsigned char*buf, c;1543enum object_type type;1544unsigned long in_pack_size;15451546 buf =use_pack(p, &w_curs, entry->in_pack_offset, &avail);15471548/*1549 * We want in_pack_type even if we do not reuse delta1550 * since non-delta representations could still be reused.1551 */1552 used =unpack_object_header_buffer(buf, avail,1553&type,1554&in_pack_size);1555if(used ==0)1556goto give_up;15571558if(type <0)1559BUG("invalid type%d", type);1560 entry->in_pack_type = type;15611562/*1563 * Determine if this is a delta and if so whether we can1564 * reuse it or not. Otherwise let's find out as cheaply as1565 * possible what the actual type and size for this object is.1566 */1567switch(entry->in_pack_type) {1568default:1569/* Not a delta hence we've already got all we need. */1570oe_set_type(entry, entry->in_pack_type);1571SET_SIZE(entry, in_pack_size);1572 entry->in_pack_header_size = used;1573if(oe_type(entry) < OBJ_COMMIT ||oe_type(entry) > OBJ_BLOB)1574goto give_up;1575unuse_pack(&w_curs);1576return;1577case OBJ_REF_DELTA:1578if(reuse_delta && !entry->preferred_base)1579 base_ref =use_pack(p, &w_curs,1580 entry->in_pack_offset + used, NULL);1581 entry->in_pack_header_size = used + the_hash_algo->rawsz;1582break;1583case OBJ_OFS_DELTA:1584 buf =use_pack(p, &w_curs,1585 entry->in_pack_offset + used, NULL);1586 used_0 =0;1587 c = buf[used_0++];1588 ofs = c &127;1589while(c &128) {1590 ofs +=1;1591if(!ofs ||MSB(ofs,7)) {1592error(_("delta base offset overflow in pack for%s"),1593oid_to_hex(&entry->idx.oid));1594goto give_up;1595}1596 c = buf[used_0++];1597 ofs = (ofs <<7) + (c &127);1598}1599 ofs = entry->in_pack_offset - ofs;1600if(ofs <=0|| ofs >= entry->in_pack_offset) {1601error(_("delta base offset out of bound for%s"),1602oid_to_hex(&entry->idx.oid));1603goto give_up;1604}1605if(reuse_delta && !entry->preferred_base) {1606struct revindex_entry *revidx;1607 revidx =find_pack_revindex(p, ofs);1608if(!revidx)1609goto give_up;1610 base_ref =nth_packed_object_sha1(p, revidx->nr);1611}1612 entry->in_pack_header_size = used + used_0;1613break;1614}16151616if(can_reuse_delta(base_ref, entry, &base_entry)) {1617oe_set_type(entry, entry->in_pack_type);1618SET_SIZE(entry, in_pack_size);/* delta size */1619SET_DELTA_SIZE(entry, in_pack_size);16201621if(base_entry) {1622SET_DELTA(entry, base_entry);1623 entry->delta_sibling_idx = base_entry->delta_child_idx;1624SET_DELTA_CHILD(base_entry, entry);1625}else{1626SET_DELTA_EXT(entry, base_ref);1627}16281629unuse_pack(&w_curs);1630return;1631}16321633if(oe_type(entry)) {1634 off_t delta_pos;16351636/*1637 * This must be a delta and we already know what the1638 * final object type is. Let's extract the actual1639 * object size from the delta header.1640 */1641 delta_pos = entry->in_pack_offset + entry->in_pack_header_size;1642 canonical_size =get_size_from_delta(p, &w_curs, delta_pos);1643if(canonical_size ==0)1644goto give_up;1645SET_SIZE(entry, canonical_size);1646unuse_pack(&w_curs);1647return;1648}16491650/*1651 * No choice but to fall back to the recursive delta walk1652 * with oid_object_info() to find about the object type1653 * at this point...1654 */1655 give_up:1656unuse_pack(&w_curs);1657}16581659oe_set_type(entry,1660oid_object_info(the_repository, &entry->idx.oid, &canonical_size));1661if(entry->type_valid) {1662SET_SIZE(entry, canonical_size);1663}else{1664/*1665 * Bad object type is checked in prepare_pack(). This is1666 * to permit a missing preferred base object to be ignored1667 * as a preferred base. Doing so can result in a larger1668 * pack file, but the transfer will still take place.1669 */1670}1671}16721673static intpack_offset_sort(const void*_a,const void*_b)1674{1675const struct object_entry *a = *(struct object_entry **)_a;1676const struct object_entry *b = *(struct object_entry **)_b;1677const struct packed_git *a_in_pack =IN_PACK(a);1678const struct packed_git *b_in_pack =IN_PACK(b);16791680/* avoid filesystem trashing with loose objects */1681if(!a_in_pack && !b_in_pack)1682returnoidcmp(&a->idx.oid, &b->idx.oid);16831684if(a_in_pack < b_in_pack)1685return-1;1686if(a_in_pack > b_in_pack)1687return1;1688return a->in_pack_offset < b->in_pack_offset ? -1:1689(a->in_pack_offset > b->in_pack_offset);1690}16911692/*1693 * Drop an on-disk delta we were planning to reuse. Naively, this would1694 * just involve blanking out the "delta" field, but we have to deal1695 * with some extra book-keeping:1696 *1697 * 1. Removing ourselves from the delta_sibling linked list.1698 *1699 * 2. Updating our size/type to the non-delta representation. These were1700 * either not recorded initially (size) or overwritten with the delta type1701 * (type) when check_object() decided to reuse the delta.1702 *1703 * 3. Resetting our delta depth, as we are now a base object.1704 */1705static voiddrop_reused_delta(struct object_entry *entry)1706{1707unsigned*idx = &to_pack.objects[entry->delta_idx -1].delta_child_idx;1708struct object_info oi = OBJECT_INFO_INIT;1709enum object_type type;1710unsigned long size;17111712while(*idx) {1713struct object_entry *oe = &to_pack.objects[*idx -1];17141715if(oe == entry)1716*idx = oe->delta_sibling_idx;1717else1718 idx = &oe->delta_sibling_idx;1719}1720SET_DELTA(entry, NULL);1721 entry->depth =0;17221723 oi.sizep = &size;1724 oi.typep = &type;1725if(packed_object_info(the_repository,IN_PACK(entry), entry->in_pack_offset, &oi) <0) {1726/*1727 * We failed to get the info from this pack for some reason;1728 * fall back to oid_object_info, which may find another copy.1729 * And if that fails, the error will be recorded in oe_type(entry)1730 * and dealt with in prepare_pack().1731 */1732oe_set_type(entry,1733oid_object_info(the_repository, &entry->idx.oid, &size));1734}else{1735oe_set_type(entry, type);1736}1737SET_SIZE(entry, size);1738}17391740/*1741 * Follow the chain of deltas from this entry onward, throwing away any links1742 * that cause us to hit a cycle (as determined by the DFS state flags in1743 * the entries).1744 *1745 * We also detect too-long reused chains that would violate our --depth1746 * limit.1747 */1748static voidbreak_delta_chains(struct object_entry *entry)1749{1750/*1751 * The actual depth of each object we will write is stored as an int,1752 * as it cannot exceed our int "depth" limit. But before we break1753 * changes based no that limit, we may potentially go as deep as the1754 * number of objects, which is elsewhere bounded to a uint32_t.1755 */1756uint32_t total_depth;1757struct object_entry *cur, *next;17581759for(cur = entry, total_depth =0;1760 cur;1761 cur =DELTA(cur), total_depth++) {1762if(cur->dfs_state == DFS_DONE) {1763/*1764 * We've already seen this object and know it isn't1765 * part of a cycle. We do need to append its depth1766 * to our count.1767 */1768 total_depth += cur->depth;1769break;1770}17711772/*1773 * We break cycles before looping, so an ACTIVE state (or any1774 * other cruft which made its way into the state variable)1775 * is a bug.1776 */1777if(cur->dfs_state != DFS_NONE)1778BUG("confusing delta dfs state in first pass:%d",1779 cur->dfs_state);17801781/*1782 * Now we know this is the first time we've seen the object. If1783 * it's not a delta, we're done traversing, but we'll mark it1784 * done to save time on future traversals.1785 */1786if(!DELTA(cur)) {1787 cur->dfs_state = DFS_DONE;1788break;1789}17901791/*1792 * Mark ourselves as active and see if the next step causes1793 * us to cycle to another active object. It's important to do1794 * this _before_ we loop, because it impacts where we make the1795 * cut, and thus how our total_depth counter works.1796 * E.g., We may see a partial loop like:1797 *1798 * A -> B -> C -> D -> B1799 *1800 * Cutting B->C breaks the cycle. But now the depth of A is1801 * only 1, and our total_depth counter is at 3. The size of the1802 * error is always one less than the size of the cycle we1803 * broke. Commits C and D were "lost" from A's chain.1804 *1805 * If we instead cut D->B, then the depth of A is correct at 3.1806 * We keep all commits in the chain that we examined.1807 */1808 cur->dfs_state = DFS_ACTIVE;1809if(DELTA(cur)->dfs_state == DFS_ACTIVE) {1810drop_reused_delta(cur);1811 cur->dfs_state = DFS_DONE;1812break;1813}1814}18151816/*1817 * And now that we've gone all the way to the bottom of the chain, we1818 * need to clear the active flags and set the depth fields as1819 * appropriate. Unlike the loop above, which can quit when it drops a1820 * delta, we need to keep going to look for more depth cuts. So we need1821 * an extra "next" pointer to keep going after we reset cur->delta.1822 */1823for(cur = entry; cur; cur = next) {1824 next =DELTA(cur);18251826/*1827 * We should have a chain of zero or more ACTIVE states down to1828 * a final DONE. We can quit after the DONE, because either it1829 * has no bases, or we've already handled them in a previous1830 * call.1831 */1832if(cur->dfs_state == DFS_DONE)1833break;1834else if(cur->dfs_state != DFS_ACTIVE)1835BUG("confusing delta dfs state in second pass:%d",1836 cur->dfs_state);18371838/*1839 * If the total_depth is more than depth, then we need to snip1840 * the chain into two or more smaller chains that don't exceed1841 * the maximum depth. Most of the resulting chains will contain1842 * (depth + 1) entries (i.e., depth deltas plus one base), and1843 * the last chain (i.e., the one containing entry) will contain1844 * whatever entries are left over, namely1845 * (total_depth % (depth + 1)) of them.1846 *1847 * Since we are iterating towards decreasing depth, we need to1848 * decrement total_depth as we go, and we need to write to the1849 * entry what its final depth will be after all of the1850 * snipping. Since we're snipping into chains of length (depth1851 * + 1) entries, the final depth of an entry will be its1852 * original depth modulo (depth + 1). Any time we encounter an1853 * entry whose final depth is supposed to be zero, we snip it1854 * from its delta base, thereby making it so.1855 */1856 cur->depth = (total_depth--) % (depth +1);1857if(!cur->depth)1858drop_reused_delta(cur);18591860 cur->dfs_state = DFS_DONE;1861}1862}18631864static voidget_object_details(void)1865{1866uint32_t i;1867struct object_entry **sorted_by_offset;18681869if(progress)1870 progress_state =start_progress(_("Counting objects"),1871 to_pack.nr_objects);18721873 sorted_by_offset =xcalloc(to_pack.nr_objects,sizeof(struct object_entry *));1874for(i =0; i < to_pack.nr_objects; i++)1875 sorted_by_offset[i] = to_pack.objects + i;1876QSORT(sorted_by_offset, to_pack.nr_objects, pack_offset_sort);18771878for(i =0; i < to_pack.nr_objects; i++) {1879struct object_entry *entry = sorted_by_offset[i];1880check_object(entry);1881if(entry->type_valid &&1882oe_size_greater_than(&to_pack, entry, big_file_threshold))1883 entry->no_try_delta =1;1884display_progress(progress_state, i +1);1885}1886stop_progress(&progress_state);18871888/*1889 * This must happen in a second pass, since we rely on the delta1890 * information for the whole list being completed.1891 */1892for(i =0; i < to_pack.nr_objects; i++)1893break_delta_chains(&to_pack.objects[i]);18941895free(sorted_by_offset);1896}18971898/*1899 * We search for deltas in a list sorted by type, by filename hash, and then1900 * by size, so that we see progressively smaller and smaller files.1901 * That's because we prefer deltas to be from the bigger file1902 * to the smaller -- deletes are potentially cheaper, but perhaps1903 * more importantly, the bigger file is likely the more recent1904 * one. The deepest deltas are therefore the oldest objects which are1905 * less susceptible to be accessed often.1906 */1907static inttype_size_sort(const void*_a,const void*_b)1908{1909const struct object_entry *a = *(struct object_entry **)_a;1910const struct object_entry *b = *(struct object_entry **)_b;1911const enum object_type a_type =oe_type(a);1912const enum object_type b_type =oe_type(b);1913const unsigned long a_size =SIZE(a);1914const unsigned long b_size =SIZE(b);19151916if(a_type > b_type)1917return-1;1918if(a_type < b_type)1919return1;1920if(a->hash > b->hash)1921return-1;1922if(a->hash < b->hash)1923return1;1924if(a->preferred_base > b->preferred_base)1925return-1;1926if(a->preferred_base < b->preferred_base)1927return1;1928if(use_delta_islands) {1929const int island_cmp =island_delta_cmp(&a->idx.oid, &b->idx.oid);1930if(island_cmp)1931return island_cmp;1932}1933if(a_size > b_size)1934return-1;1935if(a_size < b_size)1936return1;1937return a < b ? -1: (a > b);/* newest first */1938}19391940struct unpacked {1941struct object_entry *entry;1942void*data;1943struct delta_index *index;1944unsigned depth;1945};19461947static intdelta_cacheable(unsigned long src_size,unsigned long trg_size,1948unsigned long delta_size)1949{1950if(max_delta_cache_size && delta_cache_size + delta_size > max_delta_cache_size)1951return0;19521953if(delta_size < cache_max_small_delta_size)1954return1;19551956/* cache delta, if objects are large enough compared to delta size */1957if((src_size >>20) + (trg_size >>21) > (delta_size >>10))1958return1;19591960return0;1961}19621963/* Protect delta_cache_size */1964static pthread_mutex_t cache_mutex;1965#define cache_lock() pthread_mutex_lock(&cache_mutex)1966#define cache_unlock() pthread_mutex_unlock(&cache_mutex)19671968/*1969 * Protect object list partitioning (e.g. struct thread_param) and1970 * progress_state1971 */1972static pthread_mutex_t progress_mutex;1973#define progress_lock() pthread_mutex_lock(&progress_mutex)1974#define progress_unlock() pthread_mutex_unlock(&progress_mutex)19751976/*1977 * Access to struct object_entry is unprotected since each thread owns1978 * a portion of the main object list. Just don't access object entries1979 * ahead in the list because they can be stolen and would need1980 * progress_mutex for protection.1981 */19821983/*1984 * Return the size of the object without doing any delta1985 * reconstruction (so non-deltas are true object sizes, but deltas1986 * return the size of the delta data).1987 */1988unsigned longoe_get_size_slow(struct packing_data *pack,1989const struct object_entry *e)1990{1991struct packed_git *p;1992struct pack_window *w_curs;1993unsigned char*buf;1994enum object_type type;1995unsigned long used, avail, size;19961997if(e->type_ != OBJ_OFS_DELTA && e->type_ != OBJ_REF_DELTA) {1998packing_data_lock(&to_pack);1999if(oid_object_info(the_repository, &e->idx.oid, &size) <0)2000die(_("unable to get size of%s"),2001oid_to_hex(&e->idx.oid));2002packing_data_unlock(&to_pack);2003return size;2004}20052006 p =oe_in_pack(pack, e);2007if(!p)2008BUG("when e->type is a delta, it must belong to a pack");20092010packing_data_lock(&to_pack);2011 w_curs = NULL;2012 buf =use_pack(p, &w_curs, e->in_pack_offset, &avail);2013 used =unpack_object_header_buffer(buf, avail, &type, &size);2014if(used ==0)2015die(_("unable to parse object header of%s"),2016oid_to_hex(&e->idx.oid));20172018unuse_pack(&w_curs);2019packing_data_unlock(&to_pack);2020return size;2021}20222023static inttry_delta(struct unpacked *trg,struct unpacked *src,2024unsigned max_depth,unsigned long*mem_usage)2025{2026struct object_entry *trg_entry = trg->entry;2027struct object_entry *src_entry = src->entry;2028unsigned long trg_size, src_size, delta_size, sizediff, max_size, sz;2029unsigned ref_depth;2030enum object_type type;2031void*delta_buf;20322033/* Don't bother doing diffs between different types */2034if(oe_type(trg_entry) !=oe_type(src_entry))2035return-1;20362037/*2038 * We do not bother to try a delta that we discarded on an2039 * earlier try, but only when reusing delta data. Note that2040 * src_entry that is marked as the preferred_base should always2041 * be considered, as even if we produce a suboptimal delta against2042 * it, we will still save the transfer cost, as we already know2043 * the other side has it and we won't send src_entry at all.2044 */2045if(reuse_delta &&IN_PACK(trg_entry) &&2046IN_PACK(trg_entry) ==IN_PACK(src_entry) &&2047!src_entry->preferred_base &&2048 trg_entry->in_pack_type != OBJ_REF_DELTA &&2049 trg_entry->in_pack_type != OBJ_OFS_DELTA)2050return0;20512052/* Let's not bust the allowed depth. */2053if(src->depth >= max_depth)2054return0;20552056/* Now some size filtering heuristics. */2057 trg_size =SIZE(trg_entry);2058if(!DELTA(trg_entry)) {2059 max_size = trg_size/2- the_hash_algo->rawsz;2060 ref_depth =1;2061}else{2062 max_size =DELTA_SIZE(trg_entry);2063 ref_depth = trg->depth;2064}2065 max_size = (uint64_t)max_size * (max_depth - src->depth) /2066(max_depth - ref_depth +1);2067if(max_size ==0)2068return0;2069 src_size =SIZE(src_entry);2070 sizediff = src_size < trg_size ? trg_size - src_size :0;2071if(sizediff >= max_size)2072return0;2073if(trg_size < src_size /32)2074return0;20752076if(!in_same_island(&trg->entry->idx.oid, &src->entry->idx.oid))2077return0;20782079/* Load data if not already done */2080if(!trg->data) {2081packing_data_lock(&to_pack);2082 trg->data =read_object_file(&trg_entry->idx.oid, &type, &sz);2083packing_data_unlock(&to_pack);2084if(!trg->data)2085die(_("object%scannot be read"),2086oid_to_hex(&trg_entry->idx.oid));2087if(sz != trg_size)2088die(_("object%sinconsistent object length (%"PRIuMAX" vs %"PRIuMAX")"),2089oid_to_hex(&trg_entry->idx.oid), (uintmax_t)sz,2090(uintmax_t)trg_size);2091*mem_usage += sz;2092}2093if(!src->data) {2094packing_data_lock(&to_pack);2095 src->data =read_object_file(&src_entry->idx.oid, &type, &sz);2096packing_data_unlock(&to_pack);2097if(!src->data) {2098if(src_entry->preferred_base) {2099static int warned =0;2100if(!warned++)2101warning(_("object%scannot be read"),2102oid_to_hex(&src_entry->idx.oid));2103/*2104 * Those objects are not included in the2105 * resulting pack. Be resilient and ignore2106 * them if they can't be read, in case the2107 * pack could be created nevertheless.2108 */2109return0;2110}2111die(_("object%scannot be read"),2112oid_to_hex(&src_entry->idx.oid));2113}2114if(sz != src_size)2115die(_("object%sinconsistent object length (%"PRIuMAX" vs %"PRIuMAX")"),2116oid_to_hex(&src_entry->idx.oid), (uintmax_t)sz,2117(uintmax_t)src_size);2118*mem_usage += sz;2119}2120if(!src->index) {2121 src->index =create_delta_index(src->data, src_size);2122if(!src->index) {2123static int warned =0;2124if(!warned++)2125warning(_("suboptimal pack - out of memory"));2126return0;2127}2128*mem_usage +=sizeof_delta_index(src->index);2129}21302131 delta_buf =create_delta(src->index, trg->data, trg_size, &delta_size, max_size);2132if(!delta_buf)2133return0;21342135if(DELTA(trg_entry)) {2136/* Prefer only shallower same-sized deltas. */2137if(delta_size ==DELTA_SIZE(trg_entry) &&2138 src->depth +1>= trg->depth) {2139free(delta_buf);2140return0;2141}2142}21432144/*2145 * Handle memory allocation outside of the cache2146 * accounting lock. Compiler will optimize the strangeness2147 * away when NO_PTHREADS is defined.2148 */2149free(trg_entry->delta_data);2150cache_lock();2151if(trg_entry->delta_data) {2152 delta_cache_size -=DELTA_SIZE(trg_entry);2153 trg_entry->delta_data = NULL;2154}2155if(delta_cacheable(src_size, trg_size, delta_size)) {2156 delta_cache_size += delta_size;2157cache_unlock();2158 trg_entry->delta_data =xrealloc(delta_buf, delta_size);2159}else{2160cache_unlock();2161free(delta_buf);2162}21632164SET_DELTA(trg_entry, src_entry);2165SET_DELTA_SIZE(trg_entry, delta_size);2166 trg->depth = src->depth +1;21672168return1;2169}21702171static unsigned intcheck_delta_limit(struct object_entry *me,unsigned int n)2172{2173struct object_entry *child =DELTA_CHILD(me);2174unsigned int m = n;2175while(child) {2176const unsigned int c =check_delta_limit(child, n +1);2177if(m < c)2178 m = c;2179 child =DELTA_SIBLING(child);2180}2181return m;2182}21832184static unsigned longfree_unpacked(struct unpacked *n)2185{2186unsigned long freed_mem =sizeof_delta_index(n->index);2187free_delta_index(n->index);2188 n->index = NULL;2189if(n->data) {2190 freed_mem +=SIZE(n->entry);2191FREE_AND_NULL(n->data);2192}2193 n->entry = NULL;2194 n->depth =0;2195return freed_mem;2196}21972198static voidfind_deltas(struct object_entry **list,unsigned*list_size,2199int window,int depth,unsigned*processed)2200{2201uint32_t i, idx =0, count =0;2202struct unpacked *array;2203unsigned long mem_usage =0;22042205 array =xcalloc(window,sizeof(struct unpacked));22062207for(;;) {2208struct object_entry *entry;2209struct unpacked *n = array + idx;2210int j, max_depth, best_base = -1;22112212progress_lock();2213if(!*list_size) {2214progress_unlock();2215break;2216}2217 entry = *list++;2218(*list_size)--;2219if(!entry->preferred_base) {2220(*processed)++;2221display_progress(progress_state, *processed);2222}2223progress_unlock();22242225 mem_usage -=free_unpacked(n);2226 n->entry = entry;22272228while(window_memory_limit &&2229 mem_usage > window_memory_limit &&2230 count >1) {2231const uint32_t tail = (idx + window - count) % window;2232 mem_usage -=free_unpacked(array + tail);2233 count--;2234}22352236/* We do not compute delta to *create* objects we are not2237 * going to pack.2238 */2239if(entry->preferred_base)2240goto next;22412242/*2243 * If the current object is at pack edge, take the depth the2244 * objects that depend on the current object into account2245 * otherwise they would become too deep.2246 */2247 max_depth = depth;2248if(DELTA_CHILD(entry)) {2249 max_depth -=check_delta_limit(entry,0);2250if(max_depth <=0)2251goto next;2252}22532254 j = window;2255while(--j >0) {2256int ret;2257uint32_t other_idx = idx + j;2258struct unpacked *m;2259if(other_idx >= window)2260 other_idx -= window;2261 m = array + other_idx;2262if(!m->entry)2263break;2264 ret =try_delta(n, m, max_depth, &mem_usage);2265if(ret <0)2266break;2267else if(ret >0)2268 best_base = other_idx;2269}22702271/*2272 * If we decided to cache the delta data, then it is best2273 * to compress it right away. First because we have to do2274 * it anyway, and doing it here while we're threaded will2275 * save a lot of time in the non threaded write phase,2276 * as well as allow for caching more deltas within2277 * the same cache size limit.2278 * ...2279 * But only if not writing to stdout, since in that case2280 * the network is most likely throttling writes anyway,2281 * and therefore it is best to go to the write phase ASAP2282 * instead, as we can afford spending more time compressing2283 * between writes at that moment.2284 */2285if(entry->delta_data && !pack_to_stdout) {2286unsigned long size;22872288 size =do_compress(&entry->delta_data,DELTA_SIZE(entry));2289if(size < (1U<< OE_Z_DELTA_BITS)) {2290 entry->z_delta_size = size;2291cache_lock();2292 delta_cache_size -=DELTA_SIZE(entry);2293 delta_cache_size += entry->z_delta_size;2294cache_unlock();2295}else{2296FREE_AND_NULL(entry->delta_data);2297 entry->z_delta_size =0;2298}2299}23002301/* if we made n a delta, and if n is already at max2302 * depth, leaving it in the window is pointless. we2303 * should evict it first.2304 */2305if(DELTA(entry) && max_depth <= n->depth)2306continue;23072308/*2309 * Move the best delta base up in the window, after the2310 * currently deltified object, to keep it longer. It will2311 * be the first base object to be attempted next.2312 */2313if(DELTA(entry)) {2314struct unpacked swap = array[best_base];2315int dist = (window + idx - best_base) % window;2316int dst = best_base;2317while(dist--) {2318int src = (dst +1) % window;2319 array[dst] = array[src];2320 dst = src;2321}2322 array[dst] = swap;2323}23242325 next:2326 idx++;2327if(count +1< window)2328 count++;2329if(idx >= window)2330 idx =0;2331}23322333for(i =0; i < window; ++i) {2334free_delta_index(array[i].index);2335free(array[i].data);2336}2337free(array);2338}23392340static voidtry_to_free_from_threads(size_t size)2341{2342packing_data_lock(&to_pack);2343release_pack_memory(size);2344packing_data_unlock(&to_pack);2345}23462347static try_to_free_t old_try_to_free_routine;23482349/*2350 * The main object list is split into smaller lists, each is handed to2351 * one worker.2352 *2353 * The main thread waits on the condition that (at least) one of the workers2354 * has stopped working (which is indicated in the .working member of2355 * struct thread_params).2356 *2357 * When a work thread has completed its work, it sets .working to 0 and2358 * signals the main thread and waits on the condition that .data_ready2359 * becomes 1.2360 *2361 * The main thread steals half of the work from the worker that has2362 * most work left to hand it to the idle worker.2363 */23642365struct thread_params {2366 pthread_t thread;2367struct object_entry **list;2368unsigned list_size;2369unsigned remaining;2370int window;2371int depth;2372int working;2373int data_ready;2374 pthread_mutex_t mutex;2375 pthread_cond_t cond;2376unsigned*processed;2377};23782379static pthread_cond_t progress_cond;23802381/*2382 * Mutex and conditional variable can't be statically-initialized on Windows.2383 */2384static voidinit_threaded_search(void)2385{2386pthread_mutex_init(&cache_mutex, NULL);2387pthread_mutex_init(&progress_mutex, NULL);2388pthread_cond_init(&progress_cond, NULL);2389 old_try_to_free_routine =set_try_to_free_routine(try_to_free_from_threads);2390}23912392static voidcleanup_threaded_search(void)2393{2394set_try_to_free_routine(old_try_to_free_routine);2395pthread_cond_destroy(&progress_cond);2396pthread_mutex_destroy(&cache_mutex);2397pthread_mutex_destroy(&progress_mutex);2398}23992400static void*threaded_find_deltas(void*arg)2401{2402struct thread_params *me = arg;24032404progress_lock();2405while(me->remaining) {2406progress_unlock();24072408find_deltas(me->list, &me->remaining,2409 me->window, me->depth, me->processed);24102411progress_lock();2412 me->working =0;2413pthread_cond_signal(&progress_cond);2414progress_unlock();24152416/*2417 * We must not set ->data_ready before we wait on the2418 * condition because the main thread may have set it to 12419 * before we get here. In order to be sure that new2420 * work is available if we see 1 in ->data_ready, it2421 * was initialized to 0 before this thread was spawned2422 * and we reset it to 0 right away.2423 */2424pthread_mutex_lock(&me->mutex);2425while(!me->data_ready)2426pthread_cond_wait(&me->cond, &me->mutex);2427 me->data_ready =0;2428pthread_mutex_unlock(&me->mutex);24292430progress_lock();2431}2432progress_unlock();2433/* leave ->working 1 so that this doesn't get more work assigned */2434return NULL;2435}24362437static voidll_find_deltas(struct object_entry **list,unsigned list_size,2438int window,int depth,unsigned*processed)2439{2440struct thread_params *p;2441int i, ret, active_threads =0;24422443init_threaded_search();24442445if(delta_search_threads <=1) {2446find_deltas(list, &list_size, window, depth, processed);2447cleanup_threaded_search();2448return;2449}2450if(progress > pack_to_stdout)2451fprintf_ln(stderr,_("Delta compression using up to%dthreads"),2452 delta_search_threads);2453 p =xcalloc(delta_search_threads,sizeof(*p));24542455/* Partition the work amongst work threads. */2456for(i =0; i < delta_search_threads; i++) {2457unsigned sub_size = list_size / (delta_search_threads - i);24582459/* don't use too small segments or no deltas will be found */2460if(sub_size <2*window && i+1< delta_search_threads)2461 sub_size =0;24622463 p[i].window = window;2464 p[i].depth = depth;2465 p[i].processed = processed;2466 p[i].working =1;2467 p[i].data_ready =0;24682469/* try to split chunks on "path" boundaries */2470while(sub_size && sub_size < list_size &&2471 list[sub_size]->hash &&2472 list[sub_size]->hash == list[sub_size-1]->hash)2473 sub_size++;24742475 p[i].list = list;2476 p[i].list_size = sub_size;2477 p[i].remaining = sub_size;24782479 list += sub_size;2480 list_size -= sub_size;2481}24822483/* Start work threads. */2484for(i =0; i < delta_search_threads; i++) {2485if(!p[i].list_size)2486continue;2487pthread_mutex_init(&p[i].mutex, NULL);2488pthread_cond_init(&p[i].cond, NULL);2489 ret =pthread_create(&p[i].thread, NULL,2490 threaded_find_deltas, &p[i]);2491if(ret)2492die(_("unable to create thread:%s"),strerror(ret));2493 active_threads++;2494}24952496/*2497 * Now let's wait for work completion. Each time a thread is done2498 * with its work, we steal half of the remaining work from the2499 * thread with the largest number of unprocessed objects and give2500 * it to that newly idle thread. This ensure good load balancing2501 * until the remaining object list segments are simply too short2502 * to be worth splitting anymore.2503 */2504while(active_threads) {2505struct thread_params *target = NULL;2506struct thread_params *victim = NULL;2507unsigned sub_size =0;25082509progress_lock();2510for(;;) {2511for(i =0; !target && i < delta_search_threads; i++)2512if(!p[i].working)2513 target = &p[i];2514if(target)2515break;2516pthread_cond_wait(&progress_cond, &progress_mutex);2517}25182519for(i =0; i < delta_search_threads; i++)2520if(p[i].remaining >2*window &&2521(!victim || victim->remaining < p[i].remaining))2522 victim = &p[i];2523if(victim) {2524 sub_size = victim->remaining /2;2525 list = victim->list + victim->list_size - sub_size;2526while(sub_size && list[0]->hash &&2527 list[0]->hash == list[-1]->hash) {2528 list++;2529 sub_size--;2530}2531if(!sub_size) {2532/*2533 * It is possible for some "paths" to have2534 * so many objects that no hash boundary2535 * might be found. Let's just steal the2536 * exact half in that case.2537 */2538 sub_size = victim->remaining /2;2539 list -= sub_size;2540}2541 target->list = list;2542 victim->list_size -= sub_size;2543 victim->remaining -= sub_size;2544}2545 target->list_size = sub_size;2546 target->remaining = sub_size;2547 target->working =1;2548progress_unlock();25492550pthread_mutex_lock(&target->mutex);2551 target->data_ready =1;2552pthread_cond_signal(&target->cond);2553pthread_mutex_unlock(&target->mutex);25542555if(!sub_size) {2556pthread_join(target->thread, NULL);2557pthread_cond_destroy(&target->cond);2558pthread_mutex_destroy(&target->mutex);2559 active_threads--;2560}2561}2562cleanup_threaded_search();2563free(p);2564}25652566static voidadd_tag_chain(const struct object_id *oid)2567{2568struct tag *tag;25692570/*2571 * We catch duplicates already in add_object_entry(), but we'd2572 * prefer to do this extra check to avoid having to parse the2573 * tag at all if we already know that it's being packed (e.g., if2574 * it was included via bitmaps, we would not have parsed it2575 * previously).2576 */2577if(packlist_find(&to_pack, oid))2578return;25792580 tag =lookup_tag(the_repository, oid);2581while(1) {2582if(!tag ||parse_tag(tag) || !tag->tagged)2583die(_("unable to pack objects reachable from tag%s"),2584oid_to_hex(oid));25852586add_object_entry(&tag->object.oid, OBJ_TAG, NULL,0);25872588if(tag->tagged->type != OBJ_TAG)2589return;25902591 tag = (struct tag *)tag->tagged;2592}2593}25942595static intadd_ref_tag(const char*path,const struct object_id *oid,int flag,void*cb_data)2596{2597struct object_id peeled;25982599if(starts_with(path,"refs/tags/") &&/* is a tag? */2600!peel_ref(path, &peeled) &&/* peelable? */2601packlist_find(&to_pack, &peeled))/* object packed? */2602add_tag_chain(oid);2603return0;2604}26052606static voidprepare_pack(int window,int depth)2607{2608struct object_entry **delta_list;2609uint32_t i, nr_deltas;2610unsigned n;26112612if(use_delta_islands)2613resolve_tree_islands(the_repository, progress, &to_pack);26142615get_object_details();26162617/*2618 * If we're locally repacking then we need to be doubly careful2619 * from now on in order to make sure no stealth corruption gets2620 * propagated to the new pack. Clients receiving streamed packs2621 * should validate everything they get anyway so no need to incur2622 * the additional cost here in that case.2623 */2624if(!pack_to_stdout)2625 do_check_packed_object_crc =1;26262627if(!to_pack.nr_objects || !window || !depth)2628return;26292630ALLOC_ARRAY(delta_list, to_pack.nr_objects);2631 nr_deltas = n =0;26322633for(i =0; i < to_pack.nr_objects; i++) {2634struct object_entry *entry = to_pack.objects + i;26352636if(DELTA(entry))2637/* This happens if we decided to reuse existing2638 * delta from a pack. "reuse_delta &&" is implied.2639 */2640continue;26412642if(!entry->type_valid ||2643oe_size_less_than(&to_pack, entry,50))2644continue;26452646if(entry->no_try_delta)2647continue;26482649if(!entry->preferred_base) {2650 nr_deltas++;2651if(oe_type(entry) <0)2652die(_("unable to get type of object%s"),2653oid_to_hex(&entry->idx.oid));2654}else{2655if(oe_type(entry) <0) {2656/*2657 * This object is not found, but we2658 * don't have to include it anyway.2659 */2660continue;2661}2662}26632664 delta_list[n++] = entry;2665}26662667if(nr_deltas && n >1) {2668unsigned nr_done =0;2669if(progress)2670 progress_state =start_progress(_("Compressing objects"),2671 nr_deltas);2672QSORT(delta_list, n, type_size_sort);2673ll_find_deltas(delta_list, n, window+1, depth, &nr_done);2674stop_progress(&progress_state);2675if(nr_done != nr_deltas)2676die(_("inconsistency with delta count"));2677}2678free(delta_list);2679}26802681static intgit_pack_config(const char*k,const char*v,void*cb)2682{2683if(!strcmp(k,"pack.window")) {2684 window =git_config_int(k, v);2685return0;2686}2687if(!strcmp(k,"pack.windowmemory")) {2688 window_memory_limit =git_config_ulong(k, v);2689return0;2690}2691if(!strcmp(k,"pack.depth")) {2692 depth =git_config_int(k, v);2693return0;2694}2695if(!strcmp(k,"pack.deltacachesize")) {2696 max_delta_cache_size =git_config_int(k, v);2697return0;2698}2699if(!strcmp(k,"pack.deltacachelimit")) {2700 cache_max_small_delta_size =git_config_int(k, v);2701return0;2702}2703if(!strcmp(k,"pack.writebitmaphashcache")) {2704if(git_config_bool(k, v))2705 write_bitmap_options |= BITMAP_OPT_HASH_CACHE;2706else2707 write_bitmap_options &= ~BITMAP_OPT_HASH_CACHE;2708}2709if(!strcmp(k,"pack.usebitmaps")) {2710 use_bitmap_index_default =git_config_bool(k, v);2711return0;2712}2713if(!strcmp(k,"pack.usesparse")) {2714 sparse =git_config_bool(k, v);2715return0;2716}2717if(!strcmp(k,"pack.threads")) {2718 delta_search_threads =git_config_int(k, v);2719if(delta_search_threads <0)2720die(_("invalid number of threads specified (%d)"),2721 delta_search_threads);2722if(!HAVE_THREADS && delta_search_threads !=1) {2723warning(_("no threads support, ignoring%s"), k);2724 delta_search_threads =0;2725}2726return0;2727}2728if(!strcmp(k,"pack.indexversion")) {2729 pack_idx_opts.version =git_config_int(k, v);2730if(pack_idx_opts.version >2)2731die(_("bad pack.indexversion=%"PRIu32),2732 pack_idx_opts.version);2733return0;2734}2735returngit_default_config(k, v, cb);2736}27372738static voidread_object_list_from_stdin(void)2739{2740char line[GIT_MAX_HEXSZ +1+ PATH_MAX +2];2741struct object_id oid;2742const char*p;27432744for(;;) {2745if(!fgets(line,sizeof(line), stdin)) {2746if(feof(stdin))2747break;2748if(!ferror(stdin))2749die("BUG: fgets returned NULL, not EOF, not error!");2750if(errno != EINTR)2751die_errno("fgets");2752clearerr(stdin);2753continue;2754}2755if(line[0] =='-') {2756if(get_oid_hex(line+1, &oid))2757die(_("expected edge object ID, got garbage:\n%s"),2758 line);2759add_preferred_base(&oid);2760continue;2761}2762if(parse_oid_hex(line, &oid, &p))2763die(_("expected object ID, got garbage:\n%s"), line);27642765add_preferred_base_object(p +1);2766add_object_entry(&oid, OBJ_NONE, p +1,0);2767}2768}27692770/* Remember to update object flag allocation in object.h */2771#define OBJECT_ADDED (1u<<20)27722773static voidshow_commit(struct commit *commit,void*data)2774{2775add_object_entry(&commit->object.oid, OBJ_COMMIT, NULL,0);2776 commit->object.flags |= OBJECT_ADDED;27772778if(write_bitmap_index)2779index_commit_for_bitmap(commit);27802781if(use_delta_islands)2782propagate_island_marks(commit);2783}27842785static voidshow_object(struct object *obj,const char*name,void*data)2786{2787add_preferred_base_object(name);2788add_object_entry(&obj->oid, obj->type, name,0);2789 obj->flags |= OBJECT_ADDED;27902791if(use_delta_islands) {2792const char*p;2793unsigned depth;2794struct object_entry *ent;27952796/* the empty string is a root tree, which is depth 0 */2797 depth = *name ?1:0;2798for(p =strchr(name,'/'); p; p =strchr(p +1,'/'))2799 depth++;28002801 ent =packlist_find(&to_pack, &obj->oid);2802if(ent && depth >oe_tree_depth(&to_pack, ent))2803oe_set_tree_depth(&to_pack, ent, depth);2804}2805}28062807static voidshow_object__ma_allow_any(struct object *obj,const char*name,void*data)2808{2809assert(arg_missing_action == MA_ALLOW_ANY);28102811/*2812 * Quietly ignore ALL missing objects. This avoids problems with2813 * staging them now and getting an odd error later.2814 */2815if(!has_object_file(&obj->oid))2816return;28172818show_object(obj, name, data);2819}28202821static voidshow_object__ma_allow_promisor(struct object *obj,const char*name,void*data)2822{2823assert(arg_missing_action == MA_ALLOW_PROMISOR);28242825/*2826 * Quietly ignore EXPECTED missing objects. This avoids problems with2827 * staging them now and getting an odd error later.2828 */2829if(!has_object_file(&obj->oid) &&is_promisor_object(&obj->oid))2830return;28312832show_object(obj, name, data);2833}28342835static intoption_parse_missing_action(const struct option *opt,2836const char*arg,int unset)2837{2838assert(arg);2839assert(!unset);28402841if(!strcmp(arg,"error")) {2842 arg_missing_action = MA_ERROR;2843 fn_show_object = show_object;2844return0;2845}28462847if(!strcmp(arg,"allow-any")) {2848 arg_missing_action = MA_ALLOW_ANY;2849 fetch_if_missing =0;2850 fn_show_object = show_object__ma_allow_any;2851return0;2852}28532854if(!strcmp(arg,"allow-promisor")) {2855 arg_missing_action = MA_ALLOW_PROMISOR;2856 fetch_if_missing =0;2857 fn_show_object = show_object__ma_allow_promisor;2858return0;2859}28602861die(_("invalid value for --missing"));2862return0;2863}28642865static voidshow_edge(struct commit *commit)2866{2867add_preferred_base(&commit->object.oid);2868}28692870struct in_pack_object {2871 off_t offset;2872struct object *object;2873};28742875struct in_pack {2876unsigned int alloc;2877unsigned int nr;2878struct in_pack_object *array;2879};28802881static voidmark_in_pack_object(struct object *object,struct packed_git *p,struct in_pack *in_pack)2882{2883 in_pack->array[in_pack->nr].offset =find_pack_entry_one(object->oid.hash, p);2884 in_pack->array[in_pack->nr].object = object;2885 in_pack->nr++;2886}28872888/*2889 * Compare the objects in the offset order, in order to emulate the2890 * "git rev-list --objects" output that produced the pack originally.2891 */2892static intofscmp(const void*a_,const void*b_)2893{2894struct in_pack_object *a = (struct in_pack_object *)a_;2895struct in_pack_object *b = (struct in_pack_object *)b_;28962897if(a->offset < b->offset)2898return-1;2899else if(a->offset > b->offset)2900return1;2901else2902returnoidcmp(&a->object->oid, &b->object->oid);2903}29042905static voidadd_objects_in_unpacked_packs(void)2906{2907struct packed_git *p;2908struct in_pack in_pack;2909uint32_t i;29102911memset(&in_pack,0,sizeof(in_pack));29122913for(p =get_all_packs(the_repository); p; p = p->next) {2914struct object_id oid;2915struct object *o;29162917if(!p->pack_local || p->pack_keep || p->pack_keep_in_core)2918continue;2919if(open_pack_index(p))2920die(_("cannot open pack index"));29212922ALLOC_GROW(in_pack.array,2923 in_pack.nr + p->num_objects,2924 in_pack.alloc);29252926for(i =0; i < p->num_objects; i++) {2927nth_packed_object_oid(&oid, p, i);2928 o =lookup_unknown_object(&oid);2929if(!(o->flags & OBJECT_ADDED))2930mark_in_pack_object(o, p, &in_pack);2931 o->flags |= OBJECT_ADDED;2932}2933}29342935if(in_pack.nr) {2936QSORT(in_pack.array, in_pack.nr, ofscmp);2937for(i =0; i < in_pack.nr; i++) {2938struct object *o = in_pack.array[i].object;2939add_object_entry(&o->oid, o->type,"",0);2940}2941}2942free(in_pack.array);2943}29442945static intadd_loose_object(const struct object_id *oid,const char*path,2946void*data)2947{2948enum object_type type =oid_object_info(the_repository, oid, NULL);29492950if(type <0) {2951warning(_("loose object at%scould not be examined"), path);2952return0;2953}29542955add_object_entry(oid, type,"",0);2956return0;2957}29582959/*2960 * We actually don't even have to worry about reachability here.2961 * add_object_entry will weed out duplicates, so we just add every2962 * loose object we find.2963 */2964static voidadd_unreachable_loose_objects(void)2965{2966for_each_loose_file_in_objdir(get_object_directory(),2967 add_loose_object,2968 NULL, NULL, NULL);2969}29702971static inthas_sha1_pack_kept_or_nonlocal(const struct object_id *oid)2972{2973static struct packed_git *last_found = (void*)1;2974struct packed_git *p;29752976 p = (last_found != (void*)1) ? last_found :2977get_all_packs(the_repository);29782979while(p) {2980if((!p->pack_local || p->pack_keep ||2981 p->pack_keep_in_core) &&2982find_pack_entry_one(oid->hash, p)) {2983 last_found = p;2984return1;2985}2986if(p == last_found)2987 p =get_all_packs(the_repository);2988else2989 p = p->next;2990if(p == last_found)2991 p = p->next;2992}2993return0;2994}29952996/*2997 * Store a list of sha1s that are should not be discarded2998 * because they are either written too recently, or are2999 * reachable from another object that was.3000 *3001 * This is filled by get_object_list.3002 */3003static struct oid_array recent_objects;30043005static intloosened_object_can_be_discarded(const struct object_id *oid,3006 timestamp_t mtime)3007{3008if(!unpack_unreachable_expiration)3009return0;3010if(mtime > unpack_unreachable_expiration)3011return0;3012if(oid_array_lookup(&recent_objects, oid) >=0)3013return0;3014return1;3015}30163017static voidloosen_unused_packed_objects(void)3018{3019struct packed_git *p;3020uint32_t i;3021struct object_id oid;30223023for(p =get_all_packs(the_repository); p; p = p->next) {3024if(!p->pack_local || p->pack_keep || p->pack_keep_in_core)3025continue;30263027if(open_pack_index(p))3028die(_("cannot open pack index"));30293030for(i =0; i < p->num_objects; i++) {3031nth_packed_object_oid(&oid, p, i);3032if(!packlist_find(&to_pack, &oid) &&3033!has_sha1_pack_kept_or_nonlocal(&oid) &&3034!loosened_object_can_be_discarded(&oid, p->mtime))3035if(force_object_loose(&oid, p->mtime))3036die(_("unable to force loose object"));3037}3038}3039}30403041/*3042 * This tracks any options which pack-reuse code expects to be on, or which a3043 * reader of the pack might not understand, and which would therefore prevent3044 * blind reuse of what we have on disk.3045 */3046static intpack_options_allow_reuse(void)3047{3048return pack_to_stdout &&3049 allow_ofs_delta &&3050!ignore_packed_keep_on_disk &&3051!ignore_packed_keep_in_core &&3052(!local || !have_non_local_packs) &&3053!incremental;3054}30553056static intget_object_list_from_bitmap(struct rev_info *revs)3057{3058if(!(bitmap_git =prepare_bitmap_walk(revs)))3059return-1;30603061if(pack_options_allow_reuse() &&3062!reuse_partial_packfile_from_bitmap(3063 bitmap_git,3064&reuse_packfile,3065&reuse_packfile_objects,3066&reuse_packfile_offset)) {3067assert(reuse_packfile_objects);3068 nr_result += reuse_packfile_objects;3069display_progress(progress_state, nr_result);3070}30713072traverse_bitmap_commit_list(bitmap_git, &add_object_entry_from_bitmap);3073return0;3074}30753076static voidrecord_recent_object(struct object *obj,3077const char*name,3078void*data)3079{3080oid_array_append(&recent_objects, &obj->oid);3081}30823083static voidrecord_recent_commit(struct commit *commit,void*data)3084{3085oid_array_append(&recent_objects, &commit->object.oid);3086}30873088static voidget_object_list(int ac,const char**av)3089{3090struct rev_info revs;3091struct setup_revision_opt s_r_opt = {3092.allow_exclude_promisor_objects =1,3093};3094char line[1000];3095int flags =0;3096int save_warning;30973098repo_init_revisions(the_repository, &revs, NULL);3099 save_commit_buffer =0;3100setup_revisions(ac, av, &revs, &s_r_opt);31013102/* make sure shallows are read */3103is_repository_shallow(the_repository);31043105 save_warning = warn_on_object_refname_ambiguity;3106 warn_on_object_refname_ambiguity =0;31073108while(fgets(line,sizeof(line), stdin) != NULL) {3109int len =strlen(line);3110if(len && line[len -1] =='\n')3111 line[--len] =0;3112if(!len)3113break;3114if(*line =='-') {3115if(!strcmp(line,"--not")) {3116 flags ^= UNINTERESTING;3117 write_bitmap_index =0;3118continue;3119}3120if(starts_with(line,"--shallow ")) {3121struct object_id oid;3122if(get_oid_hex(line +10, &oid))3123die("not an SHA-1 '%s'", line +10);3124register_shallow(the_repository, &oid);3125 use_bitmap_index =0;3126continue;3127}3128die(_("not a rev '%s'"), line);3129}3130if(handle_revision_arg(line, &revs, flags, REVARG_CANNOT_BE_FILENAME))3131die(_("bad revision '%s'"), line);3132}31333134 warn_on_object_refname_ambiguity = save_warning;31353136if(use_bitmap_index && !get_object_list_from_bitmap(&revs))3137return;31383139if(use_delta_islands)3140load_delta_islands(the_repository, progress);31413142if(prepare_revision_walk(&revs))3143die(_("revision walk setup failed"));3144mark_edges_uninteresting(&revs, show_edge, sparse);31453146if(!fn_show_object)3147 fn_show_object = show_object;3148traverse_commit_list_filtered(&filter_options, &revs,3149 show_commit, fn_show_object, NULL,3150 NULL);31513152if(unpack_unreachable_expiration) {3153 revs.ignore_missing_links =1;3154if(add_unseen_recent_objects_to_traversal(&revs,3155 unpack_unreachable_expiration))3156die(_("unable to add recent objects"));3157if(prepare_revision_walk(&revs))3158die(_("revision walk setup failed"));3159traverse_commit_list(&revs, record_recent_commit,3160 record_recent_object, NULL);3161}31623163if(keep_unreachable)3164add_objects_in_unpacked_packs();3165if(pack_loose_unreachable)3166add_unreachable_loose_objects();3167if(unpack_unreachable)3168loosen_unused_packed_objects();31693170oid_array_clear(&recent_objects);3171}31723173static voidadd_extra_kept_packs(const struct string_list *names)3174{3175struct packed_git *p;31763177if(!names->nr)3178return;31793180for(p =get_all_packs(the_repository); p; p = p->next) {3181const char*name =basename(p->pack_name);3182int i;31833184if(!p->pack_local)3185continue;31863187for(i =0; i < names->nr; i++)3188if(!fspathcmp(name, names->items[i].string))3189break;31903191if(i < names->nr) {3192 p->pack_keep_in_core =1;3193 ignore_packed_keep_in_core =1;3194continue;3195}3196}3197}31983199static intoption_parse_index_version(const struct option *opt,3200const char*arg,int unset)3201{3202char*c;3203const char*val = arg;32043205BUG_ON_OPT_NEG(unset);32063207 pack_idx_opts.version =strtoul(val, &c,10);3208if(pack_idx_opts.version >2)3209die(_("unsupported index version%s"), val);3210if(*c ==','&& c[1])3211 pack_idx_opts.off32_limit =strtoul(c+1, &c,0);3212if(*c || pack_idx_opts.off32_limit &0x80000000)3213die(_("bad index version '%s'"), val);3214return0;3215}32163217static intoption_parse_unpack_unreachable(const struct option *opt,3218const char*arg,int unset)3219{3220if(unset) {3221 unpack_unreachable =0;3222 unpack_unreachable_expiration =0;3223}3224else{3225 unpack_unreachable =1;3226if(arg)3227 unpack_unreachable_expiration =approxidate(arg);3228}3229return0;3230}32313232intcmd_pack_objects(int argc,const char**argv,const char*prefix)3233{3234int use_internal_rev_list =0;3235int shallow =0;3236int all_progress_implied =0;3237struct argv_array rp = ARGV_ARRAY_INIT;3238int rev_list_unpacked =0, rev_list_all =0, rev_list_reflog =0;3239int rev_list_index =0;3240struct string_list keep_pack_list = STRING_LIST_INIT_NODUP;3241struct option pack_objects_options[] = {3242OPT_SET_INT('q',"quiet", &progress,3243N_("do not show progress meter"),0),3244OPT_SET_INT(0,"progress", &progress,3245N_("show progress meter"),1),3246OPT_SET_INT(0,"all-progress", &progress,3247N_("show progress meter during object writing phase"),2),3248OPT_BOOL(0,"all-progress-implied",3249&all_progress_implied,3250N_("similar to --all-progress when progress meter is shown")),3251{ OPTION_CALLBACK,0,"index-version", NULL,N_("<version>[,<offset>]"),3252N_("write the pack index file in the specified idx format version"),3253 PARSE_OPT_NONEG, option_parse_index_version },3254OPT_MAGNITUDE(0,"max-pack-size", &pack_size_limit,3255N_("maximum size of each output pack file")),3256OPT_BOOL(0,"local", &local,3257N_("ignore borrowed objects from alternate object store")),3258OPT_BOOL(0,"incremental", &incremental,3259N_("ignore packed objects")),3260OPT_INTEGER(0,"window", &window,3261N_("limit pack window by objects")),3262OPT_MAGNITUDE(0,"window-memory", &window_memory_limit,3263N_("limit pack window by memory in addition to object limit")),3264OPT_INTEGER(0,"depth", &depth,3265N_("maximum length of delta chain allowed in the resulting pack")),3266OPT_BOOL(0,"reuse-delta", &reuse_delta,3267N_("reuse existing deltas")),3268OPT_BOOL(0,"reuse-object", &reuse_object,3269N_("reuse existing objects")),3270OPT_BOOL(0,"delta-base-offset", &allow_ofs_delta,3271N_("use OFS_DELTA objects")),3272OPT_INTEGER(0,"threads", &delta_search_threads,3273N_("use threads when searching for best delta matches")),3274OPT_BOOL(0,"non-empty", &non_empty,3275N_("do not create an empty pack output")),3276OPT_BOOL(0,"revs", &use_internal_rev_list,3277N_("read revision arguments from standard input")),3278OPT_SET_INT_F(0,"unpacked", &rev_list_unpacked,3279N_("limit the objects to those that are not yet packed"),32801, PARSE_OPT_NONEG),3281OPT_SET_INT_F(0,"all", &rev_list_all,3282N_("include objects reachable from any reference"),32831, PARSE_OPT_NONEG),3284OPT_SET_INT_F(0,"reflog", &rev_list_reflog,3285N_("include objects referred by reflog entries"),32861, PARSE_OPT_NONEG),3287OPT_SET_INT_F(0,"indexed-objects", &rev_list_index,3288N_("include objects referred to by the index"),32891, PARSE_OPT_NONEG),3290OPT_BOOL(0,"stdout", &pack_to_stdout,3291N_("output pack to stdout")),3292OPT_BOOL(0,"include-tag", &include_tag,3293N_("include tag objects that refer to objects to be packed")),3294OPT_BOOL(0,"keep-unreachable", &keep_unreachable,3295N_("keep unreachable objects")),3296OPT_BOOL(0,"pack-loose-unreachable", &pack_loose_unreachable,3297N_("pack loose unreachable objects")),3298{ OPTION_CALLBACK,0,"unpack-unreachable", NULL,N_("time"),3299N_("unpack unreachable objects newer than <time>"),3300 PARSE_OPT_OPTARG, option_parse_unpack_unreachable },3301OPT_BOOL(0,"sparse", &sparse,3302N_("use the sparse reachability algorithm")),3303OPT_BOOL(0,"thin", &thin,3304N_("create thin packs")),3305OPT_BOOL(0,"shallow", &shallow,3306N_("create packs suitable for shallow fetches")),3307OPT_BOOL(0,"honor-pack-keep", &ignore_packed_keep_on_disk,3308N_("ignore packs that have companion .keep file")),3309OPT_STRING_LIST(0,"keep-pack", &keep_pack_list,N_("name"),3310N_("ignore this pack")),3311OPT_INTEGER(0,"compression", &pack_compression_level,3312N_("pack compression level")),3313OPT_SET_INT(0,"keep-true-parents", &grafts_replace_parents,3314N_("do not hide commits by grafts"),0),3315OPT_BOOL(0,"use-bitmap-index", &use_bitmap_index,3316N_("use a bitmap index if available to speed up counting objects")),3317OPT_SET_INT(0,"write-bitmap-index", &write_bitmap_index,3318N_("write a bitmap index together with the pack index"),3319 WRITE_BITMAP_TRUE),3320OPT_SET_INT_F(0,"write-bitmap-index-quiet",3321&write_bitmap_index,3322N_("write a bitmap index if possible"),3323 WRITE_BITMAP_QUIET, PARSE_OPT_HIDDEN),3324OPT_PARSE_LIST_OBJECTS_FILTER(&filter_options),3325{ OPTION_CALLBACK,0,"missing", NULL,N_("action"),3326N_("handling for missing objects"), PARSE_OPT_NONEG,3327 option_parse_missing_action },3328OPT_BOOL(0,"exclude-promisor-objects", &exclude_promisor_objects,3329N_("do not pack objects in promisor packfiles")),3330OPT_BOOL(0,"delta-islands", &use_delta_islands,3331N_("respect islands during delta compression")),3332OPT_END(),3333};33343335if(DFS_NUM_STATES > (1<< OE_DFS_STATE_BITS))3336BUG("too many dfs states, increase OE_DFS_STATE_BITS");33373338 read_replace_refs =0;33393340 sparse =git_env_bool("GIT_TEST_PACK_SPARSE",0);3341reset_pack_idx_option(&pack_idx_opts);3342git_config(git_pack_config, NULL);33433344 progress =isatty(2);3345 argc =parse_options(argc, argv, prefix, pack_objects_options,3346 pack_usage,0);33473348if(argc) {3349 base_name = argv[0];3350 argc--;3351}3352if(pack_to_stdout != !base_name || argc)3353usage_with_options(pack_usage, pack_objects_options);33543355if(depth >= (1<< OE_DEPTH_BITS)) {3356warning(_("delta chain depth%dis too deep, forcing%d"),3357 depth, (1<< OE_DEPTH_BITS) -1);3358 depth = (1<< OE_DEPTH_BITS) -1;3359}3360if(cache_max_small_delta_size >= (1U<< OE_Z_DELTA_BITS)) {3361warning(_("pack.deltaCacheLimit is too high, forcing%d"),3362(1U<< OE_Z_DELTA_BITS) -1);3363 cache_max_small_delta_size = (1U<< OE_Z_DELTA_BITS) -1;3364}33653366argv_array_push(&rp,"pack-objects");3367if(thin) {3368 use_internal_rev_list =1;3369argv_array_push(&rp, shallow3370?"--objects-edge-aggressive"3371:"--objects-edge");3372}else3373argv_array_push(&rp,"--objects");33743375if(rev_list_all) {3376 use_internal_rev_list =1;3377argv_array_push(&rp,"--all");3378}3379if(rev_list_reflog) {3380 use_internal_rev_list =1;3381argv_array_push(&rp,"--reflog");3382}3383if(rev_list_index) {3384 use_internal_rev_list =1;3385argv_array_push(&rp,"--indexed-objects");3386}3387if(rev_list_unpacked) {3388 use_internal_rev_list =1;3389argv_array_push(&rp,"--unpacked");3390}33913392if(exclude_promisor_objects) {3393 use_internal_rev_list =1;3394 fetch_if_missing =0;3395argv_array_push(&rp,"--exclude-promisor-objects");3396}3397if(unpack_unreachable || keep_unreachable || pack_loose_unreachable)3398 use_internal_rev_list =1;33993400if(!reuse_object)3401 reuse_delta =0;3402if(pack_compression_level == -1)3403 pack_compression_level = Z_DEFAULT_COMPRESSION;3404else if(pack_compression_level <0|| pack_compression_level > Z_BEST_COMPRESSION)3405die(_("bad pack compression level%d"), pack_compression_level);34063407if(!delta_search_threads)/* --threads=0 means autodetect */3408 delta_search_threads =online_cpus();34093410if(!HAVE_THREADS && delta_search_threads !=1)3411warning(_("no threads support, ignoring --threads"));3412if(!pack_to_stdout && !pack_size_limit)3413 pack_size_limit = pack_size_limit_cfg;3414if(pack_to_stdout && pack_size_limit)3415die(_("--max-pack-size cannot be used to build a pack for transfer"));3416if(pack_size_limit && pack_size_limit <1024*1024) {3417warning(_("minimum pack size limit is 1 MiB"));3418 pack_size_limit =1024*1024;3419}34203421if(!pack_to_stdout && thin)3422die(_("--thin cannot be used to build an indexable pack"));34233424if(keep_unreachable && unpack_unreachable)3425die(_("--keep-unreachable and --unpack-unreachable are incompatible"));3426if(!rev_list_all || !rev_list_reflog || !rev_list_index)3427 unpack_unreachable_expiration =0;34283429if(filter_options.choice) {3430if(!pack_to_stdout)3431die(_("cannot use --filter without --stdout"));3432 use_bitmap_index =0;3433}34343435/*3436 * "soft" reasons not to use bitmaps - for on-disk repack by default we want3437 *3438 * - to produce good pack (with bitmap index not-yet-packed objects are3439 * packed in suboptimal order).3440 *3441 * - to use more robust pack-generation codepath (avoiding possible3442 * bugs in bitmap code and possible bitmap index corruption).3443 */3444if(!pack_to_stdout)3445 use_bitmap_index_default =0;34463447if(use_bitmap_index <0)3448 use_bitmap_index = use_bitmap_index_default;34493450/* "hard" reasons not to use bitmaps; these just won't work at all */3451if(!use_internal_rev_list || (!pack_to_stdout && write_bitmap_index) ||is_repository_shallow(the_repository))3452 use_bitmap_index =0;34533454if(pack_to_stdout || !rev_list_all)3455 write_bitmap_index =0;34563457if(use_delta_islands)3458argv_array_push(&rp,"--topo-order");34593460if(progress && all_progress_implied)3461 progress =2;34623463add_extra_kept_packs(&keep_pack_list);3464if(ignore_packed_keep_on_disk) {3465struct packed_git *p;3466for(p =get_all_packs(the_repository); p; p = p->next)3467if(p->pack_local && p->pack_keep)3468break;3469if(!p)/* no keep-able packs found */3470 ignore_packed_keep_on_disk =0;3471}3472if(local) {3473/*3474 * unlike ignore_packed_keep_on_disk above, we do not3475 * want to unset "local" based on looking at packs, as3476 * it also covers non-local objects3477 */3478struct packed_git *p;3479for(p =get_all_packs(the_repository); p; p = p->next) {3480if(!p->pack_local) {3481 have_non_local_packs =1;3482break;3483}3484}3485}34863487trace2_region_enter("pack-objects","enumerate-objects",3488 the_repository);3489prepare_packing_data(the_repository, &to_pack);34903491if(progress)3492 progress_state =start_progress(_("Enumerating objects"),0);3493if(!use_internal_rev_list)3494read_object_list_from_stdin();3495else{3496get_object_list(rp.argc, rp.argv);3497argv_array_clear(&rp);3498}3499cleanup_preferred_base();3500if(include_tag && nr_result)3501for_each_ref(add_ref_tag, NULL);3502stop_progress(&progress_state);3503trace2_region_leave("pack-objects","enumerate-objects",3504 the_repository);35053506if(non_empty && !nr_result)3507return0;3508if(nr_result) {3509trace2_region_enter("pack-objects","prepare-pack",3510 the_repository);3511prepare_pack(window, depth);3512trace2_region_leave("pack-objects","prepare-pack",3513 the_repository);3514}35153516trace2_region_enter("pack-objects","write-pack-file", the_repository);3517write_pack_file();3518trace2_region_leave("pack-objects","write-pack-file", the_repository);35193520if(progress)3521fprintf_ln(stderr,3522_("Total %"PRIu32" (delta %"PRIu32"),"3523" reused %"PRIu32" (delta %"PRIu32")"),3524 written, written_delta, reused, reused_delta);3525return0;3526}