1#include"builtin.h" 2#include"cache.h" 3#include"repository.h" 4#include"config.h" 5#include"attr.h" 6#include"object.h" 7#include"blob.h" 8#include"commit.h" 9#include"tag.h" 10#include"tree.h" 11#include"delta.h" 12#include"pack.h" 13#include"pack-revindex.h" 14#include"csum-file.h" 15#include"tree-walk.h" 16#include"diff.h" 17#include"revision.h" 18#include"list-objects.h" 19#include"list-objects-filter.h" 20#include"list-objects-filter-options.h" 21#include"pack-objects.h" 22#include"progress.h" 23#include"refs.h" 24#include"streaming.h" 25#include"thread-utils.h" 26#include"pack-bitmap.h" 27#include"delta-islands.h" 28#include"reachable.h" 29#include"sha1-array.h" 30#include"argv-array.h" 31#include"list.h" 32#include"packfile.h" 33#include"object-store.h" 34#include"dir.h" 35#include"midx.h" 36#include"trace2.h" 37 38#define IN_PACK(obj) oe_in_pack(&to_pack, obj) 39#define SIZE(obj) oe_size(&to_pack, obj) 40#define SET_SIZE(obj,size) oe_set_size(&to_pack, obj, size) 41#define DELTA_SIZE(obj) oe_delta_size(&to_pack, obj) 42#define DELTA(obj) oe_delta(&to_pack, obj) 43#define DELTA_CHILD(obj) oe_delta_child(&to_pack, obj) 44#define DELTA_SIBLING(obj) oe_delta_sibling(&to_pack, obj) 45#define SET_DELTA(obj, val) oe_set_delta(&to_pack, obj, val) 46#define SET_DELTA_EXT(obj, oid) oe_set_delta_ext(&to_pack, obj, oid) 47#define SET_DELTA_SIZE(obj, val) oe_set_delta_size(&to_pack, obj, val) 48#define SET_DELTA_CHILD(obj, val) oe_set_delta_child(&to_pack, obj, val) 49#define SET_DELTA_SIBLING(obj, val) oe_set_delta_sibling(&to_pack, obj, val) 50 51static const char*pack_usage[] = { 52N_("git pack-objects --stdout [<options>...] [< <ref-list> | < <object-list>]"), 53N_("git pack-objects [<options>...] <base-name> [< <ref-list> | < <object-list>]"), 54 NULL 55}; 56 57/* 58 * Objects we are going to pack are collected in the `to_pack` structure. 59 * It contains an array (dynamically expanded) of the object data, and a map 60 * that can resolve SHA1s to their position in the array. 61 */ 62static struct packing_data to_pack; 63 64static struct pack_idx_entry **written_list; 65static uint32_t nr_result, nr_written, nr_seen; 66static struct bitmap_index *bitmap_git; 67static uint32_t write_layer; 68 69static int non_empty; 70static int reuse_delta =1, reuse_object =1; 71static int keep_unreachable, unpack_unreachable, include_tag; 72static timestamp_t unpack_unreachable_expiration; 73static int pack_loose_unreachable; 74static int local; 75static int have_non_local_packs; 76static int incremental; 77static int ignore_packed_keep_on_disk; 78static int ignore_packed_keep_in_core; 79static int allow_ofs_delta; 80static struct pack_idx_option pack_idx_opts; 81static const char*base_name; 82static int progress =1; 83static int window =10; 84static unsigned long pack_size_limit; 85static int depth =50; 86static int delta_search_threads; 87static int pack_to_stdout; 88static int sparse; 89static int thin; 90static int num_preferred_base; 91static struct progress *progress_state; 92 93static struct packed_git *reuse_packfile; 94static uint32_t reuse_packfile_objects; 95static off_t reuse_packfile_offset; 96 97static int use_bitmap_index_default =1; 98static int use_bitmap_index = -1; 99static enum{ 100 WRITE_BITMAP_FALSE =0, 101 WRITE_BITMAP_QUIET, 102 WRITE_BITMAP_TRUE, 103} write_bitmap_index; 104static uint16_t write_bitmap_options = BITMAP_OPT_HASH_CACHE; 105 106static int exclude_promisor_objects; 107 108static int use_delta_islands; 109 110static unsigned long delta_cache_size =0; 111static unsigned long max_delta_cache_size = DEFAULT_DELTA_CACHE_SIZE; 112static unsigned long cache_max_small_delta_size =1000; 113 114static unsigned long window_memory_limit =0; 115 116static struct list_objects_filter_options filter_options; 117 118enum missing_action { 119 MA_ERROR =0,/* fail if any missing objects are encountered */ 120 MA_ALLOW_ANY,/* silently allow ALL missing objects */ 121 MA_ALLOW_PROMISOR,/* silently allow all missing PROMISOR objects */ 122}; 123static enum missing_action arg_missing_action; 124static show_object_fn fn_show_object; 125 126/* 127 * stats 128 */ 129static uint32_t written, written_delta; 130static uint32_t reused, reused_delta; 131 132/* 133 * Indexed commits 134 */ 135static struct commit **indexed_commits; 136static unsigned int indexed_commits_nr; 137static unsigned int indexed_commits_alloc; 138 139static voidindex_commit_for_bitmap(struct commit *commit) 140{ 141if(indexed_commits_nr >= indexed_commits_alloc) { 142 indexed_commits_alloc = (indexed_commits_alloc +32) *2; 143REALLOC_ARRAY(indexed_commits, indexed_commits_alloc); 144} 145 146 indexed_commits[indexed_commits_nr++] = commit; 147} 148 149static void*get_delta(struct object_entry *entry) 150{ 151unsigned long size, base_size, delta_size; 152void*buf, *base_buf, *delta_buf; 153enum object_type type; 154 155 buf =read_object_file(&entry->idx.oid, &type, &size); 156if(!buf) 157die(_("unable to read%s"),oid_to_hex(&entry->idx.oid)); 158 base_buf =read_object_file(&DELTA(entry)->idx.oid, &type, 159&base_size); 160if(!base_buf) 161die("unable to read%s", 162oid_to_hex(&DELTA(entry)->idx.oid)); 163 delta_buf =diff_delta(base_buf, base_size, 164 buf, size, &delta_size,0); 165/* 166 * We succesfully computed this delta once but dropped it for 167 * memory reasons. Something is very wrong if this time we 168 * recompute and create a different delta. 169 */ 170if(!delta_buf || delta_size !=DELTA_SIZE(entry)) 171BUG("delta size changed"); 172free(buf); 173free(base_buf); 174return delta_buf; 175} 176 177static unsigned longdo_compress(void**pptr,unsigned long size) 178{ 179 git_zstream stream; 180void*in, *out; 181unsigned long maxsize; 182 183git_deflate_init(&stream, pack_compression_level); 184 maxsize =git_deflate_bound(&stream, size); 185 186 in = *pptr; 187 out =xmalloc(maxsize); 188*pptr = out; 189 190 stream.next_in = in; 191 stream.avail_in = size; 192 stream.next_out = out; 193 stream.avail_out = maxsize; 194while(git_deflate(&stream, Z_FINISH) == Z_OK) 195;/* nothing */ 196git_deflate_end(&stream); 197 198free(in); 199return stream.total_out; 200} 201 202static unsigned longwrite_large_blob_data(struct git_istream *st,struct hashfile *f, 203const struct object_id *oid) 204{ 205 git_zstream stream; 206unsigned char ibuf[1024*16]; 207unsigned char obuf[1024*16]; 208unsigned long olen =0; 209 210git_deflate_init(&stream, pack_compression_level); 211 212for(;;) { 213 ssize_t readlen; 214int zret = Z_OK; 215 readlen =read_istream(st, ibuf,sizeof(ibuf)); 216if(readlen == -1) 217die(_("unable to read%s"),oid_to_hex(oid)); 218 219 stream.next_in = ibuf; 220 stream.avail_in = readlen; 221while((stream.avail_in || readlen ==0) && 222(zret == Z_OK || zret == Z_BUF_ERROR)) { 223 stream.next_out = obuf; 224 stream.avail_out =sizeof(obuf); 225 zret =git_deflate(&stream, readlen ?0: Z_FINISH); 226hashwrite(f, obuf, stream.next_out - obuf); 227 olen += stream.next_out - obuf; 228} 229if(stream.avail_in) 230die(_("deflate error (%d)"), zret); 231if(readlen ==0) { 232if(zret != Z_STREAM_END) 233die(_("deflate error (%d)"), zret); 234break; 235} 236} 237git_deflate_end(&stream); 238return olen; 239} 240 241/* 242 * we are going to reuse the existing object data as is. make 243 * sure it is not corrupt. 244 */ 245static intcheck_pack_inflate(struct packed_git *p, 246struct pack_window **w_curs, 247 off_t offset, 248 off_t len, 249unsigned long expect) 250{ 251 git_zstream stream; 252unsigned char fakebuf[4096], *in; 253int st; 254 255memset(&stream,0,sizeof(stream)); 256git_inflate_init(&stream); 257do{ 258 in =use_pack(p, w_curs, offset, &stream.avail_in); 259 stream.next_in = in; 260 stream.next_out = fakebuf; 261 stream.avail_out =sizeof(fakebuf); 262 st =git_inflate(&stream, Z_FINISH); 263 offset += stream.next_in - in; 264}while(st == Z_OK || st == Z_BUF_ERROR); 265git_inflate_end(&stream); 266return(st == Z_STREAM_END && 267 stream.total_out == expect && 268 stream.total_in == len) ?0: -1; 269} 270 271static voidcopy_pack_data(struct hashfile *f, 272struct packed_git *p, 273struct pack_window **w_curs, 274 off_t offset, 275 off_t len) 276{ 277unsigned char*in; 278unsigned long avail; 279 280while(len) { 281 in =use_pack(p, w_curs, offset, &avail); 282if(avail > len) 283 avail = (unsigned long)len; 284hashwrite(f, in, avail); 285 offset += avail; 286 len -= avail; 287} 288} 289 290/* Return 0 if we will bust the pack-size limit */ 291static unsigned longwrite_no_reuse_object(struct hashfile *f,struct object_entry *entry, 292unsigned long limit,int usable_delta) 293{ 294unsigned long size, datalen; 295unsigned char header[MAX_PACK_OBJECT_HEADER], 296 dheader[MAX_PACK_OBJECT_HEADER]; 297unsigned hdrlen; 298enum object_type type; 299void*buf; 300struct git_istream *st = NULL; 301const unsigned hashsz = the_hash_algo->rawsz; 302 303if(!usable_delta) { 304if(oe_type(entry) == OBJ_BLOB && 305oe_size_greater_than(&to_pack, entry, big_file_threshold) && 306(st =open_istream(&entry->idx.oid, &type, &size, NULL)) != NULL) 307 buf = NULL; 308else{ 309 buf =read_object_file(&entry->idx.oid, &type, &size); 310if(!buf) 311die(_("unable to read%s"), 312oid_to_hex(&entry->idx.oid)); 313} 314/* 315 * make sure no cached delta data remains from a 316 * previous attempt before a pack split occurred. 317 */ 318FREE_AND_NULL(entry->delta_data); 319 entry->z_delta_size =0; 320}else if(entry->delta_data) { 321 size =DELTA_SIZE(entry); 322 buf = entry->delta_data; 323 entry->delta_data = NULL; 324 type = (allow_ofs_delta &&DELTA(entry)->idx.offset) ? 325 OBJ_OFS_DELTA : OBJ_REF_DELTA; 326}else{ 327 buf =get_delta(entry); 328 size =DELTA_SIZE(entry); 329 type = (allow_ofs_delta &&DELTA(entry)->idx.offset) ? 330 OBJ_OFS_DELTA : OBJ_REF_DELTA; 331} 332 333if(st)/* large blob case, just assume we don't compress well */ 334 datalen = size; 335else if(entry->z_delta_size) 336 datalen = entry->z_delta_size; 337else 338 datalen =do_compress(&buf, size); 339 340/* 341 * The object header is a byte of 'type' followed by zero or 342 * more bytes of length. 343 */ 344 hdrlen =encode_in_pack_object_header(header,sizeof(header), 345 type, size); 346 347if(type == OBJ_OFS_DELTA) { 348/* 349 * Deltas with relative base contain an additional 350 * encoding of the relative offset for the delta 351 * base from this object's position in the pack. 352 */ 353 off_t ofs = entry->idx.offset -DELTA(entry)->idx.offset; 354unsigned pos =sizeof(dheader) -1; 355 dheader[pos] = ofs &127; 356while(ofs >>=7) 357 dheader[--pos] =128| (--ofs &127); 358if(limit && hdrlen +sizeof(dheader) - pos + datalen + hashsz >= limit) { 359if(st) 360close_istream(st); 361free(buf); 362return0; 363} 364hashwrite(f, header, hdrlen); 365hashwrite(f, dheader + pos,sizeof(dheader) - pos); 366 hdrlen +=sizeof(dheader) - pos; 367}else if(type == OBJ_REF_DELTA) { 368/* 369 * Deltas with a base reference contain 370 * additional bytes for the base object ID. 371 */ 372if(limit && hdrlen + hashsz + datalen + hashsz >= limit) { 373if(st) 374close_istream(st); 375free(buf); 376return0; 377} 378hashwrite(f, header, hdrlen); 379hashwrite(f,DELTA(entry)->idx.oid.hash, hashsz); 380 hdrlen += hashsz; 381}else{ 382if(limit && hdrlen + datalen + hashsz >= limit) { 383if(st) 384close_istream(st); 385free(buf); 386return0; 387} 388hashwrite(f, header, hdrlen); 389} 390if(st) { 391 datalen =write_large_blob_data(st, f, &entry->idx.oid); 392close_istream(st); 393}else{ 394hashwrite(f, buf, datalen); 395free(buf); 396} 397 398return hdrlen + datalen; 399} 400 401/* Return 0 if we will bust the pack-size limit */ 402static off_t write_reuse_object(struct hashfile *f,struct object_entry *entry, 403unsigned long limit,int usable_delta) 404{ 405struct packed_git *p =IN_PACK(entry); 406struct pack_window *w_curs = NULL; 407struct revindex_entry *revidx; 408 off_t offset; 409enum object_type type =oe_type(entry); 410 off_t datalen; 411unsigned char header[MAX_PACK_OBJECT_HEADER], 412 dheader[MAX_PACK_OBJECT_HEADER]; 413unsigned hdrlen; 414const unsigned hashsz = the_hash_algo->rawsz; 415unsigned long entry_size =SIZE(entry); 416 417if(DELTA(entry)) 418 type = (allow_ofs_delta &&DELTA(entry)->idx.offset) ? 419 OBJ_OFS_DELTA : OBJ_REF_DELTA; 420 hdrlen =encode_in_pack_object_header(header,sizeof(header), 421 type, entry_size); 422 423 offset = entry->in_pack_offset; 424 revidx =find_pack_revindex(p, offset); 425 datalen = revidx[1].offset - offset; 426if(!pack_to_stdout && p->index_version >1&& 427check_pack_crc(p, &w_curs, offset, datalen, revidx->nr)) { 428error(_("bad packed object CRC for%s"), 429oid_to_hex(&entry->idx.oid)); 430unuse_pack(&w_curs); 431returnwrite_no_reuse_object(f, entry, limit, usable_delta); 432} 433 434 offset += entry->in_pack_header_size; 435 datalen -= entry->in_pack_header_size; 436 437if(!pack_to_stdout && p->index_version ==1&& 438check_pack_inflate(p, &w_curs, offset, datalen, entry_size)) { 439error(_("corrupt packed object for%s"), 440oid_to_hex(&entry->idx.oid)); 441unuse_pack(&w_curs); 442returnwrite_no_reuse_object(f, entry, limit, usable_delta); 443} 444 445if(type == OBJ_OFS_DELTA) { 446 off_t ofs = entry->idx.offset -DELTA(entry)->idx.offset; 447unsigned pos =sizeof(dheader) -1; 448 dheader[pos] = ofs &127; 449while(ofs >>=7) 450 dheader[--pos] =128| (--ofs &127); 451if(limit && hdrlen +sizeof(dheader) - pos + datalen + hashsz >= limit) { 452unuse_pack(&w_curs); 453return0; 454} 455hashwrite(f, header, hdrlen); 456hashwrite(f, dheader + pos,sizeof(dheader) - pos); 457 hdrlen +=sizeof(dheader) - pos; 458 reused_delta++; 459}else if(type == OBJ_REF_DELTA) { 460if(limit && hdrlen + hashsz + datalen + hashsz >= limit) { 461unuse_pack(&w_curs); 462return0; 463} 464hashwrite(f, header, hdrlen); 465hashwrite(f,DELTA(entry)->idx.oid.hash, hashsz); 466 hdrlen += hashsz; 467 reused_delta++; 468}else{ 469if(limit && hdrlen + datalen + hashsz >= limit) { 470unuse_pack(&w_curs); 471return0; 472} 473hashwrite(f, header, hdrlen); 474} 475copy_pack_data(f, p, &w_curs, offset, datalen); 476unuse_pack(&w_curs); 477 reused++; 478return hdrlen + datalen; 479} 480 481/* Return 0 if we will bust the pack-size limit */ 482static off_t write_object(struct hashfile *f, 483struct object_entry *entry, 484 off_t write_offset) 485{ 486unsigned long limit; 487 off_t len; 488int usable_delta, to_reuse; 489 490if(!pack_to_stdout) 491crc32_begin(f); 492 493/* apply size limit if limited packsize and not first object */ 494if(!pack_size_limit || !nr_written) 495 limit =0; 496else if(pack_size_limit <= write_offset) 497/* 498 * the earlier object did not fit the limit; avoid 499 * mistaking this with unlimited (i.e. limit = 0). 500 */ 501 limit =1; 502else 503 limit = pack_size_limit - write_offset; 504 505if(!DELTA(entry)) 506 usable_delta =0;/* no delta */ 507else if(!pack_size_limit) 508 usable_delta =1;/* unlimited packfile */ 509else if(DELTA(entry)->idx.offset == (off_t)-1) 510 usable_delta =0;/* base was written to another pack */ 511else if(DELTA(entry)->idx.offset) 512 usable_delta =1;/* base already exists in this pack */ 513else 514 usable_delta =0;/* base could end up in another pack */ 515 516if(!reuse_object) 517 to_reuse =0;/* explicit */ 518else if(!IN_PACK(entry)) 519 to_reuse =0;/* can't reuse what we don't have */ 520else if(oe_type(entry) == OBJ_REF_DELTA || 521oe_type(entry) == OBJ_OFS_DELTA) 522/* check_object() decided it for us ... */ 523 to_reuse = usable_delta; 524/* ... but pack split may override that */ 525else if(oe_type(entry) != entry->in_pack_type) 526 to_reuse =0;/* pack has delta which is unusable */ 527else if(DELTA(entry)) 528 to_reuse =0;/* we want to pack afresh */ 529else 530 to_reuse =1;/* we have it in-pack undeltified, 531 * and we do not need to deltify it. 532 */ 533 534if(!to_reuse) 535 len =write_no_reuse_object(f, entry, limit, usable_delta); 536else 537 len =write_reuse_object(f, entry, limit, usable_delta); 538if(!len) 539return0; 540 541if(usable_delta) 542 written_delta++; 543 written++; 544if(!pack_to_stdout) 545 entry->idx.crc32 =crc32_end(f); 546return len; 547} 548 549enum write_one_status { 550 WRITE_ONE_SKIP = -1,/* already written */ 551 WRITE_ONE_BREAK =0,/* writing this will bust the limit; not written */ 552 WRITE_ONE_WRITTEN =1,/* normal */ 553 WRITE_ONE_RECURSIVE =2/* already scheduled to be written */ 554}; 555 556static enum write_one_status write_one(struct hashfile *f, 557struct object_entry *e, 558 off_t *offset) 559{ 560 off_t size; 561int recursing; 562 563/* 564 * we set offset to 1 (which is an impossible value) to mark 565 * the fact that this object is involved in "write its base 566 * first before writing a deltified object" recursion. 567 */ 568 recursing = (e->idx.offset ==1); 569if(recursing) { 570warning(_("recursive delta detected for object%s"), 571oid_to_hex(&e->idx.oid)); 572return WRITE_ONE_RECURSIVE; 573}else if(e->idx.offset || e->preferred_base) { 574/* offset is non zero if object is written already. */ 575return WRITE_ONE_SKIP; 576} 577 578/* if we are deltified, write out base object first. */ 579if(DELTA(e)) { 580 e->idx.offset =1;/* now recurse */ 581switch(write_one(f,DELTA(e), offset)) { 582case WRITE_ONE_RECURSIVE: 583/* we cannot depend on this one */ 584SET_DELTA(e, NULL); 585break; 586default: 587break; 588case WRITE_ONE_BREAK: 589 e->idx.offset = recursing; 590return WRITE_ONE_BREAK; 591} 592} 593 594 e->idx.offset = *offset; 595 size =write_object(f, e, *offset); 596if(!size) { 597 e->idx.offset = recursing; 598return WRITE_ONE_BREAK; 599} 600 written_list[nr_written++] = &e->idx; 601 602/* make sure off_t is sufficiently large not to wrap */ 603if(signed_add_overflows(*offset, size)) 604die(_("pack too large for current definition of off_t")); 605*offset += size; 606return WRITE_ONE_WRITTEN; 607} 608 609static intmark_tagged(const char*path,const struct object_id *oid,int flag, 610void*cb_data) 611{ 612struct object_id peeled; 613struct object_entry *entry =packlist_find(&to_pack, oid, NULL); 614 615if(entry) 616 entry->tagged =1; 617if(!peel_ref(path, &peeled)) { 618 entry =packlist_find(&to_pack, &peeled, NULL); 619if(entry) 620 entry->tagged =1; 621} 622return0; 623} 624 625staticinlinevoidadd_to_write_order(struct object_entry **wo, 626unsigned int*endp, 627struct object_entry *e) 628{ 629if(e->filled ||oe_layer(&to_pack, e) != write_layer) 630return; 631 wo[(*endp)++] = e; 632 e->filled =1; 633} 634 635static voidadd_descendants_to_write_order(struct object_entry **wo, 636unsigned int*endp, 637struct object_entry *e) 638{ 639int add_to_order =1; 640while(e) { 641if(add_to_order) { 642struct object_entry *s; 643/* add this node... */ 644add_to_write_order(wo, endp, e); 645/* all its siblings... */ 646for(s =DELTA_SIBLING(e); s; s =DELTA_SIBLING(s)) { 647add_to_write_order(wo, endp, s); 648} 649} 650/* drop down a level to add left subtree nodes if possible */ 651if(DELTA_CHILD(e)) { 652 add_to_order =1; 653 e =DELTA_CHILD(e); 654}else{ 655 add_to_order =0; 656/* our sibling might have some children, it is next */ 657if(DELTA_SIBLING(e)) { 658 e =DELTA_SIBLING(e); 659continue; 660} 661/* go back to our parent node */ 662 e =DELTA(e); 663while(e && !DELTA_SIBLING(e)) { 664/* we're on the right side of a subtree, keep 665 * going up until we can go right again */ 666 e =DELTA(e); 667} 668if(!e) { 669/* done- we hit our original root node */ 670return; 671} 672/* pass it off to sibling at this level */ 673 e =DELTA_SIBLING(e); 674} 675}; 676} 677 678static voidadd_family_to_write_order(struct object_entry **wo, 679unsigned int*endp, 680struct object_entry *e) 681{ 682struct object_entry *root; 683 684for(root = e;DELTA(root); root =DELTA(root)) 685;/* nothing */ 686add_descendants_to_write_order(wo, endp, root); 687} 688 689static voidcompute_layer_order(struct object_entry **wo,unsigned int*wo_end) 690{ 691unsigned int i, last_untagged; 692struct object_entry *objects = to_pack.objects; 693 694for(i =0; i < to_pack.nr_objects; i++) { 695if(objects[i].tagged) 696break; 697add_to_write_order(wo, wo_end, &objects[i]); 698} 699 last_untagged = i; 700 701/* 702 * Then fill all the tagged tips. 703 */ 704for(; i < to_pack.nr_objects; i++) { 705if(objects[i].tagged) 706add_to_write_order(wo, wo_end, &objects[i]); 707} 708 709/* 710 * And then all remaining commits and tags. 711 */ 712for(i = last_untagged; i < to_pack.nr_objects; i++) { 713if(oe_type(&objects[i]) != OBJ_COMMIT && 714oe_type(&objects[i]) != OBJ_TAG) 715continue; 716add_to_write_order(wo, wo_end, &objects[i]); 717} 718 719/* 720 * And then all the trees. 721 */ 722for(i = last_untagged; i < to_pack.nr_objects; i++) { 723if(oe_type(&objects[i]) != OBJ_TREE) 724continue; 725add_to_write_order(wo, wo_end, &objects[i]); 726} 727 728/* 729 * Finally all the rest in really tight order 730 */ 731for(i = last_untagged; i < to_pack.nr_objects; i++) { 732if(!objects[i].filled &&oe_layer(&to_pack, &objects[i]) == write_layer) 733add_family_to_write_order(wo, wo_end, &objects[i]); 734} 735} 736 737static struct object_entry **compute_write_order(void) 738{ 739uint32_t max_layers =1; 740unsigned int i, wo_end; 741 742struct object_entry **wo; 743struct object_entry *objects = to_pack.objects; 744 745for(i =0; i < to_pack.nr_objects; i++) { 746 objects[i].tagged =0; 747 objects[i].filled =0; 748SET_DELTA_CHILD(&objects[i], NULL); 749SET_DELTA_SIBLING(&objects[i], NULL); 750} 751 752/* 753 * Fully connect delta_child/delta_sibling network. 754 * Make sure delta_sibling is sorted in the original 755 * recency order. 756 */ 757for(i = to_pack.nr_objects; i >0;) { 758struct object_entry *e = &objects[--i]; 759if(!DELTA(e)) 760continue; 761/* Mark me as the first child */ 762 e->delta_sibling_idx =DELTA(e)->delta_child_idx; 763SET_DELTA_CHILD(DELTA(e), e); 764} 765 766/* 767 * Mark objects that are at the tip of tags. 768 */ 769for_each_tag_ref(mark_tagged, NULL); 770 771if(use_delta_islands) 772 max_layers =compute_pack_layers(&to_pack); 773 774ALLOC_ARRAY(wo, to_pack.nr_objects); 775 wo_end =0; 776 777for(; write_layer < max_layers; ++write_layer) 778compute_layer_order(wo, &wo_end); 779 780if(wo_end != to_pack.nr_objects) 781die(_("ordered%uobjects, expected %"PRIu32), 782 wo_end, to_pack.nr_objects); 783 784return wo; 785} 786 787static off_t write_reused_pack(struct hashfile *f) 788{ 789unsigned char buffer[8192]; 790 off_t to_write, total; 791int fd; 792 793if(!is_pack_valid(reuse_packfile)) 794die(_("packfile is invalid:%s"), reuse_packfile->pack_name); 795 796 fd =git_open(reuse_packfile->pack_name); 797if(fd <0) 798die_errno(_("unable to open packfile for reuse:%s"), 799 reuse_packfile->pack_name); 800 801if(lseek(fd,sizeof(struct pack_header), SEEK_SET) == -1) 802die_errno(_("unable to seek in reused packfile")); 803 804if(reuse_packfile_offset <0) 805 reuse_packfile_offset = reuse_packfile->pack_size - the_hash_algo->rawsz; 806 807 total = to_write = reuse_packfile_offset -sizeof(struct pack_header); 808 809while(to_write) { 810int read_pack =xread(fd, buffer,sizeof(buffer)); 811 812if(read_pack <=0) 813die_errno(_("unable to read from reused packfile")); 814 815if(read_pack > to_write) 816 read_pack = to_write; 817 818hashwrite(f, buffer, read_pack); 819 to_write -= read_pack; 820 821/* 822 * We don't know the actual number of objects written, 823 * only how many bytes written, how many bytes total, and 824 * how many objects total. So we can fake it by pretending all 825 * objects we are writing are the same size. This gives us a 826 * smooth progress meter, and at the end it matches the true 827 * answer. 828 */ 829 written = reuse_packfile_objects * 830(((double)(total - to_write)) / total); 831display_progress(progress_state, written); 832} 833 834close(fd); 835 written = reuse_packfile_objects; 836display_progress(progress_state, written); 837return reuse_packfile_offset -sizeof(struct pack_header); 838} 839 840static const char no_split_warning[] =N_( 841"disabling bitmap writing, packs are split due to pack.packSizeLimit" 842); 843 844static voidwrite_pack_file(void) 845{ 846uint32_t i =0, j; 847struct hashfile *f; 848 off_t offset; 849uint32_t nr_remaining = nr_result; 850time_t last_mtime =0; 851struct object_entry **write_order; 852 853if(progress > pack_to_stdout) 854 progress_state =start_progress(_("Writing objects"), nr_result); 855ALLOC_ARRAY(written_list, to_pack.nr_objects); 856 write_order =compute_write_order(); 857 858do{ 859struct object_id oid; 860char*pack_tmp_name = NULL; 861 862if(pack_to_stdout) 863 f =hashfd_throughput(1,"<stdout>", progress_state); 864else 865 f =create_tmp_packfile(&pack_tmp_name); 866 867 offset =write_pack_header(f, nr_remaining); 868 869if(reuse_packfile) { 870 off_t packfile_size; 871assert(pack_to_stdout); 872 873 packfile_size =write_reused_pack(f); 874 offset += packfile_size; 875} 876 877 nr_written =0; 878for(; i < to_pack.nr_objects; i++) { 879struct object_entry *e = write_order[i]; 880if(write_one(f, e, &offset) == WRITE_ONE_BREAK) 881break; 882display_progress(progress_state, written); 883} 884 885/* 886 * Did we write the wrong # entries in the header? 887 * If so, rewrite it like in fast-import 888 */ 889if(pack_to_stdout) { 890finalize_hashfile(f, oid.hash, CSUM_HASH_IN_STREAM | CSUM_CLOSE); 891}else if(nr_written == nr_remaining) { 892finalize_hashfile(f, oid.hash, CSUM_HASH_IN_STREAM | CSUM_FSYNC | CSUM_CLOSE); 893}else{ 894int fd =finalize_hashfile(f, oid.hash,0); 895fixup_pack_header_footer(fd, oid.hash, pack_tmp_name, 896 nr_written, oid.hash, offset); 897close(fd); 898if(write_bitmap_index) { 899if(write_bitmap_index != WRITE_BITMAP_QUIET) 900warning(_(no_split_warning)); 901 write_bitmap_index =0; 902} 903} 904 905if(!pack_to_stdout) { 906struct stat st; 907struct strbuf tmpname = STRBUF_INIT; 908 909/* 910 * Packs are runtime accessed in their mtime 911 * order since newer packs are more likely to contain 912 * younger objects. So if we are creating multiple 913 * packs then we should modify the mtime of later ones 914 * to preserve this property. 915 */ 916if(stat(pack_tmp_name, &st) <0) { 917warning_errno(_("failed to stat%s"), pack_tmp_name); 918}else if(!last_mtime) { 919 last_mtime = st.st_mtime; 920}else{ 921struct utimbuf utb; 922 utb.actime = st.st_atime; 923 utb.modtime = --last_mtime; 924if(utime(pack_tmp_name, &utb) <0) 925warning_errno(_("failed utime() on%s"), pack_tmp_name); 926} 927 928strbuf_addf(&tmpname,"%s-", base_name); 929 930if(write_bitmap_index) { 931bitmap_writer_set_checksum(oid.hash); 932bitmap_writer_build_type_index( 933&to_pack, written_list, nr_written); 934} 935 936finish_tmp_packfile(&tmpname, pack_tmp_name, 937 written_list, nr_written, 938&pack_idx_opts, oid.hash); 939 940if(write_bitmap_index) { 941strbuf_addf(&tmpname,"%s.bitmap",oid_to_hex(&oid)); 942 943stop_progress(&progress_state); 944 945bitmap_writer_show_progress(progress); 946bitmap_writer_reuse_bitmaps(&to_pack); 947bitmap_writer_select_commits(indexed_commits, indexed_commits_nr, -1); 948bitmap_writer_build(&to_pack); 949bitmap_writer_finish(written_list, nr_written, 950 tmpname.buf, write_bitmap_options); 951 write_bitmap_index =0; 952} 953 954strbuf_release(&tmpname); 955free(pack_tmp_name); 956puts(oid_to_hex(&oid)); 957} 958 959/* mark written objects as written to previous pack */ 960for(j =0; j < nr_written; j++) { 961 written_list[j]->offset = (off_t)-1; 962} 963 nr_remaining -= nr_written; 964}while(nr_remaining && i < to_pack.nr_objects); 965 966free(written_list); 967free(write_order); 968stop_progress(&progress_state); 969if(written != nr_result) 970die(_("wrote %"PRIu32" objects while expecting %"PRIu32), 971 written, nr_result); 972trace2_data_intmax("pack-objects", the_repository, 973"write_pack_file/wrote", nr_result); 974} 975 976static intno_try_delta(const char*path) 977{ 978static struct attr_check *check; 979 980if(!check) 981 check =attr_check_initl("delta", NULL); 982git_check_attr(the_repository->index, path, check); 983if(ATTR_FALSE(check->items[0].value)) 984return1; 985return0; 986} 987 988/* 989 * When adding an object, check whether we have already added it 990 * to our packing list. If so, we can skip. However, if we are 991 * being asked to excludei t, but the previous mention was to include 992 * it, make sure to adjust its flags and tweak our numbers accordingly. 993 * 994 * As an optimization, we pass out the index position where we would have 995 * found the item, since that saves us from having to look it up again a 996 * few lines later when we want to add the new entry. 997 */ 998static inthave_duplicate_entry(const struct object_id *oid, 999int exclude,1000uint32_t*index_pos)1001{1002struct object_entry *entry;10031004 entry =packlist_find(&to_pack, oid, index_pos);1005if(!entry)1006return0;10071008if(exclude) {1009if(!entry->preferred_base)1010 nr_result--;1011 entry->preferred_base =1;1012}10131014return1;1015}10161017static intwant_found_object(int exclude,struct packed_git *p)1018{1019if(exclude)1020return1;1021if(incremental)1022return0;10231024/*1025 * When asked to do --local (do not include an object that appears in a1026 * pack we borrow from elsewhere) or --honor-pack-keep (do not include1027 * an object that appears in a pack marked with .keep), finding a pack1028 * that matches the criteria is sufficient for us to decide to omit it.1029 * However, even if this pack does not satisfy the criteria, we need to1030 * make sure no copy of this object appears in _any_ pack that makes us1031 * to omit the object, so we need to check all the packs.1032 *1033 * We can however first check whether these options can possible matter;1034 * if they do not matter we know we want the object in generated pack.1035 * Otherwise, we signal "-1" at the end to tell the caller that we do1036 * not know either way, and it needs to check more packs.1037 */1038if(!ignore_packed_keep_on_disk &&1039!ignore_packed_keep_in_core &&1040(!local || !have_non_local_packs))1041return1;10421043if(local && !p->pack_local)1044return0;1045if(p->pack_local &&1046((ignore_packed_keep_on_disk && p->pack_keep) ||1047(ignore_packed_keep_in_core && p->pack_keep_in_core)))1048return0;10491050/* we don't know yet; keep looking for more packs */1051return-1;1052}10531054/*1055 * Check whether we want the object in the pack (e.g., we do not want1056 * objects found in non-local stores if the "--local" option was used).1057 *1058 * If the caller already knows an existing pack it wants to take the object1059 * from, that is passed in *found_pack and *found_offset; otherwise this1060 * function finds if there is any pack that has the object and returns the pack1061 * and its offset in these variables.1062 */1063static intwant_object_in_pack(const struct object_id *oid,1064int exclude,1065struct packed_git **found_pack,1066 off_t *found_offset)1067{1068int want;1069struct list_head *pos;1070struct multi_pack_index *m;10711072if(!exclude && local &&has_loose_object_nonlocal(oid))1073return0;10741075/*1076 * If we already know the pack object lives in, start checks from that1077 * pack - in the usual case when neither --local was given nor .keep files1078 * are present we will determine the answer right now.1079 */1080if(*found_pack) {1081 want =want_found_object(exclude, *found_pack);1082if(want != -1)1083return want;1084}10851086for(m =get_multi_pack_index(the_repository); m; m = m->next) {1087struct pack_entry e;1088if(fill_midx_entry(the_repository, oid, &e, m)) {1089struct packed_git *p = e.p;1090 off_t offset;10911092if(p == *found_pack)1093 offset = *found_offset;1094else1095 offset =find_pack_entry_one(oid->hash, p);10961097if(offset) {1098if(!*found_pack) {1099if(!is_pack_valid(p))1100continue;1101*found_offset = offset;1102*found_pack = p;1103}1104 want =want_found_object(exclude, p);1105if(want != -1)1106return want;1107}1108}1109}11101111list_for_each(pos,get_packed_git_mru(the_repository)) {1112struct packed_git *p =list_entry(pos,struct packed_git, mru);1113 off_t offset;11141115if(p == *found_pack)1116 offset = *found_offset;1117else1118 offset =find_pack_entry_one(oid->hash, p);11191120if(offset) {1121if(!*found_pack) {1122if(!is_pack_valid(p))1123continue;1124*found_offset = offset;1125*found_pack = p;1126}1127 want =want_found_object(exclude, p);1128if(!exclude && want >0)1129list_move(&p->mru,1130get_packed_git_mru(the_repository));1131if(want != -1)1132return want;1133}1134}11351136return1;1137}11381139static voidcreate_object_entry(const struct object_id *oid,1140enum object_type type,1141uint32_t hash,1142int exclude,1143int no_try_delta,1144uint32_t index_pos,1145struct packed_git *found_pack,1146 off_t found_offset)1147{1148struct object_entry *entry;11491150 entry =packlist_alloc(&to_pack, oid, index_pos);1151 entry->hash = hash;1152oe_set_type(entry, type);1153if(exclude)1154 entry->preferred_base =1;1155else1156 nr_result++;1157if(found_pack) {1158oe_set_in_pack(&to_pack, entry, found_pack);1159 entry->in_pack_offset = found_offset;1160}11611162 entry->no_try_delta = no_try_delta;1163}11641165static const char no_closure_warning[] =N_(1166"disabling bitmap writing, as some objects are not being packed"1167);11681169static intadd_object_entry(const struct object_id *oid,enum object_type type,1170const char*name,int exclude)1171{1172struct packed_git *found_pack = NULL;1173 off_t found_offset =0;1174uint32_t index_pos;11751176display_progress(progress_state, ++nr_seen);11771178if(have_duplicate_entry(oid, exclude, &index_pos))1179return0;11801181if(!want_object_in_pack(oid, exclude, &found_pack, &found_offset)) {1182/* The pack is missing an object, so it will not have closure */1183if(write_bitmap_index) {1184if(write_bitmap_index != WRITE_BITMAP_QUIET)1185warning(_(no_closure_warning));1186 write_bitmap_index =0;1187}1188return0;1189}11901191create_object_entry(oid, type,pack_name_hash(name),1192 exclude, name &&no_try_delta(name),1193 index_pos, found_pack, found_offset);1194return1;1195}11961197static intadd_object_entry_from_bitmap(const struct object_id *oid,1198enum object_type type,1199int flags,uint32_t name_hash,1200struct packed_git *pack, off_t offset)1201{1202uint32_t index_pos;12031204display_progress(progress_state, ++nr_seen);12051206if(have_duplicate_entry(oid,0, &index_pos))1207return0;12081209if(!want_object_in_pack(oid,0, &pack, &offset))1210return0;12111212create_object_entry(oid, type, name_hash,0,0, index_pos, pack, offset);1213return1;1214}12151216struct pbase_tree_cache {1217struct object_id oid;1218int ref;1219int temporary;1220void*tree_data;1221unsigned long tree_size;1222};12231224static struct pbase_tree_cache *(pbase_tree_cache[256]);1225static intpbase_tree_cache_ix(const struct object_id *oid)1226{1227return oid->hash[0] %ARRAY_SIZE(pbase_tree_cache);1228}1229static intpbase_tree_cache_ix_incr(int ix)1230{1231return(ix+1) %ARRAY_SIZE(pbase_tree_cache);1232}12331234static struct pbase_tree {1235struct pbase_tree *next;1236/* This is a phony "cache" entry; we are not1237 * going to evict it or find it through _get()1238 * mechanism -- this is for the toplevel node that1239 * would almost always change with any commit.1240 */1241struct pbase_tree_cache pcache;1242} *pbase_tree;12431244static struct pbase_tree_cache *pbase_tree_get(const struct object_id *oid)1245{1246struct pbase_tree_cache *ent, *nent;1247void*data;1248unsigned long size;1249enum object_type type;1250int neigh;1251int my_ix =pbase_tree_cache_ix(oid);1252int available_ix = -1;12531254/* pbase-tree-cache acts as a limited hashtable.1255 * your object will be found at your index or within a few1256 * slots after that slot if it is cached.1257 */1258for(neigh =0; neigh <8; neigh++) {1259 ent = pbase_tree_cache[my_ix];1260if(ent &&oideq(&ent->oid, oid)) {1261 ent->ref++;1262return ent;1263}1264else if(((available_ix <0) && (!ent || !ent->ref)) ||1265((0<= available_ix) &&1266(!ent && pbase_tree_cache[available_ix])))1267 available_ix = my_ix;1268if(!ent)1269break;1270 my_ix =pbase_tree_cache_ix_incr(my_ix);1271}12721273/* Did not find one. Either we got a bogus request or1274 * we need to read and perhaps cache.1275 */1276 data =read_object_file(oid, &type, &size);1277if(!data)1278return NULL;1279if(type != OBJ_TREE) {1280free(data);1281return NULL;1282}12831284/* We need to either cache or return a throwaway copy */12851286if(available_ix <0)1287 ent = NULL;1288else{1289 ent = pbase_tree_cache[available_ix];1290 my_ix = available_ix;1291}12921293if(!ent) {1294 nent =xmalloc(sizeof(*nent));1295 nent->temporary = (available_ix <0);1296}1297else{1298/* evict and reuse */1299free(ent->tree_data);1300 nent = ent;1301}1302oidcpy(&nent->oid, oid);1303 nent->tree_data = data;1304 nent->tree_size = size;1305 nent->ref =1;1306if(!nent->temporary)1307 pbase_tree_cache[my_ix] = nent;1308return nent;1309}13101311static voidpbase_tree_put(struct pbase_tree_cache *cache)1312{1313if(!cache->temporary) {1314 cache->ref--;1315return;1316}1317free(cache->tree_data);1318free(cache);1319}13201321static intname_cmp_len(const char*name)1322{1323int i;1324for(i =0; name[i] && name[i] !='\n'&& name[i] !='/'; i++)1325;1326return i;1327}13281329static voidadd_pbase_object(struct tree_desc *tree,1330const char*name,1331int cmplen,1332const char*fullname)1333{1334struct name_entry entry;1335int cmp;13361337while(tree_entry(tree,&entry)) {1338if(S_ISGITLINK(entry.mode))1339continue;1340 cmp =tree_entry_len(&entry) != cmplen ?1:1341memcmp(name, entry.path, cmplen);1342if(cmp >0)1343continue;1344if(cmp <0)1345return;1346if(name[cmplen] !='/') {1347add_object_entry(&entry.oid,1348object_type(entry.mode),1349 fullname,1);1350return;1351}1352if(S_ISDIR(entry.mode)) {1353struct tree_desc sub;1354struct pbase_tree_cache *tree;1355const char*down = name+cmplen+1;1356int downlen =name_cmp_len(down);13571358 tree =pbase_tree_get(&entry.oid);1359if(!tree)1360return;1361init_tree_desc(&sub, tree->tree_data, tree->tree_size);13621363add_pbase_object(&sub, down, downlen, fullname);1364pbase_tree_put(tree);1365}1366}1367}13681369static unsigned*done_pbase_paths;1370static int done_pbase_paths_num;1371static int done_pbase_paths_alloc;1372static intdone_pbase_path_pos(unsigned hash)1373{1374int lo =0;1375int hi = done_pbase_paths_num;1376while(lo < hi) {1377int mi = lo + (hi - lo) /2;1378if(done_pbase_paths[mi] == hash)1379return mi;1380if(done_pbase_paths[mi] < hash)1381 hi = mi;1382else1383 lo = mi +1;1384}1385return-lo-1;1386}13871388static intcheck_pbase_path(unsigned hash)1389{1390int pos =done_pbase_path_pos(hash);1391if(0<= pos)1392return1;1393 pos = -pos -1;1394ALLOC_GROW(done_pbase_paths,1395 done_pbase_paths_num +1,1396 done_pbase_paths_alloc);1397 done_pbase_paths_num++;1398if(pos < done_pbase_paths_num)1399MOVE_ARRAY(done_pbase_paths + pos +1, done_pbase_paths + pos,1400 done_pbase_paths_num - pos -1);1401 done_pbase_paths[pos] = hash;1402return0;1403}14041405static voidadd_preferred_base_object(const char*name)1406{1407struct pbase_tree *it;1408int cmplen;1409unsigned hash =pack_name_hash(name);14101411if(!num_preferred_base ||check_pbase_path(hash))1412return;14131414 cmplen =name_cmp_len(name);1415for(it = pbase_tree; it; it = it->next) {1416if(cmplen ==0) {1417add_object_entry(&it->pcache.oid, OBJ_TREE, NULL,1);1418}1419else{1420struct tree_desc tree;1421init_tree_desc(&tree, it->pcache.tree_data, it->pcache.tree_size);1422add_pbase_object(&tree, name, cmplen, name);1423}1424}1425}14261427static voidadd_preferred_base(struct object_id *oid)1428{1429struct pbase_tree *it;1430void*data;1431unsigned long size;1432struct object_id tree_oid;14331434if(window <= num_preferred_base++)1435return;14361437 data =read_object_with_reference(the_repository, oid,1438 tree_type, &size, &tree_oid);1439if(!data)1440return;14411442for(it = pbase_tree; it; it = it->next) {1443if(oideq(&it->pcache.oid, &tree_oid)) {1444free(data);1445return;1446}1447}14481449 it =xcalloc(1,sizeof(*it));1450 it->next = pbase_tree;1451 pbase_tree = it;14521453oidcpy(&it->pcache.oid, &tree_oid);1454 it->pcache.tree_data = data;1455 it->pcache.tree_size = size;1456}14571458static voidcleanup_preferred_base(void)1459{1460struct pbase_tree *it;1461unsigned i;14621463 it = pbase_tree;1464 pbase_tree = NULL;1465while(it) {1466struct pbase_tree *tmp = it;1467 it = tmp->next;1468free(tmp->pcache.tree_data);1469free(tmp);1470}14711472for(i =0; i <ARRAY_SIZE(pbase_tree_cache); i++) {1473if(!pbase_tree_cache[i])1474continue;1475free(pbase_tree_cache[i]->tree_data);1476FREE_AND_NULL(pbase_tree_cache[i]);1477}14781479FREE_AND_NULL(done_pbase_paths);1480 done_pbase_paths_num = done_pbase_paths_alloc =0;1481}14821483/*1484 * Return 1 iff the object specified by "delta" can be sent1485 * literally as a delta against the base in "base_sha1". If1486 * so, then *base_out will point to the entry in our packing1487 * list, or NULL if we must use the external-base list.1488 *1489 * Depth value does not matter - find_deltas() will1490 * never consider reused delta as the base object to1491 * deltify other objects against, in order to avoid1492 * circular deltas.1493 */1494static intcan_reuse_delta(const unsigned char*base_sha1,1495struct object_entry *delta,1496struct object_entry **base_out)1497{1498struct object_entry *base;1499struct object_id base_oid;15001501if(!base_sha1)1502return0;15031504oidread(&base_oid, base_sha1);15051506/*1507 * First see if we're already sending the base (or it's explicitly in1508 * our "excluded" list).1509 */1510 base =packlist_find(&to_pack, &base_oid, NULL);1511if(base) {1512if(!in_same_island(&delta->idx.oid, &base->idx.oid))1513return0;1514*base_out = base;1515return1;1516}15171518/*1519 * Otherwise, reachability bitmaps may tell us if the receiver has it,1520 * even if it was buried too deep in history to make it into the1521 * packing list.1522 */1523if(thin &&bitmap_has_oid_in_uninteresting(bitmap_git, &base_oid)) {1524if(use_delta_islands) {1525if(!in_same_island(&delta->idx.oid, &base_oid))1526return0;1527}1528*base_out = NULL;1529return1;1530}15311532return0;1533}15341535static voidcheck_object(struct object_entry *entry)1536{1537unsigned long canonical_size;15381539if(IN_PACK(entry)) {1540struct packed_git *p =IN_PACK(entry);1541struct pack_window *w_curs = NULL;1542const unsigned char*base_ref = NULL;1543struct object_entry *base_entry;1544unsigned long used, used_0;1545unsigned long avail;1546 off_t ofs;1547unsigned char*buf, c;1548enum object_type type;1549unsigned long in_pack_size;15501551 buf =use_pack(p, &w_curs, entry->in_pack_offset, &avail);15521553/*1554 * We want in_pack_type even if we do not reuse delta1555 * since non-delta representations could still be reused.1556 */1557 used =unpack_object_header_buffer(buf, avail,1558&type,1559&in_pack_size);1560if(used ==0)1561goto give_up;15621563if(type <0)1564BUG("invalid type%d", type);1565 entry->in_pack_type = type;15661567/*1568 * Determine if this is a delta and if so whether we can1569 * reuse it or not. Otherwise let's find out as cheaply as1570 * possible what the actual type and size for this object is.1571 */1572switch(entry->in_pack_type) {1573default:1574/* Not a delta hence we've already got all we need. */1575oe_set_type(entry, entry->in_pack_type);1576SET_SIZE(entry, in_pack_size);1577 entry->in_pack_header_size = used;1578if(oe_type(entry) < OBJ_COMMIT ||oe_type(entry) > OBJ_BLOB)1579goto give_up;1580unuse_pack(&w_curs);1581return;1582case OBJ_REF_DELTA:1583if(reuse_delta && !entry->preferred_base)1584 base_ref =use_pack(p, &w_curs,1585 entry->in_pack_offset + used, NULL);1586 entry->in_pack_header_size = used + the_hash_algo->rawsz;1587break;1588case OBJ_OFS_DELTA:1589 buf =use_pack(p, &w_curs,1590 entry->in_pack_offset + used, NULL);1591 used_0 =0;1592 c = buf[used_0++];1593 ofs = c &127;1594while(c &128) {1595 ofs +=1;1596if(!ofs ||MSB(ofs,7)) {1597error(_("delta base offset overflow in pack for%s"),1598oid_to_hex(&entry->idx.oid));1599goto give_up;1600}1601 c = buf[used_0++];1602 ofs = (ofs <<7) + (c &127);1603}1604 ofs = entry->in_pack_offset - ofs;1605if(ofs <=0|| ofs >= entry->in_pack_offset) {1606error(_("delta base offset out of bound for%s"),1607oid_to_hex(&entry->idx.oid));1608goto give_up;1609}1610if(reuse_delta && !entry->preferred_base) {1611struct revindex_entry *revidx;1612 revidx =find_pack_revindex(p, ofs);1613if(!revidx)1614goto give_up;1615 base_ref =nth_packed_object_sha1(p, revidx->nr);1616}1617 entry->in_pack_header_size = used + used_0;1618break;1619}16201621if(can_reuse_delta(base_ref, entry, &base_entry)) {1622oe_set_type(entry, entry->in_pack_type);1623SET_SIZE(entry, in_pack_size);/* delta size */1624SET_DELTA_SIZE(entry, in_pack_size);16251626if(base_entry) {1627SET_DELTA(entry, base_entry);1628 entry->delta_sibling_idx = base_entry->delta_child_idx;1629SET_DELTA_CHILD(base_entry, entry);1630}else{1631SET_DELTA_EXT(entry, base_ref);1632}16331634unuse_pack(&w_curs);1635return;1636}16371638if(oe_type(entry)) {1639 off_t delta_pos;16401641/*1642 * This must be a delta and we already know what the1643 * final object type is. Let's extract the actual1644 * object size from the delta header.1645 */1646 delta_pos = entry->in_pack_offset + entry->in_pack_header_size;1647 canonical_size =get_size_from_delta(p, &w_curs, delta_pos);1648if(canonical_size ==0)1649goto give_up;1650SET_SIZE(entry, canonical_size);1651unuse_pack(&w_curs);1652return;1653}16541655/*1656 * No choice but to fall back to the recursive delta walk1657 * with oid_object_info() to find about the object type1658 * at this point...1659 */1660 give_up:1661unuse_pack(&w_curs);1662}16631664oe_set_type(entry,1665oid_object_info(the_repository, &entry->idx.oid, &canonical_size));1666if(entry->type_valid) {1667SET_SIZE(entry, canonical_size);1668}else{1669/*1670 * Bad object type is checked in prepare_pack(). This is1671 * to permit a missing preferred base object to be ignored1672 * as a preferred base. Doing so can result in a larger1673 * pack file, but the transfer will still take place.1674 */1675}1676}16771678static intpack_offset_sort(const void*_a,const void*_b)1679{1680const struct object_entry *a = *(struct object_entry **)_a;1681const struct object_entry *b = *(struct object_entry **)_b;1682const struct packed_git *a_in_pack =IN_PACK(a);1683const struct packed_git *b_in_pack =IN_PACK(b);16841685/* avoid filesystem trashing with loose objects */1686if(!a_in_pack && !b_in_pack)1687returnoidcmp(&a->idx.oid, &b->idx.oid);16881689if(a_in_pack < b_in_pack)1690return-1;1691if(a_in_pack > b_in_pack)1692return1;1693return a->in_pack_offset < b->in_pack_offset ? -1:1694(a->in_pack_offset > b->in_pack_offset);1695}16961697/*1698 * Drop an on-disk delta we were planning to reuse. Naively, this would1699 * just involve blanking out the "delta" field, but we have to deal1700 * with some extra book-keeping:1701 *1702 * 1. Removing ourselves from the delta_sibling linked list.1703 *1704 * 2. Updating our size/type to the non-delta representation. These were1705 * either not recorded initially (size) or overwritten with the delta type1706 * (type) when check_object() decided to reuse the delta.1707 *1708 * 3. Resetting our delta depth, as we are now a base object.1709 */1710static voiddrop_reused_delta(struct object_entry *entry)1711{1712unsigned*idx = &to_pack.objects[entry->delta_idx -1].delta_child_idx;1713struct object_info oi = OBJECT_INFO_INIT;1714enum object_type type;1715unsigned long size;17161717while(*idx) {1718struct object_entry *oe = &to_pack.objects[*idx -1];17191720if(oe == entry)1721*idx = oe->delta_sibling_idx;1722else1723 idx = &oe->delta_sibling_idx;1724}1725SET_DELTA(entry, NULL);1726 entry->depth =0;17271728 oi.sizep = &size;1729 oi.typep = &type;1730if(packed_object_info(the_repository,IN_PACK(entry), entry->in_pack_offset, &oi) <0) {1731/*1732 * We failed to get the info from this pack for some reason;1733 * fall back to oid_object_info, which may find another copy.1734 * And if that fails, the error will be recorded in oe_type(entry)1735 * and dealt with in prepare_pack().1736 */1737oe_set_type(entry,1738oid_object_info(the_repository, &entry->idx.oid, &size));1739}else{1740oe_set_type(entry, type);1741}1742SET_SIZE(entry, size);1743}17441745/*1746 * Follow the chain of deltas from this entry onward, throwing away any links1747 * that cause us to hit a cycle (as determined by the DFS state flags in1748 * the entries).1749 *1750 * We also detect too-long reused chains that would violate our --depth1751 * limit.1752 */1753static voidbreak_delta_chains(struct object_entry *entry)1754{1755/*1756 * The actual depth of each object we will write is stored as an int,1757 * as it cannot exceed our int "depth" limit. But before we break1758 * changes based no that limit, we may potentially go as deep as the1759 * number of objects, which is elsewhere bounded to a uint32_t.1760 */1761uint32_t total_depth;1762struct object_entry *cur, *next;17631764for(cur = entry, total_depth =0;1765 cur;1766 cur =DELTA(cur), total_depth++) {1767if(cur->dfs_state == DFS_DONE) {1768/*1769 * We've already seen this object and know it isn't1770 * part of a cycle. We do need to append its depth1771 * to our count.1772 */1773 total_depth += cur->depth;1774break;1775}17761777/*1778 * We break cycles before looping, so an ACTIVE state (or any1779 * other cruft which made its way into the state variable)1780 * is a bug.1781 */1782if(cur->dfs_state != DFS_NONE)1783BUG("confusing delta dfs state in first pass:%d",1784 cur->dfs_state);17851786/*1787 * Now we know this is the first time we've seen the object. If1788 * it's not a delta, we're done traversing, but we'll mark it1789 * done to save time on future traversals.1790 */1791if(!DELTA(cur)) {1792 cur->dfs_state = DFS_DONE;1793break;1794}17951796/*1797 * Mark ourselves as active and see if the next step causes1798 * us to cycle to another active object. It's important to do1799 * this _before_ we loop, because it impacts where we make the1800 * cut, and thus how our total_depth counter works.1801 * E.g., We may see a partial loop like:1802 *1803 * A -> B -> C -> D -> B1804 *1805 * Cutting B->C breaks the cycle. But now the depth of A is1806 * only 1, and our total_depth counter is at 3. The size of the1807 * error is always one less than the size of the cycle we1808 * broke. Commits C and D were "lost" from A's chain.1809 *1810 * If we instead cut D->B, then the depth of A is correct at 3.1811 * We keep all commits in the chain that we examined.1812 */1813 cur->dfs_state = DFS_ACTIVE;1814if(DELTA(cur)->dfs_state == DFS_ACTIVE) {1815drop_reused_delta(cur);1816 cur->dfs_state = DFS_DONE;1817break;1818}1819}18201821/*1822 * And now that we've gone all the way to the bottom of the chain, we1823 * need to clear the active flags and set the depth fields as1824 * appropriate. Unlike the loop above, which can quit when it drops a1825 * delta, we need to keep going to look for more depth cuts. So we need1826 * an extra "next" pointer to keep going after we reset cur->delta.1827 */1828for(cur = entry; cur; cur = next) {1829 next =DELTA(cur);18301831/*1832 * We should have a chain of zero or more ACTIVE states down to1833 * a final DONE. We can quit after the DONE, because either it1834 * has no bases, or we've already handled them in a previous1835 * call.1836 */1837if(cur->dfs_state == DFS_DONE)1838break;1839else if(cur->dfs_state != DFS_ACTIVE)1840BUG("confusing delta dfs state in second pass:%d",1841 cur->dfs_state);18421843/*1844 * If the total_depth is more than depth, then we need to snip1845 * the chain into two or more smaller chains that don't exceed1846 * the maximum depth. Most of the resulting chains will contain1847 * (depth + 1) entries (i.e., depth deltas plus one base), and1848 * the last chain (i.e., the one containing entry) will contain1849 * whatever entries are left over, namely1850 * (total_depth % (depth + 1)) of them.1851 *1852 * Since we are iterating towards decreasing depth, we need to1853 * decrement total_depth as we go, and we need to write to the1854 * entry what its final depth will be after all of the1855 * snipping. Since we're snipping into chains of length (depth1856 * + 1) entries, the final depth of an entry will be its1857 * original depth modulo (depth + 1). Any time we encounter an1858 * entry whose final depth is supposed to be zero, we snip it1859 * from its delta base, thereby making it so.1860 */1861 cur->depth = (total_depth--) % (depth +1);1862if(!cur->depth)1863drop_reused_delta(cur);18641865 cur->dfs_state = DFS_DONE;1866}1867}18681869static voidget_object_details(void)1870{1871uint32_t i;1872struct object_entry **sorted_by_offset;18731874if(progress)1875 progress_state =start_progress(_("Counting objects"),1876 to_pack.nr_objects);18771878 sorted_by_offset =xcalloc(to_pack.nr_objects,sizeof(struct object_entry *));1879for(i =0; i < to_pack.nr_objects; i++)1880 sorted_by_offset[i] = to_pack.objects + i;1881QSORT(sorted_by_offset, to_pack.nr_objects, pack_offset_sort);18821883for(i =0; i < to_pack.nr_objects; i++) {1884struct object_entry *entry = sorted_by_offset[i];1885check_object(entry);1886if(entry->type_valid &&1887oe_size_greater_than(&to_pack, entry, big_file_threshold))1888 entry->no_try_delta =1;1889display_progress(progress_state, i +1);1890}1891stop_progress(&progress_state);18921893/*1894 * This must happen in a second pass, since we rely on the delta1895 * information for the whole list being completed.1896 */1897for(i =0; i < to_pack.nr_objects; i++)1898break_delta_chains(&to_pack.objects[i]);18991900free(sorted_by_offset);1901}19021903/*1904 * We search for deltas in a list sorted by type, by filename hash, and then1905 * by size, so that we see progressively smaller and smaller files.1906 * That's because we prefer deltas to be from the bigger file1907 * to the smaller -- deletes are potentially cheaper, but perhaps1908 * more importantly, the bigger file is likely the more recent1909 * one. The deepest deltas are therefore the oldest objects which are1910 * less susceptible to be accessed often.1911 */1912static inttype_size_sort(const void*_a,const void*_b)1913{1914const struct object_entry *a = *(struct object_entry **)_a;1915const struct object_entry *b = *(struct object_entry **)_b;1916const enum object_type a_type =oe_type(a);1917const enum object_type b_type =oe_type(b);1918const unsigned long a_size =SIZE(a);1919const unsigned long b_size =SIZE(b);19201921if(a_type > b_type)1922return-1;1923if(a_type < b_type)1924return1;1925if(a->hash > b->hash)1926return-1;1927if(a->hash < b->hash)1928return1;1929if(a->preferred_base > b->preferred_base)1930return-1;1931if(a->preferred_base < b->preferred_base)1932return1;1933if(use_delta_islands) {1934const int island_cmp =island_delta_cmp(&a->idx.oid, &b->idx.oid);1935if(island_cmp)1936return island_cmp;1937}1938if(a_size > b_size)1939return-1;1940if(a_size < b_size)1941return1;1942return a < b ? -1: (a > b);/* newest first */1943}19441945struct unpacked {1946struct object_entry *entry;1947void*data;1948struct delta_index *index;1949unsigned depth;1950};19511952static intdelta_cacheable(unsigned long src_size,unsigned long trg_size,1953unsigned long delta_size)1954{1955if(max_delta_cache_size && delta_cache_size + delta_size > max_delta_cache_size)1956return0;19571958if(delta_size < cache_max_small_delta_size)1959return1;19601961/* cache delta, if objects are large enough compared to delta size */1962if((src_size >>20) + (trg_size >>21) > (delta_size >>10))1963return1;19641965return0;1966}19671968/* Protect delta_cache_size */1969static pthread_mutex_t cache_mutex;1970#define cache_lock() pthread_mutex_lock(&cache_mutex)1971#define cache_unlock() pthread_mutex_unlock(&cache_mutex)19721973/*1974 * Protect object list partitioning (e.g. struct thread_param) and1975 * progress_state1976 */1977static pthread_mutex_t progress_mutex;1978#define progress_lock() pthread_mutex_lock(&progress_mutex)1979#define progress_unlock() pthread_mutex_unlock(&progress_mutex)19801981/*1982 * Access to struct object_entry is unprotected since each thread owns1983 * a portion of the main object list. Just don't access object entries1984 * ahead in the list because they can be stolen and would need1985 * progress_mutex for protection.1986 */19871988/*1989 * Return the size of the object without doing any delta1990 * reconstruction (so non-deltas are true object sizes, but deltas1991 * return the size of the delta data).1992 */1993unsigned longoe_get_size_slow(struct packing_data *pack,1994const struct object_entry *e)1995{1996struct packed_git *p;1997struct pack_window *w_curs;1998unsigned char*buf;1999enum object_type type;2000unsigned long used, avail, size;20012002if(e->type_ != OBJ_OFS_DELTA && e->type_ != OBJ_REF_DELTA) {2003packing_data_lock(&to_pack);2004if(oid_object_info(the_repository, &e->idx.oid, &size) <0)2005die(_("unable to get size of%s"),2006oid_to_hex(&e->idx.oid));2007packing_data_unlock(&to_pack);2008return size;2009}20102011 p =oe_in_pack(pack, e);2012if(!p)2013BUG("when e->type is a delta, it must belong to a pack");20142015packing_data_lock(&to_pack);2016 w_curs = NULL;2017 buf =use_pack(p, &w_curs, e->in_pack_offset, &avail);2018 used =unpack_object_header_buffer(buf, avail, &type, &size);2019if(used ==0)2020die(_("unable to parse object header of%s"),2021oid_to_hex(&e->idx.oid));20222023unuse_pack(&w_curs);2024packing_data_unlock(&to_pack);2025return size;2026}20272028static inttry_delta(struct unpacked *trg,struct unpacked *src,2029unsigned max_depth,unsigned long*mem_usage)2030{2031struct object_entry *trg_entry = trg->entry;2032struct object_entry *src_entry = src->entry;2033unsigned long trg_size, src_size, delta_size, sizediff, max_size, sz;2034unsigned ref_depth;2035enum object_type type;2036void*delta_buf;20372038/* Don't bother doing diffs between different types */2039if(oe_type(trg_entry) !=oe_type(src_entry))2040return-1;20412042/*2043 * We do not bother to try a delta that we discarded on an2044 * earlier try, but only when reusing delta data. Note that2045 * src_entry that is marked as the preferred_base should always2046 * be considered, as even if we produce a suboptimal delta against2047 * it, we will still save the transfer cost, as we already know2048 * the other side has it and we won't send src_entry at all.2049 */2050if(reuse_delta &&IN_PACK(trg_entry) &&2051IN_PACK(trg_entry) ==IN_PACK(src_entry) &&2052!src_entry->preferred_base &&2053 trg_entry->in_pack_type != OBJ_REF_DELTA &&2054 trg_entry->in_pack_type != OBJ_OFS_DELTA)2055return0;20562057/* Let's not bust the allowed depth. */2058if(src->depth >= max_depth)2059return0;20602061/* Now some size filtering heuristics. */2062 trg_size =SIZE(trg_entry);2063if(!DELTA(trg_entry)) {2064 max_size = trg_size/2- the_hash_algo->rawsz;2065 ref_depth =1;2066}else{2067 max_size =DELTA_SIZE(trg_entry);2068 ref_depth = trg->depth;2069}2070 max_size = (uint64_t)max_size * (max_depth - src->depth) /2071(max_depth - ref_depth +1);2072if(max_size ==0)2073return0;2074 src_size =SIZE(src_entry);2075 sizediff = src_size < trg_size ? trg_size - src_size :0;2076if(sizediff >= max_size)2077return0;2078if(trg_size < src_size /32)2079return0;20802081if(!in_same_island(&trg->entry->idx.oid, &src->entry->idx.oid))2082return0;20832084/* Load data if not already done */2085if(!trg->data) {2086packing_data_lock(&to_pack);2087 trg->data =read_object_file(&trg_entry->idx.oid, &type, &sz);2088packing_data_unlock(&to_pack);2089if(!trg->data)2090die(_("object%scannot be read"),2091oid_to_hex(&trg_entry->idx.oid));2092if(sz != trg_size)2093die(_("object%sinconsistent object length (%"PRIuMAX" vs %"PRIuMAX")"),2094oid_to_hex(&trg_entry->idx.oid), (uintmax_t)sz,2095(uintmax_t)trg_size);2096*mem_usage += sz;2097}2098if(!src->data) {2099packing_data_lock(&to_pack);2100 src->data =read_object_file(&src_entry->idx.oid, &type, &sz);2101packing_data_unlock(&to_pack);2102if(!src->data) {2103if(src_entry->preferred_base) {2104static int warned =0;2105if(!warned++)2106warning(_("object%scannot be read"),2107oid_to_hex(&src_entry->idx.oid));2108/*2109 * Those objects are not included in the2110 * resulting pack. Be resilient and ignore2111 * them if they can't be read, in case the2112 * pack could be created nevertheless.2113 */2114return0;2115}2116die(_("object%scannot be read"),2117oid_to_hex(&src_entry->idx.oid));2118}2119if(sz != src_size)2120die(_("object%sinconsistent object length (%"PRIuMAX" vs %"PRIuMAX")"),2121oid_to_hex(&src_entry->idx.oid), (uintmax_t)sz,2122(uintmax_t)src_size);2123*mem_usage += sz;2124}2125if(!src->index) {2126 src->index =create_delta_index(src->data, src_size);2127if(!src->index) {2128static int warned =0;2129if(!warned++)2130warning(_("suboptimal pack - out of memory"));2131return0;2132}2133*mem_usage +=sizeof_delta_index(src->index);2134}21352136 delta_buf =create_delta(src->index, trg->data, trg_size, &delta_size, max_size);2137if(!delta_buf)2138return0;21392140if(DELTA(trg_entry)) {2141/* Prefer only shallower same-sized deltas. */2142if(delta_size ==DELTA_SIZE(trg_entry) &&2143 src->depth +1>= trg->depth) {2144free(delta_buf);2145return0;2146}2147}21482149/*2150 * Handle memory allocation outside of the cache2151 * accounting lock. Compiler will optimize the strangeness2152 * away when NO_PTHREADS is defined.2153 */2154free(trg_entry->delta_data);2155cache_lock();2156if(trg_entry->delta_data) {2157 delta_cache_size -=DELTA_SIZE(trg_entry);2158 trg_entry->delta_data = NULL;2159}2160if(delta_cacheable(src_size, trg_size, delta_size)) {2161 delta_cache_size += delta_size;2162cache_unlock();2163 trg_entry->delta_data =xrealloc(delta_buf, delta_size);2164}else{2165cache_unlock();2166free(delta_buf);2167}21682169SET_DELTA(trg_entry, src_entry);2170SET_DELTA_SIZE(trg_entry, delta_size);2171 trg->depth = src->depth +1;21722173return1;2174}21752176static unsigned intcheck_delta_limit(struct object_entry *me,unsigned int n)2177{2178struct object_entry *child =DELTA_CHILD(me);2179unsigned int m = n;2180while(child) {2181const unsigned int c =check_delta_limit(child, n +1);2182if(m < c)2183 m = c;2184 child =DELTA_SIBLING(child);2185}2186return m;2187}21882189static unsigned longfree_unpacked(struct unpacked *n)2190{2191unsigned long freed_mem =sizeof_delta_index(n->index);2192free_delta_index(n->index);2193 n->index = NULL;2194if(n->data) {2195 freed_mem +=SIZE(n->entry);2196FREE_AND_NULL(n->data);2197}2198 n->entry = NULL;2199 n->depth =0;2200return freed_mem;2201}22022203static voidfind_deltas(struct object_entry **list,unsigned*list_size,2204int window,int depth,unsigned*processed)2205{2206uint32_t i, idx =0, count =0;2207struct unpacked *array;2208unsigned long mem_usage =0;22092210 array =xcalloc(window,sizeof(struct unpacked));22112212for(;;) {2213struct object_entry *entry;2214struct unpacked *n = array + idx;2215int j, max_depth, best_base = -1;22162217progress_lock();2218if(!*list_size) {2219progress_unlock();2220break;2221}2222 entry = *list++;2223(*list_size)--;2224if(!entry->preferred_base) {2225(*processed)++;2226display_progress(progress_state, *processed);2227}2228progress_unlock();22292230 mem_usage -=free_unpacked(n);2231 n->entry = entry;22322233while(window_memory_limit &&2234 mem_usage > window_memory_limit &&2235 count >1) {2236const uint32_t tail = (idx + window - count) % window;2237 mem_usage -=free_unpacked(array + tail);2238 count--;2239}22402241/* We do not compute delta to *create* objects we are not2242 * going to pack.2243 */2244if(entry->preferred_base)2245goto next;22462247/*2248 * If the current object is at pack edge, take the depth the2249 * objects that depend on the current object into account2250 * otherwise they would become too deep.2251 */2252 max_depth = depth;2253if(DELTA_CHILD(entry)) {2254 max_depth -=check_delta_limit(entry,0);2255if(max_depth <=0)2256goto next;2257}22582259 j = window;2260while(--j >0) {2261int ret;2262uint32_t other_idx = idx + j;2263struct unpacked *m;2264if(other_idx >= window)2265 other_idx -= window;2266 m = array + other_idx;2267if(!m->entry)2268break;2269 ret =try_delta(n, m, max_depth, &mem_usage);2270if(ret <0)2271break;2272else if(ret >0)2273 best_base = other_idx;2274}22752276/*2277 * If we decided to cache the delta data, then it is best2278 * to compress it right away. First because we have to do2279 * it anyway, and doing it here while we're threaded will2280 * save a lot of time in the non threaded write phase,2281 * as well as allow for caching more deltas within2282 * the same cache size limit.2283 * ...2284 * But only if not writing to stdout, since in that case2285 * the network is most likely throttling writes anyway,2286 * and therefore it is best to go to the write phase ASAP2287 * instead, as we can afford spending more time compressing2288 * between writes at that moment.2289 */2290if(entry->delta_data && !pack_to_stdout) {2291unsigned long size;22922293 size =do_compress(&entry->delta_data,DELTA_SIZE(entry));2294if(size < (1U<< OE_Z_DELTA_BITS)) {2295 entry->z_delta_size = size;2296cache_lock();2297 delta_cache_size -=DELTA_SIZE(entry);2298 delta_cache_size += entry->z_delta_size;2299cache_unlock();2300}else{2301FREE_AND_NULL(entry->delta_data);2302 entry->z_delta_size =0;2303}2304}23052306/* if we made n a delta, and if n is already at max2307 * depth, leaving it in the window is pointless. we2308 * should evict it first.2309 */2310if(DELTA(entry) && max_depth <= n->depth)2311continue;23122313/*2314 * Move the best delta base up in the window, after the2315 * currently deltified object, to keep it longer. It will2316 * be the first base object to be attempted next.2317 */2318if(DELTA(entry)) {2319struct unpacked swap = array[best_base];2320int dist = (window + idx - best_base) % window;2321int dst = best_base;2322while(dist--) {2323int src = (dst +1) % window;2324 array[dst] = array[src];2325 dst = src;2326}2327 array[dst] = swap;2328}23292330 next:2331 idx++;2332if(count +1< window)2333 count++;2334if(idx >= window)2335 idx =0;2336}23372338for(i =0; i < window; ++i) {2339free_delta_index(array[i].index);2340free(array[i].data);2341}2342free(array);2343}23442345static voidtry_to_free_from_threads(size_t size)2346{2347packing_data_lock(&to_pack);2348release_pack_memory(size);2349packing_data_unlock(&to_pack);2350}23512352static try_to_free_t old_try_to_free_routine;23532354/*2355 * The main object list is split into smaller lists, each is handed to2356 * one worker.2357 *2358 * The main thread waits on the condition that (at least) one of the workers2359 * has stopped working (which is indicated in the .working member of2360 * struct thread_params).2361 *2362 * When a work thread has completed its work, it sets .working to 0 and2363 * signals the main thread and waits on the condition that .data_ready2364 * becomes 1.2365 *2366 * The main thread steals half of the work from the worker that has2367 * most work left to hand it to the idle worker.2368 */23692370struct thread_params {2371 pthread_t thread;2372struct object_entry **list;2373unsigned list_size;2374unsigned remaining;2375int window;2376int depth;2377int working;2378int data_ready;2379 pthread_mutex_t mutex;2380 pthread_cond_t cond;2381unsigned*processed;2382};23832384static pthread_cond_t progress_cond;23852386/*2387 * Mutex and conditional variable can't be statically-initialized on Windows.2388 */2389static voidinit_threaded_search(void)2390{2391pthread_mutex_init(&cache_mutex, NULL);2392pthread_mutex_init(&progress_mutex, NULL);2393pthread_cond_init(&progress_cond, NULL);2394 old_try_to_free_routine =set_try_to_free_routine(try_to_free_from_threads);2395}23962397static voidcleanup_threaded_search(void)2398{2399set_try_to_free_routine(old_try_to_free_routine);2400pthread_cond_destroy(&progress_cond);2401pthread_mutex_destroy(&cache_mutex);2402pthread_mutex_destroy(&progress_mutex);2403}24042405static void*threaded_find_deltas(void*arg)2406{2407struct thread_params *me = arg;24082409progress_lock();2410while(me->remaining) {2411progress_unlock();24122413find_deltas(me->list, &me->remaining,2414 me->window, me->depth, me->processed);24152416progress_lock();2417 me->working =0;2418pthread_cond_signal(&progress_cond);2419progress_unlock();24202421/*2422 * We must not set ->data_ready before we wait on the2423 * condition because the main thread may have set it to 12424 * before we get here. In order to be sure that new2425 * work is available if we see 1 in ->data_ready, it2426 * was initialized to 0 before this thread was spawned2427 * and we reset it to 0 right away.2428 */2429pthread_mutex_lock(&me->mutex);2430while(!me->data_ready)2431pthread_cond_wait(&me->cond, &me->mutex);2432 me->data_ready =0;2433pthread_mutex_unlock(&me->mutex);24342435progress_lock();2436}2437progress_unlock();2438/* leave ->working 1 so that this doesn't get more work assigned */2439return NULL;2440}24412442static voidll_find_deltas(struct object_entry **list,unsigned list_size,2443int window,int depth,unsigned*processed)2444{2445struct thread_params *p;2446int i, ret, active_threads =0;24472448init_threaded_search();24492450if(delta_search_threads <=1) {2451find_deltas(list, &list_size, window, depth, processed);2452cleanup_threaded_search();2453return;2454}2455if(progress > pack_to_stdout)2456fprintf_ln(stderr,_("Delta compression using up to%dthreads"),2457 delta_search_threads);2458 p =xcalloc(delta_search_threads,sizeof(*p));24592460/* Partition the work amongst work threads. */2461for(i =0; i < delta_search_threads; i++) {2462unsigned sub_size = list_size / (delta_search_threads - i);24632464/* don't use too small segments or no deltas will be found */2465if(sub_size <2*window && i+1< delta_search_threads)2466 sub_size =0;24672468 p[i].window = window;2469 p[i].depth = depth;2470 p[i].processed = processed;2471 p[i].working =1;2472 p[i].data_ready =0;24732474/* try to split chunks on "path" boundaries */2475while(sub_size && sub_size < list_size &&2476 list[sub_size]->hash &&2477 list[sub_size]->hash == list[sub_size-1]->hash)2478 sub_size++;24792480 p[i].list = list;2481 p[i].list_size = sub_size;2482 p[i].remaining = sub_size;24832484 list += sub_size;2485 list_size -= sub_size;2486}24872488/* Start work threads. */2489for(i =0; i < delta_search_threads; i++) {2490if(!p[i].list_size)2491continue;2492pthread_mutex_init(&p[i].mutex, NULL);2493pthread_cond_init(&p[i].cond, NULL);2494 ret =pthread_create(&p[i].thread, NULL,2495 threaded_find_deltas, &p[i]);2496if(ret)2497die(_("unable to create thread:%s"),strerror(ret));2498 active_threads++;2499}25002501/*2502 * Now let's wait for work completion. Each time a thread is done2503 * with its work, we steal half of the remaining work from the2504 * thread with the largest number of unprocessed objects and give2505 * it to that newly idle thread. This ensure good load balancing2506 * until the remaining object list segments are simply too short2507 * to be worth splitting anymore.2508 */2509while(active_threads) {2510struct thread_params *target = NULL;2511struct thread_params *victim = NULL;2512unsigned sub_size =0;25132514progress_lock();2515for(;;) {2516for(i =0; !target && i < delta_search_threads; i++)2517if(!p[i].working)2518 target = &p[i];2519if(target)2520break;2521pthread_cond_wait(&progress_cond, &progress_mutex);2522}25232524for(i =0; i < delta_search_threads; i++)2525if(p[i].remaining >2*window &&2526(!victim || victim->remaining < p[i].remaining))2527 victim = &p[i];2528if(victim) {2529 sub_size = victim->remaining /2;2530 list = victim->list + victim->list_size - sub_size;2531while(sub_size && list[0]->hash &&2532 list[0]->hash == list[-1]->hash) {2533 list++;2534 sub_size--;2535}2536if(!sub_size) {2537/*2538 * It is possible for some "paths" to have2539 * so many objects that no hash boundary2540 * might be found. Let's just steal the2541 * exact half in that case.2542 */2543 sub_size = victim->remaining /2;2544 list -= sub_size;2545}2546 target->list = list;2547 victim->list_size -= sub_size;2548 victim->remaining -= sub_size;2549}2550 target->list_size = sub_size;2551 target->remaining = sub_size;2552 target->working =1;2553progress_unlock();25542555pthread_mutex_lock(&target->mutex);2556 target->data_ready =1;2557pthread_cond_signal(&target->cond);2558pthread_mutex_unlock(&target->mutex);25592560if(!sub_size) {2561pthread_join(target->thread, NULL);2562pthread_cond_destroy(&target->cond);2563pthread_mutex_destroy(&target->mutex);2564 active_threads--;2565}2566}2567cleanup_threaded_search();2568free(p);2569}25702571static voidadd_tag_chain(const struct object_id *oid)2572{2573struct tag *tag;25742575/*2576 * We catch duplicates already in add_object_entry(), but we'd2577 * prefer to do this extra check to avoid having to parse the2578 * tag at all if we already know that it's being packed (e.g., if2579 * it was included via bitmaps, we would not have parsed it2580 * previously).2581 */2582if(packlist_find(&to_pack, oid, NULL))2583return;25842585 tag =lookup_tag(the_repository, oid);2586while(1) {2587if(!tag ||parse_tag(tag) || !tag->tagged)2588die(_("unable to pack objects reachable from tag%s"),2589oid_to_hex(oid));25902591add_object_entry(&tag->object.oid, OBJ_TAG, NULL,0);25922593if(tag->tagged->type != OBJ_TAG)2594return;25952596 tag = (struct tag *)tag->tagged;2597}2598}25992600static intadd_ref_tag(const char*path,const struct object_id *oid,int flag,void*cb_data)2601{2602struct object_id peeled;26032604if(starts_with(path,"refs/tags/") &&/* is a tag? */2605!peel_ref(path, &peeled) &&/* peelable? */2606packlist_find(&to_pack, &peeled, NULL))/* object packed? */2607add_tag_chain(oid);2608return0;2609}26102611static voidprepare_pack(int window,int depth)2612{2613struct object_entry **delta_list;2614uint32_t i, nr_deltas;2615unsigned n;26162617if(use_delta_islands)2618resolve_tree_islands(the_repository, progress, &to_pack);26192620get_object_details();26212622/*2623 * If we're locally repacking then we need to be doubly careful2624 * from now on in order to make sure no stealth corruption gets2625 * propagated to the new pack. Clients receiving streamed packs2626 * should validate everything they get anyway so no need to incur2627 * the additional cost here in that case.2628 */2629if(!pack_to_stdout)2630 do_check_packed_object_crc =1;26312632if(!to_pack.nr_objects || !window || !depth)2633return;26342635ALLOC_ARRAY(delta_list, to_pack.nr_objects);2636 nr_deltas = n =0;26372638for(i =0; i < to_pack.nr_objects; i++) {2639struct object_entry *entry = to_pack.objects + i;26402641if(DELTA(entry))2642/* This happens if we decided to reuse existing2643 * delta from a pack. "reuse_delta &&" is implied.2644 */2645continue;26462647if(!entry->type_valid ||2648oe_size_less_than(&to_pack, entry,50))2649continue;26502651if(entry->no_try_delta)2652continue;26532654if(!entry->preferred_base) {2655 nr_deltas++;2656if(oe_type(entry) <0)2657die(_("unable to get type of object%s"),2658oid_to_hex(&entry->idx.oid));2659}else{2660if(oe_type(entry) <0) {2661/*2662 * This object is not found, but we2663 * don't have to include it anyway.2664 */2665continue;2666}2667}26682669 delta_list[n++] = entry;2670}26712672if(nr_deltas && n >1) {2673unsigned nr_done =0;2674if(progress)2675 progress_state =start_progress(_("Compressing objects"),2676 nr_deltas);2677QSORT(delta_list, n, type_size_sort);2678ll_find_deltas(delta_list, n, window+1, depth, &nr_done);2679stop_progress(&progress_state);2680if(nr_done != nr_deltas)2681die(_("inconsistency with delta count"));2682}2683free(delta_list);2684}26852686static intgit_pack_config(const char*k,const char*v,void*cb)2687{2688if(!strcmp(k,"pack.window")) {2689 window =git_config_int(k, v);2690return0;2691}2692if(!strcmp(k,"pack.windowmemory")) {2693 window_memory_limit =git_config_ulong(k, v);2694return0;2695}2696if(!strcmp(k,"pack.depth")) {2697 depth =git_config_int(k, v);2698return0;2699}2700if(!strcmp(k,"pack.deltacachesize")) {2701 max_delta_cache_size =git_config_int(k, v);2702return0;2703}2704if(!strcmp(k,"pack.deltacachelimit")) {2705 cache_max_small_delta_size =git_config_int(k, v);2706return0;2707}2708if(!strcmp(k,"pack.writebitmaphashcache")) {2709if(git_config_bool(k, v))2710 write_bitmap_options |= BITMAP_OPT_HASH_CACHE;2711else2712 write_bitmap_options &= ~BITMAP_OPT_HASH_CACHE;2713}2714if(!strcmp(k,"pack.usebitmaps")) {2715 use_bitmap_index_default =git_config_bool(k, v);2716return0;2717}2718if(!strcmp(k,"pack.usesparse")) {2719 sparse =git_config_bool(k, v);2720return0;2721}2722if(!strcmp(k,"pack.threads")) {2723 delta_search_threads =git_config_int(k, v);2724if(delta_search_threads <0)2725die(_("invalid number of threads specified (%d)"),2726 delta_search_threads);2727if(!HAVE_THREADS && delta_search_threads !=1) {2728warning(_("no threads support, ignoring%s"), k);2729 delta_search_threads =0;2730}2731return0;2732}2733if(!strcmp(k,"pack.indexversion")) {2734 pack_idx_opts.version =git_config_int(k, v);2735if(pack_idx_opts.version >2)2736die(_("bad pack.indexversion=%"PRIu32),2737 pack_idx_opts.version);2738return0;2739}2740returngit_default_config(k, v, cb);2741}27422743static voidread_object_list_from_stdin(void)2744{2745char line[GIT_MAX_HEXSZ +1+ PATH_MAX +2];2746struct object_id oid;2747const char*p;27482749for(;;) {2750if(!fgets(line,sizeof(line), stdin)) {2751if(feof(stdin))2752break;2753if(!ferror(stdin))2754die("BUG: fgets returned NULL, not EOF, not error!");2755if(errno != EINTR)2756die_errno("fgets");2757clearerr(stdin);2758continue;2759}2760if(line[0] =='-') {2761if(get_oid_hex(line+1, &oid))2762die(_("expected edge object ID, got garbage:\n%s"),2763 line);2764add_preferred_base(&oid);2765continue;2766}2767if(parse_oid_hex(line, &oid, &p))2768die(_("expected object ID, got garbage:\n%s"), line);27692770add_preferred_base_object(p +1);2771add_object_entry(&oid, OBJ_NONE, p +1,0);2772}2773}27742775/* Remember to update object flag allocation in object.h */2776#define OBJECT_ADDED (1u<<20)27772778static voidshow_commit(struct commit *commit,void*data)2779{2780add_object_entry(&commit->object.oid, OBJ_COMMIT, NULL,0);2781 commit->object.flags |= OBJECT_ADDED;27822783if(write_bitmap_index)2784index_commit_for_bitmap(commit);27852786if(use_delta_islands)2787propagate_island_marks(commit);2788}27892790static voidshow_object(struct object *obj,const char*name,void*data)2791{2792add_preferred_base_object(name);2793add_object_entry(&obj->oid, obj->type, name,0);2794 obj->flags |= OBJECT_ADDED;27952796if(use_delta_islands) {2797const char*p;2798unsigned depth;2799struct object_entry *ent;28002801/* the empty string is a root tree, which is depth 0 */2802 depth = *name ?1:0;2803for(p =strchr(name,'/'); p; p =strchr(p +1,'/'))2804 depth++;28052806 ent =packlist_find(&to_pack, &obj->oid, NULL);2807if(ent && depth >oe_tree_depth(&to_pack, ent))2808oe_set_tree_depth(&to_pack, ent, depth);2809}2810}28112812static voidshow_object__ma_allow_any(struct object *obj,const char*name,void*data)2813{2814assert(arg_missing_action == MA_ALLOW_ANY);28152816/*2817 * Quietly ignore ALL missing objects. This avoids problems with2818 * staging them now and getting an odd error later.2819 */2820if(!has_object_file(&obj->oid))2821return;28222823show_object(obj, name, data);2824}28252826static voidshow_object__ma_allow_promisor(struct object *obj,const char*name,void*data)2827{2828assert(arg_missing_action == MA_ALLOW_PROMISOR);28292830/*2831 * Quietly ignore EXPECTED missing objects. This avoids problems with2832 * staging them now and getting an odd error later.2833 */2834if(!has_object_file(&obj->oid) &&is_promisor_object(&obj->oid))2835return;28362837show_object(obj, name, data);2838}28392840static intoption_parse_missing_action(const struct option *opt,2841const char*arg,int unset)2842{2843assert(arg);2844assert(!unset);28452846if(!strcmp(arg,"error")) {2847 arg_missing_action = MA_ERROR;2848 fn_show_object = show_object;2849return0;2850}28512852if(!strcmp(arg,"allow-any")) {2853 arg_missing_action = MA_ALLOW_ANY;2854 fetch_if_missing =0;2855 fn_show_object = show_object__ma_allow_any;2856return0;2857}28582859if(!strcmp(arg,"allow-promisor")) {2860 arg_missing_action = MA_ALLOW_PROMISOR;2861 fetch_if_missing =0;2862 fn_show_object = show_object__ma_allow_promisor;2863return0;2864}28652866die(_("invalid value for --missing"));2867return0;2868}28692870static voidshow_edge(struct commit *commit)2871{2872add_preferred_base(&commit->object.oid);2873}28742875struct in_pack_object {2876 off_t offset;2877struct object *object;2878};28792880struct in_pack {2881unsigned int alloc;2882unsigned int nr;2883struct in_pack_object *array;2884};28852886static voidmark_in_pack_object(struct object *object,struct packed_git *p,struct in_pack *in_pack)2887{2888 in_pack->array[in_pack->nr].offset =find_pack_entry_one(object->oid.hash, p);2889 in_pack->array[in_pack->nr].object = object;2890 in_pack->nr++;2891}28922893/*2894 * Compare the objects in the offset order, in order to emulate the2895 * "git rev-list --objects" output that produced the pack originally.2896 */2897static intofscmp(const void*a_,const void*b_)2898{2899struct in_pack_object *a = (struct in_pack_object *)a_;2900struct in_pack_object *b = (struct in_pack_object *)b_;29012902if(a->offset < b->offset)2903return-1;2904else if(a->offset > b->offset)2905return1;2906else2907returnoidcmp(&a->object->oid, &b->object->oid);2908}29092910static voidadd_objects_in_unpacked_packs(void)2911{2912struct packed_git *p;2913struct in_pack in_pack;2914uint32_t i;29152916memset(&in_pack,0,sizeof(in_pack));29172918for(p =get_all_packs(the_repository); p; p = p->next) {2919struct object_id oid;2920struct object *o;29212922if(!p->pack_local || p->pack_keep || p->pack_keep_in_core)2923continue;2924if(open_pack_index(p))2925die(_("cannot open pack index"));29262927ALLOC_GROW(in_pack.array,2928 in_pack.nr + p->num_objects,2929 in_pack.alloc);29302931for(i =0; i < p->num_objects; i++) {2932nth_packed_object_oid(&oid, p, i);2933 o =lookup_unknown_object(&oid);2934if(!(o->flags & OBJECT_ADDED))2935mark_in_pack_object(o, p, &in_pack);2936 o->flags |= OBJECT_ADDED;2937}2938}29392940if(in_pack.nr) {2941QSORT(in_pack.array, in_pack.nr, ofscmp);2942for(i =0; i < in_pack.nr; i++) {2943struct object *o = in_pack.array[i].object;2944add_object_entry(&o->oid, o->type,"",0);2945}2946}2947free(in_pack.array);2948}29492950static intadd_loose_object(const struct object_id *oid,const char*path,2951void*data)2952{2953enum object_type type =oid_object_info(the_repository, oid, NULL);29542955if(type <0) {2956warning(_("loose object at%scould not be examined"), path);2957return0;2958}29592960add_object_entry(oid, type,"",0);2961return0;2962}29632964/*2965 * We actually don't even have to worry about reachability here.2966 * add_object_entry will weed out duplicates, so we just add every2967 * loose object we find.2968 */2969static voidadd_unreachable_loose_objects(void)2970{2971for_each_loose_file_in_objdir(get_object_directory(),2972 add_loose_object,2973 NULL, NULL, NULL);2974}29752976static inthas_sha1_pack_kept_or_nonlocal(const struct object_id *oid)2977{2978static struct packed_git *last_found = (void*)1;2979struct packed_git *p;29802981 p = (last_found != (void*)1) ? last_found :2982get_all_packs(the_repository);29832984while(p) {2985if((!p->pack_local || p->pack_keep ||2986 p->pack_keep_in_core) &&2987find_pack_entry_one(oid->hash, p)) {2988 last_found = p;2989return1;2990}2991if(p == last_found)2992 p =get_all_packs(the_repository);2993else2994 p = p->next;2995if(p == last_found)2996 p = p->next;2997}2998return0;2999}30003001/*3002 * Store a list of sha1s that are should not be discarded3003 * because they are either written too recently, or are3004 * reachable from another object that was.3005 *3006 * This is filled by get_object_list.3007 */3008static struct oid_array recent_objects;30093010static intloosened_object_can_be_discarded(const struct object_id *oid,3011 timestamp_t mtime)3012{3013if(!unpack_unreachable_expiration)3014return0;3015if(mtime > unpack_unreachable_expiration)3016return0;3017if(oid_array_lookup(&recent_objects, oid) >=0)3018return0;3019return1;3020}30213022static voidloosen_unused_packed_objects(void)3023{3024struct packed_git *p;3025uint32_t i;3026struct object_id oid;30273028for(p =get_all_packs(the_repository); p; p = p->next) {3029if(!p->pack_local || p->pack_keep || p->pack_keep_in_core)3030continue;30313032if(open_pack_index(p))3033die(_("cannot open pack index"));30343035for(i =0; i < p->num_objects; i++) {3036nth_packed_object_oid(&oid, p, i);3037if(!packlist_find(&to_pack, &oid, NULL) &&3038!has_sha1_pack_kept_or_nonlocal(&oid) &&3039!loosened_object_can_be_discarded(&oid, p->mtime))3040if(force_object_loose(&oid, p->mtime))3041die(_("unable to force loose object"));3042}3043}3044}30453046/*3047 * This tracks any options which pack-reuse code expects to be on, or which a3048 * reader of the pack might not understand, and which would therefore prevent3049 * blind reuse of what we have on disk.3050 */3051static intpack_options_allow_reuse(void)3052{3053return pack_to_stdout &&3054 allow_ofs_delta &&3055!ignore_packed_keep_on_disk &&3056!ignore_packed_keep_in_core &&3057(!local || !have_non_local_packs) &&3058!incremental;3059}30603061static intget_object_list_from_bitmap(struct rev_info *revs)3062{3063if(!(bitmap_git =prepare_bitmap_walk(revs)))3064return-1;30653066if(pack_options_allow_reuse() &&3067!reuse_partial_packfile_from_bitmap(3068 bitmap_git,3069&reuse_packfile,3070&reuse_packfile_objects,3071&reuse_packfile_offset)) {3072assert(reuse_packfile_objects);3073 nr_result += reuse_packfile_objects;3074display_progress(progress_state, nr_result);3075}30763077traverse_bitmap_commit_list(bitmap_git, &add_object_entry_from_bitmap);3078return0;3079}30803081static voidrecord_recent_object(struct object *obj,3082const char*name,3083void*data)3084{3085oid_array_append(&recent_objects, &obj->oid);3086}30873088static voidrecord_recent_commit(struct commit *commit,void*data)3089{3090oid_array_append(&recent_objects, &commit->object.oid);3091}30923093static voidget_object_list(int ac,const char**av)3094{3095struct rev_info revs;3096struct setup_revision_opt s_r_opt = {3097.allow_exclude_promisor_objects =1,3098};3099char line[1000];3100int flags =0;3101int save_warning;31023103repo_init_revisions(the_repository, &revs, NULL);3104 save_commit_buffer =0;3105setup_revisions(ac, av, &revs, &s_r_opt);31063107/* make sure shallows are read */3108is_repository_shallow(the_repository);31093110 save_warning = warn_on_object_refname_ambiguity;3111 warn_on_object_refname_ambiguity =0;31123113while(fgets(line,sizeof(line), stdin) != NULL) {3114int len =strlen(line);3115if(len && line[len -1] =='\n')3116 line[--len] =0;3117if(!len)3118break;3119if(*line =='-') {3120if(!strcmp(line,"--not")) {3121 flags ^= UNINTERESTING;3122 write_bitmap_index =0;3123continue;3124}3125if(starts_with(line,"--shallow ")) {3126struct object_id oid;3127if(get_oid_hex(line +10, &oid))3128die("not an SHA-1 '%s'", line +10);3129register_shallow(the_repository, &oid);3130 use_bitmap_index =0;3131continue;3132}3133die(_("not a rev '%s'"), line);3134}3135if(handle_revision_arg(line, &revs, flags, REVARG_CANNOT_BE_FILENAME))3136die(_("bad revision '%s'"), line);3137}31383139 warn_on_object_refname_ambiguity = save_warning;31403141if(use_bitmap_index && !get_object_list_from_bitmap(&revs))3142return;31433144if(use_delta_islands)3145load_delta_islands(the_repository, progress);31463147if(prepare_revision_walk(&revs))3148die(_("revision walk setup failed"));3149mark_edges_uninteresting(&revs, show_edge, sparse);31503151if(!fn_show_object)3152 fn_show_object = show_object;3153traverse_commit_list_filtered(&filter_options, &revs,3154 show_commit, fn_show_object, NULL,3155 NULL);31563157if(unpack_unreachable_expiration) {3158 revs.ignore_missing_links =1;3159if(add_unseen_recent_objects_to_traversal(&revs,3160 unpack_unreachable_expiration))3161die(_("unable to add recent objects"));3162if(prepare_revision_walk(&revs))3163die(_("revision walk setup failed"));3164traverse_commit_list(&revs, record_recent_commit,3165 record_recent_object, NULL);3166}31673168if(keep_unreachable)3169add_objects_in_unpacked_packs();3170if(pack_loose_unreachable)3171add_unreachable_loose_objects();3172if(unpack_unreachable)3173loosen_unused_packed_objects();31743175oid_array_clear(&recent_objects);3176}31773178static voidadd_extra_kept_packs(const struct string_list *names)3179{3180struct packed_git *p;31813182if(!names->nr)3183return;31843185for(p =get_all_packs(the_repository); p; p = p->next) {3186const char*name =basename(p->pack_name);3187int i;31883189if(!p->pack_local)3190continue;31913192for(i =0; i < names->nr; i++)3193if(!fspathcmp(name, names->items[i].string))3194break;31953196if(i < names->nr) {3197 p->pack_keep_in_core =1;3198 ignore_packed_keep_in_core =1;3199continue;3200}3201}3202}32033204static intoption_parse_index_version(const struct option *opt,3205const char*arg,int unset)3206{3207char*c;3208const char*val = arg;32093210BUG_ON_OPT_NEG(unset);32113212 pack_idx_opts.version =strtoul(val, &c,10);3213if(pack_idx_opts.version >2)3214die(_("unsupported index version%s"), val);3215if(*c ==','&& c[1])3216 pack_idx_opts.off32_limit =strtoul(c+1, &c,0);3217if(*c || pack_idx_opts.off32_limit &0x80000000)3218die(_("bad index version '%s'"), val);3219return0;3220}32213222static intoption_parse_unpack_unreachable(const struct option *opt,3223const char*arg,int unset)3224{3225if(unset) {3226 unpack_unreachable =0;3227 unpack_unreachable_expiration =0;3228}3229else{3230 unpack_unreachable =1;3231if(arg)3232 unpack_unreachable_expiration =approxidate(arg);3233}3234return0;3235}32363237intcmd_pack_objects(int argc,const char**argv,const char*prefix)3238{3239int use_internal_rev_list =0;3240int shallow =0;3241int all_progress_implied =0;3242struct argv_array rp = ARGV_ARRAY_INIT;3243int rev_list_unpacked =0, rev_list_all =0, rev_list_reflog =0;3244int rev_list_index =0;3245struct string_list keep_pack_list = STRING_LIST_INIT_NODUP;3246struct option pack_objects_options[] = {3247OPT_SET_INT('q',"quiet", &progress,3248N_("do not show progress meter"),0),3249OPT_SET_INT(0,"progress", &progress,3250N_("show progress meter"),1),3251OPT_SET_INT(0,"all-progress", &progress,3252N_("show progress meter during object writing phase"),2),3253OPT_BOOL(0,"all-progress-implied",3254&all_progress_implied,3255N_("similar to --all-progress when progress meter is shown")),3256{ OPTION_CALLBACK,0,"index-version", NULL,N_("<version>[,<offset>]"),3257N_("write the pack index file in the specified idx format version"),3258 PARSE_OPT_NONEG, option_parse_index_version },3259OPT_MAGNITUDE(0,"max-pack-size", &pack_size_limit,3260N_("maximum size of each output pack file")),3261OPT_BOOL(0,"local", &local,3262N_("ignore borrowed objects from alternate object store")),3263OPT_BOOL(0,"incremental", &incremental,3264N_("ignore packed objects")),3265OPT_INTEGER(0,"window", &window,3266N_("limit pack window by objects")),3267OPT_MAGNITUDE(0,"window-memory", &window_memory_limit,3268N_("limit pack window by memory in addition to object limit")),3269OPT_INTEGER(0,"depth", &depth,3270N_("maximum length of delta chain allowed in the resulting pack")),3271OPT_BOOL(0,"reuse-delta", &reuse_delta,3272N_("reuse existing deltas")),3273OPT_BOOL(0,"reuse-object", &reuse_object,3274N_("reuse existing objects")),3275OPT_BOOL(0,"delta-base-offset", &allow_ofs_delta,3276N_("use OFS_DELTA objects")),3277OPT_INTEGER(0,"threads", &delta_search_threads,3278N_("use threads when searching for best delta matches")),3279OPT_BOOL(0,"non-empty", &non_empty,3280N_("do not create an empty pack output")),3281OPT_BOOL(0,"revs", &use_internal_rev_list,3282N_("read revision arguments from standard input")),3283OPT_SET_INT_F(0,"unpacked", &rev_list_unpacked,3284N_("limit the objects to those that are not yet packed"),32851, PARSE_OPT_NONEG),3286OPT_SET_INT_F(0,"all", &rev_list_all,3287N_("include objects reachable from any reference"),32881, PARSE_OPT_NONEG),3289OPT_SET_INT_F(0,"reflog", &rev_list_reflog,3290N_("include objects referred by reflog entries"),32911, PARSE_OPT_NONEG),3292OPT_SET_INT_F(0,"indexed-objects", &rev_list_index,3293N_("include objects referred to by the index"),32941, PARSE_OPT_NONEG),3295OPT_BOOL(0,"stdout", &pack_to_stdout,3296N_("output pack to stdout")),3297OPT_BOOL(0,"include-tag", &include_tag,3298N_("include tag objects that refer to objects to be packed")),3299OPT_BOOL(0,"keep-unreachable", &keep_unreachable,3300N_("keep unreachable objects")),3301OPT_BOOL(0,"pack-loose-unreachable", &pack_loose_unreachable,3302N_("pack loose unreachable objects")),3303{ OPTION_CALLBACK,0,"unpack-unreachable", NULL,N_("time"),3304N_("unpack unreachable objects newer than <time>"),3305 PARSE_OPT_OPTARG, option_parse_unpack_unreachable },3306OPT_BOOL(0,"sparse", &sparse,3307N_("use the sparse reachability algorithm")),3308OPT_BOOL(0,"thin", &thin,3309N_("create thin packs")),3310OPT_BOOL(0,"shallow", &shallow,3311N_("create packs suitable for shallow fetches")),3312OPT_BOOL(0,"honor-pack-keep", &ignore_packed_keep_on_disk,3313N_("ignore packs that have companion .keep file")),3314OPT_STRING_LIST(0,"keep-pack", &keep_pack_list,N_("name"),3315N_("ignore this pack")),3316OPT_INTEGER(0,"compression", &pack_compression_level,3317N_("pack compression level")),3318OPT_SET_INT(0,"keep-true-parents", &grafts_replace_parents,3319N_("do not hide commits by grafts"),0),3320OPT_BOOL(0,"use-bitmap-index", &use_bitmap_index,3321N_("use a bitmap index if available to speed up counting objects")),3322OPT_SET_INT(0,"write-bitmap-index", &write_bitmap_index,3323N_("write a bitmap index together with the pack index"),3324 WRITE_BITMAP_TRUE),3325OPT_SET_INT_F(0,"write-bitmap-index-quiet",3326&write_bitmap_index,3327N_("write a bitmap index if possible"),3328 WRITE_BITMAP_QUIET, PARSE_OPT_HIDDEN),3329OPT_PARSE_LIST_OBJECTS_FILTER(&filter_options),3330{ OPTION_CALLBACK,0,"missing", NULL,N_("action"),3331N_("handling for missing objects"), PARSE_OPT_NONEG,3332 option_parse_missing_action },3333OPT_BOOL(0,"exclude-promisor-objects", &exclude_promisor_objects,3334N_("do not pack objects in promisor packfiles")),3335OPT_BOOL(0,"delta-islands", &use_delta_islands,3336N_("respect islands during delta compression")),3337OPT_END(),3338};33393340if(DFS_NUM_STATES > (1<< OE_DFS_STATE_BITS))3341BUG("too many dfs states, increase OE_DFS_STATE_BITS");33423343 read_replace_refs =0;33443345 sparse =git_env_bool("GIT_TEST_PACK_SPARSE",0);3346reset_pack_idx_option(&pack_idx_opts);3347git_config(git_pack_config, NULL);33483349 progress =isatty(2);3350 argc =parse_options(argc, argv, prefix, pack_objects_options,3351 pack_usage,0);33523353if(argc) {3354 base_name = argv[0];3355 argc--;3356}3357if(pack_to_stdout != !base_name || argc)3358usage_with_options(pack_usage, pack_objects_options);33593360if(depth >= (1<< OE_DEPTH_BITS)) {3361warning(_("delta chain depth%dis too deep, forcing%d"),3362 depth, (1<< OE_DEPTH_BITS) -1);3363 depth = (1<< OE_DEPTH_BITS) -1;3364}3365if(cache_max_small_delta_size >= (1U<< OE_Z_DELTA_BITS)) {3366warning(_("pack.deltaCacheLimit is too high, forcing%d"),3367(1U<< OE_Z_DELTA_BITS) -1);3368 cache_max_small_delta_size = (1U<< OE_Z_DELTA_BITS) -1;3369}33703371argv_array_push(&rp,"pack-objects");3372if(thin) {3373 use_internal_rev_list =1;3374argv_array_push(&rp, shallow3375?"--objects-edge-aggressive"3376:"--objects-edge");3377}else3378argv_array_push(&rp,"--objects");33793380if(rev_list_all) {3381 use_internal_rev_list =1;3382argv_array_push(&rp,"--all");3383}3384if(rev_list_reflog) {3385 use_internal_rev_list =1;3386argv_array_push(&rp,"--reflog");3387}3388if(rev_list_index) {3389 use_internal_rev_list =1;3390argv_array_push(&rp,"--indexed-objects");3391}3392if(rev_list_unpacked) {3393 use_internal_rev_list =1;3394argv_array_push(&rp,"--unpacked");3395}33963397if(exclude_promisor_objects) {3398 use_internal_rev_list =1;3399 fetch_if_missing =0;3400argv_array_push(&rp,"--exclude-promisor-objects");3401}3402if(unpack_unreachable || keep_unreachable || pack_loose_unreachable)3403 use_internal_rev_list =1;34043405if(!reuse_object)3406 reuse_delta =0;3407if(pack_compression_level == -1)3408 pack_compression_level = Z_DEFAULT_COMPRESSION;3409else if(pack_compression_level <0|| pack_compression_level > Z_BEST_COMPRESSION)3410die(_("bad pack compression level%d"), pack_compression_level);34113412if(!delta_search_threads)/* --threads=0 means autodetect */3413 delta_search_threads =online_cpus();34143415if(!HAVE_THREADS && delta_search_threads !=1)3416warning(_("no threads support, ignoring --threads"));3417if(!pack_to_stdout && !pack_size_limit)3418 pack_size_limit = pack_size_limit_cfg;3419if(pack_to_stdout && pack_size_limit)3420die(_("--max-pack-size cannot be used to build a pack for transfer"));3421if(pack_size_limit && pack_size_limit <1024*1024) {3422warning(_("minimum pack size limit is 1 MiB"));3423 pack_size_limit =1024*1024;3424}34253426if(!pack_to_stdout && thin)3427die(_("--thin cannot be used to build an indexable pack"));34283429if(keep_unreachable && unpack_unreachable)3430die(_("--keep-unreachable and --unpack-unreachable are incompatible"));3431if(!rev_list_all || !rev_list_reflog || !rev_list_index)3432 unpack_unreachable_expiration =0;34333434if(filter_options.choice) {3435if(!pack_to_stdout)3436die(_("cannot use --filter without --stdout"));3437 use_bitmap_index =0;3438}34393440/*3441 * "soft" reasons not to use bitmaps - for on-disk repack by default we want3442 *3443 * - to produce good pack (with bitmap index not-yet-packed objects are3444 * packed in suboptimal order).3445 *3446 * - to use more robust pack-generation codepath (avoiding possible3447 * bugs in bitmap code and possible bitmap index corruption).3448 */3449if(!pack_to_stdout)3450 use_bitmap_index_default =0;34513452if(use_bitmap_index <0)3453 use_bitmap_index = use_bitmap_index_default;34543455/* "hard" reasons not to use bitmaps; these just won't work at all */3456if(!use_internal_rev_list || (!pack_to_stdout && write_bitmap_index) ||is_repository_shallow(the_repository))3457 use_bitmap_index =0;34583459if(pack_to_stdout || !rev_list_all)3460 write_bitmap_index =0;34613462if(use_delta_islands)3463argv_array_push(&rp,"--topo-order");34643465if(progress && all_progress_implied)3466 progress =2;34673468add_extra_kept_packs(&keep_pack_list);3469if(ignore_packed_keep_on_disk) {3470struct packed_git *p;3471for(p =get_all_packs(the_repository); p; p = p->next)3472if(p->pack_local && p->pack_keep)3473break;3474if(!p)/* no keep-able packs found */3475 ignore_packed_keep_on_disk =0;3476}3477if(local) {3478/*3479 * unlike ignore_packed_keep_on_disk above, we do not3480 * want to unset "local" based on looking at packs, as3481 * it also covers non-local objects3482 */3483struct packed_git *p;3484for(p =get_all_packs(the_repository); p; p = p->next) {3485if(!p->pack_local) {3486 have_non_local_packs =1;3487break;3488}3489}3490}34913492trace2_region_enter("pack-objects","enumerate-objects",3493 the_repository);3494prepare_packing_data(the_repository, &to_pack);34953496if(progress)3497 progress_state =start_progress(_("Enumerating objects"),0);3498if(!use_internal_rev_list)3499read_object_list_from_stdin();3500else{3501get_object_list(rp.argc, rp.argv);3502argv_array_clear(&rp);3503}3504cleanup_preferred_base();3505if(include_tag && nr_result)3506for_each_ref(add_ref_tag, NULL);3507stop_progress(&progress_state);3508trace2_region_leave("pack-objects","enumerate-objects",3509 the_repository);35103511if(non_empty && !nr_result)3512return0;3513if(nr_result) {3514trace2_region_enter("pack-objects","prepare-pack",3515 the_repository);3516prepare_pack(window, depth);3517trace2_region_leave("pack-objects","prepare-pack",3518 the_repository);3519}35203521trace2_region_enter("pack-objects","write-pack-file", the_repository);3522write_pack_file();3523trace2_region_leave("pack-objects","write-pack-file", the_repository);35243525if(progress)3526fprintf_ln(stderr,3527_("Total %"PRIu32" (delta %"PRIu32"),"3528" reused %"PRIu32" (delta %"PRIu32")"),3529 written, written_delta, reused, reused_delta);3530return0;3531}