1#include"builtin.h" 2#include"cache.h" 3#include"repository.h" 4#include"config.h" 5#include"attr.h" 6#include"object.h" 7#include"blob.h" 8#include"commit.h" 9#include"tag.h" 10#include"tree.h" 11#include"delta.h" 12#include"pack.h" 13#include"pack-revindex.h" 14#include"csum-file.h" 15#include"tree-walk.h" 16#include"diff.h" 17#include"revision.h" 18#include"list-objects.h" 19#include"list-objects-filter.h" 20#include"list-objects-filter-options.h" 21#include"pack-objects.h" 22#include"progress.h" 23#include"refs.h" 24#include"streaming.h" 25#include"thread-utils.h" 26#include"pack-bitmap.h" 27#include"delta-islands.h" 28#include"reachable.h" 29#include"sha1-array.h" 30#include"argv-array.h" 31#include"list.h" 32#include"packfile.h" 33#include"object-store.h" 34#include"dir.h" 35#include"midx.h" 36 37#define IN_PACK(obj) oe_in_pack(&to_pack, obj) 38#define SIZE(obj) oe_size(&to_pack, obj) 39#define SET_SIZE(obj,size) oe_set_size(&to_pack, obj, size) 40#define DELTA_SIZE(obj) oe_delta_size(&to_pack, obj) 41#define DELTA(obj) oe_delta(&to_pack, obj) 42#define DELTA_CHILD(obj) oe_delta_child(&to_pack, obj) 43#define DELTA_SIBLING(obj) oe_delta_sibling(&to_pack, obj) 44#define SET_DELTA(obj, val) oe_set_delta(&to_pack, obj, val) 45#define SET_DELTA_EXT(obj, oid) oe_set_delta_ext(&to_pack, obj, oid) 46#define SET_DELTA_SIZE(obj, val) oe_set_delta_size(&to_pack, obj, val) 47#define SET_DELTA_CHILD(obj, val) oe_set_delta_child(&to_pack, obj, val) 48#define SET_DELTA_SIBLING(obj, val) oe_set_delta_sibling(&to_pack, obj, val) 49 50static const char*pack_usage[] = { 51N_("git pack-objects --stdout [<options>...] [< <ref-list> | < <object-list>]"), 52N_("git pack-objects [<options>...] <base-name> [< <ref-list> | < <object-list>]"), 53 NULL 54}; 55 56/* 57 * Objects we are going to pack are collected in the `to_pack` structure. 58 * It contains an array (dynamically expanded) of the object data, and a map 59 * that can resolve SHA1s to their position in the array. 60 */ 61static struct packing_data to_pack; 62 63static struct pack_idx_entry **written_list; 64static uint32_t nr_result, nr_written, nr_seen; 65static struct bitmap_index *bitmap_git; 66static uint32_t write_layer; 67 68static int non_empty; 69static int reuse_delta =1, reuse_object =1; 70static int keep_unreachable, unpack_unreachable, include_tag; 71static timestamp_t unpack_unreachable_expiration; 72static int pack_loose_unreachable; 73static int local; 74static int have_non_local_packs; 75static int incremental; 76static int ignore_packed_keep_on_disk; 77static int ignore_packed_keep_in_core; 78static int allow_ofs_delta; 79static struct pack_idx_option pack_idx_opts; 80static const char*base_name; 81static int progress =1; 82static int window =10; 83static unsigned long pack_size_limit; 84static int depth =50; 85static int delta_search_threads; 86static int pack_to_stdout; 87static int thin; 88static int num_preferred_base; 89static struct progress *progress_state; 90 91static struct packed_git *reuse_packfile; 92static uint32_t reuse_packfile_objects; 93static off_t reuse_packfile_offset; 94 95static int use_bitmap_index_default =1; 96static int use_bitmap_index = -1; 97static int write_bitmap_index; 98static uint16_t write_bitmap_options; 99 100static int exclude_promisor_objects; 101 102static int use_delta_islands; 103 104static unsigned long delta_cache_size =0; 105static unsigned long max_delta_cache_size = DEFAULT_DELTA_CACHE_SIZE; 106static unsigned long cache_max_small_delta_size =1000; 107 108static unsigned long window_memory_limit =0; 109 110static struct list_objects_filter_options filter_options; 111 112enum missing_action { 113 MA_ERROR =0,/* fail if any missing objects are encountered */ 114 MA_ALLOW_ANY,/* silently allow ALL missing objects */ 115 MA_ALLOW_PROMISOR,/* silently allow all missing PROMISOR objects */ 116}; 117static enum missing_action arg_missing_action; 118static show_object_fn fn_show_object; 119 120/* 121 * stats 122 */ 123static uint32_t written, written_delta; 124static uint32_t reused, reused_delta; 125 126/* 127 * Indexed commits 128 */ 129static struct commit **indexed_commits; 130static unsigned int indexed_commits_nr; 131static unsigned int indexed_commits_alloc; 132 133static voidindex_commit_for_bitmap(struct commit *commit) 134{ 135if(indexed_commits_nr >= indexed_commits_alloc) { 136 indexed_commits_alloc = (indexed_commits_alloc +32) *2; 137REALLOC_ARRAY(indexed_commits, indexed_commits_alloc); 138} 139 140 indexed_commits[indexed_commits_nr++] = commit; 141} 142 143static void*get_delta(struct object_entry *entry) 144{ 145unsigned long size, base_size, delta_size; 146void*buf, *base_buf, *delta_buf; 147enum object_type type; 148 149 buf =read_object_file(&entry->idx.oid, &type, &size); 150if(!buf) 151die(_("unable to read%s"),oid_to_hex(&entry->idx.oid)); 152 base_buf =read_object_file(&DELTA(entry)->idx.oid, &type, 153&base_size); 154if(!base_buf) 155die("unable to read%s", 156oid_to_hex(&DELTA(entry)->idx.oid)); 157 delta_buf =diff_delta(base_buf, base_size, 158 buf, size, &delta_size,0); 159/* 160 * We succesfully computed this delta once but dropped it for 161 * memory reasons. Something is very wrong if this time we 162 * recompute and create a different delta. 163 */ 164if(!delta_buf || delta_size !=DELTA_SIZE(entry)) 165BUG("delta size changed"); 166free(buf); 167free(base_buf); 168return delta_buf; 169} 170 171static unsigned longdo_compress(void**pptr,unsigned long size) 172{ 173 git_zstream stream; 174void*in, *out; 175unsigned long maxsize; 176 177git_deflate_init(&stream, pack_compression_level); 178 maxsize =git_deflate_bound(&stream, size); 179 180 in = *pptr; 181 out =xmalloc(maxsize); 182*pptr = out; 183 184 stream.next_in = in; 185 stream.avail_in = size; 186 stream.next_out = out; 187 stream.avail_out = maxsize; 188while(git_deflate(&stream, Z_FINISH) == Z_OK) 189;/* nothing */ 190git_deflate_end(&stream); 191 192free(in); 193return stream.total_out; 194} 195 196static unsigned longwrite_large_blob_data(struct git_istream *st,struct hashfile *f, 197const struct object_id *oid) 198{ 199 git_zstream stream; 200unsigned char ibuf[1024*16]; 201unsigned char obuf[1024*16]; 202unsigned long olen =0; 203 204git_deflate_init(&stream, pack_compression_level); 205 206for(;;) { 207 ssize_t readlen; 208int zret = Z_OK; 209 readlen =read_istream(st, ibuf,sizeof(ibuf)); 210if(readlen == -1) 211die(_("unable to read%s"),oid_to_hex(oid)); 212 213 stream.next_in = ibuf; 214 stream.avail_in = readlen; 215while((stream.avail_in || readlen ==0) && 216(zret == Z_OK || zret == Z_BUF_ERROR)) { 217 stream.next_out = obuf; 218 stream.avail_out =sizeof(obuf); 219 zret =git_deflate(&stream, readlen ?0: Z_FINISH); 220hashwrite(f, obuf, stream.next_out - obuf); 221 olen += stream.next_out - obuf; 222} 223if(stream.avail_in) 224die(_("deflate error (%d)"), zret); 225if(readlen ==0) { 226if(zret != Z_STREAM_END) 227die(_("deflate error (%d)"), zret); 228break; 229} 230} 231git_deflate_end(&stream); 232return olen; 233} 234 235/* 236 * we are going to reuse the existing object data as is. make 237 * sure it is not corrupt. 238 */ 239static intcheck_pack_inflate(struct packed_git *p, 240struct pack_window **w_curs, 241 off_t offset, 242 off_t len, 243unsigned long expect) 244{ 245 git_zstream stream; 246unsigned char fakebuf[4096], *in; 247int st; 248 249memset(&stream,0,sizeof(stream)); 250git_inflate_init(&stream); 251do{ 252 in =use_pack(p, w_curs, offset, &stream.avail_in); 253 stream.next_in = in; 254 stream.next_out = fakebuf; 255 stream.avail_out =sizeof(fakebuf); 256 st =git_inflate(&stream, Z_FINISH); 257 offset += stream.next_in - in; 258}while(st == Z_OK || st == Z_BUF_ERROR); 259git_inflate_end(&stream); 260return(st == Z_STREAM_END && 261 stream.total_out == expect && 262 stream.total_in == len) ?0: -1; 263} 264 265static voidcopy_pack_data(struct hashfile *f, 266struct packed_git *p, 267struct pack_window **w_curs, 268 off_t offset, 269 off_t len) 270{ 271unsigned char*in; 272unsigned long avail; 273 274while(len) { 275 in =use_pack(p, w_curs, offset, &avail); 276if(avail > len) 277 avail = (unsigned long)len; 278hashwrite(f, in, avail); 279 offset += avail; 280 len -= avail; 281} 282} 283 284/* Return 0 if we will bust the pack-size limit */ 285static unsigned longwrite_no_reuse_object(struct hashfile *f,struct object_entry *entry, 286unsigned long limit,int usable_delta) 287{ 288unsigned long size, datalen; 289unsigned char header[MAX_PACK_OBJECT_HEADER], 290 dheader[MAX_PACK_OBJECT_HEADER]; 291unsigned hdrlen; 292enum object_type type; 293void*buf; 294struct git_istream *st = NULL; 295const unsigned hashsz = the_hash_algo->rawsz; 296 297if(!usable_delta) { 298if(oe_type(entry) == OBJ_BLOB && 299oe_size_greater_than(&to_pack, entry, big_file_threshold) && 300(st =open_istream(&entry->idx.oid, &type, &size, NULL)) != NULL) 301 buf = NULL; 302else{ 303 buf =read_object_file(&entry->idx.oid, &type, &size); 304if(!buf) 305die(_("unable to read%s"), 306oid_to_hex(&entry->idx.oid)); 307} 308/* 309 * make sure no cached delta data remains from a 310 * previous attempt before a pack split occurred. 311 */ 312FREE_AND_NULL(entry->delta_data); 313 entry->z_delta_size =0; 314}else if(entry->delta_data) { 315 size =DELTA_SIZE(entry); 316 buf = entry->delta_data; 317 entry->delta_data = NULL; 318 type = (allow_ofs_delta &&DELTA(entry)->idx.offset) ? 319 OBJ_OFS_DELTA : OBJ_REF_DELTA; 320}else{ 321 buf =get_delta(entry); 322 size =DELTA_SIZE(entry); 323 type = (allow_ofs_delta &&DELTA(entry)->idx.offset) ? 324 OBJ_OFS_DELTA : OBJ_REF_DELTA; 325} 326 327if(st)/* large blob case, just assume we don't compress well */ 328 datalen = size; 329else if(entry->z_delta_size) 330 datalen = entry->z_delta_size; 331else 332 datalen =do_compress(&buf, size); 333 334/* 335 * The object header is a byte of 'type' followed by zero or 336 * more bytes of length. 337 */ 338 hdrlen =encode_in_pack_object_header(header,sizeof(header), 339 type, size); 340 341if(type == OBJ_OFS_DELTA) { 342/* 343 * Deltas with relative base contain an additional 344 * encoding of the relative offset for the delta 345 * base from this object's position in the pack. 346 */ 347 off_t ofs = entry->idx.offset -DELTA(entry)->idx.offset; 348unsigned pos =sizeof(dheader) -1; 349 dheader[pos] = ofs &127; 350while(ofs >>=7) 351 dheader[--pos] =128| (--ofs &127); 352if(limit && hdrlen +sizeof(dheader) - pos + datalen + hashsz >= limit) { 353if(st) 354close_istream(st); 355free(buf); 356return0; 357} 358hashwrite(f, header, hdrlen); 359hashwrite(f, dheader + pos,sizeof(dheader) - pos); 360 hdrlen +=sizeof(dheader) - pos; 361}else if(type == OBJ_REF_DELTA) { 362/* 363 * Deltas with a base reference contain 364 * additional bytes for the base object ID. 365 */ 366if(limit && hdrlen + hashsz + datalen + hashsz >= limit) { 367if(st) 368close_istream(st); 369free(buf); 370return0; 371} 372hashwrite(f, header, hdrlen); 373hashwrite(f,DELTA(entry)->idx.oid.hash, hashsz); 374 hdrlen += hashsz; 375}else{ 376if(limit && hdrlen + datalen + hashsz >= limit) { 377if(st) 378close_istream(st); 379free(buf); 380return0; 381} 382hashwrite(f, header, hdrlen); 383} 384if(st) { 385 datalen =write_large_blob_data(st, f, &entry->idx.oid); 386close_istream(st); 387}else{ 388hashwrite(f, buf, datalen); 389free(buf); 390} 391 392return hdrlen + datalen; 393} 394 395/* Return 0 if we will bust the pack-size limit */ 396static off_t write_reuse_object(struct hashfile *f,struct object_entry *entry, 397unsigned long limit,int usable_delta) 398{ 399struct packed_git *p =IN_PACK(entry); 400struct pack_window *w_curs = NULL; 401struct revindex_entry *revidx; 402 off_t offset; 403enum object_type type =oe_type(entry); 404 off_t datalen; 405unsigned char header[MAX_PACK_OBJECT_HEADER], 406 dheader[MAX_PACK_OBJECT_HEADER]; 407unsigned hdrlen; 408const unsigned hashsz = the_hash_algo->rawsz; 409unsigned long entry_size =SIZE(entry); 410 411if(DELTA(entry)) 412 type = (allow_ofs_delta &&DELTA(entry)->idx.offset) ? 413 OBJ_OFS_DELTA : OBJ_REF_DELTA; 414 hdrlen =encode_in_pack_object_header(header,sizeof(header), 415 type, entry_size); 416 417 offset = entry->in_pack_offset; 418 revidx =find_pack_revindex(p, offset); 419 datalen = revidx[1].offset - offset; 420if(!pack_to_stdout && p->index_version >1&& 421check_pack_crc(p, &w_curs, offset, datalen, revidx->nr)) { 422error(_("bad packed object CRC for%s"), 423oid_to_hex(&entry->idx.oid)); 424unuse_pack(&w_curs); 425returnwrite_no_reuse_object(f, entry, limit, usable_delta); 426} 427 428 offset += entry->in_pack_header_size; 429 datalen -= entry->in_pack_header_size; 430 431if(!pack_to_stdout && p->index_version ==1&& 432check_pack_inflate(p, &w_curs, offset, datalen, entry_size)) { 433error(_("corrupt packed object for%s"), 434oid_to_hex(&entry->idx.oid)); 435unuse_pack(&w_curs); 436returnwrite_no_reuse_object(f, entry, limit, usable_delta); 437} 438 439if(type == OBJ_OFS_DELTA) { 440 off_t ofs = entry->idx.offset -DELTA(entry)->idx.offset; 441unsigned pos =sizeof(dheader) -1; 442 dheader[pos] = ofs &127; 443while(ofs >>=7) 444 dheader[--pos] =128| (--ofs &127); 445if(limit && hdrlen +sizeof(dheader) - pos + datalen + hashsz >= limit) { 446unuse_pack(&w_curs); 447return0; 448} 449hashwrite(f, header, hdrlen); 450hashwrite(f, dheader + pos,sizeof(dheader) - pos); 451 hdrlen +=sizeof(dheader) - pos; 452 reused_delta++; 453}else if(type == OBJ_REF_DELTA) { 454if(limit && hdrlen + hashsz + datalen + hashsz >= limit) { 455unuse_pack(&w_curs); 456return0; 457} 458hashwrite(f, header, hdrlen); 459hashwrite(f,DELTA(entry)->idx.oid.hash, hashsz); 460 hdrlen += hashsz; 461 reused_delta++; 462}else{ 463if(limit && hdrlen + datalen + hashsz >= limit) { 464unuse_pack(&w_curs); 465return0; 466} 467hashwrite(f, header, hdrlen); 468} 469copy_pack_data(f, p, &w_curs, offset, datalen); 470unuse_pack(&w_curs); 471 reused++; 472return hdrlen + datalen; 473} 474 475/* Return 0 if we will bust the pack-size limit */ 476static off_t write_object(struct hashfile *f, 477struct object_entry *entry, 478 off_t write_offset) 479{ 480unsigned long limit; 481 off_t len; 482int usable_delta, to_reuse; 483 484if(!pack_to_stdout) 485crc32_begin(f); 486 487/* apply size limit if limited packsize and not first object */ 488if(!pack_size_limit || !nr_written) 489 limit =0; 490else if(pack_size_limit <= write_offset) 491/* 492 * the earlier object did not fit the limit; avoid 493 * mistaking this with unlimited (i.e. limit = 0). 494 */ 495 limit =1; 496else 497 limit = pack_size_limit - write_offset; 498 499if(!DELTA(entry)) 500 usable_delta =0;/* no delta */ 501else if(!pack_size_limit) 502 usable_delta =1;/* unlimited packfile */ 503else if(DELTA(entry)->idx.offset == (off_t)-1) 504 usable_delta =0;/* base was written to another pack */ 505else if(DELTA(entry)->idx.offset) 506 usable_delta =1;/* base already exists in this pack */ 507else 508 usable_delta =0;/* base could end up in another pack */ 509 510if(!reuse_object) 511 to_reuse =0;/* explicit */ 512else if(!IN_PACK(entry)) 513 to_reuse =0;/* can't reuse what we don't have */ 514else if(oe_type(entry) == OBJ_REF_DELTA || 515oe_type(entry) == OBJ_OFS_DELTA) 516/* check_object() decided it for us ... */ 517 to_reuse = usable_delta; 518/* ... but pack split may override that */ 519else if(oe_type(entry) != entry->in_pack_type) 520 to_reuse =0;/* pack has delta which is unusable */ 521else if(DELTA(entry)) 522 to_reuse =0;/* we want to pack afresh */ 523else 524 to_reuse =1;/* we have it in-pack undeltified, 525 * and we do not need to deltify it. 526 */ 527 528if(!to_reuse) 529 len =write_no_reuse_object(f, entry, limit, usable_delta); 530else 531 len =write_reuse_object(f, entry, limit, usable_delta); 532if(!len) 533return0; 534 535if(usable_delta) 536 written_delta++; 537 written++; 538if(!pack_to_stdout) 539 entry->idx.crc32 =crc32_end(f); 540return len; 541} 542 543enum write_one_status { 544 WRITE_ONE_SKIP = -1,/* already written */ 545 WRITE_ONE_BREAK =0,/* writing this will bust the limit; not written */ 546 WRITE_ONE_WRITTEN =1,/* normal */ 547 WRITE_ONE_RECURSIVE =2/* already scheduled to be written */ 548}; 549 550static enum write_one_status write_one(struct hashfile *f, 551struct object_entry *e, 552 off_t *offset) 553{ 554 off_t size; 555int recursing; 556 557/* 558 * we set offset to 1 (which is an impossible value) to mark 559 * the fact that this object is involved in "write its base 560 * first before writing a deltified object" recursion. 561 */ 562 recursing = (e->idx.offset ==1); 563if(recursing) { 564warning(_("recursive delta detected for object%s"), 565oid_to_hex(&e->idx.oid)); 566return WRITE_ONE_RECURSIVE; 567}else if(e->idx.offset || e->preferred_base) { 568/* offset is non zero if object is written already. */ 569return WRITE_ONE_SKIP; 570} 571 572/* if we are deltified, write out base object first. */ 573if(DELTA(e)) { 574 e->idx.offset =1;/* now recurse */ 575switch(write_one(f,DELTA(e), offset)) { 576case WRITE_ONE_RECURSIVE: 577/* we cannot depend on this one */ 578SET_DELTA(e, NULL); 579break; 580default: 581break; 582case WRITE_ONE_BREAK: 583 e->idx.offset = recursing; 584return WRITE_ONE_BREAK; 585} 586} 587 588 e->idx.offset = *offset; 589 size =write_object(f, e, *offset); 590if(!size) { 591 e->idx.offset = recursing; 592return WRITE_ONE_BREAK; 593} 594 written_list[nr_written++] = &e->idx; 595 596/* make sure off_t is sufficiently large not to wrap */ 597if(signed_add_overflows(*offset, size)) 598die(_("pack too large for current definition of off_t")); 599*offset += size; 600return WRITE_ONE_WRITTEN; 601} 602 603static intmark_tagged(const char*path,const struct object_id *oid,int flag, 604void*cb_data) 605{ 606struct object_id peeled; 607struct object_entry *entry =packlist_find(&to_pack, oid->hash, NULL); 608 609if(entry) 610 entry->tagged =1; 611if(!peel_ref(path, &peeled)) { 612 entry =packlist_find(&to_pack, peeled.hash, NULL); 613if(entry) 614 entry->tagged =1; 615} 616return0; 617} 618 619staticinlinevoidadd_to_write_order(struct object_entry **wo, 620unsigned int*endp, 621struct object_entry *e) 622{ 623if(e->filled ||oe_layer(&to_pack, e) != write_layer) 624return; 625 wo[(*endp)++] = e; 626 e->filled =1; 627} 628 629static voidadd_descendants_to_write_order(struct object_entry **wo, 630unsigned int*endp, 631struct object_entry *e) 632{ 633int add_to_order =1; 634while(e) { 635if(add_to_order) { 636struct object_entry *s; 637/* add this node... */ 638add_to_write_order(wo, endp, e); 639/* all its siblings... */ 640for(s =DELTA_SIBLING(e); s; s =DELTA_SIBLING(s)) { 641add_to_write_order(wo, endp, s); 642} 643} 644/* drop down a level to add left subtree nodes if possible */ 645if(DELTA_CHILD(e)) { 646 add_to_order =1; 647 e =DELTA_CHILD(e); 648}else{ 649 add_to_order =0; 650/* our sibling might have some children, it is next */ 651if(DELTA_SIBLING(e)) { 652 e =DELTA_SIBLING(e); 653continue; 654} 655/* go back to our parent node */ 656 e =DELTA(e); 657while(e && !DELTA_SIBLING(e)) { 658/* we're on the right side of a subtree, keep 659 * going up until we can go right again */ 660 e =DELTA(e); 661} 662if(!e) { 663/* done- we hit our original root node */ 664return; 665} 666/* pass it off to sibling at this level */ 667 e =DELTA_SIBLING(e); 668} 669}; 670} 671 672static voidadd_family_to_write_order(struct object_entry **wo, 673unsigned int*endp, 674struct object_entry *e) 675{ 676struct object_entry *root; 677 678for(root = e;DELTA(root); root =DELTA(root)) 679;/* nothing */ 680add_descendants_to_write_order(wo, endp, root); 681} 682 683static voidcompute_layer_order(struct object_entry **wo,unsigned int*wo_end) 684{ 685unsigned int i, last_untagged; 686struct object_entry *objects = to_pack.objects; 687 688for(i =0; i < to_pack.nr_objects; i++) { 689if(objects[i].tagged) 690break; 691add_to_write_order(wo, wo_end, &objects[i]); 692} 693 last_untagged = i; 694 695/* 696 * Then fill all the tagged tips. 697 */ 698for(; i < to_pack.nr_objects; i++) { 699if(objects[i].tagged) 700add_to_write_order(wo, wo_end, &objects[i]); 701} 702 703/* 704 * And then all remaining commits and tags. 705 */ 706for(i = last_untagged; i < to_pack.nr_objects; i++) { 707if(oe_type(&objects[i]) != OBJ_COMMIT && 708oe_type(&objects[i]) != OBJ_TAG) 709continue; 710add_to_write_order(wo, wo_end, &objects[i]); 711} 712 713/* 714 * And then all the trees. 715 */ 716for(i = last_untagged; i < to_pack.nr_objects; i++) { 717if(oe_type(&objects[i]) != OBJ_TREE) 718continue; 719add_to_write_order(wo, wo_end, &objects[i]); 720} 721 722/* 723 * Finally all the rest in really tight order 724 */ 725for(i = last_untagged; i < to_pack.nr_objects; i++) { 726if(!objects[i].filled &&oe_layer(&to_pack, &objects[i]) == write_layer) 727add_family_to_write_order(wo, wo_end, &objects[i]); 728} 729} 730 731static struct object_entry **compute_write_order(void) 732{ 733uint32_t max_layers =1; 734unsigned int i, wo_end; 735 736struct object_entry **wo; 737struct object_entry *objects = to_pack.objects; 738 739for(i =0; i < to_pack.nr_objects; i++) { 740 objects[i].tagged =0; 741 objects[i].filled =0; 742SET_DELTA_CHILD(&objects[i], NULL); 743SET_DELTA_SIBLING(&objects[i], NULL); 744} 745 746/* 747 * Fully connect delta_child/delta_sibling network. 748 * Make sure delta_sibling is sorted in the original 749 * recency order. 750 */ 751for(i = to_pack.nr_objects; i >0;) { 752struct object_entry *e = &objects[--i]; 753if(!DELTA(e)) 754continue; 755/* Mark me as the first child */ 756 e->delta_sibling_idx =DELTA(e)->delta_child_idx; 757SET_DELTA_CHILD(DELTA(e), e); 758} 759 760/* 761 * Mark objects that are at the tip of tags. 762 */ 763for_each_tag_ref(mark_tagged, NULL); 764 765if(use_delta_islands) 766 max_layers =compute_pack_layers(&to_pack); 767 768ALLOC_ARRAY(wo, to_pack.nr_objects); 769 wo_end =0; 770 771for(; write_layer < max_layers; ++write_layer) 772compute_layer_order(wo, &wo_end); 773 774if(wo_end != to_pack.nr_objects) 775die(_("ordered%uobjects, expected %"PRIu32), 776 wo_end, to_pack.nr_objects); 777 778return wo; 779} 780 781static off_t write_reused_pack(struct hashfile *f) 782{ 783unsigned char buffer[8192]; 784 off_t to_write, total; 785int fd; 786 787if(!is_pack_valid(reuse_packfile)) 788die(_("packfile is invalid:%s"), reuse_packfile->pack_name); 789 790 fd =git_open(reuse_packfile->pack_name); 791if(fd <0) 792die_errno(_("unable to open packfile for reuse:%s"), 793 reuse_packfile->pack_name); 794 795if(lseek(fd,sizeof(struct pack_header), SEEK_SET) == -1) 796die_errno(_("unable to seek in reused packfile")); 797 798if(reuse_packfile_offset <0) 799 reuse_packfile_offset = reuse_packfile->pack_size - the_hash_algo->rawsz; 800 801 total = to_write = reuse_packfile_offset -sizeof(struct pack_header); 802 803while(to_write) { 804int read_pack =xread(fd, buffer,sizeof(buffer)); 805 806if(read_pack <=0) 807die_errno(_("unable to read from reused packfile")); 808 809if(read_pack > to_write) 810 read_pack = to_write; 811 812hashwrite(f, buffer, read_pack); 813 to_write -= read_pack; 814 815/* 816 * We don't know the actual number of objects written, 817 * only how many bytes written, how many bytes total, and 818 * how many objects total. So we can fake it by pretending all 819 * objects we are writing are the same size. This gives us a 820 * smooth progress meter, and at the end it matches the true 821 * answer. 822 */ 823 written = reuse_packfile_objects * 824(((double)(total - to_write)) / total); 825display_progress(progress_state, written); 826} 827 828close(fd); 829 written = reuse_packfile_objects; 830display_progress(progress_state, written); 831return reuse_packfile_offset -sizeof(struct pack_header); 832} 833 834static const char no_split_warning[] =N_( 835"disabling bitmap writing, packs are split due to pack.packSizeLimit" 836); 837 838static voidwrite_pack_file(void) 839{ 840uint32_t i =0, j; 841struct hashfile *f; 842 off_t offset; 843uint32_t nr_remaining = nr_result; 844time_t last_mtime =0; 845struct object_entry **write_order; 846 847if(progress > pack_to_stdout) 848 progress_state =start_progress(_("Writing objects"), nr_result); 849ALLOC_ARRAY(written_list, to_pack.nr_objects); 850 write_order =compute_write_order(); 851 852do{ 853struct object_id oid; 854char*pack_tmp_name = NULL; 855 856if(pack_to_stdout) 857 f =hashfd_throughput(1,"<stdout>", progress_state); 858else 859 f =create_tmp_packfile(&pack_tmp_name); 860 861 offset =write_pack_header(f, nr_remaining); 862 863if(reuse_packfile) { 864 off_t packfile_size; 865assert(pack_to_stdout); 866 867 packfile_size =write_reused_pack(f); 868 offset += packfile_size; 869} 870 871 nr_written =0; 872for(; i < to_pack.nr_objects; i++) { 873struct object_entry *e = write_order[i]; 874if(write_one(f, e, &offset) == WRITE_ONE_BREAK) 875break; 876display_progress(progress_state, written); 877} 878 879/* 880 * Did we write the wrong # entries in the header? 881 * If so, rewrite it like in fast-import 882 */ 883if(pack_to_stdout) { 884finalize_hashfile(f, oid.hash, CSUM_HASH_IN_STREAM | CSUM_CLOSE); 885}else if(nr_written == nr_remaining) { 886finalize_hashfile(f, oid.hash, CSUM_HASH_IN_STREAM | CSUM_FSYNC | CSUM_CLOSE); 887}else{ 888int fd =finalize_hashfile(f, oid.hash,0); 889fixup_pack_header_footer(fd, oid.hash, pack_tmp_name, 890 nr_written, oid.hash, offset); 891close(fd); 892if(write_bitmap_index) { 893warning(_(no_split_warning)); 894 write_bitmap_index =0; 895} 896} 897 898if(!pack_to_stdout) { 899struct stat st; 900struct strbuf tmpname = STRBUF_INIT; 901 902/* 903 * Packs are runtime accessed in their mtime 904 * order since newer packs are more likely to contain 905 * younger objects. So if we are creating multiple 906 * packs then we should modify the mtime of later ones 907 * to preserve this property. 908 */ 909if(stat(pack_tmp_name, &st) <0) { 910warning_errno(_("failed to stat%s"), pack_tmp_name); 911}else if(!last_mtime) { 912 last_mtime = st.st_mtime; 913}else{ 914struct utimbuf utb; 915 utb.actime = st.st_atime; 916 utb.modtime = --last_mtime; 917if(utime(pack_tmp_name, &utb) <0) 918warning_errno(_("failed utime() on%s"), pack_tmp_name); 919} 920 921strbuf_addf(&tmpname,"%s-", base_name); 922 923if(write_bitmap_index) { 924bitmap_writer_set_checksum(oid.hash); 925bitmap_writer_build_type_index( 926&to_pack, written_list, nr_written); 927} 928 929finish_tmp_packfile(&tmpname, pack_tmp_name, 930 written_list, nr_written, 931&pack_idx_opts, oid.hash); 932 933if(write_bitmap_index) { 934strbuf_addf(&tmpname,"%s.bitmap",oid_to_hex(&oid)); 935 936stop_progress(&progress_state); 937 938bitmap_writer_show_progress(progress); 939bitmap_writer_reuse_bitmaps(&to_pack); 940bitmap_writer_select_commits(indexed_commits, indexed_commits_nr, -1); 941bitmap_writer_build(&to_pack); 942bitmap_writer_finish(written_list, nr_written, 943 tmpname.buf, write_bitmap_options); 944 write_bitmap_index =0; 945} 946 947strbuf_release(&tmpname); 948free(pack_tmp_name); 949puts(oid_to_hex(&oid)); 950} 951 952/* mark written objects as written to previous pack */ 953for(j =0; j < nr_written; j++) { 954 written_list[j]->offset = (off_t)-1; 955} 956 nr_remaining -= nr_written; 957}while(nr_remaining && i < to_pack.nr_objects); 958 959free(written_list); 960free(write_order); 961stop_progress(&progress_state); 962if(written != nr_result) 963die(_("wrote %"PRIu32" objects while expecting %"PRIu32), 964 written, nr_result); 965} 966 967static intno_try_delta(const char*path) 968{ 969static struct attr_check *check; 970 971if(!check) 972 check =attr_check_initl("delta", NULL); 973git_check_attr(&the_index, path, check); 974if(ATTR_FALSE(check->items[0].value)) 975return1; 976return0; 977} 978 979/* 980 * When adding an object, check whether we have already added it 981 * to our packing list. If so, we can skip. However, if we are 982 * being asked to excludei t, but the previous mention was to include 983 * it, make sure to adjust its flags and tweak our numbers accordingly. 984 * 985 * As an optimization, we pass out the index position where we would have 986 * found the item, since that saves us from having to look it up again a 987 * few lines later when we want to add the new entry. 988 */ 989static inthave_duplicate_entry(const struct object_id *oid, 990int exclude, 991uint32_t*index_pos) 992{ 993struct object_entry *entry; 994 995 entry =packlist_find(&to_pack, oid->hash, index_pos); 996if(!entry) 997return0; 998 999if(exclude) {1000if(!entry->preferred_base)1001 nr_result--;1002 entry->preferred_base =1;1003}10041005return1;1006}10071008static intwant_found_object(int exclude,struct packed_git *p)1009{1010if(exclude)1011return1;1012if(incremental)1013return0;10141015/*1016 * When asked to do --local (do not include an object that appears in a1017 * pack we borrow from elsewhere) or --honor-pack-keep (do not include1018 * an object that appears in a pack marked with .keep), finding a pack1019 * that matches the criteria is sufficient for us to decide to omit it.1020 * However, even if this pack does not satisfy the criteria, we need to1021 * make sure no copy of this object appears in _any_ pack that makes us1022 * to omit the object, so we need to check all the packs.1023 *1024 * We can however first check whether these options can possible matter;1025 * if they do not matter we know we want the object in generated pack.1026 * Otherwise, we signal "-1" at the end to tell the caller that we do1027 * not know either way, and it needs to check more packs.1028 */1029if(!ignore_packed_keep_on_disk &&1030!ignore_packed_keep_in_core &&1031(!local || !have_non_local_packs))1032return1;10331034if(local && !p->pack_local)1035return0;1036if(p->pack_local &&1037((ignore_packed_keep_on_disk && p->pack_keep) ||1038(ignore_packed_keep_in_core && p->pack_keep_in_core)))1039return0;10401041/* we don't know yet; keep looking for more packs */1042return-1;1043}10441045/*1046 * Check whether we want the object in the pack (e.g., we do not want1047 * objects found in non-local stores if the "--local" option was used).1048 *1049 * If the caller already knows an existing pack it wants to take the object1050 * from, that is passed in *found_pack and *found_offset; otherwise this1051 * function finds if there is any pack that has the object and returns the pack1052 * and its offset in these variables.1053 */1054static intwant_object_in_pack(const struct object_id *oid,1055int exclude,1056struct packed_git **found_pack,1057 off_t *found_offset)1058{1059int want;1060struct list_head *pos;1061struct multi_pack_index *m;10621063if(!exclude && local &&has_loose_object_nonlocal(oid))1064return0;10651066/*1067 * If we already know the pack object lives in, start checks from that1068 * pack - in the usual case when neither --local was given nor .keep files1069 * are present we will determine the answer right now.1070 */1071if(*found_pack) {1072 want =want_found_object(exclude, *found_pack);1073if(want != -1)1074return want;1075}10761077for(m =get_multi_pack_index(the_repository); m; m = m->next) {1078struct pack_entry e;1079if(fill_midx_entry(oid, &e, m)) {1080struct packed_git *p = e.p;1081 off_t offset;10821083if(p == *found_pack)1084 offset = *found_offset;1085else1086 offset =find_pack_entry_one(oid->hash, p);10871088if(offset) {1089if(!*found_pack) {1090if(!is_pack_valid(p))1091continue;1092*found_offset = offset;1093*found_pack = p;1094}1095 want =want_found_object(exclude, p);1096if(want != -1)1097return want;1098}1099}1100}11011102list_for_each(pos,get_packed_git_mru(the_repository)) {1103struct packed_git *p =list_entry(pos,struct packed_git, mru);1104 off_t offset;11051106if(p == *found_pack)1107 offset = *found_offset;1108else1109 offset =find_pack_entry_one(oid->hash, p);11101111if(offset) {1112if(!*found_pack) {1113if(!is_pack_valid(p))1114continue;1115*found_offset = offset;1116*found_pack = p;1117}1118 want =want_found_object(exclude, p);1119if(!exclude && want >0)1120list_move(&p->mru,1121get_packed_git_mru(the_repository));1122if(want != -1)1123return want;1124}1125}11261127return1;1128}11291130static voidcreate_object_entry(const struct object_id *oid,1131enum object_type type,1132uint32_t hash,1133int exclude,1134int no_try_delta,1135uint32_t index_pos,1136struct packed_git *found_pack,1137 off_t found_offset)1138{1139struct object_entry *entry;11401141 entry =packlist_alloc(&to_pack, oid->hash, index_pos);1142 entry->hash = hash;1143oe_set_type(entry, type);1144if(exclude)1145 entry->preferred_base =1;1146else1147 nr_result++;1148if(found_pack) {1149oe_set_in_pack(&to_pack, entry, found_pack);1150 entry->in_pack_offset = found_offset;1151}11521153 entry->no_try_delta = no_try_delta;1154}11551156static const char no_closure_warning[] =N_(1157"disabling bitmap writing, as some objects are not being packed"1158);11591160static intadd_object_entry(const struct object_id *oid,enum object_type type,1161const char*name,int exclude)1162{1163struct packed_git *found_pack = NULL;1164 off_t found_offset =0;1165uint32_t index_pos;11661167display_progress(progress_state, ++nr_seen);11681169if(have_duplicate_entry(oid, exclude, &index_pos))1170return0;11711172if(!want_object_in_pack(oid, exclude, &found_pack, &found_offset)) {1173/* The pack is missing an object, so it will not have closure */1174if(write_bitmap_index) {1175warning(_(no_closure_warning));1176 write_bitmap_index =0;1177}1178return0;1179}11801181create_object_entry(oid, type,pack_name_hash(name),1182 exclude, name &&no_try_delta(name),1183 index_pos, found_pack, found_offset);1184return1;1185}11861187static intadd_object_entry_from_bitmap(const struct object_id *oid,1188enum object_type type,1189int flags,uint32_t name_hash,1190struct packed_git *pack, off_t offset)1191{1192uint32_t index_pos;11931194display_progress(progress_state, ++nr_seen);11951196if(have_duplicate_entry(oid,0, &index_pos))1197return0;11981199if(!want_object_in_pack(oid,0, &pack, &offset))1200return0;12011202create_object_entry(oid, type, name_hash,0,0, index_pos, pack, offset);1203return1;1204}12051206struct pbase_tree_cache {1207struct object_id oid;1208int ref;1209int temporary;1210void*tree_data;1211unsigned long tree_size;1212};12131214static struct pbase_tree_cache *(pbase_tree_cache[256]);1215static intpbase_tree_cache_ix(const struct object_id *oid)1216{1217return oid->hash[0] %ARRAY_SIZE(pbase_tree_cache);1218}1219static intpbase_tree_cache_ix_incr(int ix)1220{1221return(ix+1) %ARRAY_SIZE(pbase_tree_cache);1222}12231224static struct pbase_tree {1225struct pbase_tree *next;1226/* This is a phony "cache" entry; we are not1227 * going to evict it or find it through _get()1228 * mechanism -- this is for the toplevel node that1229 * would almost always change with any commit.1230 */1231struct pbase_tree_cache pcache;1232} *pbase_tree;12331234static struct pbase_tree_cache *pbase_tree_get(const struct object_id *oid)1235{1236struct pbase_tree_cache *ent, *nent;1237void*data;1238unsigned long size;1239enum object_type type;1240int neigh;1241int my_ix =pbase_tree_cache_ix(oid);1242int available_ix = -1;12431244/* pbase-tree-cache acts as a limited hashtable.1245 * your object will be found at your index or within a few1246 * slots after that slot if it is cached.1247 */1248for(neigh =0; neigh <8; neigh++) {1249 ent = pbase_tree_cache[my_ix];1250if(ent &&oideq(&ent->oid, oid)) {1251 ent->ref++;1252return ent;1253}1254else if(((available_ix <0) && (!ent || !ent->ref)) ||1255((0<= available_ix) &&1256(!ent && pbase_tree_cache[available_ix])))1257 available_ix = my_ix;1258if(!ent)1259break;1260 my_ix =pbase_tree_cache_ix_incr(my_ix);1261}12621263/* Did not find one. Either we got a bogus request or1264 * we need to read and perhaps cache.1265 */1266 data =read_object_file(oid, &type, &size);1267if(!data)1268return NULL;1269if(type != OBJ_TREE) {1270free(data);1271return NULL;1272}12731274/* We need to either cache or return a throwaway copy */12751276if(available_ix <0)1277 ent = NULL;1278else{1279 ent = pbase_tree_cache[available_ix];1280 my_ix = available_ix;1281}12821283if(!ent) {1284 nent =xmalloc(sizeof(*nent));1285 nent->temporary = (available_ix <0);1286}1287else{1288/* evict and reuse */1289free(ent->tree_data);1290 nent = ent;1291}1292oidcpy(&nent->oid, oid);1293 nent->tree_data = data;1294 nent->tree_size = size;1295 nent->ref =1;1296if(!nent->temporary)1297 pbase_tree_cache[my_ix] = nent;1298return nent;1299}13001301static voidpbase_tree_put(struct pbase_tree_cache *cache)1302{1303if(!cache->temporary) {1304 cache->ref--;1305return;1306}1307free(cache->tree_data);1308free(cache);1309}13101311static intname_cmp_len(const char*name)1312{1313int i;1314for(i =0; name[i] && name[i] !='\n'&& name[i] !='/'; i++)1315;1316return i;1317}13181319static voidadd_pbase_object(struct tree_desc *tree,1320const char*name,1321int cmplen,1322const char*fullname)1323{1324struct name_entry entry;1325int cmp;13261327while(tree_entry(tree,&entry)) {1328if(S_ISGITLINK(entry.mode))1329continue;1330 cmp =tree_entry_len(&entry) != cmplen ?1:1331memcmp(name, entry.path, cmplen);1332if(cmp >0)1333continue;1334if(cmp <0)1335return;1336if(name[cmplen] !='/') {1337add_object_entry(&entry.oid,1338object_type(entry.mode),1339 fullname,1);1340return;1341}1342if(S_ISDIR(entry.mode)) {1343struct tree_desc sub;1344struct pbase_tree_cache *tree;1345const char*down = name+cmplen+1;1346int downlen =name_cmp_len(down);13471348 tree =pbase_tree_get(&entry.oid);1349if(!tree)1350return;1351init_tree_desc(&sub, tree->tree_data, tree->tree_size);13521353add_pbase_object(&sub, down, downlen, fullname);1354pbase_tree_put(tree);1355}1356}1357}13581359static unsigned*done_pbase_paths;1360static int done_pbase_paths_num;1361static int done_pbase_paths_alloc;1362static intdone_pbase_path_pos(unsigned hash)1363{1364int lo =0;1365int hi = done_pbase_paths_num;1366while(lo < hi) {1367int mi = lo + (hi - lo) /2;1368if(done_pbase_paths[mi] == hash)1369return mi;1370if(done_pbase_paths[mi] < hash)1371 hi = mi;1372else1373 lo = mi +1;1374}1375return-lo-1;1376}13771378static intcheck_pbase_path(unsigned hash)1379{1380int pos =done_pbase_path_pos(hash);1381if(0<= pos)1382return1;1383 pos = -pos -1;1384ALLOC_GROW(done_pbase_paths,1385 done_pbase_paths_num +1,1386 done_pbase_paths_alloc);1387 done_pbase_paths_num++;1388if(pos < done_pbase_paths_num)1389MOVE_ARRAY(done_pbase_paths + pos +1, done_pbase_paths + pos,1390 done_pbase_paths_num - pos -1);1391 done_pbase_paths[pos] = hash;1392return0;1393}13941395static voidadd_preferred_base_object(const char*name)1396{1397struct pbase_tree *it;1398int cmplen;1399unsigned hash =pack_name_hash(name);14001401if(!num_preferred_base ||check_pbase_path(hash))1402return;14031404 cmplen =name_cmp_len(name);1405for(it = pbase_tree; it; it = it->next) {1406if(cmplen ==0) {1407add_object_entry(&it->pcache.oid, OBJ_TREE, NULL,1);1408}1409else{1410struct tree_desc tree;1411init_tree_desc(&tree, it->pcache.tree_data, it->pcache.tree_size);1412add_pbase_object(&tree, name, cmplen, name);1413}1414}1415}14161417static voidadd_preferred_base(struct object_id *oid)1418{1419struct pbase_tree *it;1420void*data;1421unsigned long size;1422struct object_id tree_oid;14231424if(window <= num_preferred_base++)1425return;14261427 data =read_object_with_reference(oid, tree_type, &size, &tree_oid);1428if(!data)1429return;14301431for(it = pbase_tree; it; it = it->next) {1432if(oideq(&it->pcache.oid, &tree_oid)) {1433free(data);1434return;1435}1436}14371438 it =xcalloc(1,sizeof(*it));1439 it->next = pbase_tree;1440 pbase_tree = it;14411442oidcpy(&it->pcache.oid, &tree_oid);1443 it->pcache.tree_data = data;1444 it->pcache.tree_size = size;1445}14461447static voidcleanup_preferred_base(void)1448{1449struct pbase_tree *it;1450unsigned i;14511452 it = pbase_tree;1453 pbase_tree = NULL;1454while(it) {1455struct pbase_tree *tmp = it;1456 it = tmp->next;1457free(tmp->pcache.tree_data);1458free(tmp);1459}14601461for(i =0; i <ARRAY_SIZE(pbase_tree_cache); i++) {1462if(!pbase_tree_cache[i])1463continue;1464free(pbase_tree_cache[i]->tree_data);1465FREE_AND_NULL(pbase_tree_cache[i]);1466}14671468FREE_AND_NULL(done_pbase_paths);1469 done_pbase_paths_num = done_pbase_paths_alloc =0;1470}14711472/*1473 * Return 1 iff the object specified by "delta" can be sent1474 * literally as a delta against the base in "base_sha1". If1475 * so, then *base_out will point to the entry in our packing1476 * list, or NULL if we must use the external-base list.1477 *1478 * Depth value does not matter - find_deltas() will1479 * never consider reused delta as the base object to1480 * deltify other objects against, in order to avoid1481 * circular deltas.1482 */1483static intcan_reuse_delta(const unsigned char*base_sha1,1484struct object_entry *delta,1485struct object_entry **base_out)1486{1487struct object_entry *base;14881489if(!base_sha1)1490return0;14911492/*1493 * First see if we're already sending the base (or it's explicitly in1494 * our "excluded" list).1495 */1496 base =packlist_find(&to_pack, base_sha1, NULL);1497if(base) {1498if(!in_same_island(&delta->idx.oid, &base->idx.oid))1499return0;1500*base_out = base;1501return1;1502}15031504/*1505 * Otherwise, reachability bitmaps may tell us if the receiver has it,1506 * even if it was buried too deep in history to make it into the1507 * packing list.1508 */1509if(thin &&bitmap_has_sha1_in_uninteresting(bitmap_git, base_sha1)) {1510if(use_delta_islands) {1511struct object_id base_oid;1512hashcpy(base_oid.hash, base_sha1);1513if(!in_same_island(&delta->idx.oid, &base_oid))1514return0;1515}1516*base_out = NULL;1517return1;1518}15191520return0;1521}15221523static voidcheck_object(struct object_entry *entry)1524{1525unsigned long canonical_size;15261527if(IN_PACK(entry)) {1528struct packed_git *p =IN_PACK(entry);1529struct pack_window *w_curs = NULL;1530const unsigned char*base_ref = NULL;1531struct object_entry *base_entry;1532unsigned long used, used_0;1533unsigned long avail;1534 off_t ofs;1535unsigned char*buf, c;1536enum object_type type;1537unsigned long in_pack_size;15381539 buf =use_pack(p, &w_curs, entry->in_pack_offset, &avail);15401541/*1542 * We want in_pack_type even if we do not reuse delta1543 * since non-delta representations could still be reused.1544 */1545 used =unpack_object_header_buffer(buf, avail,1546&type,1547&in_pack_size);1548if(used ==0)1549goto give_up;15501551if(type <0)1552BUG("invalid type%d", type);1553 entry->in_pack_type = type;15541555/*1556 * Determine if this is a delta and if so whether we can1557 * reuse it or not. Otherwise let's find out as cheaply as1558 * possible what the actual type and size for this object is.1559 */1560switch(entry->in_pack_type) {1561default:1562/* Not a delta hence we've already got all we need. */1563oe_set_type(entry, entry->in_pack_type);1564SET_SIZE(entry, in_pack_size);1565 entry->in_pack_header_size = used;1566if(oe_type(entry) < OBJ_COMMIT ||oe_type(entry) > OBJ_BLOB)1567goto give_up;1568unuse_pack(&w_curs);1569return;1570case OBJ_REF_DELTA:1571if(reuse_delta && !entry->preferred_base)1572 base_ref =use_pack(p, &w_curs,1573 entry->in_pack_offset + used, NULL);1574 entry->in_pack_header_size = used + the_hash_algo->rawsz;1575break;1576case OBJ_OFS_DELTA:1577 buf =use_pack(p, &w_curs,1578 entry->in_pack_offset + used, NULL);1579 used_0 =0;1580 c = buf[used_0++];1581 ofs = c &127;1582while(c &128) {1583 ofs +=1;1584if(!ofs ||MSB(ofs,7)) {1585error(_("delta base offset overflow in pack for%s"),1586oid_to_hex(&entry->idx.oid));1587goto give_up;1588}1589 c = buf[used_0++];1590 ofs = (ofs <<7) + (c &127);1591}1592 ofs = entry->in_pack_offset - ofs;1593if(ofs <=0|| ofs >= entry->in_pack_offset) {1594error(_("delta base offset out of bound for%s"),1595oid_to_hex(&entry->idx.oid));1596goto give_up;1597}1598if(reuse_delta && !entry->preferred_base) {1599struct revindex_entry *revidx;1600 revidx =find_pack_revindex(p, ofs);1601if(!revidx)1602goto give_up;1603 base_ref =nth_packed_object_sha1(p, revidx->nr);1604}1605 entry->in_pack_header_size = used + used_0;1606break;1607}16081609if(can_reuse_delta(base_ref, entry, &base_entry)) {1610oe_set_type(entry, entry->in_pack_type);1611SET_SIZE(entry, in_pack_size);/* delta size */1612SET_DELTA_SIZE(entry, in_pack_size);16131614if(base_entry) {1615SET_DELTA(entry, base_entry);1616 entry->delta_sibling_idx = base_entry->delta_child_idx;1617SET_DELTA_CHILD(base_entry, entry);1618}else{1619SET_DELTA_EXT(entry, base_ref);1620}16211622unuse_pack(&w_curs);1623return;1624}16251626if(oe_type(entry)) {1627 off_t delta_pos;16281629/*1630 * This must be a delta and we already know what the1631 * final object type is. Let's extract the actual1632 * object size from the delta header.1633 */1634 delta_pos = entry->in_pack_offset + entry->in_pack_header_size;1635 canonical_size =get_size_from_delta(p, &w_curs, delta_pos);1636if(canonical_size ==0)1637goto give_up;1638SET_SIZE(entry, canonical_size);1639unuse_pack(&w_curs);1640return;1641}16421643/*1644 * No choice but to fall back to the recursive delta walk1645 * with sha1_object_info() to find about the object type1646 * at this point...1647 */1648 give_up:1649unuse_pack(&w_curs);1650}16511652oe_set_type(entry,1653oid_object_info(the_repository, &entry->idx.oid, &canonical_size));1654if(entry->type_valid) {1655SET_SIZE(entry, canonical_size);1656}else{1657/*1658 * Bad object type is checked in prepare_pack(). This is1659 * to permit a missing preferred base object to be ignored1660 * as a preferred base. Doing so can result in a larger1661 * pack file, but the transfer will still take place.1662 */1663}1664}16651666static intpack_offset_sort(const void*_a,const void*_b)1667{1668const struct object_entry *a = *(struct object_entry **)_a;1669const struct object_entry *b = *(struct object_entry **)_b;1670const struct packed_git *a_in_pack =IN_PACK(a);1671const struct packed_git *b_in_pack =IN_PACK(b);16721673/* avoid filesystem trashing with loose objects */1674if(!a_in_pack && !b_in_pack)1675returnoidcmp(&a->idx.oid, &b->idx.oid);16761677if(a_in_pack < b_in_pack)1678return-1;1679if(a_in_pack > b_in_pack)1680return1;1681return a->in_pack_offset < b->in_pack_offset ? -1:1682(a->in_pack_offset > b->in_pack_offset);1683}16841685/*1686 * Drop an on-disk delta we were planning to reuse. Naively, this would1687 * just involve blanking out the "delta" field, but we have to deal1688 * with some extra book-keeping:1689 *1690 * 1. Removing ourselves from the delta_sibling linked list.1691 *1692 * 2. Updating our size/type to the non-delta representation. These were1693 * either not recorded initially (size) or overwritten with the delta type1694 * (type) when check_object() decided to reuse the delta.1695 *1696 * 3. Resetting our delta depth, as we are now a base object.1697 */1698static voiddrop_reused_delta(struct object_entry *entry)1699{1700unsigned*idx = &to_pack.objects[entry->delta_idx -1].delta_child_idx;1701struct object_info oi = OBJECT_INFO_INIT;1702enum object_type type;1703unsigned long size;17041705while(*idx) {1706struct object_entry *oe = &to_pack.objects[*idx -1];17071708if(oe == entry)1709*idx = oe->delta_sibling_idx;1710else1711 idx = &oe->delta_sibling_idx;1712}1713SET_DELTA(entry, NULL);1714 entry->depth =0;17151716 oi.sizep = &size;1717 oi.typep = &type;1718if(packed_object_info(the_repository,IN_PACK(entry), entry->in_pack_offset, &oi) <0) {1719/*1720 * We failed to get the info from this pack for some reason;1721 * fall back to sha1_object_info, which may find another copy.1722 * And if that fails, the error will be recorded in oe_type(entry)1723 * and dealt with in prepare_pack().1724 */1725oe_set_type(entry,1726oid_object_info(the_repository, &entry->idx.oid, &size));1727}else{1728oe_set_type(entry, type);1729}1730SET_SIZE(entry, size);1731}17321733/*1734 * Follow the chain of deltas from this entry onward, throwing away any links1735 * that cause us to hit a cycle (as determined by the DFS state flags in1736 * the entries).1737 *1738 * We also detect too-long reused chains that would violate our --depth1739 * limit.1740 */1741static voidbreak_delta_chains(struct object_entry *entry)1742{1743/*1744 * The actual depth of each object we will write is stored as an int,1745 * as it cannot exceed our int "depth" limit. But before we break1746 * changes based no that limit, we may potentially go as deep as the1747 * number of objects, which is elsewhere bounded to a uint32_t.1748 */1749uint32_t total_depth;1750struct object_entry *cur, *next;17511752for(cur = entry, total_depth =0;1753 cur;1754 cur =DELTA(cur), total_depth++) {1755if(cur->dfs_state == DFS_DONE) {1756/*1757 * We've already seen this object and know it isn't1758 * part of a cycle. We do need to append its depth1759 * to our count.1760 */1761 total_depth += cur->depth;1762break;1763}17641765/*1766 * We break cycles before looping, so an ACTIVE state (or any1767 * other cruft which made its way into the state variable)1768 * is a bug.1769 */1770if(cur->dfs_state != DFS_NONE)1771BUG("confusing delta dfs state in first pass:%d",1772 cur->dfs_state);17731774/*1775 * Now we know this is the first time we've seen the object. If1776 * it's not a delta, we're done traversing, but we'll mark it1777 * done to save time on future traversals.1778 */1779if(!DELTA(cur)) {1780 cur->dfs_state = DFS_DONE;1781break;1782}17831784/*1785 * Mark ourselves as active and see if the next step causes1786 * us to cycle to another active object. It's important to do1787 * this _before_ we loop, because it impacts where we make the1788 * cut, and thus how our total_depth counter works.1789 * E.g., We may see a partial loop like:1790 *1791 * A -> B -> C -> D -> B1792 *1793 * Cutting B->C breaks the cycle. But now the depth of A is1794 * only 1, and our total_depth counter is at 3. The size of the1795 * error is always one less than the size of the cycle we1796 * broke. Commits C and D were "lost" from A's chain.1797 *1798 * If we instead cut D->B, then the depth of A is correct at 3.1799 * We keep all commits in the chain that we examined.1800 */1801 cur->dfs_state = DFS_ACTIVE;1802if(DELTA(cur)->dfs_state == DFS_ACTIVE) {1803drop_reused_delta(cur);1804 cur->dfs_state = DFS_DONE;1805break;1806}1807}18081809/*1810 * And now that we've gone all the way to the bottom of the chain, we1811 * need to clear the active flags and set the depth fields as1812 * appropriate. Unlike the loop above, which can quit when it drops a1813 * delta, we need to keep going to look for more depth cuts. So we need1814 * an extra "next" pointer to keep going after we reset cur->delta.1815 */1816for(cur = entry; cur; cur = next) {1817 next =DELTA(cur);18181819/*1820 * We should have a chain of zero or more ACTIVE states down to1821 * a final DONE. We can quit after the DONE, because either it1822 * has no bases, or we've already handled them in a previous1823 * call.1824 */1825if(cur->dfs_state == DFS_DONE)1826break;1827else if(cur->dfs_state != DFS_ACTIVE)1828BUG("confusing delta dfs state in second pass:%d",1829 cur->dfs_state);18301831/*1832 * If the total_depth is more than depth, then we need to snip1833 * the chain into two or more smaller chains that don't exceed1834 * the maximum depth. Most of the resulting chains will contain1835 * (depth + 1) entries (i.e., depth deltas plus one base), and1836 * the last chain (i.e., the one containing entry) will contain1837 * whatever entries are left over, namely1838 * (total_depth % (depth + 1)) of them.1839 *1840 * Since we are iterating towards decreasing depth, we need to1841 * decrement total_depth as we go, and we need to write to the1842 * entry what its final depth will be after all of the1843 * snipping. Since we're snipping into chains of length (depth1844 * + 1) entries, the final depth of an entry will be its1845 * original depth modulo (depth + 1). Any time we encounter an1846 * entry whose final depth is supposed to be zero, we snip it1847 * from its delta base, thereby making it so.1848 */1849 cur->depth = (total_depth--) % (depth +1);1850if(!cur->depth)1851drop_reused_delta(cur);18521853 cur->dfs_state = DFS_DONE;1854}1855}18561857static voidget_object_details(void)1858{1859uint32_t i;1860struct object_entry **sorted_by_offset;18611862if(progress)1863 progress_state =start_progress(_("Counting objects"),1864 to_pack.nr_objects);18651866 sorted_by_offset =xcalloc(to_pack.nr_objects,sizeof(struct object_entry *));1867for(i =0; i < to_pack.nr_objects; i++)1868 sorted_by_offset[i] = to_pack.objects + i;1869QSORT(sorted_by_offset, to_pack.nr_objects, pack_offset_sort);18701871for(i =0; i < to_pack.nr_objects; i++) {1872struct object_entry *entry = sorted_by_offset[i];1873check_object(entry);1874if(entry->type_valid &&1875oe_size_greater_than(&to_pack, entry, big_file_threshold))1876 entry->no_try_delta =1;1877display_progress(progress_state, i +1);1878}1879stop_progress(&progress_state);18801881/*1882 * This must happen in a second pass, since we rely on the delta1883 * information for the whole list being completed.1884 */1885for(i =0; i < to_pack.nr_objects; i++)1886break_delta_chains(&to_pack.objects[i]);18871888free(sorted_by_offset);1889}18901891/*1892 * We search for deltas in a list sorted by type, by filename hash, and then1893 * by size, so that we see progressively smaller and smaller files.1894 * That's because we prefer deltas to be from the bigger file1895 * to the smaller -- deletes are potentially cheaper, but perhaps1896 * more importantly, the bigger file is likely the more recent1897 * one. The deepest deltas are therefore the oldest objects which are1898 * less susceptible to be accessed often.1899 */1900static inttype_size_sort(const void*_a,const void*_b)1901{1902const struct object_entry *a = *(struct object_entry **)_a;1903const struct object_entry *b = *(struct object_entry **)_b;1904enum object_type a_type =oe_type(a);1905enum object_type b_type =oe_type(b);1906unsigned long a_size =SIZE(a);1907unsigned long b_size =SIZE(b);19081909if(a_type > b_type)1910return-1;1911if(a_type < b_type)1912return1;1913if(a->hash > b->hash)1914return-1;1915if(a->hash < b->hash)1916return1;1917if(a->preferred_base > b->preferred_base)1918return-1;1919if(a->preferred_base < b->preferred_base)1920return1;1921if(use_delta_islands) {1922int island_cmp =island_delta_cmp(&a->idx.oid, &b->idx.oid);1923if(island_cmp)1924return island_cmp;1925}1926if(a_size > b_size)1927return-1;1928if(a_size < b_size)1929return1;1930return a < b ? -1: (a > b);/* newest first */1931}19321933struct unpacked {1934struct object_entry *entry;1935void*data;1936struct delta_index *index;1937unsigned depth;1938};19391940static intdelta_cacheable(unsigned long src_size,unsigned long trg_size,1941unsigned long delta_size)1942{1943if(max_delta_cache_size && delta_cache_size + delta_size > max_delta_cache_size)1944return0;19451946if(delta_size < cache_max_small_delta_size)1947return1;19481949/* cache delta, if objects are large enough compared to delta size */1950if((src_size >>20) + (trg_size >>21) > (delta_size >>10))1951return1;19521953return0;1954}19551956/* Protect delta_cache_size */1957static pthread_mutex_t cache_mutex;1958#define cache_lock() pthread_mutex_lock(&cache_mutex)1959#define cache_unlock() pthread_mutex_unlock(&cache_mutex)19601961/*1962 * Protect object list partitioning (e.g. struct thread_param) and1963 * progress_state1964 */1965static pthread_mutex_t progress_mutex;1966#define progress_lock() pthread_mutex_lock(&progress_mutex)1967#define progress_unlock() pthread_mutex_unlock(&progress_mutex)19681969/*1970 * Access to struct object_entry is unprotected since each thread owns1971 * a portion of the main object list. Just don't access object entries1972 * ahead in the list because they can be stolen and would need1973 * progress_mutex for protection.1974 */19751976/*1977 * Return the size of the object without doing any delta1978 * reconstruction (so non-deltas are true object sizes, but deltas1979 * return the size of the delta data).1980 */1981unsigned longoe_get_size_slow(struct packing_data *pack,1982const struct object_entry *e)1983{1984struct packed_git *p;1985struct pack_window *w_curs;1986unsigned char*buf;1987enum object_type type;1988unsigned long used, avail, size;19891990if(e->type_ != OBJ_OFS_DELTA && e->type_ != OBJ_REF_DELTA) {1991packing_data_lock(&to_pack);1992if(oid_object_info(the_repository, &e->idx.oid, &size) <0)1993die(_("unable to get size of%s"),1994oid_to_hex(&e->idx.oid));1995packing_data_unlock(&to_pack);1996return size;1997}19981999 p =oe_in_pack(pack, e);2000if(!p)2001BUG("when e->type is a delta, it must belong to a pack");20022003packing_data_lock(&to_pack);2004 w_curs = NULL;2005 buf =use_pack(p, &w_curs, e->in_pack_offset, &avail);2006 used =unpack_object_header_buffer(buf, avail, &type, &size);2007if(used ==0)2008die(_("unable to parse object header of%s"),2009oid_to_hex(&e->idx.oid));20102011unuse_pack(&w_curs);2012packing_data_unlock(&to_pack);2013return size;2014}20152016static inttry_delta(struct unpacked *trg,struct unpacked *src,2017unsigned max_depth,unsigned long*mem_usage)2018{2019struct object_entry *trg_entry = trg->entry;2020struct object_entry *src_entry = src->entry;2021unsigned long trg_size, src_size, delta_size, sizediff, max_size, sz;2022unsigned ref_depth;2023enum object_type type;2024void*delta_buf;20252026/* Don't bother doing diffs between different types */2027if(oe_type(trg_entry) !=oe_type(src_entry))2028return-1;20292030/*2031 * We do not bother to try a delta that we discarded on an2032 * earlier try, but only when reusing delta data. Note that2033 * src_entry that is marked as the preferred_base should always2034 * be considered, as even if we produce a suboptimal delta against2035 * it, we will still save the transfer cost, as we already know2036 * the other side has it and we won't send src_entry at all.2037 */2038if(reuse_delta &&IN_PACK(trg_entry) &&2039IN_PACK(trg_entry) ==IN_PACK(src_entry) &&2040!src_entry->preferred_base &&2041 trg_entry->in_pack_type != OBJ_REF_DELTA &&2042 trg_entry->in_pack_type != OBJ_OFS_DELTA)2043return0;20442045/* Let's not bust the allowed depth. */2046if(src->depth >= max_depth)2047return0;20482049/* Now some size filtering heuristics. */2050 trg_size =SIZE(trg_entry);2051if(!DELTA(trg_entry)) {2052 max_size = trg_size/2- the_hash_algo->rawsz;2053 ref_depth =1;2054}else{2055 max_size =DELTA_SIZE(trg_entry);2056 ref_depth = trg->depth;2057}2058 max_size = (uint64_t)max_size * (max_depth - src->depth) /2059(max_depth - ref_depth +1);2060if(max_size ==0)2061return0;2062 src_size =SIZE(src_entry);2063 sizediff = src_size < trg_size ? trg_size - src_size :0;2064if(sizediff >= max_size)2065return0;2066if(trg_size < src_size /32)2067return0;20682069if(!in_same_island(&trg->entry->idx.oid, &src->entry->idx.oid))2070return0;20712072/* Load data if not already done */2073if(!trg->data) {2074packing_data_lock(&to_pack);2075 trg->data =read_object_file(&trg_entry->idx.oid, &type, &sz);2076packing_data_unlock(&to_pack);2077if(!trg->data)2078die(_("object%scannot be read"),2079oid_to_hex(&trg_entry->idx.oid));2080if(sz != trg_size)2081die(_("object%sinconsistent object length (%"PRIuMAX" vs %"PRIuMAX")"),2082oid_to_hex(&trg_entry->idx.oid), (uintmax_t)sz,2083(uintmax_t)trg_size);2084*mem_usage += sz;2085}2086if(!src->data) {2087packing_data_lock(&to_pack);2088 src->data =read_object_file(&src_entry->idx.oid, &type, &sz);2089packing_data_unlock(&to_pack);2090if(!src->data) {2091if(src_entry->preferred_base) {2092static int warned =0;2093if(!warned++)2094warning(_("object%scannot be read"),2095oid_to_hex(&src_entry->idx.oid));2096/*2097 * Those objects are not included in the2098 * resulting pack. Be resilient and ignore2099 * them if they can't be read, in case the2100 * pack could be created nevertheless.2101 */2102return0;2103}2104die(_("object%scannot be read"),2105oid_to_hex(&src_entry->idx.oid));2106}2107if(sz != src_size)2108die(_("object%sinconsistent object length (%"PRIuMAX" vs %"PRIuMAX")"),2109oid_to_hex(&src_entry->idx.oid), (uintmax_t)sz,2110(uintmax_t)src_size);2111*mem_usage += sz;2112}2113if(!src->index) {2114 src->index =create_delta_index(src->data, src_size);2115if(!src->index) {2116static int warned =0;2117if(!warned++)2118warning(_("suboptimal pack - out of memory"));2119return0;2120}2121*mem_usage +=sizeof_delta_index(src->index);2122}21232124 delta_buf =create_delta(src->index, trg->data, trg_size, &delta_size, max_size);2125if(!delta_buf)2126return0;21272128if(DELTA(trg_entry)) {2129/* Prefer only shallower same-sized deltas. */2130if(delta_size ==DELTA_SIZE(trg_entry) &&2131 src->depth +1>= trg->depth) {2132free(delta_buf);2133return0;2134}2135}21362137/*2138 * Handle memory allocation outside of the cache2139 * accounting lock. Compiler will optimize the strangeness2140 * away when NO_PTHREADS is defined.2141 */2142free(trg_entry->delta_data);2143cache_lock();2144if(trg_entry->delta_data) {2145 delta_cache_size -=DELTA_SIZE(trg_entry);2146 trg_entry->delta_data = NULL;2147}2148if(delta_cacheable(src_size, trg_size, delta_size)) {2149 delta_cache_size += delta_size;2150cache_unlock();2151 trg_entry->delta_data =xrealloc(delta_buf, delta_size);2152}else{2153cache_unlock();2154free(delta_buf);2155}21562157SET_DELTA(trg_entry, src_entry);2158SET_DELTA_SIZE(trg_entry, delta_size);2159 trg->depth = src->depth +1;21602161return1;2162}21632164static unsigned intcheck_delta_limit(struct object_entry *me,unsigned int n)2165{2166struct object_entry *child =DELTA_CHILD(me);2167unsigned int m = n;2168while(child) {2169unsigned int c =check_delta_limit(child, n +1);2170if(m < c)2171 m = c;2172 child =DELTA_SIBLING(child);2173}2174return m;2175}21762177static unsigned longfree_unpacked(struct unpacked *n)2178{2179unsigned long freed_mem =sizeof_delta_index(n->index);2180free_delta_index(n->index);2181 n->index = NULL;2182if(n->data) {2183 freed_mem +=SIZE(n->entry);2184FREE_AND_NULL(n->data);2185}2186 n->entry = NULL;2187 n->depth =0;2188return freed_mem;2189}21902191static voidfind_deltas(struct object_entry **list,unsigned*list_size,2192int window,int depth,unsigned*processed)2193{2194uint32_t i, idx =0, count =0;2195struct unpacked *array;2196unsigned long mem_usage =0;21972198 array =xcalloc(window,sizeof(struct unpacked));21992200for(;;) {2201struct object_entry *entry;2202struct unpacked *n = array + idx;2203int j, max_depth, best_base = -1;22042205progress_lock();2206if(!*list_size) {2207progress_unlock();2208break;2209}2210 entry = *list++;2211(*list_size)--;2212if(!entry->preferred_base) {2213(*processed)++;2214display_progress(progress_state, *processed);2215}2216progress_unlock();22172218 mem_usage -=free_unpacked(n);2219 n->entry = entry;22202221while(window_memory_limit &&2222 mem_usage > window_memory_limit &&2223 count >1) {2224uint32_t tail = (idx + window - count) % window;2225 mem_usage -=free_unpacked(array + tail);2226 count--;2227}22282229/* We do not compute delta to *create* objects we are not2230 * going to pack.2231 */2232if(entry->preferred_base)2233goto next;22342235/*2236 * If the current object is at pack edge, take the depth the2237 * objects that depend on the current object into account2238 * otherwise they would become too deep.2239 */2240 max_depth = depth;2241if(DELTA_CHILD(entry)) {2242 max_depth -=check_delta_limit(entry,0);2243if(max_depth <=0)2244goto next;2245}22462247 j = window;2248while(--j >0) {2249int ret;2250uint32_t other_idx = idx + j;2251struct unpacked *m;2252if(other_idx >= window)2253 other_idx -= window;2254 m = array + other_idx;2255if(!m->entry)2256break;2257 ret =try_delta(n, m, max_depth, &mem_usage);2258if(ret <0)2259break;2260else if(ret >0)2261 best_base = other_idx;2262}22632264/*2265 * If we decided to cache the delta data, then it is best2266 * to compress it right away. First because we have to do2267 * it anyway, and doing it here while we're threaded will2268 * save a lot of time in the non threaded write phase,2269 * as well as allow for caching more deltas within2270 * the same cache size limit.2271 * ...2272 * But only if not writing to stdout, since in that case2273 * the network is most likely throttling writes anyway,2274 * and therefore it is best to go to the write phase ASAP2275 * instead, as we can afford spending more time compressing2276 * between writes at that moment.2277 */2278if(entry->delta_data && !pack_to_stdout) {2279unsigned long size;22802281 size =do_compress(&entry->delta_data,DELTA_SIZE(entry));2282if(size < (1U<< OE_Z_DELTA_BITS)) {2283 entry->z_delta_size = size;2284cache_lock();2285 delta_cache_size -=DELTA_SIZE(entry);2286 delta_cache_size += entry->z_delta_size;2287cache_unlock();2288}else{2289FREE_AND_NULL(entry->delta_data);2290 entry->z_delta_size =0;2291}2292}22932294/* if we made n a delta, and if n is already at max2295 * depth, leaving it in the window is pointless. we2296 * should evict it first.2297 */2298if(DELTA(entry) && max_depth <= n->depth)2299continue;23002301/*2302 * Move the best delta base up in the window, after the2303 * currently deltified object, to keep it longer. It will2304 * be the first base object to be attempted next.2305 */2306if(DELTA(entry)) {2307struct unpacked swap = array[best_base];2308int dist = (window + idx - best_base) % window;2309int dst = best_base;2310while(dist--) {2311int src = (dst +1) % window;2312 array[dst] = array[src];2313 dst = src;2314}2315 array[dst] = swap;2316}23172318 next:2319 idx++;2320if(count +1< window)2321 count++;2322if(idx >= window)2323 idx =0;2324}23252326for(i =0; i < window; ++i) {2327free_delta_index(array[i].index);2328free(array[i].data);2329}2330free(array);2331}23322333static voidtry_to_free_from_threads(size_t size)2334{2335packing_data_lock(&to_pack);2336release_pack_memory(size);2337packing_data_unlock(&to_pack);2338}23392340static try_to_free_t old_try_to_free_routine;23412342/*2343 * The main object list is split into smaller lists, each is handed to2344 * one worker.2345 *2346 * The main thread waits on the condition that (at least) one of the workers2347 * has stopped working (which is indicated in the .working member of2348 * struct thread_params).2349 *2350 * When a work thread has completed its work, it sets .working to 0 and2351 * signals the main thread and waits on the condition that .data_ready2352 * becomes 1.2353 *2354 * The main thread steals half of the work from the worker that has2355 * most work left to hand it to the idle worker.2356 */23572358struct thread_params {2359 pthread_t thread;2360struct object_entry **list;2361unsigned list_size;2362unsigned remaining;2363int window;2364int depth;2365int working;2366int data_ready;2367 pthread_mutex_t mutex;2368 pthread_cond_t cond;2369unsigned*processed;2370};23712372static pthread_cond_t progress_cond;23732374/*2375 * Mutex and conditional variable can't be statically-initialized on Windows.2376 */2377static voidinit_threaded_search(void)2378{2379pthread_mutex_init(&cache_mutex, NULL);2380pthread_mutex_init(&progress_mutex, NULL);2381pthread_cond_init(&progress_cond, NULL);2382 old_try_to_free_routine =set_try_to_free_routine(try_to_free_from_threads);2383}23842385static voidcleanup_threaded_search(void)2386{2387set_try_to_free_routine(old_try_to_free_routine);2388pthread_cond_destroy(&progress_cond);2389pthread_mutex_destroy(&cache_mutex);2390pthread_mutex_destroy(&progress_mutex);2391}23922393static void*threaded_find_deltas(void*arg)2394{2395struct thread_params *me = arg;23962397progress_lock();2398while(me->remaining) {2399progress_unlock();24002401find_deltas(me->list, &me->remaining,2402 me->window, me->depth, me->processed);24032404progress_lock();2405 me->working =0;2406pthread_cond_signal(&progress_cond);2407progress_unlock();24082409/*2410 * We must not set ->data_ready before we wait on the2411 * condition because the main thread may have set it to 12412 * before we get here. In order to be sure that new2413 * work is available if we see 1 in ->data_ready, it2414 * was initialized to 0 before this thread was spawned2415 * and we reset it to 0 right away.2416 */2417pthread_mutex_lock(&me->mutex);2418while(!me->data_ready)2419pthread_cond_wait(&me->cond, &me->mutex);2420 me->data_ready =0;2421pthread_mutex_unlock(&me->mutex);24222423progress_lock();2424}2425progress_unlock();2426/* leave ->working 1 so that this doesn't get more work assigned */2427return NULL;2428}24292430static voidll_find_deltas(struct object_entry **list,unsigned list_size,2431int window,int depth,unsigned*processed)2432{2433struct thread_params *p;2434int i, ret, active_threads =0;24352436init_threaded_search();24372438if(delta_search_threads <=1) {2439find_deltas(list, &list_size, window, depth, processed);2440cleanup_threaded_search();2441return;2442}2443if(progress > pack_to_stdout)2444fprintf_ln(stderr,_("Delta compression using up to%dthreads"),2445 delta_search_threads);2446 p =xcalloc(delta_search_threads,sizeof(*p));24472448/* Partition the work amongst work threads. */2449for(i =0; i < delta_search_threads; i++) {2450unsigned sub_size = list_size / (delta_search_threads - i);24512452/* don't use too small segments or no deltas will be found */2453if(sub_size <2*window && i+1< delta_search_threads)2454 sub_size =0;24552456 p[i].window = window;2457 p[i].depth = depth;2458 p[i].processed = processed;2459 p[i].working =1;2460 p[i].data_ready =0;24612462/* try to split chunks on "path" boundaries */2463while(sub_size && sub_size < list_size &&2464 list[sub_size]->hash &&2465 list[sub_size]->hash == list[sub_size-1]->hash)2466 sub_size++;24672468 p[i].list = list;2469 p[i].list_size = sub_size;2470 p[i].remaining = sub_size;24712472 list += sub_size;2473 list_size -= sub_size;2474}24752476/* Start work threads. */2477for(i =0; i < delta_search_threads; i++) {2478if(!p[i].list_size)2479continue;2480pthread_mutex_init(&p[i].mutex, NULL);2481pthread_cond_init(&p[i].cond, NULL);2482 ret =pthread_create(&p[i].thread, NULL,2483 threaded_find_deltas, &p[i]);2484if(ret)2485die(_("unable to create thread:%s"),strerror(ret));2486 active_threads++;2487}24882489/*2490 * Now let's wait for work completion. Each time a thread is done2491 * with its work, we steal half of the remaining work from the2492 * thread with the largest number of unprocessed objects and give2493 * it to that newly idle thread. This ensure good load balancing2494 * until the remaining object list segments are simply too short2495 * to be worth splitting anymore.2496 */2497while(active_threads) {2498struct thread_params *target = NULL;2499struct thread_params *victim = NULL;2500unsigned sub_size =0;25012502progress_lock();2503for(;;) {2504for(i =0; !target && i < delta_search_threads; i++)2505if(!p[i].working)2506 target = &p[i];2507if(target)2508break;2509pthread_cond_wait(&progress_cond, &progress_mutex);2510}25112512for(i =0; i < delta_search_threads; i++)2513if(p[i].remaining >2*window &&2514(!victim || victim->remaining < p[i].remaining))2515 victim = &p[i];2516if(victim) {2517 sub_size = victim->remaining /2;2518 list = victim->list + victim->list_size - sub_size;2519while(sub_size && list[0]->hash &&2520 list[0]->hash == list[-1]->hash) {2521 list++;2522 sub_size--;2523}2524if(!sub_size) {2525/*2526 * It is possible for some "paths" to have2527 * so many objects that no hash boundary2528 * might be found. Let's just steal the2529 * exact half in that case.2530 */2531 sub_size = victim->remaining /2;2532 list -= sub_size;2533}2534 target->list = list;2535 victim->list_size -= sub_size;2536 victim->remaining -= sub_size;2537}2538 target->list_size = sub_size;2539 target->remaining = sub_size;2540 target->working =1;2541progress_unlock();25422543pthread_mutex_lock(&target->mutex);2544 target->data_ready =1;2545pthread_cond_signal(&target->cond);2546pthread_mutex_unlock(&target->mutex);25472548if(!sub_size) {2549pthread_join(target->thread, NULL);2550pthread_cond_destroy(&target->cond);2551pthread_mutex_destroy(&target->mutex);2552 active_threads--;2553}2554}2555cleanup_threaded_search();2556free(p);2557}25582559static voidadd_tag_chain(const struct object_id *oid)2560{2561struct tag *tag;25622563/*2564 * We catch duplicates already in add_object_entry(), but we'd2565 * prefer to do this extra check to avoid having to parse the2566 * tag at all if we already know that it's being packed (e.g., if2567 * it was included via bitmaps, we would not have parsed it2568 * previously).2569 */2570if(packlist_find(&to_pack, oid->hash, NULL))2571return;25722573 tag =lookup_tag(the_repository, oid);2574while(1) {2575if(!tag ||parse_tag(tag) || !tag->tagged)2576die(_("unable to pack objects reachable from tag%s"),2577oid_to_hex(oid));25782579add_object_entry(&tag->object.oid, OBJ_TAG, NULL,0);25802581if(tag->tagged->type != OBJ_TAG)2582return;25832584 tag = (struct tag *)tag->tagged;2585}2586}25872588static intadd_ref_tag(const char*path,const struct object_id *oid,int flag,void*cb_data)2589{2590struct object_id peeled;25912592if(starts_with(path,"refs/tags/") &&/* is a tag? */2593!peel_ref(path, &peeled) &&/* peelable? */2594packlist_find(&to_pack, peeled.hash, NULL))/* object packed? */2595add_tag_chain(oid);2596return0;2597}25982599static voidprepare_pack(int window,int depth)2600{2601struct object_entry **delta_list;2602uint32_t i, nr_deltas;2603unsigned n;26042605if(use_delta_islands)2606resolve_tree_islands(the_repository, progress, &to_pack);26072608get_object_details();26092610/*2611 * If we're locally repacking then we need to be doubly careful2612 * from now on in order to make sure no stealth corruption gets2613 * propagated to the new pack. Clients receiving streamed packs2614 * should validate everything they get anyway so no need to incur2615 * the additional cost here in that case.2616 */2617if(!pack_to_stdout)2618 do_check_packed_object_crc =1;26192620if(!to_pack.nr_objects || !window || !depth)2621return;26222623ALLOC_ARRAY(delta_list, to_pack.nr_objects);2624 nr_deltas = n =0;26252626for(i =0; i < to_pack.nr_objects; i++) {2627struct object_entry *entry = to_pack.objects + i;26282629if(DELTA(entry))2630/* This happens if we decided to reuse existing2631 * delta from a pack. "reuse_delta &&" is implied.2632 */2633continue;26342635if(!entry->type_valid ||2636oe_size_less_than(&to_pack, entry,50))2637continue;26382639if(entry->no_try_delta)2640continue;26412642if(!entry->preferred_base) {2643 nr_deltas++;2644if(oe_type(entry) <0)2645die(_("unable to get type of object%s"),2646oid_to_hex(&entry->idx.oid));2647}else{2648if(oe_type(entry) <0) {2649/*2650 * This object is not found, but we2651 * don't have to include it anyway.2652 */2653continue;2654}2655}26562657 delta_list[n++] = entry;2658}26592660if(nr_deltas && n >1) {2661unsigned nr_done =0;2662if(progress)2663 progress_state =start_progress(_("Compressing objects"),2664 nr_deltas);2665QSORT(delta_list, n, type_size_sort);2666ll_find_deltas(delta_list, n, window+1, depth, &nr_done);2667stop_progress(&progress_state);2668if(nr_done != nr_deltas)2669die(_("inconsistency with delta count"));2670}2671free(delta_list);2672}26732674static intgit_pack_config(const char*k,const char*v,void*cb)2675{2676if(!strcmp(k,"pack.window")) {2677 window =git_config_int(k, v);2678return0;2679}2680if(!strcmp(k,"pack.windowmemory")) {2681 window_memory_limit =git_config_ulong(k, v);2682return0;2683}2684if(!strcmp(k,"pack.depth")) {2685 depth =git_config_int(k, v);2686return0;2687}2688if(!strcmp(k,"pack.deltacachesize")) {2689 max_delta_cache_size =git_config_int(k, v);2690return0;2691}2692if(!strcmp(k,"pack.deltacachelimit")) {2693 cache_max_small_delta_size =git_config_int(k, v);2694return0;2695}2696if(!strcmp(k,"pack.writebitmaphashcache")) {2697if(git_config_bool(k, v))2698 write_bitmap_options |= BITMAP_OPT_HASH_CACHE;2699else2700 write_bitmap_options &= ~BITMAP_OPT_HASH_CACHE;2701}2702if(!strcmp(k,"pack.usebitmaps")) {2703 use_bitmap_index_default =git_config_bool(k, v);2704return0;2705}2706if(!strcmp(k,"pack.threads")) {2707 delta_search_threads =git_config_int(k, v);2708if(delta_search_threads <0)2709die(_("invalid number of threads specified (%d)"),2710 delta_search_threads);2711if(!HAVE_THREADS && delta_search_threads !=1) {2712warning(_("no threads support, ignoring%s"), k);2713 delta_search_threads =0;2714}2715return0;2716}2717if(!strcmp(k,"pack.indexversion")) {2718 pack_idx_opts.version =git_config_int(k, v);2719if(pack_idx_opts.version >2)2720die(_("bad pack.indexversion=%"PRIu32),2721 pack_idx_opts.version);2722return0;2723}2724returngit_default_config(k, v, cb);2725}27262727static voidread_object_list_from_stdin(void)2728{2729char line[GIT_MAX_HEXSZ +1+ PATH_MAX +2];2730struct object_id oid;2731const char*p;27322733for(;;) {2734if(!fgets(line,sizeof(line), stdin)) {2735if(feof(stdin))2736break;2737if(!ferror(stdin))2738die("BUG: fgets returned NULL, not EOF, not error!");2739if(errno != EINTR)2740die_errno("fgets");2741clearerr(stdin);2742continue;2743}2744if(line[0] =='-') {2745if(get_oid_hex(line+1, &oid))2746die(_("expected edge object ID, got garbage:\n%s"),2747 line);2748add_preferred_base(&oid);2749continue;2750}2751if(parse_oid_hex(line, &oid, &p))2752die(_("expected object ID, got garbage:\n%s"), line);27532754add_preferred_base_object(p +1);2755add_object_entry(&oid, OBJ_NONE, p +1,0);2756}2757}27582759/* Remember to update object flag allocation in object.h */2760#define OBJECT_ADDED (1u<<20)27612762static voidshow_commit(struct commit *commit,void*data)2763{2764add_object_entry(&commit->object.oid, OBJ_COMMIT, NULL,0);2765 commit->object.flags |= OBJECT_ADDED;27662767if(write_bitmap_index)2768index_commit_for_bitmap(commit);27692770if(use_delta_islands)2771propagate_island_marks(commit);2772}27732774static voidshow_object(struct object *obj,const char*name,void*data)2775{2776add_preferred_base_object(name);2777add_object_entry(&obj->oid, obj->type, name,0);2778 obj->flags |= OBJECT_ADDED;27792780if(use_delta_islands) {2781const char*p;2782unsigned depth;2783struct object_entry *ent;27842785/* the empty string is a root tree, which is depth 0 */2786 depth = *name ?1:0;2787for(p =strchr(name,'/'); p; p =strchr(p +1,'/'))2788 depth++;27892790 ent =packlist_find(&to_pack, obj->oid.hash, NULL);2791if(ent && depth >oe_tree_depth(&to_pack, ent))2792oe_set_tree_depth(&to_pack, ent, depth);2793}2794}27952796static voidshow_object__ma_allow_any(struct object *obj,const char*name,void*data)2797{2798assert(arg_missing_action == MA_ALLOW_ANY);27992800/*2801 * Quietly ignore ALL missing objects. This avoids problems with2802 * staging them now and getting an odd error later.2803 */2804if(!has_object_file(&obj->oid))2805return;28062807show_object(obj, name, data);2808}28092810static voidshow_object__ma_allow_promisor(struct object *obj,const char*name,void*data)2811{2812assert(arg_missing_action == MA_ALLOW_PROMISOR);28132814/*2815 * Quietly ignore EXPECTED missing objects. This avoids problems with2816 * staging them now and getting an odd error later.2817 */2818if(!has_object_file(&obj->oid) &&is_promisor_object(&obj->oid))2819return;28202821show_object(obj, name, data);2822}28232824static intoption_parse_missing_action(const struct option *opt,2825const char*arg,int unset)2826{2827assert(arg);2828assert(!unset);28292830if(!strcmp(arg,"error")) {2831 arg_missing_action = MA_ERROR;2832 fn_show_object = show_object;2833return0;2834}28352836if(!strcmp(arg,"allow-any")) {2837 arg_missing_action = MA_ALLOW_ANY;2838 fetch_if_missing =0;2839 fn_show_object = show_object__ma_allow_any;2840return0;2841}28422843if(!strcmp(arg,"allow-promisor")) {2844 arg_missing_action = MA_ALLOW_PROMISOR;2845 fetch_if_missing =0;2846 fn_show_object = show_object__ma_allow_promisor;2847return0;2848}28492850die(_("invalid value for --missing"));2851return0;2852}28532854static voidshow_edge(struct commit *commit)2855{2856add_preferred_base(&commit->object.oid);2857}28582859struct in_pack_object {2860 off_t offset;2861struct object *object;2862};28632864struct in_pack {2865unsigned int alloc;2866unsigned int nr;2867struct in_pack_object *array;2868};28692870static voidmark_in_pack_object(struct object *object,struct packed_git *p,struct in_pack *in_pack)2871{2872 in_pack->array[in_pack->nr].offset =find_pack_entry_one(object->oid.hash, p);2873 in_pack->array[in_pack->nr].object = object;2874 in_pack->nr++;2875}28762877/*2878 * Compare the objects in the offset order, in order to emulate the2879 * "git rev-list --objects" output that produced the pack originally.2880 */2881static intofscmp(const void*a_,const void*b_)2882{2883struct in_pack_object *a = (struct in_pack_object *)a_;2884struct in_pack_object *b = (struct in_pack_object *)b_;28852886if(a->offset < b->offset)2887return-1;2888else if(a->offset > b->offset)2889return1;2890else2891returnoidcmp(&a->object->oid, &b->object->oid);2892}28932894static voidadd_objects_in_unpacked_packs(struct rev_info *revs)2895{2896struct packed_git *p;2897struct in_pack in_pack;2898uint32_t i;28992900memset(&in_pack,0,sizeof(in_pack));29012902for(p =get_all_packs(the_repository); p; p = p->next) {2903struct object_id oid;2904struct object *o;29052906if(!p->pack_local || p->pack_keep || p->pack_keep_in_core)2907continue;2908if(open_pack_index(p))2909die(_("cannot open pack index"));29102911ALLOC_GROW(in_pack.array,2912 in_pack.nr + p->num_objects,2913 in_pack.alloc);29142915for(i =0; i < p->num_objects; i++) {2916nth_packed_object_oid(&oid, p, i);2917 o =lookup_unknown_object(oid.hash);2918if(!(o->flags & OBJECT_ADDED))2919mark_in_pack_object(o, p, &in_pack);2920 o->flags |= OBJECT_ADDED;2921}2922}29232924if(in_pack.nr) {2925QSORT(in_pack.array, in_pack.nr, ofscmp);2926for(i =0; i < in_pack.nr; i++) {2927struct object *o = in_pack.array[i].object;2928add_object_entry(&o->oid, o->type,"",0);2929}2930}2931free(in_pack.array);2932}29332934static intadd_loose_object(const struct object_id *oid,const char*path,2935void*data)2936{2937enum object_type type =oid_object_info(the_repository, oid, NULL);29382939if(type <0) {2940warning(_("loose object at%scould not be examined"), path);2941return0;2942}29432944add_object_entry(oid, type,"",0);2945return0;2946}29472948/*2949 * We actually don't even have to worry about reachability here.2950 * add_object_entry will weed out duplicates, so we just add every2951 * loose object we find.2952 */2953static voidadd_unreachable_loose_objects(void)2954{2955for_each_loose_file_in_objdir(get_object_directory(),2956 add_loose_object,2957 NULL, NULL, NULL);2958}29592960static inthas_sha1_pack_kept_or_nonlocal(const struct object_id *oid)2961{2962static struct packed_git *last_found = (void*)1;2963struct packed_git *p;29642965 p = (last_found != (void*)1) ? last_found :2966get_all_packs(the_repository);29672968while(p) {2969if((!p->pack_local || p->pack_keep ||2970 p->pack_keep_in_core) &&2971find_pack_entry_one(oid->hash, p)) {2972 last_found = p;2973return1;2974}2975if(p == last_found)2976 p =get_all_packs(the_repository);2977else2978 p = p->next;2979if(p == last_found)2980 p = p->next;2981}2982return0;2983}29842985/*2986 * Store a list of sha1s that are should not be discarded2987 * because they are either written too recently, or are2988 * reachable from another object that was.2989 *2990 * This is filled by get_object_list.2991 */2992static struct oid_array recent_objects;29932994static intloosened_object_can_be_discarded(const struct object_id *oid,2995 timestamp_t mtime)2996{2997if(!unpack_unreachable_expiration)2998return0;2999if(mtime > unpack_unreachable_expiration)3000return0;3001if(oid_array_lookup(&recent_objects, oid) >=0)3002return0;3003return1;3004}30053006static voidloosen_unused_packed_objects(struct rev_info *revs)3007{3008struct packed_git *p;3009uint32_t i;3010struct object_id oid;30113012for(p =get_all_packs(the_repository); p; p = p->next) {3013if(!p->pack_local || p->pack_keep || p->pack_keep_in_core)3014continue;30153016if(open_pack_index(p))3017die(_("cannot open pack index"));30183019for(i =0; i < p->num_objects; i++) {3020nth_packed_object_oid(&oid, p, i);3021if(!packlist_find(&to_pack, oid.hash, NULL) &&3022!has_sha1_pack_kept_or_nonlocal(&oid) &&3023!loosened_object_can_be_discarded(&oid, p->mtime))3024if(force_object_loose(&oid, p->mtime))3025die(_("unable to force loose object"));3026}3027}3028}30293030/*3031 * This tracks any options which pack-reuse code expects to be on, or which a3032 * reader of the pack might not understand, and which would therefore prevent3033 * blind reuse of what we have on disk.3034 */3035static intpack_options_allow_reuse(void)3036{3037return pack_to_stdout &&3038 allow_ofs_delta &&3039!ignore_packed_keep_on_disk &&3040!ignore_packed_keep_in_core &&3041(!local || !have_non_local_packs) &&3042!incremental;3043}30443045static intget_object_list_from_bitmap(struct rev_info *revs)3046{3047if(!(bitmap_git =prepare_bitmap_walk(revs)))3048return-1;30493050if(pack_options_allow_reuse() &&3051!reuse_partial_packfile_from_bitmap(3052 bitmap_git,3053&reuse_packfile,3054&reuse_packfile_objects,3055&reuse_packfile_offset)) {3056assert(reuse_packfile_objects);3057 nr_result += reuse_packfile_objects;3058display_progress(progress_state, nr_result);3059}30603061traverse_bitmap_commit_list(bitmap_git, &add_object_entry_from_bitmap);3062return0;3063}30643065static voidrecord_recent_object(struct object *obj,3066const char*name,3067void*data)3068{3069oid_array_append(&recent_objects, &obj->oid);3070}30713072static voidrecord_recent_commit(struct commit *commit,void*data)3073{3074oid_array_append(&recent_objects, &commit->object.oid);3075}30763077static voidget_object_list(int ac,const char**av)3078{3079struct rev_info revs;3080struct setup_revision_opt s_r_opt = {3081.allow_exclude_promisor_objects =1,3082};3083char line[1000];3084int flags =0;3085int save_warning;30863087repo_init_revisions(the_repository, &revs, NULL);3088 save_commit_buffer =0;3089setup_revisions(ac, av, &revs, &s_r_opt);30903091/* make sure shallows are read */3092is_repository_shallow(the_repository);30933094 save_warning = warn_on_object_refname_ambiguity;3095 warn_on_object_refname_ambiguity =0;30963097while(fgets(line,sizeof(line), stdin) != NULL) {3098int len =strlen(line);3099if(len && line[len -1] =='\n')3100 line[--len] =0;3101if(!len)3102break;3103if(*line =='-') {3104if(!strcmp(line,"--not")) {3105 flags ^= UNINTERESTING;3106 write_bitmap_index =0;3107continue;3108}3109if(starts_with(line,"--shallow ")) {3110struct object_id oid;3111if(get_oid_hex(line +10, &oid))3112die("not an SHA-1 '%s'", line +10);3113register_shallow(the_repository, &oid);3114 use_bitmap_index =0;3115continue;3116}3117die(_("not a rev '%s'"), line);3118}3119if(handle_revision_arg(line, &revs, flags, REVARG_CANNOT_BE_FILENAME))3120die(_("bad revision '%s'"), line);3121}31223123 warn_on_object_refname_ambiguity = save_warning;31243125if(use_bitmap_index && !get_object_list_from_bitmap(&revs))3126return;31273128if(use_delta_islands)3129load_delta_islands(the_repository);31303131if(prepare_revision_walk(&revs))3132die(_("revision walk setup failed"));3133mark_edges_uninteresting(&revs, show_edge);31343135if(!fn_show_object)3136 fn_show_object = show_object;3137traverse_commit_list_filtered(&filter_options, &revs,3138 show_commit, fn_show_object, NULL,3139 NULL);31403141if(unpack_unreachable_expiration) {3142 revs.ignore_missing_links =1;3143if(add_unseen_recent_objects_to_traversal(&revs,3144 unpack_unreachable_expiration))3145die(_("unable to add recent objects"));3146if(prepare_revision_walk(&revs))3147die(_("revision walk setup failed"));3148traverse_commit_list(&revs, record_recent_commit,3149 record_recent_object, NULL);3150}31513152if(keep_unreachable)3153add_objects_in_unpacked_packs(&revs);3154if(pack_loose_unreachable)3155add_unreachable_loose_objects();3156if(unpack_unreachable)3157loosen_unused_packed_objects(&revs);31583159oid_array_clear(&recent_objects);3160}31613162static voidadd_extra_kept_packs(const struct string_list *names)3163{3164struct packed_git *p;31653166if(!names->nr)3167return;31683169for(p =get_all_packs(the_repository); p; p = p->next) {3170const char*name =basename(p->pack_name);3171int i;31723173if(!p->pack_local)3174continue;31753176for(i =0; i < names->nr; i++)3177if(!fspathcmp(name, names->items[i].string))3178break;31793180if(i < names->nr) {3181 p->pack_keep_in_core =1;3182 ignore_packed_keep_in_core =1;3183continue;3184}3185}3186}31873188static intoption_parse_index_version(const struct option *opt,3189const char*arg,int unset)3190{3191char*c;3192const char*val = arg;31933194BUG_ON_OPT_NEG(unset);31953196 pack_idx_opts.version =strtoul(val, &c,10);3197if(pack_idx_opts.version >2)3198die(_("unsupported index version%s"), val);3199if(*c ==','&& c[1])3200 pack_idx_opts.off32_limit =strtoul(c+1, &c,0);3201if(*c || pack_idx_opts.off32_limit &0x80000000)3202die(_("bad index version '%s'"), val);3203return0;3204}32053206static intoption_parse_unpack_unreachable(const struct option *opt,3207const char*arg,int unset)3208{3209if(unset) {3210 unpack_unreachable =0;3211 unpack_unreachable_expiration =0;3212}3213else{3214 unpack_unreachable =1;3215if(arg)3216 unpack_unreachable_expiration =approxidate(arg);3217}3218return0;3219}32203221intcmd_pack_objects(int argc,const char**argv,const char*prefix)3222{3223int use_internal_rev_list =0;3224int shallow =0;3225int all_progress_implied =0;3226struct argv_array rp = ARGV_ARRAY_INIT;3227int rev_list_unpacked =0, rev_list_all =0, rev_list_reflog =0;3228int rev_list_index =0;3229struct string_list keep_pack_list = STRING_LIST_INIT_NODUP;3230struct option pack_objects_options[] = {3231OPT_SET_INT('q',"quiet", &progress,3232N_("do not show progress meter"),0),3233OPT_SET_INT(0,"progress", &progress,3234N_("show progress meter"),1),3235OPT_SET_INT(0,"all-progress", &progress,3236N_("show progress meter during object writing phase"),2),3237OPT_BOOL(0,"all-progress-implied",3238&all_progress_implied,3239N_("similar to --all-progress when progress meter is shown")),3240{ OPTION_CALLBACK,0,"index-version", NULL,N_("<version>[,<offset>]"),3241N_("write the pack index file in the specified idx format version"),3242 PARSE_OPT_NONEG, option_parse_index_version },3243OPT_MAGNITUDE(0,"max-pack-size", &pack_size_limit,3244N_("maximum size of each output pack file")),3245OPT_BOOL(0,"local", &local,3246N_("ignore borrowed objects from alternate object store")),3247OPT_BOOL(0,"incremental", &incremental,3248N_("ignore packed objects")),3249OPT_INTEGER(0,"window", &window,3250N_("limit pack window by objects")),3251OPT_MAGNITUDE(0,"window-memory", &window_memory_limit,3252N_("limit pack window by memory in addition to object limit")),3253OPT_INTEGER(0,"depth", &depth,3254N_("maximum length of delta chain allowed in the resulting pack")),3255OPT_BOOL(0,"reuse-delta", &reuse_delta,3256N_("reuse existing deltas")),3257OPT_BOOL(0,"reuse-object", &reuse_object,3258N_("reuse existing objects")),3259OPT_BOOL(0,"delta-base-offset", &allow_ofs_delta,3260N_("use OFS_DELTA objects")),3261OPT_INTEGER(0,"threads", &delta_search_threads,3262N_("use threads when searching for best delta matches")),3263OPT_BOOL(0,"non-empty", &non_empty,3264N_("do not create an empty pack output")),3265OPT_BOOL(0,"revs", &use_internal_rev_list,3266N_("read revision arguments from standard input")),3267OPT_SET_INT_F(0,"unpacked", &rev_list_unpacked,3268N_("limit the objects to those that are not yet packed"),32691, PARSE_OPT_NONEG),3270OPT_SET_INT_F(0,"all", &rev_list_all,3271N_("include objects reachable from any reference"),32721, PARSE_OPT_NONEG),3273OPT_SET_INT_F(0,"reflog", &rev_list_reflog,3274N_("include objects referred by reflog entries"),32751, PARSE_OPT_NONEG),3276OPT_SET_INT_F(0,"indexed-objects", &rev_list_index,3277N_("include objects referred to by the index"),32781, PARSE_OPT_NONEG),3279OPT_BOOL(0,"stdout", &pack_to_stdout,3280N_("output pack to stdout")),3281OPT_BOOL(0,"include-tag", &include_tag,3282N_("include tag objects that refer to objects to be packed")),3283OPT_BOOL(0,"keep-unreachable", &keep_unreachable,3284N_("keep unreachable objects")),3285OPT_BOOL(0,"pack-loose-unreachable", &pack_loose_unreachable,3286N_("pack loose unreachable objects")),3287{ OPTION_CALLBACK,0,"unpack-unreachable", NULL,N_("time"),3288N_("unpack unreachable objects newer than <time>"),3289 PARSE_OPT_OPTARG, option_parse_unpack_unreachable },3290OPT_BOOL(0,"thin", &thin,3291N_("create thin packs")),3292OPT_BOOL(0,"shallow", &shallow,3293N_("create packs suitable for shallow fetches")),3294OPT_BOOL(0,"honor-pack-keep", &ignore_packed_keep_on_disk,3295N_("ignore packs that have companion .keep file")),3296OPT_STRING_LIST(0,"keep-pack", &keep_pack_list,N_("name"),3297N_("ignore this pack")),3298OPT_INTEGER(0,"compression", &pack_compression_level,3299N_("pack compression level")),3300OPT_SET_INT(0,"keep-true-parents", &grafts_replace_parents,3301N_("do not hide commits by grafts"),0),3302OPT_BOOL(0,"use-bitmap-index", &use_bitmap_index,3303N_("use a bitmap index if available to speed up counting objects")),3304OPT_BOOL(0,"write-bitmap-index", &write_bitmap_index,3305N_("write a bitmap index together with the pack index")),3306OPT_PARSE_LIST_OBJECTS_FILTER(&filter_options),3307{ OPTION_CALLBACK,0,"missing", NULL,N_("action"),3308N_("handling for missing objects"), PARSE_OPT_NONEG,3309 option_parse_missing_action },3310OPT_BOOL(0,"exclude-promisor-objects", &exclude_promisor_objects,3311N_("do not pack objects in promisor packfiles")),3312OPT_BOOL(0,"delta-islands", &use_delta_islands,3313N_("respect islands during delta compression")),3314OPT_END(),3315};33163317if(DFS_NUM_STATES > (1<< OE_DFS_STATE_BITS))3318BUG("too many dfs states, increase OE_DFS_STATE_BITS");33193320 read_replace_refs =0;33213322reset_pack_idx_option(&pack_idx_opts);3323git_config(git_pack_config, NULL);33243325 progress =isatty(2);3326 argc =parse_options(argc, argv, prefix, pack_objects_options,3327 pack_usage,0);33283329if(argc) {3330 base_name = argv[0];3331 argc--;3332}3333if(pack_to_stdout != !base_name || argc)3334usage_with_options(pack_usage, pack_objects_options);33353336if(depth >= (1<< OE_DEPTH_BITS)) {3337warning(_("delta chain depth%dis too deep, forcing%d"),3338 depth, (1<< OE_DEPTH_BITS) -1);3339 depth = (1<< OE_DEPTH_BITS) -1;3340}3341if(cache_max_small_delta_size >= (1U<< OE_Z_DELTA_BITS)) {3342warning(_("pack.deltaCacheLimit is too high, forcing%d"),3343(1U<< OE_Z_DELTA_BITS) -1);3344 cache_max_small_delta_size = (1U<< OE_Z_DELTA_BITS) -1;3345}33463347argv_array_push(&rp,"pack-objects");3348if(thin) {3349 use_internal_rev_list =1;3350argv_array_push(&rp, shallow3351?"--objects-edge-aggressive"3352:"--objects-edge");3353}else3354argv_array_push(&rp,"--objects");33553356if(rev_list_all) {3357 use_internal_rev_list =1;3358argv_array_push(&rp,"--all");3359}3360if(rev_list_reflog) {3361 use_internal_rev_list =1;3362argv_array_push(&rp,"--reflog");3363}3364if(rev_list_index) {3365 use_internal_rev_list =1;3366argv_array_push(&rp,"--indexed-objects");3367}3368if(rev_list_unpacked) {3369 use_internal_rev_list =1;3370argv_array_push(&rp,"--unpacked");3371}33723373if(exclude_promisor_objects) {3374 use_internal_rev_list =1;3375 fetch_if_missing =0;3376argv_array_push(&rp,"--exclude-promisor-objects");3377}3378if(unpack_unreachable || keep_unreachable || pack_loose_unreachable)3379 use_internal_rev_list =1;33803381if(!reuse_object)3382 reuse_delta =0;3383if(pack_compression_level == -1)3384 pack_compression_level = Z_DEFAULT_COMPRESSION;3385else if(pack_compression_level <0|| pack_compression_level > Z_BEST_COMPRESSION)3386die(_("bad pack compression level%d"), pack_compression_level);33873388if(!delta_search_threads)/* --threads=0 means autodetect */3389 delta_search_threads =online_cpus();33903391if(!HAVE_THREADS && delta_search_threads !=1)3392warning(_("no threads support, ignoring --threads"));3393if(!pack_to_stdout && !pack_size_limit)3394 pack_size_limit = pack_size_limit_cfg;3395if(pack_to_stdout && pack_size_limit)3396die(_("--max-pack-size cannot be used to build a pack for transfer"));3397if(pack_size_limit && pack_size_limit <1024*1024) {3398warning(_("minimum pack size limit is 1 MiB"));3399 pack_size_limit =1024*1024;3400}34013402if(!pack_to_stdout && thin)3403die(_("--thin cannot be used to build an indexable pack"));34043405if(keep_unreachable && unpack_unreachable)3406die(_("--keep-unreachable and --unpack-unreachable are incompatible"));3407if(!rev_list_all || !rev_list_reflog || !rev_list_index)3408 unpack_unreachable_expiration =0;34093410if(filter_options.choice) {3411if(!pack_to_stdout)3412die(_("cannot use --filter without --stdout"));3413 use_bitmap_index =0;3414}34153416/*3417 * "soft" reasons not to use bitmaps - for on-disk repack by default we want3418 *3419 * - to produce good pack (with bitmap index not-yet-packed objects are3420 * packed in suboptimal order).3421 *3422 * - to use more robust pack-generation codepath (avoiding possible3423 * bugs in bitmap code and possible bitmap index corruption).3424 */3425if(!pack_to_stdout)3426 use_bitmap_index_default =0;34273428if(use_bitmap_index <0)3429 use_bitmap_index = use_bitmap_index_default;34303431/* "hard" reasons not to use bitmaps; these just won't work at all */3432if(!use_internal_rev_list || (!pack_to_stdout && write_bitmap_index) ||is_repository_shallow(the_repository))3433 use_bitmap_index =0;34343435if(pack_to_stdout || !rev_list_all)3436 write_bitmap_index =0;34373438if(use_delta_islands)3439argv_array_push(&rp,"--topo-order");34403441if(progress && all_progress_implied)3442 progress =2;34433444add_extra_kept_packs(&keep_pack_list);3445if(ignore_packed_keep_on_disk) {3446struct packed_git *p;3447for(p =get_all_packs(the_repository); p; p = p->next)3448if(p->pack_local && p->pack_keep)3449break;3450if(!p)/* no keep-able packs found */3451 ignore_packed_keep_on_disk =0;3452}3453if(local) {3454/*3455 * unlike ignore_packed_keep_on_disk above, we do not3456 * want to unset "local" based on looking at packs, as3457 * it also covers non-local objects3458 */3459struct packed_git *p;3460for(p =get_all_packs(the_repository); p; p = p->next) {3461if(!p->pack_local) {3462 have_non_local_packs =1;3463break;3464}3465}3466}34673468prepare_packing_data(the_repository, &to_pack);34693470if(progress)3471 progress_state =start_progress(_("Enumerating objects"),0);3472if(!use_internal_rev_list)3473read_object_list_from_stdin();3474else{3475get_object_list(rp.argc, rp.argv);3476argv_array_clear(&rp);3477}3478cleanup_preferred_base();3479if(include_tag && nr_result)3480for_each_ref(add_ref_tag, NULL);3481stop_progress(&progress_state);34823483if(non_empty && !nr_result)3484return0;3485if(nr_result)3486prepare_pack(window, depth);3487write_pack_file();3488if(progress)3489fprintf_ln(stderr,3490_("Total %"PRIu32" (delta %"PRIu32"),"3491" reused %"PRIu32" (delta %"PRIu32")"),3492 written, written_delta, reused, reused_delta);3493return0;3494}