1#include"builtin.h" 2#include"cache.h" 3#include"repository.h" 4#include"config.h" 5#include"attr.h" 6#include"object.h" 7#include"blob.h" 8#include"commit.h" 9#include"tag.h" 10#include"tree.h" 11#include"delta.h" 12#include"pack.h" 13#include"pack-revindex.h" 14#include"csum-file.h" 15#include"tree-walk.h" 16#include"diff.h" 17#include"revision.h" 18#include"list-objects.h" 19#include"list-objects-filter.h" 20#include"list-objects-filter-options.h" 21#include"pack-objects.h" 22#include"progress.h" 23#include"refs.h" 24#include"streaming.h" 25#include"thread-utils.h" 26#include"pack-bitmap.h" 27#include"reachable.h" 28#include"sha1-array.h" 29#include"argv-array.h" 30#include"list.h" 31#include"packfile.h" 32#include"object-store.h" 33#include"dir.h" 34#include"midx.h" 35 36#define IN_PACK(obj) oe_in_pack(&to_pack, obj) 37#define SIZE(obj) oe_size(&to_pack, obj) 38#define SET_SIZE(obj,size) oe_set_size(&to_pack, obj, size) 39#define DELTA_SIZE(obj) oe_delta_size(&to_pack, obj) 40#define DELTA(obj) oe_delta(&to_pack, obj) 41#define DELTA_CHILD(obj) oe_delta_child(&to_pack, obj) 42#define DELTA_SIBLING(obj) oe_delta_sibling(&to_pack, obj) 43#define SET_DELTA(obj, val) oe_set_delta(&to_pack, obj, val) 44#define SET_DELTA_SIZE(obj, val) oe_set_delta_size(&to_pack, obj, val) 45#define SET_DELTA_CHILD(obj, val) oe_set_delta_child(&to_pack, obj, val) 46#define SET_DELTA_SIBLING(obj, val) oe_set_delta_sibling(&to_pack, obj, val) 47 48static const char*pack_usage[] = { 49N_("git pack-objects --stdout [<options>...] [< <ref-list> | < <object-list>]"), 50N_("git pack-objects [<options>...] <base-name> [< <ref-list> | < <object-list>]"), 51 NULL 52}; 53 54/* 55 * Objects we are going to pack are collected in the `to_pack` structure. 56 * It contains an array (dynamically expanded) of the object data, and a map 57 * that can resolve SHA1s to their position in the array. 58 */ 59static struct packing_data to_pack; 60 61static struct pack_idx_entry **written_list; 62static uint32_t nr_result, nr_written, nr_seen; 63 64static int non_empty; 65static int reuse_delta =1, reuse_object =1; 66static int keep_unreachable, unpack_unreachable, include_tag; 67static timestamp_t unpack_unreachable_expiration; 68static int pack_loose_unreachable; 69static int local; 70static int have_non_local_packs; 71static int incremental; 72static int ignore_packed_keep_on_disk; 73static int ignore_packed_keep_in_core; 74static int allow_ofs_delta; 75static struct pack_idx_option pack_idx_opts; 76static const char*base_name; 77static int progress =1; 78static int window =10; 79static unsigned long pack_size_limit; 80static int depth =50; 81static int delta_search_threads; 82static int pack_to_stdout; 83static int num_preferred_base; 84static struct progress *progress_state; 85 86static struct packed_git *reuse_packfile; 87static uint32_t reuse_packfile_objects; 88static off_t reuse_packfile_offset; 89 90static int use_bitmap_index_default =1; 91static int use_bitmap_index = -1; 92static int write_bitmap_index; 93static uint16_t write_bitmap_options; 94 95static int exclude_promisor_objects; 96 97static unsigned long delta_cache_size =0; 98static unsigned long max_delta_cache_size = DEFAULT_DELTA_CACHE_SIZE; 99static unsigned long cache_max_small_delta_size =1000; 100 101static unsigned long window_memory_limit =0; 102 103static struct list_objects_filter_options filter_options; 104 105enum missing_action { 106 MA_ERROR =0,/* fail if any missing objects are encountered */ 107 MA_ALLOW_ANY,/* silently allow ALL missing objects */ 108 MA_ALLOW_PROMISOR,/* silently allow all missing PROMISOR objects */ 109}; 110static enum missing_action arg_missing_action; 111static show_object_fn fn_show_object; 112 113/* 114 * stats 115 */ 116static uint32_t written, written_delta; 117static uint32_t reused, reused_delta; 118 119/* 120 * Indexed commits 121 */ 122static struct commit **indexed_commits; 123static unsigned int indexed_commits_nr; 124static unsigned int indexed_commits_alloc; 125 126static voidindex_commit_for_bitmap(struct commit *commit) 127{ 128if(indexed_commits_nr >= indexed_commits_alloc) { 129 indexed_commits_alloc = (indexed_commits_alloc +32) *2; 130REALLOC_ARRAY(indexed_commits, indexed_commits_alloc); 131} 132 133 indexed_commits[indexed_commits_nr++] = commit; 134} 135 136static void*get_delta(struct object_entry *entry) 137{ 138unsigned long size, base_size, delta_size; 139void*buf, *base_buf, *delta_buf; 140enum object_type type; 141 142 buf =read_object_file(&entry->idx.oid, &type, &size); 143if(!buf) 144die(_("unable to read%s"),oid_to_hex(&entry->idx.oid)); 145 base_buf =read_object_file(&DELTA(entry)->idx.oid, &type, 146&base_size); 147if(!base_buf) 148die("unable to read%s", 149oid_to_hex(&DELTA(entry)->idx.oid)); 150 delta_buf =diff_delta(base_buf, base_size, 151 buf, size, &delta_size,0); 152/* 153 * We succesfully computed this delta once but dropped it for 154 * memory reasons. Something is very wrong if this time we 155 * recompute and create a different delta. 156 */ 157if(!delta_buf || delta_size !=DELTA_SIZE(entry)) 158BUG("delta size changed"); 159free(buf); 160free(base_buf); 161return delta_buf; 162} 163 164static unsigned longdo_compress(void**pptr,unsigned long size) 165{ 166 git_zstream stream; 167void*in, *out; 168unsigned long maxsize; 169 170git_deflate_init(&stream, pack_compression_level); 171 maxsize =git_deflate_bound(&stream, size); 172 173 in = *pptr; 174 out =xmalloc(maxsize); 175*pptr = out; 176 177 stream.next_in = in; 178 stream.avail_in = size; 179 stream.next_out = out; 180 stream.avail_out = maxsize; 181while(git_deflate(&stream, Z_FINISH) == Z_OK) 182;/* nothing */ 183git_deflate_end(&stream); 184 185free(in); 186return stream.total_out; 187} 188 189static unsigned longwrite_large_blob_data(struct git_istream *st,struct hashfile *f, 190const struct object_id *oid) 191{ 192 git_zstream stream; 193unsigned char ibuf[1024*16]; 194unsigned char obuf[1024*16]; 195unsigned long olen =0; 196 197git_deflate_init(&stream, pack_compression_level); 198 199for(;;) { 200 ssize_t readlen; 201int zret = Z_OK; 202 readlen =read_istream(st, ibuf,sizeof(ibuf)); 203if(readlen == -1) 204die(_("unable to read%s"),oid_to_hex(oid)); 205 206 stream.next_in = ibuf; 207 stream.avail_in = readlen; 208while((stream.avail_in || readlen ==0) && 209(zret == Z_OK || zret == Z_BUF_ERROR)) { 210 stream.next_out = obuf; 211 stream.avail_out =sizeof(obuf); 212 zret =git_deflate(&stream, readlen ?0: Z_FINISH); 213hashwrite(f, obuf, stream.next_out - obuf); 214 olen += stream.next_out - obuf; 215} 216if(stream.avail_in) 217die(_("deflate error (%d)"), zret); 218if(readlen ==0) { 219if(zret != Z_STREAM_END) 220die(_("deflate error (%d)"), zret); 221break; 222} 223} 224git_deflate_end(&stream); 225return olen; 226} 227 228/* 229 * we are going to reuse the existing object data as is. make 230 * sure it is not corrupt. 231 */ 232static intcheck_pack_inflate(struct packed_git *p, 233struct pack_window **w_curs, 234 off_t offset, 235 off_t len, 236unsigned long expect) 237{ 238 git_zstream stream; 239unsigned char fakebuf[4096], *in; 240int st; 241 242memset(&stream,0,sizeof(stream)); 243git_inflate_init(&stream); 244do{ 245 in =use_pack(p, w_curs, offset, &stream.avail_in); 246 stream.next_in = in; 247 stream.next_out = fakebuf; 248 stream.avail_out =sizeof(fakebuf); 249 st =git_inflate(&stream, Z_FINISH); 250 offset += stream.next_in - in; 251}while(st == Z_OK || st == Z_BUF_ERROR); 252git_inflate_end(&stream); 253return(st == Z_STREAM_END && 254 stream.total_out == expect && 255 stream.total_in == len) ?0: -1; 256} 257 258static voidcopy_pack_data(struct hashfile *f, 259struct packed_git *p, 260struct pack_window **w_curs, 261 off_t offset, 262 off_t len) 263{ 264unsigned char*in; 265unsigned long avail; 266 267while(len) { 268 in =use_pack(p, w_curs, offset, &avail); 269if(avail > len) 270 avail = (unsigned long)len; 271hashwrite(f, in, avail); 272 offset += avail; 273 len -= avail; 274} 275} 276 277/* Return 0 if we will bust the pack-size limit */ 278static unsigned longwrite_no_reuse_object(struct hashfile *f,struct object_entry *entry, 279unsigned long limit,int usable_delta) 280{ 281unsigned long size, datalen; 282unsigned char header[MAX_PACK_OBJECT_HEADER], 283 dheader[MAX_PACK_OBJECT_HEADER]; 284unsigned hdrlen; 285enum object_type type; 286void*buf; 287struct git_istream *st = NULL; 288const unsigned hashsz = the_hash_algo->rawsz; 289 290if(!usable_delta) { 291if(oe_type(entry) == OBJ_BLOB && 292oe_size_greater_than(&to_pack, entry, big_file_threshold) && 293(st =open_istream(&entry->idx.oid, &type, &size, NULL)) != NULL) 294 buf = NULL; 295else{ 296 buf =read_object_file(&entry->idx.oid, &type, &size); 297if(!buf) 298die(_("unable to read%s"), 299oid_to_hex(&entry->idx.oid)); 300} 301/* 302 * make sure no cached delta data remains from a 303 * previous attempt before a pack split occurred. 304 */ 305FREE_AND_NULL(entry->delta_data); 306 entry->z_delta_size =0; 307}else if(entry->delta_data) { 308 size =DELTA_SIZE(entry); 309 buf = entry->delta_data; 310 entry->delta_data = NULL; 311 type = (allow_ofs_delta &&DELTA(entry)->idx.offset) ? 312 OBJ_OFS_DELTA : OBJ_REF_DELTA; 313}else{ 314 buf =get_delta(entry); 315 size =DELTA_SIZE(entry); 316 type = (allow_ofs_delta &&DELTA(entry)->idx.offset) ? 317 OBJ_OFS_DELTA : OBJ_REF_DELTA; 318} 319 320if(st)/* large blob case, just assume we don't compress well */ 321 datalen = size; 322else if(entry->z_delta_size) 323 datalen = entry->z_delta_size; 324else 325 datalen =do_compress(&buf, size); 326 327/* 328 * The object header is a byte of 'type' followed by zero or 329 * more bytes of length. 330 */ 331 hdrlen =encode_in_pack_object_header(header,sizeof(header), 332 type, size); 333 334if(type == OBJ_OFS_DELTA) { 335/* 336 * Deltas with relative base contain an additional 337 * encoding of the relative offset for the delta 338 * base from this object's position in the pack. 339 */ 340 off_t ofs = entry->idx.offset -DELTA(entry)->idx.offset; 341unsigned pos =sizeof(dheader) -1; 342 dheader[pos] = ofs &127; 343while(ofs >>=7) 344 dheader[--pos] =128| (--ofs &127); 345if(limit && hdrlen +sizeof(dheader) - pos + datalen + hashsz >= limit) { 346if(st) 347close_istream(st); 348free(buf); 349return0; 350} 351hashwrite(f, header, hdrlen); 352hashwrite(f, dheader + pos,sizeof(dheader) - pos); 353 hdrlen +=sizeof(dheader) - pos; 354}else if(type == OBJ_REF_DELTA) { 355/* 356 * Deltas with a base reference contain 357 * additional bytes for the base object ID. 358 */ 359if(limit && hdrlen + hashsz + datalen + hashsz >= limit) { 360if(st) 361close_istream(st); 362free(buf); 363return0; 364} 365hashwrite(f, header, hdrlen); 366hashwrite(f,DELTA(entry)->idx.oid.hash, hashsz); 367 hdrlen += hashsz; 368}else{ 369if(limit && hdrlen + datalen + hashsz >= limit) { 370if(st) 371close_istream(st); 372free(buf); 373return0; 374} 375hashwrite(f, header, hdrlen); 376} 377if(st) { 378 datalen =write_large_blob_data(st, f, &entry->idx.oid); 379close_istream(st); 380}else{ 381hashwrite(f, buf, datalen); 382free(buf); 383} 384 385return hdrlen + datalen; 386} 387 388/* Return 0 if we will bust the pack-size limit */ 389static off_t write_reuse_object(struct hashfile *f,struct object_entry *entry, 390unsigned long limit,int usable_delta) 391{ 392struct packed_git *p =IN_PACK(entry); 393struct pack_window *w_curs = NULL; 394struct revindex_entry *revidx; 395 off_t offset; 396enum object_type type =oe_type(entry); 397 off_t datalen; 398unsigned char header[MAX_PACK_OBJECT_HEADER], 399 dheader[MAX_PACK_OBJECT_HEADER]; 400unsigned hdrlen; 401const unsigned hashsz = the_hash_algo->rawsz; 402unsigned long entry_size =SIZE(entry); 403 404if(DELTA(entry)) 405 type = (allow_ofs_delta &&DELTA(entry)->idx.offset) ? 406 OBJ_OFS_DELTA : OBJ_REF_DELTA; 407 hdrlen =encode_in_pack_object_header(header,sizeof(header), 408 type, entry_size); 409 410 offset = entry->in_pack_offset; 411 revidx =find_pack_revindex(p, offset); 412 datalen = revidx[1].offset - offset; 413if(!pack_to_stdout && p->index_version >1&& 414check_pack_crc(p, &w_curs, offset, datalen, revidx->nr)) { 415error(_("bad packed object CRC for%s"), 416oid_to_hex(&entry->idx.oid)); 417unuse_pack(&w_curs); 418returnwrite_no_reuse_object(f, entry, limit, usable_delta); 419} 420 421 offset += entry->in_pack_header_size; 422 datalen -= entry->in_pack_header_size; 423 424if(!pack_to_stdout && p->index_version ==1&& 425check_pack_inflate(p, &w_curs, offset, datalen, entry_size)) { 426error(_("corrupt packed object for%s"), 427oid_to_hex(&entry->idx.oid)); 428unuse_pack(&w_curs); 429returnwrite_no_reuse_object(f, entry, limit, usable_delta); 430} 431 432if(type == OBJ_OFS_DELTA) { 433 off_t ofs = entry->idx.offset -DELTA(entry)->idx.offset; 434unsigned pos =sizeof(dheader) -1; 435 dheader[pos] = ofs &127; 436while(ofs >>=7) 437 dheader[--pos] =128| (--ofs &127); 438if(limit && hdrlen +sizeof(dheader) - pos + datalen + hashsz >= limit) { 439unuse_pack(&w_curs); 440return0; 441} 442hashwrite(f, header, hdrlen); 443hashwrite(f, dheader + pos,sizeof(dheader) - pos); 444 hdrlen +=sizeof(dheader) - pos; 445 reused_delta++; 446}else if(type == OBJ_REF_DELTA) { 447if(limit && hdrlen + hashsz + datalen + hashsz >= limit) { 448unuse_pack(&w_curs); 449return0; 450} 451hashwrite(f, header, hdrlen); 452hashwrite(f,DELTA(entry)->idx.oid.hash, hashsz); 453 hdrlen += hashsz; 454 reused_delta++; 455}else{ 456if(limit && hdrlen + datalen + hashsz >= limit) { 457unuse_pack(&w_curs); 458return0; 459} 460hashwrite(f, header, hdrlen); 461} 462copy_pack_data(f, p, &w_curs, offset, datalen); 463unuse_pack(&w_curs); 464 reused++; 465return hdrlen + datalen; 466} 467 468/* Return 0 if we will bust the pack-size limit */ 469static off_t write_object(struct hashfile *f, 470struct object_entry *entry, 471 off_t write_offset) 472{ 473unsigned long limit; 474 off_t len; 475int usable_delta, to_reuse; 476 477if(!pack_to_stdout) 478crc32_begin(f); 479 480/* apply size limit if limited packsize and not first object */ 481if(!pack_size_limit || !nr_written) 482 limit =0; 483else if(pack_size_limit <= write_offset) 484/* 485 * the earlier object did not fit the limit; avoid 486 * mistaking this with unlimited (i.e. limit = 0). 487 */ 488 limit =1; 489else 490 limit = pack_size_limit - write_offset; 491 492if(!DELTA(entry)) 493 usable_delta =0;/* no delta */ 494else if(!pack_size_limit) 495 usable_delta =1;/* unlimited packfile */ 496else if(DELTA(entry)->idx.offset == (off_t)-1) 497 usable_delta =0;/* base was written to another pack */ 498else if(DELTA(entry)->idx.offset) 499 usable_delta =1;/* base already exists in this pack */ 500else 501 usable_delta =0;/* base could end up in another pack */ 502 503if(!reuse_object) 504 to_reuse =0;/* explicit */ 505else if(!IN_PACK(entry)) 506 to_reuse =0;/* can't reuse what we don't have */ 507else if(oe_type(entry) == OBJ_REF_DELTA || 508oe_type(entry) == OBJ_OFS_DELTA) 509/* check_object() decided it for us ... */ 510 to_reuse = usable_delta; 511/* ... but pack split may override that */ 512else if(oe_type(entry) != entry->in_pack_type) 513 to_reuse =0;/* pack has delta which is unusable */ 514else if(DELTA(entry)) 515 to_reuse =0;/* we want to pack afresh */ 516else 517 to_reuse =1;/* we have it in-pack undeltified, 518 * and we do not need to deltify it. 519 */ 520 521if(!to_reuse) 522 len =write_no_reuse_object(f, entry, limit, usable_delta); 523else 524 len =write_reuse_object(f, entry, limit, usable_delta); 525if(!len) 526return0; 527 528if(usable_delta) 529 written_delta++; 530 written++; 531if(!pack_to_stdout) 532 entry->idx.crc32 =crc32_end(f); 533return len; 534} 535 536enum write_one_status { 537 WRITE_ONE_SKIP = -1,/* already written */ 538 WRITE_ONE_BREAK =0,/* writing this will bust the limit; not written */ 539 WRITE_ONE_WRITTEN =1,/* normal */ 540 WRITE_ONE_RECURSIVE =2/* already scheduled to be written */ 541}; 542 543static enum write_one_status write_one(struct hashfile *f, 544struct object_entry *e, 545 off_t *offset) 546{ 547 off_t size; 548int recursing; 549 550/* 551 * we set offset to 1 (which is an impossible value) to mark 552 * the fact that this object is involved in "write its base 553 * first before writing a deltified object" recursion. 554 */ 555 recursing = (e->idx.offset ==1); 556if(recursing) { 557warning(_("recursive delta detected for object%s"), 558oid_to_hex(&e->idx.oid)); 559return WRITE_ONE_RECURSIVE; 560}else if(e->idx.offset || e->preferred_base) { 561/* offset is non zero if object is written already. */ 562return WRITE_ONE_SKIP; 563} 564 565/* if we are deltified, write out base object first. */ 566if(DELTA(e)) { 567 e->idx.offset =1;/* now recurse */ 568switch(write_one(f,DELTA(e), offset)) { 569case WRITE_ONE_RECURSIVE: 570/* we cannot depend on this one */ 571SET_DELTA(e, NULL); 572break; 573default: 574break; 575case WRITE_ONE_BREAK: 576 e->idx.offset = recursing; 577return WRITE_ONE_BREAK; 578} 579} 580 581 e->idx.offset = *offset; 582 size =write_object(f, e, *offset); 583if(!size) { 584 e->idx.offset = recursing; 585return WRITE_ONE_BREAK; 586} 587 written_list[nr_written++] = &e->idx; 588 589/* make sure off_t is sufficiently large not to wrap */ 590if(signed_add_overflows(*offset, size)) 591die(_("pack too large for current definition of off_t")); 592*offset += size; 593return WRITE_ONE_WRITTEN; 594} 595 596static intmark_tagged(const char*path,const struct object_id *oid,int flag, 597void*cb_data) 598{ 599struct object_id peeled; 600struct object_entry *entry =packlist_find(&to_pack, oid->hash, NULL); 601 602if(entry) 603 entry->tagged =1; 604if(!peel_ref(path, &peeled)) { 605 entry =packlist_find(&to_pack, peeled.hash, NULL); 606if(entry) 607 entry->tagged =1; 608} 609return0; 610} 611 612staticinlinevoidadd_to_write_order(struct object_entry **wo, 613unsigned int*endp, 614struct object_entry *e) 615{ 616if(e->filled) 617return; 618 wo[(*endp)++] = e; 619 e->filled =1; 620} 621 622static voidadd_descendants_to_write_order(struct object_entry **wo, 623unsigned int*endp, 624struct object_entry *e) 625{ 626int add_to_order =1; 627while(e) { 628if(add_to_order) { 629struct object_entry *s; 630/* add this node... */ 631add_to_write_order(wo, endp, e); 632/* all its siblings... */ 633for(s =DELTA_SIBLING(e); s; s =DELTA_SIBLING(s)) { 634add_to_write_order(wo, endp, s); 635} 636} 637/* drop down a level to add left subtree nodes if possible */ 638if(DELTA_CHILD(e)) { 639 add_to_order =1; 640 e =DELTA_CHILD(e); 641}else{ 642 add_to_order =0; 643/* our sibling might have some children, it is next */ 644if(DELTA_SIBLING(e)) { 645 e =DELTA_SIBLING(e); 646continue; 647} 648/* go back to our parent node */ 649 e =DELTA(e); 650while(e && !DELTA_SIBLING(e)) { 651/* we're on the right side of a subtree, keep 652 * going up until we can go right again */ 653 e =DELTA(e); 654} 655if(!e) { 656/* done- we hit our original root node */ 657return; 658} 659/* pass it off to sibling at this level */ 660 e =DELTA_SIBLING(e); 661} 662}; 663} 664 665static voidadd_family_to_write_order(struct object_entry **wo, 666unsigned int*endp, 667struct object_entry *e) 668{ 669struct object_entry *root; 670 671for(root = e;DELTA(root); root =DELTA(root)) 672;/* nothing */ 673add_descendants_to_write_order(wo, endp, root); 674} 675 676static struct object_entry **compute_write_order(void) 677{ 678unsigned int i, wo_end, last_untagged; 679 680struct object_entry **wo; 681struct object_entry *objects = to_pack.objects; 682 683for(i =0; i < to_pack.nr_objects; i++) { 684 objects[i].tagged =0; 685 objects[i].filled =0; 686SET_DELTA_CHILD(&objects[i], NULL); 687SET_DELTA_SIBLING(&objects[i], NULL); 688} 689 690/* 691 * Fully connect delta_child/delta_sibling network. 692 * Make sure delta_sibling is sorted in the original 693 * recency order. 694 */ 695for(i = to_pack.nr_objects; i >0;) { 696struct object_entry *e = &objects[--i]; 697if(!DELTA(e)) 698continue; 699/* Mark me as the first child */ 700 e->delta_sibling_idx =DELTA(e)->delta_child_idx; 701SET_DELTA_CHILD(DELTA(e), e); 702} 703 704/* 705 * Mark objects that are at the tip of tags. 706 */ 707for_each_tag_ref(mark_tagged, NULL); 708 709/* 710 * Give the objects in the original recency order until 711 * we see a tagged tip. 712 */ 713ALLOC_ARRAY(wo, to_pack.nr_objects); 714for(i = wo_end =0; i < to_pack.nr_objects; i++) { 715if(objects[i].tagged) 716break; 717add_to_write_order(wo, &wo_end, &objects[i]); 718} 719 last_untagged = i; 720 721/* 722 * Then fill all the tagged tips. 723 */ 724for(; i < to_pack.nr_objects; i++) { 725if(objects[i].tagged) 726add_to_write_order(wo, &wo_end, &objects[i]); 727} 728 729/* 730 * And then all remaining commits and tags. 731 */ 732for(i = last_untagged; i < to_pack.nr_objects; i++) { 733if(oe_type(&objects[i]) != OBJ_COMMIT && 734oe_type(&objects[i]) != OBJ_TAG) 735continue; 736add_to_write_order(wo, &wo_end, &objects[i]); 737} 738 739/* 740 * And then all the trees. 741 */ 742for(i = last_untagged; i < to_pack.nr_objects; i++) { 743if(oe_type(&objects[i]) != OBJ_TREE) 744continue; 745add_to_write_order(wo, &wo_end, &objects[i]); 746} 747 748/* 749 * Finally all the rest in really tight order 750 */ 751for(i = last_untagged; i < to_pack.nr_objects; i++) { 752if(!objects[i].filled) 753add_family_to_write_order(wo, &wo_end, &objects[i]); 754} 755 756if(wo_end != to_pack.nr_objects) 757die(_("ordered%uobjects, expected %"PRIu32), 758 wo_end, to_pack.nr_objects); 759 760return wo; 761} 762 763static off_t write_reused_pack(struct hashfile *f) 764{ 765unsigned char buffer[8192]; 766 off_t to_write, total; 767int fd; 768 769if(!is_pack_valid(reuse_packfile)) 770die(_("packfile is invalid:%s"), reuse_packfile->pack_name); 771 772 fd =git_open(reuse_packfile->pack_name); 773if(fd <0) 774die_errno(_("unable to open packfile for reuse:%s"), 775 reuse_packfile->pack_name); 776 777if(lseek(fd,sizeof(struct pack_header), SEEK_SET) == -1) 778die_errno(_("unable to seek in reused packfile")); 779 780if(reuse_packfile_offset <0) 781 reuse_packfile_offset = reuse_packfile->pack_size - the_hash_algo->rawsz; 782 783 total = to_write = reuse_packfile_offset -sizeof(struct pack_header); 784 785while(to_write) { 786int read_pack =xread(fd, buffer,sizeof(buffer)); 787 788if(read_pack <=0) 789die_errno(_("unable to read from reused packfile")); 790 791if(read_pack > to_write) 792 read_pack = to_write; 793 794hashwrite(f, buffer, read_pack); 795 to_write -= read_pack; 796 797/* 798 * We don't know the actual number of objects written, 799 * only how many bytes written, how many bytes total, and 800 * how many objects total. So we can fake it by pretending all 801 * objects we are writing are the same size. This gives us a 802 * smooth progress meter, and at the end it matches the true 803 * answer. 804 */ 805 written = reuse_packfile_objects * 806(((double)(total - to_write)) / total); 807display_progress(progress_state, written); 808} 809 810close(fd); 811 written = reuse_packfile_objects; 812display_progress(progress_state, written); 813return reuse_packfile_offset -sizeof(struct pack_header); 814} 815 816static const char no_split_warning[] =N_( 817"disabling bitmap writing, packs are split due to pack.packSizeLimit" 818); 819 820static voidwrite_pack_file(void) 821{ 822uint32_t i =0, j; 823struct hashfile *f; 824 off_t offset; 825uint32_t nr_remaining = nr_result; 826time_t last_mtime =0; 827struct object_entry **write_order; 828 829if(progress > pack_to_stdout) 830 progress_state =start_progress(_("Writing objects"), nr_result); 831ALLOC_ARRAY(written_list, to_pack.nr_objects); 832 write_order =compute_write_order(); 833 834do{ 835struct object_id oid; 836char*pack_tmp_name = NULL; 837 838if(pack_to_stdout) 839 f =hashfd_throughput(1,"<stdout>", progress_state); 840else 841 f =create_tmp_packfile(&pack_tmp_name); 842 843 offset =write_pack_header(f, nr_remaining); 844 845if(reuse_packfile) { 846 off_t packfile_size; 847assert(pack_to_stdout); 848 849 packfile_size =write_reused_pack(f); 850 offset += packfile_size; 851} 852 853 nr_written =0; 854for(; i < to_pack.nr_objects; i++) { 855struct object_entry *e = write_order[i]; 856if(write_one(f, e, &offset) == WRITE_ONE_BREAK) 857break; 858display_progress(progress_state, written); 859} 860 861/* 862 * Did we write the wrong # entries in the header? 863 * If so, rewrite it like in fast-import 864 */ 865if(pack_to_stdout) { 866finalize_hashfile(f, oid.hash, CSUM_HASH_IN_STREAM | CSUM_CLOSE); 867}else if(nr_written == nr_remaining) { 868finalize_hashfile(f, oid.hash, CSUM_HASH_IN_STREAM | CSUM_FSYNC | CSUM_CLOSE); 869}else{ 870int fd =finalize_hashfile(f, oid.hash,0); 871fixup_pack_header_footer(fd, oid.hash, pack_tmp_name, 872 nr_written, oid.hash, offset); 873close(fd); 874if(write_bitmap_index) { 875warning(_(no_split_warning)); 876 write_bitmap_index =0; 877} 878} 879 880if(!pack_to_stdout) { 881struct stat st; 882struct strbuf tmpname = STRBUF_INIT; 883 884/* 885 * Packs are runtime accessed in their mtime 886 * order since newer packs are more likely to contain 887 * younger objects. So if we are creating multiple 888 * packs then we should modify the mtime of later ones 889 * to preserve this property. 890 */ 891if(stat(pack_tmp_name, &st) <0) { 892warning_errno(_("failed to stat%s"), pack_tmp_name); 893}else if(!last_mtime) { 894 last_mtime = st.st_mtime; 895}else{ 896struct utimbuf utb; 897 utb.actime = st.st_atime; 898 utb.modtime = --last_mtime; 899if(utime(pack_tmp_name, &utb) <0) 900warning_errno(_("failed utime() on%s"), pack_tmp_name); 901} 902 903strbuf_addf(&tmpname,"%s-", base_name); 904 905if(write_bitmap_index) { 906bitmap_writer_set_checksum(oid.hash); 907bitmap_writer_build_type_index( 908&to_pack, written_list, nr_written); 909} 910 911finish_tmp_packfile(&tmpname, pack_tmp_name, 912 written_list, nr_written, 913&pack_idx_opts, oid.hash); 914 915if(write_bitmap_index) { 916strbuf_addf(&tmpname,"%s.bitmap",oid_to_hex(&oid)); 917 918stop_progress(&progress_state); 919 920bitmap_writer_show_progress(progress); 921bitmap_writer_reuse_bitmaps(&to_pack); 922bitmap_writer_select_commits(indexed_commits, indexed_commits_nr, -1); 923bitmap_writer_build(&to_pack); 924bitmap_writer_finish(written_list, nr_written, 925 tmpname.buf, write_bitmap_options); 926 write_bitmap_index =0; 927} 928 929strbuf_release(&tmpname); 930free(pack_tmp_name); 931puts(oid_to_hex(&oid)); 932} 933 934/* mark written objects as written to previous pack */ 935for(j =0; j < nr_written; j++) { 936 written_list[j]->offset = (off_t)-1; 937} 938 nr_remaining -= nr_written; 939}while(nr_remaining && i < to_pack.nr_objects); 940 941free(written_list); 942free(write_order); 943stop_progress(&progress_state); 944if(written != nr_result) 945die(_("wrote %"PRIu32" objects while expecting %"PRIu32), 946 written, nr_result); 947} 948 949static intno_try_delta(const char*path) 950{ 951static struct attr_check *check; 952 953if(!check) 954 check =attr_check_initl("delta", NULL); 955if(git_check_attr(&the_index, path, check)) 956return0; 957if(ATTR_FALSE(check->items[0].value)) 958return1; 959return0; 960} 961 962/* 963 * When adding an object, check whether we have already added it 964 * to our packing list. If so, we can skip. However, if we are 965 * being asked to excludei t, but the previous mention was to include 966 * it, make sure to adjust its flags and tweak our numbers accordingly. 967 * 968 * As an optimization, we pass out the index position where we would have 969 * found the item, since that saves us from having to look it up again a 970 * few lines later when we want to add the new entry. 971 */ 972static inthave_duplicate_entry(const struct object_id *oid, 973int exclude, 974uint32_t*index_pos) 975{ 976struct object_entry *entry; 977 978 entry =packlist_find(&to_pack, oid->hash, index_pos); 979if(!entry) 980return0; 981 982if(exclude) { 983if(!entry->preferred_base) 984 nr_result--; 985 entry->preferred_base =1; 986} 987 988return1; 989} 990 991static intwant_found_object(int exclude,struct packed_git *p) 992{ 993if(exclude) 994return1; 995if(incremental) 996return0; 997 998/* 999 * When asked to do --local (do not include an object that appears in a1000 * pack we borrow from elsewhere) or --honor-pack-keep (do not include1001 * an object that appears in a pack marked with .keep), finding a pack1002 * that matches the criteria is sufficient for us to decide to omit it.1003 * However, even if this pack does not satisfy the criteria, we need to1004 * make sure no copy of this object appears in _any_ pack that makes us1005 * to omit the object, so we need to check all the packs.1006 *1007 * We can however first check whether these options can possible matter;1008 * if they do not matter we know we want the object in generated pack.1009 * Otherwise, we signal "-1" at the end to tell the caller that we do1010 * not know either way, and it needs to check more packs.1011 */1012if(!ignore_packed_keep_on_disk &&1013!ignore_packed_keep_in_core &&1014(!local || !have_non_local_packs))1015return1;10161017if(local && !p->pack_local)1018return0;1019if(p->pack_local &&1020((ignore_packed_keep_on_disk && p->pack_keep) ||1021(ignore_packed_keep_in_core && p->pack_keep_in_core)))1022return0;10231024/* we don't know yet; keep looking for more packs */1025return-1;1026}10271028/*1029 * Check whether we want the object in the pack (e.g., we do not want1030 * objects found in non-local stores if the "--local" option was used).1031 *1032 * If the caller already knows an existing pack it wants to take the object1033 * from, that is passed in *found_pack and *found_offset; otherwise this1034 * function finds if there is any pack that has the object and returns the pack1035 * and its offset in these variables.1036 */1037static intwant_object_in_pack(const struct object_id *oid,1038int exclude,1039struct packed_git **found_pack,1040 off_t *found_offset)1041{1042int want;1043struct list_head *pos;1044struct multi_pack_index *m;10451046if(!exclude && local &&has_loose_object_nonlocal(oid))1047return0;10481049/*1050 * If we already know the pack object lives in, start checks from that1051 * pack - in the usual case when neither --local was given nor .keep files1052 * are present we will determine the answer right now.1053 */1054if(*found_pack) {1055 want =want_found_object(exclude, *found_pack);1056if(want != -1)1057return want;1058}10591060for(m =get_multi_pack_index(the_repository); m; m = m->next) {1061struct pack_entry e;1062if(fill_midx_entry(oid, &e, m)) {1063struct packed_git *p = e.p;1064 off_t offset;10651066if(p == *found_pack)1067 offset = *found_offset;1068else1069 offset =find_pack_entry_one(oid->hash, p);10701071if(offset) {1072if(!*found_pack) {1073if(!is_pack_valid(p))1074continue;1075*found_offset = offset;1076*found_pack = p;1077}1078 want =want_found_object(exclude, p);1079if(want != -1)1080return want;1081}1082}1083}10841085list_for_each(pos,get_packed_git_mru(the_repository)) {1086struct packed_git *p =list_entry(pos,struct packed_git, mru);1087 off_t offset;10881089if(p == *found_pack)1090 offset = *found_offset;1091else1092 offset =find_pack_entry_one(oid->hash, p);10931094if(offset) {1095if(!*found_pack) {1096if(!is_pack_valid(p))1097continue;1098*found_offset = offset;1099*found_pack = p;1100}1101 want =want_found_object(exclude, p);1102if(!exclude && want >0)1103list_move(&p->mru,1104get_packed_git_mru(the_repository));1105if(want != -1)1106return want;1107}1108}11091110return1;1111}11121113static voidcreate_object_entry(const struct object_id *oid,1114enum object_type type,1115uint32_t hash,1116int exclude,1117int no_try_delta,1118uint32_t index_pos,1119struct packed_git *found_pack,1120 off_t found_offset)1121{1122struct object_entry *entry;11231124 entry =packlist_alloc(&to_pack, oid->hash, index_pos);1125 entry->hash = hash;1126oe_set_type(entry, type);1127if(exclude)1128 entry->preferred_base =1;1129else1130 nr_result++;1131if(found_pack) {1132oe_set_in_pack(&to_pack, entry, found_pack);1133 entry->in_pack_offset = found_offset;1134}11351136 entry->no_try_delta = no_try_delta;1137}11381139static const char no_closure_warning[] =N_(1140"disabling bitmap writing, as some objects are not being packed"1141);11421143static intadd_object_entry(const struct object_id *oid,enum object_type type,1144const char*name,int exclude)1145{1146struct packed_git *found_pack = NULL;1147 off_t found_offset =0;1148uint32_t index_pos;11491150display_progress(progress_state, ++nr_seen);11511152if(have_duplicate_entry(oid, exclude, &index_pos))1153return0;11541155if(!want_object_in_pack(oid, exclude, &found_pack, &found_offset)) {1156/* The pack is missing an object, so it will not have closure */1157if(write_bitmap_index) {1158warning(_(no_closure_warning));1159 write_bitmap_index =0;1160}1161return0;1162}11631164create_object_entry(oid, type,pack_name_hash(name),1165 exclude, name &&no_try_delta(name),1166 index_pos, found_pack, found_offset);1167return1;1168}11691170static intadd_object_entry_from_bitmap(const struct object_id *oid,1171enum object_type type,1172int flags,uint32_t name_hash,1173struct packed_git *pack, off_t offset)1174{1175uint32_t index_pos;11761177display_progress(progress_state, ++nr_seen);11781179if(have_duplicate_entry(oid,0, &index_pos))1180return0;11811182if(!want_object_in_pack(oid,0, &pack, &offset))1183return0;11841185create_object_entry(oid, type, name_hash,0,0, index_pos, pack, offset);1186return1;1187}11881189struct pbase_tree_cache {1190struct object_id oid;1191int ref;1192int temporary;1193void*tree_data;1194unsigned long tree_size;1195};11961197static struct pbase_tree_cache *(pbase_tree_cache[256]);1198static intpbase_tree_cache_ix(const struct object_id *oid)1199{1200return oid->hash[0] %ARRAY_SIZE(pbase_tree_cache);1201}1202static intpbase_tree_cache_ix_incr(int ix)1203{1204return(ix+1) %ARRAY_SIZE(pbase_tree_cache);1205}12061207static struct pbase_tree {1208struct pbase_tree *next;1209/* This is a phony "cache" entry; we are not1210 * going to evict it or find it through _get()1211 * mechanism -- this is for the toplevel node that1212 * would almost always change with any commit.1213 */1214struct pbase_tree_cache pcache;1215} *pbase_tree;12161217static struct pbase_tree_cache *pbase_tree_get(const struct object_id *oid)1218{1219struct pbase_tree_cache *ent, *nent;1220void*data;1221unsigned long size;1222enum object_type type;1223int neigh;1224int my_ix =pbase_tree_cache_ix(oid);1225int available_ix = -1;12261227/* pbase-tree-cache acts as a limited hashtable.1228 * your object will be found at your index or within a few1229 * slots after that slot if it is cached.1230 */1231for(neigh =0; neigh <8; neigh++) {1232 ent = pbase_tree_cache[my_ix];1233if(ent && !oidcmp(&ent->oid, oid)) {1234 ent->ref++;1235return ent;1236}1237else if(((available_ix <0) && (!ent || !ent->ref)) ||1238((0<= available_ix) &&1239(!ent && pbase_tree_cache[available_ix])))1240 available_ix = my_ix;1241if(!ent)1242break;1243 my_ix =pbase_tree_cache_ix_incr(my_ix);1244}12451246/* Did not find one. Either we got a bogus request or1247 * we need to read and perhaps cache.1248 */1249 data =read_object_file(oid, &type, &size);1250if(!data)1251return NULL;1252if(type != OBJ_TREE) {1253free(data);1254return NULL;1255}12561257/* We need to either cache or return a throwaway copy */12581259if(available_ix <0)1260 ent = NULL;1261else{1262 ent = pbase_tree_cache[available_ix];1263 my_ix = available_ix;1264}12651266if(!ent) {1267 nent =xmalloc(sizeof(*nent));1268 nent->temporary = (available_ix <0);1269}1270else{1271/* evict and reuse */1272free(ent->tree_data);1273 nent = ent;1274}1275oidcpy(&nent->oid, oid);1276 nent->tree_data = data;1277 nent->tree_size = size;1278 nent->ref =1;1279if(!nent->temporary)1280 pbase_tree_cache[my_ix] = nent;1281return nent;1282}12831284static voidpbase_tree_put(struct pbase_tree_cache *cache)1285{1286if(!cache->temporary) {1287 cache->ref--;1288return;1289}1290free(cache->tree_data);1291free(cache);1292}12931294static intname_cmp_len(const char*name)1295{1296int i;1297for(i =0; name[i] && name[i] !='\n'&& name[i] !='/'; i++)1298;1299return i;1300}13011302static voidadd_pbase_object(struct tree_desc *tree,1303const char*name,1304int cmplen,1305const char*fullname)1306{1307struct name_entry entry;1308int cmp;13091310while(tree_entry(tree,&entry)) {1311if(S_ISGITLINK(entry.mode))1312continue;1313 cmp =tree_entry_len(&entry) != cmplen ?1:1314memcmp(name, entry.path, cmplen);1315if(cmp >0)1316continue;1317if(cmp <0)1318return;1319if(name[cmplen] !='/') {1320add_object_entry(entry.oid,1321object_type(entry.mode),1322 fullname,1);1323return;1324}1325if(S_ISDIR(entry.mode)) {1326struct tree_desc sub;1327struct pbase_tree_cache *tree;1328const char*down = name+cmplen+1;1329int downlen =name_cmp_len(down);13301331 tree =pbase_tree_get(entry.oid);1332if(!tree)1333return;1334init_tree_desc(&sub, tree->tree_data, tree->tree_size);13351336add_pbase_object(&sub, down, downlen, fullname);1337pbase_tree_put(tree);1338}1339}1340}13411342static unsigned*done_pbase_paths;1343static int done_pbase_paths_num;1344static int done_pbase_paths_alloc;1345static intdone_pbase_path_pos(unsigned hash)1346{1347int lo =0;1348int hi = done_pbase_paths_num;1349while(lo < hi) {1350int mi = lo + (hi - lo) /2;1351if(done_pbase_paths[mi] == hash)1352return mi;1353if(done_pbase_paths[mi] < hash)1354 hi = mi;1355else1356 lo = mi +1;1357}1358return-lo-1;1359}13601361static intcheck_pbase_path(unsigned hash)1362{1363int pos =done_pbase_path_pos(hash);1364if(0<= pos)1365return1;1366 pos = -pos -1;1367ALLOC_GROW(done_pbase_paths,1368 done_pbase_paths_num +1,1369 done_pbase_paths_alloc);1370 done_pbase_paths_num++;1371if(pos < done_pbase_paths_num)1372MOVE_ARRAY(done_pbase_paths + pos +1, done_pbase_paths + pos,1373 done_pbase_paths_num - pos -1);1374 done_pbase_paths[pos] = hash;1375return0;1376}13771378static voidadd_preferred_base_object(const char*name)1379{1380struct pbase_tree *it;1381int cmplen;1382unsigned hash =pack_name_hash(name);13831384if(!num_preferred_base ||check_pbase_path(hash))1385return;13861387 cmplen =name_cmp_len(name);1388for(it = pbase_tree; it; it = it->next) {1389if(cmplen ==0) {1390add_object_entry(&it->pcache.oid, OBJ_TREE, NULL,1);1391}1392else{1393struct tree_desc tree;1394init_tree_desc(&tree, it->pcache.tree_data, it->pcache.tree_size);1395add_pbase_object(&tree, name, cmplen, name);1396}1397}1398}13991400static voidadd_preferred_base(struct object_id *oid)1401{1402struct pbase_tree *it;1403void*data;1404unsigned long size;1405struct object_id tree_oid;14061407if(window <= num_preferred_base++)1408return;14091410 data =read_object_with_reference(oid, tree_type, &size, &tree_oid);1411if(!data)1412return;14131414for(it = pbase_tree; it; it = it->next) {1415if(!oidcmp(&it->pcache.oid, &tree_oid)) {1416free(data);1417return;1418}1419}14201421 it =xcalloc(1,sizeof(*it));1422 it->next = pbase_tree;1423 pbase_tree = it;14241425oidcpy(&it->pcache.oid, &tree_oid);1426 it->pcache.tree_data = data;1427 it->pcache.tree_size = size;1428}14291430static voidcleanup_preferred_base(void)1431{1432struct pbase_tree *it;1433unsigned i;14341435 it = pbase_tree;1436 pbase_tree = NULL;1437while(it) {1438struct pbase_tree *tmp = it;1439 it = tmp->next;1440free(tmp->pcache.tree_data);1441free(tmp);1442}14431444for(i =0; i <ARRAY_SIZE(pbase_tree_cache); i++) {1445if(!pbase_tree_cache[i])1446continue;1447free(pbase_tree_cache[i]->tree_data);1448FREE_AND_NULL(pbase_tree_cache[i]);1449}14501451FREE_AND_NULL(done_pbase_paths);1452 done_pbase_paths_num = done_pbase_paths_alloc =0;1453}14541455static voidcheck_object(struct object_entry *entry)1456{1457unsigned long canonical_size;14581459if(IN_PACK(entry)) {1460struct packed_git *p =IN_PACK(entry);1461struct pack_window *w_curs = NULL;1462const unsigned char*base_ref = NULL;1463struct object_entry *base_entry;1464unsigned long used, used_0;1465unsigned long avail;1466 off_t ofs;1467unsigned char*buf, c;1468enum object_type type;1469unsigned long in_pack_size;14701471 buf =use_pack(p, &w_curs, entry->in_pack_offset, &avail);14721473/*1474 * We want in_pack_type even if we do not reuse delta1475 * since non-delta representations could still be reused.1476 */1477 used =unpack_object_header_buffer(buf, avail,1478&type,1479&in_pack_size);1480if(used ==0)1481goto give_up;14821483if(type <0)1484BUG("invalid type%d", type);1485 entry->in_pack_type = type;14861487/*1488 * Determine if this is a delta and if so whether we can1489 * reuse it or not. Otherwise let's find out as cheaply as1490 * possible what the actual type and size for this object is.1491 */1492switch(entry->in_pack_type) {1493default:1494/* Not a delta hence we've already got all we need. */1495oe_set_type(entry, entry->in_pack_type);1496SET_SIZE(entry, in_pack_size);1497 entry->in_pack_header_size = used;1498if(oe_type(entry) < OBJ_COMMIT ||oe_type(entry) > OBJ_BLOB)1499goto give_up;1500unuse_pack(&w_curs);1501return;1502case OBJ_REF_DELTA:1503if(reuse_delta && !entry->preferred_base)1504 base_ref =use_pack(p, &w_curs,1505 entry->in_pack_offset + used, NULL);1506 entry->in_pack_header_size = used + the_hash_algo->rawsz;1507break;1508case OBJ_OFS_DELTA:1509 buf =use_pack(p, &w_curs,1510 entry->in_pack_offset + used, NULL);1511 used_0 =0;1512 c = buf[used_0++];1513 ofs = c &127;1514while(c &128) {1515 ofs +=1;1516if(!ofs ||MSB(ofs,7)) {1517error(_("delta base offset overflow in pack for%s"),1518oid_to_hex(&entry->idx.oid));1519goto give_up;1520}1521 c = buf[used_0++];1522 ofs = (ofs <<7) + (c &127);1523}1524 ofs = entry->in_pack_offset - ofs;1525if(ofs <=0|| ofs >= entry->in_pack_offset) {1526error(_("delta base offset out of bound for%s"),1527oid_to_hex(&entry->idx.oid));1528goto give_up;1529}1530if(reuse_delta && !entry->preferred_base) {1531struct revindex_entry *revidx;1532 revidx =find_pack_revindex(p, ofs);1533if(!revidx)1534goto give_up;1535 base_ref =nth_packed_object_sha1(p, revidx->nr);1536}1537 entry->in_pack_header_size = used + used_0;1538break;1539}15401541if(base_ref && (base_entry =packlist_find(&to_pack, base_ref, NULL))) {1542/*1543 * If base_ref was set above that means we wish to1544 * reuse delta data, and we even found that base1545 * in the list of objects we want to pack. Goodie!1546 *1547 * Depth value does not matter - find_deltas() will1548 * never consider reused delta as the base object to1549 * deltify other objects against, in order to avoid1550 * circular deltas.1551 */1552oe_set_type(entry, entry->in_pack_type);1553SET_SIZE(entry, in_pack_size);/* delta size */1554SET_DELTA(entry, base_entry);1555SET_DELTA_SIZE(entry, in_pack_size);1556 entry->delta_sibling_idx = base_entry->delta_child_idx;1557SET_DELTA_CHILD(base_entry, entry);1558unuse_pack(&w_curs);1559return;1560}15611562if(oe_type(entry)) {1563 off_t delta_pos;15641565/*1566 * This must be a delta and we already know what the1567 * final object type is. Let's extract the actual1568 * object size from the delta header.1569 */1570 delta_pos = entry->in_pack_offset + entry->in_pack_header_size;1571 canonical_size =get_size_from_delta(p, &w_curs, delta_pos);1572if(canonical_size ==0)1573goto give_up;1574SET_SIZE(entry, canonical_size);1575unuse_pack(&w_curs);1576return;1577}15781579/*1580 * No choice but to fall back to the recursive delta walk1581 * with sha1_object_info() to find about the object type1582 * at this point...1583 */1584 give_up:1585unuse_pack(&w_curs);1586}15871588oe_set_type(entry,1589oid_object_info(the_repository, &entry->idx.oid, &canonical_size));1590if(entry->type_valid) {1591SET_SIZE(entry, canonical_size);1592}else{1593/*1594 * Bad object type is checked in prepare_pack(). This is1595 * to permit a missing preferred base object to be ignored1596 * as a preferred base. Doing so can result in a larger1597 * pack file, but the transfer will still take place.1598 */1599}1600}16011602static intpack_offset_sort(const void*_a,const void*_b)1603{1604const struct object_entry *a = *(struct object_entry **)_a;1605const struct object_entry *b = *(struct object_entry **)_b;1606const struct packed_git *a_in_pack =IN_PACK(a);1607const struct packed_git *b_in_pack =IN_PACK(b);16081609/* avoid filesystem trashing with loose objects */1610if(!a_in_pack && !b_in_pack)1611returnoidcmp(&a->idx.oid, &b->idx.oid);16121613if(a_in_pack < b_in_pack)1614return-1;1615if(a_in_pack > b_in_pack)1616return1;1617return a->in_pack_offset < b->in_pack_offset ? -1:1618(a->in_pack_offset > b->in_pack_offset);1619}16201621/*1622 * Drop an on-disk delta we were planning to reuse. Naively, this would1623 * just involve blanking out the "delta" field, but we have to deal1624 * with some extra book-keeping:1625 *1626 * 1. Removing ourselves from the delta_sibling linked list.1627 *1628 * 2. Updating our size/type to the non-delta representation. These were1629 * either not recorded initially (size) or overwritten with the delta type1630 * (type) when check_object() decided to reuse the delta.1631 *1632 * 3. Resetting our delta depth, as we are now a base object.1633 */1634static voiddrop_reused_delta(struct object_entry *entry)1635{1636unsigned*idx = &to_pack.objects[entry->delta_idx -1].delta_child_idx;1637struct object_info oi = OBJECT_INFO_INIT;1638enum object_type type;1639unsigned long size;16401641while(*idx) {1642struct object_entry *oe = &to_pack.objects[*idx -1];16431644if(oe == entry)1645*idx = oe->delta_sibling_idx;1646else1647 idx = &oe->delta_sibling_idx;1648}1649SET_DELTA(entry, NULL);1650 entry->depth =0;16511652 oi.sizep = &size;1653 oi.typep = &type;1654if(packed_object_info(the_repository,IN_PACK(entry), entry->in_pack_offset, &oi) <0) {1655/*1656 * We failed to get the info from this pack for some reason;1657 * fall back to sha1_object_info, which may find another copy.1658 * And if that fails, the error will be recorded in oe_type(entry)1659 * and dealt with in prepare_pack().1660 */1661oe_set_type(entry,1662oid_object_info(the_repository, &entry->idx.oid, &size));1663}else{1664oe_set_type(entry, type);1665}1666SET_SIZE(entry, size);1667}16681669/*1670 * Follow the chain of deltas from this entry onward, throwing away any links1671 * that cause us to hit a cycle (as determined by the DFS state flags in1672 * the entries).1673 *1674 * We also detect too-long reused chains that would violate our --depth1675 * limit.1676 */1677static voidbreak_delta_chains(struct object_entry *entry)1678{1679/*1680 * The actual depth of each object we will write is stored as an int,1681 * as it cannot exceed our int "depth" limit. But before we break1682 * changes based no that limit, we may potentially go as deep as the1683 * number of objects, which is elsewhere bounded to a uint32_t.1684 */1685uint32_t total_depth;1686struct object_entry *cur, *next;16871688for(cur = entry, total_depth =0;1689 cur;1690 cur =DELTA(cur), total_depth++) {1691if(cur->dfs_state == DFS_DONE) {1692/*1693 * We've already seen this object and know it isn't1694 * part of a cycle. We do need to append its depth1695 * to our count.1696 */1697 total_depth += cur->depth;1698break;1699}17001701/*1702 * We break cycles before looping, so an ACTIVE state (or any1703 * other cruft which made its way into the state variable)1704 * is a bug.1705 */1706if(cur->dfs_state != DFS_NONE)1707BUG("confusing delta dfs state in first pass:%d",1708 cur->dfs_state);17091710/*1711 * Now we know this is the first time we've seen the object. If1712 * it's not a delta, we're done traversing, but we'll mark it1713 * done to save time on future traversals.1714 */1715if(!DELTA(cur)) {1716 cur->dfs_state = DFS_DONE;1717break;1718}17191720/*1721 * Mark ourselves as active and see if the next step causes1722 * us to cycle to another active object. It's important to do1723 * this _before_ we loop, because it impacts where we make the1724 * cut, and thus how our total_depth counter works.1725 * E.g., We may see a partial loop like:1726 *1727 * A -> B -> C -> D -> B1728 *1729 * Cutting B->C breaks the cycle. But now the depth of A is1730 * only 1, and our total_depth counter is at 3. The size of the1731 * error is always one less than the size of the cycle we1732 * broke. Commits C and D were "lost" from A's chain.1733 *1734 * If we instead cut D->B, then the depth of A is correct at 3.1735 * We keep all commits in the chain that we examined.1736 */1737 cur->dfs_state = DFS_ACTIVE;1738if(DELTA(cur)->dfs_state == DFS_ACTIVE) {1739drop_reused_delta(cur);1740 cur->dfs_state = DFS_DONE;1741break;1742}1743}17441745/*1746 * And now that we've gone all the way to the bottom of the chain, we1747 * need to clear the active flags and set the depth fields as1748 * appropriate. Unlike the loop above, which can quit when it drops a1749 * delta, we need to keep going to look for more depth cuts. So we need1750 * an extra "next" pointer to keep going after we reset cur->delta.1751 */1752for(cur = entry; cur; cur = next) {1753 next =DELTA(cur);17541755/*1756 * We should have a chain of zero or more ACTIVE states down to1757 * a final DONE. We can quit after the DONE, because either it1758 * has no bases, or we've already handled them in a previous1759 * call.1760 */1761if(cur->dfs_state == DFS_DONE)1762break;1763else if(cur->dfs_state != DFS_ACTIVE)1764BUG("confusing delta dfs state in second pass:%d",1765 cur->dfs_state);17661767/*1768 * If the total_depth is more than depth, then we need to snip1769 * the chain into two or more smaller chains that don't exceed1770 * the maximum depth. Most of the resulting chains will contain1771 * (depth + 1) entries (i.e., depth deltas plus one base), and1772 * the last chain (i.e., the one containing entry) will contain1773 * whatever entries are left over, namely1774 * (total_depth % (depth + 1)) of them.1775 *1776 * Since we are iterating towards decreasing depth, we need to1777 * decrement total_depth as we go, and we need to write to the1778 * entry what its final depth will be after all of the1779 * snipping. Since we're snipping into chains of length (depth1780 * + 1) entries, the final depth of an entry will be its1781 * original depth modulo (depth + 1). Any time we encounter an1782 * entry whose final depth is supposed to be zero, we snip it1783 * from its delta base, thereby making it so.1784 */1785 cur->depth = (total_depth--) % (depth +1);1786if(!cur->depth)1787drop_reused_delta(cur);17881789 cur->dfs_state = DFS_DONE;1790}1791}17921793static voidget_object_details(void)1794{1795uint32_t i;1796struct object_entry **sorted_by_offset;17971798if(progress)1799 progress_state =start_progress(_("Counting objects"),1800 to_pack.nr_objects);18011802 sorted_by_offset =xcalloc(to_pack.nr_objects,sizeof(struct object_entry *));1803for(i =0; i < to_pack.nr_objects; i++)1804 sorted_by_offset[i] = to_pack.objects + i;1805QSORT(sorted_by_offset, to_pack.nr_objects, pack_offset_sort);18061807for(i =0; i < to_pack.nr_objects; i++) {1808struct object_entry *entry = sorted_by_offset[i];1809check_object(entry);1810if(entry->type_valid &&1811oe_size_greater_than(&to_pack, entry, big_file_threshold))1812 entry->no_try_delta =1;1813display_progress(progress_state, i +1);1814}1815stop_progress(&progress_state);18161817/*1818 * This must happen in a second pass, since we rely on the delta1819 * information for the whole list being completed.1820 */1821for(i =0; i < to_pack.nr_objects; i++)1822break_delta_chains(&to_pack.objects[i]);18231824free(sorted_by_offset);1825}18261827/*1828 * We search for deltas in a list sorted by type, by filename hash, and then1829 * by size, so that we see progressively smaller and smaller files.1830 * That's because we prefer deltas to be from the bigger file1831 * to the smaller -- deletes are potentially cheaper, but perhaps1832 * more importantly, the bigger file is likely the more recent1833 * one. The deepest deltas are therefore the oldest objects which are1834 * less susceptible to be accessed often.1835 */1836static inttype_size_sort(const void*_a,const void*_b)1837{1838const struct object_entry *a = *(struct object_entry **)_a;1839const struct object_entry *b = *(struct object_entry **)_b;1840enum object_type a_type =oe_type(a);1841enum object_type b_type =oe_type(b);1842unsigned long a_size =SIZE(a);1843unsigned long b_size =SIZE(b);18441845if(a_type > b_type)1846return-1;1847if(a_type < b_type)1848return1;1849if(a->hash > b->hash)1850return-1;1851if(a->hash < b->hash)1852return1;1853if(a->preferred_base > b->preferred_base)1854return-1;1855if(a->preferred_base < b->preferred_base)1856return1;1857if(a_size > b_size)1858return-1;1859if(a_size < b_size)1860return1;1861return a < b ? -1: (a > b);/* newest first */1862}18631864struct unpacked {1865struct object_entry *entry;1866void*data;1867struct delta_index *index;1868unsigned depth;1869};18701871static intdelta_cacheable(unsigned long src_size,unsigned long trg_size,1872unsigned long delta_size)1873{1874if(max_delta_cache_size && delta_cache_size + delta_size > max_delta_cache_size)1875return0;18761877if(delta_size < cache_max_small_delta_size)1878return1;18791880/* cache delta, if objects are large enough compared to delta size */1881if((src_size >>20) + (trg_size >>21) > (delta_size >>10))1882return1;18831884return0;1885}18861887#ifndef NO_PTHREADS18881889/* Protect access to object database */1890static pthread_mutex_t read_mutex;1891#define read_lock() pthread_mutex_lock(&read_mutex)1892#define read_unlock() pthread_mutex_unlock(&read_mutex)18931894/* Protect delta_cache_size */1895static pthread_mutex_t cache_mutex;1896#define cache_lock() pthread_mutex_lock(&cache_mutex)1897#define cache_unlock() pthread_mutex_unlock(&cache_mutex)18981899/*1900 * Protect object list partitioning (e.g. struct thread_param) and1901 * progress_state1902 */1903static pthread_mutex_t progress_mutex;1904#define progress_lock() pthread_mutex_lock(&progress_mutex)1905#define progress_unlock() pthread_mutex_unlock(&progress_mutex)19061907/*1908 * Access to struct object_entry is unprotected since each thread owns1909 * a portion of the main object list. Just don't access object entries1910 * ahead in the list because they can be stolen and would need1911 * progress_mutex for protection.1912 */1913#else19141915#define read_lock() (void)01916#define read_unlock() (void)01917#define cache_lock() (void)01918#define cache_unlock() (void)01919#define progress_lock() (void)01920#define progress_unlock() (void)019211922#endif19231924/*1925 * Return the size of the object without doing any delta1926 * reconstruction (so non-deltas are true object sizes, but deltas1927 * return the size of the delta data).1928 */1929unsigned longoe_get_size_slow(struct packing_data *pack,1930const struct object_entry *e)1931{1932struct packed_git *p;1933struct pack_window *w_curs;1934unsigned char*buf;1935enum object_type type;1936unsigned long used, avail, size;19371938if(e->type_ != OBJ_OFS_DELTA && e->type_ != OBJ_REF_DELTA) {1939read_lock();1940if(oid_object_info(the_repository, &e->idx.oid, &size) <0)1941die(_("unable to get size of%s"),1942oid_to_hex(&e->idx.oid));1943read_unlock();1944return size;1945}19461947 p =oe_in_pack(pack, e);1948if(!p)1949BUG("when e->type is a delta, it must belong to a pack");19501951read_lock();1952 w_curs = NULL;1953 buf =use_pack(p, &w_curs, e->in_pack_offset, &avail);1954 used =unpack_object_header_buffer(buf, avail, &type, &size);1955if(used ==0)1956die(_("unable to parse object header of%s"),1957oid_to_hex(&e->idx.oid));19581959unuse_pack(&w_curs);1960read_unlock();1961return size;1962}19631964static inttry_delta(struct unpacked *trg,struct unpacked *src,1965unsigned max_depth,unsigned long*mem_usage)1966{1967struct object_entry *trg_entry = trg->entry;1968struct object_entry *src_entry = src->entry;1969unsigned long trg_size, src_size, delta_size, sizediff, max_size, sz;1970unsigned ref_depth;1971enum object_type type;1972void*delta_buf;19731974/* Don't bother doing diffs between different types */1975if(oe_type(trg_entry) !=oe_type(src_entry))1976return-1;19771978/*1979 * We do not bother to try a delta that we discarded on an1980 * earlier try, but only when reusing delta data. Note that1981 * src_entry that is marked as the preferred_base should always1982 * be considered, as even if we produce a suboptimal delta against1983 * it, we will still save the transfer cost, as we already know1984 * the other side has it and we won't send src_entry at all.1985 */1986if(reuse_delta &&IN_PACK(trg_entry) &&1987IN_PACK(trg_entry) ==IN_PACK(src_entry) &&1988!src_entry->preferred_base &&1989 trg_entry->in_pack_type != OBJ_REF_DELTA &&1990 trg_entry->in_pack_type != OBJ_OFS_DELTA)1991return0;19921993/* Let's not bust the allowed depth. */1994if(src->depth >= max_depth)1995return0;19961997/* Now some size filtering heuristics. */1998 trg_size =SIZE(trg_entry);1999if(!DELTA(trg_entry)) {2000 max_size = trg_size/2- the_hash_algo->rawsz;2001 ref_depth =1;2002}else{2003 max_size =DELTA_SIZE(trg_entry);2004 ref_depth = trg->depth;2005}2006 max_size = (uint64_t)max_size * (max_depth - src->depth) /2007(max_depth - ref_depth +1);2008if(max_size ==0)2009return0;2010 src_size =SIZE(src_entry);2011 sizediff = src_size < trg_size ? trg_size - src_size :0;2012if(sizediff >= max_size)2013return0;2014if(trg_size < src_size /32)2015return0;20162017/* Load data if not already done */2018if(!trg->data) {2019read_lock();2020 trg->data =read_object_file(&trg_entry->idx.oid, &type, &sz);2021read_unlock();2022if(!trg->data)2023die(_("object%scannot be read"),2024oid_to_hex(&trg_entry->idx.oid));2025if(sz != trg_size)2026die(_("object%sinconsistent object length (%lu vs%lu)"),2027oid_to_hex(&trg_entry->idx.oid), sz,2028 trg_size);2029*mem_usage += sz;2030}2031if(!src->data) {2032read_lock();2033 src->data =read_object_file(&src_entry->idx.oid, &type, &sz);2034read_unlock();2035if(!src->data) {2036if(src_entry->preferred_base) {2037static int warned =0;2038if(!warned++)2039warning(_("object%scannot be read"),2040oid_to_hex(&src_entry->idx.oid));2041/*2042 * Those objects are not included in the2043 * resulting pack. Be resilient and ignore2044 * them if they can't be read, in case the2045 * pack could be created nevertheless.2046 */2047return0;2048}2049die(_("object%scannot be read"),2050oid_to_hex(&src_entry->idx.oid));2051}2052if(sz != src_size)2053die(_("object%sinconsistent object length (%lu vs%lu)"),2054oid_to_hex(&src_entry->idx.oid), sz,2055 src_size);2056*mem_usage += sz;2057}2058if(!src->index) {2059 src->index =create_delta_index(src->data, src_size);2060if(!src->index) {2061static int warned =0;2062if(!warned++)2063warning(_("suboptimal pack - out of memory"));2064return0;2065}2066*mem_usage +=sizeof_delta_index(src->index);2067}20682069 delta_buf =create_delta(src->index, trg->data, trg_size, &delta_size, max_size);2070if(!delta_buf)2071return0;20722073if(DELTA(trg_entry)) {2074/* Prefer only shallower same-sized deltas. */2075if(delta_size ==DELTA_SIZE(trg_entry) &&2076 src->depth +1>= trg->depth) {2077free(delta_buf);2078return0;2079}2080}20812082/*2083 * Handle memory allocation outside of the cache2084 * accounting lock. Compiler will optimize the strangeness2085 * away when NO_PTHREADS is defined.2086 */2087free(trg_entry->delta_data);2088cache_lock();2089if(trg_entry->delta_data) {2090 delta_cache_size -=DELTA_SIZE(trg_entry);2091 trg_entry->delta_data = NULL;2092}2093if(delta_cacheable(src_size, trg_size, delta_size)) {2094 delta_cache_size += delta_size;2095cache_unlock();2096 trg_entry->delta_data =xrealloc(delta_buf, delta_size);2097}else{2098cache_unlock();2099free(delta_buf);2100}21012102SET_DELTA(trg_entry, src_entry);2103SET_DELTA_SIZE(trg_entry, delta_size);2104 trg->depth = src->depth +1;21052106return1;2107}21082109static unsigned intcheck_delta_limit(struct object_entry *me,unsigned int n)2110{2111struct object_entry *child =DELTA_CHILD(me);2112unsigned int m = n;2113while(child) {2114unsigned int c =check_delta_limit(child, n +1);2115if(m < c)2116 m = c;2117 child =DELTA_SIBLING(child);2118}2119return m;2120}21212122static unsigned longfree_unpacked(struct unpacked *n)2123{2124unsigned long freed_mem =sizeof_delta_index(n->index);2125free_delta_index(n->index);2126 n->index = NULL;2127if(n->data) {2128 freed_mem +=SIZE(n->entry);2129FREE_AND_NULL(n->data);2130}2131 n->entry = NULL;2132 n->depth =0;2133return freed_mem;2134}21352136static voidfind_deltas(struct object_entry **list,unsigned*list_size,2137int window,int depth,unsigned*processed)2138{2139uint32_t i, idx =0, count =0;2140struct unpacked *array;2141unsigned long mem_usage =0;21422143 array =xcalloc(window,sizeof(struct unpacked));21442145for(;;) {2146struct object_entry *entry;2147struct unpacked *n = array + idx;2148int j, max_depth, best_base = -1;21492150progress_lock();2151if(!*list_size) {2152progress_unlock();2153break;2154}2155 entry = *list++;2156(*list_size)--;2157if(!entry->preferred_base) {2158(*processed)++;2159display_progress(progress_state, *processed);2160}2161progress_unlock();21622163 mem_usage -=free_unpacked(n);2164 n->entry = entry;21652166while(window_memory_limit &&2167 mem_usage > window_memory_limit &&2168 count >1) {2169uint32_t tail = (idx + window - count) % window;2170 mem_usage -=free_unpacked(array + tail);2171 count--;2172}21732174/* We do not compute delta to *create* objects we are not2175 * going to pack.2176 */2177if(entry->preferred_base)2178goto next;21792180/*2181 * If the current object is at pack edge, take the depth the2182 * objects that depend on the current object into account2183 * otherwise they would become too deep.2184 */2185 max_depth = depth;2186if(DELTA_CHILD(entry)) {2187 max_depth -=check_delta_limit(entry,0);2188if(max_depth <=0)2189goto next;2190}21912192 j = window;2193while(--j >0) {2194int ret;2195uint32_t other_idx = idx + j;2196struct unpacked *m;2197if(other_idx >= window)2198 other_idx -= window;2199 m = array + other_idx;2200if(!m->entry)2201break;2202 ret =try_delta(n, m, max_depth, &mem_usage);2203if(ret <0)2204break;2205else if(ret >0)2206 best_base = other_idx;2207}22082209/*2210 * If we decided to cache the delta data, then it is best2211 * to compress it right away. First because we have to do2212 * it anyway, and doing it here while we're threaded will2213 * save a lot of time in the non threaded write phase,2214 * as well as allow for caching more deltas within2215 * the same cache size limit.2216 * ...2217 * But only if not writing to stdout, since in that case2218 * the network is most likely throttling writes anyway,2219 * and therefore it is best to go to the write phase ASAP2220 * instead, as we can afford spending more time compressing2221 * between writes at that moment.2222 */2223if(entry->delta_data && !pack_to_stdout) {2224unsigned long size;22252226 size =do_compress(&entry->delta_data,DELTA_SIZE(entry));2227if(size < (1U<< OE_Z_DELTA_BITS)) {2228 entry->z_delta_size = size;2229cache_lock();2230 delta_cache_size -=DELTA_SIZE(entry);2231 delta_cache_size += entry->z_delta_size;2232cache_unlock();2233}else{2234FREE_AND_NULL(entry->delta_data);2235 entry->z_delta_size =0;2236}2237}22382239/* if we made n a delta, and if n is already at max2240 * depth, leaving it in the window is pointless. we2241 * should evict it first.2242 */2243if(DELTA(entry) && max_depth <= n->depth)2244continue;22452246/*2247 * Move the best delta base up in the window, after the2248 * currently deltified object, to keep it longer. It will2249 * be the first base object to be attempted next.2250 */2251if(DELTA(entry)) {2252struct unpacked swap = array[best_base];2253int dist = (window + idx - best_base) % window;2254int dst = best_base;2255while(dist--) {2256int src = (dst +1) % window;2257 array[dst] = array[src];2258 dst = src;2259}2260 array[dst] = swap;2261}22622263 next:2264 idx++;2265if(count +1< window)2266 count++;2267if(idx >= window)2268 idx =0;2269}22702271for(i =0; i < window; ++i) {2272free_delta_index(array[i].index);2273free(array[i].data);2274}2275free(array);2276}22772278#ifndef NO_PTHREADS22792280static voidtry_to_free_from_threads(size_t size)2281{2282read_lock();2283release_pack_memory(size);2284read_unlock();2285}22862287static try_to_free_t old_try_to_free_routine;22882289/*2290 * The main object list is split into smaller lists, each is handed to2291 * one worker.2292 *2293 * The main thread waits on the condition that (at least) one of the workers2294 * has stopped working (which is indicated in the .working member of2295 * struct thread_params).2296 *2297 * When a work thread has completed its work, it sets .working to 0 and2298 * signals the main thread and waits on the condition that .data_ready2299 * becomes 1.2300 *2301 * The main thread steals half of the work from the worker that has2302 * most work left to hand it to the idle worker.2303 */23042305struct thread_params {2306 pthread_t thread;2307struct object_entry **list;2308unsigned list_size;2309unsigned remaining;2310int window;2311int depth;2312int working;2313int data_ready;2314 pthread_mutex_t mutex;2315 pthread_cond_t cond;2316unsigned*processed;2317};23182319static pthread_cond_t progress_cond;23202321/*2322 * Mutex and conditional variable can't be statically-initialized on Windows.2323 */2324static voidinit_threaded_search(void)2325{2326init_recursive_mutex(&read_mutex);2327pthread_mutex_init(&cache_mutex, NULL);2328pthread_mutex_init(&progress_mutex, NULL);2329pthread_cond_init(&progress_cond, NULL);2330pthread_mutex_init(&to_pack.lock, NULL);2331 old_try_to_free_routine =set_try_to_free_routine(try_to_free_from_threads);2332}23332334static voidcleanup_threaded_search(void)2335{2336set_try_to_free_routine(old_try_to_free_routine);2337pthread_cond_destroy(&progress_cond);2338pthread_mutex_destroy(&read_mutex);2339pthread_mutex_destroy(&cache_mutex);2340pthread_mutex_destroy(&progress_mutex);2341}23422343static void*threaded_find_deltas(void*arg)2344{2345struct thread_params *me = arg;23462347progress_lock();2348while(me->remaining) {2349progress_unlock();23502351find_deltas(me->list, &me->remaining,2352 me->window, me->depth, me->processed);23532354progress_lock();2355 me->working =0;2356pthread_cond_signal(&progress_cond);2357progress_unlock();23582359/*2360 * We must not set ->data_ready before we wait on the2361 * condition because the main thread may have set it to 12362 * before we get here. In order to be sure that new2363 * work is available if we see 1 in ->data_ready, it2364 * was initialized to 0 before this thread was spawned2365 * and we reset it to 0 right away.2366 */2367pthread_mutex_lock(&me->mutex);2368while(!me->data_ready)2369pthread_cond_wait(&me->cond, &me->mutex);2370 me->data_ready =0;2371pthread_mutex_unlock(&me->mutex);23722373progress_lock();2374}2375progress_unlock();2376/* leave ->working 1 so that this doesn't get more work assigned */2377return NULL;2378}23792380static voidll_find_deltas(struct object_entry **list,unsigned list_size,2381int window,int depth,unsigned*processed)2382{2383struct thread_params *p;2384int i, ret, active_threads =0;23852386init_threaded_search();23872388if(delta_search_threads <=1) {2389find_deltas(list, &list_size, window, depth, processed);2390cleanup_threaded_search();2391return;2392}2393if(progress > pack_to_stdout)2394fprintf_ln(stderr,_("Delta compression using up to%dthreads"),2395 delta_search_threads);2396 p =xcalloc(delta_search_threads,sizeof(*p));23972398/* Partition the work amongst work threads. */2399for(i =0; i < delta_search_threads; i++) {2400unsigned sub_size = list_size / (delta_search_threads - i);24012402/* don't use too small segments or no deltas will be found */2403if(sub_size <2*window && i+1< delta_search_threads)2404 sub_size =0;24052406 p[i].window = window;2407 p[i].depth = depth;2408 p[i].processed = processed;2409 p[i].working =1;2410 p[i].data_ready =0;24112412/* try to split chunks on "path" boundaries */2413while(sub_size && sub_size < list_size &&2414 list[sub_size]->hash &&2415 list[sub_size]->hash == list[sub_size-1]->hash)2416 sub_size++;24172418 p[i].list = list;2419 p[i].list_size = sub_size;2420 p[i].remaining = sub_size;24212422 list += sub_size;2423 list_size -= sub_size;2424}24252426/* Start work threads. */2427for(i =0; i < delta_search_threads; i++) {2428if(!p[i].list_size)2429continue;2430pthread_mutex_init(&p[i].mutex, NULL);2431pthread_cond_init(&p[i].cond, NULL);2432 ret =pthread_create(&p[i].thread, NULL,2433 threaded_find_deltas, &p[i]);2434if(ret)2435die(_("unable to create thread:%s"),strerror(ret));2436 active_threads++;2437}24382439/*2440 * Now let's wait for work completion. Each time a thread is done2441 * with its work, we steal half of the remaining work from the2442 * thread with the largest number of unprocessed objects and give2443 * it to that newly idle thread. This ensure good load balancing2444 * until the remaining object list segments are simply too short2445 * to be worth splitting anymore.2446 */2447while(active_threads) {2448struct thread_params *target = NULL;2449struct thread_params *victim = NULL;2450unsigned sub_size =0;24512452progress_lock();2453for(;;) {2454for(i =0; !target && i < delta_search_threads; i++)2455if(!p[i].working)2456 target = &p[i];2457if(target)2458break;2459pthread_cond_wait(&progress_cond, &progress_mutex);2460}24612462for(i =0; i < delta_search_threads; i++)2463if(p[i].remaining >2*window &&2464(!victim || victim->remaining < p[i].remaining))2465 victim = &p[i];2466if(victim) {2467 sub_size = victim->remaining /2;2468 list = victim->list + victim->list_size - sub_size;2469while(sub_size && list[0]->hash &&2470 list[0]->hash == list[-1]->hash) {2471 list++;2472 sub_size--;2473}2474if(!sub_size) {2475/*2476 * It is possible for some "paths" to have2477 * so many objects that no hash boundary2478 * might be found. Let's just steal the2479 * exact half in that case.2480 */2481 sub_size = victim->remaining /2;2482 list -= sub_size;2483}2484 target->list = list;2485 victim->list_size -= sub_size;2486 victim->remaining -= sub_size;2487}2488 target->list_size = sub_size;2489 target->remaining = sub_size;2490 target->working =1;2491progress_unlock();24922493pthread_mutex_lock(&target->mutex);2494 target->data_ready =1;2495pthread_cond_signal(&target->cond);2496pthread_mutex_unlock(&target->mutex);24972498if(!sub_size) {2499pthread_join(target->thread, NULL);2500pthread_cond_destroy(&target->cond);2501pthread_mutex_destroy(&target->mutex);2502 active_threads--;2503}2504}2505cleanup_threaded_search();2506free(p);2507}25082509#else2510#define ll_find_deltas(l, s, w, d, p) find_deltas(l, &s, w, d, p)2511#endif25122513static voidadd_tag_chain(const struct object_id *oid)2514{2515struct tag *tag;25162517/*2518 * We catch duplicates already in add_object_entry(), but we'd2519 * prefer to do this extra check to avoid having to parse the2520 * tag at all if we already know that it's being packed (e.g., if2521 * it was included via bitmaps, we would not have parsed it2522 * previously).2523 */2524if(packlist_find(&to_pack, oid->hash, NULL))2525return;25262527 tag =lookup_tag(the_repository, oid);2528while(1) {2529if(!tag ||parse_tag(tag) || !tag->tagged)2530die(_("unable to pack objects reachable from tag%s"),2531oid_to_hex(oid));25322533add_object_entry(&tag->object.oid, OBJ_TAG, NULL,0);25342535if(tag->tagged->type != OBJ_TAG)2536return;25372538 tag = (struct tag *)tag->tagged;2539}2540}25412542static intadd_ref_tag(const char*path,const struct object_id *oid,int flag,void*cb_data)2543{2544struct object_id peeled;25452546if(starts_with(path,"refs/tags/") &&/* is a tag? */2547!peel_ref(path, &peeled) &&/* peelable? */2548packlist_find(&to_pack, peeled.hash, NULL))/* object packed? */2549add_tag_chain(oid);2550return0;2551}25522553static voidprepare_pack(int window,int depth)2554{2555struct object_entry **delta_list;2556uint32_t i, nr_deltas;2557unsigned n;25582559get_object_details();25602561/*2562 * If we're locally repacking then we need to be doubly careful2563 * from now on in order to make sure no stealth corruption gets2564 * propagated to the new pack. Clients receiving streamed packs2565 * should validate everything they get anyway so no need to incur2566 * the additional cost here in that case.2567 */2568if(!pack_to_stdout)2569 do_check_packed_object_crc =1;25702571if(!to_pack.nr_objects || !window || !depth)2572return;25732574ALLOC_ARRAY(delta_list, to_pack.nr_objects);2575 nr_deltas = n =0;25762577for(i =0; i < to_pack.nr_objects; i++) {2578struct object_entry *entry = to_pack.objects + i;25792580if(DELTA(entry))2581/* This happens if we decided to reuse existing2582 * delta from a pack. "reuse_delta &&" is implied.2583 */2584continue;25852586if(!entry->type_valid ||2587oe_size_less_than(&to_pack, entry,50))2588continue;25892590if(entry->no_try_delta)2591continue;25922593if(!entry->preferred_base) {2594 nr_deltas++;2595if(oe_type(entry) <0)2596die(_("unable to get type of object%s"),2597oid_to_hex(&entry->idx.oid));2598}else{2599if(oe_type(entry) <0) {2600/*2601 * This object is not found, but we2602 * don't have to include it anyway.2603 */2604continue;2605}2606}26072608 delta_list[n++] = entry;2609}26102611if(nr_deltas && n >1) {2612unsigned nr_done =0;2613if(progress)2614 progress_state =start_progress(_("Compressing objects"),2615 nr_deltas);2616QSORT(delta_list, n, type_size_sort);2617ll_find_deltas(delta_list, n, window+1, depth, &nr_done);2618stop_progress(&progress_state);2619if(nr_done != nr_deltas)2620die(_("inconsistency with delta count"));2621}2622free(delta_list);2623}26242625static intgit_pack_config(const char*k,const char*v,void*cb)2626{2627if(!strcmp(k,"pack.window")) {2628 window =git_config_int(k, v);2629return0;2630}2631if(!strcmp(k,"pack.windowmemory")) {2632 window_memory_limit =git_config_ulong(k, v);2633return0;2634}2635if(!strcmp(k,"pack.depth")) {2636 depth =git_config_int(k, v);2637return0;2638}2639if(!strcmp(k,"pack.deltacachesize")) {2640 max_delta_cache_size =git_config_int(k, v);2641return0;2642}2643if(!strcmp(k,"pack.deltacachelimit")) {2644 cache_max_small_delta_size =git_config_int(k, v);2645return0;2646}2647if(!strcmp(k,"pack.writebitmaphashcache")) {2648if(git_config_bool(k, v))2649 write_bitmap_options |= BITMAP_OPT_HASH_CACHE;2650else2651 write_bitmap_options &= ~BITMAP_OPT_HASH_CACHE;2652}2653if(!strcmp(k,"pack.usebitmaps")) {2654 use_bitmap_index_default =git_config_bool(k, v);2655return0;2656}2657if(!strcmp(k,"pack.threads")) {2658 delta_search_threads =git_config_int(k, v);2659if(delta_search_threads <0)2660die(_("invalid number of threads specified (%d)"),2661 delta_search_threads);2662#ifdef NO_PTHREADS2663if(delta_search_threads !=1) {2664warning(_("no threads support, ignoring%s"), k);2665 delta_search_threads =0;2666}2667#endif2668return0;2669}2670if(!strcmp(k,"pack.indexversion")) {2671 pack_idx_opts.version =git_config_int(k, v);2672if(pack_idx_opts.version >2)2673die(_("bad pack.indexversion=%"PRIu32),2674 pack_idx_opts.version);2675return0;2676}2677returngit_default_config(k, v, cb);2678}26792680static voidread_object_list_from_stdin(void)2681{2682char line[GIT_MAX_HEXSZ +1+ PATH_MAX +2];2683struct object_id oid;2684const char*p;26852686for(;;) {2687if(!fgets(line,sizeof(line), stdin)) {2688if(feof(stdin))2689break;2690if(!ferror(stdin))2691die("BUG: fgets returned NULL, not EOF, not error!");2692if(errno != EINTR)2693die_errno("fgets");2694clearerr(stdin);2695continue;2696}2697if(line[0] =='-') {2698if(get_oid_hex(line+1, &oid))2699die(_("expected edge object ID, got garbage:\n%s"),2700 line);2701add_preferred_base(&oid);2702continue;2703}2704if(parse_oid_hex(line, &oid, &p))2705die(_("expected object ID, got garbage:\n%s"), line);27062707add_preferred_base_object(p +1);2708add_object_entry(&oid, OBJ_NONE, p +1,0);2709}2710}27112712/* Remember to update object flag allocation in object.h */2713#define OBJECT_ADDED (1u<<20)27142715static voidshow_commit(struct commit *commit,void*data)2716{2717add_object_entry(&commit->object.oid, OBJ_COMMIT, NULL,0);2718 commit->object.flags |= OBJECT_ADDED;27192720if(write_bitmap_index)2721index_commit_for_bitmap(commit);2722}27232724static voidshow_object(struct object *obj,const char*name,void*data)2725{2726add_preferred_base_object(name);2727add_object_entry(&obj->oid, obj->type, name,0);2728 obj->flags |= OBJECT_ADDED;2729}27302731static voidshow_object__ma_allow_any(struct object *obj,const char*name,void*data)2732{2733assert(arg_missing_action == MA_ALLOW_ANY);27342735/*2736 * Quietly ignore ALL missing objects. This avoids problems with2737 * staging them now and getting an odd error later.2738 */2739if(!has_object_file(&obj->oid))2740return;27412742show_object(obj, name, data);2743}27442745static voidshow_object__ma_allow_promisor(struct object *obj,const char*name,void*data)2746{2747assert(arg_missing_action == MA_ALLOW_PROMISOR);27482749/*2750 * Quietly ignore EXPECTED missing objects. This avoids problems with2751 * staging them now and getting an odd error later.2752 */2753if(!has_object_file(&obj->oid) &&is_promisor_object(&obj->oid))2754return;27552756show_object(obj, name, data);2757}27582759static intoption_parse_missing_action(const struct option *opt,2760const char*arg,int unset)2761{2762assert(arg);2763assert(!unset);27642765if(!strcmp(arg,"error")) {2766 arg_missing_action = MA_ERROR;2767 fn_show_object = show_object;2768return0;2769}27702771if(!strcmp(arg,"allow-any")) {2772 arg_missing_action = MA_ALLOW_ANY;2773 fetch_if_missing =0;2774 fn_show_object = show_object__ma_allow_any;2775return0;2776}27772778if(!strcmp(arg,"allow-promisor")) {2779 arg_missing_action = MA_ALLOW_PROMISOR;2780 fetch_if_missing =0;2781 fn_show_object = show_object__ma_allow_promisor;2782return0;2783}27842785die(_("invalid value for --missing"));2786return0;2787}27882789static voidshow_edge(struct commit *commit)2790{2791add_preferred_base(&commit->object.oid);2792}27932794struct in_pack_object {2795 off_t offset;2796struct object *object;2797};27982799struct in_pack {2800unsigned int alloc;2801unsigned int nr;2802struct in_pack_object *array;2803};28042805static voidmark_in_pack_object(struct object *object,struct packed_git *p,struct in_pack *in_pack)2806{2807 in_pack->array[in_pack->nr].offset =find_pack_entry_one(object->oid.hash, p);2808 in_pack->array[in_pack->nr].object = object;2809 in_pack->nr++;2810}28112812/*2813 * Compare the objects in the offset order, in order to emulate the2814 * "git rev-list --objects" output that produced the pack originally.2815 */2816static intofscmp(const void*a_,const void*b_)2817{2818struct in_pack_object *a = (struct in_pack_object *)a_;2819struct in_pack_object *b = (struct in_pack_object *)b_;28202821if(a->offset < b->offset)2822return-1;2823else if(a->offset > b->offset)2824return1;2825else2826returnoidcmp(&a->object->oid, &b->object->oid);2827}28282829static voidadd_objects_in_unpacked_packs(struct rev_info *revs)2830{2831struct packed_git *p;2832struct in_pack in_pack;2833uint32_t i;28342835memset(&in_pack,0,sizeof(in_pack));28362837for(p =get_all_packs(the_repository); p; p = p->next) {2838struct object_id oid;2839struct object *o;28402841if(!p->pack_local || p->pack_keep || p->pack_keep_in_core)2842continue;2843if(open_pack_index(p))2844die(_("cannot open pack index"));28452846ALLOC_GROW(in_pack.array,2847 in_pack.nr + p->num_objects,2848 in_pack.alloc);28492850for(i =0; i < p->num_objects; i++) {2851nth_packed_object_oid(&oid, p, i);2852 o =lookup_unknown_object(oid.hash);2853if(!(o->flags & OBJECT_ADDED))2854mark_in_pack_object(o, p, &in_pack);2855 o->flags |= OBJECT_ADDED;2856}2857}28582859if(in_pack.nr) {2860QSORT(in_pack.array, in_pack.nr, ofscmp);2861for(i =0; i < in_pack.nr; i++) {2862struct object *o = in_pack.array[i].object;2863add_object_entry(&o->oid, o->type,"",0);2864}2865}2866free(in_pack.array);2867}28682869static intadd_loose_object(const struct object_id *oid,const char*path,2870void*data)2871{2872enum object_type type =oid_object_info(the_repository, oid, NULL);28732874if(type <0) {2875warning(_("loose object at%scould not be examined"), path);2876return0;2877}28782879add_object_entry(oid, type,"",0);2880return0;2881}28822883/*2884 * We actually don't even have to worry about reachability here.2885 * add_object_entry will weed out duplicates, so we just add every2886 * loose object we find.2887 */2888static voidadd_unreachable_loose_objects(void)2889{2890for_each_loose_file_in_objdir(get_object_directory(),2891 add_loose_object,2892 NULL, NULL, NULL);2893}28942895static inthas_sha1_pack_kept_or_nonlocal(const struct object_id *oid)2896{2897static struct packed_git *last_found = (void*)1;2898struct packed_git *p;28992900 p = (last_found != (void*)1) ? last_found :2901get_all_packs(the_repository);29022903while(p) {2904if((!p->pack_local || p->pack_keep ||2905 p->pack_keep_in_core) &&2906find_pack_entry_one(oid->hash, p)) {2907 last_found = p;2908return1;2909}2910if(p == last_found)2911 p =get_all_packs(the_repository);2912else2913 p = p->next;2914if(p == last_found)2915 p = p->next;2916}2917return0;2918}29192920/*2921 * Store a list of sha1s that are should not be discarded2922 * because they are either written too recently, or are2923 * reachable from another object that was.2924 *2925 * This is filled by get_object_list.2926 */2927static struct oid_array recent_objects;29282929static intloosened_object_can_be_discarded(const struct object_id *oid,2930 timestamp_t mtime)2931{2932if(!unpack_unreachable_expiration)2933return0;2934if(mtime > unpack_unreachable_expiration)2935return0;2936if(oid_array_lookup(&recent_objects, oid) >=0)2937return0;2938return1;2939}29402941static voidloosen_unused_packed_objects(struct rev_info *revs)2942{2943struct packed_git *p;2944uint32_t i;2945struct object_id oid;29462947for(p =get_all_packs(the_repository); p; p = p->next) {2948if(!p->pack_local || p->pack_keep || p->pack_keep_in_core)2949continue;29502951if(open_pack_index(p))2952die(_("cannot open pack index"));29532954for(i =0; i < p->num_objects; i++) {2955nth_packed_object_oid(&oid, p, i);2956if(!packlist_find(&to_pack, oid.hash, NULL) &&2957!has_sha1_pack_kept_or_nonlocal(&oid) &&2958!loosened_object_can_be_discarded(&oid, p->mtime))2959if(force_object_loose(&oid, p->mtime))2960die(_("unable to force loose object"));2961}2962}2963}29642965/*2966 * This tracks any options which pack-reuse code expects to be on, or which a2967 * reader of the pack might not understand, and which would therefore prevent2968 * blind reuse of what we have on disk.2969 */2970static intpack_options_allow_reuse(void)2971{2972return pack_to_stdout &&2973 allow_ofs_delta &&2974!ignore_packed_keep_on_disk &&2975!ignore_packed_keep_in_core &&2976(!local || !have_non_local_packs) &&2977!incremental;2978}29792980static intget_object_list_from_bitmap(struct rev_info *revs)2981{2982struct bitmap_index *bitmap_git;2983if(!(bitmap_git =prepare_bitmap_walk(revs)))2984return-1;29852986if(pack_options_allow_reuse() &&2987!reuse_partial_packfile_from_bitmap(2988 bitmap_git,2989&reuse_packfile,2990&reuse_packfile_objects,2991&reuse_packfile_offset)) {2992assert(reuse_packfile_objects);2993 nr_result += reuse_packfile_objects;2994display_progress(progress_state, nr_result);2995}29962997traverse_bitmap_commit_list(bitmap_git, &add_object_entry_from_bitmap);2998free_bitmap_index(bitmap_git);2999return0;3000}30013002static voidrecord_recent_object(struct object *obj,3003const char*name,3004void*data)3005{3006oid_array_append(&recent_objects, &obj->oid);3007}30083009static voidrecord_recent_commit(struct commit *commit,void*data)3010{3011oid_array_append(&recent_objects, &commit->object.oid);3012}30133014static voidget_object_list(int ac,const char**av)3015{3016struct rev_info revs;3017char line[1000];3018int flags =0;30193020init_revisions(&revs, NULL);3021 save_commit_buffer =0;3022setup_revisions(ac, av, &revs, NULL);30233024/* make sure shallows are read */3025is_repository_shallow(the_repository);30263027while(fgets(line,sizeof(line), stdin) != NULL) {3028int len =strlen(line);3029if(len && line[len -1] =='\n')3030 line[--len] =0;3031if(!len)3032break;3033if(*line =='-') {3034if(!strcmp(line,"--not")) {3035 flags ^= UNINTERESTING;3036 write_bitmap_index =0;3037continue;3038}3039if(starts_with(line,"--shallow ")) {3040struct object_id oid;3041if(get_oid_hex(line +10, &oid))3042die("not an SHA-1 '%s'", line +10);3043register_shallow(the_repository, &oid);3044 use_bitmap_index =0;3045continue;3046}3047die(_("not a rev '%s'"), line);3048}3049if(handle_revision_arg(line, &revs, flags, REVARG_CANNOT_BE_FILENAME))3050die(_("bad revision '%s'"), line);3051}30523053if(use_bitmap_index && !get_object_list_from_bitmap(&revs))3054return;30553056if(prepare_revision_walk(&revs))3057die(_("revision walk setup failed"));3058mark_edges_uninteresting(&revs, show_edge);30593060if(!fn_show_object)3061 fn_show_object = show_object;3062traverse_commit_list_filtered(&filter_options, &revs,3063 show_commit, fn_show_object, NULL,3064 NULL);30653066if(unpack_unreachable_expiration) {3067 revs.ignore_missing_links =1;3068if(add_unseen_recent_objects_to_traversal(&revs,3069 unpack_unreachable_expiration))3070die(_("unable to add recent objects"));3071if(prepare_revision_walk(&revs))3072die(_("revision walk setup failed"));3073traverse_commit_list(&revs, record_recent_commit,3074 record_recent_object, NULL);3075}30763077if(keep_unreachable)3078add_objects_in_unpacked_packs(&revs);3079if(pack_loose_unreachable)3080add_unreachable_loose_objects();3081if(unpack_unreachable)3082loosen_unused_packed_objects(&revs);30833084oid_array_clear(&recent_objects);3085}30863087static voidadd_extra_kept_packs(const struct string_list *names)3088{3089struct packed_git *p;30903091if(!names->nr)3092return;30933094for(p =get_all_packs(the_repository); p; p = p->next) {3095const char*name =basename(p->pack_name);3096int i;30973098if(!p->pack_local)3099continue;31003101for(i =0; i < names->nr; i++)3102if(!fspathcmp(name, names->items[i].string))3103break;31043105if(i < names->nr) {3106 p->pack_keep_in_core =1;3107 ignore_packed_keep_in_core =1;3108continue;3109}3110}3111}31123113static intoption_parse_index_version(const struct option *opt,3114const char*arg,int unset)3115{3116char*c;3117const char*val = arg;3118 pack_idx_opts.version =strtoul(val, &c,10);3119if(pack_idx_opts.version >2)3120die(_("unsupported index version%s"), val);3121if(*c ==','&& c[1])3122 pack_idx_opts.off32_limit =strtoul(c+1, &c,0);3123if(*c || pack_idx_opts.off32_limit &0x80000000)3124die(_("bad index version '%s'"), val);3125return0;3126}31273128static intoption_parse_unpack_unreachable(const struct option *opt,3129const char*arg,int unset)3130{3131if(unset) {3132 unpack_unreachable =0;3133 unpack_unreachable_expiration =0;3134}3135else{3136 unpack_unreachable =1;3137if(arg)3138 unpack_unreachable_expiration =approxidate(arg);3139}3140return0;3141}31423143intcmd_pack_objects(int argc,const char**argv,const char*prefix)3144{3145int use_internal_rev_list =0;3146int thin =0;3147int shallow =0;3148int all_progress_implied =0;3149struct argv_array rp = ARGV_ARRAY_INIT;3150int rev_list_unpacked =0, rev_list_all =0, rev_list_reflog =0;3151int rev_list_index =0;3152struct string_list keep_pack_list = STRING_LIST_INIT_NODUP;3153struct option pack_objects_options[] = {3154OPT_SET_INT('q',"quiet", &progress,3155N_("do not show progress meter"),0),3156OPT_SET_INT(0,"progress", &progress,3157N_("show progress meter"),1),3158OPT_SET_INT(0,"all-progress", &progress,3159N_("show progress meter during object writing phase"),2),3160OPT_BOOL(0,"all-progress-implied",3161&all_progress_implied,3162N_("similar to --all-progress when progress meter is shown")),3163{ OPTION_CALLBACK,0,"index-version", NULL,N_("<version>[,<offset>]"),3164N_("write the pack index file in the specified idx format version"),31650, option_parse_index_version },3166OPT_MAGNITUDE(0,"max-pack-size", &pack_size_limit,3167N_("maximum size of each output pack file")),3168OPT_BOOL(0,"local", &local,3169N_("ignore borrowed objects from alternate object store")),3170OPT_BOOL(0,"incremental", &incremental,3171N_("ignore packed objects")),3172OPT_INTEGER(0,"window", &window,3173N_("limit pack window by objects")),3174OPT_MAGNITUDE(0,"window-memory", &window_memory_limit,3175N_("limit pack window by memory in addition to object limit")),3176OPT_INTEGER(0,"depth", &depth,3177N_("maximum length of delta chain allowed in the resulting pack")),3178OPT_BOOL(0,"reuse-delta", &reuse_delta,3179N_("reuse existing deltas")),3180OPT_BOOL(0,"reuse-object", &reuse_object,3181N_("reuse existing objects")),3182OPT_BOOL(0,"delta-base-offset", &allow_ofs_delta,3183N_("use OFS_DELTA objects")),3184OPT_INTEGER(0,"threads", &delta_search_threads,3185N_("use threads when searching for best delta matches")),3186OPT_BOOL(0,"non-empty", &non_empty,3187N_("do not create an empty pack output")),3188OPT_BOOL(0,"revs", &use_internal_rev_list,3189N_("read revision arguments from standard input")),3190OPT_SET_INT_F(0,"unpacked", &rev_list_unpacked,3191N_("limit the objects to those that are not yet packed"),31921, PARSE_OPT_NONEG),3193OPT_SET_INT_F(0,"all", &rev_list_all,3194N_("include objects reachable from any reference"),31951, PARSE_OPT_NONEG),3196OPT_SET_INT_F(0,"reflog", &rev_list_reflog,3197N_("include objects referred by reflog entries"),31981, PARSE_OPT_NONEG),3199OPT_SET_INT_F(0,"indexed-objects", &rev_list_index,3200N_("include objects referred to by the index"),32011, PARSE_OPT_NONEG),3202OPT_BOOL(0,"stdout", &pack_to_stdout,3203N_("output pack to stdout")),3204OPT_BOOL(0,"include-tag", &include_tag,3205N_("include tag objects that refer to objects to be packed")),3206OPT_BOOL(0,"keep-unreachable", &keep_unreachable,3207N_("keep unreachable objects")),3208OPT_BOOL(0,"pack-loose-unreachable", &pack_loose_unreachable,3209N_("pack loose unreachable objects")),3210{ OPTION_CALLBACK,0,"unpack-unreachable", NULL,N_("time"),3211N_("unpack unreachable objects newer than <time>"),3212 PARSE_OPT_OPTARG, option_parse_unpack_unreachable },3213OPT_BOOL(0,"thin", &thin,3214N_("create thin packs")),3215OPT_BOOL(0,"shallow", &shallow,3216N_("create packs suitable for shallow fetches")),3217OPT_BOOL(0,"honor-pack-keep", &ignore_packed_keep_on_disk,3218N_("ignore packs that have companion .keep file")),3219OPT_STRING_LIST(0,"keep-pack", &keep_pack_list,N_("name"),3220N_("ignore this pack")),3221OPT_INTEGER(0,"compression", &pack_compression_level,3222N_("pack compression level")),3223OPT_SET_INT(0,"keep-true-parents", &grafts_replace_parents,3224N_("do not hide commits by grafts"),0),3225OPT_BOOL(0,"use-bitmap-index", &use_bitmap_index,3226N_("use a bitmap index if available to speed up counting objects")),3227OPT_BOOL(0,"write-bitmap-index", &write_bitmap_index,3228N_("write a bitmap index together with the pack index")),3229OPT_PARSE_LIST_OBJECTS_FILTER(&filter_options),3230{ OPTION_CALLBACK,0,"missing", NULL,N_("action"),3231N_("handling for missing objects"), PARSE_OPT_NONEG,3232 option_parse_missing_action },3233OPT_BOOL(0,"exclude-promisor-objects", &exclude_promisor_objects,3234N_("do not pack objects in promisor packfiles")),3235OPT_END(),3236};32373238if(DFS_NUM_STATES > (1<< OE_DFS_STATE_BITS))3239BUG("too many dfs states, increase OE_DFS_STATE_BITS");32403241 read_replace_refs =0;32423243reset_pack_idx_option(&pack_idx_opts);3244git_config(git_pack_config, NULL);32453246 progress =isatty(2);3247 argc =parse_options(argc, argv, prefix, pack_objects_options,3248 pack_usage,0);32493250if(argc) {3251 base_name = argv[0];3252 argc--;3253}3254if(pack_to_stdout != !base_name || argc)3255usage_with_options(pack_usage, pack_objects_options);32563257if(depth >= (1<< OE_DEPTH_BITS)) {3258warning(_("delta chain depth%dis too deep, forcing%d"),3259 depth, (1<< OE_DEPTH_BITS) -1);3260 depth = (1<< OE_DEPTH_BITS) -1;3261}3262if(cache_max_small_delta_size >= (1U<< OE_Z_DELTA_BITS)) {3263warning(_("pack.deltaCacheLimit is too high, forcing%d"),3264(1U<< OE_Z_DELTA_BITS) -1);3265 cache_max_small_delta_size = (1U<< OE_Z_DELTA_BITS) -1;3266}32673268argv_array_push(&rp,"pack-objects");3269if(thin) {3270 use_internal_rev_list =1;3271argv_array_push(&rp, shallow3272?"--objects-edge-aggressive"3273:"--objects-edge");3274}else3275argv_array_push(&rp,"--objects");32763277if(rev_list_all) {3278 use_internal_rev_list =1;3279argv_array_push(&rp,"--all");3280}3281if(rev_list_reflog) {3282 use_internal_rev_list =1;3283argv_array_push(&rp,"--reflog");3284}3285if(rev_list_index) {3286 use_internal_rev_list =1;3287argv_array_push(&rp,"--indexed-objects");3288}3289if(rev_list_unpacked) {3290 use_internal_rev_list =1;3291argv_array_push(&rp,"--unpacked");3292}32933294if(exclude_promisor_objects) {3295 use_internal_rev_list =1;3296 fetch_if_missing =0;3297argv_array_push(&rp,"--exclude-promisor-objects");3298}3299if(unpack_unreachable || keep_unreachable || pack_loose_unreachable)3300 use_internal_rev_list =1;33013302if(!reuse_object)3303 reuse_delta =0;3304if(pack_compression_level == -1)3305 pack_compression_level = Z_DEFAULT_COMPRESSION;3306else if(pack_compression_level <0|| pack_compression_level > Z_BEST_COMPRESSION)3307die(_("bad pack compression level%d"), pack_compression_level);33083309if(!delta_search_threads)/* --threads=0 means autodetect */3310 delta_search_threads =online_cpus();33113312#ifdef NO_PTHREADS3313if(delta_search_threads !=1)3314warning(_("no threads support, ignoring --threads"));3315#endif3316if(!pack_to_stdout && !pack_size_limit)3317 pack_size_limit = pack_size_limit_cfg;3318if(pack_to_stdout && pack_size_limit)3319die(_("--max-pack-size cannot be used to build a pack for transfer"));3320if(pack_size_limit && pack_size_limit <1024*1024) {3321warning(_("minimum pack size limit is 1 MiB"));3322 pack_size_limit =1024*1024;3323}33243325if(!pack_to_stdout && thin)3326die(_("--thin cannot be used to build an indexable pack"));33273328if(keep_unreachable && unpack_unreachable)3329die(_("--keep-unreachable and --unpack-unreachable are incompatible"));3330if(!rev_list_all || !rev_list_reflog || !rev_list_index)3331 unpack_unreachable_expiration =0;33323333if(filter_options.choice) {3334if(!pack_to_stdout)3335die(_("cannot use --filter without --stdout"));3336 use_bitmap_index =0;3337}33383339/*3340 * "soft" reasons not to use bitmaps - for on-disk repack by default we want3341 *3342 * - to produce good pack (with bitmap index not-yet-packed objects are3343 * packed in suboptimal order).3344 *3345 * - to use more robust pack-generation codepath (avoiding possible3346 * bugs in bitmap code and possible bitmap index corruption).3347 */3348if(!pack_to_stdout)3349 use_bitmap_index_default =0;33503351if(use_bitmap_index <0)3352 use_bitmap_index = use_bitmap_index_default;33533354/* "hard" reasons not to use bitmaps; these just won't work at all */3355if(!use_internal_rev_list || (!pack_to_stdout && write_bitmap_index) ||is_repository_shallow(the_repository))3356 use_bitmap_index =0;33573358if(pack_to_stdout || !rev_list_all)3359 write_bitmap_index =0;33603361if(progress && all_progress_implied)3362 progress =2;33633364add_extra_kept_packs(&keep_pack_list);3365if(ignore_packed_keep_on_disk) {3366struct packed_git *p;3367for(p =get_all_packs(the_repository); p; p = p->next)3368if(p->pack_local && p->pack_keep)3369break;3370if(!p)/* no keep-able packs found */3371 ignore_packed_keep_on_disk =0;3372}3373if(local) {3374/*3375 * unlike ignore_packed_keep_on_disk above, we do not3376 * want to unset "local" based on looking at packs, as3377 * it also covers non-local objects3378 */3379struct packed_git *p;3380for(p =get_all_packs(the_repository); p; p = p->next) {3381if(!p->pack_local) {3382 have_non_local_packs =1;3383break;3384}3385}3386}33873388prepare_packing_data(&to_pack);33893390if(progress)3391 progress_state =start_progress(_("Enumerating objects"),0);3392if(!use_internal_rev_list)3393read_object_list_from_stdin();3394else{3395get_object_list(rp.argc, rp.argv);3396argv_array_clear(&rp);3397}3398cleanup_preferred_base();3399if(include_tag && nr_result)3400for_each_ref(add_ref_tag, NULL);3401stop_progress(&progress_state);34023403if(non_empty && !nr_result)3404return0;3405if(nr_result)3406prepare_pack(window, depth);3407write_pack_file();3408if(progress)3409fprintf_ln(stderr,3410_("Total %"PRIu32" (delta %"PRIu32"),"3411" reused %"PRIu32" (delta %"PRIu32")"),3412 written, written_delta, reused, reused_delta);3413return0;3414}