1#include"builtin.h" 2#include"cache.h" 3#include"repository.h" 4#include"config.h" 5#include"attr.h" 6#include"object.h" 7#include"blob.h" 8#include"commit.h" 9#include"tag.h" 10#include"tree.h" 11#include"delta.h" 12#include"pack.h" 13#include"pack-revindex.h" 14#include"csum-file.h" 15#include"tree-walk.h" 16#include"diff.h" 17#include"revision.h" 18#include"list-objects.h" 19#include"list-objects-filter.h" 20#include"list-objects-filter-options.h" 21#include"pack-objects.h" 22#include"progress.h" 23#include"refs.h" 24#include"streaming.h" 25#include"thread-utils.h" 26#include"pack-bitmap.h" 27#include"reachable.h" 28#include"sha1-array.h" 29#include"argv-array.h" 30#include"list.h" 31#include"packfile.h" 32#include"object-store.h" 33 34#define IN_PACK(obj) oe_in_pack(&to_pack, obj) 35#define SIZE(obj) oe_size(&to_pack, obj) 36#define SET_SIZE(obj,size) oe_set_size(&to_pack, obj, size) 37#define DELTA(obj) oe_delta(&to_pack, obj) 38#define DELTA_CHILD(obj) oe_delta_child(&to_pack, obj) 39#define DELTA_SIBLING(obj) oe_delta_sibling(&to_pack, obj) 40#define SET_DELTA(obj, val) oe_set_delta(&to_pack, obj, val) 41#define SET_DELTA_CHILD(obj, val) oe_set_delta_child(&to_pack, obj, val) 42#define SET_DELTA_SIBLING(obj, val) oe_set_delta_sibling(&to_pack, obj, val) 43 44static const char*pack_usage[] = { 45N_("git pack-objects --stdout [<options>...] [< <ref-list> | < <object-list>]"), 46N_("git pack-objects [<options>...] <base-name> [< <ref-list> | < <object-list>]"), 47 NULL 48}; 49 50/* 51 * Objects we are going to pack are collected in the `to_pack` structure. 52 * It contains an array (dynamically expanded) of the object data, and a map 53 * that can resolve SHA1s to their position in the array. 54 */ 55static struct packing_data to_pack; 56 57static struct pack_idx_entry **written_list; 58static uint32_t nr_result, nr_written; 59 60static int non_empty; 61static int reuse_delta =1, reuse_object =1; 62static int keep_unreachable, unpack_unreachable, include_tag; 63static timestamp_t unpack_unreachable_expiration; 64static int pack_loose_unreachable; 65static int local; 66static int have_non_local_packs; 67static int incremental; 68static int ignore_packed_keep; 69static int allow_ofs_delta; 70static struct pack_idx_option pack_idx_opts; 71static const char*base_name; 72static int progress =1; 73static int window =10; 74static unsigned long pack_size_limit; 75static int depth =50; 76static int delta_search_threads; 77static int pack_to_stdout; 78static int num_preferred_base; 79static struct progress *progress_state; 80 81static struct packed_git *reuse_packfile; 82static uint32_t reuse_packfile_objects; 83static off_t reuse_packfile_offset; 84 85static int use_bitmap_index_default =1; 86static int use_bitmap_index = -1; 87static int write_bitmap_index; 88static uint16_t write_bitmap_options; 89 90static int exclude_promisor_objects; 91 92static unsigned long delta_cache_size =0; 93static unsigned long max_delta_cache_size =256*1024*1024; 94static unsigned long cache_max_small_delta_size =1000; 95 96static unsigned long window_memory_limit =0; 97 98static struct list_objects_filter_options filter_options; 99 100enum missing_action { 101 MA_ERROR =0,/* fail if any missing objects are encountered */ 102 MA_ALLOW_ANY,/* silently allow ALL missing objects */ 103 MA_ALLOW_PROMISOR,/* silently allow all missing PROMISOR objects */ 104}; 105static enum missing_action arg_missing_action; 106static show_object_fn fn_show_object; 107 108/* 109 * stats 110 */ 111static uint32_t written, written_delta; 112static uint32_t reused, reused_delta; 113 114/* 115 * Indexed commits 116 */ 117static struct commit **indexed_commits; 118static unsigned int indexed_commits_nr; 119static unsigned int indexed_commits_alloc; 120 121static voidindex_commit_for_bitmap(struct commit *commit) 122{ 123if(indexed_commits_nr >= indexed_commits_alloc) { 124 indexed_commits_alloc = (indexed_commits_alloc +32) *2; 125REALLOC_ARRAY(indexed_commits, indexed_commits_alloc); 126} 127 128 indexed_commits[indexed_commits_nr++] = commit; 129} 130 131static void*get_delta(struct object_entry *entry) 132{ 133unsigned long size, base_size, delta_size; 134void*buf, *base_buf, *delta_buf; 135enum object_type type; 136 137 buf =read_object_file(&entry->idx.oid, &type, &size); 138if(!buf) 139die("unable to read%s",oid_to_hex(&entry->idx.oid)); 140 base_buf =read_object_file(&DELTA(entry)->idx.oid, &type, 141&base_size); 142if(!base_buf) 143die("unable to read%s", 144oid_to_hex(&DELTA(entry)->idx.oid)); 145 delta_buf =diff_delta(base_buf, base_size, 146 buf, size, &delta_size,0); 147if(!delta_buf || delta_size != entry->delta_size) 148die("delta size changed"); 149free(buf); 150free(base_buf); 151return delta_buf; 152} 153 154static unsigned longdo_compress(void**pptr,unsigned long size) 155{ 156 git_zstream stream; 157void*in, *out; 158unsigned long maxsize; 159 160git_deflate_init(&stream, pack_compression_level); 161 maxsize =git_deflate_bound(&stream, size); 162 163 in = *pptr; 164 out =xmalloc(maxsize); 165*pptr = out; 166 167 stream.next_in = in; 168 stream.avail_in = size; 169 stream.next_out = out; 170 stream.avail_out = maxsize; 171while(git_deflate(&stream, Z_FINISH) == Z_OK) 172;/* nothing */ 173git_deflate_end(&stream); 174 175free(in); 176return stream.total_out; 177} 178 179static unsigned longwrite_large_blob_data(struct git_istream *st,struct hashfile *f, 180const struct object_id *oid) 181{ 182 git_zstream stream; 183unsigned char ibuf[1024*16]; 184unsigned char obuf[1024*16]; 185unsigned long olen =0; 186 187git_deflate_init(&stream, pack_compression_level); 188 189for(;;) { 190 ssize_t readlen; 191int zret = Z_OK; 192 readlen =read_istream(st, ibuf,sizeof(ibuf)); 193if(readlen == -1) 194die(_("unable to read%s"),oid_to_hex(oid)); 195 196 stream.next_in = ibuf; 197 stream.avail_in = readlen; 198while((stream.avail_in || readlen ==0) && 199(zret == Z_OK || zret == Z_BUF_ERROR)) { 200 stream.next_out = obuf; 201 stream.avail_out =sizeof(obuf); 202 zret =git_deflate(&stream, readlen ?0: Z_FINISH); 203hashwrite(f, obuf, stream.next_out - obuf); 204 olen += stream.next_out - obuf; 205} 206if(stream.avail_in) 207die(_("deflate error (%d)"), zret); 208if(readlen ==0) { 209if(zret != Z_STREAM_END) 210die(_("deflate error (%d)"), zret); 211break; 212} 213} 214git_deflate_end(&stream); 215return olen; 216} 217 218/* 219 * we are going to reuse the existing object data as is. make 220 * sure it is not corrupt. 221 */ 222static intcheck_pack_inflate(struct packed_git *p, 223struct pack_window **w_curs, 224 off_t offset, 225 off_t len, 226unsigned long expect) 227{ 228 git_zstream stream; 229unsigned char fakebuf[4096], *in; 230int st; 231 232memset(&stream,0,sizeof(stream)); 233git_inflate_init(&stream); 234do{ 235 in =use_pack(p, w_curs, offset, &stream.avail_in); 236 stream.next_in = in; 237 stream.next_out = fakebuf; 238 stream.avail_out =sizeof(fakebuf); 239 st =git_inflate(&stream, Z_FINISH); 240 offset += stream.next_in - in; 241}while(st == Z_OK || st == Z_BUF_ERROR); 242git_inflate_end(&stream); 243return(st == Z_STREAM_END && 244 stream.total_out == expect && 245 stream.total_in == len) ?0: -1; 246} 247 248static voidcopy_pack_data(struct hashfile *f, 249struct packed_git *p, 250struct pack_window **w_curs, 251 off_t offset, 252 off_t len) 253{ 254unsigned char*in; 255unsigned long avail; 256 257while(len) { 258 in =use_pack(p, w_curs, offset, &avail); 259if(avail > len) 260 avail = (unsigned long)len; 261hashwrite(f, in, avail); 262 offset += avail; 263 len -= avail; 264} 265} 266 267/* Return 0 if we will bust the pack-size limit */ 268static unsigned longwrite_no_reuse_object(struct hashfile *f,struct object_entry *entry, 269unsigned long limit,int usable_delta) 270{ 271unsigned long size, datalen; 272unsigned char header[MAX_PACK_OBJECT_HEADER], 273 dheader[MAX_PACK_OBJECT_HEADER]; 274unsigned hdrlen; 275enum object_type type; 276void*buf; 277struct git_istream *st = NULL; 278 279if(!usable_delta) { 280if(oe_type(entry) == OBJ_BLOB && 281oe_size_greater_than(&to_pack, entry, big_file_threshold) && 282(st =open_istream(&entry->idx.oid, &type, &size, NULL)) != NULL) 283 buf = NULL; 284else{ 285 buf =read_object_file(&entry->idx.oid, &type, &size); 286if(!buf) 287die(_("unable to read%s"), 288oid_to_hex(&entry->idx.oid)); 289} 290/* 291 * make sure no cached delta data remains from a 292 * previous attempt before a pack split occurred. 293 */ 294FREE_AND_NULL(entry->delta_data); 295 entry->z_delta_size =0; 296}else if(entry->delta_data) { 297 size = entry->delta_size; 298 buf = entry->delta_data; 299 entry->delta_data = NULL; 300 type = (allow_ofs_delta &&DELTA(entry)->idx.offset) ? 301 OBJ_OFS_DELTA : OBJ_REF_DELTA; 302}else{ 303 buf =get_delta(entry); 304 size = entry->delta_size; 305 type = (allow_ofs_delta &&DELTA(entry)->idx.offset) ? 306 OBJ_OFS_DELTA : OBJ_REF_DELTA; 307} 308 309if(st)/* large blob case, just assume we don't compress well */ 310 datalen = size; 311else if(entry->z_delta_size) 312 datalen = entry->z_delta_size; 313else 314 datalen =do_compress(&buf, size); 315 316/* 317 * The object header is a byte of 'type' followed by zero or 318 * more bytes of length. 319 */ 320 hdrlen =encode_in_pack_object_header(header,sizeof(header), 321 type, size); 322 323if(type == OBJ_OFS_DELTA) { 324/* 325 * Deltas with relative base contain an additional 326 * encoding of the relative offset for the delta 327 * base from this object's position in the pack. 328 */ 329 off_t ofs = entry->idx.offset -DELTA(entry)->idx.offset; 330unsigned pos =sizeof(dheader) -1; 331 dheader[pos] = ofs &127; 332while(ofs >>=7) 333 dheader[--pos] =128| (--ofs &127); 334if(limit && hdrlen +sizeof(dheader) - pos + datalen +20>= limit) { 335if(st) 336close_istream(st); 337free(buf); 338return0; 339} 340hashwrite(f, header, hdrlen); 341hashwrite(f, dheader + pos,sizeof(dheader) - pos); 342 hdrlen +=sizeof(dheader) - pos; 343}else if(type == OBJ_REF_DELTA) { 344/* 345 * Deltas with a base reference contain 346 * an additional 20 bytes for the base sha1. 347 */ 348if(limit && hdrlen +20+ datalen +20>= limit) { 349if(st) 350close_istream(st); 351free(buf); 352return0; 353} 354hashwrite(f, header, hdrlen); 355hashwrite(f,DELTA(entry)->idx.oid.hash,20); 356 hdrlen +=20; 357}else{ 358if(limit && hdrlen + datalen +20>= limit) { 359if(st) 360close_istream(st); 361free(buf); 362return0; 363} 364hashwrite(f, header, hdrlen); 365} 366if(st) { 367 datalen =write_large_blob_data(st, f, &entry->idx.oid); 368close_istream(st); 369}else{ 370hashwrite(f, buf, datalen); 371free(buf); 372} 373 374return hdrlen + datalen; 375} 376 377/* Return 0 if we will bust the pack-size limit */ 378static off_t write_reuse_object(struct hashfile *f,struct object_entry *entry, 379unsigned long limit,int usable_delta) 380{ 381struct packed_git *p =IN_PACK(entry); 382struct pack_window *w_curs = NULL; 383struct revindex_entry *revidx; 384 off_t offset; 385enum object_type type =oe_type(entry); 386 off_t datalen; 387unsigned char header[MAX_PACK_OBJECT_HEADER], 388 dheader[MAX_PACK_OBJECT_HEADER]; 389unsigned hdrlen; 390unsigned long entry_size =SIZE(entry); 391 392if(DELTA(entry)) 393 type = (allow_ofs_delta &&DELTA(entry)->idx.offset) ? 394 OBJ_OFS_DELTA : OBJ_REF_DELTA; 395 hdrlen =encode_in_pack_object_header(header,sizeof(header), 396 type, entry_size); 397 398 offset = entry->in_pack_offset; 399 revidx =find_pack_revindex(p, offset); 400 datalen = revidx[1].offset - offset; 401if(!pack_to_stdout && p->index_version >1&& 402check_pack_crc(p, &w_curs, offset, datalen, revidx->nr)) { 403error("bad packed object CRC for%s", 404oid_to_hex(&entry->idx.oid)); 405unuse_pack(&w_curs); 406returnwrite_no_reuse_object(f, entry, limit, usable_delta); 407} 408 409 offset += entry->in_pack_header_size; 410 datalen -= entry->in_pack_header_size; 411 412if(!pack_to_stdout && p->index_version ==1&& 413check_pack_inflate(p, &w_curs, offset, datalen, entry_size)) { 414error("corrupt packed object for%s", 415oid_to_hex(&entry->idx.oid)); 416unuse_pack(&w_curs); 417returnwrite_no_reuse_object(f, entry, limit, usable_delta); 418} 419 420if(type == OBJ_OFS_DELTA) { 421 off_t ofs = entry->idx.offset -DELTA(entry)->idx.offset; 422unsigned pos =sizeof(dheader) -1; 423 dheader[pos] = ofs &127; 424while(ofs >>=7) 425 dheader[--pos] =128| (--ofs &127); 426if(limit && hdrlen +sizeof(dheader) - pos + datalen +20>= limit) { 427unuse_pack(&w_curs); 428return0; 429} 430hashwrite(f, header, hdrlen); 431hashwrite(f, dheader + pos,sizeof(dheader) - pos); 432 hdrlen +=sizeof(dheader) - pos; 433 reused_delta++; 434}else if(type == OBJ_REF_DELTA) { 435if(limit && hdrlen +20+ datalen +20>= limit) { 436unuse_pack(&w_curs); 437return0; 438} 439hashwrite(f, header, hdrlen); 440hashwrite(f,DELTA(entry)->idx.oid.hash,20); 441 hdrlen +=20; 442 reused_delta++; 443}else{ 444if(limit && hdrlen + datalen +20>= limit) { 445unuse_pack(&w_curs); 446return0; 447} 448hashwrite(f, header, hdrlen); 449} 450copy_pack_data(f, p, &w_curs, offset, datalen); 451unuse_pack(&w_curs); 452 reused++; 453return hdrlen + datalen; 454} 455 456/* Return 0 if we will bust the pack-size limit */ 457static off_t write_object(struct hashfile *f, 458struct object_entry *entry, 459 off_t write_offset) 460{ 461unsigned long limit; 462 off_t len; 463int usable_delta, to_reuse; 464 465if(!pack_to_stdout) 466crc32_begin(f); 467 468/* apply size limit if limited packsize and not first object */ 469if(!pack_size_limit || !nr_written) 470 limit =0; 471else if(pack_size_limit <= write_offset) 472/* 473 * the earlier object did not fit the limit; avoid 474 * mistaking this with unlimited (i.e. limit = 0). 475 */ 476 limit =1; 477else 478 limit = pack_size_limit - write_offset; 479 480if(!DELTA(entry)) 481 usable_delta =0;/* no delta */ 482else if(!pack_size_limit) 483 usable_delta =1;/* unlimited packfile */ 484else if(DELTA(entry)->idx.offset == (off_t)-1) 485 usable_delta =0;/* base was written to another pack */ 486else if(DELTA(entry)->idx.offset) 487 usable_delta =1;/* base already exists in this pack */ 488else 489 usable_delta =0;/* base could end up in another pack */ 490 491if(!reuse_object) 492 to_reuse =0;/* explicit */ 493else if(!IN_PACK(entry)) 494 to_reuse =0;/* can't reuse what we don't have */ 495else if(oe_type(entry) == OBJ_REF_DELTA || 496oe_type(entry) == OBJ_OFS_DELTA) 497/* check_object() decided it for us ... */ 498 to_reuse = usable_delta; 499/* ... but pack split may override that */ 500else if(oe_type(entry) != entry->in_pack_type) 501 to_reuse =0;/* pack has delta which is unusable */ 502else if(DELTA(entry)) 503 to_reuse =0;/* we want to pack afresh */ 504else 505 to_reuse =1;/* we have it in-pack undeltified, 506 * and we do not need to deltify it. 507 */ 508 509if(!to_reuse) 510 len =write_no_reuse_object(f, entry, limit, usable_delta); 511else 512 len =write_reuse_object(f, entry, limit, usable_delta); 513if(!len) 514return0; 515 516if(usable_delta) 517 written_delta++; 518 written++; 519if(!pack_to_stdout) 520 entry->idx.crc32 =crc32_end(f); 521return len; 522} 523 524enum write_one_status { 525 WRITE_ONE_SKIP = -1,/* already written */ 526 WRITE_ONE_BREAK =0,/* writing this will bust the limit; not written */ 527 WRITE_ONE_WRITTEN =1,/* normal */ 528 WRITE_ONE_RECURSIVE =2/* already scheduled to be written */ 529}; 530 531static enum write_one_status write_one(struct hashfile *f, 532struct object_entry *e, 533 off_t *offset) 534{ 535 off_t size; 536int recursing; 537 538/* 539 * we set offset to 1 (which is an impossible value) to mark 540 * the fact that this object is involved in "write its base 541 * first before writing a deltified object" recursion. 542 */ 543 recursing = (e->idx.offset ==1); 544if(recursing) { 545warning("recursive delta detected for object%s", 546oid_to_hex(&e->idx.oid)); 547return WRITE_ONE_RECURSIVE; 548}else if(e->idx.offset || e->preferred_base) { 549/* offset is non zero if object is written already. */ 550return WRITE_ONE_SKIP; 551} 552 553/* if we are deltified, write out base object first. */ 554if(DELTA(e)) { 555 e->idx.offset =1;/* now recurse */ 556switch(write_one(f,DELTA(e), offset)) { 557case WRITE_ONE_RECURSIVE: 558/* we cannot depend on this one */ 559SET_DELTA(e, NULL); 560break; 561default: 562break; 563case WRITE_ONE_BREAK: 564 e->idx.offset = recursing; 565return WRITE_ONE_BREAK; 566} 567} 568 569 e->idx.offset = *offset; 570 size =write_object(f, e, *offset); 571if(!size) { 572 e->idx.offset = recursing; 573return WRITE_ONE_BREAK; 574} 575 written_list[nr_written++] = &e->idx; 576 577/* make sure off_t is sufficiently large not to wrap */ 578if(signed_add_overflows(*offset, size)) 579die("pack too large for current definition of off_t"); 580*offset += size; 581return WRITE_ONE_WRITTEN; 582} 583 584static intmark_tagged(const char*path,const struct object_id *oid,int flag, 585void*cb_data) 586{ 587struct object_id peeled; 588struct object_entry *entry =packlist_find(&to_pack, oid->hash, NULL); 589 590if(entry) 591 entry->tagged =1; 592if(!peel_ref(path, &peeled)) { 593 entry =packlist_find(&to_pack, peeled.hash, NULL); 594if(entry) 595 entry->tagged =1; 596} 597return0; 598} 599 600staticinlinevoidadd_to_write_order(struct object_entry **wo, 601unsigned int*endp, 602struct object_entry *e) 603{ 604if(e->filled) 605return; 606 wo[(*endp)++] = e; 607 e->filled =1; 608} 609 610static voidadd_descendants_to_write_order(struct object_entry **wo, 611unsigned int*endp, 612struct object_entry *e) 613{ 614int add_to_order =1; 615while(e) { 616if(add_to_order) { 617struct object_entry *s; 618/* add this node... */ 619add_to_write_order(wo, endp, e); 620/* all its siblings... */ 621for(s =DELTA_SIBLING(e); s; s =DELTA_SIBLING(s)) { 622add_to_write_order(wo, endp, s); 623} 624} 625/* drop down a level to add left subtree nodes if possible */ 626if(DELTA_CHILD(e)) { 627 add_to_order =1; 628 e =DELTA_CHILD(e); 629}else{ 630 add_to_order =0; 631/* our sibling might have some children, it is next */ 632if(DELTA_SIBLING(e)) { 633 e =DELTA_SIBLING(e); 634continue; 635} 636/* go back to our parent node */ 637 e =DELTA(e); 638while(e && !DELTA_SIBLING(e)) { 639/* we're on the right side of a subtree, keep 640 * going up until we can go right again */ 641 e =DELTA(e); 642} 643if(!e) { 644/* done- we hit our original root node */ 645return; 646} 647/* pass it off to sibling at this level */ 648 e =DELTA_SIBLING(e); 649} 650}; 651} 652 653static voidadd_family_to_write_order(struct object_entry **wo, 654unsigned int*endp, 655struct object_entry *e) 656{ 657struct object_entry *root; 658 659for(root = e;DELTA(root); root =DELTA(root)) 660;/* nothing */ 661add_descendants_to_write_order(wo, endp, root); 662} 663 664static struct object_entry **compute_write_order(void) 665{ 666unsigned int i, wo_end, last_untagged; 667 668struct object_entry **wo; 669struct object_entry *objects = to_pack.objects; 670 671for(i =0; i < to_pack.nr_objects; i++) { 672 objects[i].tagged =0; 673 objects[i].filled =0; 674SET_DELTA_CHILD(&objects[i], NULL); 675SET_DELTA_SIBLING(&objects[i], NULL); 676} 677 678/* 679 * Fully connect delta_child/delta_sibling network. 680 * Make sure delta_sibling is sorted in the original 681 * recency order. 682 */ 683for(i = to_pack.nr_objects; i >0;) { 684struct object_entry *e = &objects[--i]; 685if(!DELTA(e)) 686continue; 687/* Mark me as the first child */ 688 e->delta_sibling_idx =DELTA(e)->delta_child_idx; 689SET_DELTA_CHILD(DELTA(e), e); 690} 691 692/* 693 * Mark objects that are at the tip of tags. 694 */ 695for_each_tag_ref(mark_tagged, NULL); 696 697/* 698 * Give the objects in the original recency order until 699 * we see a tagged tip. 700 */ 701ALLOC_ARRAY(wo, to_pack.nr_objects); 702for(i = wo_end =0; i < to_pack.nr_objects; i++) { 703if(objects[i].tagged) 704break; 705add_to_write_order(wo, &wo_end, &objects[i]); 706} 707 last_untagged = i; 708 709/* 710 * Then fill all the tagged tips. 711 */ 712for(; i < to_pack.nr_objects; i++) { 713if(objects[i].tagged) 714add_to_write_order(wo, &wo_end, &objects[i]); 715} 716 717/* 718 * And then all remaining commits and tags. 719 */ 720for(i = last_untagged; i < to_pack.nr_objects; i++) { 721if(oe_type(&objects[i]) != OBJ_COMMIT && 722oe_type(&objects[i]) != OBJ_TAG) 723continue; 724add_to_write_order(wo, &wo_end, &objects[i]); 725} 726 727/* 728 * And then all the trees. 729 */ 730for(i = last_untagged; i < to_pack.nr_objects; i++) { 731if(oe_type(&objects[i]) != OBJ_TREE) 732continue; 733add_to_write_order(wo, &wo_end, &objects[i]); 734} 735 736/* 737 * Finally all the rest in really tight order 738 */ 739for(i = last_untagged; i < to_pack.nr_objects; i++) { 740if(!objects[i].filled) 741add_family_to_write_order(wo, &wo_end, &objects[i]); 742} 743 744if(wo_end != to_pack.nr_objects) 745die("ordered%uobjects, expected %"PRIu32, wo_end, to_pack.nr_objects); 746 747return wo; 748} 749 750static off_t write_reused_pack(struct hashfile *f) 751{ 752unsigned char buffer[8192]; 753 off_t to_write, total; 754int fd; 755 756if(!is_pack_valid(reuse_packfile)) 757die("packfile is invalid:%s", reuse_packfile->pack_name); 758 759 fd =git_open(reuse_packfile->pack_name); 760if(fd <0) 761die_errno("unable to open packfile for reuse:%s", 762 reuse_packfile->pack_name); 763 764if(lseek(fd,sizeof(struct pack_header), SEEK_SET) == -1) 765die_errno("unable to seek in reused packfile"); 766 767if(reuse_packfile_offset <0) 768 reuse_packfile_offset = reuse_packfile->pack_size -20; 769 770 total = to_write = reuse_packfile_offset -sizeof(struct pack_header); 771 772while(to_write) { 773int read_pack =xread(fd, buffer,sizeof(buffer)); 774 775if(read_pack <=0) 776die_errno("unable to read from reused packfile"); 777 778if(read_pack > to_write) 779 read_pack = to_write; 780 781hashwrite(f, buffer, read_pack); 782 to_write -= read_pack; 783 784/* 785 * We don't know the actual number of objects written, 786 * only how many bytes written, how many bytes total, and 787 * how many objects total. So we can fake it by pretending all 788 * objects we are writing are the same size. This gives us a 789 * smooth progress meter, and at the end it matches the true 790 * answer. 791 */ 792 written = reuse_packfile_objects * 793(((double)(total - to_write)) / total); 794display_progress(progress_state, written); 795} 796 797close(fd); 798 written = reuse_packfile_objects; 799display_progress(progress_state, written); 800return reuse_packfile_offset -sizeof(struct pack_header); 801} 802 803static const char no_split_warning[] =N_( 804"disabling bitmap writing, packs are split due to pack.packSizeLimit" 805); 806 807static voidwrite_pack_file(void) 808{ 809uint32_t i =0, j; 810struct hashfile *f; 811 off_t offset; 812uint32_t nr_remaining = nr_result; 813time_t last_mtime =0; 814struct object_entry **write_order; 815 816if(progress > pack_to_stdout) 817 progress_state =start_progress(_("Writing objects"), nr_result); 818ALLOC_ARRAY(written_list, to_pack.nr_objects); 819 write_order =compute_write_order(); 820 821do{ 822struct object_id oid; 823char*pack_tmp_name = NULL; 824 825if(pack_to_stdout) 826 f =hashfd_throughput(1,"<stdout>", progress_state); 827else 828 f =create_tmp_packfile(&pack_tmp_name); 829 830 offset =write_pack_header(f, nr_remaining); 831 832if(reuse_packfile) { 833 off_t packfile_size; 834assert(pack_to_stdout); 835 836 packfile_size =write_reused_pack(f); 837 offset += packfile_size; 838} 839 840 nr_written =0; 841for(; i < to_pack.nr_objects; i++) { 842struct object_entry *e = write_order[i]; 843if(write_one(f, e, &offset) == WRITE_ONE_BREAK) 844break; 845display_progress(progress_state, written); 846} 847 848/* 849 * Did we write the wrong # entries in the header? 850 * If so, rewrite it like in fast-import 851 */ 852if(pack_to_stdout) { 853hashclose(f, oid.hash, CSUM_CLOSE); 854}else if(nr_written == nr_remaining) { 855hashclose(f, oid.hash, CSUM_FSYNC); 856}else{ 857int fd =hashclose(f, oid.hash,0); 858fixup_pack_header_footer(fd, oid.hash, pack_tmp_name, 859 nr_written, oid.hash, offset); 860close(fd); 861if(write_bitmap_index) { 862warning(_(no_split_warning)); 863 write_bitmap_index =0; 864} 865} 866 867if(!pack_to_stdout) { 868struct stat st; 869struct strbuf tmpname = STRBUF_INIT; 870 871/* 872 * Packs are runtime accessed in their mtime 873 * order since newer packs are more likely to contain 874 * younger objects. So if we are creating multiple 875 * packs then we should modify the mtime of later ones 876 * to preserve this property. 877 */ 878if(stat(pack_tmp_name, &st) <0) { 879warning_errno("failed to stat%s", pack_tmp_name); 880}else if(!last_mtime) { 881 last_mtime = st.st_mtime; 882}else{ 883struct utimbuf utb; 884 utb.actime = st.st_atime; 885 utb.modtime = --last_mtime; 886if(utime(pack_tmp_name, &utb) <0) 887warning_errno("failed utime() on%s", pack_tmp_name); 888} 889 890strbuf_addf(&tmpname,"%s-", base_name); 891 892if(write_bitmap_index) { 893bitmap_writer_set_checksum(oid.hash); 894bitmap_writer_build_type_index( 895&to_pack, written_list, nr_written); 896} 897 898finish_tmp_packfile(&tmpname, pack_tmp_name, 899 written_list, nr_written, 900&pack_idx_opts, oid.hash); 901 902if(write_bitmap_index) { 903strbuf_addf(&tmpname,"%s.bitmap",oid_to_hex(&oid)); 904 905stop_progress(&progress_state); 906 907bitmap_writer_show_progress(progress); 908bitmap_writer_reuse_bitmaps(&to_pack); 909bitmap_writer_select_commits(indexed_commits, indexed_commits_nr, -1); 910bitmap_writer_build(&to_pack); 911bitmap_writer_finish(written_list, nr_written, 912 tmpname.buf, write_bitmap_options); 913 write_bitmap_index =0; 914} 915 916strbuf_release(&tmpname); 917free(pack_tmp_name); 918puts(oid_to_hex(&oid)); 919} 920 921/* mark written objects as written to previous pack */ 922for(j =0; j < nr_written; j++) { 923 written_list[j]->offset = (off_t)-1; 924} 925 nr_remaining -= nr_written; 926}while(nr_remaining && i < to_pack.nr_objects); 927 928free(written_list); 929free(write_order); 930stop_progress(&progress_state); 931if(written != nr_result) 932die("wrote %"PRIu32" objects while expecting %"PRIu32, 933 written, nr_result); 934} 935 936static intno_try_delta(const char*path) 937{ 938static struct attr_check *check; 939 940if(!check) 941 check =attr_check_initl("delta", NULL); 942if(git_check_attr(path, check)) 943return0; 944if(ATTR_FALSE(check->items[0].value)) 945return1; 946return0; 947} 948 949/* 950 * When adding an object, check whether we have already added it 951 * to our packing list. If so, we can skip. However, if we are 952 * being asked to excludei t, but the previous mention was to include 953 * it, make sure to adjust its flags and tweak our numbers accordingly. 954 * 955 * As an optimization, we pass out the index position where we would have 956 * found the item, since that saves us from having to look it up again a 957 * few lines later when we want to add the new entry. 958 */ 959static inthave_duplicate_entry(const struct object_id *oid, 960int exclude, 961uint32_t*index_pos) 962{ 963struct object_entry *entry; 964 965 entry =packlist_find(&to_pack, oid->hash, index_pos); 966if(!entry) 967return0; 968 969if(exclude) { 970if(!entry->preferred_base) 971 nr_result--; 972 entry->preferred_base =1; 973} 974 975return1; 976} 977 978static intwant_found_object(int exclude,struct packed_git *p) 979{ 980if(exclude) 981return1; 982if(incremental) 983return0; 984 985/* 986 * When asked to do --local (do not include an object that appears in a 987 * pack we borrow from elsewhere) or --honor-pack-keep (do not include 988 * an object that appears in a pack marked with .keep), finding a pack 989 * that matches the criteria is sufficient for us to decide to omit it. 990 * However, even if this pack does not satisfy the criteria, we need to 991 * make sure no copy of this object appears in _any_ pack that makes us 992 * to omit the object, so we need to check all the packs. 993 * 994 * We can however first check whether these options can possible matter; 995 * if they do not matter we know we want the object in generated pack. 996 * Otherwise, we signal "-1" at the end to tell the caller that we do 997 * not know either way, and it needs to check more packs. 998 */ 999if(!ignore_packed_keep &&1000(!local || !have_non_local_packs))1001return1;10021003if(local && !p->pack_local)1004return0;1005if(ignore_packed_keep && p->pack_local && p->pack_keep)1006return0;10071008/* we don't know yet; keep looking for more packs */1009return-1;1010}10111012/*1013 * Check whether we want the object in the pack (e.g., we do not want1014 * objects found in non-local stores if the "--local" option was used).1015 *1016 * If the caller already knows an existing pack it wants to take the object1017 * from, that is passed in *found_pack and *found_offset; otherwise this1018 * function finds if there is any pack that has the object and returns the pack1019 * and its offset in these variables.1020 */1021static intwant_object_in_pack(const struct object_id *oid,1022int exclude,1023struct packed_git **found_pack,1024 off_t *found_offset)1025{1026int want;1027struct list_head *pos;10281029if(!exclude && local &&has_loose_object_nonlocal(oid->hash))1030return0;10311032/*1033 * If we already know the pack object lives in, start checks from that1034 * pack - in the usual case when neither --local was given nor .keep files1035 * are present we will determine the answer right now.1036 */1037if(*found_pack) {1038 want =want_found_object(exclude, *found_pack);1039if(want != -1)1040return want;1041}1042list_for_each(pos,get_packed_git_mru(the_repository)) {1043struct packed_git *p =list_entry(pos,struct packed_git, mru);1044 off_t offset;10451046if(p == *found_pack)1047 offset = *found_offset;1048else1049 offset =find_pack_entry_one(oid->hash, p);10501051if(offset) {1052if(!*found_pack) {1053if(!is_pack_valid(p))1054continue;1055*found_offset = offset;1056*found_pack = p;1057}1058 want =want_found_object(exclude, p);1059if(!exclude && want >0)1060list_move(&p->mru,1061get_packed_git_mru(the_repository));1062if(want != -1)1063return want;1064}1065}10661067return1;1068}10691070static voidcreate_object_entry(const struct object_id *oid,1071enum object_type type,1072uint32_t hash,1073int exclude,1074int no_try_delta,1075uint32_t index_pos,1076struct packed_git *found_pack,1077 off_t found_offset)1078{1079struct object_entry *entry;10801081 entry =packlist_alloc(&to_pack, oid->hash, index_pos);1082 entry->hash = hash;1083oe_set_type(entry, type);1084if(exclude)1085 entry->preferred_base =1;1086else1087 nr_result++;1088if(found_pack) {1089oe_set_in_pack(&to_pack, entry, found_pack);1090 entry->in_pack_offset = found_offset;1091}10921093 entry->no_try_delta = no_try_delta;1094}10951096static const char no_closure_warning[] =N_(1097"disabling bitmap writing, as some objects are not being packed"1098);10991100static intadd_object_entry(const struct object_id *oid,enum object_type type,1101const char*name,int exclude)1102{1103struct packed_git *found_pack = NULL;1104 off_t found_offset =0;1105uint32_t index_pos;11061107if(have_duplicate_entry(oid, exclude, &index_pos))1108return0;11091110if(!want_object_in_pack(oid, exclude, &found_pack, &found_offset)) {1111/* The pack is missing an object, so it will not have closure */1112if(write_bitmap_index) {1113warning(_(no_closure_warning));1114 write_bitmap_index =0;1115}1116return0;1117}11181119create_object_entry(oid, type,pack_name_hash(name),1120 exclude, name &&no_try_delta(name),1121 index_pos, found_pack, found_offset);11221123display_progress(progress_state, nr_result);1124return1;1125}11261127static intadd_object_entry_from_bitmap(const struct object_id *oid,1128enum object_type type,1129int flags,uint32_t name_hash,1130struct packed_git *pack, off_t offset)1131{1132uint32_t index_pos;11331134if(have_duplicate_entry(oid,0, &index_pos))1135return0;11361137if(!want_object_in_pack(oid,0, &pack, &offset))1138return0;11391140create_object_entry(oid, type, name_hash,0,0, index_pos, pack, offset);11411142display_progress(progress_state, nr_result);1143return1;1144}11451146struct pbase_tree_cache {1147struct object_id oid;1148int ref;1149int temporary;1150void*tree_data;1151unsigned long tree_size;1152};11531154static struct pbase_tree_cache *(pbase_tree_cache[256]);1155static intpbase_tree_cache_ix(const struct object_id *oid)1156{1157return oid->hash[0] %ARRAY_SIZE(pbase_tree_cache);1158}1159static intpbase_tree_cache_ix_incr(int ix)1160{1161return(ix+1) %ARRAY_SIZE(pbase_tree_cache);1162}11631164static struct pbase_tree {1165struct pbase_tree *next;1166/* This is a phony "cache" entry; we are not1167 * going to evict it or find it through _get()1168 * mechanism -- this is for the toplevel node that1169 * would almost always change with any commit.1170 */1171struct pbase_tree_cache pcache;1172} *pbase_tree;11731174static struct pbase_tree_cache *pbase_tree_get(const struct object_id *oid)1175{1176struct pbase_tree_cache *ent, *nent;1177void*data;1178unsigned long size;1179enum object_type type;1180int neigh;1181int my_ix =pbase_tree_cache_ix(oid);1182int available_ix = -1;11831184/* pbase-tree-cache acts as a limited hashtable.1185 * your object will be found at your index or within a few1186 * slots after that slot if it is cached.1187 */1188for(neigh =0; neigh <8; neigh++) {1189 ent = pbase_tree_cache[my_ix];1190if(ent && !oidcmp(&ent->oid, oid)) {1191 ent->ref++;1192return ent;1193}1194else if(((available_ix <0) && (!ent || !ent->ref)) ||1195((0<= available_ix) &&1196(!ent && pbase_tree_cache[available_ix])))1197 available_ix = my_ix;1198if(!ent)1199break;1200 my_ix =pbase_tree_cache_ix_incr(my_ix);1201}12021203/* Did not find one. Either we got a bogus request or1204 * we need to read and perhaps cache.1205 */1206 data =read_object_file(oid, &type, &size);1207if(!data)1208return NULL;1209if(type != OBJ_TREE) {1210free(data);1211return NULL;1212}12131214/* We need to either cache or return a throwaway copy */12151216if(available_ix <0)1217 ent = NULL;1218else{1219 ent = pbase_tree_cache[available_ix];1220 my_ix = available_ix;1221}12221223if(!ent) {1224 nent =xmalloc(sizeof(*nent));1225 nent->temporary = (available_ix <0);1226}1227else{1228/* evict and reuse */1229free(ent->tree_data);1230 nent = ent;1231}1232oidcpy(&nent->oid, oid);1233 nent->tree_data = data;1234 nent->tree_size = size;1235 nent->ref =1;1236if(!nent->temporary)1237 pbase_tree_cache[my_ix] = nent;1238return nent;1239}12401241static voidpbase_tree_put(struct pbase_tree_cache *cache)1242{1243if(!cache->temporary) {1244 cache->ref--;1245return;1246}1247free(cache->tree_data);1248free(cache);1249}12501251static intname_cmp_len(const char*name)1252{1253int i;1254for(i =0; name[i] && name[i] !='\n'&& name[i] !='/'; i++)1255;1256return i;1257}12581259static voidadd_pbase_object(struct tree_desc *tree,1260const char*name,1261int cmplen,1262const char*fullname)1263{1264struct name_entry entry;1265int cmp;12661267while(tree_entry(tree,&entry)) {1268if(S_ISGITLINK(entry.mode))1269continue;1270 cmp =tree_entry_len(&entry) != cmplen ?1:1271memcmp(name, entry.path, cmplen);1272if(cmp >0)1273continue;1274if(cmp <0)1275return;1276if(name[cmplen] !='/') {1277add_object_entry(entry.oid,1278object_type(entry.mode),1279 fullname,1);1280return;1281}1282if(S_ISDIR(entry.mode)) {1283struct tree_desc sub;1284struct pbase_tree_cache *tree;1285const char*down = name+cmplen+1;1286int downlen =name_cmp_len(down);12871288 tree =pbase_tree_get(entry.oid);1289if(!tree)1290return;1291init_tree_desc(&sub, tree->tree_data, tree->tree_size);12921293add_pbase_object(&sub, down, downlen, fullname);1294pbase_tree_put(tree);1295}1296}1297}12981299static unsigned*done_pbase_paths;1300static int done_pbase_paths_num;1301static int done_pbase_paths_alloc;1302static intdone_pbase_path_pos(unsigned hash)1303{1304int lo =0;1305int hi = done_pbase_paths_num;1306while(lo < hi) {1307int mi = lo + (hi - lo) /2;1308if(done_pbase_paths[mi] == hash)1309return mi;1310if(done_pbase_paths[mi] < hash)1311 hi = mi;1312else1313 lo = mi +1;1314}1315return-lo-1;1316}13171318static intcheck_pbase_path(unsigned hash)1319{1320int pos =done_pbase_path_pos(hash);1321if(0<= pos)1322return1;1323 pos = -pos -1;1324ALLOC_GROW(done_pbase_paths,1325 done_pbase_paths_num +1,1326 done_pbase_paths_alloc);1327 done_pbase_paths_num++;1328if(pos < done_pbase_paths_num)1329MOVE_ARRAY(done_pbase_paths + pos +1, done_pbase_paths + pos,1330 done_pbase_paths_num - pos -1);1331 done_pbase_paths[pos] = hash;1332return0;1333}13341335static voidadd_preferred_base_object(const char*name)1336{1337struct pbase_tree *it;1338int cmplen;1339unsigned hash =pack_name_hash(name);13401341if(!num_preferred_base ||check_pbase_path(hash))1342return;13431344 cmplen =name_cmp_len(name);1345for(it = pbase_tree; it; it = it->next) {1346if(cmplen ==0) {1347add_object_entry(&it->pcache.oid, OBJ_TREE, NULL,1);1348}1349else{1350struct tree_desc tree;1351init_tree_desc(&tree, it->pcache.tree_data, it->pcache.tree_size);1352add_pbase_object(&tree, name, cmplen, name);1353}1354}1355}13561357static voidadd_preferred_base(struct object_id *oid)1358{1359struct pbase_tree *it;1360void*data;1361unsigned long size;1362struct object_id tree_oid;13631364if(window <= num_preferred_base++)1365return;13661367 data =read_object_with_reference(oid, tree_type, &size, &tree_oid);1368if(!data)1369return;13701371for(it = pbase_tree; it; it = it->next) {1372if(!oidcmp(&it->pcache.oid, &tree_oid)) {1373free(data);1374return;1375}1376}13771378 it =xcalloc(1,sizeof(*it));1379 it->next = pbase_tree;1380 pbase_tree = it;13811382oidcpy(&it->pcache.oid, &tree_oid);1383 it->pcache.tree_data = data;1384 it->pcache.tree_size = size;1385}13861387static voidcleanup_preferred_base(void)1388{1389struct pbase_tree *it;1390unsigned i;13911392 it = pbase_tree;1393 pbase_tree = NULL;1394while(it) {1395struct pbase_tree *tmp = it;1396 it = tmp->next;1397free(tmp->pcache.tree_data);1398free(tmp);1399}14001401for(i =0; i <ARRAY_SIZE(pbase_tree_cache); i++) {1402if(!pbase_tree_cache[i])1403continue;1404free(pbase_tree_cache[i]->tree_data);1405FREE_AND_NULL(pbase_tree_cache[i]);1406}14071408FREE_AND_NULL(done_pbase_paths);1409 done_pbase_paths_num = done_pbase_paths_alloc =0;1410}14111412static voidcheck_object(struct object_entry *entry)1413{1414unsigned long canonical_size;14151416if(IN_PACK(entry)) {1417struct packed_git *p =IN_PACK(entry);1418struct pack_window *w_curs = NULL;1419const unsigned char*base_ref = NULL;1420struct object_entry *base_entry;1421unsigned long used, used_0;1422unsigned long avail;1423 off_t ofs;1424unsigned char*buf, c;1425enum object_type type;1426unsigned long in_pack_size;14271428 buf =use_pack(p, &w_curs, entry->in_pack_offset, &avail);14291430/*1431 * We want in_pack_type even if we do not reuse delta1432 * since non-delta representations could still be reused.1433 */1434 used =unpack_object_header_buffer(buf, avail,1435&type,1436&in_pack_size);1437if(used ==0)1438goto give_up;14391440if(type <0)1441BUG("invalid type%d", type);1442 entry->in_pack_type = type;14431444/*1445 * Determine if this is a delta and if so whether we can1446 * reuse it or not. Otherwise let's find out as cheaply as1447 * possible what the actual type and size for this object is.1448 */1449switch(entry->in_pack_type) {1450default:1451/* Not a delta hence we've already got all we need. */1452oe_set_type(entry, entry->in_pack_type);1453SET_SIZE(entry, in_pack_size);1454 entry->in_pack_header_size = used;1455if(oe_type(entry) < OBJ_COMMIT ||oe_type(entry) > OBJ_BLOB)1456goto give_up;1457unuse_pack(&w_curs);1458return;1459case OBJ_REF_DELTA:1460if(reuse_delta && !entry->preferred_base)1461 base_ref =use_pack(p, &w_curs,1462 entry->in_pack_offset + used, NULL);1463 entry->in_pack_header_size = used +20;1464break;1465case OBJ_OFS_DELTA:1466 buf =use_pack(p, &w_curs,1467 entry->in_pack_offset + used, NULL);1468 used_0 =0;1469 c = buf[used_0++];1470 ofs = c &127;1471while(c &128) {1472 ofs +=1;1473if(!ofs ||MSB(ofs,7)) {1474error("delta base offset overflow in pack for%s",1475oid_to_hex(&entry->idx.oid));1476goto give_up;1477}1478 c = buf[used_0++];1479 ofs = (ofs <<7) + (c &127);1480}1481 ofs = entry->in_pack_offset - ofs;1482if(ofs <=0|| ofs >= entry->in_pack_offset) {1483error("delta base offset out of bound for%s",1484oid_to_hex(&entry->idx.oid));1485goto give_up;1486}1487if(reuse_delta && !entry->preferred_base) {1488struct revindex_entry *revidx;1489 revidx =find_pack_revindex(p, ofs);1490if(!revidx)1491goto give_up;1492 base_ref =nth_packed_object_sha1(p, revidx->nr);1493}1494 entry->in_pack_header_size = used + used_0;1495break;1496}14971498if(base_ref && (base_entry =packlist_find(&to_pack, base_ref, NULL))) {1499/*1500 * If base_ref was set above that means we wish to1501 * reuse delta data, and we even found that base1502 * in the list of objects we want to pack. Goodie!1503 *1504 * Depth value does not matter - find_deltas() will1505 * never consider reused delta as the base object to1506 * deltify other objects against, in order to avoid1507 * circular deltas.1508 */1509oe_set_type(entry, entry->in_pack_type);1510SET_SIZE(entry, in_pack_size);/* delta size */1511SET_DELTA(entry, base_entry);1512 entry->delta_size = in_pack_size;1513 entry->delta_sibling_idx = base_entry->delta_child_idx;1514SET_DELTA_CHILD(base_entry, entry);1515unuse_pack(&w_curs);1516return;1517}15181519if(oe_type(entry)) {1520 off_t delta_pos;15211522/*1523 * This must be a delta and we already know what the1524 * final object type is. Let's extract the actual1525 * object size from the delta header.1526 */1527 delta_pos = entry->in_pack_offset + entry->in_pack_header_size;1528 canonical_size =get_size_from_delta(p, &w_curs, delta_pos);1529if(canonical_size ==0)1530goto give_up;1531SET_SIZE(entry, canonical_size);1532unuse_pack(&w_curs);1533return;1534}15351536/*1537 * No choice but to fall back to the recursive delta walk1538 * with sha1_object_info() to find about the object type1539 * at this point...1540 */1541 give_up:1542unuse_pack(&w_curs);1543}15441545oe_set_type(entry,oid_object_info(&entry->idx.oid, &canonical_size));1546if(entry->type_valid) {1547SET_SIZE(entry, canonical_size);1548}else{1549/*1550 * Bad object type is checked in prepare_pack(). This is1551 * to permit a missing preferred base object to be ignored1552 * as a preferred base. Doing so can result in a larger1553 * pack file, but the transfer will still take place.1554 */1555}1556}15571558static intpack_offset_sort(const void*_a,const void*_b)1559{1560const struct object_entry *a = *(struct object_entry **)_a;1561const struct object_entry *b = *(struct object_entry **)_b;1562const struct packed_git *a_in_pack =IN_PACK(a);1563const struct packed_git *b_in_pack =IN_PACK(b);15641565/* avoid filesystem trashing with loose objects */1566if(!a_in_pack && !b_in_pack)1567returnoidcmp(&a->idx.oid, &b->idx.oid);15681569if(a_in_pack < b_in_pack)1570return-1;1571if(a_in_pack > b_in_pack)1572return1;1573return a->in_pack_offset < b->in_pack_offset ? -1:1574(a->in_pack_offset > b->in_pack_offset);1575}15761577/*1578 * Drop an on-disk delta we were planning to reuse. Naively, this would1579 * just involve blanking out the "delta" field, but we have to deal1580 * with some extra book-keeping:1581 *1582 * 1. Removing ourselves from the delta_sibling linked list.1583 *1584 * 2. Updating our size/type to the non-delta representation. These were1585 * either not recorded initially (size) or overwritten with the delta type1586 * (type) when check_object() decided to reuse the delta.1587 *1588 * 3. Resetting our delta depth, as we are now a base object.1589 */1590static voiddrop_reused_delta(struct object_entry *entry)1591{1592unsigned*idx = &to_pack.objects[entry->delta_idx -1].delta_child_idx;1593struct object_info oi = OBJECT_INFO_INIT;1594enum object_type type;1595unsigned long size;15961597while(*idx) {1598struct object_entry *oe = &to_pack.objects[*idx -1];15991600if(oe == entry)1601*idx = oe->delta_sibling_idx;1602else1603 idx = &oe->delta_sibling_idx;1604}1605SET_DELTA(entry, NULL);1606 entry->depth =0;16071608 oi.sizep = &size;1609 oi.typep = &type;1610if(packed_object_info(IN_PACK(entry), entry->in_pack_offset, &oi) <0) {1611/*1612 * We failed to get the info from this pack for some reason;1613 * fall back to sha1_object_info, which may find another copy.1614 * And if that fails, the error will be recorded in oe_type(entry)1615 * and dealt with in prepare_pack().1616 */1617oe_set_type(entry,oid_object_info(&entry->idx.oid, &size));1618}else{1619oe_set_type(entry, type);1620}1621SET_SIZE(entry, size);1622}16231624/*1625 * Follow the chain of deltas from this entry onward, throwing away any links1626 * that cause us to hit a cycle (as determined by the DFS state flags in1627 * the entries).1628 *1629 * We also detect too-long reused chains that would violate our --depth1630 * limit.1631 */1632static voidbreak_delta_chains(struct object_entry *entry)1633{1634/*1635 * The actual depth of each object we will write is stored as an int,1636 * as it cannot exceed our int "depth" limit. But before we break1637 * changes based no that limit, we may potentially go as deep as the1638 * number of objects, which is elsewhere bounded to a uint32_t.1639 */1640uint32_t total_depth;1641struct object_entry *cur, *next;16421643for(cur = entry, total_depth =0;1644 cur;1645 cur =DELTA(cur), total_depth++) {1646if(cur->dfs_state == DFS_DONE) {1647/*1648 * We've already seen this object and know it isn't1649 * part of a cycle. We do need to append its depth1650 * to our count.1651 */1652 total_depth += cur->depth;1653break;1654}16551656/*1657 * We break cycles before looping, so an ACTIVE state (or any1658 * other cruft which made its way into the state variable)1659 * is a bug.1660 */1661if(cur->dfs_state != DFS_NONE)1662die("BUG: confusing delta dfs state in first pass:%d",1663 cur->dfs_state);16641665/*1666 * Now we know this is the first time we've seen the object. If1667 * it's not a delta, we're done traversing, but we'll mark it1668 * done to save time on future traversals.1669 */1670if(!DELTA(cur)) {1671 cur->dfs_state = DFS_DONE;1672break;1673}16741675/*1676 * Mark ourselves as active and see if the next step causes1677 * us to cycle to another active object. It's important to do1678 * this _before_ we loop, because it impacts where we make the1679 * cut, and thus how our total_depth counter works.1680 * E.g., We may see a partial loop like:1681 *1682 * A -> B -> C -> D -> B1683 *1684 * Cutting B->C breaks the cycle. But now the depth of A is1685 * only 1, and our total_depth counter is at 3. The size of the1686 * error is always one less than the size of the cycle we1687 * broke. Commits C and D were "lost" from A's chain.1688 *1689 * If we instead cut D->B, then the depth of A is correct at 3.1690 * We keep all commits in the chain that we examined.1691 */1692 cur->dfs_state = DFS_ACTIVE;1693if(DELTA(cur)->dfs_state == DFS_ACTIVE) {1694drop_reused_delta(cur);1695 cur->dfs_state = DFS_DONE;1696break;1697}1698}16991700/*1701 * And now that we've gone all the way to the bottom of the chain, we1702 * need to clear the active flags and set the depth fields as1703 * appropriate. Unlike the loop above, which can quit when it drops a1704 * delta, we need to keep going to look for more depth cuts. So we need1705 * an extra "next" pointer to keep going after we reset cur->delta.1706 */1707for(cur = entry; cur; cur = next) {1708 next =DELTA(cur);17091710/*1711 * We should have a chain of zero or more ACTIVE states down to1712 * a final DONE. We can quit after the DONE, because either it1713 * has no bases, or we've already handled them in a previous1714 * call.1715 */1716if(cur->dfs_state == DFS_DONE)1717break;1718else if(cur->dfs_state != DFS_ACTIVE)1719die("BUG: confusing delta dfs state in second pass:%d",1720 cur->dfs_state);17211722/*1723 * If the total_depth is more than depth, then we need to snip1724 * the chain into two or more smaller chains that don't exceed1725 * the maximum depth. Most of the resulting chains will contain1726 * (depth + 1) entries (i.e., depth deltas plus one base), and1727 * the last chain (i.e., the one containing entry) will contain1728 * whatever entries are left over, namely1729 * (total_depth % (depth + 1)) of them.1730 *1731 * Since we are iterating towards decreasing depth, we need to1732 * decrement total_depth as we go, and we need to write to the1733 * entry what its final depth will be after all of the1734 * snipping. Since we're snipping into chains of length (depth1735 * + 1) entries, the final depth of an entry will be its1736 * original depth modulo (depth + 1). Any time we encounter an1737 * entry whose final depth is supposed to be zero, we snip it1738 * from its delta base, thereby making it so.1739 */1740 cur->depth = (total_depth--) % (depth +1);1741if(!cur->depth)1742drop_reused_delta(cur);17431744 cur->dfs_state = DFS_DONE;1745}1746}17471748static voidget_object_details(void)1749{1750uint32_t i;1751struct object_entry **sorted_by_offset;17521753 sorted_by_offset =xcalloc(to_pack.nr_objects,sizeof(struct object_entry *));1754for(i =0; i < to_pack.nr_objects; i++)1755 sorted_by_offset[i] = to_pack.objects + i;1756QSORT(sorted_by_offset, to_pack.nr_objects, pack_offset_sort);17571758for(i =0; i < to_pack.nr_objects; i++) {1759struct object_entry *entry = sorted_by_offset[i];1760check_object(entry);1761if(entry->type_valid &&1762oe_size_greater_than(&to_pack, entry, big_file_threshold))1763 entry->no_try_delta =1;1764}17651766/*1767 * This must happen in a second pass, since we rely on the delta1768 * information for the whole list being completed.1769 */1770for(i =0; i < to_pack.nr_objects; i++)1771break_delta_chains(&to_pack.objects[i]);17721773free(sorted_by_offset);1774}17751776/*1777 * We search for deltas in a list sorted by type, by filename hash, and then1778 * by size, so that we see progressively smaller and smaller files.1779 * That's because we prefer deltas to be from the bigger file1780 * to the smaller -- deletes are potentially cheaper, but perhaps1781 * more importantly, the bigger file is likely the more recent1782 * one. The deepest deltas are therefore the oldest objects which are1783 * less susceptible to be accessed often.1784 */1785static inttype_size_sort(const void*_a,const void*_b)1786{1787const struct object_entry *a = *(struct object_entry **)_a;1788const struct object_entry *b = *(struct object_entry **)_b;1789enum object_type a_type =oe_type(a);1790enum object_type b_type =oe_type(b);1791unsigned long a_size =SIZE(a);1792unsigned long b_size =SIZE(b);17931794if(a_type > b_type)1795return-1;1796if(a_type < b_type)1797return1;1798if(a->hash > b->hash)1799return-1;1800if(a->hash < b->hash)1801return1;1802if(a->preferred_base > b->preferred_base)1803return-1;1804if(a->preferred_base < b->preferred_base)1805return1;1806if(a_size > b_size)1807return-1;1808if(a_size < b_size)1809return1;1810return a < b ? -1: (a > b);/* newest first */1811}18121813struct unpacked {1814struct object_entry *entry;1815void*data;1816struct delta_index *index;1817unsigned depth;1818};18191820static intdelta_cacheable(unsigned long src_size,unsigned long trg_size,1821unsigned long delta_size)1822{1823if(max_delta_cache_size && delta_cache_size + delta_size > max_delta_cache_size)1824return0;18251826if(delta_size < cache_max_small_delta_size)1827return1;18281829/* cache delta, if objects are large enough compared to delta size */1830if((src_size >>20) + (trg_size >>21) > (delta_size >>10))1831return1;18321833return0;1834}18351836#ifndef NO_PTHREADS18371838static pthread_mutex_t read_mutex;1839#define read_lock() pthread_mutex_lock(&read_mutex)1840#define read_unlock() pthread_mutex_unlock(&read_mutex)18411842static pthread_mutex_t cache_mutex;1843#define cache_lock() pthread_mutex_lock(&cache_mutex)1844#define cache_unlock() pthread_mutex_unlock(&cache_mutex)18451846static pthread_mutex_t progress_mutex;1847#define progress_lock() pthread_mutex_lock(&progress_mutex)1848#define progress_unlock() pthread_mutex_unlock(&progress_mutex)18491850#else18511852#define read_lock() (void)01853#define read_unlock() (void)01854#define cache_lock() (void)01855#define cache_unlock() (void)01856#define progress_lock() (void)01857#define progress_unlock() (void)018581859#endif18601861/*1862 * Return the size of the object without doing any delta1863 * reconstruction (so non-deltas are true object sizes, but deltas1864 * return the size of the delta data).1865 */1866unsigned longoe_get_size_slow(struct packing_data *pack,1867const struct object_entry *e)1868{1869struct packed_git *p;1870struct pack_window *w_curs;1871unsigned char*buf;1872enum object_type type;1873unsigned long used, avail, size;18741875if(e->type_ != OBJ_OFS_DELTA && e->type_ != OBJ_REF_DELTA) {1876read_lock();1877if(oid_object_info(&e->idx.oid, &size) <0)1878die(_("unable to get size of%s"),1879oid_to_hex(&e->idx.oid));1880read_unlock();1881return size;1882}18831884 p =oe_in_pack(pack, e);1885if(!p)1886BUG("when e->type is a delta, it must belong to a pack");18871888read_lock();1889 w_curs = NULL;1890 buf =use_pack(p, &w_curs, e->in_pack_offset, &avail);1891 used =unpack_object_header_buffer(buf, avail, &type, &size);1892if(used ==0)1893die(_("unable to parse object header of%s"),1894oid_to_hex(&e->idx.oid));18951896unuse_pack(&w_curs);1897read_unlock();1898return size;1899}19001901static inttry_delta(struct unpacked *trg,struct unpacked *src,1902unsigned max_depth,unsigned long*mem_usage)1903{1904struct object_entry *trg_entry = trg->entry;1905struct object_entry *src_entry = src->entry;1906unsigned long trg_size, src_size, delta_size, sizediff, max_size, sz;1907unsigned ref_depth;1908enum object_type type;1909void*delta_buf;19101911/* Don't bother doing diffs between different types */1912if(oe_type(trg_entry) !=oe_type(src_entry))1913return-1;19141915/*1916 * We do not bother to try a delta that we discarded on an1917 * earlier try, but only when reusing delta data. Note that1918 * src_entry that is marked as the preferred_base should always1919 * be considered, as even if we produce a suboptimal delta against1920 * it, we will still save the transfer cost, as we already know1921 * the other side has it and we won't send src_entry at all.1922 */1923if(reuse_delta &&IN_PACK(trg_entry) &&1924IN_PACK(trg_entry) ==IN_PACK(src_entry) &&1925!src_entry->preferred_base &&1926 trg_entry->in_pack_type != OBJ_REF_DELTA &&1927 trg_entry->in_pack_type != OBJ_OFS_DELTA)1928return0;19291930/* Let's not bust the allowed depth. */1931if(src->depth >= max_depth)1932return0;19331934/* Now some size filtering heuristics. */1935 trg_size =SIZE(trg_entry);1936if(!DELTA(trg_entry)) {1937 max_size = trg_size/2-20;1938 ref_depth =1;1939}else{1940 max_size = trg_entry->delta_size;1941 ref_depth = trg->depth;1942}1943 max_size = (uint64_t)max_size * (max_depth - src->depth) /1944(max_depth - ref_depth +1);1945if(max_size ==0)1946return0;1947 src_size =SIZE(src_entry);1948 sizediff = src_size < trg_size ? trg_size - src_size :0;1949if(sizediff >= max_size)1950return0;1951if(trg_size < src_size /32)1952return0;19531954/* Load data if not already done */1955if(!trg->data) {1956read_lock();1957 trg->data =read_object_file(&trg_entry->idx.oid, &type, &sz);1958read_unlock();1959if(!trg->data)1960die("object%scannot be read",1961oid_to_hex(&trg_entry->idx.oid));1962if(sz != trg_size)1963die("object%sinconsistent object length (%lu vs%lu)",1964oid_to_hex(&trg_entry->idx.oid), sz,1965 trg_size);1966*mem_usage += sz;1967}1968if(!src->data) {1969read_lock();1970 src->data =read_object_file(&src_entry->idx.oid, &type, &sz);1971read_unlock();1972if(!src->data) {1973if(src_entry->preferred_base) {1974static int warned =0;1975if(!warned++)1976warning("object%scannot be read",1977oid_to_hex(&src_entry->idx.oid));1978/*1979 * Those objects are not included in the1980 * resulting pack. Be resilient and ignore1981 * them if they can't be read, in case the1982 * pack could be created nevertheless.1983 */1984return0;1985}1986die("object%scannot be read",1987oid_to_hex(&src_entry->idx.oid));1988}1989if(sz != src_size)1990die("object%sinconsistent object length (%lu vs%lu)",1991oid_to_hex(&src_entry->idx.oid), sz,1992 src_size);1993*mem_usage += sz;1994}1995if(!src->index) {1996 src->index =create_delta_index(src->data, src_size);1997if(!src->index) {1998static int warned =0;1999if(!warned++)2000warning("suboptimal pack - out of memory");2001return0;2002}2003*mem_usage +=sizeof_delta_index(src->index);2004}20052006 delta_buf =create_delta(src->index, trg->data, trg_size, &delta_size, max_size);2007if(!delta_buf)2008return0;20092010if(DELTA(trg_entry)) {2011/* Prefer only shallower same-sized deltas. */2012if(delta_size == trg_entry->delta_size &&2013 src->depth +1>= trg->depth) {2014free(delta_buf);2015return0;2016}2017}20182019/*2020 * Handle memory allocation outside of the cache2021 * accounting lock. Compiler will optimize the strangeness2022 * away when NO_PTHREADS is defined.2023 */2024free(trg_entry->delta_data);2025cache_lock();2026if(trg_entry->delta_data) {2027 delta_cache_size -= trg_entry->delta_size;2028 trg_entry->delta_data = NULL;2029}2030if(delta_cacheable(src_size, trg_size, delta_size)) {2031 delta_cache_size += delta_size;2032cache_unlock();2033 trg_entry->delta_data =xrealloc(delta_buf, delta_size);2034}else{2035cache_unlock();2036free(delta_buf);2037}20382039SET_DELTA(trg_entry, src_entry);2040 trg_entry->delta_size = delta_size;2041 trg->depth = src->depth +1;20422043return1;2044}20452046static unsigned intcheck_delta_limit(struct object_entry *me,unsigned int n)2047{2048struct object_entry *child =DELTA_CHILD(me);2049unsigned int m = n;2050while(child) {2051unsigned int c =check_delta_limit(child, n +1);2052if(m < c)2053 m = c;2054 child =DELTA_SIBLING(child);2055}2056return m;2057}20582059static unsigned longfree_unpacked(struct unpacked *n)2060{2061unsigned long freed_mem =sizeof_delta_index(n->index);2062free_delta_index(n->index);2063 n->index = NULL;2064if(n->data) {2065 freed_mem +=SIZE(n->entry);2066FREE_AND_NULL(n->data);2067}2068 n->entry = NULL;2069 n->depth =0;2070return freed_mem;2071}20722073static voidfind_deltas(struct object_entry **list,unsigned*list_size,2074int window,int depth,unsigned*processed)2075{2076uint32_t i, idx =0, count =0;2077struct unpacked *array;2078unsigned long mem_usage =0;20792080 array =xcalloc(window,sizeof(struct unpacked));20812082for(;;) {2083struct object_entry *entry;2084struct unpacked *n = array + idx;2085int j, max_depth, best_base = -1;20862087progress_lock();2088if(!*list_size) {2089progress_unlock();2090break;2091}2092 entry = *list++;2093(*list_size)--;2094if(!entry->preferred_base) {2095(*processed)++;2096display_progress(progress_state, *processed);2097}2098progress_unlock();20992100 mem_usage -=free_unpacked(n);2101 n->entry = entry;21022103while(window_memory_limit &&2104 mem_usage > window_memory_limit &&2105 count >1) {2106uint32_t tail = (idx + window - count) % window;2107 mem_usage -=free_unpacked(array + tail);2108 count--;2109}21102111/* We do not compute delta to *create* objects we are not2112 * going to pack.2113 */2114if(entry->preferred_base)2115goto next;21162117/*2118 * If the current object is at pack edge, take the depth the2119 * objects that depend on the current object into account2120 * otherwise they would become too deep.2121 */2122 max_depth = depth;2123if(DELTA_CHILD(entry)) {2124 max_depth -=check_delta_limit(entry,0);2125if(max_depth <=0)2126goto next;2127}21282129 j = window;2130while(--j >0) {2131int ret;2132uint32_t other_idx = idx + j;2133struct unpacked *m;2134if(other_idx >= window)2135 other_idx -= window;2136 m = array + other_idx;2137if(!m->entry)2138break;2139 ret =try_delta(n, m, max_depth, &mem_usage);2140if(ret <0)2141break;2142else if(ret >0)2143 best_base = other_idx;2144}21452146/*2147 * If we decided to cache the delta data, then it is best2148 * to compress it right away. First because we have to do2149 * it anyway, and doing it here while we're threaded will2150 * save a lot of time in the non threaded write phase,2151 * as well as allow for caching more deltas within2152 * the same cache size limit.2153 * ...2154 * But only if not writing to stdout, since in that case2155 * the network is most likely throttling writes anyway,2156 * and therefore it is best to go to the write phase ASAP2157 * instead, as we can afford spending more time compressing2158 * between writes at that moment.2159 */2160if(entry->delta_data && !pack_to_stdout) {2161unsigned long size;21622163 size =do_compress(&entry->delta_data, entry->delta_size);2164if(size < (1U<< OE_Z_DELTA_BITS)) {2165 entry->z_delta_size = size;2166cache_lock();2167 delta_cache_size -= entry->delta_size;2168 delta_cache_size += entry->z_delta_size;2169cache_unlock();2170}else{2171FREE_AND_NULL(entry->delta_data);2172 entry->z_delta_size =0;2173}2174}21752176/* if we made n a delta, and if n is already at max2177 * depth, leaving it in the window is pointless. we2178 * should evict it first.2179 */2180if(DELTA(entry) && max_depth <= n->depth)2181continue;21822183/*2184 * Move the best delta base up in the window, after the2185 * currently deltified object, to keep it longer. It will2186 * be the first base object to be attempted next.2187 */2188if(DELTA(entry)) {2189struct unpacked swap = array[best_base];2190int dist = (window + idx - best_base) % window;2191int dst = best_base;2192while(dist--) {2193int src = (dst +1) % window;2194 array[dst] = array[src];2195 dst = src;2196}2197 array[dst] = swap;2198}21992200 next:2201 idx++;2202if(count +1< window)2203 count++;2204if(idx >= window)2205 idx =0;2206}22072208for(i =0; i < window; ++i) {2209free_delta_index(array[i].index);2210free(array[i].data);2211}2212free(array);2213}22142215#ifndef NO_PTHREADS22162217static voidtry_to_free_from_threads(size_t size)2218{2219read_lock();2220release_pack_memory(size);2221read_unlock();2222}22232224static try_to_free_t old_try_to_free_routine;22252226/*2227 * The main thread waits on the condition that (at least) one of the workers2228 * has stopped working (which is indicated in the .working member of2229 * struct thread_params).2230 * When a work thread has completed its work, it sets .working to 0 and2231 * signals the main thread and waits on the condition that .data_ready2232 * becomes 1.2233 */22342235struct thread_params {2236 pthread_t thread;2237struct object_entry **list;2238unsigned list_size;2239unsigned remaining;2240int window;2241int depth;2242int working;2243int data_ready;2244 pthread_mutex_t mutex;2245 pthread_cond_t cond;2246unsigned*processed;2247};22482249static pthread_cond_t progress_cond;22502251/*2252 * Mutex and conditional variable can't be statically-initialized on Windows.2253 */2254static voidinit_threaded_search(void)2255{2256init_recursive_mutex(&read_mutex);2257pthread_mutex_init(&cache_mutex, NULL);2258pthread_mutex_init(&progress_mutex, NULL);2259pthread_cond_init(&progress_cond, NULL);2260 old_try_to_free_routine =set_try_to_free_routine(try_to_free_from_threads);2261}22622263static voidcleanup_threaded_search(void)2264{2265set_try_to_free_routine(old_try_to_free_routine);2266pthread_cond_destroy(&progress_cond);2267pthread_mutex_destroy(&read_mutex);2268pthread_mutex_destroy(&cache_mutex);2269pthread_mutex_destroy(&progress_mutex);2270}22712272static void*threaded_find_deltas(void*arg)2273{2274struct thread_params *me = arg;22752276progress_lock();2277while(me->remaining) {2278progress_unlock();22792280find_deltas(me->list, &me->remaining,2281 me->window, me->depth, me->processed);22822283progress_lock();2284 me->working =0;2285pthread_cond_signal(&progress_cond);2286progress_unlock();22872288/*2289 * We must not set ->data_ready before we wait on the2290 * condition because the main thread may have set it to 12291 * before we get here. In order to be sure that new2292 * work is available if we see 1 in ->data_ready, it2293 * was initialized to 0 before this thread was spawned2294 * and we reset it to 0 right away.2295 */2296pthread_mutex_lock(&me->mutex);2297while(!me->data_ready)2298pthread_cond_wait(&me->cond, &me->mutex);2299 me->data_ready =0;2300pthread_mutex_unlock(&me->mutex);23012302progress_lock();2303}2304progress_unlock();2305/* leave ->working 1 so that this doesn't get more work assigned */2306return NULL;2307}23082309static voidll_find_deltas(struct object_entry **list,unsigned list_size,2310int window,int depth,unsigned*processed)2311{2312struct thread_params *p;2313int i, ret, active_threads =0;23142315init_threaded_search();23162317if(delta_search_threads <=1) {2318find_deltas(list, &list_size, window, depth, processed);2319cleanup_threaded_search();2320return;2321}2322if(progress > pack_to_stdout)2323fprintf(stderr,"Delta compression using up to%dthreads.\n",2324 delta_search_threads);2325 p =xcalloc(delta_search_threads,sizeof(*p));23262327/* Partition the work amongst work threads. */2328for(i =0; i < delta_search_threads; i++) {2329unsigned sub_size = list_size / (delta_search_threads - i);23302331/* don't use too small segments or no deltas will be found */2332if(sub_size <2*window && i+1< delta_search_threads)2333 sub_size =0;23342335 p[i].window = window;2336 p[i].depth = depth;2337 p[i].processed = processed;2338 p[i].working =1;2339 p[i].data_ready =0;23402341/* try to split chunks on "path" boundaries */2342while(sub_size && sub_size < list_size &&2343 list[sub_size]->hash &&2344 list[sub_size]->hash == list[sub_size-1]->hash)2345 sub_size++;23462347 p[i].list = list;2348 p[i].list_size = sub_size;2349 p[i].remaining = sub_size;23502351 list += sub_size;2352 list_size -= sub_size;2353}23542355/* Start work threads. */2356for(i =0; i < delta_search_threads; i++) {2357if(!p[i].list_size)2358continue;2359pthread_mutex_init(&p[i].mutex, NULL);2360pthread_cond_init(&p[i].cond, NULL);2361 ret =pthread_create(&p[i].thread, NULL,2362 threaded_find_deltas, &p[i]);2363if(ret)2364die("unable to create thread:%s",strerror(ret));2365 active_threads++;2366}23672368/*2369 * Now let's wait for work completion. Each time a thread is done2370 * with its work, we steal half of the remaining work from the2371 * thread with the largest number of unprocessed objects and give2372 * it to that newly idle thread. This ensure good load balancing2373 * until the remaining object list segments are simply too short2374 * to be worth splitting anymore.2375 */2376while(active_threads) {2377struct thread_params *target = NULL;2378struct thread_params *victim = NULL;2379unsigned sub_size =0;23802381progress_lock();2382for(;;) {2383for(i =0; !target && i < delta_search_threads; i++)2384if(!p[i].working)2385 target = &p[i];2386if(target)2387break;2388pthread_cond_wait(&progress_cond, &progress_mutex);2389}23902391for(i =0; i < delta_search_threads; i++)2392if(p[i].remaining >2*window &&2393(!victim || victim->remaining < p[i].remaining))2394 victim = &p[i];2395if(victim) {2396 sub_size = victim->remaining /2;2397 list = victim->list + victim->list_size - sub_size;2398while(sub_size && list[0]->hash &&2399 list[0]->hash == list[-1]->hash) {2400 list++;2401 sub_size--;2402}2403if(!sub_size) {2404/*2405 * It is possible for some "paths" to have2406 * so many objects that no hash boundary2407 * might be found. Let's just steal the2408 * exact half in that case.2409 */2410 sub_size = victim->remaining /2;2411 list -= sub_size;2412}2413 target->list = list;2414 victim->list_size -= sub_size;2415 victim->remaining -= sub_size;2416}2417 target->list_size = sub_size;2418 target->remaining = sub_size;2419 target->working =1;2420progress_unlock();24212422pthread_mutex_lock(&target->mutex);2423 target->data_ready =1;2424pthread_cond_signal(&target->cond);2425pthread_mutex_unlock(&target->mutex);24262427if(!sub_size) {2428pthread_join(target->thread, NULL);2429pthread_cond_destroy(&target->cond);2430pthread_mutex_destroy(&target->mutex);2431 active_threads--;2432}2433}2434cleanup_threaded_search();2435free(p);2436}24372438#else2439#define ll_find_deltas(l, s, w, d, p) find_deltas(l, &s, w, d, p)2440#endif24412442static voidadd_tag_chain(const struct object_id *oid)2443{2444struct tag *tag;24452446/*2447 * We catch duplicates already in add_object_entry(), but we'd2448 * prefer to do this extra check to avoid having to parse the2449 * tag at all if we already know that it's being packed (e.g., if2450 * it was included via bitmaps, we would not have parsed it2451 * previously).2452 */2453if(packlist_find(&to_pack, oid->hash, NULL))2454return;24552456 tag =lookup_tag(oid);2457while(1) {2458if(!tag ||parse_tag(tag) || !tag->tagged)2459die("unable to pack objects reachable from tag%s",2460oid_to_hex(oid));24612462add_object_entry(&tag->object.oid, OBJ_TAG, NULL,0);24632464if(tag->tagged->type != OBJ_TAG)2465return;24662467 tag = (struct tag *)tag->tagged;2468}2469}24702471static intadd_ref_tag(const char*path,const struct object_id *oid,int flag,void*cb_data)2472{2473struct object_id peeled;24742475if(starts_with(path,"refs/tags/") &&/* is a tag? */2476!peel_ref(path, &peeled) &&/* peelable? */2477packlist_find(&to_pack, peeled.hash, NULL))/* object packed? */2478add_tag_chain(oid);2479return0;2480}24812482static voidprepare_pack(int window,int depth)2483{2484struct object_entry **delta_list;2485uint32_t i, nr_deltas;2486unsigned n;24872488get_object_details();24892490/*2491 * If we're locally repacking then we need to be doubly careful2492 * from now on in order to make sure no stealth corruption gets2493 * propagated to the new pack. Clients receiving streamed packs2494 * should validate everything they get anyway so no need to incur2495 * the additional cost here in that case.2496 */2497if(!pack_to_stdout)2498 do_check_packed_object_crc =1;24992500if(!to_pack.nr_objects || !window || !depth)2501return;25022503ALLOC_ARRAY(delta_list, to_pack.nr_objects);2504 nr_deltas = n =0;25052506for(i =0; i < to_pack.nr_objects; i++) {2507struct object_entry *entry = to_pack.objects + i;25082509if(DELTA(entry))2510/* This happens if we decided to reuse existing2511 * delta from a pack. "reuse_delta &&" is implied.2512 */2513continue;25142515if(!entry->type_valid ||2516oe_size_less_than(&to_pack, entry,50))2517continue;25182519if(entry->no_try_delta)2520continue;25212522if(!entry->preferred_base) {2523 nr_deltas++;2524if(oe_type(entry) <0)2525die("unable to get type of object%s",2526oid_to_hex(&entry->idx.oid));2527}else{2528if(oe_type(entry) <0) {2529/*2530 * This object is not found, but we2531 * don't have to include it anyway.2532 */2533continue;2534}2535}25362537 delta_list[n++] = entry;2538}25392540if(nr_deltas && n >1) {2541unsigned nr_done =0;2542if(progress)2543 progress_state =start_progress(_("Compressing objects"),2544 nr_deltas);2545QSORT(delta_list, n, type_size_sort);2546ll_find_deltas(delta_list, n, window+1, depth, &nr_done);2547stop_progress(&progress_state);2548if(nr_done != nr_deltas)2549die("inconsistency with delta count");2550}2551free(delta_list);2552}25532554static intgit_pack_config(const char*k,const char*v,void*cb)2555{2556if(!strcmp(k,"pack.window")) {2557 window =git_config_int(k, v);2558return0;2559}2560if(!strcmp(k,"pack.windowmemory")) {2561 window_memory_limit =git_config_ulong(k, v);2562return0;2563}2564if(!strcmp(k,"pack.depth")) {2565 depth =git_config_int(k, v);2566return0;2567}2568if(!strcmp(k,"pack.deltacachesize")) {2569 max_delta_cache_size =git_config_int(k, v);2570return0;2571}2572if(!strcmp(k,"pack.deltacachelimit")) {2573 cache_max_small_delta_size =git_config_int(k, v);2574return0;2575}2576if(!strcmp(k,"pack.writebitmaphashcache")) {2577if(git_config_bool(k, v))2578 write_bitmap_options |= BITMAP_OPT_HASH_CACHE;2579else2580 write_bitmap_options &= ~BITMAP_OPT_HASH_CACHE;2581}2582if(!strcmp(k,"pack.usebitmaps")) {2583 use_bitmap_index_default =git_config_bool(k, v);2584return0;2585}2586if(!strcmp(k,"pack.threads")) {2587 delta_search_threads =git_config_int(k, v);2588if(delta_search_threads <0)2589die("invalid number of threads specified (%d)",2590 delta_search_threads);2591#ifdef NO_PTHREADS2592if(delta_search_threads !=1) {2593warning("no threads support, ignoring%s", k);2594 delta_search_threads =0;2595}2596#endif2597return0;2598}2599if(!strcmp(k,"pack.indexversion")) {2600 pack_idx_opts.version =git_config_int(k, v);2601if(pack_idx_opts.version >2)2602die("bad pack.indexversion=%"PRIu32,2603 pack_idx_opts.version);2604return0;2605}2606returngit_default_config(k, v, cb);2607}26082609static voidread_object_list_from_stdin(void)2610{2611char line[GIT_MAX_HEXSZ +1+ PATH_MAX +2];2612struct object_id oid;2613const char*p;26142615for(;;) {2616if(!fgets(line,sizeof(line), stdin)) {2617if(feof(stdin))2618break;2619if(!ferror(stdin))2620die("fgets returned NULL, not EOF, not error!");2621if(errno != EINTR)2622die_errno("fgets");2623clearerr(stdin);2624continue;2625}2626if(line[0] =='-') {2627if(get_oid_hex(line+1, &oid))2628die("expected edge object ID, got garbage:\n%s",2629 line);2630add_preferred_base(&oid);2631continue;2632}2633if(parse_oid_hex(line, &oid, &p))2634die("expected object ID, got garbage:\n%s", line);26352636add_preferred_base_object(p +1);2637add_object_entry(&oid, OBJ_NONE, p +1,0);2638}2639}26402641/* Remember to update object flag allocation in object.h */2642#define OBJECT_ADDED (1u<<20)26432644static voidshow_commit(struct commit *commit,void*data)2645{2646add_object_entry(&commit->object.oid, OBJ_COMMIT, NULL,0);2647 commit->object.flags |= OBJECT_ADDED;26482649if(write_bitmap_index)2650index_commit_for_bitmap(commit);2651}26522653static voidshow_object(struct object *obj,const char*name,void*data)2654{2655add_preferred_base_object(name);2656add_object_entry(&obj->oid, obj->type, name,0);2657 obj->flags |= OBJECT_ADDED;2658}26592660static voidshow_object__ma_allow_any(struct object *obj,const char*name,void*data)2661{2662assert(arg_missing_action == MA_ALLOW_ANY);26632664/*2665 * Quietly ignore ALL missing objects. This avoids problems with2666 * staging them now and getting an odd error later.2667 */2668if(!has_object_file(&obj->oid))2669return;26702671show_object(obj, name, data);2672}26732674static voidshow_object__ma_allow_promisor(struct object *obj,const char*name,void*data)2675{2676assert(arg_missing_action == MA_ALLOW_PROMISOR);26772678/*2679 * Quietly ignore EXPECTED missing objects. This avoids problems with2680 * staging them now and getting an odd error later.2681 */2682if(!has_object_file(&obj->oid) &&is_promisor_object(&obj->oid))2683return;26842685show_object(obj, name, data);2686}26872688static intoption_parse_missing_action(const struct option *opt,2689const char*arg,int unset)2690{2691assert(arg);2692assert(!unset);26932694if(!strcmp(arg,"error")) {2695 arg_missing_action = MA_ERROR;2696 fn_show_object = show_object;2697return0;2698}26992700if(!strcmp(arg,"allow-any")) {2701 arg_missing_action = MA_ALLOW_ANY;2702 fetch_if_missing =0;2703 fn_show_object = show_object__ma_allow_any;2704return0;2705}27062707if(!strcmp(arg,"allow-promisor")) {2708 arg_missing_action = MA_ALLOW_PROMISOR;2709 fetch_if_missing =0;2710 fn_show_object = show_object__ma_allow_promisor;2711return0;2712}27132714die(_("invalid value for --missing"));2715return0;2716}27172718static voidshow_edge(struct commit *commit)2719{2720add_preferred_base(&commit->object.oid);2721}27222723struct in_pack_object {2724 off_t offset;2725struct object *object;2726};27272728struct in_pack {2729unsigned int alloc;2730unsigned int nr;2731struct in_pack_object *array;2732};27332734static voidmark_in_pack_object(struct object *object,struct packed_git *p,struct in_pack *in_pack)2735{2736 in_pack->array[in_pack->nr].offset =find_pack_entry_one(object->oid.hash, p);2737 in_pack->array[in_pack->nr].object = object;2738 in_pack->nr++;2739}27402741/*2742 * Compare the objects in the offset order, in order to emulate the2743 * "git rev-list --objects" output that produced the pack originally.2744 */2745static intofscmp(const void*a_,const void*b_)2746{2747struct in_pack_object *a = (struct in_pack_object *)a_;2748struct in_pack_object *b = (struct in_pack_object *)b_;27492750if(a->offset < b->offset)2751return-1;2752else if(a->offset > b->offset)2753return1;2754else2755returnoidcmp(&a->object->oid, &b->object->oid);2756}27572758static voidadd_objects_in_unpacked_packs(struct rev_info *revs)2759{2760struct packed_git *p;2761struct in_pack in_pack;2762uint32_t i;27632764memset(&in_pack,0,sizeof(in_pack));27652766for(p =get_packed_git(the_repository); p; p = p->next) {2767struct object_id oid;2768struct object *o;27692770if(!p->pack_local || p->pack_keep)2771continue;2772if(open_pack_index(p))2773die("cannot open pack index");27742775ALLOC_GROW(in_pack.array,2776 in_pack.nr + p->num_objects,2777 in_pack.alloc);27782779for(i =0; i < p->num_objects; i++) {2780nth_packed_object_oid(&oid, p, i);2781 o =lookup_unknown_object(oid.hash);2782if(!(o->flags & OBJECT_ADDED))2783mark_in_pack_object(o, p, &in_pack);2784 o->flags |= OBJECT_ADDED;2785}2786}27872788if(in_pack.nr) {2789QSORT(in_pack.array, in_pack.nr, ofscmp);2790for(i =0; i < in_pack.nr; i++) {2791struct object *o = in_pack.array[i].object;2792add_object_entry(&o->oid, o->type,"",0);2793}2794}2795free(in_pack.array);2796}27972798static intadd_loose_object(const struct object_id *oid,const char*path,2799void*data)2800{2801enum object_type type =oid_object_info(oid, NULL);28022803if(type <0) {2804warning("loose object at%scould not be examined", path);2805return0;2806}28072808add_object_entry(oid, type,"",0);2809return0;2810}28112812/*2813 * We actually don't even have to worry about reachability here.2814 * add_object_entry will weed out duplicates, so we just add every2815 * loose object we find.2816 */2817static voidadd_unreachable_loose_objects(void)2818{2819for_each_loose_file_in_objdir(get_object_directory(),2820 add_loose_object,2821 NULL, NULL, NULL);2822}28232824static inthas_sha1_pack_kept_or_nonlocal(const struct object_id *oid)2825{2826static struct packed_git *last_found = (void*)1;2827struct packed_git *p;28282829 p = (last_found != (void*)1) ? last_found :2830get_packed_git(the_repository);28312832while(p) {2833if((!p->pack_local || p->pack_keep) &&2834find_pack_entry_one(oid->hash, p)) {2835 last_found = p;2836return1;2837}2838if(p == last_found)2839 p =get_packed_git(the_repository);2840else2841 p = p->next;2842if(p == last_found)2843 p = p->next;2844}2845return0;2846}28472848/*2849 * Store a list of sha1s that are should not be discarded2850 * because they are either written too recently, or are2851 * reachable from another object that was.2852 *2853 * This is filled by get_object_list.2854 */2855static struct oid_array recent_objects;28562857static intloosened_object_can_be_discarded(const struct object_id *oid,2858 timestamp_t mtime)2859{2860if(!unpack_unreachable_expiration)2861return0;2862if(mtime > unpack_unreachable_expiration)2863return0;2864if(oid_array_lookup(&recent_objects, oid) >=0)2865return0;2866return1;2867}28682869static voidloosen_unused_packed_objects(struct rev_info *revs)2870{2871struct packed_git *p;2872uint32_t i;2873struct object_id oid;28742875for(p =get_packed_git(the_repository); p; p = p->next) {2876if(!p->pack_local || p->pack_keep)2877continue;28782879if(open_pack_index(p))2880die("cannot open pack index");28812882for(i =0; i < p->num_objects; i++) {2883nth_packed_object_oid(&oid, p, i);2884if(!packlist_find(&to_pack, oid.hash, NULL) &&2885!has_sha1_pack_kept_or_nonlocal(&oid) &&2886!loosened_object_can_be_discarded(&oid, p->mtime))2887if(force_object_loose(&oid, p->mtime))2888die("unable to force loose object");2889}2890}2891}28922893/*2894 * This tracks any options which pack-reuse code expects to be on, or which a2895 * reader of the pack might not understand, and which would therefore prevent2896 * blind reuse of what we have on disk.2897 */2898static intpack_options_allow_reuse(void)2899{2900return pack_to_stdout &&2901 allow_ofs_delta &&2902!ignore_packed_keep &&2903(!local || !have_non_local_packs) &&2904!incremental;2905}29062907static intget_object_list_from_bitmap(struct rev_info *revs)2908{2909if(prepare_bitmap_walk(revs) <0)2910return-1;29112912if(pack_options_allow_reuse() &&2913!reuse_partial_packfile_from_bitmap(2914&reuse_packfile,2915&reuse_packfile_objects,2916&reuse_packfile_offset)) {2917assert(reuse_packfile_objects);2918 nr_result += reuse_packfile_objects;2919display_progress(progress_state, nr_result);2920}29212922traverse_bitmap_commit_list(&add_object_entry_from_bitmap);2923return0;2924}29252926static voidrecord_recent_object(struct object *obj,2927const char*name,2928void*data)2929{2930oid_array_append(&recent_objects, &obj->oid);2931}29322933static voidrecord_recent_commit(struct commit *commit,void*data)2934{2935oid_array_append(&recent_objects, &commit->object.oid);2936}29372938static voidget_object_list(int ac,const char**av)2939{2940struct rev_info revs;2941char line[1000];2942int flags =0;29432944init_revisions(&revs, NULL);2945 save_commit_buffer =0;2946setup_revisions(ac, av, &revs, NULL);29472948/* make sure shallows are read */2949is_repository_shallow();29502951while(fgets(line,sizeof(line), stdin) != NULL) {2952int len =strlen(line);2953if(len && line[len -1] =='\n')2954 line[--len] =0;2955if(!len)2956break;2957if(*line =='-') {2958if(!strcmp(line,"--not")) {2959 flags ^= UNINTERESTING;2960 write_bitmap_index =0;2961continue;2962}2963if(starts_with(line,"--shallow ")) {2964struct object_id oid;2965if(get_oid_hex(line +10, &oid))2966die("not an SHA-1 '%s'", line +10);2967register_shallow(&oid);2968 use_bitmap_index =0;2969continue;2970}2971die("not a rev '%s'", line);2972}2973if(handle_revision_arg(line, &revs, flags, REVARG_CANNOT_BE_FILENAME))2974die("bad revision '%s'", line);2975}29762977if(use_bitmap_index && !get_object_list_from_bitmap(&revs))2978return;29792980if(prepare_revision_walk(&revs))2981die("revision walk setup failed");2982mark_edges_uninteresting(&revs, show_edge);29832984if(!fn_show_object)2985 fn_show_object = show_object;2986traverse_commit_list_filtered(&filter_options, &revs,2987 show_commit, fn_show_object, NULL,2988 NULL);29892990if(unpack_unreachable_expiration) {2991 revs.ignore_missing_links =1;2992if(add_unseen_recent_objects_to_traversal(&revs,2993 unpack_unreachable_expiration))2994die("unable to add recent objects");2995if(prepare_revision_walk(&revs))2996die("revision walk setup failed");2997traverse_commit_list(&revs, record_recent_commit,2998 record_recent_object, NULL);2999}30003001if(keep_unreachable)3002add_objects_in_unpacked_packs(&revs);3003if(pack_loose_unreachable)3004add_unreachable_loose_objects();3005if(unpack_unreachable)3006loosen_unused_packed_objects(&revs);30073008oid_array_clear(&recent_objects);3009}30103011static intoption_parse_index_version(const struct option *opt,3012const char*arg,int unset)3013{3014char*c;3015const char*val = arg;3016 pack_idx_opts.version =strtoul(val, &c,10);3017if(pack_idx_opts.version >2)3018die(_("unsupported index version%s"), val);3019if(*c ==','&& c[1])3020 pack_idx_opts.off32_limit =strtoul(c+1, &c,0);3021if(*c || pack_idx_opts.off32_limit &0x80000000)3022die(_("bad index version '%s'"), val);3023return0;3024}30253026static intoption_parse_unpack_unreachable(const struct option *opt,3027const char*arg,int unset)3028{3029if(unset) {3030 unpack_unreachable =0;3031 unpack_unreachable_expiration =0;3032}3033else{3034 unpack_unreachable =1;3035if(arg)3036 unpack_unreachable_expiration =approxidate(arg);3037}3038return0;3039}30403041intcmd_pack_objects(int argc,const char**argv,const char*prefix)3042{3043int use_internal_rev_list =0;3044int thin =0;3045int shallow =0;3046int all_progress_implied =0;3047struct argv_array rp = ARGV_ARRAY_INIT;3048int rev_list_unpacked =0, rev_list_all =0, rev_list_reflog =0;3049int rev_list_index =0;3050struct option pack_objects_options[] = {3051OPT_SET_INT('q',"quiet", &progress,3052N_("do not show progress meter"),0),3053OPT_SET_INT(0,"progress", &progress,3054N_("show progress meter"),1),3055OPT_SET_INT(0,"all-progress", &progress,3056N_("show progress meter during object writing phase"),2),3057OPT_BOOL(0,"all-progress-implied",3058&all_progress_implied,3059N_("similar to --all-progress when progress meter is shown")),3060{ OPTION_CALLBACK,0,"index-version", NULL,N_("version[,offset]"),3061N_("write the pack index file in the specified idx format version"),30620, option_parse_index_version },3063OPT_MAGNITUDE(0,"max-pack-size", &pack_size_limit,3064N_("maximum size of each output pack file")),3065OPT_BOOL(0,"local", &local,3066N_("ignore borrowed objects from alternate object store")),3067OPT_BOOL(0,"incremental", &incremental,3068N_("ignore packed objects")),3069OPT_INTEGER(0,"window", &window,3070N_("limit pack window by objects")),3071OPT_MAGNITUDE(0,"window-memory", &window_memory_limit,3072N_("limit pack window by memory in addition to object limit")),3073OPT_INTEGER(0,"depth", &depth,3074N_("maximum length of delta chain allowed in the resulting pack")),3075OPT_BOOL(0,"reuse-delta", &reuse_delta,3076N_("reuse existing deltas")),3077OPT_BOOL(0,"reuse-object", &reuse_object,3078N_("reuse existing objects")),3079OPT_BOOL(0,"delta-base-offset", &allow_ofs_delta,3080N_("use OFS_DELTA objects")),3081OPT_INTEGER(0,"threads", &delta_search_threads,3082N_("use threads when searching for best delta matches")),3083OPT_BOOL(0,"non-empty", &non_empty,3084N_("do not create an empty pack output")),3085OPT_BOOL(0,"revs", &use_internal_rev_list,3086N_("read revision arguments from standard input")),3087{ OPTION_SET_INT,0,"unpacked", &rev_list_unpacked, NULL,3088N_("limit the objects to those that are not yet packed"),3089 PARSE_OPT_NOARG | PARSE_OPT_NONEG, NULL,1},3090{ OPTION_SET_INT,0,"all", &rev_list_all, NULL,3091N_("include objects reachable from any reference"),3092 PARSE_OPT_NOARG | PARSE_OPT_NONEG, NULL,1},3093{ OPTION_SET_INT,0,"reflog", &rev_list_reflog, NULL,3094N_("include objects referred by reflog entries"),3095 PARSE_OPT_NOARG | PARSE_OPT_NONEG, NULL,1},3096{ OPTION_SET_INT,0,"indexed-objects", &rev_list_index, NULL,3097N_("include objects referred to by the index"),3098 PARSE_OPT_NOARG | PARSE_OPT_NONEG, NULL,1},3099OPT_BOOL(0,"stdout", &pack_to_stdout,3100N_("output pack to stdout")),3101OPT_BOOL(0,"include-tag", &include_tag,3102N_("include tag objects that refer to objects to be packed")),3103OPT_BOOL(0,"keep-unreachable", &keep_unreachable,3104N_("keep unreachable objects")),3105OPT_BOOL(0,"pack-loose-unreachable", &pack_loose_unreachable,3106N_("pack loose unreachable objects")),3107{ OPTION_CALLBACK,0,"unpack-unreachable", NULL,N_("time"),3108N_("unpack unreachable objects newer than <time>"),3109 PARSE_OPT_OPTARG, option_parse_unpack_unreachable },3110OPT_BOOL(0,"thin", &thin,3111N_("create thin packs")),3112OPT_BOOL(0,"shallow", &shallow,3113N_("create packs suitable for shallow fetches")),3114OPT_BOOL(0,"honor-pack-keep", &ignore_packed_keep,3115N_("ignore packs that have companion .keep file")),3116OPT_INTEGER(0,"compression", &pack_compression_level,3117N_("pack compression level")),3118OPT_SET_INT(0,"keep-true-parents", &grafts_replace_parents,3119N_("do not hide commits by grafts"),0),3120OPT_BOOL(0,"use-bitmap-index", &use_bitmap_index,3121N_("use a bitmap index if available to speed up counting objects")),3122OPT_BOOL(0,"write-bitmap-index", &write_bitmap_index,3123N_("write a bitmap index together with the pack index")),3124OPT_PARSE_LIST_OBJECTS_FILTER(&filter_options),3125{ OPTION_CALLBACK,0,"missing", NULL,N_("action"),3126N_("handling for missing objects"), PARSE_OPT_NONEG,3127 option_parse_missing_action },3128OPT_BOOL(0,"exclude-promisor-objects", &exclude_promisor_objects,3129N_("do not pack objects in promisor packfiles")),3130OPT_END(),3131};31323133if(DFS_NUM_STATES > (1<< OE_DFS_STATE_BITS))3134BUG("too many dfs states, increase OE_DFS_STATE_BITS");31353136 check_replace_refs =0;31373138reset_pack_idx_option(&pack_idx_opts);3139git_config(git_pack_config, NULL);31403141 progress =isatty(2);3142 argc =parse_options(argc, argv, prefix, pack_objects_options,3143 pack_usage,0);31443145if(argc) {3146 base_name = argv[0];3147 argc--;3148}3149if(pack_to_stdout != !base_name || argc)3150usage_with_options(pack_usage, pack_objects_options);31513152if(depth >= (1<< OE_DEPTH_BITS)) {3153warning(_("delta chain depth%dis too deep, forcing%d"),3154 depth, (1<< OE_DEPTH_BITS) -1);3155 depth = (1<< OE_DEPTH_BITS) -1;3156}3157if(cache_max_small_delta_size >= (1U<< OE_Z_DELTA_BITS)) {3158warning(_("pack.deltaCacheLimit is too high, forcing%d"),3159(1U<< OE_Z_DELTA_BITS) -1);3160 cache_max_small_delta_size = (1U<< OE_Z_DELTA_BITS) -1;3161}31623163argv_array_push(&rp,"pack-objects");3164if(thin) {3165 use_internal_rev_list =1;3166argv_array_push(&rp, shallow3167?"--objects-edge-aggressive"3168:"--objects-edge");3169}else3170argv_array_push(&rp,"--objects");31713172if(rev_list_all) {3173 use_internal_rev_list =1;3174argv_array_push(&rp,"--all");3175}3176if(rev_list_reflog) {3177 use_internal_rev_list =1;3178argv_array_push(&rp,"--reflog");3179}3180if(rev_list_index) {3181 use_internal_rev_list =1;3182argv_array_push(&rp,"--indexed-objects");3183}3184if(rev_list_unpacked) {3185 use_internal_rev_list =1;3186argv_array_push(&rp,"--unpacked");3187}31883189if(exclude_promisor_objects) {3190 use_internal_rev_list =1;3191 fetch_if_missing =0;3192argv_array_push(&rp,"--exclude-promisor-objects");3193}31943195if(!reuse_object)3196 reuse_delta =0;3197if(pack_compression_level == -1)3198 pack_compression_level = Z_DEFAULT_COMPRESSION;3199else if(pack_compression_level <0|| pack_compression_level > Z_BEST_COMPRESSION)3200die("bad pack compression level%d", pack_compression_level);32013202if(!delta_search_threads)/* --threads=0 means autodetect */3203 delta_search_threads =online_cpus();32043205#ifdef NO_PTHREADS3206if(delta_search_threads !=1)3207warning("no threads support, ignoring --threads");3208#endif3209if(!pack_to_stdout && !pack_size_limit)3210 pack_size_limit = pack_size_limit_cfg;3211if(pack_to_stdout && pack_size_limit)3212die("--max-pack-size cannot be used to build a pack for transfer.");3213if(pack_size_limit && pack_size_limit <1024*1024) {3214warning("minimum pack size limit is 1 MiB");3215 pack_size_limit =1024*1024;3216}32173218if(!pack_to_stdout && thin)3219die("--thin cannot be used to build an indexable pack.");32203221if(keep_unreachable && unpack_unreachable)3222die("--keep-unreachable and --unpack-unreachable are incompatible.");3223if(!rev_list_all || !rev_list_reflog || !rev_list_index)3224 unpack_unreachable_expiration =0;32253226if(filter_options.choice) {3227if(!pack_to_stdout)3228die("cannot use --filter without --stdout.");3229 use_bitmap_index =0;3230}32313232/*3233 * "soft" reasons not to use bitmaps - for on-disk repack by default we want3234 *3235 * - to produce good pack (with bitmap index not-yet-packed objects are3236 * packed in suboptimal order).3237 *3238 * - to use more robust pack-generation codepath (avoiding possible3239 * bugs in bitmap code and possible bitmap index corruption).3240 */3241if(!pack_to_stdout)3242 use_bitmap_index_default =0;32433244if(use_bitmap_index <0)3245 use_bitmap_index = use_bitmap_index_default;32463247/* "hard" reasons not to use bitmaps; these just won't work at all */3248if(!use_internal_rev_list || (!pack_to_stdout && write_bitmap_index) ||is_repository_shallow())3249 use_bitmap_index =0;32503251if(pack_to_stdout || !rev_list_all)3252 write_bitmap_index =0;32533254if(progress && all_progress_implied)3255 progress =2;32563257if(ignore_packed_keep) {3258struct packed_git *p;3259for(p =get_packed_git(the_repository); p; p = p->next)3260if(p->pack_local && p->pack_keep)3261break;3262if(!p)/* no keep-able packs found */3263 ignore_packed_keep =0;3264}3265if(local) {3266/*3267 * unlike ignore_packed_keep above, we do not want to3268 * unset "local" based on looking at packs, as it3269 * also covers non-local objects3270 */3271struct packed_git *p;3272for(p =get_packed_git(the_repository); p; p = p->next) {3273if(!p->pack_local) {3274 have_non_local_packs =1;3275break;3276}3277}3278}32793280prepare_packing_data(&to_pack);32813282if(progress)3283 progress_state =start_progress(_("Counting objects"),0);3284if(!use_internal_rev_list)3285read_object_list_from_stdin();3286else{3287get_object_list(rp.argc, rp.argv);3288argv_array_clear(&rp);3289}3290cleanup_preferred_base();3291if(include_tag && nr_result)3292for_each_ref(add_ref_tag, NULL);3293stop_progress(&progress_state);32943295if(non_empty && !nr_result)3296return0;3297if(nr_result)3298prepare_pack(window, depth);3299write_pack_file();3300if(progress)3301fprintf(stderr,"Total %"PRIu32" (delta %"PRIu32"),"3302" reused %"PRIu32" (delta %"PRIu32")\n",3303 written, written_delta, reused, reused_delta);3304return0;3305}