1#include"builtin.h" 2#include"cache.h" 3#include"repository.h" 4#include"config.h" 5#include"attr.h" 6#include"object.h" 7#include"blob.h" 8#include"commit.h" 9#include"tag.h" 10#include"tree.h" 11#include"delta.h" 12#include"pack.h" 13#include"pack-revindex.h" 14#include"csum-file.h" 15#include"tree-walk.h" 16#include"diff.h" 17#include"revision.h" 18#include"list-objects.h" 19#include"list-objects-filter.h" 20#include"list-objects-filter-options.h" 21#include"pack-objects.h" 22#include"progress.h" 23#include"refs.h" 24#include"streaming.h" 25#include"thread-utils.h" 26#include"pack-bitmap.h" 27#include"reachable.h" 28#include"sha1-array.h" 29#include"argv-array.h" 30#include"list.h" 31#include"packfile.h" 32#include"object-store.h" 33 34static const char*pack_usage[] = { 35N_("git pack-objects --stdout [<options>...] [< <ref-list> | < <object-list>]"), 36N_("git pack-objects [<options>...] <base-name> [< <ref-list> | < <object-list>]"), 37 NULL 38}; 39 40/* 41 * Objects we are going to pack are collected in the `to_pack` structure. 42 * It contains an array (dynamically expanded) of the object data, and a map 43 * that can resolve SHA1s to their position in the array. 44 */ 45static struct packing_data to_pack; 46 47static struct pack_idx_entry **written_list; 48static uint32_t nr_result, nr_written; 49 50static int non_empty; 51static int reuse_delta =1, reuse_object =1; 52static int keep_unreachable, unpack_unreachable, include_tag; 53static timestamp_t unpack_unreachable_expiration; 54static int pack_loose_unreachable; 55static int local; 56static int have_non_local_packs; 57static int incremental; 58static int ignore_packed_keep; 59static int allow_ofs_delta; 60static struct pack_idx_option pack_idx_opts; 61static const char*base_name; 62static int progress =1; 63static int window =10; 64static unsigned long pack_size_limit; 65static int depth =50; 66static int delta_search_threads; 67static int pack_to_stdout; 68static int num_preferred_base; 69static struct progress *progress_state; 70 71static struct packed_git *reuse_packfile; 72static uint32_t reuse_packfile_objects; 73static off_t reuse_packfile_offset; 74 75static int use_bitmap_index_default =1; 76static int use_bitmap_index = -1; 77static int write_bitmap_index; 78static uint16_t write_bitmap_options; 79 80static int exclude_promisor_objects; 81 82static unsigned long delta_cache_size =0; 83static unsigned long max_delta_cache_size =256*1024*1024; 84static unsigned long cache_max_small_delta_size =1000; 85 86static unsigned long window_memory_limit =0; 87 88static struct list_objects_filter_options filter_options; 89 90enum missing_action { 91 MA_ERROR =0,/* fail if any missing objects are encountered */ 92 MA_ALLOW_ANY,/* silently allow ALL missing objects */ 93 MA_ALLOW_PROMISOR,/* silently allow all missing PROMISOR objects */ 94}; 95static enum missing_action arg_missing_action; 96static show_object_fn fn_show_object; 97 98/* 99 * stats 100 */ 101static uint32_t written, written_delta; 102static uint32_t reused, reused_delta; 103 104/* 105 * Indexed commits 106 */ 107static struct commit **indexed_commits; 108static unsigned int indexed_commits_nr; 109static unsigned int indexed_commits_alloc; 110 111static voidindex_commit_for_bitmap(struct commit *commit) 112{ 113if(indexed_commits_nr >= indexed_commits_alloc) { 114 indexed_commits_alloc = (indexed_commits_alloc +32) *2; 115REALLOC_ARRAY(indexed_commits, indexed_commits_alloc); 116} 117 118 indexed_commits[indexed_commits_nr++] = commit; 119} 120 121static void*get_delta(struct object_entry *entry) 122{ 123unsigned long size, base_size, delta_size; 124void*buf, *base_buf, *delta_buf; 125enum object_type type; 126 127 buf =read_object_file(&entry->idx.oid, &type, &size); 128if(!buf) 129die("unable to read%s",oid_to_hex(&entry->idx.oid)); 130 base_buf =read_object_file(&entry->delta->idx.oid, &type, &base_size); 131if(!base_buf) 132die("unable to read%s", 133oid_to_hex(&entry->delta->idx.oid)); 134 delta_buf =diff_delta(base_buf, base_size, 135 buf, size, &delta_size,0); 136if(!delta_buf || delta_size != entry->delta_size) 137die("delta size changed"); 138free(buf); 139free(base_buf); 140return delta_buf; 141} 142 143static unsigned longdo_compress(void**pptr,unsigned long size) 144{ 145 git_zstream stream; 146void*in, *out; 147unsigned long maxsize; 148 149git_deflate_init(&stream, pack_compression_level); 150 maxsize =git_deflate_bound(&stream, size); 151 152 in = *pptr; 153 out =xmalloc(maxsize); 154*pptr = out; 155 156 stream.next_in = in; 157 stream.avail_in = size; 158 stream.next_out = out; 159 stream.avail_out = maxsize; 160while(git_deflate(&stream, Z_FINISH) == Z_OK) 161;/* nothing */ 162git_deflate_end(&stream); 163 164free(in); 165return stream.total_out; 166} 167 168static unsigned longwrite_large_blob_data(struct git_istream *st,struct hashfile *f, 169const struct object_id *oid) 170{ 171 git_zstream stream; 172unsigned char ibuf[1024*16]; 173unsigned char obuf[1024*16]; 174unsigned long olen =0; 175 176git_deflate_init(&stream, pack_compression_level); 177 178for(;;) { 179 ssize_t readlen; 180int zret = Z_OK; 181 readlen =read_istream(st, ibuf,sizeof(ibuf)); 182if(readlen == -1) 183die(_("unable to read%s"),oid_to_hex(oid)); 184 185 stream.next_in = ibuf; 186 stream.avail_in = readlen; 187while((stream.avail_in || readlen ==0) && 188(zret == Z_OK || zret == Z_BUF_ERROR)) { 189 stream.next_out = obuf; 190 stream.avail_out =sizeof(obuf); 191 zret =git_deflate(&stream, readlen ?0: Z_FINISH); 192hashwrite(f, obuf, stream.next_out - obuf); 193 olen += stream.next_out - obuf; 194} 195if(stream.avail_in) 196die(_("deflate error (%d)"), zret); 197if(readlen ==0) { 198if(zret != Z_STREAM_END) 199die(_("deflate error (%d)"), zret); 200break; 201} 202} 203git_deflate_end(&stream); 204return olen; 205} 206 207/* 208 * we are going to reuse the existing object data as is. make 209 * sure it is not corrupt. 210 */ 211static intcheck_pack_inflate(struct packed_git *p, 212struct pack_window **w_curs, 213 off_t offset, 214 off_t len, 215unsigned long expect) 216{ 217 git_zstream stream; 218unsigned char fakebuf[4096], *in; 219int st; 220 221memset(&stream,0,sizeof(stream)); 222git_inflate_init(&stream); 223do{ 224 in =use_pack(p, w_curs, offset, &stream.avail_in); 225 stream.next_in = in; 226 stream.next_out = fakebuf; 227 stream.avail_out =sizeof(fakebuf); 228 st =git_inflate(&stream, Z_FINISH); 229 offset += stream.next_in - in; 230}while(st == Z_OK || st == Z_BUF_ERROR); 231git_inflate_end(&stream); 232return(st == Z_STREAM_END && 233 stream.total_out == expect && 234 stream.total_in == len) ?0: -1; 235} 236 237static voidcopy_pack_data(struct hashfile *f, 238struct packed_git *p, 239struct pack_window **w_curs, 240 off_t offset, 241 off_t len) 242{ 243unsigned char*in; 244unsigned long avail; 245 246while(len) { 247 in =use_pack(p, w_curs, offset, &avail); 248if(avail > len) 249 avail = (unsigned long)len; 250hashwrite(f, in, avail); 251 offset += avail; 252 len -= avail; 253} 254} 255 256/* Return 0 if we will bust the pack-size limit */ 257static unsigned longwrite_no_reuse_object(struct hashfile *f,struct object_entry *entry, 258unsigned long limit,int usable_delta) 259{ 260unsigned long size, datalen; 261unsigned char header[MAX_PACK_OBJECT_HEADER], 262 dheader[MAX_PACK_OBJECT_HEADER]; 263unsigned hdrlen; 264enum object_type type; 265void*buf; 266struct git_istream *st = NULL; 267 268if(!usable_delta) { 269if(oe_type(entry) == OBJ_BLOB && 270 entry->size > big_file_threshold && 271(st =open_istream(&entry->idx.oid, &type, &size, NULL)) != NULL) 272 buf = NULL; 273else{ 274 buf =read_object_file(&entry->idx.oid, &type, &size); 275if(!buf) 276die(_("unable to read%s"), 277oid_to_hex(&entry->idx.oid)); 278} 279/* 280 * make sure no cached delta data remains from a 281 * previous attempt before a pack split occurred. 282 */ 283FREE_AND_NULL(entry->delta_data); 284 entry->z_delta_size =0; 285}else if(entry->delta_data) { 286 size = entry->delta_size; 287 buf = entry->delta_data; 288 entry->delta_data = NULL; 289 type = (allow_ofs_delta && entry->delta->idx.offset) ? 290 OBJ_OFS_DELTA : OBJ_REF_DELTA; 291}else{ 292 buf =get_delta(entry); 293 size = entry->delta_size; 294 type = (allow_ofs_delta && entry->delta->idx.offset) ? 295 OBJ_OFS_DELTA : OBJ_REF_DELTA; 296} 297 298if(st)/* large blob case, just assume we don't compress well */ 299 datalen = size; 300else if(entry->z_delta_size) 301 datalen = entry->z_delta_size; 302else 303 datalen =do_compress(&buf, size); 304 305/* 306 * The object header is a byte of 'type' followed by zero or 307 * more bytes of length. 308 */ 309 hdrlen =encode_in_pack_object_header(header,sizeof(header), 310 type, size); 311 312if(type == OBJ_OFS_DELTA) { 313/* 314 * Deltas with relative base contain an additional 315 * encoding of the relative offset for the delta 316 * base from this object's position in the pack. 317 */ 318 off_t ofs = entry->idx.offset - entry->delta->idx.offset; 319unsigned pos =sizeof(dheader) -1; 320 dheader[pos] = ofs &127; 321while(ofs >>=7) 322 dheader[--pos] =128| (--ofs &127); 323if(limit && hdrlen +sizeof(dheader) - pos + datalen +20>= limit) { 324if(st) 325close_istream(st); 326free(buf); 327return0; 328} 329hashwrite(f, header, hdrlen); 330hashwrite(f, dheader + pos,sizeof(dheader) - pos); 331 hdrlen +=sizeof(dheader) - pos; 332}else if(type == OBJ_REF_DELTA) { 333/* 334 * Deltas with a base reference contain 335 * an additional 20 bytes for the base sha1. 336 */ 337if(limit && hdrlen +20+ datalen +20>= limit) { 338if(st) 339close_istream(st); 340free(buf); 341return0; 342} 343hashwrite(f, header, hdrlen); 344hashwrite(f, entry->delta->idx.oid.hash,20); 345 hdrlen +=20; 346}else{ 347if(limit && hdrlen + datalen +20>= limit) { 348if(st) 349close_istream(st); 350free(buf); 351return0; 352} 353hashwrite(f, header, hdrlen); 354} 355if(st) { 356 datalen =write_large_blob_data(st, f, &entry->idx.oid); 357close_istream(st); 358}else{ 359hashwrite(f, buf, datalen); 360free(buf); 361} 362 363return hdrlen + datalen; 364} 365 366/* Return 0 if we will bust the pack-size limit */ 367static off_t write_reuse_object(struct hashfile *f,struct object_entry *entry, 368unsigned long limit,int usable_delta) 369{ 370struct packed_git *p = entry->in_pack; 371struct pack_window *w_curs = NULL; 372struct revindex_entry *revidx; 373 off_t offset; 374enum object_type type =oe_type(entry); 375 off_t datalen; 376unsigned char header[MAX_PACK_OBJECT_HEADER], 377 dheader[MAX_PACK_OBJECT_HEADER]; 378unsigned hdrlen; 379 380if(entry->delta) 381 type = (allow_ofs_delta && entry->delta->idx.offset) ? 382 OBJ_OFS_DELTA : OBJ_REF_DELTA; 383 hdrlen =encode_in_pack_object_header(header,sizeof(header), 384 type, entry->size); 385 386 offset = entry->in_pack_offset; 387 revidx =find_pack_revindex(p, offset); 388 datalen = revidx[1].offset - offset; 389if(!pack_to_stdout && p->index_version >1&& 390check_pack_crc(p, &w_curs, offset, datalen, revidx->nr)) { 391error("bad packed object CRC for%s", 392oid_to_hex(&entry->idx.oid)); 393unuse_pack(&w_curs); 394returnwrite_no_reuse_object(f, entry, limit, usable_delta); 395} 396 397 offset += entry->in_pack_header_size; 398 datalen -= entry->in_pack_header_size; 399 400if(!pack_to_stdout && p->index_version ==1&& 401check_pack_inflate(p, &w_curs, offset, datalen, entry->size)) { 402error("corrupt packed object for%s", 403oid_to_hex(&entry->idx.oid)); 404unuse_pack(&w_curs); 405returnwrite_no_reuse_object(f, entry, limit, usable_delta); 406} 407 408if(type == OBJ_OFS_DELTA) { 409 off_t ofs = entry->idx.offset - entry->delta->idx.offset; 410unsigned pos =sizeof(dheader) -1; 411 dheader[pos] = ofs &127; 412while(ofs >>=7) 413 dheader[--pos] =128| (--ofs &127); 414if(limit && hdrlen +sizeof(dheader) - pos + datalen +20>= limit) { 415unuse_pack(&w_curs); 416return0; 417} 418hashwrite(f, header, hdrlen); 419hashwrite(f, dheader + pos,sizeof(dheader) - pos); 420 hdrlen +=sizeof(dheader) - pos; 421 reused_delta++; 422}else if(type == OBJ_REF_DELTA) { 423if(limit && hdrlen +20+ datalen +20>= limit) { 424unuse_pack(&w_curs); 425return0; 426} 427hashwrite(f, header, hdrlen); 428hashwrite(f, entry->delta->idx.oid.hash,20); 429 hdrlen +=20; 430 reused_delta++; 431}else{ 432if(limit && hdrlen + datalen +20>= limit) { 433unuse_pack(&w_curs); 434return0; 435} 436hashwrite(f, header, hdrlen); 437} 438copy_pack_data(f, p, &w_curs, offset, datalen); 439unuse_pack(&w_curs); 440 reused++; 441return hdrlen + datalen; 442} 443 444/* Return 0 if we will bust the pack-size limit */ 445static off_t write_object(struct hashfile *f, 446struct object_entry *entry, 447 off_t write_offset) 448{ 449unsigned long limit; 450 off_t len; 451int usable_delta, to_reuse; 452 453if(!pack_to_stdout) 454crc32_begin(f); 455 456/* apply size limit if limited packsize and not first object */ 457if(!pack_size_limit || !nr_written) 458 limit =0; 459else if(pack_size_limit <= write_offset) 460/* 461 * the earlier object did not fit the limit; avoid 462 * mistaking this with unlimited (i.e. limit = 0). 463 */ 464 limit =1; 465else 466 limit = pack_size_limit - write_offset; 467 468if(!entry->delta) 469 usable_delta =0;/* no delta */ 470else if(!pack_size_limit) 471 usable_delta =1;/* unlimited packfile */ 472else if(entry->delta->idx.offset == (off_t)-1) 473 usable_delta =0;/* base was written to another pack */ 474else if(entry->delta->idx.offset) 475 usable_delta =1;/* base already exists in this pack */ 476else 477 usable_delta =0;/* base could end up in another pack */ 478 479if(!reuse_object) 480 to_reuse =0;/* explicit */ 481else if(!entry->in_pack) 482 to_reuse =0;/* can't reuse what we don't have */ 483else if(oe_type(entry) == OBJ_REF_DELTA || 484oe_type(entry) == OBJ_OFS_DELTA) 485/* check_object() decided it for us ... */ 486 to_reuse = usable_delta; 487/* ... but pack split may override that */ 488else if(oe_type(entry) != entry->in_pack_type) 489 to_reuse =0;/* pack has delta which is unusable */ 490else if(entry->delta) 491 to_reuse =0;/* we want to pack afresh */ 492else 493 to_reuse =1;/* we have it in-pack undeltified, 494 * and we do not need to deltify it. 495 */ 496 497if(!to_reuse) 498 len =write_no_reuse_object(f, entry, limit, usable_delta); 499else 500 len =write_reuse_object(f, entry, limit, usable_delta); 501if(!len) 502return0; 503 504if(usable_delta) 505 written_delta++; 506 written++; 507if(!pack_to_stdout) 508 entry->idx.crc32 =crc32_end(f); 509return len; 510} 511 512enum write_one_status { 513 WRITE_ONE_SKIP = -1,/* already written */ 514 WRITE_ONE_BREAK =0,/* writing this will bust the limit; not written */ 515 WRITE_ONE_WRITTEN =1,/* normal */ 516 WRITE_ONE_RECURSIVE =2/* already scheduled to be written */ 517}; 518 519static enum write_one_status write_one(struct hashfile *f, 520struct object_entry *e, 521 off_t *offset) 522{ 523 off_t size; 524int recursing; 525 526/* 527 * we set offset to 1 (which is an impossible value) to mark 528 * the fact that this object is involved in "write its base 529 * first before writing a deltified object" recursion. 530 */ 531 recursing = (e->idx.offset ==1); 532if(recursing) { 533warning("recursive delta detected for object%s", 534oid_to_hex(&e->idx.oid)); 535return WRITE_ONE_RECURSIVE; 536}else if(e->idx.offset || e->preferred_base) { 537/* offset is non zero if object is written already. */ 538return WRITE_ONE_SKIP; 539} 540 541/* if we are deltified, write out base object first. */ 542if(e->delta) { 543 e->idx.offset =1;/* now recurse */ 544switch(write_one(f, e->delta, offset)) { 545case WRITE_ONE_RECURSIVE: 546/* we cannot depend on this one */ 547 e->delta = NULL; 548break; 549default: 550break; 551case WRITE_ONE_BREAK: 552 e->idx.offset = recursing; 553return WRITE_ONE_BREAK; 554} 555} 556 557 e->idx.offset = *offset; 558 size =write_object(f, e, *offset); 559if(!size) { 560 e->idx.offset = recursing; 561return WRITE_ONE_BREAK; 562} 563 written_list[nr_written++] = &e->idx; 564 565/* make sure off_t is sufficiently large not to wrap */ 566if(signed_add_overflows(*offset, size)) 567die("pack too large for current definition of off_t"); 568*offset += size; 569return WRITE_ONE_WRITTEN; 570} 571 572static intmark_tagged(const char*path,const struct object_id *oid,int flag, 573void*cb_data) 574{ 575struct object_id peeled; 576struct object_entry *entry =packlist_find(&to_pack, oid->hash, NULL); 577 578if(entry) 579 entry->tagged =1; 580if(!peel_ref(path, &peeled)) { 581 entry =packlist_find(&to_pack, peeled.hash, NULL); 582if(entry) 583 entry->tagged =1; 584} 585return0; 586} 587 588staticinlinevoidadd_to_write_order(struct object_entry **wo, 589unsigned int*endp, 590struct object_entry *e) 591{ 592if(e->filled) 593return; 594 wo[(*endp)++] = e; 595 e->filled =1; 596} 597 598static voidadd_descendants_to_write_order(struct object_entry **wo, 599unsigned int*endp, 600struct object_entry *e) 601{ 602int add_to_order =1; 603while(e) { 604if(add_to_order) { 605struct object_entry *s; 606/* add this node... */ 607add_to_write_order(wo, endp, e); 608/* all its siblings... */ 609for(s = e->delta_sibling; s; s = s->delta_sibling) { 610add_to_write_order(wo, endp, s); 611} 612} 613/* drop down a level to add left subtree nodes if possible */ 614if(e->delta_child) { 615 add_to_order =1; 616 e = e->delta_child; 617}else{ 618 add_to_order =0; 619/* our sibling might have some children, it is next */ 620if(e->delta_sibling) { 621 e = e->delta_sibling; 622continue; 623} 624/* go back to our parent node */ 625 e = e->delta; 626while(e && !e->delta_sibling) { 627/* we're on the right side of a subtree, keep 628 * going up until we can go right again */ 629 e = e->delta; 630} 631if(!e) { 632/* done- we hit our original root node */ 633return; 634} 635/* pass it off to sibling at this level */ 636 e = e->delta_sibling; 637} 638}; 639} 640 641static voidadd_family_to_write_order(struct object_entry **wo, 642unsigned int*endp, 643struct object_entry *e) 644{ 645struct object_entry *root; 646 647for(root = e; root->delta; root = root->delta) 648;/* nothing */ 649add_descendants_to_write_order(wo, endp, root); 650} 651 652static struct object_entry **compute_write_order(void) 653{ 654unsigned int i, wo_end, last_untagged; 655 656struct object_entry **wo; 657struct object_entry *objects = to_pack.objects; 658 659for(i =0; i < to_pack.nr_objects; i++) { 660 objects[i].tagged =0; 661 objects[i].filled =0; 662 objects[i].delta_child = NULL; 663 objects[i].delta_sibling = NULL; 664} 665 666/* 667 * Fully connect delta_child/delta_sibling network. 668 * Make sure delta_sibling is sorted in the original 669 * recency order. 670 */ 671for(i = to_pack.nr_objects; i >0;) { 672struct object_entry *e = &objects[--i]; 673if(!e->delta) 674continue; 675/* Mark me as the first child */ 676 e->delta_sibling = e->delta->delta_child; 677 e->delta->delta_child = e; 678} 679 680/* 681 * Mark objects that are at the tip of tags. 682 */ 683for_each_tag_ref(mark_tagged, NULL); 684 685/* 686 * Give the objects in the original recency order until 687 * we see a tagged tip. 688 */ 689ALLOC_ARRAY(wo, to_pack.nr_objects); 690for(i = wo_end =0; i < to_pack.nr_objects; i++) { 691if(objects[i].tagged) 692break; 693add_to_write_order(wo, &wo_end, &objects[i]); 694} 695 last_untagged = i; 696 697/* 698 * Then fill all the tagged tips. 699 */ 700for(; i < to_pack.nr_objects; i++) { 701if(objects[i].tagged) 702add_to_write_order(wo, &wo_end, &objects[i]); 703} 704 705/* 706 * And then all remaining commits and tags. 707 */ 708for(i = last_untagged; i < to_pack.nr_objects; i++) { 709if(oe_type(&objects[i]) != OBJ_COMMIT && 710oe_type(&objects[i]) != OBJ_TAG) 711continue; 712add_to_write_order(wo, &wo_end, &objects[i]); 713} 714 715/* 716 * And then all the trees. 717 */ 718for(i = last_untagged; i < to_pack.nr_objects; i++) { 719if(oe_type(&objects[i]) != OBJ_TREE) 720continue; 721add_to_write_order(wo, &wo_end, &objects[i]); 722} 723 724/* 725 * Finally all the rest in really tight order 726 */ 727for(i = last_untagged; i < to_pack.nr_objects; i++) { 728if(!objects[i].filled) 729add_family_to_write_order(wo, &wo_end, &objects[i]); 730} 731 732if(wo_end != to_pack.nr_objects) 733die("ordered%uobjects, expected %"PRIu32, wo_end, to_pack.nr_objects); 734 735return wo; 736} 737 738static off_t write_reused_pack(struct hashfile *f) 739{ 740unsigned char buffer[8192]; 741 off_t to_write, total; 742int fd; 743 744if(!is_pack_valid(reuse_packfile)) 745die("packfile is invalid:%s", reuse_packfile->pack_name); 746 747 fd =git_open(reuse_packfile->pack_name); 748if(fd <0) 749die_errno("unable to open packfile for reuse:%s", 750 reuse_packfile->pack_name); 751 752if(lseek(fd,sizeof(struct pack_header), SEEK_SET) == -1) 753die_errno("unable to seek in reused packfile"); 754 755if(reuse_packfile_offset <0) 756 reuse_packfile_offset = reuse_packfile->pack_size -20; 757 758 total = to_write = reuse_packfile_offset -sizeof(struct pack_header); 759 760while(to_write) { 761int read_pack =xread(fd, buffer,sizeof(buffer)); 762 763if(read_pack <=0) 764die_errno("unable to read from reused packfile"); 765 766if(read_pack > to_write) 767 read_pack = to_write; 768 769hashwrite(f, buffer, read_pack); 770 to_write -= read_pack; 771 772/* 773 * We don't know the actual number of objects written, 774 * only how many bytes written, how many bytes total, and 775 * how many objects total. So we can fake it by pretending all 776 * objects we are writing are the same size. This gives us a 777 * smooth progress meter, and at the end it matches the true 778 * answer. 779 */ 780 written = reuse_packfile_objects * 781(((double)(total - to_write)) / total); 782display_progress(progress_state, written); 783} 784 785close(fd); 786 written = reuse_packfile_objects; 787display_progress(progress_state, written); 788return reuse_packfile_offset -sizeof(struct pack_header); 789} 790 791static const char no_split_warning[] =N_( 792"disabling bitmap writing, packs are split due to pack.packSizeLimit" 793); 794 795static voidwrite_pack_file(void) 796{ 797uint32_t i =0, j; 798struct hashfile *f; 799 off_t offset; 800uint32_t nr_remaining = nr_result; 801time_t last_mtime =0; 802struct object_entry **write_order; 803 804if(progress > pack_to_stdout) 805 progress_state =start_progress(_("Writing objects"), nr_result); 806ALLOC_ARRAY(written_list, to_pack.nr_objects); 807 write_order =compute_write_order(); 808 809do{ 810struct object_id oid; 811char*pack_tmp_name = NULL; 812 813if(pack_to_stdout) 814 f =hashfd_throughput(1,"<stdout>", progress_state); 815else 816 f =create_tmp_packfile(&pack_tmp_name); 817 818 offset =write_pack_header(f, nr_remaining); 819 820if(reuse_packfile) { 821 off_t packfile_size; 822assert(pack_to_stdout); 823 824 packfile_size =write_reused_pack(f); 825 offset += packfile_size; 826} 827 828 nr_written =0; 829for(; i < to_pack.nr_objects; i++) { 830struct object_entry *e = write_order[i]; 831if(write_one(f, e, &offset) == WRITE_ONE_BREAK) 832break; 833display_progress(progress_state, written); 834} 835 836/* 837 * Did we write the wrong # entries in the header? 838 * If so, rewrite it like in fast-import 839 */ 840if(pack_to_stdout) { 841hashclose(f, oid.hash, CSUM_CLOSE); 842}else if(nr_written == nr_remaining) { 843hashclose(f, oid.hash, CSUM_FSYNC); 844}else{ 845int fd =hashclose(f, oid.hash,0); 846fixup_pack_header_footer(fd, oid.hash, pack_tmp_name, 847 nr_written, oid.hash, offset); 848close(fd); 849if(write_bitmap_index) { 850warning(_(no_split_warning)); 851 write_bitmap_index =0; 852} 853} 854 855if(!pack_to_stdout) { 856struct stat st; 857struct strbuf tmpname = STRBUF_INIT; 858 859/* 860 * Packs are runtime accessed in their mtime 861 * order since newer packs are more likely to contain 862 * younger objects. So if we are creating multiple 863 * packs then we should modify the mtime of later ones 864 * to preserve this property. 865 */ 866if(stat(pack_tmp_name, &st) <0) { 867warning_errno("failed to stat%s", pack_tmp_name); 868}else if(!last_mtime) { 869 last_mtime = st.st_mtime; 870}else{ 871struct utimbuf utb; 872 utb.actime = st.st_atime; 873 utb.modtime = --last_mtime; 874if(utime(pack_tmp_name, &utb) <0) 875warning_errno("failed utime() on%s", pack_tmp_name); 876} 877 878strbuf_addf(&tmpname,"%s-", base_name); 879 880if(write_bitmap_index) { 881bitmap_writer_set_checksum(oid.hash); 882bitmap_writer_build_type_index( 883&to_pack, written_list, nr_written); 884} 885 886finish_tmp_packfile(&tmpname, pack_tmp_name, 887 written_list, nr_written, 888&pack_idx_opts, oid.hash); 889 890if(write_bitmap_index) { 891strbuf_addf(&tmpname,"%s.bitmap",oid_to_hex(&oid)); 892 893stop_progress(&progress_state); 894 895bitmap_writer_show_progress(progress); 896bitmap_writer_reuse_bitmaps(&to_pack); 897bitmap_writer_select_commits(indexed_commits, indexed_commits_nr, -1); 898bitmap_writer_build(&to_pack); 899bitmap_writer_finish(written_list, nr_written, 900 tmpname.buf, write_bitmap_options); 901 write_bitmap_index =0; 902} 903 904strbuf_release(&tmpname); 905free(pack_tmp_name); 906puts(oid_to_hex(&oid)); 907} 908 909/* mark written objects as written to previous pack */ 910for(j =0; j < nr_written; j++) { 911 written_list[j]->offset = (off_t)-1; 912} 913 nr_remaining -= nr_written; 914}while(nr_remaining && i < to_pack.nr_objects); 915 916free(written_list); 917free(write_order); 918stop_progress(&progress_state); 919if(written != nr_result) 920die("wrote %"PRIu32" objects while expecting %"PRIu32, 921 written, nr_result); 922} 923 924static intno_try_delta(const char*path) 925{ 926static struct attr_check *check; 927 928if(!check) 929 check =attr_check_initl("delta", NULL); 930if(git_check_attr(path, check)) 931return0; 932if(ATTR_FALSE(check->items[0].value)) 933return1; 934return0; 935} 936 937/* 938 * When adding an object, check whether we have already added it 939 * to our packing list. If so, we can skip. However, if we are 940 * being asked to excludei t, but the previous mention was to include 941 * it, make sure to adjust its flags and tweak our numbers accordingly. 942 * 943 * As an optimization, we pass out the index position where we would have 944 * found the item, since that saves us from having to look it up again a 945 * few lines later when we want to add the new entry. 946 */ 947static inthave_duplicate_entry(const struct object_id *oid, 948int exclude, 949uint32_t*index_pos) 950{ 951struct object_entry *entry; 952 953 entry =packlist_find(&to_pack, oid->hash, index_pos); 954if(!entry) 955return0; 956 957if(exclude) { 958if(!entry->preferred_base) 959 nr_result--; 960 entry->preferred_base =1; 961} 962 963return1; 964} 965 966static intwant_found_object(int exclude,struct packed_git *p) 967{ 968if(exclude) 969return1; 970if(incremental) 971return0; 972 973/* 974 * When asked to do --local (do not include an object that appears in a 975 * pack we borrow from elsewhere) or --honor-pack-keep (do not include 976 * an object that appears in a pack marked with .keep), finding a pack 977 * that matches the criteria is sufficient for us to decide to omit it. 978 * However, even if this pack does not satisfy the criteria, we need to 979 * make sure no copy of this object appears in _any_ pack that makes us 980 * to omit the object, so we need to check all the packs. 981 * 982 * We can however first check whether these options can possible matter; 983 * if they do not matter we know we want the object in generated pack. 984 * Otherwise, we signal "-1" at the end to tell the caller that we do 985 * not know either way, and it needs to check more packs. 986 */ 987if(!ignore_packed_keep && 988(!local || !have_non_local_packs)) 989return1; 990 991if(local && !p->pack_local) 992return0; 993if(ignore_packed_keep && p->pack_local && p->pack_keep) 994return0; 995 996/* we don't know yet; keep looking for more packs */ 997return-1; 998} 9991000/*1001 * Check whether we want the object in the pack (e.g., we do not want1002 * objects found in non-local stores if the "--local" option was used).1003 *1004 * If the caller already knows an existing pack it wants to take the object1005 * from, that is passed in *found_pack and *found_offset; otherwise this1006 * function finds if there is any pack that has the object and returns the pack1007 * and its offset in these variables.1008 */1009static intwant_object_in_pack(const struct object_id *oid,1010int exclude,1011struct packed_git **found_pack,1012 off_t *found_offset)1013{1014int want;1015struct list_head *pos;10161017if(!exclude && local &&has_loose_object_nonlocal(oid->hash))1018return0;10191020/*1021 * If we already know the pack object lives in, start checks from that1022 * pack - in the usual case when neither --local was given nor .keep files1023 * are present we will determine the answer right now.1024 */1025if(*found_pack) {1026 want =want_found_object(exclude, *found_pack);1027if(want != -1)1028return want;1029}1030list_for_each(pos,get_packed_git_mru(the_repository)) {1031struct packed_git *p =list_entry(pos,struct packed_git, mru);1032 off_t offset;10331034if(p == *found_pack)1035 offset = *found_offset;1036else1037 offset =find_pack_entry_one(oid->hash, p);10381039if(offset) {1040if(!*found_pack) {1041if(!is_pack_valid(p))1042continue;1043*found_offset = offset;1044*found_pack = p;1045}1046 want =want_found_object(exclude, p);1047if(!exclude && want >0)1048list_move(&p->mru,1049get_packed_git_mru(the_repository));1050if(want != -1)1051return want;1052}1053}10541055return1;1056}10571058static voidcreate_object_entry(const struct object_id *oid,1059enum object_type type,1060uint32_t hash,1061int exclude,1062int no_try_delta,1063uint32_t index_pos,1064struct packed_git *found_pack,1065 off_t found_offset)1066{1067struct object_entry *entry;10681069 entry =packlist_alloc(&to_pack, oid->hash, index_pos);1070 entry->hash = hash;1071oe_set_type(entry, type);1072if(exclude)1073 entry->preferred_base =1;1074else1075 nr_result++;1076if(found_pack) {1077 entry->in_pack = found_pack;1078 entry->in_pack_offset = found_offset;1079}10801081 entry->no_try_delta = no_try_delta;1082}10831084static const char no_closure_warning[] =N_(1085"disabling bitmap writing, as some objects are not being packed"1086);10871088static intadd_object_entry(const struct object_id *oid,enum object_type type,1089const char*name,int exclude)1090{1091struct packed_git *found_pack = NULL;1092 off_t found_offset =0;1093uint32_t index_pos;10941095if(have_duplicate_entry(oid, exclude, &index_pos))1096return0;10971098if(!want_object_in_pack(oid, exclude, &found_pack, &found_offset)) {1099/* The pack is missing an object, so it will not have closure */1100if(write_bitmap_index) {1101warning(_(no_closure_warning));1102 write_bitmap_index =0;1103}1104return0;1105}11061107create_object_entry(oid, type,pack_name_hash(name),1108 exclude, name &&no_try_delta(name),1109 index_pos, found_pack, found_offset);11101111display_progress(progress_state, nr_result);1112return1;1113}11141115static intadd_object_entry_from_bitmap(const struct object_id *oid,1116enum object_type type,1117int flags,uint32_t name_hash,1118struct packed_git *pack, off_t offset)1119{1120uint32_t index_pos;11211122if(have_duplicate_entry(oid,0, &index_pos))1123return0;11241125if(!want_object_in_pack(oid,0, &pack, &offset))1126return0;11271128create_object_entry(oid, type, name_hash,0,0, index_pos, pack, offset);11291130display_progress(progress_state, nr_result);1131return1;1132}11331134struct pbase_tree_cache {1135struct object_id oid;1136int ref;1137int temporary;1138void*tree_data;1139unsigned long tree_size;1140};11411142static struct pbase_tree_cache *(pbase_tree_cache[256]);1143static intpbase_tree_cache_ix(const struct object_id *oid)1144{1145return oid->hash[0] %ARRAY_SIZE(pbase_tree_cache);1146}1147static intpbase_tree_cache_ix_incr(int ix)1148{1149return(ix+1) %ARRAY_SIZE(pbase_tree_cache);1150}11511152static struct pbase_tree {1153struct pbase_tree *next;1154/* This is a phony "cache" entry; we are not1155 * going to evict it or find it through _get()1156 * mechanism -- this is for the toplevel node that1157 * would almost always change with any commit.1158 */1159struct pbase_tree_cache pcache;1160} *pbase_tree;11611162static struct pbase_tree_cache *pbase_tree_get(const struct object_id *oid)1163{1164struct pbase_tree_cache *ent, *nent;1165void*data;1166unsigned long size;1167enum object_type type;1168int neigh;1169int my_ix =pbase_tree_cache_ix(oid);1170int available_ix = -1;11711172/* pbase-tree-cache acts as a limited hashtable.1173 * your object will be found at your index or within a few1174 * slots after that slot if it is cached.1175 */1176for(neigh =0; neigh <8; neigh++) {1177 ent = pbase_tree_cache[my_ix];1178if(ent && !oidcmp(&ent->oid, oid)) {1179 ent->ref++;1180return ent;1181}1182else if(((available_ix <0) && (!ent || !ent->ref)) ||1183((0<= available_ix) &&1184(!ent && pbase_tree_cache[available_ix])))1185 available_ix = my_ix;1186if(!ent)1187break;1188 my_ix =pbase_tree_cache_ix_incr(my_ix);1189}11901191/* Did not find one. Either we got a bogus request or1192 * we need to read and perhaps cache.1193 */1194 data =read_object_file(oid, &type, &size);1195if(!data)1196return NULL;1197if(type != OBJ_TREE) {1198free(data);1199return NULL;1200}12011202/* We need to either cache or return a throwaway copy */12031204if(available_ix <0)1205 ent = NULL;1206else{1207 ent = pbase_tree_cache[available_ix];1208 my_ix = available_ix;1209}12101211if(!ent) {1212 nent =xmalloc(sizeof(*nent));1213 nent->temporary = (available_ix <0);1214}1215else{1216/* evict and reuse */1217free(ent->tree_data);1218 nent = ent;1219}1220oidcpy(&nent->oid, oid);1221 nent->tree_data = data;1222 nent->tree_size = size;1223 nent->ref =1;1224if(!nent->temporary)1225 pbase_tree_cache[my_ix] = nent;1226return nent;1227}12281229static voidpbase_tree_put(struct pbase_tree_cache *cache)1230{1231if(!cache->temporary) {1232 cache->ref--;1233return;1234}1235free(cache->tree_data);1236free(cache);1237}12381239static intname_cmp_len(const char*name)1240{1241int i;1242for(i =0; name[i] && name[i] !='\n'&& name[i] !='/'; i++)1243;1244return i;1245}12461247static voidadd_pbase_object(struct tree_desc *tree,1248const char*name,1249int cmplen,1250const char*fullname)1251{1252struct name_entry entry;1253int cmp;12541255while(tree_entry(tree,&entry)) {1256if(S_ISGITLINK(entry.mode))1257continue;1258 cmp =tree_entry_len(&entry) != cmplen ?1:1259memcmp(name, entry.path, cmplen);1260if(cmp >0)1261continue;1262if(cmp <0)1263return;1264if(name[cmplen] !='/') {1265add_object_entry(entry.oid,1266object_type(entry.mode),1267 fullname,1);1268return;1269}1270if(S_ISDIR(entry.mode)) {1271struct tree_desc sub;1272struct pbase_tree_cache *tree;1273const char*down = name+cmplen+1;1274int downlen =name_cmp_len(down);12751276 tree =pbase_tree_get(entry.oid);1277if(!tree)1278return;1279init_tree_desc(&sub, tree->tree_data, tree->tree_size);12801281add_pbase_object(&sub, down, downlen, fullname);1282pbase_tree_put(tree);1283}1284}1285}12861287static unsigned*done_pbase_paths;1288static int done_pbase_paths_num;1289static int done_pbase_paths_alloc;1290static intdone_pbase_path_pos(unsigned hash)1291{1292int lo =0;1293int hi = done_pbase_paths_num;1294while(lo < hi) {1295int mi = lo + (hi - lo) /2;1296if(done_pbase_paths[mi] == hash)1297return mi;1298if(done_pbase_paths[mi] < hash)1299 hi = mi;1300else1301 lo = mi +1;1302}1303return-lo-1;1304}13051306static intcheck_pbase_path(unsigned hash)1307{1308int pos =done_pbase_path_pos(hash);1309if(0<= pos)1310return1;1311 pos = -pos -1;1312ALLOC_GROW(done_pbase_paths,1313 done_pbase_paths_num +1,1314 done_pbase_paths_alloc);1315 done_pbase_paths_num++;1316if(pos < done_pbase_paths_num)1317MOVE_ARRAY(done_pbase_paths + pos +1, done_pbase_paths + pos,1318 done_pbase_paths_num - pos -1);1319 done_pbase_paths[pos] = hash;1320return0;1321}13221323static voidadd_preferred_base_object(const char*name)1324{1325struct pbase_tree *it;1326int cmplen;1327unsigned hash =pack_name_hash(name);13281329if(!num_preferred_base ||check_pbase_path(hash))1330return;13311332 cmplen =name_cmp_len(name);1333for(it = pbase_tree; it; it = it->next) {1334if(cmplen ==0) {1335add_object_entry(&it->pcache.oid, OBJ_TREE, NULL,1);1336}1337else{1338struct tree_desc tree;1339init_tree_desc(&tree, it->pcache.tree_data, it->pcache.tree_size);1340add_pbase_object(&tree, name, cmplen, name);1341}1342}1343}13441345static voidadd_preferred_base(struct object_id *oid)1346{1347struct pbase_tree *it;1348void*data;1349unsigned long size;1350struct object_id tree_oid;13511352if(window <= num_preferred_base++)1353return;13541355 data =read_object_with_reference(oid, tree_type, &size, &tree_oid);1356if(!data)1357return;13581359for(it = pbase_tree; it; it = it->next) {1360if(!oidcmp(&it->pcache.oid, &tree_oid)) {1361free(data);1362return;1363}1364}13651366 it =xcalloc(1,sizeof(*it));1367 it->next = pbase_tree;1368 pbase_tree = it;13691370oidcpy(&it->pcache.oid, &tree_oid);1371 it->pcache.tree_data = data;1372 it->pcache.tree_size = size;1373}13741375static voidcleanup_preferred_base(void)1376{1377struct pbase_tree *it;1378unsigned i;13791380 it = pbase_tree;1381 pbase_tree = NULL;1382while(it) {1383struct pbase_tree *tmp = it;1384 it = tmp->next;1385free(tmp->pcache.tree_data);1386free(tmp);1387}13881389for(i =0; i <ARRAY_SIZE(pbase_tree_cache); i++) {1390if(!pbase_tree_cache[i])1391continue;1392free(pbase_tree_cache[i]->tree_data);1393FREE_AND_NULL(pbase_tree_cache[i]);1394}13951396FREE_AND_NULL(done_pbase_paths);1397 done_pbase_paths_num = done_pbase_paths_alloc =0;1398}13991400static voidcheck_object(struct object_entry *entry)1401{1402if(entry->in_pack) {1403struct packed_git *p = entry->in_pack;1404struct pack_window *w_curs = NULL;1405const unsigned char*base_ref = NULL;1406struct object_entry *base_entry;1407unsigned long used, used_0;1408unsigned long avail;1409 off_t ofs;1410unsigned char*buf, c;1411enum object_type type;14121413 buf =use_pack(p, &w_curs, entry->in_pack_offset, &avail);14141415/*1416 * We want in_pack_type even if we do not reuse delta1417 * since non-delta representations could still be reused.1418 */1419 used =unpack_object_header_buffer(buf, avail,1420&type,1421&entry->size);1422if(used ==0)1423goto give_up;14241425if(type <0)1426BUG("invalid type%d", type);1427 entry->in_pack_type = type;14281429/*1430 * Determine if this is a delta and if so whether we can1431 * reuse it or not. Otherwise let's find out as cheaply as1432 * possible what the actual type and size for this object is.1433 */1434switch(entry->in_pack_type) {1435default:1436/* Not a delta hence we've already got all we need. */1437oe_set_type(entry, entry->in_pack_type);1438 entry->in_pack_header_size = used;1439if(oe_type(entry) < OBJ_COMMIT ||oe_type(entry) > OBJ_BLOB)1440goto give_up;1441unuse_pack(&w_curs);1442return;1443case OBJ_REF_DELTA:1444if(reuse_delta && !entry->preferred_base)1445 base_ref =use_pack(p, &w_curs,1446 entry->in_pack_offset + used, NULL);1447 entry->in_pack_header_size = used +20;1448break;1449case OBJ_OFS_DELTA:1450 buf =use_pack(p, &w_curs,1451 entry->in_pack_offset + used, NULL);1452 used_0 =0;1453 c = buf[used_0++];1454 ofs = c &127;1455while(c &128) {1456 ofs +=1;1457if(!ofs ||MSB(ofs,7)) {1458error("delta base offset overflow in pack for%s",1459oid_to_hex(&entry->idx.oid));1460goto give_up;1461}1462 c = buf[used_0++];1463 ofs = (ofs <<7) + (c &127);1464}1465 ofs = entry->in_pack_offset - ofs;1466if(ofs <=0|| ofs >= entry->in_pack_offset) {1467error("delta base offset out of bound for%s",1468oid_to_hex(&entry->idx.oid));1469goto give_up;1470}1471if(reuse_delta && !entry->preferred_base) {1472struct revindex_entry *revidx;1473 revidx =find_pack_revindex(p, ofs);1474if(!revidx)1475goto give_up;1476 base_ref =nth_packed_object_sha1(p, revidx->nr);1477}1478 entry->in_pack_header_size = used + used_0;1479break;1480}14811482if(base_ref && (base_entry =packlist_find(&to_pack, base_ref, NULL))) {1483/*1484 * If base_ref was set above that means we wish to1485 * reuse delta data, and we even found that base1486 * in the list of objects we want to pack. Goodie!1487 *1488 * Depth value does not matter - find_deltas() will1489 * never consider reused delta as the base object to1490 * deltify other objects against, in order to avoid1491 * circular deltas.1492 */1493oe_set_type(entry, entry->in_pack_type);1494 entry->delta = base_entry;1495 entry->delta_size = entry->size;1496 entry->delta_sibling = base_entry->delta_child;1497 base_entry->delta_child = entry;1498unuse_pack(&w_curs);1499return;1500}15011502if(oe_type(entry)) {1503/*1504 * This must be a delta and we already know what the1505 * final object type is. Let's extract the actual1506 * object size from the delta header.1507 */1508 entry->size =get_size_from_delta(p, &w_curs,1509 entry->in_pack_offset + entry->in_pack_header_size);1510if(entry->size ==0)1511goto give_up;1512unuse_pack(&w_curs);1513return;1514}15151516/*1517 * No choice but to fall back to the recursive delta walk1518 * with sha1_object_info() to find about the object type1519 * at this point...1520 */1521 give_up:1522unuse_pack(&w_curs);1523}15241525oe_set_type(entry,oid_object_info(&entry->idx.oid, &entry->size));1526/*1527 * The error condition is checked in prepare_pack(). This is1528 * to permit a missing preferred base object to be ignored1529 * as a preferred base. Doing so can result in a larger1530 * pack file, but the transfer will still take place.1531 */1532}15331534static intpack_offset_sort(const void*_a,const void*_b)1535{1536const struct object_entry *a = *(struct object_entry **)_a;1537const struct object_entry *b = *(struct object_entry **)_b;15381539/* avoid filesystem trashing with loose objects */1540if(!a->in_pack && !b->in_pack)1541returnoidcmp(&a->idx.oid, &b->idx.oid);15421543if(a->in_pack < b->in_pack)1544return-1;1545if(a->in_pack > b->in_pack)1546return1;1547return a->in_pack_offset < b->in_pack_offset ? -1:1548(a->in_pack_offset > b->in_pack_offset);1549}15501551/*1552 * Drop an on-disk delta we were planning to reuse. Naively, this would1553 * just involve blanking out the "delta" field, but we have to deal1554 * with some extra book-keeping:1555 *1556 * 1. Removing ourselves from the delta_sibling linked list.1557 *1558 * 2. Updating our size/type to the non-delta representation. These were1559 * either not recorded initially (size) or overwritten with the delta type1560 * (type) when check_object() decided to reuse the delta.1561 *1562 * 3. Resetting our delta depth, as we are now a base object.1563 */1564static voiddrop_reused_delta(struct object_entry *entry)1565{1566struct object_entry **p = &entry->delta->delta_child;1567struct object_info oi = OBJECT_INFO_INIT;1568enum object_type type;15691570while(*p) {1571if(*p == entry)1572*p = (*p)->delta_sibling;1573else1574 p = &(*p)->delta_sibling;1575}1576 entry->delta = NULL;1577 entry->depth =0;15781579 oi.sizep = &entry->size;1580 oi.typep = &type;1581if(packed_object_info(entry->in_pack, entry->in_pack_offset, &oi) <0) {1582/*1583 * We failed to get the info from this pack for some reason;1584 * fall back to sha1_object_info, which may find another copy.1585 * And if that fails, the error will be recorded in oe_type(entry)1586 * and dealt with in prepare_pack().1587 */1588oe_set_type(entry,oid_object_info(&entry->idx.oid,1589&entry->size));1590}else{1591oe_set_type(entry, type);1592}1593}15941595/*1596 * Follow the chain of deltas from this entry onward, throwing away any links1597 * that cause us to hit a cycle (as determined by the DFS state flags in1598 * the entries).1599 *1600 * We also detect too-long reused chains that would violate our --depth1601 * limit.1602 */1603static voidbreak_delta_chains(struct object_entry *entry)1604{1605/*1606 * The actual depth of each object we will write is stored as an int,1607 * as it cannot exceed our int "depth" limit. But before we break1608 * changes based no that limit, we may potentially go as deep as the1609 * number of objects, which is elsewhere bounded to a uint32_t.1610 */1611uint32_t total_depth;1612struct object_entry *cur, *next;16131614for(cur = entry, total_depth =0;1615 cur;1616 cur = cur->delta, total_depth++) {1617if(cur->dfs_state == DFS_DONE) {1618/*1619 * We've already seen this object and know it isn't1620 * part of a cycle. We do need to append its depth1621 * to our count.1622 */1623 total_depth += cur->depth;1624break;1625}16261627/*1628 * We break cycles before looping, so an ACTIVE state (or any1629 * other cruft which made its way into the state variable)1630 * is a bug.1631 */1632if(cur->dfs_state != DFS_NONE)1633die("BUG: confusing delta dfs state in first pass:%d",1634 cur->dfs_state);16351636/*1637 * Now we know this is the first time we've seen the object. If1638 * it's not a delta, we're done traversing, but we'll mark it1639 * done to save time on future traversals.1640 */1641if(!cur->delta) {1642 cur->dfs_state = DFS_DONE;1643break;1644}16451646/*1647 * Mark ourselves as active and see if the next step causes1648 * us to cycle to another active object. It's important to do1649 * this _before_ we loop, because it impacts where we make the1650 * cut, and thus how our total_depth counter works.1651 * E.g., We may see a partial loop like:1652 *1653 * A -> B -> C -> D -> B1654 *1655 * Cutting B->C breaks the cycle. But now the depth of A is1656 * only 1, and our total_depth counter is at 3. The size of the1657 * error is always one less than the size of the cycle we1658 * broke. Commits C and D were "lost" from A's chain.1659 *1660 * If we instead cut D->B, then the depth of A is correct at 3.1661 * We keep all commits in the chain that we examined.1662 */1663 cur->dfs_state = DFS_ACTIVE;1664if(cur->delta->dfs_state == DFS_ACTIVE) {1665drop_reused_delta(cur);1666 cur->dfs_state = DFS_DONE;1667break;1668}1669}16701671/*1672 * And now that we've gone all the way to the bottom of the chain, we1673 * need to clear the active flags and set the depth fields as1674 * appropriate. Unlike the loop above, which can quit when it drops a1675 * delta, we need to keep going to look for more depth cuts. So we need1676 * an extra "next" pointer to keep going after we reset cur->delta.1677 */1678for(cur = entry; cur; cur = next) {1679 next = cur->delta;16801681/*1682 * We should have a chain of zero or more ACTIVE states down to1683 * a final DONE. We can quit after the DONE, because either it1684 * has no bases, or we've already handled them in a previous1685 * call.1686 */1687if(cur->dfs_state == DFS_DONE)1688break;1689else if(cur->dfs_state != DFS_ACTIVE)1690die("BUG: confusing delta dfs state in second pass:%d",1691 cur->dfs_state);16921693/*1694 * If the total_depth is more than depth, then we need to snip1695 * the chain into two or more smaller chains that don't exceed1696 * the maximum depth. Most of the resulting chains will contain1697 * (depth + 1) entries (i.e., depth deltas plus one base), and1698 * the last chain (i.e., the one containing entry) will contain1699 * whatever entries are left over, namely1700 * (total_depth % (depth + 1)) of them.1701 *1702 * Since we are iterating towards decreasing depth, we need to1703 * decrement total_depth as we go, and we need to write to the1704 * entry what its final depth will be after all of the1705 * snipping. Since we're snipping into chains of length (depth1706 * + 1) entries, the final depth of an entry will be its1707 * original depth modulo (depth + 1). Any time we encounter an1708 * entry whose final depth is supposed to be zero, we snip it1709 * from its delta base, thereby making it so.1710 */1711 cur->depth = (total_depth--) % (depth +1);1712if(!cur->depth)1713drop_reused_delta(cur);17141715 cur->dfs_state = DFS_DONE;1716}1717}17181719static voidget_object_details(void)1720{1721uint32_t i;1722struct object_entry **sorted_by_offset;17231724 sorted_by_offset =xcalloc(to_pack.nr_objects,sizeof(struct object_entry *));1725for(i =0; i < to_pack.nr_objects; i++)1726 sorted_by_offset[i] = to_pack.objects + i;1727QSORT(sorted_by_offset, to_pack.nr_objects, pack_offset_sort);17281729for(i =0; i < to_pack.nr_objects; i++) {1730struct object_entry *entry = sorted_by_offset[i];1731check_object(entry);1732if(big_file_threshold < entry->size)1733 entry->no_try_delta =1;1734}17351736/*1737 * This must happen in a second pass, since we rely on the delta1738 * information for the whole list being completed.1739 */1740for(i =0; i < to_pack.nr_objects; i++)1741break_delta_chains(&to_pack.objects[i]);17421743free(sorted_by_offset);1744}17451746/*1747 * We search for deltas in a list sorted by type, by filename hash, and then1748 * by size, so that we see progressively smaller and smaller files.1749 * That's because we prefer deltas to be from the bigger file1750 * to the smaller -- deletes are potentially cheaper, but perhaps1751 * more importantly, the bigger file is likely the more recent1752 * one. The deepest deltas are therefore the oldest objects which are1753 * less susceptible to be accessed often.1754 */1755static inttype_size_sort(const void*_a,const void*_b)1756{1757const struct object_entry *a = *(struct object_entry **)_a;1758const struct object_entry *b = *(struct object_entry **)_b;1759enum object_type a_type =oe_type(a);1760enum object_type b_type =oe_type(b);17611762if(a_type > b_type)1763return-1;1764if(a_type < b_type)1765return1;1766if(a->hash > b->hash)1767return-1;1768if(a->hash < b->hash)1769return1;1770if(a->preferred_base > b->preferred_base)1771return-1;1772if(a->preferred_base < b->preferred_base)1773return1;1774if(a->size > b->size)1775return-1;1776if(a->size < b->size)1777return1;1778return a < b ? -1: (a > b);/* newest first */1779}17801781struct unpacked {1782struct object_entry *entry;1783void*data;1784struct delta_index *index;1785unsigned depth;1786};17871788static intdelta_cacheable(unsigned long src_size,unsigned long trg_size,1789unsigned long delta_size)1790{1791if(max_delta_cache_size && delta_cache_size + delta_size > max_delta_cache_size)1792return0;17931794if(delta_size < cache_max_small_delta_size)1795return1;17961797/* cache delta, if objects are large enough compared to delta size */1798if((src_size >>20) + (trg_size >>21) > (delta_size >>10))1799return1;18001801return0;1802}18031804#ifndef NO_PTHREADS18051806static pthread_mutex_t read_mutex;1807#define read_lock() pthread_mutex_lock(&read_mutex)1808#define read_unlock() pthread_mutex_unlock(&read_mutex)18091810static pthread_mutex_t cache_mutex;1811#define cache_lock() pthread_mutex_lock(&cache_mutex)1812#define cache_unlock() pthread_mutex_unlock(&cache_mutex)18131814static pthread_mutex_t progress_mutex;1815#define progress_lock() pthread_mutex_lock(&progress_mutex)1816#define progress_unlock() pthread_mutex_unlock(&progress_mutex)18171818#else18191820#define read_lock() (void)01821#define read_unlock() (void)01822#define cache_lock() (void)01823#define cache_unlock() (void)01824#define progress_lock() (void)01825#define progress_unlock() (void)018261827#endif18281829static inttry_delta(struct unpacked *trg,struct unpacked *src,1830unsigned max_depth,unsigned long*mem_usage)1831{1832struct object_entry *trg_entry = trg->entry;1833struct object_entry *src_entry = src->entry;1834unsigned long trg_size, src_size, delta_size, sizediff, max_size, sz;1835unsigned ref_depth;1836enum object_type type;1837void*delta_buf;18381839/* Don't bother doing diffs between different types */1840if(oe_type(trg_entry) !=oe_type(src_entry))1841return-1;18421843/*1844 * We do not bother to try a delta that we discarded on an1845 * earlier try, but only when reusing delta data. Note that1846 * src_entry that is marked as the preferred_base should always1847 * be considered, as even if we produce a suboptimal delta against1848 * it, we will still save the transfer cost, as we already know1849 * the other side has it and we won't send src_entry at all.1850 */1851if(reuse_delta && trg_entry->in_pack &&1852 trg_entry->in_pack == src_entry->in_pack &&1853!src_entry->preferred_base &&1854 trg_entry->in_pack_type != OBJ_REF_DELTA &&1855 trg_entry->in_pack_type != OBJ_OFS_DELTA)1856return0;18571858/* Let's not bust the allowed depth. */1859if(src->depth >= max_depth)1860return0;18611862/* Now some size filtering heuristics. */1863 trg_size = trg_entry->size;1864if(!trg_entry->delta) {1865 max_size = trg_size/2-20;1866 ref_depth =1;1867}else{1868 max_size = trg_entry->delta_size;1869 ref_depth = trg->depth;1870}1871 max_size = (uint64_t)max_size * (max_depth - src->depth) /1872(max_depth - ref_depth +1);1873if(max_size ==0)1874return0;1875 src_size = src_entry->size;1876 sizediff = src_size < trg_size ? trg_size - src_size :0;1877if(sizediff >= max_size)1878return0;1879if(trg_size < src_size /32)1880return0;18811882/* Load data if not already done */1883if(!trg->data) {1884read_lock();1885 trg->data =read_object_file(&trg_entry->idx.oid, &type, &sz);1886read_unlock();1887if(!trg->data)1888die("object%scannot be read",1889oid_to_hex(&trg_entry->idx.oid));1890if(sz != trg_size)1891die("object%sinconsistent object length (%lu vs%lu)",1892oid_to_hex(&trg_entry->idx.oid), sz,1893 trg_size);1894*mem_usage += sz;1895}1896if(!src->data) {1897read_lock();1898 src->data =read_object_file(&src_entry->idx.oid, &type, &sz);1899read_unlock();1900if(!src->data) {1901if(src_entry->preferred_base) {1902static int warned =0;1903if(!warned++)1904warning("object%scannot be read",1905oid_to_hex(&src_entry->idx.oid));1906/*1907 * Those objects are not included in the1908 * resulting pack. Be resilient and ignore1909 * them if they can't be read, in case the1910 * pack could be created nevertheless.1911 */1912return0;1913}1914die("object%scannot be read",1915oid_to_hex(&src_entry->idx.oid));1916}1917if(sz != src_size)1918die("object%sinconsistent object length (%lu vs%lu)",1919oid_to_hex(&src_entry->idx.oid), sz,1920 src_size);1921*mem_usage += sz;1922}1923if(!src->index) {1924 src->index =create_delta_index(src->data, src_size);1925if(!src->index) {1926static int warned =0;1927if(!warned++)1928warning("suboptimal pack - out of memory");1929return0;1930}1931*mem_usage +=sizeof_delta_index(src->index);1932}19331934 delta_buf =create_delta(src->index, trg->data, trg_size, &delta_size, max_size);1935if(!delta_buf)1936return0;19371938if(trg_entry->delta) {1939/* Prefer only shallower same-sized deltas. */1940if(delta_size == trg_entry->delta_size &&1941 src->depth +1>= trg->depth) {1942free(delta_buf);1943return0;1944}1945}19461947/*1948 * Handle memory allocation outside of the cache1949 * accounting lock. Compiler will optimize the strangeness1950 * away when NO_PTHREADS is defined.1951 */1952free(trg_entry->delta_data);1953cache_lock();1954if(trg_entry->delta_data) {1955 delta_cache_size -= trg_entry->delta_size;1956 trg_entry->delta_data = NULL;1957}1958if(delta_cacheable(src_size, trg_size, delta_size)) {1959 delta_cache_size += delta_size;1960cache_unlock();1961 trg_entry->delta_data =xrealloc(delta_buf, delta_size);1962}else{1963cache_unlock();1964free(delta_buf);1965}19661967 trg_entry->delta = src_entry;1968 trg_entry->delta_size = delta_size;1969 trg->depth = src->depth +1;19701971return1;1972}19731974static unsigned intcheck_delta_limit(struct object_entry *me,unsigned int n)1975{1976struct object_entry *child = me->delta_child;1977unsigned int m = n;1978while(child) {1979unsigned int c =check_delta_limit(child, n +1);1980if(m < c)1981 m = c;1982 child = child->delta_sibling;1983}1984return m;1985}19861987static unsigned longfree_unpacked(struct unpacked *n)1988{1989unsigned long freed_mem =sizeof_delta_index(n->index);1990free_delta_index(n->index);1991 n->index = NULL;1992if(n->data) {1993 freed_mem += n->entry->size;1994FREE_AND_NULL(n->data);1995}1996 n->entry = NULL;1997 n->depth =0;1998return freed_mem;1999}20002001static voidfind_deltas(struct object_entry **list,unsigned*list_size,2002int window,int depth,unsigned*processed)2003{2004uint32_t i, idx =0, count =0;2005struct unpacked *array;2006unsigned long mem_usage =0;20072008 array =xcalloc(window,sizeof(struct unpacked));20092010for(;;) {2011struct object_entry *entry;2012struct unpacked *n = array + idx;2013int j, max_depth, best_base = -1;20142015progress_lock();2016if(!*list_size) {2017progress_unlock();2018break;2019}2020 entry = *list++;2021(*list_size)--;2022if(!entry->preferred_base) {2023(*processed)++;2024display_progress(progress_state, *processed);2025}2026progress_unlock();20272028 mem_usage -=free_unpacked(n);2029 n->entry = entry;20302031while(window_memory_limit &&2032 mem_usage > window_memory_limit &&2033 count >1) {2034uint32_t tail = (idx + window - count) % window;2035 mem_usage -=free_unpacked(array + tail);2036 count--;2037}20382039/* We do not compute delta to *create* objects we are not2040 * going to pack.2041 */2042if(entry->preferred_base)2043goto next;20442045/*2046 * If the current object is at pack edge, take the depth the2047 * objects that depend on the current object into account2048 * otherwise they would become too deep.2049 */2050 max_depth = depth;2051if(entry->delta_child) {2052 max_depth -=check_delta_limit(entry,0);2053if(max_depth <=0)2054goto next;2055}20562057 j = window;2058while(--j >0) {2059int ret;2060uint32_t other_idx = idx + j;2061struct unpacked *m;2062if(other_idx >= window)2063 other_idx -= window;2064 m = array + other_idx;2065if(!m->entry)2066break;2067 ret =try_delta(n, m, max_depth, &mem_usage);2068if(ret <0)2069break;2070else if(ret >0)2071 best_base = other_idx;2072}20732074/*2075 * If we decided to cache the delta data, then it is best2076 * to compress it right away. First because we have to do2077 * it anyway, and doing it here while we're threaded will2078 * save a lot of time in the non threaded write phase,2079 * as well as allow for caching more deltas within2080 * the same cache size limit.2081 * ...2082 * But only if not writing to stdout, since in that case2083 * the network is most likely throttling writes anyway,2084 * and therefore it is best to go to the write phase ASAP2085 * instead, as we can afford spending more time compressing2086 * between writes at that moment.2087 */2088if(entry->delta_data && !pack_to_stdout) {2089 entry->z_delta_size =do_compress(&entry->delta_data,2090 entry->delta_size);2091cache_lock();2092 delta_cache_size -= entry->delta_size;2093 delta_cache_size += entry->z_delta_size;2094cache_unlock();2095}20962097/* if we made n a delta, and if n is already at max2098 * depth, leaving it in the window is pointless. we2099 * should evict it first.2100 */2101if(entry->delta && max_depth <= n->depth)2102continue;21032104/*2105 * Move the best delta base up in the window, after the2106 * currently deltified object, to keep it longer. It will2107 * be the first base object to be attempted next.2108 */2109if(entry->delta) {2110struct unpacked swap = array[best_base];2111int dist = (window + idx - best_base) % window;2112int dst = best_base;2113while(dist--) {2114int src = (dst +1) % window;2115 array[dst] = array[src];2116 dst = src;2117}2118 array[dst] = swap;2119}21202121 next:2122 idx++;2123if(count +1< window)2124 count++;2125if(idx >= window)2126 idx =0;2127}21282129for(i =0; i < window; ++i) {2130free_delta_index(array[i].index);2131free(array[i].data);2132}2133free(array);2134}21352136#ifndef NO_PTHREADS21372138static voidtry_to_free_from_threads(size_t size)2139{2140read_lock();2141release_pack_memory(size);2142read_unlock();2143}21442145static try_to_free_t old_try_to_free_routine;21462147/*2148 * The main thread waits on the condition that (at least) one of the workers2149 * has stopped working (which is indicated in the .working member of2150 * struct thread_params).2151 * When a work thread has completed its work, it sets .working to 0 and2152 * signals the main thread and waits on the condition that .data_ready2153 * becomes 1.2154 */21552156struct thread_params {2157 pthread_t thread;2158struct object_entry **list;2159unsigned list_size;2160unsigned remaining;2161int window;2162int depth;2163int working;2164int data_ready;2165 pthread_mutex_t mutex;2166 pthread_cond_t cond;2167unsigned*processed;2168};21692170static pthread_cond_t progress_cond;21712172/*2173 * Mutex and conditional variable can't be statically-initialized on Windows.2174 */2175static voidinit_threaded_search(void)2176{2177init_recursive_mutex(&read_mutex);2178pthread_mutex_init(&cache_mutex, NULL);2179pthread_mutex_init(&progress_mutex, NULL);2180pthread_cond_init(&progress_cond, NULL);2181 old_try_to_free_routine =set_try_to_free_routine(try_to_free_from_threads);2182}21832184static voidcleanup_threaded_search(void)2185{2186set_try_to_free_routine(old_try_to_free_routine);2187pthread_cond_destroy(&progress_cond);2188pthread_mutex_destroy(&read_mutex);2189pthread_mutex_destroy(&cache_mutex);2190pthread_mutex_destroy(&progress_mutex);2191}21922193static void*threaded_find_deltas(void*arg)2194{2195struct thread_params *me = arg;21962197progress_lock();2198while(me->remaining) {2199progress_unlock();22002201find_deltas(me->list, &me->remaining,2202 me->window, me->depth, me->processed);22032204progress_lock();2205 me->working =0;2206pthread_cond_signal(&progress_cond);2207progress_unlock();22082209/*2210 * We must not set ->data_ready before we wait on the2211 * condition because the main thread may have set it to 12212 * before we get here. In order to be sure that new2213 * work is available if we see 1 in ->data_ready, it2214 * was initialized to 0 before this thread was spawned2215 * and we reset it to 0 right away.2216 */2217pthread_mutex_lock(&me->mutex);2218while(!me->data_ready)2219pthread_cond_wait(&me->cond, &me->mutex);2220 me->data_ready =0;2221pthread_mutex_unlock(&me->mutex);22222223progress_lock();2224}2225progress_unlock();2226/* leave ->working 1 so that this doesn't get more work assigned */2227return NULL;2228}22292230static voidll_find_deltas(struct object_entry **list,unsigned list_size,2231int window,int depth,unsigned*processed)2232{2233struct thread_params *p;2234int i, ret, active_threads =0;22352236init_threaded_search();22372238if(delta_search_threads <=1) {2239find_deltas(list, &list_size, window, depth, processed);2240cleanup_threaded_search();2241return;2242}2243if(progress > pack_to_stdout)2244fprintf(stderr,"Delta compression using up to%dthreads.\n",2245 delta_search_threads);2246 p =xcalloc(delta_search_threads,sizeof(*p));22472248/* Partition the work amongst work threads. */2249for(i =0; i < delta_search_threads; i++) {2250unsigned sub_size = list_size / (delta_search_threads - i);22512252/* don't use too small segments or no deltas will be found */2253if(sub_size <2*window && i+1< delta_search_threads)2254 sub_size =0;22552256 p[i].window = window;2257 p[i].depth = depth;2258 p[i].processed = processed;2259 p[i].working =1;2260 p[i].data_ready =0;22612262/* try to split chunks on "path" boundaries */2263while(sub_size && sub_size < list_size &&2264 list[sub_size]->hash &&2265 list[sub_size]->hash == list[sub_size-1]->hash)2266 sub_size++;22672268 p[i].list = list;2269 p[i].list_size = sub_size;2270 p[i].remaining = sub_size;22712272 list += sub_size;2273 list_size -= sub_size;2274}22752276/* Start work threads. */2277for(i =0; i < delta_search_threads; i++) {2278if(!p[i].list_size)2279continue;2280pthread_mutex_init(&p[i].mutex, NULL);2281pthread_cond_init(&p[i].cond, NULL);2282 ret =pthread_create(&p[i].thread, NULL,2283 threaded_find_deltas, &p[i]);2284if(ret)2285die("unable to create thread:%s",strerror(ret));2286 active_threads++;2287}22882289/*2290 * Now let's wait for work completion. Each time a thread is done2291 * with its work, we steal half of the remaining work from the2292 * thread with the largest number of unprocessed objects and give2293 * it to that newly idle thread. This ensure good load balancing2294 * until the remaining object list segments are simply too short2295 * to be worth splitting anymore.2296 */2297while(active_threads) {2298struct thread_params *target = NULL;2299struct thread_params *victim = NULL;2300unsigned sub_size =0;23012302progress_lock();2303for(;;) {2304for(i =0; !target && i < delta_search_threads; i++)2305if(!p[i].working)2306 target = &p[i];2307if(target)2308break;2309pthread_cond_wait(&progress_cond, &progress_mutex);2310}23112312for(i =0; i < delta_search_threads; i++)2313if(p[i].remaining >2*window &&2314(!victim || victim->remaining < p[i].remaining))2315 victim = &p[i];2316if(victim) {2317 sub_size = victim->remaining /2;2318 list = victim->list + victim->list_size - sub_size;2319while(sub_size && list[0]->hash &&2320 list[0]->hash == list[-1]->hash) {2321 list++;2322 sub_size--;2323}2324if(!sub_size) {2325/*2326 * It is possible for some "paths" to have2327 * so many objects that no hash boundary2328 * might be found. Let's just steal the2329 * exact half in that case.2330 */2331 sub_size = victim->remaining /2;2332 list -= sub_size;2333}2334 target->list = list;2335 victim->list_size -= sub_size;2336 victim->remaining -= sub_size;2337}2338 target->list_size = sub_size;2339 target->remaining = sub_size;2340 target->working =1;2341progress_unlock();23422343pthread_mutex_lock(&target->mutex);2344 target->data_ready =1;2345pthread_cond_signal(&target->cond);2346pthread_mutex_unlock(&target->mutex);23472348if(!sub_size) {2349pthread_join(target->thread, NULL);2350pthread_cond_destroy(&target->cond);2351pthread_mutex_destroy(&target->mutex);2352 active_threads--;2353}2354}2355cleanup_threaded_search();2356free(p);2357}23582359#else2360#define ll_find_deltas(l, s, w, d, p) find_deltas(l, &s, w, d, p)2361#endif23622363static voidadd_tag_chain(const struct object_id *oid)2364{2365struct tag *tag;23662367/*2368 * We catch duplicates already in add_object_entry(), but we'd2369 * prefer to do this extra check to avoid having to parse the2370 * tag at all if we already know that it's being packed (e.g., if2371 * it was included via bitmaps, we would not have parsed it2372 * previously).2373 */2374if(packlist_find(&to_pack, oid->hash, NULL))2375return;23762377 tag =lookup_tag(oid);2378while(1) {2379if(!tag ||parse_tag(tag) || !tag->tagged)2380die("unable to pack objects reachable from tag%s",2381oid_to_hex(oid));23822383add_object_entry(&tag->object.oid, OBJ_TAG, NULL,0);23842385if(tag->tagged->type != OBJ_TAG)2386return;23872388 tag = (struct tag *)tag->tagged;2389}2390}23912392static intadd_ref_tag(const char*path,const struct object_id *oid,int flag,void*cb_data)2393{2394struct object_id peeled;23952396if(starts_with(path,"refs/tags/") &&/* is a tag? */2397!peel_ref(path, &peeled) &&/* peelable? */2398packlist_find(&to_pack, peeled.hash, NULL))/* object packed? */2399add_tag_chain(oid);2400return0;2401}24022403static voidprepare_pack(int window,int depth)2404{2405struct object_entry **delta_list;2406uint32_t i, nr_deltas;2407unsigned n;24082409get_object_details();24102411/*2412 * If we're locally repacking then we need to be doubly careful2413 * from now on in order to make sure no stealth corruption gets2414 * propagated to the new pack. Clients receiving streamed packs2415 * should validate everything they get anyway so no need to incur2416 * the additional cost here in that case.2417 */2418if(!pack_to_stdout)2419 do_check_packed_object_crc =1;24202421if(!to_pack.nr_objects || !window || !depth)2422return;24232424ALLOC_ARRAY(delta_list, to_pack.nr_objects);2425 nr_deltas = n =0;24262427for(i =0; i < to_pack.nr_objects; i++) {2428struct object_entry *entry = to_pack.objects + i;24292430if(entry->delta)2431/* This happens if we decided to reuse existing2432 * delta from a pack. "reuse_delta &&" is implied.2433 */2434continue;24352436if(entry->size <50)2437continue;24382439if(entry->no_try_delta)2440continue;24412442if(!entry->preferred_base) {2443 nr_deltas++;2444if(oe_type(entry) <0)2445die("unable to get type of object%s",2446oid_to_hex(&entry->idx.oid));2447}else{2448if(oe_type(entry) <0) {2449/*2450 * This object is not found, but we2451 * don't have to include it anyway.2452 */2453continue;2454}2455}24562457 delta_list[n++] = entry;2458}24592460if(nr_deltas && n >1) {2461unsigned nr_done =0;2462if(progress)2463 progress_state =start_progress(_("Compressing objects"),2464 nr_deltas);2465QSORT(delta_list, n, type_size_sort);2466ll_find_deltas(delta_list, n, window+1, depth, &nr_done);2467stop_progress(&progress_state);2468if(nr_done != nr_deltas)2469die("inconsistency with delta count");2470}2471free(delta_list);2472}24732474static intgit_pack_config(const char*k,const char*v,void*cb)2475{2476if(!strcmp(k,"pack.window")) {2477 window =git_config_int(k, v);2478return0;2479}2480if(!strcmp(k,"pack.windowmemory")) {2481 window_memory_limit =git_config_ulong(k, v);2482return0;2483}2484if(!strcmp(k,"pack.depth")) {2485 depth =git_config_int(k, v);2486return0;2487}2488if(!strcmp(k,"pack.deltacachesize")) {2489 max_delta_cache_size =git_config_int(k, v);2490return0;2491}2492if(!strcmp(k,"pack.deltacachelimit")) {2493 cache_max_small_delta_size =git_config_int(k, v);2494return0;2495}2496if(!strcmp(k,"pack.writebitmaphashcache")) {2497if(git_config_bool(k, v))2498 write_bitmap_options |= BITMAP_OPT_HASH_CACHE;2499else2500 write_bitmap_options &= ~BITMAP_OPT_HASH_CACHE;2501}2502if(!strcmp(k,"pack.usebitmaps")) {2503 use_bitmap_index_default =git_config_bool(k, v);2504return0;2505}2506if(!strcmp(k,"pack.threads")) {2507 delta_search_threads =git_config_int(k, v);2508if(delta_search_threads <0)2509die("invalid number of threads specified (%d)",2510 delta_search_threads);2511#ifdef NO_PTHREADS2512if(delta_search_threads !=1) {2513warning("no threads support, ignoring%s", k);2514 delta_search_threads =0;2515}2516#endif2517return0;2518}2519if(!strcmp(k,"pack.indexversion")) {2520 pack_idx_opts.version =git_config_int(k, v);2521if(pack_idx_opts.version >2)2522die("bad pack.indexversion=%"PRIu32,2523 pack_idx_opts.version);2524return0;2525}2526returngit_default_config(k, v, cb);2527}25282529static voidread_object_list_from_stdin(void)2530{2531char line[GIT_MAX_HEXSZ +1+ PATH_MAX +2];2532struct object_id oid;2533const char*p;25342535for(;;) {2536if(!fgets(line,sizeof(line), stdin)) {2537if(feof(stdin))2538break;2539if(!ferror(stdin))2540die("fgets returned NULL, not EOF, not error!");2541if(errno != EINTR)2542die_errno("fgets");2543clearerr(stdin);2544continue;2545}2546if(line[0] =='-') {2547if(get_oid_hex(line+1, &oid))2548die("expected edge object ID, got garbage:\n%s",2549 line);2550add_preferred_base(&oid);2551continue;2552}2553if(parse_oid_hex(line, &oid, &p))2554die("expected object ID, got garbage:\n%s", line);25552556add_preferred_base_object(p +1);2557add_object_entry(&oid, OBJ_NONE, p +1,0);2558}2559}25602561/* Remember to update object flag allocation in object.h */2562#define OBJECT_ADDED (1u<<20)25632564static voidshow_commit(struct commit *commit,void*data)2565{2566add_object_entry(&commit->object.oid, OBJ_COMMIT, NULL,0);2567 commit->object.flags |= OBJECT_ADDED;25682569if(write_bitmap_index)2570index_commit_for_bitmap(commit);2571}25722573static voidshow_object(struct object *obj,const char*name,void*data)2574{2575add_preferred_base_object(name);2576add_object_entry(&obj->oid, obj->type, name,0);2577 obj->flags |= OBJECT_ADDED;2578}25792580static voidshow_object__ma_allow_any(struct object *obj,const char*name,void*data)2581{2582assert(arg_missing_action == MA_ALLOW_ANY);25832584/*2585 * Quietly ignore ALL missing objects. This avoids problems with2586 * staging them now and getting an odd error later.2587 */2588if(!has_object_file(&obj->oid))2589return;25902591show_object(obj, name, data);2592}25932594static voidshow_object__ma_allow_promisor(struct object *obj,const char*name,void*data)2595{2596assert(arg_missing_action == MA_ALLOW_PROMISOR);25972598/*2599 * Quietly ignore EXPECTED missing objects. This avoids problems with2600 * staging them now and getting an odd error later.2601 */2602if(!has_object_file(&obj->oid) &&is_promisor_object(&obj->oid))2603return;26042605show_object(obj, name, data);2606}26072608static intoption_parse_missing_action(const struct option *opt,2609const char*arg,int unset)2610{2611assert(arg);2612assert(!unset);26132614if(!strcmp(arg,"error")) {2615 arg_missing_action = MA_ERROR;2616 fn_show_object = show_object;2617return0;2618}26192620if(!strcmp(arg,"allow-any")) {2621 arg_missing_action = MA_ALLOW_ANY;2622 fetch_if_missing =0;2623 fn_show_object = show_object__ma_allow_any;2624return0;2625}26262627if(!strcmp(arg,"allow-promisor")) {2628 arg_missing_action = MA_ALLOW_PROMISOR;2629 fetch_if_missing =0;2630 fn_show_object = show_object__ma_allow_promisor;2631return0;2632}26332634die(_("invalid value for --missing"));2635return0;2636}26372638static voidshow_edge(struct commit *commit)2639{2640add_preferred_base(&commit->object.oid);2641}26422643struct in_pack_object {2644 off_t offset;2645struct object *object;2646};26472648struct in_pack {2649unsigned int alloc;2650unsigned int nr;2651struct in_pack_object *array;2652};26532654static voidmark_in_pack_object(struct object *object,struct packed_git *p,struct in_pack *in_pack)2655{2656 in_pack->array[in_pack->nr].offset =find_pack_entry_one(object->oid.hash, p);2657 in_pack->array[in_pack->nr].object = object;2658 in_pack->nr++;2659}26602661/*2662 * Compare the objects in the offset order, in order to emulate the2663 * "git rev-list --objects" output that produced the pack originally.2664 */2665static intofscmp(const void*a_,const void*b_)2666{2667struct in_pack_object *a = (struct in_pack_object *)a_;2668struct in_pack_object *b = (struct in_pack_object *)b_;26692670if(a->offset < b->offset)2671return-1;2672else if(a->offset > b->offset)2673return1;2674else2675returnoidcmp(&a->object->oid, &b->object->oid);2676}26772678static voidadd_objects_in_unpacked_packs(struct rev_info *revs)2679{2680struct packed_git *p;2681struct in_pack in_pack;2682uint32_t i;26832684memset(&in_pack,0,sizeof(in_pack));26852686for(p =get_packed_git(the_repository); p; p = p->next) {2687struct object_id oid;2688struct object *o;26892690if(!p->pack_local || p->pack_keep)2691continue;2692if(open_pack_index(p))2693die("cannot open pack index");26942695ALLOC_GROW(in_pack.array,2696 in_pack.nr + p->num_objects,2697 in_pack.alloc);26982699for(i =0; i < p->num_objects; i++) {2700nth_packed_object_oid(&oid, p, i);2701 o =lookup_unknown_object(oid.hash);2702if(!(o->flags & OBJECT_ADDED))2703mark_in_pack_object(o, p, &in_pack);2704 o->flags |= OBJECT_ADDED;2705}2706}27072708if(in_pack.nr) {2709QSORT(in_pack.array, in_pack.nr, ofscmp);2710for(i =0; i < in_pack.nr; i++) {2711struct object *o = in_pack.array[i].object;2712add_object_entry(&o->oid, o->type,"",0);2713}2714}2715free(in_pack.array);2716}27172718static intadd_loose_object(const struct object_id *oid,const char*path,2719void*data)2720{2721enum object_type type =oid_object_info(oid, NULL);27222723if(type <0) {2724warning("loose object at%scould not be examined", path);2725return0;2726}27272728add_object_entry(oid, type,"",0);2729return0;2730}27312732/*2733 * We actually don't even have to worry about reachability here.2734 * add_object_entry will weed out duplicates, so we just add every2735 * loose object we find.2736 */2737static voidadd_unreachable_loose_objects(void)2738{2739for_each_loose_file_in_objdir(get_object_directory(),2740 add_loose_object,2741 NULL, NULL, NULL);2742}27432744static inthas_sha1_pack_kept_or_nonlocal(const struct object_id *oid)2745{2746static struct packed_git *last_found = (void*)1;2747struct packed_git *p;27482749 p = (last_found != (void*)1) ? last_found :2750get_packed_git(the_repository);27512752while(p) {2753if((!p->pack_local || p->pack_keep) &&2754find_pack_entry_one(oid->hash, p)) {2755 last_found = p;2756return1;2757}2758if(p == last_found)2759 p =get_packed_git(the_repository);2760else2761 p = p->next;2762if(p == last_found)2763 p = p->next;2764}2765return0;2766}27672768/*2769 * Store a list of sha1s that are should not be discarded2770 * because they are either written too recently, or are2771 * reachable from another object that was.2772 *2773 * This is filled by get_object_list.2774 */2775static struct oid_array recent_objects;27762777static intloosened_object_can_be_discarded(const struct object_id *oid,2778 timestamp_t mtime)2779{2780if(!unpack_unreachable_expiration)2781return0;2782if(mtime > unpack_unreachable_expiration)2783return0;2784if(oid_array_lookup(&recent_objects, oid) >=0)2785return0;2786return1;2787}27882789static voidloosen_unused_packed_objects(struct rev_info *revs)2790{2791struct packed_git *p;2792uint32_t i;2793struct object_id oid;27942795for(p =get_packed_git(the_repository); p; p = p->next) {2796if(!p->pack_local || p->pack_keep)2797continue;27982799if(open_pack_index(p))2800die("cannot open pack index");28012802for(i =0; i < p->num_objects; i++) {2803nth_packed_object_oid(&oid, p, i);2804if(!packlist_find(&to_pack, oid.hash, NULL) &&2805!has_sha1_pack_kept_or_nonlocal(&oid) &&2806!loosened_object_can_be_discarded(&oid, p->mtime))2807if(force_object_loose(&oid, p->mtime))2808die("unable to force loose object");2809}2810}2811}28122813/*2814 * This tracks any options which pack-reuse code expects to be on, or which a2815 * reader of the pack might not understand, and which would therefore prevent2816 * blind reuse of what we have on disk.2817 */2818static intpack_options_allow_reuse(void)2819{2820return pack_to_stdout &&2821 allow_ofs_delta &&2822!ignore_packed_keep &&2823(!local || !have_non_local_packs) &&2824!incremental;2825}28262827static intget_object_list_from_bitmap(struct rev_info *revs)2828{2829if(prepare_bitmap_walk(revs) <0)2830return-1;28312832if(pack_options_allow_reuse() &&2833!reuse_partial_packfile_from_bitmap(2834&reuse_packfile,2835&reuse_packfile_objects,2836&reuse_packfile_offset)) {2837assert(reuse_packfile_objects);2838 nr_result += reuse_packfile_objects;2839display_progress(progress_state, nr_result);2840}28412842traverse_bitmap_commit_list(&add_object_entry_from_bitmap);2843return0;2844}28452846static voidrecord_recent_object(struct object *obj,2847const char*name,2848void*data)2849{2850oid_array_append(&recent_objects, &obj->oid);2851}28522853static voidrecord_recent_commit(struct commit *commit,void*data)2854{2855oid_array_append(&recent_objects, &commit->object.oid);2856}28572858static voidget_object_list(int ac,const char**av)2859{2860struct rev_info revs;2861char line[1000];2862int flags =0;28632864init_revisions(&revs, NULL);2865 save_commit_buffer =0;2866setup_revisions(ac, av, &revs, NULL);28672868/* make sure shallows are read */2869is_repository_shallow();28702871while(fgets(line,sizeof(line), stdin) != NULL) {2872int len =strlen(line);2873if(len && line[len -1] =='\n')2874 line[--len] =0;2875if(!len)2876break;2877if(*line =='-') {2878if(!strcmp(line,"--not")) {2879 flags ^= UNINTERESTING;2880 write_bitmap_index =0;2881continue;2882}2883if(starts_with(line,"--shallow ")) {2884struct object_id oid;2885if(get_oid_hex(line +10, &oid))2886die("not an SHA-1 '%s'", line +10);2887register_shallow(&oid);2888 use_bitmap_index =0;2889continue;2890}2891die("not a rev '%s'", line);2892}2893if(handle_revision_arg(line, &revs, flags, REVARG_CANNOT_BE_FILENAME))2894die("bad revision '%s'", line);2895}28962897if(use_bitmap_index && !get_object_list_from_bitmap(&revs))2898return;28992900if(prepare_revision_walk(&revs))2901die("revision walk setup failed");2902mark_edges_uninteresting(&revs, show_edge);29032904if(!fn_show_object)2905 fn_show_object = show_object;2906traverse_commit_list_filtered(&filter_options, &revs,2907 show_commit, fn_show_object, NULL,2908 NULL);29092910if(unpack_unreachable_expiration) {2911 revs.ignore_missing_links =1;2912if(add_unseen_recent_objects_to_traversal(&revs,2913 unpack_unreachable_expiration))2914die("unable to add recent objects");2915if(prepare_revision_walk(&revs))2916die("revision walk setup failed");2917traverse_commit_list(&revs, record_recent_commit,2918 record_recent_object, NULL);2919}29202921if(keep_unreachable)2922add_objects_in_unpacked_packs(&revs);2923if(pack_loose_unreachable)2924add_unreachable_loose_objects();2925if(unpack_unreachable)2926loosen_unused_packed_objects(&revs);29272928oid_array_clear(&recent_objects);2929}29302931static intoption_parse_index_version(const struct option *opt,2932const char*arg,int unset)2933{2934char*c;2935const char*val = arg;2936 pack_idx_opts.version =strtoul(val, &c,10);2937if(pack_idx_opts.version >2)2938die(_("unsupported index version%s"), val);2939if(*c ==','&& c[1])2940 pack_idx_opts.off32_limit =strtoul(c+1, &c,0);2941if(*c || pack_idx_opts.off32_limit &0x80000000)2942die(_("bad index version '%s'"), val);2943return0;2944}29452946static intoption_parse_unpack_unreachable(const struct option *opt,2947const char*arg,int unset)2948{2949if(unset) {2950 unpack_unreachable =0;2951 unpack_unreachable_expiration =0;2952}2953else{2954 unpack_unreachable =1;2955if(arg)2956 unpack_unreachable_expiration =approxidate(arg);2957}2958return0;2959}29602961intcmd_pack_objects(int argc,const char**argv,const char*prefix)2962{2963int use_internal_rev_list =0;2964int thin =0;2965int shallow =0;2966int all_progress_implied =0;2967struct argv_array rp = ARGV_ARRAY_INIT;2968int rev_list_unpacked =0, rev_list_all =0, rev_list_reflog =0;2969int rev_list_index =0;2970struct option pack_objects_options[] = {2971OPT_SET_INT('q',"quiet", &progress,2972N_("do not show progress meter"),0),2973OPT_SET_INT(0,"progress", &progress,2974N_("show progress meter"),1),2975OPT_SET_INT(0,"all-progress", &progress,2976N_("show progress meter during object writing phase"),2),2977OPT_BOOL(0,"all-progress-implied",2978&all_progress_implied,2979N_("similar to --all-progress when progress meter is shown")),2980{ OPTION_CALLBACK,0,"index-version", NULL,N_("version[,offset]"),2981N_("write the pack index file in the specified idx format version"),29820, option_parse_index_version },2983OPT_MAGNITUDE(0,"max-pack-size", &pack_size_limit,2984N_("maximum size of each output pack file")),2985OPT_BOOL(0,"local", &local,2986N_("ignore borrowed objects from alternate object store")),2987OPT_BOOL(0,"incremental", &incremental,2988N_("ignore packed objects")),2989OPT_INTEGER(0,"window", &window,2990N_("limit pack window by objects")),2991OPT_MAGNITUDE(0,"window-memory", &window_memory_limit,2992N_("limit pack window by memory in addition to object limit")),2993OPT_INTEGER(0,"depth", &depth,2994N_("maximum length of delta chain allowed in the resulting pack")),2995OPT_BOOL(0,"reuse-delta", &reuse_delta,2996N_("reuse existing deltas")),2997OPT_BOOL(0,"reuse-object", &reuse_object,2998N_("reuse existing objects")),2999OPT_BOOL(0,"delta-base-offset", &allow_ofs_delta,3000N_("use OFS_DELTA objects")),3001OPT_INTEGER(0,"threads", &delta_search_threads,3002N_("use threads when searching for best delta matches")),3003OPT_BOOL(0,"non-empty", &non_empty,3004N_("do not create an empty pack output")),3005OPT_BOOL(0,"revs", &use_internal_rev_list,3006N_("read revision arguments from standard input")),3007{ OPTION_SET_INT,0,"unpacked", &rev_list_unpacked, NULL,3008N_("limit the objects to those that are not yet packed"),3009 PARSE_OPT_NOARG | PARSE_OPT_NONEG, NULL,1},3010{ OPTION_SET_INT,0,"all", &rev_list_all, NULL,3011N_("include objects reachable from any reference"),3012 PARSE_OPT_NOARG | PARSE_OPT_NONEG, NULL,1},3013{ OPTION_SET_INT,0,"reflog", &rev_list_reflog, NULL,3014N_("include objects referred by reflog entries"),3015 PARSE_OPT_NOARG | PARSE_OPT_NONEG, NULL,1},3016{ OPTION_SET_INT,0,"indexed-objects", &rev_list_index, NULL,3017N_("include objects referred to by the index"),3018 PARSE_OPT_NOARG | PARSE_OPT_NONEG, NULL,1},3019OPT_BOOL(0,"stdout", &pack_to_stdout,3020N_("output pack to stdout")),3021OPT_BOOL(0,"include-tag", &include_tag,3022N_("include tag objects that refer to objects to be packed")),3023OPT_BOOL(0,"keep-unreachable", &keep_unreachable,3024N_("keep unreachable objects")),3025OPT_BOOL(0,"pack-loose-unreachable", &pack_loose_unreachable,3026N_("pack loose unreachable objects")),3027{ OPTION_CALLBACK,0,"unpack-unreachable", NULL,N_("time"),3028N_("unpack unreachable objects newer than <time>"),3029 PARSE_OPT_OPTARG, option_parse_unpack_unreachable },3030OPT_BOOL(0,"thin", &thin,3031N_("create thin packs")),3032OPT_BOOL(0,"shallow", &shallow,3033N_("create packs suitable for shallow fetches")),3034OPT_BOOL(0,"honor-pack-keep", &ignore_packed_keep,3035N_("ignore packs that have companion .keep file")),3036OPT_INTEGER(0,"compression", &pack_compression_level,3037N_("pack compression level")),3038OPT_SET_INT(0,"keep-true-parents", &grafts_replace_parents,3039N_("do not hide commits by grafts"),0),3040OPT_BOOL(0,"use-bitmap-index", &use_bitmap_index,3041N_("use a bitmap index if available to speed up counting objects")),3042OPT_BOOL(0,"write-bitmap-index", &write_bitmap_index,3043N_("write a bitmap index together with the pack index")),3044OPT_PARSE_LIST_OBJECTS_FILTER(&filter_options),3045{ OPTION_CALLBACK,0,"missing", NULL,N_("action"),3046N_("handling for missing objects"), PARSE_OPT_NONEG,3047 option_parse_missing_action },3048OPT_BOOL(0,"exclude-promisor-objects", &exclude_promisor_objects,3049N_("do not pack objects in promisor packfiles")),3050OPT_END(),3051};30523053if(DFS_NUM_STATES > (1<< OE_DFS_STATE_BITS))3054BUG("too many dfs states, increase OE_DFS_STATE_BITS");30553056 check_replace_refs =0;30573058reset_pack_idx_option(&pack_idx_opts);3059git_config(git_pack_config, NULL);30603061 progress =isatty(2);3062 argc =parse_options(argc, argv, prefix, pack_objects_options,3063 pack_usage,0);30643065if(argc) {3066 base_name = argv[0];3067 argc--;3068}3069if(pack_to_stdout != !base_name || argc)3070usage_with_options(pack_usage, pack_objects_options);30713072if(depth >= (1<< OE_DEPTH_BITS)) {3073warning(_("delta chain depth%dis too deep, forcing%d"),3074 depth, (1<< OE_DEPTH_BITS) -1);3075 depth = (1<< OE_DEPTH_BITS) -1;3076}30773078argv_array_push(&rp,"pack-objects");3079if(thin) {3080 use_internal_rev_list =1;3081argv_array_push(&rp, shallow3082?"--objects-edge-aggressive"3083:"--objects-edge");3084}else3085argv_array_push(&rp,"--objects");30863087if(rev_list_all) {3088 use_internal_rev_list =1;3089argv_array_push(&rp,"--all");3090}3091if(rev_list_reflog) {3092 use_internal_rev_list =1;3093argv_array_push(&rp,"--reflog");3094}3095if(rev_list_index) {3096 use_internal_rev_list =1;3097argv_array_push(&rp,"--indexed-objects");3098}3099if(rev_list_unpacked) {3100 use_internal_rev_list =1;3101argv_array_push(&rp,"--unpacked");3102}31033104if(exclude_promisor_objects) {3105 use_internal_rev_list =1;3106 fetch_if_missing =0;3107argv_array_push(&rp,"--exclude-promisor-objects");3108}31093110if(!reuse_object)3111 reuse_delta =0;3112if(pack_compression_level == -1)3113 pack_compression_level = Z_DEFAULT_COMPRESSION;3114else if(pack_compression_level <0|| pack_compression_level > Z_BEST_COMPRESSION)3115die("bad pack compression level%d", pack_compression_level);31163117if(!delta_search_threads)/* --threads=0 means autodetect */3118 delta_search_threads =online_cpus();31193120#ifdef NO_PTHREADS3121if(delta_search_threads !=1)3122warning("no threads support, ignoring --threads");3123#endif3124if(!pack_to_stdout && !pack_size_limit)3125 pack_size_limit = pack_size_limit_cfg;3126if(pack_to_stdout && pack_size_limit)3127die("--max-pack-size cannot be used to build a pack for transfer.");3128if(pack_size_limit && pack_size_limit <1024*1024) {3129warning("minimum pack size limit is 1 MiB");3130 pack_size_limit =1024*1024;3131}31323133if(!pack_to_stdout && thin)3134die("--thin cannot be used to build an indexable pack.");31353136if(keep_unreachable && unpack_unreachable)3137die("--keep-unreachable and --unpack-unreachable are incompatible.");3138if(!rev_list_all || !rev_list_reflog || !rev_list_index)3139 unpack_unreachable_expiration =0;31403141if(filter_options.choice) {3142if(!pack_to_stdout)3143die("cannot use --filter without --stdout.");3144 use_bitmap_index =0;3145}31463147/*3148 * "soft" reasons not to use bitmaps - for on-disk repack by default we want3149 *3150 * - to produce good pack (with bitmap index not-yet-packed objects are3151 * packed in suboptimal order).3152 *3153 * - to use more robust pack-generation codepath (avoiding possible3154 * bugs in bitmap code and possible bitmap index corruption).3155 */3156if(!pack_to_stdout)3157 use_bitmap_index_default =0;31583159if(use_bitmap_index <0)3160 use_bitmap_index = use_bitmap_index_default;31613162/* "hard" reasons not to use bitmaps; these just won't work at all */3163if(!use_internal_rev_list || (!pack_to_stdout && write_bitmap_index) ||is_repository_shallow())3164 use_bitmap_index =0;31653166if(pack_to_stdout || !rev_list_all)3167 write_bitmap_index =0;31683169if(progress && all_progress_implied)3170 progress =2;31713172if(ignore_packed_keep) {3173struct packed_git *p;3174for(p =get_packed_git(the_repository); p; p = p->next)3175if(p->pack_local && p->pack_keep)3176break;3177if(!p)/* no keep-able packs found */3178 ignore_packed_keep =0;3179}3180if(local) {3181/*3182 * unlike ignore_packed_keep above, we do not want to3183 * unset "local" based on looking at packs, as it3184 * also covers non-local objects3185 */3186struct packed_git *p;3187for(p =get_packed_git(the_repository); p; p = p->next) {3188if(!p->pack_local) {3189 have_non_local_packs =1;3190break;3191}3192}3193}31943195if(progress)3196 progress_state =start_progress(_("Counting objects"),0);3197if(!use_internal_rev_list)3198read_object_list_from_stdin();3199else{3200get_object_list(rp.argc, rp.argv);3201argv_array_clear(&rp);3202}3203cleanup_preferred_base();3204if(include_tag && nr_result)3205for_each_ref(add_ref_tag, NULL);3206stop_progress(&progress_state);32073208if(non_empty && !nr_result)3209return0;3210if(nr_result)3211prepare_pack(window, depth);3212write_pack_file();3213if(progress)3214fprintf(stderr,"Total %"PRIu32" (delta %"PRIu32"),"3215" reused %"PRIu32" (delta %"PRIu32")\n",3216 written, written_delta, reused, reused_delta);3217return0;3218}