1#include"builtin.h" 2#include"cache.h" 3#include"repository.h" 4#include"config.h" 5#include"attr.h" 6#include"object.h" 7#include"blob.h" 8#include"commit.h" 9#include"tag.h" 10#include"tree.h" 11#include"delta.h" 12#include"pack.h" 13#include"pack-revindex.h" 14#include"csum-file.h" 15#include"tree-walk.h" 16#include"diff.h" 17#include"revision.h" 18#include"list-objects.h" 19#include"list-objects-filter.h" 20#include"list-objects-filter-options.h" 21#include"pack-objects.h" 22#include"progress.h" 23#include"refs.h" 24#include"streaming.h" 25#include"thread-utils.h" 26#include"pack-bitmap.h" 27#include"reachable.h" 28#include"sha1-array.h" 29#include"argv-array.h" 30#include"list.h" 31#include"packfile.h" 32#include"object-store.h" 33 34#define IN_PACK(obj) oe_in_pack(&to_pack, obj) 35 36static const char*pack_usage[] = { 37N_("git pack-objects --stdout [<options>...] [< <ref-list> | < <object-list>]"), 38N_("git pack-objects [<options>...] <base-name> [< <ref-list> | < <object-list>]"), 39 NULL 40}; 41 42/* 43 * Objects we are going to pack are collected in the `to_pack` structure. 44 * It contains an array (dynamically expanded) of the object data, and a map 45 * that can resolve SHA1s to their position in the array. 46 */ 47static struct packing_data to_pack; 48 49static struct pack_idx_entry **written_list; 50static uint32_t nr_result, nr_written; 51 52static int non_empty; 53static int reuse_delta =1, reuse_object =1; 54static int keep_unreachable, unpack_unreachable, include_tag; 55static timestamp_t unpack_unreachable_expiration; 56static int pack_loose_unreachable; 57static int local; 58static int have_non_local_packs; 59static int incremental; 60static int ignore_packed_keep; 61static int allow_ofs_delta; 62static struct pack_idx_option pack_idx_opts; 63static const char*base_name; 64static int progress =1; 65static int window =10; 66static unsigned long pack_size_limit; 67static int depth =50; 68static int delta_search_threads; 69static int pack_to_stdout; 70static int num_preferred_base; 71static struct progress *progress_state; 72 73static struct packed_git *reuse_packfile; 74static uint32_t reuse_packfile_objects; 75static off_t reuse_packfile_offset; 76 77static int use_bitmap_index_default =1; 78static int use_bitmap_index = -1; 79static int write_bitmap_index; 80static uint16_t write_bitmap_options; 81 82static int exclude_promisor_objects; 83 84static unsigned long delta_cache_size =0; 85static unsigned long max_delta_cache_size =256*1024*1024; 86static unsigned long cache_max_small_delta_size =1000; 87 88static unsigned long window_memory_limit =0; 89 90static struct list_objects_filter_options filter_options; 91 92enum missing_action { 93 MA_ERROR =0,/* fail if any missing objects are encountered */ 94 MA_ALLOW_ANY,/* silently allow ALL missing objects */ 95 MA_ALLOW_PROMISOR,/* silently allow all missing PROMISOR objects */ 96}; 97static enum missing_action arg_missing_action; 98static show_object_fn fn_show_object; 99 100/* 101 * stats 102 */ 103static uint32_t written, written_delta; 104static uint32_t reused, reused_delta; 105 106/* 107 * Indexed commits 108 */ 109static struct commit **indexed_commits; 110static unsigned int indexed_commits_nr; 111static unsigned int indexed_commits_alloc; 112 113static voidindex_commit_for_bitmap(struct commit *commit) 114{ 115if(indexed_commits_nr >= indexed_commits_alloc) { 116 indexed_commits_alloc = (indexed_commits_alloc +32) *2; 117REALLOC_ARRAY(indexed_commits, indexed_commits_alloc); 118} 119 120 indexed_commits[indexed_commits_nr++] = commit; 121} 122 123static void*get_delta(struct object_entry *entry) 124{ 125unsigned long size, base_size, delta_size; 126void*buf, *base_buf, *delta_buf; 127enum object_type type; 128 129 buf =read_object_file(&entry->idx.oid, &type, &size); 130if(!buf) 131die("unable to read%s",oid_to_hex(&entry->idx.oid)); 132 base_buf =read_object_file(&entry->delta->idx.oid, &type, &base_size); 133if(!base_buf) 134die("unable to read%s", 135oid_to_hex(&entry->delta->idx.oid)); 136 delta_buf =diff_delta(base_buf, base_size, 137 buf, size, &delta_size,0); 138if(!delta_buf || delta_size != entry->delta_size) 139die("delta size changed"); 140free(buf); 141free(base_buf); 142return delta_buf; 143} 144 145static unsigned longdo_compress(void**pptr,unsigned long size) 146{ 147 git_zstream stream; 148void*in, *out; 149unsigned long maxsize; 150 151git_deflate_init(&stream, pack_compression_level); 152 maxsize =git_deflate_bound(&stream, size); 153 154 in = *pptr; 155 out =xmalloc(maxsize); 156*pptr = out; 157 158 stream.next_in = in; 159 stream.avail_in = size; 160 stream.next_out = out; 161 stream.avail_out = maxsize; 162while(git_deflate(&stream, Z_FINISH) == Z_OK) 163;/* nothing */ 164git_deflate_end(&stream); 165 166free(in); 167return stream.total_out; 168} 169 170static unsigned longwrite_large_blob_data(struct git_istream *st,struct hashfile *f, 171const struct object_id *oid) 172{ 173 git_zstream stream; 174unsigned char ibuf[1024*16]; 175unsigned char obuf[1024*16]; 176unsigned long olen =0; 177 178git_deflate_init(&stream, pack_compression_level); 179 180for(;;) { 181 ssize_t readlen; 182int zret = Z_OK; 183 readlen =read_istream(st, ibuf,sizeof(ibuf)); 184if(readlen == -1) 185die(_("unable to read%s"),oid_to_hex(oid)); 186 187 stream.next_in = ibuf; 188 stream.avail_in = readlen; 189while((stream.avail_in || readlen ==0) && 190(zret == Z_OK || zret == Z_BUF_ERROR)) { 191 stream.next_out = obuf; 192 stream.avail_out =sizeof(obuf); 193 zret =git_deflate(&stream, readlen ?0: Z_FINISH); 194hashwrite(f, obuf, stream.next_out - obuf); 195 olen += stream.next_out - obuf; 196} 197if(stream.avail_in) 198die(_("deflate error (%d)"), zret); 199if(readlen ==0) { 200if(zret != Z_STREAM_END) 201die(_("deflate error (%d)"), zret); 202break; 203} 204} 205git_deflate_end(&stream); 206return olen; 207} 208 209/* 210 * we are going to reuse the existing object data as is. make 211 * sure it is not corrupt. 212 */ 213static intcheck_pack_inflate(struct packed_git *p, 214struct pack_window **w_curs, 215 off_t offset, 216 off_t len, 217unsigned long expect) 218{ 219 git_zstream stream; 220unsigned char fakebuf[4096], *in; 221int st; 222 223memset(&stream,0,sizeof(stream)); 224git_inflate_init(&stream); 225do{ 226 in =use_pack(p, w_curs, offset, &stream.avail_in); 227 stream.next_in = in; 228 stream.next_out = fakebuf; 229 stream.avail_out =sizeof(fakebuf); 230 st =git_inflate(&stream, Z_FINISH); 231 offset += stream.next_in - in; 232}while(st == Z_OK || st == Z_BUF_ERROR); 233git_inflate_end(&stream); 234return(st == Z_STREAM_END && 235 stream.total_out == expect && 236 stream.total_in == len) ?0: -1; 237} 238 239static voidcopy_pack_data(struct hashfile *f, 240struct packed_git *p, 241struct pack_window **w_curs, 242 off_t offset, 243 off_t len) 244{ 245unsigned char*in; 246unsigned long avail; 247 248while(len) { 249 in =use_pack(p, w_curs, offset, &avail); 250if(avail > len) 251 avail = (unsigned long)len; 252hashwrite(f, in, avail); 253 offset += avail; 254 len -= avail; 255} 256} 257 258/* Return 0 if we will bust the pack-size limit */ 259static unsigned longwrite_no_reuse_object(struct hashfile *f,struct object_entry *entry, 260unsigned long limit,int usable_delta) 261{ 262unsigned long size, datalen; 263unsigned char header[MAX_PACK_OBJECT_HEADER], 264 dheader[MAX_PACK_OBJECT_HEADER]; 265unsigned hdrlen; 266enum object_type type; 267void*buf; 268struct git_istream *st = NULL; 269 270if(!usable_delta) { 271if(oe_type(entry) == OBJ_BLOB && 272 entry->size > big_file_threshold && 273(st =open_istream(&entry->idx.oid, &type, &size, NULL)) != NULL) 274 buf = NULL; 275else{ 276 buf =read_object_file(&entry->idx.oid, &type, &size); 277if(!buf) 278die(_("unable to read%s"), 279oid_to_hex(&entry->idx.oid)); 280} 281/* 282 * make sure no cached delta data remains from a 283 * previous attempt before a pack split occurred. 284 */ 285FREE_AND_NULL(entry->delta_data); 286 entry->z_delta_size =0; 287}else if(entry->delta_data) { 288 size = entry->delta_size; 289 buf = entry->delta_data; 290 entry->delta_data = NULL; 291 type = (allow_ofs_delta && entry->delta->idx.offset) ? 292 OBJ_OFS_DELTA : OBJ_REF_DELTA; 293}else{ 294 buf =get_delta(entry); 295 size = entry->delta_size; 296 type = (allow_ofs_delta && entry->delta->idx.offset) ? 297 OBJ_OFS_DELTA : OBJ_REF_DELTA; 298} 299 300if(st)/* large blob case, just assume we don't compress well */ 301 datalen = size; 302else if(entry->z_delta_size) 303 datalen = entry->z_delta_size; 304else 305 datalen =do_compress(&buf, size); 306 307/* 308 * The object header is a byte of 'type' followed by zero or 309 * more bytes of length. 310 */ 311 hdrlen =encode_in_pack_object_header(header,sizeof(header), 312 type, size); 313 314if(type == OBJ_OFS_DELTA) { 315/* 316 * Deltas with relative base contain an additional 317 * encoding of the relative offset for the delta 318 * base from this object's position in the pack. 319 */ 320 off_t ofs = entry->idx.offset - entry->delta->idx.offset; 321unsigned pos =sizeof(dheader) -1; 322 dheader[pos] = ofs &127; 323while(ofs >>=7) 324 dheader[--pos] =128| (--ofs &127); 325if(limit && hdrlen +sizeof(dheader) - pos + datalen +20>= limit) { 326if(st) 327close_istream(st); 328free(buf); 329return0; 330} 331hashwrite(f, header, hdrlen); 332hashwrite(f, dheader + pos,sizeof(dheader) - pos); 333 hdrlen +=sizeof(dheader) - pos; 334}else if(type == OBJ_REF_DELTA) { 335/* 336 * Deltas with a base reference contain 337 * an additional 20 bytes for the base sha1. 338 */ 339if(limit && hdrlen +20+ datalen +20>= limit) { 340if(st) 341close_istream(st); 342free(buf); 343return0; 344} 345hashwrite(f, header, hdrlen); 346hashwrite(f, entry->delta->idx.oid.hash,20); 347 hdrlen +=20; 348}else{ 349if(limit && hdrlen + datalen +20>= limit) { 350if(st) 351close_istream(st); 352free(buf); 353return0; 354} 355hashwrite(f, header, hdrlen); 356} 357if(st) { 358 datalen =write_large_blob_data(st, f, &entry->idx.oid); 359close_istream(st); 360}else{ 361hashwrite(f, buf, datalen); 362free(buf); 363} 364 365return hdrlen + datalen; 366} 367 368/* Return 0 if we will bust the pack-size limit */ 369static off_t write_reuse_object(struct hashfile *f,struct object_entry *entry, 370unsigned long limit,int usable_delta) 371{ 372struct packed_git *p =IN_PACK(entry); 373struct pack_window *w_curs = NULL; 374struct revindex_entry *revidx; 375 off_t offset; 376enum object_type type =oe_type(entry); 377 off_t datalen; 378unsigned char header[MAX_PACK_OBJECT_HEADER], 379 dheader[MAX_PACK_OBJECT_HEADER]; 380unsigned hdrlen; 381 382if(entry->delta) 383 type = (allow_ofs_delta && entry->delta->idx.offset) ? 384 OBJ_OFS_DELTA : OBJ_REF_DELTA; 385 hdrlen =encode_in_pack_object_header(header,sizeof(header), 386 type, entry->size); 387 388 offset = entry->in_pack_offset; 389 revidx =find_pack_revindex(p, offset); 390 datalen = revidx[1].offset - offset; 391if(!pack_to_stdout && p->index_version >1&& 392check_pack_crc(p, &w_curs, offset, datalen, revidx->nr)) { 393error("bad packed object CRC for%s", 394oid_to_hex(&entry->idx.oid)); 395unuse_pack(&w_curs); 396returnwrite_no_reuse_object(f, entry, limit, usable_delta); 397} 398 399 offset += entry->in_pack_header_size; 400 datalen -= entry->in_pack_header_size; 401 402if(!pack_to_stdout && p->index_version ==1&& 403check_pack_inflate(p, &w_curs, offset, datalen, entry->size)) { 404error("corrupt packed object for%s", 405oid_to_hex(&entry->idx.oid)); 406unuse_pack(&w_curs); 407returnwrite_no_reuse_object(f, entry, limit, usable_delta); 408} 409 410if(type == OBJ_OFS_DELTA) { 411 off_t ofs = entry->idx.offset - entry->delta->idx.offset; 412unsigned pos =sizeof(dheader) -1; 413 dheader[pos] = ofs &127; 414while(ofs >>=7) 415 dheader[--pos] =128| (--ofs &127); 416if(limit && hdrlen +sizeof(dheader) - pos + datalen +20>= limit) { 417unuse_pack(&w_curs); 418return0; 419} 420hashwrite(f, header, hdrlen); 421hashwrite(f, dheader + pos,sizeof(dheader) - pos); 422 hdrlen +=sizeof(dheader) - pos; 423 reused_delta++; 424}else if(type == OBJ_REF_DELTA) { 425if(limit && hdrlen +20+ datalen +20>= limit) { 426unuse_pack(&w_curs); 427return0; 428} 429hashwrite(f, header, hdrlen); 430hashwrite(f, entry->delta->idx.oid.hash,20); 431 hdrlen +=20; 432 reused_delta++; 433}else{ 434if(limit && hdrlen + datalen +20>= limit) { 435unuse_pack(&w_curs); 436return0; 437} 438hashwrite(f, header, hdrlen); 439} 440copy_pack_data(f, p, &w_curs, offset, datalen); 441unuse_pack(&w_curs); 442 reused++; 443return hdrlen + datalen; 444} 445 446/* Return 0 if we will bust the pack-size limit */ 447static off_t write_object(struct hashfile *f, 448struct object_entry *entry, 449 off_t write_offset) 450{ 451unsigned long limit; 452 off_t len; 453int usable_delta, to_reuse; 454 455if(!pack_to_stdout) 456crc32_begin(f); 457 458/* apply size limit if limited packsize and not first object */ 459if(!pack_size_limit || !nr_written) 460 limit =0; 461else if(pack_size_limit <= write_offset) 462/* 463 * the earlier object did not fit the limit; avoid 464 * mistaking this with unlimited (i.e. limit = 0). 465 */ 466 limit =1; 467else 468 limit = pack_size_limit - write_offset; 469 470if(!entry->delta) 471 usable_delta =0;/* no delta */ 472else if(!pack_size_limit) 473 usable_delta =1;/* unlimited packfile */ 474else if(entry->delta->idx.offset == (off_t)-1) 475 usable_delta =0;/* base was written to another pack */ 476else if(entry->delta->idx.offset) 477 usable_delta =1;/* base already exists in this pack */ 478else 479 usable_delta =0;/* base could end up in another pack */ 480 481if(!reuse_object) 482 to_reuse =0;/* explicit */ 483else if(!IN_PACK(entry)) 484 to_reuse =0;/* can't reuse what we don't have */ 485else if(oe_type(entry) == OBJ_REF_DELTA || 486oe_type(entry) == OBJ_OFS_DELTA) 487/* check_object() decided it for us ... */ 488 to_reuse = usable_delta; 489/* ... but pack split may override that */ 490else if(oe_type(entry) != entry->in_pack_type) 491 to_reuse =0;/* pack has delta which is unusable */ 492else if(entry->delta) 493 to_reuse =0;/* we want to pack afresh */ 494else 495 to_reuse =1;/* we have it in-pack undeltified, 496 * and we do not need to deltify it. 497 */ 498 499if(!to_reuse) 500 len =write_no_reuse_object(f, entry, limit, usable_delta); 501else 502 len =write_reuse_object(f, entry, limit, usable_delta); 503if(!len) 504return0; 505 506if(usable_delta) 507 written_delta++; 508 written++; 509if(!pack_to_stdout) 510 entry->idx.crc32 =crc32_end(f); 511return len; 512} 513 514enum write_one_status { 515 WRITE_ONE_SKIP = -1,/* already written */ 516 WRITE_ONE_BREAK =0,/* writing this will bust the limit; not written */ 517 WRITE_ONE_WRITTEN =1,/* normal */ 518 WRITE_ONE_RECURSIVE =2/* already scheduled to be written */ 519}; 520 521static enum write_one_status write_one(struct hashfile *f, 522struct object_entry *e, 523 off_t *offset) 524{ 525 off_t size; 526int recursing; 527 528/* 529 * we set offset to 1 (which is an impossible value) to mark 530 * the fact that this object is involved in "write its base 531 * first before writing a deltified object" recursion. 532 */ 533 recursing = (e->idx.offset ==1); 534if(recursing) { 535warning("recursive delta detected for object%s", 536oid_to_hex(&e->idx.oid)); 537return WRITE_ONE_RECURSIVE; 538}else if(e->idx.offset || e->preferred_base) { 539/* offset is non zero if object is written already. */ 540return WRITE_ONE_SKIP; 541} 542 543/* if we are deltified, write out base object first. */ 544if(e->delta) { 545 e->idx.offset =1;/* now recurse */ 546switch(write_one(f, e->delta, offset)) { 547case WRITE_ONE_RECURSIVE: 548/* we cannot depend on this one */ 549 e->delta = NULL; 550break; 551default: 552break; 553case WRITE_ONE_BREAK: 554 e->idx.offset = recursing; 555return WRITE_ONE_BREAK; 556} 557} 558 559 e->idx.offset = *offset; 560 size =write_object(f, e, *offset); 561if(!size) { 562 e->idx.offset = recursing; 563return WRITE_ONE_BREAK; 564} 565 written_list[nr_written++] = &e->idx; 566 567/* make sure off_t is sufficiently large not to wrap */ 568if(signed_add_overflows(*offset, size)) 569die("pack too large for current definition of off_t"); 570*offset += size; 571return WRITE_ONE_WRITTEN; 572} 573 574static intmark_tagged(const char*path,const struct object_id *oid,int flag, 575void*cb_data) 576{ 577struct object_id peeled; 578struct object_entry *entry =packlist_find(&to_pack, oid->hash, NULL); 579 580if(entry) 581 entry->tagged =1; 582if(!peel_ref(path, &peeled)) { 583 entry =packlist_find(&to_pack, peeled.hash, NULL); 584if(entry) 585 entry->tagged =1; 586} 587return0; 588} 589 590staticinlinevoidadd_to_write_order(struct object_entry **wo, 591unsigned int*endp, 592struct object_entry *e) 593{ 594if(e->filled) 595return; 596 wo[(*endp)++] = e; 597 e->filled =1; 598} 599 600static voidadd_descendants_to_write_order(struct object_entry **wo, 601unsigned int*endp, 602struct object_entry *e) 603{ 604int add_to_order =1; 605while(e) { 606if(add_to_order) { 607struct object_entry *s; 608/* add this node... */ 609add_to_write_order(wo, endp, e); 610/* all its siblings... */ 611for(s = e->delta_sibling; s; s = s->delta_sibling) { 612add_to_write_order(wo, endp, s); 613} 614} 615/* drop down a level to add left subtree nodes if possible */ 616if(e->delta_child) { 617 add_to_order =1; 618 e = e->delta_child; 619}else{ 620 add_to_order =0; 621/* our sibling might have some children, it is next */ 622if(e->delta_sibling) { 623 e = e->delta_sibling; 624continue; 625} 626/* go back to our parent node */ 627 e = e->delta; 628while(e && !e->delta_sibling) { 629/* we're on the right side of a subtree, keep 630 * going up until we can go right again */ 631 e = e->delta; 632} 633if(!e) { 634/* done- we hit our original root node */ 635return; 636} 637/* pass it off to sibling at this level */ 638 e = e->delta_sibling; 639} 640}; 641} 642 643static voidadd_family_to_write_order(struct object_entry **wo, 644unsigned int*endp, 645struct object_entry *e) 646{ 647struct object_entry *root; 648 649for(root = e; root->delta; root = root->delta) 650;/* nothing */ 651add_descendants_to_write_order(wo, endp, root); 652} 653 654static struct object_entry **compute_write_order(void) 655{ 656unsigned int i, wo_end, last_untagged; 657 658struct object_entry **wo; 659struct object_entry *objects = to_pack.objects; 660 661for(i =0; i < to_pack.nr_objects; i++) { 662 objects[i].tagged =0; 663 objects[i].filled =0; 664 objects[i].delta_child = NULL; 665 objects[i].delta_sibling = NULL; 666} 667 668/* 669 * Fully connect delta_child/delta_sibling network. 670 * Make sure delta_sibling is sorted in the original 671 * recency order. 672 */ 673for(i = to_pack.nr_objects; i >0;) { 674struct object_entry *e = &objects[--i]; 675if(!e->delta) 676continue; 677/* Mark me as the first child */ 678 e->delta_sibling = e->delta->delta_child; 679 e->delta->delta_child = e; 680} 681 682/* 683 * Mark objects that are at the tip of tags. 684 */ 685for_each_tag_ref(mark_tagged, NULL); 686 687/* 688 * Give the objects in the original recency order until 689 * we see a tagged tip. 690 */ 691ALLOC_ARRAY(wo, to_pack.nr_objects); 692for(i = wo_end =0; i < to_pack.nr_objects; i++) { 693if(objects[i].tagged) 694break; 695add_to_write_order(wo, &wo_end, &objects[i]); 696} 697 last_untagged = i; 698 699/* 700 * Then fill all the tagged tips. 701 */ 702for(; i < to_pack.nr_objects; i++) { 703if(objects[i].tagged) 704add_to_write_order(wo, &wo_end, &objects[i]); 705} 706 707/* 708 * And then all remaining commits and tags. 709 */ 710for(i = last_untagged; i < to_pack.nr_objects; i++) { 711if(oe_type(&objects[i]) != OBJ_COMMIT && 712oe_type(&objects[i]) != OBJ_TAG) 713continue; 714add_to_write_order(wo, &wo_end, &objects[i]); 715} 716 717/* 718 * And then all the trees. 719 */ 720for(i = last_untagged; i < to_pack.nr_objects; i++) { 721if(oe_type(&objects[i]) != OBJ_TREE) 722continue; 723add_to_write_order(wo, &wo_end, &objects[i]); 724} 725 726/* 727 * Finally all the rest in really tight order 728 */ 729for(i = last_untagged; i < to_pack.nr_objects; i++) { 730if(!objects[i].filled) 731add_family_to_write_order(wo, &wo_end, &objects[i]); 732} 733 734if(wo_end != to_pack.nr_objects) 735die("ordered%uobjects, expected %"PRIu32, wo_end, to_pack.nr_objects); 736 737return wo; 738} 739 740static off_t write_reused_pack(struct hashfile *f) 741{ 742unsigned char buffer[8192]; 743 off_t to_write, total; 744int fd; 745 746if(!is_pack_valid(reuse_packfile)) 747die("packfile is invalid:%s", reuse_packfile->pack_name); 748 749 fd =git_open(reuse_packfile->pack_name); 750if(fd <0) 751die_errno("unable to open packfile for reuse:%s", 752 reuse_packfile->pack_name); 753 754if(lseek(fd,sizeof(struct pack_header), SEEK_SET) == -1) 755die_errno("unable to seek in reused packfile"); 756 757if(reuse_packfile_offset <0) 758 reuse_packfile_offset = reuse_packfile->pack_size -20; 759 760 total = to_write = reuse_packfile_offset -sizeof(struct pack_header); 761 762while(to_write) { 763int read_pack =xread(fd, buffer,sizeof(buffer)); 764 765if(read_pack <=0) 766die_errno("unable to read from reused packfile"); 767 768if(read_pack > to_write) 769 read_pack = to_write; 770 771hashwrite(f, buffer, read_pack); 772 to_write -= read_pack; 773 774/* 775 * We don't know the actual number of objects written, 776 * only how many bytes written, how many bytes total, and 777 * how many objects total. So we can fake it by pretending all 778 * objects we are writing are the same size. This gives us a 779 * smooth progress meter, and at the end it matches the true 780 * answer. 781 */ 782 written = reuse_packfile_objects * 783(((double)(total - to_write)) / total); 784display_progress(progress_state, written); 785} 786 787close(fd); 788 written = reuse_packfile_objects; 789display_progress(progress_state, written); 790return reuse_packfile_offset -sizeof(struct pack_header); 791} 792 793static const char no_split_warning[] =N_( 794"disabling bitmap writing, packs are split due to pack.packSizeLimit" 795); 796 797static voidwrite_pack_file(void) 798{ 799uint32_t i =0, j; 800struct hashfile *f; 801 off_t offset; 802uint32_t nr_remaining = nr_result; 803time_t last_mtime =0; 804struct object_entry **write_order; 805 806if(progress > pack_to_stdout) 807 progress_state =start_progress(_("Writing objects"), nr_result); 808ALLOC_ARRAY(written_list, to_pack.nr_objects); 809 write_order =compute_write_order(); 810 811do{ 812struct object_id oid; 813char*pack_tmp_name = NULL; 814 815if(pack_to_stdout) 816 f =hashfd_throughput(1,"<stdout>", progress_state); 817else 818 f =create_tmp_packfile(&pack_tmp_name); 819 820 offset =write_pack_header(f, nr_remaining); 821 822if(reuse_packfile) { 823 off_t packfile_size; 824assert(pack_to_stdout); 825 826 packfile_size =write_reused_pack(f); 827 offset += packfile_size; 828} 829 830 nr_written =0; 831for(; i < to_pack.nr_objects; i++) { 832struct object_entry *e = write_order[i]; 833if(write_one(f, e, &offset) == WRITE_ONE_BREAK) 834break; 835display_progress(progress_state, written); 836} 837 838/* 839 * Did we write the wrong # entries in the header? 840 * If so, rewrite it like in fast-import 841 */ 842if(pack_to_stdout) { 843hashclose(f, oid.hash, CSUM_CLOSE); 844}else if(nr_written == nr_remaining) { 845hashclose(f, oid.hash, CSUM_FSYNC); 846}else{ 847int fd =hashclose(f, oid.hash,0); 848fixup_pack_header_footer(fd, oid.hash, pack_tmp_name, 849 nr_written, oid.hash, offset); 850close(fd); 851if(write_bitmap_index) { 852warning(_(no_split_warning)); 853 write_bitmap_index =0; 854} 855} 856 857if(!pack_to_stdout) { 858struct stat st; 859struct strbuf tmpname = STRBUF_INIT; 860 861/* 862 * Packs are runtime accessed in their mtime 863 * order since newer packs are more likely to contain 864 * younger objects. So if we are creating multiple 865 * packs then we should modify the mtime of later ones 866 * to preserve this property. 867 */ 868if(stat(pack_tmp_name, &st) <0) { 869warning_errno("failed to stat%s", pack_tmp_name); 870}else if(!last_mtime) { 871 last_mtime = st.st_mtime; 872}else{ 873struct utimbuf utb; 874 utb.actime = st.st_atime; 875 utb.modtime = --last_mtime; 876if(utime(pack_tmp_name, &utb) <0) 877warning_errno("failed utime() on%s", pack_tmp_name); 878} 879 880strbuf_addf(&tmpname,"%s-", base_name); 881 882if(write_bitmap_index) { 883bitmap_writer_set_checksum(oid.hash); 884bitmap_writer_build_type_index( 885&to_pack, written_list, nr_written); 886} 887 888finish_tmp_packfile(&tmpname, pack_tmp_name, 889 written_list, nr_written, 890&pack_idx_opts, oid.hash); 891 892if(write_bitmap_index) { 893strbuf_addf(&tmpname,"%s.bitmap",oid_to_hex(&oid)); 894 895stop_progress(&progress_state); 896 897bitmap_writer_show_progress(progress); 898bitmap_writer_reuse_bitmaps(&to_pack); 899bitmap_writer_select_commits(indexed_commits, indexed_commits_nr, -1); 900bitmap_writer_build(&to_pack); 901bitmap_writer_finish(written_list, nr_written, 902 tmpname.buf, write_bitmap_options); 903 write_bitmap_index =0; 904} 905 906strbuf_release(&tmpname); 907free(pack_tmp_name); 908puts(oid_to_hex(&oid)); 909} 910 911/* mark written objects as written to previous pack */ 912for(j =0; j < nr_written; j++) { 913 written_list[j]->offset = (off_t)-1; 914} 915 nr_remaining -= nr_written; 916}while(nr_remaining && i < to_pack.nr_objects); 917 918free(written_list); 919free(write_order); 920stop_progress(&progress_state); 921if(written != nr_result) 922die("wrote %"PRIu32" objects while expecting %"PRIu32, 923 written, nr_result); 924} 925 926static intno_try_delta(const char*path) 927{ 928static struct attr_check *check; 929 930if(!check) 931 check =attr_check_initl("delta", NULL); 932if(git_check_attr(path, check)) 933return0; 934if(ATTR_FALSE(check->items[0].value)) 935return1; 936return0; 937} 938 939/* 940 * When adding an object, check whether we have already added it 941 * to our packing list. If so, we can skip. However, if we are 942 * being asked to excludei t, but the previous mention was to include 943 * it, make sure to adjust its flags and tweak our numbers accordingly. 944 * 945 * As an optimization, we pass out the index position where we would have 946 * found the item, since that saves us from having to look it up again a 947 * few lines later when we want to add the new entry. 948 */ 949static inthave_duplicate_entry(const struct object_id *oid, 950int exclude, 951uint32_t*index_pos) 952{ 953struct object_entry *entry; 954 955 entry =packlist_find(&to_pack, oid->hash, index_pos); 956if(!entry) 957return0; 958 959if(exclude) { 960if(!entry->preferred_base) 961 nr_result--; 962 entry->preferred_base =1; 963} 964 965return1; 966} 967 968static intwant_found_object(int exclude,struct packed_git *p) 969{ 970if(exclude) 971return1; 972if(incremental) 973return0; 974 975/* 976 * When asked to do --local (do not include an object that appears in a 977 * pack we borrow from elsewhere) or --honor-pack-keep (do not include 978 * an object that appears in a pack marked with .keep), finding a pack 979 * that matches the criteria is sufficient for us to decide to omit it. 980 * However, even if this pack does not satisfy the criteria, we need to 981 * make sure no copy of this object appears in _any_ pack that makes us 982 * to omit the object, so we need to check all the packs. 983 * 984 * We can however first check whether these options can possible matter; 985 * if they do not matter we know we want the object in generated pack. 986 * Otherwise, we signal "-1" at the end to tell the caller that we do 987 * not know either way, and it needs to check more packs. 988 */ 989if(!ignore_packed_keep && 990(!local || !have_non_local_packs)) 991return1; 992 993if(local && !p->pack_local) 994return0; 995if(ignore_packed_keep && p->pack_local && p->pack_keep) 996return0; 997 998/* we don't know yet; keep looking for more packs */ 999return-1;1000}10011002/*1003 * Check whether we want the object in the pack (e.g., we do not want1004 * objects found in non-local stores if the "--local" option was used).1005 *1006 * If the caller already knows an existing pack it wants to take the object1007 * from, that is passed in *found_pack and *found_offset; otherwise this1008 * function finds if there is any pack that has the object and returns the pack1009 * and its offset in these variables.1010 */1011static intwant_object_in_pack(const struct object_id *oid,1012int exclude,1013struct packed_git **found_pack,1014 off_t *found_offset)1015{1016int want;1017struct list_head *pos;10181019if(!exclude && local &&has_loose_object_nonlocal(oid->hash))1020return0;10211022/*1023 * If we already know the pack object lives in, start checks from that1024 * pack - in the usual case when neither --local was given nor .keep files1025 * are present we will determine the answer right now.1026 */1027if(*found_pack) {1028 want =want_found_object(exclude, *found_pack);1029if(want != -1)1030return want;1031}1032list_for_each(pos,get_packed_git_mru(the_repository)) {1033struct packed_git *p =list_entry(pos,struct packed_git, mru);1034 off_t offset;10351036if(p == *found_pack)1037 offset = *found_offset;1038else1039 offset =find_pack_entry_one(oid->hash, p);10401041if(offset) {1042if(!*found_pack) {1043if(!is_pack_valid(p))1044continue;1045*found_offset = offset;1046*found_pack = p;1047}1048 want =want_found_object(exclude, p);1049if(!exclude && want >0)1050list_move(&p->mru,1051get_packed_git_mru(the_repository));1052if(want != -1)1053return want;1054}1055}10561057return1;1058}10591060static voidcreate_object_entry(const struct object_id *oid,1061enum object_type type,1062uint32_t hash,1063int exclude,1064int no_try_delta,1065uint32_t index_pos,1066struct packed_git *found_pack,1067 off_t found_offset)1068{1069struct object_entry *entry;10701071 entry =packlist_alloc(&to_pack, oid->hash, index_pos);1072 entry->hash = hash;1073oe_set_type(entry, type);1074if(exclude)1075 entry->preferred_base =1;1076else1077 nr_result++;1078if(found_pack) {1079oe_set_in_pack(&to_pack, entry, found_pack);1080 entry->in_pack_offset = found_offset;1081}10821083 entry->no_try_delta = no_try_delta;1084}10851086static const char no_closure_warning[] =N_(1087"disabling bitmap writing, as some objects are not being packed"1088);10891090static intadd_object_entry(const struct object_id *oid,enum object_type type,1091const char*name,int exclude)1092{1093struct packed_git *found_pack = NULL;1094 off_t found_offset =0;1095uint32_t index_pos;10961097if(have_duplicate_entry(oid, exclude, &index_pos))1098return0;10991100if(!want_object_in_pack(oid, exclude, &found_pack, &found_offset)) {1101/* The pack is missing an object, so it will not have closure */1102if(write_bitmap_index) {1103warning(_(no_closure_warning));1104 write_bitmap_index =0;1105}1106return0;1107}11081109create_object_entry(oid, type,pack_name_hash(name),1110 exclude, name &&no_try_delta(name),1111 index_pos, found_pack, found_offset);11121113display_progress(progress_state, nr_result);1114return1;1115}11161117static intadd_object_entry_from_bitmap(const struct object_id *oid,1118enum object_type type,1119int flags,uint32_t name_hash,1120struct packed_git *pack, off_t offset)1121{1122uint32_t index_pos;11231124if(have_duplicate_entry(oid,0, &index_pos))1125return0;11261127if(!want_object_in_pack(oid,0, &pack, &offset))1128return0;11291130create_object_entry(oid, type, name_hash,0,0, index_pos, pack, offset);11311132display_progress(progress_state, nr_result);1133return1;1134}11351136struct pbase_tree_cache {1137struct object_id oid;1138int ref;1139int temporary;1140void*tree_data;1141unsigned long tree_size;1142};11431144static struct pbase_tree_cache *(pbase_tree_cache[256]);1145static intpbase_tree_cache_ix(const struct object_id *oid)1146{1147return oid->hash[0] %ARRAY_SIZE(pbase_tree_cache);1148}1149static intpbase_tree_cache_ix_incr(int ix)1150{1151return(ix+1) %ARRAY_SIZE(pbase_tree_cache);1152}11531154static struct pbase_tree {1155struct pbase_tree *next;1156/* This is a phony "cache" entry; we are not1157 * going to evict it or find it through _get()1158 * mechanism -- this is for the toplevel node that1159 * would almost always change with any commit.1160 */1161struct pbase_tree_cache pcache;1162} *pbase_tree;11631164static struct pbase_tree_cache *pbase_tree_get(const struct object_id *oid)1165{1166struct pbase_tree_cache *ent, *nent;1167void*data;1168unsigned long size;1169enum object_type type;1170int neigh;1171int my_ix =pbase_tree_cache_ix(oid);1172int available_ix = -1;11731174/* pbase-tree-cache acts as a limited hashtable.1175 * your object will be found at your index or within a few1176 * slots after that slot if it is cached.1177 */1178for(neigh =0; neigh <8; neigh++) {1179 ent = pbase_tree_cache[my_ix];1180if(ent && !oidcmp(&ent->oid, oid)) {1181 ent->ref++;1182return ent;1183}1184else if(((available_ix <0) && (!ent || !ent->ref)) ||1185((0<= available_ix) &&1186(!ent && pbase_tree_cache[available_ix])))1187 available_ix = my_ix;1188if(!ent)1189break;1190 my_ix =pbase_tree_cache_ix_incr(my_ix);1191}11921193/* Did not find one. Either we got a bogus request or1194 * we need to read and perhaps cache.1195 */1196 data =read_object_file(oid, &type, &size);1197if(!data)1198return NULL;1199if(type != OBJ_TREE) {1200free(data);1201return NULL;1202}12031204/* We need to either cache or return a throwaway copy */12051206if(available_ix <0)1207 ent = NULL;1208else{1209 ent = pbase_tree_cache[available_ix];1210 my_ix = available_ix;1211}12121213if(!ent) {1214 nent =xmalloc(sizeof(*nent));1215 nent->temporary = (available_ix <0);1216}1217else{1218/* evict and reuse */1219free(ent->tree_data);1220 nent = ent;1221}1222oidcpy(&nent->oid, oid);1223 nent->tree_data = data;1224 nent->tree_size = size;1225 nent->ref =1;1226if(!nent->temporary)1227 pbase_tree_cache[my_ix] = nent;1228return nent;1229}12301231static voidpbase_tree_put(struct pbase_tree_cache *cache)1232{1233if(!cache->temporary) {1234 cache->ref--;1235return;1236}1237free(cache->tree_data);1238free(cache);1239}12401241static intname_cmp_len(const char*name)1242{1243int i;1244for(i =0; name[i] && name[i] !='\n'&& name[i] !='/'; i++)1245;1246return i;1247}12481249static voidadd_pbase_object(struct tree_desc *tree,1250const char*name,1251int cmplen,1252const char*fullname)1253{1254struct name_entry entry;1255int cmp;12561257while(tree_entry(tree,&entry)) {1258if(S_ISGITLINK(entry.mode))1259continue;1260 cmp =tree_entry_len(&entry) != cmplen ?1:1261memcmp(name, entry.path, cmplen);1262if(cmp >0)1263continue;1264if(cmp <0)1265return;1266if(name[cmplen] !='/') {1267add_object_entry(entry.oid,1268object_type(entry.mode),1269 fullname,1);1270return;1271}1272if(S_ISDIR(entry.mode)) {1273struct tree_desc sub;1274struct pbase_tree_cache *tree;1275const char*down = name+cmplen+1;1276int downlen =name_cmp_len(down);12771278 tree =pbase_tree_get(entry.oid);1279if(!tree)1280return;1281init_tree_desc(&sub, tree->tree_data, tree->tree_size);12821283add_pbase_object(&sub, down, downlen, fullname);1284pbase_tree_put(tree);1285}1286}1287}12881289static unsigned*done_pbase_paths;1290static int done_pbase_paths_num;1291static int done_pbase_paths_alloc;1292static intdone_pbase_path_pos(unsigned hash)1293{1294int lo =0;1295int hi = done_pbase_paths_num;1296while(lo < hi) {1297int mi = lo + (hi - lo) /2;1298if(done_pbase_paths[mi] == hash)1299return mi;1300if(done_pbase_paths[mi] < hash)1301 hi = mi;1302else1303 lo = mi +1;1304}1305return-lo-1;1306}13071308static intcheck_pbase_path(unsigned hash)1309{1310int pos =done_pbase_path_pos(hash);1311if(0<= pos)1312return1;1313 pos = -pos -1;1314ALLOC_GROW(done_pbase_paths,1315 done_pbase_paths_num +1,1316 done_pbase_paths_alloc);1317 done_pbase_paths_num++;1318if(pos < done_pbase_paths_num)1319MOVE_ARRAY(done_pbase_paths + pos +1, done_pbase_paths + pos,1320 done_pbase_paths_num - pos -1);1321 done_pbase_paths[pos] = hash;1322return0;1323}13241325static voidadd_preferred_base_object(const char*name)1326{1327struct pbase_tree *it;1328int cmplen;1329unsigned hash =pack_name_hash(name);13301331if(!num_preferred_base ||check_pbase_path(hash))1332return;13331334 cmplen =name_cmp_len(name);1335for(it = pbase_tree; it; it = it->next) {1336if(cmplen ==0) {1337add_object_entry(&it->pcache.oid, OBJ_TREE, NULL,1);1338}1339else{1340struct tree_desc tree;1341init_tree_desc(&tree, it->pcache.tree_data, it->pcache.tree_size);1342add_pbase_object(&tree, name, cmplen, name);1343}1344}1345}13461347static voidadd_preferred_base(struct object_id *oid)1348{1349struct pbase_tree *it;1350void*data;1351unsigned long size;1352struct object_id tree_oid;13531354if(window <= num_preferred_base++)1355return;13561357 data =read_object_with_reference(oid, tree_type, &size, &tree_oid);1358if(!data)1359return;13601361for(it = pbase_tree; it; it = it->next) {1362if(!oidcmp(&it->pcache.oid, &tree_oid)) {1363free(data);1364return;1365}1366}13671368 it =xcalloc(1,sizeof(*it));1369 it->next = pbase_tree;1370 pbase_tree = it;13711372oidcpy(&it->pcache.oid, &tree_oid);1373 it->pcache.tree_data = data;1374 it->pcache.tree_size = size;1375}13761377static voidcleanup_preferred_base(void)1378{1379struct pbase_tree *it;1380unsigned i;13811382 it = pbase_tree;1383 pbase_tree = NULL;1384while(it) {1385struct pbase_tree *tmp = it;1386 it = tmp->next;1387free(tmp->pcache.tree_data);1388free(tmp);1389}13901391for(i =0; i <ARRAY_SIZE(pbase_tree_cache); i++) {1392if(!pbase_tree_cache[i])1393continue;1394free(pbase_tree_cache[i]->tree_data);1395FREE_AND_NULL(pbase_tree_cache[i]);1396}13971398FREE_AND_NULL(done_pbase_paths);1399 done_pbase_paths_num = done_pbase_paths_alloc =0;1400}14011402static voidcheck_object(struct object_entry *entry)1403{1404if(IN_PACK(entry)) {1405struct packed_git *p =IN_PACK(entry);1406struct pack_window *w_curs = NULL;1407const unsigned char*base_ref = NULL;1408struct object_entry *base_entry;1409unsigned long used, used_0;1410unsigned long avail;1411 off_t ofs;1412unsigned char*buf, c;1413enum object_type type;14141415 buf =use_pack(p, &w_curs, entry->in_pack_offset, &avail);14161417/*1418 * We want in_pack_type even if we do not reuse delta1419 * since non-delta representations could still be reused.1420 */1421 used =unpack_object_header_buffer(buf, avail,1422&type,1423&entry->size);1424if(used ==0)1425goto give_up;14261427if(type <0)1428BUG("invalid type%d", type);1429 entry->in_pack_type = type;14301431/*1432 * Determine if this is a delta and if so whether we can1433 * reuse it or not. Otherwise let's find out as cheaply as1434 * possible what the actual type and size for this object is.1435 */1436switch(entry->in_pack_type) {1437default:1438/* Not a delta hence we've already got all we need. */1439oe_set_type(entry, entry->in_pack_type);1440 entry->in_pack_header_size = used;1441if(oe_type(entry) < OBJ_COMMIT ||oe_type(entry) > OBJ_BLOB)1442goto give_up;1443unuse_pack(&w_curs);1444return;1445case OBJ_REF_DELTA:1446if(reuse_delta && !entry->preferred_base)1447 base_ref =use_pack(p, &w_curs,1448 entry->in_pack_offset + used, NULL);1449 entry->in_pack_header_size = used +20;1450break;1451case OBJ_OFS_DELTA:1452 buf =use_pack(p, &w_curs,1453 entry->in_pack_offset + used, NULL);1454 used_0 =0;1455 c = buf[used_0++];1456 ofs = c &127;1457while(c &128) {1458 ofs +=1;1459if(!ofs ||MSB(ofs,7)) {1460error("delta base offset overflow in pack for%s",1461oid_to_hex(&entry->idx.oid));1462goto give_up;1463}1464 c = buf[used_0++];1465 ofs = (ofs <<7) + (c &127);1466}1467 ofs = entry->in_pack_offset - ofs;1468if(ofs <=0|| ofs >= entry->in_pack_offset) {1469error("delta base offset out of bound for%s",1470oid_to_hex(&entry->idx.oid));1471goto give_up;1472}1473if(reuse_delta && !entry->preferred_base) {1474struct revindex_entry *revidx;1475 revidx =find_pack_revindex(p, ofs);1476if(!revidx)1477goto give_up;1478 base_ref =nth_packed_object_sha1(p, revidx->nr);1479}1480 entry->in_pack_header_size = used + used_0;1481break;1482}14831484if(base_ref && (base_entry =packlist_find(&to_pack, base_ref, NULL))) {1485/*1486 * If base_ref was set above that means we wish to1487 * reuse delta data, and we even found that base1488 * in the list of objects we want to pack. Goodie!1489 *1490 * Depth value does not matter - find_deltas() will1491 * never consider reused delta as the base object to1492 * deltify other objects against, in order to avoid1493 * circular deltas.1494 */1495oe_set_type(entry, entry->in_pack_type);1496 entry->delta = base_entry;1497 entry->delta_size = entry->size;1498 entry->delta_sibling = base_entry->delta_child;1499 base_entry->delta_child = entry;1500unuse_pack(&w_curs);1501return;1502}15031504if(oe_type(entry)) {1505/*1506 * This must be a delta and we already know what the1507 * final object type is. Let's extract the actual1508 * object size from the delta header.1509 */1510 entry->size =get_size_from_delta(p, &w_curs,1511 entry->in_pack_offset + entry->in_pack_header_size);1512if(entry->size ==0)1513goto give_up;1514unuse_pack(&w_curs);1515return;1516}15171518/*1519 * No choice but to fall back to the recursive delta walk1520 * with sha1_object_info() to find about the object type1521 * at this point...1522 */1523 give_up:1524unuse_pack(&w_curs);1525}15261527oe_set_type(entry,oid_object_info(&entry->idx.oid, &entry->size));1528/*1529 * The error condition is checked in prepare_pack(). This is1530 * to permit a missing preferred base object to be ignored1531 * as a preferred base. Doing so can result in a larger1532 * pack file, but the transfer will still take place.1533 */1534}15351536static intpack_offset_sort(const void*_a,const void*_b)1537{1538const struct object_entry *a = *(struct object_entry **)_a;1539const struct object_entry *b = *(struct object_entry **)_b;1540const struct packed_git *a_in_pack =IN_PACK(a);1541const struct packed_git *b_in_pack =IN_PACK(b);15421543/* avoid filesystem trashing with loose objects */1544if(!a_in_pack && !b_in_pack)1545returnoidcmp(&a->idx.oid, &b->idx.oid);15461547if(a_in_pack < b_in_pack)1548return-1;1549if(a_in_pack > b_in_pack)1550return1;1551return a->in_pack_offset < b->in_pack_offset ? -1:1552(a->in_pack_offset > b->in_pack_offset);1553}15541555/*1556 * Drop an on-disk delta we were planning to reuse. Naively, this would1557 * just involve blanking out the "delta" field, but we have to deal1558 * with some extra book-keeping:1559 *1560 * 1. Removing ourselves from the delta_sibling linked list.1561 *1562 * 2. Updating our size/type to the non-delta representation. These were1563 * either not recorded initially (size) or overwritten with the delta type1564 * (type) when check_object() decided to reuse the delta.1565 *1566 * 3. Resetting our delta depth, as we are now a base object.1567 */1568static voiddrop_reused_delta(struct object_entry *entry)1569{1570struct object_entry **p = &entry->delta->delta_child;1571struct object_info oi = OBJECT_INFO_INIT;1572enum object_type type;15731574while(*p) {1575if(*p == entry)1576*p = (*p)->delta_sibling;1577else1578 p = &(*p)->delta_sibling;1579}1580 entry->delta = NULL;1581 entry->depth =0;15821583 oi.sizep = &entry->size;1584 oi.typep = &type;1585if(packed_object_info(IN_PACK(entry), entry->in_pack_offset, &oi) <0) {1586/*1587 * We failed to get the info from this pack for some reason;1588 * fall back to sha1_object_info, which may find another copy.1589 * And if that fails, the error will be recorded in oe_type(entry)1590 * and dealt with in prepare_pack().1591 */1592oe_set_type(entry,oid_object_info(&entry->idx.oid,1593&entry->size));1594}else{1595oe_set_type(entry, type);1596}1597}15981599/*1600 * Follow the chain of deltas from this entry onward, throwing away any links1601 * that cause us to hit a cycle (as determined by the DFS state flags in1602 * the entries).1603 *1604 * We also detect too-long reused chains that would violate our --depth1605 * limit.1606 */1607static voidbreak_delta_chains(struct object_entry *entry)1608{1609/*1610 * The actual depth of each object we will write is stored as an int,1611 * as it cannot exceed our int "depth" limit. But before we break1612 * changes based no that limit, we may potentially go as deep as the1613 * number of objects, which is elsewhere bounded to a uint32_t.1614 */1615uint32_t total_depth;1616struct object_entry *cur, *next;16171618for(cur = entry, total_depth =0;1619 cur;1620 cur = cur->delta, total_depth++) {1621if(cur->dfs_state == DFS_DONE) {1622/*1623 * We've already seen this object and know it isn't1624 * part of a cycle. We do need to append its depth1625 * to our count.1626 */1627 total_depth += cur->depth;1628break;1629}16301631/*1632 * We break cycles before looping, so an ACTIVE state (or any1633 * other cruft which made its way into the state variable)1634 * is a bug.1635 */1636if(cur->dfs_state != DFS_NONE)1637die("BUG: confusing delta dfs state in first pass:%d",1638 cur->dfs_state);16391640/*1641 * Now we know this is the first time we've seen the object. If1642 * it's not a delta, we're done traversing, but we'll mark it1643 * done to save time on future traversals.1644 */1645if(!cur->delta) {1646 cur->dfs_state = DFS_DONE;1647break;1648}16491650/*1651 * Mark ourselves as active and see if the next step causes1652 * us to cycle to another active object. It's important to do1653 * this _before_ we loop, because it impacts where we make the1654 * cut, and thus how our total_depth counter works.1655 * E.g., We may see a partial loop like:1656 *1657 * A -> B -> C -> D -> B1658 *1659 * Cutting B->C breaks the cycle. But now the depth of A is1660 * only 1, and our total_depth counter is at 3. The size of the1661 * error is always one less than the size of the cycle we1662 * broke. Commits C and D were "lost" from A's chain.1663 *1664 * If we instead cut D->B, then the depth of A is correct at 3.1665 * We keep all commits in the chain that we examined.1666 */1667 cur->dfs_state = DFS_ACTIVE;1668if(cur->delta->dfs_state == DFS_ACTIVE) {1669drop_reused_delta(cur);1670 cur->dfs_state = DFS_DONE;1671break;1672}1673}16741675/*1676 * And now that we've gone all the way to the bottom of the chain, we1677 * need to clear the active flags and set the depth fields as1678 * appropriate. Unlike the loop above, which can quit when it drops a1679 * delta, we need to keep going to look for more depth cuts. So we need1680 * an extra "next" pointer to keep going after we reset cur->delta.1681 */1682for(cur = entry; cur; cur = next) {1683 next = cur->delta;16841685/*1686 * We should have a chain of zero or more ACTIVE states down to1687 * a final DONE. We can quit after the DONE, because either it1688 * has no bases, or we've already handled them in a previous1689 * call.1690 */1691if(cur->dfs_state == DFS_DONE)1692break;1693else if(cur->dfs_state != DFS_ACTIVE)1694die("BUG: confusing delta dfs state in second pass:%d",1695 cur->dfs_state);16961697/*1698 * If the total_depth is more than depth, then we need to snip1699 * the chain into two or more smaller chains that don't exceed1700 * the maximum depth. Most of the resulting chains will contain1701 * (depth + 1) entries (i.e., depth deltas plus one base), and1702 * the last chain (i.e., the one containing entry) will contain1703 * whatever entries are left over, namely1704 * (total_depth % (depth + 1)) of them.1705 *1706 * Since we are iterating towards decreasing depth, we need to1707 * decrement total_depth as we go, and we need to write to the1708 * entry what its final depth will be after all of the1709 * snipping. Since we're snipping into chains of length (depth1710 * + 1) entries, the final depth of an entry will be its1711 * original depth modulo (depth + 1). Any time we encounter an1712 * entry whose final depth is supposed to be zero, we snip it1713 * from its delta base, thereby making it so.1714 */1715 cur->depth = (total_depth--) % (depth +1);1716if(!cur->depth)1717drop_reused_delta(cur);17181719 cur->dfs_state = DFS_DONE;1720}1721}17221723static voidget_object_details(void)1724{1725uint32_t i;1726struct object_entry **sorted_by_offset;17271728 sorted_by_offset =xcalloc(to_pack.nr_objects,sizeof(struct object_entry *));1729for(i =0; i < to_pack.nr_objects; i++)1730 sorted_by_offset[i] = to_pack.objects + i;1731QSORT(sorted_by_offset, to_pack.nr_objects, pack_offset_sort);17321733for(i =0; i < to_pack.nr_objects; i++) {1734struct object_entry *entry = sorted_by_offset[i];1735check_object(entry);1736if(big_file_threshold < entry->size)1737 entry->no_try_delta =1;1738}17391740/*1741 * This must happen in a second pass, since we rely on the delta1742 * information for the whole list being completed.1743 */1744for(i =0; i < to_pack.nr_objects; i++)1745break_delta_chains(&to_pack.objects[i]);17461747free(sorted_by_offset);1748}17491750/*1751 * We search for deltas in a list sorted by type, by filename hash, and then1752 * by size, so that we see progressively smaller and smaller files.1753 * That's because we prefer deltas to be from the bigger file1754 * to the smaller -- deletes are potentially cheaper, but perhaps1755 * more importantly, the bigger file is likely the more recent1756 * one. The deepest deltas are therefore the oldest objects which are1757 * less susceptible to be accessed often.1758 */1759static inttype_size_sort(const void*_a,const void*_b)1760{1761const struct object_entry *a = *(struct object_entry **)_a;1762const struct object_entry *b = *(struct object_entry **)_b;1763enum object_type a_type =oe_type(a);1764enum object_type b_type =oe_type(b);17651766if(a_type > b_type)1767return-1;1768if(a_type < b_type)1769return1;1770if(a->hash > b->hash)1771return-1;1772if(a->hash < b->hash)1773return1;1774if(a->preferred_base > b->preferred_base)1775return-1;1776if(a->preferred_base < b->preferred_base)1777return1;1778if(a->size > b->size)1779return-1;1780if(a->size < b->size)1781return1;1782return a < b ? -1: (a > b);/* newest first */1783}17841785struct unpacked {1786struct object_entry *entry;1787void*data;1788struct delta_index *index;1789unsigned depth;1790};17911792static intdelta_cacheable(unsigned long src_size,unsigned long trg_size,1793unsigned long delta_size)1794{1795if(max_delta_cache_size && delta_cache_size + delta_size > max_delta_cache_size)1796return0;17971798if(delta_size < cache_max_small_delta_size)1799return1;18001801/* cache delta, if objects are large enough compared to delta size */1802if((src_size >>20) + (trg_size >>21) > (delta_size >>10))1803return1;18041805return0;1806}18071808#ifndef NO_PTHREADS18091810static pthread_mutex_t read_mutex;1811#define read_lock() pthread_mutex_lock(&read_mutex)1812#define read_unlock() pthread_mutex_unlock(&read_mutex)18131814static pthread_mutex_t cache_mutex;1815#define cache_lock() pthread_mutex_lock(&cache_mutex)1816#define cache_unlock() pthread_mutex_unlock(&cache_mutex)18171818static pthread_mutex_t progress_mutex;1819#define progress_lock() pthread_mutex_lock(&progress_mutex)1820#define progress_unlock() pthread_mutex_unlock(&progress_mutex)18211822#else18231824#define read_lock() (void)01825#define read_unlock() (void)01826#define cache_lock() (void)01827#define cache_unlock() (void)01828#define progress_lock() (void)01829#define progress_unlock() (void)018301831#endif18321833static inttry_delta(struct unpacked *trg,struct unpacked *src,1834unsigned max_depth,unsigned long*mem_usage)1835{1836struct object_entry *trg_entry = trg->entry;1837struct object_entry *src_entry = src->entry;1838unsigned long trg_size, src_size, delta_size, sizediff, max_size, sz;1839unsigned ref_depth;1840enum object_type type;1841void*delta_buf;18421843/* Don't bother doing diffs between different types */1844if(oe_type(trg_entry) !=oe_type(src_entry))1845return-1;18461847/*1848 * We do not bother to try a delta that we discarded on an1849 * earlier try, but only when reusing delta data. Note that1850 * src_entry that is marked as the preferred_base should always1851 * be considered, as even if we produce a suboptimal delta against1852 * it, we will still save the transfer cost, as we already know1853 * the other side has it and we won't send src_entry at all.1854 */1855if(reuse_delta &&IN_PACK(trg_entry) &&1856IN_PACK(trg_entry) ==IN_PACK(src_entry) &&1857!src_entry->preferred_base &&1858 trg_entry->in_pack_type != OBJ_REF_DELTA &&1859 trg_entry->in_pack_type != OBJ_OFS_DELTA)1860return0;18611862/* Let's not bust the allowed depth. */1863if(src->depth >= max_depth)1864return0;18651866/* Now some size filtering heuristics. */1867 trg_size = trg_entry->size;1868if(!trg_entry->delta) {1869 max_size = trg_size/2-20;1870 ref_depth =1;1871}else{1872 max_size = trg_entry->delta_size;1873 ref_depth = trg->depth;1874}1875 max_size = (uint64_t)max_size * (max_depth - src->depth) /1876(max_depth - ref_depth +1);1877if(max_size ==0)1878return0;1879 src_size = src_entry->size;1880 sizediff = src_size < trg_size ? trg_size - src_size :0;1881if(sizediff >= max_size)1882return0;1883if(trg_size < src_size /32)1884return0;18851886/* Load data if not already done */1887if(!trg->data) {1888read_lock();1889 trg->data =read_object_file(&trg_entry->idx.oid, &type, &sz);1890read_unlock();1891if(!trg->data)1892die("object%scannot be read",1893oid_to_hex(&trg_entry->idx.oid));1894if(sz != trg_size)1895die("object%sinconsistent object length (%lu vs%lu)",1896oid_to_hex(&trg_entry->idx.oid), sz,1897 trg_size);1898*mem_usage += sz;1899}1900if(!src->data) {1901read_lock();1902 src->data =read_object_file(&src_entry->idx.oid, &type, &sz);1903read_unlock();1904if(!src->data) {1905if(src_entry->preferred_base) {1906static int warned =0;1907if(!warned++)1908warning("object%scannot be read",1909oid_to_hex(&src_entry->idx.oid));1910/*1911 * Those objects are not included in the1912 * resulting pack. Be resilient and ignore1913 * them if they can't be read, in case the1914 * pack could be created nevertheless.1915 */1916return0;1917}1918die("object%scannot be read",1919oid_to_hex(&src_entry->idx.oid));1920}1921if(sz != src_size)1922die("object%sinconsistent object length (%lu vs%lu)",1923oid_to_hex(&src_entry->idx.oid), sz,1924 src_size);1925*mem_usage += sz;1926}1927if(!src->index) {1928 src->index =create_delta_index(src->data, src_size);1929if(!src->index) {1930static int warned =0;1931if(!warned++)1932warning("suboptimal pack - out of memory");1933return0;1934}1935*mem_usage +=sizeof_delta_index(src->index);1936}19371938 delta_buf =create_delta(src->index, trg->data, trg_size, &delta_size, max_size);1939if(!delta_buf)1940return0;19411942if(trg_entry->delta) {1943/* Prefer only shallower same-sized deltas. */1944if(delta_size == trg_entry->delta_size &&1945 src->depth +1>= trg->depth) {1946free(delta_buf);1947return0;1948}1949}19501951/*1952 * Handle memory allocation outside of the cache1953 * accounting lock. Compiler will optimize the strangeness1954 * away when NO_PTHREADS is defined.1955 */1956free(trg_entry->delta_data);1957cache_lock();1958if(trg_entry->delta_data) {1959 delta_cache_size -= trg_entry->delta_size;1960 trg_entry->delta_data = NULL;1961}1962if(delta_cacheable(src_size, trg_size, delta_size)) {1963 delta_cache_size += delta_size;1964cache_unlock();1965 trg_entry->delta_data =xrealloc(delta_buf, delta_size);1966}else{1967cache_unlock();1968free(delta_buf);1969}19701971 trg_entry->delta = src_entry;1972 trg_entry->delta_size = delta_size;1973 trg->depth = src->depth +1;19741975return1;1976}19771978static unsigned intcheck_delta_limit(struct object_entry *me,unsigned int n)1979{1980struct object_entry *child = me->delta_child;1981unsigned int m = n;1982while(child) {1983unsigned int c =check_delta_limit(child, n +1);1984if(m < c)1985 m = c;1986 child = child->delta_sibling;1987}1988return m;1989}19901991static unsigned longfree_unpacked(struct unpacked *n)1992{1993unsigned long freed_mem =sizeof_delta_index(n->index);1994free_delta_index(n->index);1995 n->index = NULL;1996if(n->data) {1997 freed_mem += n->entry->size;1998FREE_AND_NULL(n->data);1999}2000 n->entry = NULL;2001 n->depth =0;2002return freed_mem;2003}20042005static voidfind_deltas(struct object_entry **list,unsigned*list_size,2006int window,int depth,unsigned*processed)2007{2008uint32_t i, idx =0, count =0;2009struct unpacked *array;2010unsigned long mem_usage =0;20112012 array =xcalloc(window,sizeof(struct unpacked));20132014for(;;) {2015struct object_entry *entry;2016struct unpacked *n = array + idx;2017int j, max_depth, best_base = -1;20182019progress_lock();2020if(!*list_size) {2021progress_unlock();2022break;2023}2024 entry = *list++;2025(*list_size)--;2026if(!entry->preferred_base) {2027(*processed)++;2028display_progress(progress_state, *processed);2029}2030progress_unlock();20312032 mem_usage -=free_unpacked(n);2033 n->entry = entry;20342035while(window_memory_limit &&2036 mem_usage > window_memory_limit &&2037 count >1) {2038uint32_t tail = (idx + window - count) % window;2039 mem_usage -=free_unpacked(array + tail);2040 count--;2041}20422043/* We do not compute delta to *create* objects we are not2044 * going to pack.2045 */2046if(entry->preferred_base)2047goto next;20482049/*2050 * If the current object is at pack edge, take the depth the2051 * objects that depend on the current object into account2052 * otherwise they would become too deep.2053 */2054 max_depth = depth;2055if(entry->delta_child) {2056 max_depth -=check_delta_limit(entry,0);2057if(max_depth <=0)2058goto next;2059}20602061 j = window;2062while(--j >0) {2063int ret;2064uint32_t other_idx = idx + j;2065struct unpacked *m;2066if(other_idx >= window)2067 other_idx -= window;2068 m = array + other_idx;2069if(!m->entry)2070break;2071 ret =try_delta(n, m, max_depth, &mem_usage);2072if(ret <0)2073break;2074else if(ret >0)2075 best_base = other_idx;2076}20772078/*2079 * If we decided to cache the delta data, then it is best2080 * to compress it right away. First because we have to do2081 * it anyway, and doing it here while we're threaded will2082 * save a lot of time in the non threaded write phase,2083 * as well as allow for caching more deltas within2084 * the same cache size limit.2085 * ...2086 * But only if not writing to stdout, since in that case2087 * the network is most likely throttling writes anyway,2088 * and therefore it is best to go to the write phase ASAP2089 * instead, as we can afford spending more time compressing2090 * between writes at that moment.2091 */2092if(entry->delta_data && !pack_to_stdout) {2093 entry->z_delta_size =do_compress(&entry->delta_data,2094 entry->delta_size);2095cache_lock();2096 delta_cache_size -= entry->delta_size;2097 delta_cache_size += entry->z_delta_size;2098cache_unlock();2099}21002101/* if we made n a delta, and if n is already at max2102 * depth, leaving it in the window is pointless. we2103 * should evict it first.2104 */2105if(entry->delta && max_depth <= n->depth)2106continue;21072108/*2109 * Move the best delta base up in the window, after the2110 * currently deltified object, to keep it longer. It will2111 * be the first base object to be attempted next.2112 */2113if(entry->delta) {2114struct unpacked swap = array[best_base];2115int dist = (window + idx - best_base) % window;2116int dst = best_base;2117while(dist--) {2118int src = (dst +1) % window;2119 array[dst] = array[src];2120 dst = src;2121}2122 array[dst] = swap;2123}21242125 next:2126 idx++;2127if(count +1< window)2128 count++;2129if(idx >= window)2130 idx =0;2131}21322133for(i =0; i < window; ++i) {2134free_delta_index(array[i].index);2135free(array[i].data);2136}2137free(array);2138}21392140#ifndef NO_PTHREADS21412142static voidtry_to_free_from_threads(size_t size)2143{2144read_lock();2145release_pack_memory(size);2146read_unlock();2147}21482149static try_to_free_t old_try_to_free_routine;21502151/*2152 * The main thread waits on the condition that (at least) one of the workers2153 * has stopped working (which is indicated in the .working member of2154 * struct thread_params).2155 * When a work thread has completed its work, it sets .working to 0 and2156 * signals the main thread and waits on the condition that .data_ready2157 * becomes 1.2158 */21592160struct thread_params {2161 pthread_t thread;2162struct object_entry **list;2163unsigned list_size;2164unsigned remaining;2165int window;2166int depth;2167int working;2168int data_ready;2169 pthread_mutex_t mutex;2170 pthread_cond_t cond;2171unsigned*processed;2172};21732174static pthread_cond_t progress_cond;21752176/*2177 * Mutex and conditional variable can't be statically-initialized on Windows.2178 */2179static voidinit_threaded_search(void)2180{2181init_recursive_mutex(&read_mutex);2182pthread_mutex_init(&cache_mutex, NULL);2183pthread_mutex_init(&progress_mutex, NULL);2184pthread_cond_init(&progress_cond, NULL);2185 old_try_to_free_routine =set_try_to_free_routine(try_to_free_from_threads);2186}21872188static voidcleanup_threaded_search(void)2189{2190set_try_to_free_routine(old_try_to_free_routine);2191pthread_cond_destroy(&progress_cond);2192pthread_mutex_destroy(&read_mutex);2193pthread_mutex_destroy(&cache_mutex);2194pthread_mutex_destroy(&progress_mutex);2195}21962197static void*threaded_find_deltas(void*arg)2198{2199struct thread_params *me = arg;22002201progress_lock();2202while(me->remaining) {2203progress_unlock();22042205find_deltas(me->list, &me->remaining,2206 me->window, me->depth, me->processed);22072208progress_lock();2209 me->working =0;2210pthread_cond_signal(&progress_cond);2211progress_unlock();22122213/*2214 * We must not set ->data_ready before we wait on the2215 * condition because the main thread may have set it to 12216 * before we get here. In order to be sure that new2217 * work is available if we see 1 in ->data_ready, it2218 * was initialized to 0 before this thread was spawned2219 * and we reset it to 0 right away.2220 */2221pthread_mutex_lock(&me->mutex);2222while(!me->data_ready)2223pthread_cond_wait(&me->cond, &me->mutex);2224 me->data_ready =0;2225pthread_mutex_unlock(&me->mutex);22262227progress_lock();2228}2229progress_unlock();2230/* leave ->working 1 so that this doesn't get more work assigned */2231return NULL;2232}22332234static voidll_find_deltas(struct object_entry **list,unsigned list_size,2235int window,int depth,unsigned*processed)2236{2237struct thread_params *p;2238int i, ret, active_threads =0;22392240init_threaded_search();22412242if(delta_search_threads <=1) {2243find_deltas(list, &list_size, window, depth, processed);2244cleanup_threaded_search();2245return;2246}2247if(progress > pack_to_stdout)2248fprintf(stderr,"Delta compression using up to%dthreads.\n",2249 delta_search_threads);2250 p =xcalloc(delta_search_threads,sizeof(*p));22512252/* Partition the work amongst work threads. */2253for(i =0; i < delta_search_threads; i++) {2254unsigned sub_size = list_size / (delta_search_threads - i);22552256/* don't use too small segments or no deltas will be found */2257if(sub_size <2*window && i+1< delta_search_threads)2258 sub_size =0;22592260 p[i].window = window;2261 p[i].depth = depth;2262 p[i].processed = processed;2263 p[i].working =1;2264 p[i].data_ready =0;22652266/* try to split chunks on "path" boundaries */2267while(sub_size && sub_size < list_size &&2268 list[sub_size]->hash &&2269 list[sub_size]->hash == list[sub_size-1]->hash)2270 sub_size++;22712272 p[i].list = list;2273 p[i].list_size = sub_size;2274 p[i].remaining = sub_size;22752276 list += sub_size;2277 list_size -= sub_size;2278}22792280/* Start work threads. */2281for(i =0; i < delta_search_threads; i++) {2282if(!p[i].list_size)2283continue;2284pthread_mutex_init(&p[i].mutex, NULL);2285pthread_cond_init(&p[i].cond, NULL);2286 ret =pthread_create(&p[i].thread, NULL,2287 threaded_find_deltas, &p[i]);2288if(ret)2289die("unable to create thread:%s",strerror(ret));2290 active_threads++;2291}22922293/*2294 * Now let's wait for work completion. Each time a thread is done2295 * with its work, we steal half of the remaining work from the2296 * thread with the largest number of unprocessed objects and give2297 * it to that newly idle thread. This ensure good load balancing2298 * until the remaining object list segments are simply too short2299 * to be worth splitting anymore.2300 */2301while(active_threads) {2302struct thread_params *target = NULL;2303struct thread_params *victim = NULL;2304unsigned sub_size =0;23052306progress_lock();2307for(;;) {2308for(i =0; !target && i < delta_search_threads; i++)2309if(!p[i].working)2310 target = &p[i];2311if(target)2312break;2313pthread_cond_wait(&progress_cond, &progress_mutex);2314}23152316for(i =0; i < delta_search_threads; i++)2317if(p[i].remaining >2*window &&2318(!victim || victim->remaining < p[i].remaining))2319 victim = &p[i];2320if(victim) {2321 sub_size = victim->remaining /2;2322 list = victim->list + victim->list_size - sub_size;2323while(sub_size && list[0]->hash &&2324 list[0]->hash == list[-1]->hash) {2325 list++;2326 sub_size--;2327}2328if(!sub_size) {2329/*2330 * It is possible for some "paths" to have2331 * so many objects that no hash boundary2332 * might be found. Let's just steal the2333 * exact half in that case.2334 */2335 sub_size = victim->remaining /2;2336 list -= sub_size;2337}2338 target->list = list;2339 victim->list_size -= sub_size;2340 victim->remaining -= sub_size;2341}2342 target->list_size = sub_size;2343 target->remaining = sub_size;2344 target->working =1;2345progress_unlock();23462347pthread_mutex_lock(&target->mutex);2348 target->data_ready =1;2349pthread_cond_signal(&target->cond);2350pthread_mutex_unlock(&target->mutex);23512352if(!sub_size) {2353pthread_join(target->thread, NULL);2354pthread_cond_destroy(&target->cond);2355pthread_mutex_destroy(&target->mutex);2356 active_threads--;2357}2358}2359cleanup_threaded_search();2360free(p);2361}23622363#else2364#define ll_find_deltas(l, s, w, d, p) find_deltas(l, &s, w, d, p)2365#endif23662367static voidadd_tag_chain(const struct object_id *oid)2368{2369struct tag *tag;23702371/*2372 * We catch duplicates already in add_object_entry(), but we'd2373 * prefer to do this extra check to avoid having to parse the2374 * tag at all if we already know that it's being packed (e.g., if2375 * it was included via bitmaps, we would not have parsed it2376 * previously).2377 */2378if(packlist_find(&to_pack, oid->hash, NULL))2379return;23802381 tag =lookup_tag(oid);2382while(1) {2383if(!tag ||parse_tag(tag) || !tag->tagged)2384die("unable to pack objects reachable from tag%s",2385oid_to_hex(oid));23862387add_object_entry(&tag->object.oid, OBJ_TAG, NULL,0);23882389if(tag->tagged->type != OBJ_TAG)2390return;23912392 tag = (struct tag *)tag->tagged;2393}2394}23952396static intadd_ref_tag(const char*path,const struct object_id *oid,int flag,void*cb_data)2397{2398struct object_id peeled;23992400if(starts_with(path,"refs/tags/") &&/* is a tag? */2401!peel_ref(path, &peeled) &&/* peelable? */2402packlist_find(&to_pack, peeled.hash, NULL))/* object packed? */2403add_tag_chain(oid);2404return0;2405}24062407static voidprepare_pack(int window,int depth)2408{2409struct object_entry **delta_list;2410uint32_t i, nr_deltas;2411unsigned n;24122413get_object_details();24142415/*2416 * If we're locally repacking then we need to be doubly careful2417 * from now on in order to make sure no stealth corruption gets2418 * propagated to the new pack. Clients receiving streamed packs2419 * should validate everything they get anyway so no need to incur2420 * the additional cost here in that case.2421 */2422if(!pack_to_stdout)2423 do_check_packed_object_crc =1;24242425if(!to_pack.nr_objects || !window || !depth)2426return;24272428ALLOC_ARRAY(delta_list, to_pack.nr_objects);2429 nr_deltas = n =0;24302431for(i =0; i < to_pack.nr_objects; i++) {2432struct object_entry *entry = to_pack.objects + i;24332434if(entry->delta)2435/* This happens if we decided to reuse existing2436 * delta from a pack. "reuse_delta &&" is implied.2437 */2438continue;24392440if(entry->size <50)2441continue;24422443if(entry->no_try_delta)2444continue;24452446if(!entry->preferred_base) {2447 nr_deltas++;2448if(oe_type(entry) <0)2449die("unable to get type of object%s",2450oid_to_hex(&entry->idx.oid));2451}else{2452if(oe_type(entry) <0) {2453/*2454 * This object is not found, but we2455 * don't have to include it anyway.2456 */2457continue;2458}2459}24602461 delta_list[n++] = entry;2462}24632464if(nr_deltas && n >1) {2465unsigned nr_done =0;2466if(progress)2467 progress_state =start_progress(_("Compressing objects"),2468 nr_deltas);2469QSORT(delta_list, n, type_size_sort);2470ll_find_deltas(delta_list, n, window+1, depth, &nr_done);2471stop_progress(&progress_state);2472if(nr_done != nr_deltas)2473die("inconsistency with delta count");2474}2475free(delta_list);2476}24772478static intgit_pack_config(const char*k,const char*v,void*cb)2479{2480if(!strcmp(k,"pack.window")) {2481 window =git_config_int(k, v);2482return0;2483}2484if(!strcmp(k,"pack.windowmemory")) {2485 window_memory_limit =git_config_ulong(k, v);2486return0;2487}2488if(!strcmp(k,"pack.depth")) {2489 depth =git_config_int(k, v);2490return0;2491}2492if(!strcmp(k,"pack.deltacachesize")) {2493 max_delta_cache_size =git_config_int(k, v);2494return0;2495}2496if(!strcmp(k,"pack.deltacachelimit")) {2497 cache_max_small_delta_size =git_config_int(k, v);2498return0;2499}2500if(!strcmp(k,"pack.writebitmaphashcache")) {2501if(git_config_bool(k, v))2502 write_bitmap_options |= BITMAP_OPT_HASH_CACHE;2503else2504 write_bitmap_options &= ~BITMAP_OPT_HASH_CACHE;2505}2506if(!strcmp(k,"pack.usebitmaps")) {2507 use_bitmap_index_default =git_config_bool(k, v);2508return0;2509}2510if(!strcmp(k,"pack.threads")) {2511 delta_search_threads =git_config_int(k, v);2512if(delta_search_threads <0)2513die("invalid number of threads specified (%d)",2514 delta_search_threads);2515#ifdef NO_PTHREADS2516if(delta_search_threads !=1) {2517warning("no threads support, ignoring%s", k);2518 delta_search_threads =0;2519}2520#endif2521return0;2522}2523if(!strcmp(k,"pack.indexversion")) {2524 pack_idx_opts.version =git_config_int(k, v);2525if(pack_idx_opts.version >2)2526die("bad pack.indexversion=%"PRIu32,2527 pack_idx_opts.version);2528return0;2529}2530returngit_default_config(k, v, cb);2531}25322533static voidread_object_list_from_stdin(void)2534{2535char line[GIT_MAX_HEXSZ +1+ PATH_MAX +2];2536struct object_id oid;2537const char*p;25382539for(;;) {2540if(!fgets(line,sizeof(line), stdin)) {2541if(feof(stdin))2542break;2543if(!ferror(stdin))2544die("fgets returned NULL, not EOF, not error!");2545if(errno != EINTR)2546die_errno("fgets");2547clearerr(stdin);2548continue;2549}2550if(line[0] =='-') {2551if(get_oid_hex(line+1, &oid))2552die("expected edge object ID, got garbage:\n%s",2553 line);2554add_preferred_base(&oid);2555continue;2556}2557if(parse_oid_hex(line, &oid, &p))2558die("expected object ID, got garbage:\n%s", line);25592560add_preferred_base_object(p +1);2561add_object_entry(&oid, OBJ_NONE, p +1,0);2562}2563}25642565/* Remember to update object flag allocation in object.h */2566#define OBJECT_ADDED (1u<<20)25672568static voidshow_commit(struct commit *commit,void*data)2569{2570add_object_entry(&commit->object.oid, OBJ_COMMIT, NULL,0);2571 commit->object.flags |= OBJECT_ADDED;25722573if(write_bitmap_index)2574index_commit_for_bitmap(commit);2575}25762577static voidshow_object(struct object *obj,const char*name,void*data)2578{2579add_preferred_base_object(name);2580add_object_entry(&obj->oid, obj->type, name,0);2581 obj->flags |= OBJECT_ADDED;2582}25832584static voidshow_object__ma_allow_any(struct object *obj,const char*name,void*data)2585{2586assert(arg_missing_action == MA_ALLOW_ANY);25872588/*2589 * Quietly ignore ALL missing objects. This avoids problems with2590 * staging them now and getting an odd error later.2591 */2592if(!has_object_file(&obj->oid))2593return;25942595show_object(obj, name, data);2596}25972598static voidshow_object__ma_allow_promisor(struct object *obj,const char*name,void*data)2599{2600assert(arg_missing_action == MA_ALLOW_PROMISOR);26012602/*2603 * Quietly ignore EXPECTED missing objects. This avoids problems with2604 * staging them now and getting an odd error later.2605 */2606if(!has_object_file(&obj->oid) &&is_promisor_object(&obj->oid))2607return;26082609show_object(obj, name, data);2610}26112612static intoption_parse_missing_action(const struct option *opt,2613const char*arg,int unset)2614{2615assert(arg);2616assert(!unset);26172618if(!strcmp(arg,"error")) {2619 arg_missing_action = MA_ERROR;2620 fn_show_object = show_object;2621return0;2622}26232624if(!strcmp(arg,"allow-any")) {2625 arg_missing_action = MA_ALLOW_ANY;2626 fetch_if_missing =0;2627 fn_show_object = show_object__ma_allow_any;2628return0;2629}26302631if(!strcmp(arg,"allow-promisor")) {2632 arg_missing_action = MA_ALLOW_PROMISOR;2633 fetch_if_missing =0;2634 fn_show_object = show_object__ma_allow_promisor;2635return0;2636}26372638die(_("invalid value for --missing"));2639return0;2640}26412642static voidshow_edge(struct commit *commit)2643{2644add_preferred_base(&commit->object.oid);2645}26462647struct in_pack_object {2648 off_t offset;2649struct object *object;2650};26512652struct in_pack {2653unsigned int alloc;2654unsigned int nr;2655struct in_pack_object *array;2656};26572658static voidmark_in_pack_object(struct object *object,struct packed_git *p,struct in_pack *in_pack)2659{2660 in_pack->array[in_pack->nr].offset =find_pack_entry_one(object->oid.hash, p);2661 in_pack->array[in_pack->nr].object = object;2662 in_pack->nr++;2663}26642665/*2666 * Compare the objects in the offset order, in order to emulate the2667 * "git rev-list --objects" output that produced the pack originally.2668 */2669static intofscmp(const void*a_,const void*b_)2670{2671struct in_pack_object *a = (struct in_pack_object *)a_;2672struct in_pack_object *b = (struct in_pack_object *)b_;26732674if(a->offset < b->offset)2675return-1;2676else if(a->offset > b->offset)2677return1;2678else2679returnoidcmp(&a->object->oid, &b->object->oid);2680}26812682static voidadd_objects_in_unpacked_packs(struct rev_info *revs)2683{2684struct packed_git *p;2685struct in_pack in_pack;2686uint32_t i;26872688memset(&in_pack,0,sizeof(in_pack));26892690for(p =get_packed_git(the_repository); p; p = p->next) {2691struct object_id oid;2692struct object *o;26932694if(!p->pack_local || p->pack_keep)2695continue;2696if(open_pack_index(p))2697die("cannot open pack index");26982699ALLOC_GROW(in_pack.array,2700 in_pack.nr + p->num_objects,2701 in_pack.alloc);27022703for(i =0; i < p->num_objects; i++) {2704nth_packed_object_oid(&oid, p, i);2705 o =lookup_unknown_object(oid.hash);2706if(!(o->flags & OBJECT_ADDED))2707mark_in_pack_object(o, p, &in_pack);2708 o->flags |= OBJECT_ADDED;2709}2710}27112712if(in_pack.nr) {2713QSORT(in_pack.array, in_pack.nr, ofscmp);2714for(i =0; i < in_pack.nr; i++) {2715struct object *o = in_pack.array[i].object;2716add_object_entry(&o->oid, o->type,"",0);2717}2718}2719free(in_pack.array);2720}27212722static intadd_loose_object(const struct object_id *oid,const char*path,2723void*data)2724{2725enum object_type type =oid_object_info(oid, NULL);27262727if(type <0) {2728warning("loose object at%scould not be examined", path);2729return0;2730}27312732add_object_entry(oid, type,"",0);2733return0;2734}27352736/*2737 * We actually don't even have to worry about reachability here.2738 * add_object_entry will weed out duplicates, so we just add every2739 * loose object we find.2740 */2741static voidadd_unreachable_loose_objects(void)2742{2743for_each_loose_file_in_objdir(get_object_directory(),2744 add_loose_object,2745 NULL, NULL, NULL);2746}27472748static inthas_sha1_pack_kept_or_nonlocal(const struct object_id *oid)2749{2750static struct packed_git *last_found = (void*)1;2751struct packed_git *p;27522753 p = (last_found != (void*)1) ? last_found :2754get_packed_git(the_repository);27552756while(p) {2757if((!p->pack_local || p->pack_keep) &&2758find_pack_entry_one(oid->hash, p)) {2759 last_found = p;2760return1;2761}2762if(p == last_found)2763 p =get_packed_git(the_repository);2764else2765 p = p->next;2766if(p == last_found)2767 p = p->next;2768}2769return0;2770}27712772/*2773 * Store a list of sha1s that are should not be discarded2774 * because they are either written too recently, or are2775 * reachable from another object that was.2776 *2777 * This is filled by get_object_list.2778 */2779static struct oid_array recent_objects;27802781static intloosened_object_can_be_discarded(const struct object_id *oid,2782 timestamp_t mtime)2783{2784if(!unpack_unreachable_expiration)2785return0;2786if(mtime > unpack_unreachable_expiration)2787return0;2788if(oid_array_lookup(&recent_objects, oid) >=0)2789return0;2790return1;2791}27922793static voidloosen_unused_packed_objects(struct rev_info *revs)2794{2795struct packed_git *p;2796uint32_t i;2797struct object_id oid;27982799for(p =get_packed_git(the_repository); p; p = p->next) {2800if(!p->pack_local || p->pack_keep)2801continue;28022803if(open_pack_index(p))2804die("cannot open pack index");28052806for(i =0; i < p->num_objects; i++) {2807nth_packed_object_oid(&oid, p, i);2808if(!packlist_find(&to_pack, oid.hash, NULL) &&2809!has_sha1_pack_kept_or_nonlocal(&oid) &&2810!loosened_object_can_be_discarded(&oid, p->mtime))2811if(force_object_loose(&oid, p->mtime))2812die("unable to force loose object");2813}2814}2815}28162817/*2818 * This tracks any options which pack-reuse code expects to be on, or which a2819 * reader of the pack might not understand, and which would therefore prevent2820 * blind reuse of what we have on disk.2821 */2822static intpack_options_allow_reuse(void)2823{2824return pack_to_stdout &&2825 allow_ofs_delta &&2826!ignore_packed_keep &&2827(!local || !have_non_local_packs) &&2828!incremental;2829}28302831static intget_object_list_from_bitmap(struct rev_info *revs)2832{2833if(prepare_bitmap_walk(revs) <0)2834return-1;28352836if(pack_options_allow_reuse() &&2837!reuse_partial_packfile_from_bitmap(2838&reuse_packfile,2839&reuse_packfile_objects,2840&reuse_packfile_offset)) {2841assert(reuse_packfile_objects);2842 nr_result += reuse_packfile_objects;2843display_progress(progress_state, nr_result);2844}28452846traverse_bitmap_commit_list(&add_object_entry_from_bitmap);2847return0;2848}28492850static voidrecord_recent_object(struct object *obj,2851const char*name,2852void*data)2853{2854oid_array_append(&recent_objects, &obj->oid);2855}28562857static voidrecord_recent_commit(struct commit *commit,void*data)2858{2859oid_array_append(&recent_objects, &commit->object.oid);2860}28612862static voidget_object_list(int ac,const char**av)2863{2864struct rev_info revs;2865char line[1000];2866int flags =0;28672868init_revisions(&revs, NULL);2869 save_commit_buffer =0;2870setup_revisions(ac, av, &revs, NULL);28712872/* make sure shallows are read */2873is_repository_shallow();28742875while(fgets(line,sizeof(line), stdin) != NULL) {2876int len =strlen(line);2877if(len && line[len -1] =='\n')2878 line[--len] =0;2879if(!len)2880break;2881if(*line =='-') {2882if(!strcmp(line,"--not")) {2883 flags ^= UNINTERESTING;2884 write_bitmap_index =0;2885continue;2886}2887if(starts_with(line,"--shallow ")) {2888struct object_id oid;2889if(get_oid_hex(line +10, &oid))2890die("not an SHA-1 '%s'", line +10);2891register_shallow(&oid);2892 use_bitmap_index =0;2893continue;2894}2895die("not a rev '%s'", line);2896}2897if(handle_revision_arg(line, &revs, flags, REVARG_CANNOT_BE_FILENAME))2898die("bad revision '%s'", line);2899}29002901if(use_bitmap_index && !get_object_list_from_bitmap(&revs))2902return;29032904if(prepare_revision_walk(&revs))2905die("revision walk setup failed");2906mark_edges_uninteresting(&revs, show_edge);29072908if(!fn_show_object)2909 fn_show_object = show_object;2910traverse_commit_list_filtered(&filter_options, &revs,2911 show_commit, fn_show_object, NULL,2912 NULL);29132914if(unpack_unreachable_expiration) {2915 revs.ignore_missing_links =1;2916if(add_unseen_recent_objects_to_traversal(&revs,2917 unpack_unreachable_expiration))2918die("unable to add recent objects");2919if(prepare_revision_walk(&revs))2920die("revision walk setup failed");2921traverse_commit_list(&revs, record_recent_commit,2922 record_recent_object, NULL);2923}29242925if(keep_unreachable)2926add_objects_in_unpacked_packs(&revs);2927if(pack_loose_unreachable)2928add_unreachable_loose_objects();2929if(unpack_unreachable)2930loosen_unused_packed_objects(&revs);29312932oid_array_clear(&recent_objects);2933}29342935static intoption_parse_index_version(const struct option *opt,2936const char*arg,int unset)2937{2938char*c;2939const char*val = arg;2940 pack_idx_opts.version =strtoul(val, &c,10);2941if(pack_idx_opts.version >2)2942die(_("unsupported index version%s"), val);2943if(*c ==','&& c[1])2944 pack_idx_opts.off32_limit =strtoul(c+1, &c,0);2945if(*c || pack_idx_opts.off32_limit &0x80000000)2946die(_("bad index version '%s'"), val);2947return0;2948}29492950static intoption_parse_unpack_unreachable(const struct option *opt,2951const char*arg,int unset)2952{2953if(unset) {2954 unpack_unreachable =0;2955 unpack_unreachable_expiration =0;2956}2957else{2958 unpack_unreachable =1;2959if(arg)2960 unpack_unreachable_expiration =approxidate(arg);2961}2962return0;2963}29642965intcmd_pack_objects(int argc,const char**argv,const char*prefix)2966{2967int use_internal_rev_list =0;2968int thin =0;2969int shallow =0;2970int all_progress_implied =0;2971struct argv_array rp = ARGV_ARRAY_INIT;2972int rev_list_unpacked =0, rev_list_all =0, rev_list_reflog =0;2973int rev_list_index =0;2974struct option pack_objects_options[] = {2975OPT_SET_INT('q',"quiet", &progress,2976N_("do not show progress meter"),0),2977OPT_SET_INT(0,"progress", &progress,2978N_("show progress meter"),1),2979OPT_SET_INT(0,"all-progress", &progress,2980N_("show progress meter during object writing phase"),2),2981OPT_BOOL(0,"all-progress-implied",2982&all_progress_implied,2983N_("similar to --all-progress when progress meter is shown")),2984{ OPTION_CALLBACK,0,"index-version", NULL,N_("version[,offset]"),2985N_("write the pack index file in the specified idx format version"),29860, option_parse_index_version },2987OPT_MAGNITUDE(0,"max-pack-size", &pack_size_limit,2988N_("maximum size of each output pack file")),2989OPT_BOOL(0,"local", &local,2990N_("ignore borrowed objects from alternate object store")),2991OPT_BOOL(0,"incremental", &incremental,2992N_("ignore packed objects")),2993OPT_INTEGER(0,"window", &window,2994N_("limit pack window by objects")),2995OPT_MAGNITUDE(0,"window-memory", &window_memory_limit,2996N_("limit pack window by memory in addition to object limit")),2997OPT_INTEGER(0,"depth", &depth,2998N_("maximum length of delta chain allowed in the resulting pack")),2999OPT_BOOL(0,"reuse-delta", &reuse_delta,3000N_("reuse existing deltas")),3001OPT_BOOL(0,"reuse-object", &reuse_object,3002N_("reuse existing objects")),3003OPT_BOOL(0,"delta-base-offset", &allow_ofs_delta,3004N_("use OFS_DELTA objects")),3005OPT_INTEGER(0,"threads", &delta_search_threads,3006N_("use threads when searching for best delta matches")),3007OPT_BOOL(0,"non-empty", &non_empty,3008N_("do not create an empty pack output")),3009OPT_BOOL(0,"revs", &use_internal_rev_list,3010N_("read revision arguments from standard input")),3011{ OPTION_SET_INT,0,"unpacked", &rev_list_unpacked, NULL,3012N_("limit the objects to those that are not yet packed"),3013 PARSE_OPT_NOARG | PARSE_OPT_NONEG, NULL,1},3014{ OPTION_SET_INT,0,"all", &rev_list_all, NULL,3015N_("include objects reachable from any reference"),3016 PARSE_OPT_NOARG | PARSE_OPT_NONEG, NULL,1},3017{ OPTION_SET_INT,0,"reflog", &rev_list_reflog, NULL,3018N_("include objects referred by reflog entries"),3019 PARSE_OPT_NOARG | PARSE_OPT_NONEG, NULL,1},3020{ OPTION_SET_INT,0,"indexed-objects", &rev_list_index, NULL,3021N_("include objects referred to by the index"),3022 PARSE_OPT_NOARG | PARSE_OPT_NONEG, NULL,1},3023OPT_BOOL(0,"stdout", &pack_to_stdout,3024N_("output pack to stdout")),3025OPT_BOOL(0,"include-tag", &include_tag,3026N_("include tag objects that refer to objects to be packed")),3027OPT_BOOL(0,"keep-unreachable", &keep_unreachable,3028N_("keep unreachable objects")),3029OPT_BOOL(0,"pack-loose-unreachable", &pack_loose_unreachable,3030N_("pack loose unreachable objects")),3031{ OPTION_CALLBACK,0,"unpack-unreachable", NULL,N_("time"),3032N_("unpack unreachable objects newer than <time>"),3033 PARSE_OPT_OPTARG, option_parse_unpack_unreachable },3034OPT_BOOL(0,"thin", &thin,3035N_("create thin packs")),3036OPT_BOOL(0,"shallow", &shallow,3037N_("create packs suitable for shallow fetches")),3038OPT_BOOL(0,"honor-pack-keep", &ignore_packed_keep,3039N_("ignore packs that have companion .keep file")),3040OPT_INTEGER(0,"compression", &pack_compression_level,3041N_("pack compression level")),3042OPT_SET_INT(0,"keep-true-parents", &grafts_replace_parents,3043N_("do not hide commits by grafts"),0),3044OPT_BOOL(0,"use-bitmap-index", &use_bitmap_index,3045N_("use a bitmap index if available to speed up counting objects")),3046OPT_BOOL(0,"write-bitmap-index", &write_bitmap_index,3047N_("write a bitmap index together with the pack index")),3048OPT_PARSE_LIST_OBJECTS_FILTER(&filter_options),3049{ OPTION_CALLBACK,0,"missing", NULL,N_("action"),3050N_("handling for missing objects"), PARSE_OPT_NONEG,3051 option_parse_missing_action },3052OPT_BOOL(0,"exclude-promisor-objects", &exclude_promisor_objects,3053N_("do not pack objects in promisor packfiles")),3054OPT_END(),3055};30563057if(DFS_NUM_STATES > (1<< OE_DFS_STATE_BITS))3058BUG("too many dfs states, increase OE_DFS_STATE_BITS");30593060 check_replace_refs =0;30613062reset_pack_idx_option(&pack_idx_opts);3063git_config(git_pack_config, NULL);30643065 progress =isatty(2);3066 argc =parse_options(argc, argv, prefix, pack_objects_options,3067 pack_usage,0);30683069if(argc) {3070 base_name = argv[0];3071 argc--;3072}3073if(pack_to_stdout != !base_name || argc)3074usage_with_options(pack_usage, pack_objects_options);30753076if(depth >= (1<< OE_DEPTH_BITS)) {3077warning(_("delta chain depth%dis too deep, forcing%d"),3078 depth, (1<< OE_DEPTH_BITS) -1);3079 depth = (1<< OE_DEPTH_BITS) -1;3080}30813082argv_array_push(&rp,"pack-objects");3083if(thin) {3084 use_internal_rev_list =1;3085argv_array_push(&rp, shallow3086?"--objects-edge-aggressive"3087:"--objects-edge");3088}else3089argv_array_push(&rp,"--objects");30903091if(rev_list_all) {3092 use_internal_rev_list =1;3093argv_array_push(&rp,"--all");3094}3095if(rev_list_reflog) {3096 use_internal_rev_list =1;3097argv_array_push(&rp,"--reflog");3098}3099if(rev_list_index) {3100 use_internal_rev_list =1;3101argv_array_push(&rp,"--indexed-objects");3102}3103if(rev_list_unpacked) {3104 use_internal_rev_list =1;3105argv_array_push(&rp,"--unpacked");3106}31073108if(exclude_promisor_objects) {3109 use_internal_rev_list =1;3110 fetch_if_missing =0;3111argv_array_push(&rp,"--exclude-promisor-objects");3112}31133114if(!reuse_object)3115 reuse_delta =0;3116if(pack_compression_level == -1)3117 pack_compression_level = Z_DEFAULT_COMPRESSION;3118else if(pack_compression_level <0|| pack_compression_level > Z_BEST_COMPRESSION)3119die("bad pack compression level%d", pack_compression_level);31203121if(!delta_search_threads)/* --threads=0 means autodetect */3122 delta_search_threads =online_cpus();31233124#ifdef NO_PTHREADS3125if(delta_search_threads !=1)3126warning("no threads support, ignoring --threads");3127#endif3128if(!pack_to_stdout && !pack_size_limit)3129 pack_size_limit = pack_size_limit_cfg;3130if(pack_to_stdout && pack_size_limit)3131die("--max-pack-size cannot be used to build a pack for transfer.");3132if(pack_size_limit && pack_size_limit <1024*1024) {3133warning("minimum pack size limit is 1 MiB");3134 pack_size_limit =1024*1024;3135}31363137if(!pack_to_stdout && thin)3138die("--thin cannot be used to build an indexable pack.");31393140if(keep_unreachable && unpack_unreachable)3141die("--keep-unreachable and --unpack-unreachable are incompatible.");3142if(!rev_list_all || !rev_list_reflog || !rev_list_index)3143 unpack_unreachable_expiration =0;31443145if(filter_options.choice) {3146if(!pack_to_stdout)3147die("cannot use --filter without --stdout.");3148 use_bitmap_index =0;3149}31503151/*3152 * "soft" reasons not to use bitmaps - for on-disk repack by default we want3153 *3154 * - to produce good pack (with bitmap index not-yet-packed objects are3155 * packed in suboptimal order).3156 *3157 * - to use more robust pack-generation codepath (avoiding possible3158 * bugs in bitmap code and possible bitmap index corruption).3159 */3160if(!pack_to_stdout)3161 use_bitmap_index_default =0;31623163if(use_bitmap_index <0)3164 use_bitmap_index = use_bitmap_index_default;31653166/* "hard" reasons not to use bitmaps; these just won't work at all */3167if(!use_internal_rev_list || (!pack_to_stdout && write_bitmap_index) ||is_repository_shallow())3168 use_bitmap_index =0;31693170if(pack_to_stdout || !rev_list_all)3171 write_bitmap_index =0;31723173if(progress && all_progress_implied)3174 progress =2;31753176if(ignore_packed_keep) {3177struct packed_git *p;3178for(p =get_packed_git(the_repository); p; p = p->next)3179if(p->pack_local && p->pack_keep)3180break;3181if(!p)/* no keep-able packs found */3182 ignore_packed_keep =0;3183}3184if(local) {3185/*3186 * unlike ignore_packed_keep above, we do not want to3187 * unset "local" based on looking at packs, as it3188 * also covers non-local objects3189 */3190struct packed_git *p;3191for(p =get_packed_git(the_repository); p; p = p->next) {3192if(!p->pack_local) {3193 have_non_local_packs =1;3194break;3195}3196}3197}31983199prepare_packing_data(&to_pack);32003201if(progress)3202 progress_state =start_progress(_("Counting objects"),0);3203if(!use_internal_rev_list)3204read_object_list_from_stdin();3205else{3206get_object_list(rp.argc, rp.argv);3207argv_array_clear(&rp);3208}3209cleanup_preferred_base();3210if(include_tag && nr_result)3211for_each_ref(add_ref_tag, NULL);3212stop_progress(&progress_state);32133214if(non_empty && !nr_result)3215return0;3216if(nr_result)3217prepare_pack(window, depth);3218write_pack_file();3219if(progress)3220fprintf(stderr,"Total %"PRIu32" (delta %"PRIu32"),"3221" reused %"PRIu32" (delta %"PRIu32")\n",3222 written, written_delta, reused, reused_delta);3223return0;3224}