1#ifndef PACK_OBJECTS_H 2#define PACK_OBJECTS_H 3 4#include"object-store.h" 5 6#define DEFAULT_DELTA_CACHE_SIZE (256 * 1024 * 1024) 7 8#define OE_DFS_STATE_BITS 2 9#define OE_DEPTH_BITS 12 10#define OE_IN_PACK_BITS 10 11#define OE_Z_DELTA_BITS 20 12/* 13 * Note that oe_set_size() becomes expensive when the given size is 14 * above this limit. Don't lower it too much. 15 */ 16#define OE_SIZE_BITS 31 17#define OE_DELTA_SIZE_BITS 20 18 19/* 20 * State flags for depth-first search used for analyzing delta cycles. 21 * 22 * The depth is measured in delta-links to the base (so if A is a delta 23 * against B, then A has a depth of 1, and B a depth of 0). 24 */ 25enum dfs_state { 26 DFS_NONE =0, 27 DFS_ACTIVE, 28 DFS_DONE, 29 DFS_NUM_STATES 30}; 31 32/* 33 * The size of struct nearly determines pack-objects's memory 34 * consumption. This struct is packed tight for that reason. When you 35 * add or reorder something in this struct, think a bit about this. 36 * 37 * basic object info 38 * ----------------- 39 * idx.oid is filled up before delta searching starts. idx.crc32 is 40 * only valid after the object is written out and will be used for 41 * generating the index. idx.offset will be both gradually set and 42 * used in writing phase (base objects get offset first, then deltas 43 * refer to them) 44 * 45 * "size" is the uncompressed object size. Compressed size of the raw 46 * data for an object in a pack is not stored anywhere but is computed 47 * and made available when reverse .idx is made. Note that when a 48 * delta is reused, "size" is the uncompressed _delta_ size, not the 49 * canonical one after the delta has been applied. 50 * 51 * "hash" contains a path name hash which is used for sorting the 52 * delta list and also during delta searching. Once prepare_pack() 53 * returns it's no longer needed. 54 * 55 * source pack info 56 * ---------------- 57 * The (in_pack, in_pack_offset) tuple contains the location of the 58 * object in the source pack. in_pack_header_size allows quickly 59 * skipping the header and going straight to the zlib stream. 60 * 61 * "type" and "in_pack_type" both describe object type. in_pack_type 62 * may contain a delta type, while type is always the canonical type. 63 * 64 * deltas 65 * ------ 66 * Delta links (delta, delta_child and delta_sibling) are created to 67 * reflect that delta graph from the source pack then updated or added 68 * during delta searching phase when we find better deltas. 69 * 70 * delta_child and delta_sibling are last needed in 71 * compute_write_order(). "delta" and "delta_size" must remain valid 72 * at object writing phase in case the delta is not cached. 73 * 74 * If a delta is cached in memory and is compressed, delta_data points 75 * to the data and z_delta_size contains the compressed size. If it's 76 * uncompressed [1], z_delta_size must be zero. delta_size is always 77 * the uncompressed size and must be valid even if the delta is not 78 * cached. 79 * 80 * [1] during try_delta phase we don't bother with compressing because 81 * the delta could be quickly replaced with a better one. 82 */ 83struct object_entry { 84struct pack_idx_entry idx; 85void*delta_data;/* cached delta (uncompressed) */ 86 off_t in_pack_offset; 87uint32_t hash;/* name hint hash */ 88unsigned size_:OE_SIZE_BITS; 89unsigned size_valid:1; 90uint32_t delta_idx;/* delta base object */ 91uint32_t delta_child_idx;/* deltified objects who bases me */ 92uint32_t delta_sibling_idx;/* other deltified objects who 93 * uses the same base as me 94 */ 95unsigned delta_size_:OE_DELTA_SIZE_BITS;/* delta data size (uncompressed) */ 96unsigned delta_size_valid:1; 97unsigned in_pack_idx:OE_IN_PACK_BITS;/* already in pack */ 98unsigned z_delta_size:OE_Z_DELTA_BITS; 99unsigned type_valid:1; 100unsigned type_:TYPE_BITS; 101unsigned no_try_delta:1; 102unsigned in_pack_type:TYPE_BITS;/* could be delta */ 103unsigned preferred_base:1;/* 104 * we do not pack this, but is available 105 * to be used as the base object to delta 106 * objects against. 107 */ 108unsigned tagged:1;/* near the very tip of refs */ 109unsigned filled:1;/* assigned write-order */ 110unsigned dfs_state:OE_DFS_STATE_BITS; 111unsigned char in_pack_header_size; 112unsigned depth:OE_DEPTH_BITS; 113 114/* 115 * pahole results on 64-bit linux (gcc and clang) 116 * 117 * size: 80, bit_padding: 20 bits, holes: 8 bits 118 * 119 * and on 32-bit (gcc) 120 * 121 * size: 76, bit_padding: 20 bits, holes: 8 bits 122 */ 123}; 124 125struct packing_data { 126struct object_entry *objects; 127uint32_t nr_objects, nr_alloc; 128 129int32_t*index; 130uint32_t index_size; 131 132unsigned int*in_pack_pos; 133 134/* 135 * Only one of these can be non-NULL and they have different 136 * sizes. if in_pack_by_idx is allocated, oe_in_pack() returns 137 * the pack of an object using in_pack_idx field. If not, 138 * in_pack[] array is used the same way as in_pack_pos[] 139 */ 140struct packed_git **in_pack_by_idx; 141struct packed_git **in_pack; 142 143uintmax_t oe_size_limit; 144}; 145 146voidprepare_packing_data(struct packing_data *pdata); 147struct object_entry *packlist_alloc(struct packing_data *pdata, 148const unsigned char*sha1, 149uint32_t index_pos); 150 151struct object_entry *packlist_find(struct packing_data *pdata, 152const unsigned char*sha1, 153uint32_t*index_pos); 154 155staticinlineuint32_tpack_name_hash(const char*name) 156{ 157uint32_t c, hash =0; 158 159if(!name) 160return0; 161 162/* 163 * This effectively just creates a sortable number from the 164 * last sixteen non-whitespace characters. Last characters 165 * count "most", so things that end in ".c" sort together. 166 */ 167while((c = *name++) !=0) { 168if(isspace(c)) 169continue; 170 hash = (hash >>2) + (c <<24); 171} 172return hash; 173} 174 175staticinlineenum object_type oe_type(const struct object_entry *e) 176{ 177return e->type_valid ? e->type_ : OBJ_BAD; 178} 179 180staticinlinevoidoe_set_type(struct object_entry *e, 181enum object_type type) 182{ 183if(type >= OBJ_ANY) 184BUG("OBJ_ANY cannot be set in pack-objects code"); 185 186 e->type_valid = type >= OBJ_NONE; 187 e->type_ = (unsigned)type; 188} 189 190staticinlineunsigned intoe_in_pack_pos(const struct packing_data *pack, 191const struct object_entry *e) 192{ 193return pack->in_pack_pos[e - pack->objects]; 194} 195 196staticinlinevoidoe_set_in_pack_pos(const struct packing_data *pack, 197const struct object_entry *e, 198unsigned int pos) 199{ 200 pack->in_pack_pos[e - pack->objects] = pos; 201} 202 203staticinlinestruct packed_git *oe_in_pack(const struct packing_data *pack, 204const struct object_entry *e) 205{ 206if(pack->in_pack_by_idx) 207return pack->in_pack_by_idx[e->in_pack_idx]; 208else 209return pack->in_pack[e - pack->objects]; 210} 211 212voidoe_map_new_pack(struct packing_data *pack, 213struct packed_git *p); 214staticinlinevoidoe_set_in_pack(struct packing_data *pack, 215struct object_entry *e, 216struct packed_git *p) 217{ 218if(!p->index) 219oe_map_new_pack(pack, p); 220if(pack->in_pack_by_idx) 221 e->in_pack_idx = p->index; 222else 223 pack->in_pack[e - pack->objects] = p; 224} 225 226staticinlinestruct object_entry *oe_delta( 227const struct packing_data *pack, 228const struct object_entry *e) 229{ 230if(e->delta_idx) 231return&pack->objects[e->delta_idx -1]; 232return NULL; 233} 234 235staticinlinevoidoe_set_delta(struct packing_data *pack, 236struct object_entry *e, 237struct object_entry *delta) 238{ 239if(delta) 240 e->delta_idx = (delta - pack->objects) +1; 241else 242 e->delta_idx =0; 243} 244 245staticinlinestruct object_entry *oe_delta_child( 246const struct packing_data *pack, 247const struct object_entry *e) 248{ 249if(e->delta_child_idx) 250return&pack->objects[e->delta_child_idx -1]; 251return NULL; 252} 253 254staticinlinevoidoe_set_delta_child(struct packing_data *pack, 255struct object_entry *e, 256struct object_entry *delta) 257{ 258if(delta) 259 e->delta_child_idx = (delta - pack->objects) +1; 260else 261 e->delta_child_idx =0; 262} 263 264staticinlinestruct object_entry *oe_delta_sibling( 265const struct packing_data *pack, 266const struct object_entry *e) 267{ 268if(e->delta_sibling_idx) 269return&pack->objects[e->delta_sibling_idx -1]; 270return NULL; 271} 272 273staticinlinevoidoe_set_delta_sibling(struct packing_data *pack, 274struct object_entry *e, 275struct object_entry *delta) 276{ 277if(delta) 278 e->delta_sibling_idx = (delta - pack->objects) +1; 279else 280 e->delta_sibling_idx =0; 281} 282 283unsigned longoe_get_size_slow(struct packing_data *pack, 284const struct object_entry *e); 285staticinlineunsigned longoe_size(struct packing_data *pack, 286const struct object_entry *e) 287{ 288if(e->size_valid) 289return e->size_; 290 291returnoe_get_size_slow(pack, e); 292} 293 294staticinlineintoe_size_less_than(struct packing_data *pack, 295const struct object_entry *lhs, 296unsigned long rhs) 297{ 298if(lhs->size_valid) 299return lhs->size_ < rhs; 300if(rhs < pack->oe_size_limit)/* rhs < 2^x <= lhs ? */ 301return0; 302returnoe_get_size_slow(pack, lhs) < rhs; 303} 304 305staticinlineintoe_size_greater_than(struct packing_data *pack, 306const struct object_entry *lhs, 307unsigned long rhs) 308{ 309if(lhs->size_valid) 310return lhs->size_ > rhs; 311if(rhs < pack->oe_size_limit)/* rhs < 2^x <= lhs ? */ 312return1; 313returnoe_get_size_slow(pack, lhs) > rhs; 314} 315 316staticinlinevoidoe_set_size(struct packing_data *pack, 317struct object_entry *e, 318unsigned long size) 319{ 320if(size < pack->oe_size_limit) { 321 e->size_ = size; 322 e->size_valid =1; 323}else{ 324 e->size_valid =0; 325if(oe_get_size_slow(pack, e) != size) 326BUG("'size' is supposed to be the object size!"); 327} 328} 329 330staticinlineunsigned longoe_delta_size(struct packing_data *pack, 331const struct object_entry *e) 332{ 333if(e->delta_size_valid) 334return e->delta_size_; 335returnoe_size(pack, e); 336} 337 338staticinlinevoidoe_set_delta_size(struct packing_data *pack, 339struct object_entry *e, 340unsigned long size) 341{ 342 e->delta_size_ = size; 343 e->delta_size_valid = e->delta_size_ == size; 344if(!e->delta_size_valid && size !=oe_size(pack, e)) 345BUG("this can only happen in check_object() " 346"where delta size is the same as entry size"); 347} 348 349#endif