1#ifndef PACK_OBJECTS_H 2#define PACK_OBJECTS_H 3 4#include"object-store.h" 5#include"pack.h" 6 7#define DEFAULT_DELTA_CACHE_SIZE (256 * 1024 * 1024) 8 9#define OE_DFS_STATE_BITS 2 10#define OE_DEPTH_BITS 12 11#define OE_IN_PACK_BITS 10 12#define OE_Z_DELTA_BITS 20 13/* 14 * Note that oe_set_size() becomes expensive when the given size is 15 * above this limit. Don't lower it too much. 16 */ 17#define OE_SIZE_BITS 31 18#define OE_DELTA_SIZE_BITS 20 19 20/* 21 * State flags for depth-first search used for analyzing delta cycles. 22 * 23 * The depth is measured in delta-links to the base (so if A is a delta 24 * against B, then A has a depth of 1, and B a depth of 0). 25 */ 26enum dfs_state { 27 DFS_NONE =0, 28 DFS_ACTIVE, 29 DFS_DONE, 30 DFS_NUM_STATES 31}; 32 33/* 34 * The size of struct nearly determines pack-objects's memory 35 * consumption. This struct is packed tight for that reason. When you 36 * add or reorder something in this struct, think a bit about this. 37 * 38 * basic object info 39 * ----------------- 40 * idx.oid is filled up before delta searching starts. idx.crc32 is 41 * only valid after the object is written out and will be used for 42 * generating the index. idx.offset will be both gradually set and 43 * used in writing phase (base objects get offset first, then deltas 44 * refer to them) 45 * 46 * "size" is the uncompressed object size. Compressed size of the raw 47 * data for an object in a pack is not stored anywhere but is computed 48 * and made available when reverse .idx is made. Note that when a 49 * delta is reused, "size" is the uncompressed _delta_ size, not the 50 * canonical one after the delta has been applied. 51 * 52 * "hash" contains a path name hash which is used for sorting the 53 * delta list and also during delta searching. Once prepare_pack() 54 * returns it's no longer needed. 55 * 56 * source pack info 57 * ---------------- 58 * The (in_pack, in_pack_offset) tuple contains the location of the 59 * object in the source pack. in_pack_header_size allows quickly 60 * skipping the header and going straight to the zlib stream. 61 * 62 * "type" and "in_pack_type" both describe object type. in_pack_type 63 * may contain a delta type, while type is always the canonical type. 64 * 65 * deltas 66 * ------ 67 * Delta links (delta, delta_child and delta_sibling) are created to 68 * reflect that delta graph from the source pack then updated or added 69 * during delta searching phase when we find better deltas. 70 * 71 * delta_child and delta_sibling are last needed in 72 * compute_write_order(). "delta" and "delta_size" must remain valid 73 * at object writing phase in case the delta is not cached. 74 * 75 * If a delta is cached in memory and is compressed, delta_data points 76 * to the data and z_delta_size contains the compressed size. If it's 77 * uncompressed [1], z_delta_size must be zero. delta_size is always 78 * the uncompressed size and must be valid even if the delta is not 79 * cached. 80 * 81 * [1] during try_delta phase we don't bother with compressing because 82 * the delta could be quickly replaced with a better one. 83 */ 84struct object_entry { 85struct pack_idx_entry idx; 86void*delta_data;/* cached delta (uncompressed) */ 87 off_t in_pack_offset; 88uint32_t hash;/* name hint hash */ 89unsigned size_:OE_SIZE_BITS; 90unsigned size_valid:1; 91uint32_t delta_idx;/* delta base object */ 92uint32_t delta_child_idx;/* deltified objects who bases me */ 93uint32_t delta_sibling_idx;/* other deltified objects who 94 * uses the same base as me 95 */ 96unsigned delta_size_:OE_DELTA_SIZE_BITS;/* delta data size (uncompressed) */ 97unsigned delta_size_valid:1; 98unsigned in_pack_idx:OE_IN_PACK_BITS;/* already in pack */ 99unsigned z_delta_size:OE_Z_DELTA_BITS; 100unsigned type_valid:1; 101unsigned type_:TYPE_BITS; 102unsigned no_try_delta:1; 103unsigned in_pack_type:TYPE_BITS;/* could be delta */ 104unsigned preferred_base:1;/* 105 * we do not pack this, but is available 106 * to be used as the base object to delta 107 * objects against. 108 */ 109unsigned tagged:1;/* near the very tip of refs */ 110unsigned filled:1;/* assigned write-order */ 111unsigned dfs_state:OE_DFS_STATE_BITS; 112unsigned char in_pack_header_size; 113unsigned depth:OE_DEPTH_BITS; 114unsigned ext_base:1;/* delta_idx points outside packlist */ 115 116/* 117 * pahole results on 64-bit linux (gcc and clang) 118 * 119 * size: 80, bit_padding: 20 bits, holes: 8 bits 120 * 121 * and on 32-bit (gcc) 122 * 123 * size: 76, bit_padding: 20 bits, holes: 8 bits 124 */ 125}; 126 127struct packing_data { 128struct object_entry *objects; 129uint32_t nr_objects, nr_alloc; 130 131int32_t*index; 132uint32_t index_size; 133 134unsigned int*in_pack_pos; 135 136/* 137 * Only one of these can be non-NULL and they have different 138 * sizes. if in_pack_by_idx is allocated, oe_in_pack() returns 139 * the pack of an object using in_pack_idx field. If not, 140 * in_pack[] array is used the same way as in_pack_pos[] 141 */ 142struct packed_git **in_pack_by_idx; 143struct packed_git **in_pack; 144 145/* 146 * This list contains entries for bases which we know the other side 147 * has (e.g., via reachability bitmaps), but which aren't in our 148 * "objects" list. 149 */ 150struct object_entry *ext_bases; 151uint32_t nr_ext, alloc_ext; 152 153uintmax_t oe_size_limit; 154}; 155 156voidprepare_packing_data(struct packing_data *pdata); 157struct object_entry *packlist_alloc(struct packing_data *pdata, 158const unsigned char*sha1, 159uint32_t index_pos); 160 161struct object_entry *packlist_find(struct packing_data *pdata, 162const unsigned char*sha1, 163uint32_t*index_pos); 164 165staticinlineuint32_tpack_name_hash(const char*name) 166{ 167uint32_t c, hash =0; 168 169if(!name) 170return0; 171 172/* 173 * This effectively just creates a sortable number from the 174 * last sixteen non-whitespace characters. Last characters 175 * count "most", so things that end in ".c" sort together. 176 */ 177while((c = *name++) !=0) { 178if(isspace(c)) 179continue; 180 hash = (hash >>2) + (c <<24); 181} 182return hash; 183} 184 185staticinlineenum object_type oe_type(const struct object_entry *e) 186{ 187return e->type_valid ? e->type_ : OBJ_BAD; 188} 189 190staticinlinevoidoe_set_type(struct object_entry *e, 191enum object_type type) 192{ 193if(type >= OBJ_ANY) 194BUG("OBJ_ANY cannot be set in pack-objects code"); 195 196 e->type_valid = type >= OBJ_NONE; 197 e->type_ = (unsigned)type; 198} 199 200staticinlineunsigned intoe_in_pack_pos(const struct packing_data *pack, 201const struct object_entry *e) 202{ 203return pack->in_pack_pos[e - pack->objects]; 204} 205 206staticinlinevoidoe_set_in_pack_pos(const struct packing_data *pack, 207const struct object_entry *e, 208unsigned int pos) 209{ 210 pack->in_pack_pos[e - pack->objects] = pos; 211} 212 213staticinlinestruct packed_git *oe_in_pack(const struct packing_data *pack, 214const struct object_entry *e) 215{ 216if(pack->in_pack_by_idx) 217return pack->in_pack_by_idx[e->in_pack_idx]; 218else 219return pack->in_pack[e - pack->objects]; 220} 221 222voidoe_map_new_pack(struct packing_data *pack, 223struct packed_git *p); 224staticinlinevoidoe_set_in_pack(struct packing_data *pack, 225struct object_entry *e, 226struct packed_git *p) 227{ 228if(!p->index) 229oe_map_new_pack(pack, p); 230if(pack->in_pack_by_idx) 231 e->in_pack_idx = p->index; 232else 233 pack->in_pack[e - pack->objects] = p; 234} 235 236staticinlinestruct object_entry *oe_delta( 237const struct packing_data *pack, 238const struct object_entry *e) 239{ 240if(!e->delta_idx) 241return NULL; 242if(e->ext_base) 243return&pack->ext_bases[e->delta_idx -1]; 244else 245return&pack->objects[e->delta_idx -1]; 246} 247 248staticinlinevoidoe_set_delta(struct packing_data *pack, 249struct object_entry *e, 250struct object_entry *delta) 251{ 252if(delta) 253 e->delta_idx = (delta - pack->objects) +1; 254else 255 e->delta_idx =0; 256} 257 258voidoe_set_delta_ext(struct packing_data *pack, 259struct object_entry *e, 260const unsigned char*sha1); 261 262staticinlinestruct object_entry *oe_delta_child( 263const struct packing_data *pack, 264const struct object_entry *e) 265{ 266if(e->delta_child_idx) 267return&pack->objects[e->delta_child_idx -1]; 268return NULL; 269} 270 271staticinlinevoidoe_set_delta_child(struct packing_data *pack, 272struct object_entry *e, 273struct object_entry *delta) 274{ 275if(delta) 276 e->delta_child_idx = (delta - pack->objects) +1; 277else 278 e->delta_child_idx =0; 279} 280 281staticinlinestruct object_entry *oe_delta_sibling( 282const struct packing_data *pack, 283const struct object_entry *e) 284{ 285if(e->delta_sibling_idx) 286return&pack->objects[e->delta_sibling_idx -1]; 287return NULL; 288} 289 290staticinlinevoidoe_set_delta_sibling(struct packing_data *pack, 291struct object_entry *e, 292struct object_entry *delta) 293{ 294if(delta) 295 e->delta_sibling_idx = (delta - pack->objects) +1; 296else 297 e->delta_sibling_idx =0; 298} 299 300unsigned longoe_get_size_slow(struct packing_data *pack, 301const struct object_entry *e); 302staticinlineunsigned longoe_size(struct packing_data *pack, 303const struct object_entry *e) 304{ 305if(e->size_valid) 306return e->size_; 307 308returnoe_get_size_slow(pack, e); 309} 310 311staticinlineintoe_size_less_than(struct packing_data *pack, 312const struct object_entry *lhs, 313unsigned long rhs) 314{ 315if(lhs->size_valid) 316return lhs->size_ < rhs; 317if(rhs < pack->oe_size_limit)/* rhs < 2^x <= lhs ? */ 318return0; 319returnoe_get_size_slow(pack, lhs) < rhs; 320} 321 322staticinlineintoe_size_greater_than(struct packing_data *pack, 323const struct object_entry *lhs, 324unsigned long rhs) 325{ 326if(lhs->size_valid) 327return lhs->size_ > rhs; 328if(rhs < pack->oe_size_limit)/* rhs < 2^x <= lhs ? */ 329return1; 330returnoe_get_size_slow(pack, lhs) > rhs; 331} 332 333staticinlinevoidoe_set_size(struct packing_data *pack, 334struct object_entry *e, 335unsigned long size) 336{ 337if(size < pack->oe_size_limit) { 338 e->size_ = size; 339 e->size_valid =1; 340}else{ 341 e->size_valid =0; 342if(oe_get_size_slow(pack, e) != size) 343BUG("'size' is supposed to be the object size!"); 344} 345} 346 347staticinlineunsigned longoe_delta_size(struct packing_data *pack, 348const struct object_entry *e) 349{ 350if(e->delta_size_valid) 351return e->delta_size_; 352returnoe_size(pack, e); 353} 354 355staticinlinevoidoe_set_delta_size(struct packing_data *pack, 356struct object_entry *e, 357unsigned long size) 358{ 359 e->delta_size_ = size; 360 e->delta_size_valid = e->delta_size_ == size; 361if(!e->delta_size_valid && size !=oe_size(pack, e)) 362BUG("this can only happen in check_object() " 363"where delta size is the same as entry size"); 364} 365 366#endif