From: Junio C Hamano Date: Fri, 5 Aug 2011 21:54:57 +0000 (-0700) Subject: Merge branch 'jc/pack-order-tweak' X-Git-Tag: v1.7.7-rc0~52 X-Git-Url: https://git.lorimer.id.au/gitweb.git/diff_plain/96790ca02935791b302cd65a4d092f4cdf5cd21e?ds=inline;hp=-c Merge branch 'jc/pack-order-tweak' * jc/pack-order-tweak: pack-objects: optimize "recency order" core: log offset pack data accesses happened --- 96790ca02935791b302cd65a4d092f4cdf5cd21e diff --combined builtin/pack-objects.c index 84e6dafb12,27132bb931..27f24d3aaf --- a/builtin/pack-objects.c +++ b/builtin/pack-objects.c @@@ -51,6 -51,8 +51,8 @@@ struct object_entry * objects against. */ unsigned char no_try_delta; + unsigned char tagged; /* near the very tip of refs */ + unsigned char filled; /* assigned write-order */ }; /* @@@ -70,7 -72,6 +72,7 @@@ static int local static int incremental; static int ignore_packed_keep; static int allow_ofs_delta; +static struct pack_idx_option pack_idx_opts; static const char *base_name; static int progress = 1; static int window = 10; @@@ -96,6 -97,7 +98,7 @@@ static unsigned long window_memory_limi */ static int *object_ix; static int object_ix_hashsz; + static struct object_entry *locate_object_entry(const unsigned char *sha1); /* * stats @@@ -127,13 -129,13 +130,13 @@@ static void *get_delta(struct object_en static unsigned long do_compress(void **pptr, unsigned long size) { - z_stream stream; + git_zstream stream; void *in, *out; unsigned long maxsize; memset(&stream, 0, sizeof(stream)); - deflateInit(&stream, pack_compression_level); - maxsize = deflateBound(&stream, size); + git_deflate_init(&stream, pack_compression_level); + maxsize = git_deflate_bound(&stream, size); in = *pptr; out = xmalloc(maxsize); @@@ -143,9 -145,9 +146,9 @@@ stream.avail_in = size; stream.next_out = out; stream.avail_out = maxsize; - while (deflate(&stream, Z_FINISH) == Z_OK) + while (git_deflate(&stream, Z_FINISH) == Z_OK) ; /* nothing */ - deflateEnd(&stream); + git_deflate_end(&stream); free(in); return stream.total_out; @@@ -161,7 -163,7 +164,7 @@@ static int check_pack_inflate(struct pa off_t len, unsigned long expect) { - z_stream stream; + git_zstream stream; unsigned char fakebuf[4096], *in; int st; @@@ -188,18 -190,19 +191,19 @@@ static void copy_pack_data(struct sha1f off_t len) { unsigned char *in; - unsigned int avail; + unsigned long avail; while (len) { in = use_pack(p, w_curs, offset, &avail); if (avail > len) - avail = (unsigned int)len; + avail = (unsigned long)len; sha1write(f, in, avail); offset += avail; len -= avail; } } + /* Return 0 if we will bust the pack-size limit */ static unsigned long write_object(struct sha1file *f, struct object_entry *entry, off_t write_offset) @@@ -434,6 -437,134 +438,134 @@@ static int write_one(struct sha1file *f return 1; } + static int mark_tagged(const char *path, const unsigned char *sha1, int flag, + void *cb_data) + { + unsigned char peeled[20]; + struct object_entry *entry = locate_object_entry(sha1); + + if (entry) + entry->tagged = 1; + if (!peel_ref(path, peeled)) { + entry = locate_object_entry(peeled); + if (entry) + entry->tagged = 1; + } + return 0; + } + + static void add_to_write_order(struct object_entry **wo, + int *endp, + struct object_entry *e) + { + if (e->filled) + return; + wo[(*endp)++] = e; + e->filled = 1; + } + + static void add_descendants_to_write_order(struct object_entry **wo, + int *endp, + struct object_entry *e) + { + struct object_entry *child; + + for (child = e->delta_child; child; child = child->delta_sibling) + add_to_write_order(wo, endp, child); + for (child = e->delta_child; child; child = child->delta_sibling) + add_descendants_to_write_order(wo, endp, child); + } + + static void add_family_to_write_order(struct object_entry **wo, + int *endp, + struct object_entry *e) + { + struct object_entry *root; + + for (root = e; root->delta; root = root->delta) + ; /* nothing */ + add_to_write_order(wo, endp, root); + add_descendants_to_write_order(wo, endp, root); + } + + static struct object_entry **compute_write_order(void) + { + int i, wo_end; + + struct object_entry **wo = xmalloc(nr_objects * sizeof(*wo)); + + for (i = 0; i < nr_objects; i++) { + objects[i].tagged = 0; + objects[i].filled = 0; + objects[i].delta_child = NULL; + objects[i].delta_sibling = NULL; + } + + /* + * Fully connect delta_child/delta_sibling network. + * Make sure delta_sibling is sorted in the original + * recency order. + */ + for (i = nr_objects - 1; 0 <= i; i--) { + struct object_entry *e = &objects[i]; + if (!e->delta) + continue; + /* Mark me as the first child */ + e->delta_sibling = e->delta->delta_child; + e->delta->delta_child = e; + } + + /* + * Mark objects that are at the tip of tags. + */ + for_each_tag_ref(mark_tagged, NULL); + + /* + * Give the commits in the original recency order until + * we see a tagged tip. + */ + for (i = wo_end = 0; i < nr_objects; i++) { + if (objects[i].tagged) + break; + add_to_write_order(wo, &wo_end, &objects[i]); + } + + /* + * Then fill all the tagged tips. + */ + for (; i < nr_objects; i++) { + if (objects[i].tagged) + add_to_write_order(wo, &wo_end, &objects[i]); + } + + /* + * And then all remaining commits and tags. + */ + for (i = 0; i < nr_objects; i++) { + if (objects[i].type != OBJ_COMMIT && + objects[i].type != OBJ_TAG) + continue; + add_to_write_order(wo, &wo_end, &objects[i]); + } + + /* + * And then all the trees. + */ + for (i = 0; i < nr_objects; i++) { + if (objects[i].type != OBJ_TREE) + continue; + add_to_write_order(wo, &wo_end, &objects[i]); + } + + /* + * Finally all the rest in really tight order + */ + for (i = 0; i < nr_objects; i++) + add_family_to_write_order(wo, &wo_end, &objects[i]); + + return wo; + } + static void write_pack_file(void) { uint32_t i = 0, j; @@@ -442,10 -573,12 +574,12 @@@ struct pack_header hdr; uint32_t nr_remaining = nr_result; time_t last_mtime = 0; + struct object_entry **write_order; if (progress > pack_to_stdout) progress_state = start_progress("Writing objects", nr_result); written_list = xmalloc(nr_objects * sizeof(*written_list)); + write_order = compute_write_order(); do { unsigned char sha1[20]; @@@ -469,7 -602,8 +603,8 @@@ offset = sizeof(hdr); nr_written = 0; for (; i < nr_objects; i++) { - if (!write_one(f, objects + i, &offset)) + struct object_entry *e = write_order[i]; + if (!write_one(f, e, &offset)) break; display_progress(progress_state, written); } @@@ -494,8 -628,8 +629,8 @@@ const char *idx_tmp_name; char tmpname[PATH_MAX]; - idx_tmp_name = write_idx_file(NULL, written_list, - nr_written, sha1); + idx_tmp_name = write_idx_file(NULL, written_list, nr_written, + &pack_idx_opts, sha1); snprintf(tmpname, sizeof(tmpname), "%s-%s.pack", base_name, sha1_to_hex(sha1)); @@@ -546,6 -680,7 +681,7 @@@ } while (nr_remaining && i < nr_objects); free(written_list); + free(write_order); stop_progress(&progress_state); if (written != nr_result) die("wrote %"PRIu32" objects while expecting %"PRIu32, @@@ -995,7 -1130,7 +1131,7 @@@ static void check_object(struct object_ const unsigned char *base_ref = NULL; struct object_entry *base_entry; unsigned long used, used_0; - unsigned int avail; + unsigned long avail; off_t ofs; unsigned char *buf, c; @@@ -1885,10 -2020,10 +2021,10 @@@ static int git_pack_config(const char * return 0; } if (!strcmp(k, "pack.indexversion")) { - pack_idx_default_version = git_config_int(k, v); - if (pack_idx_default_version > 2) + pack_idx_opts.version = git_config_int(k, v); + if (pack_idx_opts.version > 2) die("bad pack.indexversion=%"PRIu32, - pack_idx_default_version); + pack_idx_opts.version); return 0; } if (!strcmp(k, "pack.packsizelimit")) { @@@ -2135,7 -2270,6 +2271,7 @@@ int cmd_pack_objects(int argc, const ch rp_av[1] = "--objects"; /* --thin will make it --objects-edge */ rp_ac = 2; + reset_pack_idx_option(&pack_idx_opts); git_config(git_pack_config, NULL); if (!pack_compression_seen && core_compression_seen) pack_compression_level = core_compression_level; @@@ -2280,12 -2414,12 +2416,12 @@@ } if (!prefixcmp(arg, "--index-version=")) { char *c; - pack_idx_default_version = strtoul(arg + 16, &c, 10); - if (pack_idx_default_version > 2) + pack_idx_opts.version = strtoul(arg + 16, &c, 10); + if (pack_idx_opts.version > 2) die("bad %s", arg); if (*c == ',') - pack_idx_off32_limit = strtoul(c+1, &c, 0); - if (*c || pack_idx_off32_limit & 0x80000000) + pack_idx_opts.off32_limit = strtoul(c+1, &c, 0); + if (*c || pack_idx_opts.off32_limit & 0x80000000) die("bad %s", arg); continue; } diff --combined cache.h index 9e12d55470,16a8c7c23c..f49eaf9263 --- a/cache.h +++ b/cache.h @@@ -6,7 -6,6 +6,7 @@@ #include "hash.h" #include "advice.h" #include "gettext.h" +#include "convert.h" #include SHA1_HEADER #ifndef git_SHA_CTX @@@ -17,27 -16,13 +17,27 @@@ #endif #include -#if defined(NO_DEFLATE_BOUND) || ZLIB_VERNUM < 0x1200 -#define deflateBound(c,s) ((s) + (((s) + 7) >> 3) + (((s) + 63) >> 6) + 11) -#endif - -void git_inflate_init(z_streamp strm); -void git_inflate_end(z_streamp strm); -int git_inflate(z_streamp strm, int flush); +typedef struct git_zstream { + z_stream z; + unsigned long avail_in; + unsigned long avail_out; + unsigned long total_in; + unsigned long total_out; + unsigned char *next_in; + unsigned char *next_out; +} git_zstream; + +void git_inflate_init(git_zstream *); +void git_inflate_init_gzip_only(git_zstream *); +void git_inflate_end(git_zstream *); +int git_inflate(git_zstream *, int flush); + +void git_deflate_init(git_zstream *, int level); +void git_deflate_init_gzip(git_zstream *, int level); +void git_deflate_end(git_zstream *); +int git_deflate_end_gently(git_zstream *); +int git_deflate(git_zstream *, int flush); +unsigned long git_deflate_bound(git_zstream *, unsigned long); #if defined(DT_UNKNOWN) && !defined(NO_D_TYPE_IN_DIRENT) #define DTYPE(de) ((de)->d_type) @@@ -597,6 -582,35 +597,6 @@@ extern int fsync_object_files extern int core_preload_index; extern int core_apply_sparse_checkout; -enum safe_crlf { - SAFE_CRLF_FALSE = 0, - SAFE_CRLF_FAIL = 1, - SAFE_CRLF_WARN = 2 -}; - -extern enum safe_crlf safe_crlf; - -enum auto_crlf { - AUTO_CRLF_FALSE = 0, - AUTO_CRLF_TRUE = 1, - AUTO_CRLF_INPUT = -1 -}; - -extern enum auto_crlf auto_crlf; - -enum eol { - EOL_UNSET, - EOL_CRLF, - EOL_LF, -#ifdef NATIVE_CRLF - EOL_NATIVE = EOL_CRLF -#else - EOL_NATIVE = EOL_LF -#endif -}; - -extern enum eol core_eol; - enum branch_track { BRANCH_TRACK_UNSPECIFIED = -1, BRANCH_TRACK_NEVER = 0, @@@ -766,13 -780,13 +766,16 @@@ extern int hash_sha1_file(const void *b extern int write_sha1_file(const void *buf, unsigned long len, const char *type, unsigned char *return_sha1); extern int pretend_sha1_file(void *, unsigned long, enum object_type, unsigned char *); extern int force_object_loose(const unsigned char *sha1, time_t mtime); +extern void *map_sha1_file(const unsigned char *sha1, unsigned long *size); +extern int unpack_sha1_header(git_zstream *stream, unsigned char *map, unsigned long mapsize, void *buffer, unsigned long bufsiz); +extern int parse_sha1_header(const char *hdr, unsigned long *sizep); /* global flag to enable extra checks when accessing packed objects */ extern int do_check_packed_object_crc; + /* for development: log offset of pack access */ + extern const char *log_pack_access; + extern int check_sha1_signature(const unsigned char *sha1, void *buf, unsigned long size, const char *type); extern int move_temp_to_file(const char *tmpfile, const char *filename); @@@ -998,7 -1012,7 +1001,7 @@@ extern struct packed_git *find_sha1_pac extern void pack_report(void); extern int open_pack_index(struct packed_git *); extern void close_pack_index(struct packed_git *); -extern unsigned char *use_pack(struct packed_git *, struct pack_window **, off_t, unsigned int *); +extern unsigned char *use_pack(struct packed_git *, struct pack_window **, off_t, unsigned long *); extern void close_pack_windows(struct packed_git *); extern void unuse_pack(struct pack_window **); extern void free_pack_by_name(const char *); @@@ -1010,36 -1024,7 +1013,36 @@@ extern off_t find_pack_entry_one(const extern void *unpack_entry(struct packed_git *, off_t, enum object_type *, unsigned long *); extern unsigned long unpack_object_header_buffer(const unsigned char *buf, unsigned long len, enum object_type *type, unsigned long *sizep); extern unsigned long get_size_from_delta(struct packed_git *, struct pack_window **, off_t); -extern const char *packed_object_info_detail(struct packed_git *, off_t, unsigned long *, unsigned long *, unsigned int *, unsigned char *); +extern int unpack_object_header(struct packed_git *, struct pack_window **, off_t *, unsigned long *); + +struct object_info { + /* Request */ + unsigned long *sizep; + + /* Response */ + enum { + OI_CACHED, + OI_LOOSE, + OI_PACKED, + OI_DBCACHED + } whence; + union { + /* + * struct { + * ... Nothing to expose in this case + * } cached; + * struct { + * ... Nothing to expose in this case + * } loose; + */ + struct { + struct packed_git *pack; + off_t offset; + unsigned int is_delta; + } packed; + } u; +}; +extern int sha1_object_info_extended(const unsigned char *, struct object_info *); /* Dumb servers support */ extern int update_server_info(int); @@@ -1081,8 -1066,6 +1084,8 @@@ extern int config_error_nonbool(const c extern const char *get_log_output_encoding(void); extern const char *get_commit_output_encoding(void); +extern int git_config_parse_parameter(const char *, config_fn_t fn, void *data); + extern const char *config_exclusive_filename; #define MAX_GITNAME (1000) @@@ -1149,6 -1132,13 +1152,6 @@@ extern void trace_strbuf(const char *ke void packet_trace_identity(const char *prog); -/* convert.c */ -/* returns 1 if *dst was used */ -extern int convert_to_git(const char *path, const char *src, size_t len, - struct strbuf *dst, enum safe_crlf checksafe); -extern int convert_to_working_tree(const char *path, const char *src, size_t len, struct strbuf *dst); -extern int renormalize_buffer(const char *path, const char *src, size_t len, struct strbuf *dst); - /* add */ /* * return 0 if success, 1 - if addition of a file failed and diff --combined config.c index e42c59b190,5ef3f397d5..4183f80262 --- a/config.c +++ b/config.c @@@ -12,18 -12,10 +12,18 @@@ #define MAXNAME (256) -static FILE *config_file; -static const char *config_file_name; -static int config_linenr; -static int config_file_eof; +typedef struct config_file { + struct config_file *prev; + FILE *f; + const char *name; + int linenr; + int eof; + struct strbuf value; + char var[MAXNAME]; +} config_file; + +static config_file *cf; + static int zlib_compression_seen; const char *config_exclusive_filename = NULL; @@@ -47,13 -39,13 +47,13 @@@ void git_config_push_parameter(const ch strbuf_release(&env); } -static int git_config_parse_parameter(const char *text, - config_fn_t fn, void *data) +int git_config_parse_parameter(const char *text, + config_fn_t fn, void *data) { - struct strbuf tmp = STRBUF_INIT; struct strbuf **pair; - strbuf_addstr(&tmp, text); - pair = strbuf_split(&tmp, '='); + pair = strbuf_split_str(text, '=', 2); + if (!pair[0]) + return error("bogus config parameter: %s", text); if (pair[0]->len && pair[0]->buf[pair[0]->len - 1] == '=') strbuf_setlen(pair[0], pair[0]->len - 1); strbuf_trim(pair[0]); @@@ -107,7 -99,7 +107,7 @@@ static int get_next_char(void FILE *f; c = '\n'; - if ((f = config_file) != NULL) { + if (cf && ((f = cf->f) != NULL)) { c = fgetc(f); if (c == '\r') { /* DOS like systems */ @@@ -118,9 -110,9 +118,9 @@@ } } if (c == '\n') - config_linenr++; + cf->linenr++; if (c == EOF) { - config_file_eof = 1; + cf->eof = 1; c = '\n'; } } @@@ -129,20 -121,21 +129,20 @@@ static char *parse_value(void) { - static struct strbuf value = STRBUF_INIT; int quote = 0, comment = 0, space = 0; - strbuf_reset(&value); + strbuf_reset(&cf->value); for (;;) { int c = get_next_char(); if (c == '\n') { if (quote) return NULL; - return value.buf; + return cf->value.buf; } if (comment) continue; if (isspace(c) && !quote) { - if (value.len) + if (cf->value.len) space++; continue; } @@@ -153,7 -146,7 +153,7 @@@ } } for (; space; space--) - strbuf_addch(&value, ' '); + strbuf_addch(&cf->value, ' '); if (c == '\\') { c = get_next_char(); switch (c) { @@@ -175,14 -168,14 +175,14 @@@ default: return NULL; } - strbuf_addch(&value, c); + strbuf_addch(&cf->value, c); continue; } if (c == '"') { quote = 1-quote; continue; } - strbuf_addch(&value, c); + strbuf_addch(&cf->value, c); } } @@@ -199,7 -192,7 +199,7 @@@ static int get_value(config_fn_t fn, vo /* Get the full name */ for (;;) { c = get_next_char(); - if (config_file_eof) + if (cf->eof) break; if (!iskeychar(c)) break; @@@ -263,7 -256,7 +263,7 @@@ static int get_base_var(char *name for (;;) { int c = get_next_char(); - if (config_file_eof) + if (cf->eof) return -1; if (c == ']') return baselen; @@@ -281,7 -274,7 +281,7 @@@ static int git_parse_file(config_fn_t f { int comment = 0; int baselen = 0; - static char var[MAXNAME]; + char *var = cf->var; /* U+FEFF Byte Order Mark in UTF8 */ static const unsigned char *utf8_bom = (unsigned char *) "\xef\xbb\xbf"; @@@ -305,7 -298,7 +305,7 @@@ } } if (c == '\n') { - if (config_file_eof) + if (cf->eof) return 0; comment = 0; continue; @@@ -330,7 -323,7 +330,7 @@@ if (get_value(fn, data, var, baselen+1) < 0) break; } - die("bad config file line %d in %s", config_linenr, config_file_name); + die("bad config file line %d in %s", cf->linenr, cf->name); } static int parse_unit_factor(const char *end, unsigned long *val) @@@ -381,8 -374,8 +381,8 @@@ int git_parse_ulong(const char *value, static void die_bad_config(const char *name) { - if (config_file_name) - die("bad config value for '%s' in %s", name, config_file_name); + if (cf && cf->name) + die("bad config value for '%s' in %s", name, cf->name); die("bad config value for '%s'", name); } @@@ -576,6 -569,9 +576,9 @@@ static int git_default_core_config(cons return 0; } + if (!strcmp(var, "core.logpackaccess")) + return git_config_string(&log_pack_access, var, value); + if (!strcmp(var, "core.autocrlf")) { if (value && !strcasecmp(value, "input")) { if (core_eol == EOL_CRLF) @@@ -802,24 -798,13 +805,24 @@@ int git_config_from_file(config_fn_t fn ret = -1; if (f) { - config_file = f; - config_file_name = filename; - config_linenr = 1; - config_file_eof = 0; + config_file top; + + /* push config-file parsing state stack */ + top.prev = cf; + top.f = f; + top.name = filename; + top.linenr = 1; + top.eof = 0; + strbuf_init(&top.value, 1024); + cf = ⊤ + ret = git_parse_file(fn, data); + + /* pop config-file parsing state stack */ + strbuf_release(&top.value); + cf = top.prev; + fclose(f); - config_file_name = NULL; } return ret; } @@@ -874,7 -859,7 +877,7 @@@ int git_config_early(config_fn_t fn, vo switch (git_config_from_parameters(fn, data)) { case -1: /* error */ - ret--; + die("unable to parse command-line config"); break; case 0: /* found nothing */ break; @@@ -927,7 -912,6 +930,7 @@@ static int store_aux(const char *key, c { const char *ep; size_t section_len; + FILE *f = cf->f; switch (store.state) { case KEY_SEEN: @@@ -939,7 -923,7 +942,7 @@@ return 1; } - store.offset[store.seen] = ftell(config_file); + store.offset[store.seen] = ftell(f); store.seen++; } break; @@@ -966,19 -950,19 +969,19 @@@ * Do not increment matches: this is no match, but we * just made sure we are in the desired section. */ - store.offset[store.seen] = ftell(config_file); + store.offset[store.seen] = ftell(f); /* fallthru */ case SECTION_END_SEEN: case START: if (matches(key, value)) { - store.offset[store.seen] = ftell(config_file); + store.offset[store.seen] = ftell(f); store.state = KEY_SEEN; store.seen++; } else { if (strrchr(key, '.') - key == store.baselen && !strncmp(key, store.key, store.baselen)) { store.state = SECTION_SEEN; - store.offset[store.seen] = ftell(config_file); + store.offset[store.seen] = ftell(f); } } } @@@ -1434,7 -1418,6 +1437,7 @@@ int git_config_rename_section(const cha struct lock_file *lock = xcalloc(sizeof(struct lock_file), 1); int out_fd; char buf[1024]; + FILE *config_file; if (config_exclusive_filename) config_filename = xstrdup(config_exclusive_filename); @@@ -1499,10 -1482,10 +1502,10 @@@ } } fclose(config_file); - unlock_and_out: +unlock_and_out: if (commit_lock_file(lock) < 0) ret = error("could not commit config file %s", config_filename); - out: +out: free(config_filename); return ret; } diff --combined sha1_file.c index 89d7e5eb57,baf5da1823..d5616dca08 --- a/sha1_file.c +++ b/sha1_file.c @@@ -834,7 -834,7 +834,7 @@@ static int in_window(struct pack_windo unsigned char *use_pack(struct packed_git *p, struct pack_window **w_cursor, off_t offset, - unsigned int *left) + unsigned long *left) { struct pack_window *win = *w_cursor; @@@ -1186,7 -1186,7 +1186,7 @@@ static int open_sha1_file(const unsigne return -1; } -static void *map_sha1_file(const unsigned char *sha1, unsigned long *size) +void *map_sha1_file(const unsigned char *sha1, unsigned long *size) { void *map; int fd; @@@ -1205,29 -1205,20 +1205,29 @@@ return map; } -static int legacy_loose_object(unsigned char *map) +/* + * There used to be a second loose object header format which + * was meant to mimic the in-pack format, allowing for direct + * copy of the object data. This format turned up not to be + * really worth it and we no longer write loose objects in that + * format. + */ +static int experimental_loose_object(unsigned char *map) { unsigned int word; /* * Is it a zlib-compressed buffer? If so, the first byte * must be 0x78 (15-bit window size, deflated), and the - * first 16-bit word is evenly divisible by 31 + * first 16-bit word is evenly divisible by 31. If so, + * we are looking at the official format, not the experimental + * one. */ word = (map[0] << 8) + map[1]; if (map[0] == 0x78 && !(word % 31)) - return 1; - else return 0; + else + return 1; } unsigned long unpack_object_header_buffer(const unsigned char *buf, @@@ -1254,7 -1245,7 +1254,7 @@@ return used; } -static int unpack_sha1_header(z_stream *stream, unsigned char *map, unsigned long mapsize, void *buffer, unsigned long bufsiz) +int unpack_sha1_header(git_zstream *stream, unsigned char *map, unsigned long mapsize, void *buffer, unsigned long bufsiz) { unsigned long size, used; static const char valid_loose_object_type[8] = { @@@ -1271,32 -1262,37 +1271,32 @@@ stream->next_out = buffer; stream->avail_out = bufsiz; - if (legacy_loose_object(map)) { - git_inflate_init(stream); - return git_inflate(stream, 0); - } - + if (experimental_loose_object(map)) { + /* + * The old experimental format we no longer produce; + * we can still read it. + */ + used = unpack_object_header_buffer(map, mapsize, &type, &size); + if (!used || !valid_loose_object_type[type]) + return -1; + map += used; + mapsize -= used; - /* - * There used to be a second loose object header format which - * was meant to mimic the in-pack format, allowing for direct - * copy of the object data. This format turned up not to be - * really worth it and we don't write it any longer. But we - * can still read it. - */ - used = unpack_object_header_buffer(map, mapsize, &type, &size); - if (!used || !valid_loose_object_type[type]) - return -1; - map += used; - mapsize -= used; + /* Set up the stream for the rest.. */ + stream->next_in = map; + stream->avail_in = mapsize; + git_inflate_init(stream); - /* Set up the stream for the rest.. */ - stream->next_in = map; - stream->avail_in = mapsize; + /* And generate the fake traditional header */ + stream->total_out = 1 + snprintf(buffer, bufsiz, "%s %lu", + typename(type), size); + return 0; + } git_inflate_init(stream); - - /* And generate the fake traditional header */ - stream->total_out = 1 + snprintf(buffer, bufsiz, "%s %lu", - typename(type), size); - return 0; + return git_inflate(stream, 0); } -static void *unpack_sha1_rest(z_stream *stream, void *buffer, unsigned long size, const unsigned char *sha1) +static void *unpack_sha1_rest(git_zstream *stream, void *buffer, unsigned long size, const unsigned char *sha1) { int bytes = strlen(buffer) + 1; unsigned char *buf = xmallocz(size); @@@ -1346,7 -1342,7 +1346,7 @@@ * too permissive for what we want to check. So do an anal * object header parse by hand. */ -static int parse_sha1_header(const char *hdr, unsigned long *sizep) +int parse_sha1_header(const char *hdr, unsigned long *sizep) { char type[10]; int i; @@@ -1395,7 -1391,7 +1395,7 @@@ static void *unpack_sha1_file(void *map, unsigned long mapsize, enum object_type *type, unsigned long *size, const unsigned char *sha1) { int ret; - z_stream stream; + git_zstream stream; char hdr[8192]; ret = unpack_sha1_header(&stream, map, mapsize, hdr, sizeof(hdr)); @@@ -1411,7 -1407,7 +1411,7 @@@ unsigned long get_size_from_delta(struc { const unsigned char *data; unsigned char delta_head[20], *in; - z_stream stream; + git_zstream stream; int st; memset(&stream, 0, sizeof(stream)); @@@ -1485,7 -1481,7 +1485,7 @@@ static off_t get_delta_base(struct pack /* forward declaration for a mutually recursive function */ static int packed_object_info(struct packed_git *p, off_t offset, - unsigned long *sizep); + unsigned long *sizep, int *rtype); static int packed_delta_info(struct packed_git *p, struct pack_window **w_curs, @@@ -1499,7 -1495,7 +1499,7 @@@ base_offset = get_delta_base(p, w_curs, &curpos, type, obj_offset); if (!base_offset) return OBJ_BAD; - type = packed_object_info(p, base_offset, NULL); + type = packed_object_info(p, base_offset, NULL, NULL); if (type <= OBJ_NONE) { struct revindex_entry *revidx; const unsigned char *base_sha1; @@@ -1527,13 -1523,13 +1527,13 @@@ return type; } -static int unpack_object_header(struct packed_git *p, - struct pack_window **w_curs, - off_t *curpos, - unsigned long *sizep) +int unpack_object_header(struct packed_git *p, + struct pack_window **w_curs, + off_t *curpos, + unsigned long *sizep) { unsigned char *base; - unsigned int left; + unsigned long left; unsigned long used; enum object_type type; @@@ -1553,8 -1549,63 +1553,8 @@@ return type; } -const char *packed_object_info_detail(struct packed_git *p, - off_t obj_offset, - unsigned long *size, - unsigned long *store_size, - unsigned int *delta_chain_length, - unsigned char *base_sha1) -{ - struct pack_window *w_curs = NULL; - off_t curpos; - unsigned long dummy; - unsigned char *next_sha1; - enum object_type type; - struct revindex_entry *revidx; - - *delta_chain_length = 0; - curpos = obj_offset; - type = unpack_object_header(p, &w_curs, &curpos, size); - - revidx = find_pack_revindex(p, obj_offset); - *store_size = revidx[1].offset - obj_offset; - - for (;;) { - switch (type) { - default: - die("pack %s contains unknown object type %d", - p->pack_name, type); - case OBJ_COMMIT: - case OBJ_TREE: - case OBJ_BLOB: - case OBJ_TAG: - unuse_pack(&w_curs); - return typename(type); - case OBJ_OFS_DELTA: - obj_offset = get_delta_base(p, &w_curs, &curpos, type, obj_offset); - if (!obj_offset) - die("pack %s contains bad delta base reference of type %s", - p->pack_name, typename(type)); - if (*delta_chain_length == 0) { - revidx = find_pack_revindex(p, obj_offset); - hashcpy(base_sha1, nth_packed_object_sha1(p, revidx->nr)); - } - break; - case OBJ_REF_DELTA: - next_sha1 = use_pack(p, &w_curs, curpos, NULL); - if (*delta_chain_length == 0) - hashcpy(base_sha1, next_sha1); - obj_offset = find_pack_entry_one(next_sha1, p); - break; - } - (*delta_chain_length)++; - curpos = obj_offset; - type = unpack_object_header(p, &w_curs, &curpos, &dummy); - } -} - static int packed_object_info(struct packed_git *p, off_t obj_offset, - unsigned long *sizep) + unsigned long *sizep, int *rtype) { struct pack_window *w_curs = NULL; unsigned long size; @@@ -1562,8 -1613,6 +1562,8 @@@ enum object_type type; type = unpack_object_header(p, &w_curs, &curpos, &size); + if (rtype) + *rtype = type; /* representation type */ switch (type) { case OBJ_OFS_DELTA: @@@ -1593,7 -1642,7 +1593,7 @@@ static void *unpack_compressed_entry(st unsigned long size) { int st; - z_stream stream; + git_zstream stream; unsigned char *buffer, *in; buffer = xmallocz(size); @@@ -1646,13 -1695,6 +1646,13 @@@ static unsigned long pack_entry_hash(st return hash % MAX_DELTA_CACHE; } +static int in_delta_base_cache(struct packed_git *p, off_t base_offset) +{ + unsigned long hash = pack_entry_hash(p, base_offset); + struct delta_base_cache_entry *ent = delta_base_cache + hash; + return (ent->data && ent->p == p && ent->base_offset == base_offset); +} + static void *cache_or_unpack_entry(struct packed_git *p, off_t base_offset, unsigned long *base_size, enum object_type *type, int keep_cache) { @@@ -1797,6 -1839,24 +1797,24 @@@ static void *unpack_delta_entry(struct return result; } + static void write_pack_access_log(struct packed_git *p, off_t obj_offset) + { + static FILE *log_file; + + if (!log_file) { + log_file = fopen(log_pack_access, "w"); + if (!log_file) { + error("cannot open pack access log '%s' for writing: %s", + log_pack_access, strerror(errno)); + log_pack_access = NULL; + return; + } + } + fprintf(log_file, "%s %"PRIuMAX"\n", + p->pack_name, (uintmax_t)obj_offset); + fflush(log_file); + } + int do_check_packed_object_crc; void *unpack_entry(struct packed_git *p, off_t obj_offset, @@@ -1806,6 -1866,9 +1824,9 @@@ off_t curpos = obj_offset; void *data; + if (log_pack_access) + write_pack_access_log(p, obj_offset); + if (do_check_packed_object_crc && p->index_version > 1) { struct revindex_entry *revidx = find_pack_revindex(p, obj_offset); unsigned long len = revidx[1].offset - obj_offset; @@@ -2033,7 -2096,7 +2054,7 @@@ static int sha1_loose_object_info(cons int status; unsigned long mapsize, size; void *map; - z_stream stream; + git_zstream stream; char hdr[32]; map = map_sha1_file(sha1, &mapsize); @@@ -2051,28 -2114,24 +2072,28 @@@ return status; } -int sha1_object_info(const unsigned char *sha1, unsigned long *sizep) +/* returns enum object_type or negative */ +int sha1_object_info_extended(const unsigned char *sha1, struct object_info *oi) { struct cached_object *co; struct pack_entry e; - int status; + int status, rtype; co = find_cached_object(sha1); if (co) { - if (sizep) - *sizep = co->size; + if (oi->sizep) + *(oi->sizep) = co->size; + oi->whence = OI_CACHED; return co->type; } if (!find_pack_entry(sha1, &e)) { /* Most likely it's a loose object. */ - status = sha1_loose_object_info(sha1, sizep); - if (status >= 0) + status = sha1_loose_object_info(sha1, oi->sizep); + if (status >= 0) { + oi->whence = OI_LOOSE; return status; + } /* Not a loose object; someone else may have just packed it. */ reprepare_packed_git(); @@@ -2080,31 -2139,15 +2101,31 @@@ return status; } - status = packed_object_info(e.p, e.offset, sizep); + status = packed_object_info(e.p, e.offset, oi->sizep, &rtype); if (status < 0) { mark_bad_packed_object(e.p, sha1); - status = sha1_object_info(sha1, sizep); + status = sha1_object_info_extended(sha1, oi); + } else if (in_delta_base_cache(e.p, e.offset)) { + oi->whence = OI_DBCACHED; + } else { + oi->whence = OI_PACKED; + oi->u.packed.offset = e.offset; + oi->u.packed.pack = e.p; + oi->u.packed.is_delta = (rtype == OBJ_REF_DELTA || + rtype == OBJ_OFS_DELTA); } return status; } +int sha1_object_info(const unsigned char *sha1, unsigned long *sizep) +{ + struct object_info oi; + + oi.sizep = sizep; + return sha1_object_info_extended(sha1, &oi); +} + static void *read_packed_sha1(const unsigned char *sha1, enum object_type *type, unsigned long *size) { @@@ -2402,7 -2445,7 +2423,7 @@@ static int write_loose_object(const uns { int fd, ret; unsigned char compressed[4096]; - z_stream stream; + git_zstream stream; git_SHA_CTX c; unsigned char parano_sha1[20]; char *filename; @@@ -2419,7 -2462,7 +2440,7 @@@ /* Set it up */ memset(&stream, 0, sizeof(stream)); - deflateInit(&stream, zlib_compression_level); + git_deflate_init(&stream, zlib_compression_level); stream.next_out = compressed; stream.avail_out = sizeof(compressed); git_SHA1_Init(&c); @@@ -2427,8 -2470,8 +2448,8 @@@ /* First header.. */ stream.next_in = (unsigned char *)hdr; stream.avail_in = hdrlen; - while (deflate(&stream, 0) == Z_OK) - /* nothing */; + while (git_deflate(&stream, 0) == Z_OK) + ; /* nothing */ git_SHA1_Update(&c, hdr, hdrlen); /* Then the data itself.. */ @@@ -2436,7 -2479,7 +2457,7 @@@ stream.avail_in = len; do { unsigned char *in0 = stream.next_in; - ret = deflate(&stream, Z_FINISH); + ret = git_deflate(&stream, Z_FINISH); git_SHA1_Update(&c, in0, stream.next_in - in0); if (write_buffer(fd, compressed, stream.next_out - compressed) < 0) die("unable to write sha1 file"); @@@ -2446,7 -2489,7 +2467,7 @@@ if (ret != Z_STREAM_END) die("unable to deflate new object %s (%d)", sha1_to_hex(sha1), ret); - ret = deflateEnd(&stream); + ret = git_deflate_end_gently(&stream); if (ret != Z_OK) die("deflateEnd on object %s failed (%d)", sha1_to_hex(sha1), ret); git_SHA1_Final(parano_sha1, &c); @@@ -2682,7 -2725,7 +2703,7 @@@ static int index_stream(unsigned char * while (size) { char buf[10240]; size_t sz = size < sizeof(buf) ? size : sizeof(buf); - size_t actual; + ssize_t actual; actual = read_in_full(fd, buf, sz); if (actual < 0)