From: Junio C Hamano Date: Thu, 13 Nov 2008 06:00:43 +0000 (-0800) Subject: Merge branch 'bc/maint-keep-pack' X-Git-Tag: v1.6.1-rc1~57 X-Git-Url: https://git.lorimer.id.au/gitweb.git/diff_plain/ecbbfb15a42987deb6620c4575c420002313af23?hp=-c Merge branch 'bc/maint-keep-pack' * bc/maint-keep-pack: t7700: test that 'repack -a' packs alternate packed objects pack-objects: extend --local to mean ignore non-local loose objects too sha1_file.c: split has_loose_object() into local and non-local counterparts t7700: demonstrate mishandling of loose objects in an alternate ODB builtin-gc.c: use new pack_keep bitfield to detect .keep file existence repack: do not fall back to incremental repacking with [-a|-A] repack: don't repack local objects in packs with .keep file pack-objects: new option --honor-pack-keep packed_git: convert pack_local flag into a bitfield and add pack_keep t7700: demonstrate mishandling of objects in packs with a .keep file --- ecbbfb15a42987deb6620c4575c420002313af23 diff --combined builtin-gc.c index 7af65bb31b,53a0d43b67..781df601c5 --- a/builtin-gc.c +++ b/builtin-gc.c @@@ -26,7 -26,7 +26,7 @@@ static int pack_refs = 1 static int aggressive_window = -1; static int gc_auto_threshold = 6700; static int gc_auto_pack_limit = 50; -static char *prune_expire = "2.weeks.ago"; +static const char *prune_expire = "2.weeks.ago"; #define MAX_ADD 10 static const char *argv_pack_refs[] = {"pack-refs", "--all", "--prune", NULL}; @@@ -57,12 -57,15 +57,12 @@@ static int gc_config(const char *var, c return 0; } if (!strcmp(var, "gc.pruneexpire")) { - if (!value) - return config_error_nonbool(var); - if (strcmp(value, "now")) { + if (value && strcmp(value, "now")) { unsigned long now = approxidate("now"); if (approxidate(value) >= now) return error("Invalid %s: '%s'", var, value); } - prune_expire = xstrdup(value); - return 0; + return git_config_string(&prune_expire, var, value); } return git_default_config(var, value, cb); } @@@ -131,19 -134,9 +131,9 @@@ static int too_many_packs(void prepare_packed_git(); for (cnt = 0, p = packed_git; p; p = p->next) { - char path[PATH_MAX]; - size_t len; - int keep; - if (!p->pack_local) continue; - len = strlen(p->pack_name); - if (PATH_MAX <= len + 1) - continue; /* oops, give up */ - memcpy(path, p->pack_name, len-5); - memcpy(path + len - 5, ".keep", 6); - keep = access(p->pack_name, F_OK) && (errno == ENOENT); - if (keep) + if (p->pack_keep) continue; /* * Perhaps check the size of the pack and count only diff --combined builtin-pack-objects.c index 95e6faf1e6,85bd795d3b..59ae64d83f --- a/builtin-pack-objects.c +++ b/builtin-pack-objects.c @@@ -71,6 -71,7 +71,7 @@@ static int reuse_delta = 1, reuse_objec static int keep_unreachable, unpack_unreachable, include_tag; static int local; static int incremental; + static int ignore_packed_keep; static int allow_ofs_delta; static const char *base_name; static int progress = 1; @@@ -245,16 -246,8 +246,16 @@@ static unsigned long write_object(struc type = entry->type; /* write limit if limited packsize and not first object */ - limit = pack_size_limit && nr_written ? - pack_size_limit - write_offset : 0; + if (!pack_size_limit || !nr_written) + limit = 0; + else if (pack_size_limit <= write_offset) + /* + * the earlier object did not fit the limit; avoid + * mistaking this with unlimited (i.e. limit = 0). + */ + limit = 1; + else + limit = pack_size_limit - write_offset; if (!entry->delta) usable_delta = 0; /* no delta */ @@@ -698,6 -691,9 +699,9 @@@ static int add_object_entry(const unsig return 0; } + if (!exclude && local && has_loose_object_nonlocal(sha1)) + return 0; + for (p = packed_git; p; p = p->next) { off_t offset = find_pack_entry_one(sha1, p); if (offset) { @@@ -711,6 -707,8 +715,8 @@@ return 0; if (local && !p->pack_local) return 0; + if (ignore_packed_keep && p->pack_local && p->pack_keep) + return 0; } } @@@ -1377,10 -1375,12 +1383,10 @@@ static void find_deltas(struct object_e int window, int depth, unsigned *processed) { uint32_t i, idx = 0, count = 0; - unsigned int array_size = window * sizeof(struct unpacked); struct unpacked *array; unsigned long mem_usage = 0; - array = xmalloc(array_size); - memset(array, 0, array_size); + array = xcalloc(window, sizeof(struct unpacked)); for (;;) { struct object_entry *entry; @@@ -1732,14 -1732,6 +1738,14 @@@ static void prepare_pack(int window, in if (entry->type < 0) die("unable to get type of object %s", sha1_to_hex(entry->idx.sha1)); + } else { + if (entry->type < 0) { + /* + * This object is not found, but we + * don't have to include it anyway. + */ + continue; + } } delta_list[n++] = entry; @@@ -2056,6 -2048,10 +2062,10 @@@ int cmd_pack_objects(int argc, const ch incremental = 1; continue; } + if (!strcmp("--honor-pack-keep", arg)) { + ignore_packed_keep = 1; + continue; + } if (!prefixcmp(arg, "--compression=")) { char *end; int level = strtoul(arg+14, &end, 0); diff --combined cache.h index 6be60ea3ff,7595c149ea..c776f2f5ab --- a/cache.h +++ b/cache.h @@@ -6,14 -6,8 +6,14 @@@ #include "hash.h" #include SHA1_HEADER -#include +#ifndef git_SHA_CTX +#define git_SHA_CTX SHA_CTX +#define git_SHA1_Init SHA1_Init +#define git_SHA1_Update SHA1_Update +#define git_SHA1_Final SHA1_Final +#endif +#include #if defined(NO_DEFLATE_BOUND) || ZLIB_VERNUM < 0x1200 #define deflateBound(c,s) ((s) + (((s) + 7) >> 3) + (((s) + 63) >> 6) + 11) #endif @@@ -132,7 -126,6 +132,7 @@@ struct cache_entry #define CE_NAMEMASK (0x0fff) #define CE_STAGEMASK (0x3000) +#define CE_EXTENDED (0x4000) #define CE_VALID (0x8000) #define CE_STAGESHIFT 12 @@@ -262,7 -255,6 +262,7 @@@ static inline void remove_name_hash(str #define read_cache() read_index(&the_index) #define read_cache_from(path) read_index_from(&the_index, (path)) +#define is_cache_unborn() is_index_unborn(&the_index) #define read_cache_unmerged() read_index_unmerged(&the_index) #define write_cache(newfd, cache, entries) write_index(&the_index, (newfd)) #define discard_cache() discard_index(&the_index) @@@ -321,7 -313,6 +321,7 @@@ extern int is_bare_repository(void) extern int is_inside_git_dir(void); extern char *git_work_tree_cfg; extern int is_inside_work_tree(void); +extern int have_git_dir(void); extern const char *get_git_dir(void); extern char *get_object_directory(void); extern char *get_index_file(void); @@@ -369,7 -360,6 +369,7 @@@ extern int init_db(const char *template /* Initialize and use the cache information */ extern int read_index(struct index_state *); extern int read_index_from(struct index_state *, const char *path); +extern int is_index_unborn(struct index_state *); extern int read_index_unmerged(struct index_state *); extern int write_index(const struct index_state *, int newfd); extern int discard_index(struct index_state *); @@@ -381,7 -371,6 +381,7 @@@ extern int index_name_pos(const struct #define ADD_CACHE_OK_TO_REPLACE 2 /* Ok to replace file/directory */ #define ADD_CACHE_SKIP_DFCHECK 4 /* Ok to skip DF conflict checks */ #define ADD_CACHE_JUST_APPEND 8 /* Append only; tree.c::read_tree() */ +#define ADD_CACHE_NEW_ONLY 16 /* Do not replace existing ones */ extern int add_index_entry(struct index_state *, struct cache_entry *ce, int option); extern struct cache_entry *refresh_cache_entry(struct cache_entry *ce, int really); extern void rename_index_entry_at(struct index_state *, int pos, const char *new_name); @@@ -390,8 -379,6 +390,8 @@@ extern int remove_file_from_index(struc #define ADD_CACHE_VERBOSE 1 #define ADD_CACHE_PRETEND 2 #define ADD_CACHE_IGNORE_ERRORS 4 +#define ADD_CACHE_IGNORE_REMOVAL 8 +#define ADD_CACHE_INTENT 16 extern int add_to_index(struct index_state *, const char *path, struct stat *, int flags); extern int add_file_to_index(struct index_state *, const char *path, int flags); extern struct cache_entry *make_cache_entry(unsigned int mode, const unsigned char *sha1, const char *path, int stage, int refresh); @@@ -407,6 -394,7 +407,6 @@@ extern int ie_modified(const struct ind extern int ce_path_match(const struct cache_entry *ce, const char **pathspec); extern int index_fd(unsigned char *sha1, int fd, struct stat *st, int write_object, enum object_type type, const char *path); -extern int index_pipe(unsigned char *sha1, int fd, const char *type, int write_object); extern int index_path(unsigned char *sha1, const char *path, struct stat *st, int write_object); extern void fill_stat_cache_info(struct cache_entry *ce, struct stat *st); @@@ -468,7 -456,6 +468,7 @@@ enum safe_crlf extern enum safe_crlf safe_crlf; enum branch_track { + BRANCH_TRACK_UNSPECIFIED = -1, BRANCH_TRACK_NEVER = 0, BRANCH_TRACK_REMOTE, BRANCH_TRACK_ALWAYS, @@@ -528,7 -515,6 +528,7 @@@ static inline void hashclr(unsigned cha { memset(hash, 0, 20); } +extern int is_empty_blob_sha1(const unsigned char *sha1); int git_mkstemp(char *path, size_t n, const char *template); @@@ -556,7 -542,6 +556,7 @@@ static inline int is_absolute_path(cons { return path[0] == '/' || has_dos_drive_prefix(path); } +int is_directory(const char *); const char *make_absolute_path(const char *path); const char *make_nonrelative_path(const char *path); const char *make_relative_path(const char *abs, const char *base); @@@ -580,6 -565,7 +580,7 @@@ extern int move_temp_to_file(const cha extern int has_sha1_pack(const unsigned char *sha1, const char **ignore); extern int has_sha1_file(const unsigned char *sha1); + extern int has_loose_object_nonlocal(const unsigned char *sha1); extern int has_pack_file(const unsigned char *sha1); extern int has_pack_index(const unsigned char *sha1); @@@ -664,8 -650,6 +665,8 @@@ extern struct alternate_object_databas } *alt_odb_list; extern void prepare_alt_odb(void); extern void add_to_alternates_file(const char *reference); +typedef int alt_odb_fn(struct alternate_object_database *, void *); +extern void foreach_alt_odb(alt_odb_fn, void*); struct pack_window { struct pack_window *next; @@@ -688,7 -672,8 +689,8 @@@ extern struct packed_git int index_version; time_t mtime; int pack_fd; - int pack_local; + unsigned pack_local:1, + pack_keep:1; unsigned char sha1[20]; /* something like ".git/objects/pack/xxxxx.pack" */ char pack_name[FLEX_ARRAY]; /* more */ @@@ -734,11 -719,7 +736,11 @@@ extern struct child_process *git_connec extern int finish_connect(struct child_process *conn); extern int path_match(const char *path, int nr, char **match); extern int get_ack(int fd, unsigned char *result_sha1); -extern struct ref **get_remote_heads(int in, struct ref **list, int nr_match, char **match, unsigned int flags); +struct extra_have_objects { + int nr, alloc; + unsigned char (*array)[20]; +}; +extern struct ref **get_remote_heads(int in, struct ref **list, int nr_match, char **match, unsigned int flags, struct extra_have_objects *); extern int server_supports(const char *feature); extern struct packed_git *parse_pack_index(unsigned char *sha1); @@@ -772,6 -753,7 +774,6 @@@ typedef int (*config_fn_t)(const char * extern int git_default_config(const char *, const char *, void *); extern int git_config_from_file(config_fn_t fn, const char *, void *); extern int git_config(config_fn_t fn, void *); -extern int git_parse_long(const char *, long *); extern int git_parse_ulong(const char *, unsigned long *); extern int git_config_int(const char *, const char *); extern unsigned long git_config_ulong(const char *, const char *); diff --combined sha1_file.c index ab2b520f03,0203de5855..491220572b --- a/sha1_file.c +++ b/sha1_file.c @@@ -99,11 -99,7 +99,11 @@@ int safe_create_leading_directories(cha pos = strchr(pos, '/'); if (!pos) break; - *pos = 0; + while (*++pos == '/') + ; + if (!*pos) + break; + *--pos = '\0'; if (!stat(path, &st)) { /* path exists */ if (!S_ISDIR(st.st_mode)) { @@@ -254,6 -250,7 +254,6 @@@ static void read_info_alternates(const */ static int link_alt_odb_entry(const char * entry, int len, const char * relative_base, int depth) { - struct stat st; const char *objdir = get_object_directory(); struct alternate_object_database *ent; struct alternate_object_database *alt; @@@ -284,7 -281,7 +284,7 @@@ ent->base[pfxlen] = ent->base[entlen-1] = 0; /* Detect cases where alternate disappeared */ - if (stat(ent->base, &st) || !S_ISDIR(st.st_mode)) { + if (!is_directory(ent->base)) { error("object directory %s does not exist; " "check .git/objects/info/alternates.", ent->base); @@@ -397,16 -394,6 +397,16 @@@ void add_to_alternates_file(const char link_alt_odb_entries(alt, alt + strlen(alt), '\n', NULL, 0); } +void foreach_alt_odb(alt_odb_fn fn, void *cb) +{ + struct alternate_object_database *ent; + + prepare_alt_odb(); + for (ent = alt_odb_list; ent; ent = ent->next) + if (fn(ent, cb)) + return; +} + void prepare_alt_odb(void) { const char *alt; @@@ -423,23 -410,30 +423,30 @@@ read_info_alternates(get_object_directory(), 0); } - static int has_loose_object(const unsigned char *sha1) + static int has_loose_object_local(const unsigned char *sha1) { char *name = sha1_file_name(sha1); - struct alternate_object_database *alt; + return !access(name, F_OK); + } - if (!access(name, F_OK)) - return 1; + int has_loose_object_nonlocal(const unsigned char *sha1) + { + struct alternate_object_database *alt; prepare_alt_odb(); for (alt = alt_odb_list; alt; alt = alt->next) { - name = alt->name; - fill_sha1_path(name, sha1); + fill_sha1_path(alt->name, sha1); if (!access(alt->base, F_OK)) return 1; } return 0; } + static int has_loose_object(const unsigned char *sha1) + { + return has_loose_object_local(sha1) || + has_loose_object_nonlocal(sha1); + } + static unsigned int pack_used_ctr; static unsigned int pack_mmap_calls; static unsigned int peak_pack_open_windows; @@@ -841,6 -835,11 +848,11 @@@ struct packed_git *add_packed_git(cons return NULL; } memcpy(p->pack_name, path, path_len); + + strcpy(p->pack_name + path_len, ".keep"); + if (!access(p->pack_name, F_OK)) + p->pack_keep = 1; + strcpy(p->pack_name + path_len, ".pack"); if (stat(p->pack_name, &st) || !S_ISREG(st.st_mode)) { free(p); @@@ -1571,9 -1570,11 +1583,9 @@@ static void *cache_or_unpack_entry(stru struct delta_base_cache_entry *ent = delta_base_cache + hash; ret = ent->data; - if (ret && ent->p == p && ent->base_offset == base_offset) - goto found_cache_entry; - return unpack_entry(p, base_offset, type, base_size); + if (!ret || ent->p != p || ent->base_offset != base_offset) + return unpack_entry(p, base_offset, type, base_size); -found_cache_entry: if (!keep_cache) { ent->data = NULL; ent->lru.next->prev = ent->lru.prev; @@@ -2130,16 -2131,16 +2142,16 @@@ static void write_sha1_file_prepare(con const char *type, unsigned char *sha1, char *hdr, int *hdrlen) { - SHA_CTX c; + git_SHA_CTX c; /* Generate the header */ *hdrlen = sprintf(hdr, "%s %lu", type, len)+1; /* Sha1.. */ - SHA1_Init(&c); - SHA1_Update(&c, hdr, *hdrlen); - SHA1_Update(&c, buf, len); - SHA1_Final(sha1, &c); + git_SHA1_Init(&c); + git_SHA1_Update(&c, hdr, *hdrlen); + git_SHA1_Update(&c, buf, len); + git_SHA1_Final(sha1, &c); } /* @@@ -2378,21 -2379,51 +2390,21 @@@ int has_sha1_file(const unsigned char * return has_loose_object(sha1); } -int index_pipe(unsigned char *sha1, int fd, const char *type, int write_object) -{ - struct strbuf buf; - int ret; - - strbuf_init(&buf, 0); - if (strbuf_read(&buf, fd, 4096) < 0) { - strbuf_release(&buf); - return -1; - } - - if (!type) - type = blob_type; - if (write_object) - ret = write_sha1_file(buf.buf, buf.len, type, sha1); - else - ret = hash_sha1_file(buf.buf, buf.len, type, sha1); - strbuf_release(&buf); - - return ret; -} - -int index_fd(unsigned char *sha1, int fd, struct stat *st, int write_object, - enum object_type type, const char *path) +static int index_mem(unsigned char *sha1, void *buf, size_t size, + int write_object, enum object_type type, const char *path) { - size_t size = xsize_t(st->st_size); - void *buf = NULL; int ret, re_allocated = 0; - if (size) - buf = xmmap(NULL, size, PROT_READ, MAP_PRIVATE, fd, 0); - close(fd); - if (!type) type = OBJ_BLOB; /* * Convert blobs to git internal format */ - if ((type == OBJ_BLOB) && S_ISREG(st->st_mode)) { - struct strbuf nbuf; - strbuf_init(&nbuf, 0); + if ((type == OBJ_BLOB) && path) { + struct strbuf nbuf = STRBUF_INIT; if (convert_to_git(path, buf, size, &nbuf, write_object ? safe_crlf : 0)) { - munmap(buf, size); buf = strbuf_detach(&nbuf, &size); re_allocated = 1; } @@@ -2402,32 -2433,12 +2414,32 @@@ ret = write_sha1_file(buf, size, typename(type), sha1); else ret = hash_sha1_file(buf, size, typename(type), sha1); - if (re_allocated) { + if (re_allocated) free(buf); - return ret; - } - if (size) + return ret; +} + +int index_fd(unsigned char *sha1, int fd, struct stat *st, int write_object, + enum object_type type, const char *path) +{ + int ret; + size_t size = xsize_t(st->st_size); + + if (!S_ISREG(st->st_mode)) { + struct strbuf sbuf = STRBUF_INIT; + if (strbuf_read(&sbuf, fd, 4096) >= 0) + ret = index_mem(sha1, sbuf.buf, sbuf.len, write_object, + type, path); + else + ret = -1; + strbuf_release(&sbuf); + } else if (size) { + void *buf = xmmap(NULL, size, PROT_READ, MAP_PRIVATE, fd, 0); + ret = index_mem(sha1, buf, size, write_object, type, path); munmap(buf, size); + } else + ret = index_mem(sha1, NULL, size, write_object, type, path); + close(fd); return ret; }