From: Junio C Hamano Date: Tue, 21 Mar 2017 22:07:17 +0000 (-0700) Subject: Merge branch 'jk/pack-name-cleanups' X-Git-Tag: v2.13.0-rc0~92 X-Git-Url: https://git.lorimer.id.au/gitweb.git/diff_plain/45cbc37c5f84fadd78cccf6b2ea82a9ef251cdc6?ds=inline;hp=-c Merge branch 'jk/pack-name-cleanups' Code clean-up. * jk/pack-name-cleanups: index-pack: make pointer-alias fallbacks safer replace snprintf with odb_pack_name() odb_pack_keep(): stop generating keepfile name sha1_file.c: make pack-name helper globally accessible move odb_* declarations out of git-compat-util.h --- 45cbc37c5f84fadd78cccf6b2ea82a9ef251cdc6 diff --combined cache.h index 9b2157f591,0c177b7acd..14f8a06c46 --- a/cache.h +++ b/cache.h @@@ -10,8 -10,8 +10,8 @@@ #include "trace.h" #include "string-list.h" #include "pack-revindex.h" +#include "hash.h" -#include SHA1_HEADER #ifndef platform_SHA_CTX /* * platform's underlying implementation of SHA-1; could be OpenSSL, @@@ -518,13 -518,6 +518,13 @@@ extern void set_git_work_tree(const cha #define ALTERNATE_DB_ENVIRONMENT "GIT_ALTERNATE_OBJECT_DIRECTORIES" extern void setup_work_tree(void); +/* + * Find GIT_DIR of the repository that contains the current working directory, + * without changing the working directory or other global state. The result is + * appended to gitdir. The return value is either NULL if no repository was + * found, or pointing to the path inside gitdir's buffer. + */ +extern const char *discover_git_directory(struct strbuf *gitdir); extern const char *setup_git_directory_gently(int *); extern const char *setup_git_directory(void); extern char *prefix_path(const char *prefix, int len, const char *path); @@@ -1052,6 -1045,9 +1052,6 @@@ static inline int is_empty_tree_oid(con return !hashcmp(oid->hash, EMPTY_TREE_SHA1_BIN); } - -int git_mkstemp(char *path, size_t n, const char *template); - /* set default permissions by passing mode arguments to open(2) */ int git_mkstemps_mode(char *pattern, int suffix_len, int mode); int git_mkstemp_mode(char *pattern, int mode); @@@ -1076,9 -1072,8 +1076,9 @@@ int adjust_shared_perm(const char *path /* * Create the directory containing the named path, using care to be - * somewhat safe against races. Return one of the scld_error values - * to indicate success/failure. + * somewhat safe against races. Return one of the scld_error values to + * indicate success/failure. On error, set errno to describe the + * problem. * * SCLD_VANISHED indicates that one of the ancestor directories of the * path existed at one point during the function call and then @@@ -1102,49 -1097,6 +1102,49 @@@ enum scld_error enum scld_error safe_create_leading_directories(char *path); enum scld_error safe_create_leading_directories_const(const char *path); +/* + * Callback function for raceproof_create_file(). This function is + * expected to do something that makes dirname(path) permanent despite + * the fact that other processes might be cleaning up empty + * directories at the same time. Usually it will create a file named + * path, but alternatively it could create another file in that + * directory, or even chdir() into that directory. The function should + * return 0 if the action was completed successfully. On error, it + * should return a nonzero result and set errno. + * raceproof_create_file() treats two errno values specially: + * + * - ENOENT -- dirname(path) does not exist. In this case, + * raceproof_create_file() tries creating dirname(path) + * (and any parent directories, if necessary) and calls + * the function again. + * + * - EISDIR -- the file already exists and is a directory. In this + * case, raceproof_create_file() removes the directory if + * it is empty (and recursively any empty directories that + * it contains) and calls the function again. + * + * Any other errno causes raceproof_create_file() to fail with the + * callback's return value and errno. + * + * Obviously, this function should be OK with being called again if it + * fails with ENOENT or EISDIR. In other scenarios it will not be + * called again. + */ +typedef int create_file_fn(const char *path, void *cb); + +/* + * Create a file in dirname(path) by calling fn, creating leading + * directories if necessary. Retry a few times in case we are racing + * with another process that is trying to clean up the directory that + * contains path. See the documentation for create_file_fn for more + * details. + * + * Return the value and set the errno that resulted from the most + * recent call of fn. fn is always called at least once, and will be + * called more than once if it returns ENOENT or EISDIR. + */ +int raceproof_create_file(const char *path, create_file_fn fn, void *cb); + int mkdir_in_gitdir(const char *path); extern char *expand_user_path(const char *path); const char *enter_repo(const char *path, int strict); @@@ -1157,7 -1109,7 +1157,7 @@@ char *strbuf_realpath(struct strbuf *re int die_on_error); const char *real_path(const char *path); const char *real_path_if_valid(const char *path); -char *real_pathdup(const char *path); +char *real_pathdup(const char *path, int die_on_error); const char *absolute_path(const char *path); char *absolute_pathdup(const char *path); const char *remove_leading_path(const char *in, const char *prefix); @@@ -1277,9 -1229,6 +1277,9 @@@ extern int has_pack_index(const unsigne extern void assert_sha1_type(const unsigned char *sha1, enum object_type expect); +/* Helper to check and "touch" a file */ +extern int check_and_freshen_file(const char *fn, int freshen); + extern const signed char hexval_table[256]; static inline unsigned int hexval(unsigned char c) { @@@ -1370,46 -1319,7 +1370,46 @@@ extern char *oid_to_hex_r(char *out, co extern char *sha1_to_hex(const unsigned char *sha1); /* static buffer result! */ extern char *oid_to_hex(const struct object_id *oid); /* same static buffer as sha1_to_hex */ -extern int interpret_branch_name(const char *str, int len, struct strbuf *); +/* + * Parse a 40-character hexadecimal object ID starting from hex, updating the + * pointer specified by end when parsing stops. The resulting object ID is + * stored in oid. Returns 0 on success. Parsing will stop on the first NUL or + * other invalid character. end is only updated on success; otherwise, it is + * unmodified. + */ +extern int parse_oid_hex(const char *hex, struct object_id *oid, const char **end); + +/* + * This reads short-hand syntax that not only evaluates to a commit + * object name, but also can act as if the end user spelled the name + * of the branch from the command line. + * + * - "@{-N}" finds the name of the Nth previous branch we were on, and + * places the name of the branch in the given buf and returns the + * number of characters parsed if successful. + * + * - "@{upstream}" finds the name of the other ref that + * is configured to merge with (missing defaults + * to the current branch), and places the name of the branch in the + * given buf and returns the number of characters parsed if + * successful. + * + * If the input is not of the accepted format, it returns a negative + * number to signal an error. + * + * If the input was ok but there are not N branch switches in the + * reflog, it returns 0. + * + * If "allowed" is non-zero, it is a treated as a bitfield of allowable + * expansions: local branches ("refs/heads/"), remote branches + * ("refs/remotes/"), or "HEAD". If no "allowed" bits are set, any expansion is + * allowed, even ones to refs outside of those namespaces. + */ +#define INTERPRET_BRANCH_LOCAL (1<<0) +#define INTERPRET_BRANCH_REMOTE (1<<1) +#define INTERPRET_BRANCH_HEAD (1<<2) +extern int interpret_branch_name(const char *str, int len, struct strbuf *, + unsigned allowed); extern int get_oid_mb(const char *str, struct object_id *oid); extern int validate_headref(const char *ref); @@@ -1653,6 -1563,27 +1653,27 @@@ extern struct packed_git *find_sha1_pac extern void pack_report(void); + /* + * Create a temporary file rooted in the object database directory. + */ + extern int odb_mkstemp(char *template, size_t limit, const char *pattern); + + /* + * Generate the filename to be used for a pack file with checksum "sha1" and + * extension "ext". The result is written into the strbuf "buf", overwriting + * any existing contents. A pointer to buf->buf is returned as a convenience. + * + * Example: odb_pack_name(out, sha1, "idx") => ".git/objects/pack/pack-1234..idx" + */ + extern char *odb_pack_name(struct strbuf *buf, const unsigned char *sha1, const char *ext); + + /* + * Create a pack .keep file named "name" (which should generally be the output + * of odb_pack_name). Returns a file descriptor opened for writing, or -1 on + * error. + */ + extern int odb_pack_keep(const char *name); + /* * mmap the index file for the specified packfile (if it is not * already mmapped). Return 0 on success. @@@ -1689,12 -1620,6 +1710,12 @@@ extern void check_pack_index_ptr(const * error. */ extern const unsigned char *nth_packed_object_sha1(struct packed_git *, uint32_t n); +/* + * Like nth_packed_object_sha1, but write the data into the object specified by + * the the first argument. Returns the first argument on success, and NULL on + * error. + */ +extern const struct object_id *nth_packed_object_oid(struct object_id *, struct packed_git *, uint32_t n); /* * Return the offset of the nth object within the specified packfile. @@@ -1736,7 -1661,7 +1757,7 @@@ extern int unpack_object_header(struct * scratch buffer, but restored to its original contents before * the function returns. */ -typedef int each_loose_object_fn(const unsigned char *sha1, +typedef int each_loose_object_fn(const struct object_id *oid, const char *path, void *data); typedef int each_loose_cruft_fn(const char *basename, @@@ -1762,7 -1687,7 +1783,7 @@@ int for_each_loose_file_in_objdir_buf(s * LOCAL_ONLY flag is set). */ #define FOR_EACH_OBJECT_LOCAL_ONLY 0x1 -typedef int each_packed_object_fn(const unsigned char *sha1, +typedef int each_packed_object_fn(const struct object_id *oid, struct packed_git *pack, uint32_t pos, void *data); @@@ -1849,7 -1774,6 +1870,7 @@@ extern int git_config_from_blob_sha1(co const unsigned char *sha1, void *data); extern void git_config_push_parameter(const char *text); extern int git_config_from_parameters(config_fn_t fn, void *data); +extern void read_early_config(config_fn_t cb, void *data); extern void git_config(config_fn_t fn, void *); extern int git_config_with_options(config_fn_t fn, void *, struct git_config_source *config_source, @@@ -1916,11 -1840,8 +1937,11 @@@ extern int git_config_include(const cha * * (i.e., what gets handed to a config_fn_t). The caller provides the section; * we return -1 if it does not match, 0 otherwise. The subsection and key - * out-parameters are filled by the function (and subsection is NULL if it is + * out-parameters are filled by the function (and *subsection is NULL if it is * missing). + * + * If the subsection pointer-to-pointer passed in is NULL, returns 0 only if + * there is no subsection at all. */ extern int parse_config_key(const char *var, const char *section, @@@ -1982,11 -1903,6 +2003,11 @@@ extern int git_config_get_bool_or_int(c extern int git_config_get_maybe_bool(const char *key, int *dest); extern int git_config_get_pathname(const char *key, const char **dest); extern int git_config_get_untracked_cache(void); +extern int git_config_get_split_index(void); +extern int git_config_get_max_percent_split_change(void); + +/* This dies if the configured or default date is in the future */ +extern int git_config_get_expiry(const char *key, const char **output); /* * This is a hack for test programs like test-dump-untracked-cache to diff --combined environment.c index 42dc3106d2,4cd20d7067..2fdba76222 --- a/environment.c +++ b/environment.c @@@ -259,7 -259,7 +259,7 @@@ void set_git_work_tree(const char *new_ return; } git_work_tree_initialized = 1; - work_tree = real_pathdup(new_work_tree); + work_tree = real_pathdup(new_work_tree, 1); } const char *get_git_work_tree(void) @@@ -296,18 -296,16 +296,16 @@@ int odb_mkstemp(char *template, size_t return xmkstemp_mode(template, mode); } - int odb_pack_keep(char *name, size_t namesz, const unsigned char *sha1) + int odb_pack_keep(const char *name) { int fd; - snprintf(name, namesz, "%s/pack/pack-%s.keep", - get_object_directory(), sha1_to_hex(sha1)); fd = open(name, O_RDWR|O_CREAT|O_EXCL, 0600); if (0 <= fd) return fd; /* slow path */ - safe_create_leading_directories(name); + safe_create_leading_directories_const(name); return open(name, O_RDWR|O_CREAT|O_EXCL, 0600); } diff --combined fast-import.c index 6c13472c42,4d5a7f58d0..41a539f97d --- a/fast-import.c +++ b/fast-import.c @@@ -940,41 -940,40 +940,40 @@@ static const char *create_index(void static char *keep_pack(const char *curr_index_name) { - static char name[PATH_MAX]; static const char *keep_msg = "fast-import"; + struct strbuf name = STRBUF_INIT; int keep_fd; - keep_fd = odb_pack_keep(name, sizeof(name), pack_data->sha1); + odb_pack_name(&name, pack_data->sha1, "keep"); + keep_fd = odb_pack_keep(name.buf); if (keep_fd < 0) die_errno("cannot create keep file"); write_or_die(keep_fd, keep_msg, strlen(keep_msg)); if (close(keep_fd)) die_errno("failed to write keep file"); - snprintf(name, sizeof(name), "%s/pack/pack-%s.pack", - get_object_directory(), sha1_to_hex(pack_data->sha1)); - if (finalize_object_file(pack_data->pack_name, name)) + odb_pack_name(&name, pack_data->sha1, "pack"); + if (finalize_object_file(pack_data->pack_name, name.buf)) die("cannot store pack file"); - snprintf(name, sizeof(name), "%s/pack/pack-%s.idx", - get_object_directory(), sha1_to_hex(pack_data->sha1)); - if (finalize_object_file(curr_index_name, name)) + odb_pack_name(&name, pack_data->sha1, "idx"); + if (finalize_object_file(curr_index_name, name.buf)) die("cannot store index file"); free((void *)curr_index_name); - return name; + return strbuf_detach(&name, NULL); } static void unkeep_all_packs(void) { - static char name[PATH_MAX]; + struct strbuf name = STRBUF_INIT; int k; for (k = 0; k < pack_id; k++) { struct packed_git *p = all_packs[k]; - snprintf(name, sizeof(name), "%s/pack/pack-%s.keep", - get_object_directory(), sha1_to_hex(p->sha1)); - unlink_or_warn(name); + odb_pack_name(&name, p->sha1, "keep"); + unlink_or_warn(name.buf); } + strbuf_release(&name); } static int loosen_small_pack(const struct packed_git *p) @@@ -1033,6 -1032,7 +1032,7 @@@ static void end_packfile(void die("core git rejected index %s", idx_name); all_packs[pack_id] = new_p; install_packed_git(new_p); + free(idx_name); /* Print the boundary */ if (pack_edges) { @@@ -1752,7 -1752,7 +1752,7 @@@ static int update_branch(struct branch if (is_null_sha1(b->sha1)) { if (b->delete) - delete_ref(b->name, NULL, 0); + delete_ref(NULL, b->name, NULL, 0); return 0; } if (read_ref(b->name, old_sha1)) diff --combined git-compat-util.h index e626851fe9,412703b4ff..8a4a3f85e7 --- a/git-compat-util.h +++ b/git-compat-util.h @@@ -639,6 -639,11 +639,6 @@@ extern int gitsetenv(const char *, cons extern char *gitmkdtemp(char *); #endif -#ifdef NO_MKSTEMPS -#define mkstemps gitmkstemps -extern int gitmkstemps(char *, int); -#endif - #ifdef NO_UNSETENV #define unsetenv gitunsetenv extern void gitunsetenv(const char *); @@@ -798,8 -803,6 +798,6 @@@ extern FILE *xfopen(const char *path, c extern FILE *xfdopen(int fd, const char *mode); extern int xmkstemp(char *template); extern int xmkstemp_mode(char *template, int mode); - extern int odb_mkstemp(char *template, size_t limit, const char *pattern); - extern int odb_pack_keep(char *name, size_t namesz, const unsigned char *sha1); extern char *xgetcwd(void); extern FILE *fopen_for_writing(const char *path); diff --combined sha1_file.c index 29bbc5f427,df98c7f0dc..71063890ff --- a/sha1_file.c +++ b/sha1_file.c @@@ -129,10 -129,8 +129,10 @@@ enum scld_error safe_create_leading_dir *slash = '\0'; if (!stat(path, &st)) { /* path exists */ - if (!S_ISDIR(st.st_mode)) + if (!S_ISDIR(st.st_mode)) { + errno = ENOTDIR; ret = SCLD_EXISTS; + } } else if (mkdir(path, 0777)) { if (errno == EEXIST && !stat(path, &st) && S_ISDIR(st.st_mode)) @@@ -160,85 -158,13 +160,85 @@@ enum scld_error safe_create_leading_directories_const(const char *path) { + int save_errno; /* path points to cache entries, so xstrdup before messing with it */ char *buf = xstrdup(path); enum scld_error result = safe_create_leading_directories(buf); + + save_errno = errno; free(buf); + errno = save_errno; return result; } +int raceproof_create_file(const char *path, create_file_fn fn, void *cb) +{ + /* + * The number of times we will try to remove empty directories + * in the way of path. This is only 1 because if another + * process is racily creating directories that conflict with + * us, we don't want to fight against them. + */ + int remove_directories_remaining = 1; + + /* + * The number of times that we will try to create the + * directories containing path. We are willing to attempt this + * more than once, because another process could be trying to + * clean up empty directories at the same time as we are + * trying to create them. + */ + int create_directories_remaining = 3; + + /* A scratch copy of path, filled lazily if we need it: */ + struct strbuf path_copy = STRBUF_INIT; + + int ret, save_errno; + + /* Sanity check: */ + assert(*path); + +retry_fn: + ret = fn(path, cb); + save_errno = errno; + if (!ret) + goto out; + + if (errno == EISDIR && remove_directories_remaining-- > 0) { + /* + * A directory is in the way. Maybe it is empty; try + * to remove it: + */ + if (!path_copy.len) + strbuf_addstr(&path_copy, path); + + if (!remove_dir_recursively(&path_copy, REMOVE_DIR_EMPTY_ONLY)) + goto retry_fn; + } else if (errno == ENOENT && create_directories_remaining-- > 0) { + /* + * Maybe the containing directory didn't exist, or + * maybe it was just deleted by a process that is + * racing with us to clean up empty directories. Try + * to create it: + */ + enum scld_error scld_result; + + if (!path_copy.len) + strbuf_addstr(&path_copy, path); + + do { + scld_result = safe_create_leading_directories(path_copy.buf); + if (scld_result == SCLD_OK) + goto retry_fn; + } while (scld_result == SCLD_VANISHED && create_directories_remaining-- > 0); + } + +out: + strbuf_release(&path_copy); + errno = save_errno; + return ret; +} + static void fill_sha1_path(struct strbuf *buf, const unsigned char *sha1) { int i; @@@ -277,31 -203,26 +277,26 @@@ static const char *alt_sha1_path(struc return buf->buf; } - /* - * Return the name of the pack or index file with the specified sha1 - * in its filename. *base and *name are scratch space that must be - * provided by the caller. which should be "pack" or "idx". - */ - static char *sha1_get_pack_name(const unsigned char *sha1, - struct strbuf *buf, - const char *which) + char *odb_pack_name(struct strbuf *buf, + const unsigned char *sha1, + const char *ext) { strbuf_reset(buf); strbuf_addf(buf, "%s/pack/pack-%s.%s", get_object_directory(), - sha1_to_hex(sha1), which); + sha1_to_hex(sha1), ext); return buf->buf; } char *sha1_pack_name(const unsigned char *sha1) { static struct strbuf buf = STRBUF_INIT; - return sha1_get_pack_name(sha1, &buf, "pack"); + return odb_pack_name(&buf, sha1, "pack"); } char *sha1_pack_index_name(const unsigned char *sha1) { static struct strbuf buf = STRBUF_INIT; - return sha1_get_pack_name(sha1, &buf, "idx"); + return odb_pack_name(&buf, sha1, "idx"); } struct alternate_object_database *alt_odb_list; @@@ -667,7 -588,7 +662,7 @@@ static int freshen_file(const char *fn * either does not exist on disk, or has a stale mtime and may be subject to * pruning). */ -static int check_and_freshen_file(const char *fn, int freshen) +int check_and_freshen_file(const char *fn, int freshen) { if (access(fn, F_OK)) return 0; @@@ -2606,7 -2527,6 +2601,7 @@@ void *unpack_entry(struct packed_git *p while (delta_stack_nr) { void *delta_data; void *base = data; + void *external_base = NULL; unsigned long delta_size, base_size = size; int i; @@@ -2633,7 -2553,6 +2628,7 @@@ p->pack_name); mark_bad_packed_object(p, base_sha1); base = read_object(base_sha1, &type, &base_size); + external_base = base; } } @@@ -2652,7 -2571,6 +2647,7 @@@ "at offset %"PRIuMAX" from %s", (uintmax_t)curpos, p->pack_name); data = NULL; + free(external_base); continue; } @@@ -2672,7 -2590,6 +2667,7 @@@ error("failed to apply delta"); free(delta_data); + free(external_base); } *final_type = type; @@@ -2706,17 -2623,6 +2701,17 @@@ const unsigned char *nth_packed_object_ } } +const struct object_id *nth_packed_object_oid(struct object_id *oid, + struct packed_git *p, + uint32_t n) +{ + const unsigned char *hash = nth_packed_object_sha1(p, n); + if (!hash) + return NULL; + hashcpy(oid->hash, hash); + return oid; +} + void check_pack_index_ptr(const struct packed_git *p, const void *vptr) { const unsigned char *ptr = vptr; @@@ -3763,15 -3669,15 +3758,15 @@@ static int for_each_file_in_obj_subdir( strbuf_setlen(path, baselen); strbuf_addf(path, "/%s", de->d_name); - if (strlen(de->d_name) == 38) { - char hex[41]; - unsigned char sha1[20]; + if (strlen(de->d_name) == GIT_SHA1_HEXSZ - 2) { + char hex[GIT_SHA1_HEXSZ+1]; + struct object_id oid; snprintf(hex, sizeof(hex), "%02x%s", subdir_nr, de->d_name); - if (!get_sha1_hex(hex, sha1)) { + if (!get_oid_hex(hex, &oid)) { if (obj_cb) { - r = obj_cb(sha1, path->buf, data); + r = obj_cb(&oid, path->buf, data); if (r) break; } @@@ -3877,13 -3783,13 +3872,13 @@@ static int for_each_object_in_pack(stru int r = 0; for (i = 0; i < p->num_objects; i++) { - const unsigned char *sha1 = nth_packed_object_sha1(p, i); + struct object_id oid; - if (!sha1) + if (!nth_packed_object_oid(&oid, p, i)) return error("unable to get sha1 of object %u in %s", i, p->pack_name); - r = cb(sha1, p, i, data); + r = cb(&oid, p, i, data); if (r) break; }