From: Junio C Hamano Date: Mon, 27 Jan 2014 18:45:33 +0000 (-0800) Subject: Merge branch 'mh/safe-create-leading-directories' X-Git-Tag: v1.9-rc1~10 X-Git-Url: https://git.lorimer.id.au/gitweb.git/diff_plain/d0956cfa8ef8b3668ea6adc94b27f5dcdc93bde0?hp=-c Merge branch 'mh/safe-create-leading-directories' Code clean-up and protection against concurrent write access to the ref namespace. * mh/safe-create-leading-directories: rename_tmp_log(): on SCLD_VANISHED, retry rename_tmp_log(): limit the number of remote_empty_directories() attempts rename_tmp_log(): handle a possible mkdir/rmdir race rename_ref(): extract function rename_tmp_log() remove_dir_recurse(): handle disappearing files and directories remove_dir_recurse(): tighten condition for removing unreadable dir lock_ref_sha1_basic(): if locking fails with ENOENT, retry lock_ref_sha1_basic(): on SCLD_VANISHED, retry safe_create_leading_directories(): add new error value SCLD_VANISHED cmd_init_db(): when creating directories, handle errors conservatively safe_create_leading_directories(): introduce enum for return values safe_create_leading_directories(): always restore slash at end of loop safe_create_leading_directories(): split on first of multiple slashes safe_create_leading_directories(): rename local variable safe_create_leading_directories(): add explicit "slash" pointer safe_create_leading_directories(): reduce scope of local variable safe_create_leading_directories(): fix format of "if" chaining --- d0956cfa8ef8b3668ea6adc94b27f5dcdc93bde0 diff --combined builtin/init-db.c index b3f03cf0d6,ceeb138ba8..c7c76bbf21 --- a/builtin/init-db.c +++ b/builtin/init-db.c @@@ -266,7 -266,7 +266,7 @@@ static int create_default_files(const c /* allow template config file to override the default */ if (log_all_ref_updates == -1) git_config_set("core.logallrefupdates", "true"); - if (prefixcmp(git_dir, work_tree) || + if (!starts_with(git_dir, work_tree) || strcmp(git_dir + strlen(work_tree), "/.git")) { git_config_set("core.worktree", work_tree); } @@@ -515,13 -515,14 +515,14 @@@ int cmd_init_db(int argc, const char ** saved = shared_repository; shared_repository = 0; switch (safe_create_leading_directories_const(argv[0])) { - case -3: + case SCLD_OK: + case SCLD_PERMS: + break; + case SCLD_EXISTS: errno = EEXIST; /* fallthru */ - case -1: - die_errno(_("cannot mkdir %s"), argv[0]); - break; default: + die_errno(_("cannot mkdir %s"), argv[0]); break; } shared_repository = saved; diff --combined cache.h index 763c961520,f34c0a7c28..dc040fb1aa --- a/cache.h +++ b/cache.h @@@ -354,7 -354,6 +354,7 @@@ static inline enum object_type object_t #define DB_ENVIRONMENT "GIT_OBJECT_DIRECTORY" #define INDEX_ENVIRONMENT "GIT_INDEX_FILE" #define GRAFT_ENVIRONMENT "GIT_GRAFT_FILE" +#define GIT_SHALLOW_FILE_ENVIRONMENT "GIT_SHALLOW_FILE" #define TEMPLATE_DIR_ENVIRONMENT "GIT_TEMPLATE_DIR" #define CONFIG_ENVIRONMENT "GIT_CONFIG" #define CONFIG_DATA_ENVIRONMENT "GIT_CONFIG_PARAMETERS" @@@ -737,8 -736,29 +737,29 @@@ enum sharedrepo }; int git_config_perm(const char *var, const char *value); int adjust_shared_perm(const char *path); - int safe_create_leading_directories(char *path); - int safe_create_leading_directories_const(const char *path); + + /* + * Create the directory containing the named path, using care to be + * somewhat safe against races. Return one of the scld_error values + * to indicate success/failure. + * + * SCLD_VANISHED indicates that one of the ancestor directories of the + * path existed at one point during the function call and then + * suddenly vanished, probably because another process pruned the + * directory while we were working. To be robust against this kind of + * race, callers might want to try invoking the function again when it + * returns SCLD_VANISHED. + */ + enum scld_error { + SCLD_OK = 0, + SCLD_FAILED = -1, + SCLD_PERMS = -2, + SCLD_EXISTS = -3, + SCLD_VANISHED = -4 + }; + enum scld_error safe_create_leading_directories(char *path); + enum scld_error safe_create_leading_directories_const(const char *path); + int mkdir_in_gitdir(const char *path); extern void home_config_paths(char **global, char **xdg, char *file); extern char *expand_user_path(const char *path); @@@ -761,11 -781,11 +782,11 @@@ int daemon_avoid_alias(const char *path int offset_1st_component(const char *path); /* object replacement */ -#define READ_SHA1_FILE_REPLACE 1 +#define LOOKUP_REPLACE_OBJECT 1 extern void *read_sha1_file_extended(const unsigned char *sha1, enum object_type *type, unsigned long *size, unsigned flag); static inline void *read_sha1_file(const unsigned char *sha1, enum object_type *type, unsigned long *size) { - return read_sha1_file_extended(sha1, type, size, READ_SHA1_FILE_REPLACE); + return read_sha1_file_extended(sha1, type, size, LOOKUP_REPLACE_OBJECT); } extern const unsigned char *do_lookup_replace_object(const unsigned char *sha1); static inline const unsigned char *lookup_replace_object(const unsigned char *sha1) @@@ -774,12 -794,6 +795,12 @@@ return sha1; return do_lookup_replace_object(sha1); } +static inline const unsigned char *lookup_replace_object_extended(const unsigned char *sha1, unsigned flag) +{ + if (!(flag & LOOKUP_REPLACE_OBJECT)) + return sha1; + return lookup_replace_object(sha1); +} /* Read and unpack a sha1 file into memory, write memory to a sha1 file */ extern int sha1_object_info(const unsigned char *, unsigned long *); @@@ -894,12 -908,9 +915,12 @@@ extern int dwim_log(const char *str, in extern int interpret_branch_name(const char *str, int len, struct strbuf *); extern int get_sha1_mb(const char *str, unsigned char *sha1); -extern int refname_match(const char *abbrev_name, const char *full_name, const char **rules); -extern const char *ref_rev_parse_rules[]; -#define ref_fetch_rules ref_rev_parse_rules +/* + * Return true iff abbrev_name is a possible abbreviation for + * full_name according to the rules defined by ref_rev_parse_rules in + * refs.c. + */ +extern int refname_match(const char *abbrev_name, const char *full_name); extern int create_symref(const char *ref, const char *refs_heads_master, const char *logmsg); extern int validate_headref(const char *ref); @@@ -1084,7 -1095,6 +1105,7 @@@ struct object_info enum object_type *typep; unsigned long *sizep; unsigned long *disk_sizep; + unsigned char *delta_base_sha1; /* Response */ enum { @@@ -1109,7 -1119,7 +1130,7 @@@ } packed; } u; }; -extern int sha1_object_info_extended(const unsigned char *, struct object_info *); +extern int sha1_object_info_extended(const unsigned char *, struct object_info *, unsigned flags); /* Dumb servers support */ extern int update_server_info(int); @@@ -1247,8 -1257,6 +1268,8 @@@ __attribute__((format (printf, 2, 3)) extern void trace_argv_printf(const char **argv, const char *format, ...); extern void trace_repo_setup(const char *prefix); extern int trace_want(const char *key); +__attribute__((format (printf, 2, 3))) +extern void trace_printf_key(const char *key, const char *fmt, ...); extern void trace_strbuf(const char *key, const struct strbuf *buf); void packet_trace_identity(const char *prog); diff --combined dir.c index d10a63f731,716b61320a..b35b6330f8 --- a/dir.c +++ b/dir.c @@@ -126,13 -126,10 +126,13 @@@ static size_t common_prefix_len(const s PATHSPEC_MAXDEPTH | PATHSPEC_LITERAL | PATHSPEC_GLOB | - PATHSPEC_ICASE); + PATHSPEC_ICASE | + PATHSPEC_EXCLUDE); for (n = 0; n < pathspec->nr; n++) { size_t i = 0, len = 0, item_len; + if (pathspec->items[n].magic & PATHSPEC_EXCLUDE) + continue; if (pathspec->items[n].magic & PATHSPEC_ICASE) item_len = pathspec->items[n].prefix; else @@@ -282,10 -279,9 +282,10 @@@ static int match_pathspec_item(const st * pathspec did not match any names, which could indicate that the * user mistyped the nth pathspec. */ -int match_pathspec_depth(const struct pathspec *ps, - const char *name, int namelen, - int prefix, char *seen) +static int match_pathspec_depth_1(const struct pathspec *ps, + const char *name, int namelen, + int prefix, char *seen, + int exclude) { int i, retval = 0; @@@ -294,8 -290,7 +294,8 @@@ PATHSPEC_MAXDEPTH | PATHSPEC_LITERAL | PATHSPEC_GLOB | - PATHSPEC_ICASE); + PATHSPEC_ICASE | + PATHSPEC_EXCLUDE); if (!ps->nr) { if (!ps->recursive || @@@ -314,19 -309,8 +314,19 @@@ for (i = ps->nr - 1; i >= 0; i--) { int how; + + if ((!exclude && ps->items[i].magic & PATHSPEC_EXCLUDE) || + ( exclude && !(ps->items[i].magic & PATHSPEC_EXCLUDE))) + continue; + if (seen && seen[i] == MATCHED_EXACTLY) continue; + /* + * Make exclude patterns optional and never report + * "pathspec ':(exclude)foo' matches no files" + */ + if (seen && ps->items[i].magic & PATHSPEC_EXCLUDE) + seen[i] = MATCHED_FNMATCH; how = match_pathspec_item(ps->items+i, prefix, name, namelen); if (ps->recursive && (ps->magic & PATHSPEC_MAXDEPTH) && @@@ -350,18 -334,6 +350,18 @@@ return retval; } +int match_pathspec_depth(const struct pathspec *ps, + const char *name, int namelen, + int prefix, char *seen) +{ + int positive, negative; + positive = match_pathspec_depth_1(ps, name, namelen, prefix, seen, 0); + if (!(ps->magic & PATHSPEC_EXCLUDE) || !positive) + return positive; + negative = match_pathspec_depth_1(ps, name, namelen, prefix, seen, 1); + return negative ? 0 : positive; +} + /* * Return the length of the "simple" part of a path match limiter. */ @@@ -1403,18 -1375,11 +1403,18 @@@ int read_directory(struct dir_struct *d PATHSPEC_MAXDEPTH | PATHSPEC_LITERAL | PATHSPEC_GLOB | - PATHSPEC_ICASE); + PATHSPEC_ICASE | + PATHSPEC_EXCLUDE); if (has_symlink_leading_path(path, len)) return dir->nr; + /* + * exclude patterns are treated like positive ones in + * create_simplify. Usually exclude patterns should be a + * subset of positive ones, which has no impacts on + * create_simplify(). + */ simplify = create_simplify(pathspec ? pathspec->_raw : NULL); if (!len || treat_leading_path(dir, path, len, simplify)) read_directory_recursive(dir, path, len, 0, simplify); @@@ -1511,8 -1476,13 +1511,13 @@@ static int remove_dir_recurse(struct st flag &= ~REMOVE_DIR_KEEP_TOPLEVEL; dir = opendir(path->buf); if (!dir) { - /* an empty dir could be removed even if it is unreadble */ - if (!keep_toplevel) + if (errno == ENOENT) + return keep_toplevel ? -1 : 0; + else if (errno == EACCES && !keep_toplevel) + /* + * An empty dir could be removable even if it + * is unreadable: + */ return rmdir(path->buf); else return -1; @@@ -1528,13 -1498,21 +1533,21 @@@ strbuf_setlen(path, len); strbuf_addstr(path, e->d_name); - if (lstat(path->buf, &st)) - ; /* fall thru */ - else if (S_ISDIR(st.st_mode)) { + if (lstat(path->buf, &st)) { + if (errno == ENOENT) + /* + * file disappeared, which is what we + * wanted anyway + */ + continue; + /* fall thru */ + } else if (S_ISDIR(st.st_mode)) { if (!remove_dir_recurse(path, flag, &kept_down)) continue; /* happy */ - } else if (!only_empty && !unlink(path->buf)) + } else if (!only_empty && + (!unlink(path->buf) || errno == ENOENT)) { continue; /* happy, too */ + } /* path too long, stat fails, or non-directory still exists */ ret = -1; @@@ -1544,7 -1522,7 +1557,7 @@@ strbuf_setlen(path, original_len); if (!ret && !keep_toplevel && !kept_down) - ret = rmdir(path->buf); + ret = (!rmdir(path->buf) || errno == ENOENT) ? 0 : -1; else if (kept_up) /* * report the uplevel that it is not an error that we diff --combined merge-recursive.c index a18bd15dd3,021e1fc453..8400a8e937 --- a/merge-recursive.c +++ b/merge-recursive.c @@@ -693,7 -693,7 +693,7 @@@ static int make_room_for_path(struct me /* Make sure leading directories are created */ status = safe_create_leading_directories_const(path); if (status) { - if (status == -3) { + if (status == SCLD_EXISTS) { /* something else exists */ error(msg, path, _(": perhaps a D/F conflict?")); return -1; @@@ -2063,13 -2063,13 +2063,13 @@@ int parse_merge_opt(struct merge_option o->recursive_variant = MERGE_RECURSIVE_THEIRS; else if (!strcmp(s, "subtree")) o->subtree_shift = ""; - else if (!prefixcmp(s, "subtree=")) + else if (starts_with(s, "subtree=")) o->subtree_shift = s + strlen("subtree="); else if (!strcmp(s, "patience")) o->xdl_opts = DIFF_WITH_ALG(o, PATIENCE_DIFF); else if (!strcmp(s, "histogram")) o->xdl_opts = DIFF_WITH_ALG(o, HISTOGRAM_DIFF); - else if (!prefixcmp(s, "diff-algorithm=")) { + else if (starts_with(s, "diff-algorithm=")) { long value = parse_algorithm_value(s + strlen("diff-algorithm=")); if (value < 0) return -1; @@@ -2088,7 -2088,7 +2088,7 @@@ o->renormalize = 1; else if (!strcmp(s, "no-renormalize")) o->renormalize = 0; - else if (!prefixcmp(s, "rename-threshold=")) { + else if (starts_with(s, "rename-threshold=")) { const char *score = s + strlen("rename-threshold="); if ((o->rename_score = parse_rename_score(&score)) == -1 || *score != 0) return -1; diff --combined refs.c index fc33ee8ffb,703b5a2f45..89228e2373 --- a/refs.c +++ b/refs.c @@@ -637,7 -637,7 +637,7 @@@ static int do_one_ref(struct ref_entry struct ref_entry *old_current_ref; int retval; - if (prefixcmp(entry->name, data->base)) + if (!starts_with(entry->name, data->base)) return 0; if (!(data->flags & DO_FOR_EACH_INCLUDE_BROKEN) && @@@ -1042,7 -1042,7 +1042,7 @@@ static void read_packed_refs(FILE *f, s if (refname) { last = create_ref_entry(refname, sha1, REF_ISPACKED, 1); if (peeled == PEELED_FULLY || - (peeled == PEELED_TAGS && !prefixcmp(refname, "refs/tags/"))) + (peeled == PEELED_TAGS && starts_with(refname, "refs/tags/"))) last->flag |= REF_KNOWS_PEELED; add_ref(dir, last); continue; @@@ -1376,7 -1376,7 +1376,7 @@@ const char *resolve_ref_unsafe(const ch return NULL; } buffer[len] = 0; - if (!prefixcmp(buffer, "refs/") && + if (starts_with(buffer, "refs/") && !check_refname_format(buffer, 0)) { strcpy(refname_buffer, buffer); refname = refname_buffer; @@@ -1415,7 -1415,7 +1415,7 @@@ /* * Is it a symbolic ref? */ - if (prefixcmp(buffer, "ref:")) { + if (!starts_with(buffer, "ref:")) { /* * Please note that FETCH_HEAD has a second * line containing other data. @@@ -1837,7 -1837,7 +1837,7 @@@ int for_each_glob_ref_in(each_ref_fn fn struct ref_filter filter; int ret; - if (!prefix && prefixcmp(pattern, "refs/")) + if (!prefix && !starts_with(pattern, "refs/")) strbuf_addstr(&real_pattern, "refs/"); else if (prefix) strbuf_addstr(&real_pattern, prefix); @@@ -1874,13 -1874,13 +1874,13 @@@ int for_each_rawref(each_ref_fn fn, voi const char *prettify_refname(const char *name) { return name + ( - !prefixcmp(name, "refs/heads/") ? 11 : - !prefixcmp(name, "refs/tags/") ? 10 : - !prefixcmp(name, "refs/remotes/") ? 13 : + starts_with(name, "refs/heads/") ? 11 : + starts_with(name, "refs/tags/") ? 10 : + starts_with(name, "refs/remotes/") ? 13 : 0); } -const char *ref_rev_parse_rules[] = { +static const char *ref_rev_parse_rules[] = { "%.*s", "refs/%.*s", "refs/tags/%.*s", @@@ -1890,12 -1890,12 +1890,12 @@@ NULL }; -int refname_match(const char *abbrev_name, const char *full_name, const char **rules) +int refname_match(const char *abbrev_name, const char *full_name) { const char **p; const int abbrev_name_len = strlen(abbrev_name); - for (p = rules; *p; p++) { + for (p = ref_rev_parse_rules; *p; p++) { if (!strcmp(full_name, mkpath(*p, abbrev_name_len, abbrev_name))) { return 1; } @@@ -2039,6 -2039,7 +2039,7 @@@ static struct ref_lock *lock_ref_sha1_b int type, lflags; int mustexist = (old_sha1 && !is_null_sha1(old_sha1)); int missing = 0; + int attempts_remaining = 3; lock = xcalloc(1, sizeof(struct ref_lock)); lock->lock_fd = -1; @@@ -2080,7 -2081,7 +2081,7 @@@ lock->lk = xcalloc(1, sizeof(struct lock_file)); - lflags = LOCK_DIE_ON_ERROR; + lflags = 0; if (flags & REF_NODEREF) { refname = orig_refname; lflags |= LOCK_NODEREF; @@@ -2093,13 -2094,32 +2094,32 @@@ if ((flags & REF_NODEREF) && (type & REF_ISSYMREF)) lock->force_write = 1; - if (safe_create_leading_directories(ref_file)) { + retry: + switch (safe_create_leading_directories(ref_file)) { + case SCLD_OK: + break; /* success */ + case SCLD_VANISHED: + if (--attempts_remaining > 0) + goto retry; + /* fall through */ + default: last_errno = errno; error("unable to create directory for %s", ref_file); goto error_return; } lock->lock_fd = hold_lock_file_for_update(lock->lk, ref_file, lflags); + if (lock->lock_fd < 0) { + if (errno == ENOENT && --attempts_remaining > 0) + /* + * Maybe somebody just deleted one of the + * directories leading to ref_file. Try + * again: + */ + goto retry; + else + unable_to_lock_index_die(ref_file, errno); + } return old_sha1 ? verify_lock(lock, old_sha1, mustexist) : lock; error_return: @@@ -2244,7 -2264,7 +2264,7 @@@ static int pack_if_possible_fn(struct r struct pack_refs_cb_data *cb = cb_data; enum peel_status peel_status; struct ref_entry *packed_entry; - int is_tag_ref = !prefixcmp(entry->name, "refs/tags/"); + int is_tag_ref = starts_with(entry->name, "refs/tags/"); /* ALWAYS pack tags */ if (!(cb->flags & PACK_REFS_ALL) && !is_tag_ref) @@@ -2508,6 -2528,51 +2528,51 @@@ int delete_ref(const char *refname, con */ #define TMP_RENAMED_LOG "logs/refs/.tmp-renamed-log" + static int rename_tmp_log(const char *newrefname) + { + int attempts_remaining = 4; + + retry: + switch (safe_create_leading_directories(git_path("logs/%s", newrefname))) { + case SCLD_OK: + break; /* success */ + case SCLD_VANISHED: + if (--attempts_remaining > 0) + goto retry; + /* fall through */ + default: + error("unable to create directory for %s", newrefname); + return -1; + } + + if (rename(git_path(TMP_RENAMED_LOG), git_path("logs/%s", newrefname))) { + if ((errno==EISDIR || errno==ENOTDIR) && --attempts_remaining > 0) { + /* + * rename(a, b) when b is an existing + * directory ought to result in ISDIR, but + * Solaris 5.8 gives ENOTDIR. Sheesh. + */ + if (remove_empty_directories(git_path("logs/%s", newrefname))) { + error("Directory not empty: logs/%s", newrefname); + return -1; + } + goto retry; + } else if (errno == ENOENT && --attempts_remaining > 0) { + /* + * Maybe another process just deleted one of + * the directories in the path to newrefname. + * Try again from the beginning. + */ + goto retry; + } else { + error("unable to move logfile "TMP_RENAMED_LOG" to logs/%s: %s", + newrefname, strerror(errno)); + return -1; + } + } + return 0; + } + int rename_ref(const char *oldrefname, const char *newrefname, const char *logmsg) { unsigned char sha1[20], orig_sha1[20]; @@@ -2555,30 -2620,9 +2620,9 @@@ } } - if (log && safe_create_leading_directories(git_path("logs/%s", newrefname))) { - error("unable to create directory for %s", newrefname); + if (log && rename_tmp_log(newrefname)) goto rollback; - } - retry: - if (log && rename(git_path(TMP_RENAMED_LOG), git_path("logs/%s", newrefname))) { - if (errno==EISDIR || errno==ENOTDIR) { - /* - * rename(a, b) when b is an existing - * directory ought to result in ISDIR, but - * Solaris 5.8 gives ENOTDIR. Sheesh. - */ - if (remove_empty_directories(git_path("logs/%s", newrefname))) { - error("Directory not empty: logs/%s", newrefname); - goto rollback; - } - goto retry; - } else { - error("unable to move logfile "TMP_RENAMED_LOG" to logs/%s: %s", - newrefname, strerror(errno)); - goto rollback; - } - } logmoved = log; lock = lock_ref_sha1_basic(newrefname, NULL, 0, NULL); @@@ -2679,9 -2723,9 +2723,9 @@@ int log_ref_setup(const char *refname, git_snpath(logfile, bufsize, "logs/%s", refname); if (log_all_ref_updates && - (!prefixcmp(refname, "refs/heads/") || - !prefixcmp(refname, "refs/remotes/") || - !prefixcmp(refname, "refs/notes/") || + (starts_with(refname, "refs/heads/") || + starts_with(refname, "refs/remotes/") || + starts_with(refname, "refs/notes/") || !strcmp(refname, "HEAD"))) { if (safe_create_leading_directories(logfile) < 0) return error("unable to create directory for %s", @@@ -2751,7 -2795,7 +2795,7 @@@ static int log_ref_write(const char *re static int is_branch(const char *refname) { - return !strcmp(refname, "HEAD") || !prefixcmp(refname, "refs/heads/"); + return !strcmp(refname, "HEAD") || starts_with(refname, "refs/heads/"); } int write_ref_sha1(struct ref_lock *lock, @@@ -3334,6 -3378,29 +3378,6 @@@ cleanup return ret; } -/* - * generate a format suitable for scanf from a ref_rev_parse_rules - * rule, that is replace the "%.*s" spec with a "%s" spec - */ -static void gen_scanf_fmt(char *scanf_fmt, const char *rule) -{ - char *spec; - - spec = strstr(rule, "%.*s"); - if (!spec || strstr(spec + 4, "%.*s")) - die("invalid rule in ref_rev_parse_rules: %s", rule); - - /* copy all until spec */ - strncpy(scanf_fmt, rule, spec - rule); - scanf_fmt[spec - rule] = '\0'; - /* copy new spec */ - strcat(scanf_fmt, "%s"); - /* copy remaining rule */ - strcat(scanf_fmt, spec + 4); - - return; -} - char *shorten_unambiguous_ref(const char *refname, int strict) { int i; @@@ -3341,29 -3408,23 +3385,29 @@@ static int nr_rules; char *short_name; - /* pre generate scanf formats from ref_rev_parse_rules[] */ if (!nr_rules) { + /* + * Pre-generate scanf formats from ref_rev_parse_rules[]. + * Generate a format suitable for scanf from a + * ref_rev_parse_rules rule by interpolating "%s" at the + * location of the "%.*s". + */ size_t total_len = 0; + size_t offset = 0; /* the rule list is NULL terminated, count them first */ for (nr_rules = 0; ref_rev_parse_rules[nr_rules]; nr_rules++) - /* no +1 because strlen("%s") < strlen("%.*s") */ - total_len += strlen(ref_rev_parse_rules[nr_rules]); + /* -2 for strlen("%.*s") - strlen("%s"); +1 for NUL */ + total_len += strlen(ref_rev_parse_rules[nr_rules]) - 2 + 1; scanf_fmts = xmalloc(nr_rules * sizeof(char *) + total_len); - total_len = 0; + offset = 0; for (i = 0; i < nr_rules; i++) { - scanf_fmts[i] = (char *)&scanf_fmts[nr_rules] - + total_len; - gen_scanf_fmt(scanf_fmts[i], ref_rev_parse_rules[i]); - total_len += strlen(ref_rev_parse_rules[i]); + assert(offset < total_len); + scanf_fmts[i] = (char *)&scanf_fmts[nr_rules] + offset; + offset += snprintf(scanf_fmts[i], total_len - offset, + ref_rev_parse_rules[i], 2, "%s") + 1; } } @@@ -3433,7 -3494,7 +3477,7 @@@ int parse_hide_refs_config(const char * { if (!strcmp("transfer.hiderefs", var) || /* NEEDSWORK: use parse_config_key() once both are merged */ - (!prefixcmp(var, section) && var[strlen(section)] == '.' && + (starts_with(var, section) && var[strlen(section)] == '.' && !strcmp(var + strlen(section), ".hiderefs"))) { char *ref; int len; @@@ -3461,7 -3522,7 +3505,7 @@@ int ref_is_hidden(const char *refname return 0; for_each_string_list_item(item, hide_refs) { int len; - if (prefixcmp(refname, item->string)) + if (!starts_with(refname, item->string)) continue; len = strlen(item->string); if (!refname[len] || refname[len] == '/') diff --combined sha1_file.c index e13bd2c3ee,ed814e546f..8b0849f931 --- a/sha1_file.c +++ b/sha1_file.c @@@ -105,50 -105,59 +105,59 @@@ int mkdir_in_gitdir(const char *path return adjust_shared_perm(path); } - int safe_create_leading_directories(char *path) + enum scld_error safe_create_leading_directories(char *path) { - char *pos = path + offset_1st_component(path); - struct stat st; + char *next_component = path + offset_1st_component(path); + enum scld_error ret = SCLD_OK; - while (pos) { - pos = strchr(pos, '/'); - if (!pos) + while (ret == SCLD_OK && next_component) { + struct stat st; + char *slash = strchr(next_component, '/'); + + if (!slash) break; - while (*++pos == '/') - ; - if (!*pos) + + next_component = slash + 1; + while (*next_component == '/') + next_component++; + if (!*next_component) break; - *--pos = '\0'; + + *slash = '\0'; if (!stat(path, &st)) { /* path exists */ - if (!S_ISDIR(st.st_mode)) { - *pos = '/'; - return -3; - } - } - else if (mkdir(path, 0777)) { + if (!S_ISDIR(st.st_mode)) + ret = SCLD_EXISTS; + } else if (mkdir(path, 0777)) { if (errno == EEXIST && - !stat(path, &st) && S_ISDIR(st.st_mode)) { + !stat(path, &st) && S_ISDIR(st.st_mode)) ; /* somebody created it since we checked */ - } else { - *pos = '/'; - return -1; - } - } - else if (adjust_shared_perm(path)) { - *pos = '/'; - return -2; + else if (errno == ENOENT) + /* + * Either mkdir() failed because + * somebody just pruned the containing + * directory, or stat() failed because + * the file that was in our way was + * just removed. Either way, inform + * the caller that it might be worth + * trying again: + */ + ret = SCLD_VANISHED; + else + ret = SCLD_FAILED; + } else if (adjust_shared_perm(path)) { + ret = SCLD_PERMS; } - *pos++ = '/'; + *slash = '/'; } - return 0; + return ret; } - int safe_create_leading_directories_const(const char *path) + enum scld_error safe_create_leading_directories_const(const char *path) { /* path points to cache entries, so xstrdup before messing with it */ char *buf = xstrdup(path); - int result = safe_create_leading_directories(buf); + enum scld_error result = safe_create_leading_directories(buf); free(buf); return result; } @@@ -807,38 -816,15 +816,38 @@@ void free_pack_by_name(const char *pack static unsigned int get_max_fd_limit(void) { #ifdef RLIMIT_NOFILE - struct rlimit lim; + { + struct rlimit lim; - if (getrlimit(RLIMIT_NOFILE, &lim)) - die_errno("cannot get RLIMIT_NOFILE"); + if (!getrlimit(RLIMIT_NOFILE, &lim)) + return lim.rlim_cur; + } +#endif - return lim.rlim_cur; -#elif defined(_SC_OPEN_MAX) - return sysconf(_SC_OPEN_MAX); -#elif defined(OPEN_MAX) +#ifdef _SC_OPEN_MAX + { + long open_max = sysconf(_SC_OPEN_MAX); + if (0 < open_max) + return open_max; + /* + * Otherwise, we got -1 for one of the two + * reasons: + * + * (1) sysconf() did not understand _SC_OPEN_MAX + * and signaled an error with -1; or + * (2) sysconf() said there is no limit. + * + * We _could_ clear errno before calling sysconf() to + * tell these two cases apart and return a huge number + * in the latter case to let the caller cap it to a + * value that is not so selfish, but letting the + * fallback OPEN_MAX codepath take care of these cases + * is a lot simpler. + */ + } +#endif + +#ifdef OPEN_MAX return OPEN_MAX; #else return 1; /* see the caller ;-) */ @@@ -1465,6 -1451,51 +1474,6 @@@ void *map_sha1_file(const unsigned cha return map; } -/* - * There used to be a second loose object header format which - * was meant to mimic the in-pack format, allowing for direct - * copy of the object data. This format turned up not to be - * really worth it and we no longer write loose objects in that - * format. - */ -static int experimental_loose_object(unsigned char *map) -{ - unsigned int word; - - /* - * We must determine if the buffer contains the standard - * zlib-deflated stream or the experimental format based - * on the in-pack object format. Compare the header byte - * for each format: - * - * RFC1950 zlib w/ deflate : 0www1000 : 0 <= www <= 7 - * Experimental pack-based : Stttssss : ttt = 1,2,3,4 - * - * If bit 7 is clear and bits 0-3 equal 8, the buffer MUST be - * in standard loose-object format, UNLESS it is a Git-pack - * format object *exactly* 8 bytes in size when inflated. - * - * However, RFC1950 also specifies that the 1st 16-bit word - * must be divisible by 31 - this checksum tells us our buffer - * is in the standard format, giving a false positive only if - * the 1st word of the Git-pack format object happens to be - * divisible by 31, ie: - * ((byte0 * 256) + byte1) % 31 = 0 - * => 0ttt10000www1000 % 31 = 0 - * - * As it happens, this case can only arise for www=3 & ttt=1 - * - ie, a Commit object, which would have to be 8 bytes in - * size. As no Commit can be that small, we find that the - * combination of these two criteria (bitmask & checksum) - * can always correctly determine the buffer format. - */ - word = (map[0] << 8) + map[1]; - if ((map[0] & 0x8F) == 0x08 && !(word % 31)) - return 0; - else - return 1; -} - unsigned long unpack_object_header_buffer(const unsigned char *buf, unsigned long len, enum object_type *type, unsigned long *sizep) { @@@ -1492,6 -1523,14 +1501,6 @@@ int unpack_sha1_header(git_zstream *stream, unsigned char *map, unsigned long mapsize, void *buffer, unsigned long bufsiz) { - unsigned long size, used; - static const char valid_loose_object_type[8] = { - 0, /* OBJ_EXT */ - 1, 1, 1, 1, /* "commit", "tree", "blob", "tag" */ - 0, /* "delta" and others are invalid in a loose object */ - }; - enum object_type type; - /* Get the data stream */ memset(stream, 0, sizeof(*stream)); stream->next_in = map; @@@ -1499,6 -1538,27 +1508,6 @@@ stream->next_out = buffer; stream->avail_out = bufsiz; - if (experimental_loose_object(map)) { - /* - * The old experimental format we no longer produce; - * we can still read it. - */ - used = unpack_object_header_buffer(map, mapsize, &type, &size); - if (!used || !valid_loose_object_type[type]) - return -1; - map += used; - mapsize -= used; - - /* Set up the stream for the rest.. */ - stream->next_in = map; - stream->avail_in = mapsize; - git_inflate_init(stream); - - /* And generate the fake traditional header */ - stream->total_out = 1 + snprintf(buffer, bufsiz, "%s %lu", - typename(type), size); - return 0; - } git_inflate_init(stream); return git_inflate(stream, 0); } @@@ -1690,38 -1750,6 +1699,38 @@@ static off_t get_delta_base(struct pack return base_offset; } +/* + * Like get_delta_base above, but we return the sha1 instead of the pack + * offset. This means it is cheaper for REF deltas (we do not have to do + * the final object lookup), but more expensive for OFS deltas (we + * have to load the revidx to convert the offset back into a sha1). + */ +static const unsigned char *get_delta_base_sha1(struct packed_git *p, + struct pack_window **w_curs, + off_t curpos, + enum object_type type, + off_t delta_obj_offset) +{ + if (type == OBJ_REF_DELTA) { + unsigned char *base = use_pack(p, w_curs, curpos, NULL); + return base; + } else if (type == OBJ_OFS_DELTA) { + struct revindex_entry *revidx; + off_t base_offset = get_delta_base(p, w_curs, &curpos, + type, delta_obj_offset); + + if (!base_offset) + return NULL; + + revidx = find_pack_revindex(p, base_offset); + if (!revidx) + return NULL; + + return nth_packed_object_sha1(p, revidx->nr); + } else + return NULL; +} + int unpack_object_header(struct packed_git *p, struct pack_window **w_curs, off_t *curpos, @@@ -1879,22 -1907,6 +1888,22 @@@ static int packed_object_info(struct pa } } + if (oi->delta_base_sha1) { + if (type == OBJ_OFS_DELTA || type == OBJ_REF_DELTA) { + const unsigned char *base; + + base = get_delta_base_sha1(p, &w_curs, curpos, + type, obj_offset); + if (!base) { + type = OBJ_BAD; + goto out; + } + + hashcpy(oi->delta_base_sha1, base); + } else + hashclr(oi->delta_base_sha1); + } + out: unuse_pack(&w_curs); return type; @@@ -2478,9 -2490,6 +2487,9 @@@ static int sha1_loose_object_info(cons git_zstream stream; char hdr[32]; + if (oi->delta_base_sha1) + hashclr(oi->delta_base_sha1); + /* * If we don't care about type or size, then we don't * need to look inside the object at all. Note that we @@@ -2517,14 -2526,13 +2526,14 @@@ return 0; } -int sha1_object_info_extended(const unsigned char *sha1, struct object_info *oi) +int sha1_object_info_extended(const unsigned char *sha1, struct object_info *oi, unsigned flags) { struct cached_object *co; struct pack_entry e; int rtype; + const unsigned char *real = lookup_replace_object_extended(sha1, flags); - co = find_cached_object(sha1); + co = find_cached_object(real); if (co) { if (oi->typep) *(oi->typep) = co->type; @@@ -2532,29 -2540,27 +2541,29 @@@ *(oi->sizep) = co->size; if (oi->disk_sizep) *(oi->disk_sizep) = 0; + if (oi->delta_base_sha1) + hashclr(oi->delta_base_sha1); oi->whence = OI_CACHED; return 0; } - if (!find_pack_entry(sha1, &e)) { + if (!find_pack_entry(real, &e)) { /* Most likely it's a loose object. */ - if (!sha1_loose_object_info(sha1, oi)) { + if (!sha1_loose_object_info(real, oi)) { oi->whence = OI_LOOSE; return 0; } /* Not a loose object; someone else may have just packed it. */ reprepare_packed_git(); - if (!find_pack_entry(sha1, &e)) + if (!find_pack_entry(real, &e)) return -1; } rtype = packed_object_info(e.p, e.offset, oi); if (rtype < 0) { - mark_bad_packed_object(e.p, sha1); - return sha1_object_info_extended(sha1, oi); + mark_bad_packed_object(e.p, real); + return sha1_object_info_extended(real, oi, 0); } else if (in_delta_base_cache(e.p, e.offset)) { oi->whence = OI_DBCACHED; } else { @@@ -2576,7 -2582,7 +2585,7 @@@ int sha1_object_info(const unsigned cha oi.typep = &type; oi.sizep = sizep; - if (sha1_object_info_extended(sha1, &oi) < 0) + if (sha1_object_info_extended(sha1, &oi, LOOKUP_REPLACE_OBJECT) < 0) return -1; return type; } @@@ -2668,7 -2674,8 +2677,7 @@@ void *read_sha1_file_extended(const uns void *data; char *path; const struct packed_git *p; - const unsigned char *repl = (flag & READ_SHA1_FILE_REPLACE) - ? lookup_replace_object(sha1) : sha1; + const unsigned char *repl = lookup_replace_object_extended(sha1, flag); errno = 0; data = read_object(repl, type, size);