Merge branch 'mh/safe-create-leading-directories'
authorJunio C Hamano <gitster@pobox.com>
Mon, 27 Jan 2014 18:45:33 +0000 (10:45 -0800)
committerJunio C Hamano <gitster@pobox.com>
Mon, 27 Jan 2014 18:45:33 +0000 (10:45 -0800)
Code clean-up and protection against concurrent write access to the
ref namespace.

* mh/safe-create-leading-directories:
rename_tmp_log(): on SCLD_VANISHED, retry
rename_tmp_log(): limit the number of remote_empty_directories() attempts
rename_tmp_log(): handle a possible mkdir/rmdir race
rename_ref(): extract function rename_tmp_log()
remove_dir_recurse(): handle disappearing files and directories
remove_dir_recurse(): tighten condition for removing unreadable dir
lock_ref_sha1_basic(): if locking fails with ENOENT, retry
lock_ref_sha1_basic(): on SCLD_VANISHED, retry
safe_create_leading_directories(): add new error value SCLD_VANISHED
cmd_init_db(): when creating directories, handle errors conservatively
safe_create_leading_directories(): introduce enum for return values
safe_create_leading_directories(): always restore slash at end of loop
safe_create_leading_directories(): split on first of multiple slashes
safe_create_leading_directories(): rename local variable
safe_create_leading_directories(): add explicit "slash" pointer
safe_create_leading_directories(): reduce scope of local variable
safe_create_leading_directories(): fix format of "if" chaining

1  2 
builtin/init-db.c
cache.h
dir.c
merge-recursive.c
refs.c
sha1_file.c
diff --combined builtin/init-db.c
index b3f03cf0d6c0f776b2eaba8ff4e1d340216a4463,ceeb138ba826a83843b3dae0d90cddcab664770a..c7c76bbf21fd5b5c9200b2535875b912e4d49000
@@@ -266,7 -266,7 +266,7 @@@ static int create_default_files(const c
                /* allow template config file to override the default */
                if (log_all_ref_updates == -1)
                    git_config_set("core.logallrefupdates", "true");
 -              if (prefixcmp(git_dir, work_tree) ||
 +              if (!starts_with(git_dir, work_tree) ||
                    strcmp(git_dir + strlen(work_tree), "/.git")) {
                        git_config_set("core.worktree", work_tree);
                }
@@@ -515,13 -515,14 +515,14 @@@ int cmd_init_db(int argc, const char **
                                saved = shared_repository;
                                shared_repository = 0;
                                switch (safe_create_leading_directories_const(argv[0])) {
-                               case -3:
+                               case SCLD_OK:
+                               case SCLD_PERMS:
+                                       break;
+                               case SCLD_EXISTS:
                                        errno = EEXIST;
                                        /* fallthru */
-                               case -1:
-                                       die_errno(_("cannot mkdir %s"), argv[0]);
-                                       break;
                                default:
+                                       die_errno(_("cannot mkdir %s"), argv[0]);
                                        break;
                                }
                                shared_repository = saved;
diff --combined cache.h
index 763c961520f93999760d8f24bdb66af2678cca04,f34c0a7c2810934dd72b566c955b2f4c3eb42c25..dc040fb1aa99b7970e85e5b175e60f862ff6a74a
+++ b/cache.h
@@@ -354,7 -354,6 +354,7 @@@ static inline enum object_type object_t
  #define DB_ENVIRONMENT "GIT_OBJECT_DIRECTORY"
  #define INDEX_ENVIRONMENT "GIT_INDEX_FILE"
  #define GRAFT_ENVIRONMENT "GIT_GRAFT_FILE"
 +#define GIT_SHALLOW_FILE_ENVIRONMENT "GIT_SHALLOW_FILE"
  #define TEMPLATE_DIR_ENVIRONMENT "GIT_TEMPLATE_DIR"
  #define CONFIG_ENVIRONMENT "GIT_CONFIG"
  #define CONFIG_DATA_ENVIRONMENT "GIT_CONFIG_PARAMETERS"
@@@ -737,8 -736,29 +737,29 @@@ enum sharedrepo 
  };
  int git_config_perm(const char *var, const char *value);
  int adjust_shared_perm(const char *path);
- int safe_create_leading_directories(char *path);
- int safe_create_leading_directories_const(const char *path);
+ /*
+  * Create the directory containing the named path, using care to be
+  * somewhat safe against races.  Return one of the scld_error values
+  * to indicate success/failure.
+  *
+  * SCLD_VANISHED indicates that one of the ancestor directories of the
+  * path existed at one point during the function call and then
+  * suddenly vanished, probably because another process pruned the
+  * directory while we were working.  To be robust against this kind of
+  * race, callers might want to try invoking the function again when it
+  * returns SCLD_VANISHED.
+  */
+ enum scld_error {
+       SCLD_OK = 0,
+       SCLD_FAILED = -1,
+       SCLD_PERMS = -2,
+       SCLD_EXISTS = -3,
+       SCLD_VANISHED = -4
+ };
+ enum scld_error safe_create_leading_directories(char *path);
+ enum scld_error safe_create_leading_directories_const(const char *path);
  int mkdir_in_gitdir(const char *path);
  extern void home_config_paths(char **global, char **xdg, char *file);
  extern char *expand_user_path(const char *path);
@@@ -761,11 -781,11 +782,11 @@@ int daemon_avoid_alias(const char *path
  int offset_1st_component(const char *path);
  
  /* object replacement */
 -#define READ_SHA1_FILE_REPLACE 1
 +#define LOOKUP_REPLACE_OBJECT 1
  extern void *read_sha1_file_extended(const unsigned char *sha1, enum object_type *type, unsigned long *size, unsigned flag);
  static inline void *read_sha1_file(const unsigned char *sha1, enum object_type *type, unsigned long *size)
  {
 -      return read_sha1_file_extended(sha1, type, size, READ_SHA1_FILE_REPLACE);
 +      return read_sha1_file_extended(sha1, type, size, LOOKUP_REPLACE_OBJECT);
  }
  extern const unsigned char *do_lookup_replace_object(const unsigned char *sha1);
  static inline const unsigned char *lookup_replace_object(const unsigned char *sha1)
                return sha1;
        return do_lookup_replace_object(sha1);
  }
 +static inline const unsigned char *lookup_replace_object_extended(const unsigned char *sha1, unsigned flag)
 +{
 +      if (!(flag & LOOKUP_REPLACE_OBJECT))
 +              return sha1;
 +      return lookup_replace_object(sha1);
 +}
  
  /* Read and unpack a sha1 file into memory, write memory to a sha1 file */
  extern int sha1_object_info(const unsigned char *, unsigned long *);
@@@ -894,12 -908,9 +915,12 @@@ extern int dwim_log(const char *str, in
  extern int interpret_branch_name(const char *str, int len, struct strbuf *);
  extern int get_sha1_mb(const char *str, unsigned char *sha1);
  
 -extern int refname_match(const char *abbrev_name, const char *full_name, const char **rules);
 -extern const char *ref_rev_parse_rules[];
 -#define ref_fetch_rules ref_rev_parse_rules
 +/*
 + * Return true iff abbrev_name is a possible abbreviation for
 + * full_name according to the rules defined by ref_rev_parse_rules in
 + * refs.c.
 + */
 +extern int refname_match(const char *abbrev_name, const char *full_name);
  
  extern int create_symref(const char *ref, const char *refs_heads_master, const char *logmsg);
  extern int validate_headref(const char *ref);
@@@ -1084,7 -1095,6 +1105,7 @@@ struct object_info 
        enum object_type *typep;
        unsigned long *sizep;
        unsigned long *disk_sizep;
 +      unsigned char *delta_base_sha1;
  
        /* Response */
        enum {
                } packed;
        } u;
  };
 -extern int sha1_object_info_extended(const unsigned char *, struct object_info *);
 +extern int sha1_object_info_extended(const unsigned char *, struct object_info *, unsigned flags);
  
  /* Dumb servers support */
  extern int update_server_info(int);
@@@ -1247,8 -1257,6 +1268,8 @@@ __attribute__((format (printf, 2, 3))
  extern void trace_argv_printf(const char **argv, const char *format, ...);
  extern void trace_repo_setup(const char *prefix);
  extern int trace_want(const char *key);
 +__attribute__((format (printf, 2, 3)))
 +extern void trace_printf_key(const char *key, const char *fmt, ...);
  extern void trace_strbuf(const char *key, const struct strbuf *buf);
  
  void packet_trace_identity(const char *prog);
diff --combined dir.c
index d10a63f731020aab99f9b25a2b96a029d772410d,716b61320a38344b091d43b39f6e17aa73553cc1..b35b6330f850f610b582b189d7e4d6a9ba4495db
--- 1/dir.c
--- 2/dir.c
+++ b/dir.c
@@@ -126,13 -126,10 +126,13 @@@ static size_t common_prefix_len(const s
                       PATHSPEC_MAXDEPTH |
                       PATHSPEC_LITERAL |
                       PATHSPEC_GLOB |
 -                     PATHSPEC_ICASE);
 +                     PATHSPEC_ICASE |
 +                     PATHSPEC_EXCLUDE);
  
        for (n = 0; n < pathspec->nr; n++) {
                size_t i = 0, len = 0, item_len;
 +              if (pathspec->items[n].magic & PATHSPEC_EXCLUDE)
 +                      continue;
                if (pathspec->items[n].magic & PATHSPEC_ICASE)
                        item_len = pathspec->items[n].prefix;
                else
@@@ -282,10 -279,9 +282,10 @@@ static int match_pathspec_item(const st
   * pathspec did not match any names, which could indicate that the
   * user mistyped the nth pathspec.
   */
 -int match_pathspec_depth(const struct pathspec *ps,
 -                       const char *name, int namelen,
 -                       int prefix, char *seen)
 +static int match_pathspec_depth_1(const struct pathspec *ps,
 +                                const char *name, int namelen,
 +                                int prefix, char *seen,
 +                                int exclude)
  {
        int i, retval = 0;
  
                       PATHSPEC_MAXDEPTH |
                       PATHSPEC_LITERAL |
                       PATHSPEC_GLOB |
 -                     PATHSPEC_ICASE);
 +                     PATHSPEC_ICASE |
 +                     PATHSPEC_EXCLUDE);
  
        if (!ps->nr) {
                if (!ps->recursive ||
  
        for (i = ps->nr - 1; i >= 0; i--) {
                int how;
 +
 +              if ((!exclude &&   ps->items[i].magic & PATHSPEC_EXCLUDE) ||
 +                  ( exclude && !(ps->items[i].magic & PATHSPEC_EXCLUDE)))
 +                      continue;
 +
                if (seen && seen[i] == MATCHED_EXACTLY)
                        continue;
 +              /*
 +               * Make exclude patterns optional and never report
 +               * "pathspec ':(exclude)foo' matches no files"
 +               */
 +              if (seen && ps->items[i].magic & PATHSPEC_EXCLUDE)
 +                      seen[i] = MATCHED_FNMATCH;
                how = match_pathspec_item(ps->items+i, prefix, name, namelen);
                if (ps->recursive &&
                    (ps->magic & PATHSPEC_MAXDEPTH) &&
        return retval;
  }
  
 +int match_pathspec_depth(const struct pathspec *ps,
 +                       const char *name, int namelen,
 +                       int prefix, char *seen)
 +{
 +      int positive, negative;
 +      positive = match_pathspec_depth_1(ps, name, namelen, prefix, seen, 0);
 +      if (!(ps->magic & PATHSPEC_EXCLUDE) || !positive)
 +              return positive;
 +      negative = match_pathspec_depth_1(ps, name, namelen, prefix, seen, 1);
 +      return negative ? 0 : positive;
 +}
 +
  /*
   * Return the length of the "simple" part of a path match limiter.
   */
@@@ -1403,18 -1375,11 +1403,18 @@@ int read_directory(struct dir_struct *d
                               PATHSPEC_MAXDEPTH |
                               PATHSPEC_LITERAL |
                               PATHSPEC_GLOB |
 -                             PATHSPEC_ICASE);
 +                             PATHSPEC_ICASE |
 +                             PATHSPEC_EXCLUDE);
  
        if (has_symlink_leading_path(path, len))
                return dir->nr;
  
 +      /*
 +       * exclude patterns are treated like positive ones in
 +       * create_simplify. Usually exclude patterns should be a
 +       * subset of positive ones, which has no impacts on
 +       * create_simplify().
 +       */
        simplify = create_simplify(pathspec ? pathspec->_raw : NULL);
        if (!len || treat_leading_path(dir, path, len, simplify))
                read_directory_recursive(dir, path, len, 0, simplify);
@@@ -1511,8 -1476,13 +1511,13 @@@ static int remove_dir_recurse(struct st
        flag &= ~REMOVE_DIR_KEEP_TOPLEVEL;
        dir = opendir(path->buf);
        if (!dir) {
-               /* an empty dir could be removed even if it is unreadble */
-               if (!keep_toplevel)
+               if (errno == ENOENT)
+                       return keep_toplevel ? -1 : 0;
+               else if (errno == EACCES && !keep_toplevel)
+                       /*
+                        * An empty dir could be removable even if it
+                        * is unreadable:
+                        */
                        return rmdir(path->buf);
                else
                        return -1;
  
                strbuf_setlen(path, len);
                strbuf_addstr(path, e->d_name);
-               if (lstat(path->buf, &st))
-                       ; /* fall thru */
-               else if (S_ISDIR(st.st_mode)) {
+               if (lstat(path->buf, &st)) {
+                       if (errno == ENOENT)
+                               /*
+                                * file disappeared, which is what we
+                                * wanted anyway
+                                */
+                               continue;
+                       /* fall thru */
+               } else if (S_ISDIR(st.st_mode)) {
                        if (!remove_dir_recurse(path, flag, &kept_down))
                                continue; /* happy */
-               } else if (!only_empty && !unlink(path->buf))
+               } else if (!only_empty &&
+                          (!unlink(path->buf) || errno == ENOENT)) {
                        continue; /* happy, too */
+               }
  
                /* path too long, stat fails, or non-directory still exists */
                ret = -1;
  
        strbuf_setlen(path, original_len);
        if (!ret && !keep_toplevel && !kept_down)
-               ret = rmdir(path->buf);
+               ret = (!rmdir(path->buf) || errno == ENOENT) ? 0 : -1;
        else if (kept_up)
                /*
                 * report the uplevel that it is not an error that we
diff --combined merge-recursive.c
index a18bd15dd3dcf303909b4b541f35815705a3f50b,021e1fc4532758d8c9577bf493d23a8e23ddc96e..8400a8e937d8303ecc2ace1136a0700187865f58
@@@ -693,7 -693,7 +693,7 @@@ static int make_room_for_path(struct me
        /* Make sure leading directories are created */
        status = safe_create_leading_directories_const(path);
        if (status) {
-               if (status == -3) {
+               if (status == SCLD_EXISTS) {
                        /* something else exists */
                        error(msg, path, _(": perhaps a D/F conflict?"));
                        return -1;
@@@ -2063,13 -2063,13 +2063,13 @@@ int parse_merge_opt(struct merge_option
                o->recursive_variant = MERGE_RECURSIVE_THEIRS;
        else if (!strcmp(s, "subtree"))
                o->subtree_shift = "";
 -      else if (!prefixcmp(s, "subtree="))
 +      else if (starts_with(s, "subtree="))
                o->subtree_shift = s + strlen("subtree=");
        else if (!strcmp(s, "patience"))
                o->xdl_opts = DIFF_WITH_ALG(o, PATIENCE_DIFF);
        else if (!strcmp(s, "histogram"))
                o->xdl_opts = DIFF_WITH_ALG(o, HISTOGRAM_DIFF);
 -      else if (!prefixcmp(s, "diff-algorithm=")) {
 +      else if (starts_with(s, "diff-algorithm=")) {
                long value = parse_algorithm_value(s + strlen("diff-algorithm="));
                if (value < 0)
                        return -1;
                o->renormalize = 1;
        else if (!strcmp(s, "no-renormalize"))
                o->renormalize = 0;
 -      else if (!prefixcmp(s, "rename-threshold=")) {
 +      else if (starts_with(s, "rename-threshold=")) {
                const char *score = s + strlen("rename-threshold=");
                if ((o->rename_score = parse_rename_score(&score)) == -1 || *score != 0)
                        return -1;
diff --combined refs.c
index fc33ee8ffbe917659e36a801ebf7caa1b89a661d,703b5a2f457e0b931cada8fc39c235ada8e48c0b..89228e23732ef7e2e83a2e84e82ffb5bd7afec3f
--- 1/refs.c
--- 2/refs.c
+++ b/refs.c
@@@ -637,7 -637,7 +637,7 @@@ static int do_one_ref(struct ref_entry 
        struct ref_entry *old_current_ref;
        int retval;
  
 -      if (prefixcmp(entry->name, data->base))
 +      if (!starts_with(entry->name, data->base))
                return 0;
  
        if (!(data->flags & DO_FOR_EACH_INCLUDE_BROKEN) &&
@@@ -1042,7 -1042,7 +1042,7 @@@ static void read_packed_refs(FILE *f, s
                if (refname) {
                        last = create_ref_entry(refname, sha1, REF_ISPACKED, 1);
                        if (peeled == PEELED_FULLY ||
 -                          (peeled == PEELED_TAGS && !prefixcmp(refname, "refs/tags/")))
 +                          (peeled == PEELED_TAGS && starts_with(refname, "refs/tags/")))
                                last->flag |= REF_KNOWS_PEELED;
                        add_ref(dir, last);
                        continue;
@@@ -1376,7 -1376,7 +1376,7 @@@ const char *resolve_ref_unsafe(const ch
                                        return NULL;
                        }
                        buffer[len] = 0;
 -                      if (!prefixcmp(buffer, "refs/") &&
 +                      if (starts_with(buffer, "refs/") &&
                                        !check_refname_format(buffer, 0)) {
                                strcpy(refname_buffer, buffer);
                                refname = refname_buffer;
                /*
                 * Is it a symbolic ref?
                 */
 -              if (prefixcmp(buffer, "ref:")) {
 +              if (!starts_with(buffer, "ref:")) {
                        /*
                         * Please note that FETCH_HEAD has a second
                         * line containing other data.
@@@ -1837,7 -1837,7 +1837,7 @@@ int for_each_glob_ref_in(each_ref_fn fn
        struct ref_filter filter;
        int ret;
  
 -      if (!prefix && prefixcmp(pattern, "refs/"))
 +      if (!prefix && !starts_with(pattern, "refs/"))
                strbuf_addstr(&real_pattern, "refs/");
        else if (prefix)
                strbuf_addstr(&real_pattern, prefix);
@@@ -1874,13 -1874,13 +1874,13 @@@ int for_each_rawref(each_ref_fn fn, voi
  const char *prettify_refname(const char *name)
  {
        return name + (
 -              !prefixcmp(name, "refs/heads/") ? 11 :
 -              !prefixcmp(name, "refs/tags/") ? 10 :
 -              !prefixcmp(name, "refs/remotes/") ? 13 :
 +              starts_with(name, "refs/heads/") ? 11 :
 +              starts_with(name, "refs/tags/") ? 10 :
 +              starts_with(name, "refs/remotes/") ? 13 :
                0);
  }
  
 -const char *ref_rev_parse_rules[] = {
 +static const char *ref_rev_parse_rules[] = {
        "%.*s",
        "refs/%.*s",
        "refs/tags/%.*s",
        NULL
  };
  
 -int refname_match(const char *abbrev_name, const char *full_name, const char **rules)
 +int refname_match(const char *abbrev_name, const char *full_name)
  {
        const char **p;
        const int abbrev_name_len = strlen(abbrev_name);
  
 -      for (p = rules; *p; p++) {
 +      for (p = ref_rev_parse_rules; *p; p++) {
                if (!strcmp(full_name, mkpath(*p, abbrev_name_len, abbrev_name))) {
                        return 1;
                }
@@@ -2039,6 -2039,7 +2039,7 @@@ static struct ref_lock *lock_ref_sha1_b
        int type, lflags;
        int mustexist = (old_sha1 && !is_null_sha1(old_sha1));
        int missing = 0;
+       int attempts_remaining = 3;
  
        lock = xcalloc(1, sizeof(struct ref_lock));
        lock->lock_fd = -1;
  
        lock->lk = xcalloc(1, sizeof(struct lock_file));
  
-       lflags = LOCK_DIE_ON_ERROR;
+       lflags = 0;
        if (flags & REF_NODEREF) {
                refname = orig_refname;
                lflags |= LOCK_NODEREF;
        if ((flags & REF_NODEREF) && (type & REF_ISSYMREF))
                lock->force_write = 1;
  
-       if (safe_create_leading_directories(ref_file)) {
+  retry:
+       switch (safe_create_leading_directories(ref_file)) {
+       case SCLD_OK:
+               break; /* success */
+       case SCLD_VANISHED:
+               if (--attempts_remaining > 0)
+                       goto retry;
+               /* fall through */
+       default:
                last_errno = errno;
                error("unable to create directory for %s", ref_file);
                goto error_return;
        }
  
        lock->lock_fd = hold_lock_file_for_update(lock->lk, ref_file, lflags);
+       if (lock->lock_fd < 0) {
+               if (errno == ENOENT && --attempts_remaining > 0)
+                       /*
+                        * Maybe somebody just deleted one of the
+                        * directories leading to ref_file.  Try
+                        * again:
+                        */
+                       goto retry;
+               else
+                       unable_to_lock_index_die(ref_file, errno);
+       }
        return old_sha1 ? verify_lock(lock, old_sha1, mustexist) : lock;
  
   error_return:
@@@ -2244,7 -2264,7 +2264,7 @@@ static int pack_if_possible_fn(struct r
        struct pack_refs_cb_data *cb = cb_data;
        enum peel_status peel_status;
        struct ref_entry *packed_entry;
 -      int is_tag_ref = !prefixcmp(entry->name, "refs/tags/");
 +      int is_tag_ref = starts_with(entry->name, "refs/tags/");
  
        /* ALWAYS pack tags */
        if (!(cb->flags & PACK_REFS_ALL) && !is_tag_ref)
@@@ -2508,6 -2528,51 +2528,51 @@@ int delete_ref(const char *refname, con
   */
  #define TMP_RENAMED_LOG  "logs/refs/.tmp-renamed-log"
  
+ static int rename_tmp_log(const char *newrefname)
+ {
+       int attempts_remaining = 4;
+  retry:
+       switch (safe_create_leading_directories(git_path("logs/%s", newrefname))) {
+       case SCLD_OK:
+               break; /* success */
+       case SCLD_VANISHED:
+               if (--attempts_remaining > 0)
+                       goto retry;
+               /* fall through */
+       default:
+               error("unable to create directory for %s", newrefname);
+               return -1;
+       }
+       if (rename(git_path(TMP_RENAMED_LOG), git_path("logs/%s", newrefname))) {
+               if ((errno==EISDIR || errno==ENOTDIR) && --attempts_remaining > 0) {
+                       /*
+                        * rename(a, b) when b is an existing
+                        * directory ought to result in ISDIR, but
+                        * Solaris 5.8 gives ENOTDIR.  Sheesh.
+                        */
+                       if (remove_empty_directories(git_path("logs/%s", newrefname))) {
+                               error("Directory not empty: logs/%s", newrefname);
+                               return -1;
+                       }
+                       goto retry;
+               } else if (errno == ENOENT && --attempts_remaining > 0) {
+                       /*
+                        * Maybe another process just deleted one of
+                        * the directories in the path to newrefname.
+                        * Try again from the beginning.
+                        */
+                       goto retry;
+               } else {
+                       error("unable to move logfile "TMP_RENAMED_LOG" to logs/%s: %s",
+                               newrefname, strerror(errno));
+                       return -1;
+               }
+       }
+       return 0;
+ }
  int rename_ref(const char *oldrefname, const char *newrefname, const char *logmsg)
  {
        unsigned char sha1[20], orig_sha1[20];
                }
        }
  
-       if (log && safe_create_leading_directories(git_path("logs/%s", newrefname))) {
-               error("unable to create directory for %s", newrefname);
+       if (log && rename_tmp_log(newrefname))
                goto rollback;
-       }
  
-  retry:
-       if (log && rename(git_path(TMP_RENAMED_LOG), git_path("logs/%s", newrefname))) {
-               if (errno==EISDIR || errno==ENOTDIR) {
-                       /*
-                        * rename(a, b) when b is an existing
-                        * directory ought to result in ISDIR, but
-                        * Solaris 5.8 gives ENOTDIR.  Sheesh.
-                        */
-                       if (remove_empty_directories(git_path("logs/%s", newrefname))) {
-                               error("Directory not empty: logs/%s", newrefname);
-                               goto rollback;
-                       }
-                       goto retry;
-               } else {
-                       error("unable to move logfile "TMP_RENAMED_LOG" to logs/%s: %s",
-                               newrefname, strerror(errno));
-                       goto rollback;
-               }
-       }
        logmoved = log;
  
        lock = lock_ref_sha1_basic(newrefname, NULL, 0, NULL);
@@@ -2679,9 -2723,9 +2723,9 @@@ int log_ref_setup(const char *refname, 
  
        git_snpath(logfile, bufsize, "logs/%s", refname);
        if (log_all_ref_updates &&
 -          (!prefixcmp(refname, "refs/heads/") ||
 -           !prefixcmp(refname, "refs/remotes/") ||
 -           !prefixcmp(refname, "refs/notes/") ||
 +          (starts_with(refname, "refs/heads/") ||
 +           starts_with(refname, "refs/remotes/") ||
 +           starts_with(refname, "refs/notes/") ||
             !strcmp(refname, "HEAD"))) {
                if (safe_create_leading_directories(logfile) < 0)
                        return error("unable to create directory for %s",
@@@ -2751,7 -2795,7 +2795,7 @@@ static int log_ref_write(const char *re
  
  static int is_branch(const char *refname)
  {
 -      return !strcmp(refname, "HEAD") || !prefixcmp(refname, "refs/heads/");
 +      return !strcmp(refname, "HEAD") || starts_with(refname, "refs/heads/");
  }
  
  int write_ref_sha1(struct ref_lock *lock,
@@@ -3334,6 -3378,29 +3378,6 @@@ cleanup
        return ret;
  }
  
 -/*
 - * generate a format suitable for scanf from a ref_rev_parse_rules
 - * rule, that is replace the "%.*s" spec with a "%s" spec
 - */
 -static void gen_scanf_fmt(char *scanf_fmt, const char *rule)
 -{
 -      char *spec;
 -
 -      spec = strstr(rule, "%.*s");
 -      if (!spec || strstr(spec + 4, "%.*s"))
 -              die("invalid rule in ref_rev_parse_rules: %s", rule);
 -
 -      /* copy all until spec */
 -      strncpy(scanf_fmt, rule, spec - rule);
 -      scanf_fmt[spec - rule] = '\0';
 -      /* copy new spec */
 -      strcat(scanf_fmt, "%s");
 -      /* copy remaining rule */
 -      strcat(scanf_fmt, spec + 4);
 -
 -      return;
 -}
 -
  char *shorten_unambiguous_ref(const char *refname, int strict)
  {
        int i;
        static int nr_rules;
        char *short_name;
  
 -      /* pre generate scanf formats from ref_rev_parse_rules[] */
        if (!nr_rules) {
 +              /*
 +               * Pre-generate scanf formats from ref_rev_parse_rules[].
 +               * Generate a format suitable for scanf from a
 +               * ref_rev_parse_rules rule by interpolating "%s" at the
 +               * location of the "%.*s".
 +               */
                size_t total_len = 0;
 +              size_t offset = 0;
  
                /* the rule list is NULL terminated, count them first */
                for (nr_rules = 0; ref_rev_parse_rules[nr_rules]; nr_rules++)
 -                      /* no +1 because strlen("%s") < strlen("%.*s") */
 -                      total_len += strlen(ref_rev_parse_rules[nr_rules]);
 +                      /* -2 for strlen("%.*s") - strlen("%s"); +1 for NUL */
 +                      total_len += strlen(ref_rev_parse_rules[nr_rules]) - 2 + 1;
  
                scanf_fmts = xmalloc(nr_rules * sizeof(char *) + total_len);
  
 -              total_len = 0;
 +              offset = 0;
                for (i = 0; i < nr_rules; i++) {
 -                      scanf_fmts[i] = (char *)&scanf_fmts[nr_rules]
 -                                      + total_len;
 -                      gen_scanf_fmt(scanf_fmts[i], ref_rev_parse_rules[i]);
 -                      total_len += strlen(ref_rev_parse_rules[i]);
 +                      assert(offset < total_len);
 +                      scanf_fmts[i] = (char *)&scanf_fmts[nr_rules] + offset;
 +                      offset += snprintf(scanf_fmts[i], total_len - offset,
 +                                         ref_rev_parse_rules[i], 2, "%s") + 1;
                }
        }
  
@@@ -3433,7 -3494,7 +3477,7 @@@ int parse_hide_refs_config(const char *
  {
        if (!strcmp("transfer.hiderefs", var) ||
            /* NEEDSWORK: use parse_config_key() once both are merged */
 -          (!prefixcmp(var, section) && var[strlen(section)] == '.' &&
 +          (starts_with(var, section) && var[strlen(section)] == '.' &&
             !strcmp(var + strlen(section), ".hiderefs"))) {
                char *ref;
                int len;
@@@ -3461,7 -3522,7 +3505,7 @@@ int ref_is_hidden(const char *refname
                return 0;
        for_each_string_list_item(item, hide_refs) {
                int len;
 -              if (prefixcmp(refname, item->string))
 +              if (!starts_with(refname, item->string))
                        continue;
                len = strlen(item->string);
                if (!refname[len] || refname[len] == '/')
diff --combined sha1_file.c
index e13bd2c3ee7edb4d157e499fbcea80535ec5752f,ed814e546f809ac190a294220fd678b8f45459fb..8b0849f931c5a2f9eea47594b50560fe4526969c
@@@ -105,50 -105,59 +105,59 @@@ int mkdir_in_gitdir(const char *path
        return adjust_shared_perm(path);
  }
  
int safe_create_leading_directories(char *path)
enum scld_error safe_create_leading_directories(char *path)
  {
-       char *pos = path + offset_1st_component(path);
-       struct stat st;
+       char *next_component = path + offset_1st_component(path);
+       enum scld_error ret = SCLD_OK;
  
-       while (pos) {
-               pos = strchr(pos, '/');
-               if (!pos)
+       while (ret == SCLD_OK && next_component) {
+               struct stat st;
+               char *slash = strchr(next_component, '/');
+               if (!slash)
                        break;
-               while (*++pos == '/')
-                       ;
-               if (!*pos)
+               next_component = slash + 1;
+               while (*next_component == '/')
+                       next_component++;
+               if (!*next_component)
                        break;
-               *--pos = '\0';
+               *slash = '\0';
                if (!stat(path, &st)) {
                        /* path exists */
-                       if (!S_ISDIR(st.st_mode)) {
-                               *pos = '/';
-                               return -3;
-                       }
-               }
-               else if (mkdir(path, 0777)) {
+                       if (!S_ISDIR(st.st_mode))
+                               ret = SCLD_EXISTS;
+               } else if (mkdir(path, 0777)) {
                        if (errno == EEXIST &&
-                           !stat(path, &st) && S_ISDIR(st.st_mode)) {
+                           !stat(path, &st) && S_ISDIR(st.st_mode))
                                ; /* somebody created it since we checked */
-                       } else {
-                               *pos = '/';
-                               return -1;
-                       }
-               }
-               else if (adjust_shared_perm(path)) {
-                       *pos = '/';
-                       return -2;
+                       else if (errno == ENOENT)
+                               /*
+                                * Either mkdir() failed because
+                                * somebody just pruned the containing
+                                * directory, or stat() failed because
+                                * the file that was in our way was
+                                * just removed.  Either way, inform
+                                * the caller that it might be worth
+                                * trying again:
+                                */
+                               ret = SCLD_VANISHED;
+                       else
+                               ret = SCLD_FAILED;
+               } else if (adjust_shared_perm(path)) {
+                       ret = SCLD_PERMS;
                }
-               *pos++ = '/';
+               *slash = '/';
        }
-       return 0;
+       return ret;
  }
  
int safe_create_leading_directories_const(const char *path)
enum scld_error safe_create_leading_directories_const(const char *path)
  {
        /* path points to cache entries, so xstrdup before messing with it */
        char *buf = xstrdup(path);
-       int result = safe_create_leading_directories(buf);
+       enum scld_error result = safe_create_leading_directories(buf);
        free(buf);
        return result;
  }
@@@ -807,38 -816,15 +816,38 @@@ void free_pack_by_name(const char *pack
  static unsigned int get_max_fd_limit(void)
  {
  #ifdef RLIMIT_NOFILE
 -      struct rlimit lim;
 +      {
 +              struct rlimit lim;
  
 -      if (getrlimit(RLIMIT_NOFILE, &lim))
 -              die_errno("cannot get RLIMIT_NOFILE");
 +              if (!getrlimit(RLIMIT_NOFILE, &lim))
 +                      return lim.rlim_cur;
 +      }
 +#endif
  
 -      return lim.rlim_cur;
 -#elif defined(_SC_OPEN_MAX)
 -      return sysconf(_SC_OPEN_MAX);
 -#elif defined(OPEN_MAX)
 +#ifdef _SC_OPEN_MAX
 +      {
 +              long open_max = sysconf(_SC_OPEN_MAX);
 +              if (0 < open_max)
 +                      return open_max;
 +              /*
 +               * Otherwise, we got -1 for one of the two
 +               * reasons:
 +               *
 +               * (1) sysconf() did not understand _SC_OPEN_MAX
 +               *     and signaled an error with -1; or
 +               * (2) sysconf() said there is no limit.
 +               *
 +               * We _could_ clear errno before calling sysconf() to
 +               * tell these two cases apart and return a huge number
 +               * in the latter case to let the caller cap it to a
 +               * value that is not so selfish, but letting the
 +               * fallback OPEN_MAX codepath take care of these cases
 +               * is a lot simpler.
 +               */
 +      }
 +#endif
 +
 +#ifdef OPEN_MAX
        return OPEN_MAX;
  #else
        return 1; /* see the caller ;-) */
@@@ -1465,6 -1451,51 +1474,6 @@@ void *map_sha1_file(const unsigned cha
        return map;
  }
  
 -/*
 - * There used to be a second loose object header format which
 - * was meant to mimic the in-pack format, allowing for direct
 - * copy of the object data.  This format turned up not to be
 - * really worth it and we no longer write loose objects in that
 - * format.
 - */
 -static int experimental_loose_object(unsigned char *map)
 -{
 -      unsigned int word;
 -
 -      /*
 -       * We must determine if the buffer contains the standard
 -       * zlib-deflated stream or the experimental format based
 -       * on the in-pack object format. Compare the header byte
 -       * for each format:
 -       *
 -       * RFC1950 zlib w/ deflate : 0www1000 : 0 <= www <= 7
 -       * Experimental pack-based : Stttssss : ttt = 1,2,3,4
 -       *
 -       * If bit 7 is clear and bits 0-3 equal 8, the buffer MUST be
 -       * in standard loose-object format, UNLESS it is a Git-pack
 -       * format object *exactly* 8 bytes in size when inflated.
 -       *
 -       * However, RFC1950 also specifies that the 1st 16-bit word
 -       * must be divisible by 31 - this checksum tells us our buffer
 -       * is in the standard format, giving a false positive only if
 -       * the 1st word of the Git-pack format object happens to be
 -       * divisible by 31, ie:
 -       *      ((byte0 * 256) + byte1) % 31 = 0
 -       *   =>        0ttt10000www1000 % 31 = 0
 -       *
 -       * As it happens, this case can only arise for www=3 & ttt=1
 -       * - ie, a Commit object, which would have to be 8 bytes in
 -       * size. As no Commit can be that small, we find that the
 -       * combination of these two criteria (bitmask & checksum)
 -       * can always correctly determine the buffer format.
 -       */
 -      word = (map[0] << 8) + map[1];
 -      if ((map[0] & 0x8F) == 0x08 && !(word % 31))
 -              return 0;
 -      else
 -              return 1;
 -}
 -
  unsigned long unpack_object_header_buffer(const unsigned char *buf,
                unsigned long len, enum object_type *type, unsigned long *sizep)
  {
  
  int unpack_sha1_header(git_zstream *stream, unsigned char *map, unsigned long mapsize, void *buffer, unsigned long bufsiz)
  {
 -      unsigned long size, used;
 -      static const char valid_loose_object_type[8] = {
 -              0, /* OBJ_EXT */
 -              1, 1, 1, 1, /* "commit", "tree", "blob", "tag" */
 -              0, /* "delta" and others are invalid in a loose object */
 -      };
 -      enum object_type type;
 -
        /* Get the data stream */
        memset(stream, 0, sizeof(*stream));
        stream->next_in = map;
        stream->next_out = buffer;
        stream->avail_out = bufsiz;
  
 -      if (experimental_loose_object(map)) {
 -              /*
 -               * The old experimental format we no longer produce;
 -               * we can still read it.
 -               */
 -              used = unpack_object_header_buffer(map, mapsize, &type, &size);
 -              if (!used || !valid_loose_object_type[type])
 -                      return -1;
 -              map += used;
 -              mapsize -= used;
 -
 -              /* Set up the stream for the rest.. */
 -              stream->next_in = map;
 -              stream->avail_in = mapsize;
 -              git_inflate_init(stream);
 -
 -              /* And generate the fake traditional header */
 -              stream->total_out = 1 + snprintf(buffer, bufsiz, "%s %lu",
 -                                               typename(type), size);
 -              return 0;
 -      }
        git_inflate_init(stream);
        return git_inflate(stream, 0);
  }
@@@ -1690,38 -1750,6 +1699,38 @@@ static off_t get_delta_base(struct pack
        return base_offset;
  }
  
 +/*
 + * Like get_delta_base above, but we return the sha1 instead of the pack
 + * offset. This means it is cheaper for REF deltas (we do not have to do
 + * the final object lookup), but more expensive for OFS deltas (we
 + * have to load the revidx to convert the offset back into a sha1).
 + */
 +static const unsigned char *get_delta_base_sha1(struct packed_git *p,
 +                                              struct pack_window **w_curs,
 +                                              off_t curpos,
 +                                              enum object_type type,
 +                                              off_t delta_obj_offset)
 +{
 +      if (type == OBJ_REF_DELTA) {
 +              unsigned char *base = use_pack(p, w_curs, curpos, NULL);
 +              return base;
 +      } else if (type == OBJ_OFS_DELTA) {
 +              struct revindex_entry *revidx;
 +              off_t base_offset = get_delta_base(p, w_curs, &curpos,
 +                                                 type, delta_obj_offset);
 +
 +              if (!base_offset)
 +                      return NULL;
 +
 +              revidx = find_pack_revindex(p, base_offset);
 +              if (!revidx)
 +                      return NULL;
 +
 +              return nth_packed_object_sha1(p, revidx->nr);
 +      } else
 +              return NULL;
 +}
 +
  int unpack_object_header(struct packed_git *p,
                         struct pack_window **w_curs,
                         off_t *curpos,
@@@ -1879,22 -1907,6 +1888,22 @@@ static int packed_object_info(struct pa
                }
        }
  
 +      if (oi->delta_base_sha1) {
 +              if (type == OBJ_OFS_DELTA || type == OBJ_REF_DELTA) {
 +                      const unsigned char *base;
 +
 +                      base = get_delta_base_sha1(p, &w_curs, curpos,
 +                                                 type, obj_offset);
 +                      if (!base) {
 +                              type = OBJ_BAD;
 +                              goto out;
 +                      }
 +
 +                      hashcpy(oi->delta_base_sha1, base);
 +              } else
 +                      hashclr(oi->delta_base_sha1);
 +      }
 +
  out:
        unuse_pack(&w_curs);
        return type;
@@@ -2478,9 -2490,6 +2487,9 @@@ static int sha1_loose_object_info(cons
        git_zstream stream;
        char hdr[32];
  
 +      if (oi->delta_base_sha1)
 +              hashclr(oi->delta_base_sha1);
 +
        /*
         * If we don't care about type or size, then we don't
         * need to look inside the object at all. Note that we
        return 0;
  }
  
 -int sha1_object_info_extended(const unsigned char *sha1, struct object_info *oi)
 +int sha1_object_info_extended(const unsigned char *sha1, struct object_info *oi, unsigned flags)
  {
        struct cached_object *co;
        struct pack_entry e;
        int rtype;
 +      const unsigned char *real = lookup_replace_object_extended(sha1, flags);
  
 -      co = find_cached_object(sha1);
 +      co = find_cached_object(real);
        if (co) {
                if (oi->typep)
                        *(oi->typep) = co->type;
                        *(oi->sizep) = co->size;
                if (oi->disk_sizep)
                        *(oi->disk_sizep) = 0;
 +              if (oi->delta_base_sha1)
 +                      hashclr(oi->delta_base_sha1);
                oi->whence = OI_CACHED;
                return 0;
        }
  
 -      if (!find_pack_entry(sha1, &e)) {
 +      if (!find_pack_entry(real, &e)) {
                /* Most likely it's a loose object. */
 -              if (!sha1_loose_object_info(sha1, oi)) {
 +              if (!sha1_loose_object_info(real, oi)) {
                        oi->whence = OI_LOOSE;
                        return 0;
                }
  
                /* Not a loose object; someone else may have just packed it. */
                reprepare_packed_git();
 -              if (!find_pack_entry(sha1, &e))
 +              if (!find_pack_entry(real, &e))
                        return -1;
        }
  
        rtype = packed_object_info(e.p, e.offset, oi);
        if (rtype < 0) {
 -              mark_bad_packed_object(e.p, sha1);
 -              return sha1_object_info_extended(sha1, oi);
 +              mark_bad_packed_object(e.p, real);
 +              return sha1_object_info_extended(real, oi, 0);
        } else if (in_delta_base_cache(e.p, e.offset)) {
                oi->whence = OI_DBCACHED;
        } else {
@@@ -2576,7 -2582,7 +2585,7 @@@ int sha1_object_info(const unsigned cha
  
        oi.typep = &type;
        oi.sizep = sizep;
 -      if (sha1_object_info_extended(sha1, &oi) < 0)
 +      if (sha1_object_info_extended(sha1, &oi, LOOKUP_REPLACE_OBJECT) < 0)
                return -1;
        return type;
  }
@@@ -2668,7 -2674,8 +2677,7 @@@ void *read_sha1_file_extended(const uns
        void *data;
        char *path;
        const struct packed_git *p;
 -      const unsigned char *repl = (flag & READ_SHA1_FILE_REPLACE)
 -              ? lookup_replace_object(sha1) : sha1;
 +      const unsigned char *repl = lookup_replace_object_extended(sha1, flag);
  
        errno = 0;
        data = read_object(repl, type, size);