Merge branch 'jk/drop-sha1-entry-pos' into next
authorJunio C Hamano <gitster@pobox.com>
Fri, 18 Aug 2017 20:52:54 +0000 (13:52 -0700)
committerJunio C Hamano <gitster@pobox.com>
Fri, 18 Aug 2017 20:52:55 +0000 (13:52 -0700)
Code clean-up.

* jk/drop-sha1-entry-pos:
sha1_file: drop experimental GIT_USE_LOOKUP search

1  2 
sha1_file.c
t/test-lib.sh
diff --combined sha1_file.c
index b60ae15f7068c157df6407933ea7ee94a53f0640,85c28bdbddb317d606b07e5ccb00d9e74312b75d..ccf6fcf4ecafb50347747ed8dce5f09157a65ad8
@@@ -7,7 -7,6 +7,7 @@@
   * creation etc.
   */
  #include "cache.h"
 +#include "config.h"
  #include "string-list.h"
  #include "lockfile.h"
  #include "delta.h"
@@@ -130,10 -129,8 +130,10 @@@ enum scld_error safe_create_leading_dir
                *slash = '\0';
                if (!stat(path, &st)) {
                        /* path exists */
 -                      if (!S_ISDIR(st.st_mode))
 +                      if (!S_ISDIR(st.st_mode)) {
 +                              errno = ENOTDIR;
                                ret = SCLD_EXISTS;
 +                      }
                } else if (mkdir(path, 0777)) {
                        if (errno == EEXIST &&
                            !stat(path, &st) && S_ISDIR(st.st_mode))
  
  enum scld_error safe_create_leading_directories_const(const char *path)
  {
 +      int save_errno;
        /* path points to cache entries, so xstrdup before messing with it */
        char *buf = xstrdup(path);
        enum scld_error result = safe_create_leading_directories(buf);
 +
 +      save_errno = errno;
        free(buf);
 +      errno = save_errno;
        return result;
  }
  
 +int raceproof_create_file(const char *path, create_file_fn fn, void *cb)
 +{
 +      /*
 +       * The number of times we will try to remove empty directories
 +       * in the way of path. This is only 1 because if another
 +       * process is racily creating directories that conflict with
 +       * us, we don't want to fight against them.
 +       */
 +      int remove_directories_remaining = 1;
 +
 +      /*
 +       * The number of times that we will try to create the
 +       * directories containing path. We are willing to attempt this
 +       * more than once, because another process could be trying to
 +       * clean up empty directories at the same time as we are
 +       * trying to create them.
 +       */
 +      int create_directories_remaining = 3;
 +
 +      /* A scratch copy of path, filled lazily if we need it: */
 +      struct strbuf path_copy = STRBUF_INIT;
 +
 +      int ret, save_errno;
 +
 +      /* Sanity check: */
 +      assert(*path);
 +
 +retry_fn:
 +      ret = fn(path, cb);
 +      save_errno = errno;
 +      if (!ret)
 +              goto out;
 +
 +      if (errno == EISDIR && remove_directories_remaining-- > 0) {
 +              /*
 +               * A directory is in the way. Maybe it is empty; try
 +               * to remove it:
 +               */
 +              if (!path_copy.len)
 +                      strbuf_addstr(&path_copy, path);
 +
 +              if (!remove_dir_recursively(&path_copy, REMOVE_DIR_EMPTY_ONLY))
 +                      goto retry_fn;
 +      } else if (errno == ENOENT && create_directories_remaining-- > 0) {
 +              /*
 +               * Maybe the containing directory didn't exist, or
 +               * maybe it was just deleted by a process that is
 +               * racing with us to clean up empty directories. Try
 +               * to create it:
 +               */
 +              enum scld_error scld_result;
 +
 +              if (!path_copy.len)
 +                      strbuf_addstr(&path_copy, path);
 +
 +              do {
 +                      scld_result = safe_create_leading_directories(path_copy.buf);
 +                      if (scld_result == SCLD_OK)
 +                              goto retry_fn;
 +              } while (scld_result == SCLD_VANISHED && create_directories_remaining-- > 0);
 +      }
 +
 +out:
 +      strbuf_release(&path_copy);
 +      errno = save_errno;
 +      return ret;
 +}
 +
  static void fill_sha1_path(struct strbuf *buf, const unsigned char *sha1)
  {
        int i;
@@@ -611,7 -536,8 +611,7 @@@ char *compute_alternate_path(const cha
  
  out:
        if (seen_error) {
 -              free(ref_git);
 -              ref_git = NULL;
 +              FREE_AND_NULL(ref_git);
        }
  
        return ref_git;
@@@ -662,7 -588,7 +662,7 @@@ static int freshen_file(const char *fn
   * either does not exist on disk, or has a stale mtime and may be subject to
   * pruning).
   */
 -static int check_and_freshen_file(const char *fn, int freshen)
 +int check_and_freshen_file(const char *fn, int freshen)
  {
        if (access(fn, F_OK))
                return 0;
@@@ -1606,7 -1532,7 +1606,7 @@@ static void mark_bad_packed_object(stru
                if (!hashcmp(sha1, p->bad_object_sha1 + GIT_SHA1_RAWSZ * i))
                        return;
        p->bad_object_sha1 = xrealloc(p->bad_object_sha1,
 -                                    st_mult(GIT_SHA1_RAWSZ,
 +                                    st_mult(GIT_MAX_RAWSZ,
                                              st_add(p->num_bad_objects, 1)));
        hashcpy(p->bad_object_sha1 + GIT_SHA1_RAWSZ * p->num_bad_objects, sha1);
        p->num_bad_objects++;
@@@ -1684,14 -1610,14 +1684,14 @@@ int git_open_cloexec(const char *name, 
                fd = open(name, flags | o_cloexec);
        }
  
 -#if defined(F_GETFL) && defined(F_SETFL) && defined(FD_CLOEXEC)
 +#if defined(F_GETFD) && defined(F_SETFD) && defined(FD_CLOEXEC)
        {
                static int fd_cloexec = FD_CLOEXEC;
  
                if (!o_cloexec && 0 <= fd && fd_cloexec) {
                        /* Opened w/o O_CLOEXEC?  try with fcntl(2) to add it */
 -                      int flags = fcntl(fd, F_GETFL);
 -                      if (fcntl(fd, F_SETFL, flags | fd_cloexec))
 +                      int flags = fcntl(fd, F_GETFD);
 +                      if (fcntl(fd, F_SETFD, flags | fd_cloexec))
                                fd_cloexec = 0;
                }
        }
@@@ -1964,7 -1890,7 +1964,7 @@@ static int parse_sha1_header_extended(c
         * we're obtaining the type using '--allow-unknown-type'
         * option.
         */
 -      if ((flags & LOOKUP_UNKNOWN_OBJECT) && (type < 0))
 +      if ((flags & OBJECT_INFO_ALLOW_UNKNOWN_TYPE) && (type < 0))
                type = 0;
        else if (type < 0)
                die("invalid object type");
@@@ -2002,7 -1928,20 +2002,7 @@@ int parse_sha1_header(const char *hdr, 
        struct object_info oi = OBJECT_INFO_INIT;
  
        oi.sizep = sizep;
 -      return parse_sha1_header_extended(hdr, &oi, LOOKUP_REPLACE_OBJECT);
 -}
 -
 -static void *unpack_sha1_file(void *map, unsigned long mapsize, enum object_type *type, unsigned long *size, const unsigned char *sha1)
 -{
 -      int ret;
 -      git_zstream stream;
 -      char hdr[8192];
 -
 -      ret = unpack_sha1_header(&stream, map, mapsize, hdr, sizeof(hdr));
 -      if (ret < Z_OK || (*type = parse_sha1_header(hdr, size)) < 0)
 -              return NULL;
 -
 -      return unpack_sha1_rest(&stream, hdr, *size, sha1);
 +      return parse_sha1_header_extended(hdr, &oi, 0);
  }
  
  unsigned long get_size_from_delta(struct packed_git *p,
@@@ -2226,6 -2165,107 +2226,6 @@@ unwind
        goto out;
  }
  
 -int packed_object_info(struct packed_git *p, off_t obj_offset,
 -                     struct object_info *oi)
 -{
 -      struct pack_window *w_curs = NULL;
 -      unsigned long size;
 -      off_t curpos = obj_offset;
 -      enum object_type type;
 -
 -      /*
 -       * We always get the representation type, but only convert it to
 -       * a "real" type later if the caller is interested.
 -       */
 -      type = unpack_object_header(p, &w_curs, &curpos, &size);
 -
 -      if (oi->sizep) {
 -              if (type == OBJ_OFS_DELTA || type == OBJ_REF_DELTA) {
 -                      off_t tmp_pos = curpos;
 -                      off_t base_offset = get_delta_base(p, &w_curs, &tmp_pos,
 -                                                         type, obj_offset);
 -                      if (!base_offset) {
 -                              type = OBJ_BAD;
 -                              goto out;
 -                      }
 -                      *oi->sizep = get_size_from_delta(p, &w_curs, tmp_pos);
 -                      if (*oi->sizep == 0) {
 -                              type = OBJ_BAD;
 -                              goto out;
 -                      }
 -              } else {
 -                      *oi->sizep = size;
 -              }
 -      }
 -
 -      if (oi->disk_sizep) {
 -              struct revindex_entry *revidx = find_pack_revindex(p, obj_offset);
 -              *oi->disk_sizep = revidx[1].offset - obj_offset;
 -      }
 -
 -      if (oi->typep) {
 -              *oi->typep = packed_to_object_type(p, obj_offset, type, &w_curs, curpos);
 -              if (*oi->typep < 0) {
 -                      type = OBJ_BAD;
 -                      goto out;
 -              }
 -      }
 -
 -      if (oi->delta_base_sha1) {
 -              if (type == OBJ_OFS_DELTA || type == OBJ_REF_DELTA) {
 -                      const unsigned char *base;
 -
 -                      base = get_delta_base_sha1(p, &w_curs, curpos,
 -                                                 type, obj_offset);
 -                      if (!base) {
 -                              type = OBJ_BAD;
 -                              goto out;
 -                      }
 -
 -                      hashcpy(oi->delta_base_sha1, base);
 -              } else
 -                      hashclr(oi->delta_base_sha1);
 -      }
 -
 -out:
 -      unuse_pack(&w_curs);
 -      return type;
 -}
 -
 -static void *unpack_compressed_entry(struct packed_git *p,
 -                                  struct pack_window **w_curs,
 -                                  off_t curpos,
 -                                  unsigned long size)
 -{
 -      int st;
 -      git_zstream stream;
 -      unsigned char *buffer, *in;
 -
 -      buffer = xmallocz_gently(size);
 -      if (!buffer)
 -              return NULL;
 -      memset(&stream, 0, sizeof(stream));
 -      stream.next_out = buffer;
 -      stream.avail_out = size + 1;
 -
 -      git_inflate_init(&stream);
 -      do {
 -              in = use_pack(p, w_curs, curpos, &stream.avail_in);
 -              stream.next_in = in;
 -              st = git_inflate(&stream, Z_FINISH);
 -              if (!stream.avail_out)
 -                      break; /* the payload is larger than it should be */
 -              curpos += stream.next_in - in;
 -      } while (st == Z_OK || st == Z_BUF_ERROR);
 -      git_inflate_end(&stream);
 -      if ((st != Z_STREAM_END) || stream.total_out != size) {
 -              free(buffer);
 -              return NULL;
 -      }
 -
 -      return buffer;
 -}
 -
  static struct hashmap delta_base_cache;
  static size_t delta_base_cached;
  
@@@ -2275,8 -2315,7 +2275,8 @@@ static int delta_base_cache_key_eq(cons
        return a->p == b->p && a->base_offset == b->base_offset;
  }
  
 -static int delta_base_cache_hash_cmp(const void *va, const void *vb,
 +static int delta_base_cache_hash_cmp(const void *unused_cmp_data,
 +                                   const void *va, const void *vb,
                                     const void *vkey)
  {
        const struct delta_base_cache_entry *a = va, *b = vb;
@@@ -2314,10 -2353,8 +2314,10 @@@ static void *cache_or_unpack_entry(stru
        if (!ent)
                return unpack_entry(p, base_offset, type, base_size);
  
 -      *type = ent->type;
 -      *base_size = ent->size;
 +      if (type)
 +              *type = ent->type;
 +      if (base_size)
 +              *base_size = ent->size;
        return xmemdupz(ent->data, ent->size);
  }
  
@@@ -2361,128 -2398,11 +2361,128 @@@ static void add_delta_base_cache(struc
        list_add_tail(&ent->lru, &delta_base_cache_lru);
  
        if (!delta_base_cache.cmpfn)
 -              hashmap_init(&delta_base_cache, delta_base_cache_hash_cmp, 0);
 +              hashmap_init(&delta_base_cache, delta_base_cache_hash_cmp, NULL, 0);
        hashmap_entry_init(ent, pack_entry_hash(p, base_offset));
        hashmap_add(&delta_base_cache, ent);
  }
  
 +int packed_object_info(struct packed_git *p, off_t obj_offset,
 +                     struct object_info *oi)
 +{
 +      struct pack_window *w_curs = NULL;
 +      unsigned long size;
 +      off_t curpos = obj_offset;
 +      enum object_type type;
 +
 +      /*
 +       * We always get the representation type, but only convert it to
 +       * a "real" type later if the caller is interested.
 +       */
 +      if (oi->contentp) {
 +              *oi->contentp = cache_or_unpack_entry(p, obj_offset, oi->sizep,
 +                                                    &type);
 +              if (!*oi->contentp)
 +                      type = OBJ_BAD;
 +      } else {
 +              type = unpack_object_header(p, &w_curs, &curpos, &size);
 +      }
 +
 +      if (!oi->contentp && oi->sizep) {
 +              if (type == OBJ_OFS_DELTA || type == OBJ_REF_DELTA) {
 +                      off_t tmp_pos = curpos;
 +                      off_t base_offset = get_delta_base(p, &w_curs, &tmp_pos,
 +                                                         type, obj_offset);
 +                      if (!base_offset) {
 +                              type = OBJ_BAD;
 +                              goto out;
 +                      }
 +                      *oi->sizep = get_size_from_delta(p, &w_curs, tmp_pos);
 +                      if (*oi->sizep == 0) {
 +                              type = OBJ_BAD;
 +                              goto out;
 +                      }
 +              } else {
 +                      *oi->sizep = size;
 +              }
 +      }
 +
 +      if (oi->disk_sizep) {
 +              struct revindex_entry *revidx = find_pack_revindex(p, obj_offset);
 +              *oi->disk_sizep = revidx[1].offset - obj_offset;
 +      }
 +
 +      if (oi->typep || oi->typename) {
 +              enum object_type ptot;
 +              ptot = packed_to_object_type(p, obj_offset, type, &w_curs,
 +                                           curpos);
 +              if (oi->typep)
 +                      *oi->typep = ptot;
 +              if (oi->typename) {
 +                      const char *tn = typename(ptot);
 +                      if (tn)
 +                              strbuf_addstr(oi->typename, tn);
 +              }
 +              if (ptot < 0) {
 +                      type = OBJ_BAD;
 +                      goto out;
 +              }
 +      }
 +
 +      if (oi->delta_base_sha1) {
 +              if (type == OBJ_OFS_DELTA || type == OBJ_REF_DELTA) {
 +                      const unsigned char *base;
 +
 +                      base = get_delta_base_sha1(p, &w_curs, curpos,
 +                                                 type, obj_offset);
 +                      if (!base) {
 +                              type = OBJ_BAD;
 +                              goto out;
 +                      }
 +
 +                      hashcpy(oi->delta_base_sha1, base);
 +              } else
 +                      hashclr(oi->delta_base_sha1);
 +      }
 +
 +out:
 +      unuse_pack(&w_curs);
 +      return type;
 +}
 +
 +static void *unpack_compressed_entry(struct packed_git *p,
 +                                  struct pack_window **w_curs,
 +                                  off_t curpos,
 +                                  unsigned long size)
 +{
 +      int st;
 +      git_zstream stream;
 +      unsigned char *buffer, *in;
 +
 +      buffer = xmallocz_gently(size);
 +      if (!buffer)
 +              return NULL;
 +      memset(&stream, 0, sizeof(stream));
 +      stream.next_out = buffer;
 +      stream.avail_out = size + 1;
 +
 +      git_inflate_init(&stream);
 +      do {
 +              in = use_pack(p, w_curs, curpos, &stream.avail_in);
 +              stream.next_in = in;
 +              st = git_inflate(&stream, Z_FINISH);
 +              if (!stream.avail_out)
 +                      break; /* the payload is larger than it should be */
 +              curpos += stream.next_in - in;
 +      } while (st == Z_OK || st == Z_BUF_ERROR);
 +      git_inflate_end(&stream);
 +      if ((st != Z_STREAM_END) || stream.total_out != size) {
 +              free(buffer);
 +              return NULL;
 +      }
 +
 +      return buffer;
 +}
 +
  static void *read_object(const unsigned char *sha1, enum object_type *type,
                         unsigned long *size);
  
@@@ -2676,10 -2596,8 +2676,10 @@@ void *unpack_entry(struct packed_git *p
                free(external_base);
        }
  
 -      *final_type = type;
 -      *final_size = size;
 +      if (final_type)
 +              *final_type = type;
 +      if (final_size)
 +              *final_size = size;
  
        unuse_pack(&w_curs);
  
@@@ -2709,17 -2627,6 +2709,17 @@@ const unsigned char *nth_packed_object_
        }
  }
  
 +const struct object_id *nth_packed_object_oid(struct object_id *oid,
 +                                            struct packed_git *p,
 +                                            uint32_t n)
 +{
 +      const unsigned char *hash = nth_packed_object_sha1(p, n);
 +      if (!hash)
 +              return NULL;
 +      hashcpy(oid->hash, hash);
 +      return oid;
 +}
 +
  void check_pack_index_ptr(const struct packed_git *p, const void *vptr)
  {
        const unsigned char *ptr = vptr;
@@@ -2759,7 -2666,6 +2759,6 @@@ off_t find_pack_entry_one(const unsigne
        const uint32_t *level1_ofs = p->index_data;
        const unsigned char *index = p->index_data;
        unsigned hi, lo, stride;
-       static int use_lookup = -1;
        static int debug_lookup = -1;
  
        if (debug_lookup < 0)
                printf("%02x%02x%02x... lo %u hi %u nr %"PRIu32"\n",
                       sha1[0], sha1[1], sha1[2], lo, hi, p->num_objects);
  
-       if (use_lookup < 0)
-               use_lookup = !!getenv("GIT_USE_LOOKUP");
-       if (use_lookup) {
-               int pos = sha1_entry_pos(index, stride, 0,
-                                        lo, hi, p->num_objects, sha1);
-               if (pos < 0)
-                       return 0;
-               return nth_packed_object_offset(p, pos);
-       }
        do {
                unsigned mi = (lo + hi) / 2;
                int cmp = hashcmp(index + mi * stride, sha1);
@@@ -2913,7 -2809,6 +2902,7 @@@ static int sha1_loose_object_info(cons
        git_zstream stream;
        char hdr[32];
        struct strbuf hdrbuf = STRBUF_INIT;
 +      unsigned long size_scratch;
  
        if (oi->delta_base_sha1)
                hashclr(oi->delta_base_sha1);
         * return value implicitly indicates whether the
         * object even exists.
         */
 -      if (!oi->typep && !oi->typename && !oi->sizep) {
 +      if (!oi->typep && !oi->typename && !oi->sizep && !oi->contentp) {
                const char *path;
                struct stat st;
                if (stat_sha1_file(sha1, &st, &path) < 0)
        map = map_sha1_file(sha1, &mapsize);
        if (!map)
                return -1;
 +
 +      if (!oi->sizep)
 +              oi->sizep = &size_scratch;
 +
        if (oi->disk_sizep)
                *oi->disk_sizep = mapsize;
 -      if ((flags & LOOKUP_UNKNOWN_OBJECT)) {
 +      if ((flags & OBJECT_INFO_ALLOW_UNKNOWN_TYPE)) {
                if (unpack_sha1_header_to_strbuf(&stream, map, mapsize, hdr, sizeof(hdr), &hdrbuf) < 0)
                        status = error("unable to unpack %s header with --allow-unknown-type",
                                       sha1_to_hex(sha1));
                                       sha1_to_hex(sha1));
        } else if ((status = parse_sha1_header_extended(hdr, oi, flags)) < 0)
                status = error("unable to parse %s header", sha1_to_hex(sha1));
 -      git_inflate_end(&stream);
 +
 +      if (status >= 0 && oi->contentp)
 +              *oi->contentp = unpack_sha1_rest(&stream, hdr,
 +                                               *oi->sizep, sha1);
 +      else
 +              git_inflate_end(&stream);
 +
        munmap(map, mapsize);
        if (status && oi->typep)
                *oi->typep = status;
 +      if (oi->sizep == &size_scratch)
 +              oi->sizep = NULL;
        strbuf_release(&hdrbuf);
 -      return 0;
 +      return (status < 0) ? status : 0;
  }
  
  int sha1_object_info_extended(const unsigned char *sha1, struct object_info *oi, unsigned flags)
  {
 -      struct cached_object *co;
 +      static struct object_info blank_oi = OBJECT_INFO_INIT;
        struct pack_entry e;
        int rtype;
 -      enum object_type real_type;
 -      const unsigned char *real = lookup_replace_object_extended(sha1, flags);
 -
 -      co = find_cached_object(real);
 -      if (co) {
 -              if (oi->typep)
 -                      *(oi->typep) = co->type;
 -              if (oi->sizep)
 -                      *(oi->sizep) = co->size;
 -              if (oi->disk_sizep)
 -                      *(oi->disk_sizep) = 0;
 -              if (oi->delta_base_sha1)
 -                      hashclr(oi->delta_base_sha1);
 -              if (oi->typename)
 -                      strbuf_addstr(oi->typename, typename(co->type));
 -              oi->whence = OI_CACHED;
 -              return 0;
 +      const unsigned char *real = (flags & OBJECT_INFO_LOOKUP_REPLACE) ?
 +                                  lookup_replace_object(sha1) :
 +                                  sha1;
 +
 +      if (!oi)
 +              oi = &blank_oi;
 +
 +      if (!(flags & OBJECT_INFO_SKIP_CACHED)) {
 +              struct cached_object *co = find_cached_object(real);
 +              if (co) {
 +                      if (oi->typep)
 +                              *(oi->typep) = co->type;
 +                      if (oi->sizep)
 +                              *(oi->sizep) = co->size;
 +                      if (oi->disk_sizep)
 +                              *(oi->disk_sizep) = 0;
 +                      if (oi->delta_base_sha1)
 +                              hashclr(oi->delta_base_sha1);
 +                      if (oi->typename)
 +                              strbuf_addstr(oi->typename, typename(co->type));
 +                      if (oi->contentp)
 +                              *oi->contentp = xmemdupz(co->buf, co->size);
 +                      oi->whence = OI_CACHED;
 +                      return 0;
 +              }
        }
  
        if (!find_pack_entry(real, &e)) {
                }
  
                /* Not a loose object; someone else may have just packed it. */
 -              reprepare_packed_git();
 -              if (!find_pack_entry(real, &e))
 +              if (flags & OBJECT_INFO_QUICK) {
                        return -1;
 +              } else {
 +                      reprepare_packed_git();
 +                      if (!find_pack_entry(real, &e))
 +                              return -1;
 +              }
        }
  
 -      /*
 -       * packed_object_info() does not follow the delta chain to
 -       * find out the real type, unless it is given oi->typep.
 -       */
 -      if (oi->typename && !oi->typep)
 -              oi->typep = &real_type;
 +      if (oi == &blank_oi)
 +              /*
 +               * We know that the caller doesn't actually need the
 +               * information below, so return early.
 +               */
 +              return 0;
  
        rtype = packed_object_info(e.p, e.offset, oi);
        if (rtype < 0) {
                mark_bad_packed_object(e.p, real);
 -              if (oi->typep == &real_type)
 -                      oi->typep = NULL;
                return sha1_object_info_extended(real, oi, 0);
        } else if (in_delta_base_cache(e.p, e.offset)) {
                oi->whence = OI_DBCACHED;
                oi->u.packed.is_delta = (rtype == OBJ_REF_DELTA ||
                                         rtype == OBJ_OFS_DELTA);
        }
 -      if (oi->typename)
 -              strbuf_addstr(oi->typename, typename(*oi->typep));
 -      if (oi->typep == &real_type)
 -              oi->typep = NULL;
  
        return 0;
  }
@@@ -3057,8 -2934,7 +3046,8 @@@ int sha1_object_info(const unsigned cha
  
        oi.typep = &type;
        oi.sizep = sizep;
 -      if (sha1_object_info_extended(sha1, &oi, LOOKUP_REPLACE_OBJECT) < 0)
 +      if (sha1_object_info_extended(sha1, &oi,
 +                                    OBJECT_INFO_LOOKUP_REPLACE) < 0)
                return -1;
        return type;
  }
@@@ -3108,15 -2984,28 +3097,15 @@@ int pretend_sha1_file(void *buf, unsign
  static void *read_object(const unsigned char *sha1, enum object_type *type,
                         unsigned long *size)
  {
 -      unsigned long mapsize;
 -      void *map, *buf;
 -      struct cached_object *co;
 -
 -      co = find_cached_object(sha1);
 -      if (co) {
 -              *type = co->type;
 -              *size = co->size;
 -              return xmemdupz(co->buf, co->size);
 -      }
 +      struct object_info oi = OBJECT_INFO_INIT;
 +      void *content;
 +      oi.typep = type;
 +      oi.sizep = size;
 +      oi.contentp = &content;
  
 -      buf = read_packed_sha1(sha1, type, size);
 -      if (buf)
 -              return buf;
 -      map = map_sha1_file(sha1, &mapsize);
 -      if (map) {
 -              buf = unpack_sha1_file(map, mapsize, type, size, sha1);
 -              munmap(map, mapsize);
 -              return buf;
 -      }
 -      reprepare_packed_git();
 -      return read_packed_sha1(sha1, type, size);
 +      if (sha1_object_info_extended(sha1, &oi, 0) < 0)
 +              return NULL;
 +      return content;
  }
  
  /*
  void *read_sha1_file_extended(const unsigned char *sha1,
                              enum object_type *type,
                              unsigned long *size,
 -                            unsigned flag)
 +                            int lookup_replace)
  {
        void *data;
        const struct packed_git *p;
        const char *path;
        struct stat st;
 -      const unsigned char *repl = lookup_replace_object_extended(sha1, flag);
 +      const unsigned char *repl = lookup_replace ? lookup_replace_object(sha1)
 +                                                 : sha1;
  
        errno = 0;
        data = read_object(repl, type, size);
@@@ -3495,10 -3383,16 +3484,10 @@@ int has_sha1_pack(const unsigned char *
  
  int has_sha1_file_with_flags(const unsigned char *sha1, int flags)
  {
 -      struct pack_entry e;
 -
 -      if (find_pack_entry(sha1, &e))
 -              return 1;
 -      if (has_loose_object(sha1))
 -              return 1;
 -      if (flags & HAS_SHA1_QUICK)
 +      if (!startup_info->have_repository)
                return 0;
 -      reprepare_packed_git();
 -      return find_pack_entry(sha1, &e);
 +      return sha1_object_info_extended(sha1, NULL,
 +                                       flags | OBJECT_INFO_SKIP_CACHED) >= 0;
  }
  
  int has_object_file(const struct object_id *oid)
@@@ -3554,7 -3448,7 +3543,7 @@@ static int index_mem(unsigned char *sha
         */
        if ((type == OBJ_BLOB) && path) {
                struct strbuf nbuf = STRBUF_INIT;
 -              if (convert_to_git(path, buf, size, &nbuf,
 +              if (convert_to_git(&the_index, path, buf, size, &nbuf,
                                   write_object ? safe_crlf : SAFE_CRLF_FALSE)) {
                        buf = strbuf_detach(&nbuf, &size);
                        re_allocated = 1;
@@@ -3588,7 -3482,7 +3577,7 @@@ static int index_stream_convert_blob(un
        assert(path);
        assert(would_convert_to_git_filter_fd(path));
  
 -      convert_to_git_filter_fd(path, fd, &sbuf,
 +      convert_to_git_filter_fd(&the_index, path, fd, &sbuf,
                                 write_object ? safe_crlf : SAFE_CRLF_FALSE);
  
        if (write_object)
@@@ -3676,7 -3570,7 +3665,7 @@@ int index_fd(unsigned char *sha1, int f
        else if (!S_ISREG(st->st_mode))
                ret = index_pipe(sha1, fd, type, path, flags);
        else if (st->st_size <= big_file_threshold || type != OBJ_BLOB ||
 -               (path && would_convert_to_git(path)))
 +               (path && would_convert_to_git(&the_index, path)))
                ret = index_core(sha1, fd, xsize_t(st->st_size), type, path,
                                 flags);
        else
@@@ -3743,32 -3637,22 +3732,32 @@@ void assert_sha1_type(const unsigned ch
                    typename(expect));
  }
  
 -static int for_each_file_in_obj_subdir(int subdir_nr,
 -                                     struct strbuf *path,
 -                                     each_loose_object_fn obj_cb,
 -                                     each_loose_cruft_fn cruft_cb,
 -                                     each_loose_subdir_fn subdir_cb,
 -                                     void *data)
 +int for_each_file_in_obj_subdir(unsigned int subdir_nr,
 +                              struct strbuf *path,
 +                              each_loose_object_fn obj_cb,
 +                              each_loose_cruft_fn cruft_cb,
 +                              each_loose_subdir_fn subdir_cb,
 +                              void *data)
  {
 -      size_t baselen = path->len;
 -      DIR *dir = opendir(path->buf);
 +      size_t origlen, baselen;
 +      DIR *dir;
        struct dirent *de;
        int r = 0;
  
 +      if (subdir_nr > 0xff)
 +              BUG("invalid loose object subdirectory: %x", subdir_nr);
 +
 +      origlen = path->len;
 +      strbuf_complete(path, '/');
 +      strbuf_addf(path, "%02x", subdir_nr);
 +      baselen = path->len;
 +
 +      dir = opendir(path->buf);
        if (!dir) {
 -              if (errno == ENOENT)
 -                      return 0;
 -              return error_errno("unable to open %s", path->buf);
 +              if (errno != ENOENT)
 +                      r = error_errno("unable to open %s", path->buf);
 +              strbuf_setlen(path, origlen);
 +              return r;
        }
  
        while ((de = readdir(dir))) {
                strbuf_setlen(path, baselen);
                strbuf_addf(path, "/%s", de->d_name);
  
 -              if (strlen(de->d_name) == 38)  {
 -                      char hex[41];
 -                      unsigned char sha1[20];
 +              if (strlen(de->d_name) == GIT_SHA1_HEXSZ - 2)  {
 +                      char hex[GIT_MAX_HEXSZ+1];
 +                      struct object_id oid;
  
 -                      snprintf(hex, sizeof(hex), "%02x%s",
 -                               subdir_nr, de->d_name);
 -                      if (!get_sha1_hex(hex, sha1)) {
 +                      xsnprintf(hex, sizeof(hex), "%02x%s",
 +                                subdir_nr, de->d_name);
 +                      if (!get_oid_hex(hex, &oid)) {
                                if (obj_cb) {
 -                                      r = obj_cb(sha1, path->buf, data);
 +                                      r = obj_cb(&oid, path->buf, data);
                                        if (r)
                                                break;
                                }
        if (!r && subdir_cb)
                r = subdir_cb(subdir_nr, path->buf, data);
  
 +      strbuf_setlen(path, origlen);
 +
        return r;
  }
  
@@@ -3817,12 -3699,15 +3806,12 @@@ int for_each_loose_file_in_objdir_buf(s
                            each_loose_subdir_fn subdir_cb,
                            void *data)
  {
 -      size_t baselen = path->len;
        int r = 0;
        int i;
  
        for (i = 0; i < 256; i++) {
 -              strbuf_addf(path, "/%02x", i);
                r = for_each_file_in_obj_subdir(i, path, obj_cb, cruft_cb,
                                                subdir_cb, data);
 -              strbuf_setlen(path, baselen);
                if (r)
                        break;
        }
@@@ -3891,13 -3776,13 +3880,13 @@@ static int for_each_object_in_pack(stru
        int r = 0;
  
        for (i = 0; i < p->num_objects; i++) {
 -              const unsigned char *sha1 = nth_packed_object_sha1(p, i);
 +              struct object_id oid;
  
 -              if (!sha1)
 +              if (!nth_packed_object_oid(&oid, p, i))
                        return error("unable to get sha1 of object %u in %s",
                                     i, p->pack_name);
  
 -              r = cb(sha1, p, i, data);
 +              r = cb(&oid, p, i, data);
                if (r)
                        break;
        }
@@@ -3932,7 -3817,7 +3921,7 @@@ static int check_stream_sha1(git_zstrea
                             const unsigned char *expected_sha1)
  {
        git_SHA_CTX c;
 -      unsigned char real_sha1[GIT_SHA1_RAWSZ];
 +      unsigned char real_sha1[GIT_MAX_RAWSZ];
        unsigned char buf[4096];
        unsigned long total_read;
        int status = Z_OK;
@@@ -3989,6 -3874,7 +3978,6 @@@ int read_loose_object(const char *path
                      void **contents)
  {
        int ret = -1;
 -      int fd = -1;
        void *map = NULL;
        unsigned long mapsize;
        git_zstream stream;
  out:
        if (map)
                munmap(map, mapsize);
 -      if (fd >= 0)
 -              close(fd);
        return ret;
  }
diff --combined t/test-lib.sh
index 1b6e53f78aef40673354e5f1911d78808ae8ef4d,2d2266ba2fa7f494e246ed7cd0186b22c9ae035b..5e3ca30ab3b4e9e74b953dc77587ae35deb60cf2
@@@ -36,14 -36,6 +36,14 @@@ the
  fi
  GIT_BUILD_DIR="$TEST_DIRECTORY"/..
  
 +# If we were built with ASAN, it may complain about leaks
 +# of program-lifetime variables. Disable it by default to lower
 +# the noise level. This needs to happen at the start of the script,
 +# before we even do our "did we build git yet" check (since we don't
 +# want that one to complain to stderr).
 +: ${ASAN_OPTIONS=detect_leaks=0:abort_on_error=1}
 +export ASAN_OPTIONS
 +
  ################################################################
  # It appears that people try to run tests without building...
  "$GIT_BUILD_DIR/git" >/dev/null
@@@ -99,7 -91,6 +99,6 @@@ unset VISUAL EMAIL LANGUAGE COLUMNS $("
        my $ok = join("|", qw(
                TRACE
                DEBUG
-               USE_LOOKUP
                TEST
                .*_TEST
                PROVE
@@@ -156,6 -147,9 +155,6 @@@ els
        }
  fi
  
 -: ${ASAN_OPTIONS=detect_leaks=0}
 -export ASAN_OPTIONS
 -
  # Protect ourselves from common misconfiguration to export
  # CDPATH into the environment
  unset CDPATH
@@@ -630,9 -624,9 +629,9 @@@ test_run_ () 
                trace=
                # 117 is magic because it is unlikely to match the exit
                # code of other programs
 -              test_eval_ "(exit 117) && $1"
 -              if test "$?" != 117; then
 -                      error "bug in the test script: broken &&-chain: $1"
 +              if test "OK-117" != "$(test_eval_ "(exit 117) && $1${LF}${LF}echo OK-\$?" 3>&1)"
 +              then
 +                      error "bug in the test script: broken &&-chain or run-away HERE-DOC: $1"
                fi
                trace=$trace_tmp
        fi
@@@ -750,36 -744,26 +749,36 @@@ test_done () 
        fi
        case "$test_failure" in
        0)
 -              # Maybe print SKIP message
 -              if test -n "$skip_all" && test $test_count -gt 0
 -              then
 -                      error "Can't use skip_all after running some tests"
 -              fi
 -              test -z "$skip_all" || skip_all=" # SKIP $skip_all"
 -
                if test $test_external_has_tap -eq 0
                then
                        if test $test_remaining -gt 0
                        then
                                say_color pass "# passed all $msg"
                        fi
 -                      say "1..$test_count$skip_all"
 +
 +                      # Maybe print SKIP message
 +                      test -z "$skip_all" || skip_all="# SKIP $skip_all"
 +                      case "$test_count" in
 +                      0)
 +                              say "1..$test_count${skip_all:+ $skip_all}"
 +                              ;;
 +                      *)
 +                              test -z "$skip_all" ||
 +                              say_color warn "$skip_all"
 +                              say "1..$test_count"
 +                              ;;
 +                      esac
                fi
  
 -              test -d "$remove_trash" &&
 -              cd "$(dirname "$remove_trash")" &&
 -              rm -rf "$(basename "$remove_trash")"
 +              if test -z "$debug"
 +              then
 +                      test -d "$TRASH_DIRECTORY" ||
 +                      error "Tests passed but trash directory already removed before test cleanup; aborting"
  
 +                      cd "$TRASH_DIRECTORY/.." &&
 +                      rm -fr "$TRASH_DIRECTORY" ||
 +                      error "Tests passed but test cleanup failed; aborting"
 +              fi
                test_at_end_hook_
  
                exit 0 ;;
@@@ -934,6 -918,7 +933,6 @@@ case "$TRASH_DIRECTORY" i
  /*) ;; # absolute path is good
   *) TRASH_DIRECTORY="$TEST_OUTPUT_DIRECTORY/$TRASH_DIRECTORY" ;;
  esac
 -test ! -z "$debug" || remove_trash=$TRASH_DIRECTORY
  rm -fr "$TRASH_DIRECTORY" || {
        GIT_EXIT_OK=t
        echo >&5 "FATAL: Cannot prepare test area"
@@@ -1023,9 -1008,8 +1022,9 @@@ esa
  
  ( COLUMNS=1 && test $COLUMNS = 1 ) && test_set_prereq COLUMNS_CAN_BE_1
  test -z "$NO_PERL" && test_set_prereq PERL
 +test -z "$NO_PTHREADS" && test_set_prereq PTHREADS
  test -z "$NO_PYTHON" && test_set_prereq PYTHON
 -test -n "$USE_LIBPCRE" && test_set_prereq LIBPCRE
 +test -n "$USE_LIBPCRE1$USE_LIBPCRE2" && test_set_prereq PCRE
  test -z "$NO_GETTEXT" && test_set_prereq GETTEXT
  
  # Can we rely on git's output in the C locale?
@@@ -1179,6 -1163,3 +1178,6 @@@ build_option () 
  test_lazy_prereq LONG_IS_64BIT '
        test 8 -le "$(build_option sizeof-long)"
  '
 +
 +test_lazy_prereq TIME_IS_64BIT 'test-date is64bit'
 +test_lazy_prereq TIME_T_IS_64BIT 'test-date time_t-is64bit'