Merge branch 'cb/open-noatime-clear-errno'
authorJunio C Hamano <gitster@pobox.com>
Tue, 25 Aug 2015 21:57:10 +0000 (14:57 -0700)
committerJunio C Hamano <gitster@pobox.com>
Tue, 25 Aug 2015 21:57:10 +0000 (14:57 -0700)
When trying to see that an object does not exist, a state errno
leaked from our "first try to open a packfile with O_NOATIME and
then if it fails retry without it" logic on a system that refuses
O_NOATIME. This confused us and caused us to die, saying that the
packfile is unreadable, when we should have just reported that the
object does not exist in that packfile to the caller.

* cb/open-noatime-clear-errno:
git_open_noatime: return with errno=0 on success

1  2 
sha1_file.c
diff --combined sha1_file.c
index ea78ba7cf062c92fe78e67ff4c0f5a0607e6aebd,3502dcf66641005ff5377d211c3f5198caf494b9..08302f5857b8ad298809adf04a6bffc136f843a4
@@@ -404,46 -404,13 +404,46 @@@ void read_info_alternates(const char * 
  void add_to_alternates_file(const char *reference)
  {
        struct lock_file *lock = xcalloc(1, sizeof(struct lock_file));
 -      int fd = hold_lock_file_for_append(lock, git_path("objects/info/alternates"), LOCK_DIE_ON_ERROR);
 -      char *alt = mkpath("%s\n", reference);
 -      write_or_die(fd, alt, strlen(alt));
 -      if (commit_lock_file(lock))
 -              die("could not close alternates file");
 -      if (alt_odb_tail)
 -              link_alt_odb_entries(alt, strlen(alt), '\n', NULL, 0);
 +      char *alts = git_pathdup("objects/info/alternates");
 +      FILE *in, *out;
 +
 +      hold_lock_file_for_update(lock, alts, LOCK_DIE_ON_ERROR);
 +      out = fdopen_lock_file(lock, "w");
 +      if (!out)
 +              die_errno("unable to fdopen alternates lockfile");
 +
 +      in = fopen(alts, "r");
 +      if (in) {
 +              struct strbuf line = STRBUF_INIT;
 +              int found = 0;
 +
 +              while (strbuf_getline(&line, in, '\n') != EOF) {
 +                      if (!strcmp(reference, line.buf)) {
 +                              found = 1;
 +                              break;
 +                      }
 +                      fprintf_or_die(out, "%s\n", line.buf);
 +              }
 +
 +              strbuf_release(&line);
 +              fclose(in);
 +
 +              if (found) {
 +                      rollback_lock_file(lock);
 +                      lock = NULL;
 +              }
 +      }
 +      else if (errno != ENOENT)
 +              die_errno("unable to read alternates file");
 +
 +      if (lock) {
 +              fprintf_or_die(out, "%s\n", reference);
 +              if (commit_lock_file(lock))
 +                      die_errno("unable to move new alternates file into place");
 +              if (alt_odb_tail)
 +                      link_alt_odb_entries(reference, strlen(reference), '\n', NULL, 0);
 +      }
 +      free(alts);
  }
  
  int foreach_alt_odb(alt_odb_fn fn, void *cb)
@@@ -476,7 -443,6 +476,7 @@@ void prepare_alt_odb(void
        read_info_alternates(get_object_directory(), 0);
  }
  
 +/* Returns 1 if we have successfully freshened the file, 0 otherwise. */
  static int freshen_file(const char *fn)
  {
        struct utimbuf t;
        return !utime(fn, &t);
  }
  
 +/*
 + * All of the check_and_freshen functions return 1 if the file exists and was
 + * freshened (if freshening was requested), 0 otherwise. If they return
 + * 0, you should not assume that it is safe to skip a write of the object (it
 + * either does not exist on disk, or has a stale mtime and may be subject to
 + * pruning).
 + */
  static int check_and_freshen_file(const char *fn, int freshen)
  {
        if (access(fn, F_OK))
                return 0;
 -      if (freshen && freshen_file(fn))
 +      if (freshen && !freshen_file(fn))
                return 0;
        return 1;
  }
@@@ -748,8 -707,8 +748,8 @@@ static void mmap_limit_check(size_t len
                    (uintmax_t)length, (uintmax_t)limit);
  }
  
 -void *xmmap(void *start, size_t length,
 -      int prot, int flags, int fd, off_t offset)
 +void *xmmap_gently(void *start, size_t length,
 +                int prot, int flags, int fd, off_t offset)
  {
        void *ret;
  
                        return NULL;
                release_pack_memory(length);
                ret = mmap(start, length, prot, flags, fd, offset);
 -              if (ret == MAP_FAILED)
 -                      die_errno("Out of memory? mmap failed");
        }
        return ret;
  }
  
 +void *xmmap(void *start, size_t length,
 +      int prot, int flags, int fd, off_t offset)
 +{
 +      void *ret = xmmap_gently(start, length, prot, flags, fd, offset);
 +      if (ret == MAP_FAILED)
 +              die_errno("mmap failed");
 +      return ret;
 +}
 +
  void close_pack_windows(struct packed_git *p)
  {
        while (p->windows) {
@@@ -1246,7 -1198,7 +1246,7 @@@ static void report_pack_garbage(struct 
        if (!report_garbage)
                return;
  
 -      sort_string_list(list);
 +      string_list_sort(list);
  
        for (i = 0; i < list->nr; i++) {
                const char *path = list->items[i].string;
@@@ -1494,7 -1446,10 +1494,10 @@@ int git_open_noatime(const char *name
        static int sha1_file_open_flag = O_NOATIME;
  
        for (;;) {
-               int fd = open(name, O_RDONLY | sha1_file_open_flag);
+               int fd;
+               errno = 0;
+               fd = open(name, O_RDONLY | sha1_file_open_flag);
                if (fd >= 0)
                        return fd;
  
@@@ -1612,40 -1567,6 +1615,40 @@@ int unpack_sha1_header(git_zstream *str
        return git_inflate(stream, 0);
  }
  
 +static int unpack_sha1_header_to_strbuf(git_zstream *stream, unsigned char *map,
 +                                      unsigned long mapsize, void *buffer,
 +                                      unsigned long bufsiz, struct strbuf *header)
 +{
 +      int status;
 +
 +      status = unpack_sha1_header(stream, map, mapsize, buffer, bufsiz);
 +
 +      /*
 +       * Check if entire header is unpacked in the first iteration.
 +       */
 +      if (memchr(buffer, '\0', stream->next_out - (unsigned char *)buffer))
 +              return 0;
 +
 +      /*
 +       * buffer[0..bufsiz] was not large enough.  Copy the partial
 +       * result out to header, and then append the result of further
 +       * reading the stream.
 +       */
 +      strbuf_add(header, buffer, stream->next_out - (unsigned char *)buffer);
 +      stream->next_out = buffer;
 +      stream->avail_out = bufsiz;
 +
 +      do {
 +              status = git_inflate(stream, 0);
 +              strbuf_add(header, buffer, stream->next_out - (unsigned char *)buffer);
 +              if (memchr(buffer, '\0', stream->next_out - (unsigned char *)buffer))
 +                      return 0;
 +              stream->next_out = buffer;
 +              stream->avail_out = bufsiz;
 +      } while (status != Z_STREAM_END);
 +      return -1;
 +}
 +
  static void *unpack_sha1_rest(git_zstream *stream, void *buffer, unsigned long size, const unsigned char *sha1)
  {
        int bytes = strlen(buffer) + 1;
   * too permissive for what we want to check. So do an anal
   * object header parse by hand.
   */
 -int parse_sha1_header(const char *hdr, unsigned long *sizep)
 +static int parse_sha1_header_extended(const char *hdr, struct object_info *oi,
 +                             unsigned int flags)
  {
 -      char type[10];
 -      int i;
 +      const char *type_buf = hdr;
        unsigned long size;
 +      int type, type_len = 0;
  
        /*
 -       * The type can be at most ten bytes (including the
 -       * terminating '\0' that we add), and is followed by
 +       * The type can be of any size but is followed by
         * a space.
         */
 -      i = 0;
        for (;;) {
                char c = *hdr++;
                if (c == ' ')
                        break;
 -              type[i++] = c;
 -              if (i >= sizeof(type))
 -                      return -1;
 +              type_len++;
        }
 -      type[i] = 0;
 +
 +      type = type_from_string_gently(type_buf, type_len, 1);
 +      if (oi->typename)
 +              strbuf_add(oi->typename, type_buf, type_len);
 +      /*
 +       * Set type to 0 if its an unknown object and
 +       * we're obtaining the type using '--allow-unkown-type'
 +       * option.
 +       */
 +      if ((flags & LOOKUP_UNKNOWN_OBJECT) && (type < 0))
 +              type = 0;
 +      else if (type < 0)
 +              die("invalid object type");
 +      if (oi->typep)
 +              *oi->typep = type;
  
        /*
         * The length must follow immediately, and be in canonical
                        size = size * 10 + c;
                }
        }
 -      *sizep = size;
 +
 +      if (oi->sizep)
 +              *oi->sizep = size;
  
        /*
         * The length must be followed by a zero byte
         */
 -      return *hdr ? -1 : type_from_string(type);
 +      return *hdr ? -1 : type;
 +}
 +
 +int parse_sha1_header(const char *hdr, unsigned long *sizep)
 +{
 +      struct object_info oi;
 +
 +      oi.sizep = sizep;
 +      oi.typename = NULL;
 +      oi.typep = NULL;
 +      return parse_sha1_header_extended(hdr, &oi, LOOKUP_REPLACE_OBJECT);
  }
  
  static void *unpack_sha1_file(void *map, unsigned long mapsize, enum object_type *type, unsigned long *size, const unsigned char *sha1)
@@@ -2578,8 -2476,10 +2581,8 @@@ static int fill_pack_entry(const unsign
         * answer, as it may have been deleted since the index was
         * loaded!
         */
 -      if (!is_pack_valid(p)) {
 -              warning("packfile %s cannot be accessed", p->pack_name);
 +      if (!is_pack_valid(p))
                return 0;
 -      }
        e->offset = offset;
        e->p = p;
        hashcpy(e->sha1, sha1);
@@@ -2627,15 -2527,13 +2630,15 @@@ struct packed_git *find_sha1_pack(cons
  }
  
  static int sha1_loose_object_info(const unsigned char *sha1,
 -                                struct object_info *oi)
 +                                struct object_info *oi,
 +                                int flags)
  {
 -      int status;
 -      unsigned long mapsize, size;
 +      int status = 0;
 +      unsigned long mapsize;
        void *map;
        git_zstream stream;
        char hdr[32];
 +      struct strbuf hdrbuf = STRBUF_INIT;
  
        if (oi->delta_base_sha1)
                hashclr(oi->delta_base_sha1);
         * return value implicitly indicates whether the
         * object even exists.
         */
 -      if (!oi->typep && !oi->sizep) {
 +      if (!oi->typep && !oi->typename && !oi->sizep) {
                struct stat st;
                if (stat_sha1_file(sha1, &st) < 0)
                        return -1;
                return -1;
        if (oi->disk_sizep)
                *oi->disk_sizep = mapsize;
 -      if (unpack_sha1_header(&stream, map, mapsize, hdr, sizeof(hdr)) < 0)
 +      if ((flags & LOOKUP_UNKNOWN_OBJECT)) {
 +              if (unpack_sha1_header_to_strbuf(&stream, map, mapsize, hdr, sizeof(hdr), &hdrbuf) < 0)
 +                      status = error("unable to unpack %s header with --allow-unknown-type",
 +                                     sha1_to_hex(sha1));
 +      } else if (unpack_sha1_header(&stream, map, mapsize, hdr, sizeof(hdr)) < 0)
                status = error("unable to unpack %s header",
                               sha1_to_hex(sha1));
 -      else if ((status = parse_sha1_header(hdr, &size)) < 0)
 +      if (status < 0)
 +              ; /* Do nothing */
 +      else if (hdrbuf.len) {
 +              if ((status = parse_sha1_header_extended(hdrbuf.buf, oi, flags)) < 0)
 +                      status = error("unable to parse %s header with --allow-unknown-type",
 +                                     sha1_to_hex(sha1));
 +      } else if ((status = parse_sha1_header_extended(hdr, oi, flags)) < 0)
                status = error("unable to parse %s header", sha1_to_hex(sha1));
 -      else if (oi->sizep)
 -              *oi->sizep = size;
        git_inflate_end(&stream);
        munmap(map, mapsize);
 -      if (oi->typep)
 +      if (status && oi->typep)
                *oi->typep = status;
 +      strbuf_release(&hdrbuf);
        return 0;
  }
  
@@@ -2690,7 -2579,6 +2693,7 @@@ int sha1_object_info_extended(const uns
        struct cached_object *co;
        struct pack_entry e;
        int rtype;
 +      enum object_type real_type;
        const unsigned char *real = lookup_replace_object_extended(sha1, flags);
  
        co = find_cached_object(real);
                        *(oi->disk_sizep) = 0;
                if (oi->delta_base_sha1)
                        hashclr(oi->delta_base_sha1);
 +              if (oi->typename)
 +                      strbuf_addstr(oi->typename, typename(co->type));
                oi->whence = OI_CACHED;
                return 0;
        }
  
        if (!find_pack_entry(real, &e)) {
                /* Most likely it's a loose object. */
 -              if (!sha1_loose_object_info(real, oi)) {
 +              if (!sha1_loose_object_info(real, oi, flags)) {
                        oi->whence = OI_LOOSE;
                        return 0;
                }
                        return -1;
        }
  
 +      /*
 +       * packed_object_info() does not follow the delta chain to
 +       * find out the real type, unless it is given oi->typep.
 +       */
 +      if (oi->typename && !oi->typep)
 +              oi->typep = &real_type;
 +
        rtype = packed_object_info(e.p, e.offset, oi);
        if (rtype < 0) {
                mark_bad_packed_object(e.p, real);
 +              if (oi->typep == &real_type)
 +                      oi->typep = NULL;
                return sha1_object_info_extended(real, oi, 0);
        } else if (in_delta_base_cache(e.p, e.offset)) {
                oi->whence = OI_DBCACHED;
                oi->u.packed.is_delta = (rtype == OBJ_REF_DELTA ||
                                         rtype == OBJ_OFS_DELTA);
        }
 +      if (oi->typename)
 +              strbuf_addstr(oi->typename, typename(*oi->typep));
 +      if (oi->typep == &real_type)
 +              oi->typep = NULL;
  
        return 0;
  }
@@@ -2941,8 -2814,11 +2944,8 @@@ static void write_sha1_file_prepare(con
  
  /*
   * Move the just written object into its final resting place.
 - * NEEDSWORK: this should be renamed to finalize_temp_file() as
 - * "moving" is only a part of what it does, when no patch between
 - * master to pu changes the call sites of this function.
   */
 -int move_temp_to_file(const char *tmpfile, const char *filename)
 +int finalize_object_file(const char *tmpfile, const char *filename)
  {
        int ret = 0;
  
@@@ -3070,6 -2946,7 +3073,6 @@@ static int write_loose_object(const uns
        }
  
        /* Set it up */
 -      memset(&stream, 0, sizeof(stream));
        git_deflate_init(&stream, zlib_compression_level);
        stream.next_out = compressed;
        stream.avail_out = sizeof(compressed);
                                tmp_file, strerror(errno));
        }
  
 -      return move_temp_to_file(tmp_file, filename);
 +      return finalize_object_file(tmp_file, filename);
  }
  
  static int freshen_loose_object(const unsigned char *sha1)
  static int freshen_packed_object(const unsigned char *sha1)
  {
        struct pack_entry e;
 -      return find_pack_entry(sha1, &e) && freshen_file(e.p->pack_name);
 +      if (!find_pack_entry(sha1, &e))
 +              return 0;
 +      if (e.p->freshened)
 +              return 1;
 +      if (!freshen_file(e.p->pack_name))
 +              return 0;
 +      e.p->freshened = 1;
 +      return 1;
  }
  
 -int write_sha1_file(const void *buf, unsigned long len, const char *type, unsigned char *returnsha1)
 +int write_sha1_file(const void *buf, unsigned long len, const char *type, unsigned char *sha1)
  {
 -      unsigned char sha1[20];
        char hdr[32];
        int hdrlen;
  
         * it out into .git/objects/??/?{38} file.
         */
        write_sha1_file_prepare(buf, len, type, sha1, hdr, &hdrlen);
 -      if (returnsha1)
 -              hashcpy(returnsha1, sha1);
 -      if (freshen_loose_object(sha1) || freshen_packed_object(sha1))
 +      if (freshen_packed_object(sha1) || freshen_loose_object(sha1))
                return 0;
        return write_loose_object(sha1, hdr, hdrlen, buf, len, 0);
  }
  
 +int hash_sha1_file_literally(const void *buf, unsigned long len, const char *type,
 +                           unsigned char *sha1, unsigned flags)
 +{
 +      char *header;
 +      int hdrlen, status = 0;
 +
 +      /* type string, SP, %lu of the length plus NUL must fit this */
 +      header = xmalloc(strlen(type) + 32);
 +      write_sha1_file_prepare(buf, len, type, sha1, header, &hdrlen);
 +
 +      if (!(flags & HASH_WRITE_OBJECT))
 +              goto cleanup;
 +      if (freshen_packed_object(sha1) || freshen_loose_object(sha1))
 +              goto cleanup;
 +      status = write_loose_object(sha1, header, hdrlen, buf, len, 0);
 +
 +cleanup:
 +      free(header);
 +      return status;
 +}
 +
  int force_object_loose(const unsigned char *sha1, time_t mtime)
  {
        void *buf;
@@@ -3206,7 -3058,7 +3209,7 @@@ int has_sha1_pack(const unsigned char *
        return find_pack_entry(sha1, &e);
  }
  
 -int has_sha1_file(const unsigned char *sha1)
 +int has_sha1_file_with_flags(const unsigned char *sha1, int flags)
  {
        struct pack_entry e;
  
                return 1;
        if (has_loose_object(sha1))
                return 1;
 +      if (flags & HAS_SHA1_QUICK)
 +              return 0;
        reprepare_packed_git();
        return find_pack_entry(sha1, &e);
  }
@@@ -3333,7 -3183,7 +3336,7 @@@ static int index_core(unsigned char *sh
        int ret;
  
        if (!size) {
 -              ret = index_mem(sha1, NULL, size, type, path, flags);
 +              ret = index_mem(sha1, "", size, type, path, flags);
        } else if (size <= SMALL_FILE_SIZE) {
                char *buf = xmalloc(size);
                if (size == read_in_full(fd, buf, size))
@@@ -3512,42 -3362,31 +3515,42 @@@ static int for_each_file_in_obj_subdir(
        return r;
  }
  
 -int for_each_loose_file_in_objdir(const char *path,
 +int for_each_loose_file_in_objdir_buf(struct strbuf *path,
                            each_loose_object_fn obj_cb,
                            each_loose_cruft_fn cruft_cb,
                            each_loose_subdir_fn subdir_cb,
                            void *data)
  {
 -      struct strbuf buf = STRBUF_INIT;
 -      size_t baselen;
 +      size_t baselen = path->len;
        int r = 0;
        int i;
  
 -      strbuf_addstr(&buf, path);
 -      strbuf_addch(&buf, '/');
 -      baselen = buf.len;
 -
        for (i = 0; i < 256; i++) {
 -              strbuf_addf(&buf, "%02x", i);
 -              r = for_each_file_in_obj_subdir(i, &buf, obj_cb, cruft_cb,
 +              strbuf_addf(path, "/%02x", i);
 +              r = for_each_file_in_obj_subdir(i, path, obj_cb, cruft_cb,
                                                subdir_cb, data);
 -              strbuf_setlen(&buf, baselen);
 +              strbuf_setlen(path, baselen);
                if (r)
                        break;
        }
  
 +      return r;
 +}
 +
 +int for_each_loose_file_in_objdir(const char *path,
 +                                each_loose_object_fn obj_cb,
 +                                each_loose_cruft_fn cruft_cb,
 +                                each_loose_subdir_fn subdir_cb,
 +                                void *data)
 +{
 +      struct strbuf buf = STRBUF_INIT;
 +      int r;
 +
 +      strbuf_addstr(&buf, path);
 +      r = for_each_loose_file_in_objdir_buf(&buf, obj_cb, cruft_cb,
 +                                            subdir_cb, data);
        strbuf_release(&buf);
 +
        return r;
  }
  
@@@ -3560,19 -3399,12 +3563,19 @@@ static int loose_from_alt_odb(struct al
                              void *vdata)
  {
        struct loose_alt_odb_data *data = vdata;
 -      return for_each_loose_file_in_objdir(alt->base,
 -                                           data->cb, NULL, NULL,
 -                                           data->data);
 +      struct strbuf buf = STRBUF_INIT;
 +      int r;
 +
 +      /* copy base not including trailing '/' */
 +      strbuf_add(&buf, alt->base, alt->name - alt->base - 1);
 +      r = for_each_loose_file_in_objdir_buf(&buf,
 +                                            data->cb, NULL, NULL,
 +                                            data->data);
 +      strbuf_release(&buf);
 +      return r;
  }
  
 -int for_each_loose_object(each_loose_object_fn cb, void *data)
 +int for_each_loose_object(each_loose_object_fn cb, void *data, unsigned flags)
  {
        struct loose_alt_odb_data alt;
        int r;
        if (r)
                return r;
  
 +      if (flags & FOR_EACH_OBJECT_LOCAL_ONLY)
 +              return 0;
 +
        alt.cb = cb;
        alt.data = data;
        return foreach_alt_odb(loose_from_alt_odb, &alt);
@@@ -3609,23 -3438,16 +3612,23 @@@ static int for_each_object_in_pack(stru
        return r;
  }
  
 -int for_each_packed_object(each_packed_object_fn cb, void *data)
 +int for_each_packed_object(each_packed_object_fn cb, void *data, unsigned flags)
  {
        struct packed_git *p;
        int r = 0;
 +      int pack_errors = 0;
  
        prepare_packed_git();
        for (p = packed_git; p; p = p->next) {
 +              if ((flags & FOR_EACH_OBJECT_LOCAL_ONLY) && !p->pack_local)
 +                      continue;
 +              if (open_pack_index(p)) {
 +                      pack_errors = 1;
 +                      continue;
 +              }
                r = for_each_object_in_pack(p, cb, data);
                if (r)
                        break;
        }
 -      return r;
 +      return r ? r : pack_errors;
  }