*/
#include "cache.h"
#include "string-list.h"
+#include "lockfile.h"
#include "delta.h"
#include "pack.h"
#include "blob.h"
#include "sha1-lookup.h"
#include "bulk-checkin.h"
#include "streaming.h"
+#include "dir.h"
#ifndef O_NOATIME
#if defined(__linux__) && (defined(__i386__) || defined(__PPC__))
0
};
+/*
+ * A pointer to the last packed_git in which an object was found.
+ * When an object is sought, we look in this packfile first, because
+ * objects that are looked up at similar times are often in the same
+ * packfile as one another.
+ */
static struct packed_git *last_found_pack;
static struct cached_object *find_cached_object(const unsigned char *sha1)
return adjust_shared_perm(path);
}
-int safe_create_leading_directories(char *path)
+enum scld_error safe_create_leading_directories(char *path)
{
- char *pos = path + offset_1st_component(path);
- struct stat st;
+ char *next_component = path + offset_1st_component(path);
+ enum scld_error ret = SCLD_OK;
- while (pos) {
- pos = strchr(pos, '/');
- if (!pos)
+ while (ret == SCLD_OK && next_component) {
+ struct stat st;
+ char *slash = next_component, slash_character;
+
+ while (*slash && !is_dir_sep(*slash))
+ slash++;
+
+ if (!*slash)
break;
- while (*++pos == '/')
- ;
- if (!*pos)
+
+ next_component = slash + 1;
+ while (is_dir_sep(*next_component))
+ next_component++;
+ if (!*next_component)
break;
- *--pos = '\0';
+
+ slash_character = *slash;
+ *slash = '\0';
if (!stat(path, &st)) {
/* path exists */
- if (!S_ISDIR(st.st_mode)) {
- *pos = '/';
- return -3;
- }
- }
- else if (mkdir(path, 0777)) {
- *pos = '/';
- return -1;
- }
- else if (adjust_shared_perm(path)) {
- *pos = '/';
- return -2;
+ if (!S_ISDIR(st.st_mode))
+ ret = SCLD_EXISTS;
+ } else if (mkdir(path, 0777)) {
+ if (errno == EEXIST &&
+ !stat(path, &st) && S_ISDIR(st.st_mode))
+ ; /* somebody created it since we checked */
+ else if (errno == ENOENT)
+ /*
+ * Either mkdir() failed because
+ * somebody just pruned the containing
+ * directory, or stat() failed because
+ * the file that was in our way was
+ * just removed. Either way, inform
+ * the caller that it might be worth
+ * trying again:
+ */
+ ret = SCLD_VANISHED;
+ else
+ ret = SCLD_FAILED;
+ } else if (adjust_shared_perm(path)) {
+ ret = SCLD_PERMS;
}
- *pos++ = '/';
+ *slash = slash_character;
}
- return 0;
+ return ret;
}
-int safe_create_leading_directories_const(const char *path)
+enum scld_error safe_create_leading_directories_const(const char *path)
{
/* path points to cache entries, so xstrdup before messing with it */
char *buf = xstrdup(path);
- int result = safe_create_leading_directories(buf);
+ enum scld_error result = safe_create_leading_directories(buf);
free(buf);
return result;
}
}
}
-/*
- * NOTE! This returns a statically allocated buffer, so you have to be
- * careful about using it. Do an "xstrdup()" if you need to save the
- * filename.
- *
- * Also note that this returns the location for creating. Reading
- * SHA1 file can happen from any alternate directory listed in the
- * DB_ENVIRONMENT environment variable if it is not found in
- * the primary object database.
- */
-char *sha1_file_name(const unsigned char *sha1)
+const char *sha1_file_name(const unsigned char *sha1)
{
static char buf[PATH_MAX];
const char *objdir;
return buf;
}
+/*
+ * Return the name of the pack or index file with the specified sha1
+ * in its filename. *base and *name are scratch space that must be
+ * provided by the caller. which should be "pack" or "idx".
+ */
static char *sha1_get_pack_name(const unsigned char *sha1,
char **name, char **base, const char *which)
{
struct alternate_object_database *alt_odb_list;
static struct alternate_object_database **alt_odb_tail;
-static int git_open_noatime(const char *name);
-
/*
* Prepare alternate object database registry.
*
* SHA1, an extra slash for the first level indirection, and the
* terminating NUL.
*/
-static int link_alt_odb_entry(const char *entry, const char *relative_base, int depth)
+static int link_alt_odb_entry(const char *entry, const char *relative_base,
+ int depth, const char *normalized_objdir)
{
- const char *objdir = get_object_directory();
struct alternate_object_database *ent;
struct alternate_object_database *alt;
int pfxlen, entlen;
* thing twice, or object directory itself.
*/
for (alt = alt_odb_list; alt; alt = alt->next) {
- if (!memcmp(ent->base, alt->base, pfxlen)) {
+ if (pfxlen == alt->name - alt->base - 1 &&
+ !memcmp(ent->base, alt->base, pfxlen)) {
free(ent);
return -1;
}
}
- if (!strcmp(ent->base, objdir)) {
+ if (!strcmp_icase(ent->base, normalized_objdir)) {
free(ent);
return -1;
}
struct string_list entries = STRING_LIST_INIT_NODUP;
char *alt_copy;
int i;
+ struct strbuf objdirbuf = STRBUF_INIT;
if (depth > 5) {
error("%s: ignoring alternate object stores, nesting too deep.",
return;
}
+ strbuf_add_absolute_path(&objdirbuf, get_object_directory());
+ normalize_path_copy(objdirbuf.buf, objdirbuf.buf);
+
alt_copy = xmemdupz(alt, len);
string_list_split_in_place(&entries, alt_copy, sep, -1);
for (i = 0; i < entries.nr; i++) {
error("%s: ignoring relative alternate object store %s",
relative_base, entry);
} else {
- link_alt_odb_entry(entry, relative_base, depth);
+ link_alt_odb_entry(entry, relative_base, depth, objdirbuf.buf);
}
}
string_list_clear(&entries, 0);
free(alt_copy);
+ strbuf_release(&objdirbuf);
}
void read_info_alternates(const char * relative_base, int depth)
link_alt_odb_entries(alt, strlen(alt), '\n', NULL, 0);
}
-void foreach_alt_odb(alt_odb_fn fn, void *cb)
+int foreach_alt_odb(alt_odb_fn fn, void *cb)
{
struct alternate_object_database *ent;
+ int r = 0;
prepare_alt_odb();
- for (ent = alt_odb_list; ent; ent = ent->next)
- if (fn(ent, cb))
- return;
+ for (ent = alt_odb_list; ent; ent = ent->next) {
+ r = fn(ent, cb);
+ if (r)
+ break;
+ }
+ return r;
}
void prepare_alt_odb(void)
read_info_alternates(get_object_directory(), 0);
}
-static int has_loose_object_local(const unsigned char *sha1)
+static int freshen_file(const char *fn)
{
- char *name = sha1_file_name(sha1);
- return !access(name, F_OK);
+ struct utimbuf t;
+ t.actime = t.modtime = time(NULL);
+ return !utime(fn, &t);
}
-int has_loose_object_nonlocal(const unsigned char *sha1)
+static int check_and_freshen_file(const char *fn, int freshen)
+{
+ if (access(fn, F_OK))
+ return 0;
+ if (freshen && freshen_file(fn))
+ return 0;
+ return 1;
+}
+
+static int check_and_freshen_local(const unsigned char *sha1, int freshen)
+{
+ return check_and_freshen_file(sha1_file_name(sha1), freshen);
+}
+
+static int check_and_freshen_nonlocal(const unsigned char *sha1, int freshen)
{
struct alternate_object_database *alt;
prepare_alt_odb();
for (alt = alt_odb_list; alt; alt = alt->next) {
fill_sha1_path(alt->name, sha1);
- if (!access(alt->base, F_OK))
+ if (check_and_freshen_file(alt->base, freshen))
return 1;
}
return 0;
}
+static int check_and_freshen(const unsigned char *sha1, int freshen)
+{
+ return check_and_freshen_local(sha1, freshen) ||
+ check_and_freshen_nonlocal(sha1, freshen);
+}
+
+int has_loose_object_nonlocal(const unsigned char *sha1)
+{
+ return check_and_freshen_nonlocal(sha1, 0);
+}
+
static int has_loose_object(const unsigned char *sha1)
{
- return has_loose_object_local(sha1) ||
- has_loose_object_nonlocal(sha1);
+ return check_and_freshen(sha1, 0);
}
static unsigned int pack_used_ctr;
sz_fmt(pack_mapped), sz_fmt(peak_pack_mapped));
}
-static int check_packed_git_idx(const char *path, struct packed_git *p)
+/*
+ * Open and mmap the index file at path, perform a couple of
+ * consistency checks, then record its information to p. Return 0 on
+ * success.
+ */
+static int check_packed_git_idx(const char *path, struct packed_git *p)
{
void *idx_map;
struct pack_idx_header *hdr;
}
}
-static int unuse_one_window(struct packed_git *current, int keep_fd)
+static int unuse_one_window(struct packed_git *current)
{
struct packed_git *p, *lru_p = NULL;
struct pack_window *lru_w = NULL, *lru_l = NULL;
pack_mapped -= lru_w->len;
if (lru_l)
lru_l->next = lru_w->next;
- else {
+ else
lru_p->windows = lru_w->next;
- if (!lru_p->windows && lru_p->pack_fd != -1
- && lru_p->pack_fd != keep_fd) {
- close(lru_p->pack_fd);
- pack_open_fds--;
- lru_p->pack_fd = -1;
- }
- }
free(lru_w);
pack_open_windows--;
return 1;
return 0;
}
-void release_pack_memory(size_t need, int fd)
+void release_pack_memory(size_t need)
{
size_t cur = pack_mapped;
- while (need >= (cur - pack_mapped) && unuse_one_window(NULL, fd))
+ while (need >= (cur - pack_mapped) && unuse_one_window(NULL))
; /* nothing */
}
+static void mmap_limit_check(size_t length)
+{
+ static size_t limit = 0;
+ if (!limit) {
+ limit = git_env_ulong("GIT_MMAP_LIMIT", 0);
+ if (!limit)
+ limit = SIZE_MAX;
+ }
+ if (length > limit)
+ die("attempting to mmap %"PRIuMAX" over limit %"PRIuMAX,
+ (uintmax_t)length, (uintmax_t)limit);
+}
+
void *xmmap(void *start, size_t length,
int prot, int flags, int fd, off_t offset)
{
- void *ret = mmap(start, length, prot, flags, fd, offset);
+ void *ret;
+
+ mmap_limit_check(length);
+ ret = mmap(start, length, prot, flags, fd, offset);
if (ret == MAP_FAILED) {
if (!length)
return NULL;
- release_pack_memory(length, fd);
+ release_pack_memory(length);
ret = mmap(start, length, prot, flags, fd, offset);
if (ret == MAP_FAILED)
die_errno("Out of memory? mmap failed");
* contain the same set of objects as an existing one. In that case
* the resulting file might be different even if its name would be the
* same. It is best to close any reference to the old pack before it is
- * replaced on disk. Of course no index pointers nor windows for given pack
+ * replaced on disk. Of course no index pointers or windows for given pack
* must subsist at this point. If ever objects from this pack are requested
* again, the new version of the pack will be reinitialized through
* reprepare_packed_git().
static unsigned int get_max_fd_limit(void)
{
#ifdef RLIMIT_NOFILE
- struct rlimit lim;
+ {
+ struct rlimit lim;
+
+ if (!getrlimit(RLIMIT_NOFILE, &lim))
+ return lim.rlim_cur;
+ }
+#endif
- if (getrlimit(RLIMIT_NOFILE, &lim))
- die_errno("cannot get RLIMIT_NOFILE");
+#ifdef _SC_OPEN_MAX
+ {
+ long open_max = sysconf(_SC_OPEN_MAX);
+ if (0 < open_max)
+ return open_max;
+ /*
+ * Otherwise, we got -1 for one of the two
+ * reasons:
+ *
+ * (1) sysconf() did not understand _SC_OPEN_MAX
+ * and signaled an error with -1; or
+ * (2) sysconf() said there is no limit.
+ *
+ * We _could_ clear errno before calling sysconf() to
+ * tell these two cases apart and return a huge number
+ * in the latter case to let the caller cap it to a
+ * value that is not so selfish, but letting the
+ * fallback OPEN_MAX codepath take care of these cases
+ * is a lot simpler.
+ */
+ }
+#endif
- return lim.rlim_cur;
-#elif defined(_SC_OPEN_MAX)
- return sysconf(_SC_OPEN_MAX);
-#elif defined(OPEN_MAX)
+#ifdef OPEN_MAX
return OPEN_MAX;
#else
return 1; /* see the caller ;-) */
win->len = (size_t)len;
pack_mapped += win->len;
while (packed_git_limit < pack_mapped
- && unuse_one_window(p, p->pack_fd))
+ && unuse_one_window(p))
; /* nothing */
win->base = xmmap(NULL, win->len,
PROT_READ, MAP_PRIVATE,
static void try_to_free_pack_memory(size_t size)
{
- release_pack_memory(size, -1);
+ release_pack_memory(size);
}
struct packed_git *add_packed_git(const char *path, int path_len, int local)
packed_git = pack;
}
+void (*report_garbage)(const char *desc, const char *path);
+
+static void report_helper(const struct string_list *list,
+ int seen_bits, int first, int last)
+{
+ const char *msg;
+ switch (seen_bits) {
+ case 0:
+ msg = "no corresponding .idx or .pack";
+ break;
+ case 1:
+ msg = "no corresponding .idx";
+ break;
+ case 2:
+ msg = "no corresponding .pack";
+ break;
+ default:
+ return;
+ }
+ for (; first < last; first++)
+ report_garbage(msg, list->items[first].string);
+}
+
+static void report_pack_garbage(struct string_list *list)
+{
+ int i, baselen = -1, first = 0, seen_bits = 0;
+
+ if (!report_garbage)
+ return;
+
+ string_list_sort(list);
+
+ for (i = 0; i < list->nr; i++) {
+ const char *path = list->items[i].string;
+ if (baselen != -1 &&
+ strncmp(path, list->items[first].string, baselen)) {
+ report_helper(list, seen_bits, first, i);
+ baselen = -1;
+ seen_bits = 0;
+ }
+ if (baselen == -1) {
+ const char *dot = strrchr(path, '.');
+ if (!dot) {
+ report_garbage("garbage found", path);
+ continue;
+ }
+ baselen = dot - path + 1;
+ first = i;
+ }
+ if (!strcmp(path + baselen, "pack"))
+ seen_bits |= 1;
+ else if (!strcmp(path + baselen, "idx"))
+ seen_bits |= 2;
+ }
+ report_helper(list, seen_bits, first, list->nr);
+}
+
static void prepare_packed_git_one(char *objdir, int local)
{
- /* Ensure that this buffer is large enough so that we can
- append "/pack/" without clobbering the stack even if
- strlen(objdir) were PATH_MAX. */
- char path[PATH_MAX + 1 + 4 + 1 + 1];
- int len;
+ struct strbuf path = STRBUF_INIT;
+ size_t dirnamelen;
DIR *dir;
struct dirent *de;
+ struct string_list garbage = STRING_LIST_INIT_DUP;
- sprintf(path, "%s/pack", objdir);
- len = strlen(path);
- dir = opendir(path);
+ strbuf_addstr(&path, objdir);
+ strbuf_addstr(&path, "/pack");
+ dir = opendir(path.buf);
if (!dir) {
if (errno != ENOENT)
error("unable to open object pack directory: %s: %s",
- path, strerror(errno));
+ path.buf, strerror(errno));
+ strbuf_release(&path);
return;
}
- path[len++] = '/';
+ strbuf_addch(&path, '/');
+ dirnamelen = path.len;
while ((de = readdir(dir)) != NULL) {
- int namelen = strlen(de->d_name);
struct packed_git *p;
+ size_t base_len;
- if (!has_extension(de->d_name, ".idx"))
- continue;
-
- if (len + namelen + 1 > sizeof(path))
+ if (is_dot_or_dotdot(de->d_name))
continue;
- /* Don't reopen a pack we already have. */
- strcpy(path + len, de->d_name);
- for (p = packed_git; p; p = p->next) {
- if (!memcmp(path, p->pack_name, len + namelen - 4))
- break;
+ strbuf_setlen(&path, dirnamelen);
+ strbuf_addstr(&path, de->d_name);
+
+ base_len = path.len;
+ if (strip_suffix_mem(path.buf, &base_len, ".idx")) {
+ /* Don't reopen a pack we already have. */
+ for (p = packed_git; p; p = p->next) {
+ size_t len;
+ if (strip_suffix(p->pack_name, ".pack", &len) &&
+ len == base_len &&
+ !memcmp(p->pack_name, path.buf, len))
+ break;
+ }
+ if (p == NULL &&
+ /*
+ * See if it really is a valid .idx file with
+ * corresponding .pack file that we can map.
+ */
+ (p = add_packed_git(path.buf, path.len, local)) != NULL)
+ install_packed_git(p);
}
- if (p)
- continue;
- /* See if it really is a valid .idx file with corresponding
- * .pack file that we can map.
- */
- p = add_packed_git(path, len + namelen, local);
- if (!p)
+
+ if (!report_garbage)
continue;
- install_packed_git(p);
+
+ if (ends_with(de->d_name, ".idx") ||
+ ends_with(de->d_name, ".pack") ||
+ ends_with(de->d_name, ".bitmap") ||
+ ends_with(de->d_name, ".keep"))
+ string_list_append(&garbage, path.buf);
+ else
+ report_garbage("garbage found", path.buf);
}
closedir(dir);
+ report_pack_garbage(&garbage);
+ string_list_clear(&garbage, 0);
+ strbuf_release(&path);
}
static int sort_pack(const void *a_, const void *b_)
void reprepare_packed_git(void)
{
- discard_revindex();
prepare_packed_git_run_once = 0;
prepare_packed_git();
}
char buf[1024 * 16];
ssize_t readlen = read_istream(st, buf, sizeof(buf));
+ if (readlen < 0) {
+ close_istream(st);
+ return -1;
+ }
if (!readlen)
break;
git_SHA1_Update(&c, buf, readlen);
return hashcmp(sha1, real_sha1) ? -1 : 0;
}
-static int git_open_noatime(const char *name)
+int git_open_noatime(const char *name)
{
static int sha1_file_open_flag = O_NOATIME;
}
}
+static int stat_sha1_file(const unsigned char *sha1, struct stat *st)
+{
+ struct alternate_object_database *alt;
+
+ if (!lstat(sha1_file_name(sha1), st))
+ return 0;
+
+ prepare_alt_odb();
+ errno = ENOENT;
+ for (alt = alt_odb_list; alt; alt = alt->next) {
+ fill_sha1_path(alt->name, sha1);
+ if (!lstat(alt->base, st))
+ return 0;
+ }
+
+ return -1;
+}
+
static int open_sha1_file(const unsigned char *sha1)
{
int fd;
- char *name = sha1_file_name(sha1);
struct alternate_object_database *alt;
+ int most_interesting_errno;
- fd = git_open_noatime(name);
+ fd = git_open_noatime(sha1_file_name(sha1));
if (fd >= 0)
return fd;
+ most_interesting_errno = errno;
prepare_alt_odb();
- errno = ENOENT;
for (alt = alt_odb_list; alt; alt = alt->next) {
- name = alt->name;
- fill_sha1_path(name, sha1);
+ fill_sha1_path(alt->name, sha1);
fd = git_open_noatime(alt->base);
if (fd >= 0)
return fd;
+ if (most_interesting_errno == ENOENT)
+ most_interesting_errno = errno;
}
+ errno = most_interesting_errno;
return -1;
}
return map;
}
-/*
- * There used to be a second loose object header format which
- * was meant to mimic the in-pack format, allowing for direct
- * copy of the object data. This format turned up not to be
- * really worth it and we no longer write loose objects in that
- * format.
- */
-static int experimental_loose_object(unsigned char *map)
-{
- unsigned int word;
-
- /*
- * We must determine if the buffer contains the standard
- * zlib-deflated stream or the experimental format based
- * on the in-pack object format. Compare the header byte
- * for each format:
- *
- * RFC1950 zlib w/ deflate : 0www1000 : 0 <= www <= 7
- * Experimental pack-based : Stttssss : ttt = 1,2,3,4
- *
- * If bit 7 is clear and bits 0-3 equal 8, the buffer MUST be
- * in standard loose-object format, UNLESS it is a Git-pack
- * format object *exactly* 8 bytes in size when inflated.
- *
- * However, RFC1950 also specifies that the 1st 16-bit word
- * must be divisible by 31 - this checksum tells us our buffer
- * is in the standard format, giving a false positive only if
- * the 1st word of the Git-pack format object happens to be
- * divisible by 31, ie:
- * ((byte0 * 256) + byte1) % 31 = 0
- * => 0ttt10000www1000 % 31 = 0
- *
- * As it happens, this case can only arise for www=3 & ttt=1
- * - ie, a Commit object, which would have to be 8 bytes in
- * size. As no Commit can be that small, we find that the
- * combination of these two criteria (bitmask & checksum)
- * can always correctly determine the buffer format.
- */
- word = (map[0] << 8) + map[1];
- if ((map[0] & 0x8F) == 0x08 && !(word % 31))
- return 0;
- else
- return 1;
-}
-
unsigned long unpack_object_header_buffer(const unsigned char *buf,
unsigned long len, enum object_type *type, unsigned long *sizep)
{
int unpack_sha1_header(git_zstream *stream, unsigned char *map, unsigned long mapsize, void *buffer, unsigned long bufsiz)
{
- unsigned long size, used;
- static const char valid_loose_object_type[8] = {
- 0, /* OBJ_EXT */
- 1, 1, 1, 1, /* "commit", "tree", "blob", "tag" */
- 0, /* "delta" and others are invalid in a loose object */
- };
- enum object_type type;
-
/* Get the data stream */
memset(stream, 0, sizeof(*stream));
stream->next_in = map;
stream->next_out = buffer;
stream->avail_out = bufsiz;
- if (experimental_loose_object(map)) {
- /*
- * The old experimental format we no longer produce;
- * we can still read it.
- */
- used = unpack_object_header_buffer(map, mapsize, &type, &size);
- if (!used || !valid_loose_object_type[type])
- return -1;
- map += used;
- mapsize -= used;
-
- /* Set up the stream for the rest.. */
- stream->next_in = map;
- stream->avail_in = mapsize;
- git_inflate_init(stream);
-
- /* And generate the fake traditional header */
- stream->total_out = 1 + snprintf(buffer, bufsiz, "%s %lu",
- typename(type), size);
- return 0;
- }
git_inflate_init(stream);
return git_inflate(stream, 0);
}
return base_offset;
}
-/* forward declaration for a mutually recursive function */
-static int packed_object_info(struct packed_git *p, off_t offset,
- unsigned long *sizep, int *rtype);
+/*
+ * Like get_delta_base above, but we return the sha1 instead of the pack
+ * offset. This means it is cheaper for REF deltas (we do not have to do
+ * the final object lookup), but more expensive for OFS deltas (we
+ * have to load the revidx to convert the offset back into a sha1).
+ */
+static const unsigned char *get_delta_base_sha1(struct packed_git *p,
+ struct pack_window **w_curs,
+ off_t curpos,
+ enum object_type type,
+ off_t delta_obj_offset)
+{
+ if (type == OBJ_REF_DELTA) {
+ unsigned char *base = use_pack(p, w_curs, curpos, NULL);
+ return base;
+ } else if (type == OBJ_OFS_DELTA) {
+ struct revindex_entry *revidx;
+ off_t base_offset = get_delta_base(p, w_curs, &curpos,
+ type, delta_obj_offset);
-static int packed_delta_info(struct packed_git *p,
- struct pack_window **w_curs,
- off_t curpos,
- enum object_type type,
- off_t obj_offset,
- unsigned long *sizep)
-{
- off_t base_offset;
+ if (!base_offset)
+ return NULL;
- base_offset = get_delta_base(p, w_curs, &curpos, type, obj_offset);
- if (!base_offset)
- return OBJ_BAD;
- type = packed_object_info(p, base_offset, NULL, NULL);
- if (type <= OBJ_NONE) {
- struct revindex_entry *revidx;
- const unsigned char *base_sha1;
revidx = find_pack_revindex(p, base_offset);
if (!revidx)
- return OBJ_BAD;
- base_sha1 = nth_packed_object_sha1(p, revidx->nr);
- mark_bad_packed_object(p, base_sha1);
- type = sha1_object_info(base_sha1, NULL);
- if (type <= OBJ_NONE)
- return OBJ_BAD;
- }
-
- /* We choose to only get the type of the base object and
- * ignore potentially corrupt pack file that expects the delta
- * based on a base with a wrong size. This saves tons of
- * inflate() calls.
- */
- if (sizep) {
- *sizep = get_size_from_delta(p, w_curs, curpos);
- if (*sizep == 0)
- type = OBJ_BAD;
- }
+ return NULL;
- return type;
+ return nth_packed_object_sha1(p, revidx->nr);
+ } else
+ return NULL;
}
int unpack_object_header(struct packed_git *p,
return type;
}
-static int packed_object_info(struct packed_git *p, off_t obj_offset,
- unsigned long *sizep, int *rtype)
+static int retry_bad_packed_offset(struct packed_git *p, off_t obj_offset)
{
- struct pack_window *w_curs = NULL;
- unsigned long size;
- off_t curpos = obj_offset;
- enum object_type type;
+ int type;
+ struct revindex_entry *revidx;
+ const unsigned char *sha1;
+ revidx = find_pack_revindex(p, obj_offset);
+ if (!revidx)
+ return OBJ_BAD;
+ sha1 = nth_packed_object_sha1(p, revidx->nr);
+ mark_bad_packed_object(p, sha1);
+ type = sha1_object_info(sha1, NULL);
+ if (type <= OBJ_NONE)
+ return OBJ_BAD;
+ return type;
+}
- type = unpack_object_header(p, &w_curs, &curpos, &size);
- if (rtype)
- *rtype = type; /* representation type */
+#define POI_STACK_PREALLOC 64
+
+static enum object_type packed_to_object_type(struct packed_git *p,
+ off_t obj_offset,
+ enum object_type type,
+ struct pack_window **w_curs,
+ off_t curpos)
+{
+ off_t small_poi_stack[POI_STACK_PREALLOC];
+ off_t *poi_stack = small_poi_stack;
+ int poi_stack_nr = 0, poi_stack_alloc = POI_STACK_PREALLOC;
+
+ while (type == OBJ_OFS_DELTA || type == OBJ_REF_DELTA) {
+ off_t base_offset;
+ unsigned long size;
+ /* Push the object we're going to leave behind */
+ if (poi_stack_nr >= poi_stack_alloc && poi_stack == small_poi_stack) {
+ poi_stack_alloc = alloc_nr(poi_stack_nr);
+ poi_stack = xmalloc(sizeof(off_t)*poi_stack_alloc);
+ memcpy(poi_stack, small_poi_stack, sizeof(off_t)*poi_stack_nr);
+ } else {
+ ALLOC_GROW(poi_stack, poi_stack_nr+1, poi_stack_alloc);
+ }
+ poi_stack[poi_stack_nr++] = obj_offset;
+ /* If parsing the base offset fails, just unwind */
+ base_offset = get_delta_base(p, w_curs, &curpos, type, obj_offset);
+ if (!base_offset)
+ goto unwind;
+ curpos = obj_offset = base_offset;
+ type = unpack_object_header(p, w_curs, &curpos, &size);
+ if (type <= OBJ_NONE) {
+ /* If getting the base itself fails, we first
+ * retry the base, otherwise unwind */
+ type = retry_bad_packed_offset(p, base_offset);
+ if (type > OBJ_NONE)
+ goto out;
+ goto unwind;
+ }
+ }
switch (type) {
- case OBJ_OFS_DELTA:
- case OBJ_REF_DELTA:
- type = packed_delta_info(p, &w_curs, curpos,
- type, obj_offset, sizep);
- break;
+ case OBJ_BAD:
case OBJ_COMMIT:
case OBJ_TREE:
case OBJ_BLOB:
case OBJ_TAG:
- if (sizep)
- *sizep = size;
break;
default:
error("unknown object type %i at offset %"PRIuMAX" in %s",
type, (uintmax_t)obj_offset, p->pack_name);
type = OBJ_BAD;
}
+
+out:
+ if (poi_stack != small_poi_stack)
+ free(poi_stack);
+ return type;
+
+unwind:
+ while (poi_stack_nr) {
+ obj_offset = poi_stack[--poi_stack_nr];
+ type = retry_bad_packed_offset(p, obj_offset);
+ if (type > OBJ_NONE)
+ goto out;
+ }
+ type = OBJ_BAD;
+ goto out;
+}
+
+static int packed_object_info(struct packed_git *p, off_t obj_offset,
+ struct object_info *oi)
+{
+ struct pack_window *w_curs = NULL;
+ unsigned long size;
+ off_t curpos = obj_offset;
+ enum object_type type;
+
+ /*
+ * We always get the representation type, but only convert it to
+ * a "real" type later if the caller is interested.
+ */
+ type = unpack_object_header(p, &w_curs, &curpos, &size);
+
+ if (oi->sizep) {
+ if (type == OBJ_OFS_DELTA || type == OBJ_REF_DELTA) {
+ off_t tmp_pos = curpos;
+ off_t base_offset = get_delta_base(p, &w_curs, &tmp_pos,
+ type, obj_offset);
+ if (!base_offset) {
+ type = OBJ_BAD;
+ goto out;
+ }
+ *oi->sizep = get_size_from_delta(p, &w_curs, tmp_pos);
+ if (*oi->sizep == 0) {
+ type = OBJ_BAD;
+ goto out;
+ }
+ } else {
+ *oi->sizep = size;
+ }
+ }
+
+ if (oi->disk_sizep) {
+ struct revindex_entry *revidx = find_pack_revindex(p, obj_offset);
+ *oi->disk_sizep = revidx[1].offset - obj_offset;
+ }
+
+ if (oi->typep) {
+ *oi->typep = packed_to_object_type(p, obj_offset, type, &w_curs, curpos);
+ if (*oi->typep < 0) {
+ type = OBJ_BAD;
+ goto out;
+ }
+ }
+
+ if (oi->delta_base_sha1) {
+ if (type == OBJ_OFS_DELTA || type == OBJ_REF_DELTA) {
+ const unsigned char *base;
+
+ base = get_delta_base_sha1(p, &w_curs, curpos,
+ type, obj_offset);
+ if (!base) {
+ type = OBJ_BAD;
+ goto out;
+ }
+
+ hashcpy(oi->delta_base_sha1, base);
+ } else
+ hashclr(oi->delta_base_sha1);
+ }
+
+out:
unuse_pack(&w_curs);
return type;
}
git_zstream stream;
unsigned char *buffer, *in;
- buffer = xmallocz(size);
+ buffer = xmallocz_gently(size);
+ if (!buffer)
+ return NULL;
memset(&stream, 0, sizeof(stream));
stream.next_out = buffer;
stream.avail_out = size + 1;
return hash % MAX_DELTA_CACHE;
}
-static int in_delta_base_cache(struct packed_git *p, off_t base_offset)
+static struct delta_base_cache_entry *
+get_delta_base_cache_entry(struct packed_git *p, off_t base_offset)
{
unsigned long hash = pack_entry_hash(p, base_offset);
- struct delta_base_cache_entry *ent = delta_base_cache + hash;
+ return delta_base_cache + hash;
+}
+
+static int eq_delta_base_cache_entry(struct delta_base_cache_entry *ent,
+ struct packed_git *p, off_t base_offset)
+{
return (ent->data && ent->p == p && ent->base_offset == base_offset);
}
+static int in_delta_base_cache(struct packed_git *p, off_t base_offset)
+{
+ struct delta_base_cache_entry *ent;
+ ent = get_delta_base_cache_entry(p, base_offset);
+ return eq_delta_base_cache_entry(ent, p, base_offset);
+}
+
+static void clear_delta_base_cache_entry(struct delta_base_cache_entry *ent)
+{
+ ent->data = NULL;
+ ent->lru.next->prev = ent->lru.prev;
+ ent->lru.prev->next = ent->lru.next;
+ delta_base_cached -= ent->size;
+}
+
static void *cache_or_unpack_entry(struct packed_git *p, off_t base_offset,
unsigned long *base_size, enum object_type *type, int keep_cache)
{
+ struct delta_base_cache_entry *ent;
void *ret;
- unsigned long hash = pack_entry_hash(p, base_offset);
- struct delta_base_cache_entry *ent = delta_base_cache + hash;
- ret = ent->data;
- if (!ret || ent->p != p || ent->base_offset != base_offset)
+ ent = get_delta_base_cache_entry(p, base_offset);
+
+ if (!eq_delta_base_cache_entry(ent, p, base_offset))
return unpack_entry(p, base_offset, type, base_size);
- if (!keep_cache) {
- ent->data = NULL;
- ent->lru.next->prev = ent->lru.prev;
- ent->lru.prev->next = ent->lru.next;
- delta_base_cached -= ent->size;
- } else {
+ ret = ent->data;
+
+ if (!keep_cache)
+ clear_delta_base_cache_entry(ent);
+ else
ret = xmemdupz(ent->data, ent->size);
- }
*type = ent->type;
*base_size = ent->size;
return ret;
static void *read_object(const unsigned char *sha1, enum object_type *type,
unsigned long *size);
-static void *unpack_delta_entry(struct packed_git *p,
- struct pack_window **w_curs,
- off_t curpos,
- unsigned long delta_size,
- off_t obj_offset,
- enum object_type *type,
- unsigned long *sizep)
-{
- void *delta_data, *result, *base;
- unsigned long base_size;
- off_t base_offset;
-
- base_offset = get_delta_base(p, w_curs, &curpos, *type, obj_offset);
- if (!base_offset) {
- error("failed to validate delta base reference "
- "at offset %"PRIuMAX" from %s",
- (uintmax_t)curpos, p->pack_name);
- return NULL;
- }
- unuse_pack(w_curs);
- base = cache_or_unpack_entry(p, base_offset, &base_size, type, 0);
- if (!base) {
- /*
- * We're probably in deep shit, but let's try to fetch
- * the required base anyway from another pack or loose.
- * This is costly but should happen only in the presence
- * of a corrupted pack, and is better than failing outright.
- */
- struct revindex_entry *revidx;
- const unsigned char *base_sha1;
- revidx = find_pack_revindex(p, base_offset);
- if (!revidx)
- return NULL;
- base_sha1 = nth_packed_object_sha1(p, revidx->nr);
- error("failed to read delta base object %s"
- " at offset %"PRIuMAX" from %s",
- sha1_to_hex(base_sha1), (uintmax_t)base_offset,
- p->pack_name);
- mark_bad_packed_object(p, base_sha1);
- base = read_object(base_sha1, type, &base_size);
- if (!base)
- return NULL;
- }
-
- delta_data = unpack_compressed_entry(p, w_curs, curpos, delta_size);
- if (!delta_data) {
- error("failed to unpack compressed delta "
- "at offset %"PRIuMAX" from %s",
- (uintmax_t)curpos, p->pack_name);
- free(base);
- return NULL;
- }
- result = patch_delta(base, base_size,
- delta_data, delta_size,
- sizep);
- if (!result)
- die("failed to apply delta");
- free(delta_data);
- add_delta_base_cache(p, base_offset, base, base_size, *type);
- return result;
-}
-
static void write_pack_access_log(struct packed_git *p, off_t obj_offset)
{
- static FILE *log_file;
-
- if (!log_file) {
- log_file = fopen(log_pack_access, "w");
- if (!log_file) {
- error("cannot open pack access log '%s' for writing: %s",
- log_pack_access, strerror(errno));
- log_pack_access = NULL;
- return;
- }
- }
- fprintf(log_file, "%s %"PRIuMAX"\n",
- p->pack_name, (uintmax_t)obj_offset);
- fflush(log_file);
+ static struct trace_key pack_access = TRACE_KEY_INIT(PACK_ACCESS);
+ trace_printf_key(&pack_access, "%s %"PRIuMAX"\n",
+ p->pack_name, (uintmax_t)obj_offset);
}
int do_check_packed_object_crc;
+#define UNPACK_ENTRY_STACK_PREALLOC 64
+struct unpack_entry_stack_ent {
+ off_t obj_offset;
+ off_t curpos;
+ unsigned long size;
+};
+
void *unpack_entry(struct packed_git *p, off_t obj_offset,
- enum object_type *type, unsigned long *sizep)
+ enum object_type *final_type, unsigned long *final_size)
{
struct pack_window *w_curs = NULL;
off_t curpos = obj_offset;
- void *data;
+ void *data = NULL;
+ unsigned long size;
+ enum object_type type;
+ struct unpack_entry_stack_ent small_delta_stack[UNPACK_ENTRY_STACK_PREALLOC];
+ struct unpack_entry_stack_ent *delta_stack = small_delta_stack;
+ int delta_stack_nr = 0, delta_stack_alloc = UNPACK_ENTRY_STACK_PREALLOC;
+ int base_from_cache = 0;
- if (log_pack_access)
- write_pack_access_log(p, obj_offset);
+ write_pack_access_log(p, obj_offset);
- if (do_check_packed_object_crc && p->index_version > 1) {
- struct revindex_entry *revidx = find_pack_revindex(p, obj_offset);
- unsigned long len = revidx[1].offset - obj_offset;
- if (check_pack_crc(p, &w_curs, obj_offset, len, revidx->nr)) {
- const unsigned char *sha1 =
- nth_packed_object_sha1(p, revidx->nr);
- error("bad packed object CRC for %s",
- sha1_to_hex(sha1));
- mark_bad_packed_object(p, sha1);
- unuse_pack(&w_curs);
- return NULL;
+ /* PHASE 1: drill down to the innermost base object */
+ for (;;) {
+ off_t base_offset;
+ int i;
+ struct delta_base_cache_entry *ent;
+
+ ent = get_delta_base_cache_entry(p, curpos);
+ if (eq_delta_base_cache_entry(ent, p, curpos)) {
+ type = ent->type;
+ data = ent->data;
+ size = ent->size;
+ clear_delta_base_cache_entry(ent);
+ base_from_cache = 1;
+ break;
+ }
+
+ if (do_check_packed_object_crc && p->index_version > 1) {
+ struct revindex_entry *revidx = find_pack_revindex(p, obj_offset);
+ unsigned long len = revidx[1].offset - obj_offset;
+ if (check_pack_crc(p, &w_curs, obj_offset, len, revidx->nr)) {
+ const unsigned char *sha1 =
+ nth_packed_object_sha1(p, revidx->nr);
+ error("bad packed object CRC for %s",
+ sha1_to_hex(sha1));
+ mark_bad_packed_object(p, sha1);
+ unuse_pack(&w_curs);
+ return NULL;
+ }
}
+
+ type = unpack_object_header(p, &w_curs, &curpos, &size);
+ if (type != OBJ_OFS_DELTA && type != OBJ_REF_DELTA)
+ break;
+
+ base_offset = get_delta_base(p, &w_curs, &curpos, type, obj_offset);
+ if (!base_offset) {
+ error("failed to validate delta base reference "
+ "at offset %"PRIuMAX" from %s",
+ (uintmax_t)curpos, p->pack_name);
+ /* bail to phase 2, in hopes of recovery */
+ data = NULL;
+ break;
+ }
+
+ /* push object, proceed to base */
+ if (delta_stack_nr >= delta_stack_alloc
+ && delta_stack == small_delta_stack) {
+ delta_stack_alloc = alloc_nr(delta_stack_nr);
+ delta_stack = xmalloc(sizeof(*delta_stack)*delta_stack_alloc);
+ memcpy(delta_stack, small_delta_stack,
+ sizeof(*delta_stack)*delta_stack_nr);
+ } else {
+ ALLOC_GROW(delta_stack, delta_stack_nr+1, delta_stack_alloc);
+ }
+ i = delta_stack_nr++;
+ delta_stack[i].obj_offset = obj_offset;
+ delta_stack[i].curpos = curpos;
+ delta_stack[i].size = size;
+
+ curpos = obj_offset = base_offset;
}
- *type = unpack_object_header(p, &w_curs, &curpos, sizep);
- switch (*type) {
+ /* PHASE 2: handle the base */
+ switch (type) {
case OBJ_OFS_DELTA:
case OBJ_REF_DELTA:
- data = unpack_delta_entry(p, &w_curs, curpos, *sizep,
- obj_offset, type, sizep);
+ if (data)
+ die("BUG in unpack_entry: left loop at a valid delta");
break;
case OBJ_COMMIT:
case OBJ_TREE:
case OBJ_BLOB:
case OBJ_TAG:
- data = unpack_compressed_entry(p, &w_curs, curpos, *sizep);
+ if (!base_from_cache)
+ data = unpack_compressed_entry(p, &w_curs, curpos, size);
break;
default:
data = NULL;
error("unknown object type %i at offset %"PRIuMAX" in %s",
- *type, (uintmax_t)obj_offset, p->pack_name);
+ type, (uintmax_t)obj_offset, p->pack_name);
+ }
+
+ /* PHASE 3: apply deltas in order */
+
+ /* invariants:
+ * 'data' holds the base data, or NULL if there was corruption
+ */
+ while (delta_stack_nr) {
+ void *delta_data;
+ void *base = data;
+ unsigned long delta_size, base_size = size;
+ int i;
+
+ data = NULL;
+
+ if (base)
+ add_delta_base_cache(p, obj_offset, base, base_size, type);
+
+ if (!base) {
+ /*
+ * We're probably in deep shit, but let's try to fetch
+ * the required base anyway from another pack or loose.
+ * This is costly but should happen only in the presence
+ * of a corrupted pack, and is better than failing outright.
+ */
+ struct revindex_entry *revidx;
+ const unsigned char *base_sha1;
+ revidx = find_pack_revindex(p, obj_offset);
+ if (revidx) {
+ base_sha1 = nth_packed_object_sha1(p, revidx->nr);
+ error("failed to read delta base object %s"
+ " at offset %"PRIuMAX" from %s",
+ sha1_to_hex(base_sha1), (uintmax_t)obj_offset,
+ p->pack_name);
+ mark_bad_packed_object(p, base_sha1);
+ base = read_object(base_sha1, &type, &base_size);
+ }
+ }
+
+ i = --delta_stack_nr;
+ obj_offset = delta_stack[i].obj_offset;
+ curpos = delta_stack[i].curpos;
+ delta_size = delta_stack[i].size;
+
+ if (!base)
+ continue;
+
+ delta_data = unpack_compressed_entry(p, &w_curs, curpos, delta_size);
+
+ if (!delta_data) {
+ error("failed to unpack compressed delta "
+ "at offset %"PRIuMAX" from %s",
+ (uintmax_t)curpos, p->pack_name);
+ data = NULL;
+ continue;
+ }
+
+ data = patch_delta(base, base_size,
+ delta_data, delta_size,
+ &size);
+
+ /*
+ * We could not apply the delta; warn the user, but keep going.
+ * Our failure will be noticed either in the next iteration of
+ * the loop, or if this is the final delta, in the caller when
+ * we return NULL. Those code paths will take care of making
+ * a more explicit warning and retrying with another copy of
+ * the object.
+ */
+ if (!data)
+ error("failed to apply delta");
+
+ free(delta_data);
}
+
+ *final_type = type;
+ *final_size = size;
+
unuse_pack(&w_curs);
+
+ if (delta_stack != small_delta_stack)
+ free(delta_stack);
+
return data;
}
return 1;
}
+/*
+ * Iff a pack file contains the object named by sha1, return true and
+ * store its location to e.
+ */
static int find_pack_entry(const unsigned char *sha1, struct pack_entry *e)
{
struct packed_git *p;
return 1;
for (p = packed_git; p; p = p->next) {
- if (p == last_found_pack || !fill_pack_entry(sha1, e, p))
- continue;
+ if (p == last_found_pack)
+ continue; /* we already checked this one */
- last_found_pack = p;
- return 1;
+ if (fill_pack_entry(sha1, e, p)) {
+ last_found_pack = p;
+ return 1;
+ }
}
return 0;
}
}
-static int sha1_loose_object_info(const unsigned char *sha1, unsigned long *sizep)
+static int sha1_loose_object_info(const unsigned char *sha1,
+ struct object_info *oi)
{
int status;
unsigned long mapsize, size;
git_zstream stream;
char hdr[32];
+ if (oi->delta_base_sha1)
+ hashclr(oi->delta_base_sha1);
+
+ /*
+ * If we don't care about type or size, then we don't
+ * need to look inside the object at all. Note that we
+ * do not optimize out the stat call, even if the
+ * caller doesn't care about the disk-size, since our
+ * return value implicitly indicates whether the
+ * object even exists.
+ */
+ if (!oi->typep && !oi->sizep) {
+ struct stat st;
+ if (stat_sha1_file(sha1, &st) < 0)
+ return -1;
+ if (oi->disk_sizep)
+ *oi->disk_sizep = st.st_size;
+ return 0;
+ }
+
map = map_sha1_file(sha1, &mapsize);
if (!map)
- return error("unable to find %s", sha1_to_hex(sha1));
+ return -1;
+ if (oi->disk_sizep)
+ *oi->disk_sizep = mapsize;
if (unpack_sha1_header(&stream, map, mapsize, hdr, sizeof(hdr)) < 0)
status = error("unable to unpack %s header",
sha1_to_hex(sha1));
else if ((status = parse_sha1_header(hdr, &size)) < 0)
status = error("unable to parse %s header", sha1_to_hex(sha1));
- else if (sizep)
- *sizep = size;
+ else if (oi->sizep)
+ *oi->sizep = size;
git_inflate_end(&stream);
munmap(map, mapsize);
- return status;
+ if (oi->typep)
+ *oi->typep = status;
+ return 0;
}
-/* returns enum object_type or negative */
-int sha1_object_info_extended(const unsigned char *sha1, struct object_info *oi)
+int sha1_object_info_extended(const unsigned char *sha1, struct object_info *oi, unsigned flags)
{
struct cached_object *co;
struct pack_entry e;
- int status, rtype;
+ int rtype;
+ const unsigned char *real = lookup_replace_object_extended(sha1, flags);
- co = find_cached_object(sha1);
+ co = find_cached_object(real);
if (co) {
+ if (oi->typep)
+ *(oi->typep) = co->type;
if (oi->sizep)
*(oi->sizep) = co->size;
+ if (oi->disk_sizep)
+ *(oi->disk_sizep) = 0;
+ if (oi->delta_base_sha1)
+ hashclr(oi->delta_base_sha1);
oi->whence = OI_CACHED;
- return co->type;
+ return 0;
}
- if (!find_pack_entry(sha1, &e)) {
+ if (!find_pack_entry(real, &e)) {
/* Most likely it's a loose object. */
- status = sha1_loose_object_info(sha1, oi->sizep);
- if (status >= 0) {
+ if (!sha1_loose_object_info(real, oi)) {
oi->whence = OI_LOOSE;
- return status;
+ return 0;
}
/* Not a loose object; someone else may have just packed it. */
reprepare_packed_git();
- if (!find_pack_entry(sha1, &e))
- return status;
+ if (!find_pack_entry(real, &e))
+ return -1;
}
- status = packed_object_info(e.p, e.offset, oi->sizep, &rtype);
- if (status < 0) {
- mark_bad_packed_object(e.p, sha1);
- status = sha1_object_info_extended(sha1, oi);
+ rtype = packed_object_info(e.p, e.offset, oi);
+ if (rtype < 0) {
+ mark_bad_packed_object(e.p, real);
+ return sha1_object_info_extended(real, oi, 0);
} else if (in_delta_base_cache(e.p, e.offset)) {
oi->whence = OI_DBCACHED;
} else {
rtype == OBJ_OFS_DELTA);
}
- return status;
+ return 0;
}
+/* returns enum object_type or negative */
int sha1_object_info(const unsigned char *sha1, unsigned long *sizep)
{
- struct object_info oi;
+ enum object_type type;
+ struct object_info oi = {NULL};
+ oi.typep = &type;
oi.sizep = sizep;
- return sha1_object_info_extended(sha1, &oi);
+ if (sha1_object_info_extended(sha1, &oi, LOOKUP_REPLACE_OBJECT) < 0)
+ return -1;
+ return type;
}
static void *read_packed_sha1(const unsigned char *sha1,
hash_sha1_file(buf, len, typename(type), sha1);
if (has_sha1_file(sha1) || find_cached_object(sha1))
return 0;
- if (cached_object_alloc <= cached_object_nr) {
- cached_object_alloc = alloc_nr(cached_object_alloc);
- cached_objects = xrealloc(cached_objects,
- sizeof(*cached_objects) *
- cached_object_alloc);
- }
+ ALLOC_GROW(cached_objects, cached_object_nr + 1, cached_object_alloc);
co = &cached_objects[cached_object_nr++];
co->size = len;
co->type = type;
unsigned flag)
{
void *data;
- char *path;
const struct packed_git *p;
- const unsigned char *repl = (flag & READ_SHA1_FILE_REPLACE)
- ? lookup_replace_object(sha1) : sha1;
+ const unsigned char *repl = lookup_replace_object_extended(sha1, flag);
errno = 0;
data = read_object(repl, type, size);
sha1_to_hex(repl), sha1_to_hex(sha1));
if (has_loose_object(repl)) {
- path = sha1_file_name(sha1);
+ const char *path = sha1_file_name(sha1);
+
die("loose object %s (stored in %s) is corrupt",
sha1_to_hex(repl), path);
}
/* Make sure the directory exists */
memcpy(buffer, filename, dirlen);
buffer[dirlen-1] = 0;
- if (mkdir(buffer, 0777) || adjust_shared_perm(buffer))
+ if (mkdir(buffer, 0777) && errno != EEXIST)
+ return -1;
+ if (adjust_shared_perm(buffer))
return -1;
/* Try again */
git_zstream stream;
git_SHA_CTX c;
unsigned char parano_sha1[20];
- char *filename;
static char tmp_file[PATH_MAX];
+ const char *filename = sha1_file_name(sha1);
- filename = sha1_file_name(sha1);
fd = create_tmpfile(tmp_file, sizeof(tmp_file), filename);
if (fd < 0) {
if (errno == EACCES)
return move_temp_to_file(tmp_file, filename);
}
+static int freshen_loose_object(const unsigned char *sha1)
+{
+ return check_and_freshen(sha1, 1);
+}
+
+static int freshen_packed_object(const unsigned char *sha1)
+{
+ struct pack_entry e;
+ return find_pack_entry(sha1, &e) && freshen_file(e.p->pack_name);
+}
+
int write_sha1_file(const void *buf, unsigned long len, const char *type, unsigned char *returnsha1)
{
unsigned char sha1[20];
write_sha1_file_prepare(buf, len, type, sha1, hdr, &hdrlen);
if (returnsha1)
hashcpy(returnsha1, sha1);
- if (has_sha1_file(sha1))
+ if (freshen_loose_object(sha1) || freshen_packed_object(sha1))
return 0;
return write_loose_object(sha1, hdr, hdrlen, buf, len, 0);
}
if (find_pack_entry(sha1, &e))
return 1;
- return has_loose_object(sha1);
+ if (has_loose_object(sha1))
+ return 1;
+ reprepare_packed_git();
+ return find_pack_entry(sha1, &e);
}
static void check_tree(const void *buf, size_t size)
return ret;
}
+static int index_stream_convert_blob(unsigned char *sha1, int fd,
+ const char *path, unsigned flags)
+{
+ int ret;
+ const int write_object = flags & HASH_WRITE_OBJECT;
+ struct strbuf sbuf = STRBUF_INIT;
+
+ assert(path);
+ assert(would_convert_to_git_filter_fd(path));
+
+ convert_to_git_filter_fd(path, fd, &sbuf,
+ write_object ? safe_crlf : SAFE_CRLF_FALSE);
+
+ if (write_object)
+ ret = write_sha1_file(sbuf.buf, sbuf.len, typename(OBJ_BLOB),
+ sha1);
+ else
+ ret = hash_sha1_file(sbuf.buf, sbuf.len, typename(OBJ_BLOB),
+ sha1);
+ strbuf_release(&sbuf);
+ return ret;
+}
+
static int index_pipe(unsigned char *sha1, int fd, enum object_type type,
const char *path, unsigned flags)
{
enum object_type type, const char *path, unsigned flags)
{
int ret;
- size_t size = xsize_t(st->st_size);
- if (!S_ISREG(st->st_mode))
+ /*
+ * Call xsize_t() only when needed to avoid potentially unnecessary
+ * die() for large files.
+ */
+ if (type == OBJ_BLOB && path && would_convert_to_git_filter_fd(path))
+ ret = index_stream_convert_blob(sha1, fd, path, flags);
+ else if (!S_ISREG(st->st_mode))
ret = index_pipe(sha1, fd, type, path, flags);
- else if (size <= big_file_threshold || type != OBJ_BLOB ||
- (path && would_convert_to_git(path, NULL, 0, 0)))
- ret = index_core(sha1, fd, size, type, path, flags);
+ else if (st->st_size <= big_file_threshold || type != OBJ_BLOB ||
+ (path && would_convert_to_git(path)))
+ ret = index_core(sha1, fd, xsize_t(st->st_size), type, path,
+ flags);
else
- ret = index_stream(sha1, fd, size, type, path, flags);
+ ret = index_stream(sha1, fd, xsize_t(st->st_size), type, path,
+ flags);
close(fd);
return ret;
}
die("%s is not a valid '%s' object", sha1_to_hex(sha1),
typename(expect));
}
+
+static int for_each_file_in_obj_subdir(int subdir_nr,
+ struct strbuf *path,
+ each_loose_object_fn obj_cb,
+ each_loose_cruft_fn cruft_cb,
+ each_loose_subdir_fn subdir_cb,
+ void *data)
+{
+ size_t baselen = path->len;
+ DIR *dir = opendir(path->buf);
+ struct dirent *de;
+ int r = 0;
+
+ if (!dir) {
+ if (errno == ENOENT)
+ return 0;
+ return error("unable to open %s: %s", path->buf, strerror(errno));
+ }
+
+ while ((de = readdir(dir))) {
+ if (is_dot_or_dotdot(de->d_name))
+ continue;
+
+ strbuf_setlen(path, baselen);
+ strbuf_addf(path, "/%s", de->d_name);
+
+ if (strlen(de->d_name) == 38) {
+ char hex[41];
+ unsigned char sha1[20];
+
+ snprintf(hex, sizeof(hex), "%02x%s",
+ subdir_nr, de->d_name);
+ if (!get_sha1_hex(hex, sha1)) {
+ if (obj_cb) {
+ r = obj_cb(sha1, path->buf, data);
+ if (r)
+ break;
+ }
+ continue;
+ }
+ }
+
+ if (cruft_cb) {
+ r = cruft_cb(de->d_name, path->buf, data);
+ if (r)
+ break;
+ }
+ }
+ strbuf_setlen(path, baselen);
+
+ if (!r && subdir_cb)
+ r = subdir_cb(subdir_nr, path->buf, data);
+
+ closedir(dir);
+ return r;
+}
+
+int for_each_loose_file_in_objdir_buf(struct strbuf *path,
+ each_loose_object_fn obj_cb,
+ each_loose_cruft_fn cruft_cb,
+ each_loose_subdir_fn subdir_cb,
+ void *data)
+{
+ size_t baselen = path->len;
+ int r = 0;
+ int i;
+
+ for (i = 0; i < 256; i++) {
+ strbuf_addf(path, "/%02x", i);
+ r = for_each_file_in_obj_subdir(i, path, obj_cb, cruft_cb,
+ subdir_cb, data);
+ strbuf_setlen(path, baselen);
+ if (r)
+ break;
+ }
+
+ return r;
+}
+
+int for_each_loose_file_in_objdir(const char *path,
+ each_loose_object_fn obj_cb,
+ each_loose_cruft_fn cruft_cb,
+ each_loose_subdir_fn subdir_cb,
+ void *data)
+{
+ struct strbuf buf = STRBUF_INIT;
+ int r;
+
+ strbuf_addstr(&buf, path);
+ r = for_each_loose_file_in_objdir_buf(&buf, obj_cb, cruft_cb,
+ subdir_cb, data);
+ strbuf_release(&buf);
+
+ return r;
+}
+
+struct loose_alt_odb_data {
+ each_loose_object_fn *cb;
+ void *data;
+};
+
+static int loose_from_alt_odb(struct alternate_object_database *alt,
+ void *vdata)
+{
+ struct loose_alt_odb_data *data = vdata;
+ struct strbuf buf = STRBUF_INIT;
+ int r;
+
+ /* copy base not including trailing '/' */
+ strbuf_add(&buf, alt->base, alt->name - alt->base - 1);
+ r = for_each_loose_file_in_objdir_buf(&buf,
+ data->cb, NULL, NULL,
+ data->data);
+ strbuf_release(&buf);
+ return r;
+}
+
+int for_each_loose_object(each_loose_object_fn cb, void *data)
+{
+ struct loose_alt_odb_data alt;
+ int r;
+
+ r = for_each_loose_file_in_objdir(get_object_directory(),
+ cb, NULL, NULL, data);
+ if (r)
+ return r;
+
+ alt.cb = cb;
+ alt.data = data;
+ return foreach_alt_odb(loose_from_alt_odb, &alt);
+}
+
+static int for_each_object_in_pack(struct packed_git *p, each_packed_object_fn cb, void *data)
+{
+ uint32_t i;
+ int r = 0;
+
+ for (i = 0; i < p->num_objects; i++) {
+ const unsigned char *sha1 = nth_packed_object_sha1(p, i);
+
+ if (!sha1)
+ return error("unable to get sha1 of object %u in %s",
+ i, p->pack_name);
+
+ r = cb(sha1, p, i, data);
+ if (r)
+ break;
+ }
+ return r;
+}
+
+int for_each_packed_object(each_packed_object_fn cb, void *data)
+{
+ struct packed_git *p;
+ int r = 0;
+
+ prepare_packed_git();
+ for (p = packed_git; p; p = p->next) {
+ r = for_each_object_in_pack(p, cb, data);
+ if (r)
+ break;
+ }
+ return r;
+}