From: Junio C Hamano Date: Wed, 25 Feb 2009 22:48:30 +0000 (-0800) Subject: Merge branch 'jc/maint-1.6.0-pack-directory' X-Git-Tag: v1.6.2-rc2~1 X-Git-Url: https://git.lorimer.id.au/gitweb.git/diff_plain/bb0cebd7d0ac9bf2ddf94fe5579603819c4a1fc7?hp=-c Merge branch 'jc/maint-1.6.0-pack-directory' * jc/maint-1.6.0-pack-directory: Make sure objects/pack exists before creating a new pack --- bb0cebd7d0ac9bf2ddf94fe5579603819c4a1fc7 diff --combined builtin-pack-objects.c index cb51916fe3,7518d536c3..bcefa52c69 --- a/builtin-pack-objects.c +++ b/builtin-pack-objects.c @@@ -78,7 -78,7 +78,7 @@@ static int progress = 1 static int window = 10; static uint32_t pack_size_limit, pack_size_limit_cfg; static int depth = 50; -static int delta_search_threads = 1; +static int delta_search_threads; static int pack_to_stdout; static int num_preferred_base; static struct progress *progress_state; @@@ -195,16 -195,16 +195,16 @@@ static int check_pack_inflate(struct pa int st; memset(&stream, 0, sizeof(stream)); - inflateInit(&stream); + git_inflate_init(&stream); do { in = use_pack(p, w_curs, offset, &stream.avail_in); stream.next_in = in; stream.next_out = fakebuf; stream.avail_out = sizeof(fakebuf); - st = inflate(&stream, Z_FINISH); + st = git_inflate(&stream, Z_FINISH); offset += stream.next_in - in; } while (st == Z_OK || st == Z_BUF_ERROR); - inflateEnd(&stream); + git_inflate_end(&stream); return (st == Z_STREAM_END && stream.total_out == expect && stream.total_in == len) ? 0 : -1; @@@ -286,7 -286,6 +286,7 @@@ static unsigned long write_object(struc */ if (!to_reuse) { + no_reuse: if (!usable_delta) { buf = read_sha1_file(entry->idx.sha1, &type, &size); if (!buf) @@@ -368,60 -367,46 +368,60 @@@ struct revindex_entry *revidx; off_t offset; - if (entry->delta) { + if (entry->delta) type = (allow_ofs_delta && entry->delta->idx.offset) ? OBJ_OFS_DELTA : OBJ_REF_DELTA; - reused_delta++; - } hdrlen = encode_header(type, entry->size, header); + offset = entry->in_pack_offset; revidx = find_pack_revindex(p, offset); datalen = revidx[1].offset - offset; if (!pack_to_stdout && p->index_version > 1 && - check_pack_crc(p, &w_curs, offset, datalen, revidx->nr)) - die("bad packed object CRC for %s", sha1_to_hex(entry->idx.sha1)); + check_pack_crc(p, &w_curs, offset, datalen, revidx->nr)) { + error("bad packed object CRC for %s", sha1_to_hex(entry->idx.sha1)); + unuse_pack(&w_curs); + goto no_reuse; + } + offset += entry->in_pack_header_size; datalen -= entry->in_pack_header_size; + if (!pack_to_stdout && p->index_version == 1 && + check_pack_inflate(p, &w_curs, offset, datalen, entry->size)) { + error("corrupt packed object for %s", sha1_to_hex(entry->idx.sha1)); + unuse_pack(&w_curs); + goto no_reuse; + } + if (type == OBJ_OFS_DELTA) { off_t ofs = entry->idx.offset - entry->delta->idx.offset; unsigned pos = sizeof(dheader) - 1; dheader[pos] = ofs & 127; while (ofs >>= 7) dheader[--pos] = 128 | (--ofs & 127); - if (limit && hdrlen + sizeof(dheader) - pos + datalen + 20 >= limit) + if (limit && hdrlen + sizeof(dheader) - pos + datalen + 20 >= limit) { + unuse_pack(&w_curs); return 0; + } sha1write(f, header, hdrlen); sha1write(f, dheader + pos, sizeof(dheader) - pos); hdrlen += sizeof(dheader) - pos; + reused_delta++; } else if (type == OBJ_REF_DELTA) { - if (limit && hdrlen + 20 + datalen + 20 >= limit) + if (limit && hdrlen + 20 + datalen + 20 >= limit) { + unuse_pack(&w_curs); return 0; + } sha1write(f, header, hdrlen); sha1write(f, entry->delta->idx.sha1, 20); hdrlen += 20; + reused_delta++; } else { - if (limit && hdrlen + datalen + 20 >= limit) + if (limit && hdrlen + datalen + 20 >= limit) { + unuse_pack(&w_curs); return 0; + } sha1write(f, header, hdrlen); } - - if (!pack_to_stdout && p->index_version == 1 && - check_pack_inflate(p, &w_curs, offset, datalen, entry->size)) - die("corrupt packed object for %s", sha1_to_hex(entry->idx.sha1)); copy_pack_data(f, p, &w_curs, offset, datalen); unuse_pack(&w_curs); reused++; @@@ -488,9 -473,8 +488,8 @@@ static void write_pack_file(void } else { char tmpname[PATH_MAX]; int fd; - snprintf(tmpname, sizeof(tmpname), - "%s/pack/tmp_pack_XXXXXX", get_object_directory()); - fd = xmkstemp(tmpname); + fd = odb_mkstemp(tmpname, sizeof(tmpname), + "pack/tmp_pack_XXXXXX"); pack_tmp_name = xstrdup(tmpname); f = sha1fd(fd, pack_tmp_name); } @@@ -1032,11 -1016,9 +1031,11 @@@ static void check_object(struct object_ * We want in_pack_type even if we do not reuse delta * since non-delta representations could still be reused. */ - used = unpack_object_header_gently(buf, avail, + used = unpack_object_header_buffer(buf, avail, &entry->in_pack_type, &entry->size); + if (used == 0) + goto give_up; /* * Determine if this is a delta and if so whether we can @@@ -1048,8 -1030,6 +1047,8 @@@ /* Not a delta hence we've already got all we need. */ entry->type = entry->in_pack_type; entry->in_pack_header_size = used; + if (entry->type < OBJ_COMMIT || entry->type > OBJ_BLOB) + goto give_up; unuse_pack(&w_curs); return; case OBJ_REF_DELTA: @@@ -1066,25 -1046,19 +1065,25 @@@ ofs = c & 127; while (c & 128) { ofs += 1; - if (!ofs || MSB(ofs, 7)) - die("delta base offset overflow in pack for %s", - sha1_to_hex(entry->idx.sha1)); + if (!ofs || MSB(ofs, 7)) { + error("delta base offset overflow in pack for %s", + sha1_to_hex(entry->idx.sha1)); + goto give_up; + } c = buf[used_0++]; ofs = (ofs << 7) + (c & 127); } - if (ofs >= entry->in_pack_offset) - die("delta base offset out of bound for %s", - sha1_to_hex(entry->idx.sha1)); ofs = entry->in_pack_offset - ofs; + if (ofs <= 0 || ofs >= entry->in_pack_offset) { + error("delta base offset out of bound for %s", + sha1_to_hex(entry->idx.sha1)); + goto give_up; + } if (reuse_delta && !entry->preferred_base) { struct revindex_entry *revidx; revidx = find_pack_revindex(p, ofs); + if (!revidx) + goto give_up; base_ref = nth_packed_object_sha1(p, revidx->nr); } entry->in_pack_header_size = used + used_0; @@@ -1104,7 -1078,6 +1103,7 @@@ */ entry->type = entry->in_pack_type; entry->delta = base_entry; + entry->delta_size = entry->size; entry->delta_sibling = base_entry->delta_child; base_entry->delta_child = entry; unuse_pack(&w_curs); @@@ -1119,8 -1092,6 +1118,8 @@@ */ entry->size = get_size_from_delta(p, &w_curs, entry->in_pack_offset + entry->in_pack_header_size); + if (entry->size == 0) + goto give_up; unuse_pack(&w_curs); return; } @@@ -1130,7 -1101,6 +1129,7 @@@ * with sha1_object_info() to find about the object type * at this point... */ + give_up: unuse_pack(&w_curs); } @@@ -1413,10 -1383,12 +1412,10 @@@ static void find_deltas(struct object_e int window, int depth, unsigned *processed) { uint32_t i, idx = 0, count = 0; - unsigned int array_size = window * sizeof(struct unpacked); struct unpacked *array; unsigned long mem_usage = 0; - array = xmalloc(array_size); - memset(array, 0, array_size); + array = xcalloc(window, sizeof(struct unpacked)); for (;;) { struct object_entry *entry; @@@ -1612,18 -1584,11 +1611,18 @@@ static void ll_find_deltas(struct objec find_deltas(list, &list_size, window, depth, processed); return; } + if (progress > pack_to_stdout) + fprintf(stderr, "Delta compression using %d threads.\n", + delta_search_threads); /* Partition the work amongst work threads. */ for (i = 0; i < delta_search_threads; i++) { unsigned sub_size = list_size / (delta_search_threads - i); + /* don't use too small segments or no deltas will be found */ + if (sub_size < 2*window && i+1 < delta_search_threads) + sub_size = 0; + p[i].window = window; p[i].depth = depth; p[i].processed = processed; @@@ -1749,16 -1714,6 +1748,16 @@@ static void prepare_pack(int window, in get_object_details(); + /* + * If we're locally repacking then we need to be doubly careful + * from now on in order to make sure no stealth corruption gets + * propagated to the new pack. Clients receiving streamed packs + * should validate everything they get anyway so no need to incur + * the additional cost here in that case. + */ + if (!pack_to_stdout) + do_check_packed_object_crc = 1; + if (!nr_objects || !window || !depth) return; @@@ -1785,14 -1740,6 +1784,14 @@@ if (entry->type < 0) die("unable to get type of object %s", sha1_to_hex(entry->idx.sha1)); + } else { + if (entry->type < 0) { + /* + * This object is not found, but we + * don't have to include it anyway. + */ + continue; + } } delta_list[n++] = entry; diff --combined fast-import.c index 3ef3413e69,f0114948ec..3748ddf48d --- a/fast-import.c +++ b/fast-import.c @@@ -150,7 -150,6 +150,7 @@@ Format of STDIN stream #include "refs.h" #include "csum-file.h" #include "quote.h" +#include "exec_cmd.h" #define PACK_ID_BITS 16 #define MAX_PACK_ID ((1<next_pool) if ((p->end - p->next_free >= len)) break; @@@ -577,6 -572,9 +577,6 @@@ } r = p->next_free; - /* round out to a 'uintmax_t' alignment */ - if (len & (sizeof(uintmax_t) - 1)) - len += sizeof(uintmax_t) - (len & (sizeof(uintmax_t) - 1)); p->next_free += len; return r; } @@@ -817,9 -815,8 +817,8 @@@ static void start_packfile(void struct pack_header hdr; int pack_fd; - snprintf(tmpfile, sizeof(tmpfile), - "%s/pack/tmp_pack_XXXXXX", get_object_directory()); - pack_fd = xmkstemp(tmpfile); + pack_fd = odb_mkstemp(tmpfile, sizeof(tmpfile), + "pack/tmp_pack_XXXXXX"); p = xcalloc(1, sizeof(*p) + strlen(tmpfile) + 2); strcpy(p->pack_name, tmpfile); p->pack_fd = pack_fd; @@@ -847,7 -844,7 +846,7 @@@ static int oecmp (const void *a_, cons static char *create_index(void) { static char tmpfile[PATH_MAX]; - SHA_CTX ctx; + git_SHA_CTX ctx; struct sha1file *f; struct object_entry **idx, **c, **last, *e; struct object_entry_pool *o; @@@ -869,7 -866,7 +868,7 @@@ /* Generate the fan-out array. */ c = idx; for (i = 0; i < 256; i++) { - struct object_entry **next = c;; + struct object_entry **next = c; while (next < last) { if ((*next)->sha1[0] != i) break; @@@ -879,22 -876,21 +878,21 @@@ c = next; } - snprintf(tmpfile, sizeof(tmpfile), - "%s/pack/tmp_idx_XXXXXX", get_object_directory()); - idx_fd = xmkstemp(tmpfile); + idx_fd = odb_mkstemp(tmpfile, sizeof(tmpfile), + "pack/tmp_idx_XXXXXX"); f = sha1fd(idx_fd, tmpfile); sha1write(f, array, 256 * sizeof(int)); - SHA1_Init(&ctx); + git_SHA1_Init(&ctx); for (c = idx; c != last; c++) { uint32_t offset = htonl((*c)->offset); sha1write(f, &offset, 4); sha1write(f, (*c)->sha1, sizeof((*c)->sha1)); - SHA1_Update(&ctx, (*c)->sha1, 20); + git_SHA1_Update(&ctx, (*c)->sha1, 20); } sha1write(f, pack_data->sha1, sizeof(pack_data->sha1)); sha1close(f, NULL, CSUM_FSYNC); free(idx); - SHA1_Final(pack_data->sha1, &ctx); + git_SHA1_Final(pack_data->sha1, &ctx); return tmpfile; } @@@ -907,9 -903,7 +905,7 @@@ static char *keep_pack(char *curr_index chmod(pack_data->pack_name, 0444); chmod(curr_index_name, 0444); - snprintf(name, sizeof(name), "%s/pack/pack-%s.keep", - get_object_directory(), sha1_to_hex(pack_data->sha1)); - keep_fd = open(name, O_RDWR|O_CREAT|O_EXCL, 0600); + keep_fd = odb_pack_keep(name, sizeof(name), pack_data->sha1); if (keep_fd < 0) die("cannot create keep file"); write_or_die(keep_fd, keep_msg, strlen(keep_msg)); @@@ -945,7 -939,6 +941,7 @@@ static void end_packfile(void { struct packed_git *old_p = pack_data, *new_p; + clear_delta_base_cache(); if (object_count) { char *idx_name; int i; @@@ -1038,15 -1031,15 +1034,15 @@@ static int store_object unsigned char hdr[96]; unsigned char sha1[20]; unsigned long hdrlen, deltalen; - SHA_CTX c; + git_SHA_CTX c; z_stream s; hdrlen = sprintf((char*)hdr,"%s %lu", typename(type), (unsigned long)dat->len) + 1; - SHA1_Init(&c); - SHA1_Update(&c, hdr, hdrlen); - SHA1_Update(&c, dat->buf, dat->len); - SHA1_Final(sha1, &c); + git_SHA1_Init(&c); + git_SHA1_Update(&c, hdr, hdrlen); + git_SHA1_Update(&c, dat->buf, dat->len); + git_SHA1_Final(sha1, &c); if (sha1out) hashcpy(sha1out, sha1); @@@ -1750,12 -1743,9 +1746,12 @@@ static int validate_raw_date(const cha { const char *orig_src = src; char *endp, sign; + unsigned long date; + + errno = 0; - strtoul(src, &endp, 10); - if (endp == src || *endp != ' ') + date = strtoul(src, &endp, 10); + if (errno || endp == src || *endp != ' ') return -1; src = endp + 1; @@@ -1763,8 -1753,8 +1759,8 @@@ return -1; sign = *src; - strtoul(src + 1, &endp, 10); - if (endp == src || *endp || (endp - orig_src) >= maxlen) + date = strtoul(src + 1, &endp, 10); + if (errno || endp == src || *endp || (endp - orig_src) >= maxlen) return -1; strcpy(result, orig_src); @@@ -1874,13 -1864,12 +1870,13 @@@ static void file_change_m(struct branc if (!p) die("Corrupt mode: %s", command_buf.buf); switch (mode) { + case 0644: + case 0755: + mode |= S_IFREG; case S_IFREG | 0644: case S_IFREG | 0755: case S_IFLNK: case S_IFGITLINK: - case 0644: - case 0755: /* ok */ break; default: @@@ -1947,7 -1936,7 +1943,7 @@@ typename(type), command_buf.buf); } - tree_content_set(&b->branch_tree, p, sha1, S_IFREG | mode, NULL); + tree_content_set(&b->branch_tree, p, sha1, mode, NULL); } static void file_change_d(struct branch *b) @@@ -2408,8 -2397,6 +2404,8 @@@ int main(int argc, const char **argv { unsigned int i, show_stats = 1; + git_extract_argv0_path(argv[0]); + setup_git_directory(); git_config(git_pack_config, NULL); if (!pack_compression_seen && core_compression_seen) diff --combined git-compat-util.h index 079cbe9440,758a880862..dcf4127750 --- a/git-compat-util.h +++ b/git-compat-util.h @@@ -85,7 -85,6 +85,7 @@@ #undef _XOPEN_SOURCE #include #define _XOPEN_SOURCE 600 +#include "compat/cygwin.h" #else #undef _ALL_SOURCE /* AIX 5.3L defines a struct list with _ALL_SOURCE. */ #include @@@ -100,11 -99,6 +100,11 @@@ #include #endif +#ifndef NO_OPENSSL +#include +#include +#endif + /* On most systems would have given us this, but * not on some systems (e.g. GNU/Hurd). */ @@@ -155,7 -149,10 +155,7 @@@ extern void die(const char *err, ...) N extern int error(const char *err, ...) __attribute__((format (printf, 1, 2))); extern void warning(const char *err, ...) __attribute__((format (printf, 1, 2))); -extern void set_usage_routine(void (*routine)(const char *err) NORETURN); extern void set_die_routine(void (*routine)(const char *err, va_list params) NORETURN); -extern void set_error_routine(void (*routine)(const char *err, va_list params)); -extern void set_warn_routine(void (*routine)(const char *warn, va_list params)); extern int prefixcmp(const char *str, const char *prefix); extern time_t tm_to_time_t(const struct tm *tm); @@@ -195,12 -192,6 +195,12 @@@ extern int git_munmap(void *start, size #endif /* NO_MMAP */ +#ifdef NO_ST_BLOCKS_IN_STRUCT_STAT +#define on_disk_bytes(st) ((st).st_size) +#else +#define on_disk_bytes(st) ((st).st_blocks * 512) +#endif + #define DEFAULT_PACKED_GIT_LIMIT \ ((1024L * 1024L) * (sizeof(void*) >= 8 ? 8192 : 256)) @@@ -303,6 -294,8 +303,8 @@@ extern ssize_t xwrite(int fd, const voi extern int xdup(int fd); extern FILE *xfdopen(int fd, const char *mode); extern int xmkstemp(char *template); + extern int odb_mkstemp(char *template, size_t limit, const char *pattern); + extern int odb_pack_keep(char *name, size_t namesz, unsigned char *sha1); static inline size_t xsize_t(off_t len) { @@@ -327,15 -320,11 +329,15 @@@ extern unsigned char sane_ctype[256] #define GIT_SPACE 0x01 #define GIT_DIGIT 0x02 #define GIT_ALPHA 0x04 +#define GIT_GLOB_SPECIAL 0x08 +#define GIT_REGEX_SPECIAL 0x10 #define sane_istest(x,mask) ((sane_ctype[(unsigned char)(x)] & (mask)) != 0) #define isspace(x) sane_istest(x,GIT_SPACE) #define isdigit(x) sane_istest(x,GIT_DIGIT) #define isalpha(x) sane_istest(x,GIT_ALPHA) #define isalnum(x) sane_istest(x,GIT_ALPHA | GIT_DIGIT) +#define is_glob_special(x) sane_istest(x,GIT_GLOB_SPECIAL) +#define is_regex_special(x) sane_istest(x,GIT_GLOB_SPECIAL | GIT_REGEX_SPECIAL) #define tolower(x) sane_case((unsigned char)(x), 0x20) #define toupper(x) sane_case((unsigned char)(x), 0) diff --combined index-pack.c index f7a38079e1,745ac0bfa0..7fee872533 --- a/index-pack.c +++ b/index-pack.c @@@ -8,7 -8,6 +8,7 @@@ #include "tree.h" #include "progress.h" #include "fsck.h" +#include "exec_cmd.h" static const char index_pack_usage[] = "git index-pack [-v] [-o ] [{ ---keep | --keep= }] [--strict] { | --stdin [--fix-thin] [] }"; @@@ -68,7 -67,7 +68,7 @@@ static struct progress *progress static unsigned char input_buffer[4096]; static unsigned int input_offset, input_len; static off_t consumed_bytes; -static SHA_CTX input_ctx; +static git_SHA_CTX input_ctx; static uint32_t input_crc32; static int input_fd, output_fd, pack_fd; @@@ -120,7 -119,7 +120,7 @@@ static void flush(void if (input_offset) { if (output_fd >= 0) write_or_die(output_fd, input_buffer, input_offset); - SHA1_Update(&input_ctx, input_buffer, input_offset); + git_SHA1_Update(&input_ctx, input_buffer, input_offset); memmove(input_buffer, input_buffer + input_offset, input_len); input_offset = 0; } @@@ -172,14 -171,13 +172,13 @@@ static char *open_pack_file(char *pack_ input_fd = 0; if (!pack_name) { static char tmpfile[PATH_MAX]; - snprintf(tmpfile, sizeof(tmpfile), - "%s/pack/tmp_pack_XXXXXX", get_object_directory()); - output_fd = xmkstemp(tmpfile); + output_fd = odb_mkstemp(tmpfile, sizeof(tmpfile), + "pack/tmp_pack_XXXXXX"); pack_name = xstrdup(tmpfile); } else output_fd = open(pack_name, O_CREAT|O_EXCL|O_RDWR, 0600); if (output_fd < 0) - die("unable to create %s: %s\n", pack_name, strerror(errno)); + die("unable to create %s: %s", pack_name, strerror(errno)); pack_fd = output_fd; } else { input_fd = open(pack_name, O_RDONLY); @@@ -189,7 -187,7 +188,7 @@@ output_fd = -1; pack_fd = input_fd; } - SHA1_Init(&input_ctx); + git_SHA1_Init(&input_ctx); return pack_name; } @@@ -222,23 -220,17 +221,23 @@@ static void bad_object(unsigned long of die("pack has bad object at offset %lu: %s", offset, buf); } +static void free_base_data(struct base_data *c) +{ + if (c->data) { + free(c->data); + c->data = NULL; + base_cache_used -= c->size; + } +} + static void prune_base_data(struct base_data *retain) { struct base_data *b = base_cache; for (b = base_cache; base_cache_used > delta_base_cache_limit && b; b = b->child) { - if (b->data && b != retain) { - free(b->data); - b->data = NULL; - base_cache_used -= b->size; - } + if (b->data && b != retain) + free_base_data(b); } } @@@ -251,8 -243,7 +250,8 @@@ static void link_base_data(struct base_ c->base = base; c->child = NULL; - base_cache_used += c->size; + if (c->data) + base_cache_used += c->size; prune_base_data(c); } @@@ -263,7 -254,10 +262,7 @@@ static void unlink_base_data(struct bas base->child = NULL; else base_cache = NULL; - if (c->data) { - free(c->data); - base_cache_used -= c->size; - } + free_base_data(c); } static void *unpack_entry_data(unsigned long offset, unsigned long size) @@@ -276,10 -270,10 +275,10 @@@ stream.avail_out = size; stream.next_in = fill(1); stream.avail_in = input_len; - inflateInit(&stream); + git_inflate_init(&stream); for (;;) { - int ret = inflate(&stream, 0); + int ret = git_inflate(&stream, 0); use(input_len - stream.avail_in); if (stream.total_out == size && ret == Z_STREAM_END) break; @@@ -288,7 -282,7 +287,7 @@@ stream.next_in = fill(1); stream.avail_in = input_len; } - inflateEnd(&stream); + git_inflate_end(&stream); return buf; } @@@ -339,7 -333,7 +338,7 @@@ static void *unpack_raw_entry(struct ob base_offset = (base_offset << 7) + (c & 127); } delta_base->offset = obj->idx.offset - base_offset; - if (delta_base->offset >= obj->idx.offset) + if (delta_base->offset <= 0 || delta_base->offset >= obj->idx.offset) bad_object(obj->idx.offset, "delta base offset is out of bound"); break; case OBJ_COMMIT: @@@ -383,9 -377,9 +382,9 @@@ static void *get_data_from_pack(struct stream.avail_out = obj->size; stream.next_in = src; stream.avail_in = len; - inflateInit(&stream); - while ((st = inflate(&stream, Z_FINISH)) == Z_OK); - inflateEnd(&stream); + git_inflate_init(&stream); + while ((st = git_inflate(&stream, Z_FINISH)) == Z_OK); + git_inflate_end(&stream); if (st != Z_STREAM_END || stream.total_out != obj->size) die("serious inflate inconsistency"); free(src); @@@ -413,24 -407,22 +412,24 @@@ static int find_delta(const union delta return -first-1; } -static int find_delta_children(const union delta_base *base, - int *first_index, int *last_index) +static void find_delta_children(const union delta_base *base, + int *first_index, int *last_index) { int first = find_delta(base); int last = first; int end = nr_deltas - 1; - if (first < 0) - return -1; + if (first < 0) { + *first_index = 0; + *last_index = -1; + return; + } while (first > 0 && !memcmp(&deltas[first - 1].base, base, UNION_BASE_SZ)) --first; while (last < end && !memcmp(&deltas[last + 1].base, base, UNION_BASE_SZ)) ++last; *first_index = first; *last_index = last; - return 0; } static void sha1_object(const void *data, unsigned long size, @@@ -501,10 -493,8 +500,10 @@@ static void *get_base_data(struct base_ free(raw); if (!c->data) bad_object(obj->idx.offset, "failed to apply delta"); - } else + } else { c->data = get_data_from_pack(obj); + c->size = obj->size; + } base_cache_used += c->size; prune_base_data(c); @@@ -513,74 -503,49 +512,74 @@@ } static void resolve_delta(struct object_entry *delta_obj, - struct base_data *base_obj, enum object_type type) + struct base_data *base, struct base_data *result) { - void *delta_data; - unsigned long delta_size; - union delta_base delta_base; - int j, first, last; - struct base_data result; + void *base_data, *delta_data; - delta_obj->real_type = type; + delta_obj->real_type = base->obj->real_type; delta_data = get_data_from_pack(delta_obj); - delta_size = delta_obj->size; - result.data = patch_delta(get_base_data(base_obj), base_obj->size, - delta_data, delta_size, - &result.size); + base_data = get_base_data(base); + result->obj = delta_obj; + result->data = patch_delta(base_data, base->size, + delta_data, delta_obj->size, &result->size); free(delta_data); - if (!result.data) + if (!result->data) bad_object(delta_obj->idx.offset, "failed to apply delta"); - sha1_object(result.data, result.size, type, delta_obj->idx.sha1); + sha1_object(result->data, result->size, delta_obj->real_type, + delta_obj->idx.sha1); nr_resolved_deltas++; +} - result.obj = delta_obj; - link_base_data(base_obj, &result); +static void find_unresolved_deltas(struct base_data *base, + struct base_data *prev_base) +{ + int i, ref_first, ref_last, ofs_first, ofs_last; + + /* + * This is a recursive function. Those brackets should help reducing + * stack usage by limiting the scope of the delta_base union. + */ + { + union delta_base base_spec; + + hashcpy(base_spec.sha1, base->obj->idx.sha1); + find_delta_children(&base_spec, &ref_first, &ref_last); + + memset(&base_spec, 0, sizeof(base_spec)); + base_spec.offset = base->obj->idx.offset; + find_delta_children(&base_spec, &ofs_first, &ofs_last); + } + + if (ref_last == -1 && ofs_last == -1) { + free(base->data); + return; + } + + link_base_data(prev_base, base); - hashcpy(delta_base.sha1, delta_obj->idx.sha1); - if (!find_delta_children(&delta_base, &first, &last)) { - for (j = first; j <= last; j++) { - struct object_entry *child = objects + deltas[j].obj_no; - if (child->real_type == OBJ_REF_DELTA) - resolve_delta(child, &result, type); + for (i = ref_first; i <= ref_last; i++) { + struct object_entry *child = objects + deltas[i].obj_no; + if (child->real_type == OBJ_REF_DELTA) { + struct base_data result; + resolve_delta(child, base, &result); + if (i == ref_last && ofs_last == -1) + free_base_data(base); + find_unresolved_deltas(&result, base); } } - memset(&delta_base, 0, sizeof(delta_base)); - delta_base.offset = delta_obj->idx.offset; - if (!find_delta_children(&delta_base, &first, &last)) { - for (j = first; j <= last; j++) { - struct object_entry *child = objects + deltas[j].obj_no; - if (child->real_type == OBJ_OFS_DELTA) - resolve_delta(child, &result, type); + for (i = ofs_first; i <= ofs_last; i++) { + struct object_entry *child = objects + deltas[i].obj_no; + if (child->real_type == OBJ_OFS_DELTA) { + struct base_data result; + resolve_delta(child, base, &result); + if (i == ofs_last) + free_base_data(base); + find_unresolved_deltas(&result, base); } } - unlink_base_data(&result); + unlink_base_data(base); } static int compare_delta_entry(const void *a, const void *b) @@@ -625,7 -590,7 +624,7 @@@ static void parse_pack_objects(unsigne /* Check pack integrity */ flush(); - SHA1_Final(sha1, &input_ctx); + git_SHA1_Final(sha1, &input_ctx); if (hashcmp(fill(20), sha1)) die("pack is corrupted (SHA1 mismatch)"); use(20); @@@ -656,13 -621,37 +655,13 @@@ progress = start_progress("Resolving deltas", nr_deltas); for (i = 0; i < nr_objects; i++) { struct object_entry *obj = &objects[i]; - union delta_base base; - int j, ref, ref_first, ref_last, ofs, ofs_first, ofs_last; struct base_data base_obj; if (obj->type == OBJ_REF_DELTA || obj->type == OBJ_OFS_DELTA) continue; - hashcpy(base.sha1, obj->idx.sha1); - ref = !find_delta_children(&base, &ref_first, &ref_last); - memset(&base, 0, sizeof(base)); - base.offset = obj->idx.offset; - ofs = !find_delta_children(&base, &ofs_first, &ofs_last); - if (!ref && !ofs) - continue; - base_obj.data = get_data_from_pack(obj); - base_obj.size = obj->size; base_obj.obj = obj; - link_base_data(NULL, &base_obj); - - if (ref) - for (j = ref_first; j <= ref_last; j++) { - struct object_entry *child = objects + deltas[j].obj_no; - if (child->real_type == OBJ_REF_DELTA) - resolve_delta(child, &base_obj, obj->type); - } - if (ofs) - for (j = ofs_first; j <= ofs_last; j++) { - struct object_entry *child = objects + deltas[j].obj_no; - if (child->real_type == OBJ_OFS_DELTA) - resolve_delta(child, &base_obj, obj->type); - } - unlink_base_data(&base_obj); + base_obj.data = NULL; + find_unresolved_deltas(&base_obj, NULL); display_progress(progress, nr_resolved_deltas); } } @@@ -755,6 -744,7 +754,6 @@@ static void fix_unresolved_deltas(struc for (i = 0; i < n; i++) { struct delta_entry *d = sorted_by_pos[i]; enum object_type type; - int j, first, last; struct base_data base_obj; if (objects[d->obj_no].real_type != OBJ_REF_DELTA) @@@ -768,7 -758,16 +767,7 @@@ die("local object %s is corrupt", sha1_to_hex(d->base.sha1)); base_obj.obj = append_obj_to_pack(f, d->base.sha1, base_obj.data, base_obj.size, type); - link_base_data(NULL, &base_obj); - - find_delta_children(&d->base, &first, &last); - for (j = first; j <= last; j++) { - struct object_entry *child = objects + deltas[j].obj_no; - if (child->real_type == OBJ_REF_DELTA) - resolve_delta(child, &base_obj, type); - } - - unlink_base_data(&base_obj); + find_unresolved_deltas(&base_obj, NULL); display_progress(progress, nr_resolved_deltas); } free(sorted_by_pos); @@@ -790,26 -789,29 +789,28 @@@ static void final(const char *final_pac err = close(output_fd); if (err) die("error while closing pack file: %s", strerror(errno)); - chmod(curr_pack_name, 0444); } if (keep_msg) { int keep_fd, keep_msg_len = strlen(keep_msg); - if (!keep_name) { - snprintf(name, sizeof(name), "%s/pack/pack-%s.keep", - get_object_directory(), sha1_to_hex(sha1)); - keep_name = name; - } - keep_fd = open(keep_name, O_RDWR|O_CREAT|O_EXCL, 0600); + + if (!keep_name) + keep_fd = odb_pack_keep(name, sizeof(name), sha1); + else + keep_fd = open(keep_name, O_RDWR|O_CREAT|O_EXCL, 0600); + if (keep_fd < 0) { if (errno != EEXIST) - die("cannot write keep file"); + die("cannot write keep file '%s' (%s)", + keep_name, strerror(errno)); } else { if (keep_msg_len > 0) { write_or_die(keep_fd, keep_msg, keep_msg_len); write_or_die(keep_fd, "\n", 1); } if (close(keep_fd) != 0) - die("cannot write keep file"); + die("cannot close written keep file '%s' (%s)", + keep_name, strerror(errno)); report = "keep"; } } @@@ -823,9 -825,8 +824,9 @@@ if (move_temp_to_file(curr_pack_name, final_pack_name)) die("cannot store pack file"); } + if (from_stdin) + chmod(final_pack_name, 0444); - chmod(curr_index_name, 0444); if (final_index_name != curr_index_name) { if (!final_index_name) { snprintf(name, sizeof(name), "%s/pack/pack-%s.idx", @@@ -835,7 -836,6 +836,7 @@@ if (move_temp_to_file(curr_index_name, final_index_name)) die("cannot store index file"); } + chmod(final_index_name, 0444); if (!from_stdin) { printf("%s\n", sha1_to_hex(sha1)); @@@ -881,8 -881,6 +882,8 @@@ int main(int argc, char **argv struct pack_idx_entry **idx_objects; unsigned char pack_sha1[20]; + git_extract_argv0_path(argv[0]); + /* * We wish to read the repository's config file if any, and * for that it is necessary to call setup_git_directory_gently(). diff --combined pack-write.c index b426006c58,e82c025457..7053538f4c --- a/pack-write.c +++ b/pack-write.c @@@ -25,7 -25,7 +25,7 @@@ char *write_idx_file(char *index_name, off_t last_obj_offset = 0; uint32_t array[256]; int i, fd; - SHA_CTX ctx; + git_SHA_CTX ctx; uint32_t index_version; if (nr_objects) { @@@ -44,9 -44,7 +44,7 @@@ if (!index_name) { static char tmpfile[PATH_MAX]; - snprintf(tmpfile, sizeof(tmpfile), - "%s/pack/tmp_idx_XXXXXX", get_object_directory()); - fd = xmkstemp(tmpfile); + fd = odb_mkstemp(tmpfile, sizeof(tmpfile), "pack/tmp_idx_XXXXXX"); index_name = xstrdup(tmpfile); } else { unlink(index_name); @@@ -86,7 -84,7 +84,7 @@@ sha1write(f, array, 256 * 4); /* compute the SHA1 hash of sorted object names. */ - SHA1_Init(&ctx); + git_SHA1_Init(&ctx); /* * Write the actual SHA1 entries.. @@@ -99,7 -97,7 +97,7 @@@ sha1write(f, &offset, 4); } sha1write(f, obj->sha1, 20); - SHA1_Update(&ctx, obj->sha1, 20); + git_SHA1_Update(&ctx, obj->sha1, 20); } if (index_version >= 2) { @@@ -140,7 -138,7 +138,7 @@@ sha1write(f, sha1, 20); sha1close(f, NULL, CSUM_FSYNC); - SHA1_Final(sha1, &ctx); + git_SHA1_Final(sha1, &ctx); return index_name; } @@@ -168,12 -166,12 +166,12 @@@ void fixup_pack_header_footer(int pack_ off_t partial_pack_offset) { int aligned_sz, buf_sz = 8 * 1024; - SHA_CTX old_sha1_ctx, new_sha1_ctx; + git_SHA_CTX old_sha1_ctx, new_sha1_ctx; struct pack_header hdr; char *buf; - SHA1_Init(&old_sha1_ctx); - SHA1_Init(&new_sha1_ctx); + git_SHA1_Init(&old_sha1_ctx); + git_SHA1_Init(&new_sha1_ctx); if (lseek(pack_fd, 0, SEEK_SET) != 0) die("Failed seeking to start of %s: %s", pack_name, strerror(errno)); @@@ -181,9 -179,9 +179,9 @@@ die("Unable to reread header of %s: %s", pack_name, strerror(errno)); if (lseek(pack_fd, 0, SEEK_SET) != 0) die("Failed seeking to start of %s: %s", pack_name, strerror(errno)); - SHA1_Update(&old_sha1_ctx, &hdr, sizeof(hdr)); + git_SHA1_Update(&old_sha1_ctx, &hdr, sizeof(hdr)); hdr.hdr_entries = htonl(object_count); - SHA1_Update(&new_sha1_ctx, &hdr, sizeof(hdr)); + git_SHA1_Update(&new_sha1_ctx, &hdr, sizeof(hdr)); write_or_die(pack_fd, &hdr, sizeof(hdr)); partial_pack_offset -= sizeof(hdr); @@@ -198,7 -196,7 +196,7 @@@ break; if (n < 0) die("Failed to checksum %s: %s", pack_name, strerror(errno)); - SHA1_Update(&new_sha1_ctx, buf, n); + git_SHA1_Update(&new_sha1_ctx, buf, n); aligned_sz -= n; if (!aligned_sz) @@@ -207,11 -205,11 +205,11 @@@ if (!partial_pack_sha1) continue; - SHA1_Update(&old_sha1_ctx, buf, n); + git_SHA1_Update(&old_sha1_ctx, buf, n); partial_pack_offset -= n; if (partial_pack_offset == 0) { unsigned char sha1[20]; - SHA1_Final(sha1, &old_sha1_ctx); + git_SHA1_Final(sha1, &old_sha1_ctx); if (hashcmp(sha1, partial_pack_sha1) != 0) die("Unexpected checksum for %s " "(disk corruption?)", pack_name); @@@ -220,7 -218,7 +218,7 @@@ * pack, which also means making partial_pack_offset * big enough not to matter anymore. */ - SHA1_Init(&old_sha1_ctx); + git_SHA1_Init(&old_sha1_ctx); partial_pack_offset = ~partial_pack_offset; partial_pack_offset -= MSB(partial_pack_offset, 1); } @@@ -228,8 -226,8 +226,8 @@@ free(buf); if (partial_pack_sha1) - SHA1_Final(partial_pack_sha1, &old_sha1_ctx); - SHA1_Final(new_pack_sha1, &new_sha1_ctx); + git_SHA1_Final(partial_pack_sha1, &old_sha1_ctx); + git_SHA1_Final(new_pack_sha1, &new_sha1_ctx); write_or_die(pack_fd, new_pack_sha1, 20); fsync_or_die(pack_fd, pack_name); } @@@ -239,7 -237,7 +237,7 @@@ char *index_pack_lockfile(int ip_out char packname[46]; /* - * The first thing we expects from index-pack's output + * The first thing we expect from index-pack's output * is "pack\t%40s\n" or "keep\t%40s\n" (46 bytes) where * %40s is the newly created pack SHA1 name. In the "keep" * case, we need it to remove the corresponding .keep file diff --combined t/t5300-pack-object.sh index 04522857ab,73d871c713..ccfc64c6ee --- a/t/t5300-pack-object.sh +++ b/t/t5300-pack-object.sh @@@ -180,6 -180,23 +180,23 @@@ test_expect_success unset GIT_OBJECT_DIRECTORY + test_expect_success 'survive missing objects/pack directory' ' + ( + rm -fr missing-pack && + mkdir missing-pack && + cd missing-pack && + git init && + GOP=.git/objects/pack + rm -fr $GOP && + git index-pack --stdin --keep=test <../test-3-${packname_3}.pack && + test -f $GOP/pack-${packname_3}.pack && + test_cmp $GOP/pack-${packname_3}.pack ../test-3-${packname_3}.pack && + test -f $GOP/pack-${packname_3}.idx && + test_cmp $GOP/pack-${packname_3}.idx ../test-3-${packname_3}.idx && + test -f $GOP/pack-${packname_3}.keep + ) + ' + test_expect_success \ 'verify pack' \ 'git verify-pack test-1-${packname_1}.idx \ @@@ -272,8 -289,7 +289,8 @@@ test_expect_success test_expect_success \ 'make sure index-pack detects the SHA1 collision' \ - 'test_must_fail git index-pack -o bad.idx test-3.pack' + 'test_must_fail git index-pack -o bad.idx test-3.pack 2>msg && + grep "SHA1 COLLISION FOUND" msg' test_expect_success \ 'honor pack.packSizeLimit' \ diff --combined wrapper.c index c85ca52ec6,231a58f1a9..b07cdf299a --- a/wrapper.c +++ b/wrapper.c @@@ -197,62 -197,34 +197,94 @@@ int xmkstemp(char *template return fd; } +/* + * zlib wrappers to make sure we don't silently miss errors + * at init time. + */ +void git_inflate_init(z_streamp strm) +{ + const char *err; + + switch (inflateInit(strm)) { + case Z_OK: + return; + + case Z_MEM_ERROR: + err = "out of memory"; + break; + case Z_VERSION_ERROR: + err = "wrong version"; + break; + default: + err = "error"; + } + die("inflateInit: %s (%s)", err, strm->msg ? strm->msg : "no message"); +} + +void git_inflate_end(z_streamp strm) +{ + if (inflateEnd(strm) != Z_OK) + error("inflateEnd: %s", strm->msg ? strm->msg : "failed"); +} + +int git_inflate(z_streamp strm, int flush) +{ + int ret = inflate(strm, flush); + const char *err; + + switch (ret) { + /* Out of memory is fatal. */ + case Z_MEM_ERROR: + die("inflate: out of memory"); + + /* Data corruption errors: we may want to recover from them (fsck) */ + case Z_NEED_DICT: + err = "needs dictionary"; break; + case Z_DATA_ERROR: + err = "data stream error"; break; + case Z_STREAM_ERROR: + err = "stream consistency error"; break; + default: + err = "unknown error"; break; + + /* Z_BUF_ERROR: normal, needs more space in the output buffer */ + case Z_BUF_ERROR: + case Z_OK: + case Z_STREAM_END: + return ret; + } + error("inflate: %s (%s)", err, strm->msg ? strm->msg : "no message"); + return ret; +} ++ + int odb_mkstemp(char *template, size_t limit, const char *pattern) + { + int fd; + + snprintf(template, limit, "%s/%s", + get_object_directory(), pattern); + fd = mkstemp(template); + if (0 <= fd) + return fd; + + /* slow path */ + safe_create_leading_directories(template); + snprintf(template, limit, "%s/%s", + get_object_directory(), pattern); + return xmkstemp(template); + } + + int odb_pack_keep(char *name, size_t namesz, unsigned char *sha1) + { + int fd; + + snprintf(name, namesz, "%s/pack/pack-%s.keep", + get_object_directory(), sha1_to_hex(sha1)); + fd = open(name, O_RDWR|O_CREAT|O_EXCL, 0600); + if (0 <= fd) + return fd; + + /* slow path */ + safe_create_leading_directories(name); + return open(name, O_RDWR|O_CREAT|O_EXCL, 0600); + }