From: Junio C Hamano Date: Tue, 19 Jul 2011 16:33:03 +0000 (-0700) Subject: Merge branch 'jc/zlib-wrap' X-Git-Tag: v1.7.7-rc0~93 X-Git-Url: https://git.lorimer.id.au/gitweb.git/diff_plain/eb4f4076aa89d38221ee513a5f28f22124029c90?hp=-c Merge branch 'jc/zlib-wrap' * jc/zlib-wrap: zlib: allow feeding more than 4GB in one go zlib: zlib can only process 4GB at a time zlib: wrap deflateBound() too zlib: wrap deflate side of the API zlib: wrap inflateInit2 used to accept only for gzip format zlib: wrap remaining calls to direct inflate/inflateEnd zlib wrapper: refactor error message formatter Conflicts: sha1_file.c --- eb4f4076aa89d38221ee513a5f28f22124029c90 diff --combined builtin/index-pack.c index e40451ffb4,f65cb373bf..81cdc28b30 --- a/builtin/index-pack.c +++ b/builtin/index-pack.c @@@ -1,4 -1,4 +1,4 @@@ -#include "cache.h" +#include "builtin.h" #include "delta.h" #include "pack.h" #include "csum-file.h" @@@ -265,7 -265,7 +265,7 @@@ static void unlink_base_data(struct bas static void *unpack_entry_data(unsigned long offset, unsigned long size) { int status; - z_stream stream; + git_zstream stream; void *buf = xmalloc(size); memset(&stream, 0, sizeof(stream)); @@@ -355,7 -355,7 +355,7 @@@ static void *get_data_from_pack(struct off_t from = obj[0].idx.offset + obj[0].hdr_size; unsigned long len = obj[1].idx.offset - from; unsigned char *data, *inbuf; - z_stream stream; + git_zstream stream; int status; data = xmalloc(obj->size); @@@ -666,26 -666,26 +666,26 @@@ static void parse_pack_objects(unsigne static int write_compressed(struct sha1file *f, void *in, unsigned int size) { - z_stream stream; + git_zstream stream; int status; unsigned char outbuf[4096]; memset(&stream, 0, sizeof(stream)); - deflateInit(&stream, zlib_compression_level); + git_deflate_init(&stream, zlib_compression_level); stream.next_in = in; stream.avail_in = size; do { stream.next_out = outbuf; stream.avail_out = sizeof(outbuf); - status = deflate(&stream, Z_FINISH); + status = git_deflate(&stream, Z_FINISH); sha1write(f, outbuf, sizeof(outbuf) - stream.avail_out); } while (status == Z_OK); if (status != Z_STREAM_END) die("unable to deflate appended object (%d)", status); size = stream.total_out; - deflateEnd(&stream); + git_deflate_end(&stream); return size; } diff --combined cache.h index bc9e5eb15e,1784bab7d5..30be900911 --- a/cache.h +++ b/cache.h @@@ -6,7 -6,6 +6,7 @@@ #include "hash.h" #include "advice.h" #include "gettext.h" +#include "convert.h" #include SHA1_HEADER #ifndef git_SHA_CTX @@@ -17,13 -16,27 +17,27 @@@ #endif #include - #if defined(NO_DEFLATE_BOUND) || ZLIB_VERNUM < 0x1200 - #define deflateBound(c,s) ((s) + (((s) + 7) >> 3) + (((s) + 63) >> 6) + 11) - #endif - - void git_inflate_init(z_streamp strm); - void git_inflate_end(z_streamp strm); - int git_inflate(z_streamp strm, int flush); + typedef struct git_zstream { + z_stream z; + unsigned long avail_in; + unsigned long avail_out; + unsigned long total_in; + unsigned long total_out; + unsigned char *next_in; + unsigned char *next_out; + } git_zstream; + + void git_inflate_init(git_zstream *); + void git_inflate_init_gzip_only(git_zstream *); + void git_inflate_end(git_zstream *); + int git_inflate(git_zstream *, int flush); + + void git_deflate_init(git_zstream *, int level); + void git_deflate_init_gzip(git_zstream *, int level); + void git_deflate_end(git_zstream *); + int git_deflate_end_gently(git_zstream *); + int git_deflate(git_zstream *, int flush); + unsigned long git_deflate_bound(git_zstream *, unsigned long); #if defined(DT_UNKNOWN) && !defined(NO_D_TYPE_IN_DIRENT) #define DTYPE(de) ((de)->d_type) @@@ -512,18 -525,15 +526,18 @@@ struct pathspec struct pathspec_item { const char *match; int len; - unsigned int has_wildcard:1; + unsigned int use_wildcard:1; } *items; }; extern int init_pathspec(struct pathspec *, const char **); extern void free_pathspec(struct pathspec *); extern int ce_path_match(const struct cache_entry *ce, const struct pathspec *pathspec); -extern int index_fd(unsigned char *sha1, int fd, struct stat *st, int write_object, enum object_type type, const char *path, int format_check); -extern int index_path(unsigned char *sha1, const char *path, struct stat *st, int write_object); + +#define HASH_WRITE_OBJECT 1 +#define HASH_FORMAT_CHECK 2 +extern int index_fd(unsigned char *sha1, int fd, struct stat *st, enum object_type type, const char *path, unsigned flags); +extern int index_path(unsigned char *sha1, const char *path, struct stat *st, unsigned flags); extern void fill_stat_cache_info(struct cache_entry *ce, struct stat *st); #define REFRESH_REALLY 0x0001 /* ignore_valid */ @@@ -583,6 -593,35 +597,6 @@@ extern int fsync_object_files extern int core_preload_index; extern int core_apply_sparse_checkout; -enum safe_crlf { - SAFE_CRLF_FALSE = 0, - SAFE_CRLF_FAIL = 1, - SAFE_CRLF_WARN = 2 -}; - -extern enum safe_crlf safe_crlf; - -enum auto_crlf { - AUTO_CRLF_FALSE = 0, - AUTO_CRLF_TRUE = 1, - AUTO_CRLF_INPUT = -1 -}; - -extern enum auto_crlf auto_crlf; - -enum eol { - EOL_UNSET, - EOL_CRLF, - EOL_LF, -#ifdef NATIVE_CRLF - EOL_NATIVE = EOL_CRLF -#else - EOL_NATIVE = EOL_LF -#endif -}; - -extern enum eol eol; - enum branch_track { BRANCH_TRACK_UNSPECIFIED = -1, BRANCH_TRACK_NEVER = 0, @@@ -651,24 -690,14 +665,24 @@@ extern char *sha1_pack_name(const unsig extern char *sha1_pack_index_name(const unsigned char *sha1); extern const char *find_unique_abbrev(const unsigned char *sha1, int); extern const unsigned char null_sha1[20]; -static inline int is_null_sha1(const unsigned char *sha1) + +static inline int hashcmp(const unsigned char *sha1, const unsigned char *sha2) { - return !memcmp(sha1, null_sha1, 20); + int i; + + for (i = 0; i < 20; i++, sha1++, sha2++) { + if (*sha1 != *sha2) + return *sha1 - *sha2; + } + + return 0; } -static inline int hashcmp(const unsigned char *sha1, const unsigned char *sha2) + +static inline int is_null_sha1(const unsigned char *sha1) { - return memcmp(sha1, sha2, 20); + return !hashcmp(sha1, null_sha1); } + static inline void hashcpy(unsigned char *sha_dst, const unsigned char *sha_src) { memcpy(sha_dst, sha_src, 20); @@@ -719,7 -748,7 +733,7 @@@ extern char *expand_user_path(const cha char *enter_repo(char *path, int strict); static inline int is_absolute_path(const char *path) { - return path[0] == '/' || has_dos_drive_prefix(path); + return is_dir_sep(path[0]) || has_dos_drive_prefix(path); } int is_directory(const char *); const char *real_path(const char *path); @@@ -731,30 -760,17 +745,30 @@@ char *strip_path_suffix(const char *pat int daemon_avoid_alias(const char *path); int offset_1st_component(const char *path); -/* Read and unpack a sha1 file into memory, write memory to a sha1 file */ -extern int sha1_object_info(const unsigned char *, unsigned long *); -extern void *read_sha1_file_repl(const unsigned char *sha1, enum object_type *type, unsigned long *size, const unsigned char **replacement); +/* object replacement */ +#define READ_SHA1_FILE_REPLACE 1 +extern void *read_sha1_file_extended(const unsigned char *sha1, enum object_type *type, unsigned long *size, unsigned flag); static inline void *read_sha1_file(const unsigned char *sha1, enum object_type *type, unsigned long *size) { - return read_sha1_file_repl(sha1, type, size, NULL); + return read_sha1_file_extended(sha1, type, size, READ_SHA1_FILE_REPLACE); +} +extern const unsigned char *do_lookup_replace_object(const unsigned char *sha1); +static inline const unsigned char *lookup_replace_object(const unsigned char *sha1) +{ + if (!read_replace_refs) + return sha1; + return do_lookup_replace_object(sha1); } + +/* Read and unpack a sha1 file into memory, write memory to a sha1 file */ +extern int sha1_object_info(const unsigned char *, unsigned long *); extern int hash_sha1_file(const void *buf, unsigned long len, const char *type, unsigned char *sha1); extern int write_sha1_file(const void *buf, unsigned long len, const char *type, unsigned char *return_sha1); extern int pretend_sha1_file(void *, unsigned long, enum object_type, unsigned char *); extern int force_object_loose(const unsigned char *sha1, time_t mtime); +extern void *map_sha1_file(const unsigned char *sha1, unsigned long *size); - extern int unpack_sha1_header(z_stream *stream, unsigned char *map, unsigned long mapsize, void *buffer, unsigned long bufsiz); ++extern int unpack_sha1_header(git_zstream *stream, unsigned char *map, unsigned long mapsize, void *buffer, unsigned long bufsiz); +extern int parse_sha1_header(const char *hdr, unsigned long *sizep); /* global flag to enable extra checks when accessing packed objects */ extern int do_check_packed_object_crc; @@@ -788,15 -804,15 +802,15 @@@ struct object_context }; extern int get_sha1(const char *str, unsigned char *sha1); -extern int get_sha1_with_mode_1(const char *str, unsigned char *sha1, unsigned *mode, int gently, const char *prefix); +extern int get_sha1_with_mode_1(const char *str, unsigned char *sha1, unsigned *mode, int only_to_die, const char *prefix); static inline int get_sha1_with_mode(const char *str, unsigned char *sha1, unsigned *mode) { - return get_sha1_with_mode_1(str, sha1, mode, 1, NULL); + return get_sha1_with_mode_1(str, sha1, mode, 0, NULL); } -extern int get_sha1_with_context_1(const char *name, unsigned char *sha1, struct object_context *orc, int gently, const char *prefix); +extern int get_sha1_with_context_1(const char *name, unsigned char *sha1, struct object_context *orc, int only_to_die, const char *prefix); static inline int get_sha1_with_context(const char *str, unsigned char *sha1, struct object_context *orc) { - return get_sha1_with_context_1(str, sha1, orc, 1, NULL); + return get_sha1_with_context_1(str, sha1, orc, 0, NULL); } extern int get_sha1_hex(const char *hex, unsigned char *sha1); extern char *sha1_to_hex(const unsigned char *sha1); /* static buffer result! */ @@@ -984,7 -1000,7 +998,7 @@@ extern struct packed_git *find_sha1_pac extern void pack_report(void); extern int open_pack_index(struct packed_git *); extern void close_pack_index(struct packed_git *); - extern unsigned char *use_pack(struct packed_git *, struct pack_window **, off_t, unsigned int *); + extern unsigned char *use_pack(struct packed_git *, struct pack_window **, off_t, unsigned long *); extern void close_pack_windows(struct packed_git *); extern void unuse_pack(struct pack_window **); extern void free_pack_by_name(const char *); @@@ -996,51 -1012,11 +1010,51 @@@ extern off_t find_pack_entry_one(const extern void *unpack_entry(struct packed_git *, off_t, enum object_type *, unsigned long *); extern unsigned long unpack_object_header_buffer(const unsigned char *buf, unsigned long len, enum object_type *type, unsigned long *sizep); extern unsigned long get_size_from_delta(struct packed_git *, struct pack_window **, off_t); -extern const char *packed_object_info_detail(struct packed_git *, off_t, unsigned long *, unsigned long *, unsigned int *, unsigned char *); +extern int packed_object_info_detail(struct packed_git *, off_t, unsigned long *, unsigned long *, unsigned int *, unsigned char *); +extern int unpack_object_header(struct packed_git *, struct pack_window **, off_t *, unsigned long *); + +struct object_info { + /* Request */ + unsigned long *sizep; + + /* Response */ + enum { + OI_CACHED, + OI_LOOSE, + OI_PACKED, + OI_DBCACHED + } whence; + union { + /* + * struct { + * ... Nothing to expose in this case + * } cached; + * struct { + * ... Nothing to expose in this case + * } loose; + */ + struct { + struct packed_git *pack; + off_t offset; + unsigned int is_delta; + } packed; + } u; +}; +extern int sha1_object_info_extended(const unsigned char *, struct object_info *); /* Dumb servers support */ extern int update_server_info(int); +/* git_config_parse_key() returns these negated: */ +#define CONFIG_INVALID_KEY 1 +#define CONFIG_NO_SECTION_OR_NAME 2 +/* git_config_set(), git_config_set_multivar() return the above or these: */ +#define CONFIG_NO_LOCK -1 +#define CONFIG_INVALID_FILE 3 +#define CONFIG_NO_WRITE 4 +#define CONFIG_NOTHING_SET 5 +#define CONFIG_INVALID_PATTERN 6 + typedef int (*config_fn_t)(const char *, const char *, void *); extern int git_default_config(const char *, const char *, void *); extern int git_config_from_file(config_fn_t fn, const char *, void *); @@@ -1134,6 -1110,13 +1148,6 @@@ extern void trace_strbuf(const char *ke void packet_trace_identity(const char *prog); -/* convert.c */ -/* returns 1 if *dst was used */ -extern int convert_to_git(const char *path, const char *src, size_t len, - struct strbuf *dst, enum safe_crlf checksafe); -extern int convert_to_working_tree(const char *path, const char *src, size_t len, struct strbuf *dst); -extern int renormalize_buffer(const char *path, const char *src, size_t len, struct strbuf *dst); - /* add */ /* * return 0 if success, 1 - if addition of a file failed and diff --combined diff.c index 918c70ca45,431873d43f..93ef9a265c --- a/diff.c +++ b/diff.c @@@ -31,7 -31,6 +31,7 @@@ static const char *external_diff_cmd_cf int diff_auto_refresh_index = 1; static int diff_mnemonic_prefix; static int diff_no_prefix; +static int diff_dirstat_permille_default = 30; static struct diff_options default_diff_options; static char diff_colors[][COLOR_MAXLEN] = { @@@ -67,58 -66,6 +67,58 @@@ static int parse_diff_color_slot(const return -1; } +static int parse_dirstat_params(struct diff_options *options, const char *params, + struct strbuf *errmsg) +{ + const char *p = params; + int p_len, ret = 0; + + while (*p) { + p_len = strchrnul(p, ',') - p; + if (!memcmp(p, "changes", p_len)) { + DIFF_OPT_CLR(options, DIRSTAT_BY_LINE); + DIFF_OPT_CLR(options, DIRSTAT_BY_FILE); + } else if (!memcmp(p, "lines", p_len)) { + DIFF_OPT_SET(options, DIRSTAT_BY_LINE); + DIFF_OPT_CLR(options, DIRSTAT_BY_FILE); + } else if (!memcmp(p, "files", p_len)) { + DIFF_OPT_CLR(options, DIRSTAT_BY_LINE); + DIFF_OPT_SET(options, DIRSTAT_BY_FILE); + } else if (!memcmp(p, "noncumulative", p_len)) { + DIFF_OPT_CLR(options, DIRSTAT_CUMULATIVE); + } else if (!memcmp(p, "cumulative", p_len)) { + DIFF_OPT_SET(options, DIRSTAT_CUMULATIVE); + } else if (isdigit(*p)) { + char *end; + int permille = strtoul(p, &end, 10) * 10; + if (*end == '.' && isdigit(*++end)) { + /* only use first digit */ + permille += *end - '0'; + /* .. and ignore any further digits */ + while (isdigit(*++end)) + ; /* nothing */ + } + if (end - p == p_len) + options->dirstat_permille = permille; + else { + strbuf_addf(errmsg, _(" Failed to parse dirstat cut-off percentage '%.*s'\n"), + p_len, p); + ret++; + } + } else { + strbuf_addf(errmsg, _(" Unknown dirstat parameter '%.*s'\n"), + p_len, p); + ret++; + } + + p += p_len; + + if (*p) + p++; /* more parameters, swallow separator */ + } + return ret; +} + static int git_config_rename(const char *var, const char *value) { if (!value) @@@ -198,17 -145,6 +198,17 @@@ int git_diff_basic_config(const char *v return 0; } + if (!strcmp(var, "diff.dirstat")) { + struct strbuf errmsg = STRBUF_INIT; + default_diff_options.dirstat_permille = diff_dirstat_permille_default; + if (parse_dirstat_params(&default_diff_options, value, &errmsg)) + warning(_("Found errors in 'diff.dirstat' config variable:\n%s"), + errmsg.buf); + strbuf_release(&errmsg); + diff_dirstat_permille_default = default_diff_options.dirstat_permille; + return 0; + } + if (!prefixcmp(var, "submodule.")) return parse_submodule_config_option(var, value); @@@ -645,14 -581,11 +645,14 @@@ static void emit_rewrite_diff(const cha line_prefix, metainfo, a_name.buf, name_a_tab, reset, line_prefix, metainfo, b_name.buf, name_b_tab, reset, line_prefix, fraginfo); - print_line_count(o->file, lc_a); + if (!o->irreversible_delete) + print_line_count(o->file, lc_a); + else + fprintf(o->file, "?,?"); fprintf(o->file, " +"); print_line_count(o->file, lc_b); fprintf(o->file, " @@%s\n", reset); - if (lc_a) + if (lc_a && !o->irreversible_delete) emit_rewrite_lines(&ecbdata, '-', data_one, size_one); if (lc_b) emit_rewrite_lines(&ecbdata, '+', data_two, size_two); @@@ -1316,10 -1249,9 +1316,10 @@@ static void show_stats(struct diffstat_ int i, len, add, del, adds = 0, dels = 0; uintmax_t max_change = 0, max_len = 0; int total_files = data->nr; - int width, name_width; + int width, name_width, count; const char *reset, *add_c, *del_c; const char *line_prefix = ""; + int extra_shown = 0; struct strbuf *msg = NULL; if (data->nr == 0) @@@ -1332,7 -1264,6 +1332,7 @@@ width = options->stat_width ? options->stat_width : 80; name_width = options->stat_name_width ? options->stat_name_width : 50; + count = options->stat_count ? options->stat_count : data->nr; /* Sanity: give at least 5 columns to the graph, * but leave at least 10 columns for the name. @@@ -1349,14 -1280,9 +1349,14 @@@ add_c = diff_get_color_opt(options, DIFF_FILE_NEW); del_c = diff_get_color_opt(options, DIFF_FILE_OLD); - for (i = 0; i < data->nr; i++) { + for (i = 0; (i < count) && (i < data->nr); i++) { struct diffstat_file *file = data->files[i]; uintmax_t change = file->added + file->deleted; + if (!data->files[i]->is_renamed && + (change == 0)) { + count++; /* not shown == room for one more */ + continue; + } fill_print_name(file); len = strlen(file->print_name); if (max_len < len) @@@ -1367,7 -1293,6 +1367,7 @@@ if (max_change < change) max_change = change; } + count = i; /* min(count, data->nr) */ /* Compute the width of the graph part; * 10 is for one blank at the beginning of the line plus @@@ -1382,18 -1307,13 +1382,18 @@@ else width = max_change; - for (i = 0; i < data->nr; i++) { + for (i = 0; i < count; i++) { const char *prefix = ""; char *name = data->files[i]->print_name; uintmax_t added = data->files[i]->added; uintmax_t deleted = data->files[i]->deleted; int name_len; + if (!data->files[i]->is_renamed && + (added + deleted == 0)) { + total_files--; + continue; + } /* * "scale" the filename */ @@@ -1428,6 -1348,11 +1428,6 @@@ fprintf(options->file, " Unmerged\n"); continue; } - else if (!data->files[i]->is_renamed && - (added + deleted == 0)) { - total_files--; - continue; - } /* * scale the add/delete @@@ -1449,20 -1374,6 +1449,20 @@@ show_graph(options->file, '-', del, del_c, reset); fprintf(options->file, "\n"); } + for (i = count; i < data->nr; i++) { + uintmax_t added = data->files[i]->added; + uintmax_t deleted = data->files[i]->deleted; + if (!data->files[i]->is_renamed && + (added + deleted == 0)) { + total_files--; + continue; + } + adds += added; + dels += deleted; + if (!extra_shown) + fprintf(options->file, "%s ...\n", line_prefix); + extra_shown = 1; + } fprintf(options->file, "%s", line_prefix); fprintf(options->file, " %d files changed, %d insertions(+), %d deletions(-)\n", @@@ -1549,7 -1460,7 +1549,7 @@@ struct dirstat_file struct dirstat_dir { struct dirstat_file *files; - int alloc, nr, percent, cumulative; + int alloc, nr, permille, cumulative; }; static long gather_dirstat(struct diff_options *opt, struct dirstat_dir *dir, @@@ -1596,11 -1507,12 +1596,11 @@@ * under this directory (sources == 1). */ if (baselen && sources != 1) { - int permille = this_dir * 1000 / changed; - if (permille) { - int percent = permille / 10; - if (percent >= dir->percent) { + if (this_dir) { + int permille = this_dir * 1000 / changed; + if (permille >= dir->permille) { fprintf(opt->file, "%s%4d.%01d%% %.*s\n", line_prefix, - percent, permille % 10, baselen, base); + permille / 10, permille % 10, baselen, base); if (!dir->cumulative) return 0; } @@@ -1626,7 -1538,7 +1626,7 @@@ static void show_dirstat(struct diff_op dir.files = NULL; dir.alloc = 0; dir.nr = 0; - dir.percent = options->dirstat_percent; + dir.permille = options->dirstat_permille; dir.cumulative = DIFF_OPT_TST(options, DIRSTAT_CUMULATIVE); changed = 0; @@@ -1715,50 -1627,6 +1715,50 @@@ found_damage gather_dirstat(options, &dir, changed, "", 0); } +static void show_dirstat_by_line(struct diffstat_t *data, struct diff_options *options) +{ + int i; + unsigned long changed; + struct dirstat_dir dir; + + if (data->nr == 0) + return; + + dir.files = NULL; + dir.alloc = 0; + dir.nr = 0; + dir.permille = options->dirstat_permille; + dir.cumulative = DIFF_OPT_TST(options, DIRSTAT_CUMULATIVE); + + changed = 0; + for (i = 0; i < data->nr; i++) { + struct diffstat_file *file = data->files[i]; + unsigned long damage = file->added + file->deleted; + if (file->is_binary) + /* + * binary files counts bytes, not lines. Must find some + * way to normalize binary bytes vs. textual lines. + * The following heuristic assumes that there are 64 + * bytes per "line". + * This is stupid and ugly, but very cheap... + */ + damage = (damage + 63) / 64; + ALLOC_GROW(dir.files, dir.nr + 1, dir.alloc); + dir.files[dir.nr].name = file->name; + dir.files[dir.nr].changed = damage; + changed += damage; + dir.nr++; + } + + /* This can happen even with many files, if everything was renames */ + if (!changed) + return; + + /* Show all directories with more than x% of the changes */ + qsort(dir.files, dir.nr, sizeof(dir.files[0]), dirstat_compare); + gather_dirstat(options, &dir, changed, "", 0); +} + static void free_diffstat_info(struct diffstat_t *diffstat) { int i; @@@ -1861,20 -1729,20 +1861,20 @@@ static unsigned char *deflate_it(char * { int bound; unsigned char *deflated; - z_stream stream; + git_zstream stream; memset(&stream, 0, sizeof(stream)); - deflateInit(&stream, zlib_compression_level); - bound = deflateBound(&stream, size); + git_deflate_init(&stream, zlib_compression_level); + bound = git_deflate_bound(&stream, size); deflated = xmalloc(bound); stream.next_out = deflated; stream.avail_out = bound; stream.next_in = (unsigned char *)data; stream.avail_in = size; - while (deflate(&stream, Z_FINISH) == Z_OK) + while (git_deflate(&stream, Z_FINISH) == Z_OK) ; /* nothing */ - deflateEnd(&stream); + git_deflate_end(&stream); *result_size = stream.total_out; return deflated; } @@@ -2006,7 -1874,19 +2006,7 @@@ struct userdiff_driver *get_textconv(st return NULL; diff_filespec_load_driver(one); - if (!one->driver->textconv) - return NULL; - - if (one->driver->textconv_want_cache && !one->driver->textconv_cache) { - struct notes_cache *c = xmalloc(sizeof(*c)); - struct strbuf name = STRBUF_INIT; - - strbuf_addf(&name, "textconv/%s", one->driver->name); - notes_cache_init(c, name.buf, one->driver->textconv); - one->driver->textconv_cache = c; - } - - return one->driver; + return userdiff_get_textconv(one->driver); } static void builtin_diff(const char *name_a, @@@ -2109,11 -1989,7 +2109,11 @@@ } } - if (!DIFF_OPT_TST(o, TEXT) && + if (o->irreversible_delete && lbl[1][0] == '/') { + fprintf(o->file, "%s", header.buf); + strbuf_reset(&header); + goto free_ab_and_return; + } else if (!DIFF_OPT_TST(o, TEXT) && ( (!textconv_one && diff_filespec_is_binary(one)) || (!textconv_two && diff_filespec_is_binary(two)) )) { if (fill_mmfile(&mf1, one) < 0 || fill_mmfile(&mf2, two) < 0) @@@ -2133,7 -2009,8 +2133,7 @@@ fprintf(o->file, "%sBinary files %s and %s differ\n", line_prefix, lbl[0], lbl[1]); o->found_changes = 1; - } - else { + } else { /* Crazy xdl interfaces.. */ const char *diffopts = getenv("GIT_DIFF_OPTS"); xpparam_t xpp; @@@ -3016,7 -2893,7 +3016,7 @@@ void diff_setup(struct diff_options *op options->line_termination = '\n'; options->break_opt = -1; options->rename_limit = -1; - options->dirstat_percent = 3; + options->dirstat_permille = diff_dirstat_permille_default; options->context = 3; options->change = diff_change; @@@ -3230,7 -3107,6 +3230,7 @@@ static int stat_opt(struct diff_option char *end; int width = options->stat_width; int name_width = options->stat_name_width; + int count = options->stat_count; int argcount = 1; arg += strlen("--stat"); @@@ -3258,24 -3134,12 +3258,24 @@@ name_width = strtoul(av[1], &end, 10); argcount = 2; } + } else if (!prefixcmp(arg, "-count")) { + arg += strlen("-count"); + if (*arg == '=') + count = strtoul(arg + 1, &end, 10); + else if (!*arg && !av[1]) + die("Option '--stat-count' requires a value"); + else if (!*arg) { + count = strtoul(av[1], &end, 10); + argcount = 2; + } } break; case '=': width = strtoul(arg+1, &end, 10); if (*end == ',') name_width = strtoul(end+1, &end, 10); + if (*end == ',') + count = strtoul(end+1, &end, 10); } /* Important! This checks all the error cases! */ @@@ -3284,25 -3148,9 +3284,25 @@@ options->output_format |= DIFF_FORMAT_DIFFSTAT; options->stat_name_width = name_width; options->stat_width = width; + options->stat_count = count; return argcount; } +static int parse_dirstat_opt(struct diff_options *options, const char *params) +{ + struct strbuf errmsg = STRBUF_INIT; + if (parse_dirstat_params(options, params, &errmsg)) + die(_("Failed to parse --dirstat/-X option parameter:\n%s"), + errmsg.buf); + strbuf_release(&errmsg); + /* + * The caller knows a dirstat-related option is given from the command + * line; allow it to say "return this_function();" + */ + options->output_format |= DIFF_FORMAT_DIRSTAT; + return 1; +} + int diff_opt_parse(struct diff_options *options, const char **av, int ac) { const char *arg = av[0]; @@@ -3322,19 -3170,15 +3322,19 @@@ options->output_format |= DIFF_FORMAT_NUMSTAT; else if (!strcmp(arg, "--shortstat")) options->output_format |= DIFF_FORMAT_SHORTSTAT; - else if (opt_arg(arg, 'X', "dirstat", &options->dirstat_percent)) - options->output_format |= DIFF_FORMAT_DIRSTAT; - else if (!strcmp(arg, "--cumulative")) { - options->output_format |= DIFF_FORMAT_DIRSTAT; - DIFF_OPT_SET(options, DIRSTAT_CUMULATIVE); - } else if (opt_arg(arg, 0, "dirstat-by-file", - &options->dirstat_percent)) { - options->output_format |= DIFF_FORMAT_DIRSTAT; - DIFF_OPT_SET(options, DIRSTAT_BY_FILE); + else if (!strcmp(arg, "-X") || !strcmp(arg, "--dirstat")) + return parse_dirstat_opt(options, ""); + else if (!prefixcmp(arg, "-X")) + return parse_dirstat_opt(options, arg + 2); + else if (!prefixcmp(arg, "--dirstat=")) + return parse_dirstat_opt(options, arg + 10); + else if (!strcmp(arg, "--cumulative")) + return parse_dirstat_opt(options, "cumulative"); + else if (!strcmp(arg, "--dirstat-by-file")) + return parse_dirstat_opt(options, "files"); + else if (!prefixcmp(arg, "--dirstat-by-file=")) { + parse_dirstat_opt(options, "files"); + return parse_dirstat_opt(options, arg + 18); } else if (!strcmp(arg, "--check")) options->output_format |= DIFF_FORMAT_CHECKDIFF; @@@ -3349,7 -3193,7 +3349,7 @@@ else if (!strcmp(arg, "-s")) options->output_format |= DIFF_FORMAT_NO_OUTPUT; else if (!prefixcmp(arg, "--stat")) - /* --stat, --stat-width, or --stat-name-width */ + /* --stat, --stat-width, --stat-name-width, or --stat-count */ return stat_opt(options, av); /* renames options */ @@@ -3364,9 -3208,6 +3364,9 @@@ return error("invalid argument to -M: %s", arg+2); options->detect_rename = DIFF_DETECT_RENAME; } + else if (!strcmp(arg, "-D") || !strcmp(arg, "--irreversible-delete")) { + options->irreversible_delete = 1; + } else if (!prefixcmp(arg, "-C") || !prefixcmp(arg, "--find-copies=") || !strcmp(arg, "--find-copies")) { if (options->detect_rename == DIFF_DETECT_COPY) @@@ -4181,7 -4022,6 +4181,7 @@@ void diff_flush(struct diff_options *op struct diff_queue_struct *q = &diff_queued_diff; int i, output_format = options->output_format; int separator = 0; + int dirstat_by_line = 0; /* * Order: raw, stat, summary, patch @@@ -4202,11 -4042,7 +4202,11 @@@ separator++; } - if (output_format & (DIFF_FORMAT_DIFFSTAT|DIFF_FORMAT_SHORTSTAT|DIFF_FORMAT_NUMSTAT)) { + if (output_format & DIFF_FORMAT_DIRSTAT && DIFF_OPT_TST(options, DIRSTAT_BY_LINE)) + dirstat_by_line = 1; + + if (output_format & (DIFF_FORMAT_DIFFSTAT|DIFF_FORMAT_SHORTSTAT|DIFF_FORMAT_NUMSTAT) || + dirstat_by_line) { struct diffstat_t diffstat; memset(&diffstat, 0, sizeof(struct diffstat_t)); @@@ -4221,12 -4057,10 +4221,12 @@@ show_stats(&diffstat, options); if (output_format & DIFF_FORMAT_SHORTSTAT) show_shortstats(&diffstat, options); + if (output_format & DIFF_FORMAT_DIRSTAT) + show_dirstat_by_line(&diffstat, options); free_diffstat_info(&diffstat); separator++; } - if (output_format & DIFF_FORMAT_DIRSTAT) + if ((output_format & DIFF_FORMAT_DIRSTAT) && !dirstat_by_line) show_dirstat(options); if (output_format & DIFF_FORMAT_SUMMARY && !is_summary_empty(q)) { @@@ -4480,13 -4314,6 +4480,13 @@@ int diff_result_code(struct diff_option return result; } +int diff_can_quit_early(struct diff_options *opt) +{ + return (DIFF_OPT_TST(opt, QUICK) && + !opt->filter && + DIFF_OPT_TST(opt, HAS_CHANGES)); +} + /* * Shall changes to this submodule be ignored? * diff --combined http-push.c index 28bfe768f7,1e8a830937..6e8f6d09ab --- a/http-push.c +++ b/http-push.c @@@ -169,7 -169,7 +169,7 @@@ enum dav_header_flag DAV_HEADER_TIMEOUT = (1u << 2) }; -static char *xml_entities(char *s) +static char *xml_entities(const char *s) { struct strbuf buf = STRBUF_INIT; while (*s) { @@@ -197,34 -197,6 +197,34 @@@ return strbuf_detach(&buf, NULL); } +static void curl_setup_http_get(CURL *curl, const char *url, + const char *custom_req) +{ + curl_easy_setopt(curl, CURLOPT_HTTPGET, 1); + curl_easy_setopt(curl, CURLOPT_URL, url); + curl_easy_setopt(curl, CURLOPT_CUSTOMREQUEST, custom_req); + curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, fwrite_null); +} + +static void curl_setup_http(CURL *curl, const char *url, + const char *custom_req, struct buffer *buffer, + curl_write_callback write_fn) +{ + curl_easy_setopt(curl, CURLOPT_PUT, 1); + curl_easy_setopt(curl, CURLOPT_URL, url); + curl_easy_setopt(curl, CURLOPT_INFILE, buffer); + curl_easy_setopt(curl, CURLOPT_INFILESIZE, buffer->buf.len); + curl_easy_setopt(curl, CURLOPT_READFUNCTION, fread_buffer); +#ifndef NO_CURL_IOCTL + curl_easy_setopt(curl, CURLOPT_IOCTLFUNCTION, ioctl_buffer); + curl_easy_setopt(curl, CURLOPT_IOCTLDATA, &buffer); +#endif + curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_fn); + curl_easy_setopt(curl, CURLOPT_NOBODY, 0); + curl_easy_setopt(curl, CURLOPT_CUSTOMREQUEST, custom_req); + curl_easy_setopt(curl, CURLOPT_UPLOAD, 1); +} + static struct curl_slist *get_dav_token_headers(struct remote_lock *lock, enum dav_header_flag options) { struct strbuf buf = STRBUF_INIT; @@@ -300,8 -272,11 +300,8 @@@ static void start_mkcol(struct transfer slot = get_active_slot(); slot->callback_func = process_response; slot->callback_data = request; - curl_easy_setopt(slot->curl, CURLOPT_HTTPGET, 1); /* undo PUT setup */ - curl_easy_setopt(slot->curl, CURLOPT_URL, request->url); + curl_setup_http_get(slot->curl, request->url, DAV_MKCOL); curl_easy_setopt(slot->curl, CURLOPT_ERRORBUFFER, request->errorstr); - curl_easy_setopt(slot->curl, CURLOPT_CUSTOMREQUEST, DAV_MKCOL); - curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite_null); if (start_active_slot(slot)) { request->slot = slot; @@@ -377,15 -352,15 +377,15 @@@ static void start_put(struct transfer_r unsigned long len; int hdrlen; ssize_t size; - z_stream stream; + git_zstream stream; unpacked = read_sha1_file(request->obj->sha1, &type, &len); hdrlen = sprintf(hdr, "%s %lu", typename(type), len) + 1; /* Set it up */ memset(&stream, 0, sizeof(stream)); - deflateInit(&stream, zlib_compression_level); - size = deflateBound(&stream, len + hdrlen); + git_deflate_init(&stream, zlib_compression_level); + size = git_deflate_bound(&stream, len + hdrlen); strbuf_init(&request->buffer.buf, size); request->buffer.posn = 0; @@@ -396,15 -371,15 +396,15 @@@ /* First header.. */ stream.next_in = (void *)hdr; stream.avail_in = hdrlen; - while (deflate(&stream, 0) == Z_OK) - /* nothing */; + while (git_deflate(&stream, 0) == Z_OK) + ; /* nothing */ /* Then the data itself.. */ stream.next_in = unpacked; stream.avail_in = len; - while (deflate(&stream, Z_FINISH) == Z_OK) - /* nothing */; - deflateEnd(&stream); + while (git_deflate(&stream, Z_FINISH) == Z_OK) + ; /* nothing */ + git_deflate_end(&stream); free(unpacked); request->buffer.buf.len = stream.total_out; @@@ -420,8 -395,19 +420,8 @@@ slot = get_active_slot(); slot->callback_func = process_response; slot->callback_data = request; - curl_easy_setopt(slot->curl, CURLOPT_INFILE, &request->buffer); - curl_easy_setopt(slot->curl, CURLOPT_INFILESIZE, request->buffer.buf.len); - curl_easy_setopt(slot->curl, CURLOPT_READFUNCTION, fread_buffer); -#ifndef NO_CURL_IOCTL - curl_easy_setopt(slot->curl, CURLOPT_IOCTLFUNCTION, ioctl_buffer); - curl_easy_setopt(slot->curl, CURLOPT_IOCTLDATA, &request->buffer); -#endif - curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite_null); - curl_easy_setopt(slot->curl, CURLOPT_NOBODY, 0); - curl_easy_setopt(slot->curl, CURLOPT_CUSTOMREQUEST, DAV_PUT); - curl_easy_setopt(slot->curl, CURLOPT_UPLOAD, 1); - curl_easy_setopt(slot->curl, CURLOPT_PUT, 1); - curl_easy_setopt(slot->curl, CURLOPT_URL, request->url); + curl_setup_http(slot->curl, request->url, DAV_PUT, + &request->buffer, fwrite_null); if (start_active_slot(slot)) { request->slot = slot; @@@ -441,10 -427,13 +441,10 @@@ static void start_move(struct transfer_ slot = get_active_slot(); slot->callback_func = process_response; slot->callback_data = request; - curl_easy_setopt(slot->curl, CURLOPT_HTTPGET, 1); /* undo PUT setup */ - curl_easy_setopt(slot->curl, CURLOPT_CUSTOMREQUEST, DAV_MOVE); + curl_setup_http_get(slot->curl, request->url, DAV_MOVE); dav_headers = curl_slist_append(dav_headers, request->dest); dav_headers = curl_slist_append(dav_headers, "Overwrite: T"); curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, dav_headers); - curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite_null); - curl_easy_setopt(slot->curl, CURLOPT_URL, request->url); if (start_active_slot(slot)) { request->slot = slot; @@@ -469,7 -458,10 +469,7 @@@ static int refresh_lock(struct remote_l slot = get_active_slot(); slot->results = &results; - curl_easy_setopt(slot->curl, CURLOPT_HTTPGET, 1); - curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite_null); - curl_easy_setopt(slot->curl, CURLOPT_URL, lock->url); - curl_easy_setopt(slot->curl, CURLOPT_CUSTOMREQUEST, DAV_LOCK); + curl_setup_http_get(slot->curl, lock->url, DAV_LOCK); curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, dav_headers); if (start_active_slot(slot)) { @@@ -805,7 -797,7 +805,7 @@@ static void handle_new_lock_ctx(struct } } -static void one_remote_ref(char *refname); +static void one_remote_ref(const char *refname); static void xml_start_tag(void *userData, const char *name, const char **atts) @@@ -884,7 -876,10 +884,7 @@@ static struct remote_lock *lock_remote( ep[1] = '\0'; slot = get_active_slot(); slot->results = &results; - curl_easy_setopt(slot->curl, CURLOPT_HTTPGET, 1); - curl_easy_setopt(slot->curl, CURLOPT_URL, url); - curl_easy_setopt(slot->curl, CURLOPT_CUSTOMREQUEST, DAV_MKCOL); - curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite_null); + curl_setup_http_get(slot->curl, url, DAV_MKCOL); if (start_active_slot(slot)) { run_active_slot(slot); if (results.curl_result != CURLE_OK && @@@ -914,9 -909,19 +914,9 @@@ slot = get_active_slot(); slot->results = &results; - curl_easy_setopt(slot->curl, CURLOPT_INFILE, &out_buffer); - curl_easy_setopt(slot->curl, CURLOPT_INFILESIZE, out_buffer.buf.len); - curl_easy_setopt(slot->curl, CURLOPT_READFUNCTION, fread_buffer); -#ifndef NO_CURL_IOCTL - curl_easy_setopt(slot->curl, CURLOPT_IOCTLFUNCTION, ioctl_buffer); - curl_easy_setopt(slot->curl, CURLOPT_IOCTLDATA, &out_buffer); -#endif - curl_easy_setopt(slot->curl, CURLOPT_FILE, &in_buffer); - curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite_buffer); - curl_easy_setopt(slot->curl, CURLOPT_URL, url); - curl_easy_setopt(slot->curl, CURLOPT_UPLOAD, 1); - curl_easy_setopt(slot->curl, CURLOPT_CUSTOMREQUEST, DAV_LOCK); + curl_setup_http(slot->curl, url, DAV_LOCK, &out_buffer, fwrite_buffer); curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, dav_headers); + curl_easy_setopt(slot->curl, CURLOPT_FILE, &in_buffer); lock = xcalloc(1, sizeof(*lock)); lock->timeout = -1; @@@ -982,7 -987,9 +982,7 @@@ static int unlock_remote(struct remote_ slot = get_active_slot(); slot->results = &results; - curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite_null); - curl_easy_setopt(slot->curl, CURLOPT_URL, lock->url); - curl_easy_setopt(slot->curl, CURLOPT_CUSTOMREQUEST, DAV_UNLOCK); + curl_setup_http_get(slot->curl, lock->url, DAV_UNLOCK); curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, dav_headers); if (start_active_slot(slot)) { @@@ -1160,10 -1167,19 +1160,10 @@@ static void remote_ls(const char *path slot = get_active_slot(); slot->results = &results; - curl_easy_setopt(slot->curl, CURLOPT_INFILE, &out_buffer); - curl_easy_setopt(slot->curl, CURLOPT_INFILESIZE, out_buffer.buf.len); - curl_easy_setopt(slot->curl, CURLOPT_READFUNCTION, fread_buffer); -#ifndef NO_CURL_IOCTL - curl_easy_setopt(slot->curl, CURLOPT_IOCTLFUNCTION, ioctl_buffer); - curl_easy_setopt(slot->curl, CURLOPT_IOCTLDATA, &out_buffer); -#endif - curl_easy_setopt(slot->curl, CURLOPT_FILE, &in_buffer); - curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite_buffer); - curl_easy_setopt(slot->curl, CURLOPT_URL, url); - curl_easy_setopt(slot->curl, CURLOPT_UPLOAD, 1); - curl_easy_setopt(slot->curl, CURLOPT_CUSTOMREQUEST, DAV_PROPFIND); + curl_setup_http(slot->curl, url, DAV_PROPFIND, + &out_buffer, fwrite_buffer); curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, dav_headers); + curl_easy_setopt(slot->curl, CURLOPT_FILE, &in_buffer); if (start_active_slot(slot)) { run_active_slot(slot); @@@ -1234,10 -1250,19 +1234,10 @@@ static int locking_available(void slot = get_active_slot(); slot->results = &results; - curl_easy_setopt(slot->curl, CURLOPT_INFILE, &out_buffer); - curl_easy_setopt(slot->curl, CURLOPT_INFILESIZE, out_buffer.buf.len); - curl_easy_setopt(slot->curl, CURLOPT_READFUNCTION, fread_buffer); -#ifndef NO_CURL_IOCTL - curl_easy_setopt(slot->curl, CURLOPT_IOCTLFUNCTION, ioctl_buffer); - curl_easy_setopt(slot->curl, CURLOPT_IOCTLDATA, &out_buffer); -#endif - curl_easy_setopt(slot->curl, CURLOPT_FILE, &in_buffer); - curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite_buffer); - curl_easy_setopt(slot->curl, CURLOPT_URL, repo->url); - curl_easy_setopt(slot->curl, CURLOPT_UPLOAD, 1); - curl_easy_setopt(slot->curl, CURLOPT_CUSTOMREQUEST, DAV_PROPFIND); + curl_setup_http(slot->curl, repo->url, DAV_PROPFIND, + &out_buffer, fwrite_buffer); curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, dav_headers); + curl_easy_setopt(slot->curl, CURLOPT_FILE, &in_buffer); if (start_active_slot(slot)) { run_active_slot(slot); @@@ -1411,9 -1436,19 +1411,9 @@@ static int update_remote(unsigned char slot = get_active_slot(); slot->results = &results; - curl_easy_setopt(slot->curl, CURLOPT_INFILE, &out_buffer); - curl_easy_setopt(slot->curl, CURLOPT_INFILESIZE, out_buffer.buf.len); - curl_easy_setopt(slot->curl, CURLOPT_READFUNCTION, fread_buffer); -#ifndef NO_CURL_IOCTL - curl_easy_setopt(slot->curl, CURLOPT_IOCTLFUNCTION, ioctl_buffer); - curl_easy_setopt(slot->curl, CURLOPT_IOCTLDATA, &out_buffer); -#endif - curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite_null); - curl_easy_setopt(slot->curl, CURLOPT_CUSTOMREQUEST, DAV_PUT); + curl_setup_http(slot->curl, lock->url, DAV_PUT, + &out_buffer, fwrite_null); curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, dav_headers); - curl_easy_setopt(slot->curl, CURLOPT_UPLOAD, 1); - curl_easy_setopt(slot->curl, CURLOPT_PUT, 1); - curl_easy_setopt(slot->curl, CURLOPT_URL, lock->url); if (start_active_slot(slot)) { run_active_slot(slot); @@@ -1436,7 -1471,7 +1436,7 @@@ static struct ref *remote_refs; -static void one_remote_ref(char *refname) +static void one_remote_ref(const char *refname) { struct ref *ref; struct object *obj; @@@ -1537,9 -1572,19 +1537,9 @@@ static void update_remote_info_refs(str slot = get_active_slot(); slot->results = &results; - curl_easy_setopt(slot->curl, CURLOPT_INFILE, &buffer); - curl_easy_setopt(slot->curl, CURLOPT_INFILESIZE, buffer.buf.len); - curl_easy_setopt(slot->curl, CURLOPT_READFUNCTION, fread_buffer); -#ifndef NO_CURL_IOCTL - curl_easy_setopt(slot->curl, CURLOPT_IOCTLFUNCTION, ioctl_buffer); - curl_easy_setopt(slot->curl, CURLOPT_IOCTLDATA, &buffer); -#endif - curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite_null); - curl_easy_setopt(slot->curl, CURLOPT_CUSTOMREQUEST, DAV_PUT); + curl_setup_http(slot->curl, lock->url, DAV_PUT, + &buffer, fwrite_null); curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, dav_headers); - curl_easy_setopt(slot->curl, CURLOPT_UPLOAD, 1); - curl_easy_setopt(slot->curl, CURLOPT_PUT, 1); - curl_easy_setopt(slot->curl, CURLOPT_URL, lock->url); if (start_active_slot(slot)) { run_active_slot(slot); @@@ -1615,7 -1660,7 +1615,7 @@@ static int verify_merge_base(unsigned c return (merge_bases && !merge_bases->next && merge_bases->item == branch); } -static int delete_remote_branch(char *pattern, int force) +static int delete_remote_branch(const char *pattern, int force) { struct ref *refs = remote_refs; struct ref *remote_ref = NULL; @@@ -1697,7 -1742,10 +1697,7 @@@ sprintf(url, "%s%s", repo->url, remote_ref->name); slot = get_active_slot(); slot->results = &results; - curl_easy_setopt(slot->curl, CURLOPT_HTTPGET, 1); - curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite_null); - curl_easy_setopt(slot->curl, CURLOPT_URL, url); - curl_easy_setopt(slot->curl, CURLOPT_CUSTOMREQUEST, DAV_DELETE); + curl_setup_http_get(slot->curl, url, DAV_DELETE); if (start_active_slot(slot)) { run_active_slot(slot); free(url); diff --combined http.h index 19b7134fa5,a39304a7ab..0bf8592dc4 --- a/http.h +++ b/http.h @@@ -66,9 -66,9 +66,9 @@@ struct buffer }; /* Curl request read/write callbacks */ -extern size_t fread_buffer(void *ptr, size_t eltsize, size_t nmemb, void *strbuf); -extern size_t fwrite_buffer(const void *ptr, size_t eltsize, size_t nmemb, void *strbuf); -extern size_t fwrite_null(const void *ptr, size_t eltsize, size_t nmemb, void *strbuf); +extern size_t fread_buffer(char *ptr, size_t eltsize, size_t nmemb, void *strbuf); +extern size_t fwrite_buffer(char *ptr, size_t eltsize, size_t nmemb, void *strbuf); +extern size_t fwrite_null(char *ptr, size_t eltsize, size_t nmemb, void *strbuf); #ifndef NO_CURL_IOCTL extern curlioerr ioctl_buffer(CURL *handle, int cmd, void *clientp); #endif @@@ -172,7 -172,7 +172,7 @@@ struct http_object_request unsigned char sha1[20]; unsigned char real_sha1[20]; git_SHA_CTX c; - z_stream stream; + git_zstream stream; int zret; int rename; struct active_request_slot *slot; diff --combined remote-curl.c index b5be25ce96,bc48a36ef5..faaeda44a9 --- a/remote-curl.c +++ b/remote-curl.c @@@ -347,7 -347,7 +347,7 @@@ static curlioerr rpc_ioctl(CURL *handle } #endif -static size_t rpc_in(const void *ptr, size_t eltsize, +static size_t rpc_in(char *ptr, size_t eltsize, size_t nmemb, void *buffer_) { size_t size = eltsize * nmemb; @@@ -471,16 -471,12 +471,12 @@@ static int post_rpc(struct rpc_state *r * the transfer time. */ size_t size; - z_stream stream; + git_zstream stream; int ret; memset(&stream, 0, sizeof(stream)); - ret = deflateInit2(&stream, Z_BEST_COMPRESSION, - Z_DEFLATED, (15 + 16), - 8, Z_DEFAULT_STRATEGY); - if (ret != Z_OK) - die("cannot deflate request; zlib init error %d", ret); - size = deflateBound(&stream, rpc->len); + git_deflate_init_gzip(&stream, Z_BEST_COMPRESSION); + size = git_deflate_bound(&stream, rpc->len); gzip_body = xmalloc(size); stream.next_in = (unsigned char *)rpc->buf; @@@ -488,11 -484,11 +484,11 @@@ stream.next_out = (unsigned char *)gzip_body; stream.avail_out = size; - ret = deflate(&stream, Z_FINISH); + ret = git_deflate(&stream, Z_FINISH); if (ret != Z_STREAM_END) die("cannot deflate request; zlib deflate error %d", ret); - ret = deflateEnd(&stream); + ret = git_deflate_end_gently(&stream); if (ret != Z_OK) die("cannot deflate request; zlib end error %d", ret); @@@ -811,21 -807,19 +807,21 @@@ static void parse_push(struct strbuf *b strbuf_reset(buf); if (strbuf_getline(buf, stdin, '\n') == EOF) - return; + goto free_specs; if (!*buf->buf) break; } while (1); if (push(nr_spec, specs)) exit(128); /* error already reported */ - for (i = 0; i < nr_spec; i++) - free(specs[i]); - free(specs); printf("\n"); fflush(stdout); + + free_specs: + for (i = 0; i < nr_spec; i++) + free(specs[i]); + free(specs); } int main(int argc, const char **argv) diff --combined sha1_file.c index bd6fb2f95d,94d431907c..a6aac70923 --- a/sha1_file.c +++ b/sha1_file.c @@@ -11,7 -11,6 +11,7 @@@ #include "pack.h" #include "blob.h" #include "commit.h" +#include "run-command.h" #include "tag.h" #include "tree.h" #include "tree-walk.h" @@@ -834,7 -833,7 +834,7 @@@ static int in_window(struct pack_windo unsigned char *use_pack(struct packed_git *p, struct pack_window **w_cursor, off_t offset, - unsigned int *left) + unsigned long *left) { struct pack_window *win = *w_cursor; @@@ -1186,7 -1185,7 +1186,7 @@@ static int open_sha1_file(const unsigne return -1; } -static void *map_sha1_file(const unsigned char *sha1, unsigned long *size) +void *map_sha1_file(const unsigned char *sha1, unsigned long *size) { void *map; int fd; @@@ -1205,29 -1204,20 +1205,29 @@@ return map; } -static int legacy_loose_object(unsigned char *map) +/* + * There used to be a second loose object header format which + * was meant to mimic the in-pack format, allowing for direct + * copy of the object data. This format turned up not to be + * really worth it and we no longer write loose objects in that + * format. + */ +static int experimental_loose_object(unsigned char *map) { unsigned int word; /* * Is it a zlib-compressed buffer? If so, the first byte * must be 0x78 (15-bit window size, deflated), and the - * first 16-bit word is evenly divisible by 31 + * first 16-bit word is evenly divisible by 31. If so, + * we are looking at the official format, not the experimental + * one. */ word = (map[0] << 8) + map[1]; if (map[0] == 0x78 && !(word % 31)) - return 1; - else return 0; + else + return 1; } unsigned long unpack_object_header_buffer(const unsigned char *buf, @@@ -1254,7 -1244,7 +1254,7 @@@ return used; } - int unpack_sha1_header(z_stream *stream, unsigned char *map, unsigned long mapsize, void *buffer, unsigned long bufsiz) -static int unpack_sha1_header(git_zstream *stream, unsigned char *map, unsigned long mapsize, void *buffer, unsigned long bufsiz) ++int unpack_sha1_header(git_zstream *stream, unsigned char *map, unsigned long mapsize, void *buffer, unsigned long bufsiz) { unsigned long size, used; static const char valid_loose_object_type[8] = { @@@ -1271,32 -1261,37 +1271,32 @@@ stream->next_out = buffer; stream->avail_out = bufsiz; - if (legacy_loose_object(map)) { - git_inflate_init(stream); - return git_inflate(stream, 0); - } - + if (experimental_loose_object(map)) { + /* + * The old experimental format we no longer produce; + * we can still read it. + */ + used = unpack_object_header_buffer(map, mapsize, &type, &size); + if (!used || !valid_loose_object_type[type]) + return -1; + map += used; + mapsize -= used; - /* - * There used to be a second loose object header format which - * was meant to mimic the in-pack format, allowing for direct - * copy of the object data. This format turned up not to be - * really worth it and we don't write it any longer. But we - * can still read it. - */ - used = unpack_object_header_buffer(map, mapsize, &type, &size); - if (!used || !valid_loose_object_type[type]) - return -1; - map += used; - mapsize -= used; + /* Set up the stream for the rest.. */ + stream->next_in = map; + stream->avail_in = mapsize; + git_inflate_init(stream); - /* Set up the stream for the rest.. */ - stream->next_in = map; - stream->avail_in = mapsize; + /* And generate the fake traditional header */ + stream->total_out = 1 + snprintf(buffer, bufsiz, "%s %lu", + typename(type), size); + return 0; + } git_inflate_init(stream); - - /* And generate the fake traditional header */ - stream->total_out = 1 + snprintf(buffer, bufsiz, "%s %lu", - typename(type), size); - return 0; + return git_inflate(stream, 0); } - static void *unpack_sha1_rest(z_stream *stream, void *buffer, unsigned long size, const unsigned char *sha1) + static void *unpack_sha1_rest(git_zstream *stream, void *buffer, unsigned long size, const unsigned char *sha1) { int bytes = strlen(buffer) + 1; unsigned char *buf = xmallocz(size); @@@ -1346,7 -1341,7 +1346,7 @@@ * too permissive for what we want to check. So do an anal * object header parse by hand. */ -static int parse_sha1_header(const char *hdr, unsigned long *sizep) +int parse_sha1_header(const char *hdr, unsigned long *sizep) { char type[10]; int i; @@@ -1395,7 -1390,7 +1395,7 @@@ static void *unpack_sha1_file(void *map, unsigned long mapsize, enum object_type *type, unsigned long *size, const unsigned char *sha1) { int ret; - z_stream stream; + git_zstream stream; char hdr[8192]; ret = unpack_sha1_header(&stream, map, mapsize, hdr, sizeof(hdr)); @@@ -1411,7 -1406,7 +1411,7 @@@ unsigned long get_size_from_delta(struc { const unsigned char *data; unsigned char delta_head[20], *in; - z_stream stream; + git_zstream stream; int st; memset(&stream, 0, sizeof(stream)); @@@ -1485,7 -1480,7 +1485,7 @@@ static off_t get_delta_base(struct pack /* forward declaration for a mutually recursive function */ static int packed_object_info(struct packed_git *p, off_t offset, - unsigned long *sizep); + unsigned long *sizep, int *rtype); static int packed_delta_info(struct packed_git *p, struct pack_window **w_curs, @@@ -1499,7 -1494,7 +1499,7 @@@ base_offset = get_delta_base(p, w_curs, &curpos, type, obj_offset); if (!base_offset) return OBJ_BAD; - type = packed_object_info(p, base_offset, NULL); + type = packed_object_info(p, base_offset, NULL, NULL); if (type <= OBJ_NONE) { struct revindex_entry *revidx; const unsigned char *base_sha1; @@@ -1527,13 -1522,13 +1527,13 @@@ return type; } -static int unpack_object_header(struct packed_git *p, - struct pack_window **w_curs, - off_t *curpos, - unsigned long *sizep) +int unpack_object_header(struct packed_git *p, + struct pack_window **w_curs, + off_t *curpos, + unsigned long *sizep) { unsigned char *base; - unsigned int left; + unsigned long left; unsigned long used; enum object_type type; @@@ -1553,7 -1548,7 +1553,7 @@@ return type; } -const char *packed_object_info_detail(struct packed_git *p, +int packed_object_info_detail(struct packed_git *p, off_t obj_offset, unsigned long *size, unsigned long *store_size, @@@ -1584,7 -1579,7 +1584,7 @@@ case OBJ_BLOB: case OBJ_TAG: unuse_pack(&w_curs); - return typename(type); + return type; case OBJ_OFS_DELTA: obj_offset = get_delta_base(p, &w_curs, &curpos, type, obj_offset); if (!obj_offset) @@@ -1609,7 -1604,7 +1609,7 @@@ } static int packed_object_info(struct packed_git *p, off_t obj_offset, - unsigned long *sizep) + unsigned long *sizep, int *rtype) { struct pack_window *w_curs = NULL; unsigned long size; @@@ -1617,8 -1612,6 +1617,8 @@@ enum object_type type; type = unpack_object_header(p, &w_curs, &curpos, &size); + if (rtype) + *rtype = type; /* representation type */ switch (type) { case OBJ_OFS_DELTA: @@@ -1648,7 -1641,7 +1648,7 @@@ static void *unpack_compressed_entry(st unsigned long size) { int st; - z_stream stream; + git_zstream stream; unsigned char *buffer, *in; buffer = xmallocz(size); @@@ -1701,13 -1694,6 +1701,13 @@@ static unsigned long pack_entry_hash(st return hash % MAX_DELTA_CACHE; } +static int in_delta_base_cache(struct packed_git *p, off_t base_offset) +{ + unsigned long hash = pack_entry_hash(p, base_offset); + struct delta_base_cache_entry *ent = delta_base_cache + hash; + return (ent->data && ent->p == p && ent->base_offset == base_offset); +} + static void *cache_or_unpack_entry(struct packed_git *p, off_t base_offset, unsigned long *base_size, enum object_type *type, int keep_cache) { @@@ -2088,7 -2074,7 +2088,7 @@@ static int sha1_loose_object_info(cons int status; unsigned long mapsize, size; void *map; - z_stream stream; + git_zstream stream; char hdr[32]; map = map_sha1_file(sha1, &mapsize); @@@ -2106,28 -2092,24 +2106,28 @@@ return status; } -int sha1_object_info(const unsigned char *sha1, unsigned long *sizep) +/* returns enum object_type or negative */ +int sha1_object_info_extended(const unsigned char *sha1, struct object_info *oi) { struct cached_object *co; struct pack_entry e; - int status; + int status, rtype; co = find_cached_object(sha1); if (co) { - if (sizep) - *sizep = co->size; + if (oi->sizep) + *(oi->sizep) = co->size; + oi->whence = OI_CACHED; return co->type; } if (!find_pack_entry(sha1, &e)) { /* Most likely it's a loose object. */ - status = sha1_loose_object_info(sha1, sizep); - if (status >= 0) + status = sha1_loose_object_info(sha1, oi->sizep); + if (status >= 0) { + oi->whence = OI_LOOSE; return status; + } /* Not a loose object; someone else may have just packed it. */ reprepare_packed_git(); @@@ -2135,31 -2117,15 +2135,31 @@@ return status; } - status = packed_object_info(e.p, e.offset, sizep); + status = packed_object_info(e.p, e.offset, oi->sizep, &rtype); if (status < 0) { mark_bad_packed_object(e.p, sha1); - status = sha1_object_info(sha1, sizep); + status = sha1_object_info_extended(sha1, oi); + } else if (in_delta_base_cache(e.p, e.offset)) { + oi->whence = OI_DBCACHED; + } else { + oi->whence = OI_PACKED; + oi->u.packed.offset = e.offset; + oi->u.packed.pack = e.p; + oi->u.packed.is_delta = (rtype == OBJ_REF_DELTA || + rtype == OBJ_OFS_DELTA); } return status; } +int sha1_object_info(const unsigned char *sha1, unsigned long *sizep) +{ + struct object_info oi; + + oi.sizep = sizep; + return sha1_object_info_extended(sha1, &oi); +} + static void *read_packed_sha1(const unsigned char *sha1, enum object_type *type, unsigned long *size) { @@@ -2239,21 -2205,23 +2239,21 @@@ static void *read_object(const unsigne * deal with them should arrange to call read_object() and give error * messages themselves. */ -void *read_sha1_file_repl(const unsigned char *sha1, - enum object_type *type, - unsigned long *size, - const unsigned char **replacement) +void *read_sha1_file_extended(const unsigned char *sha1, + enum object_type *type, + unsigned long *size, + unsigned flag) { - const unsigned char *repl = lookup_replace_object(sha1); void *data; char *path; const struct packed_git *p; + const unsigned char *repl = (flag & READ_SHA1_FILE_REPLACE) + ? lookup_replace_object(sha1) : sha1; errno = 0; data = read_object(repl, type, size); - if (data) { - if (replacement) - *replacement = repl; + if (data) return data; - } if (errno && errno != ENOENT) die_errno("failed to read object %s", sha1_to_hex(sha1)); @@@ -2457,7 -2425,7 +2457,7 @@@ static int write_loose_object(const uns { int fd, ret; unsigned char compressed[4096]; - z_stream stream; + git_zstream stream; git_SHA_CTX c; unsigned char parano_sha1[20]; char *filename; @@@ -2474,7 -2442,7 +2474,7 @@@ /* Set it up */ memset(&stream, 0, sizeof(stream)); - deflateInit(&stream, zlib_compression_level); + git_deflate_init(&stream, zlib_compression_level); stream.next_out = compressed; stream.avail_out = sizeof(compressed); git_SHA1_Init(&c); @@@ -2482,8 -2450,8 +2482,8 @@@ /* First header.. */ stream.next_in = (unsigned char *)hdr; stream.avail_in = hdrlen; - while (deflate(&stream, 0) == Z_OK) - /* nothing */; + while (git_deflate(&stream, 0) == Z_OK) + ; /* nothing */ git_SHA1_Update(&c, hdr, hdrlen); /* Then the data itself.. */ @@@ -2491,7 -2459,7 +2491,7 @@@ stream.avail_in = len; do { unsigned char *in0 = stream.next_in; - ret = deflate(&stream, Z_FINISH); + ret = git_deflate(&stream, Z_FINISH); git_SHA1_Update(&c, in0, stream.next_in - in0); if (write_buffer(fd, compressed, stream.next_out - compressed) < 0) die("unable to write sha1 file"); @@@ -2501,7 -2469,7 +2501,7 @@@ if (ret != Z_STREAM_END) die("unable to deflate new object %s (%d)", sha1_to_hex(sha1), ret); - ret = deflateEnd(&stream); + ret = git_deflate_end_gently(&stream); if (ret != Z_OK) die("deflateEnd on object %s failed (%d)", sha1_to_hex(sha1), ret); git_SHA1_Final(parano_sha1, &c); @@@ -2612,11 -2580,10 +2612,11 @@@ static void check_tag(const void *buf, } static int index_mem(unsigned char *sha1, void *buf, size_t size, - int write_object, enum object_type type, - const char *path, int format_check) + enum object_type type, + const char *path, unsigned flags) { int ret, re_allocated = 0; + int write_object = flags & HASH_WRITE_OBJECT; if (!type) type = OBJ_BLOB; @@@ -2632,7 -2599,7 +2632,7 @@@ re_allocated = 1; } } - if (format_check) { + if (flags & HASH_FORMAT_CHECK) { if (type == OBJ_TREE) check_tree(buf, size); if (type == OBJ_COMMIT) @@@ -2650,141 -2617,44 +2650,141 @@@ return ret; } +static int index_pipe(unsigned char *sha1, int fd, enum object_type type, + const char *path, unsigned flags) +{ + struct strbuf sbuf = STRBUF_INIT; + int ret; + + if (strbuf_read(&sbuf, fd, 4096) >= 0) + ret = index_mem(sha1, sbuf.buf, sbuf.len, type, path, flags); + else + ret = -1; + strbuf_release(&sbuf); + return ret; +} + #define SMALL_FILE_SIZE (32*1024) -int index_fd(unsigned char *sha1, int fd, struct stat *st, int write_object, - enum object_type type, const char *path, int format_check) +static int index_core(unsigned char *sha1, int fd, size_t size, + enum object_type type, const char *path, + unsigned flags) { int ret; - size_t size = xsize_t(st->st_size); - if (!S_ISREG(st->st_mode)) { - struct strbuf sbuf = STRBUF_INIT; - if (strbuf_read(&sbuf, fd, 4096) >= 0) - ret = index_mem(sha1, sbuf.buf, sbuf.len, write_object, - type, path, format_check); - else - ret = -1; - strbuf_release(&sbuf); - } else if (!size) { - ret = index_mem(sha1, NULL, size, write_object, type, path, - format_check); + if (!size) { + ret = index_mem(sha1, NULL, size, type, path, flags); } else if (size <= SMALL_FILE_SIZE) { char *buf = xmalloc(size); if (size == read_in_full(fd, buf, size)) - ret = index_mem(sha1, buf, size, write_object, type, - path, format_check); + ret = index_mem(sha1, buf, size, type, path, flags); else ret = error("short read %s", strerror(errno)); free(buf); } else { void *buf = xmmap(NULL, size, PROT_READ, MAP_PRIVATE, fd, 0); - ret = index_mem(sha1, buf, size, write_object, type, path, - format_check); + ret = index_mem(sha1, buf, size, type, path, flags); munmap(buf, size); } + return ret; +} + +/* + * This creates one packfile per large blob, because the caller + * immediately wants the result sha1, and fast-import can report the + * object name via marks mechanism only by closing the created + * packfile. + * + * This also bypasses the usual "convert-to-git" dance, and that is on + * purpose. We could write a streaming version of the converting + * functions and insert that before feeding the data to fast-import + * (or equivalent in-core API described above), but the primary + * motivation for trying to stream from the working tree file and to + * avoid mmaping it in core is to deal with large binary blobs, and + * by definition they do _not_ want to get any conversion. + */ +static int index_stream(unsigned char *sha1, int fd, size_t size, + enum object_type type, const char *path, + unsigned flags) +{ + struct child_process fast_import; + char export_marks[512]; + const char *argv[] = { "fast-import", "--quiet", export_marks, NULL }; + char tmpfile[512]; + char fast_import_cmd[512]; + char buf[512]; + int len, tmpfd; + + strcpy(tmpfile, git_path("hashstream_XXXXXX")); + tmpfd = git_mkstemp_mode(tmpfile, 0600); + if (tmpfd < 0) + die_errno("cannot create tempfile: %s", tmpfile); + if (close(tmpfd)) + die_errno("cannot close tempfile: %s", tmpfile); + sprintf(export_marks, "--export-marks=%s", tmpfile); + + memset(&fast_import, 0, sizeof(fast_import)); + fast_import.in = -1; + fast_import.argv = argv; + fast_import.git_cmd = 1; + if (start_command(&fast_import)) + die_errno("index-stream: git fast-import failed"); + + len = sprintf(fast_import_cmd, "blob\nmark :1\ndata %lu\n", + (unsigned long) size); + write_or_whine(fast_import.in, fast_import_cmd, len, + "index-stream: feeding fast-import"); + while (size) { + char buf[10240]; + size_t sz = size < sizeof(buf) ? size : sizeof(buf); + ssize_t actual; + + actual = read_in_full(fd, buf, sz); + if (actual < 0) + die_errno("index-stream: reading input"); + if (write_in_full(fast_import.in, buf, actual) != actual) + die_errno("index-stream: feeding fast-import"); + size -= actual; + } + if (close(fast_import.in)) + die_errno("index-stream: closing fast-import"); + if (finish_command(&fast_import)) + die_errno("index-stream: finishing fast-import"); + + tmpfd = open(tmpfile, O_RDONLY); + if (tmpfd < 0) + die_errno("index-stream: cannot open fast-import mark"); + len = read(tmpfd, buf, sizeof(buf)); + if (len < 0) + die_errno("index-stream: reading fast-import mark"); + if (close(tmpfd) < 0) + die_errno("index-stream: closing fast-import mark"); + if (unlink(tmpfile)) + die_errno("index-stream: unlinking fast-import mark"); + if (len != 44 || + memcmp(":1 ", buf, 3) || + get_sha1_hex(buf + 3, sha1)) + die_errno("index-stream: unexpected fast-import mark: <%s>", buf); + return 0; +} + +int index_fd(unsigned char *sha1, int fd, struct stat *st, + enum object_type type, const char *path, unsigned flags) +{ + int ret; + size_t size = xsize_t(st->st_size); + + if (!S_ISREG(st->st_mode)) + ret = index_pipe(sha1, fd, type, path, flags); + else if (size <= big_file_threshold || type != OBJ_BLOB) + ret = index_core(sha1, fd, size, type, path, flags); + else + ret = index_stream(sha1, fd, size, type, path, flags); close(fd); return ret; } -int index_path(unsigned char *sha1, const char *path, struct stat *st, int write_object) +int index_path(unsigned char *sha1, const char *path, struct stat *st, unsigned flags) { int fd; struct strbuf sb = STRBUF_INIT; @@@ -2795,7 -2665,7 +2795,7 @@@ if (fd < 0) return error("open(\"%s\"): %s", path, strerror(errno)); - if (index_fd(sha1, fd, st, write_object, OBJ_BLOB, path, 0) < 0) + if (index_fd(sha1, fd, st, OBJ_BLOB, path, flags) < 0) return error("%s: failed to insert into database", path); break; @@@ -2805,7 -2675,7 +2805,7 @@@ return error("readlink(\"%s\"): %s", path, errstr); } - if (!write_object) + if (!(flags & HASH_WRITE_OBJECT)) hash_sha1_file(sb.buf, sb.len, blob_type, sha1); else if (write_sha1_file(sb.buf, sb.len, blob_type, sha1)) return error("%s: failed to insert into database", diff --combined streaming.c index 91414f4592,0000000000..25c9a209bb mode 100644,000000..100644 --- a/streaming.c +++ b/streaming.c @@@ -1,489 -1,0 +1,489 @@@ +/* + * Copyright (c) 2011, Google Inc. + */ +#include "cache.h" +#include "streaming.h" + +enum input_source { + stream_error = -1, + incore = 0, + loose = 1, + pack_non_delta = 2 +}; + +typedef int (*open_istream_fn)(struct git_istream *, + struct object_info *, + const unsigned char *, + enum object_type *); +typedef int (*close_istream_fn)(struct git_istream *); +typedef ssize_t (*read_istream_fn)(struct git_istream *, char *, size_t); + +struct stream_vtbl { + close_istream_fn close; + read_istream_fn read; +}; + +#define open_method_decl(name) \ + int open_istream_ ##name \ + (struct git_istream *st, struct object_info *oi, \ + const unsigned char *sha1, \ + enum object_type *type) + +#define close_method_decl(name) \ + int close_istream_ ##name \ + (struct git_istream *st) + +#define read_method_decl(name) \ + ssize_t read_istream_ ##name \ + (struct git_istream *st, char *buf, size_t sz) + +/* forward declaration */ +static open_method_decl(incore); +static open_method_decl(loose); +static open_method_decl(pack_non_delta); +static struct git_istream *attach_stream_filter(struct git_istream *st, + struct stream_filter *filter); + + +static open_istream_fn open_istream_tbl[] = { + open_istream_incore, + open_istream_loose, + open_istream_pack_non_delta, +}; + +#define FILTER_BUFFER (1024*16) + +struct filtered_istream { + struct git_istream *upstream; + struct stream_filter *filter; + char ibuf[FILTER_BUFFER]; + char obuf[FILTER_BUFFER]; + int i_end, i_ptr; + int o_end, o_ptr; + int input_finished; +}; + +struct git_istream { + const struct stream_vtbl *vtbl; + unsigned long size; /* inflated size of full object */ - z_stream z; ++ git_zstream z; + enum { z_unused, z_used, z_done, z_error } z_state; + + union { + struct { + char *buf; /* from read_object() */ + unsigned long read_ptr; + } incore; + + struct { + void *mapped; + unsigned long mapsize; + char hdr[32]; + int hdr_avail; + int hdr_used; + } loose; + + struct { + struct packed_git *pack; + off_t pos; + } in_pack; + + struct filtered_istream filtered; + } u; +}; + +int close_istream(struct git_istream *st) +{ + return st->vtbl->close(st); +} + +ssize_t read_istream(struct git_istream *st, char *buf, size_t sz) +{ + return st->vtbl->read(st, buf, sz); +} + +static enum input_source istream_source(const unsigned char *sha1, + enum object_type *type, + struct object_info *oi) +{ + unsigned long size; + int status; + + oi->sizep = &size; + status = sha1_object_info_extended(sha1, oi); + if (status < 0) + return stream_error; + *type = status; + + switch (oi->whence) { + case OI_LOOSE: + return loose; + case OI_PACKED: + if (!oi->u.packed.is_delta && big_file_threshold <= size) + return pack_non_delta; + /* fallthru */ + default: + return incore; + } +} + +struct git_istream *open_istream(const unsigned char *sha1, + enum object_type *type, + unsigned long *size, + struct stream_filter *filter) +{ + struct git_istream *st; + struct object_info oi; + const unsigned char *real = lookup_replace_object(sha1); + enum input_source src = istream_source(real, type, &oi); + + if (src < 0) + return NULL; + + st = xmalloc(sizeof(*st)); + if (open_istream_tbl[src](st, &oi, real, type)) { + if (open_istream_incore(st, &oi, real, type)) { + free(st); + return NULL; + } + } + if (st && filter) { + /* Add "&& !is_null_stream_filter(filter)" for performance */ + struct git_istream *nst = attach_stream_filter(st, filter); + if (!nst) + close_istream(st); + st = nst; + } + + *size = st->size; + return st; +} + + +/***************************************************************** + * + * Common helpers + * + *****************************************************************/ + +static void close_deflated_stream(struct git_istream *st) +{ + if (st->z_state == z_used) + git_inflate_end(&st->z); +} + + +/***************************************************************** + * + * Filtered stream + * + *****************************************************************/ + +static close_method_decl(filtered) +{ + free_stream_filter(st->u.filtered.filter); + return close_istream(st->u.filtered.upstream); +} + +static read_method_decl(filtered) +{ + struct filtered_istream *fs = &(st->u.filtered); + size_t filled = 0; + + while (sz) { + /* do we already have filtered output? */ + if (fs->o_ptr < fs->o_end) { + size_t to_move = fs->o_end - fs->o_ptr; + if (sz < to_move) + to_move = sz; + memcpy(buf + filled, fs->obuf + fs->o_ptr, to_move); + fs->o_ptr += to_move; + sz -= to_move; + filled += to_move; + continue; + } + fs->o_end = fs->o_ptr = 0; + + /* do we have anything to feed the filter with? */ + if (fs->i_ptr < fs->i_end) { + size_t to_feed = fs->i_end - fs->i_ptr; + size_t to_receive = FILTER_BUFFER; + if (stream_filter(fs->filter, + fs->ibuf + fs->i_ptr, &to_feed, + fs->obuf, &to_receive)) + return -1; + fs->i_ptr = fs->i_end - to_feed; + fs->o_end = FILTER_BUFFER - to_receive; + continue; + } + + /* tell the filter to drain upon no more input */ + if (fs->input_finished) { + size_t to_receive = FILTER_BUFFER; + if (stream_filter(fs->filter, + NULL, NULL, + fs->obuf, &to_receive)) + return -1; + fs->o_end = FILTER_BUFFER - to_receive; + if (!fs->o_end) + break; + continue; + } + fs->i_end = fs->i_ptr = 0; + + /* refill the input from the upstream */ + if (!fs->input_finished) { + fs->i_end = read_istream(fs->upstream, fs->ibuf, FILTER_BUFFER); + if (fs->i_end < 0) + break; + if (fs->i_end) + continue; + } + fs->input_finished = 1; + } + return filled; +} + +static struct stream_vtbl filtered_vtbl = { + close_istream_filtered, + read_istream_filtered, +}; + +static struct git_istream *attach_stream_filter(struct git_istream *st, + struct stream_filter *filter) +{ + struct git_istream *ifs = xmalloc(sizeof(*ifs)); + struct filtered_istream *fs = &(ifs->u.filtered); + + ifs->vtbl = &filtered_vtbl; + fs->upstream = st; + fs->filter = filter; + fs->i_end = fs->i_ptr = 0; + fs->o_end = fs->o_ptr = 0; + fs->input_finished = 0; + ifs->size = -1; /* unknown */ + return ifs; +} + +/***************************************************************** + * + * Loose object stream + * + *****************************************************************/ + +static read_method_decl(loose) +{ + size_t total_read = 0; + + switch (st->z_state) { + case z_done: + return 0; + case z_error: + return -1; + default: + break; + } + + if (st->u.loose.hdr_used < st->u.loose.hdr_avail) { + size_t to_copy = st->u.loose.hdr_avail - st->u.loose.hdr_used; + if (sz < to_copy) + to_copy = sz; + memcpy(buf, st->u.loose.hdr + st->u.loose.hdr_used, to_copy); + st->u.loose.hdr_used += to_copy; + total_read += to_copy; + } + + while (total_read < sz) { + int status; + + st->z.next_out = (unsigned char *)buf + total_read; + st->z.avail_out = sz - total_read; + status = git_inflate(&st->z, Z_FINISH); + + total_read = st->z.next_out - (unsigned char *)buf; + + if (status == Z_STREAM_END) { + git_inflate_end(&st->z); + st->z_state = z_done; + break; + } + if (status != Z_OK && status != Z_BUF_ERROR) { + git_inflate_end(&st->z); + st->z_state = z_error; + return -1; + } + } + return total_read; +} + +static close_method_decl(loose) +{ + close_deflated_stream(st); + munmap(st->u.loose.mapped, st->u.loose.mapsize); + return 0; +} + +static struct stream_vtbl loose_vtbl = { + close_istream_loose, + read_istream_loose, +}; + +static open_method_decl(loose) +{ + st->u.loose.mapped = map_sha1_file(sha1, &st->u.loose.mapsize); + if (!st->u.loose.mapped) + return -1; + if (unpack_sha1_header(&st->z, + st->u.loose.mapped, + st->u.loose.mapsize, + st->u.loose.hdr, + sizeof(st->u.loose.hdr)) < 0) { + git_inflate_end(&st->z); + munmap(st->u.loose.mapped, st->u.loose.mapsize); + return -1; + } + + parse_sha1_header(st->u.loose.hdr, &st->size); + st->u.loose.hdr_used = strlen(st->u.loose.hdr) + 1; + st->u.loose.hdr_avail = st->z.total_out; + st->z_state = z_used; + + st->vtbl = &loose_vtbl; + return 0; +} + + +/***************************************************************** + * + * Non-delta packed object stream + * + *****************************************************************/ + +static read_method_decl(pack_non_delta) +{ + size_t total_read = 0; + + switch (st->z_state) { + case z_unused: + memset(&st->z, 0, sizeof(st->z)); + git_inflate_init(&st->z); + st->z_state = z_used; + break; + case z_done: + return 0; + case z_error: + return -1; + case z_used: + break; + } + + while (total_read < sz) { + int status; + struct pack_window *window = NULL; + unsigned char *mapped; + + mapped = use_pack(st->u.in_pack.pack, &window, + st->u.in_pack.pos, &st->z.avail_in); + + st->z.next_out = (unsigned char *)buf + total_read; + st->z.avail_out = sz - total_read; + st->z.next_in = mapped; + status = git_inflate(&st->z, Z_FINISH); + + st->u.in_pack.pos += st->z.next_in - mapped; + total_read = st->z.next_out - (unsigned char *)buf; + unuse_pack(&window); + + if (status == Z_STREAM_END) { + git_inflate_end(&st->z); + st->z_state = z_done; + break; + } + if (status != Z_OK && status != Z_BUF_ERROR) { + git_inflate_end(&st->z); + st->z_state = z_error; + return -1; + } + } + return total_read; +} + +static close_method_decl(pack_non_delta) +{ + close_deflated_stream(st); + return 0; +} + +static struct stream_vtbl pack_non_delta_vtbl = { + close_istream_pack_non_delta, + read_istream_pack_non_delta, +}; + +static open_method_decl(pack_non_delta) +{ + struct pack_window *window; + enum object_type in_pack_type; + + st->u.in_pack.pack = oi->u.packed.pack; + st->u.in_pack.pos = oi->u.packed.offset; + window = NULL; + + in_pack_type = unpack_object_header(st->u.in_pack.pack, + &window, + &st->u.in_pack.pos, + &st->size); + unuse_pack(&window); + switch (in_pack_type) { + default: + return -1; /* we do not do deltas for now */ + case OBJ_COMMIT: + case OBJ_TREE: + case OBJ_BLOB: + case OBJ_TAG: + break; + } + st->z_state = z_unused; + st->vtbl = &pack_non_delta_vtbl; + return 0; +} + + +/***************************************************************** + * + * In-core stream + * + *****************************************************************/ + +static close_method_decl(incore) +{ + free(st->u.incore.buf); + return 0; +} + +static read_method_decl(incore) +{ + size_t read_size = sz; + size_t remainder = st->size - st->u.incore.read_ptr; + + if (remainder <= read_size) + read_size = remainder; + if (read_size) { + memcpy(buf, st->u.incore.buf + st->u.incore.read_ptr, read_size); + st->u.incore.read_ptr += read_size; + } + return read_size; +} + +static struct stream_vtbl incore_vtbl = { + close_istream_incore, + read_istream_incore, +}; + +static open_method_decl(incore) +{ + st->u.incore.buf = read_sha1_file_extended(sha1, type, &st->size, 0); + st->u.incore.read_ptr = 0; + st->vtbl = &incore_vtbl; + + return st->u.incore.buf ? 0 : -1; +}