From: Junio C Hamano Date: Fri, 9 Sep 2016 04:49:50 +0000 (-0700) Subject: Merge branch 'sb/submodule-clone-rr' X-Git-Tag: v2.11.0-rc0~172 X-Git-Url: https://git.lorimer.id.au/gitweb.git/diff_plain/02c6c14d6c304f094b680f583a1dd6e4fe75736c?hp=-c Merge branch 'sb/submodule-clone-rr' "git clone --resurse-submodules --reference $path $URL" is a way to reduce network transfer cost by borrowing objects in an existing $path repository when cloning the superproject from $URL; it learned to also peek into $path for presense of corresponding repositories of submodules and borrow objects from there when able. * sb/submodule-clone-rr: clone: recursive and reference option triggers submodule alternates clone: implement optional references clone: clarify option_reference as required clone: factor out checking for an alternate path submodule--helper update-clone: allow multiple references submodule--helper module-clone: allow multiple references t7408: merge short tests, factor out testing method t7408: modernize style --- 02c6c14d6c304f094b680f583a1dd6e4fe75736c diff --combined Documentation/config.txt index 8a115b3702,e2571ea8ea..32f065ca6a --- a/Documentation/config.txt +++ b/Documentation/config.txt @@@ -1253,16 -1253,6 +1253,16 @@@ format.attach: value as the boundary. See the --attach option in linkgit:git-format-patch[1]. +format.from:: + Provides the default value for the `--from` option to format-patch. + Accepts a boolean value, or a name and email address. If false, + format-patch defaults to `--no-from`, using commit authors directly in + the "From:" field of patch mails. If true, format-patch defaults to + `--from`, using your committer identity in the "From:" field of patch + mails and including a "From:" field in the body of the patch mail if + different. If set to a non-boolean value, format-patch uses that + value instead of your committer identity. Defaults to false. + format.numbered:: A boolean which can enable or disable sequence numbers in patch subjects. It defaults to "auto" which enables it only if there @@@ -2517,12 -2507,6 +2517,12 @@@ receive.unpackLimit: especially on slow filesystems. If not set, the value of `transfer.unpackLimit` is used instead. +receive.maxInputSize:: + If the size of the incoming pack stream is larger than this + limit, then git-receive-pack will error out, instead of + accepting the pack file. If not set or set to 0, then the size + is unlimited. + receive.denyDeletes:: If set to true, git-receive-pack will deny a ref update that deletes the ref. Use this to prevent such a ref deletion via a push. @@@ -2853,6 -2837,18 +2853,18 @@@ submodule.fetchJobs: in parallel. A value of 0 will give some reasonable default. If unset, it defaults to 1. + submodule.alternateLocation:: + Specifies how the submodules obtain alternates when submodules are + cloned. Possible values are `no`, `superproject`. + By default `no` is assumed, which doesn't add references. When the + value is set to `superproject` the submodule to be cloned computes + its alternates location relative to the superprojects alternate. + + submodule.alternateErrorStrategy + Specifies how to treat errors with the alternates for a submodule + as computed via `submodule.alternateLocation`. Possible values are + `ignore`, `info`, `die`. Default is `die`. + tag.forceSignAnnotated:: A boolean to specify whether annotated tags created should be GPG signed. If `--annotate` is specified on the command line, it takes diff --combined builtin/submodule--helper.c index e79790f0bd,a3667578b4..9d79f1994a --- a/builtin/submodule--helper.c +++ b/builtin/submodule--helper.c @@@ -442,9 -442,10 +442,9 @@@ static int module_name(int argc, const } static int clone_submodule(const char *path, const char *gitdir, const char *url, - const char *depth, const char *reference, int quiet) + const char *depth, struct string_list *reference, int quiet) { - struct child_process cp; - child_process_init(&cp); + struct child_process cp = CHILD_PROCESS_INIT; argv_array_push(&cp.args, "clone"); argv_array_push(&cp.args, "--no-checkout"); @@@ -452,8 -453,12 +452,12 @@@ argv_array_push(&cp.args, "--quiet"); if (depth && *depth) argv_array_pushl(&cp.args, "--depth", depth, NULL); - if (reference && *reference) - argv_array_pushl(&cp.args, "--reference", reference, NULL); + if (reference->nr) { + struct string_list_item *item; + for_each_string_list_item(item, reference) + argv_array_pushl(&cp.args, "--reference", + item->string, NULL); + } if (gitdir && *gitdir) argv_array_pushl(&cp.args, "--separate-git-dir", gitdir, NULL); @@@ -467,15 -472,114 +471,114 @@@ return run_command(&cp); } + struct submodule_alternate_setup { + const char *submodule_name; + enum SUBMODULE_ALTERNATE_ERROR_MODE { + SUBMODULE_ALTERNATE_ERROR_DIE, + SUBMODULE_ALTERNATE_ERROR_INFO, + SUBMODULE_ALTERNATE_ERROR_IGNORE + } error_mode; + struct string_list *reference; + }; + #define SUBMODULE_ALTERNATE_SETUP_INIT { NULL, \ + SUBMODULE_ALTERNATE_ERROR_IGNORE, NULL } + + static int add_possible_reference_from_superproject( + struct alternate_object_database *alt, void *sas_cb) + { + struct submodule_alternate_setup *sas = sas_cb; + + /* directory name, minus trailing slash */ + size_t namelen = alt->name - alt->base - 1; + struct strbuf name = STRBUF_INIT; + strbuf_add(&name, alt->base, namelen); + + /* + * If the alternate object store is another repository, try the + * standard layout with .git/modules//objects + */ + if (ends_with(name.buf, ".git/objects")) { + char *sm_alternate; + struct strbuf sb = STRBUF_INIT; + struct strbuf err = STRBUF_INIT; + strbuf_add(&sb, name.buf, name.len - strlen("objects")); + /* + * We need to end the new path with '/' to mark it as a dir, + * otherwise a submodule name containing '/' will be broken + * as the last part of a missing submodule reference would + * be taken as a file name. + */ + strbuf_addf(&sb, "modules/%s/", sas->submodule_name); + + sm_alternate = compute_alternate_path(sb.buf, &err); + if (sm_alternate) { + string_list_append(sas->reference, xstrdup(sb.buf)); + free(sm_alternate); + } else { + switch (sas->error_mode) { + case SUBMODULE_ALTERNATE_ERROR_DIE: + die(_("submodule '%s' cannot add alternate: %s"), + sas->submodule_name, err.buf); + case SUBMODULE_ALTERNATE_ERROR_INFO: + fprintf(stderr, _("submodule '%s' cannot add alternate: %s"), + sas->submodule_name, err.buf); + case SUBMODULE_ALTERNATE_ERROR_IGNORE: + ; /* nothing */ + } + } + strbuf_release(&sb); + } + + strbuf_release(&name); + return 0; + } + + static void prepare_possible_alternates(const char *sm_name, + struct string_list *reference) + { + char *sm_alternate = NULL, *error_strategy = NULL; + struct submodule_alternate_setup sas = SUBMODULE_ALTERNATE_SETUP_INIT; + + git_config_get_string("submodule.alternateLocation", &sm_alternate); + if (!sm_alternate) + return; + + git_config_get_string("submodule.alternateErrorStrategy", &error_strategy); + + if (!error_strategy) + error_strategy = xstrdup("die"); + + sas.submodule_name = sm_name; + sas.reference = reference; + if (!strcmp(error_strategy, "die")) + sas.error_mode = SUBMODULE_ALTERNATE_ERROR_DIE; + else if (!strcmp(error_strategy, "info")) + sas.error_mode = SUBMODULE_ALTERNATE_ERROR_INFO; + else if (!strcmp(error_strategy, "ignore")) + sas.error_mode = SUBMODULE_ALTERNATE_ERROR_IGNORE; + else + die(_("Value '%s' for submodule.alternateErrorStrategy is not recognized"), error_strategy); + + if (!strcmp(sm_alternate, "superproject")) + foreach_alt_odb(add_possible_reference_from_superproject, &sas); + else if (!strcmp(sm_alternate, "no")) + ; /* do nothing */ + else + die(_("Value '%s' for submodule.alternateLocation is not recognized"), sm_alternate); + + free(sm_alternate); + free(error_strategy); + } + static int module_clone(int argc, const char **argv, const char *prefix) { - const char *name = NULL, *url = NULL; - const char *reference = NULL, *depth = NULL; + const char *name = NULL, *url = NULL, *depth = NULL; int quiet = 0; FILE *submodule_dot_git; char *p, *path = NULL, *sm_gitdir; struct strbuf rel_path = STRBUF_INIT; struct strbuf sb = STRBUF_INIT; + struct string_list reference = STRING_LIST_INIT_NODUP; struct option module_clone_options[] = { OPT_STRING(0, "prefix", &prefix, @@@ -490,8 -594,8 +593,8 @@@ OPT_STRING(0, "url", &url, N_("string"), N_("url where to clone the submodule from")), - OPT_STRING(0, "reference", &reference, - N_("string"), + OPT_STRING_LIST(0, "reference", &reference, + N_("repo"), N_("reference repository")), OPT_STRING(0, "depth", &depth, N_("string"), @@@ -527,7 -631,10 +630,10 @@@ if (!file_exists(sm_gitdir)) { if (safe_create_leading_directories_const(sm_gitdir) < 0) die(_("could not create directory '%s'"), sm_gitdir); - if (clone_submodule(path, sm_gitdir, url, depth, reference, quiet)) + + prepare_possible_alternates(name, &reference); + + if (clone_submodule(path, sm_gitdir, url, depth, &reference, quiet)) die(_("clone of '%s' into submodule path '%s' failed"), url, path); } else { @@@ -579,7 -686,7 +685,7 @@@ struct submodule_update_clone /* configuration parameters which are passed on to the children */ int quiet; int recommend_shallow; - const char *reference; + struct string_list references; const char *depth; const char *recursive_prefix; const char *prefix; @@@ -595,7 -702,8 +701,8 @@@ int failed_clones_nr, failed_clones_alloc; }; #define SUBMODULE_UPDATE_CLONE_INIT {0, MODULE_LIST_INIT, 0, \ - SUBMODULE_UPDATE_STRATEGY_INIT, 0, -1, NULL, NULL, NULL, NULL, \ + SUBMODULE_UPDATE_STRATEGY_INIT, 0, -1, STRING_LIST_INIT_DUP, \ + NULL, NULL, NULL, \ STRING_LIST_INIT_DUP, 0, NULL, 0, 0} @@@ -705,8 -813,11 +812,11 @@@ static int prepare_to_clone_next_submod argv_array_pushl(&child->args, "--path", sub->path, NULL); argv_array_pushl(&child->args, "--name", sub->name, NULL); argv_array_pushl(&child->args, "--url", url, NULL); - if (suc->reference) - argv_array_push(&child->args, suc->reference); + if (suc->references.nr) { + struct string_list_item *item; + for_each_string_list_item(item, &suc->references) + argv_array_pushl(&child->args, "--reference", item->string, NULL); + } if (suc->depth) argv_array_push(&child->args, suc->depth); @@@ -747,12 -858,8 +857,12 @@@ static int update_clone_get_next_task(s if (index < suc->failed_clones_nr) { int *p; ce = suc->failed_clones[index]; - if (!prepare_to_clone_next_submodule(ce, child, suc, err)) - die("BUG: ce was a submodule before?"); + if (!prepare_to_clone_next_submodule(ce, child, suc, err)) { + suc->current ++; + strbuf_addf(err, "BUG: submodule considered for cloning," + "doesn't need cloning any more?\n"); + return 0; + } p = xmalloc(sizeof(*p)); *p = suc->current; *idx_task_cb = p; @@@ -829,7 -936,7 +939,7 @@@ static int update_clone(int argc, cons OPT_STRING(0, "update", &update, N_("string"), N_("rebase, merge, checkout or none")), - OPT_STRING(0, "reference", &suc.reference, N_("repo"), + OPT_STRING_LIST(0, "reference", &suc.references, N_("repo"), N_("reference repository")), OPT_STRING(0, "depth", &suc.depth, "", N_("Create a shallow clone truncated to the " @@@ -895,64 -1002,13 +1005,64 @@@ static int resolve_relative_path(int ar { struct strbuf sb = STRBUF_INIT; if (argc != 3) - die("submodule--helper relative_path takes exactly 2 arguments, got %d", argc); + die("submodule--helper relative-path takes exactly 2 arguments, got %d", argc); printf("%s", relative_path(argv[1], argv[2], &sb)); strbuf_release(&sb); return 0; } +static const char *remote_submodule_branch(const char *path) +{ + const struct submodule *sub; + gitmodules_config(); + git_config(submodule_config, NULL); + + sub = submodule_from_path(null_sha1, path); + if (!sub) + return NULL; + + if (!sub->branch) + return "master"; + + if (!strcmp(sub->branch, ".")) { + unsigned char sha1[20]; + const char *refname = resolve_ref_unsafe("HEAD", 0, sha1, NULL); + + if (!refname) + die(_("No such ref: %s"), "HEAD"); + + /* detached HEAD */ + if (!strcmp(refname, "HEAD")) + die(_("Submodule (%s) branch configured to inherit " + "branch from superproject, but the superproject " + "is not on any branch"), sub->name); + + if (!skip_prefix(refname, "refs/heads/", &refname)) + die(_("Expecting a full ref name, got %s"), refname); + return refname; + } + + return sub->branch; +} + +static int resolve_remote_submodule_branch(int argc, const char **argv, + const char *prefix) +{ + const char *ret; + struct strbuf sb = STRBUF_INIT; + if (argc != 2) + die("submodule--helper remote-branch takes exactly one arguments, got %d", argc); + + ret = remote_submodule_branch(argv[1]); + if (!ret) + die("submodule %s doesn't exist", argv[1]); + + printf("%s", ret); + strbuf_release(&sb); + return 0; +} + struct cmd_struct { const char *cmd; int (*fn)(int, const char **, const char *); @@@ -966,8 -1022,7 +1076,8 @@@ static struct cmd_struct commands[] = {"relative-path", resolve_relative_path}, {"resolve-relative-url", resolve_relative_url}, {"resolve-relative-url-test", resolve_relative_url_test}, - {"init", module_init} + {"init", module_init}, + {"remote-branch", resolve_remote_submodule_branch} }; int cmd_submodule__helper(int argc, const char **argv, const char *prefix) diff --combined cache.h index b780a91a56,1087dcaa1f..6738050132 --- a/cache.h +++ b/cache.h @@@ -1230,8 -1230,7 +1230,8 @@@ struct date_mode DATE_ISO8601_STRICT, DATE_RFC2822, DATE_STRFTIME, - DATE_RAW + DATE_RAW, + DATE_UNIX } type; const char *strftime_fmt; int local; @@@ -1270,7 -1269,6 +1270,7 @@@ extern const char *ident_default_email( extern const char *git_editor(void); extern const char *git_pager(int stdout_is_tty); extern int git_ident_config(const char *, const char *, void *); +extern void reset_ident_date(void); struct ident_split { const char *name_begin; @@@ -1344,6 -1342,7 +1344,7 @@@ extern struct alternate_object_databas } *alt_odb_list; extern void prepare_alt_odb(void); extern void read_info_alternates(const char * relative_base, int depth); + extern char *compute_alternate_path(const char *path, struct strbuf *err); extern void add_to_alternates_file(const char *reference); typedef int alt_odb_fn(struct alternate_object_database *, void *); extern int foreach_alt_odb(alt_odb_fn, void*); @@@ -1379,13 -1378,6 +1380,13 @@@ extern struct packed_git char pack_name[FLEX_ARRAY]; /* more */ } *packed_git; +/* + * A most-recently-used ordered version of the packed_git list, which can + * be iterated instead of packed_git (and marked via mru_mark). + */ +struct mru; +extern struct mru *packed_git_mru; + struct pack_entry { off_t offset; unsigned char sha1[20]; @@@ -1425,6 -1417,7 +1426,6 @@@ extern unsigned char *use_pack(struct p extern void close_pack_windows(struct packed_git *); extern void close_all_packs(void); extern void unuse_pack(struct pack_window **); -extern void free_pack_by_name(const char *); extern void clear_delta_base_cache(void); extern struct packed_git *add_packed_git(const char *path, size_t path_len, int local); @@@ -1574,18 -1567,10 +1575,18 @@@ struct git_config_source const char *blob; }; +enum config_origin_type { + CONFIG_ORIGIN_BLOB, + CONFIG_ORIGIN_FILE, + CONFIG_ORIGIN_STDIN, + CONFIG_ORIGIN_SUBMODULE_BLOB, + CONFIG_ORIGIN_CMDLINE +}; + typedef int (*config_fn_t)(const char *, const char *, void *); extern int git_default_config(const char *, const char *, void *); extern int git_config_from_file(config_fn_t fn, const char *, void *); -extern int git_config_from_mem(config_fn_t fn, const char *origin_type, +extern int git_config_from_mem(config_fn_t fn, const enum config_origin_type, const char *name, const char *buf, size_t len, void *data); extern void git_config_push_parameter(const char *text); extern int git_config_from_parameters(config_fn_t fn, void *data); @@@ -1729,7 -1714,7 +1730,7 @@@ extern int ignore_untracked_cache_confi struct key_value_info { const char *filename; int linenr; - const char *origin_type; + enum config_origin_type origin_type; enum config_scope scope; }; @@@ -1756,6 -1741,7 +1757,6 @@@ extern int copy_file(const char *dst, c extern int copy_file_with_time(const char *dst, const char *src, int mode); extern void write_or_die(int fd, const void *buf, size_t count); -extern int write_or_whine_pipe(int fd, const void *buf, size_t count, const char *msg); extern void fsync_or_die(int fd, const char *); extern ssize_t read_in_full(int fd, void *buf, size_t count); diff --combined git-submodule.sh index b57f87de65,3b412f554b..a1cc71b521 --- a/git-submodule.sh +++ b/git-submodule.sh @@@ -479,8 -479,7 +479,8 @@@ fetch_in_submodule () '') git fetch ;; *) - git fetch $(get_default_remote) "$2" ;; + shift + git fetch $(get_default_remote) "$@" ;; esac ) @@@ -576,7 -575,7 +576,7 @@@ cmd_update( ${wt_prefix:+--prefix "$wt_prefix"} \ ${prefix:+--recursive-prefix "$prefix"} \ ${update:+--update "$update"} \ - ${reference:+--reference "$reference"} \ + ${reference:+"$reference"} \ ${depth:+--depth "$depth"} \ ${recommend_shallow:+"$recommend_shallow"} \ ${jobs:+$jobs} \ @@@ -589,6 -588,7 +589,6 @@@ name=$(git submodule--helper name "$sm_path") || exit url=$(git config submodule."$name".url) - branch=$(get_submodule_config "$name" branch master) if ! test -z "$update" then update_module=$update @@@ -614,11 -614,10 +614,11 @@@ if test -n "$remote" then + branch=$(git submodule--helper remote-branch "$sm_path") if test -z "$nofetch" then # Fetch remote before determining tracking $sha1 - fetch_in_submodule "$sm_path" || + fetch_in_submodule "$sm_path" $depth || die "$(eval_gettext "Unable to fetch in submodule path '\$sm_path'")" fi remote_name=$(sanitize_submodule_env; cd "$sm_path" && get_default_remote) @@@ -641,13 -640,13 +641,13 @@@ # Run fetch only if $sha1 isn't present or it # is not reachable from a ref. is_tip_reachable "$sm_path" "$sha1" || - fetch_in_submodule "$sm_path" || + fetch_in_submodule "$sm_path" $depth || die "$(eval_gettext "Unable to fetch in submodule path '\$displaypath'")" # Now we tried the usual fetch, but $sha1 may # not be reachable from any of the refs is_tip_reachable "$sm_path" "$sha1" || - fetch_in_submodule "$sm_path" "$sha1" || + fetch_in_submodule "$sm_path" $depth "$sha1" || die "$(eval_gettext "Fetched in submodule path '\$displaypath', but it did not contain \$sha1. Direct fetching of that commit failed.")" fi diff --combined sha1_file.c index a57b71d133,0fe5aa35a5..6f38dc6548 --- a/sha1_file.c +++ b/sha1_file.c @@@ -23,8 -23,6 +23,8 @@@ #include "bulk-checkin.h" #include "streaming.h" #include "dir.h" +#include "mru.h" +#include "list.h" #ifndef O_NOATIME #if defined(__linux__) && (defined(__i386__) || defined(__PPC__)) @@@ -61,6 -59,14 +61,6 @@@ static struct cached_object empty_tree 0 }; -/* - * A pointer to the last packed_git in which an object was found. - * When an object is sought, we look in this packfile first, because - * objects that are looked up at similar times are often in the same - * packfile as one another. - */ -static struct packed_git *last_found_pack; - static struct cached_object *find_cached_object(const unsigned char *sha1) { int i; @@@ -419,6 -425,82 +419,82 @@@ void add_to_alternates_file(const char free(alts); } + /* + * Compute the exact path an alternate is at and returns it. In case of + * error NULL is returned and the human readable error is added to `err` + * `path` may be relative and should point to $GITDIR. + * `err` must not be null. + */ + char *compute_alternate_path(const char *path, struct strbuf *err) + { + char *ref_git = NULL; + const char *repo, *ref_git_s; + int seen_error = 0; + + ref_git_s = real_path_if_valid(path); + if (!ref_git_s) { + seen_error = 1; + strbuf_addf(err, _("path '%s' does not exist"), path); + goto out; + } else + /* + * Beware: read_gitfile(), real_path() and mkpath() + * return static buffer + */ + ref_git = xstrdup(ref_git_s); + + repo = read_gitfile(ref_git); + if (!repo) + repo = read_gitfile(mkpath("%s/.git", ref_git)); + if (repo) { + free(ref_git); + ref_git = xstrdup(repo); + } + + if (!repo && is_directory(mkpath("%s/.git/objects", ref_git))) { + char *ref_git_git = mkpathdup("%s/.git", ref_git); + free(ref_git); + ref_git = ref_git_git; + } else if (!is_directory(mkpath("%s/objects", ref_git))) { + struct strbuf sb = STRBUF_INIT; + seen_error = 1; + if (get_common_dir(&sb, ref_git)) { + strbuf_addf(err, + _("reference repository '%s' as a linked " + "checkout is not supported yet."), + path); + goto out; + } + + strbuf_addf(err, _("reference repository '%s' is not a " + "local repository."), path); + goto out; + } + + if (!access(mkpath("%s/shallow", ref_git), F_OK)) { + strbuf_addf(err, _("reference repository '%s' is shallow"), + path); + seen_error = 1; + goto out; + } + + if (!access(mkpath("%s/info/grafts", ref_git), F_OK)) { + strbuf_addf(err, + _("reference repository '%s' is grafted"), + path); + seen_error = 1; + goto out; + } + + out: + if (seen_error) { + free(ref_git); + ref_git = NULL; + } + + return ref_git; + } + int foreach_alt_odb(alt_odb_fn fn, void *cb) { struct alternate_object_database *ent; @@@ -516,9 -598,6 +592,9 @@@ static size_t peak_pack_mapped static size_t pack_mapped; struct packed_git *packed_git; +static struct mru packed_git_mru_storage; +struct mru *packed_git_mru = &packed_git_mru_storage; + void pack_report(void) { fprintf(stderr, @@@ -792,7 -871,7 +868,7 @@@ void close_all_packs(void for (p = packed_git; p; p = p->next) if (p->do_not_close) - die("BUG! Want to close pack marked 'do-not-close'"); + die("BUG: want to close pack marked 'do-not-close'"); else close_pack(p); } @@@ -888,6 -967,36 +964,6 @@@ void close_pack_index(struct packed_gi } } -/* - * This is used by git-repack in case a newly created pack happens to - * contain the same set of objects as an existing one. In that case - * the resulting file might be different even if its name would be the - * same. It is best to close any reference to the old pack before it is - * replaced on disk. Of course no index pointers or windows for given pack - * must subsist at this point. If ever objects from this pack are requested - * again, the new version of the pack will be reinitialized through - * reprepare_packed_git(). - */ -void free_pack_by_name(const char *pack_name) -{ - struct packed_git *p, **pp = &packed_git; - - while (*pp) { - p = *pp; - if (strcmp(pack_name, p->pack_name) == 0) { - clear_delta_base_cache(); - close_pack(p); - free(p->bad_object_sha1); - *pp = p->next; - if (last_found_pack == p) - last_found_pack = NULL; - free(p); - return; - } - pp = &p->next; - } -} - static unsigned int get_max_fd_limit(void) { #ifdef RLIMIT_NOFILE @@@ -1352,15 -1461,6 +1428,15 @@@ static void rearrange_packed_git(void free(ary); } +static void prepare_packed_git_mru(void) +{ + struct packed_git *p; + + mru_clear(packed_git_mru); + for (p = packed_git; p; p = p->next) + mru_append(packed_git_mru, p); +} + static int prepare_packed_git_run_once = 0; void prepare_packed_git(void) { @@@ -1376,7 -1476,6 +1452,7 @@@ alt->name[-1] = '/'; } rearrange_packed_git(); + prepare_packed_git_mru(); prepare_packed_git_run_once = 1; } @@@ -1693,7 -1792,7 +1769,7 @@@ static int parse_sha1_header_extended(c strbuf_add(oi->typename, type_buf, type_len); /* * Set type to 0 if its an unknown object and - * we're obtaining the type using '--allow-unkown-type' + * we're obtaining the type using '--allow-unknown-type' * option. */ if ((flags & LOOKUP_UNKNOWN_OBJECT) && (type < 0)) @@@ -2074,142 -2173,136 +2150,142 @@@ static void *unpack_compressed_entry(st return buffer; } -#define MAX_DELTA_CACHE (256) - +static struct hashmap delta_base_cache; static size_t delta_base_cached; -static struct delta_base_cache_lru_list { - struct delta_base_cache_lru_list *prev; - struct delta_base_cache_lru_list *next; -} delta_base_cache_lru = { &delta_base_cache_lru, &delta_base_cache_lru }; +static LIST_HEAD(delta_base_cache_lru); -static struct delta_base_cache_entry { - struct delta_base_cache_lru_list lru; - void *data; +struct delta_base_cache_key { struct packed_git *p; off_t base_offset; +}; + +struct delta_base_cache_entry { + struct hashmap hash; + struct delta_base_cache_key key; + struct list_head lru; + void *data; unsigned long size; enum object_type type; -} delta_base_cache[MAX_DELTA_CACHE]; +}; -static unsigned long pack_entry_hash(struct packed_git *p, off_t base_offset) +static unsigned int pack_entry_hash(struct packed_git *p, off_t base_offset) { - unsigned long hash; + unsigned int hash; - hash = (unsigned long)(intptr_t)p + (unsigned long)base_offset; + hash = (unsigned int)(intptr_t)p + (unsigned int)base_offset; hash += (hash >> 8) + (hash >> 16); - return hash % MAX_DELTA_CACHE; + return hash; } static struct delta_base_cache_entry * get_delta_base_cache_entry(struct packed_git *p, off_t base_offset) { - unsigned long hash = pack_entry_hash(p, base_offset); - return delta_base_cache + hash; + struct hashmap_entry entry; + struct delta_base_cache_key key; + + if (!delta_base_cache.cmpfn) + return NULL; + + hashmap_entry_init(&entry, pack_entry_hash(p, base_offset)); + key.p = p; + key.base_offset = base_offset; + return hashmap_get(&delta_base_cache, &entry, &key); } -static int eq_delta_base_cache_entry(struct delta_base_cache_entry *ent, - struct packed_git *p, off_t base_offset) +static int delta_base_cache_key_eq(const struct delta_base_cache_key *a, + const struct delta_base_cache_key *b) { - return (ent->data && ent->p == p && ent->base_offset == base_offset); + return a->p == b->p && a->base_offset == b->base_offset; +} + +static int delta_base_cache_hash_cmp(const void *va, const void *vb, + const void *vkey) +{ + const struct delta_base_cache_entry *a = va, *b = vb; + const struct delta_base_cache_key *key = vkey; + if (key) + return !delta_base_cache_key_eq(&a->key, key); + else + return !delta_base_cache_key_eq(&a->key, &b->key); } static int in_delta_base_cache(struct packed_git *p, off_t base_offset) { - struct delta_base_cache_entry *ent; - ent = get_delta_base_cache_entry(p, base_offset); - return eq_delta_base_cache_entry(ent, p, base_offset); + return !!get_delta_base_cache_entry(p, base_offset); } -static void clear_delta_base_cache_entry(struct delta_base_cache_entry *ent) +/* + * Remove the entry from the cache, but do _not_ free the associated + * entry data. The caller takes ownership of the "data" buffer, and + * should copy out any fields it wants before detaching. + */ +static void detach_delta_base_cache_entry(struct delta_base_cache_entry *ent) { - ent->data = NULL; - ent->lru.next->prev = ent->lru.prev; - ent->lru.prev->next = ent->lru.next; + hashmap_remove(&delta_base_cache, ent, &ent->key); + list_del(&ent->lru); delta_base_cached -= ent->size; + free(ent); } static void *cache_or_unpack_entry(struct packed_git *p, off_t base_offset, - unsigned long *base_size, enum object_type *type, int keep_cache) + unsigned long *base_size, enum object_type *type) { struct delta_base_cache_entry *ent; - void *ret; ent = get_delta_base_cache_entry(p, base_offset); - - if (!eq_delta_base_cache_entry(ent, p, base_offset)) + if (!ent) return unpack_entry(p, base_offset, type, base_size); - ret = ent->data; - - if (!keep_cache) - clear_delta_base_cache_entry(ent); - else - ret = xmemdupz(ent->data, ent->size); *type = ent->type; *base_size = ent->size; - return ret; + return xmemdupz(ent->data, ent->size); } static inline void release_delta_base_cache(struct delta_base_cache_entry *ent) { - if (ent->data) { - free(ent->data); - ent->data = NULL; - ent->lru.next->prev = ent->lru.prev; - ent->lru.prev->next = ent->lru.next; - delta_base_cached -= ent->size; - } + free(ent->data); + detach_delta_base_cache_entry(ent); } void clear_delta_base_cache(void) { - unsigned long p; - for (p = 0; p < MAX_DELTA_CACHE; p++) - release_delta_base_cache(&delta_base_cache[p]); + struct hashmap_iter iter; + struct delta_base_cache_entry *entry; + for (entry = hashmap_iter_first(&delta_base_cache, &iter); + entry; + entry = hashmap_iter_next(&iter)) { + release_delta_base_cache(entry); + } } static void add_delta_base_cache(struct packed_git *p, off_t base_offset, void *base, unsigned long base_size, enum object_type type) { - unsigned long hash = pack_entry_hash(p, base_offset); - struct delta_base_cache_entry *ent = delta_base_cache + hash; - struct delta_base_cache_lru_list *lru; + struct delta_base_cache_entry *ent = xmalloc(sizeof(*ent)); + struct list_head *lru; - release_delta_base_cache(ent); delta_base_cached += base_size; - for (lru = delta_base_cache_lru.next; - delta_base_cached > delta_base_cache_limit - && lru != &delta_base_cache_lru; - lru = lru->next) { - struct delta_base_cache_entry *f = (void *)lru; - if (f->type == OBJ_BLOB) - release_delta_base_cache(f); - } - for (lru = delta_base_cache_lru.next; - delta_base_cached > delta_base_cache_limit - && lru != &delta_base_cache_lru; - lru = lru->next) { - struct delta_base_cache_entry *f = (void *)lru; + list_for_each(lru, &delta_base_cache_lru) { + struct delta_base_cache_entry *f = + list_entry(lru, struct delta_base_cache_entry, lru); + if (delta_base_cached <= delta_base_cache_limit) + break; release_delta_base_cache(f); } - ent->p = p; - ent->base_offset = base_offset; + ent->key.p = p; + ent->key.base_offset = base_offset; ent->type = type; ent->data = base; ent->size = base_size; - ent->lru.next = &delta_base_cache_lru; - ent->lru.prev = delta_base_cache_lru.prev; - delta_base_cache_lru.prev->next = &ent->lru; - delta_base_cache_lru.prev = &ent->lru; + list_add_tail(&ent->lru, &delta_base_cache_lru); + + if (!delta_base_cache.cmpfn) + hashmap_init(&delta_base_cache, delta_base_cache_hash_cmp, 0); + hashmap_entry_init(ent, pack_entry_hash(p, base_offset)); + hashmap_add(&delta_base_cache, ent); } static void *read_object(const unsigned char *sha1, enum object_type *type, @@@ -2253,11 -2346,11 +2329,11 @@@ void *unpack_entry(struct packed_git *p struct delta_base_cache_entry *ent; ent = get_delta_base_cache_entry(p, curpos); - if (eq_delta_base_cache_entry(ent, p, curpos)) { + if (ent) { type = ent->type; data = ent->data; size = ent->size; - clear_delta_base_cache_entry(ent); + detach_delta_base_cache_entry(ent); base_from_cache = 1; break; } @@@ -2313,7 -2406,7 +2389,7 @@@ case OBJ_OFS_DELTA: case OBJ_REF_DELTA: if (data) - die("BUG in unpack_entry: left loop at a valid delta"); + die("BUG: unpack_entry: left loop at a valid delta"); break; case OBJ_COMMIT: case OBJ_TREE: @@@ -2587,15 -2680,21 +2663,15 @@@ static int fill_pack_entry(const unsign */ static int find_pack_entry(const unsigned char *sha1, struct pack_entry *e) { - struct packed_git *p; + struct mru_entry *p; prepare_packed_git(); if (!packed_git) return 0; - if (last_found_pack && fill_pack_entry(sha1, e, last_found_pack)) - return 1; - - for (p = packed_git; p; p = p->next) { - if (p == last_found_pack) - continue; /* we already checked this one */ - - if (fill_pack_entry(sha1, e, p)) { - last_found_pack = p; + for (p = packed_git_mru->head; p; p = p->next) { + if (fill_pack_entry(sha1, e, p->item)) { + mru_mark(packed_git_mru, p); return 1; } } @@@ -2762,7 -2861,7 +2838,7 @@@ static void *read_packed_sha1(const uns if (!find_pack_entry(sha1, &e)) return NULL; - data = cache_or_unpack_entry(e.p, e.offset, size, type, 1); + data = cache_or_unpack_entry(e.p, e.offset, size, type); if (!data) { /* * We're probably in deep shit, but let's try to fetch