From: Junio C Hamano Date: Wed, 18 Sep 2019 18:50:09 +0000 (-0700) Subject: Merge branch 'md/list-objects-filter-combo' X-Git-Url: https://git.lorimer.id.au/gitweb.git/diff_plain/627b82683447e299fc2e20140318c276efbf7de2?hp=-c Merge branch 'md/list-objects-filter-combo' The list-objects-filter API (used to create a sparse/lazy clone) learned to take a combined filter specification. * md/list-objects-filter-combo: list-objects-filter-options: make parser void list-objects-filter-options: clean up use of ALLOC_GROW list-objects-filter-options: allow mult. --filter strbuf: give URL-encoding API a char predicate fn list-objects-filter-options: make filter_spec a string_list list-objects-filter-options: move error check up list-objects-filter: implement composite filters list-objects-filter-options: always supply *errbuf list-objects-filter: put omits set in filter struct list-objects-filter: encapsulate filter components --- 627b82683447e299fc2e20140318c276efbf7de2 diff --combined Documentation/rev-list-options.txt index bb1251c036,d1f080bf6d..90ff9e2bea --- a/Documentation/rev-list-options.txt +++ b/Documentation/rev-list-options.txt @@@ -182,14 -182,6 +182,14 @@@ explicitly Pretend as if all objects mentioned by reflogs are listed on the command line as ``. +--alternate-refs:: + Pretend as if all objects mentioned as ref tips of alternate + repositories were listed on the command line. An alternate + repository is any repository whose object directory is specified + in `objects/info/alternates`. The set of included objects may + be modified by `core.alternateRefsCommand`, etc. See + linkgit:git-config[1]. + --single-worktree:: By default, all working trees will be examined by the following options when there are more than one (see @@@ -716,16 -708,6 +716,16 @@@ ifdef::git-rev-list[ Only useful with `--objects`; print the object IDs that are not in packs. +--object-names:: + Only useful with `--objects`; print the names of the object IDs + that are found. This is the default behavior. + +--no-object-names:: + Only useful with `--objects`; does not print the names of the object + IDs that are found. This inverts `--object-names`. This flag allows + the output to be more easily parsed by commands such as + linkgit:git-cat-file[1]. + --filter=:: Only useful with one of the `--objects*`; omits objects (usually blobs) from the list of printed objects. The '' @@@ -756,6 -738,22 +756,22 @@@ explicitly-given commit or tree Note that the form '--filter=sparse:path=' that wants to read from an arbitrary path on the filesystem has been dropped for security reasons. + + + Multiple '--filter=' flags can be specified to combine filters. Only + objects which are accepted by every filter are included. + + + The form '--filter=combine:++...' can also be + used to combined several filters, but this is harder than just repeating + the '--filter' flag and is usually not necessary. Filters are joined by + '{plus}' and individual filters are %-encoded (i.e. URL-encoded). + Besides the '{plus}' and '%' characters, the following characters are + reserved and also must be encoded: `~!@#$^&*()[]{}\;",<>?`+'`+ + as well as all characters with ASCII code <= `0x20`, which includes + space and newline. + + + Other arbitrary characters can also be encoded. For instance, + 'combine:tree:3+blob:none' and 'combine:tree%3A3+blob%3Anone' are + equivalent. --no-filter:: Turn off any previous `--filter=` argument. diff --combined builtin/clone.c index f665b28ccc,a693e6ca44..2048b6760a --- a/builtin/clone.c +++ b/builtin/clone.c @@@ -23,8 -23,6 +23,8 @@@ #include "transport.h" #include "strbuf.h" #include "dir.h" +#include "dir-iterator.h" +#include "iterator.h" #include "sigchain.h" #include "branch.h" #include "remote.h" @@@ -396,55 -394,50 +396,55 @@@ static void copy_alternates(struct strb fclose(in); } +static void mkdir_if_missing(const char *pathname, mode_t mode) +{ + struct stat st; + + if (!mkdir(pathname, mode)) + return; + + if (errno != EEXIST) + die_errno(_("failed to create directory '%s'"), pathname); + else if (stat(pathname, &st)) + die_errno(_("failed to stat '%s'"), pathname); + else if (!S_ISDIR(st.st_mode)) + die(_("%s exists and is not a directory"), pathname); +} + static void copy_or_link_directory(struct strbuf *src, struct strbuf *dest, - const char *src_repo, int src_baselen) + const char *src_repo) { - struct dirent *de; - struct stat buf; int src_len, dest_len; - DIR *dir; - - dir = opendir(src->buf); - if (!dir) - die_errno(_("failed to open '%s'"), src->buf); - - if (mkdir(dest->buf, 0777)) { - if (errno != EEXIST) - die_errno(_("failed to create directory '%s'"), dest->buf); - else if (stat(dest->buf, &buf)) - die_errno(_("failed to stat '%s'"), dest->buf); - else if (!S_ISDIR(buf.st_mode)) - die(_("%s exists and is not a directory"), dest->buf); - } + struct dir_iterator *iter; + int iter_status; + unsigned int flags; + + mkdir_if_missing(dest->buf, 0777); + + flags = DIR_ITERATOR_PEDANTIC | DIR_ITERATOR_FOLLOW_SYMLINKS; + iter = dir_iterator_begin(src->buf, flags); + + if (!iter) + die_errno(_("failed to start iterator over '%s'"), src->buf); strbuf_addch(src, '/'); src_len = src->len; strbuf_addch(dest, '/'); dest_len = dest->len; - while ((de = readdir(dir)) != NULL) { + while ((iter_status = dir_iterator_advance(iter)) == ITER_OK) { strbuf_setlen(src, src_len); - strbuf_addstr(src, de->d_name); + strbuf_addstr(src, iter->relative_path); strbuf_setlen(dest, dest_len); - strbuf_addstr(dest, de->d_name); - if (stat(src->buf, &buf)) { - warning (_("failed to stat %s\n"), src->buf); - continue; - } - if (S_ISDIR(buf.st_mode)) { - if (de->d_name[0] != '.') - copy_or_link_directory(src, dest, - src_repo, src_baselen); + strbuf_addstr(dest, iter->relative_path); + + if (S_ISDIR(iter->st.st_mode)) { + mkdir_if_missing(dest->buf, 0777); continue; } /* Files that cannot be copied bit-for-bit... */ - if (!strcmp(src->buf + src_baselen, "/info/alternates")) { + if (!fspathcmp(iter->relative_path, "info/alternates")) { copy_alternates(src, src_repo); continue; } @@@ -452,7 -445,7 +452,7 @@@ if (unlink(dest->buf) && errno != ENOENT) die_errno(_("failed to unlink '%s'"), dest->buf); if (!option_no_hardlinks) { - if (!link(src->buf, dest->buf)) + if (!link(real_path(src->buf), dest->buf)) continue; if (option_local > 0) die_errno(_("failed to create link '%s'"), dest->buf); @@@ -461,11 -454,7 +461,11 @@@ if (copy_file_with_time(dest->buf, src->buf, 0666)) die_errno(_("failed to copy file to '%s'"), dest->buf); } - closedir(dir); + + if (iter_status != ITER_DONE) { + strbuf_setlen(src, src_len); + die(_("failed to iterate over '%s'"), src->buf); + } } static void clone_local(const char *src_repo, const char *dest_repo) @@@ -483,7 -472,7 +483,7 @@@ get_common_dir(&dest, dest_repo); strbuf_addstr(&src, "/objects"); strbuf_addstr(&dest, "/objects"); - copy_or_link_directory(&src, &dest, src_repo, src.len); + copy_or_link_directory(&src, &dest, src_repo); strbuf_release(&src); strbuf_release(&dest); } @@@ -505,7 -494,7 +505,7 @@@ static enum static const char junk_leave_repo_msg[] = N_("Clone succeeded, but checkout failed.\n" "You can inspect what was checked out with 'git status'\n" - "and retry the checkout with 'git checkout -f HEAD'\n"); + "and retry with 'git restore --source=HEAD :/'\n"); static void remove_junk(void) { @@@ -1160,13 -1149,11 +1160,11 @@@ int cmd_clone(int argc, const char **ar transport->server_options = &server_options; if (filter_options.choice) { - struct strbuf expanded_filter_spec = STRBUF_INIT; - expand_list_objects_filter_spec(&filter_options, - &expanded_filter_spec); + const char *spec = + expand_list_objects_filter_spec(&filter_options); transport_set_option(transport, TRANS_OPT_LIST_OBJECTS_FILTER, - expanded_filter_spec.buf); + spec); transport_set_option(transport, TRANS_OPT_FROM_PROMISOR, "1"); - strbuf_release(&expanded_filter_spec); } if (transport->smart_options && !deepen && !filter_options.choice) @@@ -1263,7 -1250,7 +1261,7 @@@ transport_disconnect(transport); if (option_dissociate) { - close_all_packs(the_repository->objects); + close_object_store(the_repository->objects); dissociate_from_references(); } diff --combined builtin/fetch.c index 538f0e7207,dee89e1a19..9b27ae9681 --- a/builtin/fetch.c +++ b/builtin/fetch.c @@@ -23,10 -23,6 +23,10 @@@ #include "packfile.h" #include "list-objects-filter-options.h" #include "commit-reach.h" +#include "branch.h" +#include "promisor-remote.h" + +#define FORCED_UPDATES_DELAY_WARNING_IN_MS (10 * 1000) static const char * const builtin_fetch_usage[] = { N_("git fetch [] [ [...]]"), @@@ -43,8 -39,6 +43,8 @@@ enum }; static int fetch_prune_config = -1; /* unspecified */ +static int fetch_show_forced_updates = 1; +static uint64_t forced_updates_ms = 0; static int prune = -1; /* unspecified */ #define PRUNE_BY_DEFAULT 0 /* do we prune by default? */ @@@ -52,10 -46,8 +52,10 @@@ static int fetch_prune_tags_config = -1 static int prune_tags = -1; /* unspecified */ #define PRUNE_TAGS_BY_DEFAULT 0 /* do we prune tags by default? */ -static int all, append, dry_run, force, keep, multiple, update_head_ok, verbosity, deepen_relative; +static int all, append, dry_run, force, keep, multiple, update_head_ok; +static int verbosity, deepen_relative, set_upstream; static int progress = -1; +static int enable_auto_gc = 1; static int tags = TAGS_DEFAULT, unshallow, update_shallow, deepen; static int max_children = 1; static enum transport_family family; @@@ -87,11 -79,6 +87,11 @@@ static int git_fetch_config(const char return 0; } + if (!strcmp(k, "fetch.showforcedupdates")) { + fetch_show_forced_updates = git_config_bool(k, v); + return 0; + } + if (!strcmp(k, "submodule.recurse")) { int r = git_config_bool(k, v) ? RECURSE_SUBMODULES_ON : RECURSE_SUBMODULES_OFF; @@@ -126,8 -113,6 +126,8 @@@ static struct option builtin_fetch_opti OPT__VERBOSITY(&verbosity), OPT_BOOL(0, "all", &all, N_("fetch from all remotes")), + OPT_BOOL(0, "set-upstream", &set_upstream, + N_("set upstream for git pull/fetch")), OPT_BOOL('a', "append", &append, N_("append to .git/FETCH_HEAD instead of overwriting")), OPT_STRING(0, "upload-pack", &upload_pack, N_("path"), @@@ -184,10 -169,6 +184,10 @@@ OPT_STRING_LIST(0, "negotiation-tip", &negotiation_tip, N_("revision"), N_("report that we have only objects reachable from this object")), OPT_PARSE_LIST_OBJECTS_FILTER(&filter_options), + OPT_BOOL(0, "auto-gc", &enable_auto_gc, + N_("run 'gc --auto' after fetching")), + OPT_BOOL(0, "show-forced-updates", &fetch_show_forced_updates, + N_("check for forced-updates on all updated branches")), OPT_END() }; @@@ -258,7 -239,6 +258,7 @@@ static int will_fetch(struct ref **head struct refname_hash_entry { struct hashmap_entry ent; /* must be the first member */ struct object_id oid; + int ignore; char refname[FLEX_ARRAY]; }; @@@ -307,11 -287,6 +307,11 @@@ static int refname_hash_exists(struct h return !!hashmap_get_from_hash(map, strhash(refname), refname); } +static void clear_item(struct refname_hash_entry *item) +{ + item->ignore = 1; +} + static void find_non_local_tags(const struct ref *refs, struct ref **head, struct ref ***tail) @@@ -344,7 -319,7 +344,7 @@@ !will_fetch(head, ref->old_oid.hash) && !has_object_file_with_flags(&item->oid, OBJECT_INFO_QUICK) && !will_fetch(head, item->oid.hash)) - oidclr(&item->oid); + clear_item(item); item = NULL; continue; } @@@ -358,7 -333,7 +358,7 @@@ if (item && !has_object_file_with_flags(&item->oid, OBJECT_INFO_QUICK) && !will_fetch(head, item->oid.hash)) - oidclr(&item->oid); + clear_item(item); item = NULL; @@@ -379,7 -354,7 +379,7 @@@ if (item && !has_object_file_with_flags(&item->oid, OBJECT_INFO_QUICK) && !will_fetch(head, item->oid.hash)) - oidclr(&item->oid); + clear_item(item); /* * For all the tags in the remote_refs_list, @@@ -387,21 -362,19 +387,21 @@@ */ for_each_string_list_item(remote_ref_item, &remote_refs_list) { const char *refname = remote_ref_item->string; + struct ref *rm; item = hashmap_get_from_hash(&remote_refs, strhash(refname), refname); if (!item) BUG("unseen remote ref?"); /* Unless we have already decided to ignore this item... */ - if (!is_null_oid(&item->oid)) { - struct ref *rm = alloc_ref(item->refname); - rm->peer_ref = alloc_ref(item->refname); - oidcpy(&rm->old_oid, &item->oid); - **tail = rm; - *tail = &rm->next; - } + if (item->ignore) + continue; + + rm = alloc_ref(item->refname); + rm->peer_ref = alloc_ref(item->refname); + oidcpy(&rm->old_oid, &item->oid); + **tail = rm; + *tail = &rm->next; } hashmap_free(&remote_refs, 1); string_list_clear(&remote_refs_list, 0); @@@ -726,7 -699,6 +726,7 @@@ static int update_local_ref(struct ref enum object_type type; struct branch *current_branch = branch_get(NULL); const char *pretty_ref = prettify_refname(ref->name); + int fast_forward = 0; type = oid_object_info(the_repository, &ref->new_oid, NULL); if (type < 0) @@@ -801,18 -773,9 +801,18 @@@ return r; } - if (in_merge_bases(current, updated)) { + if (fetch_show_forced_updates) { + uint64_t t_before = getnanotime(); + fast_forward = in_merge_bases(current, updated); + forced_updates_ms += (getnanotime() - t_before) / 1000000; + } else { + fast_forward = 1; + } + + if (fast_forward) { struct strbuf quickref = STRBUF_INIT; int r; + strbuf_add_unique_abbrev(&quickref, ¤t->object.oid, DEFAULT_ABBREV); strbuf_addstr(&quickref, ".."); strbuf_add_unique_abbrev(&quickref, &ref->new_oid, DEFAULT_ABBREV); @@@ -855,15 -818,6 +855,15 @@@ static int iterate_ref_map(void *cb_dat return 0; } +static const char warn_show_forced_updates[] = +N_("Fetch normally indicates which branches had a forced update,\n" + "but that check has been disabled. To re-enable, use '--show-forced-updates'\n" + "flag or run 'git config fetch.showForcedUpdates true'."); +static const char warn_time_show_forced_updates[] = +N_("It took %.2f seconds to check forced updates. You can use\n" + "'--no-show-forced-updates' or run 'git config fetch.showForcedUpdates false'\n" + " to avoid this check.\n"); + static int store_updated_refs(const char *raw_url, const char *remote_name, int connectivity_checked, struct ref *ref_map) { @@@ -1017,15 -971,6 +1017,15 @@@ " 'git remote prune %s' to remove any old, conflicting " "branches"), remote_name); + if (advice_fetch_show_forced_updates) { + if (!fetch_show_forced_updates) { + warning(_(warn_show_forced_updates)); + } else if (forced_updates_ms > FORCED_UPDATES_DELAY_WARNING_IN_MS) { + warning(_(warn_time_show_forced_updates), + forced_updates_ms / 1000.0); + } + } + abort: strbuf_release(¬e); free(url); @@@ -1243,13 -1188,10 +1243,10 @@@ static struct transport *prepare_transp if (update_shallow) set_option(transport, TRANS_OPT_UPDATE_SHALLOW, "yes"); if (filter_options.choice) { - struct strbuf expanded_filter_spec = STRBUF_INIT; - expand_list_objects_filter_spec(&filter_options, - &expanded_filter_spec); - set_option(transport, TRANS_OPT_LIST_OBJECTS_FILTER, - expanded_filter_spec.buf); + const char *spec = + expand_list_objects_filter_spec(&filter_options); + set_option(transport, TRANS_OPT_LIST_OBJECTS_FILTER, spec); set_option(transport, TRANS_OPT_FROM_PROMISOR, "1"); - strbuf_release(&expanded_filter_spec); } if (negotiation_tip.nr) { if (transport->smart_options) @@@ -1372,51 -1314,6 +1369,51 @@@ static int do_fetch(struct transport *t retcode = 1; goto cleanup; } + + if (set_upstream) { + struct branch *branch = branch_get("HEAD"); + struct ref *rm; + struct ref *source_ref = NULL; + + /* + * We're setting the upstream configuration for the + * current branch. The relevent upstream is the + * fetched branch that is meant to be merged with the + * current one, i.e. the one fetched to FETCH_HEAD. + * + * When there are several such branches, consider the + * request ambiguous and err on the safe side by doing + * nothing and just emit a warning. + */ + for (rm = ref_map; rm; rm = rm->next) { + if (!rm->peer_ref) { + if (source_ref) { + warning(_("multiple branch detected, incompatible with --set-upstream")); + goto skip; + } else { + source_ref = rm; + } + } + } + if (source_ref) { + if (!strcmp(source_ref->name, "HEAD") || + starts_with(source_ref->name, "refs/heads/")) + install_branch_config(0, + branch->name, + transport->remote->name, + source_ref->name); + else if (starts_with(source_ref->name, "refs/remotes/")) + warning(_("not setting upstream for a remote remote-tracking branch")); + else if (starts_with(source_ref->name, "refs/tags/")) + warning(_("not setting upstream for a remote tag")); + else + warning(_("unknown branch type")); + } else { + warning(_("no source branch found.\n" + "you need to specify exactly one branch with the --set-upstream option.")); + } + } + skip: free_refs(ref_map); /* if neither --no-tags nor --tags was specified, do automated tag @@@ -1524,7 -1421,7 +1521,7 @@@ static int fetch_multiple(struct string return errcode; } - argv_array_pushl(&argv, "fetch", "--append", NULL); + argv_array_pushl(&argv, "fetch", "--append", "--no-auto-gc", NULL); add_options_to_argv(&argv); for (i = 0; i < list->nr; i++) { @@@ -1560,27 -1457,37 +1557,27 @@@ static inline void fetch_one_setup_part * If no prior partial clone/fetch and the current fetch DID NOT * request a partial-fetch, do a normal fetch. */ - if (!repository_format_partial_clone && !filter_options.choice) + if (!has_promisor_remote() && !filter_options.choice) return; /* - * If this is the FIRST partial-fetch request, we enable partial - * on this repo and remember the given filter-spec as the default - * for subsequent fetches to this remote. + * If this is a partial-fetch request, we enable partial on + * this repo if not already enabled and remember the given + * filter-spec as the default for subsequent fetches to this + * remote. */ - if (!repository_format_partial_clone && filter_options.choice) { + if (filter_options.choice) { partial_clone_register(remote->name, &filter_options); return; } - /* - * We are currently limited to only ONE promisor remote and only - * allow partial-fetches from the promisor remote. - */ - if (strcmp(remote->name, repository_format_partial_clone)) { - if (filter_options.choice) - die(_("--filter can only be used with the remote " - "configured in extensions.partialClone")); - return; - } - /* * Do a partial-fetch from the promisor remote using either the * explicitly given filter-spec or inherit the filter-spec from * the config. */ if (!filter_options.choice) - partial_clone_get_default_filter_spec(&filter_options); + partial_clone_get_default_filter_spec(&filter_options, remote->name); return; } @@@ -1701,7 -1608,7 +1698,7 @@@ int cmd_fetch(int argc, const char **ar if (depth || deepen_since || deepen_not.nr) deepen = 1; - if (filter_options.choice && !repository_format_partial_clone) + if (filter_options.choice && !has_promisor_remote()) die("--filter can only be used when extensions.partialClone is set"); if (all) { @@@ -1735,7 -1642,7 +1732,7 @@@ } if (remote) { - if (filter_options.choice || repository_format_partial_clone) + if (filter_options.choice || has_promisor_remote()) fetch_one_setup_partial(remote); result = fetch_one(remote, argc, argv, prune_tags_ok); } else { @@@ -1762,15 -1669,13 +1759,15 @@@ string_list_clear(&list, 0); - close_all_packs(the_repository->objects); + close_object_store(the_repository->objects); - argv_array_pushl(&argv_gc_auto, "gc", "--auto", NULL); - if (verbosity < 0) - argv_array_push(&argv_gc_auto, "--quiet"); - run_command_v_opt(argv_gc_auto.argv, RUN_GIT_CMD); - argv_array_clear(&argv_gc_auto); + if (enable_auto_gc) { + argv_array_pushl(&argv_gc_auto, "gc", "--auto", NULL); + if (verbosity < 0) + argv_array_push(&argv_gc_auto, "--quiet"); + run_command_v_opt(argv_gc_auto.argv, RUN_GIT_CMD); + argv_array_clear(&argv_gc_auto); + } return result; } diff --combined builtin/rev-list.c index 301ccb970b,68acbe8fd2..b8dc2e1fba --- a/builtin/rev-list.c +++ b/builtin/rev-list.c @@@ -49,7 -49,6 +49,7 @@@ static const char rev_list_usage[] " --objects | --objects-edge\n" " --unpacked\n" " --header | --pretty\n" +" --[no-]object-names\n" " --abbrev= | --no-abbrev\n" " --abbrev-commit\n" " --left-right\n" @@@ -76,9 -75,6 +76,9 @@@ enum missing_action }; static enum missing_action arg_missing_action; +/* display only the oid of each object encountered */ +static int arg_show_object_names = 1; + #define DEFAULT_OIDSET_SIZE (16*1024) static void finish_commit(struct commit *commit); @@@ -259,10 -255,7 +259,10 @@@ static void show_object(struct object * display_progress(progress, ++progress_counter); if (info->flags & REV_LIST_QUIET) return; - show_object_with_name(stdout, obj, name); + if (arg_show_object_names) + show_object_with_name(stdout, obj, name); + else + printf("%s\n", oid_to_hex(&obj->oid)); } static void show_edge(struct commit *commit) @@@ -473,8 -466,10 +473,10 @@@ int cmd_rev_list(int argc, const char * die(_("object filtering requires --objects")); if (filter_options.choice == LOFC_SPARSE_OID && !filter_options.sparse_oid_value) - die(_("invalid sparse value '%s'"), - filter_options.filter_spec); + die( + _("invalid sparse value '%s'"), + list_objects_filter_spec( + &filter_options)); continue; } if (!strcmp(arg, ("--no-" CL_ARG__FILTER))) { @@@ -491,16 -486,6 +493,16 @@@ if (skip_prefix(arg, "--missing=", &arg)) continue; /* already handled above */ + if (!strcmp(arg, ("--no-object-names"))) { + arg_show_object_names = 0; + continue; + } + + if (!strcmp(arg, ("--object-names"))) { + arg_show_object_names = 1; + continue; + } + usage(rev_list_usage); } diff --combined cache.h index 3cbad5b603,cf5d70c196..5624e6c02d --- a/cache.h +++ b/cache.h @@@ -43,6 -43,30 +43,6 @@@ int git_deflate_end_gently(git_zstream int git_deflate(git_zstream *, int flush); unsigned long git_deflate_bound(git_zstream *, unsigned long); -/* The length in bytes and in hex digits of an object name (SHA-1 value). */ -#define GIT_SHA1_RAWSZ 20 -#define GIT_SHA1_HEXSZ (2 * GIT_SHA1_RAWSZ) -/* The block size of SHA-1. */ -#define GIT_SHA1_BLKSZ 64 - -/* The length in bytes and in hex digits of an object name (SHA-256 value). */ -#define GIT_SHA256_RAWSZ 32 -#define GIT_SHA256_HEXSZ (2 * GIT_SHA256_RAWSZ) -/* The block size of SHA-256. */ -#define GIT_SHA256_BLKSZ 64 - -/* The length in byte and in hex digits of the largest possible hash value. */ -#define GIT_MAX_RAWSZ GIT_SHA256_RAWSZ -#define GIT_MAX_HEXSZ GIT_SHA256_HEXSZ -/* The largest possible block size for any supported hash. */ -#define GIT_MAX_BLKSZ GIT_SHA256_BLKSZ - -struct object_id { - unsigned char hash[GIT_MAX_RAWSZ]; -}; - -#define the_hash_algo the_repository->hash_algo - #if defined(DT_UNKNOWN) && !defined(NO_D_TYPE_IN_DIRENT) #define DTYPE(de) ((de)->d_type) #else @@@ -636,6 -660,9 +636,9 @@@ int daemonize(void) * at least 'nr' entries; the number of entries currently allocated * is 'alloc', using the standard growing factor alloc_nr() macro. * + * Consider using ALLOC_GROW_BY instead of ALLOC_GROW as it has some + * added niceties. + * * DO NOT USE any expression with side-effect for 'x', 'nr', or 'alloc'. */ #define ALLOC_GROW(x, nr, alloc) \ @@@ -649,6 -676,25 +652,25 @@@ } \ } while (0) + /* + * Similar to ALLOC_GROW but handles updating of the nr value and + * zeroing the bytes of the newly-grown array elements. + * + * DO NOT USE any expression with side-effect for any of the + * arguments. + */ + #define ALLOC_GROW_BY(x, nr, increase, alloc) \ + do { \ + if (increase) { \ + size_t new_nr = nr + (increase); \ + if (new_nr < nr) \ + BUG("negative growth in ALLOC_GROW_BY"); \ + ALLOC_GROW(x, new_nr, alloc); \ + memset((x) + nr, 0, sizeof(*(x)) * (increase)); \ + nr = new_nr; \ + } \ + } while (0) + /* Initialize and use the cache information */ struct lock_file; void preload_index(struct index_state *index, @@@ -937,6 -983,8 +959,6 @@@ extern int grafts_replace_parents #define GIT_REPO_VERSION 0 #define GIT_REPO_VERSION_READ 1 extern int repository_format_precious_objects; -extern char *repository_format_partial_clone; -extern const char *core_partial_clone_filter_default; extern int repository_format_worktree_config; /* @@@ -1474,8 -1522,7 +1496,8 @@@ int df_name_compare(const char *name1, int name_compare(const char *name1, size_t len1, const char *name2, size_t len2); int cache_name_stage_compare(const char *name1, int len1, int stage1, const char *name2, int len2, int stage2); -void *read_object_with_reference(const struct object_id *oid, +void *read_object_with_reference(struct repository *r, + const struct object_id *oid, const char *required_type, unsigned long *size, struct object_id *oid_ret); @@@ -1734,7 -1781,6 +1756,7 @@@ void setup_pager(void) int pager_in_use(void); extern int pager_use_color; int term_columns(void); +void term_clear_line(void); int decimal_width(uintmax_t); int check_pager_config(const char *cmd); void prepare_pager_args(struct child_process *, const char *pager); @@@ -1761,8 -1807,8 +1783,8 @@@ int add_files_to_cache(const char *pref extern int diff_auto_refresh_index; /* match-trees.c */ -void shift_tree(const struct object_id *, const struct object_id *, struct object_id *, int); -void shift_tree_by(const struct object_id *, const struct object_id *, struct object_id *, const char *); +void shift_tree(struct repository *, const struct object_id *, const struct object_id *, struct object_id *, int); +void shift_tree_by(struct repository *, const struct object_id *, const struct object_id *, struct object_id *, const char *); /* * whitespace rules. diff --combined fetch-pack.c index d81f47c07b,72e13b0a1d..6ccc6294ea --- a/fetch-pack.c +++ b/fetch-pack.c @@@ -36,6 -36,7 +36,6 @@@ static int agent_supported static int server_supports_filtering; static struct lock_file shallow_lock; static const char *alternate_shallow_file; -static char *negotiation_algorithm; static struct strbuf fsck_msg_types = STRBUF_INIT; /* Remember to update object flag allocation in object.h */ @@@ -285,7 -286,7 +285,7 @@@ static int find_common(struct fetch_neg * we cannot trust the object flags). */ if (!args->no_dependents && - ((o = lookup_object(the_repository, remote->hash)) != NULL) && + ((o = lookup_object(the_repository, remote)) != NULL) && (o->flags & COMPLETE)) { continue; } @@@ -338,12 -339,9 +338,9 @@@ } } if (server_supports_filtering && args->filter_options.choice) { - struct strbuf expanded_filter_spec = STRBUF_INIT; - expand_list_objects_filter_spec(&args->filter_options, - &expanded_filter_spec); - packet_buf_write(&req_buf, "filter %s", - expanded_filter_spec.buf); - strbuf_release(&expanded_filter_spec); + const char *spec = + expand_list_objects_filter_spec(&args->filter_options); + packet_buf_write(&req_buf, "filter %s", spec); } packet_buf_flush(&req_buf); state_len = req_buf.len; @@@ -363,7 -361,7 +360,7 @@@ if (skip_prefix(reader.line, "unshallow ", &arg)) { if (get_oid_hex(arg, &oid)) die(_("invalid unshallow line: %s"), reader.line); - if (!lookup_object(the_repository, oid.hash)) + if (!lookup_object(the_repository, &oid)) die(_("object not found: %s"), reader.line); /* make sure that it is parsed as shallow */ if (!parse_object(the_repository, &oid)) @@@ -706,7 -704,7 +703,7 @@@ static void mark_complete_and_common_re for (ref = *refs; ref; ref = ref->next) { struct object *o = deref_tag(the_repository, lookup_object(the_repository, - ref->old_oid.hash), + &ref->old_oid), NULL, 0); if (!o || o->type != OBJ_COMMIT || !(o->flags & COMPLETE)) @@@ -733,7 -731,7 +730,7 @@@ static int everything_local(struct fetc const struct object_id *remote = &ref->old_oid; struct object *o; - o = lookup_object(the_repository, remote->hash); + o = lookup_object(the_repository, remote); if (!o || !(o->flags & COMPLETE)) { retval = 0; print_verbose(args, "want %s (%s)", oid_to_hex(remote), @@@ -891,96 -889,82 +888,96 @@@ static struct ref *do_fetch_pack(struc struct shallow_info *si, char **pack_lockfile) { + struct repository *r = the_repository; struct ref *ref = copy_ref_list(orig_ref); struct object_id oid; const char *agent_feature; int agent_len; struct fetch_negotiator negotiator; - fetch_negotiator_init(&negotiator, negotiation_algorithm); + fetch_negotiator_init(r, &negotiator); sort_ref_list(&ref, ref_compare_name); QSORT(sought, nr_sought, cmp_ref_by_name); - if ((args->depth > 0 || is_repository_shallow(the_repository)) && !server_supports("shallow")) + if ((agent_feature = server_feature_value("agent", &agent_len))) { + agent_supported = 1; + if (agent_len) + print_verbose(args, _("Server version is %.*s"), + agent_len, agent_feature); + } + + if (server_supports("shallow")) + print_verbose(args, _("Server supports %s"), "shallow"); + else if (args->depth > 0 || is_repository_shallow(r)) die(_("Server does not support shallow clients")); if (args->depth > 0 || args->deepen_since || args->deepen_not) args->deepen = 1; if (server_supports("multi_ack_detailed")) { - print_verbose(args, _("Server supports multi_ack_detailed")); + print_verbose(args, _("Server supports %s"), "multi_ack_detailed"); multi_ack = 2; if (server_supports("no-done")) { - print_verbose(args, _("Server supports no-done")); + print_verbose(args, _("Server supports %s"), "no-done"); if (args->stateless_rpc) no_done = 1; } } else if (server_supports("multi_ack")) { - print_verbose(args, _("Server supports multi_ack")); + print_verbose(args, _("Server supports %s"), "multi_ack"); multi_ack = 1; } if (server_supports("side-band-64k")) { - print_verbose(args, _("Server supports side-band-64k")); + print_verbose(args, _("Server supports %s"), "side-band-64k"); use_sideband = 2; } else if (server_supports("side-band")) { - print_verbose(args, _("Server supports side-band")); + print_verbose(args, _("Server supports %s"), "side-band"); use_sideband = 1; } if (server_supports("allow-tip-sha1-in-want")) { - print_verbose(args, _("Server supports allow-tip-sha1-in-want")); + print_verbose(args, _("Server supports %s"), "allow-tip-sha1-in-want"); allow_unadvertised_object_request |= ALLOW_TIP_SHA1; } if (server_supports("allow-reachable-sha1-in-want")) { - print_verbose(args, _("Server supports allow-reachable-sha1-in-want")); + print_verbose(args, _("Server supports %s"), "allow-reachable-sha1-in-want"); allow_unadvertised_object_request |= ALLOW_REACHABLE_SHA1; } - if (!server_supports("thin-pack")) + if (server_supports("thin-pack")) + print_verbose(args, _("Server supports %s"), "thin-pack"); + else args->use_thin_pack = 0; - if (!server_supports("no-progress")) + if (server_supports("no-progress")) + print_verbose(args, _("Server supports %s"), "no-progress"); + else args->no_progress = 0; - if (!server_supports("include-tag")) + if (server_supports("include-tag")) + print_verbose(args, _("Server supports %s"), "include-tag"); + else args->include_tag = 0; if (server_supports("ofs-delta")) - print_verbose(args, _("Server supports ofs-delta")); + print_verbose(args, _("Server supports %s"), "ofs-delta"); else prefer_ofs_delta = 0; if (server_supports("filter")) { server_supports_filtering = 1; - print_verbose(args, _("Server supports filter")); + print_verbose(args, _("Server supports %s"), "filter"); } else if (args->filter_options.choice) { warning("filtering not recognized by server, ignoring"); } - if ((agent_feature = server_feature_value("agent", &agent_len))) { - agent_supported = 1; - if (agent_len) - print_verbose(args, _("Server version is %.*s"), - agent_len, agent_feature); - } - if (server_supports("deepen-since")) + if (server_supports("deepen-since")) { + print_verbose(args, _("Server supports %s"), "deepen-since"); deepen_since_ok = 1; - else if (args->deepen_since) + } else if (args->deepen_since) die(_("Server does not support --shallow-since")); - if (server_supports("deepen-not")) + if (server_supports("deepen-not")) { + print_verbose(args, _("Server supports %s"), "deepen-not"); deepen_not_ok = 1; - else if (args->deepen_not) + } else if (args->deepen_not) die(_("Server does not support --shallow-exclude")); - if (!server_supports("deepen-relative") && args->deepen_relative) + if (server_supports("deepen-relative")) + print_verbose(args, _("Server supports %s"), "deepen-relative"); + else if (args->deepen_relative) die(_("Server does not support --deepen")); if (!args->no_dependents) { @@@ -1061,7 -1045,7 +1058,7 @@@ static void add_wants(int no_dependents * we cannot trust the object flags). */ if (!no_dependents && - ((o = lookup_object(the_repository, remote->hash)) != NULL) && + ((o = lookup_object(the_repository, remote)) != NULL) && (o->flags & COMPLETE)) { continue; } @@@ -1112,7 -1096,7 +1109,7 @@@ static int add_haves(struct fetch_negot } static int send_fetch_request(struct fetch_negotiator *negotiator, int fd_out, - const struct fetch_pack_args *args, + struct fetch_pack_args *args, const struct ref *wants, struct oidset *common, int *haves_to_send, int *in_vain, int sideband_all) @@@ -1153,13 -1137,10 +1150,10 @@@ /* Add filter */ if (server_supports_feature("fetch", "filter", 0) && args->filter_options.choice) { - struct strbuf expanded_filter_spec = STRBUF_INIT; + const char *spec = + expand_list_objects_filter_spec(&args->filter_options); print_verbose(args, _("Server supports filter")); - expand_list_objects_filter_spec(&args->filter_options, - &expanded_filter_spec); - packet_buf_write(&req_buf, "filter %s", - expanded_filter_spec.buf); - strbuf_release(&expanded_filter_spec); + packet_buf_write(&req_buf, "filter %s", spec); } else if (args->filter_options.choice) { warning("filtering not recognized by server, ignoring"); } @@@ -1288,7 -1269,7 +1282,7 @@@ static void receive_shallow_info(struc if (skip_prefix(reader->line, "unshallow ", &arg)) { if (get_oid_hex(arg, &oid)) die(_("invalid unshallow line: %s"), reader->line); - if (!lookup_object(the_repository, oid.hash)) + if (!lookup_object(the_repository, &oid)) die(_("object not found: %s"), reader->line); /* make sure that it is parsed as shallow */ if (!parse_object(the_repository, &oid)) @@@ -1379,7 -1360,6 +1373,7 @@@ static struct ref *do_fetch_pack_v2(str struct shallow_info *si, char **pack_lockfile) { + struct repository *r = the_repository; struct ref *ref = copy_ref_list(orig_ref); enum fetch_state state = FETCH_CHECK_LOCAL; struct oidset common = OIDSET_INIT; @@@ -1387,7 -1367,7 +1381,7 @@@ int in_vain = 0; int haves_to_send = INITIAL_FLUSH; struct fetch_negotiator negotiator; - fetch_negotiator_init(&negotiator, negotiation_algorithm); + fetch_negotiator_init(r, &negotiator); packet_reader_init(&reader, fd[0], NULL, 0, PACKET_READ_CHOMP_NEWLINE | PACKET_READ_DIE_ON_ERR_PACKET); @@@ -1506,6 -1486,8 +1500,6 @@@ static void fetch_pack_config(void git_config_get_bool("repack.usedeltabaseoffset", &prefer_ofs_delta); git_config_get_bool("fetch.fsckobjects", &fetch_fsck_objects); git_config_get_bool("transfer.fsckobjects", &transfer_fsck_objects); - git_config_get_string("fetch.negotiationalgorithm", - &negotiation_algorithm); git_config(fetch_pack_config_cb, NULL); } diff --combined list-objects-filter-options.c index 28c571f922,ba1425cb4a..4d88bfe64a --- a/list-objects-filter-options.c +++ b/list-objects-filter-options.c @@@ -6,7 -6,13 +6,14 @@@ #include "list-objects.h" #include "list-objects-filter.h" #include "list-objects-filter-options.h" +#include "promisor-remote.h" + #include "trace.h" + #include "url.h" + + static int parse_combine_filter( + struct list_objects_filter_options *filter_options, + const char *arg, + struct strbuf *errbuf); /* * Parse value of the argument to the "filter" keyword. @@@ -30,19 -36,8 +37,11 @@@ static int gently_parse_list_objects_fi { const char *v0; + if (!arg) + return 0; + - if (filter_options->choice) { - if (errbuf) { - strbuf_addstr( - errbuf, - _("multiple filter-specs cannot be combined")); - } - return 1; - } - - filter_options->filter_spec = strdup(arg); + if (filter_options->choice) + BUG("filter_options already populated"); if (!strcmp(arg, "blob:none")) { filter_options->choice = LOFC_BLOB_NONE; @@@ -56,11 -51,7 +55,7 @@@ } else if (skip_prefix(arg, "tree:", &v0)) { if (!git_parse_ulong(v0, &filter_options->tree_exclude_depth)) { - if (errbuf) { - strbuf_addstr( - errbuf, - _("expected 'tree:'")); - } + strbuf_addstr(errbuf, _("expected 'tree:'")); return 1; } filter_options->choice = LOFC_TREE_DEPTH; @@@ -88,104 -79,295 +83,298 @@@ _("sparse:path filters support has been dropped")); } return 1; + + } else if (skip_prefix(arg, "combine:", &v0)) { + return parse_combine_filter(filter_options, v0, errbuf); + } /* * Please update _git_fetch() in git-completion.bash when you * add new filters */ - if (errbuf) - strbuf_addf(errbuf, _("invalid filter-spec '%s'"), arg); + strbuf_addf(errbuf, _("invalid filter-spec '%s'"), arg); memset(filter_options, 0, sizeof(*filter_options)); return 1; } - int parse_list_objects_filter(struct list_objects_filter_options *filter_options, - const char *arg) + static const char *RESERVED_NON_WS = "~`!@#$^&*()[]{}\\;'\",<>?"; + + static int has_reserved_character( + struct strbuf *sub_spec, struct strbuf *errbuf) { - struct strbuf buf = STRBUF_INIT; - if (gently_parse_list_objects_filter(filter_options, arg, &buf)) - die("%s", buf.buf); + const char *c = sub_spec->buf; + while (*c) { + if (*c <= ' ' || strchr(RESERVED_NON_WS, *c)) { + strbuf_addf( + errbuf, + _("must escape char in sub-filter-spec: '%c'"), + *c); + return 1; + } + c++; + } + return 0; } + static int parse_combine_subfilter( + struct list_objects_filter_options *filter_options, + struct strbuf *subspec, + struct strbuf *errbuf) + { + size_t new_index = filter_options->sub_nr; + char *decoded; + int result; + + ALLOC_GROW_BY(filter_options->sub, filter_options->sub_nr, 1, + filter_options->sub_alloc); + + decoded = url_percent_decode(subspec->buf); + + result = has_reserved_character(subspec, errbuf) || + gently_parse_list_objects_filter( + &filter_options->sub[new_index], decoded, errbuf); + + free(decoded); + return result; + } + + static int parse_combine_filter( + struct list_objects_filter_options *filter_options, + const char *arg, + struct strbuf *errbuf) + { + struct strbuf **subspecs = strbuf_split_str(arg, '+', 0); + size_t sub; + int result = 0; + + if (!subspecs[0]) { + strbuf_addstr(errbuf, _("expected something after combine:")); + result = 1; + goto cleanup; + } + + for (sub = 0; subspecs[sub] && !result; sub++) { + if (subspecs[sub + 1]) { + /* + * This is not the last subspec. Remove trailing "+" so + * we can parse it. + */ + size_t last = subspecs[sub]->len - 1; + assert(subspecs[sub]->buf[last] == '+'); + strbuf_remove(subspecs[sub], last, 1); + } + result = parse_combine_subfilter( + filter_options, subspecs[sub], errbuf); + } + + filter_options->choice = LOFC_COMBINE; + + cleanup: + strbuf_list_free(subspecs); + if (result) { + list_objects_filter_release(filter_options); + memset(filter_options, 0, sizeof(*filter_options)); + } + return result; + } + + static int allow_unencoded(char ch) + { + if (ch <= ' ' || ch == '%' || ch == '+') + return 0; + return !strchr(RESERVED_NON_WS, ch); + } + + static void filter_spec_append_urlencode( + struct list_objects_filter_options *filter, const char *raw) + { + struct strbuf buf = STRBUF_INIT; + strbuf_addstr_urlencode(&buf, raw, allow_unencoded); + trace_printf("Add to combine filter-spec: %s\n", buf.buf); + string_list_append(&filter->filter_spec, strbuf_detach(&buf, NULL)); + } + + /* + * Changes filter_options into an equivalent LOFC_COMBINE filter options + * instance. Does not do anything if filter_options is already LOFC_COMBINE. + */ + static void transform_to_combine_type( + struct list_objects_filter_options *filter_options) + { + assert(filter_options->choice); + if (filter_options->choice == LOFC_COMBINE) + return; + { + const int initial_sub_alloc = 2; + struct list_objects_filter_options *sub_array = + xcalloc(initial_sub_alloc, sizeof(*sub_array)); + sub_array[0] = *filter_options; + memset(filter_options, 0, sizeof(*filter_options)); + filter_options->sub = sub_array; + filter_options->sub_alloc = initial_sub_alloc; + } + filter_options->sub_nr = 1; + filter_options->choice = LOFC_COMBINE; + string_list_append(&filter_options->filter_spec, xstrdup("combine:")); + filter_spec_append_urlencode( + filter_options, + list_objects_filter_spec(&filter_options->sub[0])); + /* + * We don't need the filter_spec strings for subfilter specs, only the + * top level. + */ + string_list_clear(&filter_options->sub[0].filter_spec, /*free_util=*/0); + } + + void list_objects_filter_die_if_populated( + struct list_objects_filter_options *filter_options) + { + if (filter_options->choice) + die(_("multiple filter-specs cannot be combined")); + } + + void parse_list_objects_filter( + struct list_objects_filter_options *filter_options, + const char *arg) + { + struct strbuf errbuf = STRBUF_INIT; + int parse_error; + + if (!filter_options->choice) { + string_list_append(&filter_options->filter_spec, xstrdup(arg)); + + parse_error = gently_parse_list_objects_filter( + filter_options, arg, &errbuf); + } else { + /* + * Make filter_options an LOFC_COMBINE spec so we can trivially + * add subspecs to it. + */ + transform_to_combine_type(filter_options); + + string_list_append(&filter_options->filter_spec, xstrdup("+")); + filter_spec_append_urlencode(filter_options, arg); + ALLOC_GROW_BY(filter_options->sub, filter_options->sub_nr, 1, + filter_options->sub_alloc); + + parse_error = gently_parse_list_objects_filter( + &filter_options->sub[filter_options->sub_nr - 1], arg, + &errbuf); + } + if (parse_error) + die("%s", errbuf.buf); + } + int opt_parse_list_objects_filter(const struct option *opt, const char *arg, int unset) { struct list_objects_filter_options *filter_options = opt->value; - if (unset || !arg) { + if (unset || !arg) list_objects_filter_set_no_filter(filter_options); - return 0; + else + parse_list_objects_filter(filter_options, arg); + return 0; + } + + const char *list_objects_filter_spec(struct list_objects_filter_options *filter) + { + if (!filter->filter_spec.nr) + BUG("no filter_spec available for this filter"); + if (filter->filter_spec.nr != 1) { + struct strbuf concatted = STRBUF_INIT; + strbuf_add_separated_string_list( + &concatted, "", &filter->filter_spec); + string_list_clear(&filter->filter_spec, /*free_util=*/0); + string_list_append( + &filter->filter_spec, strbuf_detach(&concatted, NULL)); } - return parse_list_objects_filter(filter_options, arg); + return filter->filter_spec.items[0].string; } - void expand_list_objects_filter_spec( - const struct list_objects_filter_options *filter, - struct strbuf *expanded_spec) + const char *expand_list_objects_filter_spec( + struct list_objects_filter_options *filter) { - strbuf_init(expanded_spec, strlen(filter->filter_spec)); - if (filter->choice == LOFC_BLOB_LIMIT) - strbuf_addf(expanded_spec, "blob:limit=%lu", + if (filter->choice == LOFC_BLOB_LIMIT) { + struct strbuf expanded_spec = STRBUF_INIT; + strbuf_addf(&expanded_spec, "blob:limit=%lu", filter->blob_limit_value); - else if (filter->choice == LOFC_TREE_DEPTH) - strbuf_addf(expanded_spec, "tree:%lu", - filter->tree_exclude_depth); - else - strbuf_addstr(expanded_spec, filter->filter_spec); + string_list_clear(&filter->filter_spec, /*free_util=*/0); + string_list_append( + &filter->filter_spec, + strbuf_detach(&expanded_spec, NULL)); + } + + return list_objects_filter_spec(filter); } void list_objects_filter_release( struct list_objects_filter_options *filter_options) { - free(filter_options->filter_spec); + size_t sub; + + if (!filter_options) + return; + string_list_clear(&filter_options->filter_spec, /*free_util=*/0); free(filter_options->sparse_oid_value); + for (sub = 0; sub < filter_options->sub_nr; sub++) + list_objects_filter_release(&filter_options->sub[sub]); + free(filter_options->sub); memset(filter_options, 0, sizeof(*filter_options)); } void partial_clone_register( const char *remote, - const struct list_objects_filter_options *filter_options) + struct list_objects_filter_options *filter_options) { - /* - * Record the name of the partial clone remote in the - * config and in the global variable -- the latter is - * used throughout to indicate that partial clone is - * enabled and to expect missing objects. - */ - if (repository_format_partial_clone && - *repository_format_partial_clone && - strcmp(remote, repository_format_partial_clone)) - die(_("cannot change partial clone promisor remote")); + char *cfg_name; + char *filter_name; - git_config_set("core.repositoryformatversion", "1"); - git_config_set("extensions.partialclone", remote); + /* Check if it is already registered */ + if (!promisor_remote_find(remote)) { + git_config_set("core.repositoryformatversion", "1"); - repository_format_partial_clone = xstrdup(remote); + /* Add promisor config for the remote */ + cfg_name = xstrfmt("remote.%s.promisor", remote); + git_config_set(cfg_name, "true"); + free(cfg_name); + } /* * Record the initial filter-spec in the config as * the default for subsequent fetches from this remote. */ - core_partial_clone_filter_default = - xstrdup(expand_list_objects_filter_spec(filter_options)); - git_config_set("core.partialclonefilter", - core_partial_clone_filter_default); + filter_name = xstrfmt("remote.%s.partialclonefilter", remote); - git_config_set(filter_name, filter_options->filter_spec); ++ /* NEEDSWORK: 'expand' result leaking??? */ ++ git_config_set(filter_name, ++ expand_list_objects_filter_spec(filter_options)); + free(filter_name); + + /* Make sure the config info are reset */ + promisor_remote_reinit(); } void partial_clone_get_default_filter_spec( - struct list_objects_filter_options *filter_options) + struct list_objects_filter_options *filter_options, + const char *remote) { + struct promisor_remote *promisor = promisor_remote_find(remote); + struct strbuf errbuf = STRBUF_INIT; /* * Parse default value, but silently ignore it if it is invalid. */ - if (promisor) - gently_parse_list_objects_filter(filter_options, - promisor->partial_clone_filter, - NULL); - if (!core_partial_clone_filter_default) ++ if (!promisor) + return; + + string_list_append(&filter_options->filter_spec, - core_partial_clone_filter_default); ++ promisor->partial_clone_filter); + gently_parse_list_objects_filter(filter_options, - core_partial_clone_filter_default, ++ promisor->partial_clone_filter, + &errbuf); + strbuf_release(&errbuf); } diff --combined list-objects-filter-options.h index 8deaa287b5,db37dfb34a..b63c5ee1a3 --- a/list-objects-filter-options.h +++ b/list-objects-filter-options.h @@@ -2,7 -2,7 +2,7 @@@ #define LIST_OBJECTS_FILTER_OPTIONS_H #include "parse-options.h" - #include "strbuf.h" + #include "string-list.h" /* * The list of defined filters for list-objects. @@@ -13,6 -13,7 +13,7 @@@ enum list_objects_filter_choice LOFC_BLOB_LIMIT, LOFC_TREE_DEPTH, LOFC_SPARSE_OID, + LOFC_COMBINE, LOFC__COUNT /* must be last */ }; @@@ -23,8 -24,10 +24,10 @@@ struct list_objects_filter_options * commands that launch filtering sub-processes, or for communication * over the network, don't use this value; use the result of * expand_list_objects_filter_spec() instead. + * To get the raw filter spec given by the user, use the result of + * list_objects_filter_spec(). */ - char *filter_spec; + struct string_list filter_spec; /* * 'choice' is determined by parsing the filter-spec. This indicates @@@ -38,19 -41,40 +41,40 @@@ unsigned int no_filter : 1; /* - * Parsed values (fields) from within the filter-spec. These are - * choice-specific; not all values will be defined for any given - * choice. + * BEGIN choice-specific parsed values from within the filter-spec. Only + * some values will be defined for any given choice. */ + struct object_id *sparse_oid_value; unsigned long blob_limit_value; unsigned long tree_exclude_depth; + + /* LOFC_COMBINE values */ + + /* This array contains all the subfilters which this filter combines. */ + size_t sub_nr, sub_alloc; + struct list_objects_filter_options *sub; + + /* + * END choice-specific parsed values. + */ }; /* Normalized command line arguments */ #define CL_ARG__FILTER "filter" - int parse_list_objects_filter( + void list_objects_filter_die_if_populated( + struct list_objects_filter_options *filter_options); + + /* + * Parses the filter spec string given by arg and either (1) simply places the + * result in filter_options if it is not yet populated or (2) combines it with + * the filter already in filter_options if it is already populated. In the case + * of (2), the filter specs are combined as if specified with 'combine:'. + * + * Dies and prints a user-facing message if an error occurs. + */ + void parse_list_objects_filter( struct list_objects_filter_options *filter_options, const char *arg); @@@ -65,13 -89,22 +89,22 @@@ int opt_parse_list_objects_filter(cons /* * Translates abbreviated numbers in the filter's filter_spec into their * fully-expanded forms (e.g., "limit:blob=1k" becomes "limit:blob=1024"). + * Returns a string owned by the list_objects_filter_options object. * - * This form should be used instead of the raw filter_spec field when - * communicating with a remote process or subprocess. + * This form should be used instead of the raw list_objects_filter_spec() + * value when communicating with a remote process or subprocess. */ - void expand_list_objects_filter_spec( - const struct list_objects_filter_options *filter, - struct strbuf *expanded_spec); + const char *expand_list_objects_filter_spec( + struct list_objects_filter_options *filter); + + /* + * Returns the filter spec string more or less in the form as the user + * entered it. This form of the filter_spec can be used in user-facing + * messages. Returns a string owned by the list_objects_filter_options + * object. + */ + const char *list_objects_filter_spec( + struct list_objects_filter_options *filter); void list_objects_filter_release( struct list_objects_filter_options *filter_options); @@@ -85,9 -118,8 +118,9 @@@ static inline void list_objects_filter_ void partial_clone_register( const char *remote, - const struct list_objects_filter_options *filter_options); + struct list_objects_filter_options *filter_options); void partial_clone_get_default_filter_spec( - struct list_objects_filter_options *filter_options); + struct list_objects_filter_options *filter_options, + const char *remote); #endif /* LIST_OBJECTS_FILTER_OPTIONS_H */ diff --combined strbuf.c index d30f916858,60ab5144f2..aa48d179a9 --- a/strbuf.c +++ b/strbuf.c @@@ -774,8 -774,10 +774,10 @@@ void strbuf_addstr_xml_quoted(struct st } } - static int is_rfc3986_reserved(char ch) + int is_rfc3986_reserved_or_unreserved(char ch) { + if (is_rfc3986_unreserved(ch)) + return 1; switch (ch) { case '!': case '*': case '\'': case '(': case ')': case ';': case ':': case '@': case '&': case '=': case '+': case '$': @@@ -785,20 -787,19 +787,19 @@@ return 0; } - static int is_rfc3986_unreserved(char ch) + int is_rfc3986_unreserved(char ch) { return isalnum(ch) || ch == '-' || ch == '_' || ch == '.' || ch == '~'; } static void strbuf_add_urlencode(struct strbuf *sb, const char *s, size_t len, - int reserved) + char_predicate allow_unencoded_fn) { strbuf_grow(sb, len); while (len--) { char ch = *s++; - if (is_rfc3986_unreserved(ch) || - (!reserved && is_rfc3986_reserved(ch))) + if (allow_unencoded_fn(ch)) strbuf_addch(sb, ch); else strbuf_addf(sb, "%%%02x", (unsigned char)ch); @@@ -806,62 -807,30 +807,62 @@@ } void strbuf_addstr_urlencode(struct strbuf *sb, const char *s, - int reserved) + char_predicate allow_unencoded_fn) { - strbuf_add_urlencode(sb, s, strlen(s), reserved); + strbuf_add_urlencode(sb, s, strlen(s), allow_unencoded_fn); } -void strbuf_humanise_bytes(struct strbuf *buf, off_t bytes) +static void strbuf_humanise(struct strbuf *buf, off_t bytes, + int humanise_rate) { if (bytes > 1 << 30) { - strbuf_addf(buf, "%u.%2.2u GiB", + strbuf_addf(buf, + humanise_rate == 0 ? + /* TRANSLATORS: IEC 80000-13:2008 gibibyte */ + _("%u.%2.2u GiB") : + /* TRANSLATORS: IEC 80000-13:2008 gibibyte/second */ + _("%u.%2.2u GiB/s"), (unsigned)(bytes >> 30), (unsigned)(bytes & ((1 << 30) - 1)) / 10737419); } else if (bytes > 1 << 20) { unsigned x = bytes + 5243; /* for rounding */ - strbuf_addf(buf, "%u.%2.2u MiB", + strbuf_addf(buf, + humanise_rate == 0 ? + /* TRANSLATORS: IEC 80000-13:2008 mebibyte */ + _("%u.%2.2u MiB") : + /* TRANSLATORS: IEC 80000-13:2008 mebibyte/second */ + _("%u.%2.2u MiB/s"), x >> 20, ((x & ((1 << 20) - 1)) * 100) >> 20); } else if (bytes > 1 << 10) { unsigned x = bytes + 5; /* for rounding */ - strbuf_addf(buf, "%u.%2.2u KiB", + strbuf_addf(buf, + humanise_rate == 0 ? + /* TRANSLATORS: IEC 80000-13:2008 kibibyte */ + _("%u.%2.2u KiB") : + /* TRANSLATORS: IEC 80000-13:2008 kibibyte/second */ + _("%u.%2.2u KiB/s"), x >> 10, ((x & ((1 << 10) - 1)) * 100) >> 10); } else { - strbuf_addf(buf, "%u bytes", (unsigned)bytes); + strbuf_addf(buf, + humanise_rate == 0 ? + /* TRANSLATORS: IEC 80000-13:2008 byte */ + Q_("%u byte", "%u bytes", (unsigned)bytes) : + /* TRANSLATORS: IEC 80000-13:2008 byte/second */ + Q_("%u byte/s", "%u bytes/s", (unsigned)bytes), + (unsigned)bytes); } } +void strbuf_humanise_bytes(struct strbuf *buf, off_t bytes) +{ + strbuf_humanise(buf, bytes, 0); +} + +void strbuf_humanise_rate(struct strbuf *buf, off_t bytes) +{ + strbuf_humanise(buf, bytes, 1); +} + void strbuf_add_absolute_path(struct strbuf *sb, const char *path) { if (!*path) diff --combined strbuf.h index f62278a0be,346d722492..84cf969721 --- a/strbuf.h +++ b/strbuf.h @@@ -372,12 -372,6 +372,12 @@@ void strbuf_addbuf_percentquote(struct */ void strbuf_humanise_bytes(struct strbuf *buf, off_t bytes); +/** + * Append the given byte rate as a human-readable string (i.e. 12.23 KiB/s, + * 3.50 MiB/s). + */ +void strbuf_humanise_rate(struct strbuf *buf, off_t bytes); + /** * Add a formatted string to the buffer. */ @@@ -672,8 -666,13 +672,13 @@@ void strbuf_branchname(struct strbuf *s */ int strbuf_check_branch_ref(struct strbuf *sb, const char *name); + typedef int (*char_predicate)(char ch); + + int is_rfc3986_unreserved(char ch); + int is_rfc3986_reserved_or_unreserved(char ch); + void strbuf_addstr_urlencode(struct strbuf *sb, const char *name, - int reserved); + char_predicate allow_unencoded_fn); __attribute__((format (printf,1,2))) int printf_ln(const char *fmt, ...); diff --combined t/t5616-partial-clone.sh index 73cd95812f,32b7d72f3c..fc634a56b2 --- a/t/t5616-partial-clone.sh +++ b/t/t5616-partial-clone.sh @@@ -42,8 -42,8 +42,8 @@@ test_expect_success 'do partial clone 1 test_cmp expect_1.oids observed.oids && test "$(git -C pc1 config --local core.repositoryformatversion)" = "1" && - test "$(git -C pc1 config --local extensions.partialclone)" = "origin" && - test "$(git -C pc1 config --local core.partialclonefilter)" = "blob:none" + test "$(git -C pc1 config --local remote.origin.promisor)" = "true" && + test "$(git -C pc1 config --local remote.origin.partialclonefilter)" = "blob:none" ' # checkout master to force dynamic object fetch of blobs at HEAD. @@@ -208,6 -208,25 +208,25 @@@ test_expect_success 'use fsck before an test_cmp unique_types.expected unique_types.observed ' + test_expect_success 'implicitly construct combine: filter with repeated flags' ' + GIT_TRACE=$(pwd)/trace git clone --bare \ + --filter=blob:none --filter=tree:1 \ + "file://$(pwd)/srv.bare" pc2 && + grep "trace:.* git pack-objects .*--filter=combine:blob:none+tree:1" \ + trace && + git -C pc2 rev-list --objects --missing=allow-any HEAD >objects && + + # We should have gotten some root trees. + grep " $" objects && + # Should not have gotten any non-root trees or blobs. + ! grep " ." objects && + + xargs -n 1 git -C pc2 cat-file -t types && + sort -u types >unique_types.actual && + test_write_lines commit tree >unique_types.expected && + test_cmp unique_types.expected unique_types.actual + ' + test_expect_success 'partial clone fetches blobs pointed to by refs even if normally filtered out' ' rm -rf src dst && git init src && @@@ -417,7 -436,4 +436,7 @@@ test_expect_success 'tolerate server se ! test -e "$HTTPD_ROOT_PATH/one-time-sed" ' +# DO NOT add non-httpd-specific tests here, because the last part of this +# test script is only executed when httpd is available and enabled. + test_done diff --combined transport-helper.c index 6b05a88faf,0a34544df0..3169111772 --- a/transport-helper.c +++ b/transport-helper.c @@@ -682,13 -682,9 +682,9 @@@ static int fetch(struct transport *tran set_helper_option(transport, "update-shallow", "true"); if (data->transport_options.filter_options.choice) { - struct strbuf expanded_filter_spec = STRBUF_INIT; - expand_list_objects_filter_spec( - &data->transport_options.filter_options, - &expanded_filter_spec); - set_helper_option(transport, "filter", - expanded_filter_spec.buf); - strbuf_release(&expanded_filter_spec); + const char *spec = expand_list_objects_filter_spec( + &data->transport_options.filter_options); + set_helper_option(transport, "filter", spec); } if (data->transport_options.negotiation_tips) @@@ -853,7 -849,6 +849,7 @@@ static int push_refs_with_push(struct t { int force_all = flags & TRANSPORT_PUSH_FORCE; int mirror = flags & TRANSPORT_PUSH_MIRROR; + int atomic = flags & TRANSPORT_PUSH_ATOMIC; struct helper_data *data = transport->data; struct strbuf buf = STRBUF_INIT; struct ref *ref; @@@ -873,11 -868,6 +869,11 @@@ case REF_STATUS_REJECT_NONFASTFORWARD: case REF_STATUS_REJECT_STALE: case REF_STATUS_REJECT_ALREADY_EXISTS: + if (atomic) { + string_list_clear(&cas_options, 0); + return 0; + } else + continue; case REF_STATUS_UPTODATE: continue; default: diff --combined transport.c index 778c60bf57,ee7dd1c062..6324cfc482 --- a/transport.c +++ b/transport.c @@@ -224,6 -224,7 +224,7 @@@ static int set_git_option(struct git_tr opts->no_dependents = !!value; return 0; } else if (!strcmp(name, TRANS_OPT_LIST_OBJECTS_FILTER)) { + list_objects_filter_die_if_populated(&opts->filter_options); parse_list_objects_filter(&opts->filter_options, value); return 0; } @@@ -1226,20 -1227,6 +1227,20 @@@ int transport_push(struct repository *r err = push_had_errors(remote_refs); ret = push_ret | err; + if ((flags & TRANSPORT_PUSH_ATOMIC) && err) { + struct ref *it; + for (it = remote_refs; it; it = it->next) + switch (it->status) { + case REF_STATUS_NONE: + case REF_STATUS_UPTODATE: + case REF_STATUS_OK: + it->status = REF_STATUS_ATOMIC_PUSH_FAILED; + break; + default: + break; + } + } + if (!quiet || err) transport_print_push_status(transport->url, remote_refs, verbose | porcelain, porcelain, @@@ -1394,3 -1381,100 +1395,3 @@@ char *transport_anonymize_url(const cha literal_copy: return xstrdup(url); } - -static void fill_alternate_refs_command(struct child_process *cmd, - const char *repo_path) -{ - const char *value; - - if (!git_config_get_value("core.alternateRefsCommand", &value)) { - cmd->use_shell = 1; - - argv_array_push(&cmd->args, value); - argv_array_push(&cmd->args, repo_path); - } else { - cmd->git_cmd = 1; - - argv_array_pushf(&cmd->args, "--git-dir=%s", repo_path); - argv_array_push(&cmd->args, "for-each-ref"); - argv_array_push(&cmd->args, "--format=%(objectname)"); - - if (!git_config_get_value("core.alternateRefsPrefixes", &value)) { - argv_array_push(&cmd->args, "--"); - argv_array_split(&cmd->args, value); - } - } - - cmd->env = local_repo_env; - cmd->out = -1; -} - -static void read_alternate_refs(const char *path, - alternate_ref_fn *cb, - void *data) -{ - struct child_process cmd = CHILD_PROCESS_INIT; - struct strbuf line = STRBUF_INIT; - FILE *fh; - - fill_alternate_refs_command(&cmd, path); - - if (start_command(&cmd)) - return; - - fh = xfdopen(cmd.out, "r"); - while (strbuf_getline_lf(&line, fh) != EOF) { - struct object_id oid; - const char *p; - - if (parse_oid_hex(line.buf, &oid, &p) || *p) { - warning(_("invalid line while parsing alternate refs: %s"), - line.buf); - break; - } - - cb(&oid, data); - } - - fclose(fh); - finish_command(&cmd); -} - -struct alternate_refs_data { - alternate_ref_fn *fn; - void *data; -}; - -static int refs_from_alternate_cb(struct object_directory *e, - void *data) -{ - struct strbuf path = STRBUF_INIT; - size_t base_len; - struct alternate_refs_data *cb = data; - - if (!strbuf_realpath(&path, e->path, 0)) - goto out; - if (!strbuf_strip_suffix(&path, "/objects")) - goto out; - base_len = path.len; - - /* Is this a git repository with refs? */ - strbuf_addstr(&path, "/refs"); - if (!is_directory(path.buf)) - goto out; - strbuf_setlen(&path, base_len); - - read_alternate_refs(path.buf, cb->fn, cb->data); - -out: - strbuf_release(&path); - return 0; -} - -void for_each_alternate_ref(alternate_ref_fn fn, void *data) -{ - struct alternate_refs_data cb; - cb.fn = fn; - cb.data = data; - foreach_alt_odb(refs_from_alternate_cb, &cb); -} diff --combined upload-pack.c index 222cd3ad89,f8a76ebda3..875db92996 --- a/upload-pack.c +++ b/upload-pack.c @@@ -140,18 -140,17 +140,17 @@@ static void create_pack_file(const stru argv_array_push(&pack_objects.args, "--delta-base-offset"); if (use_include_tag) argv_array_push(&pack_objects.args, "--include-tag"); - if (filter_options.filter_spec) { - struct strbuf expanded_filter_spec = STRBUF_INIT; - expand_list_objects_filter_spec(&filter_options, - &expanded_filter_spec); + if (filter_options.choice) { + const char *spec = + expand_list_objects_filter_spec(&filter_options); if (pack_objects.use_shell) { struct strbuf buf = STRBUF_INIT; - sq_quote_buf(&buf, expanded_filter_spec.buf); + sq_quote_buf(&buf, spec); argv_array_pushf(&pack_objects.args, "--filter=%s", buf.buf); strbuf_release(&buf); } else { argv_array_pushf(&pack_objects.args, "--filter=%s", - expanded_filter_spec.buf); + spec); } } @@@ -528,13 -527,13 +527,13 @@@ static int get_reachable_list(struct ob return -1; while ((i = read_in_full(cmd.out, namebuf, hexsz + 1)) == hexsz + 1) { - struct object_id sha1; + struct object_id oid; const char *p; - if (parse_oid_hex(namebuf, &sha1, &p) || *p != '\n') + if (parse_oid_hex(namebuf, &oid, &p) || *p != '\n') break; - o = lookup_object(the_repository, sha1.hash); + o = lookup_object(the_repository, &oid); if (o && o->type == OBJ_COMMIT) { o->flags &= ~TMP_MARK; } @@@ -722,7 -721,7 +721,7 @@@ static void deepen_by_rev_list(struct p { struct commit_list *result; - close_commit_graph(the_repository); + close_commit_graph(the_repository->objects); result = get_shallow_commits_by_rev_list(ac, av, SHALLOW, NOT_SHALLOW); send_shallow(writer, result); free_commit_list(result); @@@ -884,6 -883,7 +883,7 @@@ static void receive_needs(struct packet if (skip_prefix(reader->line, "filter ", &arg)) { if (!filter_capability_requested) die("git upload-pack: filtering capability not negotiated"); + list_objects_filter_die_if_populated(&filter_options); parse_list_objects_filter(&filter_options, arg); continue; } @@@ -960,7 -960,7 +960,7 @@@ static int mark_our_ref(const char *refname, const char *refname_full, const struct object_id *oid) { - struct object *o = lookup_unknown_object(oid->hash); + struct object *o = lookup_unknown_object(oid); if (ref_is_hidden(refname, refname_full)) { o->flags |= HIDDEN_REF; @@@ -1305,6 -1305,7 +1305,7 @@@ static void process_args(struct packet_ } if (allow_filter && skip_prefix(arg, "filter ", &p)) { + list_objects_filter_die_if_populated(&filter_options); parse_list_objects_filter(&filter_options, p); continue; }