From: Junio C Hamano Date: Wed, 27 Dec 2017 19:16:20 +0000 (-0800) Subject: Merge branch 'jh/object-filtering' X-Git-Tag: v2.16.0-rc0~37 X-Git-Url: https://git.lorimer.id.au/gitweb.git/diff_plain/61061abba7d60f555e97a22ab5775a9d53db1660?ds=inline;hp=-c Merge branch 'jh/object-filtering' In preparation for implementing narrow/partial clone, the object walking machinery has been taught a way to tell it to "filter" some objects from enumeration. * jh/object-filtering: rev-list: support --no-filter argument list-objects-filter-options: support --no-filter list-objects-filter-options: fix 'keword' typo in comment pack-objects: add list-objects filtering rev-list: add list-objects filtering support list-objects: filter objects in traverse_commit_list oidset: add iterator methods to oidset oidmap: add oidmap iterator methods dir: allow exclusions from blob in addition to file --- 61061abba7d60f555e97a22ab5775a9d53db1660 diff --combined Makefile index 9dc5a588e2,ca378a4603..80e0674d6f --- a/Makefile +++ b/Makefile @@@ -646,9 -646,7 +646,9 @@@ TEST_PROGRAMS_NEED_X += test-ctyp TEST_PROGRAMS_NEED_X += test-config TEST_PROGRAMS_NEED_X += test-date TEST_PROGRAMS_NEED_X += test-delta +TEST_PROGRAMS_NEED_X += test-drop-caches TEST_PROGRAMS_NEED_X += test-dump-cache-tree +TEST_PROGRAMS_NEED_X += test-dump-fsmonitor TEST_PROGRAMS_NEED_X += test-dump-split-index TEST_PROGRAMS_NEED_X += test-dump-untracked-cache TEST_PROGRAMS_NEED_X += test-fake-ssh @@@ -759,7 -757,6 +759,7 @@@ LIB_OBJS += branch. LIB_OBJS += bulk-checkin.o LIB_OBJS += bundle.o LIB_OBJS += cache-tree.o +LIB_OBJS += checkout.o LIB_OBJS += color.o LIB_OBJS += column.o LIB_OBJS += combine-diff.o @@@ -797,7 -794,6 +797,7 @@@ LIB_OBJS += ewah/ewah_rlw. LIB_OBJS += exec_cmd.o LIB_OBJS += fetch-pack.o LIB_OBJS += fsck.o +LIB_OBJS += fsmonitor.o LIB_OBJS += gettext.o LIB_OBJS += gpg-interface.o LIB_OBJS += graph.o @@@ -811,6 -807,8 +811,8 @@@ LIB_OBJS += levenshtein. LIB_OBJS += line-log.o LIB_OBJS += line-range.o LIB_OBJS += list-objects.o + LIB_OBJS += list-objects-filter.o + LIB_OBJS += list-objects-filter-options.o LIB_OBJS += ll-merge.o LIB_OBJS += lockfile.o LIB_OBJS += log-tree.o @@@ -850,7 -848,6 +852,7 @@@ LIB_OBJS += pretty. LIB_OBJS += prio-queue.o LIB_OBJS += progress.o LIB_OBJS += prompt.o +LIB_OBJS += protocol.o LIB_OBJS += quote.o LIB_OBJS += reachable.o LIB_OBJS += read-cache.o @@@ -1945,8 -1942,7 +1947,8 @@@ $(SCRIPT_LIB) : % : %.sh GIT-SCRIPT-DEF git.res: git.rc GIT-VERSION-FILE $(QUIET_RC)$(RC) \ - $(join -DMAJOR= -DMINOR=, $(wordlist 1,2,$(subst -, ,$(subst ., ,$(GIT_VERSION))))) \ + $(join -DMAJOR= -DMINOR= -DMICRO= -DPATCHLEVEL=, $(wordlist 1, 4, \ + $(shell echo $(GIT_VERSION) 0 0 0 0 | tr '.a-zA-Z-' ' '))) \ -DGIT_VERSION="\\\"$(GIT_VERSION)\\\"" -i $< -o $@ # This makes sure we depend on the NO_PERL setting itself. diff --combined builtin/pack-objects.c index 631de28761,45ad35d918..6b9cfc289d --- a/builtin/pack-objects.c +++ b/builtin/pack-objects.c @@@ -15,6 -15,8 +15,8 @@@ #include "diff.h" #include "revision.h" #include "list-objects.h" + #include "list-objects-filter.h" + #include "list-objects-filter-options.h" #include "pack-objects.h" #include "progress.h" #include "refs.h" @@@ -79,6 -81,15 +81,15 @@@ static unsigned long cache_max_small_de static unsigned long window_memory_limit = 0; + static struct list_objects_filter_options filter_options; + + enum missing_action { + MA_ERROR = 0, /* fail if any missing objects are encountered */ + MA_ALLOW_ANY, /* silently allow ALL missing objects */ + }; + static enum missing_action arg_missing_action; + static show_object_fn fn_show_object; + /* * stats */ @@@ -151,7 -162,7 +162,7 @@@ static unsigned long do_compress(void * } static unsigned long write_large_blob_data(struct git_istream *st, struct sha1file *f, - const unsigned char *sha1) + const struct object_id *oid) { git_zstream stream; unsigned char ibuf[1024 * 16]; @@@ -165,7 -176,7 +176,7 @@@ int zret = Z_OK; readlen = read_istream(st, ibuf, sizeof(ibuf)); if (readlen == -1) - die(_("unable to read %s"), sha1_to_hex(sha1)); + die(_("unable to read %s"), oid_to_hex(oid)); stream.next_in = ibuf; stream.avail_in = readlen; @@@ -339,7 -350,7 +350,7 @@@ static unsigned long write_no_reuse_obj sha1write(f, header, hdrlen); } if (st) { - datalen = write_large_blob_data(st, f, entry->idx.oid.hash); + datalen = write_large_blob_data(st, f, &entry->idx.oid); close_istream(st); } else { sha1write(f, buf, datalen); @@@ -557,13 -568,13 +568,13 @@@ static enum write_one_status write_one( static int mark_tagged(const char *path, const struct object_id *oid, int flag, void *cb_data) { - unsigned char peeled[20]; + struct object_id peeled; struct object_entry *entry = packlist_find(&to_pack, oid->hash, NULL); if (entry) entry->tagged = 1; - if (!peel_ref(path, peeled)) { - entry = packlist_find(&to_pack, peeled, NULL); + if (!peel_ref(path, &peeled)) { + entry = packlist_find(&to_pack, peeled.hash, NULL); if (entry) entry->tagged = 1; } @@@ -792,7 -803,7 +803,7 @@@ static void write_pack_file(void write_order = compute_write_order(); do { - unsigned char sha1[20]; + struct object_id oid; char *pack_tmp_name = NULL; if (pack_to_stdout) @@@ -823,13 -834,13 +834,13 @@@ * If so, rewrite it like in fast-import */ if (pack_to_stdout) { - sha1close(f, sha1, CSUM_CLOSE); + sha1close(f, oid.hash, CSUM_CLOSE); } else if (nr_written == nr_remaining) { - sha1close(f, sha1, CSUM_FSYNC); + sha1close(f, oid.hash, CSUM_FSYNC); } else { - int fd = sha1close(f, sha1, 0); - fixup_pack_header_footer(fd, sha1, pack_tmp_name, - nr_written, sha1, offset); + int fd = sha1close(f, oid.hash, 0); + fixup_pack_header_footer(fd, oid.hash, pack_tmp_name, + nr_written, oid.hash, offset); close(fd); if (write_bitmap_index) { warning(_(no_split_warning)); @@@ -863,16 -874,16 +874,16 @@@ strbuf_addf(&tmpname, "%s-", base_name); if (write_bitmap_index) { - bitmap_writer_set_checksum(sha1); + bitmap_writer_set_checksum(oid.hash); bitmap_writer_build_type_index(written_list, nr_written); } finish_tmp_packfile(&tmpname, pack_tmp_name, written_list, nr_written, - &pack_idx_opts, sha1); + &pack_idx_opts, oid.hash); if (write_bitmap_index) { - strbuf_addf(&tmpname, "%s.bitmap", sha1_to_hex(sha1)); + strbuf_addf(&tmpname, "%s.bitmap", oid_to_hex(&oid)); stop_progress(&progress_state); @@@ -887,7 -898,7 +898,7 @@@ strbuf_release(&tmpname); free(pack_tmp_name); - puts(sha1_to_hex(sha1)); + puts(oid_to_hex(&oid)); } /* mark written objects as written to previous pack */ @@@ -928,13 -939,13 +939,13 @@@ static int no_try_delta(const char *pat * found the item, since that saves us from having to look it up again a * few lines later when we want to add the new entry. */ -static int have_duplicate_entry(const unsigned char *sha1, +static int have_duplicate_entry(const struct object_id *oid, int exclude, uint32_t *index_pos) { struct object_entry *entry; - entry = packlist_find(&to_pack, sha1, index_pos); + entry = packlist_find(&to_pack, oid->hash, index_pos); if (!entry) return 0; @@@ -990,7 -1001,7 +1001,7 @@@ static int want_found_object(int exclud * function finds if there is any pack that has the object and returns the pack * and its offset in these variables. */ -static int want_object_in_pack(const unsigned char *sha1, +static int want_object_in_pack(const struct object_id *oid, int exclude, struct packed_git **found_pack, off_t *found_offset) @@@ -998,7 -1009,7 +1009,7 @@@ struct mru_entry *entry; int want; - if (!exclude && local && has_loose_object_nonlocal(sha1)) + if (!exclude && local && has_loose_object_nonlocal(oid->hash)) return 0; /* @@@ -1019,7 -1030,7 +1030,7 @@@ if (p == *found_pack) offset = *found_offset; else - offset = find_pack_entry_one(sha1, p); + offset = find_pack_entry_one(oid->hash, p); if (offset) { if (!*found_pack) { @@@ -1039,7 -1050,7 +1050,7 @@@ return 1; } -static void create_object_entry(const unsigned char *sha1, +static void create_object_entry(const struct object_id *oid, enum object_type type, uint32_t hash, int exclude, @@@ -1050,7 -1061,7 +1061,7 @@@ { struct object_entry *entry; - entry = packlist_alloc(&to_pack, sha1, index_pos); + entry = packlist_alloc(&to_pack, oid->hash, index_pos); entry->hash = hash; if (type) entry->type = type; @@@ -1070,17 -1081,17 +1081,17 @@@ static const char no_closure_warning[] "disabling bitmap writing, as some objects are not being packed" ); -static int add_object_entry(const unsigned char *sha1, enum object_type type, +static int add_object_entry(const struct object_id *oid, enum object_type type, const char *name, int exclude) { struct packed_git *found_pack = NULL; off_t found_offset = 0; uint32_t index_pos; - if (have_duplicate_entry(sha1, exclude, &index_pos)) + if (have_duplicate_entry(oid, exclude, &index_pos)) return 0; - if (!want_object_in_pack(sha1, exclude, &found_pack, &found_offset)) { + if (!want_object_in_pack(oid, exclude, &found_pack, &found_offset)) { /* The pack is missing an object, so it will not have closure */ if (write_bitmap_index) { warning(_(no_closure_warning)); @@@ -1089,7 -1100,7 +1100,7 @@@ return 0; } - create_object_entry(sha1, type, pack_name_hash(name), + create_object_entry(oid, type, pack_name_hash(name), exclude, name && no_try_delta(name), index_pos, found_pack, found_offset); @@@ -1097,27 -1108,27 +1108,27 @@@ return 1; } -static int add_object_entry_from_bitmap(const unsigned char *sha1, +static int add_object_entry_from_bitmap(const struct object_id *oid, enum object_type type, int flags, uint32_t name_hash, struct packed_git *pack, off_t offset) { uint32_t index_pos; - if (have_duplicate_entry(sha1, 0, &index_pos)) + if (have_duplicate_entry(oid, 0, &index_pos)) return 0; - if (!want_object_in_pack(sha1, 0, &pack, &offset)) + if (!want_object_in_pack(oid, 0, &pack, &offset)) return 0; - create_object_entry(sha1, type, name_hash, 0, 0, index_pos, pack, offset); + create_object_entry(oid, type, name_hash, 0, 0, index_pos, pack, offset); display_progress(progress_state, nr_result); return 1; } struct pbase_tree_cache { - unsigned char sha1[20]; + struct object_id oid; int ref; int temporary; void *tree_data; @@@ -1125,9 -1136,9 +1136,9 @@@ }; static struct pbase_tree_cache *(pbase_tree_cache[256]); -static int pbase_tree_cache_ix(const unsigned char *sha1) +static int pbase_tree_cache_ix(const struct object_id *oid) { - return sha1[0] % ARRAY_SIZE(pbase_tree_cache); + return oid->hash[0] % ARRAY_SIZE(pbase_tree_cache); } static int pbase_tree_cache_ix_incr(int ix) { @@@ -1144,14 -1155,14 +1155,14 @@@ static struct pbase_tree struct pbase_tree_cache pcache; } *pbase_tree; -static struct pbase_tree_cache *pbase_tree_get(const unsigned char *sha1) +static struct pbase_tree_cache *pbase_tree_get(const struct object_id *oid) { struct pbase_tree_cache *ent, *nent; void *data; unsigned long size; enum object_type type; int neigh; - int my_ix = pbase_tree_cache_ix(sha1); + int my_ix = pbase_tree_cache_ix(oid); int available_ix = -1; /* pbase-tree-cache acts as a limited hashtable. @@@ -1160,7 -1171,7 +1171,7 @@@ */ for (neigh = 0; neigh < 8; neigh++) { ent = pbase_tree_cache[my_ix]; - if (ent && !hashcmp(ent->sha1, sha1)) { + if (ent && !oidcmp(&ent->oid, oid)) { ent->ref++; return ent; } @@@ -1176,7 -1187,7 +1187,7 @@@ /* Did not find one. Either we got a bogus request or * we need to read and perhaps cache. */ - data = read_sha1_file(sha1, &type, &size); + data = read_sha1_file(oid->hash, &type, &size); if (!data) return NULL; if (type != OBJ_TREE) { @@@ -1202,7 -1213,7 +1213,7 @@@ free(ent->tree_data); nent = ent; } - hashcpy(nent->sha1, sha1); + oidcpy(&nent->oid, oid); nent->tree_data = data; nent->tree_size = size; nent->ref = 1; @@@ -1247,7 -1258,7 +1258,7 @@@ static void add_pbase_object(struct tre if (cmp < 0) return; if (name[cmplen] != '/') { - add_object_entry(entry.oid->hash, + add_object_entry(entry.oid, object_type(entry.mode), fullname, 1); return; @@@ -1258,7 -1269,7 +1269,7 @@@ const char *down = name+cmplen+1; int downlen = name_cmp_len(down); - tree = pbase_tree_get(entry.oid->hash); + tree = pbase_tree_get(entry.oid); if (!tree) return; init_tree_desc(&sub, tree->tree_data, tree->tree_size); @@@ -1317,7 -1328,7 +1328,7 @@@ static void add_preferred_base_object(c cmplen = name_cmp_len(name); for (it = pbase_tree; it; it = it->next) { if (cmplen == 0) { - add_object_entry(it->pcache.sha1, OBJ_TREE, NULL, 1); + add_object_entry(&it->pcache.oid, OBJ_TREE, NULL, 1); } else { struct tree_desc tree; @@@ -1327,22 -1338,22 +1338,22 @@@ } } -static void add_preferred_base(unsigned char *sha1) +static void add_preferred_base(struct object_id *oid) { struct pbase_tree *it; void *data; unsigned long size; - unsigned char tree_sha1[20]; + struct object_id tree_oid; if (window <= num_preferred_base++) return; - data = read_object_with_reference(sha1, tree_type, &size, tree_sha1); + data = read_object_with_reference(oid->hash, tree_type, &size, tree_oid.hash); if (!data) return; for (it = pbase_tree; it; it = it->next) { - if (!hashcmp(it->pcache.sha1, tree_sha1)) { + if (!oidcmp(&it->pcache.oid, &tree_oid)) { free(data); return; } @@@ -1352,7 -1363,7 +1363,7 @@@ it->next = pbase_tree; pbase_tree = it; - hashcpy(it->pcache.sha1, tree_sha1); + oidcpy(&it->pcache.oid, &tree_oid); it->pcache.tree_data = data; it->pcache.tree_size = size; } @@@ -2357,7 -2368,7 +2368,7 @@@ static void add_tag_chain(const struct die("unable to pack objects reachable from tag %s", oid_to_hex(oid)); - add_object_entry(tag->object.oid.hash, OBJ_TAG, NULL, 0); + add_object_entry(&tag->object.oid, OBJ_TAG, NULL, 0); if (tag->tagged->type != OBJ_TAG) return; @@@ -2371,7 -2382,7 +2382,7 @@@ static int add_ref_tag(const char *path struct object_id peeled; if (starts_with(path, "refs/tags/") && /* is a tag? */ - !peel_ref(path, peeled.hash) && /* peelable? */ + !peel_ref(path, &peeled) && /* peelable? */ packlist_find(&to_pack, peeled.hash, NULL)) /* object packed? */ add_tag_chain(oid); return 0; @@@ -2505,9 -2516,8 +2516,9 @@@ static int git_pack_config(const char * static void read_object_list_from_stdin(void) { - char line[40 + 1 + PATH_MAX + 2]; - unsigned char sha1[20]; + char line[GIT_MAX_HEXSZ + 1 + PATH_MAX + 2]; + struct object_id oid; + const char *p; for (;;) { if (!fgets(line, sizeof(line), stdin)) { @@@ -2521,17 -2531,17 +2532,17 @@@ continue; } if (line[0] == '-') { - if (get_sha1_hex(line+1, sha1)) - die("expected edge sha1, got garbage:\n %s", + if (get_oid_hex(line+1, &oid)) + die("expected edge object ID, got garbage:\n %s", line); - add_preferred_base(sha1); + add_preferred_base(&oid); continue; } - if (get_sha1_hex(line, sha1)) - die("expected sha1, got garbage:\n %s", line); + if (parse_oid_hex(line, &oid, &p)) + die("expected object ID, got garbage:\n %s", line); - add_preferred_base_object(line+41); - add_object_entry(sha1, 0, line+41, 0); + add_preferred_base_object(p + 1); + add_object_entry(&oid, 0, p + 1, 0); } } @@@ -2539,7 -2549,7 +2550,7 @@@ static void show_commit(struct commit *commit, void *data) { - add_object_entry(commit->object.oid.hash, OBJ_COMMIT, NULL, 0); + add_object_entry(&commit->object.oid, OBJ_COMMIT, NULL, 0); commit->object.flags |= OBJECT_ADDED; if (write_bitmap_index) @@@ -2549,13 -2559,49 +2560,49 @@@ static void show_object(struct object *obj, const char *name, void *data) { add_preferred_base_object(name); - add_object_entry(obj->oid.hash, obj->type, name, 0); + add_object_entry(&obj->oid, obj->type, name, 0); obj->flags |= OBJECT_ADDED; } + static void show_object__ma_allow_any(struct object *obj, const char *name, void *data) + { + assert(arg_missing_action == MA_ALLOW_ANY); + + /* + * Quietly ignore ALL missing objects. This avoids problems with + * staging them now and getting an odd error later. + */ + if (!has_object_file(&obj->oid)) + return; + + show_object(obj, name, data); + } + + static int option_parse_missing_action(const struct option *opt, + const char *arg, int unset) + { + assert(arg); + assert(!unset); + + if (!strcmp(arg, "error")) { + arg_missing_action = MA_ERROR; + fn_show_object = show_object; + return 0; + } + + if (!strcmp(arg, "allow-any")) { + arg_missing_action = MA_ALLOW_ANY; + fn_show_object = show_object__ma_allow_any; + return 0; + } + + die(_("invalid value for --missing")); + return 0; + } + static void show_edge(struct commit *commit) { - add_preferred_base(commit->object.oid.hash); + add_preferred_base(&commit->object.oid); } struct in_pack_object { @@@ -2602,7 -2648,7 +2649,7 @@@ static void add_objects_in_unpacked_pac memset(&in_pack, 0, sizeof(in_pack)); for (p = packed_git; p; p = p->next) { - const unsigned char *sha1; + struct object_id oid; struct object *o; if (!p->pack_local || p->pack_keep) @@@ -2615,8 -2661,8 +2662,8 @@@ in_pack.alloc); for (i = 0; i < p->num_objects; i++) { - sha1 = nth_packed_object_sha1(p, i); - o = lookup_unknown_object(sha1); + nth_packed_object_oid(&oid, p, i); + o = lookup_unknown_object(oid.hash); if (!(o->flags & OBJECT_ADDED)) mark_in_pack_object(o, p, &in_pack); o->flags |= OBJECT_ADDED; @@@ -2627,7 -2673,7 +2674,7 @@@ QSORT(in_pack.array, in_pack.nr, ofscmp); for (i = 0; i < in_pack.nr; i++) { struct object *o = in_pack.array[i].object; - add_object_entry(o->oid.hash, o->type, "", 0); + add_object_entry(&o->oid, o->type, "", 0); } } free(in_pack.array); @@@ -2643,7 -2689,7 +2690,7 @@@ static int add_loose_object(const struc return 0; } - add_object_entry(oid->hash, type, "", 0); + add_object_entry(oid, type, "", 0); return 0; } @@@ -2659,7 -2705,7 +2706,7 @@@ static void add_unreachable_loose_objec NULL, NULL, NULL); } -static int has_sha1_pack_kept_or_nonlocal(const unsigned char *sha1) +static int has_sha1_pack_kept_or_nonlocal(const struct object_id *oid) { static struct packed_git *last_found = (void *)1; struct packed_git *p; @@@ -2668,7 -2714,7 +2715,7 @@@ while (p) { if ((!p->pack_local || p->pack_keep) && - find_pack_entry_one(sha1, p)) { + find_pack_entry_one(oid->hash, p)) { last_found = p; return 1; } @@@ -2719,7 -2765,7 +2766,7 @@@ static void loosen_unused_packed_object for (i = 0; i < p->num_objects; i++) { nth_packed_object_oid(&oid, p, i); if (!packlist_find(&to_pack, oid.hash, NULL) && - !has_sha1_pack_kept_or_nonlocal(oid.hash) && + !has_sha1_pack_kept_or_nonlocal(&oid) && !loosened_object_can_be_discarded(&oid, p->mtime)) if (force_object_loose(oid.hash, p->mtime)) die("unable to force loose object"); @@@ -2817,7 -2863,12 +2864,12 @@@ static void get_object_list(int ac, con if (prepare_revision_walk(&revs)) die("revision walk setup failed"); mark_edges_uninteresting(&revs, show_edge); - traverse_commit_list(&revs, show_commit, show_object, NULL); + + if (!fn_show_object) + fn_show_object = show_object; + traverse_commit_list_filtered(&filter_options, &revs, + show_commit, fn_show_object, NULL, + NULL); if (unpack_unreachable_expiration) { revs.ignore_missing_links = 1; @@@ -2953,6 -3004,10 +3005,10 @@@ int cmd_pack_objects(int argc, const ch N_("use a bitmap index if available to speed up counting objects")), OPT_BOOL(0, "write-bitmap-index", &write_bitmap_index, N_("write a bitmap index together with the pack index")), + OPT_PARSE_LIST_OBJECTS_FILTER(&filter_options), + { OPTION_CALLBACK, 0, "missing", NULL, N_("action"), + N_("handling for missing objects"), PARSE_OPT_NONEG, + option_parse_missing_action }, OPT_END(), }; @@@ -3029,6 -3084,12 +3085,12 @@@ if (!rev_list_all || !rev_list_reflog || !rev_list_index) unpack_unreachable_expiration = 0; + if (filter_options.choice) { + if (!pack_to_stdout) + die("cannot use --filter without --stdout."); + use_bitmap_index = 0; + } + /* * "soft" reasons not to use bitmaps - for on-disk repack by default we want * diff --combined builtin/rev-list.c index 4032eb3811,159b035a2c..d5345b6a2e --- a/builtin/rev-list.c +++ b/builtin/rev-list.c @@@ -4,6 -4,8 +4,8 @@@ #include "diff.h" #include "revision.h" #include "list-objects.h" + #include "list-objects-filter.h" + #include "list-objects-filter-options.h" #include "pack.h" #include "pack-bitmap.h" #include "builtin.h" @@@ -12,6 -14,7 +14,7 @@@ #include "bisect.h" #include "progress.h" #include "reflog-walk.h" + #include "oidset.h" static const char rev_list_usage[] = "git rev-list [OPTION] ... [ -- paths... ]\n" @@@ -55,6 -58,20 +58,20 @@@ static struct progress *progress; static unsigned progress_counter; + static struct list_objects_filter_options filter_options; + static struct oidset omitted_objects; + static int arg_print_omitted; /* print objects omitted by filter */ + + static struct oidset missing_objects; + enum missing_action { + MA_ERROR = 0, /* fail if any missing objects are encountered */ + MA_ALLOW_ANY, /* silently allow ALL missing objects */ + MA_PRINT, /* print ALL missing objects in special section */ + }; + static enum missing_action arg_missing_action; + + #define DEFAULT_OIDSET_SIZE (16*1024) + static void finish_commit(struct commit *commit, void *data); static void show_commit(struct commit *commit, void *data) { @@@ -178,11 -195,31 +195,31 @@@ static void finish_commit(struct commi free_commit_buffer(commit); } + static inline void finish_object__ma(struct object *obj) + { + switch (arg_missing_action) { + case MA_ERROR: + die("missing blob object '%s'", oid_to_hex(&obj->oid)); + return; + + case MA_ALLOW_ANY: + return; + + case MA_PRINT: + oidset_insert(&missing_objects, &obj->oid); + return; + + default: + BUG("unhandled missing_action"); + return; + } + } + static void finish_object(struct object *obj, const char *name, void *cb_data) { struct rev_list_info *info = cb_data; if (obj->type == OBJ_BLOB && !has_object_file(&obj->oid)) - die("missing blob object '%s'", oid_to_hex(&obj->oid)); + finish_object__ma(obj); if (info->revs->verify_objects && !obj->parsed && obj->type != OBJ_COMMIT) parse_object(&obj->oid); } @@@ -258,17 -295,37 +295,37 @@@ static int show_bisect_vars(struct rev_ } static int show_object_fast( - const unsigned char *sha1, + const struct object_id *oid, enum object_type type, int exclude, uint32_t name_hash, struct packed_git *found_pack, off_t found_offset) { - fprintf(stdout, "%s\n", sha1_to_hex(sha1)); + fprintf(stdout, "%s\n", oid_to_hex(oid)); return 1; } + static inline int parse_missing_action_value(const char *value) + { + if (!strcmp(value, "error")) { + arg_missing_action = MA_ERROR; + return 1; + } + + if (!strcmp(value, "allow-any")) { + arg_missing_action = MA_ALLOW_ANY; + return 1; + } + + if (!strcmp(value, "print")) { + arg_missing_action = MA_PRINT; + return 1; + } + + return 0; + } + int cmd_rev_list(int argc, const char **argv, const char *prefix) { struct rev_info revs; @@@ -294,7 -351,7 +351,7 @@@ if (revs.bisect) bisect_list = 1; - if (DIFF_OPT_TST(&revs.diffopt, QUICK)) + if (revs.diffopt.flags.quick) info.flags |= REV_LIST_QUIET; for (i = 1 ; i < argc; i++) { const char *arg = argv[i]; @@@ -335,6 -392,30 +392,30 @@@ show_progress = arg; continue; } + + if (skip_prefix(arg, ("--" CL_ARG__FILTER "="), &arg)) { + parse_list_objects_filter(&filter_options, arg); + if (filter_options.choice && !revs.blob_objects) + die(_("object filtering requires --objects")); + if (filter_options.choice == LOFC_SPARSE_OID && + !filter_options.sparse_oid_value) + die(_("invalid sparse value '%s'"), + filter_options.filter_spec); + continue; + } + if (!strcmp(arg, ("--no-" CL_ARG__FILTER))) { + list_objects_filter_release(&filter_options); + continue; + } + if (!strcmp(arg, "--filter-print-omitted")) { + arg_print_omitted = 1; + continue; + } + + if (skip_prefix(arg, "--missing=", &arg) && + parse_missing_action_value(arg)) + continue; + usage(rev_list_usage); } @@@ -360,6 -441,9 +441,9 @@@ if (revs.show_notes) die(_("rev-list does not support display of notes")); + if (filter_options.choice && use_bitmap_index) + die(_("cannot combine --use-bitmap-index with object filtering")); + save_commit_buffer = (revs.verbose_header || revs.grep_filter.pattern_list || revs.grep_filter.header_list); @@@ -397,13 -481,38 +481,37 @@@ if (bisect_list) { int reaches = reaches, all = all; - revs.commits = find_bisection(revs.commits, &reaches, &all, - bisect_find_all); + find_bisection(&revs.commits, &reaches, &all, bisect_find_all); if (bisect_show_vars) return show_bisect_vars(&info, reaches, all); } - traverse_commit_list(&revs, show_commit, show_object, &info); + if (arg_print_omitted) + oidset_init(&omitted_objects, DEFAULT_OIDSET_SIZE); + if (arg_missing_action == MA_PRINT) + oidset_init(&missing_objects, DEFAULT_OIDSET_SIZE); + + traverse_commit_list_filtered( + &filter_options, &revs, show_commit, show_object, &info, + (arg_print_omitted ? &omitted_objects : NULL)); + + if (arg_print_omitted) { + struct oidset_iter iter; + struct object_id *oid; + oidset_iter_init(&omitted_objects, &iter); + while ((oid = oidset_iter_next(&iter))) + printf("~%s\n", oid_to_hex(oid)); + oidset_clear(&omitted_objects); + } + if (arg_missing_action == MA_PRINT) { + struct oidset_iter iter; + struct object_id *oid; + oidset_iter_init(&missing_objects, &iter); + while ((oid = oidset_iter_next(&iter))) + printf("?%s\n", oid_to_hex(oid)); + oidset_clear(&missing_objects); + } stop_progress(&progress); diff --combined dir.c index 3c54366a17,1962374d2a..7c4b45e30e --- a/dir.c +++ b/dir.c @@@ -18,7 -18,6 +18,7 @@@ #include "utf8.h" #include "varint.h" #include "ewah/ewok.h" +#include "fsmonitor.h" /* * Tells read_directory_recursive how a file or directory should be treated. @@@ -221,6 -220,57 +221,57 @@@ int within_depth(const char *name, int return 1; } + /* + * Read the contents of the blob with the given OID into a buffer. + * Append a trailing LF to the end if the last line doesn't have one. + * + * Returns: + * -1 when the OID is invalid or unknown or does not refer to a blob. + * 0 when the blob is empty. + * 1 along with { data, size } of the (possibly augmented) buffer + * when successful. + * + * Optionally updates the given sha1_stat with the given OID (when valid). + */ + static int do_read_blob(const struct object_id *oid, + struct sha1_stat *sha1_stat, + size_t *size_out, + char **data_out) + { + enum object_type type; + unsigned long sz; + char *data; + + *size_out = 0; + *data_out = NULL; + + data = read_sha1_file(oid->hash, &type, &sz); + if (!data || type != OBJ_BLOB) { + free(data); + return -1; + } + + if (sha1_stat) { + memset(&sha1_stat->stat, 0, sizeof(sha1_stat->stat)); + hashcpy(sha1_stat->sha1, oid->hash); + } + + if (sz == 0) { + free(data); + return 0; + } + + if (data[sz - 1] != '\n') { + data = xrealloc(data, st_add(sz, 1)); + data[sz++] = '\n'; + } + + *size_out = xsize_t(sz); + *data_out = data; + + return 1; + } + #define DO_MATCH_EXCLUDE (1<<0) #define DO_MATCH_DIRECTORY (1<<1) #define DO_MATCH_SUBMODULE (1<<2) @@@ -601,32 -651,22 +652,22 @@@ void add_exclude(const char *string, co x->el = el; } - static void *read_skip_worktree_file_from_index(const struct index_state *istate, - const char *path, size_t *size, - struct sha1_stat *sha1_stat) + static int read_skip_worktree_file_from_index(const struct index_state *istate, + const char *path, + size_t *size_out, + char **data_out, + struct sha1_stat *sha1_stat) { int pos, len; - unsigned long sz; - enum object_type type; - void *data; len = strlen(path); pos = index_name_pos(istate, path, len); if (pos < 0) - return NULL; + return -1; if (!ce_skip_worktree(istate->cache[pos])) - return NULL; - data = read_sha1_file(istate->cache[pos]->oid.hash, &type, &sz); - if (!data || type != OBJ_BLOB) { - free(data); - return NULL; - } - *size = xsize_t(sz); - if (sha1_stat) { - memset(&sha1_stat->stat, 0, sizeof(sha1_stat->stat)); - hashcpy(sha1_stat->sha1, istate->cache[pos]->oid.hash); - } - return data; + return -1; + + return do_read_blob(&istate->cache[pos]->oid, sha1_stat, size_out, data_out); } /* @@@ -740,6 -780,10 +781,10 @@@ static void invalidate_directory(struc dir->dirs[i]->recurse = 0; } + static int add_excludes_from_buffer(char *buf, size_t size, + const char *base, int baselen, + struct exclude_list *el); + /* * Given a file with name "fname", read it (either from disk, or from * an index if 'istate' is non-null), parse it and store the @@@ -755,9 -799,10 +800,10 @@@ static int add_excludes(const char *fna struct sha1_stat *sha1_stat) { struct stat st; - int fd, i, lineno = 1; + int r; + int fd; size_t size = 0; - char *buf, *entry; + char *buf; fd = open(fname, O_RDONLY); if (fd < 0 || fstat(fd, &st) < 0) { @@@ -765,17 -810,13 +811,13 @@@ warn_on_fopen_errors(fname); else close(fd); - if (!istate || - (buf = read_skip_worktree_file_from_index(istate, fname, &size, sha1_stat)) == NULL) + if (!istate) return -1; - if (size == 0) { - free(buf); - return 0; - } - if (buf[size-1] != '\n') { - buf = xrealloc(buf, st_add(size, 1)); - buf[size++] = '\n'; - } + r = read_skip_worktree_file_from_index(istate, fname, + &size, &buf, + sha1_stat); + if (r != 1) + return r; } else { size = xsize_t(st.st_size); if (size == 0) { @@@ -814,6 -855,17 +856,17 @@@ } } + add_excludes_from_buffer(buf, size, base, baselen, el); + return 0; + } + + static int add_excludes_from_buffer(char *buf, size_t size, + const char *base, int baselen, + struct exclude_list *el) + { + int i, lineno = 1; + char *entry; + el->filebuf = buf; if (skip_utf8_bom(&buf, size)) @@@ -842,6 -894,23 +895,23 @@@ int add_excludes_from_file_to_list(cons return add_excludes(fname, base, baselen, el, istate, NULL); } + int add_excludes_from_blob_to_list( + struct object_id *oid, + const char *base, int baselen, + struct exclude_list *el) + { + char *buf; + size_t size; + int r; + + r = do_read_blob(oid, NULL, &size, &buf); + if (r != 1) + return r; + + add_excludes_from_buffer(buf, size, base, baselen, el); + return 0; + } + struct exclude_list *add_exclude_list(struct dir_struct *dir, int group_type, const char *src) { @@@ -1390,34 -1459,10 +1460,34 @@@ static enum path_treatment treat_direct case index_nonexistent: if (dir->flags & DIR_SHOW_OTHER_DIRECTORIES) break; + if (exclude && + (dir->flags & DIR_SHOW_IGNORED_TOO) && + (dir->flags & DIR_SHOW_IGNORED_TOO_MODE_MATCHING)) { + + /* + * This is an excluded directory and we are + * showing ignored paths that match an exclude + * pattern. (e.g. show directory as ignored + * only if it matches an exclude pattern). + * This path will either be 'path_excluded` + * (if we are showing empty directories or if + * the directory is not empty), or will be + * 'path_none' (empty directory, and we are + * not showing empty directories). + */ + if (!(dir->flags & DIR_HIDE_EMPTY_DIRECTORIES)) + return path_excluded; + + if (read_directory_recursive(dir, istate, dirname, len, + untracked, 1, 1, pathspec) == path_excluded) + return path_excluded; + + return path_none; + } if (!(dir->flags & DIR_NO_GITLINKS)) { - unsigned char sha1[20]; - if (resolve_gitlink_ref(dirname, "HEAD", sha1) == 0) - return path_untracked; + struct object_id oid; + if (resolve_gitlink_ref(dirname, "HEAD", &oid) == 0) + return exclude ? path_excluded : path_untracked; } return path_recurse; } @@@ -1586,7 -1631,6 +1656,7 @@@ static enum path_treatment treat_one_pa { int exclude; int has_path_in_index = !!index_file_exists(istate, path->buf, path->len, ignore_case); + enum path_treatment path_treatment; if (dtype == DT_UNKNOWN) dtype = get_dtype(de, istate, path->buf, path->len); @@@ -1633,23 -1677,8 +1703,23 @@@ return path_none; case DT_DIR: strbuf_addch(path, '/'); - return treat_directory(dir, istate, untracked, path->buf, path->len, - baselen, exclude, pathspec); + path_treatment = treat_directory(dir, istate, untracked, + path->buf, path->len, + baselen, exclude, pathspec); + /* + * If 1) we only want to return directories that + * match an exclude pattern and 2) this directory does + * not match an exclude pattern but all of its + * contents are excluded, then indicate that we should + * recurse into this directory (instead of marking the + * directory itself as an ignored path). + */ + if (!exclude && + path_treatment == path_excluded && + (dir->flags & DIR_SHOW_IGNORED_TOO) && + (dir->flags & DIR_SHOW_IGNORED_TOO_MODE_MATCHING)) + return path_recurse; + return path_treatment; case DT_REG: case DT_LNK: return exclude ? path_excluded : path_untracked; @@@ -1734,23 -1763,17 +1804,23 @@@ static int valid_cached_dir(struct dir_ if (!untracked) return 0; - if (stat(path->len ? path->buf : ".", &st)) { - invalidate_directory(dir->untracked, untracked); - memset(&untracked->stat_data, 0, sizeof(untracked->stat_data)); - return 0; - } - if (!untracked->valid || - match_stat_data_racy(istate, &untracked->stat_data, &st)) { - if (untracked->valid) + /* + * With fsmonitor, we can trust the untracked cache's valid field. + */ + refresh_fsmonitor(istate); + if (!(dir->untracked->use_fsmonitor && untracked->valid)) { + if (stat(path->len ? path->buf : ".", &st)) { invalidate_directory(dir->untracked, untracked); - fill_stat_data(&untracked->stat_data, &st); - return 0; + memset(&untracked->stat_data, 0, sizeof(untracked->stat_data)); + return 0; + } + if (!untracked->valid || + match_stat_data_racy(istate, &untracked->stat_data, &st)) { + if (untracked->valid) + invalidate_directory(dir->untracked, untracked); + fill_stat_data(&untracked->stat_data, &st); + return 0; + } } if (untracked->check_only != !!check_only) { @@@ -2326,10 -2349,10 +2396,10 @@@ static int remove_dir_recurse(struct st int ret = 0, original_len = path->len, len, kept_down = 0; int only_empty = (flag & REMOVE_DIR_EMPTY_ONLY); int keep_toplevel = (flag & REMOVE_DIR_KEEP_TOPLEVEL); - unsigned char submodule_head[20]; + struct object_id submodule_head; if ((flag & REMOVE_DIR_KEEP_NESTED_GIT) && - !resolve_gitlink_ref(path->buf, "HEAD", submodule_head)) { + !resolve_gitlink_ref(path->buf, "HEAD", &submodule_head)) { /* Do not descend and nuke a nested git work tree. */ if (kept_up) *kept_up = 1; diff --combined dir.h index 233a2eb36b,1bcf39123a..11a047ba48 --- a/dir.h +++ b/dir.h @@@ -139,8 -139,6 +139,8 @@@ struct untracked_cache int gitignore_invalidated; int dir_invalidated; int dir_opened; + /* fsmonitor invalidation data */ + unsigned int use_fsmonitor : 1; }; struct dir_struct { @@@ -154,8 -152,7 +154,8 @@@ DIR_COLLECT_IGNORED = 1<<4, DIR_SHOW_IGNORED_TOO = 1<<5, DIR_COLLECT_KILLED_ONLY = 1<<6, - DIR_KEEP_UNTRACKED_CONTENTS = 1<<7 + DIR_KEEP_UNTRACKED_CONTENTS = 1<<7, + DIR_SHOW_IGNORED_TOO_MODE_MATCHING = 1<<8 } flags; struct dir_entry **entries; struct dir_entry **ignored; @@@ -259,6 -256,9 +259,9 @@@ extern struct exclude_list *add_exclude extern int add_excludes_from_file_to_list(const char *fname, const char *base, int baselen, struct exclude_list *el, struct index_state *istate); extern void add_excludes_from_file(struct dir_struct *, const char *fname); + extern int add_excludes_from_blob_to_list(struct object_id *oid, + const char *base, int baselen, + struct exclude_list *el); extern void parse_exclude_pattern(const char **string, int *patternlen, unsigned *flags, int *nowildcardlen); extern void add_exclude(const char *string, const char *base, int baselen, struct exclude_list *el, int srcpos);