From: Junio C Hamano Date: Tue, 10 Jan 2017 23:24:26 +0000 (-0800) Subject: Merge branch 'jc/git-open-cloexec' X-Git-Tag: v2.12.0-rc0~86 X-Git-Url: https://git.lorimer.id.au/gitweb.git/diff_plain/02d0457eb4b9bf656965c5dbb613b8bfaef3316f?ds=inline;hp=-c Merge branch 'jc/git-open-cloexec' The codeflow of setting NOATIME and CLOEXEC on file descriptors Git opens has been simplified. We may want to drop the tip one, but we'll see. * jc/git-open-cloexec: sha1_file: stop opening files with O_NOATIME git_open_cloexec(): use fcntl(2) w/ FD_CLOEXEC fallback git_open(): untangle possible NOATIME and CLOEXEC interactions --- 02d0457eb4b9bf656965c5dbb613b8bfaef3316f diff --combined cache.h index b14c0e8e38,f440d3fd1e..1b67f078dd --- a/cache.h +++ b/cache.h @@@ -409,7 -409,6 +409,7 @@@ static inline enum object_type object_t #define GIT_NAMESPACE_ENVIRONMENT "GIT_NAMESPACE" #define GIT_WORK_TREE_ENVIRONMENT "GIT_WORK_TREE" #define GIT_PREFIX_ENVIRONMENT "GIT_PREFIX" +#define GIT_SUPER_PREFIX_ENVIRONMENT "GIT_INTERNAL_SUPER_PREFIX" #define DEFAULT_GIT_DIR_ENVIRONMENT ".git" #define DB_ENVIRONMENT "GIT_OBJECT_DIRECTORY" #define INDEX_ENVIRONMENT "GIT_INDEX_FILE" @@@ -434,7 -433,6 +434,7 @@@ #define GIT_GLOB_PATHSPECS_ENVIRONMENT "GIT_GLOB_PATHSPECS" #define GIT_NOGLOB_PATHSPECS_ENVIRONMENT "GIT_NOGLOB_PATHSPECS" #define GIT_ICASE_PATHSPECS_ENVIRONMENT "GIT_ICASE_PATHSPECS" +#define GIT_QUARANTINE_ENVIRONMENT "GIT_QUARANTINE_PATH" /* * This environment variable is expected to contain a boolean indicating @@@ -477,7 -475,6 +477,7 @@@ extern int get_common_dir_noenv(struct extern int get_common_dir(struct strbuf *sb, const char *gitdir); extern const char *get_git_namespace(void); extern const char *strip_namespace(const char *namespaced_ref); +extern const char *get_super_prefix(void); extern const char *get_git_work_tree(void); /* @@@ -670,7 -667,7 +670,7 @@@ extern const char *git_attributes_file extern const char *git_hooks_path; extern int zlib_compression_level; extern int core_compression_level; -extern int core_compression_seen; +extern int pack_compression_level; extern size_t packed_git_window_size; extern size_t packed_git_limit; extern size_t delta_base_cache_limit; @@@ -903,8 -900,8 +903,8 @@@ extern char *sha1_pack_index_name(cons * The result will be at least `len` characters long, and will be NUL * terminated. * - * The non-`_r` version returns a static buffer which will be overwritten by - * subsequent calls. + * The non-`_r` version returns a static buffer which remains valid until 4 + * more calls to find_unique_abbrev are made. * * The `_r` variant writes to a buffer supplied by the caller, which must be at * least `GIT_SHA1_HEXSZ + 1` bytes. The return value is the number of bytes @@@ -1125,7 -1122,8 +1125,8 @@@ extern int write_sha1_file(const void * extern int hash_sha1_file_literally(const void *buf, unsigned long len, const char *type, unsigned char *sha1, unsigned flags); extern int pretend_sha1_file(void *, unsigned long, enum object_type, unsigned char *); extern int force_object_loose(const unsigned char *sha1, time_t mtime); - extern int git_open(const char *name); + extern int git_open_cloexec(const char *name, int flags); + #define git_open(name) git_open_cloexec(name, O_RDONLY) extern void *map_sha1_file(const unsigned char *sha1, unsigned long *size); extern int unpack_sha1_header(git_zstream *stream, unsigned char *map, unsigned long mapsize, void *buffer, unsigned long bufsiz); extern int parse_sha1_header(const char *hdr, unsigned long *sizep); @@@ -1157,7 -1155,6 +1158,7 @@@ static inline int has_sha1_file(const u /* Same as the above, except for struct object_id. */ extern int has_object_file(const struct object_id *oid); +extern int has_object_file_with_flags(const struct object_id *oid, int flags); /* * Return true iff an alternate object database has a loose object @@@ -1190,9 -1187,6 +1191,9 @@@ static inline int hex2chr(const char *s #define MINIMUM_ABBREV minimum_abbrev #define DEFAULT_ABBREV default_abbrev +/* used when the code does not know or care what the default abbrev is */ +#define FALLBACK_DEFAULT_ABBREV 7 + struct object_context { unsigned char tree[20]; char path[PATH_MAX]; @@@ -1493,12 -1487,6 +1494,12 @@@ extern void prepare_packed_git(void) extern void reprepare_packed_git(void); extern void install_packed_git(struct packed_git *pack); +/* + * Give a rough count of objects in the repository. This sacrifices accuracy + * for speed. + */ +unsigned long approximate_object_count(void); + extern struct packed_git *find_sha1_pack(const unsigned char *sha1, struct packed_git *packs); diff --combined read-cache.c index f92a912dcb,c27d3e240b..2eca639cce --- a/read-cache.c +++ b/read-cache.c @@@ -156,14 -156,7 +156,7 @@@ void fill_stat_cache_info(struct cache_ static int ce_compare_data(const struct cache_entry *ce, struct stat *st) { int match = -1; - static int cloexec = O_CLOEXEC; - int fd = open(ce->name, O_RDONLY | cloexec); - - if ((cloexec & O_CLOEXEC) && fd < 0 && errno == EINVAL) { - /* Try again w/o O_CLOEXEC: the kernel might not support it */ - cloexec &= ~O_CLOEXEC; - fd = open(ce->name, O_RDONLY | cloexec); - } + int fd = git_open_cloexec(ce->name, O_RDONLY); if (fd >= 0) { unsigned char sha1[20]; @@@ -1425,9 -1418,12 +1418,9 @@@ static int read_index_extension(struct return 0; } -int hold_locked_index(struct lock_file *lk, int die_on_error) +int hold_locked_index(struct lock_file *lk, int lock_flags) { - return hold_lock_file_for_update(lk, get_index_file(), - die_on_error - ? LOCK_DIE_ON_ERROR - : 0); + return hold_lock_file_for_update(lk, get_index_file(), lock_flags); } int read_index(struct index_state *istate) diff --combined sha1_file.c index 1173071859,4e062edea0..1eb47f6113 --- a/sha1_file.c +++ b/sha1_file.c @@@ -26,16 -26,7 +26,8 @@@ #include "mru.h" #include "list.h" #include "mergesort.h" +#include "quote.h" - #ifndef O_NOATIME - #if defined(__linux__) && (defined(__i386__) || defined(__PPC__)) - #define O_NOATIME 01000000 - #else - #define O_NOATIME 0 - #endif - #endif - #define SZ_FMT PRIuMAX static inline uintmax_t sz_fmt(size_t s) { return s; } @@@ -297,7 -288,7 +289,7 @@@ static int link_alt_odb_entry(const cha } strbuf_addstr(&pathbuf, entry); - if (strbuf_normalize_path(&pathbuf) < 0) { + if (strbuf_normalize_path(&pathbuf) < 0 && relative_base) { error("unable to normalize alternate object path: %s", pathbuf.buf); strbuf_release(&pathbuf); @@@ -330,40 -321,13 +322,40 @@@ return 0; } +static const char *parse_alt_odb_entry(const char *string, + int sep, + struct strbuf *out) +{ + const char *end; + + strbuf_reset(out); + + if (*string == '#') { + /* comment; consume up to next separator */ + end = strchrnul(string, sep); + } else if (*string == '"' && !unquote_c_style(out, string, &end)) { + /* + * quoted path; unquote_c_style has copied the + * data for us and set "end". Broken quoting (e.g., + * an entry that doesn't end with a quote) falls + * back to the unquoted case below. + */ + } else { + /* normal, unquoted path */ + end = strchrnul(string, sep); + strbuf_add(out, string, end - string); + } + + if (*end) + end++; + return end; +} + static void link_alt_odb_entries(const char *alt, int len, int sep, const char *relative_base, int depth) { - struct string_list entries = STRING_LIST_INIT_NODUP; - char *alt_copy; - int i; struct strbuf objdirbuf = STRBUF_INIT; + struct strbuf entry = STRBUF_INIT; if (depth > 5) { error("%s: ignoring alternate object stores, nesting too deep.", @@@ -376,13 -340,16 +368,13 @@@ die("unable to normalize object directory: %s", objdirbuf.buf); - alt_copy = xmemdupz(alt, len); - string_list_split_in_place(&entries, alt_copy, sep, -1); - for (i = 0; i < entries.nr; i++) { - const char *entry = entries.items[i].string; - if (entry[0] == '\0' || entry[0] == '#') + while (*alt) { + alt = parse_alt_odb_entry(alt, sep, &entry); + if (!entry.len) continue; - link_alt_odb_entry(entry, relative_base, depth, objdirbuf.buf); + link_alt_odb_entry(entry.buf, relative_base, depth, objdirbuf.buf); } - string_list_clear(&entries, 0); - free(alt_copy); + strbuf_release(&entry); strbuf_release(&objdirbuf); } @@@ -1435,32 -1402,6 +1427,32 @@@ static void prepare_packed_git_one(cha strbuf_release(&path); } +static int approximate_object_count_valid; + +/* + * Give a fast, rough count of the number of objects in the repository. This + * ignores loose objects completely. If you have a lot of them, then either + * you should repack because your performance will be awful, or they are + * all unreachable objects about to be pruned, in which case they're not really + * interesting as a measure of repo size in the first place. + */ +unsigned long approximate_object_count(void) +{ + static unsigned long count; + if (!approximate_object_count_valid) { + struct packed_git *p; + + prepare_packed_git(); + count = 0; + for (p = packed_git; p; p = p->next) { + if (open_pack_index(p)) + continue; + count += p->num_objects; + } + } + return count; +} + static void *get_next_packed_git(const void *p) { return ((const struct packed_git *)p)->next; @@@ -1532,7 -1473,6 +1524,7 @@@ void prepare_packed_git(void void reprepare_packed_git(void) { + approximate_object_count_valid = 0; prepare_packed_git_run_once = 0; prepare_packed_git(); } @@@ -1611,31 -1551,31 +1603,31 @@@ int check_sha1_signature(const unsigne return hashcmp(sha1, real_sha1) ? -1 : 0; } - int git_open(const char *name) + int git_open_cloexec(const char *name, int flags) { - static int sha1_file_open_flag = O_NOATIME | O_CLOEXEC; - - for (;;) { - int fd; - - errno = 0; - fd = open(name, O_RDONLY | sha1_file_open_flag); - if (fd >= 0) - return fd; + int fd; + static int o_cloexec = O_CLOEXEC; + fd = open(name, flags | o_cloexec); + if ((o_cloexec & O_CLOEXEC) && fd < 0 && errno == EINVAL) { /* Try again w/o O_CLOEXEC: the kernel might not support it */ - if ((sha1_file_open_flag & O_CLOEXEC) && errno == EINVAL) { - sha1_file_open_flag &= ~O_CLOEXEC; - continue; - } + o_cloexec &= ~O_CLOEXEC; + fd = open(name, flags | o_cloexec); + } - /* Might the failure be due to O_NOATIME? */ - if (errno != ENOENT && (sha1_file_open_flag & O_NOATIME)) { - sha1_file_open_flag &= ~O_NOATIME; - continue; + #if defined(F_GETFL) && defined(F_SETFL) && defined(FD_CLOEXEC) + { + static int fd_cloexec = FD_CLOEXEC; + + if (!o_cloexec && 0 <= fd && fd_cloexec) { + /* Opened w/o O_CLOEXEC? try with fcntl(2) to add it */ + int flags = fcntl(fd, F_GETFL); + if (fcntl(fd, F_SETFL, flags | fd_cloexec)) + fd_cloexec = 0; } - return -1; } + #endif + return fd; } static int stat_sha1_file(const unsigned char *sha1, struct stat *st) @@@ -3392,11 -3332,6 +3384,11 @@@ int has_object_file(const struct object return has_sha1_file(oid->hash); } +int has_object_file_with_flags(const struct object_id *oid, int flags) +{ + return has_sha1_file_with_flags(oid->hash, flags); +} + static void check_tree(const void *buf, size_t size) { struct tree_desc desc;