From: Junio C Hamano Date: Wed, 25 Jun 2014 18:49:48 +0000 (-0700) Subject: Merge branch 'ym/fix-opportunistic-index-update-race' into maint X-Git-Tag: v2.0.1~3 X-Git-Url: https://git.lorimer.id.au/gitweb.git/diff_plain/1881d2b88c4b889dcb95782ad4bc5395808438e9?ds=inline;hp=-c Merge branch 'ym/fix-opportunistic-index-update-race' into maint "git status", even though it is a read-only operation, tries to update the index with refreshed lstat(2) info to optimize future accesses to the working tree opportunistically, but this could race with a "read-write" operation that modify the index while it is running. Detect such a race and avoid overwriting the index. * ym/fix-opportunistic-index-update-race: read-cache.c: verify index file before we opportunistically update it wrapper.c: add xpread() similar to xread() --- 1881d2b88c4b889dcb95782ad4bc5395808438e9 diff --combined builtin/index-pack.c index 54b089542b,e7a6b537b4..18f57de58b --- a/builtin/index-pack.c +++ b/builtin/index-pack.c @@@ -40,13 -40,17 +40,13 @@@ struct base_data int ofs_first, ofs_last; }; -#if !defined(NO_PTHREADS) && defined(NO_THREAD_SAFE_PREAD) -/* pread() emulation is not thread-safe. Disable threading. */ -#define NO_PTHREADS -#endif - struct thread_local { #ifndef NO_PTHREADS pthread_t thread; #endif struct base_data *base_cache; size_t base_cache_used; + int pack_fd; }; /* @@@ -87,8 -91,7 +87,8 @@@ static off_t consumed_bytes static unsigned deepest_delta; static git_SHA_CTX input_ctx; static uint32_t input_crc32; -static int input_fd, output_fd, pack_fd; +static int input_fd, output_fd; +static const char *curr_pack; #ifndef NO_PTHREADS @@@ -131,7 -134,6 +131,7 @@@ static inline void unlock_mutex(pthread */ static void init_thread(void) { + int i; init_recursive_mutex(&read_mutex); pthread_mutex_init(&counter_mutex, NULL); pthread_mutex_init(&work_mutex, NULL); @@@ -139,18 -141,11 +139,18 @@@ pthread_mutex_init(&deepest_delta_mutex, NULL); pthread_key_create(&key, NULL); thread_data = xcalloc(nr_threads, sizeof(*thread_data)); + for (i = 0; i < nr_threads; i++) { + thread_data[i].pack_fd = open(curr_pack, O_RDONLY); + if (thread_data[i].pack_fd == -1) + die_errno(_("unable to open %s"), curr_pack); + } + threads_active = 1; } static void cleanup_thread(void) { + int i; if (!threads_active) return; threads_active = 0; @@@ -159,8 -154,6 +159,8 @@@ pthread_mutex_destroy(&work_mutex); if (show_stat) pthread_mutex_destroy(&deepest_delta_mutex); + for (i = 0; i < nr_threads; i++) + close(thread_data[i].pack_fd); pthread_key_delete(key); free(thread_data); } @@@ -207,13 -200,8 +207,13 @@@ static unsigned check_object(struct obj if (!(obj->flags & FLAG_CHECKED)) { unsigned long size; int type = sha1_object_info(obj->sha1, &size); - if (type != obj->type || type <= 0) - die(_("object of unexpected type")); + if (type <= 0) + die(_("did not receive expected object %s"), + sha1_to_hex(obj->sha1)); + if (type != obj->type) + die(_("object %s: expected type %s, found %s"), + sha1_to_hex(obj->sha1), + typename(obj->type), typename(type)); obj->flags |= FLAG_CHECKED; return 1; } @@@ -300,13 -288,13 +300,13 @@@ static const char *open_pack_file(cons output_fd = open(pack_name, O_CREAT|O_EXCL|O_RDWR, 0600); if (output_fd < 0) die_errno(_("unable to create '%s'"), pack_name); - pack_fd = output_fd; + nothread_data.pack_fd = output_fd; } else { input_fd = open(pack_name, O_RDONLY); if (input_fd < 0) die_errno(_("cannot open packfile '%s'"), pack_name); output_fd = -1; - pack_fd = input_fd; + nothread_data.pack_fd = input_fd; } git_SHA1_Init(&input_ctx); return pack_name; @@@ -554,7 -542,7 +554,7 @@@ static void *unpack_data(struct object_ do { ssize_t n = (len < 64*1024) ? len : 64*1024; - n = pread(get_thread_data()->pack_fd, inbuf, n, from); - n = xpread(pack_fd, inbuf, n, from); ++ n = xpread(get_thread_data()->pack_fd, inbuf, n, from); if (n < 0) die_errno(_("cannot pread pack file")); if (!n) @@@ -1303,7 -1291,7 +1303,7 @@@ static void final(const char *final_pac if (keep_fd < 0) { if (errno != EEXIST) die_errno(_("cannot write keep file '%s'"), - keep_name); + keep_name ? keep_name : name); } else { if (keep_msg_len > 0) { write_or_die(keep_fd, keep_msg, keep_msg_len); @@@ -1311,7 -1299,7 +1311,7 @@@ } if (close(keep_fd) != 0) die_errno(_("cannot close written keep file '%s'"), - keep_name); + keep_name ? keep_name : name); report = "keep"; } } @@@ -1502,7 -1490,7 +1502,7 @@@ static void show_pack_info(int stat_onl int cmd_index_pack(int argc, const char **argv, const char *prefix) { int i, fix_thin_pack = 0, verify = 0, stat_only = 0; - const char *curr_pack, *curr_index; + const char *curr_index; const char *index_name = NULL, *pack_name = NULL; const char *keep_name = NULL, *keep_msg = NULL; char *index_name_buf = NULL, *keep_name_buf = NULL; @@@ -1514,7 -1502,7 +1514,7 @@@ if (argc == 2 && !strcmp(argv[1], "-h")) usage(index_pack_usage); - read_replace_refs = 0; + check_replace_refs = 0; reset_pack_idx_option(&opts); git_config(git_index_pack_config, &opts); @@@ -1546,9 -1534,9 +1546,9 @@@ stat_only = 1; } else if (!strcmp(arg, "--keep")) { keep_msg = ""; - } else if (!prefixcmp(arg, "--keep=")) { + } else if (starts_with(arg, "--keep=")) { keep_msg = arg + 7; - } else if (!prefixcmp(arg, "--threads=")) { + } else if (starts_with(arg, "--threads=")) { char *end; nr_threads = strtoul(arg+10, &end, 0); if (!arg[10] || *end || nr_threads < 0) @@@ -1559,7 -1547,7 +1559,7 @@@ "ignoring %s"), arg); nr_threads = 1; #endif - } else if (!prefixcmp(arg, "--pack_header=")) { + } else if (starts_with(arg, "--pack_header=")) { struct pack_header *hdr; char *c; @@@ -1578,7 -1566,7 +1578,7 @@@ if (index_name || (i+1) >= argc) usage(index_pack_usage); index_name = argv[++i]; - } else if (!prefixcmp(arg, "--index-version=")) { + } else if (starts_with(arg, "--index-version=")) { char *c; opts.version = strtoul(arg + 16, &c, 10); if (opts.version > 2) diff --combined cache.h index 107ac61b68,9244c387c3..cc46be4e0f --- a/cache.h +++ b/cache.h @@@ -3,7 -3,7 +3,7 @@@ #include "git-compat-util.h" #include "strbuf.h" -#include "hash.h" +#include "hashmap.h" #include "advice.h" #include "gettext.h" #include "convert.h" @@@ -130,12 -130,12 +130,12 @@@ struct stat_data }; struct cache_entry { + struct hashmap_entry ent; struct stat_data ce_stat_data; unsigned int ce_mode; unsigned int ce_flags; unsigned int ce_namelen; unsigned char sha1[20]; - struct cache_entry *next; char name[FLEX_ARRAY]; /* more */ }; @@@ -159,6 -159,7 +159,6 @@@ #define CE_ADDED (1 << 19) #define CE_HASHED (1 << 20) -#define CE_UNHASHED (1 << 21) #define CE_WT_REMOVE (1 << 22) /* remove in work directory */ #define CE_CONFLICTED (1 << 23) @@@ -194,18 -195,17 +194,18 @@@ struct pathspec * Copy the sha1 and stat state of a cache entry from one to * another. But we never change the name, or the hash state! */ -#define CE_STATE_MASK (CE_HASHED | CE_UNHASHED) static inline void copy_cache_entry(struct cache_entry *dst, const struct cache_entry *src) { - unsigned int state = dst->ce_flags & CE_STATE_MASK; + unsigned int state = dst->ce_flags & CE_HASHED; /* Don't copy hash chain and name */ - memcpy(dst, src, offsetof(struct cache_entry, next)); + memcpy(&dst->ce_stat_data, &src->ce_stat_data, + offsetof(struct cache_entry, name) - + offsetof(struct cache_entry, ce_stat_data)); /* Restore the hash state */ - dst->ce_flags = (dst->ce_flags & ~CE_STATE_MASK) | state; + dst->ce_flags = (dst->ce_flags & ~CE_HASHED) | state; } static inline unsigned create_ce_flags(unsigned stage) @@@ -277,8 -277,9 +277,9 @@@ struct index_state struct cache_time timestamp; unsigned name_hash_initialized : 1, initialized : 1; - struct hash_table name_hash; - struct hash_table dir_hash; + struct hashmap name_hash; + struct hashmap dir_hash; + unsigned char sha1[20]; }; extern struct index_state the_index; @@@ -316,6 -317,7 +317,6 @@@ extern void free_name_hash(struct index #define ce_modified(ce, st, options) ie_modified(&the_index, (ce), (st), (options)) #define cache_dir_exists(name, namelen) index_dir_exists(&the_index, (name), (namelen)) #define cache_file_exists(name, namelen, igncase) index_file_exists(&the_index, (name), (namelen), (igncase)) -#define cache_name_exists(name, namelen, igncase) index_name_exists(&the_index, (name), (namelen), (igncase)) #define cache_name_is_other(name, namelen) index_name_is_other(&the_index, (name), (namelen)) #define resolve_undo_clear() resolve_undo_clear_index(&the_index) #define unmerge_cache_entry_at(at) unmerge_index_entry_at(&the_index, at) @@@ -353,7 -355,6 +354,7 @@@ static inline enum object_type object_t #define DB_ENVIRONMENT "GIT_OBJECT_DIRECTORY" #define INDEX_ENVIRONMENT "GIT_INDEX_FILE" #define GRAFT_ENVIRONMENT "GIT_GRAFT_FILE" +#define GIT_SHALLOW_FILE_ENVIRONMENT "GIT_SHALLOW_FILE" #define TEMPLATE_DIR_ENVIRONMENT "GIT_TEMPLATE_DIR" #define CONFIG_ENVIRONMENT "GIT_CONFIG" #define CONFIG_DATA_ENVIRONMENT "GIT_CONFIG_PARAMETERS" @@@ -433,7 -434,6 +434,7 @@@ extern int set_git_dir_init(const char extern int init_db(const char *template_dir, unsigned int flags); extern void sanitize_stdfds(void); +extern int daemonize(void); #define alloc_nr(x) (((x)+16)*3/2) @@@ -467,6 -467,7 +468,6 @@@ extern int unmerged_index(const struct extern int verify_path(const char *path); extern struct cache_entry *index_dir_exists(struct index_state *istate, const char *name, int namelen); extern struct cache_entry *index_file_exists(struct index_state *istate, const char *name, int namelen, int igncase); -extern struct cache_entry *index_name_exists(struct index_state *istate, const char *name, int namelen, int igncase); extern int index_name_pos(const struct index_state *, const char *name, int namelen); #define ADD_CACHE_OK_TO_ADD 1 /* Ok to add */ #define ADD_CACHE_OK_TO_REPLACE 2 /* Ok to replace file/directory */ @@@ -483,11 -484,11 +484,11 @@@ extern int remove_file_from_index(struc #define ADD_CACHE_IGNORE_ERRORS 4 #define ADD_CACHE_IGNORE_REMOVAL 8 #define ADD_CACHE_INTENT 16 -#define ADD_CACHE_IMPLICIT_DOT 32 /* internal to "git add -u/-A" */ extern int add_to_index(struct index_state *, const char *path, struct stat *, int flags); extern int add_file_to_index(struct index_state *, const char *path, int flags); -extern struct cache_entry *make_cache_entry(unsigned int mode, const unsigned char *sha1, const char *path, int stage, int refresh); +extern struct cache_entry *make_cache_entry(unsigned int mode, const unsigned char *sha1, const char *path, int stage, unsigned int refresh_options); extern int ce_same_name(const struct cache_entry *a, const struct cache_entry *b); +extern void set_object_name_for_intent_to_add_entry(struct cache_entry *ce); extern int index_name_is_other(const struct index_state *, const char *, int); extern void *read_blob_data_from_index(struct index_state *, const char *, unsigned long *); @@@ -497,13 -498,11 +498,13 @@@ #define CE_MATCH_RACY_IS_DIRTY 02 /* do stat comparison even if CE_SKIP_WORKTREE is true */ #define CE_MATCH_IGNORE_SKIP_WORKTREE 04 +/* ignore non-existent files during stat update */ +#define CE_MATCH_IGNORE_MISSING 0x08 +/* enable stat refresh */ +#define CE_MATCH_REFRESH 0x10 extern int ie_match_stat(const struct index_state *, const struct cache_entry *, struct stat *, unsigned int); extern int ie_modified(const struct index_state *, const struct cache_entry *, struct stat *, unsigned int); -extern int ce_path_match(const struct cache_entry *ce, const struct pathspec *pathspec); - #define HASH_WRITE_OBJECT 1 #define HASH_FORMAT_CHECK 2 extern int index_fd(unsigned char *sha1, int fd, struct stat *st, enum object_type type, const char *path, unsigned flags); @@@ -581,17 -580,7 +582,17 @@@ extern size_t packed_git_limit extern size_t delta_base_cache_limit; extern unsigned long big_file_threshold; extern unsigned long pack_size_limit_cfg; -extern int read_replace_refs; + +/* + * Do replace refs need to be checked this run? This variable is + * initialized to true unless --no-replace-object is used or + * $GIT_NO_REPLACE_OBJECTS is set, but is set to false by some + * commands that do not want replace references to be active. As an + * optimization it is also set to false if replace references have + * been sought but there were none. + */ +extern int check_replace_refs; + extern int fsync_object_files; extern int core_preload_index; extern int core_apply_sparse_checkout; @@@ -670,28 -659,9 +671,28 @@@ extern char *git_path(const char *fmt, extern char *git_path_submodule(const char *path, const char *fmt, ...) __attribute__((format (printf, 2, 3))); -extern char *sha1_file_name(const unsigned char *sha1); +/* + * Return the name of the file in the local object database that would + * be used to store a loose object with the specified sha1. The + * return value is a pointer to a statically allocated buffer that is + * overwritten each time the function is called. + */ +extern const char *sha1_file_name(const unsigned char *sha1); + +/* + * Return the name of the (local) packfile with the specified sha1 in + * its name. The return value is a pointer to memory that is + * overwritten each time this function is called. + */ extern char *sha1_pack_name(const unsigned char *sha1); + +/* + * Return the name of the (local) pack index file with the specified + * sha1 in its name. The return value is a pointer to memory that is + * overwritten each time this function is called. + */ extern char *sha1_pack_index_name(const unsigned char *sha1); + extern const char *find_unique_abbrev(const unsigned char *sha1, int); extern const unsigned char null_sha1[20]; @@@ -767,29 -737,8 +768,29 @@@ enum sharedrepo }; int git_config_perm(const char *var, const char *value); int adjust_shared_perm(const char *path); -int safe_create_leading_directories(char *path); -int safe_create_leading_directories_const(const char *path); + +/* + * Create the directory containing the named path, using care to be + * somewhat safe against races. Return one of the scld_error values + * to indicate success/failure. + * + * SCLD_VANISHED indicates that one of the ancestor directories of the + * path existed at one point during the function call and then + * suddenly vanished, probably because another process pruned the + * directory while we were working. To be robust against this kind of + * race, callers might want to try invoking the function again when it + * returns SCLD_VANISHED. + */ +enum scld_error { + SCLD_OK = 0, + SCLD_FAILED = -1, + SCLD_PERMS = -2, + SCLD_EXISTS = -3, + SCLD_VANISHED = -4 +}; +enum scld_error safe_create_leading_directories(char *path); +enum scld_error safe_create_leading_directories_const(const char *path); + int mkdir_in_gitdir(const char *path); extern void home_config_paths(char **global, char **xdg, char *file); extern char *expand_user_path(const char *path); @@@ -812,46 -761,26 +813,46 @@@ int daemon_avoid_alias(const char *path int offset_1st_component(const char *path); /* object replacement */ -#define READ_SHA1_FILE_REPLACE 1 +#define LOOKUP_REPLACE_OBJECT 1 extern void *read_sha1_file_extended(const unsigned char *sha1, enum object_type *type, unsigned long *size, unsigned flag); static inline void *read_sha1_file(const unsigned char *sha1, enum object_type *type, unsigned long *size) { - return read_sha1_file_extended(sha1, type, size, READ_SHA1_FILE_REPLACE); + return read_sha1_file_extended(sha1, type, size, LOOKUP_REPLACE_OBJECT); } + +/* + * This internal function is only declared here for the benefit of + * lookup_replace_object(). Please do not call it directly. + */ extern const unsigned char *do_lookup_replace_object(const unsigned char *sha1); + +/* + * If object sha1 should be replaced, return the replacement object's + * name (replaced recursively, if necessary). The return value is + * either sha1 or a pointer to a permanently-allocated value. When + * object replacement is suppressed, always return sha1. + */ static inline const unsigned char *lookup_replace_object(const unsigned char *sha1) { - if (!read_replace_refs) + if (!check_replace_refs) return sha1; return do_lookup_replace_object(sha1); } +static inline const unsigned char *lookup_replace_object_extended(const unsigned char *sha1, unsigned flag) +{ + if (!(flag & LOOKUP_REPLACE_OBJECT)) + return sha1; + return lookup_replace_object(sha1); +} + /* Read and unpack a sha1 file into memory, write memory to a sha1 file */ extern int sha1_object_info(const unsigned char *, unsigned long *); extern int hash_sha1_file(const void *buf, unsigned long len, const char *type, unsigned char *sha1); extern int write_sha1_file(const void *buf, unsigned long len, const char *type, unsigned char *return_sha1); extern int pretend_sha1_file(void *, unsigned long, enum object_type, unsigned char *); extern int force_object_loose(const unsigned char *sha1, time_t mtime); +extern int git_open_noatime(const char *name); extern void *map_sha1_file(const unsigned char *sha1, unsigned long *size); extern int unpack_sha1_header(git_zstream *stream, unsigned char *map, unsigned long mapsize, void *buffer, unsigned long bufsiz); extern int parse_sha1_header(const char *hdr, unsigned long *sizep); @@@ -864,19 -793,7 +865,19 @@@ extern int check_sha1_signature(const u extern int move_temp_to_file(const char *tmpfile, const char *filename); extern int has_sha1_pack(const unsigned char *sha1); + +/* + * Return true iff we have an object named sha1, whether local or in + * an alternate object database, and whether packed or loose. This + * function does not respect replace references. + */ extern int has_sha1_file(const unsigned char *sha1); + +/* + * Return true iff an alternate object database has a loose object + * with the specified name. This function does not respect replace + * references. + */ extern int has_loose_object_nonlocal(const unsigned char *sha1); extern int has_pack_index(const unsigned char *sha1); @@@ -971,12 -888,9 +972,12 @@@ extern int dwim_log(const char *str, in extern int interpret_branch_name(const char *str, int len, struct strbuf *); extern int get_sha1_mb(const char *str, unsigned char *sha1); -extern int refname_match(const char *abbrev_name, const char *full_name, const char **rules); -extern const char *ref_rev_parse_rules[]; -#define ref_fetch_rules ref_rev_parse_rules +/* + * Return true iff abbrev_name is a possible abbreviation for + * full_name according to the rules defined by ref_rev_parse_rules in + * refs.c. + */ +extern int refname_match(const char *abbrev_name, const char *full_name); extern int create_symref(const char *ref, const char *refs_heads_master, const char *logmsg); extern int validate_headref(const char *ref); @@@ -1015,7 -929,6 +1016,7 @@@ void datestamp(char *buf, int bufsize) unsigned long approxidate_careful(const char *, int *); unsigned long approxidate_relative(const char *date, const struct timeval *now); enum date_mode parse_date_format(const char *format); +int date_overflows(unsigned long date); #define IDENT_STRICT 1 #define IDENT_NO_DATE 2 @@@ -1140,46 -1053,17 +1141,46 @@@ extern struct packed_git *find_sha1_pac struct packed_git *packs); extern void pack_report(void); + +/* + * mmap the index file for the specified packfile (if it is not + * already mmapped). Return 0 on success. + */ extern int open_pack_index(struct packed_git *); + +/* + * munmap the index file for the specified packfile (if it is + * currently mmapped). + */ extern void close_pack_index(struct packed_git *); + extern unsigned char *use_pack(struct packed_git *, struct pack_window **, off_t, unsigned long *); extern void close_pack_windows(struct packed_git *); extern void unuse_pack(struct pack_window **); extern void free_pack_by_name(const char *); extern void clear_delta_base_cache(void); extern struct packed_git *add_packed_git(const char *, int, int); -extern const unsigned char *nth_packed_object_sha1(struct packed_git *, uint32_t); -extern off_t nth_packed_object_offset(const struct packed_git *, uint32_t); -extern off_t find_pack_entry_one(const unsigned char *, struct packed_git *); + +/* + * Return the SHA-1 of the nth object within the specified packfile. + * Open the index if it is not already open. The return value points + * at the SHA-1 within the mmapped index. Return NULL if there is an + * error. + */ +extern const unsigned char *nth_packed_object_sha1(struct packed_git *, uint32_t n); + +/* + * Return the offset of the nth object within the specified packfile. + * The index must already be opened. + */ +extern off_t nth_packed_object_offset(const struct packed_git *, uint32_t n); + +/* + * If the object named sha1 is present in the specified packfile, + * return its offset within the packfile; otherwise, return 0. + */ +extern off_t find_pack_entry_one(const unsigned char *sha1, struct packed_git *); + extern int is_pack_valid(struct packed_git *); extern void *unpack_entry(struct packed_git *, off_t, enum object_type *, unsigned long *); extern unsigned long unpack_object_header_buffer(const unsigned char *buf, unsigned long len, enum object_type *type, unsigned long *sizep); @@@ -1191,7 -1075,6 +1192,7 @@@ struct object_info enum object_type *typep; unsigned long *sizep; unsigned long *disk_sizep; + unsigned char *delta_base_sha1; /* Response */ enum { @@@ -1216,7 -1099,7 +1217,7 @@@ } packed; } u; }; -extern int sha1_object_info_extended(const unsigned char *, struct object_info *); +extern int sha1_object_info_extended(const unsigned char *, struct object_info *, unsigned flags); /* Dumb servers support */ extern int update_server_info(int); @@@ -1232,12 -1115,6 +1233,12 @@@ #define CONFIG_INVALID_PATTERN 6 #define CONFIG_GENERIC_ERROR 7 +struct git_config_source { + unsigned int use_stdin:1; + const char *file; + const char *blob; +}; + typedef int (*config_fn_t)(const char *, const char *, void *); extern int git_default_config(const char *, const char *, void *); extern int git_config_from_file(config_fn_t fn, const char *, void *); @@@ -1247,7 -1124,8 +1248,7 @@@ extern void git_config_push_parameter(c extern int git_config_from_parameters(config_fn_t fn, void *data); extern int git_config(config_fn_t fn, void *); extern int git_config_with_options(config_fn_t fn, void *, - const char *filename, - const char *blob_ref, + struct git_config_source *config_source, int respect_includes); extern int git_config_early(config_fn_t fn, void *, const char *repo_config); extern int git_parse_ulong(const char *, unsigned long *); @@@ -1322,6 -1200,8 +1323,8 @@@ extern void fsync_or_die(int fd, const extern ssize_t read_in_full(int fd, void *buf, size_t count); extern ssize_t write_in_full(int fd, const void *buf, size_t count); + extern ssize_t pread_in_full(int fd, void *buf, size_t count, off_t offset); + static inline ssize_t write_str_in_full(int fd, const char *str) { return write_in_full(fd, str, strlen(str)); @@@ -1359,8 -1239,6 +1362,8 @@@ __attribute__((format (printf, 2, 3)) extern void trace_argv_printf(const char **argv, const char *format, ...); extern void trace_repo_setup(const char *prefix); extern int trace_want(const char *key); +__attribute__((format (printf, 2, 3))) +extern void trace_printf_key(const char *key, const char *fmt, ...); extern void trace_strbuf(const char *key, const struct strbuf *buf); void packet_trace_identity(const char *prog); @@@ -1458,6 -1336,4 +1461,6 @@@ int stat_validity_check(struct stat_val */ void stat_validity_update(struct stat_validity *sv, int fd); +int versioncmp(const char *s1, const char *s2); + #endif /* CACHE_H */ diff --combined git-compat-util.h index f6d3a46622,9eec5fb52b..e6a4159a25 --- a/git-compat-util.h +++ b/git-compat-util.h @@@ -75,7 -75,7 +75,7 @@@ # endif #elif !defined(__APPLE__) && !defined(__FreeBSD__) && !defined(__USLC__) && \ !defined(_M_UNIX) && !defined(__sgi) && !defined(__DragonFly__) && \ - !defined(__TANDEM) && !defined(__QNX__) + !defined(__TANDEM) && !defined(__QNX__) && !defined(__MirBSD__) #define _XOPEN_SOURCE 600 /* glibc2 and AIX 5.3L need 500, OpenBSD needs 600 for S_ISLNK() */ #define _XOPEN_SOURCE_EXTENDED 1 /* AIX 5.3L needs this */ #endif @@@ -116,6 -116,9 +116,6 @@@ #include #include #include -#ifndef USE_WILDMATCH -#include -#endif #include #include #include @@@ -301,7 -304,16 +301,7 @@@ extern char *gitbasename(char *) #include "compat/bswap.h" -#ifdef USE_WILDMATCH #include "wildmatch.h" -#define FNM_PATHNAME WM_PATHNAME -#define FNM_CASEFOLD WM_CASEFOLD -#define FNM_NOMATCH WM_NOMATCH -static inline int fnmatch(const char *pattern, const char *string, int flags) -{ - return wildmatch(pattern, string, flags, NULL); -} -#endif /* General helper functions */ extern void vreportf(const char *prefix, const char *err, va_list params); @@@ -338,16 -350,13 +338,16 @@@ extern void set_die_routine(NORETURN_PT extern void set_error_routine(void (*routine)(const char *err, va_list params)); extern void set_die_is_recursing_routine(int (*routine)(void)); -extern int prefixcmp(const char *str, const char *prefix); -extern int suffixcmp(const char *str, const char *suffix); +extern int starts_with(const char *str, const char *prefix); +extern int ends_with(const char *str, const char *suffix); static inline const char *skip_prefix(const char *str, const char *prefix) { - size_t len = strlen(prefix); - return strncmp(str, prefix, len) ? NULL : str + len; + do { + if (!*prefix) + return str; + } while (*str++ == *prefix++); + return NULL; } #if defined(NO_MMAP) || defined(USE_WIN32_MMAP) @@@ -469,15 -478,9 +469,15 @@@ extern FILE *git_fopen(const char*, con #endif #ifdef SNPRINTF_RETURNS_BOGUS +#ifdef snprintf +#undef snprintf +#endif #define snprintf git_snprintf extern int git_snprintf(char *str, size_t maxsize, const char *format, ...); +#ifdef vsnprintf +#undef vsnprintf +#endif #define vsnprintf git_vsnprintf extern int git_vsnprintf(char *str, size_t maxsize, const char *format, va_list ap); @@@ -531,12 -534,13 +531,13 @@@ extern void *xcalloc(size_t nmemb, size extern void *xmmap(void *start, size_t length, int prot, int flags, int fd, off_t offset); extern ssize_t xread(int fd, void *buf, size_t len); extern ssize_t xwrite(int fd, const void *buf, size_t len); + extern ssize_t xpread(int fd, void *buf, size_t len, off_t offset); extern int xdup(int fd); extern FILE *xfdopen(int fd, const char *mode); extern int xmkstemp(char *template); extern int xmkstemp_mode(char *template, int mode); extern int odb_mkstemp(char *template, size_t limit, const char *pattern); -extern int odb_pack_keep(char *name, size_t namesz, unsigned char *sha1); +extern int odb_pack_keep(char *name, size_t namesz, const unsigned char *sha1); static inline size_t xsize_t(off_t len) { @@@ -716,11 -720,4 +717,11 @@@ void warn_on_inaccessible(const char *p /* Get the passwd entry for the UID of the current process. */ struct passwd *xgetpwuid_self(void); +#ifdef GMTIME_UNRELIABLE_ERRORS +struct tm *git_gmtime(const time_t *); +struct tm *git_gmtime_r(const time_t *, struct tm *); +#define gmtime git_gmtime +#define gmtime_r git_gmtime_r +#endif + #endif diff --combined read-cache.c index ba13353b37,f4a0d6168b..7f5645e745 --- a/read-cache.c +++ b/read-cache.c @@@ -15,8 -15,7 +15,8 @@@ #include "strbuf.h" #include "varint.h" -static struct cache_entry *refresh_cache_entry(struct cache_entry *ce, int really); +static struct cache_entry *refresh_cache_entry(struct cache_entry *ce, + unsigned int options); /* Mask for the name length in ce_flags in the on-disk index */ @@@ -48,7 -47,6 +48,7 @@@ static void replace_index_entry(struct struct cache_entry *old = istate->cache[nr]; remove_name_hash(istate, old); + free(old); set_index_entry(istate, nr, ce); istate->cache_changed = 1; } @@@ -60,7 -58,7 +60,7 @@@ void rename_index_entry_at(struct index new = xmalloc(cache_entry_size(namelen)); copy_cache_entry(new, old); - new->ce_flags &= ~CE_STATE_MASK; + new->ce_flags &= ~CE_HASHED; new->ce_namelen = namelen; memcpy(new->name, new_name, namelen + 1); @@@ -480,7 -478,6 +480,7 @@@ int remove_index_entry_at(struct index_ record_resolve_undo(istate, ce); remove_name_hash(istate, ce); + free(ce); istate->cache_changed = 1; istate->cache_nr--; if (pos >= istate->cache_nr) @@@ -502,10 -499,8 +502,10 @@@ void remove_marked_cache_entries(struc unsigned int i, j; for (i = j = 0; i < istate->cache_nr; i++) { - if (ce_array[i]->ce_flags & CE_REMOVE) + if (ce_array[i]->ce_flags & CE_REMOVE) { remove_name_hash(istate, ce_array[i]); + free(ce_array[i]); + } else ce_array[j++] = ce_array[i]; } @@@ -584,7 -579,7 +584,7 @@@ static struct cache_entry *create_alias return new; } -static void record_intent_to_add(struct cache_entry *ce) +void set_object_name_for_intent_to_add_entry(struct cache_entry *ce) { unsigned char sha1[20]; if (write_sha1_file("", 0, blob_type, sha1)) @@@ -670,7 -665,7 +670,7 @@@ int add_to_index(struct index_state *is if (index_path(ce->sha1, path, st, HASH_WRITE_OBJECT)) return error("unable to index file %s", path); } else - record_intent_to_add(ce); + set_object_name_for_intent_to_add_entry(ce); if (ignore_case && alias && different_name(ce, alias)) ce = create_alias_ce(ce, alias); @@@ -701,7 -696,7 +701,7 @@@ int add_file_to_index(struct index_stat struct cache_entry *make_cache_entry(unsigned int mode, const unsigned char *sha1, const char *path, int stage, - int refresh) + unsigned int refresh_options) { int size, len; struct cache_entry *ce; @@@ -721,7 -716,10 +721,7 @@@ ce->ce_namelen = len; ce->ce_mode = create_ce_mode(mode); - if (refresh) - return refresh_cache_entry(ce, 0); - - return ce; + return refresh_cache_entry(ce, refresh_options); } int ce_same_name(const struct cache_entry *a, const struct cache_entry *b) @@@ -730,6 -728,11 +730,6 @@@ return ce_namelen(b) == len && !memcmp(a->name, b->name, len); } -int ce_path_match(const struct cache_entry *ce, const struct pathspec *pathspec) -{ - return match_pathspec_depth(pathspec, ce->name, ce_namelen(ce), 0, NULL); -} - /* * We fundamentally don't like some paths: we don't want * dot or dot-dot anywhere, and for obvious reasons don't @@@ -990,7 -993,11 +990,7 @@@ int add_index_entry(struct index_state } /* Make sure the array is big enough .. */ - if (istate->cache_nr == istate->cache_alloc) { - istate->cache_alloc = alloc_nr(istate->cache_alloc); - istate->cache = xrealloc(istate->cache, - istate->cache_alloc * sizeof(*istate->cache)); - } + ALLOC_GROW(istate->cache, istate->cache_nr + 1, istate->cache_alloc); /* Add it in.. */ istate->cache_nr++; @@@ -1022,12 -1029,10 +1022,12 @@@ static struct cache_entry *refresh_cach struct stat st; struct cache_entry *updated; int changed, size; + int refresh = options & CE_MATCH_REFRESH; int ignore_valid = options & CE_MATCH_IGNORE_VALID; int ignore_skip_worktree = options & CE_MATCH_IGNORE_SKIP_WORKTREE; + int ignore_missing = options & CE_MATCH_IGNORE_MISSING; - if (ce_uptodate(ce)) + if (!refresh || ce_uptodate(ce)) return ce; /* @@@ -1045,8 -1050,6 +1045,8 @@@ } if (lstat(ce->name, &st) < 0) { + if (ignore_missing && errno == ENOENT) + return ce; if (err) *err = errno; return NULL; @@@ -1124,9 -1127,7 +1124,9 @@@ int refresh_index(struct index_state *i int ignore_submodules = (flags & REFRESH_IGNORE_SUBMODULES) != 0; int first = 1; int in_porcelain = (flags & REFRESH_IN_PORCELAIN); - unsigned int options = really ? CE_MATCH_IGNORE_VALID : 0; + unsigned int options = (CE_MATCH_REFRESH | + (really ? CE_MATCH_IGNORE_VALID : 0) | + (not_new ? CE_MATCH_IGNORE_MISSING : 0)); const char *modified_fmt; const char *deleted_fmt; const char *typechange_fmt; @@@ -1148,7 -1149,8 +1148,7 @@@ if (ignore_submodules && S_ISGITLINK(ce->ce_mode)) continue; - if (pathspec && - !match_pathspec_depth(pathspec, ce->name, ce_namelen(ce), 0, seen)) + if (pathspec && !ce_path_match(ce, pathspec, seen)) filtered = 1; if (ce_stage(ce)) { @@@ -1174,6 -1176,8 +1174,6 @@@ if (!new) { const char *fmt; - if (not_new && cache_errno == ENOENT) - continue; if (really && cache_errno == EINVAL) { /* If we are doing --really-refresh that * means the index is not valid anymore. @@@ -1203,10 -1207,9 +1203,10 @@@ return has_errors; } -static struct cache_entry *refresh_cache_entry(struct cache_entry *ce, int really) +static struct cache_entry *refresh_cache_entry(struct cache_entry *ce, + unsigned int options) { - return refresh_cache_ent(&the_index, ce, really, NULL, NULL); + return refresh_cache_ent(&the_index, ce, options, NULL, NULL); } @@@ -1216,42 -1219,6 +1216,42 @@@ #define INDEX_FORMAT_DEFAULT 3 +static int index_format_config(const char *var, const char *value, void *cb) +{ + unsigned int *version = cb; + if (!strcmp(var, "index.version")) { + *version = git_config_int(var, value); + return 0; + } + return 1; +} + +static unsigned int get_index_format_default(void) +{ + char *envversion = getenv("GIT_INDEX_VERSION"); + char *endp; + unsigned int version = INDEX_FORMAT_DEFAULT; + + if (!envversion) { + git_config(index_format_config, &version); + if (version < INDEX_FORMAT_LB || INDEX_FORMAT_UB < version) { + warning(_("index.version set, but the value is invalid.\n" + "Using version %i"), INDEX_FORMAT_DEFAULT); + return INDEX_FORMAT_DEFAULT; + } + return version; + } + + version = strtoul(envversion, &endp, 10); + if (*endp || + version < INDEX_FORMAT_LB || INDEX_FORMAT_UB < version) { + warning(_("GIT_INDEX_VERSION set, but the value is invalid.\n" + "Using version %i"), INDEX_FORMAT_DEFAULT); + version = INDEX_FORMAT_DEFAULT; + } + return version; +} + /* * dev/ino/uid/gid/size are also just tracked to the low 32 bits * Again - this is just a (very strong in practice) heuristic that @@@ -1346,6 -1313,26 +1346,6 @@@ int read_index(struct index_state *ista return read_index_from(istate, get_index_file()); } -#ifndef NEEDS_ALIGNED_ACCESS -#define ntoh_s(var) ntohs(var) -#define ntoh_l(var) ntohl(var) -#else -static inline uint16_t ntoh_s_force_align(void *p) -{ - uint16_t x; - memcpy(&x, p, sizeof(x)); - return ntohs(x); -} -static inline uint32_t ntoh_l_force_align(void *p) -{ - uint32_t x; - memcpy(&x, p, sizeof(x)); - return ntohl(x); -} -#define ntoh_s(var) ntoh_s_force_align(&(var)) -#define ntoh_l(var) ntoh_l_force_align(&(var)) -#endif - static struct cache_entry *cache_entry_from_ondisk(struct ondisk_cache_entry *ondisk, unsigned int flags, const char *name, @@@ -1353,16 -1340,16 +1353,16 @@@ { struct cache_entry *ce = xmalloc(cache_entry_size(len)); - ce->ce_stat_data.sd_ctime.sec = ntoh_l(ondisk->ctime.sec); - ce->ce_stat_data.sd_mtime.sec = ntoh_l(ondisk->mtime.sec); - ce->ce_stat_data.sd_ctime.nsec = ntoh_l(ondisk->ctime.nsec); - ce->ce_stat_data.sd_mtime.nsec = ntoh_l(ondisk->mtime.nsec); - ce->ce_stat_data.sd_dev = ntoh_l(ondisk->dev); - ce->ce_stat_data.sd_ino = ntoh_l(ondisk->ino); - ce->ce_mode = ntoh_l(ondisk->mode); - ce->ce_stat_data.sd_uid = ntoh_l(ondisk->uid); - ce->ce_stat_data.sd_gid = ntoh_l(ondisk->gid); - ce->ce_stat_data.sd_size = ntoh_l(ondisk->size); + ce->ce_stat_data.sd_ctime.sec = get_be32(&ondisk->ctime.sec); + ce->ce_stat_data.sd_mtime.sec = get_be32(&ondisk->mtime.sec); + ce->ce_stat_data.sd_ctime.nsec = get_be32(&ondisk->ctime.nsec); + ce->ce_stat_data.sd_mtime.nsec = get_be32(&ondisk->mtime.nsec); + ce->ce_stat_data.sd_dev = get_be32(&ondisk->dev); + ce->ce_stat_data.sd_ino = get_be32(&ondisk->ino); + ce->ce_mode = get_be32(&ondisk->mode); + ce->ce_stat_data.sd_uid = get_be32(&ondisk->uid); + ce->ce_stat_data.sd_gid = get_be32(&ondisk->gid); + ce->ce_stat_data.sd_size = get_be32(&ondisk->size); ce->ce_flags = flags & ~CE_NAMEMASK; ce->ce_namelen = len; hashcpy(ce->sha1, ondisk->sha1); @@@ -1402,14 -1389,14 +1402,14 @@@ static struct cache_entry *create_from_ unsigned int flags; /* On-disk flags are just 16 bits */ - flags = ntoh_s(ondisk->flags); + flags = get_be16(&ondisk->flags); len = flags & CE_NAMEMASK; if (flags & CE_EXTENDED) { struct ondisk_cache_entry_extended *ondisk2; int extended_flags; ondisk2 = (struct ondisk_cache_entry_extended *)ondisk; - extended_flags = ntoh_s(ondisk2->flags2) << 16; + extended_flags = get_be16(&ondisk2->flags2) << 16; /* We do not yet understand any bit out of CE_EXTENDED_FLAGS */ if (extended_flags & ~CE_EXTENDED_FLAGS) die("Unknown index entry format %08x", extended_flags); @@@ -1477,6 -1464,7 +1477,7 @@@ int read_index_from(struct index_state if (verify_hdr(hdr, mmap_size) < 0) goto unmap; + hashcpy(istate->sha1, (unsigned char *)hdr + mmap_size - 20); istate->version = ntohl(hdr->hdr_version); istate->cache_nr = ntohl(hdr->hdr_entries); istate->cache_alloc = alloc_nr(istate->cache_nr); @@@ -1760,6 -1748,50 +1761,50 @@@ static int ce_write_entry(git_SHA_CTX * return result; } + /* + * This function verifies if index_state has the correct sha1 of the + * index file. Don't die if we have any other failure, just return 0. + */ + static int verify_index_from(const struct index_state *istate, const char *path) + { + int fd; + ssize_t n; + struct stat st; + unsigned char sha1[20]; + + if (!istate->initialized) + return 0; + + fd = open(path, O_RDONLY); + if (fd < 0) + return 0; + + if (fstat(fd, &st)) + goto out; + + if (st.st_size < sizeof(struct cache_header) + 20) + goto out; + + n = pread_in_full(fd, sha1, 20, st.st_size - 20); + if (n != 20) + goto out; + + if (hashcmp(istate->sha1, sha1)) + goto out; + + close(fd); + return 1; + + out: + close(fd); + return 0; + } + + static int verify_index(const struct index_state *istate) + { + return verify_index_from(istate, get_index_file()); + } + static int has_racy_timestamp(struct index_state *istate) { int entries = istate->cache_nr; @@@ -1779,7 -1811,7 +1824,7 @@@ void update_index_if_able(struct index_state *istate, struct lock_file *lockfile) { if ((istate->cache_changed || has_racy_timestamp(istate)) && - !write_index(istate, lockfile->fd)) + verify_index(istate) && !write_index(istate, lockfile->fd)) commit_locked_index(lockfile); else rollback_lock_file(lockfile); @@@ -1808,7 -1840,7 +1853,7 @@@ int write_index(struct index_state *ist } if (!istate->version) - istate->version = INDEX_FORMAT_DEFAULT; + istate->version = get_index_format_default(); /* demote version 3 to version 2 when the latter suffices */ if (istate->version == 3 || istate->version == 2) @@@ -1907,7 -1939,7 +1952,7 @@@ int read_index_unmerged(struct index_st new_ce->ce_mode = ce->ce_mode; if (add_index_entry(istate, new_ce, 0)) return error("%s: cannot drop to stage #0", - ce->name); + new_ce->name); i = index_name_pos(istate, new_ce->name, len); } return unmerged;