From: Junio C Hamano Date: Tue, 15 Mar 2011 21:22:23 +0000 (-0700) Subject: Merge branch 'sp/maint-fd-limit' X-Git-Tag: v1.7.5-rc0~73 X-Git-Url: https://git.lorimer.id.au/gitweb.git/diff_plain/674ef90904fe651ecacafc4211a8a24014160207?ds=inline;hp=-c Merge branch 'sp/maint-fd-limit' * sp/maint-fd-limit: sha1_file.c: Don't retain open fds on small packs mingw: add minimum getrlimit() compatibility stub Limit file descriptors used by packs --- 674ef90904fe651ecacafc4211a8a24014160207 diff --combined cache.h index d0bbc9120e,663ab5804b..c4ef999762 --- a/cache.h +++ b/cache.h @@@ -500,23 -500,8 +500,23 @@@ extern int index_name_is_other(const st extern int ie_match_stat(const struct index_state *, struct cache_entry *, struct stat *, unsigned int); extern int ie_modified(const struct index_state *, struct cache_entry *, struct stat *, unsigned int); -extern int ce_path_match(const struct cache_entry *ce, const char **pathspec); -extern int index_fd(unsigned char *sha1, int fd, struct stat *st, int write_object, enum object_type type, const char *path); +struct pathspec { + const char **raw; /* get_pathspec() result, not freed by free_pathspec() */ + int nr; + int has_wildcard:1; + int recursive:1; + int max_depth; + struct pathspec_item { + const char *match; + int len; + int has_wildcard:1; + } *items; +}; + +extern int init_pathspec(struct pathspec *, const char **); +extern void free_pathspec(struct pathspec *); +extern int ce_path_match(const struct cache_entry *ce, const struct pathspec *pathspec); +extern int index_fd(unsigned char *sha1, int fd, struct stat *st, int write_object, enum object_type type, const char *path, int format_check); extern int index_path(unsigned char *sha1, const char *path, struct stat *st, int write_object); extern void fill_stat_cache_info(struct cache_entry *ce, struct stat *st); @@@ -526,7 -511,7 +526,7 @@@ #define REFRESH_IGNORE_MISSING 0x0008 /* ignore non-existent */ #define REFRESH_IGNORE_SUBMODULES 0x0010 /* ignore submodules */ #define REFRESH_IN_PORCELAIN 0x0020 /* user friendly output, not "needs update" */ -extern int refresh_index(struct index_state *, unsigned int flags, const char **pathspec, char *seen, char *header_msg); +extern int refresh_index(struct index_state *, unsigned int flags, const char **pathspec, char *seen, const char *header_msg); struct lock_file { struct lock_file *next; @@@ -560,6 -545,7 +560,6 @@@ extern int assume_unchanged extern int prefer_symlink_refs; extern int log_all_ref_updates; extern int warn_ambiguous_refs; -extern int unique_abbrev_extra_length; extern int shared_repository; extern const char *apply_default_whitespace; extern const char *apply_default_ignorewhitespace; @@@ -622,7 -608,7 +622,7 @@@ enum rebase_setup_type enum push_default_type { PUSH_DEFAULT_NOTHING = 0, PUSH_DEFAULT_MATCHING, - PUSH_DEFAULT_TRACKING, + PUSH_DEFAULT_UPSTREAM, PUSH_DEFAULT_CURRENT }; @@@ -913,7 -899,8 +913,8 @@@ extern struct packed_git time_t mtime; int pack_fd; unsigned pack_local:1, - pack_keep:1; + pack_keep:1, + do_not_close:1; unsigned char sha1[20]; /* something like ".git/objects/pack/xxxxx.pack" */ char pack_name[FLEX_ARRAY]; /* more */ @@@ -1013,7 -1000,6 +1014,7 @@@ extern int git_config_maybe_bool(const extern int git_config_string(const char **, const char *, const char *); extern int git_config_pathname(const char **, const char *, const char *); extern int git_config_set(const char *, const char *); +extern int git_config_parse_key(const char *, char **, int *); extern int git_config_set_multivar(const char *, const char *, const char *, int); extern int git_config_rename_section(const char *, const char *); extern const char *git_etc_gitconfig(void); diff --combined compat/mingw.h index fe6ba34043,14211c6214..62eccd3391 --- a/compat/mingw.h +++ b/compat/mingw.h @@@ -119,6 -119,14 +119,6 @@@ static inline int mingw_mkdir(const cha } #define mkdir mingw_mkdir -static inline int mingw_unlink(const char *pathname) -{ - /* read-only files cannot be removed */ - chmod(pathname, 0666); - return unlink(pathname); -} -#define unlink mingw_unlink - #define WNOHANG 1 pid_t waitpid(pid_t pid, int *status, unsigned options); @@@ -166,12 -174,6 +166,12 @@@ int link(const char *oldpath, const cha * replacements of existing functions */ +int mingw_unlink(const char *pathname); +#define unlink mingw_unlink + +int mingw_rmdir(const char *path); +#define rmdir mingw_rmdir + int mingw_open (const char *filename, int oflags, ...); #define open mingw_open @@@ -231,6 -233,22 +231,22 @@@ int mingw_getpagesize(void) #define getpagesize mingw_getpagesize #endif + struct rlimit { + unsigned int rlim_cur; + }; + #define RLIMIT_NOFILE 0 + + static inline int getrlimit(int resource, struct rlimit *rlp) + { + if (resource != RLIMIT_NOFILE) { + errno = EINVAL; + return -1; + } + + rlp->rlim_cur = 2048; + return 0; + } + /* Use mingw_lstat() instead of lstat()/stat() and * mingw_fstat() instead of fstat() on Windows. */ diff --combined fast-import.c index e1268b8cb4,2369a7b30a..e24ee2c63d --- a/fast-import.c +++ b/fast-import.c @@@ -24,12 -24,10 +24,12 @@@ Format of STDIN stream commit_msg ('from' sp committish lf)? ('merge' sp committish lf)* - file_change* + (file_change | ls)* lf?; commit_msg ::= data; + ls ::= 'ls' sp '"' quoted(path) '"' lf; + file_change ::= file_clr | file_del | file_rnm @@@ -134,7 -132,7 +134,7 @@@ ts ::= # time since the epoch in seconds, ascii base10 notation; tz ::= # GIT style timezone; - # note: comments and cat requests may appear anywhere + # note: comments, ls and cat requests may appear anywhere # in the input, except within a data command. Any form # of the data command always escapes the related input # from comment processing. @@@ -143,9 -141,7 +143,9 @@@ # must be the first character on that line (an lf # preceded it). # + cat_blob ::= 'cat-blob' sp (hexsha1 | idnum) lf; + ls_tree ::= 'ls' sp (hexsha1 | idnum) sp path_str lf; comment ::= '#' not_lf* lf; not_lf ::= # Any byte that is not ASCII newline (LF); @@@ -333,7 -329,6 +333,7 @@@ static struct mark_set *marks static const char *export_marks_file; static const char *import_marks_file; static int import_marks_file_from_stream; +static int import_marks_file_ignore_missing; static int relative_marks_paths; /* Our last blob */ @@@ -378,7 -373,6 +378,7 @@@ static int cat_blob_fd = STDOUT_FILENO static void parse_argv(void); static void parse_cat_blob(void); +static void parse_ls(struct branch *b); static void write_branch_report(FILE *rpt, struct branch *b) { @@@ -877,6 -871,7 +877,7 @@@ static void start_packfile(void p = xcalloc(1, sizeof(*p) + strlen(tmpfile) + 2); strcpy(p->pack_name, tmpfile); p->pack_fd = pack_fd; + p->do_not_close = 1; pack_file = sha1fd(pack_fd, p->pack_name); hdr.hdr_signature = htonl(PACK_SIGNATURE); @@@ -1801,11 -1796,7 +1802,11 @@@ static void read_marks(void { char line[512]; FILE *f = fopen(import_marks_file, "r"); - if (!f) + if (f) + ; + else if (import_marks_file_ignore_missing && errno == ENOENT) + return; /* Marks file does not exist */ + else die_errno("cannot read '%s'", import_marks_file); while (fgets(line, sizeof(line), f)) { uintmax_t mark; @@@ -2619,8 -2610,6 +2620,8 @@@ static void parse_new_commit(void note_change_n(b, prev_fanout); else if (!strcmp("deleteall", command_buf.buf)) file_change_deleteall(b); + else if (!prefixcmp(command_buf.buf, "ls ")) + parse_ls(b); else { unread_command_buf = 1; break; @@@ -2844,153 -2833,6 +2845,153 @@@ static void parse_cat_blob(void cat_blob(oe, sha1); } +static struct object_entry *dereference(struct object_entry *oe, + unsigned char sha1[20]) +{ + unsigned long size; + char *buf = NULL; + if (!oe) { + enum object_type type = sha1_object_info(sha1, NULL); + if (type < 0) + die("object not found: %s", sha1_to_hex(sha1)); + /* cache it! */ + oe = insert_object(sha1); + oe->type = type; + oe->pack_id = MAX_PACK_ID; + oe->idx.offset = 1; + } + switch (oe->type) { + case OBJ_TREE: /* easy case. */ + return oe; + case OBJ_COMMIT: + case OBJ_TAG: + break; + default: + die("Not a treeish: %s", command_buf.buf); + } + + if (oe->pack_id != MAX_PACK_ID) { /* in a pack being written */ + buf = gfi_unpack_entry(oe, &size); + } else { + enum object_type unused; + buf = read_sha1_file(sha1, &unused, &size); + } + if (!buf) + die("Can't load object %s", sha1_to_hex(sha1)); + + /* Peel one layer. */ + switch (oe->type) { + case OBJ_TAG: + if (size < 40 + strlen("object ") || + get_sha1_hex(buf + strlen("object "), sha1)) + die("Invalid SHA1 in tag: %s", command_buf.buf); + break; + case OBJ_COMMIT: + if (size < 40 + strlen("tree ") || + get_sha1_hex(buf + strlen("tree "), sha1)) + die("Invalid SHA1 in commit: %s", command_buf.buf); + } + + free(buf); + return find_object(sha1); +} + +static struct object_entry *parse_treeish_dataref(const char **p) +{ + unsigned char sha1[20]; + struct object_entry *e; + + if (**p == ':') { /* */ + char *endptr; + e = find_mark(strtoumax(*p + 1, &endptr, 10)); + if (endptr == *p + 1) + die("Invalid mark: %s", command_buf.buf); + if (!e) + die("Unknown mark: %s", command_buf.buf); + *p = endptr; + hashcpy(sha1, e->idx.sha1); + } else { /* */ + if (get_sha1_hex(*p, sha1)) + die("Invalid SHA1: %s", command_buf.buf); + e = find_object(sha1); + *p += 40; + } + + while (!e || e->type != OBJ_TREE) + e = dereference(e, sha1); + return e; +} + +static void print_ls(int mode, const unsigned char *sha1, const char *path) +{ + static struct strbuf line = STRBUF_INIT; + + /* See show_tree(). */ + const char *type = + S_ISGITLINK(mode) ? commit_type : + S_ISDIR(mode) ? tree_type : + blob_type; + + if (!mode) { + /* missing SP path LF */ + strbuf_reset(&line); + strbuf_addstr(&line, "missing "); + quote_c_style(path, &line, NULL, 0); + strbuf_addch(&line, '\n'); + } else { + /* mode SP type SP object_name TAB path LF */ + strbuf_reset(&line); + strbuf_addf(&line, "%06o %s %s\t", + mode, type, sha1_to_hex(sha1)); + quote_c_style(path, &line, NULL, 0); + strbuf_addch(&line, '\n'); + } + cat_blob_write(line.buf, line.len); +} + +static void parse_ls(struct branch *b) +{ + const char *p; + struct tree_entry *root = NULL; + struct tree_entry leaf = {0}; + + /* ls SP ( SP)? */ + p = command_buf.buf + strlen("ls "); + if (*p == '"') { + if (!b) + die("Not in a commit: %s", command_buf.buf); + root = &b->branch_tree; + } else { + struct object_entry *e = parse_treeish_dataref(&p); + root = new_tree_entry(); + hashcpy(root->versions[1].sha1, e->idx.sha1); + load_tree(root); + if (*p++ != ' ') + die("Missing space after tree-ish: %s", command_buf.buf); + } + if (*p == '"') { + static struct strbuf uq = STRBUF_INIT; + const char *endp; + strbuf_reset(&uq); + if (unquote_c_style(&uq, p, &endp)) + die("Invalid path: %s", command_buf.buf); + if (*endp) + die("Garbage after path in: %s", command_buf.buf); + p = uq.buf; + } + tree_content_get(root, p, &leaf); + /* + * A directory in preparation would have a sha1 of zero + * until it is saved. Save, for simplicity. + */ + if (S_ISDIR(leaf.versions[1].mode)) + store_tree(&leaf); + + print_ls(leaf.versions[1].mode, leaf.versions[1].sha1, p); + if (!b || root != &b->branch_tree) + release_tree_entry(root); +} + static void checkpoint(void) { checkpoint_requested = 0; @@@ -3026,8 -2868,7 +3027,8 @@@ static char* make_fast_import_path(cons return strbuf_detach(&abs_path, NULL); } -static void option_import_marks(const char *marks, int from_stream) +static void option_import_marks(const char *marks, + int from_stream, int ignore_missing) { if (import_marks_file) { if (from_stream) @@@ -3041,7 -2882,6 +3042,7 @@@ import_marks_file = make_fast_import_path(marks); safe_create_leading_directories_const(import_marks_file); import_marks_file_from_stream = from_stream; + import_marks_file_ignore_missing = ignore_missing; } static void option_date_format(const char *fmt) @@@ -3141,10 -2981,7 +3142,10 @@@ static int parse_one_feature(const cha if (!prefixcmp(feature, "date-format=")) { option_date_format(feature + 12); } else if (!prefixcmp(feature, "import-marks=")) { - option_import_marks(feature + 13, from_stream); + option_import_marks(feature + 13, from_stream, 0); + } else if (!prefixcmp(feature, "import-marks-if-exists=")) { + option_import_marks(feature + strlen("import-marks-if-exists="), + from_stream, 1); } else if (!prefixcmp(feature, "export-marks=")) { option_export_marks(feature + 13); } else if (!strcmp(feature, "cat-blob")) { @@@ -3155,7 -2992,7 +3156,7 @@@ relative_marks_paths = 0; } else if (!prefixcmp(feature, "force")) { force_update = 1; - } else if (!strcmp(feature, "notes")) { + } else if (!strcmp(feature, "notes") || !strcmp(feature, "ls")) { ; /* do nothing; we have the feature */ } else { return 0; @@@ -3296,8 -3133,6 +3297,8 @@@ int main(int argc, const char **argv while (read_next_command() != EOF) { if (!strcmp("blob", command_buf.buf)) parse_new_blob(); + else if (!prefixcmp(command_buf.buf, "ls ")) + parse_ls(NULL); else if (!prefixcmp(command_buf.buf, "commit ")) parse_new_commit(); else if (!prefixcmp(command_buf.buf, "tag ")) diff --combined sha1_file.c index d949b35c33,e194f6a128..b4fcca8ffd --- a/sha1_file.c +++ b/sha1_file.c @@@ -13,7 -13,6 +13,7 @@@ #include "commit.h" #include "tag.h" #include "tree.h" +#include "tree-walk.h" #include "refs.h" #include "pack-revindex.h" #include "sha1-lookup.h" @@@ -418,6 -417,8 +418,8 @@@ static unsigned int pack_used_ctr static unsigned int pack_mmap_calls; static unsigned int peak_pack_open_windows; static unsigned int pack_open_windows; + static unsigned int pack_open_fds; + static unsigned int pack_max_fds; static size_t peak_pack_mapped; static size_t pack_mapped; struct packed_git *packed_git; @@@ -595,8 -596,10 +597,10 @@@ static int unuse_one_window(struct pack lru_l->next = lru_w->next; else { lru_p->windows = lru_w->next; - if (!lru_p->windows && lru_p->pack_fd != keep_fd) { + if (!lru_p->windows && lru_p->pack_fd != -1 + && lru_p->pack_fd != keep_fd) { close(lru_p->pack_fd); + pack_open_fds--; lru_p->pack_fd = -1; } } @@@ -681,8 -684,10 +685,10 @@@ void free_pack_by_name(const char *pack if (strcmp(pack_name, p->pack_name) == 0) { clear_delta_base_cache(); close_pack_windows(p); - if (p->pack_fd != -1) + if (p->pack_fd != -1) { close(p->pack_fd); + pack_open_fds--; + } close_pack_index(p); free(p->bad_object_sha1); *pp = p->next; @@@ -708,9 -713,29 +714,29 @@@ static int open_packed_git_1(struct pac if (!p->index_data && open_pack_index(p)) return error("packfile %s index unavailable", p->pack_name); + if (!pack_max_fds) { + struct rlimit lim; + unsigned int max_fds; + + if (getrlimit(RLIMIT_NOFILE, &lim)) + die_errno("cannot get RLIMIT_NOFILE"); + + max_fds = lim.rlim_cur; + + /* Save 3 for stdin/stdout/stderr, 22 for work */ + if (25 < max_fds) + pack_max_fds = max_fds - 25; + else + pack_max_fds = 1; + } + + while (pack_max_fds <= pack_open_fds && unuse_one_window(NULL, -1)) + ; /* nothing */ + p->pack_fd = git_open_noatime(p->pack_name, p); if (p->pack_fd < 0 || fstat(p->pack_fd, &st)) return -1; + pack_open_fds++; /* If we created the struct before we had the pack we lack size. */ if (!p->pack_size) { @@@ -762,6 -787,7 +788,7 @@@ static int open_packed_git(struct packe return 0; if (p->pack_fd != -1) { close(p->pack_fd); + pack_open_fds--; p->pack_fd = -1; } return -1; @@@ -787,14 -813,13 +814,13 @@@ unsigned char *use_pack(struct packed_g { struct pack_window *win = *w_cursor; - if (p->pack_fd == -1 && open_packed_git(p)) - die("packfile %s cannot be accessed", p->pack_name); - /* Since packfiles end in a hash of their content and it's * pointless to ask for an offset into the middle of that * hash, and the in_window function above wouldn't match * don't allow an offset too close to the end of the file. */ + if (!p->pack_size && p->pack_fd == -1 && open_packed_git(p)) + die("packfile %s cannot be accessed", p->pack_name); if (offset > (p->pack_size - 20)) die("offset beyond end of packfile (truncated pack?)"); @@@ -808,6 -833,10 +834,10 @@@ if (!win) { size_t window_align = packed_git_window_size / 2; off_t len; + + if (p->pack_fd == -1 && open_packed_git(p)) + die("packfile %s cannot be accessed", p->pack_name); + win = xcalloc(1, sizeof(*win)); win->offset = (offset / window_align) * window_align; len = p->pack_size - win->offset; @@@ -825,6 -854,12 +855,12 @@@ die("packfile %s cannot be mapped: %s", p->pack_name, strerror(errno)); + if (!win->offset && win->len == p->pack_size + && !p->do_not_close) { + close(p->pack_fd); + pack_open_fds--; + p->pack_fd = -1; + } pack_mmap_calls++; pack_open_windows++; if (pack_mapped > peak_pack_mapped) @@@ -919,6 -954,9 +955,9 @@@ struct packed_git *parse_pack_index(uns void install_packed_git(struct packed_git *pack) { + if (pack->pack_fd != -1) + pack_open_fds++; + pack->next = packed_git; packed_git = pack; } @@@ -936,8 -974,6 +975,6 @@@ static void prepare_packed_git_one(cha sprintf(path, "%s/pack", objdir); len = strlen(path); dir = opendir(path); - while (!dir && errno == EMFILE && unuse_one_window(NULL, -1)) - dir = opendir(path); if (!dir) { if (errno != ENOENT) error("unable to open object pack directory: %s: %s", @@@ -1093,14 -1129,6 +1130,6 @@@ static int git_open_noatime(const char if (fd >= 0) return fd; - /* Might the failure be insufficient file descriptors? */ - if (errno == EMFILE) { - if (unuse_one_window(p, -1)) - continue; - else - return -1; - } - /* Might the failure be due to O_NOATIME? */ if (errno != ENOENT && sha1_file_open_flag) { sha1_file_open_flag = 0; @@@ -1932,6 -1960,27 +1961,27 @@@ off_t find_pack_entry_one(const unsigne return 0; } + static int is_pack_valid(struct packed_git *p) + { + /* An already open pack is known to be valid. */ + if (p->pack_fd != -1) + return 1; + + /* If the pack has one window completely covering the + * file size, the pack is known to be valid even if + * the descriptor is not currently open. + */ + if (p->windows) { + struct pack_window *w = p->windows; + + if (!w->offset && w->len == p->pack_size) + return 1; + } + + /* Force the pack to open to prove its valid. */ + return !open_packed_git(p); + } + static int find_pack_entry(const unsigned char *sha1, struct pack_entry *e) { static struct packed_git *last_found = (void *)1; @@@ -1961,7 -2010,7 +2011,7 @@@ * it may have been deleted since the index * was loaded! */ - if (p->pack_fd == -1 && open_packed_git(p)) { + if (!is_pack_valid(p)) { error("packfile %s cannot be accessed", p->pack_name); goto next; } @@@ -2360,8 -2409,6 +2410,6 @@@ static int write_loose_object(const uns filename = sha1_file_name(sha1); fd = create_tmpfile(tmpfile, sizeof(tmpfile), filename); - while (fd < 0 && errno == EMFILE && unuse_one_window(NULL, -1)) - fd = create_tmpfile(tmpfile, sizeof(tmpfile), filename); if (fd < 0) { if (errno == EACCES) return error("insufficient permission for adding an object to repository database %s\n", get_object_directory()); @@@ -2480,37 -2527,8 +2528,37 @@@ int has_sha1_file(const unsigned char * return has_loose_object(sha1); } +static void check_tree(const void *buf, size_t size) +{ + struct tree_desc desc; + struct name_entry entry; + + init_tree_desc(&desc, buf, size); + while (tree_entry(&desc, &entry)) + /* do nothing + * tree_entry() will die() on malformed entries */ + ; +} + +static void check_commit(const void *buf, size_t size) +{ + struct commit c; + memset(&c, 0, sizeof(c)); + if (parse_commit_buffer(&c, buf, size)) + die("corrupt commit"); +} + +static void check_tag(const void *buf, size_t size) +{ + struct tag t; + memset(&t, 0, sizeof(t)); + if (parse_tag_buffer(&t, buf, size)) + die("corrupt tag"); +} + static int index_mem(unsigned char *sha1, void *buf, size_t size, - int write_object, enum object_type type, const char *path) + int write_object, enum object_type type, + const char *path, int format_check) { int ret, re_allocated = 0; @@@ -2528,14 -2546,6 +2576,14 @@@ re_allocated = 1; } } + if (format_check) { + if (type == OBJ_TREE) + check_tree(buf, size); + if (type == OBJ_COMMIT) + check_commit(buf, size); + if (type == OBJ_TAG) + check_tag(buf, size); + } if (write_object) ret = write_sha1_file(buf, size, typename(type), sha1); @@@ -2549,7 -2559,7 +2597,7 @@@ #define SMALL_FILE_SIZE (32*1024) int index_fd(unsigned char *sha1, int fd, struct stat *st, int write_object, - enum object_type type, const char *path) + enum object_type type, const char *path, int format_check) { int ret; size_t size = xsize_t(st->st_size); @@@ -2558,25 -2568,23 +2606,25 @@@ struct strbuf sbuf = STRBUF_INIT; if (strbuf_read(&sbuf, fd, 4096) >= 0) ret = index_mem(sha1, sbuf.buf, sbuf.len, write_object, - type, path); + type, path, format_check); else ret = -1; strbuf_release(&sbuf); } else if (!size) { - ret = index_mem(sha1, NULL, size, write_object, type, path); + ret = index_mem(sha1, NULL, size, write_object, type, path, + format_check); } else if (size <= SMALL_FILE_SIZE) { char *buf = xmalloc(size); if (size == read_in_full(fd, buf, size)) ret = index_mem(sha1, buf, size, write_object, type, - path); + path, format_check); else ret = error("short read %s", strerror(errno)); free(buf); } else { void *buf = xmmap(NULL, size, PROT_READ, MAP_PRIVATE, fd, 0); - ret = index_mem(sha1, buf, size, write_object, type, path); + ret = index_mem(sha1, buf, size, write_object, type, path, + format_check); munmap(buf, size); } close(fd); @@@ -2594,7 -2602,7 +2642,7 @@@ int index_path(unsigned char *sha1, con if (fd < 0) return error("open(\"%s\"): %s", path, strerror(errno)); - if (index_fd(sha1, fd, st, write_object, OBJ_BLOB, path) < 0) + if (index_fd(sha1, fd, st, write_object, OBJ_BLOB, path, 0) < 0) return error("%s: failed to insert into database", path); break;