From: Junio C Hamano <gitster@pobox.com>
Date: Mon, 20 Jun 2016 18:01:00 +0000 (-0700)
Subject: Merge branch 'ew/fast-import-unpack-limit'
X-Git-Tag: v2.10.0-rc0~204
X-Git-Url: https://git.lorimer.id.au/gitweb.git/diff_plain/8d6a7e9a198488a1594cd178240fb56085129c78?ds=inline;hp=-c

Merge branch 'ew/fast-import-unpack-limit'

"git fast-import" learned the same performance trick to avoid
creating too small a packfile as "git fetch" and "git push" have,
using *.unpackLimit configuration.

* ew/fast-import-unpack-limit:
fast-import: invalidate pack_id references after loosening
fast-import: implement unpack limit
---

8d6a7e9a198488a1594cd178240fb56085129c78
diff --combined Documentation/config.txt
index 2e1b2e486e,283bf04091..58673cf21e
--- a/Documentation/config.txt
+++ b/Documentation/config.txt
@@@ -81,16 -81,13 +81,16 @@@ Include
  
  You can include one config file from another by setting the special
  `include.path` variable to the name of the file to be included. The
 +variable takes a pathname as its value, and is subject to tilde
 +expansion.
 +
 +The
  included file is expanded immediately, as if its contents had been
  found at the location of the include directive. If the value of the
  `include.path` variable is a relative path, the path is considered to be
  relative to the configuration file in which the include directive was
 -found. The value of `include.path` is subject to tilde expansion: `~/`
 -is expanded to the value of `$HOME`, and `~user/` to the specified
 -user's home directory. See below for examples.
 +found.  See below for examples.
 +
  
  Example
  ~~~~~~~
@@@ -117,7 -114,7 +117,7 @@@
  	[include]
  		path = /path/to/foo.inc ; include by absolute path
  		path = foo ; expand "foo" relative to the current file
 -		path = ~/foo ; expand "foo" in your $HOME directory
 +		path = ~/foo ; expand "foo" in your `$HOME` directory
  
  
  Values
@@@ -172,13 -169,6 +172,13 @@@ thing on the same output line (e.g. ope
  list of branch names in `log --decorate` output) is set to be
  painted with `bold` or some other attribute.
  
 +pathname::
 +	A variable that takes a pathname value can be given a
 +	string that begins with "`~/`" or "`~user/`", and the usual
 +	tilde expansion happens to such a string: `~/`
 +	is expanded to the value of `$HOME`, and `~user/` to the
 +	specified user's home directory.
 +
  
  Variables
  ~~~~~~~~~
@@@ -279,12 -269,6 +279,12 @@@ See linkgit:git-update-index[1]
  +
  The default is true (when core.filemode is not specified in the config file).
  
 +core.hideDotFiles::
 +	(Windows-only) If true, mark newly-created directories and files whose
 +	name starts with a dot as hidden.  If 'dotGitOnly', only the `.git/`
 +	directory is hidden, but no other files starting with a dot.  The
 +	default mode is 'dotGitOnly'.
 +
  core.ignoreCase::
  	If true, this option enables various workarounds to enable
  	Git to work better on filesystems that are not case sensitive,
@@@ -353,9 -337,9 +353,9 @@@ core.quotePath:
  
  core.eol::
  	Sets the line ending type to use in the working directory for
 -	files that have the `text` property set.  Alternatives are
 -	'lf', 'crlf' and 'native', which uses the platform's native
 -	line ending.  The default value is `native`.  See
 +	files that have the `text` property set when core.autocrlf is false.
 +	Alternatives are 'lf', 'crlf' and 'native', which uses the platform's
 +	native line ending.  The default value is `native`.  See
  	linkgit:gitattributes[5] for more information on end-of-line
  	conversion.
  
@@@ -502,10 -486,10 +502,10 @@@ repository's usual working tree)
  
  core.logAllRefUpdates::
  	Enable the reflog. Updates to a ref <ref> is logged to the file
 -	"$GIT_DIR/logs/<ref>", by appending the new and old
 +	"`$GIT_DIR/logs/<ref>`", by appending the new and old
  	SHA-1, the date/time and the reason of the update, but
  	only when the file exists.  If this configuration
 -	variable is set to true, missing "$GIT_DIR/logs/<ref>"
 +	variable is set to true, missing "`$GIT_DIR/logs/<ref>`"
  	file is automatically created for branch heads (i.e. under
  	refs/heads/), remote refs (i.e. under refs/remotes/),
  	note refs (i.e. under refs/notes/), and the symbolic ref HEAD.
@@@ -609,11 -593,12 +609,11 @@@ be delta compressed, but larger binary 
  Common unit suffixes of 'k', 'm', or 'g' are supported.
  
  core.excludesFile::
 -	In addition to '.gitignore' (per-directory) and
 -	'.git/info/exclude', Git looks into this file for patterns
 -	of files which are not meant to be tracked.  "`~/`" is expanded
 -	to the value of `$HOME` and "`~user/`" to the specified user's
 -	home directory. Its default value is $XDG_CONFIG_HOME/git/ignore.
 -	If $XDG_CONFIG_HOME is either not set or empty, $HOME/.config/git/ignore
 +	Specifies the pathname to the file that contains patterns to
 +	describe paths that are not meant to be tracked, in addition
 +	to '.gitignore' (per-directory) and '.git/info/exclude'.
 +	Defaults to `$XDG_CONFIG_HOME/git/ignore`.
 +	If `$XDG_CONFIG_HOME` is either not set or empty, `$HOME/.config/git/ignore`
  	is used instead. See linkgit:gitignore[5].
  
  core.askPass::
@@@ -630,25 -615,8 +630,25 @@@ core.attributesFile:
  	'.git/info/attributes', Git looks into this file for attributes
  	(see linkgit:gitattributes[5]). Path expansions are made the same
  	way as for `core.excludesFile`. Its default value is
 -	$XDG_CONFIG_HOME/git/attributes. If $XDG_CONFIG_HOME is either not
 -	set or empty, $HOME/.config/git/attributes is used instead.
 +	`$XDG_CONFIG_HOME/git/attributes`. If `$XDG_CONFIG_HOME` is either not
 +	set or empty, `$HOME/.config/git/attributes` is used instead.
 +
 +core.hooksPath::
 +	By default Git will look for your hooks in the
 +	'$GIT_DIR/hooks' directory. Set this to different path,
 +	e.g. '/etc/git/hooks', and Git will try to find your hooks in
 +	that directory, e.g. '/etc/git/hooks/pre-receive' instead of
 +	in '$GIT_DIR/hooks/pre-receive'.
 ++
 +The path can be either absolute or relative. A relative path is
 +taken as relative to the directory where the hooks are run (see
 +the "DESCRIPTION" section of linkgit:githooks[5]).
 ++
 +This configuration variable is useful in cases where you'd like to
 +centrally configure your Git hooks instead of configuring them on a
 +per-repository basis, or as a more flexible and centralized
 +alternative to having an `init.templateDir` where you've changed
 +default hooks.
  
  core.editor::
  	Commands such as `commit` and `tag` that lets you edit
@@@ -1138,19 -1106,15 +1138,19 @@@ commit.status:
  	message.  Defaults to true.
  
  commit.template::
 -	Specify a file to use as the template for new commit messages.
 -	"`~/`" is expanded to the value of `$HOME` and "`~user/`" to the
 -	specified user's home directory.
 +	Specify the pathname of a file to use as the template for
 +	new commit messages.
 +
 +commit.verbose::
 +	A boolean or int to specify the level of verbose with `git commit`.
 +	See linkgit:git-commit[1].
  
  credential.helper::
  	Specify an external helper to be called when a username or
  	password credential is needed; the helper may consult external
 -	storage to avoid prompting the user for the credentials. See
 -	linkgit:gitcredentials[7] for details.
 +	storage to avoid prompting the user for the credentials. Note
 +	that multiple helpers may be defined. See linkgit:gitcredentials[7]
 +	for details.
  
  credential.useHttpPath::
  	When acquiring credentials, consider the "path" component of an http
@@@ -1189,6 -1153,15 +1189,15 @@@ difftool.<tool>.cmd:
  difftool.prompt::
  	Prompt before each invocation of the diff tool.
  
+ fastimport.unpackLimit::
+ 	If the number of objects imported by linkgit:git-fast-import[1]
+ 	is below this limit, then the objects will be unpacked into
+ 	loose object files.  However if the number of imported objects
+ 	equals or exceeds this limit then the pack will be stored as a
+ 	pack.  Storing the pack from a fast-import can make the import
+ 	operation complete faster, especially on slow filesystems.  If
+ 	not set, the value of `transfer.unpackLimit` is used instead.
+ 
  fetch.recurseSubmodules::
  	This option can be either set to a boolean value or to 'on-demand'.
  	Setting it to a boolean changes the behavior of fetch and pull to
@@@ -1294,10 -1267,6 +1303,10 @@@ format.outputDirectory:
  	Set a custom directory to store the resulting files instead of the
  	current working directory.
  
 +format.useAutoBase::
 +	A boolean value which lets you enable the `--base=auto` option of
 +	format-patch by default.
 +
  filter.<driver>.clean::
  	The command which is used to convert the content of a worktree
  	file to a blob upon checkin.  See linkgit:gitattributes[5] for
@@@ -1374,7 -1343,7 +1383,7 @@@ gc.worktreePruneExpire:
  	'git worktree prune --expire 3.months.ago'.
  	This config variable can be used to set a different grace
  	period. The value "now" may be used to disable the grace
 -	period and prune $GIT_DIR/worktrees immediately, or "never"
 +	period and prune `$GIT_DIR/worktrees` immediately, or "never"
  	may be used to suppress pruning.
  
  gc.reflogExpire::
@@@ -1514,13 -1483,13 +1523,13 @@@ grep.fallbackToNoIndex:
  	is executed outside of a git repository.  Defaults to false.
  
  gpg.program::
 -	Use this custom program instead of "gpg" found on $PATH when
 +	Use this custom program instead of "`gpg`" found on `$PATH` when
  	making or verifying a PGP signature. The program must support the
  	same command-line interface as GPG, namely, to verify a detached
 -	signature, "gpg --verify $file - <$signature" is run, and the
 +	signature, "`gpg --verify $file - <$signature`" is run, and the
  	program is expected to signal a good signature by exiting with
  	code 0, and to generate an ASCII-armored detached signature, the
 -	standard input of "gpg -bsau $key" is fed with the contents to be
 +	standard input of "`gpg -bsau $key`" is fed with the contents to be
  	signed, and the program is expected to send the result to its
  	standard output.
  
@@@ -1533,7 -1502,7 +1542,7 @@@ gui.diffContext:
  	made by the linkgit:git-gui[1]. The default is "5".
  
  gui.displayUntracked::
 -	Determines if linkgit::git-gui[1] shows untracked files
 +	Determines if linkgit:git-gui[1] shows untracked files
  	in the file list. The default is "true".
  
  gui.encoding::
@@@ -1694,19 -1663,12 +1703,19 @@@ http.emptyAuth:
  	a username in the URL, as libcurl normally requires a username for
  	authentication.
  
 +http.extraHeader::
 +	Pass an additional HTTP header when communicating with a server.  If
 +	more than one such entry exists, all of them are added as extra
 +	headers.  To allow overriding the settings inherited from the system
 +	config, an empty value will reset the extra headers to the empty list.
 +
  http.cookieFile::
 -	File containing previously stored cookie lines which should be used
 +	The pathname of a file containing previously stored cookie lines,
 +	which should be used
  	in the Git http session, if they match the server. The file format
  	of the file to read cookies from should be plain HTTP headers or
 -	the Netscape/Mozilla cookie file format (see linkgit:curl[1]).
 -	NOTE that the file specified with http.cookieFile is only used as
 +	the Netscape/Mozilla cookie file format (see `curl(1)`).
 +	NOTE that the file specified with http.cookieFile is used only as
  	input unless http.saveCookies is set.
  
  http.saveCookies::
@@@ -1933,14 -1895,6 +1942,14 @@@ interactive.singleKey:
  	setting is silently ignored if portable keystroke input
  	is not available; requires the Perl module Term::ReadKey.
  
 +interactive.diffFilter::
 +	When an interactive command (such as `git add --patch`) shows
 +	a colorized diff, git will pipe the diff through the shell
 +	command defined by this configuration variable. The command may
 +	mark up the diff further for human consumption, provided that it
 +	retains a one-to-one correspondence with the lines in the
 +	original diff. Defaults to disabled (no filtering).
 +
  log.abbrevCommit::
  	If true, makes linkgit:git-log[1], linkgit:git-show[1], and
  	linkgit:git-whatchanged[1] assume `--abbrev-commit`. You may
@@@ -1956,10 -1910,7 +1965,10 @@@ log.decorate:
  	command. If 'short' is specified, the ref name prefixes 'refs/heads/',
  	'refs/tags/' and 'refs/remotes/' will not be printed. If 'full' is
  	specified, the full ref name (including prefix) will be printed.
 -	This is the same as the log commands '--decorate' option.
 +	If 'auto' is specified, then if the output is going to a terminal,
 +	the ref names are shown as if 'short' were given, otherwise no ref
 +	names are shown. This is the same as the '--decorate' option
 +	of the `git log`.
  
  log.follow::
  	If `true`, `git log` will act as if the `--follow` option was used when
@@@ -2205,11 -2156,8 +2214,11 @@@ pack.packSizeLimit:
  	The maximum size of a pack.  This setting only affects
  	packing to a file when repacking, i.e. the git:// protocol
  	is unaffected.  It can be overridden by the `--max-pack-size`
 -	option of linkgit:git-repack[1]. The minimum size allowed is
 -	limited to 1 MiB. The default is unlimited.
 +	option of linkgit:git-repack[1].  Reaching this limit results
 +	in the creation of multiple packfiles; which in turn prevents
 +	bitmaps from being created.
 +	The minimum size allowed is limited to 1 MiB.
 +	The default is unlimited.
  	Common unit suffixes of 'k', 'm', or 'g' are
  	supported.
  
@@@ -2609,9 -2557,8 +2618,9 @@@ repack.writeBitmaps:
  	objects to disk (e.g., when `git repack -a` is run).  This
  	index can speed up the "counting objects" phase of subsequent
  	packs created for clones and fetches, at the cost of some disk
 -	space and extra time spent on the initial repack.  Defaults to
 -	false.
 +	space and extra time spent on the initial repack.  This has
 +	no effect if multiple packfiles are created.
 +	Defaults to false.
  
  rerere.autoUpdate::
  	When set to true, `git-rerere` updates the index with the
@@@ -2791,17 -2738,6 +2800,17 @@@ submodule.<name>.ignore:
  	"--ignore-submodules" option. The 'git submodule' commands are not
  	affected by this setting.
  
 +submodule.fetchJobs::
 +	Specifies how many submodules are fetched/cloned at the same time.
 +	A positive integer allows up to that number of submodules fetched
 +	in parallel. A value of 0 will give some reasonable default.
 +	If unset, it defaults to 1.
 +
 +tag.forceSignAnnotated::
 +	A boolean to specify whether annotated tags created should be GPG signed.
 +	If `--annotate` is specified on the command line, it takes
 +	precedence over this option.
 +
  tag.sort::
  	This variable controls the sort ordering of tags when displayed by
  	linkgit:git-tag[1]. Without the "--sort=<value>" option provided, the
diff --combined fast-import.c
index c504ef752d,e415f517f8..59630cee14
--- a/fast-import.c
+++ b/fast-import.c
@@@ -166,6 -166,7 +166,7 @@@ Format of STDIN stream
  #include "quote.h"
  #include "exec_cmd.h"
  #include "dir.h"
+ #include "run-command.h"
  
  #define PACK_ID_BITS 16
  #define MAX_PACK_ID ((1<<PACK_ID_BITS)-1)
@@@ -282,6 -283,7 +283,7 @@@ struct recent_command 
  /* Configured limits on output */
  static unsigned long max_depth = 10;
  static off_t max_packsize;
+ static int unpack_limit = 100;
  static int force_update;
  static int pack_compression_level = Z_DEFAULT_COMPRESSION;
  static int pack_compression_seen;
@@@ -329,7 -331,6 +331,7 @@@ static const char *export_marks_file
  static const char *import_marks_file;
  static int import_marks_file_from_stream;
  static int import_marks_file_ignore_missing;
 +static int import_marks_file_done;
  static int relative_marks_paths;
  
  /* Our last blob */
@@@ -415,7 -416,7 +417,7 @@@ static void write_crash_report(const ch
  	struct recent_command *rc;
  
  	if (!rpt) {
 -		error("can't write crash report %s: %s", loc, strerror(errno));
 +		error_errno("can't write crash report %s", loc);
  		free(loc);
  		return;
  	}
@@@ -596,6 -597,33 +598,33 @@@ static struct object_entry *insert_obje
  	return e;
  }
  
+ static void invalidate_pack_id(unsigned int id)
+ {
+ 	unsigned int h;
+ 	unsigned long lu;
+ 	struct tag *t;
+ 
+ 	for (h = 0; h < ARRAY_SIZE(object_table); h++) {
+ 		struct object_entry *e;
+ 
+ 		for (e = object_table[h]; e; e = e->next)
+ 			if (e->pack_id == id)
+ 				e->pack_id = MAX_PACK_ID;
+ 	}
+ 
+ 	for (lu = 0; lu < branch_table_sz; lu++) {
+ 		struct branch *b;
+ 
+ 		for (b = branch_table[lu]; b; b = b->table_next_branch)
+ 			if (b->pack_id == id)
+ 				b->pack_id = MAX_PACK_ID;
+ 	}
+ 
+ 	for (t = first_tag; t; t = t->next_tag)
+ 		if (t->pack_id == id)
+ 			t->pack_id = MAX_PACK_ID;
+ }
+ 
  static unsigned int hc_str(const char *s, size_t len)
  {
  	unsigned int r = 0;
@@@ -951,6 -979,23 +980,23 @@@ static void unkeep_all_packs(void
  	}
  }
  
+ static int loosen_small_pack(const struct packed_git *p)
+ {
+ 	struct child_process unpack = CHILD_PROCESS_INIT;
+ 
+ 	if (lseek(p->pack_fd, 0, SEEK_SET) < 0)
+ 		die_errno("Failed seeking to start of '%s'", p->pack_name);
+ 
+ 	unpack.in = p->pack_fd;
+ 	unpack.git_cmd = 1;
+ 	unpack.stdout_to_stderr = 1;
+ 	argv_array_push(&unpack.args, "unpack-objects");
+ 	if (!show_stats)
+ 		argv_array_push(&unpack.args, "-q");
+ 
+ 	return run_command(&unpack);
+ }
+ 
  static void end_packfile(void)
  {
  	static int running;
@@@ -973,6 -1018,14 +1019,14 @@@
  		fixup_pack_header_footer(pack_data->pack_fd, pack_data->sha1,
  				    pack_data->pack_name, object_count,
  				    cur_pack_sha1, pack_size);
+ 
+ 		if (object_count <= unpack_limit) {
+ 			if (!loosen_small_pack(pack_data)) {
+ 				invalidate_pack_id(pack_id);
+ 				goto discard_pack;
+ 			}
+ 		}
+ 
  		close(pack_data->pack_fd);
  		idx_name = keep_pack(create_index());
  
@@@ -1003,6 -1056,7 +1057,7 @@@
  		pack_id++;
  	}
  	else {
+ discard_pack:
  		close(pack_data->pack_fd);
  		unlink_or_warn(pack_data->pack_name);
  	}
@@@ -1513,7 -1567,7 +1568,7 @@@ static int tree_content_set
  	t = root->tree;
  	for (i = 0; i < t->entry_count; i++) {
  		e = t->entries[i];
 -		if (e->name->str_len == n && !strncmp_icase(p, e->name->str_dat, n)) {
 +		if (e->name->str_len == n && !fspathncmp(p, e->name->str_dat, n)) {
  			if (!*slash1) {
  				if (!S_ISDIR(mode)
  						&& e->versions[1].mode == mode
@@@ -1603,7 -1657,7 +1658,7 @@@ static int tree_content_remove
  	t = root->tree;
  	for (i = 0; i < t->entry_count; i++) {
  		e = t->entries[i];
 -		if (e->name->str_len == n && !strncmp_icase(p, e->name->str_dat, n)) {
 +		if (e->name->str_len == n && !fspathncmp(p, e->name->str_dat, n)) {
  			if (*slash1 && !S_ISDIR(e->versions[1].mode))
  				/*
  				 * If p names a file in some subdirectory, and a
@@@ -1670,7 -1724,7 +1725,7 @@@ static int tree_content_get
  	t = root->tree;
  	for (i = 0; i < t->entry_count; i++) {
  		e = t->entries[i];
 -		if (e->name->str_len == n && !strncmp_icase(p, e->name->str_dat, n)) {
 +		if (e->name->str_len == n && !fspathncmp(p, e->name->str_dat, n)) {
  			if (!*slash1)
  				goto found_entry;
  			if (!S_ISDIR(e->versions[1].mode))
@@@ -1803,12 -1857,12 +1858,12 @@@ static void dump_marks(void
  	static struct lock_file mark_lock;
  	FILE *f;
  
 -	if (!export_marks_file)
 +	if (!export_marks_file || (import_marks_file && !import_marks_file_done))
  		return;
  
  	if (hold_lock_file_for_update(&mark_lock, export_marks_file, 0) < 0) {
 -		failure |= error("Unable to write marks file %s: %s",
 -			export_marks_file, strerror(errno));
 +		failure |= error_errno("Unable to write marks file %s",
 +				       export_marks_file);
  		return;
  	}
  
@@@ -1823,8 -1877,8 +1878,8 @@@
  
  	dump_marks_helper(f, 0, marks);
  	if (commit_lock_file(&mark_lock)) {
 -		failure |= error("Unable to write file %s: %s",
 -			export_marks_file, strerror(errno));
 +		failure |= error_errno("Unable to write file %s",
 +				       export_marks_file);
  		return;
  	}
  }
@@@ -1836,7 -1890,7 +1891,7 @@@ static void read_marks(void
  	if (f)
  		;
  	else if (import_marks_file_ignore_missing && errno == ENOENT)
 -		return; /* Marks file does not exist */
 +		goto done; /* Marks file does not exist */
  	else
  		die_errno("cannot read '%s'", import_marks_file);
  	while (fgets(line, sizeof(line), f)) {
@@@ -1866,8 -1920,6 +1921,8 @@@
  		insert_mark(mark, e);
  	}
  	fclose(f);
 +done:
 +	import_marks_file_done = 1;
  }
  
  
@@@ -3320,6 -3372,7 +3375,7 @@@ static void parse_option(const char *op
  static void git_pack_config(void)
  {
  	int indexversion_value;
+ 	int limit;
  	unsigned long packsizelimit_value;
  
  	if (!git_config_get_ulong("pack.depth", &max_depth)) {
@@@ -3344,6 -3397,11 +3400,11 @@@
  	if (!git_config_get_ulong("pack.packsizelimit", &packsizelimit_value))
  		max_packsize = packsizelimit_value;
  
+ 	if (!git_config_get_int("fastimport.unpacklimit", &limit))
+ 		unpack_limit = limit;
+ 	else if (!git_config_get_int("transfer.unpacklimit", &limit))
+ 		unpack_limit = limit;
+ 
  	git_config(git_default_config, NULL);
  }
  
diff --combined t/t9300-fast-import.sh
index 4bca35c259,e6a2b8a4d7..74d740de41
--- a/t/t9300-fast-import.sh
+++ b/t/t9300-fast-import.sh
@@@ -52,6 -52,7 +52,7 @@@ echo "$@"
  ###
  
  test_expect_success 'empty stream succeeds' '
+ 	git config fastimport.unpackLimit 0 &&
  	git fast-import </dev/null
  '
  
@@@ -2650,21 -2651,6 +2651,21 @@@ test_expect_success 'R: ignore non-git 
  	git fast-import <input
  '
  
 +test_expect_success 'R: corrupt lines do not mess marks file' '
 +	rm -f io.marks &&
 +	blob=$(echo hi | git hash-object --stdin) &&
 +	cat >expect <<-EOF &&
 +	:3 0000000000000000000000000000000000000000
 +	:1 $blob
 +	:2 $blob
 +	EOF
 +	cp expect io.marks &&
 +	test_must_fail git fast-import --import-marks=io.marks --export-marks=io.marks <<-\EOF &&
 +
 +	EOF
 +	test_cmp expect io.marks
 +'
 +
  ##
  ## R: very large blobs
  ##
@@@ -2690,6 -2676,7 +2691,7 @@@ test_expect_success 'R: blob bigger tha
  	echo >>input &&
  
  	test_create_repo R &&
+ 	git --git-dir=R/.git config fastimport.unpackLimit 0 &&
  	git --git-dir=R/.git fast-import --big-file-threshold=1 <input
  '