From: Junio C Hamano <gitster@pobox.com>
Date: Mon, 29 Aug 2011 04:18:47 +0000 (-0700)
Subject: Merge branch 'di/fast-import-ident'
X-Git-Tag: v1.7.7-rc1~44
X-Git-Url: https://git.lorimer.id.au/gitweb.git/diff_plain/0b98954975502f56f1de2a4cb0569f7e80fd44d0?hp=-c

Merge branch 'di/fast-import-ident'

* di/fast-import-ident:
fsck: improve committer/author check
fsck: add a few committer name tests
fast-import: check committer name more strictly
fast-import: don't fail on omitted committer name
fast-import: add input format tests
---

0b98954975502f56f1de2a4cb0569f7e80fd44d0
diff --combined Documentation/git-fast-import.txt
index db0d75fdbc,0ca24a89f8..ec6ef31197
--- a/Documentation/git-fast-import.txt
+++ b/Documentation/git-fast-import.txt
@@@ -8,7 -8,6 +8,7 @@@ git-fast-import - Backend for fast Git 
  
  SYNOPSIS
  --------
 +[verse]
  frontend | 'git fast-import' [options]
  
  DESCRIPTION
@@@ -102,12 -101,6 +102,12 @@@ OPTION
  	when the `cat-blob` command is encountered in the stream.
  	The default behaviour is to write to `stdout`.
  
 +--done::
 +	Require a `done` command at the end of the stream.
 +	This option might be useful for detecting errors that
 +	cause the frontend to terminate before it has started to
 +	write a stream.
 +
  --export-pack-edges=<file>::
  	After creating a packfile, print a line of data to
  	<file> listing the filename of the packfile and the last
@@@ -337,11 -330,6 +337,11 @@@ and control the current import process
  	standard output.  This command is optional and is not needed
  	to perform an import.
  
 +`done`::
 +	Marks the end of the stream. This command is optional
 +	unless the `done` feature was requested using the
 +	`--done` command line option or `feature done` command.
 +
  `cat-blob`::
  	Causes fast-import to print a blob in 'cat-file --batch'
  	format to the file descriptor set with `--cat-blob-fd` or
@@@ -425,8 -413,8 +425,8 @@@ Here `<name>` is the person's display n
  (``cm@example.com'').  `LT` and `GT` are the literal less-than (\x3c)
  and greater-than (\x3e) symbols.  These are required to delimit
  the email address from the other fields in the line.  Note that
- `<name>` is free-form and may contain any sequence of bytes, except
- `LT` and `LF`.  It is typically UTF-8 encoded.
+ `<name>` and `<email>` are free-form and may contain any sequence
+ of bytes, except `LT`, `GT` and `LF`.  `<name>` is typically UTF-8 encoded.
  
  The time of the change is specified by `<when>` using the date format
  that was selected by the \--date-format=<fmt> command line option.
@@@ -1012,14 -1000,10 +1012,14 @@@ force:
  	(see OPTIONS, above).
  
  import-marks::
 +import-marks-if-exists::
  	Like --import-marks except in two respects: first, only one
 -	"feature import-marks" command is allowed per stream;
 -	second, an --import-marks= command-line option overrides
 -	any "feature import-marks" command in the stream.
 +	"feature import-marks" or "feature import-marks-if-exists"
 +	command is allowed per stream; second, an --import-marks=
 +	or --import-marks-if-exists command-line option overrides
 +	any of these "feature" commands in the stream; third,
 +	"feature import-marks-if-exists" like a corresponding
 +	command-line option silently skips a nonexistent file.
  
  cat-blob::
  ls::
@@@ -1036,11 -1020,6 +1036,11 @@@ notes:
  	Versions of fast-import not supporting notes will exit
  	with a message indicating so.
  
 +done::
 +	Error out if the stream ends without a 'done' command.
 +	Without this feature, errors causing the frontend to end
 +	abruptly at a convenient point in the stream can go
 +	undetected.
  
  `option`
  ~~~~~~~~
@@@ -1070,15 -1049,6 +1070,15 @@@ not be passed as option
  * cat-blob-fd
  * force
  
 +`done`
 +~~~~~~
 +If the `done` feature is not in use, treated as if EOF was read.
 +This can be used to tell fast-import to finish early.
 +
 +If the `--done` command line option or `feature done` command is
 +in use, the `done` command is mandatory and marks the end of the
 +stream.
 +
  Crash Reports
  -------------
  If fast-import is supplied invalid input it will terminate with a
diff --combined fast-import.c
index 7cc22625e5,967d70c700..6d491b92fe
--- a/fast-import.c
+++ b/fast-import.c
@@@ -304,7 -304,6 +304,7 @@@ static unsigned int atom_cnt
  static struct atom_str **atom_table;
  
  /* The .pack file being generated */
 +static struct pack_idx_option pack_idx_opts;
  static unsigned int pack_id;
  static struct sha1file *pack_file;
  static struct packed_git *pack_data;
@@@ -355,7 -354,6 +355,7 @@@ static unsigned int cmd_save = 100
  static uintmax_t next_mark;
  static struct strbuf new_data = STRBUF_INIT;
  static int seen_data_command;
 +static int require_explicit_termination;
  
  /* Signal handling */
  static volatile sig_atomic_t checkpoint_requested;
@@@ -898,7 -896,7 +898,7 @@@ static const char *create_index(void
  	if (c != last)
  		die("internal consistency error creating the index");
  
 -	tmpfile = write_idx_file(NULL, idx, object_count, pack_data->sha1);
 +	tmpfile = write_idx_file(NULL, idx, object_count, &pack_idx_opts, pack_data->sha1);
  	free(idx);
  	return tmpfile;
  }
@@@ -1019,7 -1017,7 +1019,7 @@@ static int store_object
  	unsigned char sha1[20];
  	unsigned long hdrlen, deltalen;
  	git_SHA_CTX c;
 -	z_stream s;
 +	git_zstream s;
  
  	hdrlen = sprintf((char *)hdr,"%s %lu", typename(type),
  		(unsigned long)dat->len) + 1;
@@@ -1052,7 -1050,7 +1052,7 @@@
  		delta = NULL;
  
  	memset(&s, 0, sizeof(s));
 -	deflateInit(&s, pack_compression_level);
 +	git_deflate_init(&s, pack_compression_level);
  	if (delta) {
  		s.next_in = delta;
  		s.avail_in = deltalen;
@@@ -1060,11 -1058,11 +1060,11 @@@
  		s.next_in = (void *)dat->buf;
  		s.avail_in = dat->len;
  	}
 -	s.avail_out = deflateBound(&s, s.avail_in);
 +	s.avail_out = git_deflate_bound(&s, s.avail_in);
  	s.next_out = out = xmalloc(s.avail_out);
 -	while (deflate(&s, Z_FINISH) == Z_OK)
 -		/* nothing */;
 -	deflateEnd(&s);
 +	while (git_deflate(&s, Z_FINISH) == Z_OK)
 +		; /* nothing */
 +	git_deflate_end(&s);
  
  	/* Determine if we should auto-checkpoint. */
  	if ((max_packsize && (pack_size + 60 + s.total_out) > max_packsize)
@@@ -1080,14 -1078,14 +1080,14 @@@
  			delta = NULL;
  
  			memset(&s, 0, sizeof(s));
 -			deflateInit(&s, pack_compression_level);
 +			git_deflate_init(&s, pack_compression_level);
  			s.next_in = (void *)dat->buf;
  			s.avail_in = dat->len;
 -			s.avail_out = deflateBound(&s, s.avail_in);
 +			s.avail_out = git_deflate_bound(&s, s.avail_in);
  			s.next_out = out = xrealloc(out, s.avail_out);
 -			while (deflate(&s, Z_FINISH) == Z_OK)
 -				/* nothing */;
 -			deflateEnd(&s);
 +			while (git_deflate(&s, Z_FINISH) == Z_OK)
 +				; /* nothing */
 +			git_deflate_end(&s);
  		}
  	}
  
@@@ -1165,7 -1163,7 +1165,7 @@@ static void stream_blob(uintmax_t len, 
  	off_t offset;
  	git_SHA_CTX c;
  	git_SHA_CTX pack_file_ctx;
 -	z_stream s;
 +	git_zstream s;
  	int status = Z_OK;
  
  	/* Determine if we should auto-checkpoint. */
@@@ -1189,7 -1187,7 +1189,7 @@@
  	crc32_begin(pack_file);
  
  	memset(&s, 0, sizeof(s));
 -	deflateInit(&s, pack_compression_level);
 +	git_deflate_init(&s, pack_compression_level);
  
  	hdrlen = encode_in_pack_object_header(OBJ_BLOB, len, out_buf);
  	if (out_sz <= hdrlen)
@@@ -1211,7 -1209,7 +1211,7 @@@
  			len -= n;
  		}
  
 -		status = deflate(&s, len ? 0 : Z_FINISH);
 +		status = git_deflate(&s, len ? 0 : Z_FINISH);
  
  		if (!s.avail_out || status == Z_STREAM_END) {
  			size_t n = s.next_out - out_buf;
@@@ -1230,7 -1228,7 +1230,7 @@@
  			die("unexpected deflate failure: %d", status);
  		}
  	}
 -	deflateEnd(&s);
 +	git_deflate_end(&s);
  	git_SHA1_Final(sha1, &c);
  
  	if (sha1out)
@@@ -1969,32 -1967,41 +1969,41 @@@ static int validate_raw_date(const cha
  
  static char *parse_ident(const char *buf)
  {
- 	const char *gt;
+ 	const char *ltgt;
  	size_t name_len;
  	char *ident;
  
- 	gt = strrchr(buf, '>');
- 	if (!gt)
+ 	/* ensure there is a space delimiter even if there is no name */
+ 	if (*buf == '<')
+ 		--buf;
+ 
+ 	ltgt = buf + strcspn(buf, "<>");
+ 	if (*ltgt != '<')
+ 		die("Missing < in ident string: %s", buf);
+ 	if (ltgt != buf && ltgt[-1] != ' ')
+ 		die("Missing space before < in ident string: %s", buf);
+ 	ltgt = ltgt + 1 + strcspn(ltgt + 1, "<>");
+ 	if (*ltgt != '>')
  		die("Missing > in ident string: %s", buf);
- 	gt++;
- 	if (*gt != ' ')
+ 	ltgt++;
+ 	if (*ltgt != ' ')
  		die("Missing space after > in ident string: %s", buf);
- 	gt++;
- 	name_len = gt - buf;
+ 	ltgt++;
+ 	name_len = ltgt - buf;
  	ident = xmalloc(name_len + 24);
  	strncpy(ident, buf, name_len);
  
  	switch (whenspec) {
  	case WHENSPEC_RAW:
- 		if (validate_raw_date(gt, ident + name_len, 24) < 0)
- 			die("Invalid raw date \"%s\" in ident: %s", gt, buf);
+ 		if (validate_raw_date(ltgt, ident + name_len, 24) < 0)
+ 			die("Invalid raw date \"%s\" in ident: %s", ltgt, buf);
  		break;
  	case WHENSPEC_RFC2822:
- 		if (parse_date(gt, ident + name_len, 24) < 0)
- 			die("Invalid rfc2822 date \"%s\" in ident: %s", gt, buf);
+ 		if (parse_date(ltgt, ident + name_len, 24) < 0)
+ 			die("Invalid rfc2822 date \"%s\" in ident: %s", ltgt, buf);
  		break;
  	case WHENSPEC_NOW:
- 		if (strcmp("now", gt))
+ 		if (strcmp("now", ltgt))
  			die("Date in ident must be 'now': %s", buf);
  		datestamp(ident + name_len, 24);
  		break;
@@@ -3141,8 -3148,6 +3150,8 @@@ static int parse_one_feature(const cha
  		relative_marks_paths = 1;
  	} else if (!strcmp(feature, "no-relative-marks")) {
  		relative_marks_paths = 0;
 +	} else if (!strcmp(feature, "done")) {
 +		require_explicit_termination = 1;
  	} else if (!strcmp(feature, "force")) {
  		force_update = 1;
  	} else if (!strcmp(feature, "notes") || !strcmp(feature, "ls")) {
@@@ -3199,10 -3204,10 +3208,10 @@@ static int git_pack_config(const char *
  		return 0;
  	}
  	if (!strcmp(k, "pack.indexversion")) {
 -		pack_idx_default_version = git_config_int(k, v);
 -		if (pack_idx_default_version > 2)
 +		pack_idx_opts.version = git_config_int(k, v);
 +		if (pack_idx_opts.version > 2)
  			die("bad pack.indexversion=%"PRIu32,
 -			    pack_idx_default_version);
 +			    pack_idx_opts.version);
  		return 0;
  	}
  	if (!strcmp(k, "pack.packsizelimit")) {
@@@ -3256,7 -3261,6 +3265,7 @@@ int main(int argc, const char **argv
  		usage(fast_import_usage);
  
  	setup_git_directory();
 +	reset_pack_idx_option(&pack_idx_opts);
  	git_config(git_pack_config, NULL);
  	if (!pack_compression_seen && core_compression_seen)
  		pack_compression_level = core_compression_level;
@@@ -3293,8 -3297,6 +3302,8 @@@
  			parse_reset_branch();
  		else if (!strcmp("checkpoint", command_buf.buf))
  			parse_checkpoint();
 +		else if (!strcmp("done", command_buf.buf))
 +			break;
  		else if (!prefixcmp(command_buf.buf, "progress "))
  			parse_progress();
  		else if (!prefixcmp(command_buf.buf, "feature "))
@@@ -3314,9 -3316,6 +3323,9 @@@
  	if (!seen_data_command)
  		parse_argv();
  
 +	if (require_explicit_termination && feof(stdin))
 +		die("stream ends early");
 +
  	end_packfile();
  
  	dump_branches();
diff --combined t/t9300-fast-import.sh
index 2cb4494262,18441f8fcb..4ef7624646
--- a/t/t9300-fast-import.sh
+++ b/t/t9300-fast-import.sh
@@@ -324,6 -324,105 +324,105 @@@ test_expect_success 
  	 test `git rev-parse master` = `git rev-parse TEMP_TAG^`'
  rm -f .git/TEMP_TAG
  
+ git gc 2>/dev/null >/dev/null
+ git prune 2>/dev/null >/dev/null
+ 
+ cat >input <<INPUT_END
+ commit refs/heads/empty-committer-1
+ committer  <> $GIT_COMMITTER_DATE
+ data <<COMMIT
+ empty commit
+ COMMIT
+ INPUT_END
+ test_expect_success 'B: accept empty committer' '
+ 	git fast-import <input &&
+ 	out=$(git fsck) &&
+ 	echo "$out" &&
+ 	test -z "$out"
+ '
+ git update-ref -d refs/heads/empty-committer-1 || true
+ 
+ git gc 2>/dev/null >/dev/null
+ git prune 2>/dev/null >/dev/null
+ 
+ cat >input <<INPUT_END
+ commit refs/heads/empty-committer-2
+ committer <a@b.com> $GIT_COMMITTER_DATE
+ data <<COMMIT
+ empty commit
+ COMMIT
+ INPUT_END
+ test_expect_success 'B: accept and fixup committer with no name' '
+ 	git fast-import <input &&
+ 	out=$(git fsck) &&
+ 	echo "$out" &&
+ 	test -z "$out"
+ '
+ git update-ref -d refs/heads/empty-committer-2 || true
+ 
+ git gc 2>/dev/null >/dev/null
+ git prune 2>/dev/null >/dev/null
+ 
+ cat >input <<INPUT_END
+ commit refs/heads/invalid-committer
+ committer Name email> $GIT_COMMITTER_DATE
+ data <<COMMIT
+ empty commit
+ COMMIT
+ INPUT_END
+ test_expect_success 'B: fail on invalid committer (1)' '
+ 	test_must_fail git fast-import <input
+ '
+ git update-ref -d refs/heads/invalid-committer || true
+ 
+ cat >input <<INPUT_END
+ commit refs/heads/invalid-committer
+ committer Name <e<mail> $GIT_COMMITTER_DATE
+ data <<COMMIT
+ empty commit
+ COMMIT
+ INPUT_END
+ test_expect_success 'B: fail on invalid committer (2)' '
+ 	test_must_fail git fast-import <input
+ '
+ git update-ref -d refs/heads/invalid-committer || true
+ 
+ cat >input <<INPUT_END
+ commit refs/heads/invalid-committer
+ committer Name <email>> $GIT_COMMITTER_DATE
+ data <<COMMIT
+ empty commit
+ COMMIT
+ INPUT_END
+ test_expect_success 'B: fail on invalid committer (3)' '
+ 	test_must_fail git fast-import <input
+ '
+ git update-ref -d refs/heads/invalid-committer || true
+ 
+ cat >input <<INPUT_END
+ commit refs/heads/invalid-committer
+ committer Name <email $GIT_COMMITTER_DATE
+ data <<COMMIT
+ empty commit
+ COMMIT
+ INPUT_END
+ test_expect_success 'B: fail on invalid committer (4)' '
+ 	test_must_fail git fast-import <input
+ '
+ git update-ref -d refs/heads/invalid-committer || true
+ 
+ cat >input <<INPUT_END
+ commit refs/heads/invalid-committer
+ committer Name<email> $GIT_COMMITTER_DATE
+ data <<COMMIT
+ empty commit
+ COMMIT
+ INPUT_END
+ test_expect_success 'B: fail on invalid committer (5)' '
+ 	test_must_fail git fast-import <input
+ '
+ git update-ref -d refs/heads/invalid-committer || true
+ 
  ###
  ### series C
  ###
@@@ -1882,53 -1981,6 +1981,53 @@@ test_expect_success 'R: --import-marks-
  	test_cmp expect io.marks
  '
  
 +test_expect_success 'R: feature import-marks-if-exists' '
 +	rm -f io.marks &&
 +	>expect &&
 +
 +	git fast-import --export-marks=io.marks <<-\EOF &&
 +	feature import-marks-if-exists=not_io.marks
 +	EOF
 +	test_cmp expect io.marks &&
 +
 +	blob=$(echo hi | git hash-object --stdin) &&
 +
 +	echo ":1 $blob" >io.marks &&
 +	echo ":1 $blob" >expect &&
 +	echo ":2 $blob" >>expect &&
 +
 +	git fast-import --export-marks=io.marks <<-\EOF &&
 +	feature import-marks-if-exists=io.marks
 +	blob
 +	mark :2
 +	data 3
 +	hi
 +
 +	EOF
 +	test_cmp expect io.marks &&
 +
 +	echo ":3 $blob" >>expect &&
 +
 +	git fast-import --import-marks=io.marks \
 +			--export-marks=io.marks <<-\EOF &&
 +	feature import-marks-if-exists=not_io.marks
 +	blob
 +	mark :3
 +	data 3
 +	hi
 +
 +	EOF
 +	test_cmp expect io.marks &&
 +
 +	>expect &&
 +
 +	git fast-import --import-marks-if-exists=not_io.marks \
 +			--export-marks=io.marks <<-\EOF
 +	feature import-marks-if-exists=io.marks
 +	EOF
 +	test_cmp expect io.marks
 +'
 +
  cat >input << EOF
  feature import-marks=marks.out
  feature export-marks=marks.new
@@@ -2244,48 -2296,6 +2343,48 @@@ test_expect_success 'R: quiet option re
      test_cmp empty output
  '
  
 +test_expect_success 'R: feature done means terminating "done" is mandatory' '
 +	echo feature done | test_must_fail git fast-import &&
 +	test_must_fail git fast-import --done </dev/null
 +'
 +
 +test_expect_success 'R: terminating "done" with trailing gibberish is ok' '
 +	git fast-import <<-\EOF &&
 +	feature done
 +	done
 +	trailing gibberish
 +	EOF
 +	git fast-import <<-\EOF
 +	done
 +	more trailing gibberish
 +	EOF
 +'
 +
 +test_expect_success 'R: terminating "done" within commit' '
 +	cat >expect <<-\EOF &&
 +	OBJID
 +	:000000 100644 OBJID OBJID A	hello.c
 +	:000000 100644 OBJID OBJID A	hello2.c
 +	EOF
 +	git fast-import <<-EOF &&
 +	commit refs/heads/done-ends
 +	committer $GIT_COMMITTER_NAME <$GIT_COMMITTER_EMAIL> $GIT_COMMITTER_DATE
 +	data <<EOT
 +	Commit terminated by "done" command
 +	EOT
 +	M 100644 inline hello.c
 +	data <<EOT
 +	Hello, world.
 +	EOT
 +	C hello.c hello2.c
 +	done
 +	EOF
 +	git rev-list done-ends |
 +	git diff-tree -r --stdin --root --always |
 +	sed -e "s/$_x40/OBJID/g" >actual &&
 +	test_cmp expect actual
 +'
 +
  cat >input <<EOF
  option git non-existing-option
  EOF