From: Junio C Hamano <junkio@cox.net>
Date: Sun, 7 Jan 2007 08:12:47 +0000 (-0800)
Subject: Merge branch 'sp/mmap'
X-Git-Tag: v1.5.0-rc1~64
X-Git-Url: https://git.lorimer.id.au/gitweb.git/diff_plain/cf2999eb4cbe39d5a40add574be9e5c019df758b?ds=inline;hp=-c

Merge branch 'sp/mmap'

* sp/mmap: (27 commits)
Spell default packedgitlimit slightly differently
Increase packedGit{Limit,WindowSize} on 64 bit systems.
Update packedGit config option documentation.
mmap: set FD_CLOEXEC for file descriptors we keep open for mmap()
pack-objects: fix use of use_pack().
Fix random segfaults in pack-objects.
Cleanup read_cache_from error handling.
Replace mmap with xmmap, better handling MAP_FAILED.
Release pack windows before reporting out of memory.
Default core.packdGitWindowSize to 1 MiB if NO_MMAP.
Test suite for sliding window mmap implementation.
Create pack_report() as a debugging aid.
Support unmapping windows on 'temporary' packfiles.
Improve error message when packfile mmap fails.
Ensure core.packedGitWindowSize cannot be less than 2 pages.
Load core configuration in git-verify-pack.
Fully activate the sliding window pack access.
Unmap individual windows rather than entire files.
Document why header parsing won't exceed a window.
Loop over pack_windows when inflating/accessing data.
...

Conflicts:

cache.h
pack-check.c
---

cf2999eb4cbe39d5a40add574be9e5c019df758b
diff --combined Documentation/config.txt
index 4318bf9334,b24d9dff62..b4aae0d0ae
--- a/Documentation/config.txt
+++ b/Documentation/config.txt
@@@ -118,6 -118,34 +118,34 @@@ core.legacyheaders:
  	database directly (where the "http://" and "rsync://" protocols
  	count as direct access).
  
+ core.packedGitWindowSize::
+ 	Number of bytes of a pack file to map into memory in a
+ 	single mapping operation.  Larger window sizes may allow
+ 	your system to process a smaller number of large pack files
+ 	more quickly.  Smaller window sizes will negatively affect
+ 	performance due to increased calls to the operating system's
+ 	memory manager, but may improve performance when accessing
+ 	a large number of large pack files.
+ +
+ Default is 1 MiB if NO_MMAP was set at compile time, otherwise 32
+ MiB on 32 bit platforms and 1 GiB on 64 bit platforms.  This should
+ be reasonable for all users/operating systems.  You probably do
+ not need to adjust this value.
+ +
+ Common unit suffixes of 'k', 'm', or 'g' are supported.
+ 
+ core.packedGitLimit::
+ 	Maximum number of bytes to map simultaneously into memory
+ 	from pack files.  If Git needs to access more than this many
+ 	bytes at once to complete an operation it will unmap existing
+ 	regions to reclaim virtual address space within the process.
+ +
+ Default is 256 MiB on 32 bit platforms and 8 GiB on 64 bit platforms.
+ This should be reasonable for all users/operating systems, except on
+ the largest projects.  You probably do not need to adjust this value.
+ +
+ Common unit suffixes of 'k', 'm', or 'g' are supported.
+ 
  alias.*::
  	Command aliases for the gitlink:git[1] command wrapper - e.g.
  	after defining "alias.last = cat-file commit HEAD", the invocation
@@@ -145,21 -173,6 +173,21 @@@ branch.<name>.merge:
  	this option, `git pull` defaults to merge the first refspec fetched.
  	Specify multiple values to get an octopus merge.
  
 +color.branch::
 +	A boolean to enable/disable color in the output of
 +	gitlink:git-branch[1]. May be set to `true` (or `always`),
 +	`false` (or `never`) or `auto`, in which case colors are used
 +	only when the output is to a terminal. Defaults to false.
 +
 +color.branch.<slot>::
 +	Use customized color for branch coloration. `<slot>` is one of
 +	`current` (the current branch), `local` (a local branch),
 +	`remote` (a tracking branch in refs/remotes/), `plain` (other
 +	refs), or `reset` (the normal terminal color).  The value for
 +	these configuration variables can be one of: `normal`, `bold`,
 +	`dim`, `ul`, `blink`, `reverse`, `reset`, `black`, `red`,
 +	`green`, `yellow`, `blue`, `magenta`, `cyan`, or `white`.
 +
  color.diff::
  	When true (or `always`), always use colors in patch.
  	When false (or `never`), never.  When set to `auto`, use
@@@ -170,8 -183,11 +198,8 @@@ color.diff.<slot>:
  	specifies which part of the patch to use the specified
  	color, and is one of `plain` (context text), `meta`
  	(metainformation), `frag` (hunk header), `old` (removed
 -	lines), or `new` (added lines).  The value for these
 -	configuration variables can be one of: `normal`, `bold`,
 -	`dim`, `ul`, `blink`, `reverse`, `reset`, `black`,
 -	`red`, `green`, `yellow`, `blue`, `magenta`, `cyan`, or
 -	`white`.
 +	lines), or `new` (added lines).  The values of these
 +	variables may be specified as in color.branch.<slot>.
  
  color.pager::
  	A boolean to enable/disable colored output when the pager is in
@@@ -189,7 -205,7 +217,7 @@@ color.status.<slot>:
  	`added` or `updated` (files which are added but not committed),
  	`changed` (files which are changed but not added in the index),
  	or `untracked` (files which are not tracked by git). The values of
 -	these variables may be specified as in color.diff.<slot>.
 +	these variables may be specified as in color.branch.<slot>.
  
  diff.renameLimit::
  	The number of files to consider when performing the copy/rename
diff --combined cache.h
index 31b0819e83,a5fc23235e..36be64e386
--- a/cache.h
+++ b/cache.h
@@@ -179,7 -179,6 +179,7 @@@ extern int refresh_cache(unsigned int f
  
  struct lock_file {
  	struct lock_file *next;
 +	char on_list;
  	char filename[PATH_MAX];
  };
  extern int hold_lock_file_for_update(struct lock_file *, const char *path, int);
@@@ -197,6 -196,8 +197,8 @@@ extern int warn_ambiguous_refs
  extern int shared_repository;
  extern const char *apply_default_whitespace;
  extern int zlib_compression_level;
+ extern size_t packed_git_window_size;
+ extern size_t packed_git_limit;
  
  #define GIT_REPO_VERSION 0
  extern int repository_format_version;
@@@ -336,14 -337,22 +338,22 @@@ extern struct alternate_object_databas
  } *alt_odb_list;
  extern void prepare_alt_odb(void);
  
+ struct pack_window {
+ 	struct pack_window *next;
+ 	unsigned char *base;
+ 	off_t offset;
+ 	size_t len;
+ 	unsigned int last_used;
+ 	unsigned int inuse_cnt;
+ };
+ 
  extern struct packed_git {
  	struct packed_git *next;
- 	unsigned long index_size;
- 	unsigned long pack_size;
+ 	struct pack_window *windows;
  	unsigned int *index_base;
- 	void *pack_base;
- 	unsigned int pack_last_used;
- 	unsigned int pack_use_cnt;
+ 	off_t index_size;
+ 	off_t pack_size;
+ 	int pack_fd;
  	int pack_local;
  	unsigned char sha1[20];
  	/* something like ".git/objects/pack/xxxxx.pack" */
@@@ -389,13 -398,14 +399,14 @@@ extern void install_packed_git(struct p
  extern struct packed_git *find_sha1_pack(const unsigned char *sha1, 
  					 struct packed_git *packs);
  
- extern int use_packed_git(struct packed_git *);
- extern void unuse_packed_git(struct packed_git *);
+ extern void pack_report();
+ extern unsigned char* use_pack(struct packed_git *, struct pack_window **, unsigned long, unsigned int *);
+ extern void unuse_pack(struct pack_window **);
  extern struct packed_git *add_packed_git(char *, int, int);
  extern int num_packed_objects(const struct packed_git *p);
  extern int nth_packed_object_sha1(const struct packed_git *, int, unsigned char*);
  extern unsigned long find_pack_entry_one(const unsigned char *, struct packed_git *);
- extern void *unpack_entry_gently(struct packed_git *, unsigned long, char *, unsigned long *);
+ extern void *unpack_entry(struct packed_git *, unsigned long, char *, unsigned long *);
  extern unsigned long unpack_object_header_gently(const unsigned char *buf, unsigned long len, enum object_type *type, unsigned long *sizep);
  extern void packed_object_info_detail(struct packed_git *, unsigned long, char *, unsigned long *, unsigned long *, unsigned int *, unsigned char *);
  
@@@ -421,7 -431,7 +432,8 @@@ extern char *git_commit_encoding
  extern char *git_log_output_encoding;
  
  extern int copy_fd(int ifd, int ofd);
+ extern void read_or_die(int fd, void *buf, size_t count);
 +extern int write_in_full(int fd, const void *buf, size_t count, const char *);
  extern void write_or_die(int fd, const void *buf, size_t count);
  extern int write_or_whine(int fd, const void *buf, size_t count, const char *msg);
  
diff --combined diff.c
index 2c2e9dcb8c,244292a70c..e1b016f47f
--- a/diff.c
+++ b/diff.c
@@@ -1341,10 -1341,8 +1341,8 @@@ int diff_populate_filespec(struct diff_
  		fd = open(s->path, O_RDONLY);
  		if (fd < 0)
  			goto err_empty;
- 		s->data = mmap(NULL, s->size, PROT_READ, MAP_PRIVATE, fd, 0);
+ 		s->data = xmmap(NULL, s->size, PROT_READ, MAP_PRIVATE, fd, 0);
  		close(fd);
- 		if (s->data == MAP_FAILED)
- 			goto err_empty;
  		s->should_munmap = 1;
  	}
  	else {
@@@ -2875,12 -2873,10 +2873,12 @@@ void diff_change(struct diff_options *o
  }
  
  void diff_unmerge(struct diff_options *options,
 -		  const char *path)
 +		  const char *path,
 +		  unsigned mode, const unsigned char *sha1)
  {
  	struct diff_filespec *one, *two;
  	one = alloc_filespec(path);
  	two = alloc_filespec(path);
 -	diff_queue(&diff_queued_diff, one, two);
 +	fill_filespec(one, sha1, mode);
 +	diff_queue(&diff_queued_diff, one, two)->is_unmerged = 1;
  }
diff --combined refs.c
index f76b4fe20d,121774cb32..52057455a2
--- a/refs.c
+++ b/refs.c
@@@ -726,6 -726,7 +726,6 @@@ static int repack_without_ref(const cha
  	}
  	if (!found)
  		return 0;
 -	memset(&packlock, 0, sizeof(packlock));
  	fd = hold_lock_file_for_update(&packlock, git_path("packed-refs"), 0);
  	if (fd < 0)
  		return error("cannot delete '%s' from packed refs", refname);
@@@ -1025,7 -1026,7 +1025,7 @@@ int read_ref_at(const char *ref, unsign
  	fstat(logfd, &st);
  	if (!st.st_size)
  		die("Log %s is empty.", logfile);
- 	logdata = mmap(NULL, st.st_size, PROT_READ, MAP_PRIVATE, logfd, 0);
+ 	logdata = xmmap(NULL, st.st_size, PROT_READ, MAP_PRIVATE, logfd, 0);
  	close(logfd);
  
  	lastrec = NULL;
diff --combined write_or_die.c
index 650f13fc01,8cf6486025..6db1d3123d
--- a/write_or_die.c
+++ b/write_or_die.c
@@@ -1,5 -1,21 +1,21 @@@
  #include "cache.h"
  
+ void read_or_die(int fd, void *buf, size_t count)
+ {
+ 	char *p = buf;
+ 	ssize_t loaded;
+ 
+ 	while (count > 0) {
+ 		loaded = xread(fd, p, count);
+ 		if (loaded == 0)
+ 			die("unexpected end of file");
+ 		else if (loaded < 0)
+ 			die("read error (%s)", strerror(errno));
+ 		count -= loaded;
+ 		p += loaded;
+ 	}
+ }
+ 
  void write_or_die(int fd, const void *buf, size_t count)
  {
  	const char *p = buf;
@@@ -43,26 -59,3 +59,26 @@@ int write_or_whine(int fd, const void *
  
  	return 1;
  }
 +
 +int write_in_full(int fd, const void *buf, size_t count, const char *msg)
 +{
 +	const char *p = buf;
 +	ssize_t written;
 +
 +	while (count > 0) {
 +		written = xwrite(fd, p, count);
 +		if (written == 0) {
 +			fprintf(stderr, "%s: disk full?\n", msg);
 +			return 0;
 +		}
 +		else if (written < 0) {
 +			fprintf(stderr, "%s: write error (%s)\n",
 +				msg, strerror(errno));
 +			return 0;
 +		}
 +		count -= written;
 +		p += written;
 +	}
 +
 +	return 1;
 +}