From: Junio C Hamano <gitster@pobox.com>
Date: Wed, 25 Feb 2015 23:40:13 +0000 (-0800)
Subject: Merge branch 'jc/max-io-size-and-ssize-max'
X-Git-Tag: v2.4.0-rc0~89
X-Git-Url: https://git.lorimer.id.au/gitweb.git/diff_plain/81a535da88617e2ae24b79a9d7413008e0ebc23b?hp=-c

Merge branch 'jc/max-io-size-and-ssize-max'

Our default I/O size (8 MiB) for large files was too large for some
platforms with smaller SSIZE_MAX, leading to read(2)/write(2)
failures.

* jc/max-io-size-and-ssize-max:
xread/xwrite: clip MAX_IO_SIZE to SSIZE_MAX
---

81a535da88617e2ae24b79a9d7413008e0ebc23b
diff --combined wrapper.c
index 007ec0d8ea,cfaf23d387..d5a6cef2be
--- a/wrapper.c
+++ b/wrapper.c
@@@ -9,24 -9,16 +9,24 @@@ static void do_nothing(size_t size
  
  static void (*try_to_free_routine)(size_t size) = do_nothing;
  
 -static void memory_limit_check(size_t size)
 +static int memory_limit_check(size_t size, int gentle)
  {
 -	static int limit = -1;
 -	if (limit == -1) {
 -		const char *env = getenv("GIT_ALLOC_LIMIT");
 -		limit = env ? atoi(env) * 1024 : 0;
 +	static size_t limit = 0;
 +	if (!limit) {
 +		limit = git_env_ulong("GIT_ALLOC_LIMIT", 0);
 +		if (!limit)
 +			limit = SIZE_MAX;
  	}
 -	if (limit && size > limit)
 -		die("attempting to allocate %"PRIuMAX" over limit %d",
 -		    (intmax_t)size, limit);
 +	if (size > limit) {
 +		if (gentle) {
 +			error("attempting to allocate %"PRIuMAX" over limit %"PRIuMAX,
 +			      (uintmax_t)size, (uintmax_t)limit);
 +			return -1;
 +		} else
 +			die("attempting to allocate %"PRIuMAX" over limit %"PRIuMAX,
 +			    (uintmax_t)size, (uintmax_t)limit);
 +	}
 +	return 0;
  }
  
  try_to_free_t set_try_to_free_routine(try_to_free_t routine)
@@@ -50,12 -42,11 +50,12 @@@ char *xstrdup(const char *str
  	return ret;
  }
  
 -void *xmalloc(size_t size)
 +static void *do_xmalloc(size_t size, int gentle)
  {
  	void *ret;
  
 -	memory_limit_check(size);
 +	if (memory_limit_check(size, gentle))
 +		return NULL;
  	ret = malloc(size);
  	if (!ret && !size)
  		ret = malloc(1);
@@@ -64,16 -55,9 +64,16 @@@
  		ret = malloc(size);
  		if (!ret && !size)
  			ret = malloc(1);
 -		if (!ret)
 -			die("Out of memory, malloc failed (tried to allocate %lu bytes)",
 -			    (unsigned long)size);
 +		if (!ret) {
 +			if (!gentle)
 +				die("Out of memory, malloc failed (tried to allocate %lu bytes)",
 +				    (unsigned long)size);
 +			else {
 +				error("Out of memory, malloc failed (tried to allocate %lu bytes)",
 +				      (unsigned long)size);
 +				return NULL;
 +			}
 +		}
  	}
  #ifdef XMALLOC_POISON
  	memset(ret, 0xA5, size);
@@@ -81,37 -65,16 +81,37 @@@
  	return ret;
  }
  
 -void *xmallocz(size_t size)
 +void *xmalloc(size_t size)
 +{
 +	return do_xmalloc(size, 0);
 +}
 +
 +static void *do_xmallocz(size_t size, int gentle)
  {
  	void *ret;
 -	if (unsigned_add_overflows(size, 1))
 -		die("Data too large to fit into virtual memory space.");
 -	ret = xmalloc(size + 1);
 -	((char*)ret)[size] = 0;
 +	if (unsigned_add_overflows(size, 1)) {
 +		if (gentle) {
 +			error("Data too large to fit into virtual memory space.");
 +			return NULL;
 +		} else
 +			die("Data too large to fit into virtual memory space.");
 +	}
 +	ret = do_xmalloc(size + 1, gentle);
 +	if (ret)
 +		((char*)ret)[size] = 0;
  	return ret;
  }
  
 +void *xmallocz(size_t size)
 +{
 +	return do_xmallocz(size, 0);
 +}
 +
 +void *xmallocz_gently(size_t size)
 +{
 +	return do_xmallocz(size, 1);
 +}
 +
  /*
   * xmemdupz() allocates (len + 1) bytes of memory, duplicates "len" bytes of
   * "data" to the allocated memory, zero terminates the allocated memory,
@@@ -133,7 -96,7 +133,7 @@@ void *xrealloc(void *ptr, size_t size
  {
  	void *ret;
  
 -	memory_limit_check(size);
 +	memory_limit_check(size, 0);
  	ret = realloc(ptr, size);
  	if (!ret && !size)
  		ret = realloc(ptr, 1);
@@@ -152,7 -115,7 +152,7 @@@ void *xcalloc(size_t nmemb, size_t size
  {
  	void *ret;
  
 -	memory_limit_check(size * nmemb);
 +	memory_limit_check(size * nmemb, 0);
  	ret = calloc(nmemb, size);
  	if (!ret && (!nmemb || !size))
  		ret = calloc(1, 1);
@@@ -170,10 -133,24 +170,24 @@@
  /*
   * Limit size of IO chunks, because huge chunks only cause pain.  OS X
   * 64-bit is buggy, returning EINVAL if len >= INT_MAX; and even in
 - * the absense of bugs, large chunks can result in bad latencies when
 + * the absence of bugs, large chunks can result in bad latencies when
   * you decide to kill the process.
+  *
+  * We pick 8 MiB as our default, but if the platform defines SSIZE_MAX
+  * that is smaller than that, clip it to SSIZE_MAX, as a call to
+  * read(2) or write(2) larger than that is allowed to fail.  As the last
+  * resort, we allow a port to pass via CFLAGS e.g. "-DMAX_IO_SIZE=value"
+  * to override this, if the definition of SSIZE_MAX given by the platform
+  * is broken.
   */
- #define MAX_IO_SIZE (8*1024*1024)
+ #ifndef MAX_IO_SIZE
+ # define MAX_IO_SIZE_DEFAULT (8*1024*1024)
+ # if defined(SSIZE_MAX) && (SSIZE_MAX < MAX_IO_SIZE_DEFAULT)
+ #  define MAX_IO_SIZE SSIZE_MAX
+ # else
+ #  define MAX_IO_SIZE MAX_IO_SIZE_DEFAULT
+ # endif
+ #endif
  
  /*
   * xread() is the same a read(), but it automatically restarts read()
@@@ -211,24 -188,6 +225,24 @@@ ssize_t xwrite(int fd, const void *buf
  	}
  }
  
 +/*
 + * xpread() is the same as pread(), but it automatically restarts pread()
 + * operations with a recoverable error (EAGAIN and EINTR). xpread() DOES
 + * NOT GUARANTEE that "len" bytes is read even if the data is available.
 + */
 +ssize_t xpread(int fd, void *buf, size_t len, off_t offset)
 +{
 +	ssize_t nr;
 +	if (len > MAX_IO_SIZE)
 +		len = MAX_IO_SIZE;
 +	while (1) {
 +		nr = pread(fd, buf, len, offset);
 +		if ((nr < 0) && (errno == EAGAIN || errno == EINTR))
 +			continue;
 +		return nr;
 +	}
 +}
 +
  ssize_t read_in_full(int fd, void *buf, size_t count)
  {
  	char *p = buf;
@@@ -269,26 -228,6 +283,26 @@@ ssize_t write_in_full(int fd, const voi
  	return total;
  }
  
 +ssize_t pread_in_full(int fd, void *buf, size_t count, off_t offset)
 +{
 +	char *p = buf;
 +	ssize_t total = 0;
 +
 +	while (count > 0) {
 +		ssize_t loaded = xpread(fd, p, count, offset);
 +		if (loaded < 0)
 +			return -1;
 +		if (loaded == 0)
 +			return total;
 +		count -= loaded;
 +		p += loaded;
 +		total += loaded;
 +		offset += loaded;
 +	}
 +
 +	return total;
 +}
 +
  int xdup(int fd)
  {
  	int ret = dup(fd);
@@@ -435,12 -374,10 +449,12 @@@ int git_mkstemp_mode(char *pattern, in
  	return git_mkstemps_mode(pattern, 0, mode);
  }
  
 +#ifdef NO_MKSTEMPS
  int gitmkstemps(char *pattern, int suffix_len)
  {
  	return git_mkstemps_mode(pattern, suffix_len, 0600);
  }
 +#endif
  
  int xmkstemp_mode(char *template, int mode)
  {
@@@ -466,29 -403,17 +480,29 @@@
  
  static int warn_if_unremovable(const char *op, const char *file, int rc)
  {
 -	if (rc < 0) {
 -		int err = errno;
 -		if (ENOENT != err) {
 -			warning("unable to %s %s: %s",
 -				op, file, strerror(errno));
 -			errno = err;
 -		}
 -	}
 +	int err;
 +	if (!rc || errno == ENOENT)
 +		return 0;
 +	err = errno;
 +	warning("unable to %s %s: %s", op, file, strerror(errno));
 +	errno = err;
  	return rc;
  }
  
 +int unlink_or_msg(const char *file, struct strbuf *err)
 +{
 +	int rc = unlink(file);
 +
 +	assert(err);
 +
 +	if (!rc || errno == ENOENT)
 +		return 0;
 +
 +	strbuf_addf(err, "unable to unlink %s: %s",
 +		    file, strerror(errno));
 +	return -1;
 +}
 +
  int unlink_or_warn(const char *file)
  {
  	return warn_if_unremovable("unlink", file, unlink(file));
@@@ -542,11 -467,3 +556,11 @@@ struct passwd *xgetpwuid_self(void
  		    errno ? strerror(errno) : _("no such user"));
  	return pw;
  }
 +
 +char *xgetcwd(void)
 +{
 +	struct strbuf sb = STRBUF_INIT;
 +	if (strbuf_getcwd(&sb))
 +		die_errno(_("unable to get current working directory"));
 +	return strbuf_detach(&sb, NULL);
 +}