http.postbuffer: allow full range of ssize_t values

Unfortunately, in order to push some large repos where a server does
not support chunked encoding, the http postbuffer must sometimes
exceed two gigabytes.  On a 64-bit system, this is OK: we just malloc
a larger buffer.

This means that we need to use CURLOPT_POSTFIELDSIZE_LARGE to set the
buffer size.

Signed-off-by: David Turner <dturner@twosigma.com>
Reviewed-by: Jonathan Nieder <jrnieder@gmail.com>
Signed-off-by: Junio C Hamano <gitster@pobox.com>

diff --git a/cache.h b/cache.h
index 5c8078291c478f579d6b8bcc69417ac6adb1ced0..3c76ba662945ad6f48dd1c87c46dc6badba74c61 100644 (file)
--- a/cache.h
+++ b/cache.h
@@ -1901,6 +1901,7 @@ extern int git_parse_maybe_bool(const char *);
 extern int git_config_int(const char *, const char *);
 extern int64_t git_config_int64(const char *, const char *);
 extern unsigned long git_config_ulong(const char *, const char *);
 extern int git_config_int(const char *, const char *);
 extern int64_t git_config_int64(const char *, const char *);
 extern unsigned long git_config_ulong(const char *, const char *);

+extern ssize_t git_config_ssize_t(const char *, const char *);

 extern int git_config_bool_or_int(const char *, const char *, int *);
 extern int git_config_bool(const char *, const char *);
 extern int git_config_maybe_bool(const char *, const char *);
 extern int git_config_bool_or_int(const char *, const char *, int *);
 extern int git_config_bool(const char *, const char *);
 extern int git_config_maybe_bool(const char *, const char *);

diff --git a/config.c b/config.c
index 1a4d85537b3c96749999250a34216257db134714..aae6dcc34e34f0844541229c63efc83d62d735a5 100644 (file)
--- a/config.c
+++ b/config.c
@@ -834,6 +834,15 @@ int git_parse_ulong(const char *value, unsigned long *ret)
        return 1;
 }
 
        return 1;
 }
 

+static int git_parse_ssize_t(const char *value, ssize_t *ret)
+{
+       intmax_t tmp;
+       if (!git_parse_signed(value, &tmp, maximum_signed_value_of_type(ssize_t)))
+               return 0;
+       *ret = tmp;
+       return 1;
+}
+

 NORETURN
 static void die_bad_number(const char *name, const char *value)
 {
 NORETURN
 static void die_bad_number(const char *name, const char *value)
 {


@@ -892,6 +901,14 @@ unsigned long git_config_ulong(const char *name, const char *value)
        return ret;
 }
 
        return ret;
 }
 

+ssize_t git_config_ssize_t(const char *name, const char *value)
+{
+       ssize_t ret;
+       if (!git_parse_ssize_t(value, &ret))
+               die_bad_number(name, value);
+       return ret;
+}
+

 int git_parse_maybe_bool(const char *value)
 {
        if (!value)
 int git_parse_maybe_bool(const char *value)
 {
        if (!value)

diff --git a/http.c b/http.c
index 96d84bbed3f66153cc3d817f2b342dc5d1600574..7bccb3645958a07df578c943a97c0dc6f266de6e 100644 (file)
--- a/http.c
+++ b/http.c
@@ -19,7 +19,7 @@ long int git_curl_ipresolve;
 #endif
 int active_requests;
 int http_is_verbose;
 #endif
 int active_requests;
 int http_is_verbose;

-size_t http_post_buffer = 16 * LARGE_PACKET_MAX;
+ssize_t http_post_buffer = 16 * LARGE_PACKET_MAX;

 
 #if LIBCURL_VERSION_NUM >= 0x070a06
 #define LIBCURL_CAN_HANDLE_AUTH_ANY
 
 #if LIBCURL_VERSION_NUM >= 0x070a06
 #define LIBCURL_CAN_HANDLE_AUTH_ANY


@@ -331,7 +331,9 @@ static int http_options(const char *var, const char *value, void *cb)
        }
 
        if (!strcmp("http.postbuffer", var)) {
        }
 
        if (!strcmp("http.postbuffer", var)) {

-               http_post_buffer = git_config_int(var, value);
+               http_post_buffer = git_config_ssize_t(var, value);
+               if (http_post_buffer < 0)
+                       warning(_("negative value for http.postbuffer; defaulting to %d"), LARGE_PACKET_MAX);

                if (http_post_buffer < LARGE_PACKET_MAX)
                        http_post_buffer = LARGE_PACKET_MAX;
                return 0;
                if (http_post_buffer < LARGE_PACKET_MAX)
                        http_post_buffer = LARGE_PACKET_MAX;
                return 0;

diff --git a/http.h b/http.h
index 02bccb7b0caf9f2b1be0b98cb16c6fc84b95cc74..f7bd3b26b0da70e44402579e4912a1b20dcef16e 100644 (file)
--- a/http.h
+++ b/http.h
@@ -111,7 +111,7 @@ extern struct curl_slist *http_copy_default_headers(void);
 extern long int git_curl_ipresolve;
 extern int active_requests;
 extern int http_is_verbose;
 extern long int git_curl_ipresolve;
 extern int active_requests;
 extern int http_is_verbose;

-extern size_t http_post_buffer;
+extern ssize_t http_post_buffer;

 extern struct credential http_auth;
 
 extern char curl_errorstr[CURL_ERROR_SIZE];
 extern struct credential http_auth;
 
 extern char curl_errorstr[CURL_ERROR_SIZE];

diff --git a/remote-curl.c b/remote-curl.c
index e953d06f66a5993b9f1a0039920c71a90afa5bc9..cf171b1bc95cd66f63c14e3337c3c91646262212 100644 (file)
--- a/remote-curl.c
+++ b/remote-curl.c
@@ -531,6 +531,12 @@ static int probe_rpc(struct rpc_state *rpc, struct slot_results *results)
        return err;
 }
 
        return err;
 }
 

+static curl_off_t xcurl_off_t(ssize_t len) {
+       if (len > maximum_signed_value_of_type(curl_off_t))
+               die("cannot handle pushes this big");
+       return (curl_off_t) len;
+}
+

 static int post_rpc(struct rpc_state *rpc)
 {
        struct active_request_slot *slot;
 static int post_rpc(struct rpc_state *rpc)
 {
        struct active_request_slot *slot;


@@ -614,7 +620,7 @@ static int post_rpc(struct rpc_state *rpc)
                 * and we just need to send it.
                 */
                curl_easy_setopt(slot->curl, CURLOPT_POSTFIELDS, gzip_body);
                 * and we just need to send it.
                 */
                curl_easy_setopt(slot->curl, CURLOPT_POSTFIELDS, gzip_body);

-               curl_easy_setopt(slot->curl, CURLOPT_POSTFIELDSIZE, gzip_size);
+               curl_easy_setopt(slot->curl, CURLOPT_POSTFIELDSIZE_LARGE, xcurl_off_t(gzip_size));

 
        } else if (use_gzip && 1024 < rpc->len) {
                /* The client backend isn't giving us compressed data so
 
        } else if (use_gzip && 1024 < rpc->len) {
                /* The client backend isn't giving us compressed data so


@@ -645,7 +651,7 @@ static int post_rpc(struct rpc_state *rpc)
 
                headers = curl_slist_append(headers, "Content-Encoding: gzip");
                curl_easy_setopt(slot->curl, CURLOPT_POSTFIELDS, gzip_body);
 
                headers = curl_slist_append(headers, "Content-Encoding: gzip");
                curl_easy_setopt(slot->curl, CURLOPT_POSTFIELDS, gzip_body);

-               curl_easy_setopt(slot->curl, CURLOPT_POSTFIELDSIZE, gzip_size);
+               curl_easy_setopt(slot->curl, CURLOPT_POSTFIELDSIZE_LARGE, xcurl_off_t(gzip_size));

 
                if (options.verbosity > 1) {
                        fprintf(stderr, "POST %s (gzip %lu to %lu bytes)\n",
 
                if (options.verbosity > 1) {
                        fprintf(stderr, "POST %s (gzip %lu to %lu bytes)\n",


@@ -658,7 +664,7 @@ static int post_rpc(struct rpc_state *rpc)
                 * more normal Content-Length approach.
                 */
                curl_easy_setopt(slot->curl, CURLOPT_POSTFIELDS, rpc->buf);
                 * more normal Content-Length approach.
                 */
                curl_easy_setopt(slot->curl, CURLOPT_POSTFIELDS, rpc->buf);

-               curl_easy_setopt(slot->curl, CURLOPT_POSTFIELDSIZE, rpc->len);
+               curl_easy_setopt(slot->curl, CURLOPT_POSTFIELDSIZE_LARGE, xcurl_off_t(rpc->len));

                if (options.verbosity > 1) {
                        fprintf(stderr, "POST %s (%lu bytes)\n",
                                rpc->service_name, (unsigned long)rpc->len);
                if (options.verbosity > 1) {
                        fprintf(stderr, "POST %s (%lu bytes)\n",
                                rpc->service_name, (unsigned long)rpc->len);