From: Junio C Hamano Date: Fri, 22 Apr 2016 22:45:09 +0000 (-0700) Subject: Merge branch 'jc/http-socks5h' X-Git-Tag: v2.9.0-rc0~105 X-Git-Url: https://git.lorimer.id.au/gitweb.git/diff_plain/1c4f47690074de3c27c1a2124779bfe2898f295d?ds=inline;hp=-c Merge branch 'jc/http-socks5h' The socks5:// proxy support added back in 2.6.4 days was not aware that socks5h:// proxies behave differently. * jc/http-socks5h: http: differentiate socks5:// and socks5h:// --- 1c4f47690074de3c27c1a2124779bfe2898f295d diff --combined http.c index 69da4454d8,b560c133cf..4304b80ad3 --- a/http.c +++ b/http.c @@@ -11,11 -11,6 +11,11 @@@ #include "gettext.h" #include "transport.h" +#if LIBCURL_VERSION_NUM >= 0x070a08 +long int git_curl_ipresolve = CURL_IPRESOLVE_WHATEVER; +#else +long int git_curl_ipresolve; +#endif int active_requests; int http_is_verbose; size_t http_post_buffer = 16 * LARGE_PACKET_MAX; @@@ -35,6 -30,7 +35,6 @@@ static CURL *curl_default #endif #define PREV_BUF_SIZE 4096 -#define RANGE_HEADER_SIZE 30 char curl_errorstr[CURL_ERROR_SIZE]; @@@ -62,41 -58,16 +62,41 @@@ static const char *ssl_key #if LIBCURL_VERSION_NUM >= 0x070908 static const char *ssl_capath; #endif +#if LIBCURL_VERSION_NUM >= 0x072c00 +static const char *ssl_pinnedkey; +#endif static const char *ssl_cainfo; static long curl_low_speed_limit = -1; static long curl_low_speed_time = -1; static int curl_ftp_no_epsv; static const char *curl_http_proxy; +static const char *curl_no_proxy; +static const char *http_proxy_authmethod; +static struct { + const char *name; + long curlauth_param; +} proxy_authmethods[] = { + { "basic", CURLAUTH_BASIC }, + { "digest", CURLAUTH_DIGEST }, + { "negotiate", CURLAUTH_GSSNEGOTIATE }, + { "ntlm", CURLAUTH_NTLM }, +#ifdef LIBCURL_CAN_HANDLE_AUTH_ANY + { "anyauth", CURLAUTH_ANY }, +#endif + /* + * CURLAUTH_DIGEST_IE has no corresponding command-line option in + * curl(1) and is not included in CURLAUTH_ANY, so we leave it out + * here, too + */ +}; +static struct credential proxy_auth = CREDENTIAL_INIT; +static const char *curl_proxyuserpwd; static const char *curl_cookie_file; static int curl_save_cookies; struct credential http_auth = CREDENTIAL_INIT; static int http_proactive_auth; static const char *user_agent; +static int curl_empty_auth; #if LIBCURL_VERSION_NUM >= 0x071700 /* Use CURLOPT_KEYPASSWD as is */ @@@ -189,9 -160,6 +189,9 @@@ static void finish_active_slot(struct a #else slot->results->auth_avail = 0; #endif + + curl_easy_getinfo(slot->curl, CURLINFO_HTTP_CONNECTCODE, + &slot->results->http_connectcode); } /* Run callback if appropriate */ @@@ -289,9 -257,6 +289,9 @@@ static int http_options(const char *var if (!strcmp("http.proxy", var)) return git_config_string(&curl_http_proxy, var, value); + if (!strcmp("http.proxyauthmethod", var)) + return git_config_string(&http_proxy_authmethod, var, value); + if (!strcmp("http.cookiefile", var)) return git_config_string(&curl_cookie_file, var, value); if (!strcmp("http.savecookies", var)) { @@@ -309,31 -274,14 +309,31 @@@ if (!strcmp("http.useragent", var)) return git_config_string(&user_agent, var, value); + if (!strcmp("http.emptyauth", var)) { + curl_empty_auth = git_config_bool(var, value); + return 0; + } + + if (!strcmp("http.pinnedpubkey", var)) { +#if LIBCURL_VERSION_NUM >= 0x072c00 + return git_config_pathname(&ssl_pinnedkey, var, value); +#else + warning(_("Public key pinning not supported with cURL < 7.44.0")); + return 0; +#endif + } + /* Fall back on the default ones */ return git_default_config(var, value, cb); } static void init_curl_http_auth(CURL *result) { - if (!http_auth.username) + if (!http_auth.username) { + if (curl_empty_auth) + curl_easy_setopt(result, CURLOPT_USERPWD, ":"); return; + } credential_fill(&http_auth); @@@ -357,64 -305,6 +357,64 @@@ #endif } +/* *var must be free-able */ +static void var_override(const char **var, char *value) +{ + if (value) { + free((void *)*var); + *var = xstrdup(value); + } +} + +static void set_proxyauth_name_password(CURL *result) +{ +#if LIBCURL_VERSION_NUM >= 0x071301 + curl_easy_setopt(result, CURLOPT_PROXYUSERNAME, + proxy_auth.username); + curl_easy_setopt(result, CURLOPT_PROXYPASSWORD, + proxy_auth.password); +#else + struct strbuf s = STRBUF_INIT; + + strbuf_addstr_urlencode(&s, proxy_auth.username, 1); + strbuf_addch(&s, ':'); + strbuf_addstr_urlencode(&s, proxy_auth.password, 1); + curl_proxyuserpwd = strbuf_detach(&s, NULL); + curl_easy_setopt(result, CURLOPT_PROXYUSERPWD, curl_proxyuserpwd); +#endif +} + +static void init_curl_proxy_auth(CURL *result) +{ + if (proxy_auth.username) { + if (!proxy_auth.password) + credential_fill(&proxy_auth); + set_proxyauth_name_password(result); + } + + var_override(&http_proxy_authmethod, getenv("GIT_HTTP_PROXY_AUTHMETHOD")); + +#if LIBCURL_VERSION_NUM >= 0x070a07 /* CURLOPT_PROXYAUTH and CURLAUTH_ANY */ + if (http_proxy_authmethod) { + int i; + for (i = 0; i < ARRAY_SIZE(proxy_authmethods); i++) { + if (!strcmp(http_proxy_authmethod, proxy_authmethods[i].name)) { + curl_easy_setopt(result, CURLOPT_PROXYAUTH, + proxy_authmethods[i].curlauth_param); + break; + } + } + if (i == ARRAY_SIZE(proxy_authmethods)) { + warning("unsupported proxy authentication method %s: using anyauth", + http_proxy_authmethod); + curl_easy_setopt(result, CURLOPT_PROXYAUTH, CURLAUTH_ANY); + } + } + else + curl_easy_setopt(result, CURLOPT_PROXYAUTH, CURLAUTH_ANY); +#endif +} + static int has_cert_password(void) { if (ssl_cert == NULL || ssl_cert_password_required != 1) @@@ -525,10 -415,6 +525,10 @@@ static CURL *get_curl_handle(void #if LIBCURL_VERSION_NUM >= 0x070908 if (ssl_capath != NULL) curl_easy_setopt(result, CURLOPT_CAPATH, ssl_capath); +#endif +#if LIBCURL_VERSION_NUM >= 0x072c00 + if (ssl_pinnedkey != NULL) + curl_easy_setopt(result, CURLOPT_PINNEDPUBLICKEY, ssl_pinnedkey); #endif if (ssl_cainfo != NULL) curl_easy_setopt(result, CURLOPT_CAINFO, ssl_cainfo); @@@ -577,35 -463,13 +577,38 @@@ curl_easy_setopt(result, CURLOPT_USE_SSL, CURLUSESSL_TRY); #endif + /* + * CURL also examines these variables as a fallback; but we need to query + * them here in order to decide whether to prompt for missing password (cf. + * init_curl_proxy_auth()). + * + * Unlike many other common environment variables, these are historically + * lowercase only. It appears that CURL did not know this and implemented + * only uppercase variants, which was later corrected to take both - with + * the exception of http_proxy, which is lowercase only also in CURL. As + * the lowercase versions are the historical quasi-standard, they take + * precedence here, as in CURL. + */ + if (!curl_http_proxy) { + if (!strcmp(http_auth.protocol, "https")) { + var_override(&curl_http_proxy, getenv("HTTPS_PROXY")); + var_override(&curl_http_proxy, getenv("https_proxy")); + } else { + var_override(&curl_http_proxy, getenv("http_proxy")); + } + if (!curl_http_proxy) { + var_override(&curl_http_proxy, getenv("ALL_PROXY")); + var_override(&curl_http_proxy, getenv("all_proxy")); + } + } + if (curl_http_proxy) { curl_easy_setopt(result, CURLOPT_PROXY, curl_http_proxy); #if LIBCURL_VERSION_NUM >= 0x071800 - if (starts_with(curl_http_proxy, "socks5")) + if (starts_with(curl_http_proxy, "socks5h")) + curl_easy_setopt(result, + CURLOPT_PROXYTYPE, CURLPROXY_SOCKS5_HOSTNAME); + else if (starts_with(curl_http_proxy, "socks5")) curl_easy_setopt(result, CURLOPT_PROXYTYPE, CURLPROXY_SOCKS5); else if (starts_with(curl_http_proxy, "socks4a")) @@@ -615,23 -479,10 +618,23 @@@ curl_easy_setopt(result, CURLOPT_PROXYTYPE, CURLPROXY_SOCKS4); #endif - } -#if LIBCURL_VERSION_NUM >= 0x070a07 - curl_easy_setopt(result, CURLOPT_PROXYAUTH, CURLAUTH_ANY); + if (strstr(curl_http_proxy, "://")) + credential_from_url(&proxy_auth, curl_http_proxy); + else { + struct strbuf url = STRBUF_INIT; + strbuf_addf(&url, "http://%s", curl_http_proxy); + credential_from_url(&proxy_auth, url.buf); + strbuf_release(&url); + } + + curl_easy_setopt(result, CURLOPT_PROXY, proxy_auth.host); +#if LIBCURL_VERSION_NUM >= 0x071304 + var_override(&curl_no_proxy, getenv("NO_PROXY")); + var_override(&curl_no_proxy, getenv("no_proxy")); + curl_easy_setopt(result, CURLOPT_NOPROXY, curl_no_proxy); #endif + } + init_curl_proxy_auth(result); set_curl_keepalive(result); @@@ -672,9 -523,6 +675,9 @@@ void http_init(struct remote *remote, c if (remote && remote->http_proxy) curl_http_proxy = xstrdup(remote->http_proxy); + if (remote) + var_override(&http_proxy_authmethod, remote->http_proxy_authmethod); + pragma_header = curl_slist_append(pragma_header, "Pragma: no-cache"); no_pragma_header = curl_slist_append(no_pragma_header, "Pragma:"); @@@ -773,18 -621,6 +776,18 @@@ void http_cleanup(void curl_http_proxy = NULL; } + if (proxy_auth.password) { + memset(proxy_auth.password, 0, strlen(proxy_auth.password)); + free(proxy_auth.password); + proxy_auth.password = NULL; + } + + free((void *)curl_proxyuserpwd); + curl_proxyuserpwd = NULL; + + free((void *)http_proxy_authmethod); + http_proxy_authmethod = NULL; + if (cert_auth.password != NULL) { memset(cert_auth.password, 0, strlen(cert_auth.password)); free(cert_auth.password); @@@ -859,15 -695,10 +862,15 @@@ struct active_request_slot *get_active_ curl_easy_setopt(slot->curl, CURLOPT_UPLOAD, 0); curl_easy_setopt(slot->curl, CURLOPT_HTTPGET, 1); curl_easy_setopt(slot->curl, CURLOPT_FAILONERROR, 1); + curl_easy_setopt(slot->curl, CURLOPT_RANGE, NULL); + +#if LIBCURL_VERSION_NUM >= 0x070a08 + curl_easy_setopt(slot->curl, CURLOPT_IPRESOLVE, git_curl_ipresolve); +#endif #ifdef LIBCURL_CAN_HANDLE_AUTH_ANY curl_easy_setopt(slot->curl, CURLOPT_HTTPAUTH, http_auth_methods); #endif - if (http_auth.password) + if (http_auth.password || curl_empty_auth) init_curl_http_auth(slot->curl); return slot; @@@ -1118,8 -949,6 +1121,8 @@@ static int handle_curl_result(struct sl if (results->curl_result == CURLE_OK) { credential_approve(&http_auth); + if (proxy_auth.password) + credential_approve(&proxy_auth); return HTTP_OK; } else if (missing_target(results)) return HTTP_MISSING_TARGET; @@@ -1134,8 -963,6 +1137,8 @@@ return HTTP_REAUTH; } } else { + if (results->http_connectcode == 407) + credential_reject(&proxy_auth); #if LIBCURL_VERSION_NUM >= 0x070c00 if (!curl_errorstr[0]) strlcpy(curl_errorstr, @@@ -1309,7 -1136,7 +1312,7 @@@ static void write_accept_language(struc decimal_places++, max_q *= 10) ; - sprintf(q_format, ";q=0.%%0%dd", decimal_places); + xsnprintf(q_format, sizeof(q_format), ";q=0.%%0%dd", decimal_places); strbuf_addstr(buf, "Accept-Language: "); @@@ -1360,13 -1187,6 +1363,13 @@@ static const char *get_accept_language( return cached_accept_language; } +static void http_opt_request_remainder(CURL *curl, off_t pos) +{ + char buf[128]; + xsnprintf(buf, sizeof(buf), "%"PRIuMAX"-", (uintmax_t)pos); + curl_easy_setopt(curl, CURLOPT_RANGE, buf); +} + /* http_request() targets */ #define HTTP_REQUEST_STRBUF 0 #define HTTP_REQUEST_FILE 1 @@@ -1392,11 -1212,14 +1395,11 @@@ static int http_request(const char *url curl_easy_setopt(slot->curl, CURLOPT_FILE, result); if (target == HTTP_REQUEST_FILE) { - long posn = ftell(result); + off_t posn = ftello(result); curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite); - if (posn > 0) { - strbuf_addf(&buf, "Range: bytes=%ld-", posn); - headers = curl_slist_append(headers, buf.buf); - strbuf_reset(&buf); - } + if (posn > 0) + http_opt_request_remainder(slot->curl, posn); } else curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite_buffer); @@@ -1578,7 -1401,7 +1581,7 @@@ int http_fetch_ref(const char *base, st if (http_get_strbuf(url, &buffer, &options) == HTTP_OK) { strbuf_rtrim(&buffer); if (buffer.len == 40) - ret = get_sha1_hex(buffer.buf, ref->old_sha1); + ret = get_oid_hex(buffer.buf, &ref->old_oid); else if (starts_with(buffer.buf, "ref: ")) { ref->symref = xstrdup(buffer.buf + 5); ret = 0; @@@ -1706,6 -1529,10 +1709,6 @@@ void release_http_pack_request(struct h fclose(preq->packfile); preq->packfile = NULL; } - if (preq->range_header != NULL) { - curl_slist_free_all(preq->range_header); - preq->range_header = NULL; - } preq->slot = NULL; free(preq->url); free(preq); @@@ -1716,7 -1543,6 +1719,7 @@@ int finish_http_pack_request(struct htt struct packed_git **lst; struct packed_git *p = preq->target; char *tmp_idx; + size_t len; struct child_process ip = CHILD_PROCESS_INIT; const char *ip_argv[8]; @@@ -1730,9 -1556,9 +1733,9 @@@ lst = &((*lst)->next); *lst = (*lst)->next; - tmp_idx = xstrdup(preq->tmpfile); - strcpy(tmp_idx + strlen(tmp_idx) - strlen(".pack.temp"), - ".idx.temp"); + if (!strip_suffix(preq->tmpfile, ".pack.temp", &len)) + die("BUG: pack tmpfile does not end in .pack.temp?"); + tmp_idx = xstrfmt("%.*s.idx.temp", (int)len, preq->tmpfile); ip_argv[0] = "index-pack"; ip_argv[1] = "-o"; @@@ -1768,7 -1594,8 +1771,7 @@@ struct http_pack_request *new_http_pack_request( struct packed_git *target, const char *base_url) { - long prev_posn = 0; - char range[RANGE_HEADER_SIZE]; + off_t prev_posn = 0; struct strbuf buf = STRBUF_INIT; struct http_pack_request *preq; @@@ -1800,13 -1627,16 +1803,13 @@@ * If there is data present from a previous transfer attempt, * resume where it left off */ - prev_posn = ftell(preq->packfile); + prev_posn = ftello(preq->packfile); if (prev_posn>0) { if (http_is_verbose) fprintf(stderr, - "Resuming fetch of pack %s at byte %ld\n", - sha1_to_hex(target->sha1), prev_posn); - sprintf(range, "Range: bytes=%ld-", prev_posn); - preq->range_header = curl_slist_append(NULL, range); - curl_easy_setopt(preq->slot->curl, CURLOPT_HTTPHEADER, - preq->range_header); + "Resuming fetch of pack %s at byte %"PRIuMAX"\n", + sha1_to_hex(target->sha1), (uintmax_t)prev_posn); + http_opt_request_remainder(preq->slot->curl, prev_posn); } return preq; @@@ -1855,7 -1685,9 +1858,7 @@@ struct http_object_request *new_http_ob int prevlocal; char prev_buf[PREV_BUF_SIZE]; ssize_t prev_read = 0; - long prev_posn = 0; - char range[RANGE_HEADER_SIZE]; - struct curl_slist *range_header = NULL; + off_t prev_posn = 0; struct http_object_request *freq; freq = xcalloc(1, sizeof(*freq)); @@@ -1959,9 -1791,12 +1962,9 @@@ if (prev_posn>0) { if (http_is_verbose) fprintf(stderr, - "Resuming fetch of object %s at byte %ld\n", - hex, prev_posn); - sprintf(range, "Range: bytes=%ld-", prev_posn); - range_header = curl_slist_append(range_header, range); - curl_easy_setopt(freq->slot->curl, - CURLOPT_HTTPHEADER, range_header); + "Resuming fetch of object %s at byte %"PRIuMAX"\n", + hex, (uintmax_t)prev_posn); + http_opt_request_remainder(freq->slot->curl, prev_posn); } return freq;