From: Junio C Hamano Date: Mon, 5 Dec 2011 23:10:28 +0000 (-0800) Subject: Merge branch 'mf/curl-select-fdset' X-Git-Tag: v1.7.9-rc0~101 X-Git-Url: https://git.lorimer.id.au/gitweb.git/diff_plain/c4c9a63b5434d008fdbfc8c41240d642cbe5147e?ds=inline;hp=-c Merge branch 'mf/curl-select-fdset' * mf/curl-select-fdset: http: drop "local" member from request struct http.c: Rely on select instead of tracking whether data was received http.c: Use timeout suggested by curl instead of fixed 50ms timeout http.c: Use curl_multi_fdset to select on curl fds instead of just sleeping --- c4c9a63b5434d008fdbfc8c41240d642cbe5147e diff --combined http.c index e6c75976e8,59592db4d4..44fcc4d178 --- a/http.c +++ b/http.c @@@ -4,7 -4,6 +4,6 @@@ #include "run-command.h" #include "url.h" - int data_received; int active_requests; int http_is_verbose; size_t http_post_buffer = 16 * LARGE_PACKET_MAX; @@@ -41,8 -40,7 +40,8 @@@ static long curl_low_speed_limit = -1 static long curl_low_speed_time = -1; static int curl_ftp_no_epsv; static const char *curl_http_proxy; -static char *user_name, *user_pass; +static const char *curl_cookie_file; +static char *user_name, *user_pass, *description; static const char *user_agent; #if LIBCURL_VERSION_NUM >= 0x071700 @@@ -99,13 -97,11 +98,11 @@@ size_t fwrite_buffer(char *ptr, size_t struct strbuf *buffer = buffer_; strbuf_add(buffer, ptr, size); - data_received++; return size; } size_t fwrite_null(char *ptr, size_t eltsize, size_t nmemb, void *strbuf) { - data_received++; return eltsize * nmemb; } @@@ -139,27 -135,6 +136,27 @@@ static void process_curl_messages(void } #endif +static char *git_getpass_with_description(const char *what, const char *desc) +{ + struct strbuf prompt = STRBUF_INIT; + char *r; + + if (desc) + strbuf_addf(&prompt, "%s for '%s': ", what, desc); + else + strbuf_addf(&prompt, "%s: ", what); + /* + * NEEDSWORK: for usernames, we should do something less magical that + * actually echoes the characters. However, we need to read from + * /dev/tty and not stdio, which is not portable (but getpass will do + * it for us). http.c uses the same workaround. + */ + r = git_getpass(prompt.buf); + + strbuf_release(&prompt); + return xstrdup(r); +} + static int http_options(const char *var, const char *value, void *cb) { if (!strcmp("http.sslverify", var)) { @@@ -213,9 -188,6 +210,9 @@@ if (!strcmp("http.proxy", var)) return git_config_string(&curl_http_proxy, var, value); + if (!strcmp("http.cookiefile", var)) + return git_config_string(&curl_cookie_file, var, value); + if (!strcmp("http.postbuffer", var)) { http_post_buffer = git_config_int(var, value); if (http_post_buffer < LARGE_PACKET_MAX) @@@ -235,7 -207,7 +232,7 @@@ static void init_curl_http_auth(CURL *r if (user_name) { struct strbuf up = STRBUF_INIT; if (!user_pass) - user_pass = xstrdup(git_getpass("Password: ")); + user_pass = xstrdup(git_getpass_with_description("Password", description)); strbuf_addf(&up, "%s:%s", user_name, user_pass); curl_easy_setopt(result, CURLOPT_USERPWD, strbuf_detach(&up, NULL)); @@@ -250,7 -222,7 +247,7 @@@ static int has_cert_password(void return 0; /* Only prompt the user once. */ ssl_cert_password_required = -1; - ssl_cert_password = git_getpass("Certificate Password: "); + ssl_cert_password = git_getpass_with_description("Certificate Password", description); if (ssl_cert_password != NULL) { ssl_cert_password = xstrdup(ssl_cert_password); return 1; @@@ -279,6 -251,8 +276,6 @@@ static CURL *get_curl_handle(void curl_easy_setopt(result, CURLOPT_HTTPAUTH, CURLAUTH_ANY); #endif - init_curl_http_auth(result); - if (ssl_cert != NULL) curl_easy_setopt(result, CURLOPT_SSLCERT, ssl_cert); if (has_cert_password()) @@@ -326,7 -300,8 +323,7 @@@ static void http_auth_init(const char *url) { - char *at, *colon, *cp, *slash, *decoded; - int len; + const char *at, *colon, *cp, *slash, *host; cp = strstr(url, "://"); if (!cp) @@@ -342,22 -317,34 +339,22 @@@ at = strchr(cp, '@'); colon = strchr(cp, ':'); slash = strchrnul(cp, '/'); - if (!at || slash <= at) - return; /* No credentials */ - if (!colon || at <= colon) { + if (!at || slash <= at) { + /* No credentials, but we may have to ask for some later */ + host = cp; + } + else if (!colon || at <= colon) { /* Only username */ - len = at - cp; - user_name = xmalloc(len + 1); - memcpy(user_name, cp, len); - user_name[len] = '\0'; - decoded = url_decode(user_name); - free(user_name); - user_name = decoded; + user_name = url_decode_mem(cp, at - cp); user_pass = NULL; + host = at + 1; } else { - len = colon - cp; - user_name = xmalloc(len + 1); - memcpy(user_name, cp, len); - user_name[len] = '\0'; - decoded = url_decode(user_name); - free(user_name); - user_name = decoded; - len = at - (colon + 1); - user_pass = xmalloc(len + 1); - memcpy(user_pass, colon + 1, len); - user_pass[len] = '\0'; - decoded = url_decode(user_pass); - free(user_pass); - user_pass = decoded; + user_name = url_decode_mem(cp, colon - cp); + user_pass = url_decode_mem(colon + 1, at - (colon + 1)); + host = at + 1; } + + description = url_decode_mem(host, slash - host); } static void set_from_env(const char **var, const char *envname) @@@ -367,7 -354,7 +364,7 @@@ *var = val; } -void http_init(struct remote *remote) +void http_init(struct remote *remote, const char *url) { char *low_speed_limit; char *low_speed_time; @@@ -431,11 -418,11 +428,11 @@@ if (getenv("GIT_CURL_FTP_NO_EPSV")) curl_ftp_no_epsv = 1; - if (remote && remote->url && remote->url[0]) { - http_auth_init(remote->url[0]); + if (url) { + http_auth_init(url); if (!ssl_cert_password_required && getenv("GIT_SSL_CERT_PASSWORD_PROTECTED") && - !prefixcmp(remote->url[0], "https://")) + !prefixcmp(url, "https://")) ssl_cert_password_required = 1; } @@@ -536,12 -523,10 +533,11 @@@ struct active_request_slot *get_active_ active_requests++; slot->in_use = 1; - slot->local = NULL; slot->results = NULL; slot->finished = NULL; slot->callback_data = NULL; slot->callback_func = NULL; + curl_easy_setopt(slot->curl, CURLOPT_COOKIEFILE, curl_cookie_file); curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, pragma_header); curl_easy_setopt(slot->curl, CURLOPT_ERRORBUFFER, curl_errorstr); curl_easy_setopt(slot->curl, CURLOPT_CUSTOMREQUEST, NULL); @@@ -640,8 -625,6 +636,6 @@@ void step_active_slots(void void run_active_slot(struct active_request_slot *slot) { #ifdef USE_CURL_MULTI - long last_pos = 0; - long current_pos; fd_set readfds; fd_set writefds; fd_set excfds; @@@ -651,25 -634,33 +645,33 @@@ slot->finished = &finished; while (!finished) { - data_received = 0; step_active_slots(); - if (!data_received && slot->local != NULL) { - current_pos = ftell(slot->local); - if (current_pos > last_pos) - data_received++; - last_pos = current_pos; - } + if (slot->in_use) { + #if LIBCURL_VERSION_NUM >= 0x070f04 + long curl_timeout; + curl_multi_timeout(curlm, &curl_timeout); + if (curl_timeout == 0) { + continue; + } else if (curl_timeout == -1) { + select_timeout.tv_sec = 0; + select_timeout.tv_usec = 50000; + } else { + select_timeout.tv_sec = curl_timeout / 1000; + select_timeout.tv_usec = (curl_timeout % 1000) * 1000; + } + #else + select_timeout.tv_sec = 0; + select_timeout.tv_usec = 50000; + #endif - if (slot->in_use && !data_received) { - max_fd = 0; + max_fd = -1; FD_ZERO(&readfds); FD_ZERO(&writefds); FD_ZERO(&excfds); - select_timeout.tv_sec = 0; - select_timeout.tv_usec = 50000; - select(max_fd, &readfds, &writefds, - &excfds, &select_timeout); + curl_multi_fdset(curlm, &readfds, &writefds, &excfds, &max_fd); + + select(max_fd+1, &readfds, &writefds, &excfds, &select_timeout); } } #else @@@ -747,6 -738,14 +749,6 @@@ static inline int needs_quote(int ch return 1; } -static inline int hex(int v) -{ - if (v < 10) - return '0' + v; - else - return 'A' + v - 10; -} - static char *quote_ref_url(const char *base, const char *ref) { struct strbuf buf = STRBUF_INIT; @@@ -814,7 -813,6 +816,6 @@@ static int http_request(const char *url headers = curl_slist_append(headers, buf.buf); strbuf_reset(&buf); } - slot->local = result; } else curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite_buffer); @@@ -836,7 -834,7 +837,7 @@@ else if (missing_target(&results)) ret = HTTP_MISSING_TARGET; else if (results.http_code == 401) { - if (user_name) { + if (user_name && user_pass) { ret = HTTP_NOAUTH; } else { /* @@@ -845,42 -843,30 +846,41 @@@ * but that is non-portable. Using git_getpass() can at least be stubbed * on other platforms with a different implementation if/when necessary. */ - user_name = xstrdup(git_getpass("Username: ")); + if (!user_name) + user_name = xstrdup(git_getpass_with_description("Username", description)); init_curl_http_auth(slot->curl); ret = HTTP_REAUTH; } - } else + } else { + if (!curl_errorstr[0]) + strlcpy(curl_errorstr, + curl_easy_strerror(results.curl_result), + sizeof(curl_errorstr)); ret = HTTP_ERROR; + } } else { error("Unable to start HTTP request for %s", url); ret = HTTP_START_FAILED; } - slot->local = NULL; curl_slist_free_all(headers); strbuf_release(&buf); return ret; } +static int http_request_reauth(const char *url, void *result, int target, + int options) +{ + int ret = http_request(url, result, target, options); + if (ret != HTTP_REAUTH) + return ret; + return http_request(url, result, target, options); +} + int http_get_strbuf(const char *url, struct strbuf *result, int options) { - int http_ret = http_request(url, result, HTTP_REQUEST_STRBUF, options); - if (http_ret == HTTP_REAUTH) { - http_ret = http_request(url, result, HTTP_REQUEST_STRBUF, options); - } - return http_ret; + return http_request_reauth(url, result, HTTP_REQUEST_STRBUF, options); } /* @@@ -903,7 -889,7 +903,7 @@@ static int http_get_file(const char *ur goto cleanup; } - ret = http_request(url, result, HTTP_REQUEST_FILE, options); + ret = http_request_reauth(url, result, HTTP_REQUEST_FILE, options); fclose(result); if ((ret == HTTP_OK) && move_temp_to_file(tmpfile.buf, filename)) @@@ -917,7 -903,7 +917,7 @@@ int http_error(const char *url, int ret { /* http_request has already handled HTTP_START_FAILED. */ if (ret != HTTP_START_FAILED) - error("%s while accessing %s\n", curl_errorstr, url); + error("%s while accessing %s", curl_errorstr, url); return ret; } @@@ -1057,7 -1043,6 +1057,6 @@@ void release_http_pack_request(struct h if (preq->packfile != NULL) { fclose(preq->packfile); preq->packfile = NULL; - preq->slot->local = NULL; } if (preq->range_header != NULL) { curl_slist_free_all(preq->range_header); @@@ -1079,7 -1064,6 +1078,6 @@@ int finish_http_pack_request(struct htt fclose(preq->packfile); preq->packfile = NULL; - preq->slot->local = NULL; lst = preq->lst; while (*lst != p) @@@ -1130,8 -1114,9 +1128,8 @@@ struct http_pack_request *new_http_pack struct strbuf buf = STRBUF_INIT; struct http_pack_request *preq; - preq = xmalloc(sizeof(*preq)); + preq = xcalloc(1, sizeof(*preq)); preq->target = target; - preq->range_header = NULL; end_url_with_slash(&buf, base_url); strbuf_addf(&buf, "objects/pack/pack-%s.pack", @@@ -1148,7 -1133,6 +1146,6 @@@ } preq->slot = get_active_slot(); - preq->slot->local = preq->packfile; curl_easy_setopt(preq->slot->curl, CURLOPT_FILE, preq->packfile); curl_easy_setopt(preq->slot->curl, CURLOPT_WRITEFUNCTION, fwrite); curl_easy_setopt(preq->slot->curl, CURLOPT_URL, preq->url); @@@ -1205,7 -1189,6 +1202,6 @@@ static size_t fwrite_sha1_file(char *pt git_SHA1_Update(&freq->c, expn, sizeof(expn) - freq->stream.avail_out); } while (freq->stream.avail_in && freq->zret == Z_OK); - data_received++; return size; } @@@ -1223,7 -1206,7 +1219,7 @@@ struct http_object_request *new_http_ob struct curl_slist *range_header = NULL; struct http_object_request *freq; - freq = xmalloc(sizeof(*freq)); + freq = xcalloc(1, sizeof(*freq)); hashcpy(freq->sha1, sha1); freq->localfile = -1; @@@ -1261,6 -1244,8 +1257,6 @@@ goto abort; } - memset(&freq->stream, 0, sizeof(freq->stream)); - git_inflate_init(&freq->stream); git_SHA1_Init(&freq->c); @@@ -1335,6 -1320,7 +1331,6 @@@ return freq; abort: - free(filename); free(freq->url); free(freq); return NULL; diff --combined http.h index 3c332a98e9,368bc0e80a..ee1606942a --- a/http.h +++ b/http.h @@@ -49,7 -49,6 +49,6 @@@ struct slot_results struct active_request_slot { CURL *curl; - FILE *local; int in_use; CURLcode curl_result; long http_code; @@@ -86,10 -85,9 +85,9 @@@ extern void add_fill_function(void *dat extern void step_active_slots(void); #endif -extern void http_init(struct remote *remote); +extern void http_init(struct remote *remote, const char *url); extern void http_cleanup(void); - extern int data_received; extern int active_requests; extern int http_is_verbose; extern size_t http_post_buffer; @@@ -172,7 -170,7 +170,7 @@@ struct http_object_request unsigned char sha1[20]; unsigned char real_sha1[20]; git_SHA_CTX c; - z_stream stream; + git_zstream stream; int zret; int rename; struct active_request_slot *slot;