Merge branch 'ew/http-walker'
authorJunio C Hamano <gitster@pobox.com>
Wed, 3 Aug 2016 22:10:24 +0000 (15:10 -0700)
committerJunio C Hamano <gitster@pobox.com>
Wed, 3 Aug 2016 22:10:24 +0000 (15:10 -0700)
Dumb http transport on the client side has been optimized.

* ew/http-walker:
list: avoid incompatibility with *BSD sys/queue.h
http-walker: reduce O(n) ops with doubly-linked list
http: avoid disconnecting on 404s for loose objects
http-walker: remove unused parameter from fetch_object

1  2 
http.c
diff --combined http.c
index d8b2bec861b14dfa16dbe99f3ccc44463375b8b3,6f12661a1411a83f2874610e6df6f21f8846e36f..e81dd132d483aa02540bac7d1f55b5e98b1ad736
--- 1/http.c
--- 2/http.c
+++ b/http.c
@@@ -11,7 -11,6 +11,7 @@@
  #include "gettext.h"
  #include "transport.h"
  
 +static struct trace_key trace_curl = TRACE_KEY_INIT(CURL);
  #if LIBCURL_VERSION_NUM >= 0x070a08
  long int git_curl_ipresolve = CURL_IPRESOLVE_WHATEVER;
  #else
@@@ -478,125 -477,6 +478,125 @@@ static void set_curl_keepalive(CURL *c
  }
  #endif
  
 +static void redact_sensitive_header(struct strbuf *header)
 +{
 +      const char *sensitive_header;
 +
 +      if (skip_prefix(header->buf, "Authorization:", &sensitive_header) ||
 +          skip_prefix(header->buf, "Proxy-Authorization:", &sensitive_header)) {
 +              /* The first token is the type, which is OK to log */
 +              while (isspace(*sensitive_header))
 +                      sensitive_header++;
 +              while (*sensitive_header && !isspace(*sensitive_header))
 +                      sensitive_header++;
 +              /* Everything else is opaque and possibly sensitive */
 +              strbuf_setlen(header,  sensitive_header - header->buf);
 +              strbuf_addstr(header, " <redacted>");
 +      }
 +}
 +
 +static void curl_dump_header(const char *text, unsigned char *ptr, size_t size, int hide_sensitive_header)
 +{
 +      struct strbuf out = STRBUF_INIT;
 +      struct strbuf **headers, **header;
 +
 +      strbuf_addf(&out, "%s, %10.10ld bytes (0x%8.8lx)\n",
 +              text, (long)size, (long)size);
 +      trace_strbuf(&trace_curl, &out);
 +      strbuf_reset(&out);
 +      strbuf_add(&out, ptr, size);
 +      headers = strbuf_split_max(&out, '\n', 0);
 +
 +      for (header = headers; *header; header++) {
 +              if (hide_sensitive_header)
 +                      redact_sensitive_header(*header);
 +              strbuf_insert((*header), 0, text, strlen(text));
 +              strbuf_insert((*header), strlen(text), ": ", 2);
 +              strbuf_rtrim((*header));
 +              strbuf_addch((*header), '\n');
 +              trace_strbuf(&trace_curl, (*header));
 +      }
 +      strbuf_list_free(headers);
 +      strbuf_release(&out);
 +}
 +
 +static void curl_dump_data(const char *text, unsigned char *ptr, size_t size)
 +{
 +      size_t i;
 +      struct strbuf out = STRBUF_INIT;
 +      unsigned int width = 60;
 +
 +      strbuf_addf(&out, "%s, %10.10ld bytes (0x%8.8lx)\n",
 +              text, (long)size, (long)size);
 +      trace_strbuf(&trace_curl, &out);
 +
 +      for (i = 0; i < size; i += width) {
 +              size_t w;
 +
 +              strbuf_reset(&out);
 +              strbuf_addf(&out, "%s: ", text);
 +              for (w = 0; (w < width) && (i + w < size); w++) {
 +                      unsigned char ch = ptr[i + w];
 +
 +                      strbuf_addch(&out,
 +                                     (ch >= 0x20) && (ch < 0x80)
 +                                     ? ch : '.');
 +              }
 +              strbuf_addch(&out, '\n');
 +              trace_strbuf(&trace_curl, &out);
 +      }
 +      strbuf_release(&out);
 +}
 +
 +static int curl_trace(CURL *handle, curl_infotype type, char *data, size_t size, void *userp)
 +{
 +      const char *text;
 +      enum { NO_FILTER = 0, DO_FILTER = 1 };
 +
 +      switch (type) {
 +      case CURLINFO_TEXT:
 +              trace_printf_key(&trace_curl, "== Info: %s", data);
 +      default:                /* we ignore unknown types by default */
 +              return 0;
 +
 +      case CURLINFO_HEADER_OUT:
 +              text = "=> Send header";
 +              curl_dump_header(text, (unsigned char *)data, size, DO_FILTER);
 +              break;
 +      case CURLINFO_DATA_OUT:
 +              text = "=> Send data";
 +              curl_dump_data(text, (unsigned char *)data, size);
 +              break;
 +      case CURLINFO_SSL_DATA_OUT:
 +              text = "=> Send SSL data";
 +              curl_dump_data(text, (unsigned char *)data, size);
 +              break;
 +      case CURLINFO_HEADER_IN:
 +              text = "<= Recv header";
 +              curl_dump_header(text, (unsigned char *)data, size, NO_FILTER);
 +              break;
 +      case CURLINFO_DATA_IN:
 +              text = "<= Recv data";
 +              curl_dump_data(text, (unsigned char *)data, size);
 +              break;
 +      case CURLINFO_SSL_DATA_IN:
 +              text = "<= Recv SSL data";
 +              curl_dump_data(text, (unsigned char *)data, size);
 +              break;
 +      }
 +      return 0;
 +}
 +
 +void setup_curl_trace(CURL *handle)
 +{
 +      if (!trace_want(&trace_curl))
 +              return;
 +      curl_easy_setopt(handle, CURLOPT_VERBOSE, 1L);
 +      curl_easy_setopt(handle, CURLOPT_DEBUGFUNCTION, curl_trace);
 +      curl_easy_setopt(handle, CURLOPT_DEBUGDATA, NULL);
 +}
 +
 +
  static CURL *get_curl_handle(void)
  {
        CURL *result = curl_easy_init();
                warning("protocol restrictions not applied to curl redirects because\n"
                        "your curl version is too old (>= 7.19.4)");
  #endif
 -
        if (getenv("GIT_CURL_VERBOSE"))
 -              curl_easy_setopt(result, CURLOPT_VERBOSE, 1);
 +              curl_easy_setopt(result, CURLOPT_VERBOSE, 1L);
 +      setup_curl_trace(result);
  
        curl_easy_setopt(result, CURLOPT_USERAGENT,
                user_agent ? user_agent : git_user_agent());
@@@ -1975,8 -1855,19 +1975,19 @@@ static size_t fwrite_sha1_file(char *pt
        unsigned char expn[4096];
        size_t size = eltsize * nmemb;
        int posn = 0;
-       struct http_object_request *freq =
-               (struct http_object_request *)data;
+       struct http_object_request *freq = data;
+       struct active_request_slot *slot = freq->slot;
+       if (slot) {
+               CURLcode c = curl_easy_getinfo(slot->curl, CURLINFO_HTTP_CODE,
+                                               &slot->http_code);
+               if (c != CURLE_OK)
+                       die("BUG: curl_easy_getinfo for HTTP code failed: %s",
+                               curl_easy_strerror(c));
+               if (slot->http_code >= 400)
+                       return size;
+       }
        do {
                ssize_t retval = xwrite(freq->localfile,
                                        (char *) ptr + posn, size - posn);
@@@ -2097,6 -1988,7 +2108,7 @@@ struct http_object_request *new_http_ob
        freq->slot = get_active_slot();
  
        curl_easy_setopt(freq->slot->curl, CURLOPT_FILE, freq);
+       curl_easy_setopt(freq->slot->curl, CURLOPT_FAILONERROR, 0);
        curl_easy_setopt(freq->slot->curl, CURLOPT_WRITEFUNCTION, fwrite_sha1_file);
        curl_easy_setopt(freq->slot->curl, CURLOPT_ERRORBUFFER, freq->errorstr);
        curl_easy_setopt(freq->slot->curl, CURLOPT_URL, freq->url);