http.con commit imap-send: return with error if curl failed (200bc38)
   1#include "git-compat-util.h"
   2#include "http.h"
   3#include "config.h"
   4#include "pack.h"
   5#include "sideband.h"
   6#include "run-command.h"
   7#include "url.h"
   8#include "urlmatch.h"
   9#include "credential.h"
  10#include "version.h"
  11#include "pkt-line.h"
  12#include "gettext.h"
  13#include "transport.h"
  14#include "packfile.h"
  15
  16static struct trace_key trace_curl = TRACE_KEY_INIT(CURL);
  17#if LIBCURL_VERSION_NUM >= 0x070a08
  18long int git_curl_ipresolve = CURL_IPRESOLVE_WHATEVER;
  19#else
  20long int git_curl_ipresolve;
  21#endif
  22int active_requests;
  23int http_is_verbose;
  24ssize_t http_post_buffer = 16 * LARGE_PACKET_MAX;
  25
  26#if LIBCURL_VERSION_NUM >= 0x070a06
  27#define LIBCURL_CAN_HANDLE_AUTH_ANY
  28#endif
  29
  30static int min_curl_sessions = 1;
  31static int curl_session_count;
  32#ifdef USE_CURL_MULTI
  33static int max_requests = -1;
  34static CURLM *curlm;
  35#endif
  36#ifndef NO_CURL_EASY_DUPHANDLE
  37static CURL *curl_default;
  38#endif
  39
  40#define PREV_BUF_SIZE 4096
  41
  42char curl_errorstr[CURL_ERROR_SIZE];
  43
  44static int curl_ssl_verify = -1;
  45static int curl_ssl_try;
  46static const char *ssl_cert;
  47static const char *ssl_cipherlist;
  48static const char *ssl_version;
  49static struct {
  50        const char *name;
  51        long ssl_version;
  52} sslversions[] = {
  53        { "sslv2", CURL_SSLVERSION_SSLv2 },
  54        { "sslv3", CURL_SSLVERSION_SSLv3 },
  55        { "tlsv1", CURL_SSLVERSION_TLSv1 },
  56#if LIBCURL_VERSION_NUM >= 0x072200
  57        { "tlsv1.0", CURL_SSLVERSION_TLSv1_0 },
  58        { "tlsv1.1", CURL_SSLVERSION_TLSv1_1 },
  59        { "tlsv1.2", CURL_SSLVERSION_TLSv1_2 },
  60#endif
  61};
  62#if LIBCURL_VERSION_NUM >= 0x070903
  63static const char *ssl_key;
  64#endif
  65#if LIBCURL_VERSION_NUM >= 0x070908
  66static const char *ssl_capath;
  67#endif
  68#if LIBCURL_VERSION_NUM >= 0x072c00
  69static const char *ssl_pinnedkey;
  70#endif
  71static const char *ssl_cainfo;
  72static long curl_low_speed_limit = -1;
  73static long curl_low_speed_time = -1;
  74static int curl_ftp_no_epsv;
  75static const char *curl_http_proxy;
  76static const char *curl_no_proxy;
  77static const char *http_proxy_authmethod;
  78static struct {
  79        const char *name;
  80        long curlauth_param;
  81} proxy_authmethods[] = {
  82        { "basic", CURLAUTH_BASIC },
  83        { "digest", CURLAUTH_DIGEST },
  84        { "negotiate", CURLAUTH_GSSNEGOTIATE },
  85        { "ntlm", CURLAUTH_NTLM },
  86#ifdef LIBCURL_CAN_HANDLE_AUTH_ANY
  87        { "anyauth", CURLAUTH_ANY },
  88#endif
  89        /*
  90         * CURLAUTH_DIGEST_IE has no corresponding command-line option in
  91         * curl(1) and is not included in CURLAUTH_ANY, so we leave it out
  92         * here, too
  93         */
  94};
  95#ifdef CURLGSSAPI_DELEGATION_FLAG
  96static const char *curl_deleg;
  97static struct {
  98        const char *name;
  99        long curl_deleg_param;
 100} curl_deleg_levels[] = {
 101        { "none", CURLGSSAPI_DELEGATION_NONE },
 102        { "policy", CURLGSSAPI_DELEGATION_POLICY_FLAG },
 103        { "always", CURLGSSAPI_DELEGATION_FLAG },
 104};
 105#endif
 106
 107static struct credential proxy_auth = CREDENTIAL_INIT;
 108static const char *curl_proxyuserpwd;
 109static const char *curl_cookie_file;
 110static int curl_save_cookies;
 111struct credential http_auth = CREDENTIAL_INIT;
 112static int http_proactive_auth;
 113static const char *user_agent;
 114static int curl_empty_auth = -1;
 115
 116enum http_follow_config http_follow_config = HTTP_FOLLOW_INITIAL;
 117
 118#if LIBCURL_VERSION_NUM >= 0x071700
 119/* Use CURLOPT_KEYPASSWD as is */
 120#elif LIBCURL_VERSION_NUM >= 0x070903
 121#define CURLOPT_KEYPASSWD CURLOPT_SSLKEYPASSWD
 122#else
 123#define CURLOPT_KEYPASSWD CURLOPT_SSLCERTPASSWD
 124#endif
 125
 126static struct credential cert_auth = CREDENTIAL_INIT;
 127static int ssl_cert_password_required;
 128#ifdef LIBCURL_CAN_HANDLE_AUTH_ANY
 129static unsigned long http_auth_methods = CURLAUTH_ANY;
 130static int http_auth_methods_restricted;
 131/* Modes for which empty_auth cannot actually help us. */
 132static unsigned long empty_auth_useless =
 133        CURLAUTH_BASIC
 134#ifdef CURLAUTH_DIGEST_IE
 135        | CURLAUTH_DIGEST_IE
 136#endif
 137        | CURLAUTH_DIGEST;
 138#endif
 139
 140static struct curl_slist *pragma_header;
 141static struct curl_slist *no_pragma_header;
 142static struct curl_slist *extra_http_headers;
 143
 144static struct active_request_slot *active_queue_head;
 145
 146static char *cached_accept_language;
 147
 148size_t fread_buffer(char *ptr, size_t eltsize, size_t nmemb, void *buffer_)
 149{
 150        size_t size = eltsize * nmemb;
 151        struct buffer *buffer = buffer_;
 152
 153        if (size > buffer->buf.len - buffer->posn)
 154                size = buffer->buf.len - buffer->posn;
 155        memcpy(ptr, buffer->buf.buf + buffer->posn, size);
 156        buffer->posn += size;
 157
 158        return size;
 159}
 160
 161#ifndef NO_CURL_IOCTL
 162curlioerr ioctl_buffer(CURL *handle, int cmd, void *clientp)
 163{
 164        struct buffer *buffer = clientp;
 165
 166        switch (cmd) {
 167        case CURLIOCMD_NOP:
 168                return CURLIOE_OK;
 169
 170        case CURLIOCMD_RESTARTREAD:
 171                buffer->posn = 0;
 172                return CURLIOE_OK;
 173
 174        default:
 175                return CURLIOE_UNKNOWNCMD;
 176        }
 177}
 178#endif
 179
 180size_t fwrite_buffer(char *ptr, size_t eltsize, size_t nmemb, void *buffer_)
 181{
 182        size_t size = eltsize * nmemb;
 183        struct strbuf *buffer = buffer_;
 184
 185        strbuf_add(buffer, ptr, size);
 186        return size;
 187}
 188
 189size_t fwrite_null(char *ptr, size_t eltsize, size_t nmemb, void *strbuf)
 190{
 191        return eltsize * nmemb;
 192}
 193
 194static void closedown_active_slot(struct active_request_slot *slot)
 195{
 196        active_requests--;
 197        slot->in_use = 0;
 198}
 199
 200static void finish_active_slot(struct active_request_slot *slot)
 201{
 202        closedown_active_slot(slot);
 203        curl_easy_getinfo(slot->curl, CURLINFO_HTTP_CODE, &slot->http_code);
 204
 205        if (slot->finished != NULL)
 206                (*slot->finished) = 1;
 207
 208        /* Store slot results so they can be read after the slot is reused */
 209        if (slot->results != NULL) {
 210                slot->results->curl_result = slot->curl_result;
 211                slot->results->http_code = slot->http_code;
 212#if LIBCURL_VERSION_NUM >= 0x070a08
 213                curl_easy_getinfo(slot->curl, CURLINFO_HTTPAUTH_AVAIL,
 214                                  &slot->results->auth_avail);
 215#else
 216                slot->results->auth_avail = 0;
 217#endif
 218
 219                curl_easy_getinfo(slot->curl, CURLINFO_HTTP_CONNECTCODE,
 220                        &slot->results->http_connectcode);
 221        }
 222
 223        /* Run callback if appropriate */
 224        if (slot->callback_func != NULL)
 225                slot->callback_func(slot->callback_data);
 226}
 227
 228static void xmulti_remove_handle(struct active_request_slot *slot)
 229{
 230#ifdef USE_CURL_MULTI
 231        curl_multi_remove_handle(curlm, slot->curl);
 232#endif
 233}
 234
 235#ifdef USE_CURL_MULTI
 236static void process_curl_messages(void)
 237{
 238        int num_messages;
 239        struct active_request_slot *slot;
 240        CURLMsg *curl_message = curl_multi_info_read(curlm, &num_messages);
 241
 242        while (curl_message != NULL) {
 243                if (curl_message->msg == CURLMSG_DONE) {
 244                        int curl_result = curl_message->data.result;
 245                        slot = active_queue_head;
 246                        while (slot != NULL &&
 247                               slot->curl != curl_message->easy_handle)
 248                                slot = slot->next;
 249                        if (slot != NULL) {
 250                                xmulti_remove_handle(slot);
 251                                slot->curl_result = curl_result;
 252                                finish_active_slot(slot);
 253                        } else {
 254                                fprintf(stderr, "Received DONE message for unknown request!\n");
 255                        }
 256                } else {
 257                        fprintf(stderr, "Unknown CURL message received: %d\n",
 258                                (int)curl_message->msg);
 259                }
 260                curl_message = curl_multi_info_read(curlm, &num_messages);
 261        }
 262}
 263#endif
 264
 265static int http_options(const char *var, const char *value, void *cb)
 266{
 267        if (!strcmp("http.sslverify", var)) {
 268                curl_ssl_verify = git_config_bool(var, value);
 269                return 0;
 270        }
 271        if (!strcmp("http.sslcipherlist", var))
 272                return git_config_string(&ssl_cipherlist, var, value);
 273        if (!strcmp("http.sslversion", var))
 274                return git_config_string(&ssl_version, var, value);
 275        if (!strcmp("http.sslcert", var))
 276                return git_config_pathname(&ssl_cert, var, value);
 277#if LIBCURL_VERSION_NUM >= 0x070903
 278        if (!strcmp("http.sslkey", var))
 279                return git_config_pathname(&ssl_key, var, value);
 280#endif
 281#if LIBCURL_VERSION_NUM >= 0x070908
 282        if (!strcmp("http.sslcapath", var))
 283                return git_config_pathname(&ssl_capath, var, value);
 284#endif
 285        if (!strcmp("http.sslcainfo", var))
 286                return git_config_pathname(&ssl_cainfo, var, value);
 287        if (!strcmp("http.sslcertpasswordprotected", var)) {
 288                ssl_cert_password_required = git_config_bool(var, value);
 289                return 0;
 290        }
 291        if (!strcmp("http.ssltry", var)) {
 292                curl_ssl_try = git_config_bool(var, value);
 293                return 0;
 294        }
 295        if (!strcmp("http.minsessions", var)) {
 296                min_curl_sessions = git_config_int(var, value);
 297#ifndef USE_CURL_MULTI
 298                if (min_curl_sessions > 1)
 299                        min_curl_sessions = 1;
 300#endif
 301                return 0;
 302        }
 303#ifdef USE_CURL_MULTI
 304        if (!strcmp("http.maxrequests", var)) {
 305                max_requests = git_config_int(var, value);
 306                return 0;
 307        }
 308#endif
 309        if (!strcmp("http.lowspeedlimit", var)) {
 310                curl_low_speed_limit = (long)git_config_int(var, value);
 311                return 0;
 312        }
 313        if (!strcmp("http.lowspeedtime", var)) {
 314                curl_low_speed_time = (long)git_config_int(var, value);
 315                return 0;
 316        }
 317
 318        if (!strcmp("http.noepsv", var)) {
 319                curl_ftp_no_epsv = git_config_bool(var, value);
 320                return 0;
 321        }
 322        if (!strcmp("http.proxy", var))
 323                return git_config_string(&curl_http_proxy, var, value);
 324
 325        if (!strcmp("http.proxyauthmethod", var))
 326                return git_config_string(&http_proxy_authmethod, var, value);
 327
 328        if (!strcmp("http.cookiefile", var))
 329                return git_config_pathname(&curl_cookie_file, var, value);
 330        if (!strcmp("http.savecookies", var)) {
 331                curl_save_cookies = git_config_bool(var, value);
 332                return 0;
 333        }
 334
 335        if (!strcmp("http.postbuffer", var)) {
 336                http_post_buffer = git_config_ssize_t(var, value);
 337                if (http_post_buffer < 0)
 338                        warning(_("negative value for http.postbuffer; defaulting to %d"), LARGE_PACKET_MAX);
 339                if (http_post_buffer < LARGE_PACKET_MAX)
 340                        http_post_buffer = LARGE_PACKET_MAX;
 341                return 0;
 342        }
 343
 344        if (!strcmp("http.useragent", var))
 345                return git_config_string(&user_agent, var, value);
 346
 347        if (!strcmp("http.emptyauth", var)) {
 348                if (value && !strcmp("auto", value))
 349                        curl_empty_auth = -1;
 350                else
 351                        curl_empty_auth = git_config_bool(var, value);
 352                return 0;
 353        }
 354
 355        if (!strcmp("http.delegation", var)) {
 356#ifdef CURLGSSAPI_DELEGATION_FLAG
 357                return git_config_string(&curl_deleg, var, value);
 358#else
 359                warning(_("Delegation control is not supported with cURL < 7.22.0"));
 360                return 0;
 361#endif
 362        }
 363
 364        if (!strcmp("http.pinnedpubkey", var)) {
 365#if LIBCURL_VERSION_NUM >= 0x072c00
 366                return git_config_pathname(&ssl_pinnedkey, var, value);
 367#else
 368                warning(_("Public key pinning not supported with cURL < 7.44.0"));
 369                return 0;
 370#endif
 371        }
 372
 373        if (!strcmp("http.extraheader", var)) {
 374                if (!value) {
 375                        return config_error_nonbool(var);
 376                } else if (!*value) {
 377                        curl_slist_free_all(extra_http_headers);
 378                        extra_http_headers = NULL;
 379                } else {
 380                        extra_http_headers =
 381                                curl_slist_append(extra_http_headers, value);
 382                }
 383                return 0;
 384        }
 385
 386        if (!strcmp("http.followredirects", var)) {
 387                if (value && !strcmp(value, "initial"))
 388                        http_follow_config = HTTP_FOLLOW_INITIAL;
 389                else if (git_config_bool(var, value))
 390                        http_follow_config = HTTP_FOLLOW_ALWAYS;
 391                else
 392                        http_follow_config = HTTP_FOLLOW_NONE;
 393                return 0;
 394        }
 395
 396        /* Fall back on the default ones */
 397        return git_default_config(var, value, cb);
 398}
 399
 400static int curl_empty_auth_enabled(void)
 401{
 402        if (curl_empty_auth >= 0)
 403                return curl_empty_auth;
 404
 405#ifndef LIBCURL_CAN_HANDLE_AUTH_ANY
 406        /*
 407         * Our libcurl is too old to do AUTH_ANY in the first place;
 408         * just default to turning the feature off.
 409         */
 410#else
 411        /*
 412         * In the automatic case, kick in the empty-auth
 413         * hack as long as we would potentially try some
 414         * method more exotic than "Basic" or "Digest".
 415         *
 416         * But only do this when this is our second or
 417         * subsequent request, as by then we know what
 418         * methods are available.
 419         */
 420        if (http_auth_methods_restricted &&
 421            (http_auth_methods & ~empty_auth_useless))
 422                return 1;
 423#endif
 424        return 0;
 425}
 426
 427static void init_curl_http_auth(CURL *result)
 428{
 429        if (!http_auth.username || !*http_auth.username) {
 430                if (curl_empty_auth_enabled())
 431                        curl_easy_setopt(result, CURLOPT_USERPWD, ":");
 432                return;
 433        }
 434
 435        credential_fill(&http_auth);
 436
 437#if LIBCURL_VERSION_NUM >= 0x071301
 438        curl_easy_setopt(result, CURLOPT_USERNAME, http_auth.username);
 439        curl_easy_setopt(result, CURLOPT_PASSWORD, http_auth.password);
 440#else
 441        {
 442                static struct strbuf up = STRBUF_INIT;
 443                /*
 444                 * Note that we assume we only ever have a single set of
 445                 * credentials in a given program run, so we do not have
 446                 * to worry about updating this buffer, only setting its
 447                 * initial value.
 448                 */
 449                if (!up.len)
 450                        strbuf_addf(&up, "%s:%s",
 451                                http_auth.username, http_auth.password);
 452                curl_easy_setopt(result, CURLOPT_USERPWD, up.buf);
 453        }
 454#endif
 455}
 456
 457/* *var must be free-able */
 458static void var_override(const char **var, char *value)
 459{
 460        if (value) {
 461                free((void *)*var);
 462                *var = xstrdup(value);
 463        }
 464}
 465
 466static void set_proxyauth_name_password(CURL *result)
 467{
 468#if LIBCURL_VERSION_NUM >= 0x071301
 469                curl_easy_setopt(result, CURLOPT_PROXYUSERNAME,
 470                        proxy_auth.username);
 471                curl_easy_setopt(result, CURLOPT_PROXYPASSWORD,
 472                        proxy_auth.password);
 473#else
 474                struct strbuf s = STRBUF_INIT;
 475
 476                strbuf_addstr_urlencode(&s, proxy_auth.username, 1);
 477                strbuf_addch(&s, ':');
 478                strbuf_addstr_urlencode(&s, proxy_auth.password, 1);
 479                curl_proxyuserpwd = strbuf_detach(&s, NULL);
 480                curl_easy_setopt(result, CURLOPT_PROXYUSERPWD, curl_proxyuserpwd);
 481#endif
 482}
 483
 484static void init_curl_proxy_auth(CURL *result)
 485{
 486        if (proxy_auth.username) {
 487                if (!proxy_auth.password)
 488                        credential_fill(&proxy_auth);
 489                set_proxyauth_name_password(result);
 490        }
 491
 492        var_override(&http_proxy_authmethod, getenv("GIT_HTTP_PROXY_AUTHMETHOD"));
 493
 494#if LIBCURL_VERSION_NUM >= 0x070a07 /* CURLOPT_PROXYAUTH and CURLAUTH_ANY */
 495        if (http_proxy_authmethod) {
 496                int i;
 497                for (i = 0; i < ARRAY_SIZE(proxy_authmethods); i++) {
 498                        if (!strcmp(http_proxy_authmethod, proxy_authmethods[i].name)) {
 499                                curl_easy_setopt(result, CURLOPT_PROXYAUTH,
 500                                                proxy_authmethods[i].curlauth_param);
 501                                break;
 502                        }
 503                }
 504                if (i == ARRAY_SIZE(proxy_authmethods)) {
 505                        warning("unsupported proxy authentication method %s: using anyauth",
 506                                        http_proxy_authmethod);
 507                        curl_easy_setopt(result, CURLOPT_PROXYAUTH, CURLAUTH_ANY);
 508                }
 509        }
 510        else
 511                curl_easy_setopt(result, CURLOPT_PROXYAUTH, CURLAUTH_ANY);
 512#endif
 513}
 514
 515static int has_cert_password(void)
 516{
 517        if (ssl_cert == NULL || ssl_cert_password_required != 1)
 518                return 0;
 519        if (!cert_auth.password) {
 520                cert_auth.protocol = xstrdup("cert");
 521                cert_auth.username = xstrdup("");
 522                cert_auth.path = xstrdup(ssl_cert);
 523                credential_fill(&cert_auth);
 524        }
 525        return 1;
 526}
 527
 528#if LIBCURL_VERSION_NUM >= 0x071900
 529static void set_curl_keepalive(CURL *c)
 530{
 531        curl_easy_setopt(c, CURLOPT_TCP_KEEPALIVE, 1);
 532}
 533
 534#elif LIBCURL_VERSION_NUM >= 0x071000
 535static int sockopt_callback(void *client, curl_socket_t fd, curlsocktype type)
 536{
 537        int ka = 1;
 538        int rc;
 539        socklen_t len = (socklen_t)sizeof(ka);
 540
 541        if (type != CURLSOCKTYPE_IPCXN)
 542                return 0;
 543
 544        rc = setsockopt(fd, SOL_SOCKET, SO_KEEPALIVE, (void *)&ka, len);
 545        if (rc < 0)
 546                warning_errno("unable to set SO_KEEPALIVE on socket");
 547
 548        return 0; /* CURL_SOCKOPT_OK only exists since curl 7.21.5 */
 549}
 550
 551static void set_curl_keepalive(CURL *c)
 552{
 553        curl_easy_setopt(c, CURLOPT_SOCKOPTFUNCTION, sockopt_callback);
 554}
 555
 556#else
 557static void set_curl_keepalive(CURL *c)
 558{
 559        /* not supported on older curl versions */
 560}
 561#endif
 562
 563static void redact_sensitive_header(struct strbuf *header)
 564{
 565        const char *sensitive_header;
 566
 567        if (skip_prefix(header->buf, "Authorization:", &sensitive_header) ||
 568            skip_prefix(header->buf, "Proxy-Authorization:", &sensitive_header)) {
 569                /* The first token is the type, which is OK to log */
 570                while (isspace(*sensitive_header))
 571                        sensitive_header++;
 572                while (*sensitive_header && !isspace(*sensitive_header))
 573                        sensitive_header++;
 574                /* Everything else is opaque and possibly sensitive */
 575                strbuf_setlen(header,  sensitive_header - header->buf);
 576                strbuf_addstr(header, " <redacted>");
 577        }
 578}
 579
 580static void curl_dump_header(const char *text, unsigned char *ptr, size_t size, int hide_sensitive_header)
 581{
 582        struct strbuf out = STRBUF_INIT;
 583        struct strbuf **headers, **header;
 584
 585        strbuf_addf(&out, "%s, %10.10ld bytes (0x%8.8lx)\n",
 586                text, (long)size, (long)size);
 587        trace_strbuf(&trace_curl, &out);
 588        strbuf_reset(&out);
 589        strbuf_add(&out, ptr, size);
 590        headers = strbuf_split_max(&out, '\n', 0);
 591
 592        for (header = headers; *header; header++) {
 593                if (hide_sensitive_header)
 594                        redact_sensitive_header(*header);
 595                strbuf_insert((*header), 0, text, strlen(text));
 596                strbuf_insert((*header), strlen(text), ": ", 2);
 597                strbuf_rtrim((*header));
 598                strbuf_addch((*header), '\n');
 599                trace_strbuf(&trace_curl, (*header));
 600        }
 601        strbuf_list_free(headers);
 602        strbuf_release(&out);
 603}
 604
 605static void curl_dump_data(const char *text, unsigned char *ptr, size_t size)
 606{
 607        size_t i;
 608        struct strbuf out = STRBUF_INIT;
 609        unsigned int width = 60;
 610
 611        strbuf_addf(&out, "%s, %10.10ld bytes (0x%8.8lx)\n",
 612                text, (long)size, (long)size);
 613        trace_strbuf(&trace_curl, &out);
 614
 615        for (i = 0; i < size; i += width) {
 616                size_t w;
 617
 618                strbuf_reset(&out);
 619                strbuf_addf(&out, "%s: ", text);
 620                for (w = 0; (w < width) && (i + w < size); w++) {
 621                        unsigned char ch = ptr[i + w];
 622
 623                        strbuf_addch(&out,
 624                                       (ch >= 0x20) && (ch < 0x80)
 625                                       ? ch : '.');
 626                }
 627                strbuf_addch(&out, '\n');
 628                trace_strbuf(&trace_curl, &out);
 629        }
 630        strbuf_release(&out);
 631}
 632
 633static int curl_trace(CURL *handle, curl_infotype type, char *data, size_t size, void *userp)
 634{
 635        const char *text;
 636        enum { NO_FILTER = 0, DO_FILTER = 1 };
 637
 638        switch (type) {
 639        case CURLINFO_TEXT:
 640                trace_printf_key(&trace_curl, "== Info: %s", data);
 641        default:                /* we ignore unknown types by default */
 642                return 0;
 643
 644        case CURLINFO_HEADER_OUT:
 645                text = "=> Send header";
 646                curl_dump_header(text, (unsigned char *)data, size, DO_FILTER);
 647                break;
 648        case CURLINFO_DATA_OUT:
 649                text = "=> Send data";
 650                curl_dump_data(text, (unsigned char *)data, size);
 651                break;
 652        case CURLINFO_SSL_DATA_OUT:
 653                text = "=> Send SSL data";
 654                curl_dump_data(text, (unsigned char *)data, size);
 655                break;
 656        case CURLINFO_HEADER_IN:
 657                text = "<= Recv header";
 658                curl_dump_header(text, (unsigned char *)data, size, NO_FILTER);
 659                break;
 660        case CURLINFO_DATA_IN:
 661                text = "<= Recv data";
 662                curl_dump_data(text, (unsigned char *)data, size);
 663                break;
 664        case CURLINFO_SSL_DATA_IN:
 665                text = "<= Recv SSL data";
 666                curl_dump_data(text, (unsigned char *)data, size);
 667                break;
 668        }
 669        return 0;
 670}
 671
 672void setup_curl_trace(CURL *handle)
 673{
 674        if (!trace_want(&trace_curl))
 675                return;
 676        curl_easy_setopt(handle, CURLOPT_VERBOSE, 1L);
 677        curl_easy_setopt(handle, CURLOPT_DEBUGFUNCTION, curl_trace);
 678        curl_easy_setopt(handle, CURLOPT_DEBUGDATA, NULL);
 679}
 680
 681#ifdef CURLPROTO_HTTP
 682static long get_curl_allowed_protocols(int from_user)
 683{
 684        long allowed_protocols = 0;
 685
 686        if (is_transport_allowed("http", from_user))
 687                allowed_protocols |= CURLPROTO_HTTP;
 688        if (is_transport_allowed("https", from_user))
 689                allowed_protocols |= CURLPROTO_HTTPS;
 690        if (is_transport_allowed("ftp", from_user))
 691                allowed_protocols |= CURLPROTO_FTP;
 692        if (is_transport_allowed("ftps", from_user))
 693                allowed_protocols |= CURLPROTO_FTPS;
 694
 695        return allowed_protocols;
 696}
 697#endif
 698
 699static CURL *get_curl_handle(void)
 700{
 701        CURL *result = curl_easy_init();
 702
 703        if (!result)
 704                die("curl_easy_init failed");
 705
 706        if (!curl_ssl_verify) {
 707                curl_easy_setopt(result, CURLOPT_SSL_VERIFYPEER, 0);
 708                curl_easy_setopt(result, CURLOPT_SSL_VERIFYHOST, 0);
 709        } else {
 710                /* Verify authenticity of the peer's certificate */
 711                curl_easy_setopt(result, CURLOPT_SSL_VERIFYPEER, 1);
 712                /* The name in the cert must match whom we tried to connect */
 713                curl_easy_setopt(result, CURLOPT_SSL_VERIFYHOST, 2);
 714        }
 715
 716#if LIBCURL_VERSION_NUM >= 0x070907
 717        curl_easy_setopt(result, CURLOPT_NETRC, CURL_NETRC_OPTIONAL);
 718#endif
 719#ifdef LIBCURL_CAN_HANDLE_AUTH_ANY
 720        curl_easy_setopt(result, CURLOPT_HTTPAUTH, CURLAUTH_ANY);
 721#endif
 722
 723#ifdef CURLGSSAPI_DELEGATION_FLAG
 724        if (curl_deleg) {
 725                int i;
 726                for (i = 0; i < ARRAY_SIZE(curl_deleg_levels); i++) {
 727                        if (!strcmp(curl_deleg, curl_deleg_levels[i].name)) {
 728                                curl_easy_setopt(result, CURLOPT_GSSAPI_DELEGATION,
 729                                                curl_deleg_levels[i].curl_deleg_param);
 730                                break;
 731                        }
 732                }
 733                if (i == ARRAY_SIZE(curl_deleg_levels))
 734                        warning("Unknown delegation method '%s': using default",
 735                                curl_deleg);
 736        }
 737#endif
 738
 739        if (http_proactive_auth)
 740                init_curl_http_auth(result);
 741
 742        if (getenv("GIT_SSL_VERSION"))
 743                ssl_version = getenv("GIT_SSL_VERSION");
 744        if (ssl_version && *ssl_version) {
 745                int i;
 746                for (i = 0; i < ARRAY_SIZE(sslversions); i++) {
 747                        if (!strcmp(ssl_version, sslversions[i].name)) {
 748                                curl_easy_setopt(result, CURLOPT_SSLVERSION,
 749                                                 sslversions[i].ssl_version);
 750                                break;
 751                        }
 752                }
 753                if (i == ARRAY_SIZE(sslversions))
 754                        warning("unsupported ssl version %s: using default",
 755                                ssl_version);
 756        }
 757
 758        if (getenv("GIT_SSL_CIPHER_LIST"))
 759                ssl_cipherlist = getenv("GIT_SSL_CIPHER_LIST");
 760        if (ssl_cipherlist != NULL && *ssl_cipherlist)
 761                curl_easy_setopt(result, CURLOPT_SSL_CIPHER_LIST,
 762                                ssl_cipherlist);
 763
 764        if (ssl_cert != NULL)
 765                curl_easy_setopt(result, CURLOPT_SSLCERT, ssl_cert);
 766        if (has_cert_password())
 767                curl_easy_setopt(result, CURLOPT_KEYPASSWD, cert_auth.password);
 768#if LIBCURL_VERSION_NUM >= 0x070903
 769        if (ssl_key != NULL)
 770                curl_easy_setopt(result, CURLOPT_SSLKEY, ssl_key);
 771#endif
 772#if LIBCURL_VERSION_NUM >= 0x070908
 773        if (ssl_capath != NULL)
 774                curl_easy_setopt(result, CURLOPT_CAPATH, ssl_capath);
 775#endif
 776#if LIBCURL_VERSION_NUM >= 0x072c00
 777        if (ssl_pinnedkey != NULL)
 778                curl_easy_setopt(result, CURLOPT_PINNEDPUBLICKEY, ssl_pinnedkey);
 779#endif
 780        if (ssl_cainfo != NULL)
 781                curl_easy_setopt(result, CURLOPT_CAINFO, ssl_cainfo);
 782
 783        if (curl_low_speed_limit > 0 && curl_low_speed_time > 0) {
 784                curl_easy_setopt(result, CURLOPT_LOW_SPEED_LIMIT,
 785                                 curl_low_speed_limit);
 786                curl_easy_setopt(result, CURLOPT_LOW_SPEED_TIME,
 787                                 curl_low_speed_time);
 788        }
 789
 790        curl_easy_setopt(result, CURLOPT_MAXREDIRS, 20);
 791#if LIBCURL_VERSION_NUM >= 0x071301
 792        curl_easy_setopt(result, CURLOPT_POSTREDIR, CURL_REDIR_POST_ALL);
 793#elif LIBCURL_VERSION_NUM >= 0x071101
 794        curl_easy_setopt(result, CURLOPT_POST301, 1);
 795#endif
 796#ifdef CURLPROTO_HTTP
 797        curl_easy_setopt(result, CURLOPT_REDIR_PROTOCOLS,
 798                         get_curl_allowed_protocols(0));
 799        curl_easy_setopt(result, CURLOPT_PROTOCOLS,
 800                         get_curl_allowed_protocols(-1));
 801#else
 802        warning("protocol restrictions not applied to curl redirects because\n"
 803                "your curl version is too old (>= 7.19.4)");
 804#endif
 805        if (getenv("GIT_CURL_VERBOSE"))
 806                curl_easy_setopt(result, CURLOPT_VERBOSE, 1L);
 807        setup_curl_trace(result);
 808
 809        curl_easy_setopt(result, CURLOPT_USERAGENT,
 810                user_agent ? user_agent : git_user_agent());
 811
 812        if (curl_ftp_no_epsv)
 813                curl_easy_setopt(result, CURLOPT_FTP_USE_EPSV, 0);
 814
 815#ifdef CURLOPT_USE_SSL
 816        if (curl_ssl_try)
 817                curl_easy_setopt(result, CURLOPT_USE_SSL, CURLUSESSL_TRY);
 818#endif
 819
 820        /*
 821         * CURL also examines these variables as a fallback; but we need to query
 822         * them here in order to decide whether to prompt for missing password (cf.
 823         * init_curl_proxy_auth()).
 824         *
 825         * Unlike many other common environment variables, these are historically
 826         * lowercase only. It appears that CURL did not know this and implemented
 827         * only uppercase variants, which was later corrected to take both - with
 828         * the exception of http_proxy, which is lowercase only also in CURL. As
 829         * the lowercase versions are the historical quasi-standard, they take
 830         * precedence here, as in CURL.
 831         */
 832        if (!curl_http_proxy) {
 833                if (http_auth.protocol && !strcmp(http_auth.protocol, "https")) {
 834                        var_override(&curl_http_proxy, getenv("HTTPS_PROXY"));
 835                        var_override(&curl_http_proxy, getenv("https_proxy"));
 836                } else {
 837                        var_override(&curl_http_proxy, getenv("http_proxy"));
 838                }
 839                if (!curl_http_proxy) {
 840                        var_override(&curl_http_proxy, getenv("ALL_PROXY"));
 841                        var_override(&curl_http_proxy, getenv("all_proxy"));
 842                }
 843        }
 844
 845        if (curl_http_proxy && curl_http_proxy[0] == '\0') {
 846                /*
 847                 * Handle case with the empty http.proxy value here to keep
 848                 * common code clean.
 849                 * NB: empty option disables proxying at all.
 850                 */
 851                curl_easy_setopt(result, CURLOPT_PROXY, "");
 852        } else if (curl_http_proxy) {
 853#if LIBCURL_VERSION_NUM >= 0x071800
 854                if (starts_with(curl_http_proxy, "socks5h"))
 855                        curl_easy_setopt(result,
 856                                CURLOPT_PROXYTYPE, CURLPROXY_SOCKS5_HOSTNAME);
 857                else if (starts_with(curl_http_proxy, "socks5"))
 858                        curl_easy_setopt(result,
 859                                CURLOPT_PROXYTYPE, CURLPROXY_SOCKS5);
 860                else if (starts_with(curl_http_proxy, "socks4a"))
 861                        curl_easy_setopt(result,
 862                                CURLOPT_PROXYTYPE, CURLPROXY_SOCKS4A);
 863                else if (starts_with(curl_http_proxy, "socks"))
 864                        curl_easy_setopt(result,
 865                                CURLOPT_PROXYTYPE, CURLPROXY_SOCKS4);
 866#endif
 867                if (strstr(curl_http_proxy, "://"))
 868                        credential_from_url(&proxy_auth, curl_http_proxy);
 869                else {
 870                        struct strbuf url = STRBUF_INIT;
 871                        strbuf_addf(&url, "http://%s", curl_http_proxy);
 872                        credential_from_url(&proxy_auth, url.buf);
 873                        strbuf_release(&url);
 874                }
 875
 876                if (!proxy_auth.host)
 877                        die("Invalid proxy URL '%s'", curl_http_proxy);
 878
 879                curl_easy_setopt(result, CURLOPT_PROXY, proxy_auth.host);
 880#if LIBCURL_VERSION_NUM >= 0x071304
 881                var_override(&curl_no_proxy, getenv("NO_PROXY"));
 882                var_override(&curl_no_proxy, getenv("no_proxy"));
 883                curl_easy_setopt(result, CURLOPT_NOPROXY, curl_no_proxy);
 884#endif
 885        }
 886        init_curl_proxy_auth(result);
 887
 888        set_curl_keepalive(result);
 889
 890        return result;
 891}
 892
 893static void set_from_env(const char **var, const char *envname)
 894{
 895        const char *val = getenv(envname);
 896        if (val)
 897                *var = val;
 898}
 899
 900void http_init(struct remote *remote, const char *url, int proactive_auth)
 901{
 902        char *low_speed_limit;
 903        char *low_speed_time;
 904        char *normalized_url;
 905        struct urlmatch_config config = { STRING_LIST_INIT_DUP };
 906
 907        config.section = "http";
 908        config.key = NULL;
 909        config.collect_fn = http_options;
 910        config.cascade_fn = git_default_config;
 911        config.cb = NULL;
 912
 913        http_is_verbose = 0;
 914        normalized_url = url_normalize(url, &config.url);
 915
 916        git_config(urlmatch_config_entry, &config);
 917        free(normalized_url);
 918
 919        if (curl_global_init(CURL_GLOBAL_ALL) != CURLE_OK)
 920                die("curl_global_init failed");
 921
 922        http_proactive_auth = proactive_auth;
 923
 924        if (remote && remote->http_proxy)
 925                curl_http_proxy = xstrdup(remote->http_proxy);
 926
 927        if (remote)
 928                var_override(&http_proxy_authmethod, remote->http_proxy_authmethod);
 929
 930        pragma_header = curl_slist_append(http_copy_default_headers(),
 931                "Pragma: no-cache");
 932        no_pragma_header = curl_slist_append(http_copy_default_headers(),
 933                "Pragma:");
 934
 935#ifdef USE_CURL_MULTI
 936        {
 937                char *http_max_requests = getenv("GIT_HTTP_MAX_REQUESTS");
 938                if (http_max_requests != NULL)
 939                        max_requests = atoi(http_max_requests);
 940        }
 941
 942        curlm = curl_multi_init();
 943        if (!curlm)
 944                die("curl_multi_init failed");
 945#endif
 946
 947        if (getenv("GIT_SSL_NO_VERIFY"))
 948                curl_ssl_verify = 0;
 949
 950        set_from_env(&ssl_cert, "GIT_SSL_CERT");
 951#if LIBCURL_VERSION_NUM >= 0x070903
 952        set_from_env(&ssl_key, "GIT_SSL_KEY");
 953#endif
 954#if LIBCURL_VERSION_NUM >= 0x070908
 955        set_from_env(&ssl_capath, "GIT_SSL_CAPATH");
 956#endif
 957        set_from_env(&ssl_cainfo, "GIT_SSL_CAINFO");
 958
 959        set_from_env(&user_agent, "GIT_HTTP_USER_AGENT");
 960
 961        low_speed_limit = getenv("GIT_HTTP_LOW_SPEED_LIMIT");
 962        if (low_speed_limit != NULL)
 963                curl_low_speed_limit = strtol(low_speed_limit, NULL, 10);
 964        low_speed_time = getenv("GIT_HTTP_LOW_SPEED_TIME");
 965        if (low_speed_time != NULL)
 966                curl_low_speed_time = strtol(low_speed_time, NULL, 10);
 967
 968        if (curl_ssl_verify == -1)
 969                curl_ssl_verify = 1;
 970
 971        curl_session_count = 0;
 972#ifdef USE_CURL_MULTI
 973        if (max_requests < 1)
 974                max_requests = DEFAULT_MAX_REQUESTS;
 975#endif
 976
 977        if (getenv("GIT_CURL_FTP_NO_EPSV"))
 978                curl_ftp_no_epsv = 1;
 979
 980        if (url) {
 981                credential_from_url(&http_auth, url);
 982                if (!ssl_cert_password_required &&
 983                    getenv("GIT_SSL_CERT_PASSWORD_PROTECTED") &&
 984                    starts_with(url, "https://"))
 985                        ssl_cert_password_required = 1;
 986        }
 987
 988#ifndef NO_CURL_EASY_DUPHANDLE
 989        curl_default = get_curl_handle();
 990#endif
 991}
 992
 993void http_cleanup(void)
 994{
 995        struct active_request_slot *slot = active_queue_head;
 996
 997        while (slot != NULL) {
 998                struct active_request_slot *next = slot->next;
 999                if (slot->curl != NULL) {
1000                        xmulti_remove_handle(slot);
1001                        curl_easy_cleanup(slot->curl);
1002                }
1003                free(slot);
1004                slot = next;
1005        }
1006        active_queue_head = NULL;
1007
1008#ifndef NO_CURL_EASY_DUPHANDLE
1009        curl_easy_cleanup(curl_default);
1010#endif
1011
1012#ifdef USE_CURL_MULTI
1013        curl_multi_cleanup(curlm);
1014#endif
1015        curl_global_cleanup();
1016
1017        curl_slist_free_all(extra_http_headers);
1018        extra_http_headers = NULL;
1019
1020        curl_slist_free_all(pragma_header);
1021        pragma_header = NULL;
1022
1023        curl_slist_free_all(no_pragma_header);
1024        no_pragma_header = NULL;
1025
1026        if (curl_http_proxy) {
1027                free((void *)curl_http_proxy);
1028                curl_http_proxy = NULL;
1029        }
1030
1031        if (proxy_auth.password) {
1032                memset(proxy_auth.password, 0, strlen(proxy_auth.password));
1033                FREE_AND_NULL(proxy_auth.password);
1034        }
1035
1036        free((void *)curl_proxyuserpwd);
1037        curl_proxyuserpwd = NULL;
1038
1039        free((void *)http_proxy_authmethod);
1040        http_proxy_authmethod = NULL;
1041
1042        if (cert_auth.password != NULL) {
1043                memset(cert_auth.password, 0, strlen(cert_auth.password));
1044                FREE_AND_NULL(cert_auth.password);
1045        }
1046        ssl_cert_password_required = 0;
1047
1048        FREE_AND_NULL(cached_accept_language);
1049}
1050
1051struct active_request_slot *get_active_slot(void)
1052{
1053        struct active_request_slot *slot = active_queue_head;
1054        struct active_request_slot *newslot;
1055
1056#ifdef USE_CURL_MULTI
1057        int num_transfers;
1058
1059        /* Wait for a slot to open up if the queue is full */
1060        while (active_requests >= max_requests) {
1061                curl_multi_perform(curlm, &num_transfers);
1062                if (num_transfers < active_requests)
1063                        process_curl_messages();
1064        }
1065#endif
1066
1067        while (slot != NULL && slot->in_use)
1068                slot = slot->next;
1069
1070        if (slot == NULL) {
1071                newslot = xmalloc(sizeof(*newslot));
1072                newslot->curl = NULL;
1073                newslot->in_use = 0;
1074                newslot->next = NULL;
1075
1076                slot = active_queue_head;
1077                if (slot == NULL) {
1078                        active_queue_head = newslot;
1079                } else {
1080                        while (slot->next != NULL)
1081                                slot = slot->next;
1082                        slot->next = newslot;
1083                }
1084                slot = newslot;
1085        }
1086
1087        if (slot->curl == NULL) {
1088#ifdef NO_CURL_EASY_DUPHANDLE
1089                slot->curl = get_curl_handle();
1090#else
1091                slot->curl = curl_easy_duphandle(curl_default);
1092#endif
1093                curl_session_count++;
1094        }
1095
1096        active_requests++;
1097        slot->in_use = 1;
1098        slot->results = NULL;
1099        slot->finished = NULL;
1100        slot->callback_data = NULL;
1101        slot->callback_func = NULL;
1102        curl_easy_setopt(slot->curl, CURLOPT_COOKIEFILE, curl_cookie_file);
1103        if (curl_save_cookies)
1104                curl_easy_setopt(slot->curl, CURLOPT_COOKIEJAR, curl_cookie_file);
1105        curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, pragma_header);
1106        curl_easy_setopt(slot->curl, CURLOPT_ERRORBUFFER, curl_errorstr);
1107        curl_easy_setopt(slot->curl, CURLOPT_CUSTOMREQUEST, NULL);
1108        curl_easy_setopt(slot->curl, CURLOPT_READFUNCTION, NULL);
1109        curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, NULL);
1110        curl_easy_setopt(slot->curl, CURLOPT_POSTFIELDS, NULL);
1111        curl_easy_setopt(slot->curl, CURLOPT_UPLOAD, 0);
1112        curl_easy_setopt(slot->curl, CURLOPT_HTTPGET, 1);
1113        curl_easy_setopt(slot->curl, CURLOPT_FAILONERROR, 1);
1114        curl_easy_setopt(slot->curl, CURLOPT_RANGE, NULL);
1115
1116        /*
1117         * Default following to off unless "ALWAYS" is configured; this gives
1118         * callers a sane starting point, and they can tweak for individual
1119         * HTTP_FOLLOW_* cases themselves.
1120         */
1121        if (http_follow_config == HTTP_FOLLOW_ALWAYS)
1122                curl_easy_setopt(slot->curl, CURLOPT_FOLLOWLOCATION, 1);
1123        else
1124                curl_easy_setopt(slot->curl, CURLOPT_FOLLOWLOCATION, 0);
1125
1126#if LIBCURL_VERSION_NUM >= 0x070a08
1127        curl_easy_setopt(slot->curl, CURLOPT_IPRESOLVE, git_curl_ipresolve);
1128#endif
1129#ifdef LIBCURL_CAN_HANDLE_AUTH_ANY
1130        curl_easy_setopt(slot->curl, CURLOPT_HTTPAUTH, http_auth_methods);
1131#endif
1132        if (http_auth.password || curl_empty_auth_enabled())
1133                init_curl_http_auth(slot->curl);
1134
1135        return slot;
1136}
1137
1138int start_active_slot(struct active_request_slot *slot)
1139{
1140#ifdef USE_CURL_MULTI
1141        CURLMcode curlm_result = curl_multi_add_handle(curlm, slot->curl);
1142        int num_transfers;
1143
1144        if (curlm_result != CURLM_OK &&
1145            curlm_result != CURLM_CALL_MULTI_PERFORM) {
1146                warning("curl_multi_add_handle failed: %s",
1147                        curl_multi_strerror(curlm_result));
1148                active_requests--;
1149                slot->in_use = 0;
1150                return 0;
1151        }
1152
1153        /*
1154         * We know there must be something to do, since we just added
1155         * something.
1156         */
1157        curl_multi_perform(curlm, &num_transfers);
1158#endif
1159        return 1;
1160}
1161
1162#ifdef USE_CURL_MULTI
1163struct fill_chain {
1164        void *data;
1165        int (*fill)(void *);
1166        struct fill_chain *next;
1167};
1168
1169static struct fill_chain *fill_cfg;
1170
1171void add_fill_function(void *data, int (*fill)(void *))
1172{
1173        struct fill_chain *new = xmalloc(sizeof(*new));
1174        struct fill_chain **linkp = &fill_cfg;
1175        new->data = data;
1176        new->fill = fill;
1177        new->next = NULL;
1178        while (*linkp)
1179                linkp = &(*linkp)->next;
1180        *linkp = new;
1181}
1182
1183void fill_active_slots(void)
1184{
1185        struct active_request_slot *slot = active_queue_head;
1186
1187        while (active_requests < max_requests) {
1188                struct fill_chain *fill;
1189                for (fill = fill_cfg; fill; fill = fill->next)
1190                        if (fill->fill(fill->data))
1191                                break;
1192
1193                if (!fill)
1194                        break;
1195        }
1196
1197        while (slot != NULL) {
1198                if (!slot->in_use && slot->curl != NULL
1199                        && curl_session_count > min_curl_sessions) {
1200                        curl_easy_cleanup(slot->curl);
1201                        slot->curl = NULL;
1202                        curl_session_count--;
1203                }
1204                slot = slot->next;
1205        }
1206}
1207
1208void step_active_slots(void)
1209{
1210        int num_transfers;
1211        CURLMcode curlm_result;
1212
1213        do {
1214                curlm_result = curl_multi_perform(curlm, &num_transfers);
1215        } while (curlm_result == CURLM_CALL_MULTI_PERFORM);
1216        if (num_transfers < active_requests) {
1217                process_curl_messages();
1218                fill_active_slots();
1219        }
1220}
1221#endif
1222
1223void run_active_slot(struct active_request_slot *slot)
1224{
1225#ifdef USE_CURL_MULTI
1226        fd_set readfds;
1227        fd_set writefds;
1228        fd_set excfds;
1229        int max_fd;
1230        struct timeval select_timeout;
1231        int finished = 0;
1232
1233        slot->finished = &finished;
1234        while (!finished) {
1235                step_active_slots();
1236
1237                if (slot->in_use) {
1238#if LIBCURL_VERSION_NUM >= 0x070f04
1239                        long curl_timeout;
1240                        curl_multi_timeout(curlm, &curl_timeout);
1241                        if (curl_timeout == 0) {
1242                                continue;
1243                        } else if (curl_timeout == -1) {
1244                                select_timeout.tv_sec  = 0;
1245                                select_timeout.tv_usec = 50000;
1246                        } else {
1247                                select_timeout.tv_sec  =  curl_timeout / 1000;
1248                                select_timeout.tv_usec = (curl_timeout % 1000) * 1000;
1249                        }
1250#else
1251                        select_timeout.tv_sec  = 0;
1252                        select_timeout.tv_usec = 50000;
1253#endif
1254
1255                        max_fd = -1;
1256                        FD_ZERO(&readfds);
1257                        FD_ZERO(&writefds);
1258                        FD_ZERO(&excfds);
1259                        curl_multi_fdset(curlm, &readfds, &writefds, &excfds, &max_fd);
1260
1261                        /*
1262                         * It can happen that curl_multi_timeout returns a pathologically
1263                         * long timeout when curl_multi_fdset returns no file descriptors
1264                         * to read.  See commit message for more details.
1265                         */
1266                        if (max_fd < 0 &&
1267                            (select_timeout.tv_sec > 0 ||
1268                             select_timeout.tv_usec > 50000)) {
1269                                select_timeout.tv_sec  = 0;
1270                                select_timeout.tv_usec = 50000;
1271                        }
1272
1273                        select(max_fd+1, &readfds, &writefds, &excfds, &select_timeout);
1274                }
1275        }
1276#else
1277        while (slot->in_use) {
1278                slot->curl_result = curl_easy_perform(slot->curl);
1279                finish_active_slot(slot);
1280        }
1281#endif
1282}
1283
1284static void release_active_slot(struct active_request_slot *slot)
1285{
1286        closedown_active_slot(slot);
1287        if (slot->curl) {
1288                xmulti_remove_handle(slot);
1289                if (curl_session_count > min_curl_sessions) {
1290                        curl_easy_cleanup(slot->curl);
1291                        slot->curl = NULL;
1292                        curl_session_count--;
1293                }
1294        }
1295#ifdef USE_CURL_MULTI
1296        fill_active_slots();
1297#endif
1298}
1299
1300void finish_all_active_slots(void)
1301{
1302        struct active_request_slot *slot = active_queue_head;
1303
1304        while (slot != NULL)
1305                if (slot->in_use) {
1306                        run_active_slot(slot);
1307                        slot = active_queue_head;
1308                } else {
1309                        slot = slot->next;
1310                }
1311}
1312
1313/* Helpers for modifying and creating URLs */
1314static inline int needs_quote(int ch)
1315{
1316        if (((ch >= 'A') && (ch <= 'Z'))
1317                        || ((ch >= 'a') && (ch <= 'z'))
1318                        || ((ch >= '0') && (ch <= '9'))
1319                        || (ch == '/')
1320                        || (ch == '-')
1321                        || (ch == '.'))
1322                return 0;
1323        return 1;
1324}
1325
1326static char *quote_ref_url(const char *base, const char *ref)
1327{
1328        struct strbuf buf = STRBUF_INIT;
1329        const char *cp;
1330        int ch;
1331
1332        end_url_with_slash(&buf, base);
1333
1334        for (cp = ref; (ch = *cp) != 0; cp++)
1335                if (needs_quote(ch))
1336                        strbuf_addf(&buf, "%%%02x", ch);
1337                else
1338                        strbuf_addch(&buf, *cp);
1339
1340        return strbuf_detach(&buf, NULL);
1341}
1342
1343void append_remote_object_url(struct strbuf *buf, const char *url,
1344                              const char *hex,
1345                              int only_two_digit_prefix)
1346{
1347        end_url_with_slash(buf, url);
1348
1349        strbuf_addf(buf, "objects/%.*s/", 2, hex);
1350        if (!only_two_digit_prefix)
1351                strbuf_addstr(buf, hex + 2);
1352}
1353
1354char *get_remote_object_url(const char *url, const char *hex,
1355                            int only_two_digit_prefix)
1356{
1357        struct strbuf buf = STRBUF_INIT;
1358        append_remote_object_url(&buf, url, hex, only_two_digit_prefix);
1359        return strbuf_detach(&buf, NULL);
1360}
1361
1362static int handle_curl_result(struct slot_results *results)
1363{
1364        /*
1365         * If we see a failing http code with CURLE_OK, we have turned off
1366         * FAILONERROR (to keep the server's custom error response), and should
1367         * translate the code into failure here.
1368         *
1369         * Likewise, if we see a redirect (30x code), that means we turned off
1370         * redirect-following, and we should treat the result as an error.
1371         */
1372        if (results->curl_result == CURLE_OK &&
1373            results->http_code >= 300) {
1374                results->curl_result = CURLE_HTTP_RETURNED_ERROR;
1375                /*
1376                 * Normally curl will already have put the "reason phrase"
1377                 * from the server into curl_errorstr; unfortunately without
1378                 * FAILONERROR it is lost, so we can give only the numeric
1379                 * status code.
1380                 */
1381                xsnprintf(curl_errorstr, sizeof(curl_errorstr),
1382                          "The requested URL returned error: %ld",
1383                          results->http_code);
1384        }
1385
1386        if (results->curl_result == CURLE_OK) {
1387                credential_approve(&http_auth);
1388                if (proxy_auth.password)
1389                        credential_approve(&proxy_auth);
1390                return HTTP_OK;
1391        } else if (missing_target(results))
1392                return HTTP_MISSING_TARGET;
1393        else if (results->http_code == 401) {
1394                if (http_auth.username && http_auth.password) {
1395                        credential_reject(&http_auth);
1396                        return HTTP_NOAUTH;
1397                } else {
1398#ifdef LIBCURL_CAN_HANDLE_AUTH_ANY
1399                        http_auth_methods &= ~CURLAUTH_GSSNEGOTIATE;
1400                        if (results->auth_avail) {
1401                                http_auth_methods &= results->auth_avail;
1402                                http_auth_methods_restricted = 1;
1403                        }
1404#endif
1405                        return HTTP_REAUTH;
1406                }
1407        } else {
1408                if (results->http_connectcode == 407)
1409                        credential_reject(&proxy_auth);
1410#if LIBCURL_VERSION_NUM >= 0x070c00
1411                if (!curl_errorstr[0])
1412                        strlcpy(curl_errorstr,
1413                                curl_easy_strerror(results->curl_result),
1414                                sizeof(curl_errorstr));
1415#endif
1416                return HTTP_ERROR;
1417        }
1418}
1419
1420int run_one_slot(struct active_request_slot *slot,
1421                 struct slot_results *results)
1422{
1423        slot->results = results;
1424        if (!start_active_slot(slot)) {
1425                xsnprintf(curl_errorstr, sizeof(curl_errorstr),
1426                          "failed to start HTTP request");
1427                return HTTP_START_FAILED;
1428        }
1429
1430        run_active_slot(slot);
1431        return handle_curl_result(results);
1432}
1433
1434struct curl_slist *http_copy_default_headers(void)
1435{
1436        struct curl_slist *headers = NULL, *h;
1437
1438        for (h = extra_http_headers; h; h = h->next)
1439                headers = curl_slist_append(headers, h->data);
1440
1441        return headers;
1442}
1443
1444static CURLcode curlinfo_strbuf(CURL *curl, CURLINFO info, struct strbuf *buf)
1445{
1446        char *ptr;
1447        CURLcode ret;
1448
1449        strbuf_reset(buf);
1450        ret = curl_easy_getinfo(curl, info, &ptr);
1451        if (!ret && ptr)
1452                strbuf_addstr(buf, ptr);
1453        return ret;
1454}
1455
1456/*
1457 * Check for and extract a content-type parameter. "raw"
1458 * should be positioned at the start of the potential
1459 * parameter, with any whitespace already removed.
1460 *
1461 * "name" is the name of the parameter. The value is appended
1462 * to "out".
1463 */
1464static int extract_param(const char *raw, const char *name,
1465                         struct strbuf *out)
1466{
1467        size_t len = strlen(name);
1468
1469        if (strncasecmp(raw, name, len))
1470                return -1;
1471        raw += len;
1472
1473        if (*raw != '=')
1474                return -1;
1475        raw++;
1476
1477        while (*raw && !isspace(*raw) && *raw != ';')
1478                strbuf_addch(out, *raw++);
1479        return 0;
1480}
1481
1482/*
1483 * Extract a normalized version of the content type, with any
1484 * spaces suppressed, all letters lowercased, and no trailing ";"
1485 * or parameters.
1486 *
1487 * Note that we will silently remove even invalid whitespace. For
1488 * example, "text / plain" is specifically forbidden by RFC 2616,
1489 * but "text/plain" is the only reasonable output, and this keeps
1490 * our code simple.
1491 *
1492 * If the "charset" argument is not NULL, store the value of any
1493 * charset parameter there.
1494 *
1495 * Example:
1496 *   "TEXT/PLAIN; charset=utf-8" -> "text/plain", "utf-8"
1497 *   "text / plain" -> "text/plain"
1498 */
1499static void extract_content_type(struct strbuf *raw, struct strbuf *type,
1500                                 struct strbuf *charset)
1501{
1502        const char *p;
1503
1504        strbuf_reset(type);
1505        strbuf_grow(type, raw->len);
1506        for (p = raw->buf; *p; p++) {
1507                if (isspace(*p))
1508                        continue;
1509                if (*p == ';') {
1510                        p++;
1511                        break;
1512                }
1513                strbuf_addch(type, tolower(*p));
1514        }
1515
1516        if (!charset)
1517                return;
1518
1519        strbuf_reset(charset);
1520        while (*p) {
1521                while (isspace(*p) || *p == ';')
1522                        p++;
1523                if (!extract_param(p, "charset", charset))
1524                        return;
1525                while (*p && !isspace(*p))
1526                        p++;
1527        }
1528
1529        if (!charset->len && starts_with(type->buf, "text/"))
1530                strbuf_addstr(charset, "ISO-8859-1");
1531}
1532
1533static void write_accept_language(struct strbuf *buf)
1534{
1535        /*
1536         * MAX_DECIMAL_PLACES must not be larger than 3. If it is larger than
1537         * that, q-value will be smaller than 0.001, the minimum q-value the
1538         * HTTP specification allows. See
1539         * http://tools.ietf.org/html/rfc7231#section-5.3.1 for q-value.
1540         */
1541        const int MAX_DECIMAL_PLACES = 3;
1542        const int MAX_LANGUAGE_TAGS = 1000;
1543        const int MAX_ACCEPT_LANGUAGE_HEADER_SIZE = 4000;
1544        char **language_tags = NULL;
1545        int num_langs = 0;
1546        const char *s = get_preferred_languages();
1547        int i;
1548        struct strbuf tag = STRBUF_INIT;
1549
1550        /* Don't add Accept-Language header if no language is preferred. */
1551        if (!s)
1552                return;
1553
1554        /*
1555         * Split the colon-separated string of preferred languages into
1556         * language_tags array.
1557         */
1558        do {
1559                /* collect language tag */
1560                for (; *s && (isalnum(*s) || *s == '_'); s++)
1561                        strbuf_addch(&tag, *s == '_' ? '-' : *s);
1562
1563                /* skip .codeset, @modifier and any other unnecessary parts */
1564                while (*s && *s != ':')
1565                        s++;
1566
1567                if (tag.len) {
1568                        num_langs++;
1569                        REALLOC_ARRAY(language_tags, num_langs);
1570                        language_tags[num_langs - 1] = strbuf_detach(&tag, NULL);
1571                        if (num_langs >= MAX_LANGUAGE_TAGS - 1) /* -1 for '*' */
1572                                break;
1573                }
1574        } while (*s++);
1575
1576        /* write Accept-Language header into buf */
1577        if (num_langs) {
1578                int last_buf_len = 0;
1579                int max_q;
1580                int decimal_places;
1581                char q_format[32];
1582
1583                /* add '*' */
1584                REALLOC_ARRAY(language_tags, num_langs + 1);
1585                language_tags[num_langs++] = "*"; /* it's OK; this won't be freed */
1586
1587                /* compute decimal_places */
1588                for (max_q = 1, decimal_places = 0;
1589                     max_q < num_langs && decimal_places <= MAX_DECIMAL_PLACES;
1590                     decimal_places++, max_q *= 10)
1591                        ;
1592
1593                xsnprintf(q_format, sizeof(q_format), ";q=0.%%0%dd", decimal_places);
1594
1595                strbuf_addstr(buf, "Accept-Language: ");
1596
1597                for (i = 0; i < num_langs; i++) {
1598                        if (i > 0)
1599                                strbuf_addstr(buf, ", ");
1600
1601                        strbuf_addstr(buf, language_tags[i]);
1602
1603                        if (i > 0)
1604                                strbuf_addf(buf, q_format, max_q - i);
1605
1606                        if (buf->len > MAX_ACCEPT_LANGUAGE_HEADER_SIZE) {
1607                                strbuf_remove(buf, last_buf_len, buf->len - last_buf_len);
1608                                break;
1609                        }
1610
1611                        last_buf_len = buf->len;
1612                }
1613        }
1614
1615        /* free language tags -- last one is a static '*' */
1616        for (i = 0; i < num_langs - 1; i++)
1617                free(language_tags[i]);
1618        free(language_tags);
1619}
1620
1621/*
1622 * Get an Accept-Language header which indicates user's preferred languages.
1623 *
1624 * Examples:
1625 *   LANGUAGE= -> ""
1626 *   LANGUAGE=ko:en -> "Accept-Language: ko, en; q=0.9, *; q=0.1"
1627 *   LANGUAGE=ko_KR.UTF-8:sr@latin -> "Accept-Language: ko-KR, sr; q=0.9, *; q=0.1"
1628 *   LANGUAGE=ko LANG=en_US.UTF-8 -> "Accept-Language: ko, *; q=0.1"
1629 *   LANGUAGE= LANG=en_US.UTF-8 -> "Accept-Language: en-US, *; q=0.1"
1630 *   LANGUAGE= LANG=C -> ""
1631 */
1632static const char *get_accept_language(void)
1633{
1634        if (!cached_accept_language) {
1635                struct strbuf buf = STRBUF_INIT;
1636                write_accept_language(&buf);
1637                if (buf.len > 0)
1638                        cached_accept_language = strbuf_detach(&buf, NULL);
1639        }
1640
1641        return cached_accept_language;
1642}
1643
1644static void http_opt_request_remainder(CURL *curl, off_t pos)
1645{
1646        char buf[128];
1647        xsnprintf(buf, sizeof(buf), "%"PRIuMAX"-", (uintmax_t)pos);
1648        curl_easy_setopt(curl, CURLOPT_RANGE, buf);
1649}
1650
1651/* http_request() targets */
1652#define HTTP_REQUEST_STRBUF     0
1653#define HTTP_REQUEST_FILE       1
1654
1655static int http_request(const char *url,
1656                        void *result, int target,
1657                        const struct http_get_options *options)
1658{
1659        struct active_request_slot *slot;
1660        struct slot_results results;
1661        struct curl_slist *headers = http_copy_default_headers();
1662        struct strbuf buf = STRBUF_INIT;
1663        const char *accept_language;
1664        int ret;
1665
1666        slot = get_active_slot();
1667        curl_easy_setopt(slot->curl, CURLOPT_HTTPGET, 1);
1668
1669        if (result == NULL) {
1670                curl_easy_setopt(slot->curl, CURLOPT_NOBODY, 1);
1671        } else {
1672                curl_easy_setopt(slot->curl, CURLOPT_NOBODY, 0);
1673                curl_easy_setopt(slot->curl, CURLOPT_FILE, result);
1674
1675                if (target == HTTP_REQUEST_FILE) {
1676                        off_t posn = ftello(result);
1677                        curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION,
1678                                         fwrite);
1679                        if (posn > 0)
1680                                http_opt_request_remainder(slot->curl, posn);
1681                } else
1682                        curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION,
1683                                         fwrite_buffer);
1684        }
1685
1686        accept_language = get_accept_language();
1687
1688        if (accept_language)
1689                headers = curl_slist_append(headers, accept_language);
1690
1691        strbuf_addstr(&buf, "Pragma:");
1692        if (options && options->no_cache)
1693                strbuf_addstr(&buf, " no-cache");
1694        if (options && options->keep_error)
1695                curl_easy_setopt(slot->curl, CURLOPT_FAILONERROR, 0);
1696        if (options && options->initial_request &&
1697            http_follow_config == HTTP_FOLLOW_INITIAL)
1698                curl_easy_setopt(slot->curl, CURLOPT_FOLLOWLOCATION, 1);
1699
1700        headers = curl_slist_append(headers, buf.buf);
1701
1702        curl_easy_setopt(slot->curl, CURLOPT_URL, url);
1703        curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, headers);
1704        curl_easy_setopt(slot->curl, CURLOPT_ENCODING, "gzip");
1705
1706        ret = run_one_slot(slot, &results);
1707
1708        if (options && options->content_type) {
1709                struct strbuf raw = STRBUF_INIT;
1710                curlinfo_strbuf(slot->curl, CURLINFO_CONTENT_TYPE, &raw);
1711                extract_content_type(&raw, options->content_type,
1712                                     options->charset);
1713                strbuf_release(&raw);
1714        }
1715
1716        if (options && options->effective_url)
1717                curlinfo_strbuf(slot->curl, CURLINFO_EFFECTIVE_URL,
1718                                options->effective_url);
1719
1720        curl_slist_free_all(headers);
1721        strbuf_release(&buf);
1722
1723        return ret;
1724}
1725
1726/*
1727 * Update the "base" url to a more appropriate value, as deduced by
1728 * redirects seen when requesting a URL starting with "url".
1729 *
1730 * The "asked" parameter is a URL that we asked curl to access, and must begin
1731 * with "base".
1732 *
1733 * The "got" parameter is the URL that curl reported to us as where we ended
1734 * up.
1735 *
1736 * Returns 1 if we updated the base url, 0 otherwise.
1737 *
1738 * Our basic strategy is to compare "base" and "asked" to find the bits
1739 * specific to our request. We then strip those bits off of "got" to yield the
1740 * new base. So for example, if our base is "http://example.com/foo.git",
1741 * and we ask for "http://example.com/foo.git/info/refs", we might end up
1742 * with "https://other.example.com/foo.git/info/refs". We would want the
1743 * new URL to become "https://other.example.com/foo.git".
1744 *
1745 * Note that this assumes a sane redirect scheme. It's entirely possible
1746 * in the example above to end up at a URL that does not even end in
1747 * "info/refs".  In such a case we die. There's not much we can do, such a
1748 * scheme is unlikely to represent a real git repository, and failing to
1749 * rewrite the base opens options for malicious redirects to do funny things.
1750 */
1751static int update_url_from_redirect(struct strbuf *base,
1752                                    const char *asked,
1753                                    const struct strbuf *got)
1754{
1755        const char *tail;
1756        size_t new_len;
1757
1758        if (!strcmp(asked, got->buf))
1759                return 0;
1760
1761        if (!skip_prefix(asked, base->buf, &tail))
1762                die("BUG: update_url_from_redirect: %s is not a superset of %s",
1763                    asked, base->buf);
1764
1765        new_len = got->len;
1766        if (!strip_suffix_mem(got->buf, &new_len, tail))
1767                die(_("unable to update url base from redirection:\n"
1768                      "  asked for: %s\n"
1769                      "   redirect: %s"),
1770                    asked, got->buf);
1771
1772        strbuf_reset(base);
1773        strbuf_add(base, got->buf, new_len);
1774
1775        return 1;
1776}
1777
1778static int http_request_reauth(const char *url,
1779                               void *result, int target,
1780                               struct http_get_options *options)
1781{
1782        int ret = http_request(url, result, target, options);
1783
1784        if (ret != HTTP_OK && ret != HTTP_REAUTH)
1785                return ret;
1786
1787        if (options && options->effective_url && options->base_url) {
1788                if (update_url_from_redirect(options->base_url,
1789                                             url, options->effective_url)) {
1790                        credential_from_url(&http_auth, options->base_url->buf);
1791                        url = options->effective_url->buf;
1792                }
1793        }
1794
1795        if (ret != HTTP_REAUTH)
1796                return ret;
1797
1798        /*
1799         * If we are using KEEP_ERROR, the previous request may have
1800         * put cruft into our output stream; we should clear it out before
1801         * making our next request. We only know how to do this for
1802         * the strbuf case, but that is enough to satisfy current callers.
1803         */
1804        if (options && options->keep_error) {
1805                switch (target) {
1806                case HTTP_REQUEST_STRBUF:
1807                        strbuf_reset(result);
1808                        break;
1809                default:
1810                        die("BUG: HTTP_KEEP_ERROR is only supported with strbufs");
1811                }
1812        }
1813
1814        credential_fill(&http_auth);
1815
1816        return http_request(url, result, target, options);
1817}
1818
1819int http_get_strbuf(const char *url,
1820                    struct strbuf *result,
1821                    struct http_get_options *options)
1822{
1823        return http_request_reauth(url, result, HTTP_REQUEST_STRBUF, options);
1824}
1825
1826/*
1827 * Downloads a URL and stores the result in the given file.
1828 *
1829 * If a previous interrupted download is detected (i.e. a previous temporary
1830 * file is still around) the download is resumed.
1831 */
1832static int http_get_file(const char *url, const char *filename,
1833                         struct http_get_options *options)
1834{
1835        int ret;
1836        struct strbuf tmpfile = STRBUF_INIT;
1837        FILE *result;
1838
1839        strbuf_addf(&tmpfile, "%s.temp", filename);
1840        result = fopen(tmpfile.buf, "a");
1841        if (!result) {
1842                error("Unable to open local file %s", tmpfile.buf);
1843                ret = HTTP_ERROR;
1844                goto cleanup;
1845        }
1846
1847        ret = http_request_reauth(url, result, HTTP_REQUEST_FILE, options);
1848        fclose(result);
1849
1850        if (ret == HTTP_OK && finalize_object_file(tmpfile.buf, filename))
1851                ret = HTTP_ERROR;
1852cleanup:
1853        strbuf_release(&tmpfile);
1854        return ret;
1855}
1856
1857int http_fetch_ref(const char *base, struct ref *ref)
1858{
1859        struct http_get_options options = {0};
1860        char *url;
1861        struct strbuf buffer = STRBUF_INIT;
1862        int ret = -1;
1863
1864        options.no_cache = 1;
1865
1866        url = quote_ref_url(base, ref->name);
1867        if (http_get_strbuf(url, &buffer, &options) == HTTP_OK) {
1868                strbuf_rtrim(&buffer);
1869                if (buffer.len == 40)
1870                        ret = get_oid_hex(buffer.buf, &ref->old_oid);
1871                else if (starts_with(buffer.buf, "ref: ")) {
1872                        ref->symref = xstrdup(buffer.buf + 5);
1873                        ret = 0;
1874                }
1875        }
1876
1877        strbuf_release(&buffer);
1878        free(url);
1879        return ret;
1880}
1881
1882/* Helpers for fetching packs */
1883static char *fetch_pack_index(unsigned char *sha1, const char *base_url)
1884{
1885        char *url, *tmp;
1886        struct strbuf buf = STRBUF_INIT;
1887
1888        if (http_is_verbose)
1889                fprintf(stderr, "Getting index for pack %s\n", sha1_to_hex(sha1));
1890
1891        end_url_with_slash(&buf, base_url);
1892        strbuf_addf(&buf, "objects/pack/pack-%s.idx", sha1_to_hex(sha1));
1893        url = strbuf_detach(&buf, NULL);
1894
1895        strbuf_addf(&buf, "%s.temp", sha1_pack_index_name(sha1));
1896        tmp = strbuf_detach(&buf, NULL);
1897
1898        if (http_get_file(url, tmp, NULL) != HTTP_OK) {
1899                error("Unable to get pack index %s", url);
1900                FREE_AND_NULL(tmp);
1901        }
1902
1903        free(url);
1904        return tmp;
1905}
1906
1907static int fetch_and_setup_pack_index(struct packed_git **packs_head,
1908        unsigned char *sha1, const char *base_url)
1909{
1910        struct packed_git *new_pack;
1911        char *tmp_idx = NULL;
1912        int ret;
1913
1914        if (has_pack_index(sha1)) {
1915                new_pack = parse_pack_index(sha1, sha1_pack_index_name(sha1));
1916                if (!new_pack)
1917                        return -1; /* parse_pack_index() already issued error message */
1918                goto add_pack;
1919        }
1920
1921        tmp_idx = fetch_pack_index(sha1, base_url);
1922        if (!tmp_idx)
1923                return -1;
1924
1925        new_pack = parse_pack_index(sha1, tmp_idx);
1926        if (!new_pack) {
1927                unlink(tmp_idx);
1928                free(tmp_idx);
1929
1930                return -1; /* parse_pack_index() already issued error message */
1931        }
1932
1933        ret = verify_pack_index(new_pack);
1934        if (!ret) {
1935                close_pack_index(new_pack);
1936                ret = finalize_object_file(tmp_idx, sha1_pack_index_name(sha1));
1937        }
1938        free(tmp_idx);
1939        if (ret)
1940                return -1;
1941
1942add_pack:
1943        new_pack->next = *packs_head;
1944        *packs_head = new_pack;
1945        return 0;
1946}
1947
1948int http_get_info_packs(const char *base_url, struct packed_git **packs_head)
1949{
1950        struct http_get_options options = {0};
1951        int ret = 0, i = 0;
1952        char *url, *data;
1953        struct strbuf buf = STRBUF_INIT;
1954        unsigned char sha1[20];
1955
1956        end_url_with_slash(&buf, base_url);
1957        strbuf_addstr(&buf, "objects/info/packs");
1958        url = strbuf_detach(&buf, NULL);
1959
1960        options.no_cache = 1;
1961        ret = http_get_strbuf(url, &buf, &options);
1962        if (ret != HTTP_OK)
1963                goto cleanup;
1964
1965        data = buf.buf;
1966        while (i < buf.len) {
1967                switch (data[i]) {
1968                case 'P':
1969                        i++;
1970                        if (i + 52 <= buf.len &&
1971                            starts_with(data + i, " pack-") &&
1972                            starts_with(data + i + 46, ".pack\n")) {
1973                                get_sha1_hex(data + i + 6, sha1);
1974                                fetch_and_setup_pack_index(packs_head, sha1,
1975                                                      base_url);
1976                                i += 51;
1977                                break;
1978                        }
1979                default:
1980                        while (i < buf.len && data[i] != '\n')
1981                                i++;
1982                }
1983                i++;
1984        }
1985
1986cleanup:
1987        free(url);
1988        return ret;
1989}
1990
1991void release_http_pack_request(struct http_pack_request *preq)
1992{
1993        if (preq->packfile != NULL) {
1994                fclose(preq->packfile);
1995                preq->packfile = NULL;
1996        }
1997        preq->slot = NULL;
1998        free(preq->url);
1999        free(preq);
2000}
2001
2002int finish_http_pack_request(struct http_pack_request *preq)
2003{
2004        struct packed_git **lst;
2005        struct packed_git *p = preq->target;
2006        char *tmp_idx;
2007        size_t len;
2008        struct child_process ip = CHILD_PROCESS_INIT;
2009        const char *ip_argv[8];
2010
2011        close_pack_index(p);
2012
2013        fclose(preq->packfile);
2014        preq->packfile = NULL;
2015
2016        lst = preq->lst;
2017        while (*lst != p)
2018                lst = &((*lst)->next);
2019        *lst = (*lst)->next;
2020
2021        if (!strip_suffix(preq->tmpfile, ".pack.temp", &len))
2022                die("BUG: pack tmpfile does not end in .pack.temp?");
2023        tmp_idx = xstrfmt("%.*s.idx.temp", (int)len, preq->tmpfile);
2024
2025        ip_argv[0] = "index-pack";
2026        ip_argv[1] = "-o";
2027        ip_argv[2] = tmp_idx;
2028        ip_argv[3] = preq->tmpfile;
2029        ip_argv[4] = NULL;
2030
2031        ip.argv = ip_argv;
2032        ip.git_cmd = 1;
2033        ip.no_stdin = 1;
2034        ip.no_stdout = 1;
2035
2036        if (run_command(&ip)) {
2037                unlink(preq->tmpfile);
2038                unlink(tmp_idx);
2039                free(tmp_idx);
2040                return -1;
2041        }
2042
2043        unlink(sha1_pack_index_name(p->sha1));
2044
2045        if (finalize_object_file(preq->tmpfile, sha1_pack_name(p->sha1))
2046         || finalize_object_file(tmp_idx, sha1_pack_index_name(p->sha1))) {
2047                free(tmp_idx);
2048                return -1;
2049        }
2050
2051        install_packed_git(p);
2052        free(tmp_idx);
2053        return 0;
2054}
2055
2056struct http_pack_request *new_http_pack_request(
2057        struct packed_git *target, const char *base_url)
2058{
2059        off_t prev_posn = 0;
2060        struct strbuf buf = STRBUF_INIT;
2061        struct http_pack_request *preq;
2062
2063        preq = xcalloc(1, sizeof(*preq));
2064        preq->target = target;
2065
2066        end_url_with_slash(&buf, base_url);
2067        strbuf_addf(&buf, "objects/pack/pack-%s.pack",
2068                sha1_to_hex(target->sha1));
2069        preq->url = strbuf_detach(&buf, NULL);
2070
2071        snprintf(preq->tmpfile, sizeof(preq->tmpfile), "%s.temp",
2072                sha1_pack_name(target->sha1));
2073        preq->packfile = fopen(preq->tmpfile, "a");
2074        if (!preq->packfile) {
2075                error("Unable to open local file %s for pack",
2076                      preq->tmpfile);
2077                goto abort;
2078        }
2079
2080        preq->slot = get_active_slot();
2081        curl_easy_setopt(preq->slot->curl, CURLOPT_FILE, preq->packfile);
2082        curl_easy_setopt(preq->slot->curl, CURLOPT_WRITEFUNCTION, fwrite);
2083        curl_easy_setopt(preq->slot->curl, CURLOPT_URL, preq->url);
2084        curl_easy_setopt(preq->slot->curl, CURLOPT_HTTPHEADER,
2085                no_pragma_header);
2086
2087        /*
2088         * If there is data present from a previous transfer attempt,
2089         * resume where it left off
2090         */
2091        prev_posn = ftello(preq->packfile);
2092        if (prev_posn>0) {
2093                if (http_is_verbose)
2094                        fprintf(stderr,
2095                                "Resuming fetch of pack %s at byte %"PRIuMAX"\n",
2096                                sha1_to_hex(target->sha1), (uintmax_t)prev_posn);
2097                http_opt_request_remainder(preq->slot->curl, prev_posn);
2098        }
2099
2100        return preq;
2101
2102abort:
2103        free(preq->url);
2104        free(preq);
2105        return NULL;
2106}
2107
2108/* Helpers for fetching objects (loose) */
2109static size_t fwrite_sha1_file(char *ptr, size_t eltsize, size_t nmemb,
2110                               void *data)
2111{
2112        unsigned char expn[4096];
2113        size_t size = eltsize * nmemb;
2114        int posn = 0;
2115        struct http_object_request *freq = data;
2116        struct active_request_slot *slot = freq->slot;
2117
2118        if (slot) {
2119                CURLcode c = curl_easy_getinfo(slot->curl, CURLINFO_HTTP_CODE,
2120                                                &slot->http_code);
2121                if (c != CURLE_OK)
2122                        die("BUG: curl_easy_getinfo for HTTP code failed: %s",
2123                                curl_easy_strerror(c));
2124                if (slot->http_code >= 300)
2125                        return size;
2126        }
2127
2128        do {
2129                ssize_t retval = xwrite(freq->localfile,
2130                                        (char *) ptr + posn, size - posn);
2131                if (retval < 0)
2132                        return posn;
2133                posn += retval;
2134        } while (posn < size);
2135
2136        freq->stream.avail_in = size;
2137        freq->stream.next_in = (void *)ptr;
2138        do {
2139                freq->stream.next_out = expn;
2140                freq->stream.avail_out = sizeof(expn);
2141                freq->zret = git_inflate(&freq->stream, Z_SYNC_FLUSH);
2142                git_SHA1_Update(&freq->c, expn,
2143                                sizeof(expn) - freq->stream.avail_out);
2144        } while (freq->stream.avail_in && freq->zret == Z_OK);
2145        return size;
2146}
2147
2148struct http_object_request *new_http_object_request(const char *base_url,
2149        unsigned char *sha1)
2150{
2151        char *hex = sha1_to_hex(sha1);
2152        const char *filename;
2153        char prevfile[PATH_MAX];
2154        int prevlocal;
2155        char prev_buf[PREV_BUF_SIZE];
2156        ssize_t prev_read = 0;
2157        off_t prev_posn = 0;
2158        struct http_object_request *freq;
2159
2160        freq = xcalloc(1, sizeof(*freq));
2161        hashcpy(freq->sha1, sha1);
2162        freq->localfile = -1;
2163
2164        filename = sha1_file_name(sha1);
2165        snprintf(freq->tmpfile, sizeof(freq->tmpfile),
2166                 "%s.temp", filename);
2167
2168        snprintf(prevfile, sizeof(prevfile), "%s.prev", filename);
2169        unlink_or_warn(prevfile);
2170        rename(freq->tmpfile, prevfile);
2171        unlink_or_warn(freq->tmpfile);
2172
2173        if (freq->localfile != -1)
2174                error("fd leakage in start: %d", freq->localfile);
2175        freq->localfile = open(freq->tmpfile,
2176                               O_WRONLY | O_CREAT | O_EXCL, 0666);
2177        /*
2178         * This could have failed due to the "lazy directory creation";
2179         * try to mkdir the last path component.
2180         */
2181        if (freq->localfile < 0 && errno == ENOENT) {
2182                char *dir = strrchr(freq->tmpfile, '/');
2183                if (dir) {
2184                        *dir = 0;
2185                        mkdir(freq->tmpfile, 0777);
2186                        *dir = '/';
2187                }
2188                freq->localfile = open(freq->tmpfile,
2189                                       O_WRONLY | O_CREAT | O_EXCL, 0666);
2190        }
2191
2192        if (freq->localfile < 0) {
2193                error_errno("Couldn't create temporary file %s", freq->tmpfile);
2194                goto abort;
2195        }
2196
2197        git_inflate_init(&freq->stream);
2198
2199        git_SHA1_Init(&freq->c);
2200
2201        freq->url = get_remote_object_url(base_url, hex, 0);
2202
2203        /*
2204         * If a previous temp file is present, process what was already
2205         * fetched.
2206         */
2207        prevlocal = open(prevfile, O_RDONLY);
2208        if (prevlocal != -1) {
2209                do {
2210                        prev_read = xread(prevlocal, prev_buf, PREV_BUF_SIZE);
2211                        if (prev_read>0) {
2212                                if (fwrite_sha1_file(prev_buf,
2213                                                     1,
2214                                                     prev_read,
2215                                                     freq) == prev_read) {
2216                                        prev_posn += prev_read;
2217                                } else {
2218                                        prev_read = -1;
2219                                }
2220                        }
2221                } while (prev_read > 0);
2222                close(prevlocal);
2223        }
2224        unlink_or_warn(prevfile);
2225
2226        /*
2227         * Reset inflate/SHA1 if there was an error reading the previous temp
2228         * file; also rewind to the beginning of the local file.
2229         */
2230        if (prev_read == -1) {
2231                memset(&freq->stream, 0, sizeof(freq->stream));
2232                git_inflate_init(&freq->stream);
2233                git_SHA1_Init(&freq->c);
2234                if (prev_posn>0) {
2235                        prev_posn = 0;
2236                        lseek(freq->localfile, 0, SEEK_SET);
2237                        if (ftruncate(freq->localfile, 0) < 0) {
2238                                error_errno("Couldn't truncate temporary file %s",
2239                                            freq->tmpfile);
2240                                goto abort;
2241                        }
2242                }
2243        }
2244
2245        freq->slot = get_active_slot();
2246
2247        curl_easy_setopt(freq->slot->curl, CURLOPT_FILE, freq);
2248        curl_easy_setopt(freq->slot->curl, CURLOPT_FAILONERROR, 0);
2249        curl_easy_setopt(freq->slot->curl, CURLOPT_WRITEFUNCTION, fwrite_sha1_file);
2250        curl_easy_setopt(freq->slot->curl, CURLOPT_ERRORBUFFER, freq->errorstr);
2251        curl_easy_setopt(freq->slot->curl, CURLOPT_URL, freq->url);
2252        curl_easy_setopt(freq->slot->curl, CURLOPT_HTTPHEADER, no_pragma_header);
2253
2254        /*
2255         * If we have successfully processed data from a previous fetch
2256         * attempt, only fetch the data we don't already have.
2257         */
2258        if (prev_posn>0) {
2259                if (http_is_verbose)
2260                        fprintf(stderr,
2261                                "Resuming fetch of object %s at byte %"PRIuMAX"\n",
2262                                hex, (uintmax_t)prev_posn);
2263                http_opt_request_remainder(freq->slot->curl, prev_posn);
2264        }
2265
2266        return freq;
2267
2268abort:
2269        free(freq->url);
2270        free(freq);
2271        return NULL;
2272}
2273
2274void process_http_object_request(struct http_object_request *freq)
2275{
2276        if (freq->slot == NULL)
2277                return;
2278        freq->curl_result = freq->slot->curl_result;
2279        freq->http_code = freq->slot->http_code;
2280        freq->slot = NULL;
2281}
2282
2283int finish_http_object_request(struct http_object_request *freq)
2284{
2285        struct stat st;
2286
2287        close(freq->localfile);
2288        freq->localfile = -1;
2289
2290        process_http_object_request(freq);
2291
2292        if (freq->http_code == 416) {
2293                warning("requested range invalid; we may already have all the data.");
2294        } else if (freq->curl_result != CURLE_OK) {
2295                if (stat(freq->tmpfile, &st) == 0)
2296                        if (st.st_size == 0)
2297                                unlink_or_warn(freq->tmpfile);
2298                return -1;
2299        }
2300
2301        git_inflate_end(&freq->stream);
2302        git_SHA1_Final(freq->real_sha1, &freq->c);
2303        if (freq->zret != Z_STREAM_END) {
2304                unlink_or_warn(freq->tmpfile);
2305                return -1;
2306        }
2307        if (hashcmp(freq->sha1, freq->real_sha1)) {
2308                unlink_or_warn(freq->tmpfile);
2309                return -1;
2310        }
2311        freq->rename =
2312                finalize_object_file(freq->tmpfile, sha1_file_name(freq->sha1));
2313
2314        return freq->rename;
2315}
2316
2317void abort_http_object_request(struct http_object_request *freq)
2318{
2319        unlink_or_warn(freq->tmpfile);
2320
2321        release_http_object_request(freq);
2322}
2323
2324void release_http_object_request(struct http_object_request *freq)
2325{
2326        if (freq->localfile != -1) {
2327                close(freq->localfile);
2328                freq->localfile = -1;
2329        }
2330        if (freq->url != NULL) {
2331                FREE_AND_NULL(freq->url);
2332        }
2333        if (freq->slot != NULL) {
2334                freq->slot->callback_func = NULL;
2335                freq->slot->callback_data = NULL;
2336                release_active_slot(freq->slot);
2337                freq->slot = NULL;
2338        }
2339}