088cf70bf039d57adf5c9179238b6e4d138a51cb
   1#include "git-compat-util.h"
   2#include "http.h"
   3#include "config.h"
   4#include "pack.h"
   5#include "sideband.h"
   6#include "run-command.h"
   7#include "url.h"
   8#include "urlmatch.h"
   9#include "credential.h"
  10#include "version.h"
  11#include "pkt-line.h"
  12#include "gettext.h"
  13#include "transport.h"
  14#include "packfile.h"
  15#include "protocol.h"
  16#include "string-list.h"
  17
  18static struct trace_key trace_curl = TRACE_KEY_INIT(CURL);
  19static struct string_list cookies_to_redact = STRING_LIST_INIT_DUP;
  20#if LIBCURL_VERSION_NUM >= 0x070a08
  21long int git_curl_ipresolve = CURL_IPRESOLVE_WHATEVER;
  22#else
  23long int git_curl_ipresolve;
  24#endif
  25int active_requests;
  26int http_is_verbose;
  27ssize_t http_post_buffer = 16 * LARGE_PACKET_MAX;
  28
  29#if LIBCURL_VERSION_NUM >= 0x070a06
  30#define LIBCURL_CAN_HANDLE_AUTH_ANY
  31#endif
  32
  33static int min_curl_sessions = 1;
  34static int curl_session_count;
  35#ifdef USE_CURL_MULTI
  36static int max_requests = -1;
  37static CURLM *curlm;
  38#endif
  39#ifndef NO_CURL_EASY_DUPHANDLE
  40static CURL *curl_default;
  41#endif
  42
  43#define PREV_BUF_SIZE 4096
  44
  45char curl_errorstr[CURL_ERROR_SIZE];
  46
  47static int curl_ssl_verify = -1;
  48static int curl_ssl_try;
  49static const char *ssl_cert;
  50static const char *ssl_cipherlist;
  51static const char *ssl_version;
  52static struct {
  53        const char *name;
  54        long ssl_version;
  55} sslversions[] = {
  56        { "sslv2", CURL_SSLVERSION_SSLv2 },
  57        { "sslv3", CURL_SSLVERSION_SSLv3 },
  58        { "tlsv1", CURL_SSLVERSION_TLSv1 },
  59#if LIBCURL_VERSION_NUM >= 0x072200
  60        { "tlsv1.0", CURL_SSLVERSION_TLSv1_0 },
  61        { "tlsv1.1", CURL_SSLVERSION_TLSv1_1 },
  62        { "tlsv1.2", CURL_SSLVERSION_TLSv1_2 },
  63#endif
  64};
  65#if LIBCURL_VERSION_NUM >= 0x070903
  66static const char *ssl_key;
  67#endif
  68#if LIBCURL_VERSION_NUM >= 0x070908
  69static const char *ssl_capath;
  70#endif
  71#if LIBCURL_VERSION_NUM >= 0x072c00
  72static const char *ssl_pinnedkey;
  73#endif
  74static const char *ssl_cainfo;
  75static long curl_low_speed_limit = -1;
  76static long curl_low_speed_time = -1;
  77static int curl_ftp_no_epsv;
  78static const char *curl_http_proxy;
  79static const char *curl_no_proxy;
  80static const char *http_proxy_authmethod;
  81static struct {
  82        const char *name;
  83        long curlauth_param;
  84} proxy_authmethods[] = {
  85        { "basic", CURLAUTH_BASIC },
  86        { "digest", CURLAUTH_DIGEST },
  87        { "negotiate", CURLAUTH_GSSNEGOTIATE },
  88        { "ntlm", CURLAUTH_NTLM },
  89#ifdef LIBCURL_CAN_HANDLE_AUTH_ANY
  90        { "anyauth", CURLAUTH_ANY },
  91#endif
  92        /*
  93         * CURLAUTH_DIGEST_IE has no corresponding command-line option in
  94         * curl(1) and is not included in CURLAUTH_ANY, so we leave it out
  95         * here, too
  96         */
  97};
  98#ifdef CURLGSSAPI_DELEGATION_FLAG
  99static const char *curl_deleg;
 100static struct {
 101        const char *name;
 102        long curl_deleg_param;
 103} curl_deleg_levels[] = {
 104        { "none", CURLGSSAPI_DELEGATION_NONE },
 105        { "policy", CURLGSSAPI_DELEGATION_POLICY_FLAG },
 106        { "always", CURLGSSAPI_DELEGATION_FLAG },
 107};
 108#endif
 109
 110static struct credential proxy_auth = CREDENTIAL_INIT;
 111static const char *curl_proxyuserpwd;
 112static const char *curl_cookie_file;
 113static int curl_save_cookies;
 114struct credential http_auth = CREDENTIAL_INIT;
 115static int http_proactive_auth;
 116static const char *user_agent;
 117static int curl_empty_auth = -1;
 118
 119enum http_follow_config http_follow_config = HTTP_FOLLOW_INITIAL;
 120
 121#if LIBCURL_VERSION_NUM >= 0x071700
 122/* Use CURLOPT_KEYPASSWD as is */
 123#elif LIBCURL_VERSION_NUM >= 0x070903
 124#define CURLOPT_KEYPASSWD CURLOPT_SSLKEYPASSWD
 125#else
 126#define CURLOPT_KEYPASSWD CURLOPT_SSLCERTPASSWD
 127#endif
 128
 129static struct credential cert_auth = CREDENTIAL_INIT;
 130static int ssl_cert_password_required;
 131#ifdef LIBCURL_CAN_HANDLE_AUTH_ANY
 132static unsigned long http_auth_methods = CURLAUTH_ANY;
 133static int http_auth_methods_restricted;
 134/* Modes for which empty_auth cannot actually help us. */
 135static unsigned long empty_auth_useless =
 136        CURLAUTH_BASIC
 137#ifdef CURLAUTH_DIGEST_IE
 138        | CURLAUTH_DIGEST_IE
 139#endif
 140        | CURLAUTH_DIGEST;
 141#endif
 142
 143static struct curl_slist *pragma_header;
 144static struct curl_slist *no_pragma_header;
 145static struct curl_slist *extra_http_headers;
 146
 147static struct active_request_slot *active_queue_head;
 148
 149static char *cached_accept_language;
 150
 151size_t fread_buffer(char *ptr, size_t eltsize, size_t nmemb, void *buffer_)
 152{
 153        size_t size = eltsize * nmemb;
 154        struct buffer *buffer = buffer_;
 155
 156        if (size > buffer->buf.len - buffer->posn)
 157                size = buffer->buf.len - buffer->posn;
 158        memcpy(ptr, buffer->buf.buf + buffer->posn, size);
 159        buffer->posn += size;
 160
 161        return size;
 162}
 163
 164#ifndef NO_CURL_IOCTL
 165curlioerr ioctl_buffer(CURL *handle, int cmd, void *clientp)
 166{
 167        struct buffer *buffer = clientp;
 168
 169        switch (cmd) {
 170        case CURLIOCMD_NOP:
 171                return CURLIOE_OK;
 172
 173        case CURLIOCMD_RESTARTREAD:
 174                buffer->posn = 0;
 175                return CURLIOE_OK;
 176
 177        default:
 178                return CURLIOE_UNKNOWNCMD;
 179        }
 180}
 181#endif
 182
 183size_t fwrite_buffer(char *ptr, size_t eltsize, size_t nmemb, void *buffer_)
 184{
 185        size_t size = eltsize * nmemb;
 186        struct strbuf *buffer = buffer_;
 187
 188        strbuf_add(buffer, ptr, size);
 189        return size;
 190}
 191
 192size_t fwrite_null(char *ptr, size_t eltsize, size_t nmemb, void *strbuf)
 193{
 194        return eltsize * nmemb;
 195}
 196
 197static void closedown_active_slot(struct active_request_slot *slot)
 198{
 199        active_requests--;
 200        slot->in_use = 0;
 201}
 202
 203static void finish_active_slot(struct active_request_slot *slot)
 204{
 205        closedown_active_slot(slot);
 206        curl_easy_getinfo(slot->curl, CURLINFO_HTTP_CODE, &slot->http_code);
 207
 208        if (slot->finished != NULL)
 209                (*slot->finished) = 1;
 210
 211        /* Store slot results so they can be read after the slot is reused */
 212        if (slot->results != NULL) {
 213                slot->results->curl_result = slot->curl_result;
 214                slot->results->http_code = slot->http_code;
 215#if LIBCURL_VERSION_NUM >= 0x070a08
 216                curl_easy_getinfo(slot->curl, CURLINFO_HTTPAUTH_AVAIL,
 217                                  &slot->results->auth_avail);
 218#else
 219                slot->results->auth_avail = 0;
 220#endif
 221
 222                curl_easy_getinfo(slot->curl, CURLINFO_HTTP_CONNECTCODE,
 223                        &slot->results->http_connectcode);
 224        }
 225
 226        /* Run callback if appropriate */
 227        if (slot->callback_func != NULL)
 228                slot->callback_func(slot->callback_data);
 229}
 230
 231static void xmulti_remove_handle(struct active_request_slot *slot)
 232{
 233#ifdef USE_CURL_MULTI
 234        curl_multi_remove_handle(curlm, slot->curl);
 235#endif
 236}
 237
 238#ifdef USE_CURL_MULTI
 239static void process_curl_messages(void)
 240{
 241        int num_messages;
 242        struct active_request_slot *slot;
 243        CURLMsg *curl_message = curl_multi_info_read(curlm, &num_messages);
 244
 245        while (curl_message != NULL) {
 246                if (curl_message->msg == CURLMSG_DONE) {
 247                        int curl_result = curl_message->data.result;
 248                        slot = active_queue_head;
 249                        while (slot != NULL &&
 250                               slot->curl != curl_message->easy_handle)
 251                                slot = slot->next;
 252                        if (slot != NULL) {
 253                                xmulti_remove_handle(slot);
 254                                slot->curl_result = curl_result;
 255                                finish_active_slot(slot);
 256                        } else {
 257                                fprintf(stderr, "Received DONE message for unknown request!\n");
 258                        }
 259                } else {
 260                        fprintf(stderr, "Unknown CURL message received: %d\n",
 261                                (int)curl_message->msg);
 262                }
 263                curl_message = curl_multi_info_read(curlm, &num_messages);
 264        }
 265}
 266#endif
 267
 268static int http_options(const char *var, const char *value, void *cb)
 269{
 270        if (!strcmp("http.sslverify", var)) {
 271                curl_ssl_verify = git_config_bool(var, value);
 272                return 0;
 273        }
 274        if (!strcmp("http.sslcipherlist", var))
 275                return git_config_string(&ssl_cipherlist, var, value);
 276        if (!strcmp("http.sslversion", var))
 277                return git_config_string(&ssl_version, var, value);
 278        if (!strcmp("http.sslcert", var))
 279                return git_config_pathname(&ssl_cert, var, value);
 280#if LIBCURL_VERSION_NUM >= 0x070903
 281        if (!strcmp("http.sslkey", var))
 282                return git_config_pathname(&ssl_key, var, value);
 283#endif
 284#if LIBCURL_VERSION_NUM >= 0x070908
 285        if (!strcmp("http.sslcapath", var))
 286                return git_config_pathname(&ssl_capath, var, value);
 287#endif
 288        if (!strcmp("http.sslcainfo", var))
 289                return git_config_pathname(&ssl_cainfo, var, value);
 290        if (!strcmp("http.sslcertpasswordprotected", var)) {
 291                ssl_cert_password_required = git_config_bool(var, value);
 292                return 0;
 293        }
 294        if (!strcmp("http.ssltry", var)) {
 295                curl_ssl_try = git_config_bool(var, value);
 296                return 0;
 297        }
 298        if (!strcmp("http.minsessions", var)) {
 299                min_curl_sessions = git_config_int(var, value);
 300#ifndef USE_CURL_MULTI
 301                if (min_curl_sessions > 1)
 302                        min_curl_sessions = 1;
 303#endif
 304                return 0;
 305        }
 306#ifdef USE_CURL_MULTI
 307        if (!strcmp("http.maxrequests", var)) {
 308                max_requests = git_config_int(var, value);
 309                return 0;
 310        }
 311#endif
 312        if (!strcmp("http.lowspeedlimit", var)) {
 313                curl_low_speed_limit = (long)git_config_int(var, value);
 314                return 0;
 315        }
 316        if (!strcmp("http.lowspeedtime", var)) {
 317                curl_low_speed_time = (long)git_config_int(var, value);
 318                return 0;
 319        }
 320
 321        if (!strcmp("http.noepsv", var)) {
 322                curl_ftp_no_epsv = git_config_bool(var, value);
 323                return 0;
 324        }
 325        if (!strcmp("http.proxy", var))
 326                return git_config_string(&curl_http_proxy, var, value);
 327
 328        if (!strcmp("http.proxyauthmethod", var))
 329                return git_config_string(&http_proxy_authmethod, var, value);
 330
 331        if (!strcmp("http.cookiefile", var))
 332                return git_config_pathname(&curl_cookie_file, var, value);
 333        if (!strcmp("http.savecookies", var)) {
 334                curl_save_cookies = git_config_bool(var, value);
 335                return 0;
 336        }
 337
 338        if (!strcmp("http.postbuffer", var)) {
 339                http_post_buffer = git_config_ssize_t(var, value);
 340                if (http_post_buffer < 0)
 341                        warning(_("negative value for http.postbuffer; defaulting to %d"), LARGE_PACKET_MAX);
 342                if (http_post_buffer < LARGE_PACKET_MAX)
 343                        http_post_buffer = LARGE_PACKET_MAX;
 344                return 0;
 345        }
 346
 347        if (!strcmp("http.useragent", var))
 348                return git_config_string(&user_agent, var, value);
 349
 350        if (!strcmp("http.emptyauth", var)) {
 351                if (value && !strcmp("auto", value))
 352                        curl_empty_auth = -1;
 353                else
 354                        curl_empty_auth = git_config_bool(var, value);
 355                return 0;
 356        }
 357
 358        if (!strcmp("http.delegation", var)) {
 359#ifdef CURLGSSAPI_DELEGATION_FLAG
 360                return git_config_string(&curl_deleg, var, value);
 361#else
 362                warning(_("Delegation control is not supported with cURL < 7.22.0"));
 363                return 0;
 364#endif
 365        }
 366
 367        if (!strcmp("http.pinnedpubkey", var)) {
 368#if LIBCURL_VERSION_NUM >= 0x072c00
 369                return git_config_pathname(&ssl_pinnedkey, var, value);
 370#else
 371                warning(_("Public key pinning not supported with cURL < 7.44.0"));
 372                return 0;
 373#endif
 374        }
 375
 376        if (!strcmp("http.extraheader", var)) {
 377                if (!value) {
 378                        return config_error_nonbool(var);
 379                } else if (!*value) {
 380                        curl_slist_free_all(extra_http_headers);
 381                        extra_http_headers = NULL;
 382                } else {
 383                        extra_http_headers =
 384                                curl_slist_append(extra_http_headers, value);
 385                }
 386                return 0;
 387        }
 388
 389        if (!strcmp("http.followredirects", var)) {
 390                if (value && !strcmp(value, "initial"))
 391                        http_follow_config = HTTP_FOLLOW_INITIAL;
 392                else if (git_config_bool(var, value))
 393                        http_follow_config = HTTP_FOLLOW_ALWAYS;
 394                else
 395                        http_follow_config = HTTP_FOLLOW_NONE;
 396                return 0;
 397        }
 398
 399        /* Fall back on the default ones */
 400        return git_default_config(var, value, cb);
 401}
 402
 403static int curl_empty_auth_enabled(void)
 404{
 405        if (curl_empty_auth >= 0)
 406                return curl_empty_auth;
 407
 408#ifndef LIBCURL_CAN_HANDLE_AUTH_ANY
 409        /*
 410         * Our libcurl is too old to do AUTH_ANY in the first place;
 411         * just default to turning the feature off.
 412         */
 413#else
 414        /*
 415         * In the automatic case, kick in the empty-auth
 416         * hack as long as we would potentially try some
 417         * method more exotic than "Basic" or "Digest".
 418         *
 419         * But only do this when this is our second or
 420         * subsequent request, as by then we know what
 421         * methods are available.
 422         */
 423        if (http_auth_methods_restricted &&
 424            (http_auth_methods & ~empty_auth_useless))
 425                return 1;
 426#endif
 427        return 0;
 428}
 429
 430static void init_curl_http_auth(CURL *result)
 431{
 432        if (!http_auth.username || !*http_auth.username) {
 433                if (curl_empty_auth_enabled())
 434                        curl_easy_setopt(result, CURLOPT_USERPWD, ":");
 435                return;
 436        }
 437
 438        credential_fill(&http_auth);
 439
 440#if LIBCURL_VERSION_NUM >= 0x071301
 441        curl_easy_setopt(result, CURLOPT_USERNAME, http_auth.username);
 442        curl_easy_setopt(result, CURLOPT_PASSWORD, http_auth.password);
 443#else
 444        {
 445                static struct strbuf up = STRBUF_INIT;
 446                /*
 447                 * Note that we assume we only ever have a single set of
 448                 * credentials in a given program run, so we do not have
 449                 * to worry about updating this buffer, only setting its
 450                 * initial value.
 451                 */
 452                if (!up.len)
 453                        strbuf_addf(&up, "%s:%s",
 454                                http_auth.username, http_auth.password);
 455                curl_easy_setopt(result, CURLOPT_USERPWD, up.buf);
 456        }
 457#endif
 458}
 459
 460/* *var must be free-able */
 461static void var_override(const char **var, char *value)
 462{
 463        if (value) {
 464                free((void *)*var);
 465                *var = xstrdup(value);
 466        }
 467}
 468
 469static void set_proxyauth_name_password(CURL *result)
 470{
 471#if LIBCURL_VERSION_NUM >= 0x071301
 472                curl_easy_setopt(result, CURLOPT_PROXYUSERNAME,
 473                        proxy_auth.username);
 474                curl_easy_setopt(result, CURLOPT_PROXYPASSWORD,
 475                        proxy_auth.password);
 476#else
 477                struct strbuf s = STRBUF_INIT;
 478
 479                strbuf_addstr_urlencode(&s, proxy_auth.username, 1);
 480                strbuf_addch(&s, ':');
 481                strbuf_addstr_urlencode(&s, proxy_auth.password, 1);
 482                curl_proxyuserpwd = strbuf_detach(&s, NULL);
 483                curl_easy_setopt(result, CURLOPT_PROXYUSERPWD, curl_proxyuserpwd);
 484#endif
 485}
 486
 487static void init_curl_proxy_auth(CURL *result)
 488{
 489        if (proxy_auth.username) {
 490                if (!proxy_auth.password)
 491                        credential_fill(&proxy_auth);
 492                set_proxyauth_name_password(result);
 493        }
 494
 495        var_override(&http_proxy_authmethod, getenv("GIT_HTTP_PROXY_AUTHMETHOD"));
 496
 497#if LIBCURL_VERSION_NUM >= 0x070a07 /* CURLOPT_PROXYAUTH and CURLAUTH_ANY */
 498        if (http_proxy_authmethod) {
 499                int i;
 500                for (i = 0; i < ARRAY_SIZE(proxy_authmethods); i++) {
 501                        if (!strcmp(http_proxy_authmethod, proxy_authmethods[i].name)) {
 502                                curl_easy_setopt(result, CURLOPT_PROXYAUTH,
 503                                                proxy_authmethods[i].curlauth_param);
 504                                break;
 505                        }
 506                }
 507                if (i == ARRAY_SIZE(proxy_authmethods)) {
 508                        warning("unsupported proxy authentication method %s: using anyauth",
 509                                        http_proxy_authmethod);
 510                        curl_easy_setopt(result, CURLOPT_PROXYAUTH, CURLAUTH_ANY);
 511                }
 512        }
 513        else
 514                curl_easy_setopt(result, CURLOPT_PROXYAUTH, CURLAUTH_ANY);
 515#endif
 516}
 517
 518static int has_cert_password(void)
 519{
 520        if (ssl_cert == NULL || ssl_cert_password_required != 1)
 521                return 0;
 522        if (!cert_auth.password) {
 523                cert_auth.protocol = xstrdup("cert");
 524                cert_auth.username = xstrdup("");
 525                cert_auth.path = xstrdup(ssl_cert);
 526                credential_fill(&cert_auth);
 527        }
 528        return 1;
 529}
 530
 531#if LIBCURL_VERSION_NUM >= 0x071900
 532static void set_curl_keepalive(CURL *c)
 533{
 534        curl_easy_setopt(c, CURLOPT_TCP_KEEPALIVE, 1);
 535}
 536
 537#elif LIBCURL_VERSION_NUM >= 0x071000
 538static int sockopt_callback(void *client, curl_socket_t fd, curlsocktype type)
 539{
 540        int ka = 1;
 541        int rc;
 542        socklen_t len = (socklen_t)sizeof(ka);
 543
 544        if (type != CURLSOCKTYPE_IPCXN)
 545                return 0;
 546
 547        rc = setsockopt(fd, SOL_SOCKET, SO_KEEPALIVE, (void *)&ka, len);
 548        if (rc < 0)
 549                warning_errno("unable to set SO_KEEPALIVE on socket");
 550
 551        return 0; /* CURL_SOCKOPT_OK only exists since curl 7.21.5 */
 552}
 553
 554static void set_curl_keepalive(CURL *c)
 555{
 556        curl_easy_setopt(c, CURLOPT_SOCKOPTFUNCTION, sockopt_callback);
 557}
 558
 559#else
 560static void set_curl_keepalive(CURL *c)
 561{
 562        /* not supported on older curl versions */
 563}
 564#endif
 565
 566static void redact_sensitive_header(struct strbuf *header)
 567{
 568        const char *sensitive_header;
 569
 570        if (skip_prefix(header->buf, "Authorization:", &sensitive_header) ||
 571            skip_prefix(header->buf, "Proxy-Authorization:", &sensitive_header)) {
 572                /* The first token is the type, which is OK to log */
 573                while (isspace(*sensitive_header))
 574                        sensitive_header++;
 575                while (*sensitive_header && !isspace(*sensitive_header))
 576                        sensitive_header++;
 577                /* Everything else is opaque and possibly sensitive */
 578                strbuf_setlen(header,  sensitive_header - header->buf);
 579                strbuf_addstr(header, " <redacted>");
 580        } else if (cookies_to_redact.nr &&
 581                   skip_prefix(header->buf, "Cookie:", &sensitive_header)) {
 582                struct strbuf redacted_header = STRBUF_INIT;
 583                char *cookie;
 584
 585                while (isspace(*sensitive_header))
 586                        sensitive_header++;
 587
 588                /*
 589                 * The contents of header starting from sensitive_header will
 590                 * subsequently be overridden, so it is fine to mutate this
 591                 * string (hence the assignment to "char *").
 592                 */
 593                cookie = (char *) sensitive_header;
 594
 595                while (cookie) {
 596                        char *equals;
 597                        char *semicolon = strstr(cookie, "; ");
 598                        if (semicolon)
 599                                *semicolon = 0;
 600                        equals = strchrnul(cookie, '=');
 601                        if (!equals) {
 602                                /* invalid cookie, just append and continue */
 603                                strbuf_addstr(&redacted_header, cookie);
 604                                continue;
 605                        }
 606                        *equals = 0; /* temporarily set to NUL for lookup */
 607                        if (string_list_lookup(&cookies_to_redact, cookie)) {
 608                                strbuf_addstr(&redacted_header, cookie);
 609                                strbuf_addstr(&redacted_header, "=<redacted>");
 610                        } else {
 611                                *equals = '=';
 612                                strbuf_addstr(&redacted_header, cookie);
 613                        }
 614                        if (semicolon) {
 615                                /*
 616                                 * There are more cookies. (Or, for some
 617                                 * reason, the input string ends in "; ".)
 618                                 */
 619                                strbuf_addstr(&redacted_header, "; ");
 620                                cookie = semicolon + strlen("; ");
 621                        } else {
 622                                cookie = NULL;
 623                        }
 624                }
 625
 626                strbuf_setlen(header, sensitive_header - header->buf);
 627                strbuf_addbuf(header, &redacted_header);
 628        }
 629}
 630
 631static void curl_dump_header(const char *text, unsigned char *ptr, size_t size, int hide_sensitive_header)
 632{
 633        struct strbuf out = STRBUF_INIT;
 634        struct strbuf **headers, **header;
 635
 636        strbuf_addf(&out, "%s, %10.10ld bytes (0x%8.8lx)\n",
 637                text, (long)size, (long)size);
 638        trace_strbuf(&trace_curl, &out);
 639        strbuf_reset(&out);
 640        strbuf_add(&out, ptr, size);
 641        headers = strbuf_split_max(&out, '\n', 0);
 642
 643        for (header = headers; *header; header++) {
 644                if (hide_sensitive_header)
 645                        redact_sensitive_header(*header);
 646                strbuf_insert((*header), 0, text, strlen(text));
 647                strbuf_insert((*header), strlen(text), ": ", 2);
 648                strbuf_rtrim((*header));
 649                strbuf_addch((*header), '\n');
 650                trace_strbuf(&trace_curl, (*header));
 651        }
 652        strbuf_list_free(headers);
 653        strbuf_release(&out);
 654}
 655
 656static void curl_dump_data(const char *text, unsigned char *ptr, size_t size)
 657{
 658        size_t i;
 659        struct strbuf out = STRBUF_INIT;
 660        unsigned int width = 60;
 661
 662        strbuf_addf(&out, "%s, %10.10ld bytes (0x%8.8lx)\n",
 663                text, (long)size, (long)size);
 664        trace_strbuf(&trace_curl, &out);
 665
 666        for (i = 0; i < size; i += width) {
 667                size_t w;
 668
 669                strbuf_reset(&out);
 670                strbuf_addf(&out, "%s: ", text);
 671                for (w = 0; (w < width) && (i + w < size); w++) {
 672                        unsigned char ch = ptr[i + w];
 673
 674                        strbuf_addch(&out,
 675                                       (ch >= 0x20) && (ch < 0x80)
 676                                       ? ch : '.');
 677                }
 678                strbuf_addch(&out, '\n');
 679                trace_strbuf(&trace_curl, &out);
 680        }
 681        strbuf_release(&out);
 682}
 683
 684static int curl_trace(CURL *handle, curl_infotype type, char *data, size_t size, void *userp)
 685{
 686        const char *text;
 687        enum { NO_FILTER = 0, DO_FILTER = 1 };
 688
 689        switch (type) {
 690        case CURLINFO_TEXT:
 691                trace_printf_key(&trace_curl, "== Info: %s", data);
 692                break;
 693        case CURLINFO_HEADER_OUT:
 694                text = "=> Send header";
 695                curl_dump_header(text, (unsigned char *)data, size, DO_FILTER);
 696                break;
 697        case CURLINFO_DATA_OUT:
 698                text = "=> Send data";
 699                curl_dump_data(text, (unsigned char *)data, size);
 700                break;
 701        case CURLINFO_SSL_DATA_OUT:
 702                text = "=> Send SSL data";
 703                curl_dump_data(text, (unsigned char *)data, size);
 704                break;
 705        case CURLINFO_HEADER_IN:
 706                text = "<= Recv header";
 707                curl_dump_header(text, (unsigned char *)data, size, NO_FILTER);
 708                break;
 709        case CURLINFO_DATA_IN:
 710                text = "<= Recv data";
 711                curl_dump_data(text, (unsigned char *)data, size);
 712                break;
 713        case CURLINFO_SSL_DATA_IN:
 714                text = "<= Recv SSL data";
 715                curl_dump_data(text, (unsigned char *)data, size);
 716                break;
 717
 718        default:                /* we ignore unknown types by default */
 719                return 0;
 720        }
 721        return 0;
 722}
 723
 724void setup_curl_trace(CURL *handle)
 725{
 726        if (!trace_want(&trace_curl))
 727                return;
 728        curl_easy_setopt(handle, CURLOPT_VERBOSE, 1L);
 729        curl_easy_setopt(handle, CURLOPT_DEBUGFUNCTION, curl_trace);
 730        curl_easy_setopt(handle, CURLOPT_DEBUGDATA, NULL);
 731}
 732
 733#ifdef CURLPROTO_HTTP
 734static long get_curl_allowed_protocols(int from_user)
 735{
 736        long allowed_protocols = 0;
 737
 738        if (is_transport_allowed("http", from_user))
 739                allowed_protocols |= CURLPROTO_HTTP;
 740        if (is_transport_allowed("https", from_user))
 741                allowed_protocols |= CURLPROTO_HTTPS;
 742        if (is_transport_allowed("ftp", from_user))
 743                allowed_protocols |= CURLPROTO_FTP;
 744        if (is_transport_allowed("ftps", from_user))
 745                allowed_protocols |= CURLPROTO_FTPS;
 746
 747        return allowed_protocols;
 748}
 749#endif
 750
 751static CURL *get_curl_handle(void)
 752{
 753        CURL *result = curl_easy_init();
 754
 755        if (!result)
 756                die("curl_easy_init failed");
 757
 758        if (!curl_ssl_verify) {
 759                curl_easy_setopt(result, CURLOPT_SSL_VERIFYPEER, 0);
 760                curl_easy_setopt(result, CURLOPT_SSL_VERIFYHOST, 0);
 761        } else {
 762                /* Verify authenticity of the peer's certificate */
 763                curl_easy_setopt(result, CURLOPT_SSL_VERIFYPEER, 1);
 764                /* The name in the cert must match whom we tried to connect */
 765                curl_easy_setopt(result, CURLOPT_SSL_VERIFYHOST, 2);
 766        }
 767
 768#if LIBCURL_VERSION_NUM >= 0x070907
 769        curl_easy_setopt(result, CURLOPT_NETRC, CURL_NETRC_OPTIONAL);
 770#endif
 771#ifdef LIBCURL_CAN_HANDLE_AUTH_ANY
 772        curl_easy_setopt(result, CURLOPT_HTTPAUTH, CURLAUTH_ANY);
 773#endif
 774
 775#ifdef CURLGSSAPI_DELEGATION_FLAG
 776        if (curl_deleg) {
 777                int i;
 778                for (i = 0; i < ARRAY_SIZE(curl_deleg_levels); i++) {
 779                        if (!strcmp(curl_deleg, curl_deleg_levels[i].name)) {
 780                                curl_easy_setopt(result, CURLOPT_GSSAPI_DELEGATION,
 781                                                curl_deleg_levels[i].curl_deleg_param);
 782                                break;
 783                        }
 784                }
 785                if (i == ARRAY_SIZE(curl_deleg_levels))
 786                        warning("Unknown delegation method '%s': using default",
 787                                curl_deleg);
 788        }
 789#endif
 790
 791        if (http_proactive_auth)
 792                init_curl_http_auth(result);
 793
 794        if (getenv("GIT_SSL_VERSION"))
 795                ssl_version = getenv("GIT_SSL_VERSION");
 796        if (ssl_version && *ssl_version) {
 797                int i;
 798                for (i = 0; i < ARRAY_SIZE(sslversions); i++) {
 799                        if (!strcmp(ssl_version, sslversions[i].name)) {
 800                                curl_easy_setopt(result, CURLOPT_SSLVERSION,
 801                                                 sslversions[i].ssl_version);
 802                                break;
 803                        }
 804                }
 805                if (i == ARRAY_SIZE(sslversions))
 806                        warning("unsupported ssl version %s: using default",
 807                                ssl_version);
 808        }
 809
 810        if (getenv("GIT_SSL_CIPHER_LIST"))
 811                ssl_cipherlist = getenv("GIT_SSL_CIPHER_LIST");
 812        if (ssl_cipherlist != NULL && *ssl_cipherlist)
 813                curl_easy_setopt(result, CURLOPT_SSL_CIPHER_LIST,
 814                                ssl_cipherlist);
 815
 816        if (ssl_cert != NULL)
 817                curl_easy_setopt(result, CURLOPT_SSLCERT, ssl_cert);
 818        if (has_cert_password())
 819                curl_easy_setopt(result, CURLOPT_KEYPASSWD, cert_auth.password);
 820#if LIBCURL_VERSION_NUM >= 0x070903
 821        if (ssl_key != NULL)
 822                curl_easy_setopt(result, CURLOPT_SSLKEY, ssl_key);
 823#endif
 824#if LIBCURL_VERSION_NUM >= 0x070908
 825        if (ssl_capath != NULL)
 826                curl_easy_setopt(result, CURLOPT_CAPATH, ssl_capath);
 827#endif
 828#if LIBCURL_VERSION_NUM >= 0x072c00
 829        if (ssl_pinnedkey != NULL)
 830                curl_easy_setopt(result, CURLOPT_PINNEDPUBLICKEY, ssl_pinnedkey);
 831#endif
 832        if (ssl_cainfo != NULL)
 833                curl_easy_setopt(result, CURLOPT_CAINFO, ssl_cainfo);
 834
 835        if (curl_low_speed_limit > 0 && curl_low_speed_time > 0) {
 836                curl_easy_setopt(result, CURLOPT_LOW_SPEED_LIMIT,
 837                                 curl_low_speed_limit);
 838                curl_easy_setopt(result, CURLOPT_LOW_SPEED_TIME,
 839                                 curl_low_speed_time);
 840        }
 841
 842        curl_easy_setopt(result, CURLOPT_MAXREDIRS, 20);
 843#if LIBCURL_VERSION_NUM >= 0x071301
 844        curl_easy_setopt(result, CURLOPT_POSTREDIR, CURL_REDIR_POST_ALL);
 845#elif LIBCURL_VERSION_NUM >= 0x071101
 846        curl_easy_setopt(result, CURLOPT_POST301, 1);
 847#endif
 848#ifdef CURLPROTO_HTTP
 849        curl_easy_setopt(result, CURLOPT_REDIR_PROTOCOLS,
 850                         get_curl_allowed_protocols(0));
 851        curl_easy_setopt(result, CURLOPT_PROTOCOLS,
 852                         get_curl_allowed_protocols(-1));
 853#else
 854        warning("protocol restrictions not applied to curl redirects because\n"
 855                "your curl version is too old (>= 7.19.4)");
 856#endif
 857        if (getenv("GIT_CURL_VERBOSE"))
 858                curl_easy_setopt(result, CURLOPT_VERBOSE, 1L);
 859        setup_curl_trace(result);
 860        if (getenv("GIT_REDACT_COOKIES")) {
 861                string_list_split(&cookies_to_redact,
 862                                  getenv("GIT_REDACT_COOKIES"), ',', -1);
 863                string_list_sort(&cookies_to_redact);
 864        }
 865
 866        curl_easy_setopt(result, CURLOPT_USERAGENT,
 867                user_agent ? user_agent : git_user_agent());
 868
 869        if (curl_ftp_no_epsv)
 870                curl_easy_setopt(result, CURLOPT_FTP_USE_EPSV, 0);
 871
 872#ifdef CURLOPT_USE_SSL
 873        if (curl_ssl_try)
 874                curl_easy_setopt(result, CURLOPT_USE_SSL, CURLUSESSL_TRY);
 875#endif
 876
 877        /*
 878         * CURL also examines these variables as a fallback; but we need to query
 879         * them here in order to decide whether to prompt for missing password (cf.
 880         * init_curl_proxy_auth()).
 881         *
 882         * Unlike many other common environment variables, these are historically
 883         * lowercase only. It appears that CURL did not know this and implemented
 884         * only uppercase variants, which was later corrected to take both - with
 885         * the exception of http_proxy, which is lowercase only also in CURL. As
 886         * the lowercase versions are the historical quasi-standard, they take
 887         * precedence here, as in CURL.
 888         */
 889        if (!curl_http_proxy) {
 890                if (http_auth.protocol && !strcmp(http_auth.protocol, "https")) {
 891                        var_override(&curl_http_proxy, getenv("HTTPS_PROXY"));
 892                        var_override(&curl_http_proxy, getenv("https_proxy"));
 893                } else {
 894                        var_override(&curl_http_proxy, getenv("http_proxy"));
 895                }
 896                if (!curl_http_proxy) {
 897                        var_override(&curl_http_proxy, getenv("ALL_PROXY"));
 898                        var_override(&curl_http_proxy, getenv("all_proxy"));
 899                }
 900        }
 901
 902        if (curl_http_proxy && curl_http_proxy[0] == '\0') {
 903                /*
 904                 * Handle case with the empty http.proxy value here to keep
 905                 * common code clean.
 906                 * NB: empty option disables proxying at all.
 907                 */
 908                curl_easy_setopt(result, CURLOPT_PROXY, "");
 909        } else if (curl_http_proxy) {
 910#if LIBCURL_VERSION_NUM >= 0x071800
 911                if (starts_with(curl_http_proxy, "socks5h"))
 912                        curl_easy_setopt(result,
 913                                CURLOPT_PROXYTYPE, CURLPROXY_SOCKS5_HOSTNAME);
 914                else if (starts_with(curl_http_proxy, "socks5"))
 915                        curl_easy_setopt(result,
 916                                CURLOPT_PROXYTYPE, CURLPROXY_SOCKS5);
 917                else if (starts_with(curl_http_proxy, "socks4a"))
 918                        curl_easy_setopt(result,
 919                                CURLOPT_PROXYTYPE, CURLPROXY_SOCKS4A);
 920                else if (starts_with(curl_http_proxy, "socks"))
 921                        curl_easy_setopt(result,
 922                                CURLOPT_PROXYTYPE, CURLPROXY_SOCKS4);
 923#endif
 924#if LIBCURL_VERSION_NUM >= 0x073400
 925                else if (starts_with(curl_http_proxy, "https"))
 926                        curl_easy_setopt(result,
 927                                CURLOPT_PROXYTYPE, CURLPROXY_HTTPS);
 928#endif
 929                if (strstr(curl_http_proxy, "://"))
 930                        credential_from_url(&proxy_auth, curl_http_proxy);
 931                else {
 932                        struct strbuf url = STRBUF_INIT;
 933                        strbuf_addf(&url, "http://%s", curl_http_proxy);
 934                        credential_from_url(&proxy_auth, url.buf);
 935                        strbuf_release(&url);
 936                }
 937
 938                if (!proxy_auth.host)
 939                        die("Invalid proxy URL '%s'", curl_http_proxy);
 940
 941                curl_easy_setopt(result, CURLOPT_PROXY, proxy_auth.host);
 942#if LIBCURL_VERSION_NUM >= 0x071304
 943                var_override(&curl_no_proxy, getenv("NO_PROXY"));
 944                var_override(&curl_no_proxy, getenv("no_proxy"));
 945                curl_easy_setopt(result, CURLOPT_NOPROXY, curl_no_proxy);
 946#endif
 947        }
 948        init_curl_proxy_auth(result);
 949
 950        set_curl_keepalive(result);
 951
 952        return result;
 953}
 954
 955static void set_from_env(const char **var, const char *envname)
 956{
 957        const char *val = getenv(envname);
 958        if (val)
 959                *var = val;
 960}
 961
 962static void protocol_http_header(void)
 963{
 964        if (get_protocol_version_config() > 0) {
 965                struct strbuf protocol_header = STRBUF_INIT;
 966
 967                strbuf_addf(&protocol_header, GIT_PROTOCOL_HEADER ": version=%d",
 968                            get_protocol_version_config());
 969
 970
 971                extra_http_headers = curl_slist_append(extra_http_headers,
 972                                                       protocol_header.buf);
 973                strbuf_release(&protocol_header);
 974        }
 975}
 976
 977void http_init(struct remote *remote, const char *url, int proactive_auth)
 978{
 979        char *low_speed_limit;
 980        char *low_speed_time;
 981        char *normalized_url;
 982        struct urlmatch_config config = { STRING_LIST_INIT_DUP };
 983
 984        config.section = "http";
 985        config.key = NULL;
 986        config.collect_fn = http_options;
 987        config.cascade_fn = git_default_config;
 988        config.cb = NULL;
 989
 990        http_is_verbose = 0;
 991        normalized_url = url_normalize(url, &config.url);
 992
 993        git_config(urlmatch_config_entry, &config);
 994        free(normalized_url);
 995
 996        if (curl_global_init(CURL_GLOBAL_ALL) != CURLE_OK)
 997                die("curl_global_init failed");
 998
 999        http_proactive_auth = proactive_auth;
1000
1001        if (remote && remote->http_proxy)
1002                curl_http_proxy = xstrdup(remote->http_proxy);
1003
1004        if (remote)
1005                var_override(&http_proxy_authmethod, remote->http_proxy_authmethod);
1006
1007        protocol_http_header();
1008
1009        pragma_header = curl_slist_append(http_copy_default_headers(),
1010                "Pragma: no-cache");
1011        no_pragma_header = curl_slist_append(http_copy_default_headers(),
1012                "Pragma:");
1013
1014#ifdef USE_CURL_MULTI
1015        {
1016                char *http_max_requests = getenv("GIT_HTTP_MAX_REQUESTS");
1017                if (http_max_requests != NULL)
1018                        max_requests = atoi(http_max_requests);
1019        }
1020
1021        curlm = curl_multi_init();
1022        if (!curlm)
1023                die("curl_multi_init failed");
1024#endif
1025
1026        if (getenv("GIT_SSL_NO_VERIFY"))
1027                curl_ssl_verify = 0;
1028
1029        set_from_env(&ssl_cert, "GIT_SSL_CERT");
1030#if LIBCURL_VERSION_NUM >= 0x070903
1031        set_from_env(&ssl_key, "GIT_SSL_KEY");
1032#endif
1033#if LIBCURL_VERSION_NUM >= 0x070908
1034        set_from_env(&ssl_capath, "GIT_SSL_CAPATH");
1035#endif
1036        set_from_env(&ssl_cainfo, "GIT_SSL_CAINFO");
1037
1038        set_from_env(&user_agent, "GIT_HTTP_USER_AGENT");
1039
1040        low_speed_limit = getenv("GIT_HTTP_LOW_SPEED_LIMIT");
1041        if (low_speed_limit != NULL)
1042                curl_low_speed_limit = strtol(low_speed_limit, NULL, 10);
1043        low_speed_time = getenv("GIT_HTTP_LOW_SPEED_TIME");
1044        if (low_speed_time != NULL)
1045                curl_low_speed_time = strtol(low_speed_time, NULL, 10);
1046
1047        if (curl_ssl_verify == -1)
1048                curl_ssl_verify = 1;
1049
1050        curl_session_count = 0;
1051#ifdef USE_CURL_MULTI
1052        if (max_requests < 1)
1053                max_requests = DEFAULT_MAX_REQUESTS;
1054#endif
1055
1056        if (getenv("GIT_CURL_FTP_NO_EPSV"))
1057                curl_ftp_no_epsv = 1;
1058
1059        if (url) {
1060                credential_from_url(&http_auth, url);
1061                if (!ssl_cert_password_required &&
1062                    getenv("GIT_SSL_CERT_PASSWORD_PROTECTED") &&
1063                    starts_with(url, "https://"))
1064                        ssl_cert_password_required = 1;
1065        }
1066
1067#ifndef NO_CURL_EASY_DUPHANDLE
1068        curl_default = get_curl_handle();
1069#endif
1070}
1071
1072void http_cleanup(void)
1073{
1074        struct active_request_slot *slot = active_queue_head;
1075
1076        while (slot != NULL) {
1077                struct active_request_slot *next = slot->next;
1078                if (slot->curl != NULL) {
1079                        xmulti_remove_handle(slot);
1080                        curl_easy_cleanup(slot->curl);
1081                }
1082                free(slot);
1083                slot = next;
1084        }
1085        active_queue_head = NULL;
1086
1087#ifndef NO_CURL_EASY_DUPHANDLE
1088        curl_easy_cleanup(curl_default);
1089#endif
1090
1091#ifdef USE_CURL_MULTI
1092        curl_multi_cleanup(curlm);
1093#endif
1094        curl_global_cleanup();
1095
1096        curl_slist_free_all(extra_http_headers);
1097        extra_http_headers = NULL;
1098
1099        curl_slist_free_all(pragma_header);
1100        pragma_header = NULL;
1101
1102        curl_slist_free_all(no_pragma_header);
1103        no_pragma_header = NULL;
1104
1105        if (curl_http_proxy) {
1106                free((void *)curl_http_proxy);
1107                curl_http_proxy = NULL;
1108        }
1109
1110        if (proxy_auth.password) {
1111                memset(proxy_auth.password, 0, strlen(proxy_auth.password));
1112                FREE_AND_NULL(proxy_auth.password);
1113        }
1114
1115        free((void *)curl_proxyuserpwd);
1116        curl_proxyuserpwd = NULL;
1117
1118        free((void *)http_proxy_authmethod);
1119        http_proxy_authmethod = NULL;
1120
1121        if (cert_auth.password != NULL) {
1122                memset(cert_auth.password, 0, strlen(cert_auth.password));
1123                FREE_AND_NULL(cert_auth.password);
1124        }
1125        ssl_cert_password_required = 0;
1126
1127        FREE_AND_NULL(cached_accept_language);
1128}
1129
1130struct active_request_slot *get_active_slot(void)
1131{
1132        struct active_request_slot *slot = active_queue_head;
1133        struct active_request_slot *newslot;
1134
1135#ifdef USE_CURL_MULTI
1136        int num_transfers;
1137
1138        /* Wait for a slot to open up if the queue is full */
1139        while (active_requests >= max_requests) {
1140                curl_multi_perform(curlm, &num_transfers);
1141                if (num_transfers < active_requests)
1142                        process_curl_messages();
1143        }
1144#endif
1145
1146        while (slot != NULL && slot->in_use)
1147                slot = slot->next;
1148
1149        if (slot == NULL) {
1150                newslot = xmalloc(sizeof(*newslot));
1151                newslot->curl = NULL;
1152                newslot->in_use = 0;
1153                newslot->next = NULL;
1154
1155                slot = active_queue_head;
1156                if (slot == NULL) {
1157                        active_queue_head = newslot;
1158                } else {
1159                        while (slot->next != NULL)
1160                                slot = slot->next;
1161                        slot->next = newslot;
1162                }
1163                slot = newslot;
1164        }
1165
1166        if (slot->curl == NULL) {
1167#ifdef NO_CURL_EASY_DUPHANDLE
1168                slot->curl = get_curl_handle();
1169#else
1170                slot->curl = curl_easy_duphandle(curl_default);
1171#endif
1172                curl_session_count++;
1173        }
1174
1175        active_requests++;
1176        slot->in_use = 1;
1177        slot->results = NULL;
1178        slot->finished = NULL;
1179        slot->callback_data = NULL;
1180        slot->callback_func = NULL;
1181        curl_easy_setopt(slot->curl, CURLOPT_COOKIEFILE, curl_cookie_file);
1182        if (curl_save_cookies)
1183                curl_easy_setopt(slot->curl, CURLOPT_COOKIEJAR, curl_cookie_file);
1184        curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, pragma_header);
1185        curl_easy_setopt(slot->curl, CURLOPT_ERRORBUFFER, curl_errorstr);
1186        curl_easy_setopt(slot->curl, CURLOPT_CUSTOMREQUEST, NULL);
1187        curl_easy_setopt(slot->curl, CURLOPT_READFUNCTION, NULL);
1188        curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, NULL);
1189        curl_easy_setopt(slot->curl, CURLOPT_POSTFIELDS, NULL);
1190        curl_easy_setopt(slot->curl, CURLOPT_UPLOAD, 0);
1191        curl_easy_setopt(slot->curl, CURLOPT_HTTPGET, 1);
1192        curl_easy_setopt(slot->curl, CURLOPT_FAILONERROR, 1);
1193        curl_easy_setopt(slot->curl, CURLOPT_RANGE, NULL);
1194
1195        /*
1196         * Default following to off unless "ALWAYS" is configured; this gives
1197         * callers a sane starting point, and they can tweak for individual
1198         * HTTP_FOLLOW_* cases themselves.
1199         */
1200        if (http_follow_config == HTTP_FOLLOW_ALWAYS)
1201                curl_easy_setopt(slot->curl, CURLOPT_FOLLOWLOCATION, 1);
1202        else
1203                curl_easy_setopt(slot->curl, CURLOPT_FOLLOWLOCATION, 0);
1204
1205#if LIBCURL_VERSION_NUM >= 0x070a08
1206        curl_easy_setopt(slot->curl, CURLOPT_IPRESOLVE, git_curl_ipresolve);
1207#endif
1208#ifdef LIBCURL_CAN_HANDLE_AUTH_ANY
1209        curl_easy_setopt(slot->curl, CURLOPT_HTTPAUTH, http_auth_methods);
1210#endif
1211        if (http_auth.password || curl_empty_auth_enabled())
1212                init_curl_http_auth(slot->curl);
1213
1214        return slot;
1215}
1216
1217int start_active_slot(struct active_request_slot *slot)
1218{
1219#ifdef USE_CURL_MULTI
1220        CURLMcode curlm_result = curl_multi_add_handle(curlm, slot->curl);
1221        int num_transfers;
1222
1223        if (curlm_result != CURLM_OK &&
1224            curlm_result != CURLM_CALL_MULTI_PERFORM) {
1225                warning("curl_multi_add_handle failed: %s",
1226                        curl_multi_strerror(curlm_result));
1227                active_requests--;
1228                slot->in_use = 0;
1229                return 0;
1230        }
1231
1232        /*
1233         * We know there must be something to do, since we just added
1234         * something.
1235         */
1236        curl_multi_perform(curlm, &num_transfers);
1237#endif
1238        return 1;
1239}
1240
1241#ifdef USE_CURL_MULTI
1242struct fill_chain {
1243        void *data;
1244        int (*fill)(void *);
1245        struct fill_chain *next;
1246};
1247
1248static struct fill_chain *fill_cfg;
1249
1250void add_fill_function(void *data, int (*fill)(void *))
1251{
1252        struct fill_chain *new = xmalloc(sizeof(*new));
1253        struct fill_chain **linkp = &fill_cfg;
1254        new->data = data;
1255        new->fill = fill;
1256        new->next = NULL;
1257        while (*linkp)
1258                linkp = &(*linkp)->next;
1259        *linkp = new;
1260}
1261
1262void fill_active_slots(void)
1263{
1264        struct active_request_slot *slot = active_queue_head;
1265
1266        while (active_requests < max_requests) {
1267                struct fill_chain *fill;
1268                for (fill = fill_cfg; fill; fill = fill->next)
1269                        if (fill->fill(fill->data))
1270                                break;
1271
1272                if (!fill)
1273                        break;
1274        }
1275
1276        while (slot != NULL) {
1277                if (!slot->in_use && slot->curl != NULL
1278                        && curl_session_count > min_curl_sessions) {
1279                        curl_easy_cleanup(slot->curl);
1280                        slot->curl = NULL;
1281                        curl_session_count--;
1282                }
1283                slot = slot->next;
1284        }
1285}
1286
1287void step_active_slots(void)
1288{
1289        int num_transfers;
1290        CURLMcode curlm_result;
1291
1292        do {
1293                curlm_result = curl_multi_perform(curlm, &num_transfers);
1294        } while (curlm_result == CURLM_CALL_MULTI_PERFORM);
1295        if (num_transfers < active_requests) {
1296                process_curl_messages();
1297                fill_active_slots();
1298        }
1299}
1300#endif
1301
1302void run_active_slot(struct active_request_slot *slot)
1303{
1304#ifdef USE_CURL_MULTI
1305        fd_set readfds;
1306        fd_set writefds;
1307        fd_set excfds;
1308        int max_fd;
1309        struct timeval select_timeout;
1310        int finished = 0;
1311
1312        slot->finished = &finished;
1313        while (!finished) {
1314                step_active_slots();
1315
1316                if (slot->in_use) {
1317#if LIBCURL_VERSION_NUM >= 0x070f04
1318                        long curl_timeout;
1319                        curl_multi_timeout(curlm, &curl_timeout);
1320                        if (curl_timeout == 0) {
1321                                continue;
1322                        } else if (curl_timeout == -1) {
1323                                select_timeout.tv_sec  = 0;
1324                                select_timeout.tv_usec = 50000;
1325                        } else {
1326                                select_timeout.tv_sec  =  curl_timeout / 1000;
1327                                select_timeout.tv_usec = (curl_timeout % 1000) * 1000;
1328                        }
1329#else
1330                        select_timeout.tv_sec  = 0;
1331                        select_timeout.tv_usec = 50000;
1332#endif
1333
1334                        max_fd = -1;
1335                        FD_ZERO(&readfds);
1336                        FD_ZERO(&writefds);
1337                        FD_ZERO(&excfds);
1338                        curl_multi_fdset(curlm, &readfds, &writefds, &excfds, &max_fd);
1339
1340                        /*
1341                         * It can happen that curl_multi_timeout returns a pathologically
1342                         * long timeout when curl_multi_fdset returns no file descriptors
1343                         * to read.  See commit message for more details.
1344                         */
1345                        if (max_fd < 0 &&
1346                            (select_timeout.tv_sec > 0 ||
1347                             select_timeout.tv_usec > 50000)) {
1348                                select_timeout.tv_sec  = 0;
1349                                select_timeout.tv_usec = 50000;
1350                        }
1351
1352                        select(max_fd+1, &readfds, &writefds, &excfds, &select_timeout);
1353                }
1354        }
1355#else
1356        while (slot->in_use) {
1357                slot->curl_result = curl_easy_perform(slot->curl);
1358                finish_active_slot(slot);
1359        }
1360#endif
1361}
1362
1363static void release_active_slot(struct active_request_slot *slot)
1364{
1365        closedown_active_slot(slot);
1366        if (slot->curl) {
1367                xmulti_remove_handle(slot);
1368                if (curl_session_count > min_curl_sessions) {
1369                        curl_easy_cleanup(slot->curl);
1370                        slot->curl = NULL;
1371                        curl_session_count--;
1372                }
1373        }
1374#ifdef USE_CURL_MULTI
1375        fill_active_slots();
1376#endif
1377}
1378
1379void finish_all_active_slots(void)
1380{
1381        struct active_request_slot *slot = active_queue_head;
1382
1383        while (slot != NULL)
1384                if (slot->in_use) {
1385                        run_active_slot(slot);
1386                        slot = active_queue_head;
1387                } else {
1388                        slot = slot->next;
1389                }
1390}
1391
1392/* Helpers for modifying and creating URLs */
1393static inline int needs_quote(int ch)
1394{
1395        if (((ch >= 'A') && (ch <= 'Z'))
1396                        || ((ch >= 'a') && (ch <= 'z'))
1397                        || ((ch >= '0') && (ch <= '9'))
1398                        || (ch == '/')
1399                        || (ch == '-')
1400                        || (ch == '.'))
1401                return 0;
1402        return 1;
1403}
1404
1405static char *quote_ref_url(const char *base, const char *ref)
1406{
1407        struct strbuf buf = STRBUF_INIT;
1408        const char *cp;
1409        int ch;
1410
1411        end_url_with_slash(&buf, base);
1412
1413        for (cp = ref; (ch = *cp) != 0; cp++)
1414                if (needs_quote(ch))
1415                        strbuf_addf(&buf, "%%%02x", ch);
1416                else
1417                        strbuf_addch(&buf, *cp);
1418
1419        return strbuf_detach(&buf, NULL);
1420}
1421
1422void append_remote_object_url(struct strbuf *buf, const char *url,
1423                              const char *hex,
1424                              int only_two_digit_prefix)
1425{
1426        end_url_with_slash(buf, url);
1427
1428        strbuf_addf(buf, "objects/%.*s/", 2, hex);
1429        if (!only_two_digit_prefix)
1430                strbuf_addstr(buf, hex + 2);
1431}
1432
1433char *get_remote_object_url(const char *url, const char *hex,
1434                            int only_two_digit_prefix)
1435{
1436        struct strbuf buf = STRBUF_INIT;
1437        append_remote_object_url(&buf, url, hex, only_two_digit_prefix);
1438        return strbuf_detach(&buf, NULL);
1439}
1440
1441static int handle_curl_result(struct slot_results *results)
1442{
1443        /*
1444         * If we see a failing http code with CURLE_OK, we have turned off
1445         * FAILONERROR (to keep the server's custom error response), and should
1446         * translate the code into failure here.
1447         *
1448         * Likewise, if we see a redirect (30x code), that means we turned off
1449         * redirect-following, and we should treat the result as an error.
1450         */
1451        if (results->curl_result == CURLE_OK &&
1452            results->http_code >= 300) {
1453                results->curl_result = CURLE_HTTP_RETURNED_ERROR;
1454                /*
1455                 * Normally curl will already have put the "reason phrase"
1456                 * from the server into curl_errorstr; unfortunately without
1457                 * FAILONERROR it is lost, so we can give only the numeric
1458                 * status code.
1459                 */
1460                xsnprintf(curl_errorstr, sizeof(curl_errorstr),
1461                          "The requested URL returned error: %ld",
1462                          results->http_code);
1463        }
1464
1465        if (results->curl_result == CURLE_OK) {
1466                credential_approve(&http_auth);
1467                if (proxy_auth.password)
1468                        credential_approve(&proxy_auth);
1469                return HTTP_OK;
1470        } else if (missing_target(results))
1471                return HTTP_MISSING_TARGET;
1472        else if (results->http_code == 401) {
1473                if (http_auth.username && http_auth.password) {
1474                        credential_reject(&http_auth);
1475                        return HTTP_NOAUTH;
1476                } else {
1477#ifdef LIBCURL_CAN_HANDLE_AUTH_ANY
1478                        http_auth_methods &= ~CURLAUTH_GSSNEGOTIATE;
1479                        if (results->auth_avail) {
1480                                http_auth_methods &= results->auth_avail;
1481                                http_auth_methods_restricted = 1;
1482                        }
1483#endif
1484                        return HTTP_REAUTH;
1485                }
1486        } else {
1487                if (results->http_connectcode == 407)
1488                        credential_reject(&proxy_auth);
1489#if LIBCURL_VERSION_NUM >= 0x070c00
1490                if (!curl_errorstr[0])
1491                        strlcpy(curl_errorstr,
1492                                curl_easy_strerror(results->curl_result),
1493                                sizeof(curl_errorstr));
1494#endif
1495                return HTTP_ERROR;
1496        }
1497}
1498
1499int run_one_slot(struct active_request_slot *slot,
1500                 struct slot_results *results)
1501{
1502        slot->results = results;
1503        if (!start_active_slot(slot)) {
1504                xsnprintf(curl_errorstr, sizeof(curl_errorstr),
1505                          "failed to start HTTP request");
1506                return HTTP_START_FAILED;
1507        }
1508
1509        run_active_slot(slot);
1510        return handle_curl_result(results);
1511}
1512
1513struct curl_slist *http_copy_default_headers(void)
1514{
1515        struct curl_slist *headers = NULL, *h;
1516
1517        for (h = extra_http_headers; h; h = h->next)
1518                headers = curl_slist_append(headers, h->data);
1519
1520        return headers;
1521}
1522
1523static CURLcode curlinfo_strbuf(CURL *curl, CURLINFO info, struct strbuf *buf)
1524{
1525        char *ptr;
1526        CURLcode ret;
1527
1528        strbuf_reset(buf);
1529        ret = curl_easy_getinfo(curl, info, &ptr);
1530        if (!ret && ptr)
1531                strbuf_addstr(buf, ptr);
1532        return ret;
1533}
1534
1535/*
1536 * Check for and extract a content-type parameter. "raw"
1537 * should be positioned at the start of the potential
1538 * parameter, with any whitespace already removed.
1539 *
1540 * "name" is the name of the parameter. The value is appended
1541 * to "out".
1542 */
1543static int extract_param(const char *raw, const char *name,
1544                         struct strbuf *out)
1545{
1546        size_t len = strlen(name);
1547
1548        if (strncasecmp(raw, name, len))
1549                return -1;
1550        raw += len;
1551
1552        if (*raw != '=')
1553                return -1;
1554        raw++;
1555
1556        while (*raw && !isspace(*raw) && *raw != ';')
1557                strbuf_addch(out, *raw++);
1558        return 0;
1559}
1560
1561/*
1562 * Extract a normalized version of the content type, with any
1563 * spaces suppressed, all letters lowercased, and no trailing ";"
1564 * or parameters.
1565 *
1566 * Note that we will silently remove even invalid whitespace. For
1567 * example, "text / plain" is specifically forbidden by RFC 2616,
1568 * but "text/plain" is the only reasonable output, and this keeps
1569 * our code simple.
1570 *
1571 * If the "charset" argument is not NULL, store the value of any
1572 * charset parameter there.
1573 *
1574 * Example:
1575 *   "TEXT/PLAIN; charset=utf-8" -> "text/plain", "utf-8"
1576 *   "text / plain" -> "text/plain"
1577 */
1578static void extract_content_type(struct strbuf *raw, struct strbuf *type,
1579                                 struct strbuf *charset)
1580{
1581        const char *p;
1582
1583        strbuf_reset(type);
1584        strbuf_grow(type, raw->len);
1585        for (p = raw->buf; *p; p++) {
1586                if (isspace(*p))
1587                        continue;
1588                if (*p == ';') {
1589                        p++;
1590                        break;
1591                }
1592                strbuf_addch(type, tolower(*p));
1593        }
1594
1595        if (!charset)
1596                return;
1597
1598        strbuf_reset(charset);
1599        while (*p) {
1600                while (isspace(*p) || *p == ';')
1601                        p++;
1602                if (!extract_param(p, "charset", charset))
1603                        return;
1604                while (*p && !isspace(*p))
1605                        p++;
1606        }
1607
1608        if (!charset->len && starts_with(type->buf, "text/"))
1609                strbuf_addstr(charset, "ISO-8859-1");
1610}
1611
1612static void write_accept_language(struct strbuf *buf)
1613{
1614        /*
1615         * MAX_DECIMAL_PLACES must not be larger than 3. If it is larger than
1616         * that, q-value will be smaller than 0.001, the minimum q-value the
1617         * HTTP specification allows. See
1618         * http://tools.ietf.org/html/rfc7231#section-5.3.1 for q-value.
1619         */
1620        const int MAX_DECIMAL_PLACES = 3;
1621        const int MAX_LANGUAGE_TAGS = 1000;
1622        const int MAX_ACCEPT_LANGUAGE_HEADER_SIZE = 4000;
1623        char **language_tags = NULL;
1624        int num_langs = 0;
1625        const char *s = get_preferred_languages();
1626        int i;
1627        struct strbuf tag = STRBUF_INIT;
1628
1629        /* Don't add Accept-Language header if no language is preferred. */
1630        if (!s)
1631                return;
1632
1633        /*
1634         * Split the colon-separated string of preferred languages into
1635         * language_tags array.
1636         */
1637        do {
1638                /* collect language tag */
1639                for (; *s && (isalnum(*s) || *s == '_'); s++)
1640                        strbuf_addch(&tag, *s == '_' ? '-' : *s);
1641
1642                /* skip .codeset, @modifier and any other unnecessary parts */
1643                while (*s && *s != ':')
1644                        s++;
1645
1646                if (tag.len) {
1647                        num_langs++;
1648                        REALLOC_ARRAY(language_tags, num_langs);
1649                        language_tags[num_langs - 1] = strbuf_detach(&tag, NULL);
1650                        if (num_langs >= MAX_LANGUAGE_TAGS - 1) /* -1 for '*' */
1651                                break;
1652                }
1653        } while (*s++);
1654
1655        /* write Accept-Language header into buf */
1656        if (num_langs) {
1657                int last_buf_len = 0;
1658                int max_q;
1659                int decimal_places;
1660                char q_format[32];
1661
1662                /* add '*' */
1663                REALLOC_ARRAY(language_tags, num_langs + 1);
1664                language_tags[num_langs++] = "*"; /* it's OK; this won't be freed */
1665
1666                /* compute decimal_places */
1667                for (max_q = 1, decimal_places = 0;
1668                     max_q < num_langs && decimal_places <= MAX_DECIMAL_PLACES;
1669                     decimal_places++, max_q *= 10)
1670                        ;
1671
1672                xsnprintf(q_format, sizeof(q_format), ";q=0.%%0%dd", decimal_places);
1673
1674                strbuf_addstr(buf, "Accept-Language: ");
1675
1676                for (i = 0; i < num_langs; i++) {
1677                        if (i > 0)
1678                                strbuf_addstr(buf, ", ");
1679
1680                        strbuf_addstr(buf, language_tags[i]);
1681
1682                        if (i > 0)
1683                                strbuf_addf(buf, q_format, max_q - i);
1684
1685                        if (buf->len > MAX_ACCEPT_LANGUAGE_HEADER_SIZE) {
1686                                strbuf_remove(buf, last_buf_len, buf->len - last_buf_len);
1687                                break;
1688                        }
1689
1690                        last_buf_len = buf->len;
1691                }
1692        }
1693
1694        /* free language tags -- last one is a static '*' */
1695        for (i = 0; i < num_langs - 1; i++)
1696                free(language_tags[i]);
1697        free(language_tags);
1698}
1699
1700/*
1701 * Get an Accept-Language header which indicates user's preferred languages.
1702 *
1703 * Examples:
1704 *   LANGUAGE= -> ""
1705 *   LANGUAGE=ko:en -> "Accept-Language: ko, en; q=0.9, *; q=0.1"
1706 *   LANGUAGE=ko_KR.UTF-8:sr@latin -> "Accept-Language: ko-KR, sr; q=0.9, *; q=0.1"
1707 *   LANGUAGE=ko LANG=en_US.UTF-8 -> "Accept-Language: ko, *; q=0.1"
1708 *   LANGUAGE= LANG=en_US.UTF-8 -> "Accept-Language: en-US, *; q=0.1"
1709 *   LANGUAGE= LANG=C -> ""
1710 */
1711static const char *get_accept_language(void)
1712{
1713        if (!cached_accept_language) {
1714                struct strbuf buf = STRBUF_INIT;
1715                write_accept_language(&buf);
1716                if (buf.len > 0)
1717                        cached_accept_language = strbuf_detach(&buf, NULL);
1718        }
1719
1720        return cached_accept_language;
1721}
1722
1723static void http_opt_request_remainder(CURL *curl, off_t pos)
1724{
1725        char buf[128];
1726        xsnprintf(buf, sizeof(buf), "%"PRIuMAX"-", (uintmax_t)pos);
1727        curl_easy_setopt(curl, CURLOPT_RANGE, buf);
1728}
1729
1730/* http_request() targets */
1731#define HTTP_REQUEST_STRBUF     0
1732#define HTTP_REQUEST_FILE       1
1733
1734static int http_request(const char *url,
1735                        void *result, int target,
1736                        const struct http_get_options *options)
1737{
1738        struct active_request_slot *slot;
1739        struct slot_results results;
1740        struct curl_slist *headers = http_copy_default_headers();
1741        struct strbuf buf = STRBUF_INIT;
1742        const char *accept_language;
1743        int ret;
1744
1745        slot = get_active_slot();
1746        curl_easy_setopt(slot->curl, CURLOPT_HTTPGET, 1);
1747
1748        if (result == NULL) {
1749                curl_easy_setopt(slot->curl, CURLOPT_NOBODY, 1);
1750        } else {
1751                curl_easy_setopt(slot->curl, CURLOPT_NOBODY, 0);
1752                curl_easy_setopt(slot->curl, CURLOPT_FILE, result);
1753
1754                if (target == HTTP_REQUEST_FILE) {
1755                        off_t posn = ftello(result);
1756                        curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION,
1757                                         fwrite);
1758                        if (posn > 0)
1759                                http_opt_request_remainder(slot->curl, posn);
1760                } else
1761                        curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION,
1762                                         fwrite_buffer);
1763        }
1764
1765        accept_language = get_accept_language();
1766
1767        if (accept_language)
1768                headers = curl_slist_append(headers, accept_language);
1769
1770        strbuf_addstr(&buf, "Pragma:");
1771        if (options && options->no_cache)
1772                strbuf_addstr(&buf, " no-cache");
1773        if (options && options->keep_error)
1774                curl_easy_setopt(slot->curl, CURLOPT_FAILONERROR, 0);
1775        if (options && options->initial_request &&
1776            http_follow_config == HTTP_FOLLOW_INITIAL)
1777                curl_easy_setopt(slot->curl, CURLOPT_FOLLOWLOCATION, 1);
1778
1779        headers = curl_slist_append(headers, buf.buf);
1780
1781        curl_easy_setopt(slot->curl, CURLOPT_URL, url);
1782        curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, headers);
1783        curl_easy_setopt(slot->curl, CURLOPT_ENCODING, "gzip");
1784
1785        ret = run_one_slot(slot, &results);
1786
1787        if (options && options->content_type) {
1788                struct strbuf raw = STRBUF_INIT;
1789                curlinfo_strbuf(slot->curl, CURLINFO_CONTENT_TYPE, &raw);
1790                extract_content_type(&raw, options->content_type,
1791                                     options->charset);
1792                strbuf_release(&raw);
1793        }
1794
1795        if (options && options->effective_url)
1796                curlinfo_strbuf(slot->curl, CURLINFO_EFFECTIVE_URL,
1797                                options->effective_url);
1798
1799        curl_slist_free_all(headers);
1800        strbuf_release(&buf);
1801
1802        return ret;
1803}
1804
1805/*
1806 * Update the "base" url to a more appropriate value, as deduced by
1807 * redirects seen when requesting a URL starting with "url".
1808 *
1809 * The "asked" parameter is a URL that we asked curl to access, and must begin
1810 * with "base".
1811 *
1812 * The "got" parameter is the URL that curl reported to us as where we ended
1813 * up.
1814 *
1815 * Returns 1 if we updated the base url, 0 otherwise.
1816 *
1817 * Our basic strategy is to compare "base" and "asked" to find the bits
1818 * specific to our request. We then strip those bits off of "got" to yield the
1819 * new base. So for example, if our base is "http://example.com/foo.git",
1820 * and we ask for "http://example.com/foo.git/info/refs", we might end up
1821 * with "https://other.example.com/foo.git/info/refs". We would want the
1822 * new URL to become "https://other.example.com/foo.git".
1823 *
1824 * Note that this assumes a sane redirect scheme. It's entirely possible
1825 * in the example above to end up at a URL that does not even end in
1826 * "info/refs".  In such a case we die. There's not much we can do, such a
1827 * scheme is unlikely to represent a real git repository, and failing to
1828 * rewrite the base opens options for malicious redirects to do funny things.
1829 */
1830static int update_url_from_redirect(struct strbuf *base,
1831                                    const char *asked,
1832                                    const struct strbuf *got)
1833{
1834        const char *tail;
1835        size_t new_len;
1836
1837        if (!strcmp(asked, got->buf))
1838                return 0;
1839
1840        if (!skip_prefix(asked, base->buf, &tail))
1841                die("BUG: update_url_from_redirect: %s is not a superset of %s",
1842                    asked, base->buf);
1843
1844        new_len = got->len;
1845        if (!strip_suffix_mem(got->buf, &new_len, tail))
1846                die(_("unable to update url base from redirection:\n"
1847                      "  asked for: %s\n"
1848                      "   redirect: %s"),
1849                    asked, got->buf);
1850
1851        strbuf_reset(base);
1852        strbuf_add(base, got->buf, new_len);
1853
1854        return 1;
1855}
1856
1857static int http_request_reauth(const char *url,
1858                               void *result, int target,
1859                               struct http_get_options *options)
1860{
1861        int ret = http_request(url, result, target, options);
1862
1863        if (ret != HTTP_OK && ret != HTTP_REAUTH)
1864                return ret;
1865
1866        if (options && options->effective_url && options->base_url) {
1867                if (update_url_from_redirect(options->base_url,
1868                                             url, options->effective_url)) {
1869                        credential_from_url(&http_auth, options->base_url->buf);
1870                        url = options->effective_url->buf;
1871                }
1872        }
1873
1874        if (ret != HTTP_REAUTH)
1875                return ret;
1876
1877        /*
1878         * If we are using KEEP_ERROR, the previous request may have
1879         * put cruft into our output stream; we should clear it out before
1880         * making our next request. We only know how to do this for
1881         * the strbuf case, but that is enough to satisfy current callers.
1882         */
1883        if (options && options->keep_error) {
1884                switch (target) {
1885                case HTTP_REQUEST_STRBUF:
1886                        strbuf_reset(result);
1887                        break;
1888                default:
1889                        die("BUG: HTTP_KEEP_ERROR is only supported with strbufs");
1890                }
1891        }
1892
1893        credential_fill(&http_auth);
1894
1895        return http_request(url, result, target, options);
1896}
1897
1898int http_get_strbuf(const char *url,
1899                    struct strbuf *result,
1900                    struct http_get_options *options)
1901{
1902        return http_request_reauth(url, result, HTTP_REQUEST_STRBUF, options);
1903}
1904
1905/*
1906 * Downloads a URL and stores the result in the given file.
1907 *
1908 * If a previous interrupted download is detected (i.e. a previous temporary
1909 * file is still around) the download is resumed.
1910 */
1911static int http_get_file(const char *url, const char *filename,
1912                         struct http_get_options *options)
1913{
1914        int ret;
1915        struct strbuf tmpfile = STRBUF_INIT;
1916        FILE *result;
1917
1918        strbuf_addf(&tmpfile, "%s.temp", filename);
1919        result = fopen(tmpfile.buf, "a");
1920        if (!result) {
1921                error("Unable to open local file %s", tmpfile.buf);
1922                ret = HTTP_ERROR;
1923                goto cleanup;
1924        }
1925
1926        ret = http_request_reauth(url, result, HTTP_REQUEST_FILE, options);
1927        fclose(result);
1928
1929        if (ret == HTTP_OK && finalize_object_file(tmpfile.buf, filename))
1930                ret = HTTP_ERROR;
1931cleanup:
1932        strbuf_release(&tmpfile);
1933        return ret;
1934}
1935
1936int http_fetch_ref(const char *base, struct ref *ref)
1937{
1938        struct http_get_options options = {0};
1939        char *url;
1940        struct strbuf buffer = STRBUF_INIT;
1941        int ret = -1;
1942
1943        options.no_cache = 1;
1944
1945        url = quote_ref_url(base, ref->name);
1946        if (http_get_strbuf(url, &buffer, &options) == HTTP_OK) {
1947                strbuf_rtrim(&buffer);
1948                if (buffer.len == 40)
1949                        ret = get_oid_hex(buffer.buf, &ref->old_oid);
1950                else if (starts_with(buffer.buf, "ref: ")) {
1951                        ref->symref = xstrdup(buffer.buf + 5);
1952                        ret = 0;
1953                }
1954        }
1955
1956        strbuf_release(&buffer);
1957        free(url);
1958        return ret;
1959}
1960
1961/* Helpers for fetching packs */
1962static char *fetch_pack_index(unsigned char *sha1, const char *base_url)
1963{
1964        char *url, *tmp;
1965        struct strbuf buf = STRBUF_INIT;
1966
1967        if (http_is_verbose)
1968                fprintf(stderr, "Getting index for pack %s\n", sha1_to_hex(sha1));
1969
1970        end_url_with_slash(&buf, base_url);
1971        strbuf_addf(&buf, "objects/pack/pack-%s.idx", sha1_to_hex(sha1));
1972        url = strbuf_detach(&buf, NULL);
1973
1974        strbuf_addf(&buf, "%s.temp", sha1_pack_index_name(sha1));
1975        tmp = strbuf_detach(&buf, NULL);
1976
1977        if (http_get_file(url, tmp, NULL) != HTTP_OK) {
1978                error("Unable to get pack index %s", url);
1979                FREE_AND_NULL(tmp);
1980        }
1981
1982        free(url);
1983        return tmp;
1984}
1985
1986static int fetch_and_setup_pack_index(struct packed_git **packs_head,
1987        unsigned char *sha1, const char *base_url)
1988{
1989        struct packed_git *new_pack;
1990        char *tmp_idx = NULL;
1991        int ret;
1992
1993        if (has_pack_index(sha1)) {
1994                new_pack = parse_pack_index(sha1, sha1_pack_index_name(sha1));
1995                if (!new_pack)
1996                        return -1; /* parse_pack_index() already issued error message */
1997                goto add_pack;
1998        }
1999
2000        tmp_idx = fetch_pack_index(sha1, base_url);
2001        if (!tmp_idx)
2002                return -1;
2003
2004        new_pack = parse_pack_index(sha1, tmp_idx);
2005        if (!new_pack) {
2006                unlink(tmp_idx);
2007                free(tmp_idx);
2008
2009                return -1; /* parse_pack_index() already issued error message */
2010        }
2011
2012        ret = verify_pack_index(new_pack);
2013        if (!ret) {
2014                close_pack_index(new_pack);
2015                ret = finalize_object_file(tmp_idx, sha1_pack_index_name(sha1));
2016        }
2017        free(tmp_idx);
2018        if (ret)
2019                return -1;
2020
2021add_pack:
2022        new_pack->next = *packs_head;
2023        *packs_head = new_pack;
2024        return 0;
2025}
2026
2027int http_get_info_packs(const char *base_url, struct packed_git **packs_head)
2028{
2029        struct http_get_options options = {0};
2030        int ret = 0, i = 0;
2031        char *url, *data;
2032        struct strbuf buf = STRBUF_INIT;
2033        unsigned char sha1[20];
2034
2035        end_url_with_slash(&buf, base_url);
2036        strbuf_addstr(&buf, "objects/info/packs");
2037        url = strbuf_detach(&buf, NULL);
2038
2039        options.no_cache = 1;
2040        ret = http_get_strbuf(url, &buf, &options);
2041        if (ret != HTTP_OK)
2042                goto cleanup;
2043
2044        data = buf.buf;
2045        while (i < buf.len) {
2046                switch (data[i]) {
2047                case 'P':
2048                        i++;
2049                        if (i + 52 <= buf.len &&
2050                            starts_with(data + i, " pack-") &&
2051                            starts_with(data + i + 46, ".pack\n")) {
2052                                get_sha1_hex(data + i + 6, sha1);
2053                                fetch_and_setup_pack_index(packs_head, sha1,
2054                                                      base_url);
2055                                i += 51;
2056                                break;
2057                        }
2058                default:
2059                        while (i < buf.len && data[i] != '\n')
2060                                i++;
2061                }
2062                i++;
2063        }
2064
2065cleanup:
2066        free(url);
2067        return ret;
2068}
2069
2070void release_http_pack_request(struct http_pack_request *preq)
2071{
2072        if (preq->packfile != NULL) {
2073                fclose(preq->packfile);
2074                preq->packfile = NULL;
2075        }
2076        preq->slot = NULL;
2077        free(preq->url);
2078        free(preq);
2079}
2080
2081int finish_http_pack_request(struct http_pack_request *preq)
2082{
2083        struct packed_git **lst;
2084        struct packed_git *p = preq->target;
2085        char *tmp_idx;
2086        size_t len;
2087        struct child_process ip = CHILD_PROCESS_INIT;
2088
2089        close_pack_index(p);
2090
2091        fclose(preq->packfile);
2092        preq->packfile = NULL;
2093
2094        lst = preq->lst;
2095        while (*lst != p)
2096                lst = &((*lst)->next);
2097        *lst = (*lst)->next;
2098
2099        if (!strip_suffix(preq->tmpfile, ".pack.temp", &len))
2100                die("BUG: pack tmpfile does not end in .pack.temp?");
2101        tmp_idx = xstrfmt("%.*s.idx.temp", (int)len, preq->tmpfile);
2102
2103        argv_array_push(&ip.args, "index-pack");
2104        argv_array_pushl(&ip.args, "-o", tmp_idx, NULL);
2105        argv_array_push(&ip.args, preq->tmpfile);
2106        ip.git_cmd = 1;
2107        ip.no_stdin = 1;
2108        ip.no_stdout = 1;
2109
2110        if (run_command(&ip)) {
2111                unlink(preq->tmpfile);
2112                unlink(tmp_idx);
2113                free(tmp_idx);
2114                return -1;
2115        }
2116
2117        unlink(sha1_pack_index_name(p->sha1));
2118
2119        if (finalize_object_file(preq->tmpfile, sha1_pack_name(p->sha1))
2120         || finalize_object_file(tmp_idx, sha1_pack_index_name(p->sha1))) {
2121                free(tmp_idx);
2122                return -1;
2123        }
2124
2125        install_packed_git(p);
2126        free(tmp_idx);
2127        return 0;
2128}
2129
2130struct http_pack_request *new_http_pack_request(
2131        struct packed_git *target, const char *base_url)
2132{
2133        off_t prev_posn = 0;
2134        struct strbuf buf = STRBUF_INIT;
2135        struct http_pack_request *preq;
2136
2137        preq = xcalloc(1, sizeof(*preq));
2138        preq->target = target;
2139
2140        end_url_with_slash(&buf, base_url);
2141        strbuf_addf(&buf, "objects/pack/pack-%s.pack",
2142                sha1_to_hex(target->sha1));
2143        preq->url = strbuf_detach(&buf, NULL);
2144
2145        snprintf(preq->tmpfile, sizeof(preq->tmpfile), "%s.temp",
2146                sha1_pack_name(target->sha1));
2147        preq->packfile = fopen(preq->tmpfile, "a");
2148        if (!preq->packfile) {
2149                error("Unable to open local file %s for pack",
2150                      preq->tmpfile);
2151                goto abort;
2152        }
2153
2154        preq->slot = get_active_slot();
2155        curl_easy_setopt(preq->slot->curl, CURLOPT_FILE, preq->packfile);
2156        curl_easy_setopt(preq->slot->curl, CURLOPT_WRITEFUNCTION, fwrite);
2157        curl_easy_setopt(preq->slot->curl, CURLOPT_URL, preq->url);
2158        curl_easy_setopt(preq->slot->curl, CURLOPT_HTTPHEADER,
2159                no_pragma_header);
2160
2161        /*
2162         * If there is data present from a previous transfer attempt,
2163         * resume where it left off
2164         */
2165        prev_posn = ftello(preq->packfile);
2166        if (prev_posn>0) {
2167                if (http_is_verbose)
2168                        fprintf(stderr,
2169                                "Resuming fetch of pack %s at byte %"PRIuMAX"\n",
2170                                sha1_to_hex(target->sha1), (uintmax_t)prev_posn);
2171                http_opt_request_remainder(preq->slot->curl, prev_posn);
2172        }
2173
2174        return preq;
2175
2176abort:
2177        free(preq->url);
2178        free(preq);
2179        return NULL;
2180}
2181
2182/* Helpers for fetching objects (loose) */
2183static size_t fwrite_sha1_file(char *ptr, size_t eltsize, size_t nmemb,
2184                               void *data)
2185{
2186        unsigned char expn[4096];
2187        size_t size = eltsize * nmemb;
2188        int posn = 0;
2189        struct http_object_request *freq = data;
2190        struct active_request_slot *slot = freq->slot;
2191
2192        if (slot) {
2193                CURLcode c = curl_easy_getinfo(slot->curl, CURLINFO_HTTP_CODE,
2194                                                &slot->http_code);
2195                if (c != CURLE_OK)
2196                        die("BUG: curl_easy_getinfo for HTTP code failed: %s",
2197                                curl_easy_strerror(c));
2198                if (slot->http_code >= 300)
2199                        return size;
2200        }
2201
2202        do {
2203                ssize_t retval = xwrite(freq->localfile,
2204                                        (char *) ptr + posn, size - posn);
2205                if (retval < 0)
2206                        return posn;
2207                posn += retval;
2208        } while (posn < size);
2209
2210        freq->stream.avail_in = size;
2211        freq->stream.next_in = (void *)ptr;
2212        do {
2213                freq->stream.next_out = expn;
2214                freq->stream.avail_out = sizeof(expn);
2215                freq->zret = git_inflate(&freq->stream, Z_SYNC_FLUSH);
2216                git_SHA1_Update(&freq->c, expn,
2217                                sizeof(expn) - freq->stream.avail_out);
2218        } while (freq->stream.avail_in && freq->zret == Z_OK);
2219        return size;
2220}
2221
2222struct http_object_request *new_http_object_request(const char *base_url,
2223        unsigned char *sha1)
2224{
2225        char *hex = sha1_to_hex(sha1);
2226        const char *filename;
2227        char prevfile[PATH_MAX];
2228        int prevlocal;
2229        char prev_buf[PREV_BUF_SIZE];
2230        ssize_t prev_read = 0;
2231        off_t prev_posn = 0;
2232        struct http_object_request *freq;
2233
2234        freq = xcalloc(1, sizeof(*freq));
2235        hashcpy(freq->sha1, sha1);
2236        freq->localfile = -1;
2237
2238        filename = sha1_file_name(sha1);
2239        snprintf(freq->tmpfile, sizeof(freq->tmpfile),
2240                 "%s.temp", filename);
2241
2242        snprintf(prevfile, sizeof(prevfile), "%s.prev", filename);
2243        unlink_or_warn(prevfile);
2244        rename(freq->tmpfile, prevfile);
2245        unlink_or_warn(freq->tmpfile);
2246
2247        if (freq->localfile != -1)
2248                error("fd leakage in start: %d", freq->localfile);
2249        freq->localfile = open(freq->tmpfile,
2250                               O_WRONLY | O_CREAT | O_EXCL, 0666);
2251        /*
2252         * This could have failed due to the "lazy directory creation";
2253         * try to mkdir the last path component.
2254         */
2255        if (freq->localfile < 0 && errno == ENOENT) {
2256                char *dir = strrchr(freq->tmpfile, '/');
2257                if (dir) {
2258                        *dir = 0;
2259                        mkdir(freq->tmpfile, 0777);
2260                        *dir = '/';
2261                }
2262                freq->localfile = open(freq->tmpfile,
2263                                       O_WRONLY | O_CREAT | O_EXCL, 0666);
2264        }
2265
2266        if (freq->localfile < 0) {
2267                error_errno("Couldn't create temporary file %s", freq->tmpfile);
2268                goto abort;
2269        }
2270
2271        git_inflate_init(&freq->stream);
2272
2273        git_SHA1_Init(&freq->c);
2274
2275        freq->url = get_remote_object_url(base_url, hex, 0);
2276
2277        /*
2278         * If a previous temp file is present, process what was already
2279         * fetched.
2280         */
2281        prevlocal = open(prevfile, O_RDONLY);
2282        if (prevlocal != -1) {
2283                do {
2284                        prev_read = xread(prevlocal, prev_buf, PREV_BUF_SIZE);
2285                        if (prev_read>0) {
2286                                if (fwrite_sha1_file(prev_buf,
2287                                                     1,
2288                                                     prev_read,
2289                                                     freq) == prev_read) {
2290                                        prev_posn += prev_read;
2291                                } else {
2292                                        prev_read = -1;
2293                                }
2294                        }
2295                } while (prev_read > 0);
2296                close(prevlocal);
2297        }
2298        unlink_or_warn(prevfile);
2299
2300        /*
2301         * Reset inflate/SHA1 if there was an error reading the previous temp
2302         * file; also rewind to the beginning of the local file.
2303         */
2304        if (prev_read == -1) {
2305                memset(&freq->stream, 0, sizeof(freq->stream));
2306                git_inflate_init(&freq->stream);
2307                git_SHA1_Init(&freq->c);
2308                if (prev_posn>0) {
2309                        prev_posn = 0;
2310                        lseek(freq->localfile, 0, SEEK_SET);
2311                        if (ftruncate(freq->localfile, 0) < 0) {
2312                                error_errno("Couldn't truncate temporary file %s",
2313                                            freq->tmpfile);
2314                                goto abort;
2315                        }
2316                }
2317        }
2318
2319        freq->slot = get_active_slot();
2320
2321        curl_easy_setopt(freq->slot->curl, CURLOPT_FILE, freq);
2322        curl_easy_setopt(freq->slot->curl, CURLOPT_FAILONERROR, 0);
2323        curl_easy_setopt(freq->slot->curl, CURLOPT_WRITEFUNCTION, fwrite_sha1_file);
2324        curl_easy_setopt(freq->slot->curl, CURLOPT_ERRORBUFFER, freq->errorstr);
2325        curl_easy_setopt(freq->slot->curl, CURLOPT_URL, freq->url);
2326        curl_easy_setopt(freq->slot->curl, CURLOPT_HTTPHEADER, no_pragma_header);
2327
2328        /*
2329         * If we have successfully processed data from a previous fetch
2330         * attempt, only fetch the data we don't already have.
2331         */
2332        if (prev_posn>0) {
2333                if (http_is_verbose)
2334                        fprintf(stderr,
2335                                "Resuming fetch of object %s at byte %"PRIuMAX"\n",
2336                                hex, (uintmax_t)prev_posn);
2337                http_opt_request_remainder(freq->slot->curl, prev_posn);
2338        }
2339
2340        return freq;
2341
2342abort:
2343        free(freq->url);
2344        free(freq);
2345        return NULL;
2346}
2347
2348void process_http_object_request(struct http_object_request *freq)
2349{
2350        if (freq->slot == NULL)
2351                return;
2352        freq->curl_result = freq->slot->curl_result;
2353        freq->http_code = freq->slot->http_code;
2354        freq->slot = NULL;
2355}
2356
2357int finish_http_object_request(struct http_object_request *freq)
2358{
2359        struct stat st;
2360
2361        close(freq->localfile);
2362        freq->localfile = -1;
2363
2364        process_http_object_request(freq);
2365
2366        if (freq->http_code == 416) {
2367                warning("requested range invalid; we may already have all the data.");
2368        } else if (freq->curl_result != CURLE_OK) {
2369                if (stat(freq->tmpfile, &st) == 0)
2370                        if (st.st_size == 0)
2371                                unlink_or_warn(freq->tmpfile);
2372                return -1;
2373        }
2374
2375        git_inflate_end(&freq->stream);
2376        git_SHA1_Final(freq->real_sha1, &freq->c);
2377        if (freq->zret != Z_STREAM_END) {
2378                unlink_or_warn(freq->tmpfile);
2379                return -1;
2380        }
2381        if (hashcmp(freq->sha1, freq->real_sha1)) {
2382                unlink_or_warn(freq->tmpfile);
2383                return -1;
2384        }
2385        freq->rename =
2386                finalize_object_file(freq->tmpfile, sha1_file_name(freq->sha1));
2387
2388        return freq->rename;
2389}
2390
2391void abort_http_object_request(struct http_object_request *freq)
2392{
2393        unlink_or_warn(freq->tmpfile);
2394
2395        release_http_object_request(freq);
2396}
2397
2398void release_http_object_request(struct http_object_request *freq)
2399{
2400        if (freq->localfile != -1) {
2401                close(freq->localfile);
2402                freq->localfile = -1;
2403        }
2404        if (freq->url != NULL) {
2405                FREE_AND_NULL(freq->url);
2406        }
2407        if (freq->slot != NULL) {
2408                freq->slot->callback_func = NULL;
2409                freq->slot->callback_data = NULL;
2410                release_active_slot(freq->slot);
2411                freq->slot = NULL;
2412        }
2413}