Andrew's git / gitweb.git / http.c
?
http.con commit Add an option not to use link(src, dest) && unlink(src) when that is unreliable (be66a6c)
   1#include "http.h"
   2
   3int data_received;
   4int active_requests;
   5
   6#ifdef USE_CURL_MULTI
   7static int max_requests = -1;
   8static CURLM *curlm;
   9#endif
  10#ifndef NO_CURL_EASY_DUPHANDLE
  11static CURL *curl_default;
  12#endif
  13char curl_errorstr[CURL_ERROR_SIZE];
  14
  15static int curl_ssl_verify = -1;
  16static const char *ssl_cert;
  17#if LIBCURL_VERSION_NUM >= 0x070902
  18static const char *ssl_key;
  19#endif
  20#if LIBCURL_VERSION_NUM >= 0x070908
  21static const char *ssl_capath;
  22#endif
  23static const char *ssl_cainfo;
  24static long curl_low_speed_limit = -1;
  25static long curl_low_speed_time = -1;
  26static int curl_ftp_no_epsv;
  27static const char *curl_http_proxy;
  28static char *user_name, *user_pass;
  29
  30static struct curl_slist *pragma_header;
  31
  32static struct active_request_slot *active_queue_head;
  33
  34size_t fread_buffer(void *ptr, size_t eltsize, size_t nmemb, void *buffer_)
  35{
  36        size_t size = eltsize * nmemb;
  37        struct buffer *buffer = buffer_;
  38
  39        if (size > buffer->buf.len - buffer->posn)
  40                size = buffer->buf.len - buffer->posn;
  41        memcpy(ptr, buffer->buf.buf + buffer->posn, size);
  42        buffer->posn += size;
  43
  44        return size;
  45}
  46
  47#ifndef NO_CURL_IOCTL
  48curlioerr ioctl_buffer(CURL *handle, int cmd, void *clientp)
  49{
  50        struct buffer *buffer = clientp;
  51
  52        switch (cmd) {
  53        case CURLIOCMD_NOP:
  54                return CURLIOE_OK;
  55
  56        case CURLIOCMD_RESTARTREAD:
  57                buffer->posn = 0;
  58                return CURLIOE_OK;
  59
  60        default:
  61                return CURLIOE_UNKNOWNCMD;
  62        }
  63}
  64#endif
  65
  66size_t fwrite_buffer(const void *ptr, size_t eltsize, size_t nmemb, void *buffer_)
  67{
  68        size_t size = eltsize * nmemb;
  69        struct strbuf *buffer = buffer_;
  70
  71        strbuf_add(buffer, ptr, size);
  72        data_received++;
  73        return size;
  74}
  75
  76size_t fwrite_null(const void *ptr, size_t eltsize, size_t nmemb, void *strbuf)
  77{
  78        data_received++;
  79        return eltsize * nmemb;
  80}
  81
  82static void finish_active_slot(struct active_request_slot *slot);
  83
  84#ifdef USE_CURL_MULTI
  85static void process_curl_messages(void)
  86{
  87        int num_messages;
  88        struct active_request_slot *slot;
  89        CURLMsg *curl_message = curl_multi_info_read(curlm, &num_messages);
  90
  91        while (curl_message != NULL) {
  92                if (curl_message->msg == CURLMSG_DONE) {
  93                        int curl_result = curl_message->data.result;
  94                        slot = active_queue_head;
  95                        while (slot != NULL &&
  96                               slot->curl != curl_message->easy_handle)
  97                                slot = slot->next;
  98                        if (slot != NULL) {
  99                                curl_multi_remove_handle(curlm, slot->curl);
 100                                slot->curl_result = curl_result;
 101                                finish_active_slot(slot);
 102                        } else {
 103                                fprintf(stderr, "Received DONE message for unknown request!\n");
 104                        }
 105                } else {
 106                        fprintf(stderr, "Unknown CURL message received: %d\n",
 107                                (int)curl_message->msg);
 108                }
 109                curl_message = curl_multi_info_read(curlm, &num_messages);
 110        }
 111}
 112#endif
 113
 114static int http_options(const char *var, const char *value, void *cb)
 115{
 116        if (!strcmp("http.sslverify", var)) {
 117                curl_ssl_verify = git_config_bool(var, value);
 118                return 0;
 119        }
 120        if (!strcmp("http.sslcert", var))
 121                return git_config_string(&ssl_cert, var, value);
 122#if LIBCURL_VERSION_NUM >= 0x070902
 123        if (!strcmp("http.sslkey", var))
 124                return git_config_string(&ssl_key, var, value);
 125#endif
 126#if LIBCURL_VERSION_NUM >= 0x070908
 127        if (!strcmp("http.sslcapath", var))
 128                return git_config_string(&ssl_capath, var, value);
 129#endif
 130        if (!strcmp("http.sslcainfo", var))
 131                return git_config_string(&ssl_cainfo, var, value);
 132#ifdef USE_CURL_MULTI
 133        if (!strcmp("http.maxrequests", var)) {
 134                max_requests = git_config_int(var, value);
 135                return 0;
 136        }
 137#endif
 138        if (!strcmp("http.lowspeedlimit", var)) {
 139                curl_low_speed_limit = (long)git_config_int(var, value);
 140                return 0;
 141        }
 142        if (!strcmp("http.lowspeedtime", var)) {
 143                curl_low_speed_time = (long)git_config_int(var, value);
 144                return 0;
 145        }
 146
 147        if (!strcmp("http.noepsv", var)) {
 148                curl_ftp_no_epsv = git_config_bool(var, value);
 149                return 0;
 150        }
 151        if (!strcmp("http.proxy", var))
 152                return git_config_string(&curl_http_proxy, var, value);
 153
 154        /* Fall back on the default ones */
 155        return git_default_config(var, value, cb);
 156}
 157
 158static void init_curl_http_auth(CURL *result)
 159{
 160        if (user_name) {
 161                struct strbuf up = STRBUF_INIT;
 162                if (!user_pass)
 163                        user_pass = xstrdup(getpass("Password: "));
 164                strbuf_addf(&up, "%s:%s", user_name, user_pass);
 165                curl_easy_setopt(result, CURLOPT_USERPWD,
 166                                 strbuf_detach(&up, NULL));
 167        }
 168}
 169
 170static CURL *get_curl_handle(void)
 171{
 172        CURL *result = curl_easy_init();
 173
 174        if (!curl_ssl_verify) {
 175                curl_easy_setopt(result, CURLOPT_SSL_VERIFYPEER, 0);
 176                curl_easy_setopt(result, CURLOPT_SSL_VERIFYHOST, 0);
 177        } else {
 178                /* Verify authenticity of the peer's certificate */
 179                curl_easy_setopt(result, CURLOPT_SSL_VERIFYPEER, 1);
 180                /* The name in the cert must match whom we tried to connect */
 181                curl_easy_setopt(result, CURLOPT_SSL_VERIFYHOST, 2);
 182        }
 183
 184#if LIBCURL_VERSION_NUM >= 0x070907
 185        curl_easy_setopt(result, CURLOPT_NETRC, CURL_NETRC_OPTIONAL);
 186#endif
 187
 188        init_curl_http_auth(result);
 189
 190        if (ssl_cert != NULL)
 191                curl_easy_setopt(result, CURLOPT_SSLCERT, ssl_cert);
 192#if LIBCURL_VERSION_NUM >= 0x070902
 193        if (ssl_key != NULL)
 194                curl_easy_setopt(result, CURLOPT_SSLKEY, ssl_key);
 195#endif
 196#if LIBCURL_VERSION_NUM >= 0x070908
 197        if (ssl_capath != NULL)
 198                curl_easy_setopt(result, CURLOPT_CAPATH, ssl_capath);
 199#endif
 200        if (ssl_cainfo != NULL)
 201                curl_easy_setopt(result, CURLOPT_CAINFO, ssl_cainfo);
 202        curl_easy_setopt(result, CURLOPT_FAILONERROR, 1);
 203
 204        if (curl_low_speed_limit > 0 && curl_low_speed_time > 0) {
 205                curl_easy_setopt(result, CURLOPT_LOW_SPEED_LIMIT,
 206                                 curl_low_speed_limit);
 207                curl_easy_setopt(result, CURLOPT_LOW_SPEED_TIME,
 208                                 curl_low_speed_time);
 209        }
 210
 211        curl_easy_setopt(result, CURLOPT_FOLLOWLOCATION, 1);
 212
 213        if (getenv("GIT_CURL_VERBOSE"))
 214                curl_easy_setopt(result, CURLOPT_VERBOSE, 1);
 215
 216        curl_easy_setopt(result, CURLOPT_USERAGENT, GIT_USER_AGENT);
 217
 218        if (curl_ftp_no_epsv)
 219                curl_easy_setopt(result, CURLOPT_FTP_USE_EPSV, 0);
 220
 221        if (curl_http_proxy)
 222                curl_easy_setopt(result, CURLOPT_PROXY, curl_http_proxy);
 223
 224        return result;
 225}
 226
 227static void http_auth_init(const char *url)
 228{
 229        char *at, *colon, *cp, *slash;
 230        int len;
 231
 232        cp = strstr(url, "://");
 233        if (!cp)
 234                return;
 235
 236        /*
 237         * Ok, the URL looks like "proto://something".  Which one?
 238         * "proto://<user>:<pass>@<host>/...",
 239         * "proto://<user>@<host>/...", or just
 240         * "proto://<host>/..."?
 241         */
 242        cp += 3;
 243        at = strchr(cp, '@');
 244        colon = strchr(cp, ':');
 245        slash = strchrnul(cp, '/');
 246        if (!at || slash <= at)
 247                return; /* No credentials */
 248        if (!colon || at <= colon) {
 249                /* Only username */
 250                len = at - cp;
 251                user_name = xmalloc(len + 1);
 252                memcpy(user_name, cp, len);
 253                user_name[len] = '\0';
 254                user_pass = NULL;
 255        } else {
 256                len = colon - cp;
 257                user_name = xmalloc(len + 1);
 258                memcpy(user_name, cp, len);
 259                user_name[len] = '\0';
 260                len = at - (colon + 1);
 261                user_pass = xmalloc(len + 1);
 262                memcpy(user_pass, colon + 1, len);
 263                user_pass[len] = '\0';
 264        }
 265}
 266
 267static void set_from_env(const char **var, const char *envname)
 268{
 269        const char *val = getenv(envname);
 270        if (val)
 271                *var = val;
 272}
 273
 274void http_init(struct remote *remote)
 275{
 276        char *low_speed_limit;
 277        char *low_speed_time;
 278
 279        git_config(http_options, NULL);
 280
 281        curl_global_init(CURL_GLOBAL_ALL);
 282
 283        if (remote && remote->http_proxy)
 284                curl_http_proxy = xstrdup(remote->http_proxy);
 285
 286        pragma_header = curl_slist_append(pragma_header, "Pragma: no-cache");
 287
 288#ifdef USE_CURL_MULTI
 289        {
 290                char *http_max_requests = getenv("GIT_HTTP_MAX_REQUESTS");
 291                if (http_max_requests != NULL)
 292                        max_requests = atoi(http_max_requests);
 293        }
 294
 295        curlm = curl_multi_init();
 296        if (curlm == NULL) {
 297                fprintf(stderr, "Error creating curl multi handle.\n");
 298                exit(1);
 299        }
 300#endif
 301
 302        if (getenv("GIT_SSL_NO_VERIFY"))
 303                curl_ssl_verify = 0;
 304
 305        set_from_env(&ssl_cert, "GIT_SSL_CERT");
 306#if LIBCURL_VERSION_NUM >= 0x070902
 307        set_from_env(&ssl_key, "GIT_SSL_KEY");
 308#endif
 309#if LIBCURL_VERSION_NUM >= 0x070908
 310        set_from_env(&ssl_capath, "GIT_SSL_CAPATH");
 311#endif
 312        set_from_env(&ssl_cainfo, "GIT_SSL_CAINFO");
 313
 314        low_speed_limit = getenv("GIT_HTTP_LOW_SPEED_LIMIT");
 315        if (low_speed_limit != NULL)
 316                curl_low_speed_limit = strtol(low_speed_limit, NULL, 10);
 317        low_speed_time = getenv("GIT_HTTP_LOW_SPEED_TIME");
 318        if (low_speed_time != NULL)
 319                curl_low_speed_time = strtol(low_speed_time, NULL, 10);
 320
 321        if (curl_ssl_verify == -1)
 322                curl_ssl_verify = 1;
 323
 324#ifdef USE_CURL_MULTI
 325        if (max_requests < 1)
 326                max_requests = DEFAULT_MAX_REQUESTS;
 327#endif
 328
 329        if (getenv("GIT_CURL_FTP_NO_EPSV"))
 330                curl_ftp_no_epsv = 1;
 331
 332        if (remote && remote->url && remote->url[0])
 333                http_auth_init(remote->url[0]);
 334
 335#ifndef NO_CURL_EASY_DUPHANDLE
 336        curl_default = get_curl_handle();
 337#endif
 338}
 339
 340void http_cleanup(void)
 341{
 342        struct active_request_slot *slot = active_queue_head;
 343
 344        while (slot != NULL) {
 345                struct active_request_slot *next = slot->next;
 346                if (slot->curl != NULL) {
 347#ifdef USE_CURL_MULTI
 348                        curl_multi_remove_handle(curlm, slot->curl);
 349#endif
 350                        curl_easy_cleanup(slot->curl);
 351                }
 352                free(slot);
 353                slot = next;
 354        }
 355        active_queue_head = NULL;
 356
 357#ifndef NO_CURL_EASY_DUPHANDLE
 358        curl_easy_cleanup(curl_default);
 359#endif
 360
 361#ifdef USE_CURL_MULTI
 362        curl_multi_cleanup(curlm);
 363#endif
 364        curl_global_cleanup();
 365
 366        curl_slist_free_all(pragma_header);
 367        pragma_header = NULL;
 368
 369        if (curl_http_proxy) {
 370                free((void *)curl_http_proxy);
 371                curl_http_proxy = NULL;
 372        }
 373}
 374
 375struct active_request_slot *get_active_slot(void)
 376{
 377        struct active_request_slot *slot = active_queue_head;
 378        struct active_request_slot *newslot;
 379
 380#ifdef USE_CURL_MULTI
 381        int num_transfers;
 382
 383        /* Wait for a slot to open up if the queue is full */
 384        while (active_requests >= max_requests) {
 385                curl_multi_perform(curlm, &num_transfers);
 386                if (num_transfers < active_requests)
 387                        process_curl_messages();
 388        }
 389#endif
 390
 391        while (slot != NULL && slot->in_use)
 392                slot = slot->next;
 393
 394        if (slot == NULL) {
 395                newslot = xmalloc(sizeof(*newslot));
 396                newslot->curl = NULL;
 397                newslot->in_use = 0;
 398                newslot->next = NULL;
 399
 400                slot = active_queue_head;
 401                if (slot == NULL) {
 402                        active_queue_head = newslot;
 403                } else {
 404                        while (slot->next != NULL)
 405                                slot = slot->next;
 406                        slot->next = newslot;
 407                }
 408                slot = newslot;
 409        }
 410
 411        if (slot->curl == NULL) {
 412#ifdef NO_CURL_EASY_DUPHANDLE
 413                slot->curl = get_curl_handle();
 414#else
 415                slot->curl = curl_easy_duphandle(curl_default);
 416#endif
 417        }
 418
 419        active_requests++;
 420        slot->in_use = 1;
 421        slot->local = NULL;
 422        slot->results = NULL;
 423        slot->finished = NULL;
 424        slot->callback_data = NULL;
 425        slot->callback_func = NULL;
 426        curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, pragma_header);
 427        curl_easy_setopt(slot->curl, CURLOPT_ERRORBUFFER, curl_errorstr);
 428        curl_easy_setopt(slot->curl, CURLOPT_CUSTOMREQUEST, NULL);
 429        curl_easy_setopt(slot->curl, CURLOPT_READFUNCTION, NULL);
 430        curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, NULL);
 431        curl_easy_setopt(slot->curl, CURLOPT_UPLOAD, 0);
 432        curl_easy_setopt(slot->curl, CURLOPT_HTTPGET, 1);
 433
 434        return slot;
 435}
 436
 437int start_active_slot(struct active_request_slot *slot)
 438{
 439#ifdef USE_CURL_MULTI
 440        CURLMcode curlm_result = curl_multi_add_handle(curlm, slot->curl);
 441        int num_transfers;
 442
 443        if (curlm_result != CURLM_OK &&
 444            curlm_result != CURLM_CALL_MULTI_PERFORM) {
 445                active_requests--;
 446                slot->in_use = 0;
 447                return 0;
 448        }
 449
 450        /*
 451         * We know there must be something to do, since we just added
 452         * something.
 453         */
 454        curl_multi_perform(curlm, &num_transfers);
 455#endif
 456        return 1;
 457}
 458
 459#ifdef USE_CURL_MULTI
 460struct fill_chain {
 461        void *data;
 462        int (*fill)(void *);
 463        struct fill_chain *next;
 464};
 465
 466static struct fill_chain *fill_cfg;
 467
 468void add_fill_function(void *data, int (*fill)(void *))
 469{
 470        struct fill_chain *new = xmalloc(sizeof(*new));
 471        struct fill_chain **linkp = &fill_cfg;
 472        new->data = data;
 473        new->fill = fill;
 474        new->next = NULL;
 475        while (*linkp)
 476                linkp = &(*linkp)->next;
 477        *linkp = new;
 478}
 479
 480void fill_active_slots(void)
 481{
 482        struct active_request_slot *slot = active_queue_head;
 483
 484        while (active_requests < max_requests) {
 485                struct fill_chain *fill;
 486                for (fill = fill_cfg; fill; fill = fill->next)
 487                        if (fill->fill(fill->data))
 488                                break;
 489
 490                if (!fill)
 491                        break;
 492        }
 493
 494        while (slot != NULL) {
 495                if (!slot->in_use && slot->curl != NULL) {
 496                        curl_easy_cleanup(slot->curl);
 497                        slot->curl = NULL;
 498                }
 499                slot = slot->next;
 500        }
 501}
 502
 503void step_active_slots(void)
 504{
 505        int num_transfers;
 506        CURLMcode curlm_result;
 507
 508        do {
 509                curlm_result = curl_multi_perform(curlm, &num_transfers);
 510        } while (curlm_result == CURLM_CALL_MULTI_PERFORM);
 511        if (num_transfers < active_requests) {
 512                process_curl_messages();
 513                fill_active_slots();
 514        }
 515}
 516#endif
 517
 518void run_active_slot(struct active_request_slot *slot)
 519{
 520#ifdef USE_CURL_MULTI
 521        long last_pos = 0;
 522        long current_pos;
 523        fd_set readfds;
 524        fd_set writefds;
 525        fd_set excfds;
 526        int max_fd;
 527        struct timeval select_timeout;
 528        int finished = 0;
 529
 530        slot->finished = &finished;
 531        while (!finished) {
 532                data_received = 0;
 533                step_active_slots();
 534
 535                if (!data_received && slot->local != NULL) {
 536                        current_pos = ftell(slot->local);
 537                        if (current_pos > last_pos)
 538                                data_received++;
 539                        last_pos = current_pos;
 540                }
 541
 542                if (slot->in_use && !data_received) {
 543                        max_fd = 0;
 544                        FD_ZERO(&readfds);
 545                        FD_ZERO(&writefds);
 546                        FD_ZERO(&excfds);
 547                        select_timeout.tv_sec = 0;
 548                        select_timeout.tv_usec = 50000;
 549                        select(max_fd, &readfds, &writefds,
 550                               &excfds, &select_timeout);
 551                }
 552        }
 553#else
 554        while (slot->in_use) {
 555                slot->curl_result = curl_easy_perform(slot->curl);
 556                finish_active_slot(slot);
 557        }
 558#endif
 559}
 560
 561static void closedown_active_slot(struct active_request_slot *slot)
 562{
 563        active_requests--;
 564        slot->in_use = 0;
 565}
 566
 567void release_active_slot(struct active_request_slot *slot)
 568{
 569        closedown_active_slot(slot);
 570        if (slot->curl) {
 571#ifdef USE_CURL_MULTI
 572                curl_multi_remove_handle(curlm, slot->curl);
 573#endif
 574                curl_easy_cleanup(slot->curl);
 575                slot->curl = NULL;
 576        }
 577#ifdef USE_CURL_MULTI
 578        fill_active_slots();
 579#endif
 580}
 581
 582static void finish_active_slot(struct active_request_slot *slot)
 583{
 584        closedown_active_slot(slot);
 585        curl_easy_getinfo(slot->curl, CURLINFO_HTTP_CODE, &slot->http_code);
 586
 587        if (slot->finished != NULL)
 588                (*slot->finished) = 1;
 589
 590        /* Store slot results so they can be read after the slot is reused */
 591        if (slot->results != NULL) {
 592                slot->results->curl_result = slot->curl_result;
 593                slot->results->http_code = slot->http_code;
 594        }
 595
 596        /* Run callback if appropriate */
 597        if (slot->callback_func != NULL)
 598                slot->callback_func(slot->callback_data);
 599}
 600
 601void finish_all_active_slots(void)
 602{
 603        struct active_request_slot *slot = active_queue_head;
 604
 605        while (slot != NULL)
 606                if (slot->in_use) {
 607                        run_active_slot(slot);
 608                        slot = active_queue_head;
 609                } else {
 610                        slot = slot->next;
 611                }
 612}
 613
 614static inline int needs_quote(int ch)
 615{
 616        if (((ch >= 'A') && (ch <= 'Z'))
 617                        || ((ch >= 'a') && (ch <= 'z'))
 618                        || ((ch >= '0') && (ch <= '9'))
 619                        || (ch == '/')
 620                        || (ch == '-')
 621                        || (ch == '.'))
 622                return 0;
 623        return 1;
 624}
 625
 626static inline int hex(int v)
 627{
 628        if (v < 10)
 629                return '0' + v;
 630        else
 631                return 'A' + v - 10;
 632}
 633
 634static char *quote_ref_url(const char *base, const char *ref)
 635{
 636        struct strbuf buf = STRBUF_INIT;
 637        const char *cp;
 638        int ch;
 639
 640        strbuf_addstr(&buf, base);
 641        if (buf.len && buf.buf[buf.len - 1] != '/' && *ref != '/')
 642                strbuf_addstr(&buf, "/");
 643
 644        for (cp = ref; (ch = *cp) != 0; cp++)
 645                if (needs_quote(ch))
 646                        strbuf_addf(&buf, "%%%02x", ch);
 647                else
 648                        strbuf_addch(&buf, *cp);
 649
 650        return strbuf_detach(&buf, NULL);
 651}
 652
 653int http_fetch_ref(const char *base, struct ref *ref)
 654{
 655        char *url;
 656        struct strbuf buffer = STRBUF_INIT;
 657        struct active_request_slot *slot;
 658        struct slot_results results;
 659        int ret;
 660
 661        url = quote_ref_url(base, ref->name);
 662        slot = get_active_slot();
 663        slot->results = &results;
 664        curl_easy_setopt(slot->curl, CURLOPT_FILE, &buffer);
 665        curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite_buffer);
 666        curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, NULL);
 667        curl_easy_setopt(slot->curl, CURLOPT_URL, url);
 668        if (start_active_slot(slot)) {
 669                run_active_slot(slot);
 670                if (results.curl_result == CURLE_OK) {
 671                        strbuf_rtrim(&buffer);
 672                        if (buffer.len == 40)
 673                                ret = get_sha1_hex(buffer.buf, ref->old_sha1);
 674                        else if (!prefixcmp(buffer.buf, "ref: ")) {
 675                                ref->symref = xstrdup(buffer.buf + 5);
 676                                ret = 0;
 677                        } else
 678                                ret = 1;
 679                } else {
 680                        ret = error("Couldn't get %s for %s\n%s",
 681                                    url, ref->name, curl_errorstr);
 682                }
 683        } else {
 684                ret = error("Unable to start request");
 685        }
 686
 687        strbuf_release(&buffer);
 688        free(url);
 689        return ret;
 690}