Andrew's git / gitweb.git / http-fetch.c
?
http-fetch.con commit Strip any trailing slash on destination argument (f676fa7)
   1#include "cache.h"
   2#include "commit.h"
   3#include "pack.h"
   4#include "fetch.h"
   5
   6#include <curl/curl.h>
   7#include <curl/easy.h>
   8
   9#if LIBCURL_VERSION_NUM >= 0x070908
  10#define USE_CURL_MULTI
  11#define DEFAULT_MAX_REQUESTS 5
  12#endif
  13
  14#if LIBCURL_VERSION_NUM < 0x070704
  15#define curl_global_cleanup() do { /* nothing */ } while(0)
  16#endif
  17#if LIBCURL_VERSION_NUM < 0x070800
  18#define curl_global_init(a) do { /* nothing */ } while(0)
  19#endif
  20
  21#if LIBCURL_VERSION_NUM < 0x070c04
  22#define NO_CURL_EASY_DUPHANDLE
  23#endif
  24
  25#define PREV_BUF_SIZE 4096
  26#define RANGE_HEADER_SIZE 30
  27
  28static int got_alternates = 0;
  29static int active_requests = 0;
  30static int data_received;
  31
  32#ifdef USE_CURL_MULTI
  33static int max_requests = -1;
  34static CURLM *curlm;
  35#endif
  36#ifndef NO_CURL_EASY_DUPHANDLE
  37static CURL *curl_default;
  38#endif
  39static struct curl_slist *pragma_header;
  40static struct curl_slist *no_pragma_header;
  41static struct curl_slist *no_range_header;
  42static char curl_errorstr[CURL_ERROR_SIZE];
  43
  44struct alt_base
  45{
  46        char *base;
  47        int got_indices;
  48        struct packed_git *packs;
  49        struct alt_base *next;
  50};
  51
  52static struct alt_base *alt = NULL;
  53
  54enum transfer_state {
  55        WAITING,
  56        ABORTED,
  57        ACTIVE,
  58        COMPLETE,
  59};
  60
  61struct transfer_request
  62{
  63        unsigned char sha1[20];
  64        struct alt_base *repo;
  65        char *url;
  66        char filename[PATH_MAX];
  67        char tmpfile[PATH_MAX];
  68        int local;
  69        enum transfer_state state;
  70        CURLcode curl_result;
  71        char errorstr[CURL_ERROR_SIZE];
  72        long http_code;
  73        unsigned char real_sha1[20];
  74        SHA_CTX c;
  75        z_stream stream;
  76        int zret;
  77        int rename;
  78        struct active_request_slot *slot;
  79        struct transfer_request *next;
  80};
  81
  82struct active_request_slot
  83{
  84        CURL *curl;
  85        FILE *local;
  86        int in_use;
  87        int done;
  88        CURLcode curl_result;
  89        long http_code;
  90        struct active_request_slot *next;
  91};
  92
  93static struct transfer_request *request_queue_head = NULL;
  94static struct active_request_slot *active_queue_head = NULL;
  95
  96static int curl_ssl_verify = -1;
  97static char *ssl_cert = NULL;
  98#if LIBCURL_VERSION_NUM >= 0x070902
  99static char *ssl_key = NULL;
 100#endif
 101#if LIBCURL_VERSION_NUM >= 0x070908
 102static char *ssl_capath = NULL;
 103#endif
 104static char *ssl_cainfo = NULL;
 105static long curl_low_speed_limit = -1;
 106static long curl_low_speed_time = -1;
 107
 108struct buffer
 109{
 110        size_t posn;
 111        size_t size;
 112        void *buffer;
 113};
 114
 115static int http_options(const char *var, const char *value)
 116{
 117        if (!strcmp("http.sslverify", var)) {
 118                if (curl_ssl_verify == -1) {
 119                        curl_ssl_verify = git_config_bool(var, value);
 120                }
 121                return 0;
 122        }
 123
 124        if (!strcmp("http.sslcert", var)) {
 125                if (ssl_cert == NULL) {
 126                        ssl_cert = xmalloc(strlen(value)+1);
 127                        strcpy(ssl_cert, value);
 128                }
 129                return 0;
 130        }
 131#if LIBCURL_VERSION_NUM >= 0x070902
 132        if (!strcmp("http.sslkey", var)) {
 133                if (ssl_key == NULL) {
 134                        ssl_key = xmalloc(strlen(value)+1);
 135                        strcpy(ssl_key, value);
 136                }
 137                return 0;
 138        }
 139#endif
 140#if LIBCURL_VERSION_NUM >= 0x070908
 141        if (!strcmp("http.sslcapath", var)) {
 142                if (ssl_capath == NULL) {
 143                        ssl_capath = xmalloc(strlen(value)+1);
 144                        strcpy(ssl_capath, value);
 145                }
 146                return 0;
 147        }
 148#endif
 149        if (!strcmp("http.sslcainfo", var)) {
 150                if (ssl_cainfo == NULL) {
 151                        ssl_cainfo = xmalloc(strlen(value)+1);
 152                        strcpy(ssl_cainfo, value);
 153                }
 154                return 0;
 155        }
 156
 157#ifdef USE_CURL_MULTI   
 158        if (!strcmp("http.maxrequests", var)) {
 159                if (max_requests == -1)
 160                        max_requests = git_config_int(var, value);
 161                return 0;
 162        }
 163#endif
 164
 165        if (!strcmp("http.lowspeedlimit", var)) {
 166                if (curl_low_speed_limit == -1)
 167                        curl_low_speed_limit = (long)git_config_int(var, value);
 168                return 0;
 169        }
 170        if (!strcmp("http.lowspeedtime", var)) {
 171                if (curl_low_speed_time == -1)
 172                        curl_low_speed_time = (long)git_config_int(var, value);
 173                return 0;
 174        }
 175
 176        /* Fall back on the default ones */
 177        return git_default_config(var, value);
 178}
 179
 180static size_t fwrite_buffer(void *ptr, size_t eltsize, size_t nmemb,
 181                            struct buffer *buffer)
 182{
 183        size_t size = eltsize * nmemb;
 184        if (size > buffer->size - buffer->posn)
 185                size = buffer->size - buffer->posn;
 186        memcpy(buffer->buffer + buffer->posn, ptr, size);
 187        buffer->posn += size;
 188        data_received++;
 189        return size;
 190}
 191
 192static size_t fwrite_buffer_dynamic(const void *ptr, size_t eltsize,
 193                                    size_t nmemb, struct buffer *buffer)
 194{
 195        size_t size = eltsize * nmemb;
 196        if (size > buffer->size - buffer->posn) {
 197                buffer->size = buffer->size * 3 / 2;
 198                if (buffer->size < buffer->posn + size)
 199                        buffer->size = buffer->posn + size;
 200                buffer->buffer = xrealloc(buffer->buffer, buffer->size);
 201        }
 202        memcpy(buffer->buffer + buffer->posn, ptr, size);
 203        buffer->posn += size;
 204        data_received++;
 205        return size;
 206}
 207
 208static size_t fwrite_sha1_file(void *ptr, size_t eltsize, size_t nmemb,
 209                               void *data)
 210{
 211        unsigned char expn[4096];
 212        size_t size = eltsize * nmemb;
 213        int posn = 0;
 214        struct transfer_request *request = (struct transfer_request *)data;
 215        do {
 216                ssize_t retval = write(request->local,
 217                                       ptr + posn, size - posn);
 218                if (retval < 0)
 219                        return posn;
 220                posn += retval;
 221        } while (posn < size);
 222
 223        request->stream.avail_in = size;
 224        request->stream.next_in = ptr;
 225        do {
 226                request->stream.next_out = expn;
 227                request->stream.avail_out = sizeof(expn);
 228                request->zret = inflate(&request->stream, Z_SYNC_FLUSH);
 229                SHA1_Update(&request->c, expn,
 230                            sizeof(expn) - request->stream.avail_out);
 231        } while (request->stream.avail_in && request->zret == Z_OK);
 232        data_received++;
 233        return size;
 234}
 235
 236#ifdef USE_CURL_MULTI
 237static void process_curl_messages(void);
 238static void process_request_queue(void);
 239#endif
 240static int fetch_alternates(char *base);
 241
 242static CURL* get_curl_handle(void)
 243{
 244        CURL* result = curl_easy_init();
 245
 246        curl_easy_setopt(result, CURLOPT_SSL_VERIFYPEER, curl_ssl_verify);
 247#if LIBCURL_VERSION_NUM >= 0x070907
 248        curl_easy_setopt(result, CURLOPT_NETRC, CURL_NETRC_OPTIONAL);
 249#endif
 250
 251        if (ssl_cert != NULL)
 252                curl_easy_setopt(result, CURLOPT_SSLCERT, ssl_cert);
 253#if LIBCURL_VERSION_NUM >= 0x070902
 254        if (ssl_key != NULL)
 255                curl_easy_setopt(result, CURLOPT_SSLKEY, ssl_key);
 256#endif
 257#if LIBCURL_VERSION_NUM >= 0x070908
 258        if (ssl_capath != NULL)
 259                curl_easy_setopt(result, CURLOPT_CAPATH, ssl_capath);
 260#endif
 261        if (ssl_cainfo != NULL)
 262                curl_easy_setopt(result, CURLOPT_CAINFO, ssl_cainfo);
 263        curl_easy_setopt(result, CURLOPT_FAILONERROR, 1);
 264
 265        if (curl_low_speed_limit > 0 && curl_low_speed_time > 0) {
 266                curl_easy_setopt(result, CURLOPT_LOW_SPEED_LIMIT,
 267                                 curl_low_speed_limit);
 268                curl_easy_setopt(result, CURLOPT_LOW_SPEED_TIME,
 269                                 curl_low_speed_time);
 270        }
 271
 272        return result;
 273}
 274
 275static struct active_request_slot *get_active_slot(void)
 276{
 277        struct active_request_slot *slot = active_queue_head;
 278        struct active_request_slot *newslot;
 279
 280#ifdef USE_CURL_MULTI
 281        int num_transfers;
 282
 283        /* Wait for a slot to open up if the queue is full */
 284        while (active_requests >= max_requests) {
 285                curl_multi_perform(curlm, &num_transfers);
 286                if (num_transfers < active_requests) {
 287                        process_curl_messages();
 288                }
 289        }
 290#endif
 291
 292        while (slot != NULL && slot->in_use) {
 293                slot = slot->next;
 294        }
 295        if (slot == NULL) {
 296                newslot = xmalloc(sizeof(*newslot));
 297                newslot->curl = NULL;
 298                newslot->in_use = 0;
 299                newslot->next = NULL;
 300
 301                slot = active_queue_head;
 302                if (slot == NULL) {
 303                        active_queue_head = newslot;
 304                } else {
 305                        while (slot->next != NULL) {
 306                                slot = slot->next;
 307                        }
 308                        slot->next = newslot;
 309                }
 310                slot = newslot;
 311        }
 312
 313        if (slot->curl == NULL) {
 314#ifdef NO_CURL_EASY_DUPHANDLE
 315                slot->curl = get_curl_handle();
 316#else
 317                slot->curl = curl_easy_duphandle(curl_default);
 318#endif
 319        }
 320
 321        active_requests++;
 322        slot->in_use = 1;
 323        slot->done = 0;
 324        slot->local = NULL;
 325        curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, pragma_header);
 326        curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, no_range_header);
 327        curl_easy_setopt(slot->curl, CURLOPT_ERRORBUFFER, curl_errorstr);
 328
 329        return slot;
 330}
 331
 332static int start_active_slot(struct active_request_slot *slot)
 333{
 334#ifdef USE_CURL_MULTI
 335        CURLMcode curlm_result = curl_multi_add_handle(curlm, slot->curl);
 336
 337        if (curlm_result != CURLM_OK &&
 338            curlm_result != CURLM_CALL_MULTI_PERFORM) {
 339                active_requests--;
 340                slot->in_use = 0;
 341                return 0;
 342        }
 343#endif
 344        return 1;
 345}
 346
 347static void run_active_slot(struct active_request_slot *slot)
 348{
 349#ifdef USE_CURL_MULTI
 350        int num_transfers;
 351        long last_pos = 0;
 352        long current_pos;
 353        fd_set readfds;
 354        fd_set writefds;
 355        fd_set excfds;
 356        int max_fd;
 357        struct timeval select_timeout;
 358        CURLMcode curlm_result;
 359
 360        while (!slot->done) {
 361                data_received = 0;
 362                do {
 363                        curlm_result = curl_multi_perform(curlm,
 364                                                          &num_transfers);
 365                } while (curlm_result == CURLM_CALL_MULTI_PERFORM);
 366                if (num_transfers < active_requests) {
 367                        process_curl_messages();
 368                        process_request_queue();
 369                }
 370
 371                if (!data_received && slot->local != NULL) {
 372                        current_pos = ftell(slot->local);
 373                        if (current_pos > last_pos)
 374                                data_received++;
 375                        last_pos = current_pos;
 376                }
 377
 378                if (!slot->done && !data_received) {
 379                        max_fd = 0;
 380                        FD_ZERO(&readfds);
 381                        FD_ZERO(&writefds);
 382                        FD_ZERO(&excfds);
 383                        select_timeout.tv_sec = 0;
 384                        select_timeout.tv_usec = 50000;
 385                        select(max_fd, &readfds, &writefds,
 386                               &excfds, &select_timeout);
 387                }
 388        }
 389#else
 390        slot->curl_result = curl_easy_perform(slot->curl);
 391        active_requests--;
 392#endif
 393}
 394
 395static void start_request(struct transfer_request *request)
 396{
 397        char *hex = sha1_to_hex(request->sha1);
 398        char prevfile[PATH_MAX];
 399        char *url;
 400        char *posn;
 401        int prevlocal;
 402        unsigned char prev_buf[PREV_BUF_SIZE];
 403        ssize_t prev_read = 0;
 404        long prev_posn = 0;
 405        char range[RANGE_HEADER_SIZE];
 406        struct curl_slist *range_header = NULL;
 407        struct active_request_slot *slot;
 408
 409        snprintf(prevfile, sizeof(prevfile), "%s.prev", request->filename);
 410        unlink(prevfile);
 411        rename(request->tmpfile, prevfile);
 412        unlink(request->tmpfile);
 413
 414        request->local = open(request->tmpfile,
 415                              O_WRONLY | O_CREAT | O_EXCL, 0666);
 416        /* This could have failed due to the "lazy directory creation";
 417         * try to mkdir the last path component.
 418         */
 419        if (request->local < 0 && errno == ENOENT) {
 420                char *dir = strrchr(request->tmpfile, '/');
 421                if (dir) {
 422                        *dir = 0;
 423                        mkdir(request->tmpfile, 0777);
 424                        *dir = '/';
 425                }
 426                request->local = open(request->tmpfile,
 427                                      O_WRONLY | O_CREAT | O_EXCL, 0666);
 428        }
 429
 430        if (request->local < 0) {
 431                request->state = ABORTED;
 432                error("Couldn't create temporary file %s for %s: %s\n",
 433                      request->tmpfile, request->filename, strerror(errno));
 434                return;
 435        }
 436
 437        memset(&request->stream, 0, sizeof(request->stream));
 438
 439        inflateInit(&request->stream);
 440
 441        SHA1_Init(&request->c);
 442
 443        url = xmalloc(strlen(request->repo->base) + 50);
 444        request->url = xmalloc(strlen(request->repo->base) + 50);
 445        strcpy(url, request->repo->base);
 446        posn = url + strlen(request->repo->base);
 447        strcpy(posn, "objects/");
 448        posn += 8;
 449        memcpy(posn, hex, 2);
 450        posn += 2;
 451        *(posn++) = '/';
 452        strcpy(posn, hex + 2);
 453        strcpy(request->url, url);
 454
 455        /* If a previous temp file is present, process what was already
 456           fetched. */
 457        prevlocal = open(prevfile, O_RDONLY);
 458        if (prevlocal != -1) {
 459                do {
 460                        prev_read = read(prevlocal, prev_buf, PREV_BUF_SIZE);
 461                        if (prev_read>0) {
 462                                if (fwrite_sha1_file(prev_buf,
 463                                                     1,
 464                                                     prev_read,
 465                                                     request) == prev_read) {
 466                                        prev_posn += prev_read;
 467                                } else {
 468                                        prev_read = -1;
 469                                }
 470                        }
 471                } while (prev_read > 0);
 472                close(prevlocal);
 473        }
 474        unlink(prevfile);
 475
 476        /* Reset inflate/SHA1 if there was an error reading the previous temp
 477           file; also rewind to the beginning of the local file. */
 478        if (prev_read == -1) {
 479                memset(&request->stream, 0, sizeof(request->stream));
 480                inflateInit(&request->stream);
 481                SHA1_Init(&request->c);
 482                if (prev_posn>0) {
 483                        prev_posn = 0;
 484                        lseek(request->local, SEEK_SET, 0);
 485                        ftruncate(request->local, 0);
 486                }
 487        }
 488
 489        slot = get_active_slot();
 490        curl_easy_setopt(slot->curl, CURLOPT_FILE, request);
 491        curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite_sha1_file);
 492        curl_easy_setopt(slot->curl, CURLOPT_ERRORBUFFER, request->errorstr);
 493        curl_easy_setopt(slot->curl, CURLOPT_URL, url);
 494        curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, no_pragma_header);
 495
 496        /* If we have successfully processed data from a previous fetch
 497           attempt, only fetch the data we don't already have. */
 498        if (prev_posn>0) {
 499                if (get_verbosely)
 500                        fprintf(stderr,
 501                                "Resuming fetch of object %s at byte %ld\n",
 502                                hex, prev_posn);
 503                sprintf(range, "Range: bytes=%ld-", prev_posn);
 504                range_header = curl_slist_append(range_header, range);
 505                curl_easy_setopt(slot->curl,
 506                                 CURLOPT_HTTPHEADER, range_header);
 507        }
 508
 509        /* Try to get the request started, abort the request on error */
 510        if (!start_active_slot(slot)) {
 511                request->state = ABORTED;
 512                close(request->local);
 513                free(request->url);
 514                return;
 515        }
 516        
 517        request->slot = slot;
 518        request->state = ACTIVE;
 519}
 520
 521static void finish_request(struct transfer_request *request)
 522{
 523        fchmod(request->local, 0444);
 524        close(request->local);
 525
 526        if (request->http_code == 416) {
 527                fprintf(stderr, "Warning: requested range invalid; we may already have all the data.\n");
 528        } else if (request->curl_result != CURLE_OK) {
 529                return;
 530        }
 531
 532        inflateEnd(&request->stream);
 533        SHA1_Final(request->real_sha1, &request->c);
 534        if (request->zret != Z_STREAM_END) {
 535                unlink(request->tmpfile);
 536                return;
 537        }
 538        if (memcmp(request->sha1, request->real_sha1, 20)) {
 539                unlink(request->tmpfile);
 540                return;
 541        }
 542        request->rename =
 543                move_temp_to_file(request->tmpfile, request->filename);
 544
 545        if (request->rename == 0)
 546                pull_say("got %s\n", sha1_to_hex(request->sha1));
 547}
 548
 549static void release_request(struct transfer_request *request)
 550{
 551        struct transfer_request *entry = request_queue_head;
 552
 553        if (request == request_queue_head) {
 554                request_queue_head = request->next;
 555        } else {
 556                while (entry->next != NULL && entry->next != request)
 557                        entry = entry->next;
 558                if (entry->next == request)
 559                        entry->next = entry->next->next;
 560        }
 561
 562        free(request->url);
 563        free(request);
 564}
 565
 566#ifdef USE_CURL_MULTI
 567void process_curl_messages(void)
 568{
 569        int num_messages;
 570        struct active_request_slot *slot;
 571        struct transfer_request *request = NULL;
 572        CURLMsg *curl_message = curl_multi_info_read(curlm, &num_messages);
 573
 574        while (curl_message != NULL) {
 575                if (curl_message->msg == CURLMSG_DONE) {
 576                        slot = active_queue_head;
 577                        while (slot != NULL &&
 578                               slot->curl != curl_message->easy_handle)
 579                                slot = slot->next;
 580                        if (slot != NULL) {
 581                                curl_multi_remove_handle(curlm, slot->curl);
 582                                active_requests--;
 583                                slot->done = 1;
 584                                slot->in_use = 0;
 585                                slot->curl_result = curl_message->data.result;
 586                                curl_easy_getinfo(slot->curl,
 587                                                  CURLINFO_HTTP_CODE,
 588                                                  &slot->http_code);
 589                                request = request_queue_head;
 590                                while (request != NULL &&
 591                                       request->slot != slot)
 592                                        request = request->next;
 593                        } else {
 594                                fprintf(stderr, "Received DONE message for unknown request!\n");
 595                        }
 596                        if (request != NULL) {
 597                                request->curl_result =
 598                                        curl_message->data.result;
 599                                request->http_code = slot->http_code;
 600                                request->slot = NULL;
 601                                request->state = COMPLETE;
 602
 603                                /* Use alternates if necessary */
 604                                if (request->http_code == 404) {
 605                                        fetch_alternates(alt->base);
 606                                        if (request->repo->next != NULL) {
 607                                                request->repo =
 608                                                        request->repo->next;
 609                                                start_request(request);
 610                                        }
 611                                } else {
 612                                        finish_request(request);
 613                                }
 614                        }
 615                } else {
 616                        fprintf(stderr, "Unknown CURL message received: %d\n",
 617                                (int)curl_message->msg);
 618                }
 619                curl_message = curl_multi_info_read(curlm, &num_messages);
 620        }
 621}
 622
 623void process_request_queue(void)
 624{
 625        struct transfer_request *request = request_queue_head;
 626        struct active_request_slot *slot = active_queue_head;
 627        int num_transfers;
 628
 629        while (active_requests < max_requests && request != NULL) {
 630                if (request->state == WAITING) {
 631                        if (has_sha1_file(request->sha1))
 632                                release_request(request);
 633                        else
 634                                start_request(request);
 635                        curl_multi_perform(curlm, &num_transfers);
 636                }
 637                request = request->next;
 638        }
 639
 640        while (slot != NULL) {
 641                if (!slot->in_use && slot->curl != NULL) {
 642                        curl_easy_cleanup(slot->curl);
 643                        slot->curl = NULL;
 644                }
 645                slot = slot->next;
 646        }                               
 647}
 648#endif
 649
 650void prefetch(unsigned char *sha1)
 651{
 652        struct transfer_request *newreq;
 653        struct transfer_request *tail;
 654        char *filename = sha1_file_name(sha1);
 655
 656        newreq = xmalloc(sizeof(*newreq));
 657        memcpy(newreq->sha1, sha1, 20);
 658        newreq->repo = alt;
 659        newreq->url = NULL;
 660        newreq->local = -1;
 661        newreq->state = WAITING;
 662        snprintf(newreq->filename, sizeof(newreq->filename), "%s", filename);
 663        snprintf(newreq->tmpfile, sizeof(newreq->tmpfile),
 664                 "%s.temp", filename);
 665        newreq->next = NULL;
 666
 667        if (request_queue_head == NULL) {
 668                request_queue_head = newreq;
 669        } else {
 670                tail = request_queue_head;
 671                while (tail->next != NULL) {
 672                        tail = tail->next;
 673                }
 674                tail->next = newreq;
 675        }
 676#ifdef USE_CURL_MULTI
 677        process_request_queue();
 678        process_curl_messages();
 679#endif
 680}
 681
 682static int fetch_index(struct alt_base *repo, unsigned char *sha1)
 683{
 684        char *hex = sha1_to_hex(sha1);
 685        char *filename;
 686        char *url;
 687        char tmpfile[PATH_MAX];
 688        long prev_posn = 0;
 689        char range[RANGE_HEADER_SIZE];
 690        struct curl_slist *range_header = NULL;
 691
 692        FILE *indexfile;
 693        struct active_request_slot *slot;
 694
 695        if (has_pack_index(sha1))
 696                return 0;
 697
 698        if (get_verbosely)
 699                fprintf(stderr, "Getting index for pack %s\n", hex);
 700        
 701        url = xmalloc(strlen(repo->base) + 64);
 702        sprintf(url, "%s/objects/pack/pack-%s.idx", repo->base, hex);
 703        
 704        filename = sha1_pack_index_name(sha1);
 705        snprintf(tmpfile, sizeof(tmpfile), "%s.temp", filename);
 706        indexfile = fopen(tmpfile, "a");
 707        if (!indexfile)
 708                return error("Unable to open local file %s for pack index",
 709                             filename);
 710
 711        slot = get_active_slot();
 712        curl_easy_setopt(slot->curl, CURLOPT_FILE, indexfile);
 713        curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite);
 714        curl_easy_setopt(slot->curl, CURLOPT_URL, url);
 715        curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, no_pragma_header);
 716        slot->local = indexfile;
 717
 718        /* If there is data present from a previous transfer attempt,
 719           resume where it left off */
 720        prev_posn = ftell(indexfile);
 721        if (prev_posn>0) {
 722                if (get_verbosely)
 723                        fprintf(stderr,
 724                                "Resuming fetch of index for pack %s at byte %ld\n",
 725                                hex, prev_posn);
 726                sprintf(range, "Range: bytes=%ld-", prev_posn);
 727                range_header = curl_slist_append(range_header, range);
 728                curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, range_header);
 729        }
 730
 731        if (start_active_slot(slot)) {
 732                run_active_slot(slot);
 733                if (slot->curl_result != CURLE_OK) {
 734                        fclose(indexfile);
 735                        return error("Unable to get pack index %s\n%s", url,
 736                                     curl_errorstr);
 737                }
 738        } else {
 739                return error("Unable to start request");
 740        }
 741
 742        fclose(indexfile);
 743
 744        return move_temp_to_file(tmpfile, filename);
 745}
 746
 747static int setup_index(struct alt_base *repo, unsigned char *sha1)
 748{
 749        struct packed_git *new_pack;
 750        if (has_pack_file(sha1))
 751                return 0; // don't list this as something we can get
 752
 753        if (fetch_index(repo, sha1))
 754                return -1;
 755
 756        new_pack = parse_pack_index(sha1);
 757        new_pack->next = repo->packs;
 758        repo->packs = new_pack;
 759        return 0;
 760}
 761
 762static int fetch_alternates(char *base)
 763{
 764        int ret = 0;
 765        struct buffer buffer;
 766        char *url;
 767        char *data;
 768        int i = 0;
 769        int http_specific = 1;
 770        struct alt_base *tail = alt;
 771        static const char null_byte = '\0';
 772
 773        struct active_request_slot *slot;
 774
 775        if (got_alternates)
 776                return 0;
 777
 778        data = xmalloc(4096);
 779        buffer.size = 4096;
 780        buffer.posn = 0;
 781        buffer.buffer = data;
 782
 783        if (get_verbosely)
 784                fprintf(stderr, "Getting alternates list\n");
 785        
 786        url = xmalloc(strlen(base) + 31);
 787        sprintf(url, "%s/objects/info/http-alternates", base);
 788
 789        slot = get_active_slot();
 790        curl_easy_setopt(slot->curl, CURLOPT_FILE, &buffer);
 791        curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION,
 792                         fwrite_buffer_dynamic);
 793        curl_easy_setopt(slot->curl, CURLOPT_URL, url);
 794        if (start_active_slot(slot)) {
 795                run_active_slot(slot);
 796                if (slot->curl_result != CURLE_OK || !buffer.posn) {
 797                        http_specific = 0;
 798
 799                        sprintf(url, "%s/objects/info/alternates", base);
 800
 801                        slot = get_active_slot();
 802                        curl_easy_setopt(slot->curl, CURLOPT_FILE, &buffer);
 803                        curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION,
 804                                         fwrite_buffer_dynamic);
 805                        curl_easy_setopt(slot->curl, CURLOPT_URL, url);
 806                        if (start_active_slot(slot)) {
 807                                run_active_slot(slot);
 808                                if (slot->curl_result != CURLE_OK) {
 809                                        free(buffer.buffer);
 810                                        if (slot->http_code == 404)
 811                                                got_alternates = 1;
 812                                        return 0;
 813                                }
 814                        }
 815                }
 816        } else {
 817                free(buffer.buffer);
 818                return 0;
 819        }
 820
 821        fwrite_buffer_dynamic(&null_byte, 1, 1, &buffer);
 822        buffer.posn--;
 823        data = buffer.buffer;
 824
 825        while (i < buffer.posn) {
 826                int posn = i;
 827                while (posn < buffer.posn && data[posn] != '\n')
 828                        posn++;
 829                if (data[posn] == '\n') {
 830                        int okay = 0;
 831                        int serverlen = 0;
 832                        struct alt_base *newalt;
 833                        char *target = NULL;
 834                        if (data[i] == '/') {
 835                                serverlen = strchr(base + 8, '/') - base;
 836                                okay = 1;
 837                        } else if (!memcmp(data + i, "../", 3)) {
 838                                i += 3;
 839                                serverlen = strlen(base);
 840                                while (i + 2 < posn && 
 841                                       !memcmp(data + i, "../", 3)) {
 842                                        do {
 843                                                serverlen--;
 844                                        } while (serverlen &&
 845                                                 base[serverlen - 1] != '/');
 846                                        i += 3;
 847                                }
 848                                // If the server got removed, give up.
 849                                okay = strchr(base, ':') - base + 3 < 
 850                                        serverlen;
 851                        } else if (http_specific) {
 852                                char *colon = strchr(data + i, ':');
 853                                char *slash = strchr(data + i, '/');
 854                                if (colon && slash && colon < data + posn &&
 855                                    slash < data + posn && colon < slash) {
 856                                        okay = 1;
 857                                }
 858                        }
 859                        // skip 'objects' at end
 860                        if (okay) {
 861                                target = xmalloc(serverlen + posn - i - 6);
 862                                strncpy(target, base, serverlen);
 863                                strncpy(target + serverlen, data + i,
 864                                        posn - i - 7);
 865                                target[serverlen + posn - i - 7] = '\0';
 866                                if (get_verbosely)
 867                                        fprintf(stderr, 
 868                                                "Also look at %s\n", target);
 869                                newalt = xmalloc(sizeof(*newalt));
 870                                newalt->next = NULL;
 871                                newalt->base = target;
 872                                newalt->got_indices = 0;
 873                                newalt->packs = NULL;
 874                                while (tail->next != NULL)
 875                                        tail = tail->next;
 876                                tail->next = newalt;
 877                                ret++;
 878                        }
 879                }
 880                i = posn + 1;
 881        }
 882
 883        got_alternates = 1;
 884        free(buffer.buffer);
 885        return ret;
 886}
 887
 888static int fetch_indices(struct alt_base *repo)
 889{
 890        unsigned char sha1[20];
 891        char *url;
 892        struct buffer buffer;
 893        char *data;
 894        int i = 0;
 895
 896        struct active_request_slot *slot;
 897
 898        if (repo->got_indices)
 899                return 0;
 900
 901        data = xmalloc(4096);
 902        buffer.size = 4096;
 903        buffer.posn = 0;
 904        buffer.buffer = data;
 905
 906        if (get_verbosely)
 907                fprintf(stderr, "Getting pack list\n");
 908        
 909        url = xmalloc(strlen(repo->base) + 21);
 910        sprintf(url, "%s/objects/info/packs", repo->base);
 911
 912        slot = get_active_slot();
 913        curl_easy_setopt(slot->curl, CURLOPT_FILE, &buffer);
 914        curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION,
 915                         fwrite_buffer_dynamic);
 916        curl_easy_setopt(slot->curl, CURLOPT_URL, url);
 917        curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, NULL);
 918        if (start_active_slot(slot)) {
 919                run_active_slot(slot);
 920                if (slot->curl_result != CURLE_OK) {
 921                        free(buffer.buffer);
 922                        return error("%s", curl_errorstr);
 923                }
 924        } else {
 925                free(buffer.buffer);
 926                return error("Unable to start request");
 927        }
 928
 929        data = buffer.buffer;
 930        while (i < buffer.posn) {
 931                switch (data[i]) {
 932                case 'P':
 933                        i++;
 934                        if (i + 52 < buffer.posn &&
 935                            !strncmp(data + i, " pack-", 6) &&
 936                            !strncmp(data + i + 46, ".pack\n", 6)) {
 937                                get_sha1_hex(data + i + 6, sha1);
 938                                setup_index(repo, sha1);
 939                                i += 51;
 940                                break;
 941                        }
 942                default:
 943                        while (data[i] != '\n')
 944                                i++;
 945                }
 946                i++;
 947        }
 948
 949        free(buffer.buffer);
 950        repo->got_indices = 1;
 951        return 0;
 952}
 953
 954static int fetch_pack(struct alt_base *repo, unsigned char *sha1)
 955{
 956        char *url;
 957        struct packed_git *target;
 958        struct packed_git **lst;
 959        FILE *packfile;
 960        char *filename;
 961        char tmpfile[PATH_MAX];
 962        int ret;
 963        long prev_posn = 0;
 964        char range[RANGE_HEADER_SIZE];
 965        struct curl_slist *range_header = NULL;
 966
 967        struct active_request_slot *slot;
 968
 969        if (fetch_indices(repo))
 970                return -1;
 971        target = find_sha1_pack(sha1, repo->packs);
 972        if (!target)
 973                return -1;
 974
 975        if (get_verbosely) {
 976                fprintf(stderr, "Getting pack %s\n",
 977                        sha1_to_hex(target->sha1));
 978                fprintf(stderr, " which contains %s\n",
 979                        sha1_to_hex(sha1));
 980        }
 981
 982        url = xmalloc(strlen(repo->base) + 65);
 983        sprintf(url, "%s/objects/pack/pack-%s.pack",
 984                repo->base, sha1_to_hex(target->sha1));
 985
 986        filename = sha1_pack_name(target->sha1);
 987        snprintf(tmpfile, sizeof(tmpfile), "%s.temp", filename);
 988        packfile = fopen(tmpfile, "a");
 989        if (!packfile)
 990                return error("Unable to open local file %s for pack",
 991                             filename);
 992
 993        slot = get_active_slot();
 994        curl_easy_setopt(slot->curl, CURLOPT_FILE, packfile);
 995        curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite);
 996        curl_easy_setopt(slot->curl, CURLOPT_URL, url);
 997        curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, no_pragma_header);
 998        slot->local = packfile;
 999
1000        /* If there is data present from a previous transfer attempt,
1001           resume where it left off */
1002        prev_posn = ftell(packfile);
1003        if (prev_posn>0) {
1004                if (get_verbosely)
1005                        fprintf(stderr,
1006                                "Resuming fetch of pack %s at byte %ld\n",
1007                                sha1_to_hex(target->sha1), prev_posn);
1008                sprintf(range, "Range: bytes=%ld-", prev_posn);
1009                range_header = curl_slist_append(range_header, range);
1010                curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, range_header);
1011        }
1012
1013        if (start_active_slot(slot)) {
1014                run_active_slot(slot);
1015                if (slot->curl_result != CURLE_OK) {
1016                        fclose(packfile);
1017                        return error("Unable to get pack file %s\n%s", url,
1018                                     curl_errorstr);
1019                }
1020        } else {
1021                return error("Unable to start request");
1022        }
1023
1024        fclose(packfile);
1025
1026        ret = move_temp_to_file(tmpfile, filename);
1027        if (ret)
1028                return ret;
1029
1030        lst = &repo->packs;
1031        while (*lst != target)
1032                lst = &((*lst)->next);
1033        *lst = (*lst)->next;
1034
1035        if (verify_pack(target, 0))
1036                return -1;
1037        install_packed_git(target);
1038
1039        return 0;
1040}
1041
1042static int fetch_object(struct alt_base *repo, unsigned char *sha1)
1043{
1044        char *hex = sha1_to_hex(sha1);
1045        int ret;
1046        struct transfer_request *request = request_queue_head;
1047
1048        while (request != NULL && memcmp(request->sha1, sha1, 20))
1049                request = request->next;
1050        if (request == NULL)
1051                return error("Couldn't find request for %s in the queue", hex);
1052
1053        if (has_sha1_file(request->sha1)) {
1054                release_request(request);
1055                return 0;
1056        }
1057
1058#ifdef USE_CURL_MULTI
1059        while (request->state == WAITING) {
1060                int num_transfers;
1061                curl_multi_perform(curlm, &num_transfers);
1062                if (num_transfers < active_requests) {
1063                        process_curl_messages();
1064                        process_request_queue();
1065                }
1066        }
1067#else
1068        start_request(request);
1069#endif
1070
1071        while (request->state == ACTIVE) {
1072                run_active_slot(request->slot);
1073#ifndef USE_CURL_MULTI
1074                request->curl_result = request->slot->curl_result;
1075                request->http_code = request->slot->http_code;
1076                request->slot = NULL;
1077
1078                /* Use alternates if necessary */
1079                if (request->http_code == 404) {
1080                        fetch_alternates(alt->base);
1081                        if (request->repo->next != NULL) {
1082                                request->repo = request->repo->next;
1083                                start_request(request);
1084                        }
1085                } else {
1086                        finish_request(request);
1087                        request->state = COMPLETE;
1088                }
1089#endif
1090        }
1091
1092        if (request->state == ABORTED) {
1093                release_request(request);
1094                return error("Request for %s aborted", hex);
1095        }
1096
1097        if (request->curl_result != CURLE_OK && request->http_code != 416) {
1098                if (request->http_code == 404)
1099                        ret = -1; /* Be silent, it is probably in a pack. */
1100                else
1101                        ret = error("%s (curl_result = %d, http_code = %ld, sha1 = %s)",
1102                                    request->errorstr, request->curl_result,
1103                                    request->http_code, hex);
1104                release_request(request);
1105                return ret;
1106        }
1107
1108        if (request->zret != Z_STREAM_END) {
1109                ret = error("File %s (%s) corrupt\n", hex, request->url);
1110                release_request(request);
1111                return ret;
1112        }
1113
1114        if (memcmp(request->sha1, request->real_sha1, 20)) {
1115                release_request(request);
1116                return error("File %s has bad hash\n", hex);
1117        }
1118
1119        if (request->rename < 0) {
1120                ret = error("unable to write sha1 filename %s: %s",
1121                            request->filename,
1122                            strerror(request->rename));
1123                release_request(request);
1124                return ret;
1125        }
1126
1127        release_request(request);
1128        return 0;
1129}
1130
1131int fetch(unsigned char *sha1)
1132{
1133        struct alt_base *altbase = alt;
1134
1135        if (!fetch_object(altbase, sha1))
1136                return 0;
1137        while (altbase) {
1138                if (!fetch_pack(altbase, sha1))
1139                        return 0;
1140                fetch_alternates(alt->base);
1141                altbase = altbase->next;
1142        }
1143        return error("Unable to find %s under %s\n", sha1_to_hex(sha1), 
1144                     alt->base);
1145}
1146
1147static inline int needs_quote(int ch)
1148{
1149        switch (ch) {
1150        case '/': case '-': case '.':
1151        case 'A'...'Z': case 'a'...'z': case '0'...'9':
1152                return 0;
1153        default:
1154                return 1;
1155        }
1156}
1157
1158static inline int hex(int v)
1159{
1160        if (v < 10) return '0' + v;
1161        else return 'A' + v - 10;
1162}
1163
1164static char *quote_ref_url(const char *base, const char *ref)
1165{
1166        const char *cp;
1167        char *dp, *qref;
1168        int len, baselen, ch;
1169
1170        baselen = strlen(base);
1171        len = baselen + 6; /* "refs/" + NUL */
1172        for (cp = ref; (ch = *cp) != 0; cp++, len++)
1173                if (needs_quote(ch))
1174                        len += 2; /* extra two hex plus replacement % */
1175        qref = xmalloc(len);
1176        memcpy(qref, base, baselen);
1177        memcpy(qref + baselen, "refs/", 5);
1178        for (cp = ref, dp = qref + baselen + 5; (ch = *cp) != 0; cp++) {
1179                if (needs_quote(ch)) {
1180                        *dp++ = '%';
1181                        *dp++ = hex((ch >> 4) & 0xF);
1182                        *dp++ = hex(ch & 0xF);
1183                }
1184                else
1185                        *dp++ = ch;
1186        }
1187        *dp = 0;
1188
1189        return qref;
1190}
1191
1192int fetch_ref(char *ref, unsigned char *sha1)
1193{
1194        char *url;
1195        char hex[42];
1196        struct buffer buffer;
1197        char *base = alt->base;
1198        struct active_request_slot *slot;
1199        buffer.size = 41;
1200        buffer.posn = 0;
1201        buffer.buffer = hex;
1202        hex[41] = '\0';
1203        
1204        url = quote_ref_url(base, ref);
1205        slot = get_active_slot();
1206        curl_easy_setopt(slot->curl, CURLOPT_FILE, &buffer);
1207        curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite_buffer);
1208        curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, NULL);
1209        curl_easy_setopt(slot->curl, CURLOPT_URL, url);
1210        if (start_active_slot(slot)) {
1211                run_active_slot(slot);
1212                if (slot->curl_result != CURLE_OK)
1213                        return error("Couldn't get %s for %s\n%s",
1214                                     url, ref, curl_errorstr);
1215        } else {
1216                return error("Unable to start request");
1217        }
1218
1219        hex[40] = '\0';
1220        get_sha1_hex(hex, sha1);
1221        return 0;
1222}
1223
1224int main(int argc, char **argv)
1225{
1226        char *commit_id;
1227        char *url;
1228        int arg = 1;
1229        struct active_request_slot *slot;
1230        char *low_speed_limit;
1231        char *low_speed_time;
1232        char *wait_url;
1233        int rc = 0;
1234
1235        while (arg < argc && argv[arg][0] == '-') {
1236                if (argv[arg][1] == 't') {
1237                        get_tree = 1;
1238                } else if (argv[arg][1] == 'c') {
1239                        get_history = 1;
1240                } else if (argv[arg][1] == 'a') {
1241                        get_all = 1;
1242                        get_tree = 1;
1243                        get_history = 1;
1244                } else if (argv[arg][1] == 'v') {
1245                        get_verbosely = 1;
1246                } else if (argv[arg][1] == 'w') {
1247                        write_ref = argv[arg + 1];
1248                        arg++;
1249                } else if (!strcmp(argv[arg], "--recover")) {
1250                        get_recover = 1;
1251                }
1252                arg++;
1253        }
1254        if (argc < arg + 2) {
1255                usage("git-http-fetch [-c] [-t] [-a] [-d] [-v] [--recover] [-w ref] commit-id url");
1256                return 1;
1257        }
1258        commit_id = argv[arg];
1259        url = argv[arg + 1];
1260
1261        curl_global_init(CURL_GLOBAL_ALL);
1262
1263#ifdef USE_CURL_MULTI
1264        {
1265                char *http_max_requests = getenv("GIT_HTTP_MAX_REQUESTS");
1266                if (http_max_requests != NULL)
1267                        max_requests = atoi(http_max_requests);
1268        }
1269
1270        curlm = curl_multi_init();
1271        if (curlm == NULL) {
1272                fprintf(stderr, "Error creating curl multi handle.\n");
1273                return 1;
1274        }
1275#endif
1276
1277        if (getenv("GIT_SSL_NO_VERIFY"))
1278                curl_ssl_verify = 0;
1279
1280        ssl_cert = getenv("GIT_SSL_CERT");
1281#if LIBCURL_VERSION_NUM >= 0x070902
1282        ssl_key = getenv("GIT_SSL_KEY");
1283#endif
1284#if LIBCURL_VERSION_NUM >= 0x070908
1285        ssl_capath = getenv("GIT_SSL_CAPATH");
1286#endif
1287        ssl_cainfo = getenv("GIT_SSL_CAINFO");
1288
1289        low_speed_limit = getenv("GIT_HTTP_LOW_SPEED_LIMIT");
1290        if (low_speed_limit != NULL)
1291                curl_low_speed_limit = strtol(low_speed_limit, NULL, 10);
1292        low_speed_time = getenv("GIT_HTTP_LOW_SPEED_TIME");
1293        if (low_speed_time != NULL)
1294                curl_low_speed_time = strtol(low_speed_time, NULL, 10);
1295
1296        git_config(http_options);
1297
1298        if (curl_ssl_verify == -1)
1299                curl_ssl_verify = 1;
1300
1301#ifdef USE_CURL_MULTI
1302        if (max_requests < 1)
1303                max_requests = DEFAULT_MAX_REQUESTS;
1304#endif
1305
1306        pragma_header = curl_slist_append(pragma_header, "Pragma: no-cache");
1307        no_pragma_header = curl_slist_append(no_pragma_header, "Pragma:");
1308        no_range_header = curl_slist_append(no_range_header, "Range:");
1309
1310#ifndef NO_CURL_EASY_DUPHANDLE
1311        curl_default = get_curl_handle();
1312#endif
1313
1314        alt = xmalloc(sizeof(*alt));
1315        alt->base = url;
1316        alt->got_indices = 0;
1317        alt->packs = NULL;
1318        alt->next = NULL;
1319
1320        if (pull(commit_id))
1321                rc = 1;
1322
1323        curl_slist_free_all(pragma_header);
1324        curl_slist_free_all(no_pragma_header);
1325        curl_slist_free_all(no_range_header);
1326#ifndef NO_CURL_EASY_DUPHANDLE
1327        curl_easy_cleanup(curl_default);
1328#endif
1329        slot = active_queue_head;
1330        while (slot != NULL) {
1331                if (slot->in_use) {
1332                        if (get_verbosely) {
1333                                curl_easy_getinfo(slot->curl,
1334                                                  CURLINFO_EFFECTIVE_URL,
1335                                                  &wait_url);
1336                                fprintf(stderr, "Waiting for %s\n", wait_url);
1337                        }
1338                        run_active_slot(slot);
1339                }
1340                if (slot->curl != NULL)
1341                        curl_easy_cleanup(slot->curl);
1342                slot = slot->next;
1343        }
1344#ifdef USE_CURL_MULTI
1345        curl_multi_cleanup(curlm);
1346#endif
1347        curl_global_cleanup();
1348        return rc;
1349}