Andrew's git / gitweb.git / http-fetch.c
?
http-fetch.con commit Isolate shared HTTP request functionality (29508e1)
   1#include "cache.h"
   2#include "commit.h"
   3#include "pack.h"
   4#include "fetch.h"
   5#include "http.h"
   6
   7#define PREV_BUF_SIZE 4096
   8#define RANGE_HEADER_SIZE 30
   9
  10static int got_alternates = -1;
  11
  12static struct curl_slist *no_pragma_header;
  13
  14struct alt_base
  15{
  16        char *base;
  17        int got_indices;
  18        struct packed_git *packs;
  19        struct alt_base *next;
  20};
  21
  22static struct alt_base *alt = NULL;
  23
  24enum transfer_state {
  25        WAITING,
  26        ABORTED,
  27        ACTIVE,
  28        COMPLETE,
  29};
  30
  31struct transfer_request
  32{
  33        unsigned char sha1[20];
  34        struct alt_base *repo;
  35        char *url;
  36        char filename[PATH_MAX];
  37        char tmpfile[PATH_MAX];
  38        int local;
  39        enum transfer_state state;
  40        CURLcode curl_result;
  41        char errorstr[CURL_ERROR_SIZE];
  42        long http_code;
  43        unsigned char real_sha1[20];
  44        SHA_CTX c;
  45        z_stream stream;
  46        int zret;
  47        int rename;
  48        struct active_request_slot *slot;
  49        struct transfer_request *next;
  50};
  51
  52struct alt_request {
  53        char *base;
  54        char *url;
  55        struct buffer *buffer;
  56        struct active_request_slot *slot;
  57        int http_specific;
  58};
  59
  60static struct transfer_request *request_queue_head = NULL;
  61
  62static size_t fwrite_sha1_file(void *ptr, size_t eltsize, size_t nmemb,
  63                               void *data)
  64{
  65        unsigned char expn[4096];
  66        size_t size = eltsize * nmemb;
  67        int posn = 0;
  68        struct transfer_request *request = (struct transfer_request *)data;
  69        do {
  70                ssize_t retval = write(request->local,
  71                                       ptr + posn, size - posn);
  72                if (retval < 0)
  73                        return posn;
  74                posn += retval;
  75        } while (posn < size);
  76
  77        request->stream.avail_in = size;
  78        request->stream.next_in = ptr;
  79        do {
  80                request->stream.next_out = expn;
  81                request->stream.avail_out = sizeof(expn);
  82                request->zret = inflate(&request->stream, Z_SYNC_FLUSH);
  83                SHA1_Update(&request->c, expn,
  84                            sizeof(expn) - request->stream.avail_out);
  85        } while (request->stream.avail_in && request->zret == Z_OK);
  86        data_received++;
  87        return size;
  88}
  89
  90static void fetch_alternates(char *base);
  91
  92static void process_object_response(void *callback_data);
  93
  94static void start_request(struct transfer_request *request)
  95{
  96        char *hex = sha1_to_hex(request->sha1);
  97        char prevfile[PATH_MAX];
  98        char *url;
  99        char *posn;
 100        int prevlocal;
 101        unsigned char prev_buf[PREV_BUF_SIZE];
 102        ssize_t prev_read = 0;
 103        long prev_posn = 0;
 104        char range[RANGE_HEADER_SIZE];
 105        struct curl_slist *range_header = NULL;
 106        struct active_request_slot *slot;
 107
 108        snprintf(prevfile, sizeof(prevfile), "%s.prev", request->filename);
 109        unlink(prevfile);
 110        rename(request->tmpfile, prevfile);
 111        unlink(request->tmpfile);
 112
 113        if (request->local != -1)
 114                error("fd leakage in start: %d", request->local);
 115        request->local = open(request->tmpfile,
 116                              O_WRONLY | O_CREAT | O_EXCL, 0666);
 117        /* This could have failed due to the "lazy directory creation";
 118         * try to mkdir the last path component.
 119         */
 120        if (request->local < 0 && errno == ENOENT) {
 121                char *dir = strrchr(request->tmpfile, '/');
 122                if (dir) {
 123                        *dir = 0;
 124                        mkdir(request->tmpfile, 0777);
 125                        *dir = '/';
 126                }
 127                request->local = open(request->tmpfile,
 128                                      O_WRONLY | O_CREAT | O_EXCL, 0666);
 129        }
 130
 131        if (request->local < 0) {
 132                request->state = ABORTED;
 133                error("Couldn't create temporary file %s for %s: %s\n",
 134                      request->tmpfile, request->filename, strerror(errno));
 135                return;
 136        }
 137
 138        memset(&request->stream, 0, sizeof(request->stream));
 139
 140        inflateInit(&request->stream);
 141
 142        SHA1_Init(&request->c);
 143
 144        url = xmalloc(strlen(request->repo->base) + 50);
 145        request->url = xmalloc(strlen(request->repo->base) + 50);
 146        strcpy(url, request->repo->base);
 147        posn = url + strlen(request->repo->base);
 148        strcpy(posn, "objects/");
 149        posn += 8;
 150        memcpy(posn, hex, 2);
 151        posn += 2;
 152        *(posn++) = '/';
 153        strcpy(posn, hex + 2);
 154        strcpy(request->url, url);
 155
 156        /* If a previous temp file is present, process what was already
 157           fetched. */
 158        prevlocal = open(prevfile, O_RDONLY);
 159        if (prevlocal != -1) {
 160                do {
 161                        prev_read = read(prevlocal, prev_buf, PREV_BUF_SIZE);
 162                        if (prev_read>0) {
 163                                if (fwrite_sha1_file(prev_buf,
 164                                                     1,
 165                                                     prev_read,
 166                                                     request) == prev_read) {
 167                                        prev_posn += prev_read;
 168                                } else {
 169                                        prev_read = -1;
 170                                }
 171                        }
 172                } while (prev_read > 0);
 173                close(prevlocal);
 174        }
 175        unlink(prevfile);
 176
 177        /* Reset inflate/SHA1 if there was an error reading the previous temp
 178           file; also rewind to the beginning of the local file. */
 179        if (prev_read == -1) {
 180                memset(&request->stream, 0, sizeof(request->stream));
 181                inflateInit(&request->stream);
 182                SHA1_Init(&request->c);
 183                if (prev_posn>0) {
 184                        prev_posn = 0;
 185                        lseek(request->local, SEEK_SET, 0);
 186                        ftruncate(request->local, 0);
 187                }
 188        }
 189
 190        slot = get_active_slot();
 191        slot->callback_func = process_object_response;
 192        slot->callback_data = request;
 193        request->slot = slot;
 194
 195        curl_easy_setopt(slot->curl, CURLOPT_FILE, request);
 196        curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite_sha1_file);
 197        curl_easy_setopt(slot->curl, CURLOPT_ERRORBUFFER, request->errorstr);
 198        curl_easy_setopt(slot->curl, CURLOPT_URL, url);
 199        curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, no_pragma_header);
 200
 201        /* If we have successfully processed data from a previous fetch
 202           attempt, only fetch the data we don't already have. */
 203        if (prev_posn>0) {
 204                if (get_verbosely)
 205                        fprintf(stderr,
 206                                "Resuming fetch of object %s at byte %ld\n",
 207                                hex, prev_posn);
 208                sprintf(range, "Range: bytes=%ld-", prev_posn);
 209                range_header = curl_slist_append(range_header, range);
 210                curl_easy_setopt(slot->curl,
 211                                 CURLOPT_HTTPHEADER, range_header);
 212        }
 213
 214        /* Try to get the request started, abort the request on error */
 215        request->state = ACTIVE;
 216        if (!start_active_slot(slot)) {
 217                request->state = ABORTED;
 218                request->slot = NULL;
 219                close(request->local); request->local = -1;
 220                free(request->url);
 221        }
 222}
 223
 224static void finish_request(struct transfer_request *request)
 225{
 226        struct stat st;
 227
 228        fchmod(request->local, 0444);
 229        close(request->local); request->local = -1;
 230
 231        if (request->http_code == 416) {
 232                fprintf(stderr, "Warning: requested range invalid; we may already have all the data.\n");
 233        } else if (request->curl_result != CURLE_OK) {
 234                if (stat(request->tmpfile, &st) == 0)
 235                        if (st.st_size == 0)
 236                                unlink(request->tmpfile);
 237                return;
 238        }
 239
 240        inflateEnd(&request->stream);
 241        SHA1_Final(request->real_sha1, &request->c);
 242        if (request->zret != Z_STREAM_END) {
 243                unlink(request->tmpfile);
 244                return;
 245        }
 246        if (memcmp(request->sha1, request->real_sha1, 20)) {
 247                unlink(request->tmpfile);
 248                return;
 249        }
 250        request->rename =
 251                move_temp_to_file(request->tmpfile, request->filename);
 252
 253        if (request->rename == 0)
 254                pull_say("got %s\n", sha1_to_hex(request->sha1));
 255}
 256
 257static void process_object_response(void *callback_data)
 258{
 259        struct transfer_request *request =
 260                (struct transfer_request *)callback_data;
 261
 262        request->curl_result = request->slot->curl_result;
 263        request->http_code = request->slot->http_code;
 264        request->slot = NULL;
 265        request->state = COMPLETE;
 266
 267        /* Use alternates if necessary */
 268        if (request->http_code == 404) {
 269                fetch_alternates(alt->base);
 270                if (request->repo->next != NULL) {
 271                        request->repo =
 272                                request->repo->next;
 273                        close(request->local);
 274                        request->local = -1;
 275                        start_request(request);
 276                        return;
 277                }
 278        }
 279
 280        finish_request(request);
 281}
 282
 283static void release_request(struct transfer_request *request)
 284{
 285        struct transfer_request *entry = request_queue_head;
 286
 287        if (request->local != -1)
 288                error("fd leakage in release: %d", request->local);
 289        if (request == request_queue_head) {
 290                request_queue_head = request->next;
 291        } else {
 292                while (entry->next != NULL && entry->next != request)
 293                        entry = entry->next;
 294                if (entry->next == request)
 295                        entry->next = entry->next->next;
 296        }
 297
 298        free(request->url);
 299        free(request);
 300}
 301
 302#ifdef USE_CURL_MULTI
 303void fill_active_slots(void)
 304{
 305        struct transfer_request *request = request_queue_head;
 306        struct active_request_slot *slot = active_queue_head;
 307        int num_transfers;
 308
 309        while (active_requests < max_requests && request != NULL) {
 310                if (request->state == WAITING) {
 311                        if (has_sha1_file(request->sha1))
 312                                release_request(request);
 313                        else
 314                                start_request(request);
 315                        curl_multi_perform(curlm, &num_transfers);
 316                }
 317                request = request->next;
 318        }
 319
 320        while (slot != NULL) {
 321                if (!slot->in_use && slot->curl != NULL) {
 322                        curl_easy_cleanup(slot->curl);
 323                        slot->curl = NULL;
 324                }
 325                slot = slot->next;
 326        }                               
 327}
 328#endif
 329
 330void prefetch(unsigned char *sha1)
 331{
 332        struct transfer_request *newreq;
 333        struct transfer_request *tail;
 334        char *filename = sha1_file_name(sha1);
 335
 336        newreq = xmalloc(sizeof(*newreq));
 337        memcpy(newreq->sha1, sha1, 20);
 338        newreq->repo = alt;
 339        newreq->url = NULL;
 340        newreq->local = -1;
 341        newreq->state = WAITING;
 342        snprintf(newreq->filename, sizeof(newreq->filename), "%s", filename);
 343        snprintf(newreq->tmpfile, sizeof(newreq->tmpfile),
 344                 "%s.temp", filename);
 345        newreq->next = NULL;
 346
 347        if (request_queue_head == NULL) {
 348                request_queue_head = newreq;
 349        } else {
 350                tail = request_queue_head;
 351                while (tail->next != NULL) {
 352                        tail = tail->next;
 353                }
 354                tail->next = newreq;
 355        }
 356
 357#ifdef USE_CURL_MULTI
 358        fill_active_slots();
 359        step_active_slots();
 360#endif
 361}
 362
 363static int fetch_index(struct alt_base *repo, unsigned char *sha1)
 364{
 365        char *hex = sha1_to_hex(sha1);
 366        char *filename;
 367        char *url;
 368        char tmpfile[PATH_MAX];
 369        long prev_posn = 0;
 370        char range[RANGE_HEADER_SIZE];
 371        struct curl_slist *range_header = NULL;
 372
 373        FILE *indexfile;
 374        struct active_request_slot *slot;
 375
 376        if (has_pack_index(sha1))
 377                return 0;
 378
 379        if (get_verbosely)
 380                fprintf(stderr, "Getting index for pack %s\n", hex);
 381        
 382        url = xmalloc(strlen(repo->base) + 64);
 383        sprintf(url, "%s/objects/pack/pack-%s.idx", repo->base, hex);
 384        
 385        filename = sha1_pack_index_name(sha1);
 386        snprintf(tmpfile, sizeof(tmpfile), "%s.temp", filename);
 387        indexfile = fopen(tmpfile, "a");
 388        if (!indexfile)
 389                return error("Unable to open local file %s for pack index",
 390                             filename);
 391
 392        slot = get_active_slot();
 393        curl_easy_setopt(slot->curl, CURLOPT_FILE, indexfile);
 394        curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite);
 395        curl_easy_setopt(slot->curl, CURLOPT_URL, url);
 396        curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, no_pragma_header);
 397        slot->local = indexfile;
 398
 399        /* If there is data present from a previous transfer attempt,
 400           resume where it left off */
 401        prev_posn = ftell(indexfile);
 402        if (prev_posn>0) {
 403                if (get_verbosely)
 404                        fprintf(stderr,
 405                                "Resuming fetch of index for pack %s at byte %ld\n",
 406                                hex, prev_posn);
 407                sprintf(range, "Range: bytes=%ld-", prev_posn);
 408                range_header = curl_slist_append(range_header, range);
 409                curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, range_header);
 410        }
 411
 412        if (start_active_slot(slot)) {
 413                run_active_slot(slot);
 414                if (slot->curl_result != CURLE_OK) {
 415                        fclose(indexfile);
 416                        return error("Unable to get pack index %s\n%s", url,
 417                                     curl_errorstr);
 418                }
 419        } else {
 420                fclose(indexfile);
 421                return error("Unable to start request");
 422        }
 423
 424        fclose(indexfile);
 425
 426        return move_temp_to_file(tmpfile, filename);
 427}
 428
 429static int setup_index(struct alt_base *repo, unsigned char *sha1)
 430{
 431        struct packed_git *new_pack;
 432        if (has_pack_file(sha1))
 433                return 0; // don't list this as something we can get
 434
 435        if (fetch_index(repo, sha1))
 436                return -1;
 437
 438        new_pack = parse_pack_index(sha1);
 439        new_pack->next = repo->packs;
 440        repo->packs = new_pack;
 441        return 0;
 442}
 443
 444static void process_alternates(void *callback_data)
 445{
 446        struct alt_request *alt_req = (struct alt_request *)callback_data;
 447        struct active_request_slot *slot = alt_req->slot;
 448        struct alt_base *tail = alt;
 449        char *base = alt_req->base;
 450        static const char null_byte = '\0';
 451        char *data;
 452        int i = 0;
 453
 454        if (alt_req->http_specific) {
 455                if (slot->curl_result != CURLE_OK ||
 456                    !alt_req->buffer->posn) {
 457
 458                        /* Try reusing the slot to get non-http alternates */
 459                        alt_req->http_specific = 0;
 460                        sprintf(alt_req->url, "%s/objects/info/alternates",
 461                                base);
 462                        curl_easy_setopt(slot->curl, CURLOPT_URL,
 463                                         alt_req->url);
 464                        active_requests++;
 465                        slot->in_use = 1;
 466                        if (start_active_slot(slot)) {
 467                                return;
 468                        } else {
 469                                got_alternates = -1;
 470                                slot->in_use = 0;
 471                                return;
 472                        }
 473                }
 474        } else if (slot->curl_result != CURLE_OK) {
 475                if (slot->http_code != 404) {
 476                        got_alternates = -1;
 477                        return;
 478                }
 479        }
 480
 481        fwrite_buffer(&null_byte, 1, 1, alt_req->buffer);
 482        alt_req->buffer->posn--;
 483        data = alt_req->buffer->buffer;
 484
 485        while (i < alt_req->buffer->posn) {
 486                int posn = i;
 487                while (posn < alt_req->buffer->posn && data[posn] != '\n')
 488                        posn++;
 489                if (data[posn] == '\n') {
 490                        int okay = 0;
 491                        int serverlen = 0;
 492                        struct alt_base *newalt;
 493                        char *target = NULL;
 494                        if (data[i] == '/') {
 495                                serverlen = strchr(base + 8, '/') - base;
 496                                okay = 1;
 497                        } else if (!memcmp(data + i, "../", 3)) {
 498                                i += 3;
 499                                serverlen = strlen(base);
 500                                while (i + 2 < posn && 
 501                                       !memcmp(data + i, "../", 3)) {
 502                                        do {
 503                                                serverlen--;
 504                                        } while (serverlen &&
 505                                                 base[serverlen - 1] != '/');
 506                                        i += 3;
 507                                }
 508                                // If the server got removed, give up.
 509                                okay = strchr(base, ':') - base + 3 < 
 510                                        serverlen;
 511                        } else if (alt_req->http_specific) {
 512                                char *colon = strchr(data + i, ':');
 513                                char *slash = strchr(data + i, '/');
 514                                if (colon && slash && colon < data + posn &&
 515                                    slash < data + posn && colon < slash) {
 516                                        okay = 1;
 517                                }
 518                        }
 519                        // skip 'objects' at end
 520                        if (okay) {
 521                                target = xmalloc(serverlen + posn - i - 6);
 522                                strncpy(target, base, serverlen);
 523                                strncpy(target + serverlen, data + i,
 524                                        posn - i - 7);
 525                                target[serverlen + posn - i - 7] = '\0';
 526                                if (get_verbosely)
 527                                        fprintf(stderr, 
 528                                                "Also look at %s\n", target);
 529                                newalt = xmalloc(sizeof(*newalt));
 530                                newalt->next = NULL;
 531                                newalt->base = target;
 532                                newalt->got_indices = 0;
 533                                newalt->packs = NULL;
 534                                while (tail->next != NULL)
 535                                        tail = tail->next;
 536                                tail->next = newalt;
 537                        }
 538                }
 539                i = posn + 1;
 540        }
 541
 542        got_alternates = 1;
 543}
 544
 545static void fetch_alternates(char *base)
 546{
 547        struct buffer buffer;
 548        char *url;
 549        char *data;
 550        struct active_request_slot *slot;
 551        static struct alt_request alt_req;
 552
 553        /* If another request has already started fetching alternates,
 554           wait for them to arrive and return to processing this request's
 555           curl message */
 556#ifdef USE_CURL_MULTI
 557        while (got_alternates == 0) {
 558                step_active_slots();
 559        }
 560#endif
 561
 562        /* Nothing to do if they've already been fetched */
 563        if (got_alternates == 1)
 564                return;
 565
 566        /* Start the fetch */
 567        got_alternates = 0;
 568
 569        data = xmalloc(4096);
 570        buffer.size = 4096;
 571        buffer.posn = 0;
 572        buffer.buffer = data;
 573
 574        if (get_verbosely)
 575                fprintf(stderr, "Getting alternates list for %s\n", base);
 576        
 577        url = xmalloc(strlen(base) + 31);
 578        sprintf(url, "%s/objects/info/http-alternates", base);
 579
 580        /* Use a callback to process the result, since another request
 581           may fail and need to have alternates loaded before continuing */
 582        slot = get_active_slot();
 583        slot->callback_func = process_alternates;
 584        slot->callback_data = &alt_req;
 585
 586        curl_easy_setopt(slot->curl, CURLOPT_FILE, &buffer);
 587        curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite_buffer);
 588        curl_easy_setopt(slot->curl, CURLOPT_URL, url);
 589
 590        alt_req.base = base;
 591        alt_req.url = url;
 592        alt_req.buffer = &buffer;
 593        alt_req.http_specific = 1;
 594        alt_req.slot = slot;
 595
 596        if (start_active_slot(slot))
 597                run_active_slot(slot);
 598        else
 599                got_alternates = -1;
 600
 601        free(data);
 602        free(url);
 603}
 604
 605static int fetch_indices(struct alt_base *repo)
 606{
 607        unsigned char sha1[20];
 608        char *url;
 609        struct buffer buffer;
 610        char *data;
 611        int i = 0;
 612
 613        struct active_request_slot *slot;
 614
 615        if (repo->got_indices)
 616                return 0;
 617
 618        data = xmalloc(4096);
 619        buffer.size = 4096;
 620        buffer.posn = 0;
 621        buffer.buffer = data;
 622
 623        if (get_verbosely)
 624                fprintf(stderr, "Getting pack list for %s\n", repo->base);
 625        
 626        url = xmalloc(strlen(repo->base) + 21);
 627        sprintf(url, "%s/objects/info/packs", repo->base);
 628
 629        slot = get_active_slot();
 630        curl_easy_setopt(slot->curl, CURLOPT_FILE, &buffer);
 631        curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite_buffer);
 632        curl_easy_setopt(slot->curl, CURLOPT_URL, url);
 633        curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, NULL);
 634        if (start_active_slot(slot)) {
 635                run_active_slot(slot);
 636                if (slot->curl_result != CURLE_OK) {
 637                        free(buffer.buffer);
 638                        return error("%s", curl_errorstr);
 639                }
 640        } else {
 641                free(buffer.buffer);
 642                return error("Unable to start request");
 643        }
 644
 645        data = buffer.buffer;
 646        while (i < buffer.posn) {
 647                switch (data[i]) {
 648                case 'P':
 649                        i++;
 650                        if (i + 52 < buffer.posn &&
 651                            !strncmp(data + i, " pack-", 6) &&
 652                            !strncmp(data + i + 46, ".pack\n", 6)) {
 653                                get_sha1_hex(data + i + 6, sha1);
 654                                setup_index(repo, sha1);
 655                                i += 51;
 656                                break;
 657                        }
 658                default:
 659                        while (data[i] != '\n')
 660                                i++;
 661                }
 662                i++;
 663        }
 664
 665        free(buffer.buffer);
 666        repo->got_indices = 1;
 667        return 0;
 668}
 669
 670static int fetch_pack(struct alt_base *repo, unsigned char *sha1)
 671{
 672        char *url;
 673        struct packed_git *target;
 674        struct packed_git **lst;
 675        FILE *packfile;
 676        char *filename;
 677        char tmpfile[PATH_MAX];
 678        int ret;
 679        long prev_posn = 0;
 680        char range[RANGE_HEADER_SIZE];
 681        struct curl_slist *range_header = NULL;
 682
 683        struct active_request_slot *slot;
 684
 685        if (fetch_indices(repo))
 686                return -1;
 687        target = find_sha1_pack(sha1, repo->packs);
 688        if (!target)
 689                return -1;
 690
 691        if (get_verbosely) {
 692                fprintf(stderr, "Getting pack %s\n",
 693                        sha1_to_hex(target->sha1));
 694                fprintf(stderr, " which contains %s\n",
 695                        sha1_to_hex(sha1));
 696        }
 697
 698        url = xmalloc(strlen(repo->base) + 65);
 699        sprintf(url, "%s/objects/pack/pack-%s.pack",
 700                repo->base, sha1_to_hex(target->sha1));
 701
 702        filename = sha1_pack_name(target->sha1);
 703        snprintf(tmpfile, sizeof(tmpfile), "%s.temp", filename);
 704        packfile = fopen(tmpfile, "a");
 705        if (!packfile)
 706                return error("Unable to open local file %s for pack",
 707                             filename);
 708
 709        slot = get_active_slot();
 710        curl_easy_setopt(slot->curl, CURLOPT_FILE, packfile);
 711        curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite);
 712        curl_easy_setopt(slot->curl, CURLOPT_URL, url);
 713        curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, no_pragma_header);
 714        slot->local = packfile;
 715
 716        /* If there is data present from a previous transfer attempt,
 717           resume where it left off */
 718        prev_posn = ftell(packfile);
 719        if (prev_posn>0) {
 720                if (get_verbosely)
 721                        fprintf(stderr,
 722                                "Resuming fetch of pack %s at byte %ld\n",
 723                                sha1_to_hex(target->sha1), prev_posn);
 724                sprintf(range, "Range: bytes=%ld-", prev_posn);
 725                range_header = curl_slist_append(range_header, range);
 726                curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, range_header);
 727        }
 728
 729        if (start_active_slot(slot)) {
 730                run_active_slot(slot);
 731                if (slot->curl_result != CURLE_OK) {
 732                        fclose(packfile);
 733                        return error("Unable to get pack file %s\n%s", url,
 734                                     curl_errorstr);
 735                }
 736        } else {
 737                fclose(packfile);
 738                return error("Unable to start request");
 739        }
 740
 741        fclose(packfile);
 742
 743        ret = move_temp_to_file(tmpfile, filename);
 744        if (ret)
 745                return ret;
 746
 747        lst = &repo->packs;
 748        while (*lst != target)
 749                lst = &((*lst)->next);
 750        *lst = (*lst)->next;
 751
 752        if (verify_pack(target, 0))
 753                return -1;
 754        install_packed_git(target);
 755
 756        return 0;
 757}
 758
 759static int fetch_object(struct alt_base *repo, unsigned char *sha1)
 760{
 761        char *hex = sha1_to_hex(sha1);
 762        int ret = 0;
 763        struct transfer_request *request = request_queue_head;
 764
 765        while (request != NULL && memcmp(request->sha1, sha1, 20))
 766                request = request->next;
 767        if (request == NULL)
 768                return error("Couldn't find request for %s in the queue", hex);
 769
 770        if (has_sha1_file(request->sha1)) {
 771                release_request(request);
 772                return 0;
 773        }
 774
 775#ifdef USE_CURL_MULTI
 776        while (request->state == WAITING) {
 777                step_active_slots();
 778        }
 779#else
 780        start_request(request);
 781#endif
 782
 783        while (request->state == ACTIVE) {
 784                run_active_slot(request->slot);
 785        }
 786        if (request->local != -1) {
 787                close(request->local); request->local = -1;
 788        }
 789
 790        if (request->state == ABORTED) {
 791                ret = error("Request for %s aborted", hex);
 792        } else if (request->curl_result != CURLE_OK &&
 793                   request->http_code != 416) {
 794                if (request->http_code == 404)
 795                        ret = -1; /* Be silent, it is probably in a pack. */
 796                else
 797                        ret = error("%s (curl_result = %d, http_code = %ld, sha1 = %s)",
 798                                    request->errorstr, request->curl_result,
 799                                    request->http_code, hex);
 800        } else if (request->zret != Z_STREAM_END) {
 801                ret = error("File %s (%s) corrupt\n", hex, request->url);
 802        } else if (memcmp(request->sha1, request->real_sha1, 20)) {
 803                ret = error("File %s has bad hash\n", hex);
 804        } else if (request->rename < 0) {
 805                ret = error("unable to write sha1 filename %s: %s",
 806                            request->filename,
 807                            strerror(request->rename));
 808        }
 809
 810        release_request(request);
 811        return ret;
 812}
 813
 814int fetch(unsigned char *sha1)
 815{
 816        struct alt_base *altbase = alt;
 817
 818        if (!fetch_object(altbase, sha1))
 819                return 0;
 820        while (altbase) {
 821                if (!fetch_pack(altbase, sha1))
 822                        return 0;
 823                fetch_alternates(alt->base);
 824                altbase = altbase->next;
 825        }
 826        return error("Unable to find %s under %s\n", sha1_to_hex(sha1), 
 827                     alt->base);
 828}
 829
 830static inline int needs_quote(int ch)
 831{
 832        switch (ch) {
 833        case '/': case '-': case '.':
 834        case 'A'...'Z': case 'a'...'z': case '0'...'9':
 835                return 0;
 836        default:
 837                return 1;
 838        }
 839}
 840
 841static inline int hex(int v)
 842{
 843        if (v < 10) return '0' + v;
 844        else return 'A' + v - 10;
 845}
 846
 847static char *quote_ref_url(const char *base, const char *ref)
 848{
 849        const char *cp;
 850        char *dp, *qref;
 851        int len, baselen, ch;
 852
 853        baselen = strlen(base);
 854        len = baselen + 6; /* "refs/" + NUL */
 855        for (cp = ref; (ch = *cp) != 0; cp++, len++)
 856                if (needs_quote(ch))
 857                        len += 2; /* extra two hex plus replacement % */
 858        qref = xmalloc(len);
 859        memcpy(qref, base, baselen);
 860        memcpy(qref + baselen, "refs/", 5);
 861        for (cp = ref, dp = qref + baselen + 5; (ch = *cp) != 0; cp++) {
 862                if (needs_quote(ch)) {
 863                        *dp++ = '%';
 864                        *dp++ = hex((ch >> 4) & 0xF);
 865                        *dp++ = hex(ch & 0xF);
 866                }
 867                else
 868                        *dp++ = ch;
 869        }
 870        *dp = 0;
 871
 872        return qref;
 873}
 874
 875int fetch_ref(char *ref, unsigned char *sha1)
 876{
 877        char *url;
 878        char hex[42];
 879        struct buffer buffer;
 880        char *base = alt->base;
 881        struct active_request_slot *slot;
 882        buffer.size = 41;
 883        buffer.posn = 0;
 884        buffer.buffer = hex;
 885        hex[41] = '\0';
 886        
 887        url = quote_ref_url(base, ref);
 888        slot = get_active_slot();
 889        curl_easy_setopt(slot->curl, CURLOPT_FILE, &buffer);
 890        curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite_buffer);
 891        curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, NULL);
 892        curl_easy_setopt(slot->curl, CURLOPT_URL, url);
 893        if (start_active_slot(slot)) {
 894                run_active_slot(slot);
 895                if (slot->curl_result != CURLE_OK)
 896                        return error("Couldn't get %s for %s\n%s",
 897                                     url, ref, curl_errorstr);
 898        } else {
 899                return error("Unable to start request");
 900        }
 901
 902        hex[40] = '\0';
 903        get_sha1_hex(hex, sha1);
 904        return 0;
 905}
 906
 907int main(int argc, char **argv)
 908{
 909        char *commit_id;
 910        char *url;
 911        int arg = 1;
 912        int rc = 0;
 913
 914        while (arg < argc && argv[arg][0] == '-') {
 915                if (argv[arg][1] == 't') {
 916                        get_tree = 1;
 917                } else if (argv[arg][1] == 'c') {
 918                        get_history = 1;
 919                } else if (argv[arg][1] == 'a') {
 920                        get_all = 1;
 921                        get_tree = 1;
 922                        get_history = 1;
 923                } else if (argv[arg][1] == 'v') {
 924                        get_verbosely = 1;
 925                } else if (argv[arg][1] == 'w') {
 926                        write_ref = argv[arg + 1];
 927                        arg++;
 928                } else if (!strcmp(argv[arg], "--recover")) {
 929                        get_recover = 1;
 930                }
 931                arg++;
 932        }
 933        if (argc < arg + 2) {
 934                usage("git-http-fetch [-c] [-t] [-a] [-d] [-v] [--recover] [-w ref] commit-id url");
 935                return 1;
 936        }
 937        commit_id = argv[arg];
 938        url = argv[arg + 1];
 939
 940        http_init();
 941
 942        no_pragma_header = curl_slist_append(no_pragma_header, "Pragma:");
 943
 944        alt = xmalloc(sizeof(*alt));
 945        alt->base = url;
 946        alt->got_indices = 0;
 947        alt->packs = NULL;
 948        alt->next = NULL;
 949
 950        if (pull(commit_id))
 951                rc = 1;
 952
 953        curl_slist_free_all(no_pragma_header);
 954
 955        http_cleanup();
 956
 957        return rc;
 958}