Andrew's git / gitweb.git / http-fetch.c
?
http-fetch.con commit Merge branch 'maint' of git://linux-nfs.org/~bfields/git into maint (f3caeb9)
   1#include "cache.h"
   2#include "commit.h"
   3#include "pack.h"
   4#include "fetch.h"
   5#include "http.h"
   6
   7#define PREV_BUF_SIZE 4096
   8#define RANGE_HEADER_SIZE 30
   9
  10static int commits_on_stdin;
  11
  12static int got_alternates = -1;
  13static int corrupt_object_found;
  14
  15static struct curl_slist *no_pragma_header;
  16
  17struct alt_base
  18{
  19        char *base;
  20        int got_indices;
  21        struct packed_git *packs;
  22        struct alt_base *next;
  23};
  24
  25static struct alt_base *alt;
  26
  27enum object_request_state {
  28        WAITING,
  29        ABORTED,
  30        ACTIVE,
  31        COMPLETE,
  32};
  33
  34struct object_request
  35{
  36        unsigned char sha1[20];
  37        struct alt_base *repo;
  38        char *url;
  39        char filename[PATH_MAX];
  40        char tmpfile[PATH_MAX];
  41        int local;
  42        enum object_request_state state;
  43        CURLcode curl_result;
  44        char errorstr[CURL_ERROR_SIZE];
  45        long http_code;
  46        unsigned char real_sha1[20];
  47        SHA_CTX c;
  48        z_stream stream;
  49        int zret;
  50        int rename;
  51        struct active_request_slot *slot;
  52        struct object_request *next;
  53};
  54
  55struct alternates_request {
  56        const char *base;
  57        char *url;
  58        struct buffer *buffer;
  59        struct active_request_slot *slot;
  60        int http_specific;
  61};
  62
  63static struct object_request *object_queue_head;
  64
  65static size_t fwrite_sha1_file(void *ptr, size_t eltsize, size_t nmemb,
  66                               void *data)
  67{
  68        unsigned char expn[4096];
  69        size_t size = eltsize * nmemb;
  70        int posn = 0;
  71        struct object_request *obj_req = (struct object_request *)data;
  72        do {
  73                ssize_t retval = xwrite(obj_req->local,
  74                                       (char *) ptr + posn, size - posn);
  75                if (retval < 0)
  76                        return posn;
  77                posn += retval;
  78        } while (posn < size);
  79
  80        obj_req->stream.avail_in = size;
  81        obj_req->stream.next_in = ptr;
  82        do {
  83                obj_req->stream.next_out = expn;
  84                obj_req->stream.avail_out = sizeof(expn);
  85                obj_req->zret = inflate(&obj_req->stream, Z_SYNC_FLUSH);
  86                SHA1_Update(&obj_req->c, expn,
  87                            sizeof(expn) - obj_req->stream.avail_out);
  88        } while (obj_req->stream.avail_in && obj_req->zret == Z_OK);
  89        data_received++;
  90        return size;
  91}
  92
  93static int missing__target(int code, int result)
  94{
  95        return  /* file:// URL -- do we ever use one??? */
  96                (result == CURLE_FILE_COULDNT_READ_FILE) ||
  97                /* http:// and https:// URL */
  98                (code == 404 && result == CURLE_HTTP_RETURNED_ERROR) ||
  99                /* ftp:// URL */
 100                (code == 550 && result == CURLE_FTP_COULDNT_RETR_FILE)
 101                ;
 102}
 103
 104#define missing_target(a) missing__target((a)->http_code, (a)->curl_result)
 105
 106static void fetch_alternates(const char *base);
 107
 108static void process_object_response(void *callback_data);
 109
 110static void start_object_request(struct object_request *obj_req)
 111{
 112        char *hex = sha1_to_hex(obj_req->sha1);
 113        char prevfile[PATH_MAX];
 114        char *url;
 115        char *posn;
 116        int prevlocal;
 117        unsigned char prev_buf[PREV_BUF_SIZE];
 118        ssize_t prev_read = 0;
 119        long prev_posn = 0;
 120        char range[RANGE_HEADER_SIZE];
 121        struct curl_slist *range_header = NULL;
 122        struct active_request_slot *slot;
 123
 124        snprintf(prevfile, sizeof(prevfile), "%s.prev", obj_req->filename);
 125        unlink(prevfile);
 126        rename(obj_req->tmpfile, prevfile);
 127        unlink(obj_req->tmpfile);
 128
 129        if (obj_req->local != -1)
 130                error("fd leakage in start: %d", obj_req->local);
 131        obj_req->local = open(obj_req->tmpfile,
 132                              O_WRONLY | O_CREAT | O_EXCL, 0666);
 133        /* This could have failed due to the "lazy directory creation";
 134         * try to mkdir the last path component.
 135         */
 136        if (obj_req->local < 0 && errno == ENOENT) {
 137                char *dir = strrchr(obj_req->tmpfile, '/');
 138                if (dir) {
 139                        *dir = 0;
 140                        mkdir(obj_req->tmpfile, 0777);
 141                        *dir = '/';
 142                }
 143                obj_req->local = open(obj_req->tmpfile,
 144                                      O_WRONLY | O_CREAT | O_EXCL, 0666);
 145        }
 146
 147        if (obj_req->local < 0) {
 148                obj_req->state = ABORTED;
 149                error("Couldn't create temporary file %s for %s: %s",
 150                      obj_req->tmpfile, obj_req->filename, strerror(errno));
 151                return;
 152        }
 153
 154        memset(&obj_req->stream, 0, sizeof(obj_req->stream));
 155
 156        inflateInit(&obj_req->stream);
 157
 158        SHA1_Init(&obj_req->c);
 159
 160        url = xmalloc(strlen(obj_req->repo->base) + 51);
 161        obj_req->url = xmalloc(strlen(obj_req->repo->base) + 51);
 162        strcpy(url, obj_req->repo->base);
 163        posn = url + strlen(obj_req->repo->base);
 164        strcpy(posn, "/objects/");
 165        posn += 9;
 166        memcpy(posn, hex, 2);
 167        posn += 2;
 168        *(posn++) = '/';
 169        strcpy(posn, hex + 2);
 170        strcpy(obj_req->url, url);
 171
 172        /* If a previous temp file is present, process what was already
 173           fetched. */
 174        prevlocal = open(prevfile, O_RDONLY);
 175        if (prevlocal != -1) {
 176                do {
 177                        prev_read = xread(prevlocal, prev_buf, PREV_BUF_SIZE);
 178                        if (prev_read>0) {
 179                                if (fwrite_sha1_file(prev_buf,
 180                                                     1,
 181                                                     prev_read,
 182                                                     obj_req) == prev_read) {
 183                                        prev_posn += prev_read;
 184                                } else {
 185                                        prev_read = -1;
 186                                }
 187                        }
 188                } while (prev_read > 0);
 189                close(prevlocal);
 190        }
 191        unlink(prevfile);
 192
 193        /* Reset inflate/SHA1 if there was an error reading the previous temp
 194           file; also rewind to the beginning of the local file. */
 195        if (prev_read == -1) {
 196                memset(&obj_req->stream, 0, sizeof(obj_req->stream));
 197                inflateInit(&obj_req->stream);
 198                SHA1_Init(&obj_req->c);
 199                if (prev_posn>0) {
 200                        prev_posn = 0;
 201                        lseek(obj_req->local, 0, SEEK_SET);
 202                        ftruncate(obj_req->local, 0);
 203                }
 204        }
 205
 206        slot = get_active_slot();
 207        slot->callback_func = process_object_response;
 208        slot->callback_data = obj_req;
 209        obj_req->slot = slot;
 210
 211        curl_easy_setopt(slot->curl, CURLOPT_FILE, obj_req);
 212        curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite_sha1_file);
 213        curl_easy_setopt(slot->curl, CURLOPT_ERRORBUFFER, obj_req->errorstr);
 214        curl_easy_setopt(slot->curl, CURLOPT_URL, url);
 215        curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, no_pragma_header);
 216
 217        /* If we have successfully processed data from a previous fetch
 218           attempt, only fetch the data we don't already have. */
 219        if (prev_posn>0) {
 220                if (get_verbosely)
 221                        fprintf(stderr,
 222                                "Resuming fetch of object %s at byte %ld\n",
 223                                hex, prev_posn);
 224                sprintf(range, "Range: bytes=%ld-", prev_posn);
 225                range_header = curl_slist_append(range_header, range);
 226                curl_easy_setopt(slot->curl,
 227                                 CURLOPT_HTTPHEADER, range_header);
 228        }
 229
 230        /* Try to get the request started, abort the request on error */
 231        obj_req->state = ACTIVE;
 232        if (!start_active_slot(slot)) {
 233                obj_req->state = ABORTED;
 234                obj_req->slot = NULL;
 235                close(obj_req->local); obj_req->local = -1;
 236                free(obj_req->url);
 237                return;
 238        }
 239}
 240
 241static void finish_object_request(struct object_request *obj_req)
 242{
 243        struct stat st;
 244
 245        fchmod(obj_req->local, 0444);
 246        close(obj_req->local); obj_req->local = -1;
 247
 248        if (obj_req->http_code == 416) {
 249                fprintf(stderr, "Warning: requested range invalid; we may already have all the data.\n");
 250        } else if (obj_req->curl_result != CURLE_OK) {
 251                if (stat(obj_req->tmpfile, &st) == 0)
 252                        if (st.st_size == 0)
 253                                unlink(obj_req->tmpfile);
 254                return;
 255        }
 256
 257        inflateEnd(&obj_req->stream);
 258        SHA1_Final(obj_req->real_sha1, &obj_req->c);
 259        if (obj_req->zret != Z_STREAM_END) {
 260                unlink(obj_req->tmpfile);
 261                return;
 262        }
 263        if (hashcmp(obj_req->sha1, obj_req->real_sha1)) {
 264                unlink(obj_req->tmpfile);
 265                return;
 266        }
 267        obj_req->rename =
 268                move_temp_to_file(obj_req->tmpfile, obj_req->filename);
 269
 270        if (obj_req->rename == 0)
 271                pull_say("got %s\n", sha1_to_hex(obj_req->sha1));
 272}
 273
 274static void process_object_response(void *callback_data)
 275{
 276        struct object_request *obj_req =
 277                (struct object_request *)callback_data;
 278
 279        obj_req->curl_result = obj_req->slot->curl_result;
 280        obj_req->http_code = obj_req->slot->http_code;
 281        obj_req->slot = NULL;
 282        obj_req->state = COMPLETE;
 283
 284        /* Use alternates if necessary */
 285        if (missing_target(obj_req)) {
 286                fetch_alternates(alt->base);
 287                if (obj_req->repo->next != NULL) {
 288                        obj_req->repo =
 289                                obj_req->repo->next;
 290                        close(obj_req->local);
 291                        obj_req->local = -1;
 292                        start_object_request(obj_req);
 293                        return;
 294                }
 295        }
 296
 297        finish_object_request(obj_req);
 298}
 299
 300static void release_object_request(struct object_request *obj_req)
 301{
 302        struct object_request *entry = object_queue_head;
 303
 304        if (obj_req->local != -1)
 305                error("fd leakage in release: %d", obj_req->local);
 306        if (obj_req == object_queue_head) {
 307                object_queue_head = obj_req->next;
 308        } else {
 309                while (entry->next != NULL && entry->next != obj_req)
 310                        entry = entry->next;
 311                if (entry->next == obj_req)
 312                        entry->next = entry->next->next;
 313        }
 314
 315        free(obj_req->url);
 316        free(obj_req);
 317}
 318
 319#ifdef USE_CURL_MULTI
 320void fill_active_slots(void)
 321{
 322        struct object_request *obj_req = object_queue_head;
 323        struct active_request_slot *slot = active_queue_head;
 324        int num_transfers;
 325
 326        while (active_requests < max_requests && obj_req != NULL) {
 327                if (obj_req->state == WAITING) {
 328                        if (has_sha1_file(obj_req->sha1))
 329                                obj_req->state = COMPLETE;
 330                        else
 331                                start_object_request(obj_req);
 332                        curl_multi_perform(curlm, &num_transfers);
 333                }
 334                obj_req = obj_req->next;
 335        }
 336
 337        while (slot != NULL) {
 338                if (!slot->in_use && slot->curl != NULL) {
 339                        curl_easy_cleanup(slot->curl);
 340                        slot->curl = NULL;
 341                }
 342                slot = slot->next;
 343        }
 344}
 345#endif
 346
 347void prefetch(unsigned char *sha1)
 348{
 349        struct object_request *newreq;
 350        struct object_request *tail;
 351        char *filename = sha1_file_name(sha1);
 352
 353        newreq = xmalloc(sizeof(*newreq));
 354        hashcpy(newreq->sha1, sha1);
 355        newreq->repo = alt;
 356        newreq->url = NULL;
 357        newreq->local = -1;
 358        newreq->state = WAITING;
 359        snprintf(newreq->filename, sizeof(newreq->filename), "%s", filename);
 360        snprintf(newreq->tmpfile, sizeof(newreq->tmpfile),
 361                 "%s.temp", filename);
 362        newreq->slot = NULL;
 363        newreq->next = NULL;
 364
 365        if (object_queue_head == NULL) {
 366                object_queue_head = newreq;
 367        } else {
 368                tail = object_queue_head;
 369                while (tail->next != NULL) {
 370                        tail = tail->next;
 371                }
 372                tail->next = newreq;
 373        }
 374
 375#ifdef USE_CURL_MULTI
 376        fill_active_slots();
 377        step_active_slots();
 378#endif
 379}
 380
 381static int fetch_index(struct alt_base *repo, unsigned char *sha1)
 382{
 383        char *hex = sha1_to_hex(sha1);
 384        char *filename;
 385        char *url;
 386        char tmpfile[PATH_MAX];
 387        long prev_posn = 0;
 388        char range[RANGE_HEADER_SIZE];
 389        struct curl_slist *range_header = NULL;
 390
 391        FILE *indexfile;
 392        struct active_request_slot *slot;
 393        struct slot_results results;
 394
 395        if (has_pack_index(sha1))
 396                return 0;
 397
 398        if (get_verbosely)
 399                fprintf(stderr, "Getting index for pack %s\n", hex);
 400
 401        url = xmalloc(strlen(repo->base) + 64);
 402        sprintf(url, "%s/objects/pack/pack-%s.idx", repo->base, hex);
 403
 404        filename = sha1_pack_index_name(sha1);
 405        snprintf(tmpfile, sizeof(tmpfile), "%s.temp", filename);
 406        indexfile = fopen(tmpfile, "a");
 407        if (!indexfile)
 408                return error("Unable to open local file %s for pack index",
 409                             filename);
 410
 411        slot = get_active_slot();
 412        slot->results = &results;
 413        curl_easy_setopt(slot->curl, CURLOPT_FILE, indexfile);
 414        curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite);
 415        curl_easy_setopt(slot->curl, CURLOPT_URL, url);
 416        curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, no_pragma_header);
 417        slot->local = indexfile;
 418
 419        /* If there is data present from a previous transfer attempt,
 420           resume where it left off */
 421        prev_posn = ftell(indexfile);
 422        if (prev_posn>0) {
 423                if (get_verbosely)
 424                        fprintf(stderr,
 425                                "Resuming fetch of index for pack %s at byte %ld\n",
 426                                hex, prev_posn);
 427                sprintf(range, "Range: bytes=%ld-", prev_posn);
 428                range_header = curl_slist_append(range_header, range);
 429                curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, range_header);
 430        }
 431
 432        if (start_active_slot(slot)) {
 433                run_active_slot(slot);
 434                if (results.curl_result != CURLE_OK) {
 435                        fclose(indexfile);
 436                        return error("Unable to get pack index %s\n%s", url,
 437                                     curl_errorstr);
 438                }
 439        } else {
 440                fclose(indexfile);
 441                return error("Unable to start request");
 442        }
 443
 444        fclose(indexfile);
 445
 446        return move_temp_to_file(tmpfile, filename);
 447}
 448
 449static int setup_index(struct alt_base *repo, unsigned char *sha1)
 450{
 451        struct packed_git *new_pack;
 452        if (has_pack_file(sha1))
 453                return 0; /* don't list this as something we can get */
 454
 455        if (fetch_index(repo, sha1))
 456                return -1;
 457
 458        new_pack = parse_pack_index(sha1);
 459        new_pack->next = repo->packs;
 460        repo->packs = new_pack;
 461        return 0;
 462}
 463
 464static void process_alternates_response(void *callback_data)
 465{
 466        struct alternates_request *alt_req =
 467                (struct alternates_request *)callback_data;
 468        struct active_request_slot *slot = alt_req->slot;
 469        struct alt_base *tail = alt;
 470        const char *base = alt_req->base;
 471        static const char null_byte = '\0';
 472        char *data;
 473        int i = 0;
 474
 475        if (alt_req->http_specific) {
 476                if (slot->curl_result != CURLE_OK ||
 477                    !alt_req->buffer->posn) {
 478
 479                        /* Try reusing the slot to get non-http alternates */
 480                        alt_req->http_specific = 0;
 481                        sprintf(alt_req->url, "%s/objects/info/alternates",
 482                                base);
 483                        curl_easy_setopt(slot->curl, CURLOPT_URL,
 484                                         alt_req->url);
 485                        active_requests++;
 486                        slot->in_use = 1;
 487                        if (slot->finished != NULL)
 488                                (*slot->finished) = 0;
 489                        if (!start_active_slot(slot)) {
 490                                got_alternates = -1;
 491                                slot->in_use = 0;
 492                                if (slot->finished != NULL)
 493                                        (*slot->finished) = 1;
 494                        }
 495                        return;
 496                }
 497        } else if (slot->curl_result != CURLE_OK) {
 498                if (!missing_target(slot)) {
 499                        got_alternates = -1;
 500                        return;
 501                }
 502        }
 503
 504        fwrite_buffer(&null_byte, 1, 1, alt_req->buffer);
 505        alt_req->buffer->posn--;
 506        data = alt_req->buffer->buffer;
 507
 508        while (i < alt_req->buffer->posn) {
 509                int posn = i;
 510                while (posn < alt_req->buffer->posn && data[posn] != '\n')
 511                        posn++;
 512                if (data[posn] == '\n') {
 513                        int okay = 0;
 514                        int serverlen = 0;
 515                        struct alt_base *newalt;
 516                        char *target = NULL;
 517                        if (data[i] == '/') {
 518                                /* This counts
 519                                 * http://git.host/pub/scm/linux.git/
 520                                 * -----------here^
 521                                 * so memcpy(dst, base, serverlen) will
 522                                 * copy up to "...git.host".
 523                                 */
 524                                const char *colon_ss = strstr(base,"://");
 525                                if (colon_ss) {
 526                                        serverlen = (strchr(colon_ss + 3, '/')
 527                                                     - base);
 528                                        okay = 1;
 529                                }
 530                        } else if (!memcmp(data + i, "../", 3)) {
 531                                /* Relative URL; chop the corresponding
 532                                 * number of subpath from base (and ../
 533                                 * from data), and concatenate the result.
 534                                 *
 535                                 * The code first drops ../ from data, and
 536                                 * then drops one ../ from data and one path
 537                                 * from base.  IOW, one extra ../ is dropped
 538                                 * from data than path is dropped from base.
 539                                 *
 540                                 * This is not wrong.  The alternate in
 541                                 *     http://git.host/pub/scm/linux.git/
 542                                 * to borrow from
 543                                 *     http://git.host/pub/scm/linus.git/
 544                                 * is ../../linus.git/objects/.  You need
 545                                 * two ../../ to borrow from your direct
 546                                 * neighbour.
 547                                 */
 548                                i += 3;
 549                                serverlen = strlen(base);
 550                                while (i + 2 < posn &&
 551                                       !memcmp(data + i, "../", 3)) {
 552                                        do {
 553                                                serverlen--;
 554                                        } while (serverlen &&
 555                                                 base[serverlen - 1] != '/');
 556                                        i += 3;
 557                                }
 558                                /* If the server got removed, give up. */
 559                                okay = strchr(base, ':') - base + 3 <
 560                                        serverlen;
 561                        } else if (alt_req->http_specific) {
 562                                char *colon = strchr(data + i, ':');
 563                                char *slash = strchr(data + i, '/');
 564                                if (colon && slash && colon < data + posn &&
 565                                    slash < data + posn && colon < slash) {
 566                                        okay = 1;
 567                                }
 568                        }
 569                        /* skip "objects\n" at end */
 570                        if (okay) {
 571                                target = xmalloc(serverlen + posn - i - 6);
 572                                memcpy(target, base, serverlen);
 573                                memcpy(target + serverlen, data + i,
 574                                       posn - i - 7);
 575                                target[serverlen + posn - i - 7] = 0;
 576                                if (get_verbosely)
 577                                        fprintf(stderr,
 578                                                "Also look at %s\n", target);
 579                                newalt = xmalloc(sizeof(*newalt));
 580                                newalt->next = NULL;
 581                                newalt->base = target;
 582                                newalt->got_indices = 0;
 583                                newalt->packs = NULL;
 584
 585                                while (tail->next != NULL)
 586                                        tail = tail->next;
 587                                tail->next = newalt;
 588                        }
 589                }
 590                i = posn + 1;
 591        }
 592
 593        got_alternates = 1;
 594}
 595
 596static void fetch_alternates(const char *base)
 597{
 598        struct buffer buffer;
 599        char *url;
 600        char *data;
 601        struct active_request_slot *slot;
 602        struct alternates_request alt_req;
 603
 604        /* If another request has already started fetching alternates,
 605           wait for them to arrive and return to processing this request's
 606           curl message */
 607#ifdef USE_CURL_MULTI
 608        while (got_alternates == 0) {
 609                step_active_slots();
 610        }
 611#endif
 612
 613        /* Nothing to do if they've already been fetched */
 614        if (got_alternates == 1)
 615                return;
 616
 617        /* Start the fetch */
 618        got_alternates = 0;
 619
 620        data = xmalloc(4096);
 621        buffer.size = 4096;
 622        buffer.posn = 0;
 623        buffer.buffer = data;
 624
 625        if (get_verbosely)
 626                fprintf(stderr, "Getting alternates list for %s\n", base);
 627
 628        url = xmalloc(strlen(base) + 31);
 629        sprintf(url, "%s/objects/info/http-alternates", base);
 630
 631        /* Use a callback to process the result, since another request
 632           may fail and need to have alternates loaded before continuing */
 633        slot = get_active_slot();
 634        slot->callback_func = process_alternates_response;
 635        slot->callback_data = &alt_req;
 636
 637        curl_easy_setopt(slot->curl, CURLOPT_FILE, &buffer);
 638        curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite_buffer);
 639        curl_easy_setopt(slot->curl, CURLOPT_URL, url);
 640
 641        alt_req.base = base;
 642        alt_req.url = url;
 643        alt_req.buffer = &buffer;
 644        alt_req.http_specific = 1;
 645        alt_req.slot = slot;
 646
 647        if (start_active_slot(slot))
 648                run_active_slot(slot);
 649        else
 650                got_alternates = -1;
 651
 652        free(data);
 653        free(url);
 654}
 655
 656static int fetch_indices(struct alt_base *repo)
 657{
 658        unsigned char sha1[20];
 659        char *url;
 660        struct buffer buffer;
 661        char *data;
 662        int i = 0;
 663
 664        struct active_request_slot *slot;
 665        struct slot_results results;
 666
 667        if (repo->got_indices)
 668                return 0;
 669
 670        data = xmalloc(4096);
 671        buffer.size = 4096;
 672        buffer.posn = 0;
 673        buffer.buffer = data;
 674
 675        if (get_verbosely)
 676                fprintf(stderr, "Getting pack list for %s\n", repo->base);
 677
 678        url = xmalloc(strlen(repo->base) + 21);
 679        sprintf(url, "%s/objects/info/packs", repo->base);
 680
 681        slot = get_active_slot();
 682        slot->results = &results;
 683        curl_easy_setopt(slot->curl, CURLOPT_FILE, &buffer);
 684        curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite_buffer);
 685        curl_easy_setopt(slot->curl, CURLOPT_URL, url);
 686        curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, NULL);
 687        if (start_active_slot(slot)) {
 688                run_active_slot(slot);
 689                if (results.curl_result != CURLE_OK) {
 690                        if (missing_target(&results)) {
 691                                repo->got_indices = 1;
 692                                free(buffer.buffer);
 693                                return 0;
 694                        } else {
 695                                repo->got_indices = 0;
 696                                free(buffer.buffer);
 697                                return error("%s", curl_errorstr);
 698                        }
 699                }
 700        } else {
 701                repo->got_indices = 0;
 702                free(buffer.buffer);
 703                return error("Unable to start request");
 704        }
 705
 706        data = buffer.buffer;
 707        while (i < buffer.posn) {
 708                switch (data[i]) {
 709                case 'P':
 710                        i++;
 711                        if (i + 52 <= buffer.posn &&
 712                            !prefixcmp(data + i, " pack-") &&
 713                            !prefixcmp(data + i + 46, ".pack\n")) {
 714                                get_sha1_hex(data + i + 6, sha1);
 715                                setup_index(repo, sha1);
 716                                i += 51;
 717                                break;
 718                        }
 719                default:
 720                        while (i < buffer.posn && data[i] != '\n')
 721                                i++;
 722                }
 723                i++;
 724        }
 725
 726        free(buffer.buffer);
 727        repo->got_indices = 1;
 728        return 0;
 729}
 730
 731static int fetch_pack(struct alt_base *repo, unsigned char *sha1)
 732{
 733        char *url;
 734        struct packed_git *target;
 735        struct packed_git **lst;
 736        FILE *packfile;
 737        char *filename;
 738        char tmpfile[PATH_MAX];
 739        int ret;
 740        long prev_posn = 0;
 741        char range[RANGE_HEADER_SIZE];
 742        struct curl_slist *range_header = NULL;
 743
 744        struct active_request_slot *slot;
 745        struct slot_results results;
 746
 747        if (fetch_indices(repo))
 748                return -1;
 749        target = find_sha1_pack(sha1, repo->packs);
 750        if (!target)
 751                return -1;
 752
 753        if (get_verbosely) {
 754                fprintf(stderr, "Getting pack %s\n",
 755                        sha1_to_hex(target->sha1));
 756                fprintf(stderr, " which contains %s\n",
 757                        sha1_to_hex(sha1));
 758        }
 759
 760        url = xmalloc(strlen(repo->base) + 65);
 761        sprintf(url, "%s/objects/pack/pack-%s.pack",
 762                repo->base, sha1_to_hex(target->sha1));
 763
 764        filename = sha1_pack_name(target->sha1);
 765        snprintf(tmpfile, sizeof(tmpfile), "%s.temp", filename);
 766        packfile = fopen(tmpfile, "a");
 767        if (!packfile)
 768                return error("Unable to open local file %s for pack",
 769                             filename);
 770
 771        slot = get_active_slot();
 772        slot->results = &results;
 773        curl_easy_setopt(slot->curl, CURLOPT_FILE, packfile);
 774        curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite);
 775        curl_easy_setopt(slot->curl, CURLOPT_URL, url);
 776        curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, no_pragma_header);
 777        slot->local = packfile;
 778
 779        /* If there is data present from a previous transfer attempt,
 780           resume where it left off */
 781        prev_posn = ftell(packfile);
 782        if (prev_posn>0) {
 783                if (get_verbosely)
 784                        fprintf(stderr,
 785                                "Resuming fetch of pack %s at byte %ld\n",
 786                                sha1_to_hex(target->sha1), prev_posn);
 787                sprintf(range, "Range: bytes=%ld-", prev_posn);
 788                range_header = curl_slist_append(range_header, range);
 789                curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, range_header);
 790        }
 791
 792        if (start_active_slot(slot)) {
 793                run_active_slot(slot);
 794                if (results.curl_result != CURLE_OK) {
 795                        fclose(packfile);
 796                        return error("Unable to get pack file %s\n%s", url,
 797                                     curl_errorstr);
 798                }
 799        } else {
 800                fclose(packfile);
 801                return error("Unable to start request");
 802        }
 803
 804        target->pack_size = ftell(packfile);
 805        fclose(packfile);
 806
 807        ret = move_temp_to_file(tmpfile, filename);
 808        if (ret)
 809                return ret;
 810
 811        lst = &repo->packs;
 812        while (*lst != target)
 813                lst = &((*lst)->next);
 814        *lst = (*lst)->next;
 815
 816        if (verify_pack(target, 0))
 817                return -1;
 818        install_packed_git(target);
 819
 820        return 0;
 821}
 822
 823static void abort_object_request(struct object_request *obj_req)
 824{
 825        if (obj_req->local >= 0) {
 826                close(obj_req->local);
 827                obj_req->local = -1;
 828        }
 829        unlink(obj_req->tmpfile);
 830        if (obj_req->slot) {
 831                release_active_slot(obj_req->slot);
 832                obj_req->slot = NULL;
 833        }
 834        release_object_request(obj_req);
 835}
 836
 837static int fetch_object(struct alt_base *repo, unsigned char *sha1)
 838{
 839        char *hex = sha1_to_hex(sha1);
 840        int ret = 0;
 841        struct object_request *obj_req = object_queue_head;
 842
 843        while (obj_req != NULL && hashcmp(obj_req->sha1, sha1))
 844                obj_req = obj_req->next;
 845        if (obj_req == NULL)
 846                return error("Couldn't find request for %s in the queue", hex);
 847
 848        if (has_sha1_file(obj_req->sha1)) {
 849                abort_object_request(obj_req);
 850                return 0;
 851        }
 852
 853#ifdef USE_CURL_MULTI
 854        while (obj_req->state == WAITING) {
 855                step_active_slots();
 856        }
 857#else
 858        start_object_request(obj_req);
 859#endif
 860
 861        while (obj_req->state == ACTIVE) {
 862                run_active_slot(obj_req->slot);
 863        }
 864        if (obj_req->local != -1) {
 865                close(obj_req->local); obj_req->local = -1;
 866        }
 867
 868        if (obj_req->state == ABORTED) {
 869                ret = error("Request for %s aborted", hex);
 870        } else if (obj_req->curl_result != CURLE_OK &&
 871                   obj_req->http_code != 416) {
 872                if (missing_target(obj_req))
 873                        ret = -1; /* Be silent, it is probably in a pack. */
 874                else
 875                        ret = error("%s (curl_result = %d, http_code = %ld, sha1 = %s)",
 876                                    obj_req->errorstr, obj_req->curl_result,
 877                                    obj_req->http_code, hex);
 878        } else if (obj_req->zret != Z_STREAM_END) {
 879                corrupt_object_found++;
 880                ret = error("File %s (%s) corrupt", hex, obj_req->url);
 881        } else if (hashcmp(obj_req->sha1, obj_req->real_sha1)) {
 882                ret = error("File %s has bad hash", hex);
 883        } else if (obj_req->rename < 0) {
 884                ret = error("unable to write sha1 filename %s",
 885                            obj_req->filename);
 886        }
 887
 888        release_object_request(obj_req);
 889        return ret;
 890}
 891
 892int fetch(unsigned char *sha1)
 893{
 894        struct alt_base *altbase = alt;
 895
 896        if (!fetch_object(altbase, sha1))
 897                return 0;
 898        while (altbase) {
 899                if (!fetch_pack(altbase, sha1))
 900                        return 0;
 901                fetch_alternates(alt->base);
 902                altbase = altbase->next;
 903        }
 904        return error("Unable to find %s under %s", sha1_to_hex(sha1),
 905                     alt->base);
 906}
 907
 908static inline int needs_quote(int ch)
 909{
 910        if (((ch >= 'A') && (ch <= 'Z'))
 911                        || ((ch >= 'a') && (ch <= 'z'))
 912                        || ((ch >= '0') && (ch <= '9'))
 913                        || (ch == '/')
 914                        || (ch == '-')
 915                        || (ch == '.'))
 916                return 0;
 917        return 1;
 918}
 919
 920static inline int hex(int v)
 921{
 922        if (v < 10) return '0' + v;
 923        else return 'A' + v - 10;
 924}
 925
 926static char *quote_ref_url(const char *base, const char *ref)
 927{
 928        const char *cp;
 929        char *dp, *qref;
 930        int len, baselen, ch;
 931
 932        baselen = strlen(base);
 933        len = baselen + 7; /* "/refs/" + NUL */
 934        for (cp = ref; (ch = *cp) != 0; cp++, len++)
 935                if (needs_quote(ch))
 936                        len += 2; /* extra two hex plus replacement % */
 937        qref = xmalloc(len);
 938        memcpy(qref, base, baselen);
 939        memcpy(qref + baselen, "/refs/", 6);
 940        for (cp = ref, dp = qref + baselen + 6; (ch = *cp) != 0; cp++) {
 941                if (needs_quote(ch)) {
 942                        *dp++ = '%';
 943                        *dp++ = hex((ch >> 4) & 0xF);
 944                        *dp++ = hex(ch & 0xF);
 945                }
 946                else
 947                        *dp++ = ch;
 948        }
 949        *dp = 0;
 950
 951        return qref;
 952}
 953
 954int fetch_ref(char *ref, unsigned char *sha1)
 955{
 956        char *url;
 957        char hex[42];
 958        struct buffer buffer;
 959        const char *base = alt->base;
 960        struct active_request_slot *slot;
 961        struct slot_results results;
 962        buffer.size = 41;
 963        buffer.posn = 0;
 964        buffer.buffer = hex;
 965        hex[41] = '\0';
 966
 967        url = quote_ref_url(base, ref);
 968        slot = get_active_slot();
 969        slot->results = &results;
 970        curl_easy_setopt(slot->curl, CURLOPT_FILE, &buffer);
 971        curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite_buffer);
 972        curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, NULL);
 973        curl_easy_setopt(slot->curl, CURLOPT_URL, url);
 974        if (start_active_slot(slot)) {
 975                run_active_slot(slot);
 976                if (results.curl_result != CURLE_OK)
 977                        return error("Couldn't get %s for %s\n%s",
 978                                     url, ref, curl_errorstr);
 979        } else {
 980                return error("Unable to start request");
 981        }
 982
 983        hex[40] = '\0';
 984        get_sha1_hex(hex, sha1);
 985        return 0;
 986}
 987
 988int main(int argc, const char **argv)
 989{
 990        int commits;
 991        const char **write_ref = NULL;
 992        char **commit_id;
 993        const char *url;
 994        char *s;
 995        int arg = 1;
 996        int rc = 0;
 997
 998        setup_git_directory();
 999        git_config(git_default_config);
1000
1001        while (arg < argc && argv[arg][0] == '-') {
1002                if (argv[arg][1] == 't') {
1003                        get_tree = 1;
1004                } else if (argv[arg][1] == 'c') {
1005                        get_history = 1;
1006                } else if (argv[arg][1] == 'a') {
1007                        get_all = 1;
1008                        get_tree = 1;
1009                        get_history = 1;
1010                } else if (argv[arg][1] == 'v') {
1011                        get_verbosely = 1;
1012                } else if (argv[arg][1] == 'w') {
1013                        write_ref = &argv[arg + 1];
1014                        arg++;
1015                } else if (!strcmp(argv[arg], "--recover")) {
1016                        get_recover = 1;
1017                } else if (!strcmp(argv[arg], "--stdin")) {
1018                        commits_on_stdin = 1;
1019                }
1020                arg++;
1021        }
1022        if (argc < arg + 2 - commits_on_stdin) {
1023                usage("git-http-fetch [-c] [-t] [-a] [-v] [--recover] [-w ref] [--stdin] commit-id url");
1024                return 1;
1025        }
1026        if (commits_on_stdin) {
1027                commits = pull_targets_stdin(&commit_id, &write_ref);
1028        } else {
1029                commit_id = (char **) &argv[arg++];
1030                commits = 1;
1031        }
1032        url = argv[arg];
1033
1034        http_init();
1035
1036        no_pragma_header = curl_slist_append(no_pragma_header, "Pragma:");
1037
1038        alt = xmalloc(sizeof(*alt));
1039        alt->base = xmalloc(strlen(url) + 1);
1040        strcpy(alt->base, url);
1041        for (s = alt->base + strlen(alt->base) - 1; *s == '/'; --s)
1042                *s = 0;
1043        alt->got_indices = 0;
1044        alt->packs = NULL;
1045        alt->next = NULL;
1046
1047        if (pull(commits, commit_id, write_ref, url))
1048                rc = 1;
1049
1050        http_cleanup();
1051
1052        curl_slist_free_all(no_pragma_header);
1053
1054        if (commits_on_stdin)
1055                pull_targets_free(commits, commit_id, write_ref);
1056
1057        if (corrupt_object_found) {
1058                fprintf(stderr,
1059"Some loose object were found to be corrupt, but they might be just\n"
1060"a false '404 Not Found' error message sent with incorrect HTTP\n"
1061"status code.  Suggest running git-fsck.\n");
1062        }
1063        return rc;
1064}