Andrew's git / gitweb.git / http-walker.c
?
http-walker.con commit transport.c::get_refs_via_curl(): use the new http API (28307b9)
   1#include "cache.h"
   2#include "commit.h"
   3#include "pack.h"
   4#include "walker.h"
   5#include "http.h"
   6
   7#define PREV_BUF_SIZE 4096
   8
   9struct alt_base
  10{
  11        char *base;
  12        int got_indices;
  13        struct packed_git *packs;
  14        struct alt_base *next;
  15};
  16
  17enum object_request_state {
  18        WAITING,
  19        ABORTED,
  20        ACTIVE,
  21        COMPLETE,
  22};
  23
  24struct object_request
  25{
  26        struct walker *walker;
  27        unsigned char sha1[20];
  28        struct alt_base *repo;
  29        char *url;
  30        char filename[PATH_MAX];
  31        char tmpfile[PATH_MAX];
  32        int local;
  33        enum object_request_state state;
  34        CURLcode curl_result;
  35        char errorstr[CURL_ERROR_SIZE];
  36        long http_code;
  37        unsigned char real_sha1[20];
  38        git_SHA_CTX c;
  39        z_stream stream;
  40        int zret;
  41        int rename;
  42        struct active_request_slot *slot;
  43        struct object_request *next;
  44};
  45
  46struct alternates_request {
  47        struct walker *walker;
  48        const char *base;
  49        char *url;
  50        struct strbuf *buffer;
  51        struct active_request_slot *slot;
  52        int http_specific;
  53};
  54
  55struct walker_data {
  56        const char *url;
  57        int got_alternates;
  58        struct alt_base *alt;
  59};
  60
  61static struct object_request *object_queue_head;
  62
  63static size_t fwrite_sha1_file(void *ptr, size_t eltsize, size_t nmemb,
  64                               void *data)
  65{
  66        unsigned char expn[4096];
  67        size_t size = eltsize * nmemb;
  68        int posn = 0;
  69        struct object_request *obj_req = (struct object_request *)data;
  70        do {
  71                ssize_t retval = xwrite(obj_req->local,
  72                                        (char *) ptr + posn, size - posn);
  73                if (retval < 0)
  74                        return posn;
  75                posn += retval;
  76        } while (posn < size);
  77
  78        obj_req->stream.avail_in = size;
  79        obj_req->stream.next_in = ptr;
  80        do {
  81                obj_req->stream.next_out = expn;
  82                obj_req->stream.avail_out = sizeof(expn);
  83                obj_req->zret = git_inflate(&obj_req->stream, Z_SYNC_FLUSH);
  84                git_SHA1_Update(&obj_req->c, expn,
  85                                sizeof(expn) - obj_req->stream.avail_out);
  86        } while (obj_req->stream.avail_in && obj_req->zret == Z_OK);
  87        data_received++;
  88        return size;
  89}
  90
  91static void fetch_alternates(struct walker *walker, const char *base);
  92
  93static void process_object_response(void *callback_data);
  94
  95static void start_object_request(struct walker *walker,
  96                                 struct object_request *obj_req)
  97{
  98        char *hex = sha1_to_hex(obj_req->sha1);
  99        char prevfile[PATH_MAX];
 100        char *url;
 101        char *posn;
 102        int prevlocal;
 103        unsigned char prev_buf[PREV_BUF_SIZE];
 104        ssize_t prev_read = 0;
 105        long prev_posn = 0;
 106        char range[RANGE_HEADER_SIZE];
 107        struct curl_slist *range_header = NULL;
 108        struct active_request_slot *slot;
 109
 110        snprintf(prevfile, sizeof(prevfile), "%s.prev", obj_req->filename);
 111        unlink_or_warn(prevfile);
 112        rename(obj_req->tmpfile, prevfile);
 113        unlink_or_warn(obj_req->tmpfile);
 114
 115        if (obj_req->local != -1)
 116                error("fd leakage in start: %d", obj_req->local);
 117        obj_req->local = open(obj_req->tmpfile,
 118                              O_WRONLY | O_CREAT | O_EXCL, 0666);
 119        /*
 120         * This could have failed due to the "lazy directory creation";
 121         * try to mkdir the last path component.
 122         */
 123        if (obj_req->local < 0 && errno == ENOENT) {
 124                char *dir = strrchr(obj_req->tmpfile, '/');
 125                if (dir) {
 126                        *dir = 0;
 127                        mkdir(obj_req->tmpfile, 0777);
 128                        *dir = '/';
 129                }
 130                obj_req->local = open(obj_req->tmpfile,
 131                                      O_WRONLY | O_CREAT | O_EXCL, 0666);
 132        }
 133
 134        if (obj_req->local < 0) {
 135                obj_req->state = ABORTED;
 136                error("Couldn't create temporary file %s for %s: %s",
 137                      obj_req->tmpfile, obj_req->filename, strerror(errno));
 138                return;
 139        }
 140
 141        memset(&obj_req->stream, 0, sizeof(obj_req->stream));
 142
 143        git_inflate_init(&obj_req->stream);
 144
 145        git_SHA1_Init(&obj_req->c);
 146
 147        url = xmalloc(strlen(obj_req->repo->base) + 51);
 148        obj_req->url = xmalloc(strlen(obj_req->repo->base) + 51);
 149        strcpy(url, obj_req->repo->base);
 150        posn = url + strlen(obj_req->repo->base);
 151        strcpy(posn, "/objects/");
 152        posn += 9;
 153        memcpy(posn, hex, 2);
 154        posn += 2;
 155        *(posn++) = '/';
 156        strcpy(posn, hex + 2);
 157        strcpy(obj_req->url, url);
 158
 159        /*
 160         * If a previous temp file is present, process what was already
 161         * fetched.
 162         */
 163        prevlocal = open(prevfile, O_RDONLY);
 164        if (prevlocal != -1) {
 165                do {
 166                        prev_read = xread(prevlocal, prev_buf, PREV_BUF_SIZE);
 167                        if (prev_read>0) {
 168                                if (fwrite_sha1_file(prev_buf,
 169                                                     1,
 170                                                     prev_read,
 171                                                     obj_req) == prev_read)
 172                                        prev_posn += prev_read;
 173                                else
 174                                        prev_read = -1;
 175                        }
 176                } while (prev_read > 0);
 177                close(prevlocal);
 178        }
 179        unlink_or_warn(prevfile);
 180
 181        /*
 182         * Reset inflate/SHA1 if there was an error reading the previous temp
 183         * file; also rewind to the beginning of the local file.
 184         */
 185        if (prev_read == -1) {
 186                memset(&obj_req->stream, 0, sizeof(obj_req->stream));
 187                git_inflate_init(&obj_req->stream);
 188                git_SHA1_Init(&obj_req->c);
 189                if (prev_posn>0) {
 190                        prev_posn = 0;
 191                        lseek(obj_req->local, 0, SEEK_SET);
 192                        ftruncate(obj_req->local, 0);
 193                }
 194        }
 195
 196        slot = get_active_slot();
 197        slot->callback_func = process_object_response;
 198        slot->callback_data = obj_req;
 199        obj_req->slot = slot;
 200
 201        curl_easy_setopt(slot->curl, CURLOPT_FILE, obj_req);
 202        curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite_sha1_file);
 203        curl_easy_setopt(slot->curl, CURLOPT_ERRORBUFFER, obj_req->errorstr);
 204        curl_easy_setopt(slot->curl, CURLOPT_URL, url);
 205        curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, no_pragma_header);
 206
 207        /*
 208         * If we have successfully processed data from a previous fetch
 209         * attempt, only fetch the data we don't already have.
 210         */
 211        if (prev_posn>0) {
 212                if (walker->get_verbosely)
 213                        fprintf(stderr,
 214                                "Resuming fetch of object %s at byte %ld\n",
 215                                hex, prev_posn);
 216                sprintf(range, "Range: bytes=%ld-", prev_posn);
 217                range_header = curl_slist_append(range_header, range);
 218                curl_easy_setopt(slot->curl,
 219                                 CURLOPT_HTTPHEADER, range_header);
 220        }
 221
 222        /* Try to get the request started, abort the request on error */
 223        obj_req->state = ACTIVE;
 224        if (!start_active_slot(slot)) {
 225                obj_req->state = ABORTED;
 226                obj_req->slot = NULL;
 227                close(obj_req->local);
 228                obj_req->local = -1;
 229                free(obj_req->url);
 230                return;
 231        }
 232}
 233
 234static void finish_object_request(struct object_request *obj_req)
 235{
 236        struct stat st;
 237
 238        close(obj_req->local);
 239        obj_req->local = -1;
 240
 241        if (obj_req->http_code == 416) {
 242                fprintf(stderr, "Warning: requested range invalid; we may already have all the data.\n");
 243        } else if (obj_req->curl_result != CURLE_OK) {
 244                if (stat(obj_req->tmpfile, &st) == 0)
 245                        if (st.st_size == 0)
 246                                unlink_or_warn(obj_req->tmpfile);
 247                return;
 248        }
 249
 250        git_inflate_end(&obj_req->stream);
 251        git_SHA1_Final(obj_req->real_sha1, &obj_req->c);
 252        if (obj_req->zret != Z_STREAM_END) {
 253                unlink_or_warn(obj_req->tmpfile);
 254                return;
 255        }
 256        if (hashcmp(obj_req->sha1, obj_req->real_sha1)) {
 257                unlink_or_warn(obj_req->tmpfile);
 258                return;
 259        }
 260        obj_req->rename =
 261                move_temp_to_file(obj_req->tmpfile, obj_req->filename);
 262
 263        if (obj_req->rename == 0)
 264                walker_say(obj_req->walker, "got %s\n", sha1_to_hex(obj_req->sha1));
 265}
 266
 267static void process_object_response(void *callback_data)
 268{
 269        struct object_request *obj_req =
 270                (struct object_request *)callback_data;
 271        struct walker *walker = obj_req->walker;
 272        struct walker_data *data = walker->data;
 273        struct alt_base *alt = data->alt;
 274
 275        obj_req->curl_result = obj_req->slot->curl_result;
 276        obj_req->http_code = obj_req->slot->http_code;
 277        obj_req->slot = NULL;
 278        obj_req->state = COMPLETE;
 279
 280        /* Use alternates if necessary */
 281        if (missing_target(obj_req)) {
 282                fetch_alternates(walker, alt->base);
 283                if (obj_req->repo->next != NULL) {
 284                        obj_req->repo =
 285                                obj_req->repo->next;
 286                        close(obj_req->local);
 287                        obj_req->local = -1;
 288                        start_object_request(walker, obj_req);
 289                        return;
 290                }
 291        }
 292
 293        finish_object_request(obj_req);
 294}
 295
 296static void release_object_request(struct object_request *obj_req)
 297{
 298        struct object_request *entry = object_queue_head;
 299
 300        if (obj_req->local != -1)
 301                error("fd leakage in release: %d", obj_req->local);
 302        if (obj_req == object_queue_head) {
 303                object_queue_head = obj_req->next;
 304        } else {
 305                while (entry->next != NULL && entry->next != obj_req)
 306                        entry = entry->next;
 307                if (entry->next == obj_req)
 308                        entry->next = entry->next->next;
 309        }
 310
 311        free(obj_req->url);
 312        free(obj_req);
 313}
 314
 315#ifdef USE_CURL_MULTI
 316static int fill_active_slot(struct walker *walker)
 317{
 318        struct object_request *obj_req;
 319
 320        for (obj_req = object_queue_head; obj_req; obj_req = obj_req->next) {
 321                if (obj_req->state == WAITING) {
 322                        if (has_sha1_file(obj_req->sha1))
 323                                obj_req->state = COMPLETE;
 324                        else {
 325                                start_object_request(walker, obj_req);
 326                                return 1;
 327                        }
 328                }
 329        }
 330        return 0;
 331}
 332#endif
 333
 334static void prefetch(struct walker *walker, unsigned char *sha1)
 335{
 336        struct object_request *newreq;
 337        struct object_request *tail;
 338        struct walker_data *data = walker->data;
 339        char *filename = sha1_file_name(sha1);
 340
 341        newreq = xmalloc(sizeof(*newreq));
 342        newreq->walker = walker;
 343        hashcpy(newreq->sha1, sha1);
 344        newreq->repo = data->alt;
 345        newreq->url = NULL;
 346        newreq->local = -1;
 347        newreq->state = WAITING;
 348        snprintf(newreq->filename, sizeof(newreq->filename), "%s", filename);
 349        snprintf(newreq->tmpfile, sizeof(newreq->tmpfile),
 350                 "%s.temp", filename);
 351        newreq->slot = NULL;
 352        newreq->next = NULL;
 353
 354        http_is_verbose = walker->get_verbosely;
 355
 356        if (object_queue_head == NULL) {
 357                object_queue_head = newreq;
 358        } else {
 359                tail = object_queue_head;
 360                while (tail->next != NULL)
 361                        tail = tail->next;
 362                tail->next = newreq;
 363        }
 364
 365#ifdef USE_CURL_MULTI
 366        fill_active_slots();
 367        step_active_slots();
 368#endif
 369}
 370
 371static int fetch_index(struct walker *walker, struct alt_base *repo, unsigned char *sha1)
 372{
 373        int ret = 0;
 374        char *hex = xstrdup(sha1_to_hex(sha1));
 375        char *filename;
 376        char *url;
 377        char tmpfile[PATH_MAX];
 378        long prev_posn = 0;
 379        char range[RANGE_HEADER_SIZE];
 380        struct curl_slist *range_header = NULL;
 381
 382        FILE *indexfile;
 383        struct active_request_slot *slot;
 384        struct slot_results results;
 385
 386        /* Don't use the index if the pack isn't there */
 387        url = xmalloc(strlen(repo->base) + 64);
 388        sprintf(url, "%s/objects/pack/pack-%s.pack", repo->base, hex);
 389        slot = get_active_slot();
 390        slot->results = &results;
 391        curl_easy_setopt(slot->curl, CURLOPT_URL, url);
 392        curl_easy_setopt(slot->curl, CURLOPT_NOBODY, 1);
 393        if (start_active_slot(slot)) {
 394                run_active_slot(slot);
 395                if (results.curl_result != CURLE_OK) {
 396                        ret = error("Unable to verify pack %s is available",
 397                                     hex);
 398                        goto cleanup_pack;
 399                }
 400        } else {
 401                ret = error("Unable to start request");
 402                goto cleanup_pack;
 403        }
 404
 405        if (has_pack_index(sha1)) {
 406                ret = 0;
 407                goto cleanup_pack;
 408        }
 409
 410        if (walker->get_verbosely)
 411                fprintf(stderr, "Getting index for pack %s\n", hex);
 412
 413        sprintf(url, "%s/objects/pack/pack-%s.idx", repo->base, hex);
 414
 415        filename = sha1_pack_index_name(sha1);
 416        snprintf(tmpfile, sizeof(tmpfile), "%s.temp", filename);
 417        indexfile = fopen(tmpfile, "a");
 418        if (!indexfile) {
 419                ret = error("Unable to open local file %s for pack index",
 420                            tmpfile);
 421                goto cleanup_pack;
 422        }
 423
 424        slot = get_active_slot();
 425        slot->results = &results;
 426        curl_easy_setopt(slot->curl, CURLOPT_NOBODY, 0);
 427        curl_easy_setopt(slot->curl, CURLOPT_HTTPGET, 1);
 428        curl_easy_setopt(slot->curl, CURLOPT_FILE, indexfile);
 429        curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite);
 430        curl_easy_setopt(slot->curl, CURLOPT_URL, url);
 431        curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, no_pragma_header);
 432        slot->local = indexfile;
 433
 434        /*
 435         * If there is data present from a previous transfer attempt,
 436         * resume where it left off
 437         */
 438        prev_posn = ftell(indexfile);
 439        if (prev_posn>0) {
 440                if (walker->get_verbosely)
 441                        fprintf(stderr,
 442                                "Resuming fetch of index for pack %s at byte %ld\n",
 443                                hex, prev_posn);
 444                sprintf(range, "Range: bytes=%ld-", prev_posn);
 445                range_header = curl_slist_append(range_header, range);
 446                curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, range_header);
 447        }
 448
 449        if (start_active_slot(slot)) {
 450                run_active_slot(slot);
 451                if (results.curl_result != CURLE_OK) {
 452                        ret = error("Unable to get pack index %s\n%s", url,
 453                                    curl_errorstr);
 454                        goto cleanup_index;
 455                }
 456        } else {
 457                ret = error("Unable to start request");
 458                goto cleanup_index;
 459        }
 460
 461        ret = move_temp_to_file(tmpfile, filename);
 462
 463cleanup_index:
 464        fclose(indexfile);
 465        slot->local = NULL;
 466cleanup_pack:
 467        free(url);
 468        free(hex);
 469        return ret;
 470}
 471
 472static int setup_index(struct walker *walker, struct alt_base *repo, unsigned char *sha1)
 473{
 474        struct packed_git *new_pack;
 475        if (has_pack_file(sha1))
 476                return 0; /* don't list this as something we can get */
 477
 478        if (fetch_index(walker, repo, sha1))
 479                return -1;
 480
 481        new_pack = parse_pack_index(sha1);
 482        if (!new_pack)
 483                return -1; /* parse_pack_index() already issued error message */
 484        new_pack->next = repo->packs;
 485        repo->packs = new_pack;
 486        return 0;
 487}
 488
 489static void process_alternates_response(void *callback_data)
 490{
 491        struct alternates_request *alt_req =
 492                (struct alternates_request *)callback_data;
 493        struct walker *walker = alt_req->walker;
 494        struct walker_data *cdata = walker->data;
 495        struct active_request_slot *slot = alt_req->slot;
 496        struct alt_base *tail = cdata->alt;
 497        const char *base = alt_req->base;
 498        static const char null_byte = '\0';
 499        char *data;
 500        int i = 0;
 501
 502        if (alt_req->http_specific) {
 503                if (slot->curl_result != CURLE_OK ||
 504                    !alt_req->buffer->len) {
 505
 506                        /* Try reusing the slot to get non-http alternates */
 507                        alt_req->http_specific = 0;
 508                        sprintf(alt_req->url, "%s/objects/info/alternates",
 509                                base);
 510                        curl_easy_setopt(slot->curl, CURLOPT_URL,
 511                                         alt_req->url);
 512                        active_requests++;
 513                        slot->in_use = 1;
 514                        if (slot->finished != NULL)
 515                                (*slot->finished) = 0;
 516                        if (!start_active_slot(slot)) {
 517                                cdata->got_alternates = -1;
 518                                slot->in_use = 0;
 519                                if (slot->finished != NULL)
 520                                        (*slot->finished) = 1;
 521                        }
 522                        return;
 523                }
 524        } else if (slot->curl_result != CURLE_OK) {
 525                if (!missing_target(slot)) {
 526                        cdata->got_alternates = -1;
 527                        return;
 528                }
 529        }
 530
 531        fwrite_buffer(&null_byte, 1, 1, alt_req->buffer);
 532        alt_req->buffer->len--;
 533        data = alt_req->buffer->buf;
 534
 535        while (i < alt_req->buffer->len) {
 536                int posn = i;
 537                while (posn < alt_req->buffer->len && data[posn] != '\n')
 538                        posn++;
 539                if (data[posn] == '\n') {
 540                        int okay = 0;
 541                        int serverlen = 0;
 542                        struct alt_base *newalt;
 543                        char *target = NULL;
 544                        if (data[i] == '/') {
 545                                /*
 546                                 * This counts
 547                                 * http://git.host/pub/scm/linux.git/
 548                                 * -----------here^
 549                                 * so memcpy(dst, base, serverlen) will
 550                                 * copy up to "...git.host".
 551                                 */
 552                                const char *colon_ss = strstr(base,"://");
 553                                if (colon_ss) {
 554                                        serverlen = (strchr(colon_ss + 3, '/')
 555                                                     - base);
 556                                        okay = 1;
 557                                }
 558                        } else if (!memcmp(data + i, "../", 3)) {
 559                                /*
 560                                 * Relative URL; chop the corresponding
 561                                 * number of subpath from base (and ../
 562                                 * from data), and concatenate the result.
 563                                 *
 564                                 * The code first drops ../ from data, and
 565                                 * then drops one ../ from data and one path
 566                                 * from base.  IOW, one extra ../ is dropped
 567                                 * from data than path is dropped from base.
 568                                 *
 569                                 * This is not wrong.  The alternate in
 570                                 *     http://git.host/pub/scm/linux.git/
 571                                 * to borrow from
 572                                 *     http://git.host/pub/scm/linus.git/
 573                                 * is ../../linus.git/objects/.  You need
 574                                 * two ../../ to borrow from your direct
 575                                 * neighbour.
 576                                 */
 577                                i += 3;
 578                                serverlen = strlen(base);
 579                                while (i + 2 < posn &&
 580                                       !memcmp(data + i, "../", 3)) {
 581                                        do {
 582                                                serverlen--;
 583                                        } while (serverlen &&
 584                                                 base[serverlen - 1] != '/');
 585                                        i += 3;
 586                                }
 587                                /* If the server got removed, give up. */
 588                                okay = strchr(base, ':') - base + 3 <
 589                                       serverlen;
 590                        } else if (alt_req->http_specific) {
 591                                char *colon = strchr(data + i, ':');
 592                                char *slash = strchr(data + i, '/');
 593                                if (colon && slash && colon < data + posn &&
 594                                    slash < data + posn && colon < slash) {
 595                                        okay = 1;
 596                                }
 597                        }
 598                        /* skip "objects\n" at end */
 599                        if (okay) {
 600                                target = xmalloc(serverlen + posn - i - 6);
 601                                memcpy(target, base, serverlen);
 602                                memcpy(target + serverlen, data + i,
 603                                       posn - i - 7);
 604                                target[serverlen + posn - i - 7] = 0;
 605                                if (walker->get_verbosely)
 606                                        fprintf(stderr,
 607                                                "Also look at %s\n", target);
 608                                newalt = xmalloc(sizeof(*newalt));
 609                                newalt->next = NULL;
 610                                newalt->base = target;
 611                                newalt->got_indices = 0;
 612                                newalt->packs = NULL;
 613
 614                                while (tail->next != NULL)
 615                                        tail = tail->next;
 616                                tail->next = newalt;
 617                        }
 618                }
 619                i = posn + 1;
 620        }
 621
 622        cdata->got_alternates = 1;
 623}
 624
 625static void fetch_alternates(struct walker *walker, const char *base)
 626{
 627        struct strbuf buffer = STRBUF_INIT;
 628        char *url;
 629        struct active_request_slot *slot;
 630        struct alternates_request alt_req;
 631        struct walker_data *cdata = walker->data;
 632
 633        /*
 634         * If another request has already started fetching alternates,
 635         * wait for them to arrive and return to processing this request's
 636         * curl message
 637         */
 638#ifdef USE_CURL_MULTI
 639        while (cdata->got_alternates == 0) {
 640                step_active_slots();
 641        }
 642#endif
 643
 644        /* Nothing to do if they've already been fetched */
 645        if (cdata->got_alternates == 1)
 646                return;
 647
 648        /* Start the fetch */
 649        cdata->got_alternates = 0;
 650
 651        if (walker->get_verbosely)
 652                fprintf(stderr, "Getting alternates list for %s\n", base);
 653
 654        url = xmalloc(strlen(base) + 31);
 655        sprintf(url, "%s/objects/info/http-alternates", base);
 656
 657        /*
 658         * Use a callback to process the result, since another request
 659         * may fail and need to have alternates loaded before continuing
 660         */
 661        slot = get_active_slot();
 662        slot->callback_func = process_alternates_response;
 663        alt_req.walker = walker;
 664        slot->callback_data = &alt_req;
 665
 666        curl_easy_setopt(slot->curl, CURLOPT_FILE, &buffer);
 667        curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite_buffer);
 668        curl_easy_setopt(slot->curl, CURLOPT_URL, url);
 669
 670        alt_req.base = base;
 671        alt_req.url = url;
 672        alt_req.buffer = &buffer;
 673        alt_req.http_specific = 1;
 674        alt_req.slot = slot;
 675
 676        if (start_active_slot(slot))
 677                run_active_slot(slot);
 678        else
 679                cdata->got_alternates = -1;
 680
 681        strbuf_release(&buffer);
 682        free(url);
 683}
 684
 685static int fetch_indices(struct walker *walker, struct alt_base *repo)
 686{
 687        unsigned char sha1[20];
 688        char *url;
 689        struct strbuf buffer = STRBUF_INIT;
 690        char *data;
 691        int i = 0;
 692        int ret = 0;
 693
 694        struct active_request_slot *slot;
 695        struct slot_results results;
 696
 697        if (repo->got_indices)
 698                return 0;
 699
 700        if (walker->get_verbosely)
 701                fprintf(stderr, "Getting pack list for %s\n", repo->base);
 702
 703        url = xmalloc(strlen(repo->base) + 21);
 704        sprintf(url, "%s/objects/info/packs", repo->base);
 705
 706        slot = get_active_slot();
 707        slot->results = &results;
 708        curl_easy_setopt(slot->curl, CURLOPT_FILE, &buffer);
 709        curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite_buffer);
 710        curl_easy_setopt(slot->curl, CURLOPT_URL, url);
 711        curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, NULL);
 712        if (start_active_slot(slot)) {
 713                run_active_slot(slot);
 714                if (results.curl_result != CURLE_OK) {
 715                        if (missing_target(&results)) {
 716                                repo->got_indices = 1;
 717                                goto cleanup;
 718                        } else {
 719                                repo->got_indices = 0;
 720                                ret = error("%s", curl_errorstr);
 721                                goto cleanup;
 722                        }
 723                }
 724        } else {
 725                repo->got_indices = 0;
 726                ret = error("Unable to start request");
 727                goto cleanup;
 728        }
 729
 730        data = buffer.buf;
 731        while (i < buffer.len) {
 732                switch (data[i]) {
 733                case 'P':
 734                        i++;
 735                        if (i + 52 <= buffer.len &&
 736                            !prefixcmp(data + i, " pack-") &&
 737                            !prefixcmp(data + i + 46, ".pack\n")) {
 738                                get_sha1_hex(data + i + 6, sha1);
 739                                setup_index(walker, repo, sha1);
 740                                i += 51;
 741                                break;
 742                        }
 743                default:
 744                        while (i < buffer.len && data[i] != '\n')
 745                                i++;
 746                }
 747                i++;
 748        }
 749
 750        repo->got_indices = 1;
 751cleanup:
 752        strbuf_release(&buffer);
 753        free(url);
 754        return ret;
 755}
 756
 757static int fetch_pack(struct walker *walker, struct alt_base *repo, unsigned char *sha1)
 758{
 759        char *url;
 760        struct packed_git *target;
 761        struct packed_git **lst;
 762        FILE *packfile;
 763        char *filename;
 764        char tmpfile[PATH_MAX];
 765        int ret;
 766        long prev_posn = 0;
 767        char range[RANGE_HEADER_SIZE];
 768        struct curl_slist *range_header = NULL;
 769
 770        struct active_request_slot *slot;
 771        struct slot_results results;
 772
 773        if (fetch_indices(walker, repo))
 774                return -1;
 775        target = find_sha1_pack(sha1, repo->packs);
 776        if (!target)
 777                return -1;
 778
 779        if (walker->get_verbosely) {
 780                fprintf(stderr, "Getting pack %s\n",
 781                        sha1_to_hex(target->sha1));
 782                fprintf(stderr, " which contains %s\n",
 783                        sha1_to_hex(sha1));
 784        }
 785
 786        url = xmalloc(strlen(repo->base) + 65);
 787        sprintf(url, "%s/objects/pack/pack-%s.pack",
 788                repo->base, sha1_to_hex(target->sha1));
 789
 790        filename = sha1_pack_name(target->sha1);
 791        snprintf(tmpfile, sizeof(tmpfile), "%s.temp", filename);
 792        packfile = fopen(tmpfile, "a");
 793        if (!packfile)
 794                return error("Unable to open local file %s for pack",
 795                             tmpfile);
 796
 797        slot = get_active_slot();
 798        slot->results = &results;
 799        curl_easy_setopt(slot->curl, CURLOPT_FILE, packfile);
 800        curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite);
 801        curl_easy_setopt(slot->curl, CURLOPT_URL, url);
 802        curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, no_pragma_header);
 803        slot->local = packfile;
 804
 805        /*
 806         * If there is data present from a previous transfer attempt,
 807         * resume where it left off
 808         */
 809        prev_posn = ftell(packfile);
 810        if (prev_posn>0) {
 811                if (walker->get_verbosely)
 812                        fprintf(stderr,
 813                                "Resuming fetch of pack %s at byte %ld\n",
 814                                sha1_to_hex(target->sha1), prev_posn);
 815                sprintf(range, "Range: bytes=%ld-", prev_posn);
 816                range_header = curl_slist_append(range_header, range);
 817                curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, range_header);
 818        }
 819
 820        if (start_active_slot(slot)) {
 821                run_active_slot(slot);
 822                if (results.curl_result != CURLE_OK) {
 823                        fclose(packfile);
 824                        slot->local = NULL;
 825                        return error("Unable to get pack file %s\n%s", url,
 826                                     curl_errorstr);
 827                }
 828        } else {
 829                fclose(packfile);
 830                slot->local = NULL;
 831                return error("Unable to start request");
 832        }
 833
 834        target->pack_size = ftell(packfile);
 835        fclose(packfile);
 836        slot->local = NULL;
 837
 838        ret = move_temp_to_file(tmpfile, filename);
 839        if (ret)
 840                return ret;
 841
 842        lst = &repo->packs;
 843        while (*lst != target)
 844                lst = &((*lst)->next);
 845        *lst = (*lst)->next;
 846
 847        if (verify_pack(target))
 848                return -1;
 849        install_packed_git(target);
 850
 851        return 0;
 852}
 853
 854static void abort_object_request(struct object_request *obj_req)
 855{
 856        if (obj_req->local >= 0) {
 857                close(obj_req->local);
 858                obj_req->local = -1;
 859        }
 860        unlink_or_warn(obj_req->tmpfile);
 861        if (obj_req->slot) {
 862                release_active_slot(obj_req->slot);
 863                obj_req->slot = NULL;
 864        }
 865        release_object_request(obj_req);
 866}
 867
 868static int fetch_object(struct walker *walker, struct alt_base *repo, unsigned char *sha1)
 869{
 870        char *hex = sha1_to_hex(sha1);
 871        int ret = 0;
 872        struct object_request *obj_req = object_queue_head;
 873
 874        while (obj_req != NULL && hashcmp(obj_req->sha1, sha1))
 875                obj_req = obj_req->next;
 876        if (obj_req == NULL)
 877                return error("Couldn't find request for %s in the queue", hex);
 878
 879        if (has_sha1_file(obj_req->sha1)) {
 880                abort_object_request(obj_req);
 881                return 0;
 882        }
 883
 884#ifdef USE_CURL_MULTI
 885        while (obj_req->state == WAITING)
 886                step_active_slots();
 887#else
 888        start_object_request(walker, obj_req);
 889#endif
 890
 891        while (obj_req->state == ACTIVE)
 892                run_active_slot(obj_req->slot);
 893
 894        if (obj_req->local != -1) {
 895                close(obj_req->local);
 896                obj_req->local = -1;
 897        }
 898
 899        if (obj_req->state == ABORTED) {
 900                ret = error("Request for %s aborted", hex);
 901        } else if (obj_req->curl_result != CURLE_OK &&
 902                   obj_req->http_code != 416) {
 903                if (missing_target(obj_req))
 904                        ret = -1; /* Be silent, it is probably in a pack. */
 905                else
 906                        ret = error("%s (curl_result = %d, http_code = %ld, sha1 = %s)",
 907                                    obj_req->errorstr, obj_req->curl_result,
 908                                    obj_req->http_code, hex);
 909        } else if (obj_req->zret != Z_STREAM_END) {
 910                walker->corrupt_object_found++;
 911                ret = error("File %s (%s) corrupt", hex, obj_req->url);
 912        } else if (hashcmp(obj_req->sha1, obj_req->real_sha1)) {
 913                ret = error("File %s has bad hash", hex);
 914        } else if (obj_req->rename < 0) {
 915                ret = error("unable to write sha1 filename %s",
 916                            obj_req->filename);
 917        }
 918
 919        release_object_request(obj_req);
 920        return ret;
 921}
 922
 923static int fetch(struct walker *walker, unsigned char *sha1)
 924{
 925        struct walker_data *data = walker->data;
 926        struct alt_base *altbase = data->alt;
 927
 928        if (!fetch_object(walker, altbase, sha1))
 929                return 0;
 930        while (altbase) {
 931                if (!fetch_pack(walker, altbase, sha1))
 932                        return 0;
 933                fetch_alternates(walker, data->alt->base);
 934                altbase = altbase->next;
 935        }
 936        return error("Unable to find %s under %s", sha1_to_hex(sha1),
 937                     data->alt->base);
 938}
 939
 940static int fetch_ref(struct walker *walker, struct ref *ref)
 941{
 942        struct walker_data *data = walker->data;
 943        return http_fetch_ref(data->alt->base, ref);
 944}
 945
 946static void cleanup(struct walker *walker)
 947{
 948        http_cleanup();
 949}
 950
 951struct walker *get_http_walker(const char *url, struct remote *remote)
 952{
 953        char *s;
 954        struct walker_data *data = xmalloc(sizeof(struct walker_data));
 955        struct walker *walker = xmalloc(sizeof(struct walker));
 956
 957        http_init(remote);
 958
 959        data->alt = xmalloc(sizeof(*data->alt));
 960        data->alt->base = xmalloc(strlen(url) + 1);
 961        strcpy(data->alt->base, url);
 962        for (s = data->alt->base + strlen(data->alt->base) - 1; *s == '/'; --s)
 963                *s = 0;
 964
 965        data->alt->got_indices = 0;
 966        data->alt->packs = NULL;
 967        data->alt->next = NULL;
 968        data->got_alternates = -1;
 969
 970        walker->corrupt_object_found = 0;
 971        walker->fetch = fetch;
 972        walker->fetch_ref = fetch_ref;
 973        walker->prefetch = prefetch;
 974        walker->cleanup = cleanup;
 975        walker->data = data;
 976
 977#ifdef USE_CURL_MULTI
 978        add_fill_function(walker, (int (*)(void *)) fill_active_slot);
 979#endif
 980
 981        return walker;
 982}