Andrew's git - gitweb.git/blob

e9b9f307f8f0a38326df56e9a827b261990734fe
   1#include "cache.h"
   2#include "commit.h"
   3#include "pack.h"
   4#include "fetch.h"
   5#include "http.h"
   6
   7#define PREV_BUF_SIZE 4096
   8#define RANGE_HEADER_SIZE 30
   9
  10static int commits_on_stdin;
  11
  12static int got_alternates = -1;
  13static int corrupt_object_found;
  14
  15static struct curl_slist *no_pragma_header;
  16
  17struct alt_base
  18{
  19        char *base;
  20        int got_indices;
  21        struct packed_git *packs;
  22        struct alt_base *next;
  23};
  24
  25static struct alt_base *alt;
  26
  27enum object_request_state {
  28        WAITING,
  29        ABORTED,
  30        ACTIVE,
  31        COMPLETE,
  32};
  33
  34struct object_request
  35{
  36        unsigned char sha1[20];
  37        struct alt_base *repo;
  38        char *url;
  39        char filename[PATH_MAX];
  40        char tmpfile[PATH_MAX];
  41        int local;
  42        enum object_request_state state;
  43        CURLcode curl_result;
  44        char errorstr[CURL_ERROR_SIZE];
  45        long http_code;
  46        unsigned char real_sha1[20];
  47        SHA_CTX c;
  48        z_stream stream;
  49        int zret;
  50        int rename;
  51        struct active_request_slot *slot;
  52        struct object_request *next;
  53};
  54
  55struct alternates_request {
  56        const char *base;
  57        char *url;
  58        struct buffer *buffer;
  59        struct active_request_slot *slot;
  60        int http_specific;
  61};
  62
  63static struct object_request *object_queue_head;
  64
  65static size_t fwrite_sha1_file(void *ptr, size_t eltsize, size_t nmemb,
  66                               void *data)
  67{
  68        unsigned char expn[4096];
  69        size_t size = eltsize * nmemb;
  70        int posn = 0;
  71        struct object_request *obj_req = (struct object_request *)data;
  72        do {
  73                ssize_t retval = xwrite(obj_req->local,
  74                                       (char *) ptr + posn, size - posn);
  75                if (retval < 0)
  76                        return posn;
  77                posn += retval;
  78        } while (posn < size);
  79
  80        obj_req->stream.avail_in = size;
  81        obj_req->stream.next_in = ptr;
  82        do {
  83                obj_req->stream.next_out = expn;
  84                obj_req->stream.avail_out = sizeof(expn);
  85                obj_req->zret = inflate(&obj_req->stream, Z_SYNC_FLUSH);
  86                SHA1_Update(&obj_req->c, expn,
  87                            sizeof(expn) - obj_req->stream.avail_out);
  88        } while (obj_req->stream.avail_in && obj_req->zret == Z_OK);
  89        data_received++;
  90        return size;
  91}
  92
  93static int missing__target(int code, int result)
  94{
  95        return  /* file:// URL -- do we ever use one??? */
  96                (result == CURLE_FILE_COULDNT_READ_FILE) ||
  97                /* http:// and https:// URL */
  98                (code == 404 && result == CURLE_HTTP_RETURNED_ERROR) ||
  99                /* ftp:// URL */
 100                (code == 550 && result == CURLE_FTP_COULDNT_RETR_FILE)
 101                ;
 102}
 103
 104#define missing_target(a) missing__target((a)->http_code, (a)->curl_result)
 105
 106static void fetch_alternates(const char *base);
 107
 108static void process_object_response(void *callback_data);
 109
 110static void start_object_request(struct object_request *obj_req)
 111{
 112        char *hex = sha1_to_hex(obj_req->sha1);
 113        char prevfile[PATH_MAX];
 114        char *url;
 115        char *posn;
 116        int prevlocal;
 117        unsigned char prev_buf[PREV_BUF_SIZE];
 118        ssize_t prev_read = 0;
 119        long prev_posn = 0;
 120        char range[RANGE_HEADER_SIZE];
 121        struct curl_slist *range_header = NULL;
 122        struct active_request_slot *slot;
 123
 124        snprintf(prevfile, sizeof(prevfile), "%s.prev", obj_req->filename);
 125        unlink(prevfile);
 126        rename(obj_req->tmpfile, prevfile);
 127        unlink(obj_req->tmpfile);
 128
 129        if (obj_req->local != -1)
 130                error("fd leakage in start: %d", obj_req->local);
 131        obj_req->local = open(obj_req->tmpfile,
 132                              O_WRONLY | O_CREAT | O_EXCL, 0666);
 133        /* This could have failed due to the "lazy directory creation";
 134         * try to mkdir the last path component.
 135         */
 136        if (obj_req->local < 0 && errno == ENOENT) {
 137                char *dir = strrchr(obj_req->tmpfile, '/');
 138                if (dir) {
 139                        *dir = 0;
 140                        mkdir(obj_req->tmpfile, 0777);
 141                        *dir = '/';
 142                }
 143                obj_req->local = open(obj_req->tmpfile,
 144                                      O_WRONLY | O_CREAT | O_EXCL, 0666);
 145        }
 146
 147        if (obj_req->local < 0) {
 148                obj_req->state = ABORTED;
 149                error("Couldn't create temporary file %s for %s: %s",
 150                      obj_req->tmpfile, obj_req->filename, strerror(errno));
 151                return;
 152        }
 153
 154        memset(&obj_req->stream, 0, sizeof(obj_req->stream));
 155
 156        inflateInit(&obj_req->stream);
 157
 158        SHA1_Init(&obj_req->c);
 159
 160        url = xmalloc(strlen(obj_req->repo->base) + 51);
 161        obj_req->url = xmalloc(strlen(obj_req->repo->base) + 51);
 162        strcpy(url, obj_req->repo->base);
 163        posn = url + strlen(obj_req->repo->base);
 164        strcpy(posn, "/objects/");
 165        posn += 9;
 166        memcpy(posn, hex, 2);
 167        posn += 2;
 168        *(posn++) = '/';
 169        strcpy(posn, hex + 2);
 170        strcpy(obj_req->url, url);
 171
 172        /* If a previous temp file is present, process what was already
 173           fetched. */
 174        prevlocal = open(prevfile, O_RDONLY);
 175        if (prevlocal != -1) {
 176                do {
 177                        prev_read = xread(prevlocal, prev_buf, PREV_BUF_SIZE);
 178                        if (prev_read>0) {
 179                                if (fwrite_sha1_file(prev_buf,
 180                                                     1,
 181                                                     prev_read,
 182                                                     obj_req) == prev_read) {
 183                                        prev_posn += prev_read;
 184                                } else {
 185                                        prev_read = -1;
 186                                }
 187                        }
 188                } while (prev_read > 0);
 189                close(prevlocal);
 190        }
 191        unlink(prevfile);
 192
 193        /* Reset inflate/SHA1 if there was an error reading the previous temp
 194           file; also rewind to the beginning of the local file. */
 195        if (prev_read == -1) {
 196                memset(&obj_req->stream, 0, sizeof(obj_req->stream));
 197                inflateInit(&obj_req->stream);
 198                SHA1_Init(&obj_req->c);
 199                if (prev_posn>0) {
 200                        prev_posn = 0;
 201                        lseek(obj_req->local, 0, SEEK_SET);
 202                        ftruncate(obj_req->local, 0);
 203                }
 204        }
 205
 206        slot = get_active_slot();
 207        slot->callback_func = process_object_response;
 208        slot->callback_data = obj_req;
 209        obj_req->slot = slot;
 210
 211        curl_easy_setopt(slot->curl, CURLOPT_FILE, obj_req);
 212        curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite_sha1_file);
 213        curl_easy_setopt(slot->curl, CURLOPT_ERRORBUFFER, obj_req->errorstr);
 214        curl_easy_setopt(slot->curl, CURLOPT_URL, url);
 215        curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, no_pragma_header);
 216
 217        /* If we have successfully processed data from a previous fetch
 218           attempt, only fetch the data we don't already have. */
 219        if (prev_posn>0) {
 220                if (get_verbosely)
 221                        fprintf(stderr,
 222                                "Resuming fetch of object %s at byte %ld\n",
 223                                hex, prev_posn);
 224                sprintf(range, "Range: bytes=%ld-", prev_posn);
 225                range_header = curl_slist_append(range_header, range);
 226                curl_easy_setopt(slot->curl,
 227                                 CURLOPT_HTTPHEADER, range_header);
 228        }
 229
 230        /* Try to get the request started, abort the request on error */
 231        obj_req->state = ACTIVE;
 232        if (!start_active_slot(slot)) {
 233                obj_req->state = ABORTED;
 234                obj_req->slot = NULL;
 235                close(obj_req->local); obj_req->local = -1;
 236                free(obj_req->url);
 237                return;
 238        }
 239}
 240
 241static void finish_object_request(struct object_request *obj_req)
 242{
 243        struct stat st;
 244
 245        fchmod(obj_req->local, 0444);
 246        close(obj_req->local); obj_req->local = -1;
 247
 248        if (obj_req->http_code == 416) {
 249                fprintf(stderr, "Warning: requested range invalid; we may already have all the data.\n");
 250        } else if (obj_req->curl_result != CURLE_OK) {
 251                if (stat(obj_req->tmpfile, &st) == 0)
 252                        if (st.st_size == 0)
 253                                unlink(obj_req->tmpfile);
 254                return;
 255        }
 256
 257        inflateEnd(&obj_req->stream);
 258        SHA1_Final(obj_req->real_sha1, &obj_req->c);
 259        if (obj_req->zret != Z_STREAM_END) {
 260                unlink(obj_req->tmpfile);
 261                return;
 262        }
 263        if (hashcmp(obj_req->sha1, obj_req->real_sha1)) {
 264                unlink(obj_req->tmpfile);
 265                return;
 266        }
 267        obj_req->rename =
 268                move_temp_to_file(obj_req->tmpfile, obj_req->filename);
 269
 270        if (obj_req->rename == 0)
 271                pull_say("got %s\n", sha1_to_hex(obj_req->sha1));
 272}
 273
 274static void process_object_response(void *callback_data)
 275{
 276        struct object_request *obj_req =
 277                (struct object_request *)callback_data;
 278
 279        obj_req->curl_result = obj_req->slot->curl_result;
 280        obj_req->http_code = obj_req->slot->http_code;
 281        obj_req->slot = NULL;
 282        obj_req->state = COMPLETE;
 283
 284        /* Use alternates if necessary */
 285        if (missing_target(obj_req)) {
 286                fetch_alternates(alt->base);
 287                if (obj_req->repo->next != NULL) {
 288                        obj_req->repo =
 289                                obj_req->repo->next;
 290                        close(obj_req->local);
 291                        obj_req->local = -1;
 292                        start_object_request(obj_req);
 293                        return;
 294                }
 295        }
 296
 297        finish_object_request(obj_req);
 298}
 299
 300static void release_object_request(struct object_request *obj_req)
 301{
 302        struct object_request *entry = object_queue_head;
 303
 304        if (obj_req->local != -1)
 305                error("fd leakage in release: %d", obj_req->local);
 306        if (obj_req == object_queue_head) {
 307                object_queue_head = obj_req->next;
 308        } else {
 309                while (entry->next != NULL && entry->next != obj_req)
 310                        entry = entry->next;
 311                if (entry->next == obj_req)
 312                        entry->next = entry->next->next;
 313        }
 314
 315        free(obj_req->url);
 316        free(obj_req);
 317}
 318
 319#ifdef USE_CURL_MULTI
 320int fill_active_slot(void)
 321{
 322        struct object_request *obj_req;
 323
 324        for (obj_req = object_queue_head; obj_req; obj_req = obj_req->next) {
 325                if (obj_req->state == WAITING) {
 326                        if (has_sha1_file(obj_req->sha1))
 327                                obj_req->state = COMPLETE;
 328                        else {
 329                                start_object_request(obj_req);
 330                                return 1;
 331                        }
 332                }
 333        }
 334        return 0;
 335}
 336#endif
 337
 338void prefetch(unsigned char *sha1)
 339{
 340        struct object_request *newreq;
 341        struct object_request *tail;
 342        char *filename = sha1_file_name(sha1);
 343
 344        newreq = xmalloc(sizeof(*newreq));
 345        hashcpy(newreq->sha1, sha1);
 346        newreq->repo = alt;
 347        newreq->url = NULL;
 348        newreq->local = -1;
 349        newreq->state = WAITING;
 350        snprintf(newreq->filename, sizeof(newreq->filename), "%s", filename);
 351        snprintf(newreq->tmpfile, sizeof(newreq->tmpfile),
 352                 "%s.temp", filename);
 353        newreq->slot = NULL;
 354        newreq->next = NULL;
 355
 356        if (object_queue_head == NULL) {
 357                object_queue_head = newreq;
 358        } else {
 359                tail = object_queue_head;
 360                while (tail->next != NULL) {
 361                        tail = tail->next;
 362                }
 363                tail->next = newreq;
 364        }
 365
 366#ifdef USE_CURL_MULTI
 367        fill_active_slots();
 368        step_active_slots();
 369#endif
 370}
 371
 372static int fetch_index(struct alt_base *repo, unsigned char *sha1)
 373{
 374        char *hex = sha1_to_hex(sha1);
 375        char *filename;
 376        char *url;
 377        char tmpfile[PATH_MAX];
 378        long prev_posn = 0;
 379        char range[RANGE_HEADER_SIZE];
 380        struct curl_slist *range_header = NULL;
 381
 382        FILE *indexfile;
 383        struct active_request_slot *slot;
 384        struct slot_results results;
 385
 386        if (has_pack_index(sha1))
 387                return 0;
 388
 389        if (get_verbosely)
 390                fprintf(stderr, "Getting index for pack %s\n", hex);
 391
 392        url = xmalloc(strlen(repo->base) + 64);
 393        sprintf(url, "%s/objects/pack/pack-%s.idx", repo->base, hex);
 394
 395        filename = sha1_pack_index_name(sha1);
 396        snprintf(tmpfile, sizeof(tmpfile), "%s.temp", filename);
 397        indexfile = fopen(tmpfile, "a");
 398        if (!indexfile)
 399                return error("Unable to open local file %s for pack index",
 400                             filename);
 401
 402        slot = get_active_slot();
 403        slot->results = &results;
 404        curl_easy_setopt(slot->curl, CURLOPT_FILE, indexfile);
 405        curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite);
 406        curl_easy_setopt(slot->curl, CURLOPT_URL, url);
 407        curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, no_pragma_header);
 408        slot->local = indexfile;
 409
 410        /* If there is data present from a previous transfer attempt,
 411           resume where it left off */
 412        prev_posn = ftell(indexfile);
 413        if (prev_posn>0) {
 414                if (get_verbosely)
 415                        fprintf(stderr,
 416                                "Resuming fetch of index for pack %s at byte %ld\n",
 417                                hex, prev_posn);
 418                sprintf(range, "Range: bytes=%ld-", prev_posn);
 419                range_header = curl_slist_append(range_header, range);
 420                curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, range_header);
 421        }
 422
 423        if (start_active_slot(slot)) {
 424                run_active_slot(slot);
 425                if (results.curl_result != CURLE_OK) {
 426                        fclose(indexfile);
 427                        return error("Unable to get pack index %s\n%s", url,
 428                                     curl_errorstr);
 429                }
 430        } else {
 431                fclose(indexfile);
 432                return error("Unable to start request");
 433        }
 434
 435        fclose(indexfile);
 436
 437        return move_temp_to_file(tmpfile, filename);
 438}
 439
 440static int setup_index(struct alt_base *repo, unsigned char *sha1)
 441{
 442        struct packed_git *new_pack;
 443        if (has_pack_file(sha1))
 444                return 0; /* don't list this as something we can get */
 445
 446        if (fetch_index(repo, sha1))
 447                return -1;
 448
 449        new_pack = parse_pack_index(sha1);
 450        new_pack->next = repo->packs;
 451        repo->packs = new_pack;
 452        return 0;
 453}
 454
 455static void process_alternates_response(void *callback_data)
 456{
 457        struct alternates_request *alt_req =
 458                (struct alternates_request *)callback_data;
 459        struct active_request_slot *slot = alt_req->slot;
 460        struct alt_base *tail = alt;
 461        const char *base = alt_req->base;
 462        static const char null_byte = '\0';
 463        char *data;
 464        int i = 0;
 465
 466        if (alt_req->http_specific) {
 467                if (slot->curl_result != CURLE_OK ||
 468                    !alt_req->buffer->posn) {
 469
 470                        /* Try reusing the slot to get non-http alternates */
 471                        alt_req->http_specific = 0;
 472                        sprintf(alt_req->url, "%s/objects/info/alternates",
 473                                base);
 474                        curl_easy_setopt(slot->curl, CURLOPT_URL,
 475                                         alt_req->url);
 476                        active_requests++;
 477                        slot->in_use = 1;
 478                        if (slot->finished != NULL)
 479                                (*slot->finished) = 0;
 480                        if (!start_active_slot(slot)) {
 481                                got_alternates = -1;
 482                                slot->in_use = 0;
 483                                if (slot->finished != NULL)
 484                                        (*slot->finished) = 1;
 485                        }
 486                        return;
 487                }
 488        } else if (slot->curl_result != CURLE_OK) {
 489                if (!missing_target(slot)) {
 490                        got_alternates = -1;
 491                        return;
 492                }
 493        }
 494
 495        fwrite_buffer(&null_byte, 1, 1, alt_req->buffer);
 496        alt_req->buffer->posn--;
 497        data = alt_req->buffer->buffer;
 498
 499        while (i < alt_req->buffer->posn) {
 500                int posn = i;
 501                while (posn < alt_req->buffer->posn && data[posn] != '\n')
 502                        posn++;
 503                if (data[posn] == '\n') {
 504                        int okay = 0;
 505                        int serverlen = 0;
 506                        struct alt_base *newalt;
 507                        char *target = NULL;
 508                        if (data[i] == '/') {
 509                                /* This counts
 510                                 * http://git.host/pub/scm/linux.git/
 511                                 * -----------here^
 512                                 * so memcpy(dst, base, serverlen) will
 513                                 * copy up to "...git.host".
 514                                 */
 515                                const char *colon_ss = strstr(base,"://");
 516                                if (colon_ss) {
 517                                        serverlen = (strchr(colon_ss + 3, '/')
 518                                                     - base);
 519                                        okay = 1;
 520                                }
 521                        } else if (!memcmp(data + i, "../", 3)) {
 522                                /* Relative URL; chop the corresponding
 523                                 * number of subpath from base (and ../
 524                                 * from data), and concatenate the result.
 525                                 *
 526                                 * The code first drops ../ from data, and
 527                                 * then drops one ../ from data and one path
 528                                 * from base.  IOW, one extra ../ is dropped
 529                                 * from data than path is dropped from base.
 530                                 *
 531                                 * This is not wrong.  The alternate in
 532                                 *     http://git.host/pub/scm/linux.git/
 533                                 * to borrow from
 534                                 *     http://git.host/pub/scm/linus.git/
 535                                 * is ../../linus.git/objects/.  You need
 536                                 * two ../../ to borrow from your direct
 537                                 * neighbour.
 538                                 */
 539                                i += 3;
 540                                serverlen = strlen(base);
 541                                while (i + 2 < posn &&
 542                                       !memcmp(data + i, "../", 3)) {
 543                                        do {
 544                                                serverlen--;
 545                                        } while (serverlen &&
 546                                                 base[serverlen - 1] != '/');
 547                                        i += 3;
 548                                }
 549                                /* If the server got removed, give up. */
 550                                okay = strchr(base, ':') - base + 3 <
 551                                        serverlen;
 552                        } else if (alt_req->http_specific) {
 553                                char *colon = strchr(data + i, ':');
 554                                char *slash = strchr(data + i, '/');
 555                                if (colon && slash && colon < data + posn &&
 556                                    slash < data + posn && colon < slash) {
 557                                        okay = 1;
 558                                }
 559                        }
 560                        /* skip "objects\n" at end */
 561                        if (okay) {
 562                                target = xmalloc(serverlen + posn - i - 6);
 563                                memcpy(target, base, serverlen);
 564                                memcpy(target + serverlen, data + i,
 565                                       posn - i - 7);
 566                                target[serverlen + posn - i - 7] = 0;
 567                                if (get_verbosely)
 568                                        fprintf(stderr,
 569                                                "Also look at %s\n", target);
 570                                newalt = xmalloc(sizeof(*newalt));
 571                                newalt->next = NULL;
 572                                newalt->base = target;
 573                                newalt->got_indices = 0;
 574                                newalt->packs = NULL;
 575
 576                                while (tail->next != NULL)
 577                                        tail = tail->next;
 578                                tail->next = newalt;
 579                        }
 580                }
 581                i = posn + 1;
 582        }
 583
 584        got_alternates = 1;
 585}
 586
 587static void fetch_alternates(const char *base)
 588{
 589        struct buffer buffer;
 590        char *url;
 591        char *data;
 592        struct active_request_slot *slot;
 593        struct alternates_request alt_req;
 594
 595        /* If another request has already started fetching alternates,
 596           wait for them to arrive and return to processing this request's
 597           curl message */
 598#ifdef USE_CURL_MULTI
 599        while (got_alternates == 0) {
 600                step_active_slots();
 601        }
 602#endif
 603
 604        /* Nothing to do if they've already been fetched */
 605        if (got_alternates == 1)
 606                return;
 607
 608        /* Start the fetch */
 609        got_alternates = 0;
 610
 611        data = xmalloc(4096);
 612        buffer.size = 4096;
 613        buffer.posn = 0;
 614        buffer.buffer = data;
 615
 616        if (get_verbosely)
 617                fprintf(stderr, "Getting alternates list for %s\n", base);
 618
 619        url = xmalloc(strlen(base) + 31);
 620        sprintf(url, "%s/objects/info/http-alternates", base);
 621
 622        /* Use a callback to process the result, since another request
 623           may fail and need to have alternates loaded before continuing */
 624        slot = get_active_slot();
 625        slot->callback_func = process_alternates_response;
 626        slot->callback_data = &alt_req;
 627
 628        curl_easy_setopt(slot->curl, CURLOPT_FILE, &buffer);
 629        curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite_buffer);
 630        curl_easy_setopt(slot->curl, CURLOPT_URL, url);
 631
 632        alt_req.base = base;
 633        alt_req.url = url;
 634        alt_req.buffer = &buffer;
 635        alt_req.http_specific = 1;
 636        alt_req.slot = slot;
 637
 638        if (start_active_slot(slot))
 639                run_active_slot(slot);
 640        else
 641                got_alternates = -1;
 642
 643        free(data);
 644        free(url);
 645}
 646
 647static int fetch_indices(struct alt_base *repo)
 648{
 649        unsigned char sha1[20];
 650        char *url;
 651        struct buffer buffer;
 652        char *data;
 653        int i = 0;
 654
 655        struct active_request_slot *slot;
 656        struct slot_results results;
 657
 658        if (repo->got_indices)
 659                return 0;
 660
 661        data = xmalloc(4096);
 662        buffer.size = 4096;
 663        buffer.posn = 0;
 664        buffer.buffer = data;
 665
 666        if (get_verbosely)
 667                fprintf(stderr, "Getting pack list for %s\n", repo->base);
 668
 669        url = xmalloc(strlen(repo->base) + 21);
 670        sprintf(url, "%s/objects/info/packs", repo->base);
 671
 672        slot = get_active_slot();
 673        slot->results = &results;
 674        curl_easy_setopt(slot->curl, CURLOPT_FILE, &buffer);
 675        curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite_buffer);
 676        curl_easy_setopt(slot->curl, CURLOPT_URL, url);
 677        curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, NULL);
 678        if (start_active_slot(slot)) {
 679                run_active_slot(slot);
 680                if (results.curl_result != CURLE_OK) {
 681                        if (missing_target(&results)) {
 682                                repo->got_indices = 1;
 683                                free(buffer.buffer);
 684                                return 0;
 685                        } else {
 686                                repo->got_indices = 0;
 687                                free(buffer.buffer);
 688                                return error("%s", curl_errorstr);
 689                        }
 690                }
 691        } else {
 692                repo->got_indices = 0;
 693                free(buffer.buffer);
 694                return error("Unable to start request");
 695        }
 696
 697        data = buffer.buffer;
 698        while (i < buffer.posn) {
 699                switch (data[i]) {
 700                case 'P':
 701                        i++;
 702                        if (i + 52 <= buffer.posn &&
 703                            !prefixcmp(data + i, " pack-") &&
 704                            !prefixcmp(data + i + 46, ".pack\n")) {
 705                                get_sha1_hex(data + i + 6, sha1);
 706                                setup_index(repo, sha1);
 707                                i += 51;
 708                                break;
 709                        }
 710                default:
 711                        while (i < buffer.posn && data[i] != '\n')
 712                                i++;
 713                }
 714                i++;
 715        }
 716
 717        free(buffer.buffer);
 718        repo->got_indices = 1;
 719        return 0;
 720}
 721
 722static int fetch_pack(struct alt_base *repo, unsigned char *sha1)
 723{
 724        char *url;
 725        struct packed_git *target;
 726        struct packed_git **lst;
 727        FILE *packfile;
 728        char *filename;
 729        char tmpfile[PATH_MAX];
 730        int ret;
 731        long prev_posn = 0;
 732        char range[RANGE_HEADER_SIZE];
 733        struct curl_slist *range_header = NULL;
 734
 735        struct active_request_slot *slot;
 736        struct slot_results results;
 737
 738        if (fetch_indices(repo))
 739                return -1;
 740        target = find_sha1_pack(sha1, repo->packs);
 741        if (!target)
 742                return -1;
 743
 744        if (get_verbosely) {
 745                fprintf(stderr, "Getting pack %s\n",
 746                        sha1_to_hex(target->sha1));
 747                fprintf(stderr, " which contains %s\n",
 748                        sha1_to_hex(sha1));
 749        }
 750
 751        url = xmalloc(strlen(repo->base) + 65);
 752        sprintf(url, "%s/objects/pack/pack-%s.pack",
 753                repo->base, sha1_to_hex(target->sha1));
 754
 755        filename = sha1_pack_name(target->sha1);
 756        snprintf(tmpfile, sizeof(tmpfile), "%s.temp", filename);
 757        packfile = fopen(tmpfile, "a");
 758        if (!packfile)
 759                return error("Unable to open local file %s for pack",
 760                             filename);
 761
 762        slot = get_active_slot();
 763        slot->results = &results;
 764        curl_easy_setopt(slot->curl, CURLOPT_FILE, packfile);
 765        curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite);
 766        curl_easy_setopt(slot->curl, CURLOPT_URL, url);
 767        curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, no_pragma_header);
 768        slot->local = packfile;
 769
 770        /* If there is data present from a previous transfer attempt,
 771           resume where it left off */
 772        prev_posn = ftell(packfile);
 773        if (prev_posn>0) {
 774                if (get_verbosely)
 775                        fprintf(stderr,
 776                                "Resuming fetch of pack %s at byte %ld\n",
 777                                sha1_to_hex(target->sha1), prev_posn);
 778                sprintf(range, "Range: bytes=%ld-", prev_posn);
 779                range_header = curl_slist_append(range_header, range);
 780                curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, range_header);
 781        }
 782
 783        if (start_active_slot(slot)) {
 784                run_active_slot(slot);
 785                if (results.curl_result != CURLE_OK) {
 786                        fclose(packfile);
 787                        return error("Unable to get pack file %s\n%s", url,
 788                                     curl_errorstr);
 789                }
 790        } else {
 791                fclose(packfile);
 792                return error("Unable to start request");
 793        }
 794
 795        target->pack_size = ftell(packfile);
 796        fclose(packfile);
 797
 798        ret = move_temp_to_file(tmpfile, filename);
 799        if (ret)
 800                return ret;
 801
 802        lst = &repo->packs;
 803        while (*lst != target)
 804                lst = &((*lst)->next);
 805        *lst = (*lst)->next;
 806
 807        if (verify_pack(target, 0))
 808                return -1;
 809        install_packed_git(target);
 810
 811        return 0;
 812}
 813
 814static void abort_object_request(struct object_request *obj_req)
 815{
 816        if (obj_req->local >= 0) {
 817                close(obj_req->local);
 818                obj_req->local = -1;
 819        }
 820        unlink(obj_req->tmpfile);
 821        if (obj_req->slot) {
 822                release_active_slot(obj_req->slot);
 823                obj_req->slot = NULL;
 824        }
 825        release_object_request(obj_req);
 826}
 827
 828static int fetch_object(struct alt_base *repo, unsigned char *sha1)
 829{
 830        char *hex = sha1_to_hex(sha1);
 831        int ret = 0;
 832        struct object_request *obj_req = object_queue_head;
 833
 834        while (obj_req != NULL && hashcmp(obj_req->sha1, sha1))
 835                obj_req = obj_req->next;
 836        if (obj_req == NULL)
 837                return error("Couldn't find request for %s in the queue", hex);
 838
 839        if (has_sha1_file(obj_req->sha1)) {
 840                abort_object_request(obj_req);
 841                return 0;
 842        }
 843
 844#ifdef USE_CURL_MULTI
 845        while (obj_req->state == WAITING) {
 846                step_active_slots();
 847        }
 848#else
 849        start_object_request(obj_req);
 850#endif
 851
 852        while (obj_req->state == ACTIVE) {
 853                run_active_slot(obj_req->slot);
 854        }
 855        if (obj_req->local != -1) {
 856                close(obj_req->local); obj_req->local = -1;
 857        }
 858
 859        if (obj_req->state == ABORTED) {
 860                ret = error("Request for %s aborted", hex);
 861        } else if (obj_req->curl_result != CURLE_OK &&
 862                   obj_req->http_code != 416) {
 863                if (missing_target(obj_req))
 864                        ret = -1; /* Be silent, it is probably in a pack. */
 865                else
 866                        ret = error("%s (curl_result = %d, http_code = %ld, sha1 = %s)",
 867                                    obj_req->errorstr, obj_req->curl_result,
 868                                    obj_req->http_code, hex);
 869        } else if (obj_req->zret != Z_STREAM_END) {
 870                corrupt_object_found++;
 871                ret = error("File %s (%s) corrupt", hex, obj_req->url);
 872        } else if (hashcmp(obj_req->sha1, obj_req->real_sha1)) {
 873                ret = error("File %s has bad hash", hex);
 874        } else if (obj_req->rename < 0) {
 875                ret = error("unable to write sha1 filename %s",
 876                            obj_req->filename);
 877        }
 878
 879        release_object_request(obj_req);
 880        return ret;
 881}
 882
 883int fetch(unsigned char *sha1)
 884{
 885        struct alt_base *altbase = alt;
 886
 887        if (!fetch_object(altbase, sha1))
 888                return 0;
 889        while (altbase) {
 890                if (!fetch_pack(altbase, sha1))
 891                        return 0;
 892                fetch_alternates(alt->base);
 893                altbase = altbase->next;
 894        }
 895        return error("Unable to find %s under %s", sha1_to_hex(sha1),
 896                     alt->base);
 897}
 898
 899static inline int needs_quote(int ch)
 900{
 901        if (((ch >= 'A') && (ch <= 'Z'))
 902                        || ((ch >= 'a') && (ch <= 'z'))
 903                        || ((ch >= '0') && (ch <= '9'))
 904                        || (ch == '/')
 905                        || (ch == '-')
 906                        || (ch == '.'))
 907                return 0;
 908        return 1;
 909}
 910
 911static inline int hex(int v)
 912{
 913        if (v < 10) return '0' + v;
 914        else return 'A' + v - 10;
 915}
 916
 917static char *quote_ref_url(const char *base, const char *ref)
 918{
 919        const char *cp;
 920        char *dp, *qref;
 921        int len, baselen, ch;
 922
 923        baselen = strlen(base);
 924        len = baselen + 7; /* "/refs/" + NUL */
 925        for (cp = ref; (ch = *cp) != 0; cp++, len++)
 926                if (needs_quote(ch))
 927                        len += 2; /* extra two hex plus replacement % */
 928        qref = xmalloc(len);
 929        memcpy(qref, base, baselen);
 930        memcpy(qref + baselen, "/refs/", 6);
 931        for (cp = ref, dp = qref + baselen + 6; (ch = *cp) != 0; cp++) {
 932                if (needs_quote(ch)) {
 933                        *dp++ = '%';
 934                        *dp++ = hex((ch >> 4) & 0xF);
 935                        *dp++ = hex(ch & 0xF);
 936                }
 937                else
 938                        *dp++ = ch;
 939        }
 940        *dp = 0;
 941
 942        return qref;
 943}
 944
 945int fetch_ref(char *ref, unsigned char *sha1)
 946{
 947        char *url;
 948        char hex[42];
 949        struct buffer buffer;
 950        const char *base = alt->base;
 951        struct active_request_slot *slot;
 952        struct slot_results results;
 953        buffer.size = 41;
 954        buffer.posn = 0;
 955        buffer.buffer = hex;
 956        hex[41] = '\0';
 957
 958        url = quote_ref_url(base, ref);
 959        slot = get_active_slot();
 960        slot->results = &results;
 961        curl_easy_setopt(slot->curl, CURLOPT_FILE, &buffer);
 962        curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite_buffer);
 963        curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, NULL);
 964        curl_easy_setopt(slot->curl, CURLOPT_URL, url);
 965        if (start_active_slot(slot)) {
 966                run_active_slot(slot);
 967                if (results.curl_result != CURLE_OK)
 968                        return error("Couldn't get %s for %s\n%s",
 969                                     url, ref, curl_errorstr);
 970        } else {
 971                return error("Unable to start request");
 972        }
 973
 974        hex[40] = '\0';
 975        get_sha1_hex(hex, sha1);
 976        return 0;
 977}
 978
 979int main(int argc, const char **argv)
 980{
 981        int commits;
 982        const char **write_ref = NULL;
 983        char **commit_id;
 984        const char *url;
 985        char *s;
 986        int arg = 1;
 987        int rc = 0;
 988
 989        setup_git_directory();
 990        git_config(git_default_config);
 991
 992        while (arg < argc && argv[arg][0] == '-') {
 993                if (argv[arg][1] == 't') {
 994                        get_tree = 1;
 995                } else if (argv[arg][1] == 'c') {
 996                        get_history = 1;
 997                } else if (argv[arg][1] == 'a') {
 998                        get_all = 1;
 999                        get_tree = 1;
1000                        get_history = 1;
1001                } else if (argv[arg][1] == 'v') {
1002                        get_verbosely = 1;
1003                } else if (argv[arg][1] == 'w') {
1004                        write_ref = &argv[arg + 1];
1005                        arg++;
1006                } else if (!strcmp(argv[arg], "--recover")) {
1007                        get_recover = 1;
1008                } else if (!strcmp(argv[arg], "--stdin")) {
1009                        commits_on_stdin = 1;
1010                }
1011                arg++;
1012        }
1013        if (argc < arg + 2 - commits_on_stdin) {
1014                usage("git-http-fetch [-c] [-t] [-a] [-v] [--recover] [-w ref] [--stdin] commit-id url");
1015                return 1;
1016        }
1017        if (commits_on_stdin) {
1018                commits = pull_targets_stdin(&commit_id, &write_ref);
1019        } else {
1020                commit_id = (char **) &argv[arg++];
1021                commits = 1;
1022        }
1023        url = argv[arg];
1024
1025        http_init();
1026
1027        no_pragma_header = curl_slist_append(no_pragma_header, "Pragma:");
1028
1029        alt = xmalloc(sizeof(*alt));
1030        alt->base = xmalloc(strlen(url) + 1);
1031        strcpy(alt->base, url);
1032        for (s = alt->base + strlen(alt->base) - 1; *s == '/'; --s)
1033                *s = 0;
1034        alt->got_indices = 0;
1035        alt->packs = NULL;
1036        alt->next = NULL;
1037
1038        if (pull(commits, commit_id, write_ref, url))
1039                rc = 1;
1040
1041        http_cleanup();
1042
1043        curl_slist_free_all(no_pragma_header);
1044
1045        if (commits_on_stdin)
1046                pull_targets_free(commits, commit_id, write_ref);
1047
1048        if (corrupt_object_found) {
1049                fprintf(stderr,
1050"Some loose object were found to be corrupt, but they might be just\n"
1051"a false '404 Not Found' error message sent with incorrect HTTP\n"
1052"status code.  Suggest running git-fsck.\n");
1053        }
1054        return rc;
1055}