Andrew's git / gitweb.git / http-fetch.c
?
http-fetch.con commit Ignore *.exe files (11b1a5d)
   1#include "cache.h"
   2#include "commit.h"
   3
   4#include "fetch.h"
   5
   6#include <curl/curl.h>
   7#include <curl/easy.h>
   8
   9#if LIBCURL_VERSION_NUM < 0x070704
  10#define curl_global_cleanup() do { /* nothing */ } while(0)
  11#endif
  12#if LIBCURL_VERSION_NUM < 0x070800
  13#define curl_global_init(a) do { /* nothing */ } while(0)
  14#endif
  15
  16static CURL *curl;
  17static struct curl_slist *no_pragma_header;
  18static char curl_errorstr[CURL_ERROR_SIZE];
  19
  20static char *initial_base;
  21
  22struct alt_base
  23{
  24        char *base;
  25        int got_indices;
  26        struct packed_git *packs;
  27        struct alt_base *next;
  28};
  29
  30static struct alt_base *alt = NULL;
  31
  32static SHA_CTX c;
  33static z_stream stream;
  34
  35static int local;
  36static int zret;
  37
  38static int curl_ssl_verify;
  39static char *ssl_cert;
  40static char *ssl_key;
  41static char *ssl_capath;
  42static char *ssl_cainfo;
  43
  44struct buffer
  45{
  46        size_t posn;
  47        size_t size;
  48        void *buffer;
  49};
  50
  51static size_t fwrite_buffer(void *ptr, size_t eltsize, size_t nmemb,
  52                            struct buffer *buffer)
  53{
  54        size_t size = eltsize * nmemb;
  55        if (size > buffer->size - buffer->posn)
  56                size = buffer->size - buffer->posn;
  57        memcpy(buffer->buffer + buffer->posn, ptr, size);
  58        buffer->posn += size;
  59        return size;
  60}
  61
  62static size_t fwrite_sha1_file(void *ptr, size_t eltsize, size_t nmemb,
  63                               void *data)
  64{
  65        unsigned char expn[4096];
  66        size_t size = eltsize * nmemb;
  67        int posn = 0;
  68        do {
  69                ssize_t retval = write(local, ptr + posn, size - posn);
  70                if (retval < 0)
  71                        return posn;
  72                posn += retval;
  73        } while (posn < size);
  74
  75        stream.avail_in = size;
  76        stream.next_in = ptr;
  77        do {
  78                stream.next_out = expn;
  79                stream.avail_out = sizeof(expn);
  80                zret = inflate(&stream, Z_SYNC_FLUSH);
  81                SHA1_Update(&c, expn, sizeof(expn) - stream.avail_out);
  82        } while (stream.avail_in && zret == Z_OK);
  83        return size;
  84}
  85
  86void prefetch(unsigned char *sha1)
  87{
  88}
  89
  90static int got_alternates = 0;
  91
  92static int fetch_index(struct alt_base *repo, unsigned char *sha1)
  93{
  94        char *filename;
  95        char *url;
  96
  97        FILE *indexfile;
  98
  99        if (has_pack_index(sha1))
 100                return 0;
 101
 102        if (get_verbosely)
 103                fprintf(stderr, "Getting index for pack %s\n",
 104                        sha1_to_hex(sha1));
 105        
 106        url = xmalloc(strlen(repo->base) + 64);
 107        sprintf(url, "%s/objects/pack/pack-%s.idx",
 108                repo->base, sha1_to_hex(sha1));
 109        
 110        filename = sha1_pack_index_name(sha1);
 111        indexfile = fopen(filename, "w");
 112        if (!indexfile)
 113                return error("Unable to open local file %s for pack index",
 114                             filename);
 115
 116        curl_easy_setopt(curl, CURLOPT_FILE, indexfile);
 117        curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, fwrite);
 118        curl_easy_setopt(curl, CURLOPT_URL, url);
 119        curl_easy_setopt(curl, CURLOPT_HTTPHEADER, no_pragma_header);
 120        curl_easy_setopt(curl, CURLOPT_ERRORBUFFER, curl_errorstr);
 121        
 122        if (curl_easy_perform(curl)) {
 123                fclose(indexfile);
 124                return error("Unable to get pack index %s\n%s", url,
 125                             curl_errorstr);
 126        }
 127
 128        fclose(indexfile);
 129        return 0;
 130}
 131
 132static int setup_index(struct alt_base *repo, unsigned char *sha1)
 133{
 134        struct packed_git *new_pack;
 135        if (has_pack_file(sha1))
 136                return 0; // don't list this as something we can get
 137
 138        if (fetch_index(repo, sha1))
 139                return -1;
 140
 141        new_pack = parse_pack_index(sha1);
 142        new_pack->next = repo->packs;
 143        repo->packs = new_pack;
 144        return 0;
 145}
 146
 147static int fetch_alternates(char *base)
 148{
 149        int ret = 0;
 150        struct buffer buffer;
 151        char *url;
 152        char *data;
 153        int i = 0;
 154        int http_specific = 1;
 155        if (got_alternates)
 156                return 0;
 157        data = xmalloc(4096);
 158        buffer.size = 4095;
 159        buffer.posn = 0;
 160        buffer.buffer = data;
 161
 162        if (get_verbosely)
 163                fprintf(stderr, "Getting alternates list\n");
 164        
 165        url = xmalloc(strlen(base) + 31);
 166        sprintf(url, "%s/objects/info/http-alternates", base);
 167
 168        curl_easy_setopt(curl, CURLOPT_FILE, &buffer);
 169        curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, fwrite_buffer);
 170        curl_easy_setopt(curl, CURLOPT_URL, url);
 171
 172        if (curl_easy_perform(curl) || !buffer.posn) {
 173                http_specific = 0;
 174
 175                sprintf(url, "%s/objects/info/alternates", base);
 176                
 177                curl_easy_setopt(curl, CURLOPT_FILE, &buffer);
 178                curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, fwrite_buffer);
 179                curl_easy_setopt(curl, CURLOPT_URL, url);
 180                
 181                if (curl_easy_perform(curl)) {
 182                        return 0;
 183                }
 184        }
 185
 186        data[buffer.posn] = '\0';
 187
 188        while (i < buffer.posn) {
 189                int posn = i;
 190                while (posn < buffer.posn && data[posn] != '\n')
 191                        posn++;
 192                if (data[posn] == '\n') {
 193                        int okay = 0;
 194                        int serverlen = 0;
 195                        struct alt_base *newalt;
 196                        char *target = NULL;
 197                        if (data[i] == '/') {
 198                                serverlen = strchr(base + 8, '/') - base;
 199                                okay = 1;
 200                        } else if (!memcmp(data + i, "../", 3)) {
 201                                i += 3;
 202                                serverlen = strlen(base);
 203                                while (i + 2 < posn && 
 204                                       !memcmp(data + i, "../", 3)) {
 205                                        do {
 206                                                serverlen--;
 207                                        } while (serverlen &&
 208                                                 base[serverlen - 1] != '/');
 209                                        i += 3;
 210                                }
 211                                // If the server got removed, give up.
 212                                okay = strchr(base, ':') - base + 3 < 
 213                                        serverlen;
 214                        } else if (http_specific) {
 215                                char *colon = strchr(data + i, ':');
 216                                char *slash = strchr(data + i, '/');
 217                                if (colon && slash && colon < data + posn &&
 218                                    slash < data + posn && colon < slash) {
 219                                        okay = 1;
 220                                }
 221                        }
 222                        // skip 'objects' at end
 223                        if (okay) {
 224                                target = xmalloc(serverlen + posn - i - 6);
 225                                strncpy(target, base, serverlen);
 226                                strncpy(target + serverlen, data + i,
 227                                        posn - i - 7);
 228                                target[serverlen + posn - i - 7] = '\0';
 229                                if (get_verbosely)
 230                                        fprintf(stderr, 
 231                                                "Also look at %s\n", target);
 232                                newalt = xmalloc(sizeof(*newalt));
 233                                newalt->next = alt;
 234                                newalt->base = target;
 235                                newalt->got_indices = 0;
 236                                newalt->packs = NULL;
 237                                alt = newalt;
 238                                ret++;
 239                        }
 240                }
 241                i = posn + 1;
 242        }
 243        got_alternates = 1;
 244        
 245        return ret;
 246}
 247
 248static int fetch_indices(struct alt_base *repo)
 249{
 250        unsigned char sha1[20];
 251        char *url;
 252        struct buffer buffer;
 253        char *data;
 254        int i = 0;
 255
 256        if (repo->got_indices)
 257                return 0;
 258
 259        data = xmalloc(4096);
 260        buffer.size = 4096;
 261        buffer.posn = 0;
 262        buffer.buffer = data;
 263
 264        if (get_verbosely)
 265                fprintf(stderr, "Getting pack list\n");
 266        
 267        url = xmalloc(strlen(repo->base) + 21);
 268        sprintf(url, "%s/objects/info/packs", repo->base);
 269
 270        curl_easy_setopt(curl, CURLOPT_FILE, &buffer);
 271        curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, fwrite_buffer);
 272        curl_easy_setopt(curl, CURLOPT_URL, url);
 273        curl_easy_setopt(curl, CURLOPT_HTTPHEADER, NULL);
 274        curl_easy_setopt(curl, CURLOPT_ERRORBUFFER, curl_errorstr);
 275        
 276        if (curl_easy_perform(curl))
 277                return error("%s", curl_errorstr);
 278
 279        while (i < buffer.posn) {
 280                switch (data[i]) {
 281                case 'P':
 282                        i++;
 283                        if (i + 52 < buffer.posn &&
 284                            !strncmp(data + i, " pack-", 6) &&
 285                            !strncmp(data + i + 46, ".pack\n", 6)) {
 286                                get_sha1_hex(data + i + 6, sha1);
 287                                setup_index(repo, sha1);
 288                                i += 51;
 289                                break;
 290                        }
 291                default:
 292                        while (data[i] != '\n')
 293                                i++;
 294                }
 295                i++;
 296        }
 297
 298        repo->got_indices = 1;
 299        return 0;
 300}
 301
 302static int fetch_pack(struct alt_base *repo, unsigned char *sha1)
 303{
 304        char *url;
 305        struct packed_git *target;
 306        struct packed_git **lst;
 307        FILE *packfile;
 308        char *filename;
 309
 310        if (fetch_indices(repo))
 311                return -1;
 312        target = find_sha1_pack(sha1, repo->packs);
 313        if (!target)
 314                return -1;
 315
 316        if (get_verbosely) {
 317                fprintf(stderr, "Getting pack %s\n",
 318                        sha1_to_hex(target->sha1));
 319                fprintf(stderr, " which contains %s\n",
 320                        sha1_to_hex(sha1));
 321        }
 322
 323        url = xmalloc(strlen(repo->base) + 65);
 324        sprintf(url, "%s/objects/pack/pack-%s.pack",
 325                repo->base, sha1_to_hex(target->sha1));
 326
 327        filename = sha1_pack_name(target->sha1);
 328        packfile = fopen(filename, "w");
 329        if (!packfile)
 330                return error("Unable to open local file %s for pack",
 331                             filename);
 332
 333        curl_easy_setopt(curl, CURLOPT_FILE, packfile);
 334        curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, fwrite);
 335        curl_easy_setopt(curl, CURLOPT_URL, url);
 336        curl_easy_setopt(curl, CURLOPT_HTTPHEADER, no_pragma_header);
 337        curl_easy_setopt(curl, CURLOPT_ERRORBUFFER, curl_errorstr);
 338
 339        if (curl_easy_perform(curl)) {
 340                fclose(packfile);
 341                return error("Unable to get pack file %s\n%s", url,
 342                             curl_errorstr);
 343        }
 344
 345        fclose(packfile);
 346
 347        lst = &repo->packs;
 348        while (*lst != target)
 349                lst = &((*lst)->next);
 350        *lst = (*lst)->next;
 351
 352        install_packed_git(target);
 353
 354        return 0;
 355}
 356
 357static int fetch_object(struct alt_base *repo, unsigned char *sha1)
 358{
 359        char *hex = sha1_to_hex(sha1);
 360        char *filename = sha1_file_name(sha1);
 361        unsigned char real_sha1[20];
 362        char tmpfile[PATH_MAX];
 363        int ret;
 364        char *url;
 365        char *posn;
 366
 367        snprintf(tmpfile, sizeof(tmpfile), "%s/obj_XXXXXX",
 368                 get_object_directory());
 369
 370        local = mkstemp(tmpfile);
 371        if (local < 0)
 372                return error("Couldn't create temporary file %s for %s: %s\n",
 373                             tmpfile, filename, strerror(errno));
 374
 375        memset(&stream, 0, sizeof(stream));
 376
 377        inflateInit(&stream);
 378
 379        SHA1_Init(&c);
 380
 381        curl_easy_setopt(curl, CURLOPT_FAILONERROR, 1);
 382        curl_easy_setopt(curl, CURLOPT_FILE, NULL);
 383        curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, fwrite_sha1_file);
 384        curl_easy_setopt(curl, CURLOPT_HTTPHEADER, no_pragma_header);
 385        curl_easy_setopt(curl, CURLOPT_ERRORBUFFER, curl_errorstr);
 386
 387        url = xmalloc(strlen(repo->base) + 50);
 388        strcpy(url, repo->base);
 389        posn = url + strlen(repo->base);
 390        strcpy(posn, "objects/");
 391        posn += 8;
 392        memcpy(posn, hex, 2);
 393        posn += 2;
 394        *(posn++) = '/';
 395        strcpy(posn, hex + 2);
 396
 397        curl_easy_setopt(curl, CURLOPT_URL, url);
 398
 399        if (curl_easy_perform(curl)) {
 400                unlink(filename);
 401                return error("%s", curl_errorstr);
 402        }
 403
 404        fchmod(local, 0444);
 405        close(local);
 406        inflateEnd(&stream);
 407        SHA1_Final(real_sha1, &c);
 408        if (zret != Z_STREAM_END) {
 409                unlink(tmpfile);
 410                return error("File %s (%s) corrupt\n", hex, url);
 411        }
 412        if (memcmp(sha1, real_sha1, 20)) {
 413                unlink(tmpfile);
 414                return error("File %s has bad hash\n", hex);
 415        }
 416        ret = link(tmpfile, filename);
 417        if (ret < 0) {
 418                /* Same Coda hack as in write_sha1_file(sha1_file.c) */
 419                ret = errno;
 420                if (ret == EXDEV && !rename(tmpfile, filename))
 421                        goto out;
 422        }
 423        unlink(tmpfile);
 424        if (ret) {
 425                if (ret != EEXIST)
 426                        return error("unable to write sha1 filename %s: %s",
 427                                     filename, strerror(ret));
 428        }
 429 out:
 430        pull_say("got %s\n", hex);
 431        return 0;
 432}
 433
 434int fetch(unsigned char *sha1)
 435{
 436        struct alt_base *altbase = alt;
 437        while (altbase) {
 438                if (!fetch_object(altbase, sha1))
 439                        return 0;
 440                if (!fetch_pack(altbase, sha1))
 441                        return 0;
 442                if (fetch_alternates(altbase->base) > 0) {
 443                        altbase = alt;
 444                        continue;
 445                }
 446                altbase = altbase->next;
 447        }
 448        return error("Unable to find %s under %s\n", sha1_to_hex(sha1), 
 449                     initial_base);
 450}
 451
 452int fetch_ref(char *ref, unsigned char *sha1)
 453{
 454        char *url, *posn;
 455        char hex[42];
 456        struct buffer buffer;
 457        char *base = initial_base;
 458        buffer.size = 41;
 459        buffer.posn = 0;
 460        buffer.buffer = hex;
 461        hex[41] = '\0';
 462        
 463        curl_easy_setopt(curl, CURLOPT_FILE, &buffer);
 464        curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, fwrite_buffer);
 465        curl_easy_setopt(curl, CURLOPT_HTTPHEADER, NULL);
 466        curl_easy_setopt(curl, CURLOPT_ERRORBUFFER, curl_errorstr);
 467
 468        url = xmalloc(strlen(base) + 6 + strlen(ref));
 469        strcpy(url, base);
 470        posn = url + strlen(base);
 471        strcpy(posn, "refs/");
 472        posn += 5;
 473        strcpy(posn, ref);
 474
 475        curl_easy_setopt(curl, CURLOPT_URL, url);
 476
 477        if (curl_easy_perform(curl))
 478                return error("Couldn't get %s for %s\n%s",
 479                             url, ref, curl_errorstr);
 480
 481        hex[40] = '\0';
 482        get_sha1_hex(hex, sha1);
 483        return 0;
 484}
 485
 486int main(int argc, char **argv)
 487{
 488        char *commit_id;
 489        char *url;
 490        int arg = 1;
 491
 492        while (arg < argc && argv[arg][0] == '-') {
 493                if (argv[arg][1] == 't') {
 494                        get_tree = 1;
 495                } else if (argv[arg][1] == 'c') {
 496                        get_history = 1;
 497                } else if (argv[arg][1] == 'a') {
 498                        get_all = 1;
 499                        get_tree = 1;
 500                        get_history = 1;
 501                } else if (argv[arg][1] == 'v') {
 502                        get_verbosely = 1;
 503                } else if (argv[arg][1] == 'w') {
 504                        write_ref = argv[arg + 1];
 505                        arg++;
 506                } else if (!strcmp(argv[arg], "--recover")) {
 507                        get_recover = 1;
 508                }
 509                arg++;
 510        }
 511        if (argc < arg + 2) {
 512                usage("git-http-fetch [-c] [-t] [-a] [-d] [-v] [--recover] [-w ref] commit-id url");
 513                return 1;
 514        }
 515        commit_id = argv[arg];
 516        url = argv[arg + 1];
 517
 518        curl_global_init(CURL_GLOBAL_ALL);
 519
 520        curl = curl_easy_init();
 521        no_pragma_header = curl_slist_append(no_pragma_header, "Pragma:");
 522
 523        curl_ssl_verify = getenv("GIT_SSL_NO_VERIFY") ? 0 : 1;
 524        curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, curl_ssl_verify);
 525#if LIBCURL_VERSION_NUM >= 0x070907
 526        curl_easy_setopt(curl, CURLOPT_NETRC, CURL_NETRC_OPTIONAL);
 527#endif
 528
 529        if ((ssl_cert = getenv("GIT_SSL_CERT")) != NULL) {
 530                curl_easy_setopt(curl, CURLOPT_SSLCERT, ssl_cert);
 531        }
 532        if ((ssl_key = getenv("GIT_SSL_KEY")) != NULL) {
 533                curl_easy_setopt(curl, CURLOPT_SSLKEY, ssl_key);
 534        }
 535#if LIBCURL_VERSION_NUM >= 0x070908
 536        if ((ssl_capath = getenv("GIT_SSL_CAPATH")) != NULL) {
 537                curl_easy_setopt(curl, CURLOPT_CAPATH, ssl_capath);
 538        }
 539#endif
 540        if ((ssl_cainfo = getenv("GIT_SSL_CAINFO")) != NULL) {
 541                curl_easy_setopt(curl, CURLOPT_CAINFO, ssl_cainfo);
 542        }
 543
 544        alt = xmalloc(sizeof(*alt));
 545        alt->base = url;
 546        alt->got_indices = 0;
 547        alt->packs = NULL;
 548        alt->next = NULL;
 549        initial_base = url;
 550
 551        if (pull(commit_id))
 552                return 1;
 553
 554        curl_slist_free_all(no_pragma_header);
 555        curl_global_cleanup();
 556        return 0;
 557}