Andrew's git / gitweb.git / http-fetch.c
?
http-fetch.con commit [PATCH] git-http-fetch: Allow caching of retrieved objects by proxy servers (1db69b5)
   1#include "cache.h"
   2#include "commit.h"
   3
   4#include "fetch.h"
   5
   6#include <curl/curl.h>
   7#include <curl/easy.h>
   8
   9#if LIBCURL_VERSION_NUM < 0x070704
  10#define curl_global_cleanup() do { /* nothing */ } while(0)
  11#endif
  12#if LIBCURL_VERSION_NUM < 0x070800
  13#define curl_global_init(a) do { /* nothing */ } while(0)
  14#endif
  15
  16static CURL *curl;
  17static struct curl_slist *no_pragma_header;
  18
  19static char *base;
  20
  21static SHA_CTX c;
  22static z_stream stream;
  23
  24static int local;
  25static int zret;
  26
  27static int curl_ssl_verify;
  28
  29struct buffer
  30{
  31        size_t posn;
  32        size_t size;
  33        void *buffer;
  34};
  35
  36static size_t fwrite_buffer(void *ptr, size_t eltsize, size_t nmemb,
  37                            struct buffer *buffer)
  38{
  39        size_t size = eltsize * nmemb;
  40        if (size > buffer->size - buffer->posn)
  41                size = buffer->size - buffer->posn;
  42        memcpy(buffer->buffer + buffer->posn, ptr, size);
  43        buffer->posn += size;
  44        return size;
  45}
  46
  47static size_t fwrite_sha1_file(void *ptr, size_t eltsize, size_t nmemb,
  48                               void *data)
  49{
  50        unsigned char expn[4096];
  51        size_t size = eltsize * nmemb;
  52        int posn = 0;
  53        do {
  54                ssize_t retval = write(local, ptr + posn, size - posn);
  55                if (retval < 0)
  56                        return posn;
  57                posn += retval;
  58        } while (posn < size);
  59
  60        stream.avail_in = size;
  61        stream.next_in = ptr;
  62        do {
  63                stream.next_out = expn;
  64                stream.avail_out = sizeof(expn);
  65                zret = inflate(&stream, Z_SYNC_FLUSH);
  66                SHA1_Update(&c, expn, sizeof(expn) - stream.avail_out);
  67        } while (stream.avail_in && zret == Z_OK);
  68        return size;
  69}
  70
  71void prefetch(unsigned char *sha1)
  72{
  73}
  74
  75static int got_indices = 0;
  76
  77static struct packed_git *packs = NULL;
  78
  79static int fetch_index(unsigned char *sha1)
  80{
  81        char *filename;
  82        char *url;
  83
  84        FILE *indexfile;
  85
  86        if (has_pack_index(sha1))
  87                return 0;
  88
  89        if (get_verbosely)
  90                fprintf(stderr, "Getting index for pack %s\n",
  91                        sha1_to_hex(sha1));
  92        
  93        url = xmalloc(strlen(base) + 64);
  94        sprintf(url, "%s/objects/pack/pack-%s.idx",
  95                base, sha1_to_hex(sha1));
  96        
  97        filename = sha1_pack_index_name(sha1);
  98        indexfile = fopen(filename, "w");
  99        if (!indexfile)
 100                return error("Unable to open local file %s for pack index",
 101                             filename);
 102
 103        curl_easy_setopt(curl, CURLOPT_FILE, indexfile);
 104        curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, fwrite);
 105        curl_easy_setopt(curl, CURLOPT_URL, url);
 106        curl_easy_setopt(curl, CURLOPT_HTTPHEADER, no_pragma_header);
 107        
 108        if (curl_easy_perform(curl)) {
 109                fclose(indexfile);
 110                return error("Unable to get pack index %s", url);
 111        }
 112
 113        fclose(indexfile);
 114        return 0;
 115}
 116
 117static int setup_index(unsigned char *sha1)
 118{
 119        struct packed_git *new_pack;
 120        if (has_pack_file(sha1))
 121                return 0; // don't list this as something we can get
 122
 123        if (fetch_index(sha1))
 124                return -1;
 125
 126        new_pack = parse_pack_index(sha1);
 127        new_pack->next = packs;
 128        packs = new_pack;
 129        return 0;
 130}
 131
 132static int fetch_indices(void)
 133{
 134        unsigned char sha1[20];
 135        char *url;
 136        struct buffer buffer;
 137        char *data;
 138        int i = 0;
 139
 140        if (got_indices)
 141                return 0;
 142
 143        data = xmalloc(4096);
 144        buffer.size = 4096;
 145        buffer.posn = 0;
 146        buffer.buffer = data;
 147
 148        if (get_verbosely)
 149                fprintf(stderr, "Getting pack list\n");
 150        
 151        url = xmalloc(strlen(base) + 21);
 152        sprintf(url, "%s/objects/info/packs", base);
 153
 154        curl_easy_setopt(curl, CURLOPT_FILE, &buffer);
 155        curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, fwrite_buffer);
 156        curl_easy_setopt(curl, CURLOPT_URL, url);
 157        curl_easy_setopt(curl, CURLOPT_HTTPHEADER, NULL);
 158        
 159        if (curl_easy_perform(curl)) {
 160                return error("Unable to get pack index %s", url);
 161        }
 162
 163        do {
 164                switch (data[i]) {
 165                case 'P':
 166                        i++;
 167                        if (i + 52 < buffer.posn &&
 168                            !strncmp(data + i, " pack-", 6) &&
 169                            !strncmp(data + i + 46, ".pack\n", 6)) {
 170                                get_sha1_hex(data + i + 6, sha1);
 171                                setup_index(sha1);
 172                                i += 51;
 173                                break;
 174                        }
 175                default:
 176                        while (data[i] != '\n')
 177                                i++;
 178                }
 179                i++;
 180        } while (i < buffer.posn);
 181
 182        got_indices = 1;
 183        return 0;
 184}
 185
 186static int fetch_pack(unsigned char *sha1)
 187{
 188        char *url;
 189        struct packed_git *target;
 190        struct packed_git **lst;
 191        FILE *packfile;
 192        char *filename;
 193
 194        if (fetch_indices())
 195                return -1;
 196        target = find_sha1_pack(sha1, packs);
 197        if (!target)
 198                return error("Couldn't get %s: not separate or in any pack",
 199                             sha1_to_hex(sha1));
 200
 201        if (get_verbosely) {
 202                fprintf(stderr, "Getting pack %s\n",
 203                        sha1_to_hex(target->sha1));
 204                fprintf(stderr, " which contains %s\n",
 205                        sha1_to_hex(sha1));
 206        }
 207
 208        url = xmalloc(strlen(base) + 65);
 209        sprintf(url, "%s/objects/pack/pack-%s.pack",
 210                base, sha1_to_hex(target->sha1));
 211
 212        filename = sha1_pack_name(target->sha1);
 213        packfile = fopen(filename, "w");
 214        if (!packfile)
 215                return error("Unable to open local file %s for pack",
 216                             filename);
 217
 218        curl_easy_setopt(curl, CURLOPT_FILE, packfile);
 219        curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, fwrite);
 220        curl_easy_setopt(curl, CURLOPT_URL, url);
 221        curl_easy_setopt(curl, CURLOPT_HTTPHEADER, no_pragma_header);
 222        
 223        if (curl_easy_perform(curl)) {
 224                fclose(packfile);
 225                return error("Unable to get pack file %s", url);
 226        }
 227
 228        fclose(packfile);
 229
 230        lst = &packs;
 231        while (*lst != target)
 232                lst = &((*lst)->next);
 233        *lst = (*lst)->next;
 234
 235        install_packed_git(target);
 236
 237        return 0;
 238}
 239
 240int fetch(unsigned char *sha1)
 241{
 242        char *hex = sha1_to_hex(sha1);
 243        char *filename = sha1_file_name(sha1);
 244        unsigned char real_sha1[20];
 245        char *url;
 246        char *posn;
 247
 248        local = open(filename, O_WRONLY | O_CREAT | O_EXCL, 0666);
 249
 250        if (local < 0)
 251                return error("Couldn't open local object %s\n", filename);
 252
 253        memset(&stream, 0, sizeof(stream));
 254
 255        inflateInit(&stream);
 256
 257        SHA1_Init(&c);
 258
 259        curl_easy_setopt(curl, CURLOPT_FAILONERROR, 1);
 260        curl_easy_setopt(curl, CURLOPT_FILE, NULL);
 261        curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, fwrite_sha1_file);
 262        curl_easy_setopt(curl, CURLOPT_HTTPHEADER, no_pragma_header);
 263
 264        url = xmalloc(strlen(base) + 50);
 265        strcpy(url, base);
 266        posn = url + strlen(base);
 267        strcpy(posn, "objects/");
 268        posn += 8;
 269        memcpy(posn, hex, 2);
 270        posn += 2;
 271        *(posn++) = '/';
 272        strcpy(posn, hex + 2);
 273
 274        curl_easy_setopt(curl, CURLOPT_URL, url);
 275
 276        if (curl_easy_perform(curl)) {
 277                unlink(filename);
 278                if (fetch_pack(sha1))
 279                        return error("Tried %s", url);
 280                return 0;
 281        }
 282
 283        close(local);
 284        inflateEnd(&stream);
 285        SHA1_Final(real_sha1, &c);
 286        if (zret != Z_STREAM_END) {
 287                unlink(filename);
 288                return error("File %s (%s) corrupt\n", hex, url);
 289        }
 290        if (memcmp(sha1, real_sha1, 20)) {
 291                unlink(filename);
 292                return error("File %s has bad hash\n", hex);
 293        }
 294        
 295        pull_say("got %s\n", hex);
 296        return 0;
 297}
 298
 299int fetch_ref(char *ref, unsigned char *sha1)
 300{
 301        char *url, *posn;
 302        char hex[42];
 303        struct buffer buffer;
 304        buffer.size = 41;
 305        buffer.posn = 0;
 306        buffer.buffer = hex;
 307        hex[41] = '\0';
 308        
 309        curl_easy_setopt(curl, CURLOPT_FILE, &buffer);
 310        curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, fwrite_buffer);
 311        curl_easy_setopt(curl, CURLOPT_HTTPHEADER, NULL);
 312
 313        url = xmalloc(strlen(base) + 6 + strlen(ref));
 314        strcpy(url, base);
 315        posn = url + strlen(base);
 316        strcpy(posn, "refs/");
 317        posn += 5;
 318        strcpy(posn, ref);
 319
 320        curl_easy_setopt(curl, CURLOPT_URL, url);
 321
 322        if (curl_easy_perform(curl))
 323                return error("Couldn't get %s for %s\n", url, ref);
 324
 325        hex[40] = '\0';
 326        get_sha1_hex(hex, sha1);
 327        return 0;
 328}
 329
 330int main(int argc, char **argv)
 331{
 332        char *commit_id;
 333        char *url;
 334        int arg = 1;
 335
 336        while (arg < argc && argv[arg][0] == '-') {
 337                if (argv[arg][1] == 't') {
 338                        get_tree = 1;
 339                } else if (argv[arg][1] == 'c') {
 340                        get_history = 1;
 341                } else if (argv[arg][1] == 'a') {
 342                        get_all = 1;
 343                        get_tree = 1;
 344                        get_history = 1;
 345                } else if (argv[arg][1] == 'v') {
 346                        get_verbosely = 1;
 347                } else if (argv[arg][1] == 'w') {
 348                        write_ref = argv[arg + 1];
 349                        arg++;
 350                }
 351                arg++;
 352        }
 353        if (argc < arg + 2) {
 354                usage("git-http-fetch [-c] [-t] [-a] [-d] [-v] [--recover] [-w ref] commit-id url");
 355                return 1;
 356        }
 357        commit_id = argv[arg];
 358        url = argv[arg + 1];
 359
 360        curl_global_init(CURL_GLOBAL_ALL);
 361
 362        curl = curl_easy_init();
 363        no_pragma_header = curl_slist_append(no_pragma_header, "Pragma:");
 364
 365        curl_ssl_verify = getenv("GIT_SSL_NO_VERIFY") ? 0 : 1;
 366        curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, curl_ssl_verify);
 367#if LIBCURL_VERSION_NUM >= 0x070907
 368        curl_easy_setopt(curl, CURLOPT_NETRC, CURL_NETRC_OPTIONAL);
 369#endif
 370
 371        base = url;
 372
 373        if (pull(commit_id))
 374                return 1;
 375
 376        curl_slist_free_all(no_pragma_header);
 377        curl_global_cleanup();
 378        return 0;
 379}