1#include "cache.h"
2#include "commit.h"
3#include "pack.h"
4#include "fetch.h"
5
6#include <curl/curl.h>
7#include <curl/easy.h>
8
9#if LIBCURL_VERSION_NUM >= 0x070908
10#define USE_CURL_MULTI
11#define DEFAULT_MAX_REQUESTS 5
12#endif
13
14#if LIBCURL_VERSION_NUM < 0x070704
15#define curl_global_cleanup() do { /* nothing */ } while(0)
16#endif
17#if LIBCURL_VERSION_NUM < 0x070800
18#define curl_global_init(a) do { /* nothing */ } while(0)
19#endif
20
21#if LIBCURL_VERSION_NUM < 0x070c04
22#define NO_CURL_EASY_DUPHANDLE
23#endif
24
25#define PREV_BUF_SIZE 4096
26#define RANGE_HEADER_SIZE 30
27
28static int active_requests = 0;
29static int data_received;
30
31#ifdef USE_CURL_MULTI
32static int max_requests = -1;
33static CURLM *curlm;
34#endif
35#ifndef NO_CURL_EASY_DUPHANDLE
36static CURL *curl_default;
37#endif
38static struct curl_slist *pragma_header;
39static struct curl_slist *no_pragma_header;
40static struct curl_slist *no_range_header;
41static char curl_errorstr[CURL_ERROR_SIZE];
42
43struct alt_base
44{
45 char *base;
46 int got_indices;
47 struct packed_git *packs;
48 struct alt_base *next;
49};
50
51static struct alt_base *alt = NULL;
52
53enum transfer_state {
54 WAITING,
55 ABORTED,
56 ACTIVE,
57 COMPLETE,
58};
59
60struct transfer_request
61{
62 unsigned char sha1[20];
63 struct alt_base *repo;
64 char *url;
65 char filename[PATH_MAX];
66 char tmpfile[PATH_MAX];
67 int local;
68 enum transfer_state state;
69 CURLcode curl_result;
70 char errorstr[CURL_ERROR_SIZE];
71 long http_code;
72 unsigned char real_sha1[20];
73 SHA_CTX c;
74 z_stream stream;
75 int zret;
76 int rename;
77 struct active_request_slot *slot;
78 struct transfer_request *next;
79};
80
81struct active_request_slot
82{
83 CURL *curl;
84 FILE *local;
85 int in_use;
86 int done;
87 CURLcode curl_result;
88 struct active_request_slot *next;
89};
90
91static struct transfer_request *request_queue_head = NULL;
92static struct active_request_slot *active_queue_head = NULL;
93
94static int curl_ssl_verify = -1;
95static char *ssl_cert = NULL;
96#if LIBCURL_VERSION_NUM >= 0x070902
97static char *ssl_key = NULL;
98#endif
99#if LIBCURL_VERSION_NUM >= 0x070908
100static char *ssl_capath = NULL;
101#endif
102static char *ssl_cainfo = NULL;
103
104struct buffer
105{
106 size_t posn;
107 size_t size;
108 void *buffer;
109};
110
111static int http_options(const char *var, const char *value)
112{
113 if (!strcmp("http.sslverify", var)) {
114 if (curl_ssl_verify == -1) {
115 curl_ssl_verify = git_config_bool(var, value);
116 }
117 return 0;
118 }
119
120 if (!strcmp("http.sslcert", var)) {
121 if (ssl_cert == NULL) {
122 ssl_cert = xmalloc(strlen(value)+1);
123 strcpy(ssl_cert, value);
124 }
125 return 0;
126 }
127#if LIBCURL_VERSION_NUM >= 0x070902
128 if (!strcmp("http.sslkey", var)) {
129 if (ssl_key == NULL) {
130 ssl_key = xmalloc(strlen(value)+1);
131 strcpy(ssl_key, value);
132 }
133 return 0;
134 }
135#endif
136#if LIBCURL_VERSION_NUM >= 0x070908
137 if (!strcmp("http.sslcapath", var)) {
138 if (ssl_capath == NULL) {
139 ssl_capath = xmalloc(strlen(value)+1);
140 strcpy(ssl_capath, value);
141 }
142 return 0;
143 }
144#endif
145 if (!strcmp("http.sslcainfo", var)) {
146 if (ssl_cainfo == NULL) {
147 ssl_cainfo = xmalloc(strlen(value)+1);
148 strcpy(ssl_cainfo, value);
149 }
150 return 0;
151 }
152
153#ifdef USE_CURL_MULTI
154 if (!strcmp("http.maxrequests", var)) {
155 if (max_requests == -1)
156 max_requests = git_config_int(var, value);
157 return 0;
158 }
159#endif
160
161 /* Fall back on the default ones */
162 return git_default_config(var, value);
163}
164
165static size_t fwrite_buffer(void *ptr, size_t eltsize, size_t nmemb,
166 struct buffer *buffer)
167{
168 size_t size = eltsize * nmemb;
169 if (size > buffer->size - buffer->posn)
170 size = buffer->size - buffer->posn;
171 memcpy(buffer->buffer + buffer->posn, ptr, size);
172 buffer->posn += size;
173 data_received++;
174 return size;
175}
176
177static size_t fwrite_buffer_dynamic(const void *ptr, size_t eltsize,
178 size_t nmemb, struct buffer *buffer)
179{
180 size_t size = eltsize * nmemb;
181 if (size > buffer->size - buffer->posn) {
182 buffer->size = buffer->size * 3 / 2;
183 if (buffer->size < buffer->posn + size)
184 buffer->size = buffer->posn + size;
185 buffer->buffer = xrealloc(buffer->buffer, buffer->size);
186 }
187 memcpy(buffer->buffer + buffer->posn, ptr, size);
188 buffer->posn += size;
189 data_received++;
190 return size;
191}
192
193static size_t fwrite_sha1_file(void *ptr, size_t eltsize, size_t nmemb,
194 void *data)
195{
196 unsigned char expn[4096];
197 size_t size = eltsize * nmemb;
198 int posn = 0;
199 struct transfer_request *request = (struct transfer_request *)data;
200 do {
201 ssize_t retval = write(request->local,
202 ptr + posn, size - posn);
203 if (retval < 0)
204 return posn;
205 posn += retval;
206 } while (posn < size);
207
208 request->stream.avail_in = size;
209 request->stream.next_in = ptr;
210 do {
211 request->stream.next_out = expn;
212 request->stream.avail_out = sizeof(expn);
213 request->zret = inflate(&request->stream, Z_SYNC_FLUSH);
214 SHA1_Update(&request->c, expn,
215 sizeof(expn) - request->stream.avail_out);
216 } while (request->stream.avail_in && request->zret == Z_OK);
217 data_received++;
218 return size;
219}
220
221#ifdef USE_CURL_MULTI
222static void process_curl_messages(void);
223static void process_request_queue(void);
224#endif
225
226static CURL* get_curl_handle(void)
227{
228 CURL* result = curl_easy_init();
229
230 curl_easy_setopt(result, CURLOPT_SSL_VERIFYPEER, curl_ssl_verify);
231#if LIBCURL_VERSION_NUM >= 0x070907
232 curl_easy_setopt(result, CURLOPT_NETRC, CURL_NETRC_OPTIONAL);
233#endif
234
235 if (ssl_cert != NULL)
236 curl_easy_setopt(result, CURLOPT_SSLCERT, ssl_cert);
237#if LIBCURL_VERSION_NUM >= 0x070902
238 if (ssl_key != NULL)
239 curl_easy_setopt(result, CURLOPT_SSLKEY, ssl_key);
240#endif
241#if LIBCURL_VERSION_NUM >= 0x070908
242 if (ssl_capath != NULL)
243 curl_easy_setopt(result, CURLOPT_CAPATH, ssl_capath);
244#endif
245 if (ssl_cainfo != NULL)
246 curl_easy_setopt(result, CURLOPT_CAINFO, ssl_cainfo);
247 curl_easy_setopt(result, CURLOPT_FAILONERROR, 1);
248
249 return result;
250}
251
252static struct active_request_slot *get_active_slot(void)
253{
254 struct active_request_slot *slot = active_queue_head;
255 struct active_request_slot *newslot;
256
257#ifdef USE_CURL_MULTI
258 int num_transfers;
259
260 /* Wait for a slot to open up if the queue is full */
261 while (active_requests >= max_requests) {
262 curl_multi_perform(curlm, &num_transfers);
263 if (num_transfers < active_requests) {
264 process_curl_messages();
265 }
266 }
267#endif
268
269 while (slot != NULL && slot->in_use) {
270 slot = slot->next;
271 }
272 if (slot == NULL) {
273 newslot = xmalloc(sizeof(*newslot));
274#ifdef NO_CURL_EASY_DUPHANDLE
275 newslot->curl = get_curl_handle();
276#else
277 newslot->curl = curl_easy_duphandle(curl_default);
278#endif
279 newslot->in_use = 0;
280 newslot->next = NULL;
281
282 slot = active_queue_head;
283 if (slot == NULL) {
284 active_queue_head = newslot;
285 } else {
286 while (slot->next != NULL) {
287 slot = slot->next;
288 }
289 slot->next = newslot;
290 }
291 slot = newslot;
292 }
293
294 active_requests++;
295 slot->in_use = 1;
296 slot->done = 0;
297 slot->local = NULL;
298 curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, pragma_header);
299 curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, no_range_header);
300 curl_easy_setopt(slot->curl, CURLOPT_ERRORBUFFER, curl_errorstr);
301
302 return slot;
303}
304
305static int start_active_slot(struct active_request_slot *slot)
306{
307#ifdef USE_CURL_MULTI
308 CURLMcode curlm_result = curl_multi_add_handle(curlm, slot->curl);
309
310 if (curlm_result != CURLM_OK &&
311 curlm_result != CURLM_CALL_MULTI_PERFORM) {
312 active_requests--;
313 slot->in_use = 0;
314 return 0;
315 }
316#endif
317 return 1;
318}
319
320static void run_active_slot(struct active_request_slot *slot)
321{
322#ifdef USE_CURL_MULTI
323 int num_transfers;
324 long last_pos = 0;
325 long current_pos;
326 fd_set readfds;
327 fd_set writefds;
328 fd_set excfds;
329 int max_fd;
330 struct timeval select_timeout;
331 CURLMcode curlm_result;
332
333 while (!slot->done) {
334 data_received = 0;
335 do {
336 curlm_result = curl_multi_perform(curlm,
337 &num_transfers);
338 } while (curlm_result == CURLM_CALL_MULTI_PERFORM);
339 if (num_transfers < active_requests) {
340 process_curl_messages();
341 process_request_queue();
342 }
343
344 if (!data_received && slot->local != NULL) {
345 current_pos = ftell(slot->local);
346 if (current_pos > last_pos)
347 data_received++;
348 last_pos = current_pos;
349 }
350
351 if (!slot->done && !data_received) {
352 max_fd = 0;
353 FD_ZERO(&readfds);
354 FD_ZERO(&writefds);
355 FD_ZERO(&excfds);
356 select_timeout.tv_sec = 0;
357 select_timeout.tv_usec = 50000;
358 select(max_fd, &readfds, &writefds,
359 &excfds, &select_timeout);
360 }
361 }
362#else
363 slot->curl_result = curl_easy_perform(slot->curl);
364 active_requests--;
365#endif
366}
367
368static void start_request(struct transfer_request *request)
369{
370 char *hex = sha1_to_hex(request->sha1);
371 char prevfile[PATH_MAX];
372 char *url;
373 char *posn;
374 int prevlocal;
375 unsigned char prev_buf[PREV_BUF_SIZE];
376 ssize_t prev_read = 0;
377 long prev_posn = 0;
378 char range[RANGE_HEADER_SIZE];
379 struct curl_slist *range_header = NULL;
380 struct active_request_slot *slot;
381
382 snprintf(prevfile, sizeof(prevfile), "%s.prev", request->filename);
383 unlink(prevfile);
384 rename(request->tmpfile, prevfile);
385 unlink(request->tmpfile);
386
387 request->local = open(request->tmpfile,
388 O_WRONLY | O_CREAT | O_EXCL, 0666);
389 /* This could have failed due to the "lazy directory creation";
390 * try to mkdir the last path component.
391 */
392 if (request->local < 0 && errno == ENOENT) {
393 char *dir = strrchr(request->tmpfile, '/');
394 if (dir) {
395 *dir = 0;
396 mkdir(request->tmpfile, 0777);
397 *dir = '/';
398 }
399 request->local = open(request->tmpfile,
400 O_WRONLY | O_CREAT | O_EXCL, 0666);
401 }
402
403 if (request->local < 0) {
404 request->state = ABORTED;
405 error("Couldn't create temporary file %s for %s: %s\n",
406 request->tmpfile, request->filename, strerror(errno));
407 return;
408 }
409
410 memset(&request->stream, 0, sizeof(request->stream));
411
412 inflateInit(&request->stream);
413
414 SHA1_Init(&request->c);
415
416 url = xmalloc(strlen(request->repo->base) + 50);
417 request->url = xmalloc(strlen(request->repo->base) + 50);
418 strcpy(url, request->repo->base);
419 posn = url + strlen(request->repo->base);
420 strcpy(posn, "objects/");
421 posn += 8;
422 memcpy(posn, hex, 2);
423 posn += 2;
424 *(posn++) = '/';
425 strcpy(posn, hex + 2);
426 strcpy(request->url, url);
427
428 /* If a previous temp file is present, process what was already
429 fetched. */
430 prevlocal = open(prevfile, O_RDONLY);
431 if (prevlocal != -1) {
432 do {
433 prev_read = read(prevlocal, prev_buf, PREV_BUF_SIZE);
434 if (prev_read>0) {
435 if (fwrite_sha1_file(prev_buf,
436 1,
437 prev_read,
438 request) == prev_read) {
439 prev_posn += prev_read;
440 } else {
441 prev_read = -1;
442 }
443 }
444 } while (prev_read > 0);
445 close(prevlocal);
446 }
447 unlink(prevfile);
448
449 /* Reset inflate/SHA1 if there was an error reading the previous temp
450 file; also rewind to the beginning of the local file. */
451 if (prev_read == -1) {
452 memset(&request->stream, 0, sizeof(request->stream));
453 inflateInit(&request->stream);
454 SHA1_Init(&request->c);
455 if (prev_posn>0) {
456 prev_posn = 0;
457 lseek(request->local, SEEK_SET, 0);
458 ftruncate(request->local, 0);
459 }
460 }
461
462 slot = get_active_slot();
463 curl_easy_setopt(slot->curl, CURLOPT_FILE, request);
464 curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite_sha1_file);
465 curl_easy_setopt(slot->curl, CURLOPT_ERRORBUFFER, request->errorstr);
466 curl_easy_setopt(slot->curl, CURLOPT_URL, url);
467 curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, no_pragma_header);
468
469 /* If we have successfully processed data from a previous fetch
470 attempt, only fetch the data we don't already have. */
471 if (prev_posn>0) {
472 if (get_verbosely)
473 fprintf(stderr,
474 "Resuming fetch of object %s at byte %ld\n",
475 hex, prev_posn);
476 sprintf(range, "Range: bytes=%ld-", prev_posn);
477 range_header = curl_slist_append(range_header, range);
478 curl_easy_setopt(slot->curl,
479 CURLOPT_HTTPHEADER, range_header);
480 }
481
482 /* Try to get the request started, abort the request on error */
483 if (!start_active_slot(slot)) {
484 request->state = ABORTED;
485 close(request->local);
486 free(request->url);
487 return;
488 }
489
490 request->slot = slot;
491 request->state = ACTIVE;
492}
493
494static void finish_request(struct transfer_request *request)
495{
496 fchmod(request->local, 0444);
497 close(request->local);
498
499 if (request->http_code == 416) {
500 fprintf(stderr, "Warning: requested range invalid; we may already have all the data.\n");
501 } else if (request->curl_result != CURLE_OK) {
502 return;
503 }
504
505 inflateEnd(&request->stream);
506 SHA1_Final(request->real_sha1, &request->c);
507 if (request->zret != Z_STREAM_END) {
508 unlink(request->tmpfile);
509 return;
510 }
511 if (memcmp(request->sha1, request->real_sha1, 20)) {
512 unlink(request->tmpfile);
513 return;
514 }
515 request->rename =
516 move_temp_to_file(request->tmpfile, request->filename);
517
518 if (request->rename == 0)
519 pull_say("got %s\n", sha1_to_hex(request->sha1));
520}
521
522static void release_request(struct transfer_request *request)
523{
524 struct transfer_request *entry = request_queue_head;
525
526 if (request == request_queue_head) {
527 request_queue_head = request->next;
528 } else {
529 while (entry->next != NULL && entry->next != request)
530 entry = entry->next;
531 if (entry->next == request)
532 entry->next = entry->next->next;
533 }
534
535 free(request->url);
536 free(request);
537}
538
539#ifdef USE_CURL_MULTI
540void process_curl_messages(void)
541{
542 int num_messages;
543 struct active_request_slot *slot;
544 struct transfer_request *request = NULL;
545 CURLMsg *curl_message = curl_multi_info_read(curlm, &num_messages);
546
547 while (curl_message != NULL) {
548 if (curl_message->msg == CURLMSG_DONE) {
549 slot = active_queue_head;
550 while (slot != NULL &&
551 slot->curl != curl_message->easy_handle)
552 slot = slot->next;
553 if (slot != NULL) {
554 curl_multi_remove_handle(curlm, slot->curl);
555 active_requests--;
556 slot->done = 1;
557 slot->in_use = 0;
558 slot->curl_result = curl_message->data.result;
559 request = request_queue_head;
560 while (request != NULL &&
561 request->slot != slot)
562 request = request->next;
563 } else {
564 fprintf(stderr, "Received DONE message for unknown request!\n");
565 }
566 if (request != NULL) {
567 request->curl_result =
568 curl_message->data.result;
569 curl_easy_getinfo(slot->curl,
570 CURLINFO_HTTP_CODE,
571 &request->http_code);
572 request->slot = NULL;
573
574 /* Use alternates if necessary */
575 if (request->http_code == 404 &&
576 request->repo->next != NULL) {
577 request->repo = request->repo->next;
578 start_request(request);
579 } else {
580 finish_request(request);
581 request->state = COMPLETE;
582 }
583 }
584 } else {
585 fprintf(stderr, "Unknown CURL message received: %d\n",
586 (int)curl_message->msg);
587 }
588 curl_message = curl_multi_info_read(curlm, &num_messages);
589 }
590}
591
592void process_request_queue(void)
593{
594 struct transfer_request *request = request_queue_head;
595 int num_transfers;
596
597 while (active_requests < max_requests && request != NULL) {
598 if (request->state == WAITING) {
599 if (has_sha1_file(request->sha1))
600 release_request(request);
601 else
602 start_request(request);
603 curl_multi_perform(curlm, &num_transfers);
604 }
605 request = request->next;
606 }
607}
608#endif
609
610void prefetch(unsigned char *sha1)
611{
612 struct transfer_request *newreq;
613 struct transfer_request *tail;
614 char *filename = sha1_file_name(sha1);
615
616 newreq = xmalloc(sizeof(*newreq));
617 memcpy(newreq->sha1, sha1, 20);
618 newreq->repo = alt;
619 newreq->url = NULL;
620 newreq->local = -1;
621 newreq->state = WAITING;
622 snprintf(newreq->filename, sizeof(newreq->filename), "%s", filename);
623 snprintf(newreq->tmpfile, sizeof(newreq->tmpfile),
624 "%s.temp", filename);
625 newreq->next = NULL;
626
627 if (request_queue_head == NULL) {
628 request_queue_head = newreq;
629 } else {
630 tail = request_queue_head;
631 while (tail->next != NULL) {
632 tail = tail->next;
633 }
634 tail->next = newreq;
635 }
636#ifdef USE_CURL_MULTI
637 process_request_queue();
638 process_curl_messages();
639#endif
640}
641
642static int fetch_index(struct alt_base *repo, unsigned char *sha1)
643{
644 char *hex = sha1_to_hex(sha1);
645 char *filename;
646 char *url;
647 char tmpfile[PATH_MAX];
648 long prev_posn = 0;
649 char range[RANGE_HEADER_SIZE];
650 struct curl_slist *range_header = NULL;
651
652 FILE *indexfile;
653 struct active_request_slot *slot;
654
655 if (has_pack_index(sha1))
656 return 0;
657
658 if (get_verbosely)
659 fprintf(stderr, "Getting index for pack %s\n", hex);
660
661 url = xmalloc(strlen(repo->base) + 64);
662 sprintf(url, "%s/objects/pack/pack-%s.idx", repo->base, hex);
663
664 filename = sha1_pack_index_name(sha1);
665 snprintf(tmpfile, sizeof(tmpfile), "%s.temp", filename);
666 indexfile = fopen(tmpfile, "a");
667 if (!indexfile)
668 return error("Unable to open local file %s for pack index",
669 filename);
670
671 slot = get_active_slot();
672 curl_easy_setopt(slot->curl, CURLOPT_FILE, indexfile);
673 curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite);
674 curl_easy_setopt(slot->curl, CURLOPT_URL, url);
675 curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, no_pragma_header);
676 slot->local = indexfile;
677
678 /* If there is data present from a previous transfer attempt,
679 resume where it left off */
680 prev_posn = ftell(indexfile);
681 if (prev_posn>0) {
682 if (get_verbosely)
683 fprintf(stderr,
684 "Resuming fetch of index for pack %s at byte %ld\n",
685 hex, prev_posn);
686 sprintf(range, "Range: bytes=%ld-", prev_posn);
687 range_header = curl_slist_append(range_header, range);
688 curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, range_header);
689 }
690
691 if (start_active_slot(slot)) {
692 run_active_slot(slot);
693 if (slot->curl_result != CURLE_OK) {
694 fclose(indexfile);
695 return error("Unable to get pack index %s\n%s", url,
696 curl_errorstr);
697 }
698 } else {
699 return error("Unable to start request");
700 }
701
702 fclose(indexfile);
703
704 return move_temp_to_file(tmpfile, filename);
705}
706
707static int setup_index(struct alt_base *repo, unsigned char *sha1)
708{
709 struct packed_git *new_pack;
710 if (has_pack_file(sha1))
711 return 0; // don't list this as something we can get
712
713 if (fetch_index(repo, sha1))
714 return -1;
715
716 new_pack = parse_pack_index(sha1);
717 new_pack->next = repo->packs;
718 repo->packs = new_pack;
719 return 0;
720}
721
722static int fetch_alternates(char *base)
723{
724 int ret = 0;
725 struct buffer buffer;
726 char *url;
727 char *data;
728 int i = 0;
729 int http_specific = 1;
730 struct alt_base *tail = alt;
731 static const char null_byte = '\0';
732
733 struct active_request_slot *slot;
734
735 data = xmalloc(4096);
736 buffer.size = 4096;
737 buffer.posn = 0;
738 buffer.buffer = data;
739
740 if (get_verbosely)
741 fprintf(stderr, "Getting alternates list\n");
742
743 url = xmalloc(strlen(base) + 31);
744 sprintf(url, "%s/objects/info/http-alternates", base);
745
746 slot = get_active_slot();
747 curl_easy_setopt(slot->curl, CURLOPT_FILE, &buffer);
748 curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION,
749 fwrite_buffer_dynamic);
750 curl_easy_setopt(slot->curl, CURLOPT_URL, url);
751 if (start_active_slot(slot)) {
752 run_active_slot(slot);
753 if (slot->curl_result != CURLE_OK || !buffer.posn) {
754 http_specific = 0;
755
756 sprintf(url, "%s/objects/info/alternates", base);
757
758 slot = get_active_slot();
759 curl_easy_setopt(slot->curl, CURLOPT_FILE, &buffer);
760 curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION,
761 fwrite_buffer_dynamic);
762 curl_easy_setopt(slot->curl, CURLOPT_URL, url);
763 if (start_active_slot(slot)) {
764 run_active_slot(slot);
765 if (slot->curl_result != CURLE_OK) {
766 free(buffer.buffer);
767 return 0;
768 }
769 }
770 }
771 } else {
772 free(buffer.buffer);
773 return 0;
774 }
775
776 fwrite_buffer_dynamic(&null_byte, 1, 1, &buffer);
777 buffer.posn--;
778 data = buffer.buffer;
779
780 while (i < buffer.posn) {
781 int posn = i;
782 while (posn < buffer.posn && data[posn] != '\n')
783 posn++;
784 if (data[posn] == '\n') {
785 int okay = 0;
786 int serverlen = 0;
787 struct alt_base *newalt;
788 char *target = NULL;
789 if (data[i] == '/') {
790 serverlen = strchr(base + 8, '/') - base;
791 okay = 1;
792 } else if (!memcmp(data + i, "../", 3)) {
793 i += 3;
794 serverlen = strlen(base);
795 while (i + 2 < posn &&
796 !memcmp(data + i, "../", 3)) {
797 do {
798 serverlen--;
799 } while (serverlen &&
800 base[serverlen - 1] != '/');
801 i += 3;
802 }
803 // If the server got removed, give up.
804 okay = strchr(base, ':') - base + 3 <
805 serverlen;
806 } else if (http_specific) {
807 char *colon = strchr(data + i, ':');
808 char *slash = strchr(data + i, '/');
809 if (colon && slash && colon < data + posn &&
810 slash < data + posn && colon < slash) {
811 okay = 1;
812 }
813 }
814 // skip 'objects' at end
815 if (okay) {
816 target = xmalloc(serverlen + posn - i - 6);
817 strncpy(target, base, serverlen);
818 strncpy(target + serverlen, data + i,
819 posn - i - 7);
820 target[serverlen + posn - i - 7] = '\0';
821 if (get_verbosely)
822 fprintf(stderr,
823 "Also look at %s\n", target);
824 newalt = xmalloc(sizeof(*newalt));
825 newalt->next = NULL;
826 newalt->base = target;
827 newalt->got_indices = 0;
828 newalt->packs = NULL;
829 while (tail->next != NULL)
830 tail = tail->next;
831 tail->next = newalt;
832 ret++;
833 }
834 }
835 i = posn + 1;
836 }
837
838 free(buffer.buffer);
839 return ret;
840}
841
842static int fetch_indices(struct alt_base *repo)
843{
844 unsigned char sha1[20];
845 char *url;
846 struct buffer buffer;
847 char *data;
848 int i = 0;
849
850 struct active_request_slot *slot;
851
852 if (repo->got_indices)
853 return 0;
854
855 data = xmalloc(4096);
856 buffer.size = 4096;
857 buffer.posn = 0;
858 buffer.buffer = data;
859
860 if (get_verbosely)
861 fprintf(stderr, "Getting pack list\n");
862
863 url = xmalloc(strlen(repo->base) + 21);
864 sprintf(url, "%s/objects/info/packs", repo->base);
865
866 slot = get_active_slot();
867 curl_easy_setopt(slot->curl, CURLOPT_FILE, &buffer);
868 curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION,
869 fwrite_buffer_dynamic);
870 curl_easy_setopt(slot->curl, CURLOPT_URL, url);
871 curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, NULL);
872 if (start_active_slot(slot)) {
873 run_active_slot(slot);
874 if (slot->curl_result != CURLE_OK) {
875 free(buffer.buffer);
876 return error("%s", curl_errorstr);
877 }
878 } else {
879 free(buffer.buffer);
880 return error("Unable to start request");
881 }
882
883 data = buffer.buffer;
884 while (i < buffer.posn) {
885 switch (data[i]) {
886 case 'P':
887 i++;
888 if (i + 52 < buffer.posn &&
889 !strncmp(data + i, " pack-", 6) &&
890 !strncmp(data + i + 46, ".pack\n", 6)) {
891 get_sha1_hex(data + i + 6, sha1);
892 setup_index(repo, sha1);
893 i += 51;
894 break;
895 }
896 default:
897 while (data[i] != '\n')
898 i++;
899 }
900 i++;
901 }
902
903 free(buffer.buffer);
904 repo->got_indices = 1;
905 return 0;
906}
907
908static int fetch_pack(struct alt_base *repo, unsigned char *sha1)
909{
910 char *url;
911 struct packed_git *target;
912 struct packed_git **lst;
913 FILE *packfile;
914 char *filename;
915 char tmpfile[PATH_MAX];
916 int ret;
917 long prev_posn = 0;
918 char range[RANGE_HEADER_SIZE];
919 struct curl_slist *range_header = NULL;
920
921 struct active_request_slot *slot;
922
923 if (fetch_indices(repo))
924 return -1;
925 target = find_sha1_pack(sha1, repo->packs);
926 if (!target)
927 return -1;
928
929 if (get_verbosely) {
930 fprintf(stderr, "Getting pack %s\n",
931 sha1_to_hex(target->sha1));
932 fprintf(stderr, " which contains %s\n",
933 sha1_to_hex(sha1));
934 }
935
936 url = xmalloc(strlen(repo->base) + 65);
937 sprintf(url, "%s/objects/pack/pack-%s.pack",
938 repo->base, sha1_to_hex(target->sha1));
939
940 filename = sha1_pack_name(target->sha1);
941 snprintf(tmpfile, sizeof(tmpfile), "%s.temp", filename);
942 packfile = fopen(tmpfile, "a");
943 if (!packfile)
944 return error("Unable to open local file %s for pack",
945 filename);
946
947 slot = get_active_slot();
948 curl_easy_setopt(slot->curl, CURLOPT_FILE, packfile);
949 curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite);
950 curl_easy_setopt(slot->curl, CURLOPT_URL, url);
951 curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, no_pragma_header);
952 slot->local = packfile;
953
954 /* If there is data present from a previous transfer attempt,
955 resume where it left off */
956 prev_posn = ftell(packfile);
957 if (prev_posn>0) {
958 if (get_verbosely)
959 fprintf(stderr,
960 "Resuming fetch of pack %s at byte %ld\n",
961 sha1_to_hex(target->sha1), prev_posn);
962 sprintf(range, "Range: bytes=%ld-", prev_posn);
963 range_header = curl_slist_append(range_header, range);
964 curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, range_header);
965 }
966
967 if (start_active_slot(slot)) {
968 run_active_slot(slot);
969 if (slot->curl_result != CURLE_OK) {
970 fclose(packfile);
971 return error("Unable to get pack file %s\n%s", url,
972 curl_errorstr);
973 }
974 } else {
975 return error("Unable to start request");
976 }
977
978 fclose(packfile);
979
980 ret = move_temp_to_file(tmpfile, filename);
981 if (ret)
982 return ret;
983
984 lst = &repo->packs;
985 while (*lst != target)
986 lst = &((*lst)->next);
987 *lst = (*lst)->next;
988
989 if (verify_pack(target, 0))
990 return -1;
991 install_packed_git(target);
992
993 return 0;
994}
995
996static int fetch_object(struct alt_base *repo, unsigned char *sha1)
997{
998 char *hex = sha1_to_hex(sha1);
999 int ret;
1000 struct transfer_request *request = request_queue_head;
1001
1002 while (request != NULL && memcmp(request->sha1, sha1, 20))
1003 request = request->next;
1004 if (request == NULL)
1005 return error("Couldn't find request for %s in the queue", hex);
1006
1007 if (has_sha1_file(request->sha1)) {
1008 release_request(request);
1009 return 0;
1010 }
1011
1012#ifdef USE_CURL_MULTI
1013 while (request->state == WAITING) {
1014 int num_transfers;
1015 curl_multi_perform(curlm, &num_transfers);
1016 if (num_transfers < active_requests) {
1017 process_curl_messages();
1018 process_request_queue();
1019 }
1020 }
1021#else
1022 start_request(request);
1023#endif
1024
1025 while (request->state == ACTIVE) {
1026 run_active_slot(request->slot);
1027#ifndef USE_CURL_MULTI
1028 request->curl_result = request->slot->curl_result;
1029 curl_easy_getinfo(request->slot->curl,
1030 CURLINFO_HTTP_CODE,
1031 &request->http_code);
1032 request->slot = NULL;
1033
1034 /* Use alternates if necessary */
1035 if (request->http_code == 404 &&
1036 request->repo->next != NULL) {
1037 request->repo = request->repo->next;
1038 start_request(request);
1039 } else {
1040 finish_request(request);
1041 request->state = COMPLETE;
1042 }
1043#endif
1044 }
1045
1046 if (request->state == ABORTED) {
1047 release_request(request);
1048 return error("Request for %s aborted", hex);
1049 }
1050
1051 if (request->curl_result != CURLE_OK && request->http_code != 416) {
1052 ret = error("%s (curl_result = %d, http_code = %ld, sha1 = %s)",
1053 request->errorstr, request->curl_result,
1054 request->http_code, hex);
1055 release_request(request);
1056 return ret;
1057 }
1058
1059 if (request->zret != Z_STREAM_END) {
1060 ret = error("File %s (%s) corrupt\n", hex, request->url);
1061 release_request(request);
1062 return ret;
1063 }
1064
1065 if (memcmp(request->sha1, request->real_sha1, 20)) {
1066 release_request(request);
1067 return error("File %s has bad hash\n", hex);
1068 }
1069
1070 if (request->rename < 0) {
1071 ret = error("unable to write sha1 filename %s: %s",
1072 request->filename,
1073 strerror(request->rename));
1074 release_request(request);
1075 return ret;
1076 }
1077
1078 release_request(request);
1079 return 0;
1080}
1081
1082int fetch(unsigned char *sha1)
1083{
1084 struct alt_base *altbase = alt;
1085
1086 if (!fetch_object(altbase, sha1))
1087 return 0;
1088 while (altbase) {
1089 if (!fetch_pack(altbase, sha1))
1090 return 0;
1091 altbase = altbase->next;
1092 }
1093 return error("Unable to find %s under %s\n", sha1_to_hex(sha1),
1094 alt->base);
1095}
1096
1097static inline int needs_quote(int ch)
1098{
1099 switch (ch) {
1100 case '/': case '-': case '.':
1101 case 'A'...'Z': case 'a'...'z': case '0'...'9':
1102 return 0;
1103 default:
1104 return 1;
1105 }
1106}
1107
1108static inline int hex(int v)
1109{
1110 if (v < 10) return '0' + v;
1111 else return 'A' + v - 10;
1112}
1113
1114static char *quote_ref_url(const char *base, const char *ref)
1115{
1116 const char *cp;
1117 char *dp, *qref;
1118 int len, baselen, ch;
1119
1120 baselen = strlen(base);
1121 len = baselen + 6; /* "refs/" + NUL */
1122 for (cp = ref; (ch = *cp) != 0; cp++, len++)
1123 if (needs_quote(ch))
1124 len += 2; /* extra two hex plus replacement % */
1125 qref = xmalloc(len);
1126 memcpy(qref, base, baselen);
1127 memcpy(qref + baselen, "refs/", 5);
1128 for (cp = ref, dp = qref + baselen + 5; (ch = *cp) != 0; cp++) {
1129 if (needs_quote(ch)) {
1130 *dp++ = '%';
1131 *dp++ = hex((ch >> 4) & 0xF);
1132 *dp++ = hex(ch & 0xF);
1133 }
1134 else
1135 *dp++ = ch;
1136 }
1137 *dp = 0;
1138
1139 return qref;
1140}
1141
1142int fetch_ref(char *ref, unsigned char *sha1)
1143{
1144 char *url;
1145 char hex[42];
1146 struct buffer buffer;
1147 char *base = alt->base;
1148 struct active_request_slot *slot;
1149 buffer.size = 41;
1150 buffer.posn = 0;
1151 buffer.buffer = hex;
1152 hex[41] = '\0';
1153
1154 url = quote_ref_url(base, ref);
1155 slot = get_active_slot();
1156 curl_easy_setopt(slot->curl, CURLOPT_FILE, &buffer);
1157 curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite_buffer);
1158 curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, NULL);
1159 curl_easy_setopt(slot->curl, CURLOPT_URL, url);
1160 if (start_active_slot(slot)) {
1161 run_active_slot(slot);
1162 if (slot->curl_result != CURLE_OK)
1163 return error("Couldn't get %s for %s\n%s",
1164 url, ref, curl_errorstr);
1165 } else {
1166 return error("Unable to start request");
1167 }
1168
1169 hex[40] = '\0';
1170 get_sha1_hex(hex, sha1);
1171 return 0;
1172}
1173
1174int main(int argc, char **argv)
1175{
1176 char *commit_id;
1177 char *url;
1178 int arg = 1;
1179 struct active_request_slot *slot;
1180
1181 while (arg < argc && argv[arg][0] == '-') {
1182 if (argv[arg][1] == 't') {
1183 get_tree = 1;
1184 } else if (argv[arg][1] == 'c') {
1185 get_history = 1;
1186 } else if (argv[arg][1] == 'a') {
1187 get_all = 1;
1188 get_tree = 1;
1189 get_history = 1;
1190 } else if (argv[arg][1] == 'v') {
1191 get_verbosely = 1;
1192 } else if (argv[arg][1] == 'w') {
1193 write_ref = argv[arg + 1];
1194 arg++;
1195 } else if (!strcmp(argv[arg], "--recover")) {
1196 get_recover = 1;
1197 }
1198 arg++;
1199 }
1200 if (argc < arg + 2) {
1201 usage("git-http-fetch [-c] [-t] [-a] [-d] [-v] [--recover] [-w ref] commit-id url");
1202 return 1;
1203 }
1204 commit_id = argv[arg];
1205 url = argv[arg + 1];
1206
1207 curl_global_init(CURL_GLOBAL_ALL);
1208
1209#ifdef USE_CURL_MULTI
1210 {
1211 char *http_max_requests = getenv("GIT_HTTP_MAX_REQUESTS");
1212 if (http_max_requests != NULL)
1213 max_requests = atoi(http_max_requests);
1214 }
1215
1216 curlm = curl_multi_init();
1217 if (curlm == NULL) {
1218 fprintf(stderr, "Error creating curl multi handle.\n");
1219 return 1;
1220 }
1221#endif
1222
1223 if (getenv("GIT_SSL_NO_VERIFY"))
1224 curl_ssl_verify = 0;
1225
1226 ssl_cert = getenv("GIT_SSL_CERT");
1227#if LIBCURL_VERSION_NUM >= 0x070902
1228 ssl_key = getenv("GIT_SSL_KEY");
1229#endif
1230#if LIBCURL_VERSION_NUM >= 0x070908
1231 ssl_capath = getenv("GIT_SSL_CAPATH");
1232#endif
1233 ssl_cainfo = getenv("GIT_SSL_CAINFO");
1234
1235 git_config(http_options);
1236
1237 if (curl_ssl_verify == -1)
1238 curl_ssl_verify = 1;
1239
1240#ifdef USE_CURL_MULTI
1241 if (max_requests < 1)
1242 max_requests = DEFAULT_MAX_REQUESTS;
1243#endif
1244
1245 pragma_header = curl_slist_append(pragma_header, "Pragma: no-cache");
1246 no_pragma_header = curl_slist_append(no_pragma_header, "Pragma:");
1247 no_range_header = curl_slist_append(no_range_header, "Range:");
1248
1249#ifndef NO_CURL_EASY_DUPHANDLE
1250 curl_default = get_curl_handle();
1251#endif
1252
1253 alt = xmalloc(sizeof(*alt));
1254 alt->base = url;
1255 alt->got_indices = 0;
1256 alt->packs = NULL;
1257 alt->next = NULL;
1258 fetch_alternates(alt->base);
1259
1260 if (pull(commit_id))
1261 return 1;
1262
1263 curl_slist_free_all(pragma_header);
1264 curl_slist_free_all(no_pragma_header);
1265 curl_slist_free_all(no_range_header);
1266#ifndef NO_CURL_EASY_DUPHANDLE
1267 curl_easy_cleanup(curl_default);
1268#endif
1269 slot = active_queue_head;
1270 while (slot != NULL) {
1271 curl_easy_cleanup(slot->curl);
1272 slot = slot->next;
1273 }
1274#ifdef USE_CURL_MULTI
1275 curl_multi_cleanup(curlm);
1276#endif
1277 curl_global_cleanup();
1278 return 0;
1279}