1#include "cache.h"
2#include "commit.h"
3#include "pack.h"
4#include "fetch.h"
5
6#include <curl/curl.h>
7#include <curl/easy.h>
8
9#if LIBCURL_VERSION_NUM >= 0x070908
10#define USE_CURL_MULTI
11#define DEFAULT_MAX_REQUESTS 5
12#endif
13
14#if LIBCURL_VERSION_NUM < 0x070704
15#define curl_global_cleanup() do { /* nothing */ } while(0)
16#endif
17#if LIBCURL_VERSION_NUM < 0x070800
18#define curl_global_init(a) do { /* nothing */ } while(0)
19#endif
20
21#define PREV_BUF_SIZE 4096
22#define RANGE_HEADER_SIZE 30
23
24static int active_requests = 0;
25static int data_received;
26
27#ifdef USE_CURL_MULTI
28static int max_requests = DEFAULT_MAX_REQUESTS;
29static CURLM *curlm;
30#endif
31static CURL *curl_default;
32static struct curl_slist *no_pragma_header;
33static struct curl_slist *no_range_header;
34static char curl_errorstr[CURL_ERROR_SIZE];
35
36struct alt_base
37{
38 char *base;
39 int got_indices;
40 struct packed_git *packs;
41 struct alt_base *next;
42};
43
44static struct alt_base *alt = NULL;
45
46enum transfer_state {
47 WAITING,
48 ABORTED,
49 ACTIVE,
50 COMPLETE,
51};
52
53struct transfer_request
54{
55 unsigned char sha1[20];
56 struct alt_base *repo;
57 char *url;
58 char filename[PATH_MAX];
59 char tmpfile[PATH_MAX];
60 int local;
61 enum transfer_state state;
62 CURLcode curl_result;
63 char errorstr[CURL_ERROR_SIZE];
64 long http_code;
65 unsigned char real_sha1[20];
66 SHA_CTX c;
67 z_stream stream;
68 int zret;
69 int rename;
70 struct active_request_slot *slot;
71 struct transfer_request *next;
72};
73
74struct active_request_slot
75{
76 CURL *curl;
77 FILE *local;
78 int in_use;
79 int done;
80 CURLcode curl_result;
81 struct active_request_slot *next;
82};
83
84static struct transfer_request *request_queue_head = NULL;
85static struct active_request_slot *active_queue_head = NULL;
86
87static int curl_ssl_verify;
88static char *ssl_cert;
89static char *ssl_key;
90static char *ssl_capath;
91static char *ssl_cainfo;
92
93struct buffer
94{
95 size_t posn;
96 size_t size;
97 void *buffer;
98};
99
100static size_t fwrite_buffer(void *ptr, size_t eltsize, size_t nmemb,
101 struct buffer *buffer)
102{
103 size_t size = eltsize * nmemb;
104 if (size > buffer->size - buffer->posn)
105 size = buffer->size - buffer->posn;
106 memcpy(buffer->buffer + buffer->posn, ptr, size);
107 buffer->posn += size;
108 data_received++;
109 return size;
110}
111
112static size_t fwrite_sha1_file(void *ptr, size_t eltsize, size_t nmemb,
113 void *data)
114{
115 unsigned char expn[4096];
116 size_t size = eltsize * nmemb;
117 int posn = 0;
118 struct transfer_request *request = (struct transfer_request *)data;
119 do {
120 ssize_t retval = write(request->local,
121 ptr + posn, size - posn);
122 if (retval < 0)
123 return posn;
124 posn += retval;
125 } while (posn < size);
126
127 request->stream.avail_in = size;
128 request->stream.next_in = ptr;
129 do {
130 request->stream.next_out = expn;
131 request->stream.avail_out = sizeof(expn);
132 request->zret = inflate(&request->stream, Z_SYNC_FLUSH);
133 SHA1_Update(&request->c, expn,
134 sizeof(expn) - request->stream.avail_out);
135 } while (request->stream.avail_in && request->zret == Z_OK);
136 data_received++;
137 return size;
138}
139
140int relink_or_rename(char *old, char *new) {
141 int ret;
142
143 ret = link(old, new);
144 if (ret < 0) {
145 /* Same Coda hack as in write_sha1_file(sha1_file.c) */
146 ret = errno;
147 if (ret == EXDEV && !rename(old, new))
148 return 0;
149 }
150 unlink(old);
151 if (ret) {
152 if (ret != EEXIST)
153 return ret;
154 }
155
156 return 0;
157}
158
159#ifdef USE_CURL_MULTI
160void process_curl_messages();
161void process_request_queue();
162#endif
163
164struct active_request_slot *get_active_slot()
165{
166 struct active_request_slot *slot = active_queue_head;
167 struct active_request_slot *newslot;
168
169#ifdef USE_CURL_MULTI
170 int num_transfers;
171
172 /* Wait for a slot to open up if the queue is full */
173 while (active_requests >= max_requests) {
174 curl_multi_perform(curlm, &num_transfers);
175 if (num_transfers < active_requests) {
176 process_curl_messages();
177 }
178 }
179#endif
180
181 while (slot != NULL && slot->in_use) {
182 slot = slot->next;
183 }
184 if (slot == NULL) {
185 newslot = xmalloc(sizeof(*newslot));
186 newslot->curl = curl_easy_duphandle(curl_default);
187 newslot->in_use = 0;
188 newslot->next = NULL;
189
190 slot = active_queue_head;
191 if (slot == NULL) {
192 active_queue_head = newslot;
193 } else {
194 while (slot->next != NULL) {
195 slot = slot->next;
196 }
197 slot->next = newslot;
198 }
199 slot = newslot;
200 }
201
202 active_requests++;
203 slot->in_use = 1;
204 slot->done = 0;
205 slot->local = NULL;
206 curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, no_pragma_header);
207 curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, no_range_header);
208 curl_easy_setopt(slot->curl, CURLOPT_ERRORBUFFER, curl_errorstr);
209
210 return slot;
211}
212
213int start_active_slot(struct active_request_slot *slot)
214{
215#ifdef USE_CURL_MULTI
216 CURLMcode curlm_result = curl_multi_add_handle(curlm, slot->curl);
217
218 if (curlm_result != CURLM_OK &&
219 curlm_result != CURLM_CALL_MULTI_PERFORM) {
220 active_requests--;
221 slot->in_use = 0;
222 return 0;
223 }
224#endif
225 return 1;
226}
227
228void run_active_slot(struct active_request_slot *slot)
229{
230#ifdef USE_CURL_MULTI
231 int num_transfers;
232 long last_pos = 0;
233 long current_pos;
234 fd_set readfds;
235 fd_set writefds;
236 fd_set excfds;
237 int max_fd;
238 struct timeval select_timeout;
239 CURLMcode curlm_result;
240
241 while (!slot->done) {
242 data_received = 0;
243 do {
244 curlm_result = curl_multi_perform(curlm,
245 &num_transfers);
246 } while (curlm_result == CURLM_CALL_MULTI_PERFORM);
247 if (num_transfers < active_requests) {
248 process_curl_messages();
249 process_request_queue();
250 }
251
252 if (!data_received && slot->local != NULL) {
253 current_pos = ftell(slot->local);
254 if (current_pos > last_pos)
255 data_received++;
256 last_pos = current_pos;
257 }
258
259 if (!slot->done && !data_received) {
260 max_fd = 0;
261 FD_ZERO(&readfds);
262 FD_ZERO(&writefds);
263 FD_ZERO(&excfds);
264 select_timeout.tv_sec = 0;
265 select_timeout.tv_usec = 50000;
266 select(max_fd, &readfds, &writefds,
267 &excfds, &select_timeout);
268 }
269 }
270#else
271 slot->curl_result = curl_easy_perform(slot->curl);
272 active_requests--;
273#endif
274}
275
276void start_request(struct transfer_request *request)
277{
278 char *hex = sha1_to_hex(request->sha1);
279 char prevfile[PATH_MAX];
280 char *url;
281 char *posn;
282 int prevlocal;
283 unsigned char prev_buf[PREV_BUF_SIZE];
284 ssize_t prev_read = 0;
285 long prev_posn = 0;
286 char range[RANGE_HEADER_SIZE];
287 struct curl_slist *range_header = NULL;
288 struct active_request_slot *slot;
289
290 snprintf(prevfile, sizeof(prevfile), "%s.prev", request->filename);
291 unlink(prevfile);
292 rename(request->tmpfile, prevfile);
293 unlink(request->tmpfile);
294
295 request->local = open(request->tmpfile,
296 O_WRONLY | O_CREAT | O_EXCL, 0666);
297 if (request->local < 0) {
298 request->state = ABORTED;
299 error("Couldn't create temporary file %s for %s: %s\n",
300 request->tmpfile, request->filename, strerror(errno));
301 return;
302 }
303
304 memset(&request->stream, 0, sizeof(request->stream));
305
306 inflateInit(&request->stream);
307
308 SHA1_Init(&request->c);
309
310 url = xmalloc(strlen(request->repo->base) + 50);
311 request->url = xmalloc(strlen(request->repo->base) + 50);
312 strcpy(url, request->repo->base);
313 posn = url + strlen(request->repo->base);
314 strcpy(posn, "objects/");
315 posn += 8;
316 memcpy(posn, hex, 2);
317 posn += 2;
318 *(posn++) = '/';
319 strcpy(posn, hex + 2);
320 strcpy(request->url, url);
321
322 /* If a previous temp file is present, process what was already
323 fetched. */
324 prevlocal = open(prevfile, O_RDONLY);
325 if (prevlocal != -1) {
326 do {
327 prev_read = read(prevlocal, prev_buf, PREV_BUF_SIZE);
328 if (prev_read>0) {
329 if (fwrite_sha1_file(prev_buf,
330 1,
331 prev_read,
332 request) == prev_read) {
333 prev_posn += prev_read;
334 } else {
335 prev_read = -1;
336 }
337 }
338 } while (prev_read > 0);
339 close(prevlocal);
340 }
341 unlink(prevfile);
342
343 /* Reset inflate/SHA1 if there was an error reading the previous temp
344 file; also rewind to the beginning of the local file. */
345 if (prev_read == -1) {
346 memset(&request->stream, 0, sizeof(request->stream));
347 inflateInit(&request->stream);
348 SHA1_Init(&request->c);
349 if (prev_posn>0) {
350 prev_posn = 0;
351 lseek(request->local, SEEK_SET, 0);
352 ftruncate(request->local, 0);
353 }
354 }
355
356 slot = get_active_slot();
357 curl_easy_setopt(slot->curl, CURLOPT_FILE, request);
358 curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite_sha1_file);
359 curl_easy_setopt(slot->curl, CURLOPT_ERRORBUFFER, request->errorstr);
360 curl_easy_setopt(slot->curl, CURLOPT_URL, url);
361
362 /* If we have successfully processed data from a previous fetch
363 attempt, only fetch the data we don't already have. */
364 if (prev_posn>0) {
365 if (get_verbosely)
366 fprintf(stderr,
367 "Resuming fetch of object %s at byte %ld\n",
368 hex, prev_posn);
369 sprintf(range, "Range: bytes=%ld-", prev_posn);
370 range_header = curl_slist_append(range_header, range);
371 curl_easy_setopt(slot->curl,
372 CURLOPT_HTTPHEADER, range_header);
373 }
374
375 /* Try to get the request started, abort the request on error */
376 if (!start_active_slot(slot)) {
377 request->state = ABORTED;
378 close(request->local);
379 free(request->url);
380 return;
381 }
382
383 request->slot = slot;
384 request->state = ACTIVE;
385}
386
387void finish_request(struct transfer_request *request)
388{
389 fchmod(request->local, 0444);
390 close(request->local);
391
392 if (request->http_code == 416) {
393 fprintf(stderr, "Warning: requested range invalid; we may already have all the data.\n");
394 } else if (request->curl_result != CURLE_OK) {
395 return;
396 }
397
398 inflateEnd(&request->stream);
399 SHA1_Final(request->real_sha1, &request->c);
400 if (request->zret != Z_STREAM_END) {
401 unlink(request->tmpfile);
402 return;
403 }
404 if (memcmp(request->sha1, request->real_sha1, 20)) {
405 unlink(request->tmpfile);
406 return;
407 }
408 request->rename =
409 relink_or_rename(request->tmpfile, request->filename);
410
411 if (request->rename == 0)
412 pull_say("got %s\n", sha1_to_hex(request->sha1));
413}
414
415void release_request(struct transfer_request *request)
416{
417 struct transfer_request *entry = request_queue_head;
418
419 if (request == request_queue_head) {
420 request_queue_head = request->next;
421 } else {
422 while (entry->next != NULL && entry->next != request)
423 entry = entry->next;
424 if (entry->next == request)
425 entry->next = entry->next->next;
426 }
427
428 free(request->url);
429 free(request);
430}
431
432#ifdef USE_CURL_MULTI
433void process_curl_messages()
434{
435 int num_messages;
436 struct active_request_slot *slot;
437 struct transfer_request *request = NULL;
438 CURLMsg *curl_message = curl_multi_info_read(curlm, &num_messages);
439
440 while (curl_message != NULL) {
441 if (curl_message->msg == CURLMSG_DONE) {
442 slot = active_queue_head;
443 while (slot != NULL &&
444 slot->curl != curl_message->easy_handle)
445 slot = slot->next;
446 if (slot != NULL) {
447 curl_multi_remove_handle(curlm, slot->curl);
448 active_requests--;
449 slot->done = 1;
450 slot->in_use = 0;
451 slot->curl_result = curl_message->data.result;
452 request = request_queue_head;
453 while (request != NULL &&
454 request->slot != slot)
455 request = request->next;
456 } else {
457 fprintf(stderr, "Received DONE message for unknown request!\n");
458 }
459 if (request != NULL) {
460 request->curl_result =
461 curl_message->data.result;
462 curl_easy_getinfo(slot->curl,
463 CURLINFO_HTTP_CODE,
464 &request->http_code);
465 request->slot = NULL;
466
467 /* Use alternates if necessary */
468 if (request->http_code == 404 &&
469 request->repo->next != NULL) {
470 request->repo = request->repo->next;
471 start_request(request);
472 } else {
473 finish_request(request);
474 request->state = COMPLETE;
475 }
476 }
477 } else {
478 fprintf(stderr, "Unknown CURL message received: %d\n",
479 (int)curl_message->msg);
480 }
481 curl_message = curl_multi_info_read(curlm, &num_messages);
482 }
483}
484
485void process_request_queue()
486{
487 struct transfer_request *request = request_queue_head;
488 int num_transfers;
489
490 while (active_requests < max_requests && request != NULL) {
491 if (request->state == WAITING) {
492 if (has_sha1_file(request->sha1))
493 release_request(request);
494 else
495 start_request(request);
496 curl_multi_perform(curlm, &num_transfers);
497 }
498 request = request->next;
499 }
500}
501#endif
502
503void prefetch(unsigned char *sha1)
504{
505 struct transfer_request *newreq;
506 struct transfer_request *tail;
507 char *filename = sha1_file_name(sha1);
508
509 newreq = xmalloc(sizeof(*newreq));
510 memcpy(newreq->sha1, sha1, 20);
511 newreq->repo = alt;
512 newreq->url = NULL;
513 newreq->local = -1;
514 newreq->state = WAITING;
515 snprintf(newreq->filename, sizeof(newreq->filename), "%s", filename);
516 snprintf(newreq->tmpfile, sizeof(newreq->tmpfile),
517 "%s.temp", filename);
518 newreq->next = NULL;
519
520 if (request_queue_head == NULL) {
521 request_queue_head = newreq;
522 } else {
523 tail = request_queue_head;
524 while (tail->next != NULL) {
525 tail = tail->next;
526 }
527 tail->next = newreq;
528 }
529#ifdef USE_CURL_MULTI
530 process_request_queue();
531 process_curl_messages();
532#endif
533}
534
535static int got_alternates = 0;
536
537static int fetch_index(struct alt_base *repo, unsigned char *sha1)
538{
539 char *hex = sha1_to_hex(sha1);
540 char *filename;
541 char *url;
542 char tmpfile[PATH_MAX];
543 int ret;
544 long prev_posn = 0;
545 char range[RANGE_HEADER_SIZE];
546 struct curl_slist *range_header = NULL;
547
548 FILE *indexfile;
549 struct active_request_slot *slot;
550
551 if (has_pack_index(sha1))
552 return 0;
553
554 if (get_verbosely)
555 fprintf(stderr, "Getting index for pack %s\n", hex);
556
557 url = xmalloc(strlen(repo->base) + 64);
558 sprintf(url, "%s/objects/pack/pack-%s.idx", repo->base, hex);
559
560 filename = sha1_pack_index_name(sha1);
561 snprintf(tmpfile, sizeof(tmpfile), "%s.temp", filename);
562 indexfile = fopen(tmpfile, "a");
563 if (!indexfile)
564 return error("Unable to open local file %s for pack index",
565 filename);
566
567 slot = get_active_slot();
568 curl_easy_setopt(slot->curl, CURLOPT_FILE, indexfile);
569 curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite);
570 curl_easy_setopt(slot->curl, CURLOPT_URL, url);
571 slot->local = indexfile;
572
573 /* If there is data present from a previous transfer attempt,
574 resume where it left off */
575 prev_posn = ftell(indexfile);
576 if (prev_posn>0) {
577 if (get_verbosely)
578 fprintf(stderr,
579 "Resuming fetch of index for pack %s at byte %ld\n",
580 hex, prev_posn);
581 sprintf(range, "Range: bytes=%ld-", prev_posn);
582 range_header = curl_slist_append(range_header, range);
583 curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, range_header);
584 }
585
586 if (start_active_slot(slot)) {
587 run_active_slot(slot);
588 if (slot->curl_result != CURLE_OK) {
589 fclose(indexfile);
590 return error("Unable to get pack index %s\n%s", url,
591 curl_errorstr);
592 }
593 } else {
594 return error("Unable to start request");
595 }
596
597 fclose(indexfile);
598
599 ret = relink_or_rename(tmpfile, filename);
600 if (ret)
601 return error("unable to write index filename %s: %s",
602 filename, strerror(ret));
603
604 return 0;
605}
606
607static int setup_index(struct alt_base *repo, unsigned char *sha1)
608{
609 struct packed_git *new_pack;
610 if (has_pack_file(sha1))
611 return 0; // don't list this as something we can get
612
613 if (fetch_index(repo, sha1))
614 return -1;
615
616 new_pack = parse_pack_index(sha1);
617 new_pack->next = repo->packs;
618 repo->packs = new_pack;
619 return 0;
620}
621
622static int fetch_alternates(char *base)
623{
624 int ret = 0;
625 struct buffer buffer;
626 char *url;
627 char *data;
628 int i = 0;
629 int http_specific = 1;
630 struct alt_base *tail = alt;
631
632 struct active_request_slot *slot;
633 if (got_alternates)
634 return 0;
635 data = xmalloc(4096);
636 buffer.size = 4095;
637 buffer.posn = 0;
638 buffer.buffer = data;
639
640 if (get_verbosely)
641 fprintf(stderr, "Getting alternates list\n");
642
643 url = xmalloc(strlen(base) + 31);
644 sprintf(url, "%s/objects/info/http-alternates", base);
645
646 slot = get_active_slot();
647 curl_easy_setopt(slot->curl, CURLOPT_FILE, &buffer);
648 curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite_buffer);
649 curl_easy_setopt(slot->curl, CURLOPT_URL, url);
650 if (start_active_slot(slot)) {
651 run_active_slot(slot);
652 if (slot->curl_result != CURLE_OK || !buffer.posn) {
653 http_specific = 0;
654
655 sprintf(url, "%s/objects/info/alternates", base);
656
657 slot = get_active_slot();
658 curl_easy_setopt(slot->curl, CURLOPT_FILE, &buffer);
659 curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION,
660 fwrite_buffer);
661 curl_easy_setopt(slot->curl, CURLOPT_URL, url);
662 if (start_active_slot(slot)) {
663 run_active_slot(slot);
664 if (slot->curl_result != CURLE_OK) {
665 return 0;
666 }
667 }
668 }
669 } else {
670 return 0;
671 }
672
673 data[buffer.posn] = '\0';
674
675 while (i < buffer.posn) {
676 int posn = i;
677 while (posn < buffer.posn && data[posn] != '\n')
678 posn++;
679 if (data[posn] == '\n') {
680 int okay = 0;
681 int serverlen = 0;
682 struct alt_base *newalt;
683 char *target = NULL;
684 if (data[i] == '/') {
685 serverlen = strchr(base + 8, '/') - base;
686 okay = 1;
687 } else if (!memcmp(data + i, "../", 3)) {
688 i += 3;
689 serverlen = strlen(base);
690 while (i + 2 < posn &&
691 !memcmp(data + i, "../", 3)) {
692 do {
693 serverlen--;
694 } while (serverlen &&
695 base[serverlen - 1] != '/');
696 i += 3;
697 }
698 // If the server got removed, give up.
699 okay = strchr(base, ':') - base + 3 <
700 serverlen;
701 } else if (http_specific) {
702 char *colon = strchr(data + i, ':');
703 char *slash = strchr(data + i, '/');
704 if (colon && slash && colon < data + posn &&
705 slash < data + posn && colon < slash) {
706 okay = 1;
707 }
708 }
709 // skip 'objects' at end
710 if (okay) {
711 target = xmalloc(serverlen + posn - i - 6);
712 strncpy(target, base, serverlen);
713 strncpy(target + serverlen, data + i,
714 posn - i - 7);
715 target[serverlen + posn - i - 7] = '\0';
716 if (get_verbosely)
717 fprintf(stderr,
718 "Also look at %s\n", target);
719 newalt = xmalloc(sizeof(*newalt));
720 newalt->next = NULL;
721 newalt->base = target;
722 newalt->got_indices = 0;
723 newalt->packs = NULL;
724 while (tail->next != NULL)
725 tail = tail->next;
726 tail->next = newalt;
727 ret++;
728 }
729 }
730 i = posn + 1;
731 }
732 got_alternates = 1;
733
734 return ret;
735}
736
737static int fetch_indices(struct alt_base *repo)
738{
739 unsigned char sha1[20];
740 char *url;
741 struct buffer buffer;
742 char *data;
743 int i = 0;
744
745 struct active_request_slot *slot;
746
747 if (repo->got_indices)
748 return 0;
749
750 data = xmalloc(4096);
751 buffer.size = 4096;
752 buffer.posn = 0;
753 buffer.buffer = data;
754
755 if (get_verbosely)
756 fprintf(stderr, "Getting pack list\n");
757
758 url = xmalloc(strlen(repo->base) + 21);
759 sprintf(url, "%s/objects/info/packs", repo->base);
760
761 slot = get_active_slot();
762 curl_easy_setopt(slot->curl, CURLOPT_FILE, &buffer);
763 curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite_buffer);
764 curl_easy_setopt(slot->curl, CURLOPT_URL, url);
765 curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, NULL);
766 if (start_active_slot(slot)) {
767 run_active_slot(slot);
768 if (slot->curl_result != CURLE_OK)
769 return error("%s", curl_errorstr);
770 } else {
771 return error("Unable to start request");
772 }
773
774 while (i < buffer.posn) {
775 switch (data[i]) {
776 case 'P':
777 i++;
778 if (i + 52 < buffer.posn &&
779 !strncmp(data + i, " pack-", 6) &&
780 !strncmp(data + i + 46, ".pack\n", 6)) {
781 get_sha1_hex(data + i + 6, sha1);
782 setup_index(repo, sha1);
783 i += 51;
784 break;
785 }
786 default:
787 while (data[i] != '\n')
788 i++;
789 }
790 i++;
791 }
792
793 repo->got_indices = 1;
794 return 0;
795}
796
797static int fetch_pack(struct alt_base *repo, unsigned char *sha1)
798{
799 char *url;
800 struct packed_git *target;
801 struct packed_git **lst;
802 FILE *packfile;
803 char *filename;
804 char tmpfile[PATH_MAX];
805 int ret;
806 long prev_posn = 0;
807 char range[RANGE_HEADER_SIZE];
808 struct curl_slist *range_header = NULL;
809
810 struct active_request_slot *slot;
811
812 if (fetch_indices(repo))
813 return -1;
814 target = find_sha1_pack(sha1, repo->packs);
815 if (!target)
816 return -1;
817
818 if (get_verbosely) {
819 fprintf(stderr, "Getting pack %s\n",
820 sha1_to_hex(target->sha1));
821 fprintf(stderr, " which contains %s\n",
822 sha1_to_hex(sha1));
823 }
824
825 url = xmalloc(strlen(repo->base) + 65);
826 sprintf(url, "%s/objects/pack/pack-%s.pack",
827 repo->base, sha1_to_hex(target->sha1));
828
829 filename = sha1_pack_name(target->sha1);
830 snprintf(tmpfile, sizeof(tmpfile), "%s.temp", filename);
831 packfile = fopen(tmpfile, "a");
832 if (!packfile)
833 return error("Unable to open local file %s for pack",
834 filename);
835
836 slot = get_active_slot();
837 curl_easy_setopt(slot->curl, CURLOPT_FILE, packfile);
838 curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite);
839 curl_easy_setopt(slot->curl, CURLOPT_URL, url);
840 slot->local = packfile;
841
842 /* If there is data present from a previous transfer attempt,
843 resume where it left off */
844 prev_posn = ftell(packfile);
845 if (prev_posn>0) {
846 if (get_verbosely)
847 fprintf(stderr,
848 "Resuming fetch of pack %s at byte %ld\n",
849 sha1_to_hex(target->sha1), prev_posn);
850 sprintf(range, "Range: bytes=%ld-", prev_posn);
851 range_header = curl_slist_append(range_header, range);
852 curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, range_header);
853 }
854
855 if (start_active_slot(slot)) {
856 run_active_slot(slot);
857 if (slot->curl_result != CURLE_OK) {
858 fclose(packfile);
859 return error("Unable to get pack file %s\n%s", url,
860 curl_errorstr);
861 }
862 } else {
863 return error("Unable to start request");
864 }
865
866 fclose(packfile);
867
868 ret = relink_or_rename(tmpfile, filename);
869 if (ret)
870 return error("unable to write pack filename %s: %s",
871 filename, strerror(ret));
872
873 lst = &repo->packs;
874 while (*lst != target)
875 lst = &((*lst)->next);
876 *lst = (*lst)->next;
877
878 if (verify_pack(target, 0))
879 return -1;
880 install_packed_git(target);
881
882 return 0;
883}
884
885static int fetch_object(struct alt_base *repo, unsigned char *sha1)
886{
887 char *hex = sha1_to_hex(sha1);
888 int ret;
889 struct transfer_request *request = request_queue_head;
890
891 while (request != NULL && memcmp(request->sha1, sha1, 20))
892 request = request->next;
893 if (request == NULL)
894 return error("Couldn't find request for %s in the queue", hex);
895
896 if (has_sha1_file(request->sha1)) {
897 release_request(request);
898 return 0;
899 }
900
901#ifdef USE_CURL_MULTI
902 int num_transfers;
903 while (request->state == WAITING) {
904 curl_multi_perform(curlm, &num_transfers);
905 if (num_transfers < active_requests) {
906 process_curl_messages();
907 process_request_queue();
908 }
909 }
910#else
911 start_request(request);
912#endif
913
914 while (request->state == ACTIVE) {
915 run_active_slot(request->slot);
916#ifndef USE_CURL_MULTI
917 request->curl_result = request->slot->curl_result;
918 curl_easy_getinfo(request->slot->curl,
919 CURLINFO_HTTP_CODE,
920 &request->http_code);
921 request->slot = NULL;
922
923 /* Use alternates if necessary */
924 if (request->http_code == 404 &&
925 request->repo->next != NULL) {
926 request->repo = request->repo->next;
927 start_request(request);
928 } else {
929 finish_request(request);
930 request->state = COMPLETE;
931 }
932#endif
933 }
934
935 if (request->state == ABORTED) {
936 release_request(request);
937 return error("Request for %s aborted", hex);
938 }
939
940 if (request->curl_result != CURLE_OK && request->http_code != 416) {
941 ret = error("%s", request->errorstr);
942 release_request(request);
943 return ret;
944 }
945
946 if (request->zret != Z_STREAM_END) {
947 ret = error("File %s (%s) corrupt\n", hex, request->url);
948 release_request(request);
949 return ret;
950 }
951
952 if (memcmp(request->sha1, request->real_sha1, 20)) {
953 release_request(request);
954 return error("File %s has bad hash\n", hex);
955 }
956
957 if (request->rename < 0) {
958 ret = error("unable to write sha1 filename %s: %s",
959 request->filename,
960 strerror(request->rename));
961 release_request(request);
962 return ret;
963 }
964
965 release_request(request);
966 return 0;
967}
968
969int fetch(unsigned char *sha1)
970{
971 struct alt_base *altbase = alt;
972
973 if (!fetch_object(altbase, sha1))
974 return 0;
975 while (altbase) {
976 if (!fetch_pack(altbase, sha1))
977 return 0;
978 altbase = altbase->next;
979 }
980 return error("Unable to find %s under %s\n", sha1_to_hex(sha1),
981 alt->base);
982}
983
984int fetch_ref(char *ref, unsigned char *sha1)
985{
986 char *url, *posn;
987 char hex[42];
988 struct buffer buffer;
989 char *base = alt->base;
990 struct active_request_slot *slot;
991 buffer.size = 41;
992 buffer.posn = 0;
993 buffer.buffer = hex;
994 hex[41] = '\0';
995
996 url = xmalloc(strlen(base) + 6 + strlen(ref));
997 strcpy(url, base);
998 posn = url + strlen(base);
999 strcpy(posn, "refs/");
1000 posn += 5;
1001 strcpy(posn, ref);
1002
1003 slot = get_active_slot();
1004 curl_easy_setopt(slot->curl, CURLOPT_FILE, &buffer);
1005 curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite_buffer);
1006 curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, NULL);
1007 curl_easy_setopt(slot->curl, CURLOPT_URL, url);
1008 if (start_active_slot(slot)) {
1009 run_active_slot(slot);
1010 if (slot->curl_result != CURLE_OK)
1011 return error("Couldn't get %s for %s\n%s",
1012 url, ref, curl_errorstr);
1013 } else {
1014 return error("Unable to start request");
1015 }
1016
1017 hex[40] = '\0';
1018 get_sha1_hex(hex, sha1);
1019 return 0;
1020}
1021
1022int main(int argc, char **argv)
1023{
1024 char *commit_id;
1025 char *url;
1026 int arg = 1;
1027 struct active_request_slot *slot;
1028
1029 while (arg < argc && argv[arg][0] == '-') {
1030 if (argv[arg][1] == 't') {
1031 get_tree = 1;
1032 } else if (argv[arg][1] == 'c') {
1033 get_history = 1;
1034 } else if (argv[arg][1] == 'a') {
1035 get_all = 1;
1036 get_tree = 1;
1037 get_history = 1;
1038 } else if (argv[arg][1] == 'v') {
1039 get_verbosely = 1;
1040 } else if (argv[arg][1] == 'w') {
1041 write_ref = argv[arg + 1];
1042 arg++;
1043 } else if (!strcmp(argv[arg], "--recover")) {
1044 get_recover = 1;
1045 }
1046 arg++;
1047 }
1048 if (argc < arg + 2) {
1049 usage("git-http-fetch [-c] [-t] [-a] [-d] [-v] [--recover] [-w ref] commit-id url");
1050 return 1;
1051 }
1052 commit_id = argv[arg];
1053 url = argv[arg + 1];
1054
1055 curl_global_init(CURL_GLOBAL_ALL);
1056
1057#ifdef USE_CURL_MULTI
1058 char *http_max_requests = getenv("GIT_HTTP_MAX_REQUESTS");
1059 if (http_max_requests != NULL)
1060 max_requests = atoi(http_max_requests);
1061 if (max_requests < 1)
1062 max_requests = DEFAULT_MAX_REQUESTS;
1063
1064 curlm = curl_multi_init();
1065 if (curlm == NULL) {
1066 fprintf(stderr, "Error creating curl multi handle.\n");
1067 return 1;
1068 }
1069#endif
1070 no_pragma_header = curl_slist_append(no_pragma_header, "Pragma:");
1071 no_range_header = curl_slist_append(no_range_header, "Range:");
1072
1073 curl_default = curl_easy_init();
1074
1075 curl_ssl_verify = getenv("GIT_SSL_NO_VERIFY") ? 0 : 1;
1076 curl_easy_setopt(curl_default, CURLOPT_SSL_VERIFYPEER, curl_ssl_verify);
1077#if LIBCURL_VERSION_NUM >= 0x070907
1078 curl_easy_setopt(curl_default, CURLOPT_NETRC, CURL_NETRC_OPTIONAL);
1079#endif
1080
1081 if ((ssl_cert = getenv("GIT_SSL_CERT")) != NULL) {
1082 curl_easy_setopt(curl_default, CURLOPT_SSLCERT, ssl_cert);
1083 }
1084#if LIBCURL_VERSION_NUM >= 0x070902
1085 if ((ssl_key = getenv("GIT_SSL_KEY")) != NULL) {
1086 curl_easy_setopt(curl_default, CURLOPT_SSLKEY, ssl_key);
1087 }
1088#endif
1089#if LIBCURL_VERSION_NUM >= 0x070908
1090 if ((ssl_capath = getenv("GIT_SSL_CAPATH")) != NULL) {
1091 curl_easy_setopt(curl_default, CURLOPT_CAPATH, ssl_capath);
1092 }
1093#endif
1094 if ((ssl_cainfo = getenv("GIT_SSL_CAINFO")) != NULL) {
1095 curl_easy_setopt(curl_default, CURLOPT_CAINFO, ssl_cainfo);
1096 }
1097 curl_easy_setopt(curl_default, CURLOPT_FAILONERROR, 1);
1098
1099 alt = xmalloc(sizeof(*alt));
1100 alt->base = url;
1101 alt->got_indices = 0;
1102 alt->packs = NULL;
1103 alt->next = NULL;
1104 fetch_alternates(alt->base);
1105
1106 if (pull(commit_id))
1107 return 1;
1108
1109 curl_slist_free_all(no_pragma_header);
1110 curl_slist_free_all(no_range_header);
1111 curl_easy_cleanup(curl_default);
1112 slot = active_queue_head;
1113 while (slot != NULL) {
1114 curl_easy_cleanup(slot->curl);
1115 slot = slot->next;
1116 }
1117#ifdef USE_CURL_MULTI
1118 curl_multi_cleanup(curlm);
1119#endif
1120 curl_global_cleanup();
1121 return 0;
1122}