bbc3023e40021877a9fd1bba9e36d97ca6e06a23
1#include "cache.h"
2#include "commit.h"
3#include "pack.h"
4#include "walker.h"
5#include "http.h"
6
7#define PREV_BUF_SIZE 4096
8#define RANGE_HEADER_SIZE 30
9
10struct alt_base
11{
12 char *base;
13 int got_indices;
14 struct packed_git *packs;
15 struct alt_base *next;
16};
17
18enum object_request_state {
19 WAITING,
20 ABORTED,
21 ACTIVE,
22 COMPLETE,
23};
24
25struct object_request
26{
27 struct walker *walker;
28 unsigned char sha1[20];
29 struct alt_base *repo;
30 char *url;
31 char filename[PATH_MAX];
32 char tmpfile[PATH_MAX];
33 int local;
34 enum object_request_state state;
35 CURLcode curl_result;
36 char errorstr[CURL_ERROR_SIZE];
37 long http_code;
38 unsigned char real_sha1[20];
39 git_SHA_CTX c;
40 z_stream stream;
41 int zret;
42 int rename;
43 struct active_request_slot *slot;
44 struct object_request *next;
45};
46
47struct alternates_request {
48 struct walker *walker;
49 const char *base;
50 char *url;
51 struct strbuf *buffer;
52 struct active_request_slot *slot;
53 int http_specific;
54};
55
56struct walker_data {
57 const char *url;
58 int got_alternates;
59 struct alt_base *alt;
60 struct curl_slist *no_pragma_header;
61};
62
63static struct object_request *object_queue_head;
64
65static size_t fwrite_sha1_file(void *ptr, size_t eltsize, size_t nmemb,
66 void *data)
67{
68 unsigned char expn[4096];
69 size_t size = eltsize * nmemb;
70 int posn = 0;
71 struct object_request *obj_req = (struct object_request *)data;
72 do {
73 ssize_t retval = xwrite(obj_req->local,
74 (char *) ptr + posn, size - posn);
75 if (retval < 0)
76 return posn;
77 posn += retval;
78 } while (posn < size);
79
80 obj_req->stream.avail_in = size;
81 obj_req->stream.next_in = ptr;
82 do {
83 obj_req->stream.next_out = expn;
84 obj_req->stream.avail_out = sizeof(expn);
85 obj_req->zret = git_inflate(&obj_req->stream, Z_SYNC_FLUSH);
86 git_SHA1_Update(&obj_req->c, expn,
87 sizeof(expn) - obj_req->stream.avail_out);
88 } while (obj_req->stream.avail_in && obj_req->zret == Z_OK);
89 data_received++;
90 return size;
91}
92
93static void fetch_alternates(struct walker *walker, const char *base);
94
95static void process_object_response(void *callback_data);
96
97static void start_object_request(struct walker *walker,
98 struct object_request *obj_req)
99{
100 char *hex = sha1_to_hex(obj_req->sha1);
101 char prevfile[PATH_MAX];
102 char *url;
103 char *posn;
104 int prevlocal;
105 unsigned char prev_buf[PREV_BUF_SIZE];
106 ssize_t prev_read = 0;
107 long prev_posn = 0;
108 char range[RANGE_HEADER_SIZE];
109 struct curl_slist *range_header = NULL;
110 struct active_request_slot *slot;
111 struct walker_data *data = walker->data;
112
113 snprintf(prevfile, sizeof(prevfile), "%s.prev", obj_req->filename);
114 unlink_or_warn(prevfile);
115 rename(obj_req->tmpfile, prevfile);
116 unlink_or_warn(obj_req->tmpfile);
117
118 if (obj_req->local != -1)
119 error("fd leakage in start: %d", obj_req->local);
120 obj_req->local = open(obj_req->tmpfile,
121 O_WRONLY | O_CREAT | O_EXCL, 0666);
122 /*
123 * This could have failed due to the "lazy directory creation";
124 * try to mkdir the last path component.
125 */
126 if (obj_req->local < 0 && errno == ENOENT) {
127 char *dir = strrchr(obj_req->tmpfile, '/');
128 if (dir) {
129 *dir = 0;
130 mkdir(obj_req->tmpfile, 0777);
131 *dir = '/';
132 }
133 obj_req->local = open(obj_req->tmpfile,
134 O_WRONLY | O_CREAT | O_EXCL, 0666);
135 }
136
137 if (obj_req->local < 0) {
138 obj_req->state = ABORTED;
139 error("Couldn't create temporary file %s for %s: %s",
140 obj_req->tmpfile, obj_req->filename, strerror(errno));
141 return;
142 }
143
144 memset(&obj_req->stream, 0, sizeof(obj_req->stream));
145
146 git_inflate_init(&obj_req->stream);
147
148 git_SHA1_Init(&obj_req->c);
149
150 url = xmalloc(strlen(obj_req->repo->base) + 51);
151 obj_req->url = xmalloc(strlen(obj_req->repo->base) + 51);
152 strcpy(url, obj_req->repo->base);
153 posn = url + strlen(obj_req->repo->base);
154 strcpy(posn, "/objects/");
155 posn += 9;
156 memcpy(posn, hex, 2);
157 posn += 2;
158 *(posn++) = '/';
159 strcpy(posn, hex + 2);
160 strcpy(obj_req->url, url);
161
162 /*
163 * If a previous temp file is present, process what was already
164 * fetched.
165 */
166 prevlocal = open(prevfile, O_RDONLY);
167 if (prevlocal != -1) {
168 do {
169 prev_read = xread(prevlocal, prev_buf, PREV_BUF_SIZE);
170 if (prev_read>0) {
171 if (fwrite_sha1_file(prev_buf,
172 1,
173 prev_read,
174 obj_req) == prev_read)
175 prev_posn += prev_read;
176 else
177 prev_read = -1;
178 }
179 } while (prev_read > 0);
180 close(prevlocal);
181 }
182 unlink_or_warn(prevfile);
183
184 /*
185 * Reset inflate/SHA1 if there was an error reading the previous temp
186 * file; also rewind to the beginning of the local file.
187 */
188 if (prev_read == -1) {
189 memset(&obj_req->stream, 0, sizeof(obj_req->stream));
190 git_inflate_init(&obj_req->stream);
191 git_SHA1_Init(&obj_req->c);
192 if (prev_posn>0) {
193 prev_posn = 0;
194 lseek(obj_req->local, 0, SEEK_SET);
195 ftruncate(obj_req->local, 0);
196 }
197 }
198
199 slot = get_active_slot();
200 slot->callback_func = process_object_response;
201 slot->callback_data = obj_req;
202 obj_req->slot = slot;
203
204 curl_easy_setopt(slot->curl, CURLOPT_FILE, obj_req);
205 curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite_sha1_file);
206 curl_easy_setopt(slot->curl, CURLOPT_ERRORBUFFER, obj_req->errorstr);
207 curl_easy_setopt(slot->curl, CURLOPT_URL, url);
208 curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, data->no_pragma_header);
209
210 /*
211 * If we have successfully processed data from a previous fetch
212 * attempt, only fetch the data we don't already have.
213 */
214 if (prev_posn>0) {
215 if (walker->get_verbosely)
216 fprintf(stderr,
217 "Resuming fetch of object %s at byte %ld\n",
218 hex, prev_posn);
219 sprintf(range, "Range: bytes=%ld-", prev_posn);
220 range_header = curl_slist_append(range_header, range);
221 curl_easy_setopt(slot->curl,
222 CURLOPT_HTTPHEADER, range_header);
223 }
224
225 /* Try to get the request started, abort the request on error */
226 obj_req->state = ACTIVE;
227 if (!start_active_slot(slot)) {
228 obj_req->state = ABORTED;
229 obj_req->slot = NULL;
230 close(obj_req->local);
231 obj_req->local = -1;
232 free(obj_req->url);
233 return;
234 }
235}
236
237static void finish_object_request(struct object_request *obj_req)
238{
239 struct stat st;
240
241 close(obj_req->local);
242 obj_req->local = -1;
243
244 if (obj_req->http_code == 416) {
245 fprintf(stderr, "Warning: requested range invalid; we may already have all the data.\n");
246 } else if (obj_req->curl_result != CURLE_OK) {
247 if (stat(obj_req->tmpfile, &st) == 0)
248 if (st.st_size == 0)
249 unlink_or_warn(obj_req->tmpfile);
250 return;
251 }
252
253 git_inflate_end(&obj_req->stream);
254 git_SHA1_Final(obj_req->real_sha1, &obj_req->c);
255 if (obj_req->zret != Z_STREAM_END) {
256 unlink_or_warn(obj_req->tmpfile);
257 return;
258 }
259 if (hashcmp(obj_req->sha1, obj_req->real_sha1)) {
260 unlink_or_warn(obj_req->tmpfile);
261 return;
262 }
263 obj_req->rename =
264 move_temp_to_file(obj_req->tmpfile, obj_req->filename);
265
266 if (obj_req->rename == 0)
267 walker_say(obj_req->walker, "got %s\n", sha1_to_hex(obj_req->sha1));
268}
269
270static void process_object_response(void *callback_data)
271{
272 struct object_request *obj_req =
273 (struct object_request *)callback_data;
274 struct walker *walker = obj_req->walker;
275 struct walker_data *data = walker->data;
276 struct alt_base *alt = data->alt;
277
278 obj_req->curl_result = obj_req->slot->curl_result;
279 obj_req->http_code = obj_req->slot->http_code;
280 obj_req->slot = NULL;
281 obj_req->state = COMPLETE;
282
283 /* Use alternates if necessary */
284 if (missing_target(obj_req)) {
285 fetch_alternates(walker, alt->base);
286 if (obj_req->repo->next != NULL) {
287 obj_req->repo =
288 obj_req->repo->next;
289 close(obj_req->local);
290 obj_req->local = -1;
291 start_object_request(walker, obj_req);
292 return;
293 }
294 }
295
296 finish_object_request(obj_req);
297}
298
299static void release_object_request(struct object_request *obj_req)
300{
301 struct object_request *entry = object_queue_head;
302
303 if (obj_req->local != -1)
304 error("fd leakage in release: %d", obj_req->local);
305 if (obj_req == object_queue_head) {
306 object_queue_head = obj_req->next;
307 } else {
308 while (entry->next != NULL && entry->next != obj_req)
309 entry = entry->next;
310 if (entry->next == obj_req)
311 entry->next = entry->next->next;
312 }
313
314 free(obj_req->url);
315 free(obj_req);
316}
317
318#ifdef USE_CURL_MULTI
319static int fill_active_slot(struct walker *walker)
320{
321 struct object_request *obj_req;
322
323 for (obj_req = object_queue_head; obj_req; obj_req = obj_req->next) {
324 if (obj_req->state == WAITING) {
325 if (has_sha1_file(obj_req->sha1))
326 obj_req->state = COMPLETE;
327 else {
328 start_object_request(walker, obj_req);
329 return 1;
330 }
331 }
332 }
333 return 0;
334}
335#endif
336
337static void prefetch(struct walker *walker, unsigned char *sha1)
338{
339 struct object_request *newreq;
340 struct object_request *tail;
341 struct walker_data *data = walker->data;
342 char *filename = sha1_file_name(sha1);
343
344 newreq = xmalloc(sizeof(*newreq));
345 newreq->walker = walker;
346 hashcpy(newreq->sha1, sha1);
347 newreq->repo = data->alt;
348 newreq->url = NULL;
349 newreq->local = -1;
350 newreq->state = WAITING;
351 snprintf(newreq->filename, sizeof(newreq->filename), "%s", filename);
352 snprintf(newreq->tmpfile, sizeof(newreq->tmpfile),
353 "%s.temp", filename);
354 newreq->slot = NULL;
355 newreq->next = NULL;
356
357 if (object_queue_head == NULL) {
358 object_queue_head = newreq;
359 } else {
360 tail = object_queue_head;
361 while (tail->next != NULL)
362 tail = tail->next;
363 tail->next = newreq;
364 }
365
366#ifdef USE_CURL_MULTI
367 fill_active_slots();
368 step_active_slots();
369#endif
370}
371
372static int fetch_index(struct walker *walker, struct alt_base *repo, unsigned char *sha1)
373{
374 char *hex = sha1_to_hex(sha1);
375 char *filename;
376 char *url;
377 char tmpfile[PATH_MAX];
378 long prev_posn = 0;
379 char range[RANGE_HEADER_SIZE];
380 struct curl_slist *range_header = NULL;
381 struct walker_data *data = walker->data;
382
383 FILE *indexfile;
384 struct active_request_slot *slot;
385 struct slot_results results;
386
387 /* Don't use the index if the pack isn't there */
388 url = xmalloc(strlen(repo->base) + 64);
389 sprintf(url, "%s/objects/pack/pack-%s.pack", repo->base, hex);
390 slot = get_active_slot();
391 slot->results = &results;
392 curl_easy_setopt(slot->curl, CURLOPT_URL, url);
393 curl_easy_setopt(slot->curl, CURLOPT_NOBODY, 1);
394 if (start_active_slot(slot)) {
395 run_active_slot(slot);
396 if (results.curl_result != CURLE_OK) {
397 free(url);
398 return error("Unable to verify pack %s is available",
399 hex);
400 }
401 } else {
402 free(url);
403 return error("Unable to start request");
404 }
405
406 if (has_pack_index(sha1)) {
407 free(url);
408 return 0;
409 }
410
411 if (walker->get_verbosely)
412 fprintf(stderr, "Getting index for pack %s\n", hex);
413
414 sprintf(url, "%s/objects/pack/pack-%s.idx", repo->base, hex);
415
416 filename = sha1_pack_index_name(sha1);
417 snprintf(tmpfile, sizeof(tmpfile), "%s.temp", filename);
418 indexfile = fopen(tmpfile, "a");
419 if (!indexfile) {
420 free(url);
421 return error("Unable to open local file %s for pack index",
422 tmpfile);
423 }
424
425 slot = get_active_slot();
426 slot->results = &results;
427 curl_easy_setopt(slot->curl, CURLOPT_NOBODY, 0);
428 curl_easy_setopt(slot->curl, CURLOPT_HTTPGET, 1);
429 curl_easy_setopt(slot->curl, CURLOPT_FILE, indexfile);
430 curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite);
431 curl_easy_setopt(slot->curl, CURLOPT_URL, url);
432 curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, data->no_pragma_header);
433 slot->local = indexfile;
434
435 /*
436 * If there is data present from a previous transfer attempt,
437 * resume where it left off
438 */
439 prev_posn = ftell(indexfile);
440 if (prev_posn>0) {
441 if (walker->get_verbosely)
442 fprintf(stderr,
443 "Resuming fetch of index for pack %s at byte %ld\n",
444 hex, prev_posn);
445 sprintf(range, "Range: bytes=%ld-", prev_posn);
446 range_header = curl_slist_append(range_header, range);
447 curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, range_header);
448 }
449
450 if (start_active_slot(slot)) {
451 run_active_slot(slot);
452 if (results.curl_result != CURLE_OK) {
453 free(url);
454 fclose(indexfile);
455 slot->local = NULL;
456 return error("Unable to get pack index %s\n%s", url,
457 curl_errorstr);
458 }
459 } else {
460 free(url);
461 fclose(indexfile);
462 slot->local = NULL;
463 return error("Unable to start request");
464 }
465
466 free(url);
467 fclose(indexfile);
468 slot->local = NULL;
469
470 return move_temp_to_file(tmpfile, filename);
471}
472
473static int setup_index(struct walker *walker, struct alt_base *repo, unsigned char *sha1)
474{
475 struct packed_git *new_pack;
476 if (has_pack_file(sha1))
477 return 0; /* don't list this as something we can get */
478
479 if (fetch_index(walker, repo, sha1))
480 return -1;
481
482 new_pack = parse_pack_index(sha1);
483 if (!new_pack)
484 return -1; /* parse_pack_index() already issued error message */
485 new_pack->next = repo->packs;
486 repo->packs = new_pack;
487 return 0;
488}
489
490static void process_alternates_response(void *callback_data)
491{
492 struct alternates_request *alt_req =
493 (struct alternates_request *)callback_data;
494 struct walker *walker = alt_req->walker;
495 struct walker_data *cdata = walker->data;
496 struct active_request_slot *slot = alt_req->slot;
497 struct alt_base *tail = cdata->alt;
498 const char *base = alt_req->base;
499 static const char null_byte = '\0';
500 char *data;
501 int i = 0;
502
503 if (alt_req->http_specific) {
504 if (slot->curl_result != CURLE_OK ||
505 !alt_req->buffer->len) {
506
507 /* Try reusing the slot to get non-http alternates */
508 alt_req->http_specific = 0;
509 sprintf(alt_req->url, "%s/objects/info/alternates",
510 base);
511 curl_easy_setopt(slot->curl, CURLOPT_URL,
512 alt_req->url);
513 active_requests++;
514 slot->in_use = 1;
515 if (slot->finished != NULL)
516 (*slot->finished) = 0;
517 if (!start_active_slot(slot)) {
518 cdata->got_alternates = -1;
519 slot->in_use = 0;
520 if (slot->finished != NULL)
521 (*slot->finished) = 1;
522 }
523 return;
524 }
525 } else if (slot->curl_result != CURLE_OK) {
526 if (!missing_target(slot)) {
527 cdata->got_alternates = -1;
528 return;
529 }
530 }
531
532 fwrite_buffer(&null_byte, 1, 1, alt_req->buffer);
533 alt_req->buffer->len--;
534 data = alt_req->buffer->buf;
535
536 while (i < alt_req->buffer->len) {
537 int posn = i;
538 while (posn < alt_req->buffer->len && data[posn] != '\n')
539 posn++;
540 if (data[posn] == '\n') {
541 int okay = 0;
542 int serverlen = 0;
543 struct alt_base *newalt;
544 char *target = NULL;
545 if (data[i] == '/') {
546 /*
547 * This counts
548 * http://git.host/pub/scm/linux.git/
549 * -----------here^
550 * so memcpy(dst, base, serverlen) will
551 * copy up to "...git.host".
552 */
553 const char *colon_ss = strstr(base,"://");
554 if (colon_ss) {
555 serverlen = (strchr(colon_ss + 3, '/')
556 - base);
557 okay = 1;
558 }
559 } else if (!memcmp(data + i, "../", 3)) {
560 /*
561 * Relative URL; chop the corresponding
562 * number of subpath from base (and ../
563 * from data), and concatenate the result.
564 *
565 * The code first drops ../ from data, and
566 * then drops one ../ from data and one path
567 * from base. IOW, one extra ../ is dropped
568 * from data than path is dropped from base.
569 *
570 * This is not wrong. The alternate in
571 * http://git.host/pub/scm/linux.git/
572 * to borrow from
573 * http://git.host/pub/scm/linus.git/
574 * is ../../linus.git/objects/. You need
575 * two ../../ to borrow from your direct
576 * neighbour.
577 */
578 i += 3;
579 serverlen = strlen(base);
580 while (i + 2 < posn &&
581 !memcmp(data + i, "../", 3)) {
582 do {
583 serverlen--;
584 } while (serverlen &&
585 base[serverlen - 1] != '/');
586 i += 3;
587 }
588 /* If the server got removed, give up. */
589 okay = strchr(base, ':') - base + 3 <
590 serverlen;
591 } else if (alt_req->http_specific) {
592 char *colon = strchr(data + i, ':');
593 char *slash = strchr(data + i, '/');
594 if (colon && slash && colon < data + posn &&
595 slash < data + posn && colon < slash) {
596 okay = 1;
597 }
598 }
599 /* skip "objects\n" at end */
600 if (okay) {
601 target = xmalloc(serverlen + posn - i - 6);
602 memcpy(target, base, serverlen);
603 memcpy(target + serverlen, data + i,
604 posn - i - 7);
605 target[serverlen + posn - i - 7] = 0;
606 if (walker->get_verbosely)
607 fprintf(stderr,
608 "Also look at %s\n", target);
609 newalt = xmalloc(sizeof(*newalt));
610 newalt->next = NULL;
611 newalt->base = target;
612 newalt->got_indices = 0;
613 newalt->packs = NULL;
614
615 while (tail->next != NULL)
616 tail = tail->next;
617 tail->next = newalt;
618 }
619 }
620 i = posn + 1;
621 }
622
623 cdata->got_alternates = 1;
624}
625
626static void fetch_alternates(struct walker *walker, const char *base)
627{
628 struct strbuf buffer = STRBUF_INIT;
629 char *url;
630 struct active_request_slot *slot;
631 struct alternates_request alt_req;
632 struct walker_data *cdata = walker->data;
633
634 /*
635 * If another request has already started fetching alternates,
636 * wait for them to arrive and return to processing this request's
637 * curl message
638 */
639#ifdef USE_CURL_MULTI
640 while (cdata->got_alternates == 0) {
641 step_active_slots();
642 }
643#endif
644
645 /* Nothing to do if they've already been fetched */
646 if (cdata->got_alternates == 1)
647 return;
648
649 /* Start the fetch */
650 cdata->got_alternates = 0;
651
652 if (walker->get_verbosely)
653 fprintf(stderr, "Getting alternates list for %s\n", base);
654
655 url = xmalloc(strlen(base) + 31);
656 sprintf(url, "%s/objects/info/http-alternates", base);
657
658 /*
659 * Use a callback to process the result, since another request
660 * may fail and need to have alternates loaded before continuing
661 */
662 slot = get_active_slot();
663 slot->callback_func = process_alternates_response;
664 alt_req.walker = walker;
665 slot->callback_data = &alt_req;
666
667 curl_easy_setopt(slot->curl, CURLOPT_FILE, &buffer);
668 curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite_buffer);
669 curl_easy_setopt(slot->curl, CURLOPT_URL, url);
670
671 alt_req.base = base;
672 alt_req.url = url;
673 alt_req.buffer = &buffer;
674 alt_req.http_specific = 1;
675 alt_req.slot = slot;
676
677 if (start_active_slot(slot))
678 run_active_slot(slot);
679 else
680 cdata->got_alternates = -1;
681
682 strbuf_release(&buffer);
683 free(url);
684}
685
686static int fetch_indices(struct walker *walker, struct alt_base *repo)
687{
688 unsigned char sha1[20];
689 char *url;
690 struct strbuf buffer = STRBUF_INIT;
691 char *data;
692 int i = 0;
693 int ret = 0;
694
695 struct active_request_slot *slot;
696 struct slot_results results;
697
698 if (repo->got_indices)
699 return 0;
700
701 if (walker->get_verbosely)
702 fprintf(stderr, "Getting pack list for %s\n", repo->base);
703
704 url = xmalloc(strlen(repo->base) + 21);
705 sprintf(url, "%s/objects/info/packs", repo->base);
706
707 slot = get_active_slot();
708 slot->results = &results;
709 curl_easy_setopt(slot->curl, CURLOPT_FILE, &buffer);
710 curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite_buffer);
711 curl_easy_setopt(slot->curl, CURLOPT_URL, url);
712 curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, NULL);
713 if (start_active_slot(slot)) {
714 run_active_slot(slot);
715 if (results.curl_result != CURLE_OK) {
716 if (missing_target(&results)) {
717 repo->got_indices = 1;
718 goto cleanup;
719 } else {
720 repo->got_indices = 0;
721 ret = error("%s", curl_errorstr);
722 goto cleanup;
723 }
724 }
725 } else {
726 repo->got_indices = 0;
727 ret = error("Unable to start request");
728 goto cleanup;
729 }
730
731 data = buffer.buf;
732 while (i < buffer.len) {
733 switch (data[i]) {
734 case 'P':
735 i++;
736 if (i + 52 <= buffer.len &&
737 !prefixcmp(data + i, " pack-") &&
738 !prefixcmp(data + i + 46, ".pack\n")) {
739 get_sha1_hex(data + i + 6, sha1);
740 setup_index(walker, repo, sha1);
741 i += 51;
742 break;
743 }
744 default:
745 while (i < buffer.len && data[i] != '\n')
746 i++;
747 }
748 i++;
749 }
750
751 repo->got_indices = 1;
752cleanup:
753 strbuf_release(&buffer);
754 free(url);
755 return ret;
756}
757
758static int fetch_pack(struct walker *walker, struct alt_base *repo, unsigned char *sha1)
759{
760 char *url;
761 struct packed_git *target;
762 struct packed_git **lst;
763 FILE *packfile;
764 char *filename;
765 char tmpfile[PATH_MAX];
766 int ret;
767 long prev_posn = 0;
768 char range[RANGE_HEADER_SIZE];
769 struct curl_slist *range_header = NULL;
770 struct walker_data *data = walker->data;
771
772 struct active_request_slot *slot;
773 struct slot_results results;
774
775 if (fetch_indices(walker, repo))
776 return -1;
777 target = find_sha1_pack(sha1, repo->packs);
778 if (!target)
779 return -1;
780
781 if (walker->get_verbosely) {
782 fprintf(stderr, "Getting pack %s\n",
783 sha1_to_hex(target->sha1));
784 fprintf(stderr, " which contains %s\n",
785 sha1_to_hex(sha1));
786 }
787
788 url = xmalloc(strlen(repo->base) + 65);
789 sprintf(url, "%s/objects/pack/pack-%s.pack",
790 repo->base, sha1_to_hex(target->sha1));
791
792 filename = sha1_pack_name(target->sha1);
793 snprintf(tmpfile, sizeof(tmpfile), "%s.temp", filename);
794 packfile = fopen(tmpfile, "a");
795 if (!packfile)
796 return error("Unable to open local file %s for pack",
797 tmpfile);
798
799 slot = get_active_slot();
800 slot->results = &results;
801 curl_easy_setopt(slot->curl, CURLOPT_FILE, packfile);
802 curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite);
803 curl_easy_setopt(slot->curl, CURLOPT_URL, url);
804 curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, data->no_pragma_header);
805 slot->local = packfile;
806
807 /*
808 * If there is data present from a previous transfer attempt,
809 * resume where it left off
810 */
811 prev_posn = ftell(packfile);
812 if (prev_posn>0) {
813 if (walker->get_verbosely)
814 fprintf(stderr,
815 "Resuming fetch of pack %s at byte %ld\n",
816 sha1_to_hex(target->sha1), prev_posn);
817 sprintf(range, "Range: bytes=%ld-", prev_posn);
818 range_header = curl_slist_append(range_header, range);
819 curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, range_header);
820 }
821
822 if (start_active_slot(slot)) {
823 run_active_slot(slot);
824 if (results.curl_result != CURLE_OK) {
825 fclose(packfile);
826 slot->local = NULL;
827 return error("Unable to get pack file %s\n%s", url,
828 curl_errorstr);
829 }
830 } else {
831 fclose(packfile);
832 slot->local = NULL;
833 return error("Unable to start request");
834 }
835
836 target->pack_size = ftell(packfile);
837 fclose(packfile);
838 slot->local = NULL;
839
840 ret = move_temp_to_file(tmpfile, filename);
841 if (ret)
842 return ret;
843
844 lst = &repo->packs;
845 while (*lst != target)
846 lst = &((*lst)->next);
847 *lst = (*lst)->next;
848
849 if (verify_pack(target))
850 return -1;
851 install_packed_git(target);
852
853 return 0;
854}
855
856static void abort_object_request(struct object_request *obj_req)
857{
858 if (obj_req->local >= 0) {
859 close(obj_req->local);
860 obj_req->local = -1;
861 }
862 unlink_or_warn(obj_req->tmpfile);
863 if (obj_req->slot) {
864 release_active_slot(obj_req->slot);
865 obj_req->slot = NULL;
866 }
867 release_object_request(obj_req);
868}
869
870static int fetch_object(struct walker *walker, struct alt_base *repo, unsigned char *sha1)
871{
872 char *hex = sha1_to_hex(sha1);
873 int ret = 0;
874 struct object_request *obj_req = object_queue_head;
875
876 while (obj_req != NULL && hashcmp(obj_req->sha1, sha1))
877 obj_req = obj_req->next;
878 if (obj_req == NULL)
879 return error("Couldn't find request for %s in the queue", hex);
880
881 if (has_sha1_file(obj_req->sha1)) {
882 abort_object_request(obj_req);
883 return 0;
884 }
885
886#ifdef USE_CURL_MULTI
887 while (obj_req->state == WAITING)
888 step_active_slots();
889#else
890 start_object_request(walker, obj_req);
891#endif
892
893 while (obj_req->state == ACTIVE)
894 run_active_slot(obj_req->slot);
895
896 if (obj_req->local != -1) {
897 close(obj_req->local);
898 obj_req->local = -1;
899 }
900
901 if (obj_req->state == ABORTED) {
902 ret = error("Request for %s aborted", hex);
903 } else if (obj_req->curl_result != CURLE_OK &&
904 obj_req->http_code != 416) {
905 if (missing_target(obj_req))
906 ret = -1; /* Be silent, it is probably in a pack. */
907 else
908 ret = error("%s (curl_result = %d, http_code = %ld, sha1 = %s)",
909 obj_req->errorstr, obj_req->curl_result,
910 obj_req->http_code, hex);
911 } else if (obj_req->zret != Z_STREAM_END) {
912 walker->corrupt_object_found++;
913 ret = error("File %s (%s) corrupt", hex, obj_req->url);
914 } else if (hashcmp(obj_req->sha1, obj_req->real_sha1)) {
915 ret = error("File %s has bad hash", hex);
916 } else if (obj_req->rename < 0) {
917 ret = error("unable to write sha1 filename %s",
918 obj_req->filename);
919 }
920
921 release_object_request(obj_req);
922 return ret;
923}
924
925static int fetch(struct walker *walker, unsigned char *sha1)
926{
927 struct walker_data *data = walker->data;
928 struct alt_base *altbase = data->alt;
929
930 if (!fetch_object(walker, altbase, sha1))
931 return 0;
932 while (altbase) {
933 if (!fetch_pack(walker, altbase, sha1))
934 return 0;
935 fetch_alternates(walker, data->alt->base);
936 altbase = altbase->next;
937 }
938 return error("Unable to find %s under %s", sha1_to_hex(sha1),
939 data->alt->base);
940}
941
942static int fetch_ref(struct walker *walker, struct ref *ref)
943{
944 struct walker_data *data = walker->data;
945 return http_fetch_ref(data->alt->base, ref);
946}
947
948static void cleanup(struct walker *walker)
949{
950 struct walker_data *data = walker->data;
951 http_cleanup();
952
953 curl_slist_free_all(data->no_pragma_header);
954}
955
956struct walker *get_http_walker(const char *url, struct remote *remote)
957{
958 char *s;
959 struct walker_data *data = xmalloc(sizeof(struct walker_data));
960 struct walker *walker = xmalloc(sizeof(struct walker));
961
962 http_init(remote);
963
964 data->no_pragma_header = curl_slist_append(NULL, "Pragma:");
965
966 data->alt = xmalloc(sizeof(*data->alt));
967 data->alt->base = xmalloc(strlen(url) + 1);
968 strcpy(data->alt->base, url);
969 for (s = data->alt->base + strlen(data->alt->base) - 1; *s == '/'; --s)
970 *s = 0;
971
972 data->alt->got_indices = 0;
973 data->alt->packs = NULL;
974 data->alt->next = NULL;
975 data->got_alternates = -1;
976
977 walker->corrupt_object_found = 0;
978 walker->fetch = fetch;
979 walker->fetch_ref = fetch_ref;
980 walker->prefetch = prefetch;
981 walker->cleanup = cleanup;
982 walker->data = data;
983
984#ifdef USE_CURL_MULTI
985 add_fill_function(walker, (int (*)(void *)) fill_active_slot);
986#endif
987
988 return walker;
989}