1b86a3ae71af4a34fcc1de28ece728cc0a21588d
1/*
2 * GIT - The information manager from hell
3 *
4 * Copyright (C) Linus Torvalds, 2005
5 *
6 * This handles basic git sha1 object files - packing, unpacking,
7 * creation etc.
8 */
9#include "cache.h"
10#include "string-list.h"
11#include "lockfile.h"
12#include "delta.h"
13#include "pack.h"
14#include "blob.h"
15#include "commit.h"
16#include "run-command.h"
17#include "tag.h"
18#include "tree.h"
19#include "tree-walk.h"
20#include "refs.h"
21#include "pack-revindex.h"
22#include "sha1-lookup.h"
23#include "bulk-checkin.h"
24#include "streaming.h"
25#include "dir.h"
26#include "mru.h"
27#include "list.h"
28#include "mergesort.h"
29
30#ifndef O_NOATIME
31#if defined(__linux__) && (defined(__i386__) || defined(__PPC__))
32#define O_NOATIME 01000000
33#else
34#define O_NOATIME 0
35#endif
36#endif
37
38#define SZ_FMT PRIuMAX
39static inline uintmax_t sz_fmt(size_t s) { return s; }
40
41const unsigned char null_sha1[20];
42const struct object_id null_oid;
43const struct object_id empty_tree_oid = {
44 EMPTY_TREE_SHA1_BIN_LITERAL
45};
46const struct object_id empty_blob_oid = {
47 EMPTY_BLOB_SHA1_BIN_LITERAL
48};
49
50/*
51 * This is meant to hold a *small* number of objects that you would
52 * want read_sha1_file() to be able to return, but yet you do not want
53 * to write them into the object store (e.g. a browse-only
54 * application).
55 */
56static struct cached_object {
57 unsigned char sha1[20];
58 enum object_type type;
59 void *buf;
60 unsigned long size;
61} *cached_objects;
62static int cached_object_nr, cached_object_alloc;
63
64static struct cached_object empty_tree = {
65 EMPTY_TREE_SHA1_BIN_LITERAL,
66 OBJ_TREE,
67 "",
68 0
69};
70
71static struct cached_object *find_cached_object(const unsigned char *sha1)
72{
73 int i;
74 struct cached_object *co = cached_objects;
75
76 for (i = 0; i < cached_object_nr; i++, co++) {
77 if (!hashcmp(co->sha1, sha1))
78 return co;
79 }
80 if (!hashcmp(sha1, empty_tree.sha1))
81 return &empty_tree;
82 return NULL;
83}
84
85int mkdir_in_gitdir(const char *path)
86{
87 if (mkdir(path, 0777)) {
88 int saved_errno = errno;
89 struct stat st;
90 struct strbuf sb = STRBUF_INIT;
91
92 if (errno != EEXIST)
93 return -1;
94 /*
95 * Are we looking at a path in a symlinked worktree
96 * whose original repository does not yet have it?
97 * e.g. .git/rr-cache pointing at its original
98 * repository in which the user hasn't performed any
99 * conflict resolution yet?
100 */
101 if (lstat(path, &st) || !S_ISLNK(st.st_mode) ||
102 strbuf_readlink(&sb, path, st.st_size) ||
103 !is_absolute_path(sb.buf) ||
104 mkdir(sb.buf, 0777)) {
105 strbuf_release(&sb);
106 errno = saved_errno;
107 return -1;
108 }
109 strbuf_release(&sb);
110 }
111 return adjust_shared_perm(path);
112}
113
114enum scld_error safe_create_leading_directories(char *path)
115{
116 char *next_component = path + offset_1st_component(path);
117 enum scld_error ret = SCLD_OK;
118
119 while (ret == SCLD_OK && next_component) {
120 struct stat st;
121 char *slash = next_component, slash_character;
122
123 while (*slash && !is_dir_sep(*slash))
124 slash++;
125
126 if (!*slash)
127 break;
128
129 next_component = slash + 1;
130 while (is_dir_sep(*next_component))
131 next_component++;
132 if (!*next_component)
133 break;
134
135 slash_character = *slash;
136 *slash = '\0';
137 if (!stat(path, &st)) {
138 /* path exists */
139 if (!S_ISDIR(st.st_mode))
140 ret = SCLD_EXISTS;
141 } else if (mkdir(path, 0777)) {
142 if (errno == EEXIST &&
143 !stat(path, &st) && S_ISDIR(st.st_mode))
144 ; /* somebody created it since we checked */
145 else if (errno == ENOENT)
146 /*
147 * Either mkdir() failed because
148 * somebody just pruned the containing
149 * directory, or stat() failed because
150 * the file that was in our way was
151 * just removed. Either way, inform
152 * the caller that it might be worth
153 * trying again:
154 */
155 ret = SCLD_VANISHED;
156 else
157 ret = SCLD_FAILED;
158 } else if (adjust_shared_perm(path)) {
159 ret = SCLD_PERMS;
160 }
161 *slash = slash_character;
162 }
163 return ret;
164}
165
166enum scld_error safe_create_leading_directories_const(const char *path)
167{
168 /* path points to cache entries, so xstrdup before messing with it */
169 char *buf = xstrdup(path);
170 enum scld_error result = safe_create_leading_directories(buf);
171 free(buf);
172 return result;
173}
174
175static void fill_sha1_path(struct strbuf *buf, const unsigned char *sha1)
176{
177 int i;
178 for (i = 0; i < 20; i++) {
179 static char hex[] = "0123456789abcdef";
180 unsigned int val = sha1[i];
181 strbuf_addch(buf, hex[val >> 4]);
182 strbuf_addch(buf, hex[val & 0xf]);
183 if (!i)
184 strbuf_addch(buf, '/');
185 }
186}
187
188const char *sha1_file_name(const unsigned char *sha1)
189{
190 static struct strbuf buf = STRBUF_INIT;
191
192 strbuf_reset(&buf);
193 strbuf_addf(&buf, "%s/", get_object_directory());
194
195 fill_sha1_path(&buf, sha1);
196 return buf.buf;
197}
198
199struct strbuf *alt_scratch_buf(struct alternate_object_database *alt)
200{
201 strbuf_setlen(&alt->scratch, alt->base_len);
202 return &alt->scratch;
203}
204
205static const char *alt_sha1_path(struct alternate_object_database *alt,
206 const unsigned char *sha1)
207{
208 struct strbuf *buf = alt_scratch_buf(alt);
209 fill_sha1_path(buf, sha1);
210 return buf->buf;
211}
212
213/*
214 * Return the name of the pack or index file with the specified sha1
215 * in its filename. *base and *name are scratch space that must be
216 * provided by the caller. which should be "pack" or "idx".
217 */
218static char *sha1_get_pack_name(const unsigned char *sha1,
219 struct strbuf *buf,
220 const char *which)
221{
222 strbuf_reset(buf);
223 strbuf_addf(buf, "%s/pack/pack-%s.%s", get_object_directory(),
224 sha1_to_hex(sha1), which);
225 return buf->buf;
226}
227
228char *sha1_pack_name(const unsigned char *sha1)
229{
230 static struct strbuf buf = STRBUF_INIT;
231 return sha1_get_pack_name(sha1, &buf, "pack");
232}
233
234char *sha1_pack_index_name(const unsigned char *sha1)
235{
236 static struct strbuf buf = STRBUF_INIT;
237 return sha1_get_pack_name(sha1, &buf, "idx");
238}
239
240struct alternate_object_database *alt_odb_list;
241static struct alternate_object_database **alt_odb_tail;
242
243/*
244 * Return non-zero iff the path is usable as an alternate object database.
245 */
246static int alt_odb_usable(struct strbuf *path, const char *normalized_objdir)
247{
248 struct alternate_object_database *alt;
249
250 /* Detect cases where alternate disappeared */
251 if (!is_directory(path->buf)) {
252 error("object directory %s does not exist; "
253 "check .git/objects/info/alternates.",
254 path->buf);
255 return 0;
256 }
257
258 /*
259 * Prevent the common mistake of listing the same
260 * thing twice, or object directory itself.
261 */
262 for (alt = alt_odb_list; alt; alt = alt->next) {
263 if (!strcmp(path->buf, alt->path))
264 return 0;
265 }
266 if (!fspathcmp(path->buf, normalized_objdir))
267 return 0;
268
269 return 1;
270}
271
272/*
273 * Prepare alternate object database registry.
274 *
275 * The variable alt_odb_list points at the list of struct
276 * alternate_object_database. The elements on this list come from
277 * non-empty elements from colon separated ALTERNATE_DB_ENVIRONMENT
278 * environment variable, and $GIT_OBJECT_DIRECTORY/info/alternates,
279 * whose contents is similar to that environment variable but can be
280 * LF separated. Its base points at a statically allocated buffer that
281 * contains "/the/directory/corresponding/to/.git/objects/...", while
282 * its name points just after the slash at the end of ".git/objects/"
283 * in the example above, and has enough space to hold 40-byte hex
284 * SHA1, an extra slash for the first level indirection, and the
285 * terminating NUL.
286 */
287static int link_alt_odb_entry(const char *entry, const char *relative_base,
288 int depth, const char *normalized_objdir)
289{
290 struct alternate_object_database *ent;
291 struct strbuf pathbuf = STRBUF_INIT;
292
293 if (!is_absolute_path(entry) && relative_base) {
294 strbuf_addstr(&pathbuf, real_path(relative_base));
295 strbuf_addch(&pathbuf, '/');
296 }
297 strbuf_addstr(&pathbuf, entry);
298
299 if (strbuf_normalize_path(&pathbuf) < 0) {
300 error("unable to normalize alternate object path: %s",
301 pathbuf.buf);
302 strbuf_release(&pathbuf);
303 return -1;
304 }
305
306 /*
307 * The trailing slash after the directory name is given by
308 * this function at the end. Remove duplicates.
309 */
310 while (pathbuf.len && pathbuf.buf[pathbuf.len - 1] == '/')
311 strbuf_setlen(&pathbuf, pathbuf.len - 1);
312
313 if (!alt_odb_usable(&pathbuf, normalized_objdir)) {
314 strbuf_release(&pathbuf);
315 return -1;
316 }
317
318 ent = alloc_alt_odb(pathbuf.buf);
319
320 /* add the alternate entry */
321 *alt_odb_tail = ent;
322 alt_odb_tail = &(ent->next);
323 ent->next = NULL;
324
325 /* recursively add alternates */
326 read_info_alternates(pathbuf.buf, depth + 1);
327
328 strbuf_release(&pathbuf);
329 return 0;
330}
331
332static void link_alt_odb_entries(const char *alt, int len, int sep,
333 const char *relative_base, int depth)
334{
335 struct string_list entries = STRING_LIST_INIT_NODUP;
336 char *alt_copy;
337 int i;
338 struct strbuf objdirbuf = STRBUF_INIT;
339
340 if (depth > 5) {
341 error("%s: ignoring alternate object stores, nesting too deep.",
342 relative_base);
343 return;
344 }
345
346 strbuf_add_absolute_path(&objdirbuf, get_object_directory());
347 if (strbuf_normalize_path(&objdirbuf) < 0)
348 die("unable to normalize object directory: %s",
349 objdirbuf.buf);
350
351 alt_copy = xmemdupz(alt, len);
352 string_list_split_in_place(&entries, alt_copy, sep, -1);
353 for (i = 0; i < entries.nr; i++) {
354 const char *entry = entries.items[i].string;
355 if (entry[0] == '\0' || entry[0] == '#')
356 continue;
357 if (!is_absolute_path(entry) && depth) {
358 error("%s: ignoring relative alternate object store %s",
359 relative_base, entry);
360 } else {
361 link_alt_odb_entry(entry, relative_base, depth, objdirbuf.buf);
362 }
363 }
364 string_list_clear(&entries, 0);
365 free(alt_copy);
366 strbuf_release(&objdirbuf);
367}
368
369void read_info_alternates(const char * relative_base, int depth)
370{
371 char *map;
372 size_t mapsz;
373 struct stat st;
374 char *path;
375 int fd;
376
377 path = xstrfmt("%s/info/alternates", relative_base);
378 fd = git_open_noatime(path);
379 free(path);
380 if (fd < 0)
381 return;
382 if (fstat(fd, &st) || (st.st_size == 0)) {
383 close(fd);
384 return;
385 }
386 mapsz = xsize_t(st.st_size);
387 map = xmmap(NULL, mapsz, PROT_READ, MAP_PRIVATE, fd, 0);
388 close(fd);
389
390 link_alt_odb_entries(map, mapsz, '\n', relative_base, depth);
391
392 munmap(map, mapsz);
393}
394
395struct alternate_object_database *alloc_alt_odb(const char *dir)
396{
397 struct alternate_object_database *ent;
398
399 FLEX_ALLOC_STR(ent, path, dir);
400 strbuf_init(&ent->scratch, 0);
401 strbuf_addf(&ent->scratch, "%s/", dir);
402 ent->base_len = ent->scratch.len;
403
404 return ent;
405}
406
407void add_to_alternates_file(const char *reference)
408{
409 struct lock_file *lock = xcalloc(1, sizeof(struct lock_file));
410 char *alts = git_pathdup("objects/info/alternates");
411 FILE *in, *out;
412
413 hold_lock_file_for_update(lock, alts, LOCK_DIE_ON_ERROR);
414 out = fdopen_lock_file(lock, "w");
415 if (!out)
416 die_errno("unable to fdopen alternates lockfile");
417
418 in = fopen(alts, "r");
419 if (in) {
420 struct strbuf line = STRBUF_INIT;
421 int found = 0;
422
423 while (strbuf_getline(&line, in) != EOF) {
424 if (!strcmp(reference, line.buf)) {
425 found = 1;
426 break;
427 }
428 fprintf_or_die(out, "%s\n", line.buf);
429 }
430
431 strbuf_release(&line);
432 fclose(in);
433
434 if (found) {
435 rollback_lock_file(lock);
436 lock = NULL;
437 }
438 }
439 else if (errno != ENOENT)
440 die_errno("unable to read alternates file");
441
442 if (lock) {
443 fprintf_or_die(out, "%s\n", reference);
444 if (commit_lock_file(lock))
445 die_errno("unable to move new alternates file into place");
446 if (alt_odb_tail)
447 link_alt_odb_entries(reference, strlen(reference), '\n', NULL, 0);
448 }
449 free(alts);
450}
451
452void add_to_alternates_memory(const char *reference)
453{
454 /*
455 * Make sure alternates are initialized, or else our entry may be
456 * overwritten when they are.
457 */
458 prepare_alt_odb();
459
460 link_alt_odb_entries(reference, strlen(reference), '\n', NULL, 0);
461}
462
463/*
464 * Compute the exact path an alternate is at and returns it. In case of
465 * error NULL is returned and the human readable error is added to `err`
466 * `path` may be relative and should point to $GITDIR.
467 * `err` must not be null.
468 */
469char *compute_alternate_path(const char *path, struct strbuf *err)
470{
471 char *ref_git = NULL;
472 const char *repo, *ref_git_s;
473 int seen_error = 0;
474
475 ref_git_s = real_path_if_valid(path);
476 if (!ref_git_s) {
477 seen_error = 1;
478 strbuf_addf(err, _("path '%s' does not exist"), path);
479 goto out;
480 } else
481 /*
482 * Beware: read_gitfile(), real_path() and mkpath()
483 * return static buffer
484 */
485 ref_git = xstrdup(ref_git_s);
486
487 repo = read_gitfile(ref_git);
488 if (!repo)
489 repo = read_gitfile(mkpath("%s/.git", ref_git));
490 if (repo) {
491 free(ref_git);
492 ref_git = xstrdup(repo);
493 }
494
495 if (!repo && is_directory(mkpath("%s/.git/objects", ref_git))) {
496 char *ref_git_git = mkpathdup("%s/.git", ref_git);
497 free(ref_git);
498 ref_git = ref_git_git;
499 } else if (!is_directory(mkpath("%s/objects", ref_git))) {
500 struct strbuf sb = STRBUF_INIT;
501 seen_error = 1;
502 if (get_common_dir(&sb, ref_git)) {
503 strbuf_addf(err,
504 _("reference repository '%s' as a linked "
505 "checkout is not supported yet."),
506 path);
507 goto out;
508 }
509
510 strbuf_addf(err, _("reference repository '%s' is not a "
511 "local repository."), path);
512 goto out;
513 }
514
515 if (!access(mkpath("%s/shallow", ref_git), F_OK)) {
516 strbuf_addf(err, _("reference repository '%s' is shallow"),
517 path);
518 seen_error = 1;
519 goto out;
520 }
521
522 if (!access(mkpath("%s/info/grafts", ref_git), F_OK)) {
523 strbuf_addf(err,
524 _("reference repository '%s' is grafted"),
525 path);
526 seen_error = 1;
527 goto out;
528 }
529
530out:
531 if (seen_error) {
532 free(ref_git);
533 ref_git = NULL;
534 }
535
536 return ref_git;
537}
538
539int foreach_alt_odb(alt_odb_fn fn, void *cb)
540{
541 struct alternate_object_database *ent;
542 int r = 0;
543
544 prepare_alt_odb();
545 for (ent = alt_odb_list; ent; ent = ent->next) {
546 r = fn(ent, cb);
547 if (r)
548 break;
549 }
550 return r;
551}
552
553void prepare_alt_odb(void)
554{
555 const char *alt;
556
557 if (alt_odb_tail)
558 return;
559
560 alt = getenv(ALTERNATE_DB_ENVIRONMENT);
561 if (!alt) alt = "";
562
563 alt_odb_tail = &alt_odb_list;
564 link_alt_odb_entries(alt, strlen(alt), PATH_SEP, NULL, 0);
565
566 read_info_alternates(get_object_directory(), 0);
567}
568
569/* Returns 1 if we have successfully freshened the file, 0 otherwise. */
570static int freshen_file(const char *fn)
571{
572 struct utimbuf t;
573 t.actime = t.modtime = time(NULL);
574 return !utime(fn, &t);
575}
576
577/*
578 * All of the check_and_freshen functions return 1 if the file exists and was
579 * freshened (if freshening was requested), 0 otherwise. If they return
580 * 0, you should not assume that it is safe to skip a write of the object (it
581 * either does not exist on disk, or has a stale mtime and may be subject to
582 * pruning).
583 */
584static int check_and_freshen_file(const char *fn, int freshen)
585{
586 if (access(fn, F_OK))
587 return 0;
588 if (freshen && !freshen_file(fn))
589 return 0;
590 return 1;
591}
592
593static int check_and_freshen_local(const unsigned char *sha1, int freshen)
594{
595 return check_and_freshen_file(sha1_file_name(sha1), freshen);
596}
597
598static int check_and_freshen_nonlocal(const unsigned char *sha1, int freshen)
599{
600 struct alternate_object_database *alt;
601 prepare_alt_odb();
602 for (alt = alt_odb_list; alt; alt = alt->next) {
603 const char *path = alt_sha1_path(alt, sha1);
604 if (check_and_freshen_file(path, freshen))
605 return 1;
606 }
607 return 0;
608}
609
610static int check_and_freshen(const unsigned char *sha1, int freshen)
611{
612 return check_and_freshen_local(sha1, freshen) ||
613 check_and_freshen_nonlocal(sha1, freshen);
614}
615
616int has_loose_object_nonlocal(const unsigned char *sha1)
617{
618 return check_and_freshen_nonlocal(sha1, 0);
619}
620
621static int has_loose_object(const unsigned char *sha1)
622{
623 return check_and_freshen(sha1, 0);
624}
625
626static unsigned int pack_used_ctr;
627static unsigned int pack_mmap_calls;
628static unsigned int peak_pack_open_windows;
629static unsigned int pack_open_windows;
630static unsigned int pack_open_fds;
631static unsigned int pack_max_fds;
632static size_t peak_pack_mapped;
633static size_t pack_mapped;
634struct packed_git *packed_git;
635
636static struct mru packed_git_mru_storage;
637struct mru *packed_git_mru = &packed_git_mru_storage;
638
639void pack_report(void)
640{
641 fprintf(stderr,
642 "pack_report: getpagesize() = %10" SZ_FMT "\n"
643 "pack_report: core.packedGitWindowSize = %10" SZ_FMT "\n"
644 "pack_report: core.packedGitLimit = %10" SZ_FMT "\n",
645 sz_fmt(getpagesize()),
646 sz_fmt(packed_git_window_size),
647 sz_fmt(packed_git_limit));
648 fprintf(stderr,
649 "pack_report: pack_used_ctr = %10u\n"
650 "pack_report: pack_mmap_calls = %10u\n"
651 "pack_report: pack_open_windows = %10u / %10u\n"
652 "pack_report: pack_mapped = "
653 "%10" SZ_FMT " / %10" SZ_FMT "\n",
654 pack_used_ctr,
655 pack_mmap_calls,
656 pack_open_windows, peak_pack_open_windows,
657 sz_fmt(pack_mapped), sz_fmt(peak_pack_mapped));
658}
659
660/*
661 * Open and mmap the index file at path, perform a couple of
662 * consistency checks, then record its information to p. Return 0 on
663 * success.
664 */
665static int check_packed_git_idx(const char *path, struct packed_git *p)
666{
667 void *idx_map;
668 struct pack_idx_header *hdr;
669 size_t idx_size;
670 uint32_t version, nr, i, *index;
671 int fd = git_open_noatime(path);
672 struct stat st;
673
674 if (fd < 0)
675 return -1;
676 if (fstat(fd, &st)) {
677 close(fd);
678 return -1;
679 }
680 idx_size = xsize_t(st.st_size);
681 if (idx_size < 4 * 256 + 20 + 20) {
682 close(fd);
683 return error("index file %s is too small", path);
684 }
685 idx_map = xmmap(NULL, idx_size, PROT_READ, MAP_PRIVATE, fd, 0);
686 close(fd);
687
688 hdr = idx_map;
689 if (hdr->idx_signature == htonl(PACK_IDX_SIGNATURE)) {
690 version = ntohl(hdr->idx_version);
691 if (version < 2 || version > 2) {
692 munmap(idx_map, idx_size);
693 return error("index file %s is version %"PRIu32
694 " and is not supported by this binary"
695 " (try upgrading GIT to a newer version)",
696 path, version);
697 }
698 } else
699 version = 1;
700
701 nr = 0;
702 index = idx_map;
703 if (version > 1)
704 index += 2; /* skip index header */
705 for (i = 0; i < 256; i++) {
706 uint32_t n = ntohl(index[i]);
707 if (n < nr) {
708 munmap(idx_map, idx_size);
709 return error("non-monotonic index %s", path);
710 }
711 nr = n;
712 }
713
714 if (version == 1) {
715 /*
716 * Total size:
717 * - 256 index entries 4 bytes each
718 * - 24-byte entries * nr (20-byte sha1 + 4-byte offset)
719 * - 20-byte SHA1 of the packfile
720 * - 20-byte SHA1 file checksum
721 */
722 if (idx_size != 4*256 + nr * 24 + 20 + 20) {
723 munmap(idx_map, idx_size);
724 return error("wrong index v1 file size in %s", path);
725 }
726 } else if (version == 2) {
727 /*
728 * Minimum size:
729 * - 8 bytes of header
730 * - 256 index entries 4 bytes each
731 * - 20-byte sha1 entry * nr
732 * - 4-byte crc entry * nr
733 * - 4-byte offset entry * nr
734 * - 20-byte SHA1 of the packfile
735 * - 20-byte SHA1 file checksum
736 * And after the 4-byte offset table might be a
737 * variable sized table containing 8-byte entries
738 * for offsets larger than 2^31.
739 */
740 unsigned long min_size = 8 + 4*256 + nr*(20 + 4 + 4) + 20 + 20;
741 unsigned long max_size = min_size;
742 if (nr)
743 max_size += (nr - 1)*8;
744 if (idx_size < min_size || idx_size > max_size) {
745 munmap(idx_map, idx_size);
746 return error("wrong index v2 file size in %s", path);
747 }
748 if (idx_size != min_size &&
749 /*
750 * make sure we can deal with large pack offsets.
751 * 31-bit signed offset won't be enough, neither
752 * 32-bit unsigned one will be.
753 */
754 (sizeof(off_t) <= 4)) {
755 munmap(idx_map, idx_size);
756 return error("pack too large for current definition of off_t in %s", path);
757 }
758 }
759
760 p->index_version = version;
761 p->index_data = idx_map;
762 p->index_size = idx_size;
763 p->num_objects = nr;
764 return 0;
765}
766
767int open_pack_index(struct packed_git *p)
768{
769 char *idx_name;
770 size_t len;
771 int ret;
772
773 if (p->index_data)
774 return 0;
775
776 if (!strip_suffix(p->pack_name, ".pack", &len))
777 die("BUG: pack_name does not end in .pack");
778 idx_name = xstrfmt("%.*s.idx", (int)len, p->pack_name);
779 ret = check_packed_git_idx(idx_name, p);
780 free(idx_name);
781 return ret;
782}
783
784static void scan_windows(struct packed_git *p,
785 struct packed_git **lru_p,
786 struct pack_window **lru_w,
787 struct pack_window **lru_l)
788{
789 struct pack_window *w, *w_l;
790
791 for (w_l = NULL, w = p->windows; w; w = w->next) {
792 if (!w->inuse_cnt) {
793 if (!*lru_w || w->last_used < (*lru_w)->last_used) {
794 *lru_p = p;
795 *lru_w = w;
796 *lru_l = w_l;
797 }
798 }
799 w_l = w;
800 }
801}
802
803static int unuse_one_window(struct packed_git *current)
804{
805 struct packed_git *p, *lru_p = NULL;
806 struct pack_window *lru_w = NULL, *lru_l = NULL;
807
808 if (current)
809 scan_windows(current, &lru_p, &lru_w, &lru_l);
810 for (p = packed_git; p; p = p->next)
811 scan_windows(p, &lru_p, &lru_w, &lru_l);
812 if (lru_p) {
813 munmap(lru_w->base, lru_w->len);
814 pack_mapped -= lru_w->len;
815 if (lru_l)
816 lru_l->next = lru_w->next;
817 else
818 lru_p->windows = lru_w->next;
819 free(lru_w);
820 pack_open_windows--;
821 return 1;
822 }
823 return 0;
824}
825
826void release_pack_memory(size_t need)
827{
828 size_t cur = pack_mapped;
829 while (need >= (cur - pack_mapped) && unuse_one_window(NULL))
830 ; /* nothing */
831}
832
833static void mmap_limit_check(size_t length)
834{
835 static size_t limit = 0;
836 if (!limit) {
837 limit = git_env_ulong("GIT_MMAP_LIMIT", 0);
838 if (!limit)
839 limit = SIZE_MAX;
840 }
841 if (length > limit)
842 die("attempting to mmap %"PRIuMAX" over limit %"PRIuMAX,
843 (uintmax_t)length, (uintmax_t)limit);
844}
845
846void *xmmap_gently(void *start, size_t length,
847 int prot, int flags, int fd, off_t offset)
848{
849 void *ret;
850
851 mmap_limit_check(length);
852 ret = mmap(start, length, prot, flags, fd, offset);
853 if (ret == MAP_FAILED) {
854 if (!length)
855 return NULL;
856 release_pack_memory(length);
857 ret = mmap(start, length, prot, flags, fd, offset);
858 }
859 return ret;
860}
861
862void *xmmap(void *start, size_t length,
863 int prot, int flags, int fd, off_t offset)
864{
865 void *ret = xmmap_gently(start, length, prot, flags, fd, offset);
866 if (ret == MAP_FAILED)
867 die_errno("mmap failed");
868 return ret;
869}
870
871void close_pack_windows(struct packed_git *p)
872{
873 while (p->windows) {
874 struct pack_window *w = p->windows;
875
876 if (w->inuse_cnt)
877 die("pack '%s' still has open windows to it",
878 p->pack_name);
879 munmap(w->base, w->len);
880 pack_mapped -= w->len;
881 pack_open_windows--;
882 p->windows = w->next;
883 free(w);
884 }
885}
886
887static int close_pack_fd(struct packed_git *p)
888{
889 if (p->pack_fd < 0)
890 return 0;
891
892 close(p->pack_fd);
893 pack_open_fds--;
894 p->pack_fd = -1;
895
896 return 1;
897}
898
899static void close_pack(struct packed_git *p)
900{
901 close_pack_windows(p);
902 close_pack_fd(p);
903 close_pack_index(p);
904}
905
906void close_all_packs(void)
907{
908 struct packed_git *p;
909
910 for (p = packed_git; p; p = p->next)
911 if (p->do_not_close)
912 die("BUG: want to close pack marked 'do-not-close'");
913 else
914 close_pack(p);
915}
916
917
918/*
919 * The LRU pack is the one with the oldest MRU window, preferring packs
920 * with no used windows, or the oldest mtime if it has no windows allocated.
921 */
922static void find_lru_pack(struct packed_git *p, struct packed_git **lru_p, struct pack_window **mru_w, int *accept_windows_inuse)
923{
924 struct pack_window *w, *this_mru_w;
925 int has_windows_inuse = 0;
926
927 /*
928 * Reject this pack if it has windows and the previously selected
929 * one does not. If this pack does not have windows, reject
930 * it if the pack file is newer than the previously selected one.
931 */
932 if (*lru_p && !*mru_w && (p->windows || p->mtime > (*lru_p)->mtime))
933 return;
934
935 for (w = this_mru_w = p->windows; w; w = w->next) {
936 /*
937 * Reject this pack if any of its windows are in use,
938 * but the previously selected pack did not have any
939 * inuse windows. Otherwise, record that this pack
940 * has windows in use.
941 */
942 if (w->inuse_cnt) {
943 if (*accept_windows_inuse)
944 has_windows_inuse = 1;
945 else
946 return;
947 }
948
949 if (w->last_used > this_mru_w->last_used)
950 this_mru_w = w;
951
952 /*
953 * Reject this pack if it has windows that have been
954 * used more recently than the previously selected pack.
955 * If the previously selected pack had windows inuse and
956 * we have not encountered a window in this pack that is
957 * inuse, skip this check since we prefer a pack with no
958 * inuse windows to one that has inuse windows.
959 */
960 if (*mru_w && *accept_windows_inuse == has_windows_inuse &&
961 this_mru_w->last_used > (*mru_w)->last_used)
962 return;
963 }
964
965 /*
966 * Select this pack.
967 */
968 *mru_w = this_mru_w;
969 *lru_p = p;
970 *accept_windows_inuse = has_windows_inuse;
971}
972
973static int close_one_pack(void)
974{
975 struct packed_git *p, *lru_p = NULL;
976 struct pack_window *mru_w = NULL;
977 int accept_windows_inuse = 1;
978
979 for (p = packed_git; p; p = p->next) {
980 if (p->pack_fd == -1)
981 continue;
982 find_lru_pack(p, &lru_p, &mru_w, &accept_windows_inuse);
983 }
984
985 if (lru_p)
986 return close_pack_fd(lru_p);
987
988 return 0;
989}
990
991void unuse_pack(struct pack_window **w_cursor)
992{
993 struct pack_window *w = *w_cursor;
994 if (w) {
995 w->inuse_cnt--;
996 *w_cursor = NULL;
997 }
998}
999
1000void close_pack_index(struct packed_git *p)
1001{
1002 if (p->index_data) {
1003 munmap((void *)p->index_data, p->index_size);
1004 p->index_data = NULL;
1005 }
1006}
1007
1008static unsigned int get_max_fd_limit(void)
1009{
1010#ifdef RLIMIT_NOFILE
1011 {
1012 struct rlimit lim;
1013
1014 if (!getrlimit(RLIMIT_NOFILE, &lim))
1015 return lim.rlim_cur;
1016 }
1017#endif
1018
1019#ifdef _SC_OPEN_MAX
1020 {
1021 long open_max = sysconf(_SC_OPEN_MAX);
1022 if (0 < open_max)
1023 return open_max;
1024 /*
1025 * Otherwise, we got -1 for one of the two
1026 * reasons:
1027 *
1028 * (1) sysconf() did not understand _SC_OPEN_MAX
1029 * and signaled an error with -1; or
1030 * (2) sysconf() said there is no limit.
1031 *
1032 * We _could_ clear errno before calling sysconf() to
1033 * tell these two cases apart and return a huge number
1034 * in the latter case to let the caller cap it to a
1035 * value that is not so selfish, but letting the
1036 * fallback OPEN_MAX codepath take care of these cases
1037 * is a lot simpler.
1038 */
1039 }
1040#endif
1041
1042#ifdef OPEN_MAX
1043 return OPEN_MAX;
1044#else
1045 return 1; /* see the caller ;-) */
1046#endif
1047}
1048
1049/*
1050 * Do not call this directly as this leaks p->pack_fd on error return;
1051 * call open_packed_git() instead.
1052 */
1053static int open_packed_git_1(struct packed_git *p)
1054{
1055 struct stat st;
1056 struct pack_header hdr;
1057 unsigned char sha1[20];
1058 unsigned char *idx_sha1;
1059 long fd_flag;
1060
1061 if (!p->index_data && open_pack_index(p))
1062 return error("packfile %s index unavailable", p->pack_name);
1063
1064 if (!pack_max_fds) {
1065 unsigned int max_fds = get_max_fd_limit();
1066
1067 /* Save 3 for stdin/stdout/stderr, 22 for work */
1068 if (25 < max_fds)
1069 pack_max_fds = max_fds - 25;
1070 else
1071 pack_max_fds = 1;
1072 }
1073
1074 while (pack_max_fds <= pack_open_fds && close_one_pack())
1075 ; /* nothing */
1076
1077 p->pack_fd = git_open_noatime(p->pack_name);
1078 if (p->pack_fd < 0 || fstat(p->pack_fd, &st))
1079 return -1;
1080 pack_open_fds++;
1081
1082 /* If we created the struct before we had the pack we lack size. */
1083 if (!p->pack_size) {
1084 if (!S_ISREG(st.st_mode))
1085 return error("packfile %s not a regular file", p->pack_name);
1086 p->pack_size = st.st_size;
1087 } else if (p->pack_size != st.st_size)
1088 return error("packfile %s size changed", p->pack_name);
1089
1090 /* We leave these file descriptors open with sliding mmap;
1091 * there is no point keeping them open across exec(), though.
1092 */
1093 fd_flag = fcntl(p->pack_fd, F_GETFD, 0);
1094 if (fd_flag < 0)
1095 return error("cannot determine file descriptor flags");
1096 fd_flag |= FD_CLOEXEC;
1097 if (fcntl(p->pack_fd, F_SETFD, fd_flag) == -1)
1098 return error("cannot set FD_CLOEXEC");
1099
1100 /* Verify we recognize this pack file format. */
1101 if (read_in_full(p->pack_fd, &hdr, sizeof(hdr)) != sizeof(hdr))
1102 return error("file %s is far too short to be a packfile", p->pack_name);
1103 if (hdr.hdr_signature != htonl(PACK_SIGNATURE))
1104 return error("file %s is not a GIT packfile", p->pack_name);
1105 if (!pack_version_ok(hdr.hdr_version))
1106 return error("packfile %s is version %"PRIu32" and not"
1107 " supported (try upgrading GIT to a newer version)",
1108 p->pack_name, ntohl(hdr.hdr_version));
1109
1110 /* Verify the pack matches its index. */
1111 if (p->num_objects != ntohl(hdr.hdr_entries))
1112 return error("packfile %s claims to have %"PRIu32" objects"
1113 " while index indicates %"PRIu32" objects",
1114 p->pack_name, ntohl(hdr.hdr_entries),
1115 p->num_objects);
1116 if (lseek(p->pack_fd, p->pack_size - sizeof(sha1), SEEK_SET) == -1)
1117 return error("end of packfile %s is unavailable", p->pack_name);
1118 if (read_in_full(p->pack_fd, sha1, sizeof(sha1)) != sizeof(sha1))
1119 return error("packfile %s signature is unavailable", p->pack_name);
1120 idx_sha1 = ((unsigned char *)p->index_data) + p->index_size - 40;
1121 if (hashcmp(sha1, idx_sha1))
1122 return error("packfile %s does not match index", p->pack_name);
1123 return 0;
1124}
1125
1126static int open_packed_git(struct packed_git *p)
1127{
1128 if (!open_packed_git_1(p))
1129 return 0;
1130 close_pack_fd(p);
1131 return -1;
1132}
1133
1134static int in_window(struct pack_window *win, off_t offset)
1135{
1136 /* We must promise at least 20 bytes (one hash) after the
1137 * offset is available from this window, otherwise the offset
1138 * is not actually in this window and a different window (which
1139 * has that one hash excess) must be used. This is to support
1140 * the object header and delta base parsing routines below.
1141 */
1142 off_t win_off = win->offset;
1143 return win_off <= offset
1144 && (offset + 20) <= (win_off + win->len);
1145}
1146
1147unsigned char *use_pack(struct packed_git *p,
1148 struct pack_window **w_cursor,
1149 off_t offset,
1150 unsigned long *left)
1151{
1152 struct pack_window *win = *w_cursor;
1153
1154 /* Since packfiles end in a hash of their content and it's
1155 * pointless to ask for an offset into the middle of that
1156 * hash, and the in_window function above wouldn't match
1157 * don't allow an offset too close to the end of the file.
1158 */
1159 if (!p->pack_size && p->pack_fd == -1 && open_packed_git(p))
1160 die("packfile %s cannot be accessed", p->pack_name);
1161 if (offset > (p->pack_size - 20))
1162 die("offset beyond end of packfile (truncated pack?)");
1163 if (offset < 0)
1164 die(_("offset before end of packfile (broken .idx?)"));
1165
1166 if (!win || !in_window(win, offset)) {
1167 if (win)
1168 win->inuse_cnt--;
1169 for (win = p->windows; win; win = win->next) {
1170 if (in_window(win, offset))
1171 break;
1172 }
1173 if (!win) {
1174 size_t window_align = packed_git_window_size / 2;
1175 off_t len;
1176
1177 if (p->pack_fd == -1 && open_packed_git(p))
1178 die("packfile %s cannot be accessed", p->pack_name);
1179
1180 win = xcalloc(1, sizeof(*win));
1181 win->offset = (offset / window_align) * window_align;
1182 len = p->pack_size - win->offset;
1183 if (len > packed_git_window_size)
1184 len = packed_git_window_size;
1185 win->len = (size_t)len;
1186 pack_mapped += win->len;
1187 while (packed_git_limit < pack_mapped
1188 && unuse_one_window(p))
1189 ; /* nothing */
1190 win->base = xmmap(NULL, win->len,
1191 PROT_READ, MAP_PRIVATE,
1192 p->pack_fd, win->offset);
1193 if (win->base == MAP_FAILED)
1194 die_errno("packfile %s cannot be mapped",
1195 p->pack_name);
1196 if (!win->offset && win->len == p->pack_size
1197 && !p->do_not_close)
1198 close_pack_fd(p);
1199 pack_mmap_calls++;
1200 pack_open_windows++;
1201 if (pack_mapped > peak_pack_mapped)
1202 peak_pack_mapped = pack_mapped;
1203 if (pack_open_windows > peak_pack_open_windows)
1204 peak_pack_open_windows = pack_open_windows;
1205 win->next = p->windows;
1206 p->windows = win;
1207 }
1208 }
1209 if (win != *w_cursor) {
1210 win->last_used = pack_used_ctr++;
1211 win->inuse_cnt++;
1212 *w_cursor = win;
1213 }
1214 offset -= win->offset;
1215 if (left)
1216 *left = win->len - xsize_t(offset);
1217 return win->base + offset;
1218}
1219
1220static struct packed_git *alloc_packed_git(int extra)
1221{
1222 struct packed_git *p = xmalloc(st_add(sizeof(*p), extra));
1223 memset(p, 0, sizeof(*p));
1224 p->pack_fd = -1;
1225 return p;
1226}
1227
1228static void try_to_free_pack_memory(size_t size)
1229{
1230 release_pack_memory(size);
1231}
1232
1233struct packed_git *add_packed_git(const char *path, size_t path_len, int local)
1234{
1235 static int have_set_try_to_free_routine;
1236 struct stat st;
1237 size_t alloc;
1238 struct packed_git *p;
1239
1240 if (!have_set_try_to_free_routine) {
1241 have_set_try_to_free_routine = 1;
1242 set_try_to_free_routine(try_to_free_pack_memory);
1243 }
1244
1245 /*
1246 * Make sure a corresponding .pack file exists and that
1247 * the index looks sane.
1248 */
1249 if (!strip_suffix_mem(path, &path_len, ".idx"))
1250 return NULL;
1251
1252 /*
1253 * ".pack" is long enough to hold any suffix we're adding (and
1254 * the use xsnprintf double-checks that)
1255 */
1256 alloc = st_add3(path_len, strlen(".pack"), 1);
1257 p = alloc_packed_git(alloc);
1258 memcpy(p->pack_name, path, path_len);
1259
1260 xsnprintf(p->pack_name + path_len, alloc - path_len, ".keep");
1261 if (!access(p->pack_name, F_OK))
1262 p->pack_keep = 1;
1263
1264 xsnprintf(p->pack_name + path_len, alloc - path_len, ".pack");
1265 if (stat(p->pack_name, &st) || !S_ISREG(st.st_mode)) {
1266 free(p);
1267 return NULL;
1268 }
1269
1270 /* ok, it looks sane as far as we can check without
1271 * actually mapping the pack file.
1272 */
1273 p->pack_size = st.st_size;
1274 p->pack_local = local;
1275 p->mtime = st.st_mtime;
1276 if (path_len < 40 || get_sha1_hex(path + path_len - 40, p->sha1))
1277 hashclr(p->sha1);
1278 return p;
1279}
1280
1281struct packed_git *parse_pack_index(unsigned char *sha1, const char *idx_path)
1282{
1283 const char *path = sha1_pack_name(sha1);
1284 size_t alloc = st_add(strlen(path), 1);
1285 struct packed_git *p = alloc_packed_git(alloc);
1286
1287 memcpy(p->pack_name, path, alloc); /* includes NUL */
1288 hashcpy(p->sha1, sha1);
1289 if (check_packed_git_idx(idx_path, p)) {
1290 free(p);
1291 return NULL;
1292 }
1293
1294 return p;
1295}
1296
1297void install_packed_git(struct packed_git *pack)
1298{
1299 if (pack->pack_fd != -1)
1300 pack_open_fds++;
1301
1302 pack->next = packed_git;
1303 packed_git = pack;
1304}
1305
1306void (*report_garbage)(unsigned seen_bits, const char *path);
1307
1308static void report_helper(const struct string_list *list,
1309 int seen_bits, int first, int last)
1310{
1311 if (seen_bits == (PACKDIR_FILE_PACK|PACKDIR_FILE_IDX))
1312 return;
1313
1314 for (; first < last; first++)
1315 report_garbage(seen_bits, list->items[first].string);
1316}
1317
1318static void report_pack_garbage(struct string_list *list)
1319{
1320 int i, baselen = -1, first = 0, seen_bits = 0;
1321
1322 if (!report_garbage)
1323 return;
1324
1325 string_list_sort(list);
1326
1327 for (i = 0; i < list->nr; i++) {
1328 const char *path = list->items[i].string;
1329 if (baselen != -1 &&
1330 strncmp(path, list->items[first].string, baselen)) {
1331 report_helper(list, seen_bits, first, i);
1332 baselen = -1;
1333 seen_bits = 0;
1334 }
1335 if (baselen == -1) {
1336 const char *dot = strrchr(path, '.');
1337 if (!dot) {
1338 report_garbage(PACKDIR_FILE_GARBAGE, path);
1339 continue;
1340 }
1341 baselen = dot - path + 1;
1342 first = i;
1343 }
1344 if (!strcmp(path + baselen, "pack"))
1345 seen_bits |= 1;
1346 else if (!strcmp(path + baselen, "idx"))
1347 seen_bits |= 2;
1348 }
1349 report_helper(list, seen_bits, first, list->nr);
1350}
1351
1352static void prepare_packed_git_one(char *objdir, int local)
1353{
1354 struct strbuf path = STRBUF_INIT;
1355 size_t dirnamelen;
1356 DIR *dir;
1357 struct dirent *de;
1358 struct string_list garbage = STRING_LIST_INIT_DUP;
1359
1360 strbuf_addstr(&path, objdir);
1361 strbuf_addstr(&path, "/pack");
1362 dir = opendir(path.buf);
1363 if (!dir) {
1364 if (errno != ENOENT)
1365 error_errno("unable to open object pack directory: %s",
1366 path.buf);
1367 strbuf_release(&path);
1368 return;
1369 }
1370 strbuf_addch(&path, '/');
1371 dirnamelen = path.len;
1372 while ((de = readdir(dir)) != NULL) {
1373 struct packed_git *p;
1374 size_t base_len;
1375
1376 if (is_dot_or_dotdot(de->d_name))
1377 continue;
1378
1379 strbuf_setlen(&path, dirnamelen);
1380 strbuf_addstr(&path, de->d_name);
1381
1382 base_len = path.len;
1383 if (strip_suffix_mem(path.buf, &base_len, ".idx")) {
1384 /* Don't reopen a pack we already have. */
1385 for (p = packed_git; p; p = p->next) {
1386 size_t len;
1387 if (strip_suffix(p->pack_name, ".pack", &len) &&
1388 len == base_len &&
1389 !memcmp(p->pack_name, path.buf, len))
1390 break;
1391 }
1392 if (p == NULL &&
1393 /*
1394 * See if it really is a valid .idx file with
1395 * corresponding .pack file that we can map.
1396 */
1397 (p = add_packed_git(path.buf, path.len, local)) != NULL)
1398 install_packed_git(p);
1399 }
1400
1401 if (!report_garbage)
1402 continue;
1403
1404 if (ends_with(de->d_name, ".idx") ||
1405 ends_with(de->d_name, ".pack") ||
1406 ends_with(de->d_name, ".bitmap") ||
1407 ends_with(de->d_name, ".keep"))
1408 string_list_append(&garbage, path.buf);
1409 else
1410 report_garbage(PACKDIR_FILE_GARBAGE, path.buf);
1411 }
1412 closedir(dir);
1413 report_pack_garbage(&garbage);
1414 string_list_clear(&garbage, 0);
1415 strbuf_release(&path);
1416}
1417
1418static void *get_next_packed_git(const void *p)
1419{
1420 return ((const struct packed_git *)p)->next;
1421}
1422
1423static void set_next_packed_git(void *p, void *next)
1424{
1425 ((struct packed_git *)p)->next = next;
1426}
1427
1428static int sort_pack(const void *a_, const void *b_)
1429{
1430 const struct packed_git *a = a_;
1431 const struct packed_git *b = b_;
1432 int st;
1433
1434 /*
1435 * Local packs tend to contain objects specific to our
1436 * variant of the project than remote ones. In addition,
1437 * remote ones could be on a network mounted filesystem.
1438 * Favor local ones for these reasons.
1439 */
1440 st = a->pack_local - b->pack_local;
1441 if (st)
1442 return -st;
1443
1444 /*
1445 * Younger packs tend to contain more recent objects,
1446 * and more recent objects tend to get accessed more
1447 * often.
1448 */
1449 if (a->mtime < b->mtime)
1450 return 1;
1451 else if (a->mtime == b->mtime)
1452 return 0;
1453 return -1;
1454}
1455
1456static void rearrange_packed_git(void)
1457{
1458 packed_git = llist_mergesort(packed_git, get_next_packed_git,
1459 set_next_packed_git, sort_pack);
1460}
1461
1462static void prepare_packed_git_mru(void)
1463{
1464 struct packed_git *p;
1465
1466 mru_clear(packed_git_mru);
1467 for (p = packed_git; p; p = p->next)
1468 mru_append(packed_git_mru, p);
1469}
1470
1471static int prepare_packed_git_run_once = 0;
1472void prepare_packed_git(void)
1473{
1474 struct alternate_object_database *alt;
1475
1476 if (prepare_packed_git_run_once)
1477 return;
1478 prepare_packed_git_one(get_object_directory(), 1);
1479 prepare_alt_odb();
1480 for (alt = alt_odb_list; alt; alt = alt->next)
1481 prepare_packed_git_one(alt->path, 0);
1482 rearrange_packed_git();
1483 prepare_packed_git_mru();
1484 prepare_packed_git_run_once = 1;
1485}
1486
1487void reprepare_packed_git(void)
1488{
1489 prepare_packed_git_run_once = 0;
1490 prepare_packed_git();
1491}
1492
1493static void mark_bad_packed_object(struct packed_git *p,
1494 const unsigned char *sha1)
1495{
1496 unsigned i;
1497 for (i = 0; i < p->num_bad_objects; i++)
1498 if (!hashcmp(sha1, p->bad_object_sha1 + GIT_SHA1_RAWSZ * i))
1499 return;
1500 p->bad_object_sha1 = xrealloc(p->bad_object_sha1,
1501 st_mult(GIT_SHA1_RAWSZ,
1502 st_add(p->num_bad_objects, 1)));
1503 hashcpy(p->bad_object_sha1 + GIT_SHA1_RAWSZ * p->num_bad_objects, sha1);
1504 p->num_bad_objects++;
1505}
1506
1507static const struct packed_git *has_packed_and_bad(const unsigned char *sha1)
1508{
1509 struct packed_git *p;
1510 unsigned i;
1511
1512 for (p = packed_git; p; p = p->next)
1513 for (i = 0; i < p->num_bad_objects; i++)
1514 if (!hashcmp(sha1, p->bad_object_sha1 + 20 * i))
1515 return p;
1516 return NULL;
1517}
1518
1519/*
1520 * With an in-core object data in "map", rehash it to make sure the
1521 * object name actually matches "sha1" to detect object corruption.
1522 * With "map" == NULL, try reading the object named with "sha1" using
1523 * the streaming interface and rehash it to do the same.
1524 */
1525int check_sha1_signature(const unsigned char *sha1, void *map,
1526 unsigned long size, const char *type)
1527{
1528 unsigned char real_sha1[20];
1529 enum object_type obj_type;
1530 struct git_istream *st;
1531 git_SHA_CTX c;
1532 char hdr[32];
1533 int hdrlen;
1534
1535 if (map) {
1536 hash_sha1_file(map, size, type, real_sha1);
1537 return hashcmp(sha1, real_sha1) ? -1 : 0;
1538 }
1539
1540 st = open_istream(sha1, &obj_type, &size, NULL);
1541 if (!st)
1542 return -1;
1543
1544 /* Generate the header */
1545 hdrlen = xsnprintf(hdr, sizeof(hdr), "%s %lu", typename(obj_type), size) + 1;
1546
1547 /* Sha1.. */
1548 git_SHA1_Init(&c);
1549 git_SHA1_Update(&c, hdr, hdrlen);
1550 for (;;) {
1551 char buf[1024 * 16];
1552 ssize_t readlen = read_istream(st, buf, sizeof(buf));
1553
1554 if (readlen < 0) {
1555 close_istream(st);
1556 return -1;
1557 }
1558 if (!readlen)
1559 break;
1560 git_SHA1_Update(&c, buf, readlen);
1561 }
1562 git_SHA1_Final(real_sha1, &c);
1563 close_istream(st);
1564 return hashcmp(sha1, real_sha1) ? -1 : 0;
1565}
1566
1567int git_open_noatime(const char *name)
1568{
1569 static int sha1_file_open_flag = O_NOATIME;
1570
1571 for (;;) {
1572 int fd;
1573
1574 errno = 0;
1575 fd = open(name, O_RDONLY | sha1_file_open_flag);
1576 if (fd >= 0)
1577 return fd;
1578
1579 /* Might the failure be due to O_NOATIME? */
1580 if (errno != ENOENT && sha1_file_open_flag) {
1581 sha1_file_open_flag = 0;
1582 continue;
1583 }
1584
1585 return -1;
1586 }
1587}
1588
1589static int stat_sha1_file(const unsigned char *sha1, struct stat *st)
1590{
1591 struct alternate_object_database *alt;
1592
1593 if (!lstat(sha1_file_name(sha1), st))
1594 return 0;
1595
1596 prepare_alt_odb();
1597 errno = ENOENT;
1598 for (alt = alt_odb_list; alt; alt = alt->next) {
1599 const char *path = alt_sha1_path(alt, sha1);
1600 if (!lstat(path, st))
1601 return 0;
1602 }
1603
1604 return -1;
1605}
1606
1607static int open_sha1_file(const unsigned char *sha1)
1608{
1609 int fd;
1610 struct alternate_object_database *alt;
1611 int most_interesting_errno;
1612
1613 fd = git_open_noatime(sha1_file_name(sha1));
1614 if (fd >= 0)
1615 return fd;
1616 most_interesting_errno = errno;
1617
1618 prepare_alt_odb();
1619 for (alt = alt_odb_list; alt; alt = alt->next) {
1620 const char *path = alt_sha1_path(alt, sha1);
1621 fd = git_open_noatime(path);
1622 if (fd >= 0)
1623 return fd;
1624 if (most_interesting_errno == ENOENT)
1625 most_interesting_errno = errno;
1626 }
1627 errno = most_interesting_errno;
1628 return -1;
1629}
1630
1631void *map_sha1_file(const unsigned char *sha1, unsigned long *size)
1632{
1633 void *map;
1634 int fd;
1635
1636 fd = open_sha1_file(sha1);
1637 map = NULL;
1638 if (fd >= 0) {
1639 struct stat st;
1640
1641 if (!fstat(fd, &st)) {
1642 *size = xsize_t(st.st_size);
1643 if (!*size) {
1644 /* mmap() is forbidden on empty files */
1645 error("object file %s is empty", sha1_file_name(sha1));
1646 return NULL;
1647 }
1648 map = xmmap(NULL, *size, PROT_READ, MAP_PRIVATE, fd, 0);
1649 }
1650 close(fd);
1651 }
1652 return map;
1653}
1654
1655unsigned long unpack_object_header_buffer(const unsigned char *buf,
1656 unsigned long len, enum object_type *type, unsigned long *sizep)
1657{
1658 unsigned shift;
1659 unsigned long size, c;
1660 unsigned long used = 0;
1661
1662 c = buf[used++];
1663 *type = (c >> 4) & 7;
1664 size = c & 15;
1665 shift = 4;
1666 while (c & 0x80) {
1667 if (len <= used || bitsizeof(long) <= shift) {
1668 error("bad object header");
1669 size = used = 0;
1670 break;
1671 }
1672 c = buf[used++];
1673 size += (c & 0x7f) << shift;
1674 shift += 7;
1675 }
1676 *sizep = size;
1677 return used;
1678}
1679
1680static int unpack_sha1_short_header(git_zstream *stream,
1681 unsigned char *map, unsigned long mapsize,
1682 void *buffer, unsigned long bufsiz)
1683{
1684 /* Get the data stream */
1685 memset(stream, 0, sizeof(*stream));
1686 stream->next_in = map;
1687 stream->avail_in = mapsize;
1688 stream->next_out = buffer;
1689 stream->avail_out = bufsiz;
1690
1691 git_inflate_init(stream);
1692 return git_inflate(stream, 0);
1693}
1694
1695int unpack_sha1_header(git_zstream *stream,
1696 unsigned char *map, unsigned long mapsize,
1697 void *buffer, unsigned long bufsiz)
1698{
1699 int status = unpack_sha1_short_header(stream, map, mapsize,
1700 buffer, bufsiz);
1701
1702 if (status < Z_OK)
1703 return status;
1704
1705 /* Make sure we have the terminating NUL */
1706 if (!memchr(buffer, '\0', stream->next_out - (unsigned char *)buffer))
1707 return -1;
1708 return 0;
1709}
1710
1711static int unpack_sha1_header_to_strbuf(git_zstream *stream, unsigned char *map,
1712 unsigned long mapsize, void *buffer,
1713 unsigned long bufsiz, struct strbuf *header)
1714{
1715 int status;
1716
1717 status = unpack_sha1_short_header(stream, map, mapsize, buffer, bufsiz);
1718 if (status < Z_OK)
1719 return -1;
1720
1721 /*
1722 * Check if entire header is unpacked in the first iteration.
1723 */
1724 if (memchr(buffer, '\0', stream->next_out - (unsigned char *)buffer))
1725 return 0;
1726
1727 /*
1728 * buffer[0..bufsiz] was not large enough. Copy the partial
1729 * result out to header, and then append the result of further
1730 * reading the stream.
1731 */
1732 strbuf_add(header, buffer, stream->next_out - (unsigned char *)buffer);
1733 stream->next_out = buffer;
1734 stream->avail_out = bufsiz;
1735
1736 do {
1737 status = git_inflate(stream, 0);
1738 strbuf_add(header, buffer, stream->next_out - (unsigned char *)buffer);
1739 if (memchr(buffer, '\0', stream->next_out - (unsigned char *)buffer))
1740 return 0;
1741 stream->next_out = buffer;
1742 stream->avail_out = bufsiz;
1743 } while (status != Z_STREAM_END);
1744 return -1;
1745}
1746
1747static void *unpack_sha1_rest(git_zstream *stream, void *buffer, unsigned long size, const unsigned char *sha1)
1748{
1749 int bytes = strlen(buffer) + 1;
1750 unsigned char *buf = xmallocz(size);
1751 unsigned long n;
1752 int status = Z_OK;
1753
1754 n = stream->total_out - bytes;
1755 if (n > size)
1756 n = size;
1757 memcpy(buf, (char *) buffer + bytes, n);
1758 bytes = n;
1759 if (bytes <= size) {
1760 /*
1761 * The above condition must be (bytes <= size), not
1762 * (bytes < size). In other words, even though we
1763 * expect no more output and set avail_out to zero,
1764 * the input zlib stream may have bytes that express
1765 * "this concludes the stream", and we *do* want to
1766 * eat that input.
1767 *
1768 * Otherwise we would not be able to test that we
1769 * consumed all the input to reach the expected size;
1770 * we also want to check that zlib tells us that all
1771 * went well with status == Z_STREAM_END at the end.
1772 */
1773 stream->next_out = buf + bytes;
1774 stream->avail_out = size - bytes;
1775 while (status == Z_OK)
1776 status = git_inflate(stream, Z_FINISH);
1777 }
1778 if (status == Z_STREAM_END && !stream->avail_in) {
1779 git_inflate_end(stream);
1780 return buf;
1781 }
1782
1783 if (status < 0)
1784 error("corrupt loose object '%s'", sha1_to_hex(sha1));
1785 else if (stream->avail_in)
1786 error("garbage at end of loose object '%s'",
1787 sha1_to_hex(sha1));
1788 free(buf);
1789 return NULL;
1790}
1791
1792/*
1793 * We used to just use "sscanf()", but that's actually way
1794 * too permissive for what we want to check. So do an anal
1795 * object header parse by hand.
1796 */
1797static int parse_sha1_header_extended(const char *hdr, struct object_info *oi,
1798 unsigned int flags)
1799{
1800 const char *type_buf = hdr;
1801 unsigned long size;
1802 int type, type_len = 0;
1803
1804 /*
1805 * The type can be of any size but is followed by
1806 * a space.
1807 */
1808 for (;;) {
1809 char c = *hdr++;
1810 if (!c)
1811 return -1;
1812 if (c == ' ')
1813 break;
1814 type_len++;
1815 }
1816
1817 type = type_from_string_gently(type_buf, type_len, 1);
1818 if (oi->typename)
1819 strbuf_add(oi->typename, type_buf, type_len);
1820 /*
1821 * Set type to 0 if its an unknown object and
1822 * we're obtaining the type using '--allow-unknown-type'
1823 * option.
1824 */
1825 if ((flags & LOOKUP_UNKNOWN_OBJECT) && (type < 0))
1826 type = 0;
1827 else if (type < 0)
1828 die("invalid object type");
1829 if (oi->typep)
1830 *oi->typep = type;
1831
1832 /*
1833 * The length must follow immediately, and be in canonical
1834 * decimal format (ie "010" is not valid).
1835 */
1836 size = *hdr++ - '0';
1837 if (size > 9)
1838 return -1;
1839 if (size) {
1840 for (;;) {
1841 unsigned long c = *hdr - '0';
1842 if (c > 9)
1843 break;
1844 hdr++;
1845 size = size * 10 + c;
1846 }
1847 }
1848
1849 if (oi->sizep)
1850 *oi->sizep = size;
1851
1852 /*
1853 * The length must be followed by a zero byte
1854 */
1855 return *hdr ? -1 : type;
1856}
1857
1858int parse_sha1_header(const char *hdr, unsigned long *sizep)
1859{
1860 struct object_info oi;
1861
1862 oi.sizep = sizep;
1863 oi.typename = NULL;
1864 oi.typep = NULL;
1865 return parse_sha1_header_extended(hdr, &oi, LOOKUP_REPLACE_OBJECT);
1866}
1867
1868static void *unpack_sha1_file(void *map, unsigned long mapsize, enum object_type *type, unsigned long *size, const unsigned char *sha1)
1869{
1870 int ret;
1871 git_zstream stream;
1872 char hdr[8192];
1873
1874 ret = unpack_sha1_header(&stream, map, mapsize, hdr, sizeof(hdr));
1875 if (ret < Z_OK || (*type = parse_sha1_header(hdr, size)) < 0)
1876 return NULL;
1877
1878 return unpack_sha1_rest(&stream, hdr, *size, sha1);
1879}
1880
1881unsigned long get_size_from_delta(struct packed_git *p,
1882 struct pack_window **w_curs,
1883 off_t curpos)
1884{
1885 const unsigned char *data;
1886 unsigned char delta_head[20], *in;
1887 git_zstream stream;
1888 int st;
1889
1890 memset(&stream, 0, sizeof(stream));
1891 stream.next_out = delta_head;
1892 stream.avail_out = sizeof(delta_head);
1893
1894 git_inflate_init(&stream);
1895 do {
1896 in = use_pack(p, w_curs, curpos, &stream.avail_in);
1897 stream.next_in = in;
1898 st = git_inflate(&stream, Z_FINISH);
1899 curpos += stream.next_in - in;
1900 } while ((st == Z_OK || st == Z_BUF_ERROR) &&
1901 stream.total_out < sizeof(delta_head));
1902 git_inflate_end(&stream);
1903 if ((st != Z_STREAM_END) && stream.total_out != sizeof(delta_head)) {
1904 error("delta data unpack-initial failed");
1905 return 0;
1906 }
1907
1908 /* Examine the initial part of the delta to figure out
1909 * the result size.
1910 */
1911 data = delta_head;
1912
1913 /* ignore base size */
1914 get_delta_hdr_size(&data, delta_head+sizeof(delta_head));
1915
1916 /* Read the result size */
1917 return get_delta_hdr_size(&data, delta_head+sizeof(delta_head));
1918}
1919
1920static off_t get_delta_base(struct packed_git *p,
1921 struct pack_window **w_curs,
1922 off_t *curpos,
1923 enum object_type type,
1924 off_t delta_obj_offset)
1925{
1926 unsigned char *base_info = use_pack(p, w_curs, *curpos, NULL);
1927 off_t base_offset;
1928
1929 /* use_pack() assured us we have [base_info, base_info + 20)
1930 * as a range that we can look at without walking off the
1931 * end of the mapped window. Its actually the hash size
1932 * that is assured. An OFS_DELTA longer than the hash size
1933 * is stupid, as then a REF_DELTA would be smaller to store.
1934 */
1935 if (type == OBJ_OFS_DELTA) {
1936 unsigned used = 0;
1937 unsigned char c = base_info[used++];
1938 base_offset = c & 127;
1939 while (c & 128) {
1940 base_offset += 1;
1941 if (!base_offset || MSB(base_offset, 7))
1942 return 0; /* overflow */
1943 c = base_info[used++];
1944 base_offset = (base_offset << 7) + (c & 127);
1945 }
1946 base_offset = delta_obj_offset - base_offset;
1947 if (base_offset <= 0 || base_offset >= delta_obj_offset)
1948 return 0; /* out of bound */
1949 *curpos += used;
1950 } else if (type == OBJ_REF_DELTA) {
1951 /* The base entry _must_ be in the same pack */
1952 base_offset = find_pack_entry_one(base_info, p);
1953 *curpos += 20;
1954 } else
1955 die("I am totally screwed");
1956 return base_offset;
1957}
1958
1959/*
1960 * Like get_delta_base above, but we return the sha1 instead of the pack
1961 * offset. This means it is cheaper for REF deltas (we do not have to do
1962 * the final object lookup), but more expensive for OFS deltas (we
1963 * have to load the revidx to convert the offset back into a sha1).
1964 */
1965static const unsigned char *get_delta_base_sha1(struct packed_git *p,
1966 struct pack_window **w_curs,
1967 off_t curpos,
1968 enum object_type type,
1969 off_t delta_obj_offset)
1970{
1971 if (type == OBJ_REF_DELTA) {
1972 unsigned char *base = use_pack(p, w_curs, curpos, NULL);
1973 return base;
1974 } else if (type == OBJ_OFS_DELTA) {
1975 struct revindex_entry *revidx;
1976 off_t base_offset = get_delta_base(p, w_curs, &curpos,
1977 type, delta_obj_offset);
1978
1979 if (!base_offset)
1980 return NULL;
1981
1982 revidx = find_pack_revindex(p, base_offset);
1983 if (!revidx)
1984 return NULL;
1985
1986 return nth_packed_object_sha1(p, revidx->nr);
1987 } else
1988 return NULL;
1989}
1990
1991int unpack_object_header(struct packed_git *p,
1992 struct pack_window **w_curs,
1993 off_t *curpos,
1994 unsigned long *sizep)
1995{
1996 unsigned char *base;
1997 unsigned long left;
1998 unsigned long used;
1999 enum object_type type;
2000
2001 /* use_pack() assures us we have [base, base + 20) available
2002 * as a range that we can look at. (Its actually the hash
2003 * size that is assured.) With our object header encoding
2004 * the maximum deflated object size is 2^137, which is just
2005 * insane, so we know won't exceed what we have been given.
2006 */
2007 base = use_pack(p, w_curs, *curpos, &left);
2008 used = unpack_object_header_buffer(base, left, &type, sizep);
2009 if (!used) {
2010 type = OBJ_BAD;
2011 } else
2012 *curpos += used;
2013
2014 return type;
2015}
2016
2017static int retry_bad_packed_offset(struct packed_git *p, off_t obj_offset)
2018{
2019 int type;
2020 struct revindex_entry *revidx;
2021 const unsigned char *sha1;
2022 revidx = find_pack_revindex(p, obj_offset);
2023 if (!revidx)
2024 return OBJ_BAD;
2025 sha1 = nth_packed_object_sha1(p, revidx->nr);
2026 mark_bad_packed_object(p, sha1);
2027 type = sha1_object_info(sha1, NULL);
2028 if (type <= OBJ_NONE)
2029 return OBJ_BAD;
2030 return type;
2031}
2032
2033#define POI_STACK_PREALLOC 64
2034
2035static enum object_type packed_to_object_type(struct packed_git *p,
2036 off_t obj_offset,
2037 enum object_type type,
2038 struct pack_window **w_curs,
2039 off_t curpos)
2040{
2041 off_t small_poi_stack[POI_STACK_PREALLOC];
2042 off_t *poi_stack = small_poi_stack;
2043 int poi_stack_nr = 0, poi_stack_alloc = POI_STACK_PREALLOC;
2044
2045 while (type == OBJ_OFS_DELTA || type == OBJ_REF_DELTA) {
2046 off_t base_offset;
2047 unsigned long size;
2048 /* Push the object we're going to leave behind */
2049 if (poi_stack_nr >= poi_stack_alloc && poi_stack == small_poi_stack) {
2050 poi_stack_alloc = alloc_nr(poi_stack_nr);
2051 ALLOC_ARRAY(poi_stack, poi_stack_alloc);
2052 memcpy(poi_stack, small_poi_stack, sizeof(off_t)*poi_stack_nr);
2053 } else {
2054 ALLOC_GROW(poi_stack, poi_stack_nr+1, poi_stack_alloc);
2055 }
2056 poi_stack[poi_stack_nr++] = obj_offset;
2057 /* If parsing the base offset fails, just unwind */
2058 base_offset = get_delta_base(p, w_curs, &curpos, type, obj_offset);
2059 if (!base_offset)
2060 goto unwind;
2061 curpos = obj_offset = base_offset;
2062 type = unpack_object_header(p, w_curs, &curpos, &size);
2063 if (type <= OBJ_NONE) {
2064 /* If getting the base itself fails, we first
2065 * retry the base, otherwise unwind */
2066 type = retry_bad_packed_offset(p, base_offset);
2067 if (type > OBJ_NONE)
2068 goto out;
2069 goto unwind;
2070 }
2071 }
2072
2073 switch (type) {
2074 case OBJ_BAD:
2075 case OBJ_COMMIT:
2076 case OBJ_TREE:
2077 case OBJ_BLOB:
2078 case OBJ_TAG:
2079 break;
2080 default:
2081 error("unknown object type %i at offset %"PRIuMAX" in %s",
2082 type, (uintmax_t)obj_offset, p->pack_name);
2083 type = OBJ_BAD;
2084 }
2085
2086out:
2087 if (poi_stack != small_poi_stack)
2088 free(poi_stack);
2089 return type;
2090
2091unwind:
2092 while (poi_stack_nr) {
2093 obj_offset = poi_stack[--poi_stack_nr];
2094 type = retry_bad_packed_offset(p, obj_offset);
2095 if (type > OBJ_NONE)
2096 goto out;
2097 }
2098 type = OBJ_BAD;
2099 goto out;
2100}
2101
2102static int packed_object_info(struct packed_git *p, off_t obj_offset,
2103 struct object_info *oi)
2104{
2105 struct pack_window *w_curs = NULL;
2106 unsigned long size;
2107 off_t curpos = obj_offset;
2108 enum object_type type;
2109
2110 /*
2111 * We always get the representation type, but only convert it to
2112 * a "real" type later if the caller is interested.
2113 */
2114 type = unpack_object_header(p, &w_curs, &curpos, &size);
2115
2116 if (oi->sizep) {
2117 if (type == OBJ_OFS_DELTA || type == OBJ_REF_DELTA) {
2118 off_t tmp_pos = curpos;
2119 off_t base_offset = get_delta_base(p, &w_curs, &tmp_pos,
2120 type, obj_offset);
2121 if (!base_offset) {
2122 type = OBJ_BAD;
2123 goto out;
2124 }
2125 *oi->sizep = get_size_from_delta(p, &w_curs, tmp_pos);
2126 if (*oi->sizep == 0) {
2127 type = OBJ_BAD;
2128 goto out;
2129 }
2130 } else {
2131 *oi->sizep = size;
2132 }
2133 }
2134
2135 if (oi->disk_sizep) {
2136 struct revindex_entry *revidx = find_pack_revindex(p, obj_offset);
2137 *oi->disk_sizep = revidx[1].offset - obj_offset;
2138 }
2139
2140 if (oi->typep) {
2141 *oi->typep = packed_to_object_type(p, obj_offset, type, &w_curs, curpos);
2142 if (*oi->typep < 0) {
2143 type = OBJ_BAD;
2144 goto out;
2145 }
2146 }
2147
2148 if (oi->delta_base_sha1) {
2149 if (type == OBJ_OFS_DELTA || type == OBJ_REF_DELTA) {
2150 const unsigned char *base;
2151
2152 base = get_delta_base_sha1(p, &w_curs, curpos,
2153 type, obj_offset);
2154 if (!base) {
2155 type = OBJ_BAD;
2156 goto out;
2157 }
2158
2159 hashcpy(oi->delta_base_sha1, base);
2160 } else
2161 hashclr(oi->delta_base_sha1);
2162 }
2163
2164out:
2165 unuse_pack(&w_curs);
2166 return type;
2167}
2168
2169static void *unpack_compressed_entry(struct packed_git *p,
2170 struct pack_window **w_curs,
2171 off_t curpos,
2172 unsigned long size)
2173{
2174 int st;
2175 git_zstream stream;
2176 unsigned char *buffer, *in;
2177
2178 buffer = xmallocz_gently(size);
2179 if (!buffer)
2180 return NULL;
2181 memset(&stream, 0, sizeof(stream));
2182 stream.next_out = buffer;
2183 stream.avail_out = size + 1;
2184
2185 git_inflate_init(&stream);
2186 do {
2187 in = use_pack(p, w_curs, curpos, &stream.avail_in);
2188 stream.next_in = in;
2189 st = git_inflate(&stream, Z_FINISH);
2190 if (!stream.avail_out)
2191 break; /* the payload is larger than it should be */
2192 curpos += stream.next_in - in;
2193 } while (st == Z_OK || st == Z_BUF_ERROR);
2194 git_inflate_end(&stream);
2195 if ((st != Z_STREAM_END) || stream.total_out != size) {
2196 free(buffer);
2197 return NULL;
2198 }
2199
2200 return buffer;
2201}
2202
2203static struct hashmap delta_base_cache;
2204static size_t delta_base_cached;
2205
2206static LIST_HEAD(delta_base_cache_lru);
2207
2208struct delta_base_cache_key {
2209 struct packed_git *p;
2210 off_t base_offset;
2211};
2212
2213struct delta_base_cache_entry {
2214 struct hashmap hash;
2215 struct delta_base_cache_key key;
2216 struct list_head lru;
2217 void *data;
2218 unsigned long size;
2219 enum object_type type;
2220};
2221
2222static unsigned int pack_entry_hash(struct packed_git *p, off_t base_offset)
2223{
2224 unsigned int hash;
2225
2226 hash = (unsigned int)(intptr_t)p + (unsigned int)base_offset;
2227 hash += (hash >> 8) + (hash >> 16);
2228 return hash;
2229}
2230
2231static struct delta_base_cache_entry *
2232get_delta_base_cache_entry(struct packed_git *p, off_t base_offset)
2233{
2234 struct hashmap_entry entry;
2235 struct delta_base_cache_key key;
2236
2237 if (!delta_base_cache.cmpfn)
2238 return NULL;
2239
2240 hashmap_entry_init(&entry, pack_entry_hash(p, base_offset));
2241 key.p = p;
2242 key.base_offset = base_offset;
2243 return hashmap_get(&delta_base_cache, &entry, &key);
2244}
2245
2246static int delta_base_cache_key_eq(const struct delta_base_cache_key *a,
2247 const struct delta_base_cache_key *b)
2248{
2249 return a->p == b->p && a->base_offset == b->base_offset;
2250}
2251
2252static int delta_base_cache_hash_cmp(const void *va, const void *vb,
2253 const void *vkey)
2254{
2255 const struct delta_base_cache_entry *a = va, *b = vb;
2256 const struct delta_base_cache_key *key = vkey;
2257 if (key)
2258 return !delta_base_cache_key_eq(&a->key, key);
2259 else
2260 return !delta_base_cache_key_eq(&a->key, &b->key);
2261}
2262
2263static int in_delta_base_cache(struct packed_git *p, off_t base_offset)
2264{
2265 return !!get_delta_base_cache_entry(p, base_offset);
2266}
2267
2268/*
2269 * Remove the entry from the cache, but do _not_ free the associated
2270 * entry data. The caller takes ownership of the "data" buffer, and
2271 * should copy out any fields it wants before detaching.
2272 */
2273static void detach_delta_base_cache_entry(struct delta_base_cache_entry *ent)
2274{
2275 hashmap_remove(&delta_base_cache, ent, &ent->key);
2276 list_del(&ent->lru);
2277 delta_base_cached -= ent->size;
2278 free(ent);
2279}
2280
2281static void *cache_or_unpack_entry(struct packed_git *p, off_t base_offset,
2282 unsigned long *base_size, enum object_type *type)
2283{
2284 struct delta_base_cache_entry *ent;
2285
2286 ent = get_delta_base_cache_entry(p, base_offset);
2287 if (!ent)
2288 return unpack_entry(p, base_offset, type, base_size);
2289
2290 *type = ent->type;
2291 *base_size = ent->size;
2292 return xmemdupz(ent->data, ent->size);
2293}
2294
2295static inline void release_delta_base_cache(struct delta_base_cache_entry *ent)
2296{
2297 free(ent->data);
2298 detach_delta_base_cache_entry(ent);
2299}
2300
2301void clear_delta_base_cache(void)
2302{
2303 struct hashmap_iter iter;
2304 struct delta_base_cache_entry *entry;
2305 for (entry = hashmap_iter_first(&delta_base_cache, &iter);
2306 entry;
2307 entry = hashmap_iter_next(&iter)) {
2308 release_delta_base_cache(entry);
2309 }
2310}
2311
2312static void add_delta_base_cache(struct packed_git *p, off_t base_offset,
2313 void *base, unsigned long base_size, enum object_type type)
2314{
2315 struct delta_base_cache_entry *ent = xmalloc(sizeof(*ent));
2316 struct list_head *lru, *tmp;
2317
2318 delta_base_cached += base_size;
2319
2320 list_for_each_safe(lru, tmp, &delta_base_cache_lru) {
2321 struct delta_base_cache_entry *f =
2322 list_entry(lru, struct delta_base_cache_entry, lru);
2323 if (delta_base_cached <= delta_base_cache_limit)
2324 break;
2325 release_delta_base_cache(f);
2326 }
2327
2328 ent->key.p = p;
2329 ent->key.base_offset = base_offset;
2330 ent->type = type;
2331 ent->data = base;
2332 ent->size = base_size;
2333 list_add_tail(&ent->lru, &delta_base_cache_lru);
2334
2335 if (!delta_base_cache.cmpfn)
2336 hashmap_init(&delta_base_cache, delta_base_cache_hash_cmp, 0);
2337 hashmap_entry_init(ent, pack_entry_hash(p, base_offset));
2338 hashmap_add(&delta_base_cache, ent);
2339}
2340
2341static void *read_object(const unsigned char *sha1, enum object_type *type,
2342 unsigned long *size);
2343
2344static void write_pack_access_log(struct packed_git *p, off_t obj_offset)
2345{
2346 static struct trace_key pack_access = TRACE_KEY_INIT(PACK_ACCESS);
2347 trace_printf_key(&pack_access, "%s %"PRIuMAX"\n",
2348 p->pack_name, (uintmax_t)obj_offset);
2349}
2350
2351int do_check_packed_object_crc;
2352
2353#define UNPACK_ENTRY_STACK_PREALLOC 64
2354struct unpack_entry_stack_ent {
2355 off_t obj_offset;
2356 off_t curpos;
2357 unsigned long size;
2358};
2359
2360void *unpack_entry(struct packed_git *p, off_t obj_offset,
2361 enum object_type *final_type, unsigned long *final_size)
2362{
2363 struct pack_window *w_curs = NULL;
2364 off_t curpos = obj_offset;
2365 void *data = NULL;
2366 unsigned long size;
2367 enum object_type type;
2368 struct unpack_entry_stack_ent small_delta_stack[UNPACK_ENTRY_STACK_PREALLOC];
2369 struct unpack_entry_stack_ent *delta_stack = small_delta_stack;
2370 int delta_stack_nr = 0, delta_stack_alloc = UNPACK_ENTRY_STACK_PREALLOC;
2371 int base_from_cache = 0;
2372
2373 write_pack_access_log(p, obj_offset);
2374
2375 /* PHASE 1: drill down to the innermost base object */
2376 for (;;) {
2377 off_t base_offset;
2378 int i;
2379 struct delta_base_cache_entry *ent;
2380
2381 ent = get_delta_base_cache_entry(p, curpos);
2382 if (ent) {
2383 type = ent->type;
2384 data = ent->data;
2385 size = ent->size;
2386 detach_delta_base_cache_entry(ent);
2387 base_from_cache = 1;
2388 break;
2389 }
2390
2391 if (do_check_packed_object_crc && p->index_version > 1) {
2392 struct revindex_entry *revidx = find_pack_revindex(p, obj_offset);
2393 off_t len = revidx[1].offset - obj_offset;
2394 if (check_pack_crc(p, &w_curs, obj_offset, len, revidx->nr)) {
2395 const unsigned char *sha1 =
2396 nth_packed_object_sha1(p, revidx->nr);
2397 error("bad packed object CRC for %s",
2398 sha1_to_hex(sha1));
2399 mark_bad_packed_object(p, sha1);
2400 unuse_pack(&w_curs);
2401 return NULL;
2402 }
2403 }
2404
2405 type = unpack_object_header(p, &w_curs, &curpos, &size);
2406 if (type != OBJ_OFS_DELTA && type != OBJ_REF_DELTA)
2407 break;
2408
2409 base_offset = get_delta_base(p, &w_curs, &curpos, type, obj_offset);
2410 if (!base_offset) {
2411 error("failed to validate delta base reference "
2412 "at offset %"PRIuMAX" from %s",
2413 (uintmax_t)curpos, p->pack_name);
2414 /* bail to phase 2, in hopes of recovery */
2415 data = NULL;
2416 break;
2417 }
2418
2419 /* push object, proceed to base */
2420 if (delta_stack_nr >= delta_stack_alloc
2421 && delta_stack == small_delta_stack) {
2422 delta_stack_alloc = alloc_nr(delta_stack_nr);
2423 ALLOC_ARRAY(delta_stack, delta_stack_alloc);
2424 memcpy(delta_stack, small_delta_stack,
2425 sizeof(*delta_stack)*delta_stack_nr);
2426 } else {
2427 ALLOC_GROW(delta_stack, delta_stack_nr+1, delta_stack_alloc);
2428 }
2429 i = delta_stack_nr++;
2430 delta_stack[i].obj_offset = obj_offset;
2431 delta_stack[i].curpos = curpos;
2432 delta_stack[i].size = size;
2433
2434 curpos = obj_offset = base_offset;
2435 }
2436
2437 /* PHASE 2: handle the base */
2438 switch (type) {
2439 case OBJ_OFS_DELTA:
2440 case OBJ_REF_DELTA:
2441 if (data)
2442 die("BUG: unpack_entry: left loop at a valid delta");
2443 break;
2444 case OBJ_COMMIT:
2445 case OBJ_TREE:
2446 case OBJ_BLOB:
2447 case OBJ_TAG:
2448 if (!base_from_cache)
2449 data = unpack_compressed_entry(p, &w_curs, curpos, size);
2450 break;
2451 default:
2452 data = NULL;
2453 error("unknown object type %i at offset %"PRIuMAX" in %s",
2454 type, (uintmax_t)obj_offset, p->pack_name);
2455 }
2456
2457 /* PHASE 3: apply deltas in order */
2458
2459 /* invariants:
2460 * 'data' holds the base data, or NULL if there was corruption
2461 */
2462 while (delta_stack_nr) {
2463 void *delta_data;
2464 void *base = data;
2465 unsigned long delta_size, base_size = size;
2466 int i;
2467
2468 data = NULL;
2469
2470 if (base)
2471 add_delta_base_cache(p, obj_offset, base, base_size, type);
2472
2473 if (!base) {
2474 /*
2475 * We're probably in deep shit, but let's try to fetch
2476 * the required base anyway from another pack or loose.
2477 * This is costly but should happen only in the presence
2478 * of a corrupted pack, and is better than failing outright.
2479 */
2480 struct revindex_entry *revidx;
2481 const unsigned char *base_sha1;
2482 revidx = find_pack_revindex(p, obj_offset);
2483 if (revidx) {
2484 base_sha1 = nth_packed_object_sha1(p, revidx->nr);
2485 error("failed to read delta base object %s"
2486 " at offset %"PRIuMAX" from %s",
2487 sha1_to_hex(base_sha1), (uintmax_t)obj_offset,
2488 p->pack_name);
2489 mark_bad_packed_object(p, base_sha1);
2490 base = read_object(base_sha1, &type, &base_size);
2491 }
2492 }
2493
2494 i = --delta_stack_nr;
2495 obj_offset = delta_stack[i].obj_offset;
2496 curpos = delta_stack[i].curpos;
2497 delta_size = delta_stack[i].size;
2498
2499 if (!base)
2500 continue;
2501
2502 delta_data = unpack_compressed_entry(p, &w_curs, curpos, delta_size);
2503
2504 if (!delta_data) {
2505 error("failed to unpack compressed delta "
2506 "at offset %"PRIuMAX" from %s",
2507 (uintmax_t)curpos, p->pack_name);
2508 data = NULL;
2509 continue;
2510 }
2511
2512 data = patch_delta(base, base_size,
2513 delta_data, delta_size,
2514 &size);
2515
2516 /*
2517 * We could not apply the delta; warn the user, but keep going.
2518 * Our failure will be noticed either in the next iteration of
2519 * the loop, or if this is the final delta, in the caller when
2520 * we return NULL. Those code paths will take care of making
2521 * a more explicit warning and retrying with another copy of
2522 * the object.
2523 */
2524 if (!data)
2525 error("failed to apply delta");
2526
2527 free(delta_data);
2528 }
2529
2530 *final_type = type;
2531 *final_size = size;
2532
2533 unuse_pack(&w_curs);
2534
2535 if (delta_stack != small_delta_stack)
2536 free(delta_stack);
2537
2538 return data;
2539}
2540
2541const unsigned char *nth_packed_object_sha1(struct packed_git *p,
2542 uint32_t n)
2543{
2544 const unsigned char *index = p->index_data;
2545 if (!index) {
2546 if (open_pack_index(p))
2547 return NULL;
2548 index = p->index_data;
2549 }
2550 if (n >= p->num_objects)
2551 return NULL;
2552 index += 4 * 256;
2553 if (p->index_version == 1) {
2554 return index + 24 * n + 4;
2555 } else {
2556 index += 8;
2557 return index + 20 * n;
2558 }
2559}
2560
2561void check_pack_index_ptr(const struct packed_git *p, const void *vptr)
2562{
2563 const unsigned char *ptr = vptr;
2564 const unsigned char *start = p->index_data;
2565 const unsigned char *end = start + p->index_size;
2566 if (ptr < start)
2567 die(_("offset before start of pack index for %s (corrupt index?)"),
2568 p->pack_name);
2569 /* No need to check for underflow; .idx files must be at least 8 bytes */
2570 if (ptr >= end - 8)
2571 die(_("offset beyond end of pack index for %s (truncated index?)"),
2572 p->pack_name);
2573}
2574
2575off_t nth_packed_object_offset(const struct packed_git *p, uint32_t n)
2576{
2577 const unsigned char *index = p->index_data;
2578 index += 4 * 256;
2579 if (p->index_version == 1) {
2580 return ntohl(*((uint32_t *)(index + 24 * n)));
2581 } else {
2582 uint32_t off;
2583 index += 8 + p->num_objects * (20 + 4);
2584 off = ntohl(*((uint32_t *)(index + 4 * n)));
2585 if (!(off & 0x80000000))
2586 return off;
2587 index += p->num_objects * 4 + (off & 0x7fffffff) * 8;
2588 check_pack_index_ptr(p, index);
2589 return (((uint64_t)ntohl(*((uint32_t *)(index + 0)))) << 32) |
2590 ntohl(*((uint32_t *)(index + 4)));
2591 }
2592}
2593
2594off_t find_pack_entry_one(const unsigned char *sha1,
2595 struct packed_git *p)
2596{
2597 const uint32_t *level1_ofs = p->index_data;
2598 const unsigned char *index = p->index_data;
2599 unsigned hi, lo, stride;
2600 static int use_lookup = -1;
2601 static int debug_lookup = -1;
2602
2603 if (debug_lookup < 0)
2604 debug_lookup = !!getenv("GIT_DEBUG_LOOKUP");
2605
2606 if (!index) {
2607 if (open_pack_index(p))
2608 return 0;
2609 level1_ofs = p->index_data;
2610 index = p->index_data;
2611 }
2612 if (p->index_version > 1) {
2613 level1_ofs += 2;
2614 index += 8;
2615 }
2616 index += 4 * 256;
2617 hi = ntohl(level1_ofs[*sha1]);
2618 lo = ((*sha1 == 0x0) ? 0 : ntohl(level1_ofs[*sha1 - 1]));
2619 if (p->index_version > 1) {
2620 stride = 20;
2621 } else {
2622 stride = 24;
2623 index += 4;
2624 }
2625
2626 if (debug_lookup)
2627 printf("%02x%02x%02x... lo %u hi %u nr %"PRIu32"\n",
2628 sha1[0], sha1[1], sha1[2], lo, hi, p->num_objects);
2629
2630 if (use_lookup < 0)
2631 use_lookup = !!getenv("GIT_USE_LOOKUP");
2632 if (use_lookup) {
2633 int pos = sha1_entry_pos(index, stride, 0,
2634 lo, hi, p->num_objects, sha1);
2635 if (pos < 0)
2636 return 0;
2637 return nth_packed_object_offset(p, pos);
2638 }
2639
2640 do {
2641 unsigned mi = (lo + hi) / 2;
2642 int cmp = hashcmp(index + mi * stride, sha1);
2643
2644 if (debug_lookup)
2645 printf("lo %u hi %u rg %u mi %u\n",
2646 lo, hi, hi - lo, mi);
2647 if (!cmp)
2648 return nth_packed_object_offset(p, mi);
2649 if (cmp > 0)
2650 hi = mi;
2651 else
2652 lo = mi+1;
2653 } while (lo < hi);
2654 return 0;
2655}
2656
2657int is_pack_valid(struct packed_git *p)
2658{
2659 /* An already open pack is known to be valid. */
2660 if (p->pack_fd != -1)
2661 return 1;
2662
2663 /* If the pack has one window completely covering the
2664 * file size, the pack is known to be valid even if
2665 * the descriptor is not currently open.
2666 */
2667 if (p->windows) {
2668 struct pack_window *w = p->windows;
2669
2670 if (!w->offset && w->len == p->pack_size)
2671 return 1;
2672 }
2673
2674 /* Force the pack to open to prove its valid. */
2675 return !open_packed_git(p);
2676}
2677
2678static int fill_pack_entry(const unsigned char *sha1,
2679 struct pack_entry *e,
2680 struct packed_git *p)
2681{
2682 off_t offset;
2683
2684 if (p->num_bad_objects) {
2685 unsigned i;
2686 for (i = 0; i < p->num_bad_objects; i++)
2687 if (!hashcmp(sha1, p->bad_object_sha1 + 20 * i))
2688 return 0;
2689 }
2690
2691 offset = find_pack_entry_one(sha1, p);
2692 if (!offset)
2693 return 0;
2694
2695 /*
2696 * We are about to tell the caller where they can locate the
2697 * requested object. We better make sure the packfile is
2698 * still here and can be accessed before supplying that
2699 * answer, as it may have been deleted since the index was
2700 * loaded!
2701 */
2702 if (!is_pack_valid(p))
2703 return 0;
2704 e->offset = offset;
2705 e->p = p;
2706 hashcpy(e->sha1, sha1);
2707 return 1;
2708}
2709
2710/*
2711 * Iff a pack file contains the object named by sha1, return true and
2712 * store its location to e.
2713 */
2714static int find_pack_entry(const unsigned char *sha1, struct pack_entry *e)
2715{
2716 struct mru_entry *p;
2717
2718 prepare_packed_git();
2719 if (!packed_git)
2720 return 0;
2721
2722 for (p = packed_git_mru->head; p; p = p->next) {
2723 if (fill_pack_entry(sha1, e, p->item)) {
2724 mru_mark(packed_git_mru, p);
2725 return 1;
2726 }
2727 }
2728 return 0;
2729}
2730
2731struct packed_git *find_sha1_pack(const unsigned char *sha1,
2732 struct packed_git *packs)
2733{
2734 struct packed_git *p;
2735
2736 for (p = packs; p; p = p->next) {
2737 if (find_pack_entry_one(sha1, p))
2738 return p;
2739 }
2740 return NULL;
2741
2742}
2743
2744static int sha1_loose_object_info(const unsigned char *sha1,
2745 struct object_info *oi,
2746 int flags)
2747{
2748 int status = 0;
2749 unsigned long mapsize;
2750 void *map;
2751 git_zstream stream;
2752 char hdr[32];
2753 struct strbuf hdrbuf = STRBUF_INIT;
2754
2755 if (oi->delta_base_sha1)
2756 hashclr(oi->delta_base_sha1);
2757
2758 /*
2759 * If we don't care about type or size, then we don't
2760 * need to look inside the object at all. Note that we
2761 * do not optimize out the stat call, even if the
2762 * caller doesn't care about the disk-size, since our
2763 * return value implicitly indicates whether the
2764 * object even exists.
2765 */
2766 if (!oi->typep && !oi->typename && !oi->sizep) {
2767 struct stat st;
2768 if (stat_sha1_file(sha1, &st) < 0)
2769 return -1;
2770 if (oi->disk_sizep)
2771 *oi->disk_sizep = st.st_size;
2772 return 0;
2773 }
2774
2775 map = map_sha1_file(sha1, &mapsize);
2776 if (!map)
2777 return -1;
2778 if (oi->disk_sizep)
2779 *oi->disk_sizep = mapsize;
2780 if ((flags & LOOKUP_UNKNOWN_OBJECT)) {
2781 if (unpack_sha1_header_to_strbuf(&stream, map, mapsize, hdr, sizeof(hdr), &hdrbuf) < 0)
2782 status = error("unable to unpack %s header with --allow-unknown-type",
2783 sha1_to_hex(sha1));
2784 } else if (unpack_sha1_header(&stream, map, mapsize, hdr, sizeof(hdr)) < 0)
2785 status = error("unable to unpack %s header",
2786 sha1_to_hex(sha1));
2787 if (status < 0)
2788 ; /* Do nothing */
2789 else if (hdrbuf.len) {
2790 if ((status = parse_sha1_header_extended(hdrbuf.buf, oi, flags)) < 0)
2791 status = error("unable to parse %s header with --allow-unknown-type",
2792 sha1_to_hex(sha1));
2793 } else if ((status = parse_sha1_header_extended(hdr, oi, flags)) < 0)
2794 status = error("unable to parse %s header", sha1_to_hex(sha1));
2795 git_inflate_end(&stream);
2796 munmap(map, mapsize);
2797 if (status && oi->typep)
2798 *oi->typep = status;
2799 strbuf_release(&hdrbuf);
2800 return 0;
2801}
2802
2803int sha1_object_info_extended(const unsigned char *sha1, struct object_info *oi, unsigned flags)
2804{
2805 struct cached_object *co;
2806 struct pack_entry e;
2807 int rtype;
2808 enum object_type real_type;
2809 const unsigned char *real = lookup_replace_object_extended(sha1, flags);
2810
2811 co = find_cached_object(real);
2812 if (co) {
2813 if (oi->typep)
2814 *(oi->typep) = co->type;
2815 if (oi->sizep)
2816 *(oi->sizep) = co->size;
2817 if (oi->disk_sizep)
2818 *(oi->disk_sizep) = 0;
2819 if (oi->delta_base_sha1)
2820 hashclr(oi->delta_base_sha1);
2821 if (oi->typename)
2822 strbuf_addstr(oi->typename, typename(co->type));
2823 oi->whence = OI_CACHED;
2824 return 0;
2825 }
2826
2827 if (!find_pack_entry(real, &e)) {
2828 /* Most likely it's a loose object. */
2829 if (!sha1_loose_object_info(real, oi, flags)) {
2830 oi->whence = OI_LOOSE;
2831 return 0;
2832 }
2833
2834 /* Not a loose object; someone else may have just packed it. */
2835 reprepare_packed_git();
2836 if (!find_pack_entry(real, &e))
2837 return -1;
2838 }
2839
2840 /*
2841 * packed_object_info() does not follow the delta chain to
2842 * find out the real type, unless it is given oi->typep.
2843 */
2844 if (oi->typename && !oi->typep)
2845 oi->typep = &real_type;
2846
2847 rtype = packed_object_info(e.p, e.offset, oi);
2848 if (rtype < 0) {
2849 mark_bad_packed_object(e.p, real);
2850 if (oi->typep == &real_type)
2851 oi->typep = NULL;
2852 return sha1_object_info_extended(real, oi, 0);
2853 } else if (in_delta_base_cache(e.p, e.offset)) {
2854 oi->whence = OI_DBCACHED;
2855 } else {
2856 oi->whence = OI_PACKED;
2857 oi->u.packed.offset = e.offset;
2858 oi->u.packed.pack = e.p;
2859 oi->u.packed.is_delta = (rtype == OBJ_REF_DELTA ||
2860 rtype == OBJ_OFS_DELTA);
2861 }
2862 if (oi->typename)
2863 strbuf_addstr(oi->typename, typename(*oi->typep));
2864 if (oi->typep == &real_type)
2865 oi->typep = NULL;
2866
2867 return 0;
2868}
2869
2870/* returns enum object_type or negative */
2871int sha1_object_info(const unsigned char *sha1, unsigned long *sizep)
2872{
2873 enum object_type type;
2874 struct object_info oi = {NULL};
2875
2876 oi.typep = &type;
2877 oi.sizep = sizep;
2878 if (sha1_object_info_extended(sha1, &oi, LOOKUP_REPLACE_OBJECT) < 0)
2879 return -1;
2880 return type;
2881}
2882
2883static void *read_packed_sha1(const unsigned char *sha1,
2884 enum object_type *type, unsigned long *size)
2885{
2886 struct pack_entry e;
2887 void *data;
2888
2889 if (!find_pack_entry(sha1, &e))
2890 return NULL;
2891 data = cache_or_unpack_entry(e.p, e.offset, size, type);
2892 if (!data) {
2893 /*
2894 * We're probably in deep shit, but let's try to fetch
2895 * the required object anyway from another pack or loose.
2896 * This should happen only in the presence of a corrupted
2897 * pack, and is better than failing outright.
2898 */
2899 error("failed to read object %s at offset %"PRIuMAX" from %s",
2900 sha1_to_hex(sha1), (uintmax_t)e.offset, e.p->pack_name);
2901 mark_bad_packed_object(e.p, sha1);
2902 data = read_object(sha1, type, size);
2903 }
2904 return data;
2905}
2906
2907int pretend_sha1_file(void *buf, unsigned long len, enum object_type type,
2908 unsigned char *sha1)
2909{
2910 struct cached_object *co;
2911
2912 hash_sha1_file(buf, len, typename(type), sha1);
2913 if (has_sha1_file(sha1) || find_cached_object(sha1))
2914 return 0;
2915 ALLOC_GROW(cached_objects, cached_object_nr + 1, cached_object_alloc);
2916 co = &cached_objects[cached_object_nr++];
2917 co->size = len;
2918 co->type = type;
2919 co->buf = xmalloc(len);
2920 memcpy(co->buf, buf, len);
2921 hashcpy(co->sha1, sha1);
2922 return 0;
2923}
2924
2925static void *read_object(const unsigned char *sha1, enum object_type *type,
2926 unsigned long *size)
2927{
2928 unsigned long mapsize;
2929 void *map, *buf;
2930 struct cached_object *co;
2931
2932 co = find_cached_object(sha1);
2933 if (co) {
2934 *type = co->type;
2935 *size = co->size;
2936 return xmemdupz(co->buf, co->size);
2937 }
2938
2939 buf = read_packed_sha1(sha1, type, size);
2940 if (buf)
2941 return buf;
2942 map = map_sha1_file(sha1, &mapsize);
2943 if (map) {
2944 buf = unpack_sha1_file(map, mapsize, type, size, sha1);
2945 munmap(map, mapsize);
2946 return buf;
2947 }
2948 reprepare_packed_git();
2949 return read_packed_sha1(sha1, type, size);
2950}
2951
2952/*
2953 * This function dies on corrupt objects; the callers who want to
2954 * deal with them should arrange to call read_object() and give error
2955 * messages themselves.
2956 */
2957void *read_sha1_file_extended(const unsigned char *sha1,
2958 enum object_type *type,
2959 unsigned long *size,
2960 unsigned flag)
2961{
2962 void *data;
2963 const struct packed_git *p;
2964 const unsigned char *repl = lookup_replace_object_extended(sha1, flag);
2965
2966 errno = 0;
2967 data = read_object(repl, type, size);
2968 if (data)
2969 return data;
2970
2971 if (errno && errno != ENOENT)
2972 die_errno("failed to read object %s", sha1_to_hex(sha1));
2973
2974 /* die if we replaced an object with one that does not exist */
2975 if (repl != sha1)
2976 die("replacement %s not found for %s",
2977 sha1_to_hex(repl), sha1_to_hex(sha1));
2978
2979 if (has_loose_object(repl)) {
2980 const char *path = sha1_file_name(sha1);
2981
2982 die("loose object %s (stored in %s) is corrupt",
2983 sha1_to_hex(repl), path);
2984 }
2985
2986 if ((p = has_packed_and_bad(repl)) != NULL)
2987 die("packed object %s (stored in %s) is corrupt",
2988 sha1_to_hex(repl), p->pack_name);
2989
2990 return NULL;
2991}
2992
2993void *read_object_with_reference(const unsigned char *sha1,
2994 const char *required_type_name,
2995 unsigned long *size,
2996 unsigned char *actual_sha1_return)
2997{
2998 enum object_type type, required_type;
2999 void *buffer;
3000 unsigned long isize;
3001 unsigned char actual_sha1[20];
3002
3003 required_type = type_from_string(required_type_name);
3004 hashcpy(actual_sha1, sha1);
3005 while (1) {
3006 int ref_length = -1;
3007 const char *ref_type = NULL;
3008
3009 buffer = read_sha1_file(actual_sha1, &type, &isize);
3010 if (!buffer)
3011 return NULL;
3012 if (type == required_type) {
3013 *size = isize;
3014 if (actual_sha1_return)
3015 hashcpy(actual_sha1_return, actual_sha1);
3016 return buffer;
3017 }
3018 /* Handle references */
3019 else if (type == OBJ_COMMIT)
3020 ref_type = "tree ";
3021 else if (type == OBJ_TAG)
3022 ref_type = "object ";
3023 else {
3024 free(buffer);
3025 return NULL;
3026 }
3027 ref_length = strlen(ref_type);
3028
3029 if (ref_length + 40 > isize ||
3030 memcmp(buffer, ref_type, ref_length) ||
3031 get_sha1_hex((char *) buffer + ref_length, actual_sha1)) {
3032 free(buffer);
3033 return NULL;
3034 }
3035 free(buffer);
3036 /* Now we have the ID of the referred-to object in
3037 * actual_sha1. Check again. */
3038 }
3039}
3040
3041static void write_sha1_file_prepare(const void *buf, unsigned long len,
3042 const char *type, unsigned char *sha1,
3043 char *hdr, int *hdrlen)
3044{
3045 git_SHA_CTX c;
3046
3047 /* Generate the header */
3048 *hdrlen = xsnprintf(hdr, *hdrlen, "%s %lu", type, len)+1;
3049
3050 /* Sha1.. */
3051 git_SHA1_Init(&c);
3052 git_SHA1_Update(&c, hdr, *hdrlen);
3053 git_SHA1_Update(&c, buf, len);
3054 git_SHA1_Final(sha1, &c);
3055}
3056
3057/*
3058 * Move the just written object into its final resting place.
3059 */
3060int finalize_object_file(const char *tmpfile, const char *filename)
3061{
3062 int ret = 0;
3063
3064 if (object_creation_mode == OBJECT_CREATION_USES_RENAMES)
3065 goto try_rename;
3066 else if (link(tmpfile, filename))
3067 ret = errno;
3068
3069 /*
3070 * Coda hack - coda doesn't like cross-directory links,
3071 * so we fall back to a rename, which will mean that it
3072 * won't be able to check collisions, but that's not a
3073 * big deal.
3074 *
3075 * The same holds for FAT formatted media.
3076 *
3077 * When this succeeds, we just return. We have nothing
3078 * left to unlink.
3079 */
3080 if (ret && ret != EEXIST) {
3081 try_rename:
3082 if (!rename(tmpfile, filename))
3083 goto out;
3084 ret = errno;
3085 }
3086 unlink_or_warn(tmpfile);
3087 if (ret) {
3088 if (ret != EEXIST) {
3089 return error_errno("unable to write sha1 filename %s", filename);
3090 }
3091 /* FIXME!!! Collision check here ? */
3092 }
3093
3094out:
3095 if (adjust_shared_perm(filename))
3096 return error("unable to set permission to '%s'", filename);
3097 return 0;
3098}
3099
3100static int write_buffer(int fd, const void *buf, size_t len)
3101{
3102 if (write_in_full(fd, buf, len) < 0)
3103 return error_errno("file write error");
3104 return 0;
3105}
3106
3107int hash_sha1_file(const void *buf, unsigned long len, const char *type,
3108 unsigned char *sha1)
3109{
3110 char hdr[32];
3111 int hdrlen = sizeof(hdr);
3112 write_sha1_file_prepare(buf, len, type, sha1, hdr, &hdrlen);
3113 return 0;
3114}
3115
3116/* Finalize a file on disk, and close it. */
3117static void close_sha1_file(int fd)
3118{
3119 if (fsync_object_files)
3120 fsync_or_die(fd, "sha1 file");
3121 if (close(fd) != 0)
3122 die_errno("error when closing sha1 file");
3123}
3124
3125/* Size of directory component, including the ending '/' */
3126static inline int directory_size(const char *filename)
3127{
3128 const char *s = strrchr(filename, '/');
3129 if (!s)
3130 return 0;
3131 return s - filename + 1;
3132}
3133
3134/*
3135 * This creates a temporary file in the same directory as the final
3136 * 'filename'
3137 *
3138 * We want to avoid cross-directory filename renames, because those
3139 * can have problems on various filesystems (FAT, NFS, Coda).
3140 */
3141static int create_tmpfile(struct strbuf *tmp, const char *filename)
3142{
3143 int fd, dirlen = directory_size(filename);
3144
3145 strbuf_reset(tmp);
3146 strbuf_add(tmp, filename, dirlen);
3147 strbuf_addstr(tmp, "tmp_obj_XXXXXX");
3148 fd = git_mkstemp_mode(tmp->buf, 0444);
3149 if (fd < 0 && dirlen && errno == ENOENT) {
3150 /*
3151 * Make sure the directory exists; note that the contents
3152 * of the buffer are undefined after mkstemp returns an
3153 * error, so we have to rewrite the whole buffer from
3154 * scratch.
3155 */
3156 strbuf_reset(tmp);
3157 strbuf_add(tmp, filename, dirlen - 1);
3158 if (mkdir(tmp->buf, 0777) && errno != EEXIST)
3159 return -1;
3160 if (adjust_shared_perm(tmp->buf))
3161 return -1;
3162
3163 /* Try again */
3164 strbuf_addstr(tmp, "/tmp_obj_XXXXXX");
3165 fd = git_mkstemp_mode(tmp->buf, 0444);
3166 }
3167 return fd;
3168}
3169
3170static int write_loose_object(const unsigned char *sha1, char *hdr, int hdrlen,
3171 const void *buf, unsigned long len, time_t mtime)
3172{
3173 int fd, ret;
3174 unsigned char compressed[4096];
3175 git_zstream stream;
3176 git_SHA_CTX c;
3177 unsigned char parano_sha1[20];
3178 static struct strbuf tmp_file = STRBUF_INIT;
3179 const char *filename = sha1_file_name(sha1);
3180
3181 fd = create_tmpfile(&tmp_file, filename);
3182 if (fd < 0) {
3183 if (errno == EACCES)
3184 return error("insufficient permission for adding an object to repository database %s", get_object_directory());
3185 else
3186 return error_errno("unable to create temporary file");
3187 }
3188
3189 /* Set it up */
3190 git_deflate_init(&stream, zlib_compression_level);
3191 stream.next_out = compressed;
3192 stream.avail_out = sizeof(compressed);
3193 git_SHA1_Init(&c);
3194
3195 /* First header.. */
3196 stream.next_in = (unsigned char *)hdr;
3197 stream.avail_in = hdrlen;
3198 while (git_deflate(&stream, 0) == Z_OK)
3199 ; /* nothing */
3200 git_SHA1_Update(&c, hdr, hdrlen);
3201
3202 /* Then the data itself.. */
3203 stream.next_in = (void *)buf;
3204 stream.avail_in = len;
3205 do {
3206 unsigned char *in0 = stream.next_in;
3207 ret = git_deflate(&stream, Z_FINISH);
3208 git_SHA1_Update(&c, in0, stream.next_in - in0);
3209 if (write_buffer(fd, compressed, stream.next_out - compressed) < 0)
3210 die("unable to write sha1 file");
3211 stream.next_out = compressed;
3212 stream.avail_out = sizeof(compressed);
3213 } while (ret == Z_OK);
3214
3215 if (ret != Z_STREAM_END)
3216 die("unable to deflate new object %s (%d)", sha1_to_hex(sha1), ret);
3217 ret = git_deflate_end_gently(&stream);
3218 if (ret != Z_OK)
3219 die("deflateEnd on object %s failed (%d)", sha1_to_hex(sha1), ret);
3220 git_SHA1_Final(parano_sha1, &c);
3221 if (hashcmp(sha1, parano_sha1) != 0)
3222 die("confused by unstable object source data for %s", sha1_to_hex(sha1));
3223
3224 close_sha1_file(fd);
3225
3226 if (mtime) {
3227 struct utimbuf utb;
3228 utb.actime = mtime;
3229 utb.modtime = mtime;
3230 if (utime(tmp_file.buf, &utb) < 0)
3231 warning_errno("failed utime() on %s", tmp_file.buf);
3232 }
3233
3234 return finalize_object_file(tmp_file.buf, filename);
3235}
3236
3237static int freshen_loose_object(const unsigned char *sha1)
3238{
3239 return check_and_freshen(sha1, 1);
3240}
3241
3242static int freshen_packed_object(const unsigned char *sha1)
3243{
3244 struct pack_entry e;
3245 if (!find_pack_entry(sha1, &e))
3246 return 0;
3247 if (e.p->freshened)
3248 return 1;
3249 if (!freshen_file(e.p->pack_name))
3250 return 0;
3251 e.p->freshened = 1;
3252 return 1;
3253}
3254
3255int write_sha1_file(const void *buf, unsigned long len, const char *type, unsigned char *sha1)
3256{
3257 char hdr[32];
3258 int hdrlen = sizeof(hdr);
3259
3260 /* Normally if we have it in the pack then we do not bother writing
3261 * it out into .git/objects/??/?{38} file.
3262 */
3263 write_sha1_file_prepare(buf, len, type, sha1, hdr, &hdrlen);
3264 if (freshen_packed_object(sha1) || freshen_loose_object(sha1))
3265 return 0;
3266 return write_loose_object(sha1, hdr, hdrlen, buf, len, 0);
3267}
3268
3269int hash_sha1_file_literally(const void *buf, unsigned long len, const char *type,
3270 unsigned char *sha1, unsigned flags)
3271{
3272 char *header;
3273 int hdrlen, status = 0;
3274
3275 /* type string, SP, %lu of the length plus NUL must fit this */
3276 hdrlen = strlen(type) + 32;
3277 header = xmalloc(hdrlen);
3278 write_sha1_file_prepare(buf, len, type, sha1, header, &hdrlen);
3279
3280 if (!(flags & HASH_WRITE_OBJECT))
3281 goto cleanup;
3282 if (freshen_packed_object(sha1) || freshen_loose_object(sha1))
3283 goto cleanup;
3284 status = write_loose_object(sha1, header, hdrlen, buf, len, 0);
3285
3286cleanup:
3287 free(header);
3288 return status;
3289}
3290
3291int force_object_loose(const unsigned char *sha1, time_t mtime)
3292{
3293 void *buf;
3294 unsigned long len;
3295 enum object_type type;
3296 char hdr[32];
3297 int hdrlen;
3298 int ret;
3299
3300 if (has_loose_object(sha1))
3301 return 0;
3302 buf = read_packed_sha1(sha1, &type, &len);
3303 if (!buf)
3304 return error("cannot read sha1_file for %s", sha1_to_hex(sha1));
3305 hdrlen = xsnprintf(hdr, sizeof(hdr), "%s %lu", typename(type), len) + 1;
3306 ret = write_loose_object(sha1, hdr, hdrlen, buf, len, mtime);
3307 free(buf);
3308
3309 return ret;
3310}
3311
3312int has_pack_index(const unsigned char *sha1)
3313{
3314 struct stat st;
3315 if (stat(sha1_pack_index_name(sha1), &st))
3316 return 0;
3317 return 1;
3318}
3319
3320int has_sha1_pack(const unsigned char *sha1)
3321{
3322 struct pack_entry e;
3323 return find_pack_entry(sha1, &e);
3324}
3325
3326int has_sha1_file_with_flags(const unsigned char *sha1, int flags)
3327{
3328 struct pack_entry e;
3329
3330 if (find_pack_entry(sha1, &e))
3331 return 1;
3332 if (has_loose_object(sha1))
3333 return 1;
3334 if (flags & HAS_SHA1_QUICK)
3335 return 0;
3336 reprepare_packed_git();
3337 return find_pack_entry(sha1, &e);
3338}
3339
3340int has_object_file(const struct object_id *oid)
3341{
3342 return has_sha1_file(oid->hash);
3343}
3344
3345static void check_tree(const void *buf, size_t size)
3346{
3347 struct tree_desc desc;
3348 struct name_entry entry;
3349
3350 init_tree_desc(&desc, buf, size);
3351 while (tree_entry(&desc, &entry))
3352 /* do nothing
3353 * tree_entry() will die() on malformed entries */
3354 ;
3355}
3356
3357static void check_commit(const void *buf, size_t size)
3358{
3359 struct commit c;
3360 memset(&c, 0, sizeof(c));
3361 if (parse_commit_buffer(&c, buf, size))
3362 die("corrupt commit");
3363}
3364
3365static void check_tag(const void *buf, size_t size)
3366{
3367 struct tag t;
3368 memset(&t, 0, sizeof(t));
3369 if (parse_tag_buffer(&t, buf, size))
3370 die("corrupt tag");
3371}
3372
3373static int index_mem(unsigned char *sha1, void *buf, size_t size,
3374 enum object_type type,
3375 const char *path, unsigned flags)
3376{
3377 int ret, re_allocated = 0;
3378 int write_object = flags & HASH_WRITE_OBJECT;
3379
3380 if (!type)
3381 type = OBJ_BLOB;
3382
3383 /*
3384 * Convert blobs to git internal format
3385 */
3386 if ((type == OBJ_BLOB) && path) {
3387 struct strbuf nbuf = STRBUF_INIT;
3388 if (convert_to_git(path, buf, size, &nbuf,
3389 write_object ? safe_crlf : SAFE_CRLF_FALSE)) {
3390 buf = strbuf_detach(&nbuf, &size);
3391 re_allocated = 1;
3392 }
3393 }
3394 if (flags & HASH_FORMAT_CHECK) {
3395 if (type == OBJ_TREE)
3396 check_tree(buf, size);
3397 if (type == OBJ_COMMIT)
3398 check_commit(buf, size);
3399 if (type == OBJ_TAG)
3400 check_tag(buf, size);
3401 }
3402
3403 if (write_object)
3404 ret = write_sha1_file(buf, size, typename(type), sha1);
3405 else
3406 ret = hash_sha1_file(buf, size, typename(type), sha1);
3407 if (re_allocated)
3408 free(buf);
3409 return ret;
3410}
3411
3412static int index_stream_convert_blob(unsigned char *sha1, int fd,
3413 const char *path, unsigned flags)
3414{
3415 int ret;
3416 const int write_object = flags & HASH_WRITE_OBJECT;
3417 struct strbuf sbuf = STRBUF_INIT;
3418
3419 assert(path);
3420 assert(would_convert_to_git_filter_fd(path));
3421
3422 convert_to_git_filter_fd(path, fd, &sbuf,
3423 write_object ? safe_crlf : SAFE_CRLF_FALSE);
3424
3425 if (write_object)
3426 ret = write_sha1_file(sbuf.buf, sbuf.len, typename(OBJ_BLOB),
3427 sha1);
3428 else
3429 ret = hash_sha1_file(sbuf.buf, sbuf.len, typename(OBJ_BLOB),
3430 sha1);
3431 strbuf_release(&sbuf);
3432 return ret;
3433}
3434
3435static int index_pipe(unsigned char *sha1, int fd, enum object_type type,
3436 const char *path, unsigned flags)
3437{
3438 struct strbuf sbuf = STRBUF_INIT;
3439 int ret;
3440
3441 if (strbuf_read(&sbuf, fd, 4096) >= 0)
3442 ret = index_mem(sha1, sbuf.buf, sbuf.len, type, path, flags);
3443 else
3444 ret = -1;
3445 strbuf_release(&sbuf);
3446 return ret;
3447}
3448
3449#define SMALL_FILE_SIZE (32*1024)
3450
3451static int index_core(unsigned char *sha1, int fd, size_t size,
3452 enum object_type type, const char *path,
3453 unsigned flags)
3454{
3455 int ret;
3456
3457 if (!size) {
3458 ret = index_mem(sha1, "", size, type, path, flags);
3459 } else if (size <= SMALL_FILE_SIZE) {
3460 char *buf = xmalloc(size);
3461 if (size == read_in_full(fd, buf, size))
3462 ret = index_mem(sha1, buf, size, type, path, flags);
3463 else
3464 ret = error_errno("short read");
3465 free(buf);
3466 } else {
3467 void *buf = xmmap(NULL, size, PROT_READ, MAP_PRIVATE, fd, 0);
3468 ret = index_mem(sha1, buf, size, type, path, flags);
3469 munmap(buf, size);
3470 }
3471 return ret;
3472}
3473
3474/*
3475 * This creates one packfile per large blob unless bulk-checkin
3476 * machinery is "plugged".
3477 *
3478 * This also bypasses the usual "convert-to-git" dance, and that is on
3479 * purpose. We could write a streaming version of the converting
3480 * functions and insert that before feeding the data to fast-import
3481 * (or equivalent in-core API described above). However, that is
3482 * somewhat complicated, as we do not know the size of the filter
3483 * result, which we need to know beforehand when writing a git object.
3484 * Since the primary motivation for trying to stream from the working
3485 * tree file and to avoid mmaping it in core is to deal with large
3486 * binary blobs, they generally do not want to get any conversion, and
3487 * callers should avoid this code path when filters are requested.
3488 */
3489static int index_stream(unsigned char *sha1, int fd, size_t size,
3490 enum object_type type, const char *path,
3491 unsigned flags)
3492{
3493 return index_bulk_checkin(sha1, fd, size, type, path, flags);
3494}
3495
3496int index_fd(unsigned char *sha1, int fd, struct stat *st,
3497 enum object_type type, const char *path, unsigned flags)
3498{
3499 int ret;
3500
3501 /*
3502 * Call xsize_t() only when needed to avoid potentially unnecessary
3503 * die() for large files.
3504 */
3505 if (type == OBJ_BLOB && path && would_convert_to_git_filter_fd(path))
3506 ret = index_stream_convert_blob(sha1, fd, path, flags);
3507 else if (!S_ISREG(st->st_mode))
3508 ret = index_pipe(sha1, fd, type, path, flags);
3509 else if (st->st_size <= big_file_threshold || type != OBJ_BLOB ||
3510 (path && would_convert_to_git(path)))
3511 ret = index_core(sha1, fd, xsize_t(st->st_size), type, path,
3512 flags);
3513 else
3514 ret = index_stream(sha1, fd, xsize_t(st->st_size), type, path,
3515 flags);
3516 close(fd);
3517 return ret;
3518}
3519
3520int index_path(unsigned char *sha1, const char *path, struct stat *st, unsigned flags)
3521{
3522 int fd;
3523 struct strbuf sb = STRBUF_INIT;
3524
3525 switch (st->st_mode & S_IFMT) {
3526 case S_IFREG:
3527 fd = open(path, O_RDONLY);
3528 if (fd < 0)
3529 return error_errno("open(\"%s\")", path);
3530 if (index_fd(sha1, fd, st, OBJ_BLOB, path, flags) < 0)
3531 return error("%s: failed to insert into database",
3532 path);
3533 break;
3534 case S_IFLNK:
3535 if (strbuf_readlink(&sb, path, st->st_size))
3536 return error_errno("readlink(\"%s\")", path);
3537 if (!(flags & HASH_WRITE_OBJECT))
3538 hash_sha1_file(sb.buf, sb.len, blob_type, sha1);
3539 else if (write_sha1_file(sb.buf, sb.len, blob_type, sha1))
3540 return error("%s: failed to insert into database",
3541 path);
3542 strbuf_release(&sb);
3543 break;
3544 case S_IFDIR:
3545 return resolve_gitlink_ref(path, "HEAD", sha1);
3546 default:
3547 return error("%s: unsupported file type", path);
3548 }
3549 return 0;
3550}
3551
3552int read_pack_header(int fd, struct pack_header *header)
3553{
3554 if (read_in_full(fd, header, sizeof(*header)) < sizeof(*header))
3555 /* "eof before pack header was fully read" */
3556 return PH_ERROR_EOF;
3557
3558 if (header->hdr_signature != htonl(PACK_SIGNATURE))
3559 /* "protocol error (pack signature mismatch detected)" */
3560 return PH_ERROR_PACK_SIGNATURE;
3561 if (!pack_version_ok(header->hdr_version))
3562 /* "protocol error (pack version unsupported)" */
3563 return PH_ERROR_PROTOCOL;
3564 return 0;
3565}
3566
3567void assert_sha1_type(const unsigned char *sha1, enum object_type expect)
3568{
3569 enum object_type type = sha1_object_info(sha1, NULL);
3570 if (type < 0)
3571 die("%s is not a valid object", sha1_to_hex(sha1));
3572 if (type != expect)
3573 die("%s is not a valid '%s' object", sha1_to_hex(sha1),
3574 typename(expect));
3575}
3576
3577static int for_each_file_in_obj_subdir(int subdir_nr,
3578 struct strbuf *path,
3579 each_loose_object_fn obj_cb,
3580 each_loose_cruft_fn cruft_cb,
3581 each_loose_subdir_fn subdir_cb,
3582 void *data)
3583{
3584 size_t baselen = path->len;
3585 DIR *dir = opendir(path->buf);
3586 struct dirent *de;
3587 int r = 0;
3588
3589 if (!dir) {
3590 if (errno == ENOENT)
3591 return 0;
3592 return error_errno("unable to open %s", path->buf);
3593 }
3594
3595 while ((de = readdir(dir))) {
3596 if (is_dot_or_dotdot(de->d_name))
3597 continue;
3598
3599 strbuf_setlen(path, baselen);
3600 strbuf_addf(path, "/%s", de->d_name);
3601
3602 if (strlen(de->d_name) == 38) {
3603 char hex[41];
3604 unsigned char sha1[20];
3605
3606 snprintf(hex, sizeof(hex), "%02x%s",
3607 subdir_nr, de->d_name);
3608 if (!get_sha1_hex(hex, sha1)) {
3609 if (obj_cb) {
3610 r = obj_cb(sha1, path->buf, data);
3611 if (r)
3612 break;
3613 }
3614 continue;
3615 }
3616 }
3617
3618 if (cruft_cb) {
3619 r = cruft_cb(de->d_name, path->buf, data);
3620 if (r)
3621 break;
3622 }
3623 }
3624 closedir(dir);
3625
3626 strbuf_setlen(path, baselen);
3627 if (!r && subdir_cb)
3628 r = subdir_cb(subdir_nr, path->buf, data);
3629
3630 return r;
3631}
3632
3633int for_each_loose_file_in_objdir_buf(struct strbuf *path,
3634 each_loose_object_fn obj_cb,
3635 each_loose_cruft_fn cruft_cb,
3636 each_loose_subdir_fn subdir_cb,
3637 void *data)
3638{
3639 size_t baselen = path->len;
3640 int r = 0;
3641 int i;
3642
3643 for (i = 0; i < 256; i++) {
3644 strbuf_addf(path, "/%02x", i);
3645 r = for_each_file_in_obj_subdir(i, path, obj_cb, cruft_cb,
3646 subdir_cb, data);
3647 strbuf_setlen(path, baselen);
3648 if (r)
3649 break;
3650 }
3651
3652 return r;
3653}
3654
3655int for_each_loose_file_in_objdir(const char *path,
3656 each_loose_object_fn obj_cb,
3657 each_loose_cruft_fn cruft_cb,
3658 each_loose_subdir_fn subdir_cb,
3659 void *data)
3660{
3661 struct strbuf buf = STRBUF_INIT;
3662 int r;
3663
3664 strbuf_addstr(&buf, path);
3665 r = for_each_loose_file_in_objdir_buf(&buf, obj_cb, cruft_cb,
3666 subdir_cb, data);
3667 strbuf_release(&buf);
3668
3669 return r;
3670}
3671
3672struct loose_alt_odb_data {
3673 each_loose_object_fn *cb;
3674 void *data;
3675};
3676
3677static int loose_from_alt_odb(struct alternate_object_database *alt,
3678 void *vdata)
3679{
3680 struct loose_alt_odb_data *data = vdata;
3681 struct strbuf buf = STRBUF_INIT;
3682 int r;
3683
3684 strbuf_addstr(&buf, alt->path);
3685 r = for_each_loose_file_in_objdir_buf(&buf,
3686 data->cb, NULL, NULL,
3687 data->data);
3688 strbuf_release(&buf);
3689 return r;
3690}
3691
3692int for_each_loose_object(each_loose_object_fn cb, void *data, unsigned flags)
3693{
3694 struct loose_alt_odb_data alt;
3695 int r;
3696
3697 r = for_each_loose_file_in_objdir(get_object_directory(),
3698 cb, NULL, NULL, data);
3699 if (r)
3700 return r;
3701
3702 if (flags & FOR_EACH_OBJECT_LOCAL_ONLY)
3703 return 0;
3704
3705 alt.cb = cb;
3706 alt.data = data;
3707 return foreach_alt_odb(loose_from_alt_odb, &alt);
3708}
3709
3710static int for_each_object_in_pack(struct packed_git *p, each_packed_object_fn cb, void *data)
3711{
3712 uint32_t i;
3713 int r = 0;
3714
3715 for (i = 0; i < p->num_objects; i++) {
3716 const unsigned char *sha1 = nth_packed_object_sha1(p, i);
3717
3718 if (!sha1)
3719 return error("unable to get sha1 of object %u in %s",
3720 i, p->pack_name);
3721
3722 r = cb(sha1, p, i, data);
3723 if (r)
3724 break;
3725 }
3726 return r;
3727}
3728
3729int for_each_packed_object(each_packed_object_fn cb, void *data, unsigned flags)
3730{
3731 struct packed_git *p;
3732 int r = 0;
3733 int pack_errors = 0;
3734
3735 prepare_packed_git();
3736 for (p = packed_git; p; p = p->next) {
3737 if ((flags & FOR_EACH_OBJECT_LOCAL_ONLY) && !p->pack_local)
3738 continue;
3739 if (open_pack_index(p)) {
3740 pack_errors = 1;
3741 continue;
3742 }
3743 r = for_each_object_in_pack(p, cb, data);
3744 if (r)
3745 break;
3746 }
3747 return r ? r : pack_errors;
3748}