1/*
2 * GIT - The information manager from hell
3 *
4 * Copyright (C) Linus Torvalds, 2005
5 *
6 * This handles basic git sha1 object files - packing, unpacking,
7 * creation etc.
8 */
9#include "cache.h"
10#include "string-list.h"
11#include "lockfile.h"
12#include "delta.h"
13#include "pack.h"
14#include "blob.h"
15#include "commit.h"
16#include "run-command.h"
17#include "tag.h"
18#include "tree.h"
19#include "tree-walk.h"
20#include "refs.h"
21#include "pack-revindex.h"
22#include "sha1-lookup.h"
23#include "bulk-checkin.h"
24#include "streaming.h"
25#include "dir.h"
26#include "mru.h"
27#include "list.h"
28#include "mergesort.h"
29#include "quote.h"
30
31#ifndef O_NOATIME
32#if defined(__linux__) && (defined(__i386__) || defined(__PPC__))
33#define O_NOATIME 01000000
34#else
35#define O_NOATIME 0
36#endif
37#endif
38
39#define SZ_FMT PRIuMAX
40static inline uintmax_t sz_fmt(size_t s) { return s; }
41
42const unsigned char null_sha1[20];
43const struct object_id null_oid;
44const struct object_id empty_tree_oid = {
45 EMPTY_TREE_SHA1_BIN_LITERAL
46};
47const struct object_id empty_blob_oid = {
48 EMPTY_BLOB_SHA1_BIN_LITERAL
49};
50
51/*
52 * This is meant to hold a *small* number of objects that you would
53 * want read_sha1_file() to be able to return, but yet you do not want
54 * to write them into the object store (e.g. a browse-only
55 * application).
56 */
57static struct cached_object {
58 unsigned char sha1[20];
59 enum object_type type;
60 void *buf;
61 unsigned long size;
62} *cached_objects;
63static int cached_object_nr, cached_object_alloc;
64
65static struct cached_object empty_tree = {
66 EMPTY_TREE_SHA1_BIN_LITERAL,
67 OBJ_TREE,
68 "",
69 0
70};
71
72static struct cached_object *find_cached_object(const unsigned char *sha1)
73{
74 int i;
75 struct cached_object *co = cached_objects;
76
77 for (i = 0; i < cached_object_nr; i++, co++) {
78 if (!hashcmp(co->sha1, sha1))
79 return co;
80 }
81 if (!hashcmp(sha1, empty_tree.sha1))
82 return &empty_tree;
83 return NULL;
84}
85
86int mkdir_in_gitdir(const char *path)
87{
88 if (mkdir(path, 0777)) {
89 int saved_errno = errno;
90 struct stat st;
91 struct strbuf sb = STRBUF_INIT;
92
93 if (errno != EEXIST)
94 return -1;
95 /*
96 * Are we looking at a path in a symlinked worktree
97 * whose original repository does not yet have it?
98 * e.g. .git/rr-cache pointing at its original
99 * repository in which the user hasn't performed any
100 * conflict resolution yet?
101 */
102 if (lstat(path, &st) || !S_ISLNK(st.st_mode) ||
103 strbuf_readlink(&sb, path, st.st_size) ||
104 !is_absolute_path(sb.buf) ||
105 mkdir(sb.buf, 0777)) {
106 strbuf_release(&sb);
107 errno = saved_errno;
108 return -1;
109 }
110 strbuf_release(&sb);
111 }
112 return adjust_shared_perm(path);
113}
114
115enum scld_error safe_create_leading_directories(char *path)
116{
117 char *next_component = path + offset_1st_component(path);
118 enum scld_error ret = SCLD_OK;
119
120 while (ret == SCLD_OK && next_component) {
121 struct stat st;
122 char *slash = next_component, slash_character;
123
124 while (*slash && !is_dir_sep(*slash))
125 slash++;
126
127 if (!*slash)
128 break;
129
130 next_component = slash + 1;
131 while (is_dir_sep(*next_component))
132 next_component++;
133 if (!*next_component)
134 break;
135
136 slash_character = *slash;
137 *slash = '\0';
138 if (!stat(path, &st)) {
139 /* path exists */
140 if (!S_ISDIR(st.st_mode))
141 ret = SCLD_EXISTS;
142 } else if (mkdir(path, 0777)) {
143 if (errno == EEXIST &&
144 !stat(path, &st) && S_ISDIR(st.st_mode))
145 ; /* somebody created it since we checked */
146 else if (errno == ENOENT)
147 /*
148 * Either mkdir() failed because
149 * somebody just pruned the containing
150 * directory, or stat() failed because
151 * the file that was in our way was
152 * just removed. Either way, inform
153 * the caller that it might be worth
154 * trying again:
155 */
156 ret = SCLD_VANISHED;
157 else
158 ret = SCLD_FAILED;
159 } else if (adjust_shared_perm(path)) {
160 ret = SCLD_PERMS;
161 }
162 *slash = slash_character;
163 }
164 return ret;
165}
166
167enum scld_error safe_create_leading_directories_const(const char *path)
168{
169 /* path points to cache entries, so xstrdup before messing with it */
170 char *buf = xstrdup(path);
171 enum scld_error result = safe_create_leading_directories(buf);
172 free(buf);
173 return result;
174}
175
176static void fill_sha1_path(struct strbuf *buf, const unsigned char *sha1)
177{
178 int i;
179 for (i = 0; i < 20; i++) {
180 static char hex[] = "0123456789abcdef";
181 unsigned int val = sha1[i];
182 strbuf_addch(buf, hex[val >> 4]);
183 strbuf_addch(buf, hex[val & 0xf]);
184 if (!i)
185 strbuf_addch(buf, '/');
186 }
187}
188
189const char *sha1_file_name(const unsigned char *sha1)
190{
191 static struct strbuf buf = STRBUF_INIT;
192
193 strbuf_reset(&buf);
194 strbuf_addf(&buf, "%s/", get_object_directory());
195
196 fill_sha1_path(&buf, sha1);
197 return buf.buf;
198}
199
200struct strbuf *alt_scratch_buf(struct alternate_object_database *alt)
201{
202 strbuf_setlen(&alt->scratch, alt->base_len);
203 return &alt->scratch;
204}
205
206static const char *alt_sha1_path(struct alternate_object_database *alt,
207 const unsigned char *sha1)
208{
209 struct strbuf *buf = alt_scratch_buf(alt);
210 fill_sha1_path(buf, sha1);
211 return buf->buf;
212}
213
214/*
215 * Return the name of the pack or index file with the specified sha1
216 * in its filename. *base and *name are scratch space that must be
217 * provided by the caller. which should be "pack" or "idx".
218 */
219static char *sha1_get_pack_name(const unsigned char *sha1,
220 struct strbuf *buf,
221 const char *which)
222{
223 strbuf_reset(buf);
224 strbuf_addf(buf, "%s/pack/pack-%s.%s", get_object_directory(),
225 sha1_to_hex(sha1), which);
226 return buf->buf;
227}
228
229char *sha1_pack_name(const unsigned char *sha1)
230{
231 static struct strbuf buf = STRBUF_INIT;
232 return sha1_get_pack_name(sha1, &buf, "pack");
233}
234
235char *sha1_pack_index_name(const unsigned char *sha1)
236{
237 static struct strbuf buf = STRBUF_INIT;
238 return sha1_get_pack_name(sha1, &buf, "idx");
239}
240
241struct alternate_object_database *alt_odb_list;
242static struct alternate_object_database **alt_odb_tail;
243
244/*
245 * Return non-zero iff the path is usable as an alternate object database.
246 */
247static int alt_odb_usable(struct strbuf *path, const char *normalized_objdir)
248{
249 struct alternate_object_database *alt;
250
251 /* Detect cases where alternate disappeared */
252 if (!is_directory(path->buf)) {
253 error("object directory %s does not exist; "
254 "check .git/objects/info/alternates.",
255 path->buf);
256 return 0;
257 }
258
259 /*
260 * Prevent the common mistake of listing the same
261 * thing twice, or object directory itself.
262 */
263 for (alt = alt_odb_list; alt; alt = alt->next) {
264 if (!fspathcmp(path->buf, alt->path))
265 return 0;
266 }
267 if (!fspathcmp(path->buf, normalized_objdir))
268 return 0;
269
270 return 1;
271}
272
273/*
274 * Prepare alternate object database registry.
275 *
276 * The variable alt_odb_list points at the list of struct
277 * alternate_object_database. The elements on this list come from
278 * non-empty elements from colon separated ALTERNATE_DB_ENVIRONMENT
279 * environment variable, and $GIT_OBJECT_DIRECTORY/info/alternates,
280 * whose contents is similar to that environment variable but can be
281 * LF separated. Its base points at a statically allocated buffer that
282 * contains "/the/directory/corresponding/to/.git/objects/...", while
283 * its name points just after the slash at the end of ".git/objects/"
284 * in the example above, and has enough space to hold 40-byte hex
285 * SHA1, an extra slash for the first level indirection, and the
286 * terminating NUL.
287 */
288static int link_alt_odb_entry(const char *entry, const char *relative_base,
289 int depth, const char *normalized_objdir)
290{
291 struct alternate_object_database *ent;
292 struct strbuf pathbuf = STRBUF_INIT;
293
294 if (!is_absolute_path(entry) && relative_base) {
295 strbuf_addstr(&pathbuf, real_path(relative_base));
296 strbuf_addch(&pathbuf, '/');
297 }
298 strbuf_addstr(&pathbuf, entry);
299
300 if (strbuf_normalize_path(&pathbuf) < 0 && relative_base) {
301 error("unable to normalize alternate object path: %s",
302 pathbuf.buf);
303 strbuf_release(&pathbuf);
304 return -1;
305 }
306
307 /*
308 * The trailing slash after the directory name is given by
309 * this function at the end. Remove duplicates.
310 */
311 while (pathbuf.len && pathbuf.buf[pathbuf.len - 1] == '/')
312 strbuf_setlen(&pathbuf, pathbuf.len - 1);
313
314 if (!alt_odb_usable(&pathbuf, normalized_objdir)) {
315 strbuf_release(&pathbuf);
316 return -1;
317 }
318
319 ent = alloc_alt_odb(pathbuf.buf);
320
321 /* add the alternate entry */
322 *alt_odb_tail = ent;
323 alt_odb_tail = &(ent->next);
324 ent->next = NULL;
325
326 /* recursively add alternates */
327 read_info_alternates(pathbuf.buf, depth + 1);
328
329 strbuf_release(&pathbuf);
330 return 0;
331}
332
333static const char *parse_alt_odb_entry(const char *string,
334 int sep,
335 struct strbuf *out)
336{
337 const char *end;
338
339 strbuf_reset(out);
340
341 if (*string == '#') {
342 /* comment; consume up to next separator */
343 end = strchrnul(string, sep);
344 } else if (*string == '"' && !unquote_c_style(out, string, &end)) {
345 /*
346 * quoted path; unquote_c_style has copied the
347 * data for us and set "end". Broken quoting (e.g.,
348 * an entry that doesn't end with a quote) falls
349 * back to the unquoted case below.
350 */
351 } else {
352 /* normal, unquoted path */
353 end = strchrnul(string, sep);
354 strbuf_add(out, string, end - string);
355 }
356
357 if (*end)
358 end++;
359 return end;
360}
361
362static void link_alt_odb_entries(const char *alt, int sep,
363 const char *relative_base, int depth)
364{
365 struct strbuf objdirbuf = STRBUF_INIT;
366 struct strbuf entry = STRBUF_INIT;
367
368 if (depth > 5) {
369 error("%s: ignoring alternate object stores, nesting too deep.",
370 relative_base);
371 return;
372 }
373
374 strbuf_add_absolute_path(&objdirbuf, get_object_directory());
375 if (strbuf_normalize_path(&objdirbuf) < 0)
376 die("unable to normalize object directory: %s",
377 objdirbuf.buf);
378
379 while (*alt) {
380 alt = parse_alt_odb_entry(alt, sep, &entry);
381 if (!entry.len)
382 continue;
383 link_alt_odb_entry(entry.buf, relative_base, depth, objdirbuf.buf);
384 }
385 strbuf_release(&entry);
386 strbuf_release(&objdirbuf);
387}
388
389void read_info_alternates(const char * relative_base, int depth)
390{
391 char *path;
392 struct strbuf buf = STRBUF_INIT;
393
394 path = xstrfmt("%s/info/alternates", relative_base);
395 if (strbuf_read_file(&buf, path, 1024) < 0) {
396 free(path);
397 return;
398 }
399
400 link_alt_odb_entries(buf.buf, '\n', relative_base, depth);
401 strbuf_release(&buf);
402 free(path);
403}
404
405struct alternate_object_database *alloc_alt_odb(const char *dir)
406{
407 struct alternate_object_database *ent;
408
409 FLEX_ALLOC_STR(ent, path, dir);
410 strbuf_init(&ent->scratch, 0);
411 strbuf_addf(&ent->scratch, "%s/", dir);
412 ent->base_len = ent->scratch.len;
413
414 return ent;
415}
416
417void add_to_alternates_file(const char *reference)
418{
419 struct lock_file *lock = xcalloc(1, sizeof(struct lock_file));
420 char *alts = git_pathdup("objects/info/alternates");
421 FILE *in, *out;
422
423 hold_lock_file_for_update(lock, alts, LOCK_DIE_ON_ERROR);
424 out = fdopen_lock_file(lock, "w");
425 if (!out)
426 die_errno("unable to fdopen alternates lockfile");
427
428 in = fopen(alts, "r");
429 if (in) {
430 struct strbuf line = STRBUF_INIT;
431 int found = 0;
432
433 while (strbuf_getline(&line, in) != EOF) {
434 if (!strcmp(reference, line.buf)) {
435 found = 1;
436 break;
437 }
438 fprintf_or_die(out, "%s\n", line.buf);
439 }
440
441 strbuf_release(&line);
442 fclose(in);
443
444 if (found) {
445 rollback_lock_file(lock);
446 lock = NULL;
447 }
448 }
449 else if (errno != ENOENT)
450 die_errno("unable to read alternates file");
451
452 if (lock) {
453 fprintf_or_die(out, "%s\n", reference);
454 if (commit_lock_file(lock))
455 die_errno("unable to move new alternates file into place");
456 if (alt_odb_tail)
457 link_alt_odb_entries(reference, '\n', NULL, 0);
458 }
459 free(alts);
460}
461
462void add_to_alternates_memory(const char *reference)
463{
464 /*
465 * Make sure alternates are initialized, or else our entry may be
466 * overwritten when they are.
467 */
468 prepare_alt_odb();
469
470 link_alt_odb_entries(reference, '\n', NULL, 0);
471}
472
473/*
474 * Compute the exact path an alternate is at and returns it. In case of
475 * error NULL is returned and the human readable error is added to `err`
476 * `path` may be relative and should point to $GITDIR.
477 * `err` must not be null.
478 */
479char *compute_alternate_path(const char *path, struct strbuf *err)
480{
481 char *ref_git = NULL;
482 const char *repo, *ref_git_s;
483 int seen_error = 0;
484
485 ref_git_s = real_path_if_valid(path);
486 if (!ref_git_s) {
487 seen_error = 1;
488 strbuf_addf(err, _("path '%s' does not exist"), path);
489 goto out;
490 } else
491 /*
492 * Beware: read_gitfile(), real_path() and mkpath()
493 * return static buffer
494 */
495 ref_git = xstrdup(ref_git_s);
496
497 repo = read_gitfile(ref_git);
498 if (!repo)
499 repo = read_gitfile(mkpath("%s/.git", ref_git));
500 if (repo) {
501 free(ref_git);
502 ref_git = xstrdup(repo);
503 }
504
505 if (!repo && is_directory(mkpath("%s/.git/objects", ref_git))) {
506 char *ref_git_git = mkpathdup("%s/.git", ref_git);
507 free(ref_git);
508 ref_git = ref_git_git;
509 } else if (!is_directory(mkpath("%s/objects", ref_git))) {
510 struct strbuf sb = STRBUF_INIT;
511 seen_error = 1;
512 if (get_common_dir(&sb, ref_git)) {
513 strbuf_addf(err,
514 _("reference repository '%s' as a linked "
515 "checkout is not supported yet."),
516 path);
517 goto out;
518 }
519
520 strbuf_addf(err, _("reference repository '%s' is not a "
521 "local repository."), path);
522 goto out;
523 }
524
525 if (!access(mkpath("%s/shallow", ref_git), F_OK)) {
526 strbuf_addf(err, _("reference repository '%s' is shallow"),
527 path);
528 seen_error = 1;
529 goto out;
530 }
531
532 if (!access(mkpath("%s/info/grafts", ref_git), F_OK)) {
533 strbuf_addf(err,
534 _("reference repository '%s' is grafted"),
535 path);
536 seen_error = 1;
537 goto out;
538 }
539
540out:
541 if (seen_error) {
542 free(ref_git);
543 ref_git = NULL;
544 }
545
546 return ref_git;
547}
548
549int foreach_alt_odb(alt_odb_fn fn, void *cb)
550{
551 struct alternate_object_database *ent;
552 int r = 0;
553
554 prepare_alt_odb();
555 for (ent = alt_odb_list; ent; ent = ent->next) {
556 r = fn(ent, cb);
557 if (r)
558 break;
559 }
560 return r;
561}
562
563void prepare_alt_odb(void)
564{
565 const char *alt;
566
567 if (alt_odb_tail)
568 return;
569
570 alt = getenv(ALTERNATE_DB_ENVIRONMENT);
571 if (!alt) alt = "";
572
573 alt_odb_tail = &alt_odb_list;
574 link_alt_odb_entries(alt, PATH_SEP, NULL, 0);
575
576 read_info_alternates(get_object_directory(), 0);
577}
578
579/* Returns 1 if we have successfully freshened the file, 0 otherwise. */
580static int freshen_file(const char *fn)
581{
582 struct utimbuf t;
583 t.actime = t.modtime = time(NULL);
584 return !utime(fn, &t);
585}
586
587/*
588 * All of the check_and_freshen functions return 1 if the file exists and was
589 * freshened (if freshening was requested), 0 otherwise. If they return
590 * 0, you should not assume that it is safe to skip a write of the object (it
591 * either does not exist on disk, or has a stale mtime and may be subject to
592 * pruning).
593 */
594static int check_and_freshen_file(const char *fn, int freshen)
595{
596 if (access(fn, F_OK))
597 return 0;
598 if (freshen && !freshen_file(fn))
599 return 0;
600 return 1;
601}
602
603static int check_and_freshen_local(const unsigned char *sha1, int freshen)
604{
605 return check_and_freshen_file(sha1_file_name(sha1), freshen);
606}
607
608static int check_and_freshen_nonlocal(const unsigned char *sha1, int freshen)
609{
610 struct alternate_object_database *alt;
611 prepare_alt_odb();
612 for (alt = alt_odb_list; alt; alt = alt->next) {
613 const char *path = alt_sha1_path(alt, sha1);
614 if (check_and_freshen_file(path, freshen))
615 return 1;
616 }
617 return 0;
618}
619
620static int check_and_freshen(const unsigned char *sha1, int freshen)
621{
622 return check_and_freshen_local(sha1, freshen) ||
623 check_and_freshen_nonlocal(sha1, freshen);
624}
625
626int has_loose_object_nonlocal(const unsigned char *sha1)
627{
628 return check_and_freshen_nonlocal(sha1, 0);
629}
630
631static int has_loose_object(const unsigned char *sha1)
632{
633 return check_and_freshen(sha1, 0);
634}
635
636static unsigned int pack_used_ctr;
637static unsigned int pack_mmap_calls;
638static unsigned int peak_pack_open_windows;
639static unsigned int pack_open_windows;
640static unsigned int pack_open_fds;
641static unsigned int pack_max_fds;
642static size_t peak_pack_mapped;
643static size_t pack_mapped;
644struct packed_git *packed_git;
645
646static struct mru packed_git_mru_storage;
647struct mru *packed_git_mru = &packed_git_mru_storage;
648
649void pack_report(void)
650{
651 fprintf(stderr,
652 "pack_report: getpagesize() = %10" SZ_FMT "\n"
653 "pack_report: core.packedGitWindowSize = %10" SZ_FMT "\n"
654 "pack_report: core.packedGitLimit = %10" SZ_FMT "\n",
655 sz_fmt(getpagesize()),
656 sz_fmt(packed_git_window_size),
657 sz_fmt(packed_git_limit));
658 fprintf(stderr,
659 "pack_report: pack_used_ctr = %10u\n"
660 "pack_report: pack_mmap_calls = %10u\n"
661 "pack_report: pack_open_windows = %10u / %10u\n"
662 "pack_report: pack_mapped = "
663 "%10" SZ_FMT " / %10" SZ_FMT "\n",
664 pack_used_ctr,
665 pack_mmap_calls,
666 pack_open_windows, peak_pack_open_windows,
667 sz_fmt(pack_mapped), sz_fmt(peak_pack_mapped));
668}
669
670/*
671 * Open and mmap the index file at path, perform a couple of
672 * consistency checks, then record its information to p. Return 0 on
673 * success.
674 */
675static int check_packed_git_idx(const char *path, struct packed_git *p)
676{
677 void *idx_map;
678 struct pack_idx_header *hdr;
679 size_t idx_size;
680 uint32_t version, nr, i, *index;
681 int fd = git_open(path);
682 struct stat st;
683
684 if (fd < 0)
685 return -1;
686 if (fstat(fd, &st)) {
687 close(fd);
688 return -1;
689 }
690 idx_size = xsize_t(st.st_size);
691 if (idx_size < 4 * 256 + 20 + 20) {
692 close(fd);
693 return error("index file %s is too small", path);
694 }
695 idx_map = xmmap(NULL, idx_size, PROT_READ, MAP_PRIVATE, fd, 0);
696 close(fd);
697
698 hdr = idx_map;
699 if (hdr->idx_signature == htonl(PACK_IDX_SIGNATURE)) {
700 version = ntohl(hdr->idx_version);
701 if (version < 2 || version > 2) {
702 munmap(idx_map, idx_size);
703 return error("index file %s is version %"PRIu32
704 " and is not supported by this binary"
705 " (try upgrading GIT to a newer version)",
706 path, version);
707 }
708 } else
709 version = 1;
710
711 nr = 0;
712 index = idx_map;
713 if (version > 1)
714 index += 2; /* skip index header */
715 for (i = 0; i < 256; i++) {
716 uint32_t n = ntohl(index[i]);
717 if (n < nr) {
718 munmap(idx_map, idx_size);
719 return error("non-monotonic index %s", path);
720 }
721 nr = n;
722 }
723
724 if (version == 1) {
725 /*
726 * Total size:
727 * - 256 index entries 4 bytes each
728 * - 24-byte entries * nr (20-byte sha1 + 4-byte offset)
729 * - 20-byte SHA1 of the packfile
730 * - 20-byte SHA1 file checksum
731 */
732 if (idx_size != 4*256 + nr * 24 + 20 + 20) {
733 munmap(idx_map, idx_size);
734 return error("wrong index v1 file size in %s", path);
735 }
736 } else if (version == 2) {
737 /*
738 * Minimum size:
739 * - 8 bytes of header
740 * - 256 index entries 4 bytes each
741 * - 20-byte sha1 entry * nr
742 * - 4-byte crc entry * nr
743 * - 4-byte offset entry * nr
744 * - 20-byte SHA1 of the packfile
745 * - 20-byte SHA1 file checksum
746 * And after the 4-byte offset table might be a
747 * variable sized table containing 8-byte entries
748 * for offsets larger than 2^31.
749 */
750 unsigned long min_size = 8 + 4*256 + nr*(20 + 4 + 4) + 20 + 20;
751 unsigned long max_size = min_size;
752 if (nr)
753 max_size += (nr - 1)*8;
754 if (idx_size < min_size || idx_size > max_size) {
755 munmap(idx_map, idx_size);
756 return error("wrong index v2 file size in %s", path);
757 }
758 if (idx_size != min_size &&
759 /*
760 * make sure we can deal with large pack offsets.
761 * 31-bit signed offset won't be enough, neither
762 * 32-bit unsigned one will be.
763 */
764 (sizeof(off_t) <= 4)) {
765 munmap(idx_map, idx_size);
766 return error("pack too large for current definition of off_t in %s", path);
767 }
768 }
769
770 p->index_version = version;
771 p->index_data = idx_map;
772 p->index_size = idx_size;
773 p->num_objects = nr;
774 return 0;
775}
776
777int open_pack_index(struct packed_git *p)
778{
779 char *idx_name;
780 size_t len;
781 int ret;
782
783 if (p->index_data)
784 return 0;
785
786 if (!strip_suffix(p->pack_name, ".pack", &len))
787 die("BUG: pack_name does not end in .pack");
788 idx_name = xstrfmt("%.*s.idx", (int)len, p->pack_name);
789 ret = check_packed_git_idx(idx_name, p);
790 free(idx_name);
791 return ret;
792}
793
794static void scan_windows(struct packed_git *p,
795 struct packed_git **lru_p,
796 struct pack_window **lru_w,
797 struct pack_window **lru_l)
798{
799 struct pack_window *w, *w_l;
800
801 for (w_l = NULL, w = p->windows; w; w = w->next) {
802 if (!w->inuse_cnt) {
803 if (!*lru_w || w->last_used < (*lru_w)->last_used) {
804 *lru_p = p;
805 *lru_w = w;
806 *lru_l = w_l;
807 }
808 }
809 w_l = w;
810 }
811}
812
813static int unuse_one_window(struct packed_git *current)
814{
815 struct packed_git *p, *lru_p = NULL;
816 struct pack_window *lru_w = NULL, *lru_l = NULL;
817
818 if (current)
819 scan_windows(current, &lru_p, &lru_w, &lru_l);
820 for (p = packed_git; p; p = p->next)
821 scan_windows(p, &lru_p, &lru_w, &lru_l);
822 if (lru_p) {
823 munmap(lru_w->base, lru_w->len);
824 pack_mapped -= lru_w->len;
825 if (lru_l)
826 lru_l->next = lru_w->next;
827 else
828 lru_p->windows = lru_w->next;
829 free(lru_w);
830 pack_open_windows--;
831 return 1;
832 }
833 return 0;
834}
835
836void release_pack_memory(size_t need)
837{
838 size_t cur = pack_mapped;
839 while (need >= (cur - pack_mapped) && unuse_one_window(NULL))
840 ; /* nothing */
841}
842
843static void mmap_limit_check(size_t length)
844{
845 static size_t limit = 0;
846 if (!limit) {
847 limit = git_env_ulong("GIT_MMAP_LIMIT", 0);
848 if (!limit)
849 limit = SIZE_MAX;
850 }
851 if (length > limit)
852 die("attempting to mmap %"PRIuMAX" over limit %"PRIuMAX,
853 (uintmax_t)length, (uintmax_t)limit);
854}
855
856void *xmmap_gently(void *start, size_t length,
857 int prot, int flags, int fd, off_t offset)
858{
859 void *ret;
860
861 mmap_limit_check(length);
862 ret = mmap(start, length, prot, flags, fd, offset);
863 if (ret == MAP_FAILED) {
864 if (!length)
865 return NULL;
866 release_pack_memory(length);
867 ret = mmap(start, length, prot, flags, fd, offset);
868 }
869 return ret;
870}
871
872void *xmmap(void *start, size_t length,
873 int prot, int flags, int fd, off_t offset)
874{
875 void *ret = xmmap_gently(start, length, prot, flags, fd, offset);
876 if (ret == MAP_FAILED)
877 die_errno("mmap failed");
878 return ret;
879}
880
881void close_pack_windows(struct packed_git *p)
882{
883 while (p->windows) {
884 struct pack_window *w = p->windows;
885
886 if (w->inuse_cnt)
887 die("pack '%s' still has open windows to it",
888 p->pack_name);
889 munmap(w->base, w->len);
890 pack_mapped -= w->len;
891 pack_open_windows--;
892 p->windows = w->next;
893 free(w);
894 }
895}
896
897static int close_pack_fd(struct packed_git *p)
898{
899 if (p->pack_fd < 0)
900 return 0;
901
902 close(p->pack_fd);
903 pack_open_fds--;
904 p->pack_fd = -1;
905
906 return 1;
907}
908
909static void close_pack(struct packed_git *p)
910{
911 close_pack_windows(p);
912 close_pack_fd(p);
913 close_pack_index(p);
914}
915
916void close_all_packs(void)
917{
918 struct packed_git *p;
919
920 for (p = packed_git; p; p = p->next)
921 if (p->do_not_close)
922 die("BUG: want to close pack marked 'do-not-close'");
923 else
924 close_pack(p);
925}
926
927
928/*
929 * The LRU pack is the one with the oldest MRU window, preferring packs
930 * with no used windows, or the oldest mtime if it has no windows allocated.
931 */
932static void find_lru_pack(struct packed_git *p, struct packed_git **lru_p, struct pack_window **mru_w, int *accept_windows_inuse)
933{
934 struct pack_window *w, *this_mru_w;
935 int has_windows_inuse = 0;
936
937 /*
938 * Reject this pack if it has windows and the previously selected
939 * one does not. If this pack does not have windows, reject
940 * it if the pack file is newer than the previously selected one.
941 */
942 if (*lru_p && !*mru_w && (p->windows || p->mtime > (*lru_p)->mtime))
943 return;
944
945 for (w = this_mru_w = p->windows; w; w = w->next) {
946 /*
947 * Reject this pack if any of its windows are in use,
948 * but the previously selected pack did not have any
949 * inuse windows. Otherwise, record that this pack
950 * has windows in use.
951 */
952 if (w->inuse_cnt) {
953 if (*accept_windows_inuse)
954 has_windows_inuse = 1;
955 else
956 return;
957 }
958
959 if (w->last_used > this_mru_w->last_used)
960 this_mru_w = w;
961
962 /*
963 * Reject this pack if it has windows that have been
964 * used more recently than the previously selected pack.
965 * If the previously selected pack had windows inuse and
966 * we have not encountered a window in this pack that is
967 * inuse, skip this check since we prefer a pack with no
968 * inuse windows to one that has inuse windows.
969 */
970 if (*mru_w && *accept_windows_inuse == has_windows_inuse &&
971 this_mru_w->last_used > (*mru_w)->last_used)
972 return;
973 }
974
975 /*
976 * Select this pack.
977 */
978 *mru_w = this_mru_w;
979 *lru_p = p;
980 *accept_windows_inuse = has_windows_inuse;
981}
982
983static int close_one_pack(void)
984{
985 struct packed_git *p, *lru_p = NULL;
986 struct pack_window *mru_w = NULL;
987 int accept_windows_inuse = 1;
988
989 for (p = packed_git; p; p = p->next) {
990 if (p->pack_fd == -1)
991 continue;
992 find_lru_pack(p, &lru_p, &mru_w, &accept_windows_inuse);
993 }
994
995 if (lru_p)
996 return close_pack_fd(lru_p);
997
998 return 0;
999}
1000
1001void unuse_pack(struct pack_window **w_cursor)
1002{
1003 struct pack_window *w = *w_cursor;
1004 if (w) {
1005 w->inuse_cnt--;
1006 *w_cursor = NULL;
1007 }
1008}
1009
1010void close_pack_index(struct packed_git *p)
1011{
1012 if (p->index_data) {
1013 munmap((void *)p->index_data, p->index_size);
1014 p->index_data = NULL;
1015 }
1016}
1017
1018static unsigned int get_max_fd_limit(void)
1019{
1020#ifdef RLIMIT_NOFILE
1021 {
1022 struct rlimit lim;
1023
1024 if (!getrlimit(RLIMIT_NOFILE, &lim))
1025 return lim.rlim_cur;
1026 }
1027#endif
1028
1029#ifdef _SC_OPEN_MAX
1030 {
1031 long open_max = sysconf(_SC_OPEN_MAX);
1032 if (0 < open_max)
1033 return open_max;
1034 /*
1035 * Otherwise, we got -1 for one of the two
1036 * reasons:
1037 *
1038 * (1) sysconf() did not understand _SC_OPEN_MAX
1039 * and signaled an error with -1; or
1040 * (2) sysconf() said there is no limit.
1041 *
1042 * We _could_ clear errno before calling sysconf() to
1043 * tell these two cases apart and return a huge number
1044 * in the latter case to let the caller cap it to a
1045 * value that is not so selfish, but letting the
1046 * fallback OPEN_MAX codepath take care of these cases
1047 * is a lot simpler.
1048 */
1049 }
1050#endif
1051
1052#ifdef OPEN_MAX
1053 return OPEN_MAX;
1054#else
1055 return 1; /* see the caller ;-) */
1056#endif
1057}
1058
1059/*
1060 * Do not call this directly as this leaks p->pack_fd on error return;
1061 * call open_packed_git() instead.
1062 */
1063static int open_packed_git_1(struct packed_git *p)
1064{
1065 struct stat st;
1066 struct pack_header hdr;
1067 unsigned char sha1[20];
1068 unsigned char *idx_sha1;
1069 long fd_flag;
1070
1071 if (!p->index_data && open_pack_index(p))
1072 return error("packfile %s index unavailable", p->pack_name);
1073
1074 if (!pack_max_fds) {
1075 unsigned int max_fds = get_max_fd_limit();
1076
1077 /* Save 3 for stdin/stdout/stderr, 22 for work */
1078 if (25 < max_fds)
1079 pack_max_fds = max_fds - 25;
1080 else
1081 pack_max_fds = 1;
1082 }
1083
1084 while (pack_max_fds <= pack_open_fds && close_one_pack())
1085 ; /* nothing */
1086
1087 p->pack_fd = git_open(p->pack_name);
1088 if (p->pack_fd < 0 || fstat(p->pack_fd, &st))
1089 return -1;
1090 pack_open_fds++;
1091
1092 /* If we created the struct before we had the pack we lack size. */
1093 if (!p->pack_size) {
1094 if (!S_ISREG(st.st_mode))
1095 return error("packfile %s not a regular file", p->pack_name);
1096 p->pack_size = st.st_size;
1097 } else if (p->pack_size != st.st_size)
1098 return error("packfile %s size changed", p->pack_name);
1099
1100 /* We leave these file descriptors open with sliding mmap;
1101 * there is no point keeping them open across exec(), though.
1102 */
1103 fd_flag = fcntl(p->pack_fd, F_GETFD, 0);
1104 if (fd_flag < 0)
1105 return error("cannot determine file descriptor flags");
1106 fd_flag |= FD_CLOEXEC;
1107 if (fcntl(p->pack_fd, F_SETFD, fd_flag) == -1)
1108 return error("cannot set FD_CLOEXEC");
1109
1110 /* Verify we recognize this pack file format. */
1111 if (read_in_full(p->pack_fd, &hdr, sizeof(hdr)) != sizeof(hdr))
1112 return error("file %s is far too short to be a packfile", p->pack_name);
1113 if (hdr.hdr_signature != htonl(PACK_SIGNATURE))
1114 return error("file %s is not a GIT packfile", p->pack_name);
1115 if (!pack_version_ok(hdr.hdr_version))
1116 return error("packfile %s is version %"PRIu32" and not"
1117 " supported (try upgrading GIT to a newer version)",
1118 p->pack_name, ntohl(hdr.hdr_version));
1119
1120 /* Verify the pack matches its index. */
1121 if (p->num_objects != ntohl(hdr.hdr_entries))
1122 return error("packfile %s claims to have %"PRIu32" objects"
1123 " while index indicates %"PRIu32" objects",
1124 p->pack_name, ntohl(hdr.hdr_entries),
1125 p->num_objects);
1126 if (lseek(p->pack_fd, p->pack_size - sizeof(sha1), SEEK_SET) == -1)
1127 return error("end of packfile %s is unavailable", p->pack_name);
1128 if (read_in_full(p->pack_fd, sha1, sizeof(sha1)) != sizeof(sha1))
1129 return error("packfile %s signature is unavailable", p->pack_name);
1130 idx_sha1 = ((unsigned char *)p->index_data) + p->index_size - 40;
1131 if (hashcmp(sha1, idx_sha1))
1132 return error("packfile %s does not match index", p->pack_name);
1133 return 0;
1134}
1135
1136static int open_packed_git(struct packed_git *p)
1137{
1138 if (!open_packed_git_1(p))
1139 return 0;
1140 close_pack_fd(p);
1141 return -1;
1142}
1143
1144static int in_window(struct pack_window *win, off_t offset)
1145{
1146 /* We must promise at least 20 bytes (one hash) after the
1147 * offset is available from this window, otherwise the offset
1148 * is not actually in this window and a different window (which
1149 * has that one hash excess) must be used. This is to support
1150 * the object header and delta base parsing routines below.
1151 */
1152 off_t win_off = win->offset;
1153 return win_off <= offset
1154 && (offset + 20) <= (win_off + win->len);
1155}
1156
1157unsigned char *use_pack(struct packed_git *p,
1158 struct pack_window **w_cursor,
1159 off_t offset,
1160 unsigned long *left)
1161{
1162 struct pack_window *win = *w_cursor;
1163
1164 /* Since packfiles end in a hash of their content and it's
1165 * pointless to ask for an offset into the middle of that
1166 * hash, and the in_window function above wouldn't match
1167 * don't allow an offset too close to the end of the file.
1168 */
1169 if (!p->pack_size && p->pack_fd == -1 && open_packed_git(p))
1170 die("packfile %s cannot be accessed", p->pack_name);
1171 if (offset > (p->pack_size - 20))
1172 die("offset beyond end of packfile (truncated pack?)");
1173 if (offset < 0)
1174 die(_("offset before end of packfile (broken .idx?)"));
1175
1176 if (!win || !in_window(win, offset)) {
1177 if (win)
1178 win->inuse_cnt--;
1179 for (win = p->windows; win; win = win->next) {
1180 if (in_window(win, offset))
1181 break;
1182 }
1183 if (!win) {
1184 size_t window_align = packed_git_window_size / 2;
1185 off_t len;
1186
1187 if (p->pack_fd == -1 && open_packed_git(p))
1188 die("packfile %s cannot be accessed", p->pack_name);
1189
1190 win = xcalloc(1, sizeof(*win));
1191 win->offset = (offset / window_align) * window_align;
1192 len = p->pack_size - win->offset;
1193 if (len > packed_git_window_size)
1194 len = packed_git_window_size;
1195 win->len = (size_t)len;
1196 pack_mapped += win->len;
1197 while (packed_git_limit < pack_mapped
1198 && unuse_one_window(p))
1199 ; /* nothing */
1200 win->base = xmmap(NULL, win->len,
1201 PROT_READ, MAP_PRIVATE,
1202 p->pack_fd, win->offset);
1203 if (win->base == MAP_FAILED)
1204 die_errno("packfile %s cannot be mapped",
1205 p->pack_name);
1206 if (!win->offset && win->len == p->pack_size
1207 && !p->do_not_close)
1208 close_pack_fd(p);
1209 pack_mmap_calls++;
1210 pack_open_windows++;
1211 if (pack_mapped > peak_pack_mapped)
1212 peak_pack_mapped = pack_mapped;
1213 if (pack_open_windows > peak_pack_open_windows)
1214 peak_pack_open_windows = pack_open_windows;
1215 win->next = p->windows;
1216 p->windows = win;
1217 }
1218 }
1219 if (win != *w_cursor) {
1220 win->last_used = pack_used_ctr++;
1221 win->inuse_cnt++;
1222 *w_cursor = win;
1223 }
1224 offset -= win->offset;
1225 if (left)
1226 *left = win->len - xsize_t(offset);
1227 return win->base + offset;
1228}
1229
1230static struct packed_git *alloc_packed_git(int extra)
1231{
1232 struct packed_git *p = xmalloc(st_add(sizeof(*p), extra));
1233 memset(p, 0, sizeof(*p));
1234 p->pack_fd = -1;
1235 return p;
1236}
1237
1238static void try_to_free_pack_memory(size_t size)
1239{
1240 release_pack_memory(size);
1241}
1242
1243struct packed_git *add_packed_git(const char *path, size_t path_len, int local)
1244{
1245 static int have_set_try_to_free_routine;
1246 struct stat st;
1247 size_t alloc;
1248 struct packed_git *p;
1249
1250 if (!have_set_try_to_free_routine) {
1251 have_set_try_to_free_routine = 1;
1252 set_try_to_free_routine(try_to_free_pack_memory);
1253 }
1254
1255 /*
1256 * Make sure a corresponding .pack file exists and that
1257 * the index looks sane.
1258 */
1259 if (!strip_suffix_mem(path, &path_len, ".idx"))
1260 return NULL;
1261
1262 /*
1263 * ".pack" is long enough to hold any suffix we're adding (and
1264 * the use xsnprintf double-checks that)
1265 */
1266 alloc = st_add3(path_len, strlen(".pack"), 1);
1267 p = alloc_packed_git(alloc);
1268 memcpy(p->pack_name, path, path_len);
1269
1270 xsnprintf(p->pack_name + path_len, alloc - path_len, ".keep");
1271 if (!access(p->pack_name, F_OK))
1272 p->pack_keep = 1;
1273
1274 xsnprintf(p->pack_name + path_len, alloc - path_len, ".pack");
1275 if (stat(p->pack_name, &st) || !S_ISREG(st.st_mode)) {
1276 free(p);
1277 return NULL;
1278 }
1279
1280 /* ok, it looks sane as far as we can check without
1281 * actually mapping the pack file.
1282 */
1283 p->pack_size = st.st_size;
1284 p->pack_local = local;
1285 p->mtime = st.st_mtime;
1286 if (path_len < 40 || get_sha1_hex(path + path_len - 40, p->sha1))
1287 hashclr(p->sha1);
1288 return p;
1289}
1290
1291struct packed_git *parse_pack_index(unsigned char *sha1, const char *idx_path)
1292{
1293 const char *path = sha1_pack_name(sha1);
1294 size_t alloc = st_add(strlen(path), 1);
1295 struct packed_git *p = alloc_packed_git(alloc);
1296
1297 memcpy(p->pack_name, path, alloc); /* includes NUL */
1298 hashcpy(p->sha1, sha1);
1299 if (check_packed_git_idx(idx_path, p)) {
1300 free(p);
1301 return NULL;
1302 }
1303
1304 return p;
1305}
1306
1307void install_packed_git(struct packed_git *pack)
1308{
1309 if (pack->pack_fd != -1)
1310 pack_open_fds++;
1311
1312 pack->next = packed_git;
1313 packed_git = pack;
1314}
1315
1316void (*report_garbage)(unsigned seen_bits, const char *path);
1317
1318static void report_helper(const struct string_list *list,
1319 int seen_bits, int first, int last)
1320{
1321 if (seen_bits == (PACKDIR_FILE_PACK|PACKDIR_FILE_IDX))
1322 return;
1323
1324 for (; first < last; first++)
1325 report_garbage(seen_bits, list->items[first].string);
1326}
1327
1328static void report_pack_garbage(struct string_list *list)
1329{
1330 int i, baselen = -1, first = 0, seen_bits = 0;
1331
1332 if (!report_garbage)
1333 return;
1334
1335 string_list_sort(list);
1336
1337 for (i = 0; i < list->nr; i++) {
1338 const char *path = list->items[i].string;
1339 if (baselen != -1 &&
1340 strncmp(path, list->items[first].string, baselen)) {
1341 report_helper(list, seen_bits, first, i);
1342 baselen = -1;
1343 seen_bits = 0;
1344 }
1345 if (baselen == -1) {
1346 const char *dot = strrchr(path, '.');
1347 if (!dot) {
1348 report_garbage(PACKDIR_FILE_GARBAGE, path);
1349 continue;
1350 }
1351 baselen = dot - path + 1;
1352 first = i;
1353 }
1354 if (!strcmp(path + baselen, "pack"))
1355 seen_bits |= 1;
1356 else if (!strcmp(path + baselen, "idx"))
1357 seen_bits |= 2;
1358 }
1359 report_helper(list, seen_bits, first, list->nr);
1360}
1361
1362static void prepare_packed_git_one(char *objdir, int local)
1363{
1364 struct strbuf path = STRBUF_INIT;
1365 size_t dirnamelen;
1366 DIR *dir;
1367 struct dirent *de;
1368 struct string_list garbage = STRING_LIST_INIT_DUP;
1369
1370 strbuf_addstr(&path, objdir);
1371 strbuf_addstr(&path, "/pack");
1372 dir = opendir(path.buf);
1373 if (!dir) {
1374 if (errno != ENOENT)
1375 error_errno("unable to open object pack directory: %s",
1376 path.buf);
1377 strbuf_release(&path);
1378 return;
1379 }
1380 strbuf_addch(&path, '/');
1381 dirnamelen = path.len;
1382 while ((de = readdir(dir)) != NULL) {
1383 struct packed_git *p;
1384 size_t base_len;
1385
1386 if (is_dot_or_dotdot(de->d_name))
1387 continue;
1388
1389 strbuf_setlen(&path, dirnamelen);
1390 strbuf_addstr(&path, de->d_name);
1391
1392 base_len = path.len;
1393 if (strip_suffix_mem(path.buf, &base_len, ".idx")) {
1394 /* Don't reopen a pack we already have. */
1395 for (p = packed_git; p; p = p->next) {
1396 size_t len;
1397 if (strip_suffix(p->pack_name, ".pack", &len) &&
1398 len == base_len &&
1399 !memcmp(p->pack_name, path.buf, len))
1400 break;
1401 }
1402 if (p == NULL &&
1403 /*
1404 * See if it really is a valid .idx file with
1405 * corresponding .pack file that we can map.
1406 */
1407 (p = add_packed_git(path.buf, path.len, local)) != NULL)
1408 install_packed_git(p);
1409 }
1410
1411 if (!report_garbage)
1412 continue;
1413
1414 if (ends_with(de->d_name, ".idx") ||
1415 ends_with(de->d_name, ".pack") ||
1416 ends_with(de->d_name, ".bitmap") ||
1417 ends_with(de->d_name, ".keep"))
1418 string_list_append(&garbage, path.buf);
1419 else
1420 report_garbage(PACKDIR_FILE_GARBAGE, path.buf);
1421 }
1422 closedir(dir);
1423 report_pack_garbage(&garbage);
1424 string_list_clear(&garbage, 0);
1425 strbuf_release(&path);
1426}
1427
1428static int approximate_object_count_valid;
1429
1430/*
1431 * Give a fast, rough count of the number of objects in the repository. This
1432 * ignores loose objects completely. If you have a lot of them, then either
1433 * you should repack because your performance will be awful, or they are
1434 * all unreachable objects about to be pruned, in which case they're not really
1435 * interesting as a measure of repo size in the first place.
1436 */
1437unsigned long approximate_object_count(void)
1438{
1439 static unsigned long count;
1440 if (!approximate_object_count_valid) {
1441 struct packed_git *p;
1442
1443 prepare_packed_git();
1444 count = 0;
1445 for (p = packed_git; p; p = p->next) {
1446 if (open_pack_index(p))
1447 continue;
1448 count += p->num_objects;
1449 }
1450 }
1451 return count;
1452}
1453
1454static void *get_next_packed_git(const void *p)
1455{
1456 return ((const struct packed_git *)p)->next;
1457}
1458
1459static void set_next_packed_git(void *p, void *next)
1460{
1461 ((struct packed_git *)p)->next = next;
1462}
1463
1464static int sort_pack(const void *a_, const void *b_)
1465{
1466 const struct packed_git *a = a_;
1467 const struct packed_git *b = b_;
1468 int st;
1469
1470 /*
1471 * Local packs tend to contain objects specific to our
1472 * variant of the project than remote ones. In addition,
1473 * remote ones could be on a network mounted filesystem.
1474 * Favor local ones for these reasons.
1475 */
1476 st = a->pack_local - b->pack_local;
1477 if (st)
1478 return -st;
1479
1480 /*
1481 * Younger packs tend to contain more recent objects,
1482 * and more recent objects tend to get accessed more
1483 * often.
1484 */
1485 if (a->mtime < b->mtime)
1486 return 1;
1487 else if (a->mtime == b->mtime)
1488 return 0;
1489 return -1;
1490}
1491
1492static void rearrange_packed_git(void)
1493{
1494 packed_git = llist_mergesort(packed_git, get_next_packed_git,
1495 set_next_packed_git, sort_pack);
1496}
1497
1498static void prepare_packed_git_mru(void)
1499{
1500 struct packed_git *p;
1501
1502 mru_clear(packed_git_mru);
1503 for (p = packed_git; p; p = p->next)
1504 mru_append(packed_git_mru, p);
1505}
1506
1507static int prepare_packed_git_run_once = 0;
1508void prepare_packed_git(void)
1509{
1510 struct alternate_object_database *alt;
1511
1512 if (prepare_packed_git_run_once)
1513 return;
1514 prepare_packed_git_one(get_object_directory(), 1);
1515 prepare_alt_odb();
1516 for (alt = alt_odb_list; alt; alt = alt->next)
1517 prepare_packed_git_one(alt->path, 0);
1518 rearrange_packed_git();
1519 prepare_packed_git_mru();
1520 prepare_packed_git_run_once = 1;
1521}
1522
1523void reprepare_packed_git(void)
1524{
1525 approximate_object_count_valid = 0;
1526 prepare_packed_git_run_once = 0;
1527 prepare_packed_git();
1528}
1529
1530static void mark_bad_packed_object(struct packed_git *p,
1531 const unsigned char *sha1)
1532{
1533 unsigned i;
1534 for (i = 0; i < p->num_bad_objects; i++)
1535 if (!hashcmp(sha1, p->bad_object_sha1 + GIT_SHA1_RAWSZ * i))
1536 return;
1537 p->bad_object_sha1 = xrealloc(p->bad_object_sha1,
1538 st_mult(GIT_SHA1_RAWSZ,
1539 st_add(p->num_bad_objects, 1)));
1540 hashcpy(p->bad_object_sha1 + GIT_SHA1_RAWSZ * p->num_bad_objects, sha1);
1541 p->num_bad_objects++;
1542}
1543
1544static const struct packed_git *has_packed_and_bad(const unsigned char *sha1)
1545{
1546 struct packed_git *p;
1547 unsigned i;
1548
1549 for (p = packed_git; p; p = p->next)
1550 for (i = 0; i < p->num_bad_objects; i++)
1551 if (!hashcmp(sha1, p->bad_object_sha1 + 20 * i))
1552 return p;
1553 return NULL;
1554}
1555
1556/*
1557 * With an in-core object data in "map", rehash it to make sure the
1558 * object name actually matches "sha1" to detect object corruption.
1559 * With "map" == NULL, try reading the object named with "sha1" using
1560 * the streaming interface and rehash it to do the same.
1561 */
1562int check_sha1_signature(const unsigned char *sha1, void *map,
1563 unsigned long size, const char *type)
1564{
1565 unsigned char real_sha1[20];
1566 enum object_type obj_type;
1567 struct git_istream *st;
1568 git_SHA_CTX c;
1569 char hdr[32];
1570 int hdrlen;
1571
1572 if (map) {
1573 hash_sha1_file(map, size, type, real_sha1);
1574 return hashcmp(sha1, real_sha1) ? -1 : 0;
1575 }
1576
1577 st = open_istream(sha1, &obj_type, &size, NULL);
1578 if (!st)
1579 return -1;
1580
1581 /* Generate the header */
1582 hdrlen = xsnprintf(hdr, sizeof(hdr), "%s %lu", typename(obj_type), size) + 1;
1583
1584 /* Sha1.. */
1585 git_SHA1_Init(&c);
1586 git_SHA1_Update(&c, hdr, hdrlen);
1587 for (;;) {
1588 char buf[1024 * 16];
1589 ssize_t readlen = read_istream(st, buf, sizeof(buf));
1590
1591 if (readlen < 0) {
1592 close_istream(st);
1593 return -1;
1594 }
1595 if (!readlen)
1596 break;
1597 git_SHA1_Update(&c, buf, readlen);
1598 }
1599 git_SHA1_Final(real_sha1, &c);
1600 close_istream(st);
1601 return hashcmp(sha1, real_sha1) ? -1 : 0;
1602}
1603
1604int git_open(const char *name)
1605{
1606 static int sha1_file_open_flag = O_NOATIME | O_CLOEXEC;
1607
1608 for (;;) {
1609 int fd;
1610
1611 errno = 0;
1612 fd = open(name, O_RDONLY | sha1_file_open_flag);
1613 if (fd >= 0)
1614 return fd;
1615
1616 /* Try again w/o O_CLOEXEC: the kernel might not support it */
1617 if ((sha1_file_open_flag & O_CLOEXEC) && errno == EINVAL) {
1618 sha1_file_open_flag &= ~O_CLOEXEC;
1619 continue;
1620 }
1621
1622 /* Might the failure be due to O_NOATIME? */
1623 if (errno != ENOENT && (sha1_file_open_flag & O_NOATIME)) {
1624 sha1_file_open_flag &= ~O_NOATIME;
1625 continue;
1626 }
1627 return -1;
1628 }
1629}
1630
1631static int stat_sha1_file(const unsigned char *sha1, struct stat *st)
1632{
1633 struct alternate_object_database *alt;
1634
1635 if (!lstat(sha1_file_name(sha1), st))
1636 return 0;
1637
1638 prepare_alt_odb();
1639 errno = ENOENT;
1640 for (alt = alt_odb_list; alt; alt = alt->next) {
1641 const char *path = alt_sha1_path(alt, sha1);
1642 if (!lstat(path, st))
1643 return 0;
1644 }
1645
1646 return -1;
1647}
1648
1649static int open_sha1_file(const unsigned char *sha1)
1650{
1651 int fd;
1652 struct alternate_object_database *alt;
1653 int most_interesting_errno;
1654
1655 fd = git_open(sha1_file_name(sha1));
1656 if (fd >= 0)
1657 return fd;
1658 most_interesting_errno = errno;
1659
1660 prepare_alt_odb();
1661 for (alt = alt_odb_list; alt; alt = alt->next) {
1662 const char *path = alt_sha1_path(alt, sha1);
1663 fd = git_open(path);
1664 if (fd >= 0)
1665 return fd;
1666 if (most_interesting_errno == ENOENT)
1667 most_interesting_errno = errno;
1668 }
1669 errno = most_interesting_errno;
1670 return -1;
1671}
1672
1673void *map_sha1_file(const unsigned char *sha1, unsigned long *size)
1674{
1675 void *map;
1676 int fd;
1677
1678 fd = open_sha1_file(sha1);
1679 map = NULL;
1680 if (fd >= 0) {
1681 struct stat st;
1682
1683 if (!fstat(fd, &st)) {
1684 *size = xsize_t(st.st_size);
1685 if (!*size) {
1686 /* mmap() is forbidden on empty files */
1687 error("object file %s is empty", sha1_file_name(sha1));
1688 return NULL;
1689 }
1690 map = xmmap(NULL, *size, PROT_READ, MAP_PRIVATE, fd, 0);
1691 }
1692 close(fd);
1693 }
1694 return map;
1695}
1696
1697unsigned long unpack_object_header_buffer(const unsigned char *buf,
1698 unsigned long len, enum object_type *type, unsigned long *sizep)
1699{
1700 unsigned shift;
1701 unsigned long size, c;
1702 unsigned long used = 0;
1703
1704 c = buf[used++];
1705 *type = (c >> 4) & 7;
1706 size = c & 15;
1707 shift = 4;
1708 while (c & 0x80) {
1709 if (len <= used || bitsizeof(long) <= shift) {
1710 error("bad object header");
1711 size = used = 0;
1712 break;
1713 }
1714 c = buf[used++];
1715 size += (c & 0x7f) << shift;
1716 shift += 7;
1717 }
1718 *sizep = size;
1719 return used;
1720}
1721
1722static int unpack_sha1_short_header(git_zstream *stream,
1723 unsigned char *map, unsigned long mapsize,
1724 void *buffer, unsigned long bufsiz)
1725{
1726 /* Get the data stream */
1727 memset(stream, 0, sizeof(*stream));
1728 stream->next_in = map;
1729 stream->avail_in = mapsize;
1730 stream->next_out = buffer;
1731 stream->avail_out = bufsiz;
1732
1733 git_inflate_init(stream);
1734 return git_inflate(stream, 0);
1735}
1736
1737int unpack_sha1_header(git_zstream *stream,
1738 unsigned char *map, unsigned long mapsize,
1739 void *buffer, unsigned long bufsiz)
1740{
1741 int status = unpack_sha1_short_header(stream, map, mapsize,
1742 buffer, bufsiz);
1743
1744 if (status < Z_OK)
1745 return status;
1746
1747 /* Make sure we have the terminating NUL */
1748 if (!memchr(buffer, '\0', stream->next_out - (unsigned char *)buffer))
1749 return -1;
1750 return 0;
1751}
1752
1753static int unpack_sha1_header_to_strbuf(git_zstream *stream, unsigned char *map,
1754 unsigned long mapsize, void *buffer,
1755 unsigned long bufsiz, struct strbuf *header)
1756{
1757 int status;
1758
1759 status = unpack_sha1_short_header(stream, map, mapsize, buffer, bufsiz);
1760 if (status < Z_OK)
1761 return -1;
1762
1763 /*
1764 * Check if entire header is unpacked in the first iteration.
1765 */
1766 if (memchr(buffer, '\0', stream->next_out - (unsigned char *)buffer))
1767 return 0;
1768
1769 /*
1770 * buffer[0..bufsiz] was not large enough. Copy the partial
1771 * result out to header, and then append the result of further
1772 * reading the stream.
1773 */
1774 strbuf_add(header, buffer, stream->next_out - (unsigned char *)buffer);
1775 stream->next_out = buffer;
1776 stream->avail_out = bufsiz;
1777
1778 do {
1779 status = git_inflate(stream, 0);
1780 strbuf_add(header, buffer, stream->next_out - (unsigned char *)buffer);
1781 if (memchr(buffer, '\0', stream->next_out - (unsigned char *)buffer))
1782 return 0;
1783 stream->next_out = buffer;
1784 stream->avail_out = bufsiz;
1785 } while (status != Z_STREAM_END);
1786 return -1;
1787}
1788
1789static void *unpack_sha1_rest(git_zstream *stream, void *buffer, unsigned long size, const unsigned char *sha1)
1790{
1791 int bytes = strlen(buffer) + 1;
1792 unsigned char *buf = xmallocz(size);
1793 unsigned long n;
1794 int status = Z_OK;
1795
1796 n = stream->total_out - bytes;
1797 if (n > size)
1798 n = size;
1799 memcpy(buf, (char *) buffer + bytes, n);
1800 bytes = n;
1801 if (bytes <= size) {
1802 /*
1803 * The above condition must be (bytes <= size), not
1804 * (bytes < size). In other words, even though we
1805 * expect no more output and set avail_out to zero,
1806 * the input zlib stream may have bytes that express
1807 * "this concludes the stream", and we *do* want to
1808 * eat that input.
1809 *
1810 * Otherwise we would not be able to test that we
1811 * consumed all the input to reach the expected size;
1812 * we also want to check that zlib tells us that all
1813 * went well with status == Z_STREAM_END at the end.
1814 */
1815 stream->next_out = buf + bytes;
1816 stream->avail_out = size - bytes;
1817 while (status == Z_OK)
1818 status = git_inflate(stream, Z_FINISH);
1819 }
1820 if (status == Z_STREAM_END && !stream->avail_in) {
1821 git_inflate_end(stream);
1822 return buf;
1823 }
1824
1825 if (status < 0)
1826 error("corrupt loose object '%s'", sha1_to_hex(sha1));
1827 else if (stream->avail_in)
1828 error("garbage at end of loose object '%s'",
1829 sha1_to_hex(sha1));
1830 free(buf);
1831 return NULL;
1832}
1833
1834/*
1835 * We used to just use "sscanf()", but that's actually way
1836 * too permissive for what we want to check. So do an anal
1837 * object header parse by hand.
1838 */
1839static int parse_sha1_header_extended(const char *hdr, struct object_info *oi,
1840 unsigned int flags)
1841{
1842 const char *type_buf = hdr;
1843 unsigned long size;
1844 int type, type_len = 0;
1845
1846 /*
1847 * The type can be of any size but is followed by
1848 * a space.
1849 */
1850 for (;;) {
1851 char c = *hdr++;
1852 if (!c)
1853 return -1;
1854 if (c == ' ')
1855 break;
1856 type_len++;
1857 }
1858
1859 type = type_from_string_gently(type_buf, type_len, 1);
1860 if (oi->typename)
1861 strbuf_add(oi->typename, type_buf, type_len);
1862 /*
1863 * Set type to 0 if its an unknown object and
1864 * we're obtaining the type using '--allow-unknown-type'
1865 * option.
1866 */
1867 if ((flags & LOOKUP_UNKNOWN_OBJECT) && (type < 0))
1868 type = 0;
1869 else if (type < 0)
1870 die("invalid object type");
1871 if (oi->typep)
1872 *oi->typep = type;
1873
1874 /*
1875 * The length must follow immediately, and be in canonical
1876 * decimal format (ie "010" is not valid).
1877 */
1878 size = *hdr++ - '0';
1879 if (size > 9)
1880 return -1;
1881 if (size) {
1882 for (;;) {
1883 unsigned long c = *hdr - '0';
1884 if (c > 9)
1885 break;
1886 hdr++;
1887 size = size * 10 + c;
1888 }
1889 }
1890
1891 if (oi->sizep)
1892 *oi->sizep = size;
1893
1894 /*
1895 * The length must be followed by a zero byte
1896 */
1897 return *hdr ? -1 : type;
1898}
1899
1900int parse_sha1_header(const char *hdr, unsigned long *sizep)
1901{
1902 struct object_info oi = OBJECT_INFO_INIT;
1903
1904 oi.sizep = sizep;
1905 return parse_sha1_header_extended(hdr, &oi, LOOKUP_REPLACE_OBJECT);
1906}
1907
1908static void *unpack_sha1_file(void *map, unsigned long mapsize, enum object_type *type, unsigned long *size, const unsigned char *sha1)
1909{
1910 int ret;
1911 git_zstream stream;
1912 char hdr[8192];
1913
1914 ret = unpack_sha1_header(&stream, map, mapsize, hdr, sizeof(hdr));
1915 if (ret < Z_OK || (*type = parse_sha1_header(hdr, size)) < 0)
1916 return NULL;
1917
1918 return unpack_sha1_rest(&stream, hdr, *size, sha1);
1919}
1920
1921unsigned long get_size_from_delta(struct packed_git *p,
1922 struct pack_window **w_curs,
1923 off_t curpos)
1924{
1925 const unsigned char *data;
1926 unsigned char delta_head[20], *in;
1927 git_zstream stream;
1928 int st;
1929
1930 memset(&stream, 0, sizeof(stream));
1931 stream.next_out = delta_head;
1932 stream.avail_out = sizeof(delta_head);
1933
1934 git_inflate_init(&stream);
1935 do {
1936 in = use_pack(p, w_curs, curpos, &stream.avail_in);
1937 stream.next_in = in;
1938 st = git_inflate(&stream, Z_FINISH);
1939 curpos += stream.next_in - in;
1940 } while ((st == Z_OK || st == Z_BUF_ERROR) &&
1941 stream.total_out < sizeof(delta_head));
1942 git_inflate_end(&stream);
1943 if ((st != Z_STREAM_END) && stream.total_out != sizeof(delta_head)) {
1944 error("delta data unpack-initial failed");
1945 return 0;
1946 }
1947
1948 /* Examine the initial part of the delta to figure out
1949 * the result size.
1950 */
1951 data = delta_head;
1952
1953 /* ignore base size */
1954 get_delta_hdr_size(&data, delta_head+sizeof(delta_head));
1955
1956 /* Read the result size */
1957 return get_delta_hdr_size(&data, delta_head+sizeof(delta_head));
1958}
1959
1960static off_t get_delta_base(struct packed_git *p,
1961 struct pack_window **w_curs,
1962 off_t *curpos,
1963 enum object_type type,
1964 off_t delta_obj_offset)
1965{
1966 unsigned char *base_info = use_pack(p, w_curs, *curpos, NULL);
1967 off_t base_offset;
1968
1969 /* use_pack() assured us we have [base_info, base_info + 20)
1970 * as a range that we can look at without walking off the
1971 * end of the mapped window. Its actually the hash size
1972 * that is assured. An OFS_DELTA longer than the hash size
1973 * is stupid, as then a REF_DELTA would be smaller to store.
1974 */
1975 if (type == OBJ_OFS_DELTA) {
1976 unsigned used = 0;
1977 unsigned char c = base_info[used++];
1978 base_offset = c & 127;
1979 while (c & 128) {
1980 base_offset += 1;
1981 if (!base_offset || MSB(base_offset, 7))
1982 return 0; /* overflow */
1983 c = base_info[used++];
1984 base_offset = (base_offset << 7) + (c & 127);
1985 }
1986 base_offset = delta_obj_offset - base_offset;
1987 if (base_offset <= 0 || base_offset >= delta_obj_offset)
1988 return 0; /* out of bound */
1989 *curpos += used;
1990 } else if (type == OBJ_REF_DELTA) {
1991 /* The base entry _must_ be in the same pack */
1992 base_offset = find_pack_entry_one(base_info, p);
1993 *curpos += 20;
1994 } else
1995 die("I am totally screwed");
1996 return base_offset;
1997}
1998
1999/*
2000 * Like get_delta_base above, but we return the sha1 instead of the pack
2001 * offset. This means it is cheaper for REF deltas (we do not have to do
2002 * the final object lookup), but more expensive for OFS deltas (we
2003 * have to load the revidx to convert the offset back into a sha1).
2004 */
2005static const unsigned char *get_delta_base_sha1(struct packed_git *p,
2006 struct pack_window **w_curs,
2007 off_t curpos,
2008 enum object_type type,
2009 off_t delta_obj_offset)
2010{
2011 if (type == OBJ_REF_DELTA) {
2012 unsigned char *base = use_pack(p, w_curs, curpos, NULL);
2013 return base;
2014 } else if (type == OBJ_OFS_DELTA) {
2015 struct revindex_entry *revidx;
2016 off_t base_offset = get_delta_base(p, w_curs, &curpos,
2017 type, delta_obj_offset);
2018
2019 if (!base_offset)
2020 return NULL;
2021
2022 revidx = find_pack_revindex(p, base_offset);
2023 if (!revidx)
2024 return NULL;
2025
2026 return nth_packed_object_sha1(p, revidx->nr);
2027 } else
2028 return NULL;
2029}
2030
2031int unpack_object_header(struct packed_git *p,
2032 struct pack_window **w_curs,
2033 off_t *curpos,
2034 unsigned long *sizep)
2035{
2036 unsigned char *base;
2037 unsigned long left;
2038 unsigned long used;
2039 enum object_type type;
2040
2041 /* use_pack() assures us we have [base, base + 20) available
2042 * as a range that we can look at. (Its actually the hash
2043 * size that is assured.) With our object header encoding
2044 * the maximum deflated object size is 2^137, which is just
2045 * insane, so we know won't exceed what we have been given.
2046 */
2047 base = use_pack(p, w_curs, *curpos, &left);
2048 used = unpack_object_header_buffer(base, left, &type, sizep);
2049 if (!used) {
2050 type = OBJ_BAD;
2051 } else
2052 *curpos += used;
2053
2054 return type;
2055}
2056
2057static int retry_bad_packed_offset(struct packed_git *p, off_t obj_offset)
2058{
2059 int type;
2060 struct revindex_entry *revidx;
2061 const unsigned char *sha1;
2062 revidx = find_pack_revindex(p, obj_offset);
2063 if (!revidx)
2064 return OBJ_BAD;
2065 sha1 = nth_packed_object_sha1(p, revidx->nr);
2066 mark_bad_packed_object(p, sha1);
2067 type = sha1_object_info(sha1, NULL);
2068 if (type <= OBJ_NONE)
2069 return OBJ_BAD;
2070 return type;
2071}
2072
2073#define POI_STACK_PREALLOC 64
2074
2075static enum object_type packed_to_object_type(struct packed_git *p,
2076 off_t obj_offset,
2077 enum object_type type,
2078 struct pack_window **w_curs,
2079 off_t curpos)
2080{
2081 off_t small_poi_stack[POI_STACK_PREALLOC];
2082 off_t *poi_stack = small_poi_stack;
2083 int poi_stack_nr = 0, poi_stack_alloc = POI_STACK_PREALLOC;
2084
2085 while (type == OBJ_OFS_DELTA || type == OBJ_REF_DELTA) {
2086 off_t base_offset;
2087 unsigned long size;
2088 /* Push the object we're going to leave behind */
2089 if (poi_stack_nr >= poi_stack_alloc && poi_stack == small_poi_stack) {
2090 poi_stack_alloc = alloc_nr(poi_stack_nr);
2091 ALLOC_ARRAY(poi_stack, poi_stack_alloc);
2092 memcpy(poi_stack, small_poi_stack, sizeof(off_t)*poi_stack_nr);
2093 } else {
2094 ALLOC_GROW(poi_stack, poi_stack_nr+1, poi_stack_alloc);
2095 }
2096 poi_stack[poi_stack_nr++] = obj_offset;
2097 /* If parsing the base offset fails, just unwind */
2098 base_offset = get_delta_base(p, w_curs, &curpos, type, obj_offset);
2099 if (!base_offset)
2100 goto unwind;
2101 curpos = obj_offset = base_offset;
2102 type = unpack_object_header(p, w_curs, &curpos, &size);
2103 if (type <= OBJ_NONE) {
2104 /* If getting the base itself fails, we first
2105 * retry the base, otherwise unwind */
2106 type = retry_bad_packed_offset(p, base_offset);
2107 if (type > OBJ_NONE)
2108 goto out;
2109 goto unwind;
2110 }
2111 }
2112
2113 switch (type) {
2114 case OBJ_BAD:
2115 case OBJ_COMMIT:
2116 case OBJ_TREE:
2117 case OBJ_BLOB:
2118 case OBJ_TAG:
2119 break;
2120 default:
2121 error("unknown object type %i at offset %"PRIuMAX" in %s",
2122 type, (uintmax_t)obj_offset, p->pack_name);
2123 type = OBJ_BAD;
2124 }
2125
2126out:
2127 if (poi_stack != small_poi_stack)
2128 free(poi_stack);
2129 return type;
2130
2131unwind:
2132 while (poi_stack_nr) {
2133 obj_offset = poi_stack[--poi_stack_nr];
2134 type = retry_bad_packed_offset(p, obj_offset);
2135 if (type > OBJ_NONE)
2136 goto out;
2137 }
2138 type = OBJ_BAD;
2139 goto out;
2140}
2141
2142int packed_object_info(struct packed_git *p, off_t obj_offset,
2143 struct object_info *oi)
2144{
2145 struct pack_window *w_curs = NULL;
2146 unsigned long size;
2147 off_t curpos = obj_offset;
2148 enum object_type type;
2149
2150 /*
2151 * We always get the representation type, but only convert it to
2152 * a "real" type later if the caller is interested.
2153 */
2154 type = unpack_object_header(p, &w_curs, &curpos, &size);
2155
2156 if (oi->sizep) {
2157 if (type == OBJ_OFS_DELTA || type == OBJ_REF_DELTA) {
2158 off_t tmp_pos = curpos;
2159 off_t base_offset = get_delta_base(p, &w_curs, &tmp_pos,
2160 type, obj_offset);
2161 if (!base_offset) {
2162 type = OBJ_BAD;
2163 goto out;
2164 }
2165 *oi->sizep = get_size_from_delta(p, &w_curs, tmp_pos);
2166 if (*oi->sizep == 0) {
2167 type = OBJ_BAD;
2168 goto out;
2169 }
2170 } else {
2171 *oi->sizep = size;
2172 }
2173 }
2174
2175 if (oi->disk_sizep) {
2176 struct revindex_entry *revidx = find_pack_revindex(p, obj_offset);
2177 *oi->disk_sizep = revidx[1].offset - obj_offset;
2178 }
2179
2180 if (oi->typep) {
2181 *oi->typep = packed_to_object_type(p, obj_offset, type, &w_curs, curpos);
2182 if (*oi->typep < 0) {
2183 type = OBJ_BAD;
2184 goto out;
2185 }
2186 }
2187
2188 if (oi->delta_base_sha1) {
2189 if (type == OBJ_OFS_DELTA || type == OBJ_REF_DELTA) {
2190 const unsigned char *base;
2191
2192 base = get_delta_base_sha1(p, &w_curs, curpos,
2193 type, obj_offset);
2194 if (!base) {
2195 type = OBJ_BAD;
2196 goto out;
2197 }
2198
2199 hashcpy(oi->delta_base_sha1, base);
2200 } else
2201 hashclr(oi->delta_base_sha1);
2202 }
2203
2204out:
2205 unuse_pack(&w_curs);
2206 return type;
2207}
2208
2209static void *unpack_compressed_entry(struct packed_git *p,
2210 struct pack_window **w_curs,
2211 off_t curpos,
2212 unsigned long size)
2213{
2214 int st;
2215 git_zstream stream;
2216 unsigned char *buffer, *in;
2217
2218 buffer = xmallocz_gently(size);
2219 if (!buffer)
2220 return NULL;
2221 memset(&stream, 0, sizeof(stream));
2222 stream.next_out = buffer;
2223 stream.avail_out = size + 1;
2224
2225 git_inflate_init(&stream);
2226 do {
2227 in = use_pack(p, w_curs, curpos, &stream.avail_in);
2228 stream.next_in = in;
2229 st = git_inflate(&stream, Z_FINISH);
2230 if (!stream.avail_out)
2231 break; /* the payload is larger than it should be */
2232 curpos += stream.next_in - in;
2233 } while (st == Z_OK || st == Z_BUF_ERROR);
2234 git_inflate_end(&stream);
2235 if ((st != Z_STREAM_END) || stream.total_out != size) {
2236 free(buffer);
2237 return NULL;
2238 }
2239
2240 return buffer;
2241}
2242
2243static struct hashmap delta_base_cache;
2244static size_t delta_base_cached;
2245
2246static LIST_HEAD(delta_base_cache_lru);
2247
2248struct delta_base_cache_key {
2249 struct packed_git *p;
2250 off_t base_offset;
2251};
2252
2253struct delta_base_cache_entry {
2254 struct hashmap hash;
2255 struct delta_base_cache_key key;
2256 struct list_head lru;
2257 void *data;
2258 unsigned long size;
2259 enum object_type type;
2260};
2261
2262static unsigned int pack_entry_hash(struct packed_git *p, off_t base_offset)
2263{
2264 unsigned int hash;
2265
2266 hash = (unsigned int)(intptr_t)p + (unsigned int)base_offset;
2267 hash += (hash >> 8) + (hash >> 16);
2268 return hash;
2269}
2270
2271static struct delta_base_cache_entry *
2272get_delta_base_cache_entry(struct packed_git *p, off_t base_offset)
2273{
2274 struct hashmap_entry entry;
2275 struct delta_base_cache_key key;
2276
2277 if (!delta_base_cache.cmpfn)
2278 return NULL;
2279
2280 hashmap_entry_init(&entry, pack_entry_hash(p, base_offset));
2281 key.p = p;
2282 key.base_offset = base_offset;
2283 return hashmap_get(&delta_base_cache, &entry, &key);
2284}
2285
2286static int delta_base_cache_key_eq(const struct delta_base_cache_key *a,
2287 const struct delta_base_cache_key *b)
2288{
2289 return a->p == b->p && a->base_offset == b->base_offset;
2290}
2291
2292static int delta_base_cache_hash_cmp(const void *va, const void *vb,
2293 const void *vkey)
2294{
2295 const struct delta_base_cache_entry *a = va, *b = vb;
2296 const struct delta_base_cache_key *key = vkey;
2297 if (key)
2298 return !delta_base_cache_key_eq(&a->key, key);
2299 else
2300 return !delta_base_cache_key_eq(&a->key, &b->key);
2301}
2302
2303static int in_delta_base_cache(struct packed_git *p, off_t base_offset)
2304{
2305 return !!get_delta_base_cache_entry(p, base_offset);
2306}
2307
2308/*
2309 * Remove the entry from the cache, but do _not_ free the associated
2310 * entry data. The caller takes ownership of the "data" buffer, and
2311 * should copy out any fields it wants before detaching.
2312 */
2313static void detach_delta_base_cache_entry(struct delta_base_cache_entry *ent)
2314{
2315 hashmap_remove(&delta_base_cache, ent, &ent->key);
2316 list_del(&ent->lru);
2317 delta_base_cached -= ent->size;
2318 free(ent);
2319}
2320
2321static void *cache_or_unpack_entry(struct packed_git *p, off_t base_offset,
2322 unsigned long *base_size, enum object_type *type)
2323{
2324 struct delta_base_cache_entry *ent;
2325
2326 ent = get_delta_base_cache_entry(p, base_offset);
2327 if (!ent)
2328 return unpack_entry(p, base_offset, type, base_size);
2329
2330 *type = ent->type;
2331 *base_size = ent->size;
2332 return xmemdupz(ent->data, ent->size);
2333}
2334
2335static inline void release_delta_base_cache(struct delta_base_cache_entry *ent)
2336{
2337 free(ent->data);
2338 detach_delta_base_cache_entry(ent);
2339}
2340
2341void clear_delta_base_cache(void)
2342{
2343 struct hashmap_iter iter;
2344 struct delta_base_cache_entry *entry;
2345 for (entry = hashmap_iter_first(&delta_base_cache, &iter);
2346 entry;
2347 entry = hashmap_iter_next(&iter)) {
2348 release_delta_base_cache(entry);
2349 }
2350}
2351
2352static void add_delta_base_cache(struct packed_git *p, off_t base_offset,
2353 void *base, unsigned long base_size, enum object_type type)
2354{
2355 struct delta_base_cache_entry *ent = xmalloc(sizeof(*ent));
2356 struct list_head *lru, *tmp;
2357
2358 delta_base_cached += base_size;
2359
2360 list_for_each_safe(lru, tmp, &delta_base_cache_lru) {
2361 struct delta_base_cache_entry *f =
2362 list_entry(lru, struct delta_base_cache_entry, lru);
2363 if (delta_base_cached <= delta_base_cache_limit)
2364 break;
2365 release_delta_base_cache(f);
2366 }
2367
2368 ent->key.p = p;
2369 ent->key.base_offset = base_offset;
2370 ent->type = type;
2371 ent->data = base;
2372 ent->size = base_size;
2373 list_add_tail(&ent->lru, &delta_base_cache_lru);
2374
2375 if (!delta_base_cache.cmpfn)
2376 hashmap_init(&delta_base_cache, delta_base_cache_hash_cmp, 0);
2377 hashmap_entry_init(ent, pack_entry_hash(p, base_offset));
2378 hashmap_add(&delta_base_cache, ent);
2379}
2380
2381static void *read_object(const unsigned char *sha1, enum object_type *type,
2382 unsigned long *size);
2383
2384static void write_pack_access_log(struct packed_git *p, off_t obj_offset)
2385{
2386 static struct trace_key pack_access = TRACE_KEY_INIT(PACK_ACCESS);
2387 trace_printf_key(&pack_access, "%s %"PRIuMAX"\n",
2388 p->pack_name, (uintmax_t)obj_offset);
2389}
2390
2391int do_check_packed_object_crc;
2392
2393#define UNPACK_ENTRY_STACK_PREALLOC 64
2394struct unpack_entry_stack_ent {
2395 off_t obj_offset;
2396 off_t curpos;
2397 unsigned long size;
2398};
2399
2400void *unpack_entry(struct packed_git *p, off_t obj_offset,
2401 enum object_type *final_type, unsigned long *final_size)
2402{
2403 struct pack_window *w_curs = NULL;
2404 off_t curpos = obj_offset;
2405 void *data = NULL;
2406 unsigned long size;
2407 enum object_type type;
2408 struct unpack_entry_stack_ent small_delta_stack[UNPACK_ENTRY_STACK_PREALLOC];
2409 struct unpack_entry_stack_ent *delta_stack = small_delta_stack;
2410 int delta_stack_nr = 0, delta_stack_alloc = UNPACK_ENTRY_STACK_PREALLOC;
2411 int base_from_cache = 0;
2412
2413 write_pack_access_log(p, obj_offset);
2414
2415 /* PHASE 1: drill down to the innermost base object */
2416 for (;;) {
2417 off_t base_offset;
2418 int i;
2419 struct delta_base_cache_entry *ent;
2420
2421 ent = get_delta_base_cache_entry(p, curpos);
2422 if (ent) {
2423 type = ent->type;
2424 data = ent->data;
2425 size = ent->size;
2426 detach_delta_base_cache_entry(ent);
2427 base_from_cache = 1;
2428 break;
2429 }
2430
2431 if (do_check_packed_object_crc && p->index_version > 1) {
2432 struct revindex_entry *revidx = find_pack_revindex(p, obj_offset);
2433 off_t len = revidx[1].offset - obj_offset;
2434 if (check_pack_crc(p, &w_curs, obj_offset, len, revidx->nr)) {
2435 const unsigned char *sha1 =
2436 nth_packed_object_sha1(p, revidx->nr);
2437 error("bad packed object CRC for %s",
2438 sha1_to_hex(sha1));
2439 mark_bad_packed_object(p, sha1);
2440 unuse_pack(&w_curs);
2441 return NULL;
2442 }
2443 }
2444
2445 type = unpack_object_header(p, &w_curs, &curpos, &size);
2446 if (type != OBJ_OFS_DELTA && type != OBJ_REF_DELTA)
2447 break;
2448
2449 base_offset = get_delta_base(p, &w_curs, &curpos, type, obj_offset);
2450 if (!base_offset) {
2451 error("failed to validate delta base reference "
2452 "at offset %"PRIuMAX" from %s",
2453 (uintmax_t)curpos, p->pack_name);
2454 /* bail to phase 2, in hopes of recovery */
2455 data = NULL;
2456 break;
2457 }
2458
2459 /* push object, proceed to base */
2460 if (delta_stack_nr >= delta_stack_alloc
2461 && delta_stack == small_delta_stack) {
2462 delta_stack_alloc = alloc_nr(delta_stack_nr);
2463 ALLOC_ARRAY(delta_stack, delta_stack_alloc);
2464 memcpy(delta_stack, small_delta_stack,
2465 sizeof(*delta_stack)*delta_stack_nr);
2466 } else {
2467 ALLOC_GROW(delta_stack, delta_stack_nr+1, delta_stack_alloc);
2468 }
2469 i = delta_stack_nr++;
2470 delta_stack[i].obj_offset = obj_offset;
2471 delta_stack[i].curpos = curpos;
2472 delta_stack[i].size = size;
2473
2474 curpos = obj_offset = base_offset;
2475 }
2476
2477 /* PHASE 2: handle the base */
2478 switch (type) {
2479 case OBJ_OFS_DELTA:
2480 case OBJ_REF_DELTA:
2481 if (data)
2482 die("BUG: unpack_entry: left loop at a valid delta");
2483 break;
2484 case OBJ_COMMIT:
2485 case OBJ_TREE:
2486 case OBJ_BLOB:
2487 case OBJ_TAG:
2488 if (!base_from_cache)
2489 data = unpack_compressed_entry(p, &w_curs, curpos, size);
2490 break;
2491 default:
2492 data = NULL;
2493 error("unknown object type %i at offset %"PRIuMAX" in %s",
2494 type, (uintmax_t)obj_offset, p->pack_name);
2495 }
2496
2497 /* PHASE 3: apply deltas in order */
2498
2499 /* invariants:
2500 * 'data' holds the base data, or NULL if there was corruption
2501 */
2502 while (delta_stack_nr) {
2503 void *delta_data;
2504 void *base = data;
2505 unsigned long delta_size, base_size = size;
2506 int i;
2507
2508 data = NULL;
2509
2510 if (base)
2511 add_delta_base_cache(p, obj_offset, base, base_size, type);
2512
2513 if (!base) {
2514 /*
2515 * We're probably in deep shit, but let's try to fetch
2516 * the required base anyway from another pack or loose.
2517 * This is costly but should happen only in the presence
2518 * of a corrupted pack, and is better than failing outright.
2519 */
2520 struct revindex_entry *revidx;
2521 const unsigned char *base_sha1;
2522 revidx = find_pack_revindex(p, obj_offset);
2523 if (revidx) {
2524 base_sha1 = nth_packed_object_sha1(p, revidx->nr);
2525 error("failed to read delta base object %s"
2526 " at offset %"PRIuMAX" from %s",
2527 sha1_to_hex(base_sha1), (uintmax_t)obj_offset,
2528 p->pack_name);
2529 mark_bad_packed_object(p, base_sha1);
2530 base = read_object(base_sha1, &type, &base_size);
2531 }
2532 }
2533
2534 i = --delta_stack_nr;
2535 obj_offset = delta_stack[i].obj_offset;
2536 curpos = delta_stack[i].curpos;
2537 delta_size = delta_stack[i].size;
2538
2539 if (!base)
2540 continue;
2541
2542 delta_data = unpack_compressed_entry(p, &w_curs, curpos, delta_size);
2543
2544 if (!delta_data) {
2545 error("failed to unpack compressed delta "
2546 "at offset %"PRIuMAX" from %s",
2547 (uintmax_t)curpos, p->pack_name);
2548 data = NULL;
2549 continue;
2550 }
2551
2552 data = patch_delta(base, base_size,
2553 delta_data, delta_size,
2554 &size);
2555
2556 /*
2557 * We could not apply the delta; warn the user, but keep going.
2558 * Our failure will be noticed either in the next iteration of
2559 * the loop, or if this is the final delta, in the caller when
2560 * we return NULL. Those code paths will take care of making
2561 * a more explicit warning and retrying with another copy of
2562 * the object.
2563 */
2564 if (!data)
2565 error("failed to apply delta");
2566
2567 free(delta_data);
2568 }
2569
2570 *final_type = type;
2571 *final_size = size;
2572
2573 unuse_pack(&w_curs);
2574
2575 if (delta_stack != small_delta_stack)
2576 free(delta_stack);
2577
2578 return data;
2579}
2580
2581const unsigned char *nth_packed_object_sha1(struct packed_git *p,
2582 uint32_t n)
2583{
2584 const unsigned char *index = p->index_data;
2585 if (!index) {
2586 if (open_pack_index(p))
2587 return NULL;
2588 index = p->index_data;
2589 }
2590 if (n >= p->num_objects)
2591 return NULL;
2592 index += 4 * 256;
2593 if (p->index_version == 1) {
2594 return index + 24 * n + 4;
2595 } else {
2596 index += 8;
2597 return index + 20 * n;
2598 }
2599}
2600
2601void check_pack_index_ptr(const struct packed_git *p, const void *vptr)
2602{
2603 const unsigned char *ptr = vptr;
2604 const unsigned char *start = p->index_data;
2605 const unsigned char *end = start + p->index_size;
2606 if (ptr < start)
2607 die(_("offset before start of pack index for %s (corrupt index?)"),
2608 p->pack_name);
2609 /* No need to check for underflow; .idx files must be at least 8 bytes */
2610 if (ptr >= end - 8)
2611 die(_("offset beyond end of pack index for %s (truncated index?)"),
2612 p->pack_name);
2613}
2614
2615off_t nth_packed_object_offset(const struct packed_git *p, uint32_t n)
2616{
2617 const unsigned char *index = p->index_data;
2618 index += 4 * 256;
2619 if (p->index_version == 1) {
2620 return ntohl(*((uint32_t *)(index + 24 * n)));
2621 } else {
2622 uint32_t off;
2623 index += 8 + p->num_objects * (20 + 4);
2624 off = ntohl(*((uint32_t *)(index + 4 * n)));
2625 if (!(off & 0x80000000))
2626 return off;
2627 index += p->num_objects * 4 + (off & 0x7fffffff) * 8;
2628 check_pack_index_ptr(p, index);
2629 return (((uint64_t)ntohl(*((uint32_t *)(index + 0)))) << 32) |
2630 ntohl(*((uint32_t *)(index + 4)));
2631 }
2632}
2633
2634off_t find_pack_entry_one(const unsigned char *sha1,
2635 struct packed_git *p)
2636{
2637 const uint32_t *level1_ofs = p->index_data;
2638 const unsigned char *index = p->index_data;
2639 unsigned hi, lo, stride;
2640 static int use_lookup = -1;
2641 static int debug_lookup = -1;
2642
2643 if (debug_lookup < 0)
2644 debug_lookup = !!getenv("GIT_DEBUG_LOOKUP");
2645
2646 if (!index) {
2647 if (open_pack_index(p))
2648 return 0;
2649 level1_ofs = p->index_data;
2650 index = p->index_data;
2651 }
2652 if (p->index_version > 1) {
2653 level1_ofs += 2;
2654 index += 8;
2655 }
2656 index += 4 * 256;
2657 hi = ntohl(level1_ofs[*sha1]);
2658 lo = ((*sha1 == 0x0) ? 0 : ntohl(level1_ofs[*sha1 - 1]));
2659 if (p->index_version > 1) {
2660 stride = 20;
2661 } else {
2662 stride = 24;
2663 index += 4;
2664 }
2665
2666 if (debug_lookup)
2667 printf("%02x%02x%02x... lo %u hi %u nr %"PRIu32"\n",
2668 sha1[0], sha1[1], sha1[2], lo, hi, p->num_objects);
2669
2670 if (use_lookup < 0)
2671 use_lookup = !!getenv("GIT_USE_LOOKUP");
2672 if (use_lookup) {
2673 int pos = sha1_entry_pos(index, stride, 0,
2674 lo, hi, p->num_objects, sha1);
2675 if (pos < 0)
2676 return 0;
2677 return nth_packed_object_offset(p, pos);
2678 }
2679
2680 do {
2681 unsigned mi = (lo + hi) / 2;
2682 int cmp = hashcmp(index + mi * stride, sha1);
2683
2684 if (debug_lookup)
2685 printf("lo %u hi %u rg %u mi %u\n",
2686 lo, hi, hi - lo, mi);
2687 if (!cmp)
2688 return nth_packed_object_offset(p, mi);
2689 if (cmp > 0)
2690 hi = mi;
2691 else
2692 lo = mi+1;
2693 } while (lo < hi);
2694 return 0;
2695}
2696
2697int is_pack_valid(struct packed_git *p)
2698{
2699 /* An already open pack is known to be valid. */
2700 if (p->pack_fd != -1)
2701 return 1;
2702
2703 /* If the pack has one window completely covering the
2704 * file size, the pack is known to be valid even if
2705 * the descriptor is not currently open.
2706 */
2707 if (p->windows) {
2708 struct pack_window *w = p->windows;
2709
2710 if (!w->offset && w->len == p->pack_size)
2711 return 1;
2712 }
2713
2714 /* Force the pack to open to prove its valid. */
2715 return !open_packed_git(p);
2716}
2717
2718static int fill_pack_entry(const unsigned char *sha1,
2719 struct pack_entry *e,
2720 struct packed_git *p)
2721{
2722 off_t offset;
2723
2724 if (p->num_bad_objects) {
2725 unsigned i;
2726 for (i = 0; i < p->num_bad_objects; i++)
2727 if (!hashcmp(sha1, p->bad_object_sha1 + 20 * i))
2728 return 0;
2729 }
2730
2731 offset = find_pack_entry_one(sha1, p);
2732 if (!offset)
2733 return 0;
2734
2735 /*
2736 * We are about to tell the caller where they can locate the
2737 * requested object. We better make sure the packfile is
2738 * still here and can be accessed before supplying that
2739 * answer, as it may have been deleted since the index was
2740 * loaded!
2741 */
2742 if (!is_pack_valid(p))
2743 return 0;
2744 e->offset = offset;
2745 e->p = p;
2746 hashcpy(e->sha1, sha1);
2747 return 1;
2748}
2749
2750/*
2751 * Iff a pack file contains the object named by sha1, return true and
2752 * store its location to e.
2753 */
2754static int find_pack_entry(const unsigned char *sha1, struct pack_entry *e)
2755{
2756 struct mru_entry *p;
2757
2758 prepare_packed_git();
2759 if (!packed_git)
2760 return 0;
2761
2762 for (p = packed_git_mru->head; p; p = p->next) {
2763 if (fill_pack_entry(sha1, e, p->item)) {
2764 mru_mark(packed_git_mru, p);
2765 return 1;
2766 }
2767 }
2768 return 0;
2769}
2770
2771struct packed_git *find_sha1_pack(const unsigned char *sha1,
2772 struct packed_git *packs)
2773{
2774 struct packed_git *p;
2775
2776 for (p = packs; p; p = p->next) {
2777 if (find_pack_entry_one(sha1, p))
2778 return p;
2779 }
2780 return NULL;
2781
2782}
2783
2784static int sha1_loose_object_info(const unsigned char *sha1,
2785 struct object_info *oi,
2786 int flags)
2787{
2788 int status = 0;
2789 unsigned long mapsize;
2790 void *map;
2791 git_zstream stream;
2792 char hdr[32];
2793 struct strbuf hdrbuf = STRBUF_INIT;
2794
2795 if (oi->delta_base_sha1)
2796 hashclr(oi->delta_base_sha1);
2797
2798 /*
2799 * If we don't care about type or size, then we don't
2800 * need to look inside the object at all. Note that we
2801 * do not optimize out the stat call, even if the
2802 * caller doesn't care about the disk-size, since our
2803 * return value implicitly indicates whether the
2804 * object even exists.
2805 */
2806 if (!oi->typep && !oi->typename && !oi->sizep) {
2807 struct stat st;
2808 if (stat_sha1_file(sha1, &st) < 0)
2809 return -1;
2810 if (oi->disk_sizep)
2811 *oi->disk_sizep = st.st_size;
2812 return 0;
2813 }
2814
2815 map = map_sha1_file(sha1, &mapsize);
2816 if (!map)
2817 return -1;
2818 if (oi->disk_sizep)
2819 *oi->disk_sizep = mapsize;
2820 if ((flags & LOOKUP_UNKNOWN_OBJECT)) {
2821 if (unpack_sha1_header_to_strbuf(&stream, map, mapsize, hdr, sizeof(hdr), &hdrbuf) < 0)
2822 status = error("unable to unpack %s header with --allow-unknown-type",
2823 sha1_to_hex(sha1));
2824 } else if (unpack_sha1_header(&stream, map, mapsize, hdr, sizeof(hdr)) < 0)
2825 status = error("unable to unpack %s header",
2826 sha1_to_hex(sha1));
2827 if (status < 0)
2828 ; /* Do nothing */
2829 else if (hdrbuf.len) {
2830 if ((status = parse_sha1_header_extended(hdrbuf.buf, oi, flags)) < 0)
2831 status = error("unable to parse %s header with --allow-unknown-type",
2832 sha1_to_hex(sha1));
2833 } else if ((status = parse_sha1_header_extended(hdr, oi, flags)) < 0)
2834 status = error("unable to parse %s header", sha1_to_hex(sha1));
2835 git_inflate_end(&stream);
2836 munmap(map, mapsize);
2837 if (status && oi->typep)
2838 *oi->typep = status;
2839 strbuf_release(&hdrbuf);
2840 return 0;
2841}
2842
2843int sha1_object_info_extended(const unsigned char *sha1, struct object_info *oi, unsigned flags)
2844{
2845 struct cached_object *co;
2846 struct pack_entry e;
2847 int rtype;
2848 enum object_type real_type;
2849 const unsigned char *real = lookup_replace_object_extended(sha1, flags);
2850
2851 co = find_cached_object(real);
2852 if (co) {
2853 if (oi->typep)
2854 *(oi->typep) = co->type;
2855 if (oi->sizep)
2856 *(oi->sizep) = co->size;
2857 if (oi->disk_sizep)
2858 *(oi->disk_sizep) = 0;
2859 if (oi->delta_base_sha1)
2860 hashclr(oi->delta_base_sha1);
2861 if (oi->typename)
2862 strbuf_addstr(oi->typename, typename(co->type));
2863 oi->whence = OI_CACHED;
2864 return 0;
2865 }
2866
2867 if (!find_pack_entry(real, &e)) {
2868 /* Most likely it's a loose object. */
2869 if (!sha1_loose_object_info(real, oi, flags)) {
2870 oi->whence = OI_LOOSE;
2871 return 0;
2872 }
2873
2874 /* Not a loose object; someone else may have just packed it. */
2875 reprepare_packed_git();
2876 if (!find_pack_entry(real, &e))
2877 return -1;
2878 }
2879
2880 /*
2881 * packed_object_info() does not follow the delta chain to
2882 * find out the real type, unless it is given oi->typep.
2883 */
2884 if (oi->typename && !oi->typep)
2885 oi->typep = &real_type;
2886
2887 rtype = packed_object_info(e.p, e.offset, oi);
2888 if (rtype < 0) {
2889 mark_bad_packed_object(e.p, real);
2890 if (oi->typep == &real_type)
2891 oi->typep = NULL;
2892 return sha1_object_info_extended(real, oi, 0);
2893 } else if (in_delta_base_cache(e.p, e.offset)) {
2894 oi->whence = OI_DBCACHED;
2895 } else {
2896 oi->whence = OI_PACKED;
2897 oi->u.packed.offset = e.offset;
2898 oi->u.packed.pack = e.p;
2899 oi->u.packed.is_delta = (rtype == OBJ_REF_DELTA ||
2900 rtype == OBJ_OFS_DELTA);
2901 }
2902 if (oi->typename)
2903 strbuf_addstr(oi->typename, typename(*oi->typep));
2904 if (oi->typep == &real_type)
2905 oi->typep = NULL;
2906
2907 return 0;
2908}
2909
2910/* returns enum object_type or negative */
2911int sha1_object_info(const unsigned char *sha1, unsigned long *sizep)
2912{
2913 enum object_type type;
2914 struct object_info oi = OBJECT_INFO_INIT;
2915
2916 oi.typep = &type;
2917 oi.sizep = sizep;
2918 if (sha1_object_info_extended(sha1, &oi, LOOKUP_REPLACE_OBJECT) < 0)
2919 return -1;
2920 return type;
2921}
2922
2923static void *read_packed_sha1(const unsigned char *sha1,
2924 enum object_type *type, unsigned long *size)
2925{
2926 struct pack_entry e;
2927 void *data;
2928
2929 if (!find_pack_entry(sha1, &e))
2930 return NULL;
2931 data = cache_or_unpack_entry(e.p, e.offset, size, type);
2932 if (!data) {
2933 /*
2934 * We're probably in deep shit, but let's try to fetch
2935 * the required object anyway from another pack or loose.
2936 * This should happen only in the presence of a corrupted
2937 * pack, and is better than failing outright.
2938 */
2939 error("failed to read object %s at offset %"PRIuMAX" from %s",
2940 sha1_to_hex(sha1), (uintmax_t)e.offset, e.p->pack_name);
2941 mark_bad_packed_object(e.p, sha1);
2942 data = read_object(sha1, type, size);
2943 }
2944 return data;
2945}
2946
2947int pretend_sha1_file(void *buf, unsigned long len, enum object_type type,
2948 unsigned char *sha1)
2949{
2950 struct cached_object *co;
2951
2952 hash_sha1_file(buf, len, typename(type), sha1);
2953 if (has_sha1_file(sha1) || find_cached_object(sha1))
2954 return 0;
2955 ALLOC_GROW(cached_objects, cached_object_nr + 1, cached_object_alloc);
2956 co = &cached_objects[cached_object_nr++];
2957 co->size = len;
2958 co->type = type;
2959 co->buf = xmalloc(len);
2960 memcpy(co->buf, buf, len);
2961 hashcpy(co->sha1, sha1);
2962 return 0;
2963}
2964
2965static void *read_object(const unsigned char *sha1, enum object_type *type,
2966 unsigned long *size)
2967{
2968 unsigned long mapsize;
2969 void *map, *buf;
2970 struct cached_object *co;
2971
2972 co = find_cached_object(sha1);
2973 if (co) {
2974 *type = co->type;
2975 *size = co->size;
2976 return xmemdupz(co->buf, co->size);
2977 }
2978
2979 buf = read_packed_sha1(sha1, type, size);
2980 if (buf)
2981 return buf;
2982 map = map_sha1_file(sha1, &mapsize);
2983 if (map) {
2984 buf = unpack_sha1_file(map, mapsize, type, size, sha1);
2985 munmap(map, mapsize);
2986 return buf;
2987 }
2988 reprepare_packed_git();
2989 return read_packed_sha1(sha1, type, size);
2990}
2991
2992/*
2993 * This function dies on corrupt objects; the callers who want to
2994 * deal with them should arrange to call read_object() and give error
2995 * messages themselves.
2996 */
2997void *read_sha1_file_extended(const unsigned char *sha1,
2998 enum object_type *type,
2999 unsigned long *size,
3000 unsigned flag)
3001{
3002 void *data;
3003 const struct packed_git *p;
3004 const unsigned char *repl = lookup_replace_object_extended(sha1, flag);
3005
3006 errno = 0;
3007 data = read_object(repl, type, size);
3008 if (data)
3009 return data;
3010
3011 if (errno && errno != ENOENT)
3012 die_errno("failed to read object %s", sha1_to_hex(sha1));
3013
3014 /* die if we replaced an object with one that does not exist */
3015 if (repl != sha1)
3016 die("replacement %s not found for %s",
3017 sha1_to_hex(repl), sha1_to_hex(sha1));
3018
3019 if (has_loose_object(repl)) {
3020 const char *path = sha1_file_name(sha1);
3021
3022 die("loose object %s (stored in %s) is corrupt",
3023 sha1_to_hex(repl), path);
3024 }
3025
3026 if ((p = has_packed_and_bad(repl)) != NULL)
3027 die("packed object %s (stored in %s) is corrupt",
3028 sha1_to_hex(repl), p->pack_name);
3029
3030 return NULL;
3031}
3032
3033void *read_object_with_reference(const unsigned char *sha1,
3034 const char *required_type_name,
3035 unsigned long *size,
3036 unsigned char *actual_sha1_return)
3037{
3038 enum object_type type, required_type;
3039 void *buffer;
3040 unsigned long isize;
3041 unsigned char actual_sha1[20];
3042
3043 required_type = type_from_string(required_type_name);
3044 hashcpy(actual_sha1, sha1);
3045 while (1) {
3046 int ref_length = -1;
3047 const char *ref_type = NULL;
3048
3049 buffer = read_sha1_file(actual_sha1, &type, &isize);
3050 if (!buffer)
3051 return NULL;
3052 if (type == required_type) {
3053 *size = isize;
3054 if (actual_sha1_return)
3055 hashcpy(actual_sha1_return, actual_sha1);
3056 return buffer;
3057 }
3058 /* Handle references */
3059 else if (type == OBJ_COMMIT)
3060 ref_type = "tree ";
3061 else if (type == OBJ_TAG)
3062 ref_type = "object ";
3063 else {
3064 free(buffer);
3065 return NULL;
3066 }
3067 ref_length = strlen(ref_type);
3068
3069 if (ref_length + 40 > isize ||
3070 memcmp(buffer, ref_type, ref_length) ||
3071 get_sha1_hex((char *) buffer + ref_length, actual_sha1)) {
3072 free(buffer);
3073 return NULL;
3074 }
3075 free(buffer);
3076 /* Now we have the ID of the referred-to object in
3077 * actual_sha1. Check again. */
3078 }
3079}
3080
3081static void write_sha1_file_prepare(const void *buf, unsigned long len,
3082 const char *type, unsigned char *sha1,
3083 char *hdr, int *hdrlen)
3084{
3085 git_SHA_CTX c;
3086
3087 /* Generate the header */
3088 *hdrlen = xsnprintf(hdr, *hdrlen, "%s %lu", type, len)+1;
3089
3090 /* Sha1.. */
3091 git_SHA1_Init(&c);
3092 git_SHA1_Update(&c, hdr, *hdrlen);
3093 git_SHA1_Update(&c, buf, len);
3094 git_SHA1_Final(sha1, &c);
3095}
3096
3097/*
3098 * Move the just written object into its final resting place.
3099 */
3100int finalize_object_file(const char *tmpfile, const char *filename)
3101{
3102 int ret = 0;
3103
3104 if (object_creation_mode == OBJECT_CREATION_USES_RENAMES)
3105 goto try_rename;
3106 else if (link(tmpfile, filename))
3107 ret = errno;
3108
3109 /*
3110 * Coda hack - coda doesn't like cross-directory links,
3111 * so we fall back to a rename, which will mean that it
3112 * won't be able to check collisions, but that's not a
3113 * big deal.
3114 *
3115 * The same holds for FAT formatted media.
3116 *
3117 * When this succeeds, we just return. We have nothing
3118 * left to unlink.
3119 */
3120 if (ret && ret != EEXIST) {
3121 try_rename:
3122 if (!rename(tmpfile, filename))
3123 goto out;
3124 ret = errno;
3125 }
3126 unlink_or_warn(tmpfile);
3127 if (ret) {
3128 if (ret != EEXIST) {
3129 return error_errno("unable to write sha1 filename %s", filename);
3130 }
3131 /* FIXME!!! Collision check here ? */
3132 }
3133
3134out:
3135 if (adjust_shared_perm(filename))
3136 return error("unable to set permission to '%s'", filename);
3137 return 0;
3138}
3139
3140static int write_buffer(int fd, const void *buf, size_t len)
3141{
3142 if (write_in_full(fd, buf, len) < 0)
3143 return error_errno("file write error");
3144 return 0;
3145}
3146
3147int hash_sha1_file(const void *buf, unsigned long len, const char *type,
3148 unsigned char *sha1)
3149{
3150 char hdr[32];
3151 int hdrlen = sizeof(hdr);
3152 write_sha1_file_prepare(buf, len, type, sha1, hdr, &hdrlen);
3153 return 0;
3154}
3155
3156/* Finalize a file on disk, and close it. */
3157static void close_sha1_file(int fd)
3158{
3159 if (fsync_object_files)
3160 fsync_or_die(fd, "sha1 file");
3161 if (close(fd) != 0)
3162 die_errno("error when closing sha1 file");
3163}
3164
3165/* Size of directory component, including the ending '/' */
3166static inline int directory_size(const char *filename)
3167{
3168 const char *s = strrchr(filename, '/');
3169 if (!s)
3170 return 0;
3171 return s - filename + 1;
3172}
3173
3174/*
3175 * This creates a temporary file in the same directory as the final
3176 * 'filename'
3177 *
3178 * We want to avoid cross-directory filename renames, because those
3179 * can have problems on various filesystems (FAT, NFS, Coda).
3180 */
3181static int create_tmpfile(struct strbuf *tmp, const char *filename)
3182{
3183 int fd, dirlen = directory_size(filename);
3184
3185 strbuf_reset(tmp);
3186 strbuf_add(tmp, filename, dirlen);
3187 strbuf_addstr(tmp, "tmp_obj_XXXXXX");
3188 fd = git_mkstemp_mode(tmp->buf, 0444);
3189 if (fd < 0 && dirlen && errno == ENOENT) {
3190 /*
3191 * Make sure the directory exists; note that the contents
3192 * of the buffer are undefined after mkstemp returns an
3193 * error, so we have to rewrite the whole buffer from
3194 * scratch.
3195 */
3196 strbuf_reset(tmp);
3197 strbuf_add(tmp, filename, dirlen - 1);
3198 if (mkdir(tmp->buf, 0777) && errno != EEXIST)
3199 return -1;
3200 if (adjust_shared_perm(tmp->buf))
3201 return -1;
3202
3203 /* Try again */
3204 strbuf_addstr(tmp, "/tmp_obj_XXXXXX");
3205 fd = git_mkstemp_mode(tmp->buf, 0444);
3206 }
3207 return fd;
3208}
3209
3210static int write_loose_object(const unsigned char *sha1, char *hdr, int hdrlen,
3211 const void *buf, unsigned long len, time_t mtime)
3212{
3213 int fd, ret;
3214 unsigned char compressed[4096];
3215 git_zstream stream;
3216 git_SHA_CTX c;
3217 unsigned char parano_sha1[20];
3218 static struct strbuf tmp_file = STRBUF_INIT;
3219 const char *filename = sha1_file_name(sha1);
3220
3221 fd = create_tmpfile(&tmp_file, filename);
3222 if (fd < 0) {
3223 if (errno == EACCES)
3224 return error("insufficient permission for adding an object to repository database %s", get_object_directory());
3225 else
3226 return error_errno("unable to create temporary file");
3227 }
3228
3229 /* Set it up */
3230 git_deflate_init(&stream, zlib_compression_level);
3231 stream.next_out = compressed;
3232 stream.avail_out = sizeof(compressed);
3233 git_SHA1_Init(&c);
3234
3235 /* First header.. */
3236 stream.next_in = (unsigned char *)hdr;
3237 stream.avail_in = hdrlen;
3238 while (git_deflate(&stream, 0) == Z_OK)
3239 ; /* nothing */
3240 git_SHA1_Update(&c, hdr, hdrlen);
3241
3242 /* Then the data itself.. */
3243 stream.next_in = (void *)buf;
3244 stream.avail_in = len;
3245 do {
3246 unsigned char *in0 = stream.next_in;
3247 ret = git_deflate(&stream, Z_FINISH);
3248 git_SHA1_Update(&c, in0, stream.next_in - in0);
3249 if (write_buffer(fd, compressed, stream.next_out - compressed) < 0)
3250 die("unable to write sha1 file");
3251 stream.next_out = compressed;
3252 stream.avail_out = sizeof(compressed);
3253 } while (ret == Z_OK);
3254
3255 if (ret != Z_STREAM_END)
3256 die("unable to deflate new object %s (%d)", sha1_to_hex(sha1), ret);
3257 ret = git_deflate_end_gently(&stream);
3258 if (ret != Z_OK)
3259 die("deflateEnd on object %s failed (%d)", sha1_to_hex(sha1), ret);
3260 git_SHA1_Final(parano_sha1, &c);
3261 if (hashcmp(sha1, parano_sha1) != 0)
3262 die("confused by unstable object source data for %s", sha1_to_hex(sha1));
3263
3264 close_sha1_file(fd);
3265
3266 if (mtime) {
3267 struct utimbuf utb;
3268 utb.actime = mtime;
3269 utb.modtime = mtime;
3270 if (utime(tmp_file.buf, &utb) < 0)
3271 warning_errno("failed utime() on %s", tmp_file.buf);
3272 }
3273
3274 return finalize_object_file(tmp_file.buf, filename);
3275}
3276
3277static int freshen_loose_object(const unsigned char *sha1)
3278{
3279 return check_and_freshen(sha1, 1);
3280}
3281
3282static int freshen_packed_object(const unsigned char *sha1)
3283{
3284 struct pack_entry e;
3285 if (!find_pack_entry(sha1, &e))
3286 return 0;
3287 if (e.p->freshened)
3288 return 1;
3289 if (!freshen_file(e.p->pack_name))
3290 return 0;
3291 e.p->freshened = 1;
3292 return 1;
3293}
3294
3295int write_sha1_file(const void *buf, unsigned long len, const char *type, unsigned char *sha1)
3296{
3297 char hdr[32];
3298 int hdrlen = sizeof(hdr);
3299
3300 /* Normally if we have it in the pack then we do not bother writing
3301 * it out into .git/objects/??/?{38} file.
3302 */
3303 write_sha1_file_prepare(buf, len, type, sha1, hdr, &hdrlen);
3304 if (freshen_packed_object(sha1) || freshen_loose_object(sha1))
3305 return 0;
3306 return write_loose_object(sha1, hdr, hdrlen, buf, len, 0);
3307}
3308
3309int hash_sha1_file_literally(const void *buf, unsigned long len, const char *type,
3310 unsigned char *sha1, unsigned flags)
3311{
3312 char *header;
3313 int hdrlen, status = 0;
3314
3315 /* type string, SP, %lu of the length plus NUL must fit this */
3316 hdrlen = strlen(type) + 32;
3317 header = xmalloc(hdrlen);
3318 write_sha1_file_prepare(buf, len, type, sha1, header, &hdrlen);
3319
3320 if (!(flags & HASH_WRITE_OBJECT))
3321 goto cleanup;
3322 if (freshen_packed_object(sha1) || freshen_loose_object(sha1))
3323 goto cleanup;
3324 status = write_loose_object(sha1, header, hdrlen, buf, len, 0);
3325
3326cleanup:
3327 free(header);
3328 return status;
3329}
3330
3331int force_object_loose(const unsigned char *sha1, time_t mtime)
3332{
3333 void *buf;
3334 unsigned long len;
3335 enum object_type type;
3336 char hdr[32];
3337 int hdrlen;
3338 int ret;
3339
3340 if (has_loose_object(sha1))
3341 return 0;
3342 buf = read_packed_sha1(sha1, &type, &len);
3343 if (!buf)
3344 return error("cannot read sha1_file for %s", sha1_to_hex(sha1));
3345 hdrlen = xsnprintf(hdr, sizeof(hdr), "%s %lu", typename(type), len) + 1;
3346 ret = write_loose_object(sha1, hdr, hdrlen, buf, len, mtime);
3347 free(buf);
3348
3349 return ret;
3350}
3351
3352int has_pack_index(const unsigned char *sha1)
3353{
3354 struct stat st;
3355 if (stat(sha1_pack_index_name(sha1), &st))
3356 return 0;
3357 return 1;
3358}
3359
3360int has_sha1_pack(const unsigned char *sha1)
3361{
3362 struct pack_entry e;
3363 return find_pack_entry(sha1, &e);
3364}
3365
3366int has_sha1_file_with_flags(const unsigned char *sha1, int flags)
3367{
3368 struct pack_entry e;
3369
3370 if (find_pack_entry(sha1, &e))
3371 return 1;
3372 if (has_loose_object(sha1))
3373 return 1;
3374 if (flags & HAS_SHA1_QUICK)
3375 return 0;
3376 reprepare_packed_git();
3377 return find_pack_entry(sha1, &e);
3378}
3379
3380int has_object_file(const struct object_id *oid)
3381{
3382 return has_sha1_file(oid->hash);
3383}
3384
3385int has_object_file_with_flags(const struct object_id *oid, int flags)
3386{
3387 return has_sha1_file_with_flags(oid->hash, flags);
3388}
3389
3390static void check_tree(const void *buf, size_t size)
3391{
3392 struct tree_desc desc;
3393 struct name_entry entry;
3394
3395 init_tree_desc(&desc, buf, size);
3396 while (tree_entry(&desc, &entry))
3397 /* do nothing
3398 * tree_entry() will die() on malformed entries */
3399 ;
3400}
3401
3402static void check_commit(const void *buf, size_t size)
3403{
3404 struct commit c;
3405 memset(&c, 0, sizeof(c));
3406 if (parse_commit_buffer(&c, buf, size))
3407 die("corrupt commit");
3408}
3409
3410static void check_tag(const void *buf, size_t size)
3411{
3412 struct tag t;
3413 memset(&t, 0, sizeof(t));
3414 if (parse_tag_buffer(&t, buf, size))
3415 die("corrupt tag");
3416}
3417
3418static int index_mem(unsigned char *sha1, void *buf, size_t size,
3419 enum object_type type,
3420 const char *path, unsigned flags)
3421{
3422 int ret, re_allocated = 0;
3423 int write_object = flags & HASH_WRITE_OBJECT;
3424
3425 if (!type)
3426 type = OBJ_BLOB;
3427
3428 /*
3429 * Convert blobs to git internal format
3430 */
3431 if ((type == OBJ_BLOB) && path) {
3432 struct strbuf nbuf = STRBUF_INIT;
3433 if (convert_to_git(path, buf, size, &nbuf,
3434 write_object ? safe_crlf : SAFE_CRLF_FALSE)) {
3435 buf = strbuf_detach(&nbuf, &size);
3436 re_allocated = 1;
3437 }
3438 }
3439 if (flags & HASH_FORMAT_CHECK) {
3440 if (type == OBJ_TREE)
3441 check_tree(buf, size);
3442 if (type == OBJ_COMMIT)
3443 check_commit(buf, size);
3444 if (type == OBJ_TAG)
3445 check_tag(buf, size);
3446 }
3447
3448 if (write_object)
3449 ret = write_sha1_file(buf, size, typename(type), sha1);
3450 else
3451 ret = hash_sha1_file(buf, size, typename(type), sha1);
3452 if (re_allocated)
3453 free(buf);
3454 return ret;
3455}
3456
3457static int index_stream_convert_blob(unsigned char *sha1, int fd,
3458 const char *path, unsigned flags)
3459{
3460 int ret;
3461 const int write_object = flags & HASH_WRITE_OBJECT;
3462 struct strbuf sbuf = STRBUF_INIT;
3463
3464 assert(path);
3465 assert(would_convert_to_git_filter_fd(path));
3466
3467 convert_to_git_filter_fd(path, fd, &sbuf,
3468 write_object ? safe_crlf : SAFE_CRLF_FALSE);
3469
3470 if (write_object)
3471 ret = write_sha1_file(sbuf.buf, sbuf.len, typename(OBJ_BLOB),
3472 sha1);
3473 else
3474 ret = hash_sha1_file(sbuf.buf, sbuf.len, typename(OBJ_BLOB),
3475 sha1);
3476 strbuf_release(&sbuf);
3477 return ret;
3478}
3479
3480static int index_pipe(unsigned char *sha1, int fd, enum object_type type,
3481 const char *path, unsigned flags)
3482{
3483 struct strbuf sbuf = STRBUF_INIT;
3484 int ret;
3485
3486 if (strbuf_read(&sbuf, fd, 4096) >= 0)
3487 ret = index_mem(sha1, sbuf.buf, sbuf.len, type, path, flags);
3488 else
3489 ret = -1;
3490 strbuf_release(&sbuf);
3491 return ret;
3492}
3493
3494#define SMALL_FILE_SIZE (32*1024)
3495
3496static int index_core(unsigned char *sha1, int fd, size_t size,
3497 enum object_type type, const char *path,
3498 unsigned flags)
3499{
3500 int ret;
3501
3502 if (!size) {
3503 ret = index_mem(sha1, "", size, type, path, flags);
3504 } else if (size <= SMALL_FILE_SIZE) {
3505 char *buf = xmalloc(size);
3506 if (size == read_in_full(fd, buf, size))
3507 ret = index_mem(sha1, buf, size, type, path, flags);
3508 else
3509 ret = error_errno("short read");
3510 free(buf);
3511 } else {
3512 void *buf = xmmap(NULL, size, PROT_READ, MAP_PRIVATE, fd, 0);
3513 ret = index_mem(sha1, buf, size, type, path, flags);
3514 munmap(buf, size);
3515 }
3516 return ret;
3517}
3518
3519/*
3520 * This creates one packfile per large blob unless bulk-checkin
3521 * machinery is "plugged".
3522 *
3523 * This also bypasses the usual "convert-to-git" dance, and that is on
3524 * purpose. We could write a streaming version of the converting
3525 * functions and insert that before feeding the data to fast-import
3526 * (or equivalent in-core API described above). However, that is
3527 * somewhat complicated, as we do not know the size of the filter
3528 * result, which we need to know beforehand when writing a git object.
3529 * Since the primary motivation for trying to stream from the working
3530 * tree file and to avoid mmaping it in core is to deal with large
3531 * binary blobs, they generally do not want to get any conversion, and
3532 * callers should avoid this code path when filters are requested.
3533 */
3534static int index_stream(unsigned char *sha1, int fd, size_t size,
3535 enum object_type type, const char *path,
3536 unsigned flags)
3537{
3538 return index_bulk_checkin(sha1, fd, size, type, path, flags);
3539}
3540
3541int index_fd(unsigned char *sha1, int fd, struct stat *st,
3542 enum object_type type, const char *path, unsigned flags)
3543{
3544 int ret;
3545
3546 /*
3547 * Call xsize_t() only when needed to avoid potentially unnecessary
3548 * die() for large files.
3549 */
3550 if (type == OBJ_BLOB && path && would_convert_to_git_filter_fd(path))
3551 ret = index_stream_convert_blob(sha1, fd, path, flags);
3552 else if (!S_ISREG(st->st_mode))
3553 ret = index_pipe(sha1, fd, type, path, flags);
3554 else if (st->st_size <= big_file_threshold || type != OBJ_BLOB ||
3555 (path && would_convert_to_git(path)))
3556 ret = index_core(sha1, fd, xsize_t(st->st_size), type, path,
3557 flags);
3558 else
3559 ret = index_stream(sha1, fd, xsize_t(st->st_size), type, path,
3560 flags);
3561 close(fd);
3562 return ret;
3563}
3564
3565int index_path(unsigned char *sha1, const char *path, struct stat *st, unsigned flags)
3566{
3567 int fd;
3568 struct strbuf sb = STRBUF_INIT;
3569
3570 switch (st->st_mode & S_IFMT) {
3571 case S_IFREG:
3572 fd = open(path, O_RDONLY);
3573 if (fd < 0)
3574 return error_errno("open(\"%s\")", path);
3575 if (index_fd(sha1, fd, st, OBJ_BLOB, path, flags) < 0)
3576 return error("%s: failed to insert into database",
3577 path);
3578 break;
3579 case S_IFLNK:
3580 if (strbuf_readlink(&sb, path, st->st_size))
3581 return error_errno("readlink(\"%s\")", path);
3582 if (!(flags & HASH_WRITE_OBJECT))
3583 hash_sha1_file(sb.buf, sb.len, blob_type, sha1);
3584 else if (write_sha1_file(sb.buf, sb.len, blob_type, sha1))
3585 return error("%s: failed to insert into database",
3586 path);
3587 strbuf_release(&sb);
3588 break;
3589 case S_IFDIR:
3590 return resolve_gitlink_ref(path, "HEAD", sha1);
3591 default:
3592 return error("%s: unsupported file type", path);
3593 }
3594 return 0;
3595}
3596
3597int read_pack_header(int fd, struct pack_header *header)
3598{
3599 if (read_in_full(fd, header, sizeof(*header)) < sizeof(*header))
3600 /* "eof before pack header was fully read" */
3601 return PH_ERROR_EOF;
3602
3603 if (header->hdr_signature != htonl(PACK_SIGNATURE))
3604 /* "protocol error (pack signature mismatch detected)" */
3605 return PH_ERROR_PACK_SIGNATURE;
3606 if (!pack_version_ok(header->hdr_version))
3607 /* "protocol error (pack version unsupported)" */
3608 return PH_ERROR_PROTOCOL;
3609 return 0;
3610}
3611
3612void assert_sha1_type(const unsigned char *sha1, enum object_type expect)
3613{
3614 enum object_type type = sha1_object_info(sha1, NULL);
3615 if (type < 0)
3616 die("%s is not a valid object", sha1_to_hex(sha1));
3617 if (type != expect)
3618 die("%s is not a valid '%s' object", sha1_to_hex(sha1),
3619 typename(expect));
3620}
3621
3622static int for_each_file_in_obj_subdir(int subdir_nr,
3623 struct strbuf *path,
3624 each_loose_object_fn obj_cb,
3625 each_loose_cruft_fn cruft_cb,
3626 each_loose_subdir_fn subdir_cb,
3627 void *data)
3628{
3629 size_t baselen = path->len;
3630 DIR *dir = opendir(path->buf);
3631 struct dirent *de;
3632 int r = 0;
3633
3634 if (!dir) {
3635 if (errno == ENOENT)
3636 return 0;
3637 return error_errno("unable to open %s", path->buf);
3638 }
3639
3640 while ((de = readdir(dir))) {
3641 if (is_dot_or_dotdot(de->d_name))
3642 continue;
3643
3644 strbuf_setlen(path, baselen);
3645 strbuf_addf(path, "/%s", de->d_name);
3646
3647 if (strlen(de->d_name) == 38) {
3648 char hex[41];
3649 unsigned char sha1[20];
3650
3651 snprintf(hex, sizeof(hex), "%02x%s",
3652 subdir_nr, de->d_name);
3653 if (!get_sha1_hex(hex, sha1)) {
3654 if (obj_cb) {
3655 r = obj_cb(sha1, path->buf, data);
3656 if (r)
3657 break;
3658 }
3659 continue;
3660 }
3661 }
3662
3663 if (cruft_cb) {
3664 r = cruft_cb(de->d_name, path->buf, data);
3665 if (r)
3666 break;
3667 }
3668 }
3669 closedir(dir);
3670
3671 strbuf_setlen(path, baselen);
3672 if (!r && subdir_cb)
3673 r = subdir_cb(subdir_nr, path->buf, data);
3674
3675 return r;
3676}
3677
3678int for_each_loose_file_in_objdir_buf(struct strbuf *path,
3679 each_loose_object_fn obj_cb,
3680 each_loose_cruft_fn cruft_cb,
3681 each_loose_subdir_fn subdir_cb,
3682 void *data)
3683{
3684 size_t baselen = path->len;
3685 int r = 0;
3686 int i;
3687
3688 for (i = 0; i < 256; i++) {
3689 strbuf_addf(path, "/%02x", i);
3690 r = for_each_file_in_obj_subdir(i, path, obj_cb, cruft_cb,
3691 subdir_cb, data);
3692 strbuf_setlen(path, baselen);
3693 if (r)
3694 break;
3695 }
3696
3697 return r;
3698}
3699
3700int for_each_loose_file_in_objdir(const char *path,
3701 each_loose_object_fn obj_cb,
3702 each_loose_cruft_fn cruft_cb,
3703 each_loose_subdir_fn subdir_cb,
3704 void *data)
3705{
3706 struct strbuf buf = STRBUF_INIT;
3707 int r;
3708
3709 strbuf_addstr(&buf, path);
3710 r = for_each_loose_file_in_objdir_buf(&buf, obj_cb, cruft_cb,
3711 subdir_cb, data);
3712 strbuf_release(&buf);
3713
3714 return r;
3715}
3716
3717struct loose_alt_odb_data {
3718 each_loose_object_fn *cb;
3719 void *data;
3720};
3721
3722static int loose_from_alt_odb(struct alternate_object_database *alt,
3723 void *vdata)
3724{
3725 struct loose_alt_odb_data *data = vdata;
3726 struct strbuf buf = STRBUF_INIT;
3727 int r;
3728
3729 strbuf_addstr(&buf, alt->path);
3730 r = for_each_loose_file_in_objdir_buf(&buf,
3731 data->cb, NULL, NULL,
3732 data->data);
3733 strbuf_release(&buf);
3734 return r;
3735}
3736
3737int for_each_loose_object(each_loose_object_fn cb, void *data, unsigned flags)
3738{
3739 struct loose_alt_odb_data alt;
3740 int r;
3741
3742 r = for_each_loose_file_in_objdir(get_object_directory(),
3743 cb, NULL, NULL, data);
3744 if (r)
3745 return r;
3746
3747 if (flags & FOR_EACH_OBJECT_LOCAL_ONLY)
3748 return 0;
3749
3750 alt.cb = cb;
3751 alt.data = data;
3752 return foreach_alt_odb(loose_from_alt_odb, &alt);
3753}
3754
3755static int for_each_object_in_pack(struct packed_git *p, each_packed_object_fn cb, void *data)
3756{
3757 uint32_t i;
3758 int r = 0;
3759
3760 for (i = 0; i < p->num_objects; i++) {
3761 const unsigned char *sha1 = nth_packed_object_sha1(p, i);
3762
3763 if (!sha1)
3764 return error("unable to get sha1 of object %u in %s",
3765 i, p->pack_name);
3766
3767 r = cb(sha1, p, i, data);
3768 if (r)
3769 break;
3770 }
3771 return r;
3772}
3773
3774int for_each_packed_object(each_packed_object_fn cb, void *data, unsigned flags)
3775{
3776 struct packed_git *p;
3777 int r = 0;
3778 int pack_errors = 0;
3779
3780 prepare_packed_git();
3781 for (p = packed_git; p; p = p->next) {
3782 if ((flags & FOR_EACH_OBJECT_LOCAL_ONLY) && !p->pack_local)
3783 continue;
3784 if (open_pack_index(p)) {
3785 pack_errors = 1;
3786 continue;
3787 }
3788 r = for_each_object_in_pack(p, cb, data);
3789 if (r)
3790 break;
3791 }
3792 return r ? r : pack_errors;
3793}