1/*
2 * GIT - The information manager from hell
3 *
4 * Copyright (C) Linus Torvalds, 2005
5 *
6 * This handles basic git sha1 object files - packing, unpacking,
7 * creation etc.
8 */
9#include "cache.h"
10#include "string-list.h"
11#include "lockfile.h"
12#include "delta.h"
13#include "pack.h"
14#include "blob.h"
15#include "commit.h"
16#include "run-command.h"
17#include "tag.h"
18#include "tree.h"
19#include "tree-walk.h"
20#include "refs.h"
21#include "pack-revindex.h"
22#include "sha1-lookup.h"
23#include "bulk-checkin.h"
24#include "streaming.h"
25#include "dir.h"
26#include "mru.h"
27#include "list.h"
28#include "mergesort.h"
29
30#define SZ_FMT PRIuMAX
31static inline uintmax_t sz_fmt(size_t s) { return s; }
32
33const unsigned char null_sha1[20];
34const struct object_id null_oid;
35const struct object_id empty_tree_oid = {
36 EMPTY_TREE_SHA1_BIN_LITERAL
37};
38const struct object_id empty_blob_oid = {
39 EMPTY_BLOB_SHA1_BIN_LITERAL
40};
41
42/*
43 * This is meant to hold a *small* number of objects that you would
44 * want read_sha1_file() to be able to return, but yet you do not want
45 * to write them into the object store (e.g. a browse-only
46 * application).
47 */
48static struct cached_object {
49 unsigned char sha1[20];
50 enum object_type type;
51 void *buf;
52 unsigned long size;
53} *cached_objects;
54static int cached_object_nr, cached_object_alloc;
55
56static struct cached_object empty_tree = {
57 EMPTY_TREE_SHA1_BIN_LITERAL,
58 OBJ_TREE,
59 "",
60 0
61};
62
63static struct cached_object *find_cached_object(const unsigned char *sha1)
64{
65 int i;
66 struct cached_object *co = cached_objects;
67
68 for (i = 0; i < cached_object_nr; i++, co++) {
69 if (!hashcmp(co->sha1, sha1))
70 return co;
71 }
72 if (!hashcmp(sha1, empty_tree.sha1))
73 return &empty_tree;
74 return NULL;
75}
76
77int mkdir_in_gitdir(const char *path)
78{
79 if (mkdir(path, 0777)) {
80 int saved_errno = errno;
81 struct stat st;
82 struct strbuf sb = STRBUF_INIT;
83
84 if (errno != EEXIST)
85 return -1;
86 /*
87 * Are we looking at a path in a symlinked worktree
88 * whose original repository does not yet have it?
89 * e.g. .git/rr-cache pointing at its original
90 * repository in which the user hasn't performed any
91 * conflict resolution yet?
92 */
93 if (lstat(path, &st) || !S_ISLNK(st.st_mode) ||
94 strbuf_readlink(&sb, path, st.st_size) ||
95 !is_absolute_path(sb.buf) ||
96 mkdir(sb.buf, 0777)) {
97 strbuf_release(&sb);
98 errno = saved_errno;
99 return -1;
100 }
101 strbuf_release(&sb);
102 }
103 return adjust_shared_perm(path);
104}
105
106enum scld_error safe_create_leading_directories(char *path)
107{
108 char *next_component = path + offset_1st_component(path);
109 enum scld_error ret = SCLD_OK;
110
111 while (ret == SCLD_OK && next_component) {
112 struct stat st;
113 char *slash = next_component, slash_character;
114
115 while (*slash && !is_dir_sep(*slash))
116 slash++;
117
118 if (!*slash)
119 break;
120
121 next_component = slash + 1;
122 while (is_dir_sep(*next_component))
123 next_component++;
124 if (!*next_component)
125 break;
126
127 slash_character = *slash;
128 *slash = '\0';
129 if (!stat(path, &st)) {
130 /* path exists */
131 if (!S_ISDIR(st.st_mode))
132 ret = SCLD_EXISTS;
133 } else if (mkdir(path, 0777)) {
134 if (errno == EEXIST &&
135 !stat(path, &st) && S_ISDIR(st.st_mode))
136 ; /* somebody created it since we checked */
137 else if (errno == ENOENT)
138 /*
139 * Either mkdir() failed because
140 * somebody just pruned the containing
141 * directory, or stat() failed because
142 * the file that was in our way was
143 * just removed. Either way, inform
144 * the caller that it might be worth
145 * trying again:
146 */
147 ret = SCLD_VANISHED;
148 else
149 ret = SCLD_FAILED;
150 } else if (adjust_shared_perm(path)) {
151 ret = SCLD_PERMS;
152 }
153 *slash = slash_character;
154 }
155 return ret;
156}
157
158enum scld_error safe_create_leading_directories_const(const char *path)
159{
160 /* path points to cache entries, so xstrdup before messing with it */
161 char *buf = xstrdup(path);
162 enum scld_error result = safe_create_leading_directories(buf);
163 free(buf);
164 return result;
165}
166
167static void fill_sha1_path(struct strbuf *buf, const unsigned char *sha1)
168{
169 int i;
170 for (i = 0; i < 20; i++) {
171 static char hex[] = "0123456789abcdef";
172 unsigned int val = sha1[i];
173 strbuf_addch(buf, hex[val >> 4]);
174 strbuf_addch(buf, hex[val & 0xf]);
175 if (!i)
176 strbuf_addch(buf, '/');
177 }
178}
179
180const char *sha1_file_name(const unsigned char *sha1)
181{
182 static struct strbuf buf = STRBUF_INIT;
183
184 strbuf_reset(&buf);
185 strbuf_addf(&buf, "%s/", get_object_directory());
186
187 fill_sha1_path(&buf, sha1);
188 return buf.buf;
189}
190
191struct strbuf *alt_scratch_buf(struct alternate_object_database *alt)
192{
193 strbuf_setlen(&alt->scratch, alt->base_len);
194 return &alt->scratch;
195}
196
197static const char *alt_sha1_path(struct alternate_object_database *alt,
198 const unsigned char *sha1)
199{
200 struct strbuf *buf = alt_scratch_buf(alt);
201 fill_sha1_path(buf, sha1);
202 return buf->buf;
203}
204
205/*
206 * Return the name of the pack or index file with the specified sha1
207 * in its filename. *base and *name are scratch space that must be
208 * provided by the caller. which should be "pack" or "idx".
209 */
210static char *sha1_get_pack_name(const unsigned char *sha1,
211 struct strbuf *buf,
212 const char *which)
213{
214 strbuf_reset(buf);
215 strbuf_addf(buf, "%s/pack/pack-%s.%s", get_object_directory(),
216 sha1_to_hex(sha1), which);
217 return buf->buf;
218}
219
220char *sha1_pack_name(const unsigned char *sha1)
221{
222 static struct strbuf buf = STRBUF_INIT;
223 return sha1_get_pack_name(sha1, &buf, "pack");
224}
225
226char *sha1_pack_index_name(const unsigned char *sha1)
227{
228 static struct strbuf buf = STRBUF_INIT;
229 return sha1_get_pack_name(sha1, &buf, "idx");
230}
231
232struct alternate_object_database *alt_odb_list;
233static struct alternate_object_database **alt_odb_tail;
234
235/*
236 * Return non-zero iff the path is usable as an alternate object database.
237 */
238static int alt_odb_usable(struct strbuf *path, const char *normalized_objdir)
239{
240 struct alternate_object_database *alt;
241
242 /* Detect cases where alternate disappeared */
243 if (!is_directory(path->buf)) {
244 error("object directory %s does not exist; "
245 "check .git/objects/info/alternates.",
246 path->buf);
247 return 0;
248 }
249
250 /*
251 * Prevent the common mistake of listing the same
252 * thing twice, or object directory itself.
253 */
254 for (alt = alt_odb_list; alt; alt = alt->next) {
255 if (!fspathcmp(path->buf, alt->path))
256 return 0;
257 }
258 if (!fspathcmp(path->buf, normalized_objdir))
259 return 0;
260
261 return 1;
262}
263
264/*
265 * Prepare alternate object database registry.
266 *
267 * The variable alt_odb_list points at the list of struct
268 * alternate_object_database. The elements on this list come from
269 * non-empty elements from colon separated ALTERNATE_DB_ENVIRONMENT
270 * environment variable, and $GIT_OBJECT_DIRECTORY/info/alternates,
271 * whose contents is similar to that environment variable but can be
272 * LF separated. Its base points at a statically allocated buffer that
273 * contains "/the/directory/corresponding/to/.git/objects/...", while
274 * its name points just after the slash at the end of ".git/objects/"
275 * in the example above, and has enough space to hold 40-byte hex
276 * SHA1, an extra slash for the first level indirection, and the
277 * terminating NUL.
278 */
279static int link_alt_odb_entry(const char *entry, const char *relative_base,
280 int depth, const char *normalized_objdir)
281{
282 struct alternate_object_database *ent;
283 struct strbuf pathbuf = STRBUF_INIT;
284
285 if (!is_absolute_path(entry) && relative_base) {
286 strbuf_addstr(&pathbuf, real_path(relative_base));
287 strbuf_addch(&pathbuf, '/');
288 }
289 strbuf_addstr(&pathbuf, entry);
290
291 if (strbuf_normalize_path(&pathbuf) < 0) {
292 error("unable to normalize alternate object path: %s",
293 pathbuf.buf);
294 strbuf_release(&pathbuf);
295 return -1;
296 }
297
298 /*
299 * The trailing slash after the directory name is given by
300 * this function at the end. Remove duplicates.
301 */
302 while (pathbuf.len && pathbuf.buf[pathbuf.len - 1] == '/')
303 strbuf_setlen(&pathbuf, pathbuf.len - 1);
304
305 if (!alt_odb_usable(&pathbuf, normalized_objdir)) {
306 strbuf_release(&pathbuf);
307 return -1;
308 }
309
310 ent = alloc_alt_odb(pathbuf.buf);
311
312 /* add the alternate entry */
313 *alt_odb_tail = ent;
314 alt_odb_tail = &(ent->next);
315 ent->next = NULL;
316
317 /* recursively add alternates */
318 read_info_alternates(pathbuf.buf, depth + 1);
319
320 strbuf_release(&pathbuf);
321 return 0;
322}
323
324static void link_alt_odb_entries(const char *alt, int len, int sep,
325 const char *relative_base, int depth)
326{
327 struct string_list entries = STRING_LIST_INIT_NODUP;
328 char *alt_copy;
329 int i;
330 struct strbuf objdirbuf = STRBUF_INIT;
331
332 if (depth > 5) {
333 error("%s: ignoring alternate object stores, nesting too deep.",
334 relative_base);
335 return;
336 }
337
338 strbuf_add_absolute_path(&objdirbuf, get_object_directory());
339 if (strbuf_normalize_path(&objdirbuf) < 0)
340 die("unable to normalize object directory: %s",
341 objdirbuf.buf);
342
343 alt_copy = xmemdupz(alt, len);
344 string_list_split_in_place(&entries, alt_copy, sep, -1);
345 for (i = 0; i < entries.nr; i++) {
346 const char *entry = entries.items[i].string;
347 if (entry[0] == '\0' || entry[0] == '#')
348 continue;
349 link_alt_odb_entry(entry, relative_base, depth, objdirbuf.buf);
350 }
351 string_list_clear(&entries, 0);
352 free(alt_copy);
353 strbuf_release(&objdirbuf);
354}
355
356void read_info_alternates(const char * relative_base, int depth)
357{
358 char *map;
359 size_t mapsz;
360 struct stat st;
361 char *path;
362 int fd;
363
364 path = xstrfmt("%s/info/alternates", relative_base);
365 fd = git_open(path);
366 free(path);
367 if (fd < 0)
368 return;
369 if (fstat(fd, &st) || (st.st_size == 0)) {
370 close(fd);
371 return;
372 }
373 mapsz = xsize_t(st.st_size);
374 map = xmmap(NULL, mapsz, PROT_READ, MAP_PRIVATE, fd, 0);
375 close(fd);
376
377 link_alt_odb_entries(map, mapsz, '\n', relative_base, depth);
378
379 munmap(map, mapsz);
380}
381
382struct alternate_object_database *alloc_alt_odb(const char *dir)
383{
384 struct alternate_object_database *ent;
385
386 FLEX_ALLOC_STR(ent, path, dir);
387 strbuf_init(&ent->scratch, 0);
388 strbuf_addf(&ent->scratch, "%s/", dir);
389 ent->base_len = ent->scratch.len;
390
391 return ent;
392}
393
394void add_to_alternates_file(const char *reference)
395{
396 struct lock_file *lock = xcalloc(1, sizeof(struct lock_file));
397 char *alts = git_pathdup("objects/info/alternates");
398 FILE *in, *out;
399
400 hold_lock_file_for_update(lock, alts, LOCK_DIE_ON_ERROR);
401 out = fdopen_lock_file(lock, "w");
402 if (!out)
403 die_errno("unable to fdopen alternates lockfile");
404
405 in = fopen(alts, "r");
406 if (in) {
407 struct strbuf line = STRBUF_INIT;
408 int found = 0;
409
410 while (strbuf_getline(&line, in) != EOF) {
411 if (!strcmp(reference, line.buf)) {
412 found = 1;
413 break;
414 }
415 fprintf_or_die(out, "%s\n", line.buf);
416 }
417
418 strbuf_release(&line);
419 fclose(in);
420
421 if (found) {
422 rollback_lock_file(lock);
423 lock = NULL;
424 }
425 }
426 else if (errno != ENOENT)
427 die_errno("unable to read alternates file");
428
429 if (lock) {
430 fprintf_or_die(out, "%s\n", reference);
431 if (commit_lock_file(lock))
432 die_errno("unable to move new alternates file into place");
433 if (alt_odb_tail)
434 link_alt_odb_entries(reference, strlen(reference), '\n', NULL, 0);
435 }
436 free(alts);
437}
438
439void add_to_alternates_memory(const char *reference)
440{
441 /*
442 * Make sure alternates are initialized, or else our entry may be
443 * overwritten when they are.
444 */
445 prepare_alt_odb();
446
447 link_alt_odb_entries(reference, strlen(reference), '\n', NULL, 0);
448}
449
450/*
451 * Compute the exact path an alternate is at and returns it. In case of
452 * error NULL is returned and the human readable error is added to `err`
453 * `path` may be relative and should point to $GITDIR.
454 * `err` must not be null.
455 */
456char *compute_alternate_path(const char *path, struct strbuf *err)
457{
458 char *ref_git = NULL;
459 const char *repo, *ref_git_s;
460 int seen_error = 0;
461
462 ref_git_s = real_path_if_valid(path);
463 if (!ref_git_s) {
464 seen_error = 1;
465 strbuf_addf(err, _("path '%s' does not exist"), path);
466 goto out;
467 } else
468 /*
469 * Beware: read_gitfile(), real_path() and mkpath()
470 * return static buffer
471 */
472 ref_git = xstrdup(ref_git_s);
473
474 repo = read_gitfile(ref_git);
475 if (!repo)
476 repo = read_gitfile(mkpath("%s/.git", ref_git));
477 if (repo) {
478 free(ref_git);
479 ref_git = xstrdup(repo);
480 }
481
482 if (!repo && is_directory(mkpath("%s/.git/objects", ref_git))) {
483 char *ref_git_git = mkpathdup("%s/.git", ref_git);
484 free(ref_git);
485 ref_git = ref_git_git;
486 } else if (!is_directory(mkpath("%s/objects", ref_git))) {
487 struct strbuf sb = STRBUF_INIT;
488 seen_error = 1;
489 if (get_common_dir(&sb, ref_git)) {
490 strbuf_addf(err,
491 _("reference repository '%s' as a linked "
492 "checkout is not supported yet."),
493 path);
494 goto out;
495 }
496
497 strbuf_addf(err, _("reference repository '%s' is not a "
498 "local repository."), path);
499 goto out;
500 }
501
502 if (!access(mkpath("%s/shallow", ref_git), F_OK)) {
503 strbuf_addf(err, _("reference repository '%s' is shallow"),
504 path);
505 seen_error = 1;
506 goto out;
507 }
508
509 if (!access(mkpath("%s/info/grafts", ref_git), F_OK)) {
510 strbuf_addf(err,
511 _("reference repository '%s' is grafted"),
512 path);
513 seen_error = 1;
514 goto out;
515 }
516
517out:
518 if (seen_error) {
519 free(ref_git);
520 ref_git = NULL;
521 }
522
523 return ref_git;
524}
525
526int foreach_alt_odb(alt_odb_fn fn, void *cb)
527{
528 struct alternate_object_database *ent;
529 int r = 0;
530
531 prepare_alt_odb();
532 for (ent = alt_odb_list; ent; ent = ent->next) {
533 r = fn(ent, cb);
534 if (r)
535 break;
536 }
537 return r;
538}
539
540void prepare_alt_odb(void)
541{
542 const char *alt;
543
544 if (alt_odb_tail)
545 return;
546
547 alt = getenv(ALTERNATE_DB_ENVIRONMENT);
548 if (!alt) alt = "";
549
550 alt_odb_tail = &alt_odb_list;
551 link_alt_odb_entries(alt, strlen(alt), PATH_SEP, NULL, 0);
552
553 read_info_alternates(get_object_directory(), 0);
554}
555
556/* Returns 1 if we have successfully freshened the file, 0 otherwise. */
557static int freshen_file(const char *fn)
558{
559 struct utimbuf t;
560 t.actime = t.modtime = time(NULL);
561 return !utime(fn, &t);
562}
563
564/*
565 * All of the check_and_freshen functions return 1 if the file exists and was
566 * freshened (if freshening was requested), 0 otherwise. If they return
567 * 0, you should not assume that it is safe to skip a write of the object (it
568 * either does not exist on disk, or has a stale mtime and may be subject to
569 * pruning).
570 */
571static int check_and_freshen_file(const char *fn, int freshen)
572{
573 if (access(fn, F_OK))
574 return 0;
575 if (freshen && !freshen_file(fn))
576 return 0;
577 return 1;
578}
579
580static int check_and_freshen_local(const unsigned char *sha1, int freshen)
581{
582 return check_and_freshen_file(sha1_file_name(sha1), freshen);
583}
584
585static int check_and_freshen_nonlocal(const unsigned char *sha1, int freshen)
586{
587 struct alternate_object_database *alt;
588 prepare_alt_odb();
589 for (alt = alt_odb_list; alt; alt = alt->next) {
590 const char *path = alt_sha1_path(alt, sha1);
591 if (check_and_freshen_file(path, freshen))
592 return 1;
593 }
594 return 0;
595}
596
597static int check_and_freshen(const unsigned char *sha1, int freshen)
598{
599 return check_and_freshen_local(sha1, freshen) ||
600 check_and_freshen_nonlocal(sha1, freshen);
601}
602
603int has_loose_object_nonlocal(const unsigned char *sha1)
604{
605 return check_and_freshen_nonlocal(sha1, 0);
606}
607
608static int has_loose_object(const unsigned char *sha1)
609{
610 return check_and_freshen(sha1, 0);
611}
612
613static unsigned int pack_used_ctr;
614static unsigned int pack_mmap_calls;
615static unsigned int peak_pack_open_windows;
616static unsigned int pack_open_windows;
617static unsigned int pack_open_fds;
618static unsigned int pack_max_fds;
619static size_t peak_pack_mapped;
620static size_t pack_mapped;
621struct packed_git *packed_git;
622
623static struct mru packed_git_mru_storage;
624struct mru *packed_git_mru = &packed_git_mru_storage;
625
626void pack_report(void)
627{
628 fprintf(stderr,
629 "pack_report: getpagesize() = %10" SZ_FMT "\n"
630 "pack_report: core.packedGitWindowSize = %10" SZ_FMT "\n"
631 "pack_report: core.packedGitLimit = %10" SZ_FMT "\n",
632 sz_fmt(getpagesize()),
633 sz_fmt(packed_git_window_size),
634 sz_fmt(packed_git_limit));
635 fprintf(stderr,
636 "pack_report: pack_used_ctr = %10u\n"
637 "pack_report: pack_mmap_calls = %10u\n"
638 "pack_report: pack_open_windows = %10u / %10u\n"
639 "pack_report: pack_mapped = "
640 "%10" SZ_FMT " / %10" SZ_FMT "\n",
641 pack_used_ctr,
642 pack_mmap_calls,
643 pack_open_windows, peak_pack_open_windows,
644 sz_fmt(pack_mapped), sz_fmt(peak_pack_mapped));
645}
646
647/*
648 * Open and mmap the index file at path, perform a couple of
649 * consistency checks, then record its information to p. Return 0 on
650 * success.
651 */
652static int check_packed_git_idx(const char *path, struct packed_git *p)
653{
654 void *idx_map;
655 struct pack_idx_header *hdr;
656 size_t idx_size;
657 uint32_t version, nr, i, *index;
658 int fd = git_open(path);
659 struct stat st;
660
661 if (fd < 0)
662 return -1;
663 if (fstat(fd, &st)) {
664 close(fd);
665 return -1;
666 }
667 idx_size = xsize_t(st.st_size);
668 if (idx_size < 4 * 256 + 20 + 20) {
669 close(fd);
670 return error("index file %s is too small", path);
671 }
672 idx_map = xmmap(NULL, idx_size, PROT_READ, MAP_PRIVATE, fd, 0);
673 close(fd);
674
675 hdr = idx_map;
676 if (hdr->idx_signature == htonl(PACK_IDX_SIGNATURE)) {
677 version = ntohl(hdr->idx_version);
678 if (version < 2 || version > 2) {
679 munmap(idx_map, idx_size);
680 return error("index file %s is version %"PRIu32
681 " and is not supported by this binary"
682 " (try upgrading GIT to a newer version)",
683 path, version);
684 }
685 } else
686 version = 1;
687
688 nr = 0;
689 index = idx_map;
690 if (version > 1)
691 index += 2; /* skip index header */
692 for (i = 0; i < 256; i++) {
693 uint32_t n = ntohl(index[i]);
694 if (n < nr) {
695 munmap(idx_map, idx_size);
696 return error("non-monotonic index %s", path);
697 }
698 nr = n;
699 }
700
701 if (version == 1) {
702 /*
703 * Total size:
704 * - 256 index entries 4 bytes each
705 * - 24-byte entries * nr (20-byte sha1 + 4-byte offset)
706 * - 20-byte SHA1 of the packfile
707 * - 20-byte SHA1 file checksum
708 */
709 if (idx_size != 4*256 + nr * 24 + 20 + 20) {
710 munmap(idx_map, idx_size);
711 return error("wrong index v1 file size in %s", path);
712 }
713 } else if (version == 2) {
714 /*
715 * Minimum size:
716 * - 8 bytes of header
717 * - 256 index entries 4 bytes each
718 * - 20-byte sha1 entry * nr
719 * - 4-byte crc entry * nr
720 * - 4-byte offset entry * nr
721 * - 20-byte SHA1 of the packfile
722 * - 20-byte SHA1 file checksum
723 * And after the 4-byte offset table might be a
724 * variable sized table containing 8-byte entries
725 * for offsets larger than 2^31.
726 */
727 unsigned long min_size = 8 + 4*256 + nr*(20 + 4 + 4) + 20 + 20;
728 unsigned long max_size = min_size;
729 if (nr)
730 max_size += (nr - 1)*8;
731 if (idx_size < min_size || idx_size > max_size) {
732 munmap(idx_map, idx_size);
733 return error("wrong index v2 file size in %s", path);
734 }
735 if (idx_size != min_size &&
736 /*
737 * make sure we can deal with large pack offsets.
738 * 31-bit signed offset won't be enough, neither
739 * 32-bit unsigned one will be.
740 */
741 (sizeof(off_t) <= 4)) {
742 munmap(idx_map, idx_size);
743 return error("pack too large for current definition of off_t in %s", path);
744 }
745 }
746
747 p->index_version = version;
748 p->index_data = idx_map;
749 p->index_size = idx_size;
750 p->num_objects = nr;
751 return 0;
752}
753
754int open_pack_index(struct packed_git *p)
755{
756 char *idx_name;
757 size_t len;
758 int ret;
759
760 if (p->index_data)
761 return 0;
762
763 if (!strip_suffix(p->pack_name, ".pack", &len))
764 die("BUG: pack_name does not end in .pack");
765 idx_name = xstrfmt("%.*s.idx", (int)len, p->pack_name);
766 ret = check_packed_git_idx(idx_name, p);
767 free(idx_name);
768 return ret;
769}
770
771static void scan_windows(struct packed_git *p,
772 struct packed_git **lru_p,
773 struct pack_window **lru_w,
774 struct pack_window **lru_l)
775{
776 struct pack_window *w, *w_l;
777
778 for (w_l = NULL, w = p->windows; w; w = w->next) {
779 if (!w->inuse_cnt) {
780 if (!*lru_w || w->last_used < (*lru_w)->last_used) {
781 *lru_p = p;
782 *lru_w = w;
783 *lru_l = w_l;
784 }
785 }
786 w_l = w;
787 }
788}
789
790static int unuse_one_window(struct packed_git *current)
791{
792 struct packed_git *p, *lru_p = NULL;
793 struct pack_window *lru_w = NULL, *lru_l = NULL;
794
795 if (current)
796 scan_windows(current, &lru_p, &lru_w, &lru_l);
797 for (p = packed_git; p; p = p->next)
798 scan_windows(p, &lru_p, &lru_w, &lru_l);
799 if (lru_p) {
800 munmap(lru_w->base, lru_w->len);
801 pack_mapped -= lru_w->len;
802 if (lru_l)
803 lru_l->next = lru_w->next;
804 else
805 lru_p->windows = lru_w->next;
806 free(lru_w);
807 pack_open_windows--;
808 return 1;
809 }
810 return 0;
811}
812
813void release_pack_memory(size_t need)
814{
815 size_t cur = pack_mapped;
816 while (need >= (cur - pack_mapped) && unuse_one_window(NULL))
817 ; /* nothing */
818}
819
820static void mmap_limit_check(size_t length)
821{
822 static size_t limit = 0;
823 if (!limit) {
824 limit = git_env_ulong("GIT_MMAP_LIMIT", 0);
825 if (!limit)
826 limit = SIZE_MAX;
827 }
828 if (length > limit)
829 die("attempting to mmap %"PRIuMAX" over limit %"PRIuMAX,
830 (uintmax_t)length, (uintmax_t)limit);
831}
832
833void *xmmap_gently(void *start, size_t length,
834 int prot, int flags, int fd, off_t offset)
835{
836 void *ret;
837
838 mmap_limit_check(length);
839 ret = mmap(start, length, prot, flags, fd, offset);
840 if (ret == MAP_FAILED) {
841 if (!length)
842 return NULL;
843 release_pack_memory(length);
844 ret = mmap(start, length, prot, flags, fd, offset);
845 }
846 return ret;
847}
848
849void *xmmap(void *start, size_t length,
850 int prot, int flags, int fd, off_t offset)
851{
852 void *ret = xmmap_gently(start, length, prot, flags, fd, offset);
853 if (ret == MAP_FAILED)
854 die_errno("mmap failed");
855 return ret;
856}
857
858void close_pack_windows(struct packed_git *p)
859{
860 while (p->windows) {
861 struct pack_window *w = p->windows;
862
863 if (w->inuse_cnt)
864 die("pack '%s' still has open windows to it",
865 p->pack_name);
866 munmap(w->base, w->len);
867 pack_mapped -= w->len;
868 pack_open_windows--;
869 p->windows = w->next;
870 free(w);
871 }
872}
873
874static int close_pack_fd(struct packed_git *p)
875{
876 if (p->pack_fd < 0)
877 return 0;
878
879 close(p->pack_fd);
880 pack_open_fds--;
881 p->pack_fd = -1;
882
883 return 1;
884}
885
886static void close_pack(struct packed_git *p)
887{
888 close_pack_windows(p);
889 close_pack_fd(p);
890 close_pack_index(p);
891}
892
893void close_all_packs(void)
894{
895 struct packed_git *p;
896
897 for (p = packed_git; p; p = p->next)
898 if (p->do_not_close)
899 die("BUG: want to close pack marked 'do-not-close'");
900 else
901 close_pack(p);
902}
903
904
905/*
906 * The LRU pack is the one with the oldest MRU window, preferring packs
907 * with no used windows, or the oldest mtime if it has no windows allocated.
908 */
909static void find_lru_pack(struct packed_git *p, struct packed_git **lru_p, struct pack_window **mru_w, int *accept_windows_inuse)
910{
911 struct pack_window *w, *this_mru_w;
912 int has_windows_inuse = 0;
913
914 /*
915 * Reject this pack if it has windows and the previously selected
916 * one does not. If this pack does not have windows, reject
917 * it if the pack file is newer than the previously selected one.
918 */
919 if (*lru_p && !*mru_w && (p->windows || p->mtime > (*lru_p)->mtime))
920 return;
921
922 for (w = this_mru_w = p->windows; w; w = w->next) {
923 /*
924 * Reject this pack if any of its windows are in use,
925 * but the previously selected pack did not have any
926 * inuse windows. Otherwise, record that this pack
927 * has windows in use.
928 */
929 if (w->inuse_cnt) {
930 if (*accept_windows_inuse)
931 has_windows_inuse = 1;
932 else
933 return;
934 }
935
936 if (w->last_used > this_mru_w->last_used)
937 this_mru_w = w;
938
939 /*
940 * Reject this pack if it has windows that have been
941 * used more recently than the previously selected pack.
942 * If the previously selected pack had windows inuse and
943 * we have not encountered a window in this pack that is
944 * inuse, skip this check since we prefer a pack with no
945 * inuse windows to one that has inuse windows.
946 */
947 if (*mru_w && *accept_windows_inuse == has_windows_inuse &&
948 this_mru_w->last_used > (*mru_w)->last_used)
949 return;
950 }
951
952 /*
953 * Select this pack.
954 */
955 *mru_w = this_mru_w;
956 *lru_p = p;
957 *accept_windows_inuse = has_windows_inuse;
958}
959
960static int close_one_pack(void)
961{
962 struct packed_git *p, *lru_p = NULL;
963 struct pack_window *mru_w = NULL;
964 int accept_windows_inuse = 1;
965
966 for (p = packed_git; p; p = p->next) {
967 if (p->pack_fd == -1)
968 continue;
969 find_lru_pack(p, &lru_p, &mru_w, &accept_windows_inuse);
970 }
971
972 if (lru_p)
973 return close_pack_fd(lru_p);
974
975 return 0;
976}
977
978void unuse_pack(struct pack_window **w_cursor)
979{
980 struct pack_window *w = *w_cursor;
981 if (w) {
982 w->inuse_cnt--;
983 *w_cursor = NULL;
984 }
985}
986
987void close_pack_index(struct packed_git *p)
988{
989 if (p->index_data) {
990 munmap((void *)p->index_data, p->index_size);
991 p->index_data = NULL;
992 }
993}
994
995static unsigned int get_max_fd_limit(void)
996{
997#ifdef RLIMIT_NOFILE
998 {
999 struct rlimit lim;
1000
1001 if (!getrlimit(RLIMIT_NOFILE, &lim))
1002 return lim.rlim_cur;
1003 }
1004#endif
1005
1006#ifdef _SC_OPEN_MAX
1007 {
1008 long open_max = sysconf(_SC_OPEN_MAX);
1009 if (0 < open_max)
1010 return open_max;
1011 /*
1012 * Otherwise, we got -1 for one of the two
1013 * reasons:
1014 *
1015 * (1) sysconf() did not understand _SC_OPEN_MAX
1016 * and signaled an error with -1; or
1017 * (2) sysconf() said there is no limit.
1018 *
1019 * We _could_ clear errno before calling sysconf() to
1020 * tell these two cases apart and return a huge number
1021 * in the latter case to let the caller cap it to a
1022 * value that is not so selfish, but letting the
1023 * fallback OPEN_MAX codepath take care of these cases
1024 * is a lot simpler.
1025 */
1026 }
1027#endif
1028
1029#ifdef OPEN_MAX
1030 return OPEN_MAX;
1031#else
1032 return 1; /* see the caller ;-) */
1033#endif
1034}
1035
1036/*
1037 * Do not call this directly as this leaks p->pack_fd on error return;
1038 * call open_packed_git() instead.
1039 */
1040static int open_packed_git_1(struct packed_git *p)
1041{
1042 struct stat st;
1043 struct pack_header hdr;
1044 unsigned char sha1[20];
1045 unsigned char *idx_sha1;
1046 long fd_flag;
1047
1048 if (!p->index_data && open_pack_index(p))
1049 return error("packfile %s index unavailable", p->pack_name);
1050
1051 if (!pack_max_fds) {
1052 unsigned int max_fds = get_max_fd_limit();
1053
1054 /* Save 3 for stdin/stdout/stderr, 22 for work */
1055 if (25 < max_fds)
1056 pack_max_fds = max_fds - 25;
1057 else
1058 pack_max_fds = 1;
1059 }
1060
1061 while (pack_max_fds <= pack_open_fds && close_one_pack())
1062 ; /* nothing */
1063
1064 p->pack_fd = git_open(p->pack_name);
1065 if (p->pack_fd < 0 || fstat(p->pack_fd, &st))
1066 return -1;
1067 pack_open_fds++;
1068
1069 /* If we created the struct before we had the pack we lack size. */
1070 if (!p->pack_size) {
1071 if (!S_ISREG(st.st_mode))
1072 return error("packfile %s not a regular file", p->pack_name);
1073 p->pack_size = st.st_size;
1074 } else if (p->pack_size != st.st_size)
1075 return error("packfile %s size changed", p->pack_name);
1076
1077 /* We leave these file descriptors open with sliding mmap;
1078 * there is no point keeping them open across exec(), though.
1079 */
1080 fd_flag = fcntl(p->pack_fd, F_GETFD, 0);
1081 if (fd_flag < 0)
1082 return error("cannot determine file descriptor flags");
1083 fd_flag |= FD_CLOEXEC;
1084 if (fcntl(p->pack_fd, F_SETFD, fd_flag) == -1)
1085 return error("cannot set FD_CLOEXEC");
1086
1087 /* Verify we recognize this pack file format. */
1088 if (read_in_full(p->pack_fd, &hdr, sizeof(hdr)) != sizeof(hdr))
1089 return error("file %s is far too short to be a packfile", p->pack_name);
1090 if (hdr.hdr_signature != htonl(PACK_SIGNATURE))
1091 return error("file %s is not a GIT packfile", p->pack_name);
1092 if (!pack_version_ok(hdr.hdr_version))
1093 return error("packfile %s is version %"PRIu32" and not"
1094 " supported (try upgrading GIT to a newer version)",
1095 p->pack_name, ntohl(hdr.hdr_version));
1096
1097 /* Verify the pack matches its index. */
1098 if (p->num_objects != ntohl(hdr.hdr_entries))
1099 return error("packfile %s claims to have %"PRIu32" objects"
1100 " while index indicates %"PRIu32" objects",
1101 p->pack_name, ntohl(hdr.hdr_entries),
1102 p->num_objects);
1103 if (lseek(p->pack_fd, p->pack_size - sizeof(sha1), SEEK_SET) == -1)
1104 return error("end of packfile %s is unavailable", p->pack_name);
1105 if (read_in_full(p->pack_fd, sha1, sizeof(sha1)) != sizeof(sha1))
1106 return error("packfile %s signature is unavailable", p->pack_name);
1107 idx_sha1 = ((unsigned char *)p->index_data) + p->index_size - 40;
1108 if (hashcmp(sha1, idx_sha1))
1109 return error("packfile %s does not match index", p->pack_name);
1110 return 0;
1111}
1112
1113static int open_packed_git(struct packed_git *p)
1114{
1115 if (!open_packed_git_1(p))
1116 return 0;
1117 close_pack_fd(p);
1118 return -1;
1119}
1120
1121static int in_window(struct pack_window *win, off_t offset)
1122{
1123 /* We must promise at least 20 bytes (one hash) after the
1124 * offset is available from this window, otherwise the offset
1125 * is not actually in this window and a different window (which
1126 * has that one hash excess) must be used. This is to support
1127 * the object header and delta base parsing routines below.
1128 */
1129 off_t win_off = win->offset;
1130 return win_off <= offset
1131 && (offset + 20) <= (win_off + win->len);
1132}
1133
1134unsigned char *use_pack(struct packed_git *p,
1135 struct pack_window **w_cursor,
1136 off_t offset,
1137 unsigned long *left)
1138{
1139 struct pack_window *win = *w_cursor;
1140
1141 /* Since packfiles end in a hash of their content and it's
1142 * pointless to ask for an offset into the middle of that
1143 * hash, and the in_window function above wouldn't match
1144 * don't allow an offset too close to the end of the file.
1145 */
1146 if (!p->pack_size && p->pack_fd == -1 && open_packed_git(p))
1147 die("packfile %s cannot be accessed", p->pack_name);
1148 if (offset > (p->pack_size - 20))
1149 die("offset beyond end of packfile (truncated pack?)");
1150 if (offset < 0)
1151 die(_("offset before end of packfile (broken .idx?)"));
1152
1153 if (!win || !in_window(win, offset)) {
1154 if (win)
1155 win->inuse_cnt--;
1156 for (win = p->windows; win; win = win->next) {
1157 if (in_window(win, offset))
1158 break;
1159 }
1160 if (!win) {
1161 size_t window_align = packed_git_window_size / 2;
1162 off_t len;
1163
1164 if (p->pack_fd == -1 && open_packed_git(p))
1165 die("packfile %s cannot be accessed", p->pack_name);
1166
1167 win = xcalloc(1, sizeof(*win));
1168 win->offset = (offset / window_align) * window_align;
1169 len = p->pack_size - win->offset;
1170 if (len > packed_git_window_size)
1171 len = packed_git_window_size;
1172 win->len = (size_t)len;
1173 pack_mapped += win->len;
1174 while (packed_git_limit < pack_mapped
1175 && unuse_one_window(p))
1176 ; /* nothing */
1177 win->base = xmmap(NULL, win->len,
1178 PROT_READ, MAP_PRIVATE,
1179 p->pack_fd, win->offset);
1180 if (win->base == MAP_FAILED)
1181 die_errno("packfile %s cannot be mapped",
1182 p->pack_name);
1183 if (!win->offset && win->len == p->pack_size
1184 && !p->do_not_close)
1185 close_pack_fd(p);
1186 pack_mmap_calls++;
1187 pack_open_windows++;
1188 if (pack_mapped > peak_pack_mapped)
1189 peak_pack_mapped = pack_mapped;
1190 if (pack_open_windows > peak_pack_open_windows)
1191 peak_pack_open_windows = pack_open_windows;
1192 win->next = p->windows;
1193 p->windows = win;
1194 }
1195 }
1196 if (win != *w_cursor) {
1197 win->last_used = pack_used_ctr++;
1198 win->inuse_cnt++;
1199 *w_cursor = win;
1200 }
1201 offset -= win->offset;
1202 if (left)
1203 *left = win->len - xsize_t(offset);
1204 return win->base + offset;
1205}
1206
1207static struct packed_git *alloc_packed_git(int extra)
1208{
1209 struct packed_git *p = xmalloc(st_add(sizeof(*p), extra));
1210 memset(p, 0, sizeof(*p));
1211 p->pack_fd = -1;
1212 return p;
1213}
1214
1215static void try_to_free_pack_memory(size_t size)
1216{
1217 release_pack_memory(size);
1218}
1219
1220struct packed_git *add_packed_git(const char *path, size_t path_len, int local)
1221{
1222 static int have_set_try_to_free_routine;
1223 struct stat st;
1224 size_t alloc;
1225 struct packed_git *p;
1226
1227 if (!have_set_try_to_free_routine) {
1228 have_set_try_to_free_routine = 1;
1229 set_try_to_free_routine(try_to_free_pack_memory);
1230 }
1231
1232 /*
1233 * Make sure a corresponding .pack file exists and that
1234 * the index looks sane.
1235 */
1236 if (!strip_suffix_mem(path, &path_len, ".idx"))
1237 return NULL;
1238
1239 /*
1240 * ".pack" is long enough to hold any suffix we're adding (and
1241 * the use xsnprintf double-checks that)
1242 */
1243 alloc = st_add3(path_len, strlen(".pack"), 1);
1244 p = alloc_packed_git(alloc);
1245 memcpy(p->pack_name, path, path_len);
1246
1247 xsnprintf(p->pack_name + path_len, alloc - path_len, ".keep");
1248 if (!access(p->pack_name, F_OK))
1249 p->pack_keep = 1;
1250
1251 xsnprintf(p->pack_name + path_len, alloc - path_len, ".pack");
1252 if (stat(p->pack_name, &st) || !S_ISREG(st.st_mode)) {
1253 free(p);
1254 return NULL;
1255 }
1256
1257 /* ok, it looks sane as far as we can check without
1258 * actually mapping the pack file.
1259 */
1260 p->pack_size = st.st_size;
1261 p->pack_local = local;
1262 p->mtime = st.st_mtime;
1263 if (path_len < 40 || get_sha1_hex(path + path_len - 40, p->sha1))
1264 hashclr(p->sha1);
1265 return p;
1266}
1267
1268struct packed_git *parse_pack_index(unsigned char *sha1, const char *idx_path)
1269{
1270 const char *path = sha1_pack_name(sha1);
1271 size_t alloc = st_add(strlen(path), 1);
1272 struct packed_git *p = alloc_packed_git(alloc);
1273
1274 memcpy(p->pack_name, path, alloc); /* includes NUL */
1275 hashcpy(p->sha1, sha1);
1276 if (check_packed_git_idx(idx_path, p)) {
1277 free(p);
1278 return NULL;
1279 }
1280
1281 return p;
1282}
1283
1284void install_packed_git(struct packed_git *pack)
1285{
1286 if (pack->pack_fd != -1)
1287 pack_open_fds++;
1288
1289 pack->next = packed_git;
1290 packed_git = pack;
1291}
1292
1293void (*report_garbage)(unsigned seen_bits, const char *path);
1294
1295static void report_helper(const struct string_list *list,
1296 int seen_bits, int first, int last)
1297{
1298 if (seen_bits == (PACKDIR_FILE_PACK|PACKDIR_FILE_IDX))
1299 return;
1300
1301 for (; first < last; first++)
1302 report_garbage(seen_bits, list->items[first].string);
1303}
1304
1305static void report_pack_garbage(struct string_list *list)
1306{
1307 int i, baselen = -1, first = 0, seen_bits = 0;
1308
1309 if (!report_garbage)
1310 return;
1311
1312 string_list_sort(list);
1313
1314 for (i = 0; i < list->nr; i++) {
1315 const char *path = list->items[i].string;
1316 if (baselen != -1 &&
1317 strncmp(path, list->items[first].string, baselen)) {
1318 report_helper(list, seen_bits, first, i);
1319 baselen = -1;
1320 seen_bits = 0;
1321 }
1322 if (baselen == -1) {
1323 const char *dot = strrchr(path, '.');
1324 if (!dot) {
1325 report_garbage(PACKDIR_FILE_GARBAGE, path);
1326 continue;
1327 }
1328 baselen = dot - path + 1;
1329 first = i;
1330 }
1331 if (!strcmp(path + baselen, "pack"))
1332 seen_bits |= 1;
1333 else if (!strcmp(path + baselen, "idx"))
1334 seen_bits |= 2;
1335 }
1336 report_helper(list, seen_bits, first, list->nr);
1337}
1338
1339static void prepare_packed_git_one(char *objdir, int local)
1340{
1341 struct strbuf path = STRBUF_INIT;
1342 size_t dirnamelen;
1343 DIR *dir;
1344 struct dirent *de;
1345 struct string_list garbage = STRING_LIST_INIT_DUP;
1346
1347 strbuf_addstr(&path, objdir);
1348 strbuf_addstr(&path, "/pack");
1349 dir = opendir(path.buf);
1350 if (!dir) {
1351 if (errno != ENOENT)
1352 error_errno("unable to open object pack directory: %s",
1353 path.buf);
1354 strbuf_release(&path);
1355 return;
1356 }
1357 strbuf_addch(&path, '/');
1358 dirnamelen = path.len;
1359 while ((de = readdir(dir)) != NULL) {
1360 struct packed_git *p;
1361 size_t base_len;
1362
1363 if (is_dot_or_dotdot(de->d_name))
1364 continue;
1365
1366 strbuf_setlen(&path, dirnamelen);
1367 strbuf_addstr(&path, de->d_name);
1368
1369 base_len = path.len;
1370 if (strip_suffix_mem(path.buf, &base_len, ".idx")) {
1371 /* Don't reopen a pack we already have. */
1372 for (p = packed_git; p; p = p->next) {
1373 size_t len;
1374 if (strip_suffix(p->pack_name, ".pack", &len) &&
1375 len == base_len &&
1376 !memcmp(p->pack_name, path.buf, len))
1377 break;
1378 }
1379 if (p == NULL &&
1380 /*
1381 * See if it really is a valid .idx file with
1382 * corresponding .pack file that we can map.
1383 */
1384 (p = add_packed_git(path.buf, path.len, local)) != NULL)
1385 install_packed_git(p);
1386 }
1387
1388 if (!report_garbage)
1389 continue;
1390
1391 if (ends_with(de->d_name, ".idx") ||
1392 ends_with(de->d_name, ".pack") ||
1393 ends_with(de->d_name, ".bitmap") ||
1394 ends_with(de->d_name, ".keep"))
1395 string_list_append(&garbage, path.buf);
1396 else
1397 report_garbage(PACKDIR_FILE_GARBAGE, path.buf);
1398 }
1399 closedir(dir);
1400 report_pack_garbage(&garbage);
1401 string_list_clear(&garbage, 0);
1402 strbuf_release(&path);
1403}
1404
1405static void *get_next_packed_git(const void *p)
1406{
1407 return ((const struct packed_git *)p)->next;
1408}
1409
1410static void set_next_packed_git(void *p, void *next)
1411{
1412 ((struct packed_git *)p)->next = next;
1413}
1414
1415static int sort_pack(const void *a_, const void *b_)
1416{
1417 const struct packed_git *a = a_;
1418 const struct packed_git *b = b_;
1419 int st;
1420
1421 /*
1422 * Local packs tend to contain objects specific to our
1423 * variant of the project than remote ones. In addition,
1424 * remote ones could be on a network mounted filesystem.
1425 * Favor local ones for these reasons.
1426 */
1427 st = a->pack_local - b->pack_local;
1428 if (st)
1429 return -st;
1430
1431 /*
1432 * Younger packs tend to contain more recent objects,
1433 * and more recent objects tend to get accessed more
1434 * often.
1435 */
1436 if (a->mtime < b->mtime)
1437 return 1;
1438 else if (a->mtime == b->mtime)
1439 return 0;
1440 return -1;
1441}
1442
1443static void rearrange_packed_git(void)
1444{
1445 packed_git = llist_mergesort(packed_git, get_next_packed_git,
1446 set_next_packed_git, sort_pack);
1447}
1448
1449static void prepare_packed_git_mru(void)
1450{
1451 struct packed_git *p;
1452
1453 mru_clear(packed_git_mru);
1454 for (p = packed_git; p; p = p->next)
1455 mru_append(packed_git_mru, p);
1456}
1457
1458static int prepare_packed_git_run_once = 0;
1459void prepare_packed_git(void)
1460{
1461 struct alternate_object_database *alt;
1462
1463 if (prepare_packed_git_run_once)
1464 return;
1465 prepare_packed_git_one(get_object_directory(), 1);
1466 prepare_alt_odb();
1467 for (alt = alt_odb_list; alt; alt = alt->next)
1468 prepare_packed_git_one(alt->path, 0);
1469 rearrange_packed_git();
1470 prepare_packed_git_mru();
1471 prepare_packed_git_run_once = 1;
1472}
1473
1474void reprepare_packed_git(void)
1475{
1476 prepare_packed_git_run_once = 0;
1477 prepare_packed_git();
1478}
1479
1480static void mark_bad_packed_object(struct packed_git *p,
1481 const unsigned char *sha1)
1482{
1483 unsigned i;
1484 for (i = 0; i < p->num_bad_objects; i++)
1485 if (!hashcmp(sha1, p->bad_object_sha1 + GIT_SHA1_RAWSZ * i))
1486 return;
1487 p->bad_object_sha1 = xrealloc(p->bad_object_sha1,
1488 st_mult(GIT_SHA1_RAWSZ,
1489 st_add(p->num_bad_objects, 1)));
1490 hashcpy(p->bad_object_sha1 + GIT_SHA1_RAWSZ * p->num_bad_objects, sha1);
1491 p->num_bad_objects++;
1492}
1493
1494static const struct packed_git *has_packed_and_bad(const unsigned char *sha1)
1495{
1496 struct packed_git *p;
1497 unsigned i;
1498
1499 for (p = packed_git; p; p = p->next)
1500 for (i = 0; i < p->num_bad_objects; i++)
1501 if (!hashcmp(sha1, p->bad_object_sha1 + 20 * i))
1502 return p;
1503 return NULL;
1504}
1505
1506/*
1507 * With an in-core object data in "map", rehash it to make sure the
1508 * object name actually matches "sha1" to detect object corruption.
1509 * With "map" == NULL, try reading the object named with "sha1" using
1510 * the streaming interface and rehash it to do the same.
1511 */
1512int check_sha1_signature(const unsigned char *sha1, void *map,
1513 unsigned long size, const char *type)
1514{
1515 unsigned char real_sha1[20];
1516 enum object_type obj_type;
1517 struct git_istream *st;
1518 git_SHA_CTX c;
1519 char hdr[32];
1520 int hdrlen;
1521
1522 if (map) {
1523 hash_sha1_file(map, size, type, real_sha1);
1524 return hashcmp(sha1, real_sha1) ? -1 : 0;
1525 }
1526
1527 st = open_istream(sha1, &obj_type, &size, NULL);
1528 if (!st)
1529 return -1;
1530
1531 /* Generate the header */
1532 hdrlen = xsnprintf(hdr, sizeof(hdr), "%s %lu", typename(obj_type), size) + 1;
1533
1534 /* Sha1.. */
1535 git_SHA1_Init(&c);
1536 git_SHA1_Update(&c, hdr, hdrlen);
1537 for (;;) {
1538 char buf[1024 * 16];
1539 ssize_t readlen = read_istream(st, buf, sizeof(buf));
1540
1541 if (readlen < 0) {
1542 close_istream(st);
1543 return -1;
1544 }
1545 if (!readlen)
1546 break;
1547 git_SHA1_Update(&c, buf, readlen);
1548 }
1549 git_SHA1_Final(real_sha1, &c);
1550 close_istream(st);
1551 return hashcmp(sha1, real_sha1) ? -1 : 0;
1552}
1553
1554int git_open_cloexec(const char *name, int flags)
1555{
1556 int fd;
1557 static int o_cloexec = O_CLOEXEC;
1558
1559 fd = open(name, flags | o_cloexec);
1560 if ((o_cloexec & O_CLOEXEC) && fd < 0 && errno == EINVAL) {
1561 /* Try again w/o O_CLOEXEC: the kernel might not support it */
1562 o_cloexec &= ~O_CLOEXEC;
1563 fd = open(name, flags | o_cloexec);
1564 }
1565
1566#if defined(F_GETFL) && defined(F_SETFL) && defined(FD_CLOEXEC)
1567 {
1568 static int fd_cloexec = FD_CLOEXEC;
1569
1570 if (!o_cloexec && 0 <= fd && fd_cloexec) {
1571 /* Opened w/o O_CLOEXEC? try with fcntl(2) to add it */
1572 int flags = fcntl(fd, F_GETFL);
1573 if (fcntl(fd, F_SETFL, flags | fd_cloexec))
1574 fd_cloexec = 0;
1575 }
1576 }
1577#endif
1578 return fd;
1579}
1580
1581static int stat_sha1_file(const unsigned char *sha1, struct stat *st)
1582{
1583 struct alternate_object_database *alt;
1584
1585 if (!lstat(sha1_file_name(sha1), st))
1586 return 0;
1587
1588 prepare_alt_odb();
1589 errno = ENOENT;
1590 for (alt = alt_odb_list; alt; alt = alt->next) {
1591 const char *path = alt_sha1_path(alt, sha1);
1592 if (!lstat(path, st))
1593 return 0;
1594 }
1595
1596 return -1;
1597}
1598
1599static int open_sha1_file(const unsigned char *sha1)
1600{
1601 int fd;
1602 struct alternate_object_database *alt;
1603 int most_interesting_errno;
1604
1605 fd = git_open(sha1_file_name(sha1));
1606 if (fd >= 0)
1607 return fd;
1608 most_interesting_errno = errno;
1609
1610 prepare_alt_odb();
1611 for (alt = alt_odb_list; alt; alt = alt->next) {
1612 const char *path = alt_sha1_path(alt, sha1);
1613 fd = git_open(path);
1614 if (fd >= 0)
1615 return fd;
1616 if (most_interesting_errno == ENOENT)
1617 most_interesting_errno = errno;
1618 }
1619 errno = most_interesting_errno;
1620 return -1;
1621}
1622
1623void *map_sha1_file(const unsigned char *sha1, unsigned long *size)
1624{
1625 void *map;
1626 int fd;
1627
1628 fd = open_sha1_file(sha1);
1629 map = NULL;
1630 if (fd >= 0) {
1631 struct stat st;
1632
1633 if (!fstat(fd, &st)) {
1634 *size = xsize_t(st.st_size);
1635 if (!*size) {
1636 /* mmap() is forbidden on empty files */
1637 error("object file %s is empty", sha1_file_name(sha1));
1638 return NULL;
1639 }
1640 map = xmmap(NULL, *size, PROT_READ, MAP_PRIVATE, fd, 0);
1641 }
1642 close(fd);
1643 }
1644 return map;
1645}
1646
1647unsigned long unpack_object_header_buffer(const unsigned char *buf,
1648 unsigned long len, enum object_type *type, unsigned long *sizep)
1649{
1650 unsigned shift;
1651 unsigned long size, c;
1652 unsigned long used = 0;
1653
1654 c = buf[used++];
1655 *type = (c >> 4) & 7;
1656 size = c & 15;
1657 shift = 4;
1658 while (c & 0x80) {
1659 if (len <= used || bitsizeof(long) <= shift) {
1660 error("bad object header");
1661 size = used = 0;
1662 break;
1663 }
1664 c = buf[used++];
1665 size += (c & 0x7f) << shift;
1666 shift += 7;
1667 }
1668 *sizep = size;
1669 return used;
1670}
1671
1672static int unpack_sha1_short_header(git_zstream *stream,
1673 unsigned char *map, unsigned long mapsize,
1674 void *buffer, unsigned long bufsiz)
1675{
1676 /* Get the data stream */
1677 memset(stream, 0, sizeof(*stream));
1678 stream->next_in = map;
1679 stream->avail_in = mapsize;
1680 stream->next_out = buffer;
1681 stream->avail_out = bufsiz;
1682
1683 git_inflate_init(stream);
1684 return git_inflate(stream, 0);
1685}
1686
1687int unpack_sha1_header(git_zstream *stream,
1688 unsigned char *map, unsigned long mapsize,
1689 void *buffer, unsigned long bufsiz)
1690{
1691 int status = unpack_sha1_short_header(stream, map, mapsize,
1692 buffer, bufsiz);
1693
1694 if (status < Z_OK)
1695 return status;
1696
1697 /* Make sure we have the terminating NUL */
1698 if (!memchr(buffer, '\0', stream->next_out - (unsigned char *)buffer))
1699 return -1;
1700 return 0;
1701}
1702
1703static int unpack_sha1_header_to_strbuf(git_zstream *stream, unsigned char *map,
1704 unsigned long mapsize, void *buffer,
1705 unsigned long bufsiz, struct strbuf *header)
1706{
1707 int status;
1708
1709 status = unpack_sha1_short_header(stream, map, mapsize, buffer, bufsiz);
1710 if (status < Z_OK)
1711 return -1;
1712
1713 /*
1714 * Check if entire header is unpacked in the first iteration.
1715 */
1716 if (memchr(buffer, '\0', stream->next_out - (unsigned char *)buffer))
1717 return 0;
1718
1719 /*
1720 * buffer[0..bufsiz] was not large enough. Copy the partial
1721 * result out to header, and then append the result of further
1722 * reading the stream.
1723 */
1724 strbuf_add(header, buffer, stream->next_out - (unsigned char *)buffer);
1725 stream->next_out = buffer;
1726 stream->avail_out = bufsiz;
1727
1728 do {
1729 status = git_inflate(stream, 0);
1730 strbuf_add(header, buffer, stream->next_out - (unsigned char *)buffer);
1731 if (memchr(buffer, '\0', stream->next_out - (unsigned char *)buffer))
1732 return 0;
1733 stream->next_out = buffer;
1734 stream->avail_out = bufsiz;
1735 } while (status != Z_STREAM_END);
1736 return -1;
1737}
1738
1739static void *unpack_sha1_rest(git_zstream *stream, void *buffer, unsigned long size, const unsigned char *sha1)
1740{
1741 int bytes = strlen(buffer) + 1;
1742 unsigned char *buf = xmallocz(size);
1743 unsigned long n;
1744 int status = Z_OK;
1745
1746 n = stream->total_out - bytes;
1747 if (n > size)
1748 n = size;
1749 memcpy(buf, (char *) buffer + bytes, n);
1750 bytes = n;
1751 if (bytes <= size) {
1752 /*
1753 * The above condition must be (bytes <= size), not
1754 * (bytes < size). In other words, even though we
1755 * expect no more output and set avail_out to zero,
1756 * the input zlib stream may have bytes that express
1757 * "this concludes the stream", and we *do* want to
1758 * eat that input.
1759 *
1760 * Otherwise we would not be able to test that we
1761 * consumed all the input to reach the expected size;
1762 * we also want to check that zlib tells us that all
1763 * went well with status == Z_STREAM_END at the end.
1764 */
1765 stream->next_out = buf + bytes;
1766 stream->avail_out = size - bytes;
1767 while (status == Z_OK)
1768 status = git_inflate(stream, Z_FINISH);
1769 }
1770 if (status == Z_STREAM_END && !stream->avail_in) {
1771 git_inflate_end(stream);
1772 return buf;
1773 }
1774
1775 if (status < 0)
1776 error("corrupt loose object '%s'", sha1_to_hex(sha1));
1777 else if (stream->avail_in)
1778 error("garbage at end of loose object '%s'",
1779 sha1_to_hex(sha1));
1780 free(buf);
1781 return NULL;
1782}
1783
1784/*
1785 * We used to just use "sscanf()", but that's actually way
1786 * too permissive for what we want to check. So do an anal
1787 * object header parse by hand.
1788 */
1789static int parse_sha1_header_extended(const char *hdr, struct object_info *oi,
1790 unsigned int flags)
1791{
1792 const char *type_buf = hdr;
1793 unsigned long size;
1794 int type, type_len = 0;
1795
1796 /*
1797 * The type can be of any size but is followed by
1798 * a space.
1799 */
1800 for (;;) {
1801 char c = *hdr++;
1802 if (!c)
1803 return -1;
1804 if (c == ' ')
1805 break;
1806 type_len++;
1807 }
1808
1809 type = type_from_string_gently(type_buf, type_len, 1);
1810 if (oi->typename)
1811 strbuf_add(oi->typename, type_buf, type_len);
1812 /*
1813 * Set type to 0 if its an unknown object and
1814 * we're obtaining the type using '--allow-unknown-type'
1815 * option.
1816 */
1817 if ((flags & LOOKUP_UNKNOWN_OBJECT) && (type < 0))
1818 type = 0;
1819 else if (type < 0)
1820 die("invalid object type");
1821 if (oi->typep)
1822 *oi->typep = type;
1823
1824 /*
1825 * The length must follow immediately, and be in canonical
1826 * decimal format (ie "010" is not valid).
1827 */
1828 size = *hdr++ - '0';
1829 if (size > 9)
1830 return -1;
1831 if (size) {
1832 for (;;) {
1833 unsigned long c = *hdr - '0';
1834 if (c > 9)
1835 break;
1836 hdr++;
1837 size = size * 10 + c;
1838 }
1839 }
1840
1841 if (oi->sizep)
1842 *oi->sizep = size;
1843
1844 /*
1845 * The length must be followed by a zero byte
1846 */
1847 return *hdr ? -1 : type;
1848}
1849
1850int parse_sha1_header(const char *hdr, unsigned long *sizep)
1851{
1852 struct object_info oi = OBJECT_INFO_INIT;
1853
1854 oi.sizep = sizep;
1855 return parse_sha1_header_extended(hdr, &oi, LOOKUP_REPLACE_OBJECT);
1856}
1857
1858static void *unpack_sha1_file(void *map, unsigned long mapsize, enum object_type *type, unsigned long *size, const unsigned char *sha1)
1859{
1860 int ret;
1861 git_zstream stream;
1862 char hdr[8192];
1863
1864 ret = unpack_sha1_header(&stream, map, mapsize, hdr, sizeof(hdr));
1865 if (ret < Z_OK || (*type = parse_sha1_header(hdr, size)) < 0)
1866 return NULL;
1867
1868 return unpack_sha1_rest(&stream, hdr, *size, sha1);
1869}
1870
1871unsigned long get_size_from_delta(struct packed_git *p,
1872 struct pack_window **w_curs,
1873 off_t curpos)
1874{
1875 const unsigned char *data;
1876 unsigned char delta_head[20], *in;
1877 git_zstream stream;
1878 int st;
1879
1880 memset(&stream, 0, sizeof(stream));
1881 stream.next_out = delta_head;
1882 stream.avail_out = sizeof(delta_head);
1883
1884 git_inflate_init(&stream);
1885 do {
1886 in = use_pack(p, w_curs, curpos, &stream.avail_in);
1887 stream.next_in = in;
1888 st = git_inflate(&stream, Z_FINISH);
1889 curpos += stream.next_in - in;
1890 } while ((st == Z_OK || st == Z_BUF_ERROR) &&
1891 stream.total_out < sizeof(delta_head));
1892 git_inflate_end(&stream);
1893 if ((st != Z_STREAM_END) && stream.total_out != sizeof(delta_head)) {
1894 error("delta data unpack-initial failed");
1895 return 0;
1896 }
1897
1898 /* Examine the initial part of the delta to figure out
1899 * the result size.
1900 */
1901 data = delta_head;
1902
1903 /* ignore base size */
1904 get_delta_hdr_size(&data, delta_head+sizeof(delta_head));
1905
1906 /* Read the result size */
1907 return get_delta_hdr_size(&data, delta_head+sizeof(delta_head));
1908}
1909
1910static off_t get_delta_base(struct packed_git *p,
1911 struct pack_window **w_curs,
1912 off_t *curpos,
1913 enum object_type type,
1914 off_t delta_obj_offset)
1915{
1916 unsigned char *base_info = use_pack(p, w_curs, *curpos, NULL);
1917 off_t base_offset;
1918
1919 /* use_pack() assured us we have [base_info, base_info + 20)
1920 * as a range that we can look at without walking off the
1921 * end of the mapped window. Its actually the hash size
1922 * that is assured. An OFS_DELTA longer than the hash size
1923 * is stupid, as then a REF_DELTA would be smaller to store.
1924 */
1925 if (type == OBJ_OFS_DELTA) {
1926 unsigned used = 0;
1927 unsigned char c = base_info[used++];
1928 base_offset = c & 127;
1929 while (c & 128) {
1930 base_offset += 1;
1931 if (!base_offset || MSB(base_offset, 7))
1932 return 0; /* overflow */
1933 c = base_info[used++];
1934 base_offset = (base_offset << 7) + (c & 127);
1935 }
1936 base_offset = delta_obj_offset - base_offset;
1937 if (base_offset <= 0 || base_offset >= delta_obj_offset)
1938 return 0; /* out of bound */
1939 *curpos += used;
1940 } else if (type == OBJ_REF_DELTA) {
1941 /* The base entry _must_ be in the same pack */
1942 base_offset = find_pack_entry_one(base_info, p);
1943 *curpos += 20;
1944 } else
1945 die("I am totally screwed");
1946 return base_offset;
1947}
1948
1949/*
1950 * Like get_delta_base above, but we return the sha1 instead of the pack
1951 * offset. This means it is cheaper for REF deltas (we do not have to do
1952 * the final object lookup), but more expensive for OFS deltas (we
1953 * have to load the revidx to convert the offset back into a sha1).
1954 */
1955static const unsigned char *get_delta_base_sha1(struct packed_git *p,
1956 struct pack_window **w_curs,
1957 off_t curpos,
1958 enum object_type type,
1959 off_t delta_obj_offset)
1960{
1961 if (type == OBJ_REF_DELTA) {
1962 unsigned char *base = use_pack(p, w_curs, curpos, NULL);
1963 return base;
1964 } else if (type == OBJ_OFS_DELTA) {
1965 struct revindex_entry *revidx;
1966 off_t base_offset = get_delta_base(p, w_curs, &curpos,
1967 type, delta_obj_offset);
1968
1969 if (!base_offset)
1970 return NULL;
1971
1972 revidx = find_pack_revindex(p, base_offset);
1973 if (!revidx)
1974 return NULL;
1975
1976 return nth_packed_object_sha1(p, revidx->nr);
1977 } else
1978 return NULL;
1979}
1980
1981int unpack_object_header(struct packed_git *p,
1982 struct pack_window **w_curs,
1983 off_t *curpos,
1984 unsigned long *sizep)
1985{
1986 unsigned char *base;
1987 unsigned long left;
1988 unsigned long used;
1989 enum object_type type;
1990
1991 /* use_pack() assures us we have [base, base + 20) available
1992 * as a range that we can look at. (Its actually the hash
1993 * size that is assured.) With our object header encoding
1994 * the maximum deflated object size is 2^137, which is just
1995 * insane, so we know won't exceed what we have been given.
1996 */
1997 base = use_pack(p, w_curs, *curpos, &left);
1998 used = unpack_object_header_buffer(base, left, &type, sizep);
1999 if (!used) {
2000 type = OBJ_BAD;
2001 } else
2002 *curpos += used;
2003
2004 return type;
2005}
2006
2007static int retry_bad_packed_offset(struct packed_git *p, off_t obj_offset)
2008{
2009 int type;
2010 struct revindex_entry *revidx;
2011 const unsigned char *sha1;
2012 revidx = find_pack_revindex(p, obj_offset);
2013 if (!revidx)
2014 return OBJ_BAD;
2015 sha1 = nth_packed_object_sha1(p, revidx->nr);
2016 mark_bad_packed_object(p, sha1);
2017 type = sha1_object_info(sha1, NULL);
2018 if (type <= OBJ_NONE)
2019 return OBJ_BAD;
2020 return type;
2021}
2022
2023#define POI_STACK_PREALLOC 64
2024
2025static enum object_type packed_to_object_type(struct packed_git *p,
2026 off_t obj_offset,
2027 enum object_type type,
2028 struct pack_window **w_curs,
2029 off_t curpos)
2030{
2031 off_t small_poi_stack[POI_STACK_PREALLOC];
2032 off_t *poi_stack = small_poi_stack;
2033 int poi_stack_nr = 0, poi_stack_alloc = POI_STACK_PREALLOC;
2034
2035 while (type == OBJ_OFS_DELTA || type == OBJ_REF_DELTA) {
2036 off_t base_offset;
2037 unsigned long size;
2038 /* Push the object we're going to leave behind */
2039 if (poi_stack_nr >= poi_stack_alloc && poi_stack == small_poi_stack) {
2040 poi_stack_alloc = alloc_nr(poi_stack_nr);
2041 ALLOC_ARRAY(poi_stack, poi_stack_alloc);
2042 memcpy(poi_stack, small_poi_stack, sizeof(off_t)*poi_stack_nr);
2043 } else {
2044 ALLOC_GROW(poi_stack, poi_stack_nr+1, poi_stack_alloc);
2045 }
2046 poi_stack[poi_stack_nr++] = obj_offset;
2047 /* If parsing the base offset fails, just unwind */
2048 base_offset = get_delta_base(p, w_curs, &curpos, type, obj_offset);
2049 if (!base_offset)
2050 goto unwind;
2051 curpos = obj_offset = base_offset;
2052 type = unpack_object_header(p, w_curs, &curpos, &size);
2053 if (type <= OBJ_NONE) {
2054 /* If getting the base itself fails, we first
2055 * retry the base, otherwise unwind */
2056 type = retry_bad_packed_offset(p, base_offset);
2057 if (type > OBJ_NONE)
2058 goto out;
2059 goto unwind;
2060 }
2061 }
2062
2063 switch (type) {
2064 case OBJ_BAD:
2065 case OBJ_COMMIT:
2066 case OBJ_TREE:
2067 case OBJ_BLOB:
2068 case OBJ_TAG:
2069 break;
2070 default:
2071 error("unknown object type %i at offset %"PRIuMAX" in %s",
2072 type, (uintmax_t)obj_offset, p->pack_name);
2073 type = OBJ_BAD;
2074 }
2075
2076out:
2077 if (poi_stack != small_poi_stack)
2078 free(poi_stack);
2079 return type;
2080
2081unwind:
2082 while (poi_stack_nr) {
2083 obj_offset = poi_stack[--poi_stack_nr];
2084 type = retry_bad_packed_offset(p, obj_offset);
2085 if (type > OBJ_NONE)
2086 goto out;
2087 }
2088 type = OBJ_BAD;
2089 goto out;
2090}
2091
2092int packed_object_info(struct packed_git *p, off_t obj_offset,
2093 struct object_info *oi)
2094{
2095 struct pack_window *w_curs = NULL;
2096 unsigned long size;
2097 off_t curpos = obj_offset;
2098 enum object_type type;
2099
2100 /*
2101 * We always get the representation type, but only convert it to
2102 * a "real" type later if the caller is interested.
2103 */
2104 type = unpack_object_header(p, &w_curs, &curpos, &size);
2105
2106 if (oi->sizep) {
2107 if (type == OBJ_OFS_DELTA || type == OBJ_REF_DELTA) {
2108 off_t tmp_pos = curpos;
2109 off_t base_offset = get_delta_base(p, &w_curs, &tmp_pos,
2110 type, obj_offset);
2111 if (!base_offset) {
2112 type = OBJ_BAD;
2113 goto out;
2114 }
2115 *oi->sizep = get_size_from_delta(p, &w_curs, tmp_pos);
2116 if (*oi->sizep == 0) {
2117 type = OBJ_BAD;
2118 goto out;
2119 }
2120 } else {
2121 *oi->sizep = size;
2122 }
2123 }
2124
2125 if (oi->disk_sizep) {
2126 struct revindex_entry *revidx = find_pack_revindex(p, obj_offset);
2127 *oi->disk_sizep = revidx[1].offset - obj_offset;
2128 }
2129
2130 if (oi->typep) {
2131 *oi->typep = packed_to_object_type(p, obj_offset, type, &w_curs, curpos);
2132 if (*oi->typep < 0) {
2133 type = OBJ_BAD;
2134 goto out;
2135 }
2136 }
2137
2138 if (oi->delta_base_sha1) {
2139 if (type == OBJ_OFS_DELTA || type == OBJ_REF_DELTA) {
2140 const unsigned char *base;
2141
2142 base = get_delta_base_sha1(p, &w_curs, curpos,
2143 type, obj_offset);
2144 if (!base) {
2145 type = OBJ_BAD;
2146 goto out;
2147 }
2148
2149 hashcpy(oi->delta_base_sha1, base);
2150 } else
2151 hashclr(oi->delta_base_sha1);
2152 }
2153
2154out:
2155 unuse_pack(&w_curs);
2156 return type;
2157}
2158
2159static void *unpack_compressed_entry(struct packed_git *p,
2160 struct pack_window **w_curs,
2161 off_t curpos,
2162 unsigned long size)
2163{
2164 int st;
2165 git_zstream stream;
2166 unsigned char *buffer, *in;
2167
2168 buffer = xmallocz_gently(size);
2169 if (!buffer)
2170 return NULL;
2171 memset(&stream, 0, sizeof(stream));
2172 stream.next_out = buffer;
2173 stream.avail_out = size + 1;
2174
2175 git_inflate_init(&stream);
2176 do {
2177 in = use_pack(p, w_curs, curpos, &stream.avail_in);
2178 stream.next_in = in;
2179 st = git_inflate(&stream, Z_FINISH);
2180 if (!stream.avail_out)
2181 break; /* the payload is larger than it should be */
2182 curpos += stream.next_in - in;
2183 } while (st == Z_OK || st == Z_BUF_ERROR);
2184 git_inflate_end(&stream);
2185 if ((st != Z_STREAM_END) || stream.total_out != size) {
2186 free(buffer);
2187 return NULL;
2188 }
2189
2190 return buffer;
2191}
2192
2193static struct hashmap delta_base_cache;
2194static size_t delta_base_cached;
2195
2196static LIST_HEAD(delta_base_cache_lru);
2197
2198struct delta_base_cache_key {
2199 struct packed_git *p;
2200 off_t base_offset;
2201};
2202
2203struct delta_base_cache_entry {
2204 struct hashmap hash;
2205 struct delta_base_cache_key key;
2206 struct list_head lru;
2207 void *data;
2208 unsigned long size;
2209 enum object_type type;
2210};
2211
2212static unsigned int pack_entry_hash(struct packed_git *p, off_t base_offset)
2213{
2214 unsigned int hash;
2215
2216 hash = (unsigned int)(intptr_t)p + (unsigned int)base_offset;
2217 hash += (hash >> 8) + (hash >> 16);
2218 return hash;
2219}
2220
2221static struct delta_base_cache_entry *
2222get_delta_base_cache_entry(struct packed_git *p, off_t base_offset)
2223{
2224 struct hashmap_entry entry;
2225 struct delta_base_cache_key key;
2226
2227 if (!delta_base_cache.cmpfn)
2228 return NULL;
2229
2230 hashmap_entry_init(&entry, pack_entry_hash(p, base_offset));
2231 key.p = p;
2232 key.base_offset = base_offset;
2233 return hashmap_get(&delta_base_cache, &entry, &key);
2234}
2235
2236static int delta_base_cache_key_eq(const struct delta_base_cache_key *a,
2237 const struct delta_base_cache_key *b)
2238{
2239 return a->p == b->p && a->base_offset == b->base_offset;
2240}
2241
2242static int delta_base_cache_hash_cmp(const void *va, const void *vb,
2243 const void *vkey)
2244{
2245 const struct delta_base_cache_entry *a = va, *b = vb;
2246 const struct delta_base_cache_key *key = vkey;
2247 if (key)
2248 return !delta_base_cache_key_eq(&a->key, key);
2249 else
2250 return !delta_base_cache_key_eq(&a->key, &b->key);
2251}
2252
2253static int in_delta_base_cache(struct packed_git *p, off_t base_offset)
2254{
2255 return !!get_delta_base_cache_entry(p, base_offset);
2256}
2257
2258/*
2259 * Remove the entry from the cache, but do _not_ free the associated
2260 * entry data. The caller takes ownership of the "data" buffer, and
2261 * should copy out any fields it wants before detaching.
2262 */
2263static void detach_delta_base_cache_entry(struct delta_base_cache_entry *ent)
2264{
2265 hashmap_remove(&delta_base_cache, ent, &ent->key);
2266 list_del(&ent->lru);
2267 delta_base_cached -= ent->size;
2268 free(ent);
2269}
2270
2271static void *cache_or_unpack_entry(struct packed_git *p, off_t base_offset,
2272 unsigned long *base_size, enum object_type *type)
2273{
2274 struct delta_base_cache_entry *ent;
2275
2276 ent = get_delta_base_cache_entry(p, base_offset);
2277 if (!ent)
2278 return unpack_entry(p, base_offset, type, base_size);
2279
2280 *type = ent->type;
2281 *base_size = ent->size;
2282 return xmemdupz(ent->data, ent->size);
2283}
2284
2285static inline void release_delta_base_cache(struct delta_base_cache_entry *ent)
2286{
2287 free(ent->data);
2288 detach_delta_base_cache_entry(ent);
2289}
2290
2291void clear_delta_base_cache(void)
2292{
2293 struct hashmap_iter iter;
2294 struct delta_base_cache_entry *entry;
2295 for (entry = hashmap_iter_first(&delta_base_cache, &iter);
2296 entry;
2297 entry = hashmap_iter_next(&iter)) {
2298 release_delta_base_cache(entry);
2299 }
2300}
2301
2302static void add_delta_base_cache(struct packed_git *p, off_t base_offset,
2303 void *base, unsigned long base_size, enum object_type type)
2304{
2305 struct delta_base_cache_entry *ent = xmalloc(sizeof(*ent));
2306 struct list_head *lru, *tmp;
2307
2308 delta_base_cached += base_size;
2309
2310 list_for_each_safe(lru, tmp, &delta_base_cache_lru) {
2311 struct delta_base_cache_entry *f =
2312 list_entry(lru, struct delta_base_cache_entry, lru);
2313 if (delta_base_cached <= delta_base_cache_limit)
2314 break;
2315 release_delta_base_cache(f);
2316 }
2317
2318 ent->key.p = p;
2319 ent->key.base_offset = base_offset;
2320 ent->type = type;
2321 ent->data = base;
2322 ent->size = base_size;
2323 list_add_tail(&ent->lru, &delta_base_cache_lru);
2324
2325 if (!delta_base_cache.cmpfn)
2326 hashmap_init(&delta_base_cache, delta_base_cache_hash_cmp, 0);
2327 hashmap_entry_init(ent, pack_entry_hash(p, base_offset));
2328 hashmap_add(&delta_base_cache, ent);
2329}
2330
2331static void *read_object(const unsigned char *sha1, enum object_type *type,
2332 unsigned long *size);
2333
2334static void write_pack_access_log(struct packed_git *p, off_t obj_offset)
2335{
2336 static struct trace_key pack_access = TRACE_KEY_INIT(PACK_ACCESS);
2337 trace_printf_key(&pack_access, "%s %"PRIuMAX"\n",
2338 p->pack_name, (uintmax_t)obj_offset);
2339}
2340
2341int do_check_packed_object_crc;
2342
2343#define UNPACK_ENTRY_STACK_PREALLOC 64
2344struct unpack_entry_stack_ent {
2345 off_t obj_offset;
2346 off_t curpos;
2347 unsigned long size;
2348};
2349
2350void *unpack_entry(struct packed_git *p, off_t obj_offset,
2351 enum object_type *final_type, unsigned long *final_size)
2352{
2353 struct pack_window *w_curs = NULL;
2354 off_t curpos = obj_offset;
2355 void *data = NULL;
2356 unsigned long size;
2357 enum object_type type;
2358 struct unpack_entry_stack_ent small_delta_stack[UNPACK_ENTRY_STACK_PREALLOC];
2359 struct unpack_entry_stack_ent *delta_stack = small_delta_stack;
2360 int delta_stack_nr = 0, delta_stack_alloc = UNPACK_ENTRY_STACK_PREALLOC;
2361 int base_from_cache = 0;
2362
2363 write_pack_access_log(p, obj_offset);
2364
2365 /* PHASE 1: drill down to the innermost base object */
2366 for (;;) {
2367 off_t base_offset;
2368 int i;
2369 struct delta_base_cache_entry *ent;
2370
2371 ent = get_delta_base_cache_entry(p, curpos);
2372 if (ent) {
2373 type = ent->type;
2374 data = ent->data;
2375 size = ent->size;
2376 detach_delta_base_cache_entry(ent);
2377 base_from_cache = 1;
2378 break;
2379 }
2380
2381 if (do_check_packed_object_crc && p->index_version > 1) {
2382 struct revindex_entry *revidx = find_pack_revindex(p, obj_offset);
2383 off_t len = revidx[1].offset - obj_offset;
2384 if (check_pack_crc(p, &w_curs, obj_offset, len, revidx->nr)) {
2385 const unsigned char *sha1 =
2386 nth_packed_object_sha1(p, revidx->nr);
2387 error("bad packed object CRC for %s",
2388 sha1_to_hex(sha1));
2389 mark_bad_packed_object(p, sha1);
2390 unuse_pack(&w_curs);
2391 return NULL;
2392 }
2393 }
2394
2395 type = unpack_object_header(p, &w_curs, &curpos, &size);
2396 if (type != OBJ_OFS_DELTA && type != OBJ_REF_DELTA)
2397 break;
2398
2399 base_offset = get_delta_base(p, &w_curs, &curpos, type, obj_offset);
2400 if (!base_offset) {
2401 error("failed to validate delta base reference "
2402 "at offset %"PRIuMAX" from %s",
2403 (uintmax_t)curpos, p->pack_name);
2404 /* bail to phase 2, in hopes of recovery */
2405 data = NULL;
2406 break;
2407 }
2408
2409 /* push object, proceed to base */
2410 if (delta_stack_nr >= delta_stack_alloc
2411 && delta_stack == small_delta_stack) {
2412 delta_stack_alloc = alloc_nr(delta_stack_nr);
2413 ALLOC_ARRAY(delta_stack, delta_stack_alloc);
2414 memcpy(delta_stack, small_delta_stack,
2415 sizeof(*delta_stack)*delta_stack_nr);
2416 } else {
2417 ALLOC_GROW(delta_stack, delta_stack_nr+1, delta_stack_alloc);
2418 }
2419 i = delta_stack_nr++;
2420 delta_stack[i].obj_offset = obj_offset;
2421 delta_stack[i].curpos = curpos;
2422 delta_stack[i].size = size;
2423
2424 curpos = obj_offset = base_offset;
2425 }
2426
2427 /* PHASE 2: handle the base */
2428 switch (type) {
2429 case OBJ_OFS_DELTA:
2430 case OBJ_REF_DELTA:
2431 if (data)
2432 die("BUG: unpack_entry: left loop at a valid delta");
2433 break;
2434 case OBJ_COMMIT:
2435 case OBJ_TREE:
2436 case OBJ_BLOB:
2437 case OBJ_TAG:
2438 if (!base_from_cache)
2439 data = unpack_compressed_entry(p, &w_curs, curpos, size);
2440 break;
2441 default:
2442 data = NULL;
2443 error("unknown object type %i at offset %"PRIuMAX" in %s",
2444 type, (uintmax_t)obj_offset, p->pack_name);
2445 }
2446
2447 /* PHASE 3: apply deltas in order */
2448
2449 /* invariants:
2450 * 'data' holds the base data, or NULL if there was corruption
2451 */
2452 while (delta_stack_nr) {
2453 void *delta_data;
2454 void *base = data;
2455 unsigned long delta_size, base_size = size;
2456 int i;
2457
2458 data = NULL;
2459
2460 if (base)
2461 add_delta_base_cache(p, obj_offset, base, base_size, type);
2462
2463 if (!base) {
2464 /*
2465 * We're probably in deep shit, but let's try to fetch
2466 * the required base anyway from another pack or loose.
2467 * This is costly but should happen only in the presence
2468 * of a corrupted pack, and is better than failing outright.
2469 */
2470 struct revindex_entry *revidx;
2471 const unsigned char *base_sha1;
2472 revidx = find_pack_revindex(p, obj_offset);
2473 if (revidx) {
2474 base_sha1 = nth_packed_object_sha1(p, revidx->nr);
2475 error("failed to read delta base object %s"
2476 " at offset %"PRIuMAX" from %s",
2477 sha1_to_hex(base_sha1), (uintmax_t)obj_offset,
2478 p->pack_name);
2479 mark_bad_packed_object(p, base_sha1);
2480 base = read_object(base_sha1, &type, &base_size);
2481 }
2482 }
2483
2484 i = --delta_stack_nr;
2485 obj_offset = delta_stack[i].obj_offset;
2486 curpos = delta_stack[i].curpos;
2487 delta_size = delta_stack[i].size;
2488
2489 if (!base)
2490 continue;
2491
2492 delta_data = unpack_compressed_entry(p, &w_curs, curpos, delta_size);
2493
2494 if (!delta_data) {
2495 error("failed to unpack compressed delta "
2496 "at offset %"PRIuMAX" from %s",
2497 (uintmax_t)curpos, p->pack_name);
2498 data = NULL;
2499 continue;
2500 }
2501
2502 data = patch_delta(base, base_size,
2503 delta_data, delta_size,
2504 &size);
2505
2506 /*
2507 * We could not apply the delta; warn the user, but keep going.
2508 * Our failure will be noticed either in the next iteration of
2509 * the loop, or if this is the final delta, in the caller when
2510 * we return NULL. Those code paths will take care of making
2511 * a more explicit warning and retrying with another copy of
2512 * the object.
2513 */
2514 if (!data)
2515 error("failed to apply delta");
2516
2517 free(delta_data);
2518 }
2519
2520 *final_type = type;
2521 *final_size = size;
2522
2523 unuse_pack(&w_curs);
2524
2525 if (delta_stack != small_delta_stack)
2526 free(delta_stack);
2527
2528 return data;
2529}
2530
2531const unsigned char *nth_packed_object_sha1(struct packed_git *p,
2532 uint32_t n)
2533{
2534 const unsigned char *index = p->index_data;
2535 if (!index) {
2536 if (open_pack_index(p))
2537 return NULL;
2538 index = p->index_data;
2539 }
2540 if (n >= p->num_objects)
2541 return NULL;
2542 index += 4 * 256;
2543 if (p->index_version == 1) {
2544 return index + 24 * n + 4;
2545 } else {
2546 index += 8;
2547 return index + 20 * n;
2548 }
2549}
2550
2551void check_pack_index_ptr(const struct packed_git *p, const void *vptr)
2552{
2553 const unsigned char *ptr = vptr;
2554 const unsigned char *start = p->index_data;
2555 const unsigned char *end = start + p->index_size;
2556 if (ptr < start)
2557 die(_("offset before start of pack index for %s (corrupt index?)"),
2558 p->pack_name);
2559 /* No need to check for underflow; .idx files must be at least 8 bytes */
2560 if (ptr >= end - 8)
2561 die(_("offset beyond end of pack index for %s (truncated index?)"),
2562 p->pack_name);
2563}
2564
2565off_t nth_packed_object_offset(const struct packed_git *p, uint32_t n)
2566{
2567 const unsigned char *index = p->index_data;
2568 index += 4 * 256;
2569 if (p->index_version == 1) {
2570 return ntohl(*((uint32_t *)(index + 24 * n)));
2571 } else {
2572 uint32_t off;
2573 index += 8 + p->num_objects * (20 + 4);
2574 off = ntohl(*((uint32_t *)(index + 4 * n)));
2575 if (!(off & 0x80000000))
2576 return off;
2577 index += p->num_objects * 4 + (off & 0x7fffffff) * 8;
2578 check_pack_index_ptr(p, index);
2579 return (((uint64_t)ntohl(*((uint32_t *)(index + 0)))) << 32) |
2580 ntohl(*((uint32_t *)(index + 4)));
2581 }
2582}
2583
2584off_t find_pack_entry_one(const unsigned char *sha1,
2585 struct packed_git *p)
2586{
2587 const uint32_t *level1_ofs = p->index_data;
2588 const unsigned char *index = p->index_data;
2589 unsigned hi, lo, stride;
2590 static int use_lookup = -1;
2591 static int debug_lookup = -1;
2592
2593 if (debug_lookup < 0)
2594 debug_lookup = !!getenv("GIT_DEBUG_LOOKUP");
2595
2596 if (!index) {
2597 if (open_pack_index(p))
2598 return 0;
2599 level1_ofs = p->index_data;
2600 index = p->index_data;
2601 }
2602 if (p->index_version > 1) {
2603 level1_ofs += 2;
2604 index += 8;
2605 }
2606 index += 4 * 256;
2607 hi = ntohl(level1_ofs[*sha1]);
2608 lo = ((*sha1 == 0x0) ? 0 : ntohl(level1_ofs[*sha1 - 1]));
2609 if (p->index_version > 1) {
2610 stride = 20;
2611 } else {
2612 stride = 24;
2613 index += 4;
2614 }
2615
2616 if (debug_lookup)
2617 printf("%02x%02x%02x... lo %u hi %u nr %"PRIu32"\n",
2618 sha1[0], sha1[1], sha1[2], lo, hi, p->num_objects);
2619
2620 if (use_lookup < 0)
2621 use_lookup = !!getenv("GIT_USE_LOOKUP");
2622 if (use_lookup) {
2623 int pos = sha1_entry_pos(index, stride, 0,
2624 lo, hi, p->num_objects, sha1);
2625 if (pos < 0)
2626 return 0;
2627 return nth_packed_object_offset(p, pos);
2628 }
2629
2630 do {
2631 unsigned mi = (lo + hi) / 2;
2632 int cmp = hashcmp(index + mi * stride, sha1);
2633
2634 if (debug_lookup)
2635 printf("lo %u hi %u rg %u mi %u\n",
2636 lo, hi, hi - lo, mi);
2637 if (!cmp)
2638 return nth_packed_object_offset(p, mi);
2639 if (cmp > 0)
2640 hi = mi;
2641 else
2642 lo = mi+1;
2643 } while (lo < hi);
2644 return 0;
2645}
2646
2647int is_pack_valid(struct packed_git *p)
2648{
2649 /* An already open pack is known to be valid. */
2650 if (p->pack_fd != -1)
2651 return 1;
2652
2653 /* If the pack has one window completely covering the
2654 * file size, the pack is known to be valid even if
2655 * the descriptor is not currently open.
2656 */
2657 if (p->windows) {
2658 struct pack_window *w = p->windows;
2659
2660 if (!w->offset && w->len == p->pack_size)
2661 return 1;
2662 }
2663
2664 /* Force the pack to open to prove its valid. */
2665 return !open_packed_git(p);
2666}
2667
2668static int fill_pack_entry(const unsigned char *sha1,
2669 struct pack_entry *e,
2670 struct packed_git *p)
2671{
2672 off_t offset;
2673
2674 if (p->num_bad_objects) {
2675 unsigned i;
2676 for (i = 0; i < p->num_bad_objects; i++)
2677 if (!hashcmp(sha1, p->bad_object_sha1 + 20 * i))
2678 return 0;
2679 }
2680
2681 offset = find_pack_entry_one(sha1, p);
2682 if (!offset)
2683 return 0;
2684
2685 /*
2686 * We are about to tell the caller where they can locate the
2687 * requested object. We better make sure the packfile is
2688 * still here and can be accessed before supplying that
2689 * answer, as it may have been deleted since the index was
2690 * loaded!
2691 */
2692 if (!is_pack_valid(p))
2693 return 0;
2694 e->offset = offset;
2695 e->p = p;
2696 hashcpy(e->sha1, sha1);
2697 return 1;
2698}
2699
2700/*
2701 * Iff a pack file contains the object named by sha1, return true and
2702 * store its location to e.
2703 */
2704static int find_pack_entry(const unsigned char *sha1, struct pack_entry *e)
2705{
2706 struct mru_entry *p;
2707
2708 prepare_packed_git();
2709 if (!packed_git)
2710 return 0;
2711
2712 for (p = packed_git_mru->head; p; p = p->next) {
2713 if (fill_pack_entry(sha1, e, p->item)) {
2714 mru_mark(packed_git_mru, p);
2715 return 1;
2716 }
2717 }
2718 return 0;
2719}
2720
2721struct packed_git *find_sha1_pack(const unsigned char *sha1,
2722 struct packed_git *packs)
2723{
2724 struct packed_git *p;
2725
2726 for (p = packs; p; p = p->next) {
2727 if (find_pack_entry_one(sha1, p))
2728 return p;
2729 }
2730 return NULL;
2731
2732}
2733
2734static int sha1_loose_object_info(const unsigned char *sha1,
2735 struct object_info *oi,
2736 int flags)
2737{
2738 int status = 0;
2739 unsigned long mapsize;
2740 void *map;
2741 git_zstream stream;
2742 char hdr[32];
2743 struct strbuf hdrbuf = STRBUF_INIT;
2744
2745 if (oi->delta_base_sha1)
2746 hashclr(oi->delta_base_sha1);
2747
2748 /*
2749 * If we don't care about type or size, then we don't
2750 * need to look inside the object at all. Note that we
2751 * do not optimize out the stat call, even if the
2752 * caller doesn't care about the disk-size, since our
2753 * return value implicitly indicates whether the
2754 * object even exists.
2755 */
2756 if (!oi->typep && !oi->typename && !oi->sizep) {
2757 struct stat st;
2758 if (stat_sha1_file(sha1, &st) < 0)
2759 return -1;
2760 if (oi->disk_sizep)
2761 *oi->disk_sizep = st.st_size;
2762 return 0;
2763 }
2764
2765 map = map_sha1_file(sha1, &mapsize);
2766 if (!map)
2767 return -1;
2768 if (oi->disk_sizep)
2769 *oi->disk_sizep = mapsize;
2770 if ((flags & LOOKUP_UNKNOWN_OBJECT)) {
2771 if (unpack_sha1_header_to_strbuf(&stream, map, mapsize, hdr, sizeof(hdr), &hdrbuf) < 0)
2772 status = error("unable to unpack %s header with --allow-unknown-type",
2773 sha1_to_hex(sha1));
2774 } else if (unpack_sha1_header(&stream, map, mapsize, hdr, sizeof(hdr)) < 0)
2775 status = error("unable to unpack %s header",
2776 sha1_to_hex(sha1));
2777 if (status < 0)
2778 ; /* Do nothing */
2779 else if (hdrbuf.len) {
2780 if ((status = parse_sha1_header_extended(hdrbuf.buf, oi, flags)) < 0)
2781 status = error("unable to parse %s header with --allow-unknown-type",
2782 sha1_to_hex(sha1));
2783 } else if ((status = parse_sha1_header_extended(hdr, oi, flags)) < 0)
2784 status = error("unable to parse %s header", sha1_to_hex(sha1));
2785 git_inflate_end(&stream);
2786 munmap(map, mapsize);
2787 if (status && oi->typep)
2788 *oi->typep = status;
2789 strbuf_release(&hdrbuf);
2790 return 0;
2791}
2792
2793int sha1_object_info_extended(const unsigned char *sha1, struct object_info *oi, unsigned flags)
2794{
2795 struct cached_object *co;
2796 struct pack_entry e;
2797 int rtype;
2798 enum object_type real_type;
2799 const unsigned char *real = lookup_replace_object_extended(sha1, flags);
2800
2801 co = find_cached_object(real);
2802 if (co) {
2803 if (oi->typep)
2804 *(oi->typep) = co->type;
2805 if (oi->sizep)
2806 *(oi->sizep) = co->size;
2807 if (oi->disk_sizep)
2808 *(oi->disk_sizep) = 0;
2809 if (oi->delta_base_sha1)
2810 hashclr(oi->delta_base_sha1);
2811 if (oi->typename)
2812 strbuf_addstr(oi->typename, typename(co->type));
2813 oi->whence = OI_CACHED;
2814 return 0;
2815 }
2816
2817 if (!find_pack_entry(real, &e)) {
2818 /* Most likely it's a loose object. */
2819 if (!sha1_loose_object_info(real, oi, flags)) {
2820 oi->whence = OI_LOOSE;
2821 return 0;
2822 }
2823
2824 /* Not a loose object; someone else may have just packed it. */
2825 reprepare_packed_git();
2826 if (!find_pack_entry(real, &e))
2827 return -1;
2828 }
2829
2830 /*
2831 * packed_object_info() does not follow the delta chain to
2832 * find out the real type, unless it is given oi->typep.
2833 */
2834 if (oi->typename && !oi->typep)
2835 oi->typep = &real_type;
2836
2837 rtype = packed_object_info(e.p, e.offset, oi);
2838 if (rtype < 0) {
2839 mark_bad_packed_object(e.p, real);
2840 if (oi->typep == &real_type)
2841 oi->typep = NULL;
2842 return sha1_object_info_extended(real, oi, 0);
2843 } else if (in_delta_base_cache(e.p, e.offset)) {
2844 oi->whence = OI_DBCACHED;
2845 } else {
2846 oi->whence = OI_PACKED;
2847 oi->u.packed.offset = e.offset;
2848 oi->u.packed.pack = e.p;
2849 oi->u.packed.is_delta = (rtype == OBJ_REF_DELTA ||
2850 rtype == OBJ_OFS_DELTA);
2851 }
2852 if (oi->typename)
2853 strbuf_addstr(oi->typename, typename(*oi->typep));
2854 if (oi->typep == &real_type)
2855 oi->typep = NULL;
2856
2857 return 0;
2858}
2859
2860/* returns enum object_type or negative */
2861int sha1_object_info(const unsigned char *sha1, unsigned long *sizep)
2862{
2863 enum object_type type;
2864 struct object_info oi = OBJECT_INFO_INIT;
2865
2866 oi.typep = &type;
2867 oi.sizep = sizep;
2868 if (sha1_object_info_extended(sha1, &oi, LOOKUP_REPLACE_OBJECT) < 0)
2869 return -1;
2870 return type;
2871}
2872
2873static void *read_packed_sha1(const unsigned char *sha1,
2874 enum object_type *type, unsigned long *size)
2875{
2876 struct pack_entry e;
2877 void *data;
2878
2879 if (!find_pack_entry(sha1, &e))
2880 return NULL;
2881 data = cache_or_unpack_entry(e.p, e.offset, size, type);
2882 if (!data) {
2883 /*
2884 * We're probably in deep shit, but let's try to fetch
2885 * the required object anyway from another pack or loose.
2886 * This should happen only in the presence of a corrupted
2887 * pack, and is better than failing outright.
2888 */
2889 error("failed to read object %s at offset %"PRIuMAX" from %s",
2890 sha1_to_hex(sha1), (uintmax_t)e.offset, e.p->pack_name);
2891 mark_bad_packed_object(e.p, sha1);
2892 data = read_object(sha1, type, size);
2893 }
2894 return data;
2895}
2896
2897int pretend_sha1_file(void *buf, unsigned long len, enum object_type type,
2898 unsigned char *sha1)
2899{
2900 struct cached_object *co;
2901
2902 hash_sha1_file(buf, len, typename(type), sha1);
2903 if (has_sha1_file(sha1) || find_cached_object(sha1))
2904 return 0;
2905 ALLOC_GROW(cached_objects, cached_object_nr + 1, cached_object_alloc);
2906 co = &cached_objects[cached_object_nr++];
2907 co->size = len;
2908 co->type = type;
2909 co->buf = xmalloc(len);
2910 memcpy(co->buf, buf, len);
2911 hashcpy(co->sha1, sha1);
2912 return 0;
2913}
2914
2915static void *read_object(const unsigned char *sha1, enum object_type *type,
2916 unsigned long *size)
2917{
2918 unsigned long mapsize;
2919 void *map, *buf;
2920 struct cached_object *co;
2921
2922 co = find_cached_object(sha1);
2923 if (co) {
2924 *type = co->type;
2925 *size = co->size;
2926 return xmemdupz(co->buf, co->size);
2927 }
2928
2929 buf = read_packed_sha1(sha1, type, size);
2930 if (buf)
2931 return buf;
2932 map = map_sha1_file(sha1, &mapsize);
2933 if (map) {
2934 buf = unpack_sha1_file(map, mapsize, type, size, sha1);
2935 munmap(map, mapsize);
2936 return buf;
2937 }
2938 reprepare_packed_git();
2939 return read_packed_sha1(sha1, type, size);
2940}
2941
2942/*
2943 * This function dies on corrupt objects; the callers who want to
2944 * deal with them should arrange to call read_object() and give error
2945 * messages themselves.
2946 */
2947void *read_sha1_file_extended(const unsigned char *sha1,
2948 enum object_type *type,
2949 unsigned long *size,
2950 unsigned flag)
2951{
2952 void *data;
2953 const struct packed_git *p;
2954 const unsigned char *repl = lookup_replace_object_extended(sha1, flag);
2955
2956 errno = 0;
2957 data = read_object(repl, type, size);
2958 if (data)
2959 return data;
2960
2961 if (errno && errno != ENOENT)
2962 die_errno("failed to read object %s", sha1_to_hex(sha1));
2963
2964 /* die if we replaced an object with one that does not exist */
2965 if (repl != sha1)
2966 die("replacement %s not found for %s",
2967 sha1_to_hex(repl), sha1_to_hex(sha1));
2968
2969 if (has_loose_object(repl)) {
2970 const char *path = sha1_file_name(sha1);
2971
2972 die("loose object %s (stored in %s) is corrupt",
2973 sha1_to_hex(repl), path);
2974 }
2975
2976 if ((p = has_packed_and_bad(repl)) != NULL)
2977 die("packed object %s (stored in %s) is corrupt",
2978 sha1_to_hex(repl), p->pack_name);
2979
2980 return NULL;
2981}
2982
2983void *read_object_with_reference(const unsigned char *sha1,
2984 const char *required_type_name,
2985 unsigned long *size,
2986 unsigned char *actual_sha1_return)
2987{
2988 enum object_type type, required_type;
2989 void *buffer;
2990 unsigned long isize;
2991 unsigned char actual_sha1[20];
2992
2993 required_type = type_from_string(required_type_name);
2994 hashcpy(actual_sha1, sha1);
2995 while (1) {
2996 int ref_length = -1;
2997 const char *ref_type = NULL;
2998
2999 buffer = read_sha1_file(actual_sha1, &type, &isize);
3000 if (!buffer)
3001 return NULL;
3002 if (type == required_type) {
3003 *size = isize;
3004 if (actual_sha1_return)
3005 hashcpy(actual_sha1_return, actual_sha1);
3006 return buffer;
3007 }
3008 /* Handle references */
3009 else if (type == OBJ_COMMIT)
3010 ref_type = "tree ";
3011 else if (type == OBJ_TAG)
3012 ref_type = "object ";
3013 else {
3014 free(buffer);
3015 return NULL;
3016 }
3017 ref_length = strlen(ref_type);
3018
3019 if (ref_length + 40 > isize ||
3020 memcmp(buffer, ref_type, ref_length) ||
3021 get_sha1_hex((char *) buffer + ref_length, actual_sha1)) {
3022 free(buffer);
3023 return NULL;
3024 }
3025 free(buffer);
3026 /* Now we have the ID of the referred-to object in
3027 * actual_sha1. Check again. */
3028 }
3029}
3030
3031static void write_sha1_file_prepare(const void *buf, unsigned long len,
3032 const char *type, unsigned char *sha1,
3033 char *hdr, int *hdrlen)
3034{
3035 git_SHA_CTX c;
3036
3037 /* Generate the header */
3038 *hdrlen = xsnprintf(hdr, *hdrlen, "%s %lu", type, len)+1;
3039
3040 /* Sha1.. */
3041 git_SHA1_Init(&c);
3042 git_SHA1_Update(&c, hdr, *hdrlen);
3043 git_SHA1_Update(&c, buf, len);
3044 git_SHA1_Final(sha1, &c);
3045}
3046
3047/*
3048 * Move the just written object into its final resting place.
3049 */
3050int finalize_object_file(const char *tmpfile, const char *filename)
3051{
3052 int ret = 0;
3053
3054 if (object_creation_mode == OBJECT_CREATION_USES_RENAMES)
3055 goto try_rename;
3056 else if (link(tmpfile, filename))
3057 ret = errno;
3058
3059 /*
3060 * Coda hack - coda doesn't like cross-directory links,
3061 * so we fall back to a rename, which will mean that it
3062 * won't be able to check collisions, but that's not a
3063 * big deal.
3064 *
3065 * The same holds for FAT formatted media.
3066 *
3067 * When this succeeds, we just return. We have nothing
3068 * left to unlink.
3069 */
3070 if (ret && ret != EEXIST) {
3071 try_rename:
3072 if (!rename(tmpfile, filename))
3073 goto out;
3074 ret = errno;
3075 }
3076 unlink_or_warn(tmpfile);
3077 if (ret) {
3078 if (ret != EEXIST) {
3079 return error_errno("unable to write sha1 filename %s", filename);
3080 }
3081 /* FIXME!!! Collision check here ? */
3082 }
3083
3084out:
3085 if (adjust_shared_perm(filename))
3086 return error("unable to set permission to '%s'", filename);
3087 return 0;
3088}
3089
3090static int write_buffer(int fd, const void *buf, size_t len)
3091{
3092 if (write_in_full(fd, buf, len) < 0)
3093 return error_errno("file write error");
3094 return 0;
3095}
3096
3097int hash_sha1_file(const void *buf, unsigned long len, const char *type,
3098 unsigned char *sha1)
3099{
3100 char hdr[32];
3101 int hdrlen = sizeof(hdr);
3102 write_sha1_file_prepare(buf, len, type, sha1, hdr, &hdrlen);
3103 return 0;
3104}
3105
3106/* Finalize a file on disk, and close it. */
3107static void close_sha1_file(int fd)
3108{
3109 if (fsync_object_files)
3110 fsync_or_die(fd, "sha1 file");
3111 if (close(fd) != 0)
3112 die_errno("error when closing sha1 file");
3113}
3114
3115/* Size of directory component, including the ending '/' */
3116static inline int directory_size(const char *filename)
3117{
3118 const char *s = strrchr(filename, '/');
3119 if (!s)
3120 return 0;
3121 return s - filename + 1;
3122}
3123
3124/*
3125 * This creates a temporary file in the same directory as the final
3126 * 'filename'
3127 *
3128 * We want to avoid cross-directory filename renames, because those
3129 * can have problems on various filesystems (FAT, NFS, Coda).
3130 */
3131static int create_tmpfile(struct strbuf *tmp, const char *filename)
3132{
3133 int fd, dirlen = directory_size(filename);
3134
3135 strbuf_reset(tmp);
3136 strbuf_add(tmp, filename, dirlen);
3137 strbuf_addstr(tmp, "tmp_obj_XXXXXX");
3138 fd = git_mkstemp_mode(tmp->buf, 0444);
3139 if (fd < 0 && dirlen && errno == ENOENT) {
3140 /*
3141 * Make sure the directory exists; note that the contents
3142 * of the buffer are undefined after mkstemp returns an
3143 * error, so we have to rewrite the whole buffer from
3144 * scratch.
3145 */
3146 strbuf_reset(tmp);
3147 strbuf_add(tmp, filename, dirlen - 1);
3148 if (mkdir(tmp->buf, 0777) && errno != EEXIST)
3149 return -1;
3150 if (adjust_shared_perm(tmp->buf))
3151 return -1;
3152
3153 /* Try again */
3154 strbuf_addstr(tmp, "/tmp_obj_XXXXXX");
3155 fd = git_mkstemp_mode(tmp->buf, 0444);
3156 }
3157 return fd;
3158}
3159
3160static int write_loose_object(const unsigned char *sha1, char *hdr, int hdrlen,
3161 const void *buf, unsigned long len, time_t mtime)
3162{
3163 int fd, ret;
3164 unsigned char compressed[4096];
3165 git_zstream stream;
3166 git_SHA_CTX c;
3167 unsigned char parano_sha1[20];
3168 static struct strbuf tmp_file = STRBUF_INIT;
3169 const char *filename = sha1_file_name(sha1);
3170
3171 fd = create_tmpfile(&tmp_file, filename);
3172 if (fd < 0) {
3173 if (errno == EACCES)
3174 return error("insufficient permission for adding an object to repository database %s", get_object_directory());
3175 else
3176 return error_errno("unable to create temporary file");
3177 }
3178
3179 /* Set it up */
3180 git_deflate_init(&stream, zlib_compression_level);
3181 stream.next_out = compressed;
3182 stream.avail_out = sizeof(compressed);
3183 git_SHA1_Init(&c);
3184
3185 /* First header.. */
3186 stream.next_in = (unsigned char *)hdr;
3187 stream.avail_in = hdrlen;
3188 while (git_deflate(&stream, 0) == Z_OK)
3189 ; /* nothing */
3190 git_SHA1_Update(&c, hdr, hdrlen);
3191
3192 /* Then the data itself.. */
3193 stream.next_in = (void *)buf;
3194 stream.avail_in = len;
3195 do {
3196 unsigned char *in0 = stream.next_in;
3197 ret = git_deflate(&stream, Z_FINISH);
3198 git_SHA1_Update(&c, in0, stream.next_in - in0);
3199 if (write_buffer(fd, compressed, stream.next_out - compressed) < 0)
3200 die("unable to write sha1 file");
3201 stream.next_out = compressed;
3202 stream.avail_out = sizeof(compressed);
3203 } while (ret == Z_OK);
3204
3205 if (ret != Z_STREAM_END)
3206 die("unable to deflate new object %s (%d)", sha1_to_hex(sha1), ret);
3207 ret = git_deflate_end_gently(&stream);
3208 if (ret != Z_OK)
3209 die("deflateEnd on object %s failed (%d)", sha1_to_hex(sha1), ret);
3210 git_SHA1_Final(parano_sha1, &c);
3211 if (hashcmp(sha1, parano_sha1) != 0)
3212 die("confused by unstable object source data for %s", sha1_to_hex(sha1));
3213
3214 close_sha1_file(fd);
3215
3216 if (mtime) {
3217 struct utimbuf utb;
3218 utb.actime = mtime;
3219 utb.modtime = mtime;
3220 if (utime(tmp_file.buf, &utb) < 0)
3221 warning_errno("failed utime() on %s", tmp_file.buf);
3222 }
3223
3224 return finalize_object_file(tmp_file.buf, filename);
3225}
3226
3227static int freshen_loose_object(const unsigned char *sha1)
3228{
3229 return check_and_freshen(sha1, 1);
3230}
3231
3232static int freshen_packed_object(const unsigned char *sha1)
3233{
3234 struct pack_entry e;
3235 if (!find_pack_entry(sha1, &e))
3236 return 0;
3237 if (e.p->freshened)
3238 return 1;
3239 if (!freshen_file(e.p->pack_name))
3240 return 0;
3241 e.p->freshened = 1;
3242 return 1;
3243}
3244
3245int write_sha1_file(const void *buf, unsigned long len, const char *type, unsigned char *sha1)
3246{
3247 char hdr[32];
3248 int hdrlen = sizeof(hdr);
3249
3250 /* Normally if we have it in the pack then we do not bother writing
3251 * it out into .git/objects/??/?{38} file.
3252 */
3253 write_sha1_file_prepare(buf, len, type, sha1, hdr, &hdrlen);
3254 if (freshen_packed_object(sha1) || freshen_loose_object(sha1))
3255 return 0;
3256 return write_loose_object(sha1, hdr, hdrlen, buf, len, 0);
3257}
3258
3259int hash_sha1_file_literally(const void *buf, unsigned long len, const char *type,
3260 unsigned char *sha1, unsigned flags)
3261{
3262 char *header;
3263 int hdrlen, status = 0;
3264
3265 /* type string, SP, %lu of the length plus NUL must fit this */
3266 hdrlen = strlen(type) + 32;
3267 header = xmalloc(hdrlen);
3268 write_sha1_file_prepare(buf, len, type, sha1, header, &hdrlen);
3269
3270 if (!(flags & HASH_WRITE_OBJECT))
3271 goto cleanup;
3272 if (freshen_packed_object(sha1) || freshen_loose_object(sha1))
3273 goto cleanup;
3274 status = write_loose_object(sha1, header, hdrlen, buf, len, 0);
3275
3276cleanup:
3277 free(header);
3278 return status;
3279}
3280
3281int force_object_loose(const unsigned char *sha1, time_t mtime)
3282{
3283 void *buf;
3284 unsigned long len;
3285 enum object_type type;
3286 char hdr[32];
3287 int hdrlen;
3288 int ret;
3289
3290 if (has_loose_object(sha1))
3291 return 0;
3292 buf = read_packed_sha1(sha1, &type, &len);
3293 if (!buf)
3294 return error("cannot read sha1_file for %s", sha1_to_hex(sha1));
3295 hdrlen = xsnprintf(hdr, sizeof(hdr), "%s %lu", typename(type), len) + 1;
3296 ret = write_loose_object(sha1, hdr, hdrlen, buf, len, mtime);
3297 free(buf);
3298
3299 return ret;
3300}
3301
3302int has_pack_index(const unsigned char *sha1)
3303{
3304 struct stat st;
3305 if (stat(sha1_pack_index_name(sha1), &st))
3306 return 0;
3307 return 1;
3308}
3309
3310int has_sha1_pack(const unsigned char *sha1)
3311{
3312 struct pack_entry e;
3313 return find_pack_entry(sha1, &e);
3314}
3315
3316int has_sha1_file_with_flags(const unsigned char *sha1, int flags)
3317{
3318 struct pack_entry e;
3319
3320 if (find_pack_entry(sha1, &e))
3321 return 1;
3322 if (has_loose_object(sha1))
3323 return 1;
3324 if (flags & HAS_SHA1_QUICK)
3325 return 0;
3326 reprepare_packed_git();
3327 return find_pack_entry(sha1, &e);
3328}
3329
3330int has_object_file(const struct object_id *oid)
3331{
3332 return has_sha1_file(oid->hash);
3333}
3334
3335static void check_tree(const void *buf, size_t size)
3336{
3337 struct tree_desc desc;
3338 struct name_entry entry;
3339
3340 init_tree_desc(&desc, buf, size);
3341 while (tree_entry(&desc, &entry))
3342 /* do nothing
3343 * tree_entry() will die() on malformed entries */
3344 ;
3345}
3346
3347static void check_commit(const void *buf, size_t size)
3348{
3349 struct commit c;
3350 memset(&c, 0, sizeof(c));
3351 if (parse_commit_buffer(&c, buf, size))
3352 die("corrupt commit");
3353}
3354
3355static void check_tag(const void *buf, size_t size)
3356{
3357 struct tag t;
3358 memset(&t, 0, sizeof(t));
3359 if (parse_tag_buffer(&t, buf, size))
3360 die("corrupt tag");
3361}
3362
3363static int index_mem(unsigned char *sha1, void *buf, size_t size,
3364 enum object_type type,
3365 const char *path, unsigned flags)
3366{
3367 int ret, re_allocated = 0;
3368 int write_object = flags & HASH_WRITE_OBJECT;
3369
3370 if (!type)
3371 type = OBJ_BLOB;
3372
3373 /*
3374 * Convert blobs to git internal format
3375 */
3376 if ((type == OBJ_BLOB) && path) {
3377 struct strbuf nbuf = STRBUF_INIT;
3378 if (convert_to_git(path, buf, size, &nbuf,
3379 write_object ? safe_crlf : SAFE_CRLF_FALSE)) {
3380 buf = strbuf_detach(&nbuf, &size);
3381 re_allocated = 1;
3382 }
3383 }
3384 if (flags & HASH_FORMAT_CHECK) {
3385 if (type == OBJ_TREE)
3386 check_tree(buf, size);
3387 if (type == OBJ_COMMIT)
3388 check_commit(buf, size);
3389 if (type == OBJ_TAG)
3390 check_tag(buf, size);
3391 }
3392
3393 if (write_object)
3394 ret = write_sha1_file(buf, size, typename(type), sha1);
3395 else
3396 ret = hash_sha1_file(buf, size, typename(type), sha1);
3397 if (re_allocated)
3398 free(buf);
3399 return ret;
3400}
3401
3402static int index_stream_convert_blob(unsigned char *sha1, int fd,
3403 const char *path, unsigned flags)
3404{
3405 int ret;
3406 const int write_object = flags & HASH_WRITE_OBJECT;
3407 struct strbuf sbuf = STRBUF_INIT;
3408
3409 assert(path);
3410 assert(would_convert_to_git_filter_fd(path));
3411
3412 convert_to_git_filter_fd(path, fd, &sbuf,
3413 write_object ? safe_crlf : SAFE_CRLF_FALSE);
3414
3415 if (write_object)
3416 ret = write_sha1_file(sbuf.buf, sbuf.len, typename(OBJ_BLOB),
3417 sha1);
3418 else
3419 ret = hash_sha1_file(sbuf.buf, sbuf.len, typename(OBJ_BLOB),
3420 sha1);
3421 strbuf_release(&sbuf);
3422 return ret;
3423}
3424
3425static int index_pipe(unsigned char *sha1, int fd, enum object_type type,
3426 const char *path, unsigned flags)
3427{
3428 struct strbuf sbuf = STRBUF_INIT;
3429 int ret;
3430
3431 if (strbuf_read(&sbuf, fd, 4096) >= 0)
3432 ret = index_mem(sha1, sbuf.buf, sbuf.len, type, path, flags);
3433 else
3434 ret = -1;
3435 strbuf_release(&sbuf);
3436 return ret;
3437}
3438
3439#define SMALL_FILE_SIZE (32*1024)
3440
3441static int index_core(unsigned char *sha1, int fd, size_t size,
3442 enum object_type type, const char *path,
3443 unsigned flags)
3444{
3445 int ret;
3446
3447 if (!size) {
3448 ret = index_mem(sha1, "", size, type, path, flags);
3449 } else if (size <= SMALL_FILE_SIZE) {
3450 char *buf = xmalloc(size);
3451 if (size == read_in_full(fd, buf, size))
3452 ret = index_mem(sha1, buf, size, type, path, flags);
3453 else
3454 ret = error_errno("short read");
3455 free(buf);
3456 } else {
3457 void *buf = xmmap(NULL, size, PROT_READ, MAP_PRIVATE, fd, 0);
3458 ret = index_mem(sha1, buf, size, type, path, flags);
3459 munmap(buf, size);
3460 }
3461 return ret;
3462}
3463
3464/*
3465 * This creates one packfile per large blob unless bulk-checkin
3466 * machinery is "plugged".
3467 *
3468 * This also bypasses the usual "convert-to-git" dance, and that is on
3469 * purpose. We could write a streaming version of the converting
3470 * functions and insert that before feeding the data to fast-import
3471 * (or equivalent in-core API described above). However, that is
3472 * somewhat complicated, as we do not know the size of the filter
3473 * result, which we need to know beforehand when writing a git object.
3474 * Since the primary motivation for trying to stream from the working
3475 * tree file and to avoid mmaping it in core is to deal with large
3476 * binary blobs, they generally do not want to get any conversion, and
3477 * callers should avoid this code path when filters are requested.
3478 */
3479static int index_stream(unsigned char *sha1, int fd, size_t size,
3480 enum object_type type, const char *path,
3481 unsigned flags)
3482{
3483 return index_bulk_checkin(sha1, fd, size, type, path, flags);
3484}
3485
3486int index_fd(unsigned char *sha1, int fd, struct stat *st,
3487 enum object_type type, const char *path, unsigned flags)
3488{
3489 int ret;
3490
3491 /*
3492 * Call xsize_t() only when needed to avoid potentially unnecessary
3493 * die() for large files.
3494 */
3495 if (type == OBJ_BLOB && path && would_convert_to_git_filter_fd(path))
3496 ret = index_stream_convert_blob(sha1, fd, path, flags);
3497 else if (!S_ISREG(st->st_mode))
3498 ret = index_pipe(sha1, fd, type, path, flags);
3499 else if (st->st_size <= big_file_threshold || type != OBJ_BLOB ||
3500 (path && would_convert_to_git(path)))
3501 ret = index_core(sha1, fd, xsize_t(st->st_size), type, path,
3502 flags);
3503 else
3504 ret = index_stream(sha1, fd, xsize_t(st->st_size), type, path,
3505 flags);
3506 close(fd);
3507 return ret;
3508}
3509
3510int index_path(unsigned char *sha1, const char *path, struct stat *st, unsigned flags)
3511{
3512 int fd;
3513 struct strbuf sb = STRBUF_INIT;
3514
3515 switch (st->st_mode & S_IFMT) {
3516 case S_IFREG:
3517 fd = open(path, O_RDONLY);
3518 if (fd < 0)
3519 return error_errno("open(\"%s\")", path);
3520 if (index_fd(sha1, fd, st, OBJ_BLOB, path, flags) < 0)
3521 return error("%s: failed to insert into database",
3522 path);
3523 break;
3524 case S_IFLNK:
3525 if (strbuf_readlink(&sb, path, st->st_size))
3526 return error_errno("readlink(\"%s\")", path);
3527 if (!(flags & HASH_WRITE_OBJECT))
3528 hash_sha1_file(sb.buf, sb.len, blob_type, sha1);
3529 else if (write_sha1_file(sb.buf, sb.len, blob_type, sha1))
3530 return error("%s: failed to insert into database",
3531 path);
3532 strbuf_release(&sb);
3533 break;
3534 case S_IFDIR:
3535 return resolve_gitlink_ref(path, "HEAD", sha1);
3536 default:
3537 return error("%s: unsupported file type", path);
3538 }
3539 return 0;
3540}
3541
3542int read_pack_header(int fd, struct pack_header *header)
3543{
3544 if (read_in_full(fd, header, sizeof(*header)) < sizeof(*header))
3545 /* "eof before pack header was fully read" */
3546 return PH_ERROR_EOF;
3547
3548 if (header->hdr_signature != htonl(PACK_SIGNATURE))
3549 /* "protocol error (pack signature mismatch detected)" */
3550 return PH_ERROR_PACK_SIGNATURE;
3551 if (!pack_version_ok(header->hdr_version))
3552 /* "protocol error (pack version unsupported)" */
3553 return PH_ERROR_PROTOCOL;
3554 return 0;
3555}
3556
3557void assert_sha1_type(const unsigned char *sha1, enum object_type expect)
3558{
3559 enum object_type type = sha1_object_info(sha1, NULL);
3560 if (type < 0)
3561 die("%s is not a valid object", sha1_to_hex(sha1));
3562 if (type != expect)
3563 die("%s is not a valid '%s' object", sha1_to_hex(sha1),
3564 typename(expect));
3565}
3566
3567static int for_each_file_in_obj_subdir(int subdir_nr,
3568 struct strbuf *path,
3569 each_loose_object_fn obj_cb,
3570 each_loose_cruft_fn cruft_cb,
3571 each_loose_subdir_fn subdir_cb,
3572 void *data)
3573{
3574 size_t baselen = path->len;
3575 DIR *dir = opendir(path->buf);
3576 struct dirent *de;
3577 int r = 0;
3578
3579 if (!dir) {
3580 if (errno == ENOENT)
3581 return 0;
3582 return error_errno("unable to open %s", path->buf);
3583 }
3584
3585 while ((de = readdir(dir))) {
3586 if (is_dot_or_dotdot(de->d_name))
3587 continue;
3588
3589 strbuf_setlen(path, baselen);
3590 strbuf_addf(path, "/%s", de->d_name);
3591
3592 if (strlen(de->d_name) == 38) {
3593 char hex[41];
3594 unsigned char sha1[20];
3595
3596 snprintf(hex, sizeof(hex), "%02x%s",
3597 subdir_nr, de->d_name);
3598 if (!get_sha1_hex(hex, sha1)) {
3599 if (obj_cb) {
3600 r = obj_cb(sha1, path->buf, data);
3601 if (r)
3602 break;
3603 }
3604 continue;
3605 }
3606 }
3607
3608 if (cruft_cb) {
3609 r = cruft_cb(de->d_name, path->buf, data);
3610 if (r)
3611 break;
3612 }
3613 }
3614 closedir(dir);
3615
3616 strbuf_setlen(path, baselen);
3617 if (!r && subdir_cb)
3618 r = subdir_cb(subdir_nr, path->buf, data);
3619
3620 return r;
3621}
3622
3623int for_each_loose_file_in_objdir_buf(struct strbuf *path,
3624 each_loose_object_fn obj_cb,
3625 each_loose_cruft_fn cruft_cb,
3626 each_loose_subdir_fn subdir_cb,
3627 void *data)
3628{
3629 size_t baselen = path->len;
3630 int r = 0;
3631 int i;
3632
3633 for (i = 0; i < 256; i++) {
3634 strbuf_addf(path, "/%02x", i);
3635 r = for_each_file_in_obj_subdir(i, path, obj_cb, cruft_cb,
3636 subdir_cb, data);
3637 strbuf_setlen(path, baselen);
3638 if (r)
3639 break;
3640 }
3641
3642 return r;
3643}
3644
3645int for_each_loose_file_in_objdir(const char *path,
3646 each_loose_object_fn obj_cb,
3647 each_loose_cruft_fn cruft_cb,
3648 each_loose_subdir_fn subdir_cb,
3649 void *data)
3650{
3651 struct strbuf buf = STRBUF_INIT;
3652 int r;
3653
3654 strbuf_addstr(&buf, path);
3655 r = for_each_loose_file_in_objdir_buf(&buf, obj_cb, cruft_cb,
3656 subdir_cb, data);
3657 strbuf_release(&buf);
3658
3659 return r;
3660}
3661
3662struct loose_alt_odb_data {
3663 each_loose_object_fn *cb;
3664 void *data;
3665};
3666
3667static int loose_from_alt_odb(struct alternate_object_database *alt,
3668 void *vdata)
3669{
3670 struct loose_alt_odb_data *data = vdata;
3671 struct strbuf buf = STRBUF_INIT;
3672 int r;
3673
3674 strbuf_addstr(&buf, alt->path);
3675 r = for_each_loose_file_in_objdir_buf(&buf,
3676 data->cb, NULL, NULL,
3677 data->data);
3678 strbuf_release(&buf);
3679 return r;
3680}
3681
3682int for_each_loose_object(each_loose_object_fn cb, void *data, unsigned flags)
3683{
3684 struct loose_alt_odb_data alt;
3685 int r;
3686
3687 r = for_each_loose_file_in_objdir(get_object_directory(),
3688 cb, NULL, NULL, data);
3689 if (r)
3690 return r;
3691
3692 if (flags & FOR_EACH_OBJECT_LOCAL_ONLY)
3693 return 0;
3694
3695 alt.cb = cb;
3696 alt.data = data;
3697 return foreach_alt_odb(loose_from_alt_odb, &alt);
3698}
3699
3700static int for_each_object_in_pack(struct packed_git *p, each_packed_object_fn cb, void *data)
3701{
3702 uint32_t i;
3703 int r = 0;
3704
3705 for (i = 0; i < p->num_objects; i++) {
3706 const unsigned char *sha1 = nth_packed_object_sha1(p, i);
3707
3708 if (!sha1)
3709 return error("unable to get sha1 of object %u in %s",
3710 i, p->pack_name);
3711
3712 r = cb(sha1, p, i, data);
3713 if (r)
3714 break;
3715 }
3716 return r;
3717}
3718
3719int for_each_packed_object(each_packed_object_fn cb, void *data, unsigned flags)
3720{
3721 struct packed_git *p;
3722 int r = 0;
3723 int pack_errors = 0;
3724
3725 prepare_packed_git();
3726 for (p = packed_git; p; p = p->next) {
3727 if ((flags & FOR_EACH_OBJECT_LOCAL_ONLY) && !p->pack_local)
3728 continue;
3729 if (open_pack_index(p)) {
3730 pack_errors = 1;
3731 continue;
3732 }
3733 r = for_each_object_in_pack(p, cb, data);
3734 if (r)
3735 break;
3736 }
3737 return r ? r : pack_errors;
3738}