int http_get_info_packs(const char *base_url, struct packed_git **packs_head)
{
struct http_get_options options = {0};
- int ret = 0, i = 0;
- char *url, *data;
+ int ret = 0;
+ char *url;
+ const char *data;
struct strbuf buf = STRBUF_INIT;
- unsigned char hash[GIT_MAX_RAWSZ];
- const unsigned hexsz = the_hash_algo->hexsz;
+ struct object_id oid;
end_url_with_slash(&buf, base_url);
strbuf_addstr(&buf, "objects/info/packs");
goto cleanup;
data = buf.buf;
- while (i < buf.len) {
- switch (data[i]) {
- case 'P':
- i++;
- if (i + hexsz + 12 <= buf.len &&
- starts_with(data + i, " pack-") &&
- starts_with(data + i + hexsz + 6, ".pack\n")) {
- get_sha1_hex(data + i + 6, hash);
- fetch_and_setup_pack_index(packs_head, hash,
- base_url);
- i += hexsz + 11;
- break;
- }
- default:
- while (i < buf.len && data[i] != '\n')
- i++;
+ while (*data) {
+ if (skip_prefix(data, "P pack-", &data) &&
+ !parse_oid_hex(data, &oid, &data) &&
+ skip_prefix(data, ".pack", &data) &&
+ (*data == '\n' || *data == '\0')) {
+ fetch_and_setup_pack_index(packs_head, oid.hash, base_url);
+ } else {
+ data = strchrnul(data, '\n');
}
- i++;
+ if (*data)
+ data++; /* skip past newline */
}
cleanup:
return nth_midxed_pack_entry(m, e, pos);
}
-int midx_contains_pack(struct multi_pack_index *m, const char *idx_name)
+/* Match "foo.idx" against either "foo.pack" _or_ "foo.idx". */
+static int cmp_idx_or_pack_name(const char *idx_or_pack_name,
+ const char *idx_name)
+{
+ /* Skip past any initial matching prefix. */
+ while (*idx_name && *idx_name == *idx_or_pack_name) {
+ idx_name++;
+ idx_or_pack_name++;
+ }
+
+ /*
+ * If we didn't match completely, we may have matched "pack-1234." and
+ * be left with "idx" and "pack" respectively, which is also OK. We do
+ * not have to check for "idx" and "idx", because that would have been
+ * a complete match (and in that case these strcmps will be false, but
+ * we'll correctly return 0 from the final strcmp() below.
+ *
+ * Technically this matches "fooidx" and "foopack", but we'd never have
+ * such names in the first place.
+ */
+ if (!strcmp(idx_name, "idx") && !strcmp(idx_or_pack_name, "pack"))
+ return 0;
+
+ /*
+ * This not only checks for a complete match, but also orders based on
+ * the first non-identical character, which means our ordering will
+ * match a raw strcmp(). That makes it OK to use this to binary search
+ * a naively-sorted list.
+ */
+ return strcmp(idx_or_pack_name, idx_name);
+}
+
+int midx_contains_pack(struct multi_pack_index *m, const char *idx_or_pack_name)
{
uint32_t first = 0, last = m->num_packs;
int cmp;
current = m->pack_names[mid];
- cmp = strcmp(idx_name, current);
+ cmp = cmp_idx_or_pack_name(idx_or_pack_name, current);
if (!cmp)
return 1;
if (cmp > 0) {
struct multi_pack_index *m,
uint32_t n);
int fill_midx_entry(const struct object_id *oid, struct pack_entry *e, struct multi_pack_index *m);
-int midx_contains_pack(struct multi_pack_index *m, const char *idx_name);
+int midx_contains_pack(struct multi_pack_index *m, const char *idx_or_pack_name);
int prepare_multi_pack_index_one(struct repository *r, const char *object_dir, int local);
int write_midx_file(const char *object_dir);
bitmap_git->bitmaps = kh_init_sha1();
bitmap_git->ext_index.positions = kh_init_sha1_pos();
- load_pack_revindex(bitmap_git->pack);
+ if (load_pack_revindex(bitmap_git->pack))
+ goto failed;
if (!(bitmap_git->commits = read_bitmap_1(bitmap_git)) ||
!(bitmap_git->trees = read_bitmap_1(bitmap_git)) ||
#include "cache.h"
#include "pack-revindex.h"
#include "object-store.h"
+#include "packfile.h"
/*
* Pack index for existing packs give us easy access to the offsets into
sort_revindex(p->revindex, num_ent, p->pack_size);
}
-void load_pack_revindex(struct packed_git *p)
+int load_pack_revindex(struct packed_git *p)
{
- if (!p->revindex)
+ if (!p->revindex) {
+ if (open_pack_index(p))
+ return -1;
create_pack_revindex(p);
+ }
+ return 0;
}
int find_revindex_position(struct packed_git *p, off_t ofs)
{
int pos;
- load_pack_revindex(p);
+ if (load_pack_revindex(p))
+ return NULL;
+
pos = find_revindex_position(p, ofs);
if (pos < 0)
unsigned int nr;
};
-void load_pack_revindex(struct packed_git *p);
+int load_pack_revindex(struct packed_git *p);
int find_revindex_position(struct packed_git *p, off_t ofs);
struct revindex_entry *find_pack_revindex(struct packed_git *p, off_t ofs);
#endif
}
+const char *pack_basename(struct packed_git *p)
+{
+ const char *ret = strrchr(p->pack_name, '/');
+ if (ret)
+ ret = ret + 1; /* skip past slash */
+ else
+ ret = p->pack_name; /* we only have a base */
+ return ret;
+}
+
/*
* Do not call this directly as this leaks p->pack_fd on error return;
* call open_packed_git() instead.
if (!p->index_data) {
struct multi_pack_index *m;
- const char *pack_name = strrchr(p->pack_name, '/');
+ const char *pack_name = pack_basename(p);
for (m = the_repository->objects->multi_pack_index;
m; m = m->next) {
uint32_t i;
int r = 0;
- if (flags & FOR_EACH_OBJECT_PACK_ORDER)
- load_pack_revindex(p);
+ if (flags & FOR_EACH_OBJECT_PACK_ORDER) {
+ if (load_pack_revindex(p))
+ return -1;
+ }
for (i = 0; i < p->num_objects; i++) {
uint32_t pos;
*
* Example: odb_pack_name(out, sha1, "idx") => ".git/objects/pack/pack-1234..idx"
*/
-extern char *odb_pack_name(struct strbuf *buf, const unsigned char *sha1, const char *ext);
+char *odb_pack_name(struct strbuf *buf, const unsigned char *sha1, const char *ext);
/*
* Return the name of the (local) packfile with the specified sha1 in
* its name. The return value is a pointer to memory that is
* overwritten each time this function is called.
*/
-extern char *sha1_pack_name(const unsigned char *sha1);
+char *sha1_pack_name(const unsigned char *sha1);
/*
* Return the name of the (local) pack index file with the specified
* sha1 in its name. The return value is a pointer to memory that is
* overwritten each time this function is called.
*/
-extern char *sha1_pack_index_name(const unsigned char *sha1);
+char *sha1_pack_index_name(const unsigned char *sha1);
-extern struct packed_git *parse_pack_index(unsigned char *sha1, const char *idx_path);
+/*
+ * Return the basename of the packfile, omitting any containing directory
+ * (e.g., "pack-1234abcd[...].pack").
+ */
+const char *pack_basename(struct packed_git *p);
+
+struct packed_git *parse_pack_index(unsigned char *sha1, const char *idx_path);
typedef void each_file_in_pack_dir_fn(const char *full_path, size_t full_path_len,
const char *file_pach, void *data);
#define PACKDIR_FILE_GARBAGE 4
extern void (*report_garbage)(unsigned seen_bits, const char *path);
-extern void reprepare_packed_git(struct repository *r);
-extern void install_packed_git(struct repository *r, struct packed_git *pack);
+void reprepare_packed_git(struct repository *r);
+void install_packed_git(struct repository *r, struct packed_git *pack);
struct packed_git *get_packed_git(struct repository *r);
struct list_head *get_packed_git_mru(struct repository *r);
*/
unsigned long approximate_object_count(void);
-extern struct packed_git *find_sha1_pack(const unsigned char *sha1,
- struct packed_git *packs);
+struct packed_git *find_sha1_pack(const unsigned char *sha1,
+ struct packed_git *packs);
-extern void pack_report(void);
+void pack_report(void);
/*
* mmap the index file for the specified packfile (if it is not
* already mmapped). Return 0 on success.
*/
-extern int open_pack_index(struct packed_git *);
+int open_pack_index(struct packed_git *);
/*
* munmap the index file for the specified packfile (if it is
* currently mmapped).
*/
-extern void close_pack_index(struct packed_git *);
+void close_pack_index(struct packed_git *);
int close_pack_fd(struct packed_git *p);
-extern uint32_t get_pack_fanout(struct packed_git *p, uint32_t value);
+uint32_t get_pack_fanout(struct packed_git *p, uint32_t value);
-extern unsigned char *use_pack(struct packed_git *, struct pack_window **, off_t, unsigned long *);
-extern void close_pack_windows(struct packed_git *);
-extern void close_pack(struct packed_git *);
-extern void close_all_packs(struct raw_object_store *o);
-extern void unuse_pack(struct pack_window **);
-extern void clear_delta_base_cache(void);
-extern struct packed_git *add_packed_git(const char *path, size_t path_len, int local);
+unsigned char *use_pack(struct packed_git *, struct pack_window **, off_t, unsigned long *);
+void close_pack_windows(struct packed_git *);
+void close_pack(struct packed_git *);
+void close_all_packs(struct raw_object_store *o);
+void unuse_pack(struct pack_window **);
+void clear_delta_base_cache(void);
+struct packed_git *add_packed_git(const char *path, size_t path_len, int local);
/*
* Make sure that a pointer access into an mmap'd index file is within bounds,
* (like the 64-bit extended offset table), as we compare the size to the
* fixed-length parts when we open the file.
*/
-extern void check_pack_index_ptr(const struct packed_git *p, const void *ptr);
+void check_pack_index_ptr(const struct packed_git *p, const void *ptr);
/*
* Perform binary search on a pack-index for a given oid. Packfile is expected to
* at the SHA-1 within the mmapped index. Return NULL if there is an
* error.
*/
-extern const unsigned char *nth_packed_object_sha1(struct packed_git *, uint32_t n);
+const unsigned char *nth_packed_object_sha1(struct packed_git *, uint32_t n);
/*
* Like nth_packed_object_sha1, but write the data into the object specified by
* the the first argument. Returns the first argument on success, and NULL on
* error.
*/
-extern const struct object_id *nth_packed_object_oid(struct object_id *, struct packed_git *, uint32_t n);
+const struct object_id *nth_packed_object_oid(struct object_id *, struct packed_git *, uint32_t n);
/*
* Return the offset of the nth object within the specified packfile.
* The index must already be opened.
*/
-extern off_t nth_packed_object_offset(const struct packed_git *, uint32_t n);
+off_t nth_packed_object_offset(const struct packed_git *, uint32_t n);
/*
* If the object named sha1 is present in the specified packfile,
* return its offset within the packfile; otherwise, return 0.
*/
-extern off_t find_pack_entry_one(const unsigned char *sha1, struct packed_git *);
+off_t find_pack_entry_one(const unsigned char *sha1, struct packed_git *);
-extern int is_pack_valid(struct packed_git *);
-extern void *unpack_entry(struct repository *r, struct packed_git *, off_t, enum object_type *, unsigned long *);
-extern unsigned long unpack_object_header_buffer(const unsigned char *buf, unsigned long len, enum object_type *type, unsigned long *sizep);
-extern unsigned long get_size_from_delta(struct packed_git *, struct pack_window **, off_t);
-extern int unpack_object_header(struct packed_git *, struct pack_window **, off_t *, unsigned long *);
+int is_pack_valid(struct packed_git *);
+void *unpack_entry(struct repository *r, struct packed_git *, off_t, enum object_type *, unsigned long *);
+unsigned long unpack_object_header_buffer(const unsigned char *buf, unsigned long len, enum object_type *type, unsigned long *sizep);
+unsigned long get_size_from_delta(struct packed_git *, struct pack_window **, off_t);
+int unpack_object_header(struct packed_git *, struct pack_window **, off_t *, unsigned long *);
-extern void release_pack_memory(size_t);
+void release_pack_memory(size_t);
/* global flag to enable extra checks when accessing packed objects */
extern int do_check_packed_object_crc;
-extern int packed_object_info(struct repository *r,
- struct packed_git *pack,
- off_t offset, struct object_info *);
+int packed_object_info(struct repository *r,
+ struct packed_git *pack,
+ off_t offset, struct object_info *);
-extern void mark_bad_packed_object(struct packed_git *p, const unsigned char *sha1);
-extern const struct packed_git *has_packed_and_bad(struct repository *r, const unsigned char *sha1);
+void mark_bad_packed_object(struct packed_git *p, const unsigned char *sha1);
+const struct packed_git *has_packed_and_bad(struct repository *r, const unsigned char *sha1);
/*
* Iff a pack file in the given repository contains the object named by sha1,
* return true and store its location to e.
*/
-extern int find_pack_entry(struct repository *r, const struct object_id *oid, struct pack_entry *e);
+int find_pack_entry(struct repository *r, const struct object_id *oid, struct pack_entry *e);
-extern int has_object_pack(const struct object_id *oid);
+int has_object_pack(const struct object_id *oid);
-extern int has_pack_index(const unsigned char *sha1);
+int has_pack_index(const unsigned char *sha1);
/*
* Return 1 if an object in a promisor packfile is or refers to the given
* object, 0 otherwise.
*/
-extern int is_promisor_object(const struct object_id *oid);
+int is_promisor_object(const struct object_id *oid);
/*
* Expose a function for fuzz testing.
* have a convenient entry-point for fuzz testing. For real uses, you should
* probably use open_pack_index() or parse_pack_index() instead.
*/
-extern int load_idx(const char *path, const unsigned int hashsz, void *idx_map,
- size_t idx_size, struct packed_git *p);
+int load_idx(const char *path, const unsigned int hashsz, void *idx_map,
+ size_t idx_size, struct packed_git *p);
#endif
return for_each_ref(add_info_ref, fp);
}
-static int update_info_refs(int force)
+static int update_info_refs(void)
{
char *path = git_pathdup("info/refs");
int ret = update_info_file(path, generate_info_refs);
struct packed_git *p;
int old_num;
int new_num;
- int nr_alloc;
} **info;
static int num_pack;
-static const char *objdir;
-static int objdirlen;
static struct pack_info *find_pack_by_name(const char *name)
{
int i;
for (i = 0; i < num_pack; i++) {
struct packed_git *p = info[i]->p;
- /* skip "/pack/" after ".git/objects" */
- if (!strcmp(p->pack_name + objdirlen + 6, name))
+ if (!strcmp(pack_basename(p), name))
return info[i];
}
return NULL;
/* Returns non-zero when we detect that the info in the
* old file is useless.
*/
-static int parse_pack_def(const char *line, int old_cnt)
+static int parse_pack_def(const char *packname, int old_cnt)
{
- struct pack_info *i = find_pack_by_name(line + 2);
+ struct pack_info *i = find_pack_by_name(packname);
if (i) {
i->old_num = old_cnt;
return 0;
static int read_pack_info_file(const char *infofile)
{
FILE *fp;
- char line[1000];
+ struct strbuf line = STRBUF_INIT;
int old_cnt = 0;
+ int stale = 1;
fp = fopen_or_warn(infofile, "r");
if (!fp)
return 1; /* nonexistent is not an error. */
- while (fgets(line, sizeof(line), fp)) {
- int len = strlen(line);
- if (len && line[len-1] == '\n')
- line[--len] = 0;
+ while (strbuf_getline(&line, fp) != EOF) {
+ const char *arg;
- if (!len)
+ if (!line.len)
continue;
- switch (line[0]) {
- case 'P': /* P name */
- if (parse_pack_def(line, old_cnt++))
+ if (skip_prefix(line.buf, "P ", &arg)) {
+ /* P name */
+ if (parse_pack_def(arg, old_cnt++))
goto out_stale;
- break;
- case 'D': /* we used to emit D but that was misguided. */
- case 'T': /* we used to emit T but nobody uses it. */
+ } else if (line.buf[0] == 'D') {
+ /* we used to emit D but that was misguided. */
goto out_stale;
- default:
- error("unrecognized: %s", line);
- break;
+ } else if (line.buf[0] == 'T') {
+ /* we used to emit T but nobody uses it. */
+ goto out_stale;
+ } else {
+ error("unrecognized: %s", line.buf);
}
}
- fclose(fp);
- return 0;
+ stale = 0;
+
out_stale:
+ strbuf_release(&line);
fclose(fp);
- return 1;
+ return stale;
}
static int compare_info(const void *a_, const void *b_)
int stale;
int i = 0;
- objdir = get_object_directory();
- objdirlen = strlen(objdir);
-
for (p = get_all_packs(the_repository); p; p = p->next) {
/* we ignore things on alternate path since they are
* not available to the pullers in general.
for (i = 0, p = get_all_packs(the_repository); p; p = p->next) {
if (!p->pack_local)
continue;
+ assert(i < num_pack);
info[i] = xcalloc(1, sizeof(struct pack_info));
info[i]->p = p;
info[i]->old_num = -1;
{
int i;
for (i = 0; i < num_pack; i++) {
- if (fprintf(fp, "P %s\n", info[i]->p->pack_name + objdirlen + 6) < 0)
+ if (fprintf(fp, "P %s\n", pack_basename(info[i]->p)) < 0)
return -1;
}
if (fputc('\n', fp) == EOF)
*/
int errs = 0;
- errs = errs | update_info_refs(force);
+ errs = errs | update_info_refs();
errs = errs | update_info_packs(force);
/* remove leftover rev-cache file if there is any */
'
midx_git_two_modes () {
+ git -c core.multiPackIndex=false $1 >expect &&
+ git -c core.multiPackIndex=true $1 >actual &&
if [ "$2" = "sorted" ]
then
- git -c core.multiPackIndex=false $1 | sort >expect &&
- git -c core.multiPackIndex=true $1 | sort >actual
- else
- git -c core.multiPackIndex=false $1 >expect &&
- git -c core.multiPackIndex=true $1 >actual
+ sort <expect >expect.sorted &&
+ mv expect.sorted expect &&
+ sort <actual >actual.sorted &&
+ mv actual.sorted actual
fi &&
test_cmp expect actual
}
midx_git_two_modes "rev-list --objects --all" &&
midx_git_two_modes "log --raw" &&
midx_git_two_modes "count-objects --verbose" &&
- midx_git_two_modes "cat-file --batch-all-objects --buffer --batch-check" &&
- midx_git_two_modes "cat-file --batch-all-objects --buffer --batch-check --unsorted" sorted
+ midx_git_two_modes "cat-file --batch-all-objects --batch-check" &&
+ midx_git_two_modes "cat-file --batch-all-objects --batch-check --unordered" sorted
'
}
compare_results_with_midx "one v2 pack"
+test_expect_success 'corrupt idx not opened' '
+ idx=$(test-tool read-midx $objdir | grep "\.idx\$") &&
+ mv $objdir/pack/$idx backup-$idx &&
+ test_when_finished "mv backup-\$idx \$objdir/pack/\$idx" &&
+
+ # This is the minimum size for a sha-1 based .idx; this lets
+ # us pass perfunctory tests, but anything that actually opens and reads
+ # the idx file will complain.
+ test_copy_bytes 1064 <backup-$idx >$objdir/pack/$idx &&
+
+ git -c core.multiPackIndex=true rev-list --objects --all 2>err &&
+ test_must_be_empty err
+'
+
test_expect_success 'add more objects' '
for i in $(test_seq 6 10)
do
test_expect_success 'fetch notices corrupt idx' '
cp -R "$GIT_DAEMON_DOCUMENT_ROOT_PATH"/repo_pack.git "$GIT_DAEMON_DOCUMENT_ROOT_PATH"/repo_bad2.git &&
(cd "$GIT_DAEMON_DOCUMENT_ROOT_PATH"/repo_bad2.git &&
+ rm -f objects/pack/multi-pack-index &&
p=$(ls objects/pack/pack-*.idx) &&
chmod u+w $p &&
printf %0256d 0 | dd of=$p bs=256 count=1 seek=1 conv=notrunc