Andrew's git - gitweb.git/blob

ca7a32bf9565961ac30e40d7c772aa54028cd06d
   1#include "cache.h"
   2#include "csum-file.h"
   3#include "dir.h"
   4#include "lockfile.h"
   5#include "packfile.h"
   6#include "object-store.h"
   7#include "midx.h"
   8
   9#define MIDX_SIGNATURE 0x4d494458 /* "MIDX" */
  10#define MIDX_VERSION 1
  11#define MIDX_BYTE_FILE_VERSION 4
  12#define MIDX_BYTE_HASH_VERSION 5
  13#define MIDX_BYTE_NUM_CHUNKS 6
  14#define MIDX_BYTE_NUM_PACKS 8
  15#define MIDX_HASH_VERSION 1
  16#define MIDX_HEADER_SIZE 12
  17#define MIDX_HASH_LEN 20
  18#define MIDX_MIN_SIZE (MIDX_HEADER_SIZE + MIDX_HASH_LEN)
  19
  20#define MIDX_MAX_CHUNKS 1
  21#define MIDX_CHUNK_ALIGNMENT 4
  22#define MIDX_CHUNKID_PACKNAMES 0x504e414d /* "PNAM" */
  23#define MIDX_CHUNKLOOKUP_WIDTH (sizeof(uint32_t) + sizeof(uint64_t))
  24
  25static char *get_midx_filename(const char *object_dir)
  26{
  27        return xstrfmt("%s/pack/multi-pack-index", object_dir);
  28}
  29
  30struct multi_pack_index *load_multi_pack_index(const char *object_dir)
  31{
  32        struct multi_pack_index *m = NULL;
  33        int fd;
  34        struct stat st;
  35        size_t midx_size;
  36        void *midx_map = NULL;
  37        uint32_t hash_version;
  38        char *midx_name = get_midx_filename(object_dir);
  39        uint32_t i;
  40
  41        fd = git_open(midx_name);
  42
  43        if (fd < 0)
  44                goto cleanup_fail;
  45        if (fstat(fd, &st)) {
  46                error_errno(_("failed to read %s"), midx_name);
  47                goto cleanup_fail;
  48        }
  49
  50        midx_size = xsize_t(st.st_size);
  51
  52        if (midx_size < MIDX_MIN_SIZE) {
  53                error(_("multi-pack-index file %s is too small"), midx_name);
  54                goto cleanup_fail;
  55        }
  56
  57        FREE_AND_NULL(midx_name);
  58
  59        midx_map = xmmap(NULL, midx_size, PROT_READ, MAP_PRIVATE, fd, 0);
  60
  61        FLEX_ALLOC_MEM(m, object_dir, object_dir, strlen(object_dir));
  62        m->fd = fd;
  63        m->data = midx_map;
  64        m->data_len = midx_size;
  65
  66        m->signature = get_be32(m->data);
  67        if (m->signature != MIDX_SIGNATURE) {
  68                error(_("multi-pack-index signature 0x%08x does not match signature 0x%08x"),
  69                      m->signature, MIDX_SIGNATURE);
  70                goto cleanup_fail;
  71        }
  72
  73        m->version = m->data[MIDX_BYTE_FILE_VERSION];
  74        if (m->version != MIDX_VERSION) {
  75                error(_("multi-pack-index version %d not recognized"),
  76                      m->version);
  77                goto cleanup_fail;
  78        }
  79
  80        hash_version = m->data[MIDX_BYTE_HASH_VERSION];
  81        if (hash_version != MIDX_HASH_VERSION) {
  82                error(_("hash version %u does not match"), hash_version);
  83                goto cleanup_fail;
  84        }
  85        m->hash_len = MIDX_HASH_LEN;
  86
  87        m->num_chunks = m->data[MIDX_BYTE_NUM_CHUNKS];
  88
  89        m->num_packs = get_be32(m->data + MIDX_BYTE_NUM_PACKS);
  90
  91        for (i = 0; i < m->num_chunks; i++) {
  92                uint32_t chunk_id = get_be32(m->data + MIDX_HEADER_SIZE +
  93                                             MIDX_CHUNKLOOKUP_WIDTH * i);
  94                uint64_t chunk_offset = get_be64(m->data + MIDX_HEADER_SIZE + 4 +
  95                                                 MIDX_CHUNKLOOKUP_WIDTH * i);
  96
  97                switch (chunk_id) {
  98                        case MIDX_CHUNKID_PACKNAMES:
  99                                m->chunk_pack_names = m->data + chunk_offset;
 100                                break;
 101
 102                        case 0:
 103                                die(_("terminating multi-pack-index chunk id appears earlier than expected"));
 104                                break;
 105
 106                        default:
 107                                /*
 108                                 * Do nothing on unrecognized chunks, allowing future
 109                                 * extensions to add optional chunks.
 110                                 */
 111                                break;
 112                }
 113        }
 114
 115        if (!m->chunk_pack_names)
 116                die(_("multi-pack-index missing required pack-name chunk"));
 117
 118        return m;
 119
 120cleanup_fail:
 121        free(m);
 122        free(midx_name);
 123        if (midx_map)
 124                munmap(midx_map, midx_size);
 125        if (0 <= fd)
 126                close(fd);
 127        return NULL;
 128}
 129
 130static size_t write_midx_header(struct hashfile *f,
 131                                unsigned char num_chunks,
 132                                uint32_t num_packs)
 133{
 134        unsigned char byte_values[4];
 135
 136        hashwrite_be32(f, MIDX_SIGNATURE);
 137        byte_values[0] = MIDX_VERSION;
 138        byte_values[1] = MIDX_HASH_VERSION;
 139        byte_values[2] = num_chunks;
 140        byte_values[3] = 0; /* unused */
 141        hashwrite(f, byte_values, sizeof(byte_values));
 142        hashwrite_be32(f, num_packs);
 143
 144        return MIDX_HEADER_SIZE;
 145}
 146
 147struct pack_list {
 148        struct packed_git **list;
 149        char **names;
 150        uint32_t nr;
 151        uint32_t alloc_list;
 152        uint32_t alloc_names;
 153        size_t pack_name_concat_len;
 154};
 155
 156static void add_pack_to_midx(const char *full_path, size_t full_path_len,
 157                             const char *file_name, void *data)
 158{
 159        struct pack_list *packs = (struct pack_list *)data;
 160
 161        if (ends_with(file_name, ".idx")) {
 162                ALLOC_GROW(packs->list, packs->nr + 1, packs->alloc_list);
 163                ALLOC_GROW(packs->names, packs->nr + 1, packs->alloc_names);
 164
 165                packs->list[packs->nr] = add_packed_git(full_path,
 166                                                        full_path_len,
 167                                                        0);
 168                if (!packs->list[packs->nr]) {
 169                        warning(_("failed to add packfile '%s'"),
 170                                full_path);
 171                        return;
 172                }
 173
 174                packs->names[packs->nr] = xstrdup(file_name);
 175                packs->pack_name_concat_len += strlen(file_name) + 1;
 176                packs->nr++;
 177        }
 178}
 179
 180struct pack_pair {
 181        uint32_t pack_int_id;
 182        char *pack_name;
 183};
 184
 185static int pack_pair_compare(const void *_a, const void *_b)
 186{
 187        struct pack_pair *a = (struct pack_pair *)_a;
 188        struct pack_pair *b = (struct pack_pair *)_b;
 189        return strcmp(a->pack_name, b->pack_name);
 190}
 191
 192static void sort_packs_by_name(char **pack_names, uint32_t nr_packs, uint32_t *perm)
 193{
 194        uint32_t i;
 195        struct pack_pair *pairs;
 196
 197        ALLOC_ARRAY(pairs, nr_packs);
 198
 199        for (i = 0; i < nr_packs; i++) {
 200                pairs[i].pack_int_id = i;
 201                pairs[i].pack_name = pack_names[i];
 202        }
 203
 204        QSORT(pairs, nr_packs, pack_pair_compare);
 205
 206        for (i = 0; i < nr_packs; i++) {
 207                pack_names[i] = pairs[i].pack_name;
 208                perm[pairs[i].pack_int_id] = i;
 209        }
 210
 211        free(pairs);
 212}
 213
 214static size_t write_midx_pack_names(struct hashfile *f,
 215                                    char **pack_names,
 216                                    uint32_t num_packs)
 217{
 218        uint32_t i;
 219        unsigned char padding[MIDX_CHUNK_ALIGNMENT];
 220        size_t written = 0;
 221
 222        for (i = 0; i < num_packs; i++) {
 223                size_t writelen = strlen(pack_names[i]) + 1;
 224
 225                if (i && strcmp(pack_names[i], pack_names[i - 1]) <= 0)
 226                        BUG("incorrect pack-file order: %s before %s",
 227                            pack_names[i - 1],
 228                            pack_names[i]);
 229
 230                hashwrite(f, pack_names[i], writelen);
 231                written += writelen;
 232        }
 233
 234        /* add padding to be aligned */
 235        i = MIDX_CHUNK_ALIGNMENT - (written % MIDX_CHUNK_ALIGNMENT);
 236        if (i < MIDX_CHUNK_ALIGNMENT) {
 237                memset(padding, 0, sizeof(padding));
 238                hashwrite(f, padding, i);
 239                written += i;
 240        }
 241
 242        return written;
 243}
 244
 245int write_midx_file(const char *object_dir)
 246{
 247        unsigned char cur_chunk, num_chunks = 0;
 248        char *midx_name;
 249        uint32_t i;
 250        struct hashfile *f = NULL;
 251        struct lock_file lk;
 252        struct pack_list packs;
 253        uint32_t *pack_perm = NULL;
 254        uint64_t written = 0;
 255        uint32_t chunk_ids[MIDX_MAX_CHUNKS + 1];
 256        uint64_t chunk_offsets[MIDX_MAX_CHUNKS + 1];
 257
 258        midx_name = get_midx_filename(object_dir);
 259        if (safe_create_leading_directories(midx_name)) {
 260                UNLEAK(midx_name);
 261                die_errno(_("unable to create leading directories of %s"),
 262                          midx_name);
 263        }
 264
 265        packs.nr = 0;
 266        packs.alloc_list = 16;
 267        packs.alloc_names = 16;
 268        packs.list = NULL;
 269        packs.pack_name_concat_len = 0;
 270        ALLOC_ARRAY(packs.list, packs.alloc_list);
 271        ALLOC_ARRAY(packs.names, packs.alloc_names);
 272
 273        for_each_file_in_pack_dir(object_dir, add_pack_to_midx, &packs);
 274
 275        if (packs.pack_name_concat_len % MIDX_CHUNK_ALIGNMENT)
 276                packs.pack_name_concat_len += MIDX_CHUNK_ALIGNMENT -
 277                                              (packs.pack_name_concat_len % MIDX_CHUNK_ALIGNMENT);
 278
 279        ALLOC_ARRAY(pack_perm, packs.nr);
 280        sort_packs_by_name(packs.names, packs.nr, pack_perm);
 281
 282        hold_lock_file_for_update(&lk, midx_name, LOCK_DIE_ON_ERROR);
 283        f = hashfd(lk.tempfile->fd, lk.tempfile->filename.buf);
 284        FREE_AND_NULL(midx_name);
 285
 286        cur_chunk = 0;
 287        num_chunks = 1;
 288
 289        written = write_midx_header(f, num_chunks, packs.nr);
 290
 291        chunk_ids[cur_chunk] = MIDX_CHUNKID_PACKNAMES;
 292        chunk_offsets[cur_chunk] = written + (num_chunks + 1) * MIDX_CHUNKLOOKUP_WIDTH;
 293
 294        cur_chunk++;
 295        chunk_ids[cur_chunk] = 0;
 296        chunk_offsets[cur_chunk] = chunk_offsets[cur_chunk - 1] + packs.pack_name_concat_len;
 297
 298        for (i = 0; i <= num_chunks; i++) {
 299                if (i && chunk_offsets[i] < chunk_offsets[i - 1])
 300                        BUG("incorrect chunk offsets: %"PRIu64" before %"PRIu64,
 301                            chunk_offsets[i - 1],
 302                            chunk_offsets[i]);
 303
 304                if (chunk_offsets[i] % MIDX_CHUNK_ALIGNMENT)
 305                        BUG("chunk offset %"PRIu64" is not properly aligned",
 306                            chunk_offsets[i]);
 307
 308                hashwrite_be32(f, chunk_ids[i]);
 309                hashwrite_be32(f, chunk_offsets[i] >> 32);
 310                hashwrite_be32(f, chunk_offsets[i]);
 311
 312                written += MIDX_CHUNKLOOKUP_WIDTH;
 313        }
 314
 315        for (i = 0; i < num_chunks; i++) {
 316                if (written != chunk_offsets[i])
 317                        BUG("incorrect chunk offset (%"PRIu64" != %"PRIu64") for chunk id %"PRIx32,
 318                            chunk_offsets[i],
 319                            written,
 320                            chunk_ids[i]);
 321
 322                switch (chunk_ids[i]) {
 323                        case MIDX_CHUNKID_PACKNAMES:
 324                                written += write_midx_pack_names(f, packs.names, packs.nr);
 325                                break;
 326
 327                        default:
 328                                BUG("trying to write unknown chunk id %"PRIx32,
 329                                    chunk_ids[i]);
 330                }
 331        }
 332
 333        if (written != chunk_offsets[num_chunks])
 334                BUG("incorrect final offset %"PRIu64" != %"PRIu64,
 335                    written,
 336                    chunk_offsets[num_chunks]);
 337
 338        finalize_hashfile(f, NULL, CSUM_FSYNC | CSUM_HASH_IN_STREAM);
 339        commit_lock_file(&lk);
 340
 341        for (i = 0; i < packs.nr; i++) {
 342                if (packs.list[i]) {
 343                        close_pack(packs.list[i]);
 344                        free(packs.list[i]);
 345                }
 346                free(packs.names[i]);
 347        }
 348
 349        free(packs.list);
 350        free(packs.names);
 351        return 0;
 352}