Andrew's git / gitweb.git / builtin/repack.c
?
builtin / repack.con commit repo-settings: consolidate some config settings (7211b9e)
   1#include "builtin.h"
   2#include "cache.h"
   3#include "config.h"
   4#include "dir.h"
   5#include "parse-options.h"
   6#include "run-command.h"
   7#include "sigchain.h"
   8#include "strbuf.h"
   9#include "string-list.h"
  10#include "argv-array.h"
  11#include "midx.h"
  12#include "packfile.h"
  13#include "object-store.h"
  14
  15static int delta_base_offset = 1;
  16static int pack_kept_objects = -1;
  17static int write_bitmaps = -1;
  18static int use_delta_islands;
  19static char *packdir, *packtmp;
  20
  21static const char *const git_repack_usage[] = {
  22        N_("git repack [<options>]"),
  23        NULL
  24};
  25
  26static const char incremental_bitmap_conflict_error[] = N_(
  27"Incremental repacks are incompatible with bitmap indexes.  Use\n"
  28"--no-write-bitmap-index or disable the pack.writebitmaps configuration."
  29);
  30
  31
  32static int repack_config(const char *var, const char *value, void *cb)
  33{
  34        if (!strcmp(var, "repack.usedeltabaseoffset")) {
  35                delta_base_offset = git_config_bool(var, value);
  36                return 0;
  37        }
  38        if (!strcmp(var, "repack.packkeptobjects")) {
  39                pack_kept_objects = git_config_bool(var, value);
  40                return 0;
  41        }
  42        if (!strcmp(var, "repack.writebitmaps") ||
  43            !strcmp(var, "pack.writebitmaps")) {
  44                write_bitmaps = git_config_bool(var, value);
  45                return 0;
  46        }
  47        if (!strcmp(var, "repack.usedeltaislands")) {
  48                use_delta_islands = git_config_bool(var, value);
  49                return 0;
  50        }
  51        return git_default_config(var, value, cb);
  52}
  53
  54/*
  55 * Remove temporary $GIT_OBJECT_DIRECTORY/pack/.tmp-$$-pack-* files.
  56 */
  57static void remove_temporary_files(void)
  58{
  59        struct strbuf buf = STRBUF_INIT;
  60        size_t dirlen, prefixlen;
  61        DIR *dir;
  62        struct dirent *e;
  63
  64        dir = opendir(packdir);
  65        if (!dir)
  66                return;
  67
  68        /* Point at the slash at the end of ".../objects/pack/" */
  69        dirlen = strlen(packdir) + 1;
  70        strbuf_addstr(&buf, packtmp);
  71        /* Hold the length of  ".tmp-%d-pack-" */
  72        prefixlen = buf.len - dirlen;
  73
  74        while ((e = readdir(dir))) {
  75                if (strncmp(e->d_name, buf.buf + dirlen, prefixlen))
  76                        continue;
  77                strbuf_setlen(&buf, dirlen);
  78                strbuf_addstr(&buf, e->d_name);
  79                unlink(buf.buf);
  80        }
  81        closedir(dir);
  82        strbuf_release(&buf);
  83}
  84
  85static void remove_pack_on_signal(int signo)
  86{
  87        remove_temporary_files();
  88        sigchain_pop(signo);
  89        raise(signo);
  90}
  91
  92static int has_pack_keep_file(void)
  93{
  94        struct packed_git *p;
  95
  96        for (p = get_all_packs(the_repository); p; p = p->next) {
  97                if (p->pack_keep)
  98                        return 1;
  99        }
 100        return 0;
 101}
 102
 103/*
 104 * Adds all packs hex strings to the fname list, which do not
 105 * have a corresponding .keep file. These packs are not to
 106 * be kept if we are going to pack everything into one file.
 107 */
 108static void get_non_kept_pack_filenames(struct string_list *fname_list,
 109                                        const struct string_list *extra_keep)
 110{
 111        DIR *dir;
 112        struct dirent *e;
 113        char *fname;
 114
 115        if (!(dir = opendir(packdir)))
 116                return;
 117
 118        while ((e = readdir(dir)) != NULL) {
 119                size_t len;
 120                int i;
 121
 122                for (i = 0; i < extra_keep->nr; i++)
 123                        if (!fspathcmp(e->d_name, extra_keep->items[i].string))
 124                                break;
 125                if (extra_keep->nr > 0 && i < extra_keep->nr)
 126                        continue;
 127
 128                if (!strip_suffix(e->d_name, ".pack", &len))
 129                        continue;
 130
 131                fname = xmemdupz(e->d_name, len);
 132
 133                if (!file_exists(mkpath("%s/%s.keep", packdir, fname)))
 134                        string_list_append_nodup(fname_list, fname);
 135                else
 136                        free(fname);
 137        }
 138        closedir(dir);
 139}
 140
 141static void remove_redundant_pack(const char *dir_name, const char *base_name)
 142{
 143        struct strbuf buf = STRBUF_INIT;
 144        strbuf_addf(&buf, "%s/%s.pack", dir_name, base_name);
 145        unlink_pack_path(buf.buf, 1);
 146        strbuf_release(&buf);
 147}
 148
 149struct pack_objects_args {
 150        const char *window;
 151        const char *window_memory;
 152        const char *depth;
 153        const char *threads;
 154        const char *max_pack_size;
 155        int no_reuse_delta;
 156        int no_reuse_object;
 157        int quiet;
 158        int local;
 159};
 160
 161static void prepare_pack_objects(struct child_process *cmd,
 162                                 const struct pack_objects_args *args)
 163{
 164        argv_array_push(&cmd->args, "pack-objects");
 165        if (args->window)
 166                argv_array_pushf(&cmd->args, "--window=%s", args->window);
 167        if (args->window_memory)
 168                argv_array_pushf(&cmd->args, "--window-memory=%s", args->window_memory);
 169        if (args->depth)
 170                argv_array_pushf(&cmd->args, "--depth=%s", args->depth);
 171        if (args->threads)
 172                argv_array_pushf(&cmd->args, "--threads=%s", args->threads);
 173        if (args->max_pack_size)
 174                argv_array_pushf(&cmd->args, "--max-pack-size=%s", args->max_pack_size);
 175        if (args->no_reuse_delta)
 176                argv_array_pushf(&cmd->args, "--no-reuse-delta");
 177        if (args->no_reuse_object)
 178                argv_array_pushf(&cmd->args, "--no-reuse-object");
 179        if (args->local)
 180                argv_array_push(&cmd->args,  "--local");
 181        if (args->quiet)
 182                argv_array_push(&cmd->args,  "--quiet");
 183        if (delta_base_offset)
 184                argv_array_push(&cmd->args,  "--delta-base-offset");
 185        argv_array_push(&cmd->args, packtmp);
 186        cmd->git_cmd = 1;
 187        cmd->out = -1;
 188}
 189
 190/*
 191 * Write oid to the given struct child_process's stdin, starting it first if
 192 * necessary.
 193 */
 194static int write_oid(const struct object_id *oid, struct packed_git *pack,
 195                     uint32_t pos, void *data)
 196{
 197        struct child_process *cmd = data;
 198
 199        if (cmd->in == -1) {
 200                if (start_command(cmd))
 201                        die(_("could not start pack-objects to repack promisor objects"));
 202        }
 203
 204        xwrite(cmd->in, oid_to_hex(oid), GIT_SHA1_HEXSZ);
 205        xwrite(cmd->in, "\n", 1);
 206        return 0;
 207}
 208
 209static void repack_promisor_objects(const struct pack_objects_args *args,
 210                                    struct string_list *names)
 211{
 212        struct child_process cmd = CHILD_PROCESS_INIT;
 213        FILE *out;
 214        struct strbuf line = STRBUF_INIT;
 215
 216        prepare_pack_objects(&cmd, args);
 217        cmd.in = -1;
 218
 219        /*
 220         * NEEDSWORK: Giving pack-objects only the OIDs without any ordering
 221         * hints may result in suboptimal deltas in the resulting pack. See if
 222         * the OIDs can be sent with fake paths such that pack-objects can use a
 223         * {type -> existing pack order} ordering when computing deltas instead
 224         * of a {type -> size} ordering, which may produce better deltas.
 225         */
 226        for_each_packed_object(write_oid, &cmd,
 227                               FOR_EACH_OBJECT_PROMISOR_ONLY);
 228
 229        if (cmd.in == -1)
 230                /* No packed objects; cmd was never started */
 231                return;
 232
 233        close(cmd.in);
 234
 235        out = xfdopen(cmd.out, "r");
 236        while (strbuf_getline_lf(&line, out) != EOF) {
 237                char *promisor_name;
 238                int fd;
 239                if (line.len != the_hash_algo->hexsz)
 240                        die(_("repack: Expecting full hex object ID lines only from pack-objects."));
 241                string_list_append(names, line.buf);
 242
 243                /*
 244                 * pack-objects creates the .pack and .idx files, but not the
 245                 * .promisor file. Create the .promisor file, which is empty.
 246                 */
 247                promisor_name = mkpathdup("%s-%s.promisor", packtmp,
 248                                          line.buf);
 249                fd = open(promisor_name, O_CREAT|O_EXCL|O_WRONLY, 0600);
 250                if (fd < 0)
 251                        die_errno(_("unable to create '%s'"), promisor_name);
 252                close(fd);
 253                free(promisor_name);
 254        }
 255        fclose(out);
 256        if (finish_command(&cmd))
 257                die(_("could not finish pack-objects to repack promisor objects"));
 258}
 259
 260#define ALL_INTO_ONE 1
 261#define LOOSEN_UNREACHABLE 2
 262
 263int cmd_repack(int argc, const char **argv, const char *prefix)
 264{
 265        struct {
 266                const char *name;
 267                unsigned optional:1;
 268        } exts[] = {
 269                {".pack"},
 270                {".idx"},
 271                {".bitmap", 1},
 272                {".promisor", 1},
 273        };
 274        struct child_process cmd = CHILD_PROCESS_INIT;
 275        struct string_list_item *item;
 276        struct string_list names = STRING_LIST_INIT_DUP;
 277        struct string_list rollback = STRING_LIST_INIT_NODUP;
 278        struct string_list existing_packs = STRING_LIST_INIT_DUP;
 279        struct strbuf line = STRBUF_INIT;
 280        int i, ext, ret, failed;
 281        FILE *out;
 282
 283        /* variables to be filled by option parsing */
 284        int pack_everything = 0;
 285        int delete_redundant = 0;
 286        const char *unpack_unreachable = NULL;
 287        int keep_unreachable = 0;
 288        struct string_list keep_pack_list = STRING_LIST_INIT_NODUP;
 289        int no_update_server_info = 0;
 290        int midx_cleared = 0;
 291        struct pack_objects_args po_args = {NULL};
 292
 293        struct option builtin_repack_options[] = {
 294                OPT_BIT('a', NULL, &pack_everything,
 295                                N_("pack everything in a single pack"), ALL_INTO_ONE),
 296                OPT_BIT('A', NULL, &pack_everything,
 297                                N_("same as -a, and turn unreachable objects loose"),
 298                                   LOOSEN_UNREACHABLE | ALL_INTO_ONE),
 299                OPT_BOOL('d', NULL, &delete_redundant,
 300                                N_("remove redundant packs, and run git-prune-packed")),
 301                OPT_BOOL('f', NULL, &po_args.no_reuse_delta,
 302                                N_("pass --no-reuse-delta to git-pack-objects")),
 303                OPT_BOOL('F', NULL, &po_args.no_reuse_object,
 304                                N_("pass --no-reuse-object to git-pack-objects")),
 305                OPT_BOOL('n', NULL, &no_update_server_info,
 306                                N_("do not run git-update-server-info")),
 307                OPT__QUIET(&po_args.quiet, N_("be quiet")),
 308                OPT_BOOL('l', "local", &po_args.local,
 309                                N_("pass --local to git-pack-objects")),
 310                OPT_BOOL('b', "write-bitmap-index", &write_bitmaps,
 311                                N_("write bitmap index")),
 312                OPT_BOOL('i', "delta-islands", &use_delta_islands,
 313                                N_("pass --delta-islands to git-pack-objects")),
 314                OPT_STRING(0, "unpack-unreachable", &unpack_unreachable, N_("approxidate"),
 315                                N_("with -A, do not loosen objects older than this")),
 316                OPT_BOOL('k', "keep-unreachable", &keep_unreachable,
 317                                N_("with -a, repack unreachable objects")),
 318                OPT_STRING(0, "window", &po_args.window, N_("n"),
 319                                N_("size of the window used for delta compression")),
 320                OPT_STRING(0, "window-memory", &po_args.window_memory, N_("bytes"),
 321                                N_("same as the above, but limit memory size instead of entries count")),
 322                OPT_STRING(0, "depth", &po_args.depth, N_("n"),
 323                                N_("limits the maximum delta depth")),
 324                OPT_STRING(0, "threads", &po_args.threads, N_("n"),
 325                                N_("limits the maximum number of threads")),
 326                OPT_STRING(0, "max-pack-size", &po_args.max_pack_size, N_("bytes"),
 327                                N_("maximum size of each packfile")),
 328                OPT_BOOL(0, "pack-kept-objects", &pack_kept_objects,
 329                                N_("repack objects in packs marked with .keep")),
 330                OPT_STRING_LIST(0, "keep-pack", &keep_pack_list, N_("name"),
 331                                N_("do not repack this pack")),
 332                OPT_END()
 333        };
 334
 335        git_config(repack_config, NULL);
 336
 337        argc = parse_options(argc, argv, prefix, builtin_repack_options,
 338                                git_repack_usage, 0);
 339
 340        if (delete_redundant && repository_format_precious_objects)
 341                die(_("cannot delete packs in a precious-objects repo"));
 342
 343        if (keep_unreachable &&
 344            (unpack_unreachable || (pack_everything & LOOSEN_UNREACHABLE)))
 345                die(_("--keep-unreachable and -A are incompatible"));
 346
 347        if (write_bitmaps < 0) {
 348                write_bitmaps = (pack_everything & ALL_INTO_ONE) &&
 349                                 is_bare_repository() &&
 350                                 keep_pack_list.nr == 0 &&
 351                                 !has_pack_keep_file();
 352        }
 353        if (pack_kept_objects < 0)
 354                pack_kept_objects = write_bitmaps;
 355
 356        if (write_bitmaps && !(pack_everything & ALL_INTO_ONE))
 357                die(_(incremental_bitmap_conflict_error));
 358
 359        packdir = mkpathdup("%s/pack", get_object_directory());
 360        packtmp = mkpathdup("%s/.tmp-%d-pack", packdir, (int)getpid());
 361
 362        sigchain_push_common(remove_pack_on_signal);
 363
 364        prepare_pack_objects(&cmd, &po_args);
 365
 366        argv_array_push(&cmd.args, "--keep-true-parents");
 367        if (!pack_kept_objects)
 368                argv_array_push(&cmd.args, "--honor-pack-keep");
 369        for (i = 0; i < keep_pack_list.nr; i++)
 370                argv_array_pushf(&cmd.args, "--keep-pack=%s",
 371                                 keep_pack_list.items[i].string);
 372        argv_array_push(&cmd.args, "--non-empty");
 373        argv_array_push(&cmd.args, "--all");
 374        argv_array_push(&cmd.args, "--reflog");
 375        argv_array_push(&cmd.args, "--indexed-objects");
 376        if (repository_format_partial_clone)
 377                argv_array_push(&cmd.args, "--exclude-promisor-objects");
 378        if (write_bitmaps)
 379                argv_array_push(&cmd.args, "--write-bitmap-index");
 380        if (use_delta_islands)
 381                argv_array_push(&cmd.args, "--delta-islands");
 382
 383        if (pack_everything & ALL_INTO_ONE) {
 384                get_non_kept_pack_filenames(&existing_packs, &keep_pack_list);
 385
 386                repack_promisor_objects(&po_args, &names);
 387
 388                if (existing_packs.nr && delete_redundant) {
 389                        if (unpack_unreachable) {
 390                                argv_array_pushf(&cmd.args,
 391                                                "--unpack-unreachable=%s",
 392                                                unpack_unreachable);
 393                                argv_array_push(&cmd.env_array, "GIT_REF_PARANOIA=1");
 394                        } else if (pack_everything & LOOSEN_UNREACHABLE) {
 395                                argv_array_push(&cmd.args,
 396                                                "--unpack-unreachable");
 397                        } else if (keep_unreachable) {
 398                                argv_array_push(&cmd.args, "--keep-unreachable");
 399                                argv_array_push(&cmd.args, "--pack-loose-unreachable");
 400                        } else {
 401                                argv_array_push(&cmd.env_array, "GIT_REF_PARANOIA=1");
 402                        }
 403                }
 404        } else {
 405                argv_array_push(&cmd.args, "--unpacked");
 406                argv_array_push(&cmd.args, "--incremental");
 407        }
 408
 409        cmd.no_stdin = 1;
 410
 411        ret = start_command(&cmd);
 412        if (ret)
 413                return ret;
 414
 415        out = xfdopen(cmd.out, "r");
 416        while (strbuf_getline_lf(&line, out) != EOF) {
 417                if (line.len != the_hash_algo->hexsz)
 418                        die(_("repack: Expecting full hex object ID lines only from pack-objects."));
 419                string_list_append(&names, line.buf);
 420        }
 421        fclose(out);
 422        ret = finish_command(&cmd);
 423        if (ret)
 424                return ret;
 425
 426        if (!names.nr && !po_args.quiet)
 427                printf_ln(_("Nothing new to pack."));
 428
 429        close_object_store(the_repository->objects);
 430
 431        /*
 432         * Ok we have prepared all new packfiles.
 433         * First see if there are packs of the same name and if so
 434         * if we can move them out of the way (this can happen if we
 435         * repacked immediately after packing fully.
 436         */
 437        failed = 0;
 438        for_each_string_list_item(item, &names) {
 439                for (ext = 0; ext < ARRAY_SIZE(exts); ext++) {
 440                        char *fname, *fname_old;
 441
 442                        if (!midx_cleared) {
 443                                clear_midx_file(the_repository);
 444                                midx_cleared = 1;
 445                        }
 446
 447                        fname = mkpathdup("%s/pack-%s%s", packdir,
 448                                                item->string, exts[ext].name);
 449                        if (!file_exists(fname)) {
 450                                free(fname);
 451                                continue;
 452                        }
 453
 454                        fname_old = mkpathdup("%s/old-%s%s", packdir,
 455                                                item->string, exts[ext].name);
 456                        if (file_exists(fname_old))
 457                                if (unlink(fname_old))
 458                                        failed = 1;
 459
 460                        if (!failed && rename(fname, fname_old)) {
 461                                free(fname);
 462                                free(fname_old);
 463                                failed = 1;
 464                                break;
 465                        } else {
 466                                string_list_append(&rollback, fname);
 467                                free(fname_old);
 468                        }
 469                }
 470                if (failed)
 471                        break;
 472        }
 473        if (failed) {
 474                struct string_list rollback_failure = STRING_LIST_INIT_DUP;
 475                for_each_string_list_item(item, &rollback) {
 476                        char *fname, *fname_old;
 477                        fname = mkpathdup("%s/%s", packdir, item->string);
 478                        fname_old = mkpathdup("%s/old-%s", packdir, item->string);
 479                        if (rename(fname_old, fname))
 480                                string_list_append(&rollback_failure, fname);
 481                        free(fname);
 482                        free(fname_old);
 483                }
 484
 485                if (rollback_failure.nr) {
 486                        int i;
 487                        fprintf(stderr,
 488                                _("WARNING: Some packs in use have been renamed by\n"
 489                                  "WARNING: prefixing old- to their name, in order to\n"
 490                                  "WARNING: replace them with the new version of the\n"
 491                                  "WARNING: file.  But the operation failed, and the\n"
 492                                  "WARNING: attempt to rename them back to their\n"
 493                                  "WARNING: original names also failed.\n"
 494                                  "WARNING: Please rename them in %s manually:\n"), packdir);
 495                        for (i = 0; i < rollback_failure.nr; i++)
 496                                fprintf(stderr, "WARNING:   old-%s -> %s\n",
 497                                        rollback_failure.items[i].string,
 498                                        rollback_failure.items[i].string);
 499                }
 500                exit(1);
 501        }
 502
 503        /* Now the ones with the same name are out of the way... */
 504        for_each_string_list_item(item, &names) {
 505                for (ext = 0; ext < ARRAY_SIZE(exts); ext++) {
 506                        char *fname, *fname_old;
 507                        struct stat statbuffer;
 508                        int exists = 0;
 509                        fname = mkpathdup("%s/pack-%s%s",
 510                                        packdir, item->string, exts[ext].name);
 511                        fname_old = mkpathdup("%s-%s%s",
 512                                        packtmp, item->string, exts[ext].name);
 513                        if (!stat(fname_old, &statbuffer)) {
 514                                statbuffer.st_mode &= ~(S_IWUSR | S_IWGRP | S_IWOTH);
 515                                chmod(fname_old, statbuffer.st_mode);
 516                                exists = 1;
 517                        }
 518                        if (exists || !exts[ext].optional) {
 519                                if (rename(fname_old, fname))
 520                                        die_errno(_("renaming '%s' failed"), fname_old);
 521                        }
 522                        free(fname);
 523                        free(fname_old);
 524                }
 525        }
 526
 527        /* Remove the "old-" files */
 528        for_each_string_list_item(item, &names) {
 529                for (ext = 0; ext < ARRAY_SIZE(exts); ext++) {
 530                        char *fname;
 531                        fname = mkpathdup("%s/old-%s%s",
 532                                          packdir,
 533                                          item->string,
 534                                          exts[ext].name);
 535                        if (remove_path(fname))
 536                                warning(_("failed to remove '%s'"), fname);
 537                        free(fname);
 538                }
 539        }
 540
 541        /* End of pack replacement. */
 542
 543        reprepare_packed_git(the_repository);
 544
 545        if (delete_redundant) {
 546                const int hexsz = the_hash_algo->hexsz;
 547                int opts = 0;
 548                string_list_sort(&names);
 549                for_each_string_list_item(item, &existing_packs) {
 550                        char *sha1;
 551                        size_t len = strlen(item->string);
 552                        if (len < hexsz)
 553                                continue;
 554                        sha1 = item->string + len - hexsz;
 555                        if (!string_list_has_string(&names, sha1))
 556                                remove_redundant_pack(packdir, item->string);
 557                }
 558                if (!po_args.quiet && isatty(2))
 559                        opts |= PRUNE_PACKED_VERBOSE;
 560                prune_packed_objects(opts);
 561
 562                if (!keep_unreachable &&
 563                    (!(pack_everything & LOOSEN_UNREACHABLE) ||
 564                     unpack_unreachable) &&
 565                    is_repository_shallow(the_repository))
 566                        prune_shallow(PRUNE_QUICK);
 567        }
 568
 569        if (!no_update_server_info)
 570                update_server_info(0);
 571        remove_temporary_files();
 572
 573        if (git_env_bool(GIT_TEST_MULTI_PACK_INDEX, 0))
 574                write_midx_file(get_object_directory());
 575
 576        string_list_clear(&names, 0);
 577        string_list_clear(&rollback, 0);
 578        string_list_clear(&existing_packs, 0);
 579        strbuf_release(&line);
 580
 581        return 0;
 582}