1#include "builtin.h"
2#include "cache.h"
3#include "config.h"
4#include "dir.h"
5#include "parse-options.h"
6#include "run-command.h"
7#include "sigchain.h"
8#include "strbuf.h"
9#include "string-list.h"
10#include "argv-array.h"
11#include "packfile.h"
12static int delta_base_offset = 1;
14static int pack_kept_objects = -1;
15static int write_bitmaps;
16static char *packdir, *packtmp;
17static const char *const git_repack_usage[] = {
19 N_("git repack [<options>]"),
20 NULL
21};
22static const char incremental_bitmap_conflict_error[] = N_(
24"Incremental repacks are incompatible with bitmap indexes. Use\n"
25"--no-write-bitmap-index or disable the pack.writebitmaps configuration."
26);
27static int repack_config(const char *var, const char *value, void *cb)
30{
31 if (!strcmp(var, "repack.usedeltabaseoffset")) {
32 delta_base_offset = git_config_bool(var, value);
33 return 0;
34 }
35 if (!strcmp(var, "repack.packkeptobjects")) {
36 pack_kept_objects = git_config_bool(var, value);
37 return 0;
38 }
39 if (!strcmp(var, "repack.writebitmaps") ||
40 !strcmp(var, "pack.writebitmaps")) {
41 write_bitmaps = git_config_bool(var, value);
42 return 0;
43 }
44 return git_default_config(var, value, cb);
45}
46/*
48 * Remove temporary $GIT_OBJECT_DIRECTORY/pack/.tmp-$$-pack-* files.
49 */
50static void remove_temporary_files(void)
51{
52 struct strbuf buf = STRBUF_INIT;
53 size_t dirlen, prefixlen;
54 DIR *dir;
55 struct dirent *e;
56 dir = opendir(packdir);
58 if (!dir)
59 return;
60 /* Point at the slash at the end of ".../objects/pack/" */
62 dirlen = strlen(packdir) + 1;
63 strbuf_addstr(&buf, packtmp);
64 /* Hold the length of ".tmp-%d-pack-" */
65 prefixlen = buf.len - dirlen;
66 while ((e = readdir(dir))) {
68 if (strncmp(e->d_name, buf.buf + dirlen, prefixlen))
69 continue;
70 strbuf_setlen(&buf, dirlen);
71 strbuf_addstr(&buf, e->d_name);
72 unlink(buf.buf);
73 }
74 closedir(dir);
75 strbuf_release(&buf);
76}
77static void remove_pack_on_signal(int signo)
79{
80 remove_temporary_files();
81 sigchain_pop(signo);
82 raise(signo);
83}
84/*
86 * Adds all packs hex strings to the fname list, which do not
87 * have a corresponding .keep file. These packs are not to
88 * be kept if we are going to pack everything into one file.
89 */
90static void get_non_kept_pack_filenames(struct string_list *fname_list,
91 const struct string_list *extra_keep)
92{
93 DIR *dir;
94 struct dirent *e;
95 char *fname;
96 if (!(dir = opendir(packdir)))
98 return;
99 while ((e = readdir(dir)) != NULL) {
101 size_t len;
102 int i;
103 for (i = 0; i < extra_keep->nr; i++)
105 if (!fspathcmp(e->d_name, extra_keep->items[i].string))
106 break;
107 if (extra_keep->nr > 0 && i < extra_keep->nr)
108 continue;
109 if (!strip_suffix(e->d_name, ".pack", &len))
111 continue;
112 fname = xmemdupz(e->d_name, len);
114 if (!file_exists(mkpath("%s/%s.keep", packdir, fname)))
116 string_list_append_nodup(fname_list, fname);
117 else
118 free(fname);
119 }
120 closedir(dir);
121}
122static void remove_redundant_pack(const char *dir_name, const char *base_name)
124{
125 const char *exts[] = {".pack", ".idx", ".keep", ".bitmap", ".promisor"};
126 int i;
127 struct strbuf buf = STRBUF_INIT;
128 size_t plen;
129 strbuf_addf(&buf, "%s/%s", dir_name, base_name);
131 plen = buf.len;
132 for (i = 0; i < ARRAY_SIZE(exts); i++) {
134 strbuf_setlen(&buf, plen);
135 strbuf_addstr(&buf, exts[i]);
136 unlink(buf.buf);
137 }
138 strbuf_release(&buf);
139}
140struct pack_objects_args {
142 const char *window;
143 const char *window_memory;
144 const char *depth;
145 const char *threads;
146 const char *max_pack_size;
147 int no_reuse_delta;
148 int no_reuse_object;
149 int quiet;
150 int local;
151};
152static void prepare_pack_objects(struct child_process *cmd,
154 const struct pack_objects_args *args)
155{
156 argv_array_push(&cmd->args, "pack-objects");
157 if (args->window)
158 argv_array_pushf(&cmd->args, "--window=%s", args->window);
159 if (args->window_memory)
160 argv_array_pushf(&cmd->args, "--window-memory=%s", args->window_memory);
161 if (args->depth)
162 argv_array_pushf(&cmd->args, "--depth=%s", args->depth);
163 if (args->threads)
164 argv_array_pushf(&cmd->args, "--threads=%s", args->threads);
165 if (args->max_pack_size)
166 argv_array_pushf(&cmd->args, "--max-pack-size=%s", args->max_pack_size);
167 if (args->no_reuse_delta)
168 argv_array_pushf(&cmd->args, "--no-reuse-delta");
169 if (args->no_reuse_object)
170 argv_array_pushf(&cmd->args, "--no-reuse-object");
171 if (args->local)
172 argv_array_push(&cmd->args, "--local");
173 if (args->quiet)
174 argv_array_push(&cmd->args, "--quiet");
175 if (delta_base_offset)
176 argv_array_push(&cmd->args, "--delta-base-offset");
177 argv_array_push(&cmd->args, packtmp);
178 cmd->git_cmd = 1;
179 cmd->out = -1;
180}
181/*
183 * Write oid to the given struct child_process's stdin, starting it first if
184 * necessary.
185 */
186static int write_oid(const struct object_id *oid, struct packed_git *pack,
187 uint32_t pos, void *data)
188{
189 struct child_process *cmd = data;
190 if (cmd->in == -1) {
192 if (start_command(cmd))
193 die("Could not start pack-objects to repack promisor objects");
194 }
195 xwrite(cmd->in, oid_to_hex(oid), GIT_SHA1_HEXSZ);
197 xwrite(cmd->in, "\n", 1);
198 return 0;
199}
200static void repack_promisor_objects(const struct pack_objects_args *args,
202 struct string_list *names)
203{
204 struct child_process cmd = CHILD_PROCESS_INIT;
205 FILE *out;
206 struct strbuf line = STRBUF_INIT;
207 prepare_pack_objects(&cmd, args);
209 cmd.in = -1;
210 /*
212 * NEEDSWORK: Giving pack-objects only the OIDs without any ordering
213 * hints may result in suboptimal deltas in the resulting pack. See if
214 * the OIDs can be sent with fake paths such that pack-objects can use a
215 * {type -> existing pack order} ordering when computing deltas instead
216 * of a {type -> size} ordering, which may produce better deltas.
217 */
218 for_each_packed_object(write_oid, &cmd,
219 FOR_EACH_OBJECT_PROMISOR_ONLY);
220 if (cmd.in == -1)
222 /* No packed objects; cmd was never started */
223 return;
224 close(cmd.in);
226 out = xfdopen(cmd.out, "r");
228 while (strbuf_getline_lf(&line, out) != EOF) {
229 char *promisor_name;
230 int fd;
231 if (line.len != 40)
232 die("repack: Expecting 40 character sha1 lines only from pack-objects.");
233 string_list_append(names, line.buf);
234 /*
236 * pack-objects creates the .pack and .idx files, but not the
237 * .promisor file. Create the .promisor file, which is empty.
238 */
239 promisor_name = mkpathdup("%s-%s.promisor", packtmp,
240 line.buf);
241 fd = open(promisor_name, O_CREAT|O_EXCL|O_WRONLY, 0600);
242 if (fd < 0)
243 die_errno("unable to create '%s'", promisor_name);
244 close(fd);
245 free(promisor_name);
246 }
247 fclose(out);
248 if (finish_command(&cmd))
249 die("Could not finish pack-objects to repack promisor objects");
250}
251#define ALL_INTO_ONE 1
253#define LOOSEN_UNREACHABLE 2
254int cmd_repack(int argc, const char **argv, const char *prefix)
256{
257 struct {
258 const char *name;
259 unsigned optional:1;
260 } exts[] = {
261 {".pack"},
262 {".idx"},
263 {".bitmap", 1},
264 {".promisor", 1},
265 };
266 struct child_process cmd = CHILD_PROCESS_INIT;
267 struct string_list_item *item;
268 struct string_list names = STRING_LIST_INIT_DUP;
269 struct string_list rollback = STRING_LIST_INIT_NODUP;
270 struct string_list existing_packs = STRING_LIST_INIT_DUP;
271 struct strbuf line = STRBUF_INIT;
272 int i, ext, ret, failed;
273 FILE *out;
274 /* variables to be filled by option parsing */
276 int pack_everything = 0;
277 int delete_redundant = 0;
278 const char *unpack_unreachable = NULL;
279 int keep_unreachable = 0;
280 struct string_list keep_pack_list = STRING_LIST_INIT_NODUP;
281 int no_update_server_info = 0;
282 struct pack_objects_args po_args = {NULL};
283 struct option builtin_repack_options[] = {
285 OPT_BIT('a', NULL, &pack_everything,
286 N_("pack everything in a single pack"), ALL_INTO_ONE),
287 OPT_BIT('A', NULL, &pack_everything,
288 N_("same as -a, and turn unreachable objects loose"),
289 LOOSEN_UNREACHABLE | ALL_INTO_ONE),
290 OPT_BOOL('d', NULL, &delete_redundant,
291 N_("remove redundant packs, and run git-prune-packed")),
292 OPT_BOOL('f', NULL, &po_args.no_reuse_delta,
293 N_("pass --no-reuse-delta to git-pack-objects")),
294 OPT_BOOL('F', NULL, &po_args.no_reuse_object,
295 N_("pass --no-reuse-object to git-pack-objects")),
296 OPT_BOOL('n', NULL, &no_update_server_info,
297 N_("do not run git-update-server-info")),
298 OPT__QUIET(&po_args.quiet, N_("be quiet")),
299 OPT_BOOL('l', "local", &po_args.local,
300 N_("pass --local to git-pack-objects")),
301 OPT_BOOL('b', "write-bitmap-index", &write_bitmaps,
302 N_("write bitmap index")),
303 OPT_STRING(0, "unpack-unreachable", &unpack_unreachable, N_("approxidate"),
304 N_("with -A, do not loosen objects older than this")),
305 OPT_BOOL('k', "keep-unreachable", &keep_unreachable,
306 N_("with -a, repack unreachable objects")),
307 OPT_STRING(0, "window", &po_args.window, N_("n"),
308 N_("size of the window used for delta compression")),
309 OPT_STRING(0, "window-memory", &po_args.window_memory, N_("bytes"),
310 N_("same as the above, but limit memory size instead of entries count")),
311 OPT_STRING(0, "depth", &po_args.depth, N_("n"),
312 N_("limits the maximum delta depth")),
313 OPT_STRING(0, "threads", &po_args.threads, N_("n"),
314 N_("limits the maximum number of threads")),
315 OPT_STRING(0, "max-pack-size", &po_args.max_pack_size, N_("bytes"),
316 N_("maximum size of each packfile")),
317 OPT_BOOL(0, "pack-kept-objects", &pack_kept_objects,
318 N_("repack objects in packs marked with .keep")),
319 OPT_STRING_LIST(0, "keep-pack", &keep_pack_list, N_("name"),
320 N_("do not repack this pack")),
321 OPT_END()
322 };
323 git_config(repack_config, NULL);
325 argc = parse_options(argc, argv, prefix, builtin_repack_options,
327 git_repack_usage, 0);
328 if (delete_redundant && repository_format_precious_objects)
330 die(_("cannot delete packs in a precious-objects repo"));
331 if (keep_unreachable &&
333 (unpack_unreachable || (pack_everything & LOOSEN_UNREACHABLE)))
334 die(_("--keep-unreachable and -A are incompatible"));
335 if (pack_kept_objects < 0)
337 pack_kept_objects = write_bitmaps;
338 if (write_bitmaps && !(pack_everything & ALL_INTO_ONE))
340 die(_(incremental_bitmap_conflict_error));
341 packdir = mkpathdup("%s/pack", get_object_directory());
343 packtmp = mkpathdup("%s/.tmp-%d-pack", packdir, (int)getpid());
344 sigchain_push_common(remove_pack_on_signal);
346 prepare_pack_objects(&cmd, &po_args);
348 argv_array_push(&cmd.args, "--keep-true-parents");
350 if (!pack_kept_objects)
351 argv_array_push(&cmd.args, "--honor-pack-keep");
352 for (i = 0; i < keep_pack_list.nr; i++)
353 argv_array_pushf(&cmd.args, "--keep-pack=%s",
354 keep_pack_list.items[i].string);
355 argv_array_push(&cmd.args, "--non-empty");
356 argv_array_push(&cmd.args, "--all");
357 argv_array_push(&cmd.args, "--reflog");
358 argv_array_push(&cmd.args, "--indexed-objects");
359 if (repository_format_partial_clone)
360 argv_array_push(&cmd.args, "--exclude-promisor-objects");
361 if (write_bitmaps)
362 argv_array_push(&cmd.args, "--write-bitmap-index");
363 if (pack_everything & ALL_INTO_ONE) {
365 get_non_kept_pack_filenames(&existing_packs, &keep_pack_list);
366 repack_promisor_objects(&po_args, &names);
368 if (existing_packs.nr && delete_redundant) {
370 if (unpack_unreachable) {
371 argv_array_pushf(&cmd.args,
372 "--unpack-unreachable=%s",
373 unpack_unreachable);
374 argv_array_push(&cmd.env_array, "GIT_REF_PARANOIA=1");
375 } else if (pack_everything & LOOSEN_UNREACHABLE) {
376 argv_array_push(&cmd.args,
377 "--unpack-unreachable");
378 } else if (keep_unreachable) {
379 argv_array_push(&cmd.args, "--keep-unreachable");
380 argv_array_push(&cmd.args, "--pack-loose-unreachable");
381 } else {
382 argv_array_push(&cmd.env_array, "GIT_REF_PARANOIA=1");
383 }
384 }
385 } else {
386 argv_array_push(&cmd.args, "--unpacked");
387 argv_array_push(&cmd.args, "--incremental");
388 }
389 cmd.no_stdin = 1;
391 ret = start_command(&cmd);
393 if (ret)
394 return ret;
395 out = xfdopen(cmd.out, "r");
397 while (strbuf_getline_lf(&line, out) != EOF) {
398 if (line.len != 40)
399 die("repack: Expecting 40 character sha1 lines only from pack-objects.");
400 string_list_append(&names, line.buf);
401 }
402 fclose(out);
403 ret = finish_command(&cmd);
404 if (ret)
405 return ret;
406 if (!names.nr && !po_args.quiet)
408 printf("Nothing new to pack.\n");
409 /*
411 * Ok we have prepared all new packfiles.
412 * First see if there are packs of the same name and if so
413 * if we can move them out of the way (this can happen if we
414 * repacked immediately after packing fully.
415 */
416 failed = 0;
417 for_each_string_list_item(item, &names) {
418 for (ext = 0; ext < ARRAY_SIZE(exts); ext++) {
419 char *fname, *fname_old;
420 fname = mkpathdup("%s/pack-%s%s", packdir,
421 item->string, exts[ext].name);
422 if (!file_exists(fname)) {
423 free(fname);
424 continue;
425 }
426 fname_old = mkpathdup("%s/old-%s%s", packdir,
428 item->string, exts[ext].name);
429 if (file_exists(fname_old))
430 if (unlink(fname_old))
431 failed = 1;
432 if (!failed && rename(fname, fname_old)) {
434 free(fname);
435 free(fname_old);
436 failed = 1;
437 break;
438 } else {
439 string_list_append(&rollback, fname);
440 free(fname_old);
441 }
442 }
443 if (failed)
444 break;
445 }
446 if (failed) {
447 struct string_list rollback_failure = STRING_LIST_INIT_DUP;
448 for_each_string_list_item(item, &rollback) {
449 char *fname, *fname_old;
450 fname = mkpathdup("%s/%s", packdir, item->string);
451 fname_old = mkpathdup("%s/old-%s", packdir, item->string);
452 if (rename(fname_old, fname))
453 string_list_append(&rollback_failure, fname);
454 free(fname);
455 free(fname_old);
456 }
457 if (rollback_failure.nr) {
459 int i;
460 fprintf(stderr,
461 "WARNING: Some packs in use have been renamed by\n"
462 "WARNING: prefixing old- to their name, in order to\n"
463 "WARNING: replace them with the new version of the\n"
464 "WARNING: file. But the operation failed, and the\n"
465 "WARNING: attempt to rename them back to their\n"
466 "WARNING: original names also failed.\n"
467 "WARNING: Please rename them in %s manually:\n", packdir);
468 for (i = 0; i < rollback_failure.nr; i++)
469 fprintf(stderr, "WARNING: old-%s -> %s\n",
470 rollback_failure.items[i].string,
471 rollback_failure.items[i].string);
472 }
473 exit(1);
474 }
475 /* Now the ones with the same name are out of the way... */
477 for_each_string_list_item(item, &names) {
478 for (ext = 0; ext < ARRAY_SIZE(exts); ext++) {
479 char *fname, *fname_old;
480 struct stat statbuffer;
481 int exists = 0;
482 fname = mkpathdup("%s/pack-%s%s",
483 packdir, item->string, exts[ext].name);
484 fname_old = mkpathdup("%s-%s%s",
485 packtmp, item->string, exts[ext].name);
486 if (!stat(fname_old, &statbuffer)) {
487 statbuffer.st_mode &= ~(S_IWUSR | S_IWGRP | S_IWOTH);
488 chmod(fname_old, statbuffer.st_mode);
489 exists = 1;
490 }
491 if (exists || !exts[ext].optional) {
492 if (rename(fname_old, fname))
493 die_errno(_("renaming '%s' failed"), fname_old);
494 }
495 free(fname);
496 free(fname_old);
497 }
498 }
499 /* Remove the "old-" files */
501 for_each_string_list_item(item, &names) {
502 for (ext = 0; ext < ARRAY_SIZE(exts); ext++) {
503 char *fname;
504 fname = mkpathdup("%s/old-%s%s",
505 packdir,
506 item->string,
507 exts[ext].name);
508 if (remove_path(fname))
509 warning(_("failed to remove '%s'"), fname);
510 free(fname);
511 }
512 }
513 /* End of pack replacement. */
515 reprepare_packed_git(the_repository);
517 if (delete_redundant) {
519 int opts = 0;
520 string_list_sort(&names);
521 for_each_string_list_item(item, &existing_packs) {
522 char *sha1;
523 size_t len = strlen(item->string);
524 if (len < 40)
525 continue;
526 sha1 = item->string + len - 40;
527 if (!string_list_has_string(&names, sha1))
528 remove_redundant_pack(packdir, item->string);
529 }
530 if (!po_args.quiet && isatty(2))
531 opts |= PRUNE_PACKED_VERBOSE;
532 prune_packed_objects(opts);
533 }
534 if (!no_update_server_info)
536 update_server_info(0);
537 remove_temporary_files();
538 string_list_clear(&names, 0);
539 string_list_clear(&rollback, 0);
540 string_list_clear(&existing_packs, 0);
541 strbuf_release(&line);
542 return 0;
544}