static int local;
static int incremental;
static int allow_ofs_delta;
-static const char *pack_tmp_name, *idx_tmp_name;
-static char tmpname[PATH_MAX];
static const char *base_name;
static int progress = 1;
static int window = 10;
static int delta_search_threads = 1;
static int pack_to_stdout;
static int num_preferred_base;
-static struct progress progress_state;
+static struct progress *progress_state;
static int pack_compression_level = Z_DEFAULT_COMPRESSION;
static int pack_compression_seen;
return offset + size;
}
-static int open_object_dir_tmp(const char *path)
-{
- snprintf(tmpname, sizeof(tmpname), "%s/%s", get_object_directory(), path);
- return xmkstemp(tmpname);
-}
-
/* forward declaration for write_pack_file */
static int adjust_perm(const char *path, mode_t mode);
uint32_t nr_remaining = nr_result;
if (do_progress)
- start_progress(&progress_state, "Writing %u objects...", "", nr_result);
+ progress_state = start_progress("Writing objects", nr_result);
written_list = xmalloc(nr_objects * sizeof(struct object_entry *));
do {
unsigned char sha1[20];
+ char *pack_tmp_name = NULL;
if (pack_to_stdout) {
- f = sha1fd(1, "<stdout>");
+ f = sha1fd_throughput(1, "<stdout>", progress_state);
} else {
- int fd = open_object_dir_tmp("tmp_pack_XXXXXX");
+ char tmpname[PATH_MAX];
+ int fd;
+ snprintf(tmpname, sizeof(tmpname),
+ "%s/tmp_pack_XXXXXX", get_object_directory());
+ fd = xmkstemp(tmpname);
pack_tmp_name = xstrdup(tmpname);
f = sha1fd(fd, pack_tmp_name);
}
if (!offset_one)
break;
offset = offset_one;
- if (do_progress)
- display_progress(&progress_state, written);
+ display_progress(progress_state, written);
}
/*
if (pack_to_stdout || nr_written == nr_remaining) {
sha1close(f, sha1, 1);
} else {
- sha1close(f, sha1, 0);
- fixup_pack_header_footer(f->fd, sha1, pack_tmp_name, nr_written);
- close(f->fd);
+ int fd = sha1close(f, NULL, 0);
+ fixup_pack_header_footer(fd, sha1, pack_tmp_name, nr_written);
+ close(fd);
}
if (!pack_to_stdout) {
mode_t mode = umask(0);
+ char *idx_tmp_name, tmpname[PATH_MAX];
umask(mode);
mode = 0444 & ~mode;
idx_tmp_name = write_idx_file(NULL,
- (struct pack_idx_entry **) written_list, nr_written, sha1);
+ (struct pack_idx_entry **) written_list,
+ nr_written, sha1);
snprintf(tmpname, sizeof(tmpname), "%s-%s.pack",
base_name, sha1_to_hex(sha1));
if (adjust_perm(pack_tmp_name, mode))
if (rename(idx_tmp_name, tmpname))
die("unable to rename temporary index file: %s",
strerror(errno));
+ free(idx_tmp_name);
+ free(pack_tmp_name);
puts(sha1_to_hex(sha1));
}
} while (nr_remaining && i < nr_objects);
free(written_list);
- if (do_progress)
- stop_progress(&progress_state);
+ stop_progress(&progress_state);
if (written != nr_result)
die("wrote %u objects while expecting %u", written, nr_result);
/*
else
object_ix[-1 - ix] = nr_objects;
- if (progress)
- display_progress(&progress_state, nr_objects);
+ display_progress(progress_state, nr_objects);
if (name && no_try_delta(name))
entry->no_try_delta = 1;
progress_lock();
(*processed)++;
- if (progress)
- display_progress(&progress_state, *processed);
+ display_progress(progress_state, *processed);
progress_unlock();
/*
delta_list[n++] = entry;
}
- if (nr_deltas) {
+ if (nr_deltas && n > 1) {
unsigned nr_done = 0;
if (progress)
- start_progress(&progress_state,
- "Deltifying %u objects...", "",
- nr_deltas);
+ progress_state = start_progress("Compressing objects",
+ nr_deltas);
qsort(delta_list, n, sizeof(*delta_list), type_size_sort);
ll_find_deltas(delta_list, n, window+1, depth, &nr_done);
- if (progress)
- stop_progress(&progress_state);
+ stop_progress(&progress_state);
if (nr_done != nr_deltas)
die("inconsistency with delta count");
}
prepare_packed_git();
if (progress)
- start_progress(&progress_state, "Generating pack...",
- "Counting objects: ", 0);
+ progress_state = start_progress("Counting objects", 0);
if (!use_internal_rev_list)
read_object_list_from_stdin();
else {
rp_av[rp_ac] = NULL;
get_object_list(rp_ac, rp_av);
}
- if (progress) {
- stop_progress(&progress_state);
- fprintf(stderr, "Done counting %u objects.\n", nr_objects);
- }
+ stop_progress(&progress_state);
if (non_empty && !nr_result)
return 0;
- if (progress && (nr_objects != nr_result))
- fprintf(stderr, "Result has %u objects.\n", nr_result);
if (nr_result)
prepare_pack(window, depth);
write_pack_file();
#include "builtin.h"
#include "cache.h"
+#include "progress.h"
static const char prune_packed_usage[] =
"git-prune-packed [-n] [-q]";
#define DRY_RUN 01
#define VERBOSE 02
+static struct progress *progress;
+
static void prune_dir(int i, DIR *dir, char *pathname, int len, int opts)
{
struct dirent *de;
printf("rm -f %s\n", pathname);
else if (unlink(pathname) < 0)
error("unable to unlink %s", pathname);
+ display_progress(progress, i + 1);
}
pathname[len] = 0;
rmdir(pathname);
const char *dir = get_object_directory();
int len = strlen(dir);
+ if (opts == VERBOSE)
+ progress = start_progress_delay("Removing duplicate objects",
+ 256, 95, 2);
+
if (len > PATH_MAX - 42)
die("impossible object directory");
memcpy(pathname, dir, len);
sprintf(pathname + len, "%02x/", i);
d = opendir(pathname);
- if (opts == VERBOSE && (d || i == 255))
- fprintf(stderr, "Removing unused objects %d%%...\015",
- ((i+1) * 100) / 256);
if (!d)
continue;
prune_dir(i, d, pathname, len + 3, opts);
closedir(d);
}
- if (opts == VERBOSE)
- fprintf(stderr, "\nDone.\n");
+ stop_progress(&progress);
}
int cmd_prune_packed(int argc, const char **argv, const char *prefix)
static void unpack_all(void)
{
int i;
- struct progress progress;
+ struct progress *progress = NULL;
struct pack_header *hdr = fill(sizeof(struct pack_header));
unsigned nr_objects = ntohl(hdr->hdr_entries);
use(sizeof(struct pack_header));
if (!quiet)
- start_progress(&progress, "Unpacking %u objects...", "", nr_objects);
+ progress = start_progress("Unpacking objects", nr_objects);
obj_list = xmalloc(nr_objects * sizeof(*obj_list));
for (i = 0; i < nr_objects; i++) {
unpack_one(i);
- if (!quiet)
- display_progress(&progress, i + 1);
+ display_progress(progress, i + 1);
}
- if (!quiet)
- stop_progress(&progress);
+ stop_progress(&progress);
if (delta_list)
die("unresolved deltas left after unpacking");
* able to verify hasn't been messed with afterwards.
*/
#include "cache.h"
+#include "progress.h"
#include "csum-file.h"
static void sha1flush(struct sha1file *f, unsigned int count)
for (;;) {
int ret = xwrite(f->fd, buf, count);
if (ret > 0) {
+ display_throughput(f->tp, ret);
buf = (char *) buf + ret;
count -= ret;
if (count)
int sha1close(struct sha1file *f, unsigned char *result, int final)
{
+ int fd;
unsigned offset = f->offset;
if (offset) {
SHA1_Update(&f->ctx, f->buffer, offset);
sha1flush(f, offset);
f->offset = 0;
}
- if (!final)
- return 0; /* only want to flush (no checksum write, no close) */
- SHA1_Final(f->buffer, &f->ctx);
- if (result)
- hashcpy(result, f->buffer);
- sha1flush(f, 20);
- if (close(f->fd))
- die("%s: sha1 file error on close (%s)", f->name, strerror(errno));
+ if (final) {
+ /* write checksum and close fd */
+ SHA1_Final(f->buffer, &f->ctx);
+ if (result)
+ hashcpy(result, f->buffer);
+ sha1flush(f, 20);
+ if (close(f->fd))
+ die("%s: sha1 file error on close (%s)",
+ f->name, strerror(errno));
+ fd = 0;
+ } else
+ fd = f->fd;
free(f);
- return 0;
+ return fd;
}
int sha1write(struct sha1file *f, void *buf, unsigned int count)
}
struct sha1file *sha1fd(int fd, const char *name)
+{
+ return sha1fd_throughput(fd, name, NULL);
+}
+
+struct sha1file *sha1fd_throughput(int fd, const char *name, struct progress *tp)
{
struct sha1file *f;
unsigned len;
f->fd = fd;
f->error = 0;
f->offset = 0;
+ f->tp = tp;
f->do_crc = 0;
SHA1_Init(&f->ctx);
return f;
#ifndef CSUM_FILE_H
#define CSUM_FILE_H
+struct progress;
+
/* A SHA1-protected file */
struct sha1file {
int fd, error;
unsigned int offset, namelen;
SHA_CTX ctx;
+ struct progress *tp;
char name[PATH_MAX];
int do_crc;
uint32_t crc32;
};
extern struct sha1file *sha1fd(int fd, const char *name);
+extern struct sha1file *sha1fd_throughput(int fd, const char *name, struct progress *tp);
extern int sha1close(struct sha1file *, unsigned char *, int);
extern int sha1write(struct sha1file *, void *, unsigned int);
extern void crc32_begin(struct sha1file *);
fullbases="$fullbases pack-$name"
chmod a-w "$PACKTMP-$name.pack"
chmod a-w "$PACKTMP-$name.idx"
- if test "$quiet" != '-q'; then
- echo "Pack pack-$name created."
- fi
mkdir -p "$PACKDIR" || exit
for sfx in pack idx
static int from_stdin;
static int verbose;
-static struct progress progress;
+static struct progress *progress;
/* We always read in 4kB chunks. */
static unsigned char input_buffer[4096];
die("early EOF");
die("read error on input: %s", strerror(errno));
}
+ if (from_stdin)
+ display_throughput(progress, ret);
input_len += ret;
} while (input_len < min);
return input_buffer;
consumed_bytes += bytes;
}
-static const char *open_pack_file(const char *pack_name)
+static char *open_pack_file(char *pack_name)
{
if (from_stdin) {
input_fd = 0;
* - remember base (SHA1 or offset) for all deltas.
*/
if (verbose)
- start_progress(&progress, "Indexing %u objects...", "", nr_objects);
+ progress = start_progress(
+ from_stdin ? "Receiving objects" : "Indexing objects",
+ nr_objects);
for (i = 0; i < nr_objects; i++) {
struct object_entry *obj = &objects[i];
data = unpack_raw_entry(obj, &delta->base);
} else
sha1_object(data, obj->size, obj->type, obj->idx.sha1);
free(data);
- if (verbose)
- display_progress(&progress, i+1);
+ display_progress(progress, i+1);
}
objects[i].idx.offset = consumed_bytes;
- if (verbose)
- stop_progress(&progress);
+ stop_progress(&progress);
/* Check pack integrity */
flush();
* for some more deltas.
*/
if (verbose)
- start_progress(&progress, "Resolving %u deltas...", "", nr_deltas);
+ progress = start_progress("Resolving deltas", nr_deltas);
for (i = 0; i < nr_objects; i++) {
struct object_entry *obj = &objects[i];
union delta_base base;
obj->size, obj->type);
}
free(data);
- if (verbose)
- display_progress(&progress, nr_resolved_deltas);
+ display_progress(progress, nr_resolved_deltas);
}
}
die("local object %s is corrupt", sha1_to_hex(d->base.sha1));
append_obj_to_pack(d->base.sha1, data, size, type);
free(data);
- if (verbose)
- display_progress(&progress, nr_resolved_deltas);
+ display_progress(progress, nr_resolved_deltas);
}
free(sorted_by_pos);
}
int main(int argc, char **argv)
{
int i, fix_thin_pack = 0;
- const char *curr_pack, *pack_name = NULL;
- const char *curr_index, *index_name = NULL;
+ char *curr_pack, *pack_name = NULL;
+ char *curr_index, *index_name = NULL;
const char *keep_name = NULL, *keep_msg = NULL;
char *index_name_buf = NULL, *keep_name_buf = NULL;
struct pack_idx_entry **idx_objects;
unsigned char sha1[20];
for (i = 1; i < argc; i++) {
- const char *arg = argv[i];
+ char *arg = argv[i];
if (*arg == '-') {
if (!strcmp(arg, "--stdin")) {
deltas = xmalloc(nr_objects * sizeof(struct delta_entry));
parse_pack_objects(sha1);
if (nr_deltas == nr_resolved_deltas) {
- if (verbose)
- stop_progress(&progress);
+ stop_progress(&progress);
/* Flush remaining pack final 20-byte SHA1. */
flush();
} else {
(nr_objects + nr_unresolved + 1)
* sizeof(*objects));
fix_unresolved_deltas(nr_unresolved);
- if (verbose) {
- stop_progress(&progress);
+ stop_progress(&progress);
+ if (verbose)
fprintf(stderr, "%d objects were added to complete this thin pack.\n",
nr_objects - nr_objects_initial);
- }
fixup_pack_header_footer(output_fd, sha1,
curr_pack, nr_objects);
}
free(objects);
free(index_name_buf);
free(keep_name_buf);
+ if (pack_name == NULL)
+ free(curr_pack);
+ if (index_name == NULL)
+ free(curr_index);
return 0;
}
* the SHA1 hash of sorted object names. The objects array passed in
* will be sorted by SHA1 on exit.
*/
-const char *write_idx_file(const char *index_name, struct pack_idx_entry **objects, int nr_objects, unsigned char *sha1)
+char *write_idx_file(char *index_name, struct pack_idx_entry **objects,
+ int nr_objects, unsigned char *sha1)
{
struct sha1file *f;
struct pack_idx_entry **sorted_by_sha, **list, **last;
off_t offset;
};
-extern const char *write_idx_file(const char *index_name, struct pack_idx_entry **objects, int nr_objects, unsigned char *sha1);
+extern char *write_idx_file(char *index_name, struct pack_idx_entry **objects, int nr_objects, unsigned char *sha1);
extern int verify_pack(struct packed_git *, int);
extern void fixup_pack_header_footer(int, unsigned char *, const char *, uint32_t);
+/*
+ * Simple text-based progress display module for GIT
+ *
+ * Copyright (c) 2007 by Nicolas Pitre <nico@cam.org>
+ *
+ * This code is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
#include "git-compat-util.h"
#include "progress.h"
+#define TP_IDX_MAX 8
+
+struct throughput {
+ struct timeval prev_tv;
+ off_t total;
+ unsigned long count;
+ unsigned long avg_bytes;
+ unsigned long last_bytes[TP_IDX_MAX];
+ unsigned int avg_misecs;
+ unsigned int last_misecs[TP_IDX_MAX];
+ unsigned int idx;
+ char display[32];
+};
+
+struct progress {
+ const char *title;
+ int last_value;
+ unsigned total;
+ unsigned last_percent;
+ unsigned delay;
+ unsigned delayed_percent_treshold;
+ struct throughput *throughput;
+};
+
static volatile sig_atomic_t progress_update;
static void progress_interval(int signum)
progress_update = 0;
}
-int display_progress(struct progress *progress, unsigned n)
+static int display(struct progress *progress, unsigned n, int done)
{
+ char *eol, *tp;
+
if (progress->delay) {
- char buf[80];
if (!progress_update || --progress->delay)
return 0;
if (progress->total) {
return 0;
}
}
- if (snprintf(buf, sizeof(buf),
- progress->delayed_title, progress->total))
- fprintf(stderr, "%s\n", buf);
}
+
+ progress->last_value = n;
+ tp = (progress->throughput) ? progress->throughput->display : "";
+ eol = done ? ", done. \n" : " \r";
if (progress->total) {
unsigned percent = n * 100 / progress->total;
if (percent != progress->last_percent || progress_update) {
progress->last_percent = percent;
- fprintf(stderr, "%s%4u%% (%u/%u) done\r",
- progress->prefix, percent, n, progress->total);
+ fprintf(stderr, "%s: %3u%% (%u/%u)%s%s",
+ progress->title, percent, n,
+ progress->total, tp, eol);
progress_update = 0;
- progress->need_lf = 1;
return 1;
}
} else if (progress_update) {
- fprintf(stderr, "%s%u\r", progress->prefix, n);
+ fprintf(stderr, "%s: %u%s%s", progress->title, n, tp, eol);
progress_update = 0;
- progress->need_lf = 1;
return 1;
}
+
return 0;
}
-void start_progress(struct progress *progress, const char *title,
- const char *prefix, unsigned total)
+void display_throughput(struct progress *progress, unsigned long n)
{
- char buf[80];
- progress->prefix = prefix;
- progress->total = total;
- progress->last_percent = -1;
- progress->delay = 0;
- progress->need_lf = 0;
- if (snprintf(buf, sizeof(buf), title, total))
- fprintf(stderr, "%s\n", buf);
- set_progress_signal();
+ struct throughput *tp;
+ struct timeval tv;
+ unsigned int misecs;
+
+ if (!progress)
+ return;
+ tp = progress->throughput;
+
+ gettimeofday(&tv, NULL);
+
+ if (!tp) {
+ progress->throughput = tp = calloc(1, sizeof(*tp));
+ if (tp)
+ tp->prev_tv = tv;
+ return;
+ }
+
+ tp->total += n;
+ tp->count += n;
+
+ /*
+ * We have x = bytes and y = microsecs. We want z = KiB/s:
+ *
+ * z = (x / 1024) / (y / 1000000)
+ * z = x / y * 1000000 / 1024
+ * z = x / (y * 1024 / 1000000)
+ * z = x / y'
+ *
+ * To simplify things we'll keep track of misecs, or 1024th of a sec
+ * obtained with:
+ *
+ * y' = y * 1024 / 1000000
+ * y' = y / (1000000 / 1024)
+ * y' = y / 977
+ */
+ misecs = (tv.tv_sec - tp->prev_tv.tv_sec) * 1024;
+ misecs += (int)(tv.tv_usec - tp->prev_tv.tv_usec) / 977;
+
+ if (misecs > 512) {
+ int l = sizeof(tp->display);
+ tp->prev_tv = tv;
+ tp->avg_bytes += tp->count;
+ tp->avg_misecs += misecs;
+
+ if (tp->total > 1 << 30) {
+ l -= snprintf(tp->display, l, ", %u.%2.2u GiB",
+ (int)(tp->total >> 30),
+ (int)(tp->total & ((1 << 30) - 1)) / 10737419);
+ } else if (tp->total > 1 << 20) {
+ l -= snprintf(tp->display, l, ", %u.%2.2u MiB",
+ (int)(tp->total >> 20),
+ ((int)(tp->total & ((1 << 20) - 1))
+ * 100) >> 20);
+ } else if (tp->total > 1 << 10) {
+ l -= snprintf(tp->display, l, ", %u.%2.2u KiB",
+ (int)(tp->total >> 10),
+ ((int)(tp->total & ((1 << 10) - 1))
+ * 100) >> 10);
+ } else {
+ l -= snprintf(tp->display, l, ", %u bytes",
+ (int)tp->total);
+ }
+ snprintf(tp->display + sizeof(tp->display) - l, l,
+ " | %lu KiB/s", tp->avg_bytes / tp->avg_misecs);
+
+ tp->avg_bytes -= tp->last_bytes[tp->idx];
+ tp->avg_misecs -= tp->last_misecs[tp->idx];
+ tp->last_bytes[tp->idx] = tp->count;
+ tp->last_misecs[tp->idx] = misecs;
+ tp->idx = (tp->idx + 1) % TP_IDX_MAX;
+ tp->count = 0;
+
+ if (progress->last_value != -1 && progress_update)
+ display(progress, progress->last_value, 0);
+ }
+}
+
+int display_progress(struct progress *progress, unsigned n)
+{
+ return progress ? display(progress, n, 0) : 0;
}
-void start_progress_delay(struct progress *progress, const char *title,
- const char *prefix, unsigned total,
- unsigned percent_treshold, unsigned delay)
+struct progress *start_progress_delay(const char *title, unsigned total,
+ unsigned percent_treshold, unsigned delay)
{
- progress->prefix = prefix;
+ struct progress *progress = malloc(sizeof(*progress));
+ if (!progress) {
+ /* unlikely, but here's a good fallback */
+ fprintf(stderr, "%s...\n", title);
+ return NULL;
+ }
+ progress->title = title;
progress->total = total;
+ progress->last_value = -1;
progress->last_percent = -1;
progress->delayed_percent_treshold = percent_treshold;
- progress->delayed_title = title;
progress->delay = delay;
- progress->need_lf = 0;
+ progress->throughput = NULL;
set_progress_signal();
+ return progress;
}
-void stop_progress(struct progress *progress)
+struct progress *start_progress(const char *title, unsigned total)
{
+ return start_progress_delay(title, total, 0, 0);
+}
+
+void stop_progress(struct progress **p_progress)
+{
+ struct progress *progress = *p_progress;
+ if (!progress)
+ return;
+ *p_progress = NULL;
+ if (progress->last_value != -1) {
+ /* Force the last update */
+ progress_update = 1;
+ display(progress, progress->last_value, 1);
+ }
clear_progress_signal();
- if (progress->need_lf)
- fputc('\n', stderr);
+ free(progress->throughput);
+ free(progress);
}
#ifndef PROGRESS_H
#define PROGRESS_H
-struct progress {
- const char *prefix;
- unsigned total;
- unsigned last_percent;
- unsigned delay;
- unsigned delayed_percent_treshold;
- const char *delayed_title;
- int need_lf;
-};
+struct progress;
+void display_throughput(struct progress *progress, unsigned long n);
int display_progress(struct progress *progress, unsigned n);
-void start_progress(struct progress *progress, const char *title,
- const char *prefix, unsigned total);
-void start_progress_delay(struct progress *progress, const char *title,
- const char *prefix, unsigned total,
- unsigned percent_treshold, unsigned delay);
-void stop_progress(struct progress *progress);
+struct progress *start_progress(const char *title, unsigned total);
+struct progress *start_progress_delay(const char *title, unsigned total,
+ unsigned percent_treshold, unsigned delay);
+void stop_progress(struct progress **progress);
#endif
strcpy(buf, "remote:");
while (1) {
int band, len;
- len = packet_read_line(in_stream, buf+7, LARGE_PACKET_MAX);
+ len = packet_read_line(in_stream, buf+7, LARGE_PACKET_MAX);
if (len == 0)
break;
if (len < 1) {
return SIDEBAND_REMOTE_ERROR;
case 2:
buf[7] = ' ';
- safe_write(err, buf, 8+len);
+ len += 8;
+ while (1) {
+ int brk = 8;
+ while (brk < len) {
+ brk++;
+ if (buf[brk-1] == '\n' ||
+ buf[brk-1] == '\r')
+ break;
+ }
+ safe_write(err, buf, brk);
+ if (brk < len) {
+ memmove(buf + 8, buf + brk, len - brk);
+ len = len - brk + 8;
+ } else
+ break;
+ }
continue;
case 1:
safe_write(out, buf+8, len);
{
unsigned short mask = htons(CE_UPDATE);
unsigned cnt = 0, total = 0;
- struct progress progress;
+ struct progress *progress = NULL;
char last_symlink[PATH_MAX];
if (o->update && o->verbose_update) {
total++;
}
- start_progress_delay(&progress, "Checking %u files out...",
- "", total, 50, 2);
+ progress = start_progress_delay("Checking out files",
+ total, 50, 2);
cnt = 0;
}
while (nr--) {
struct cache_entry *ce = *src++;
- if (total)
- if (!ce->ce_mode || ce->ce_flags & mask)
- display_progress(&progress, ++cnt);
+ if (!ce->ce_mode || ce->ce_flags & mask)
+ display_progress(progress, ++cnt);
if (!ce->ce_mode) {
if (o->update)
unlink_entry(ce->name, last_symlink);
}
}
}
- if (total)
- stop_progress(&progress);;
+ stop_progress(&progress);
}
int unpack_trees(unsigned len, struct tree_desc *t, struct unpack_trees_options *o)