revision.o pager.o tree-walk.o xdiff-interface.o \
write_or_die.o trace.o list-objects.o grep.o \
alloc.o merge-file.o path-list.o help.o unpack-trees.o $(DIFF_OBJS) \
- color.o wt-status.o archive-zip.o archive-tar.o \
- utf8.o
+ color.o wt-status.o archive-zip.o archive-tar.o shallow.o utf8.o
BUILTIN_OBJS = \
builtin-add.o \
#include "cache.h"
#include "tag.h"
#include "commit.h"
+#include "pkt-line.h"
int save_commit_buffer = 1;
return;
graft_file = get_graft_file();
read_graft_file(graft_file);
+ /* make sure shallows are read */
+ is_repository_shallow();
commit_graft_prepared = 1;
}
return commit_graft[pos];
}
+int write_shallow_commits(int fd, int use_pack_protocol)
+{
+ int i, count = 0;
+ for (i = 0; i < commit_graft_nr; i++)
+ if (commit_graft[i]->nr_parent < 0) {
+ const char *hex =
+ sha1_to_hex(commit_graft[i]->sha1);
+ count++;
+ if (use_pack_protocol)
+ packet_write(fd, "shallow %s", hex);
+ else {
+ write(fd, hex, 40);
+ write(fd, "\n", 1);
+ }
+ }
+ return count;
+}
+
+int unregister_shallow(const unsigned char *sha1)
+{
+ int pos = commit_graft_pos(sha1);
+ if (pos < 0)
+ return -1;
+ if (pos + 1 < commit_graft_nr)
+ memcpy(commit_graft + pos, commit_graft + pos + 1,
+ sizeof(struct commit_graft *)
+ * (commit_graft_nr - pos - 1));
+ commit_graft_nr--;
+ return 0;
+}
+
int parse_commit_buffer(struct commit *item, void *buffer, unsigned long size)
{
char *tail = buffer;
struct commit_graft {
unsigned char sha1[20];
- int nr_parent;
+ int nr_parent; /* < 0 if shallow commit */
unsigned char parent[FLEX_ARRAY][20]; /* more */
};
extern struct commit_list *get_merge_bases(struct commit *rev1, struct commit *rev2, int cleanup);
+extern int register_shallow(const unsigned char *sha1);
+extern int unregister_shallow(const unsigned char *sha1);
+extern int write_shallow_commits(int fd, int use_pack_protocol);
+extern int is_repository_shallow();
+extern struct commit_list *get_shallow_commits(struct object_array *heads,
+ int depth, int shallow_flag, int not_shallow_flag);
+
int in_merge_bases(struct commit *rev1, struct commit *rev2);
#endif /* COMMIT_H */
static int quiet;
static int verbose;
static int fetch_all;
+static int depth;
static const char fetch_pack_usage[] =
-"git-fetch-pack [--all] [-q] [-v] [-k] [--thin] [--exec=upload-pack] [host:]directory <refs>...";
+"git-fetch-pack [--all] [-q] [-v] [-k] [--thin] [--exec=upload-pack] [--depth=<n>] [host:]directory <refs>...";
static const char *exec = "git-upload-pack";
#define COMPLETE (1U << 0)
packet_write(fd[1], "want %s\n", sha1_to_hex(remote));
fetching++;
}
+ if (is_repository_shallow())
+ write_shallow_commits(fd[1], 1);
+ if (depth > 0)
+ packet_write(fd[1], "deepen %d", depth);
packet_flush(fd[1]);
if (!fetching)
return 1;
+ if (depth > 0) {
+ char line[1024];
+ unsigned char sha1[20];
+ int len;
+
+ while ((len = packet_read_line(fd[0], line, sizeof(line)))) {
+ if (!strncmp("shallow ", line, 8)) {
+ if (get_sha1_hex(line + 8, sha1))
+ die("invalid shallow line: %s", line);
+ register_shallow(sha1);
+ continue;
+ }
+ if (!strncmp("unshallow ", line, 10)) {
+ if (get_sha1_hex(line + 10, sha1))
+ die("invalid unshallow line: %s", line);
+ if (!lookup_object(sha1))
+ die("object not found: %s", line);
+ /* make sure that it is parsed as shallow */
+ parse_object(sha1);
+ if (unregister_shallow(sha1))
+ die("no shallow found: %s", line);
+ continue;
+ }
+ die("expected shallow/unshallow, got %s", line);
+ }
+ }
+
flushes = 0;
retval = -1;
while ((sha1 = get_rev())) {
if (!memcmp(ref->name, "refs/", 5) &&
check_ref_format(ref->name + 5))
; /* trash */
- else if (fetch_all) {
+ else if (fetch_all &&
+ (!depth || strncmp(ref->name, "refs/tags/", 10) )) {
*newtail = ref;
ref->next = NULL;
newtail = &ref->next;
}
}
- for_each_ref(mark_complete, NULL);
- if (cutoff)
- mark_recent_complete_commits(cutoff);
+ if (!depth) {
+ for_each_ref(mark_complete, NULL);
+ if (cutoff)
+ mark_recent_complete_commits(cutoff);
+ }
/*
* Mark all complete remote refs as common refs.
int status;
get_remote_heads(fd[0], &ref, 0, NULL, 0);
+ if (is_repository_shallow() && !server_supports("shallow"))
+ die("Server does not support shallow clients");
if (server_supports("multi_ack")) {
if (verbose)
fprintf(stderr, "Server supports multi_ack\n");
char *dest = NULL, **heads;
int fd[2];
pid_t pid;
+ struct stat st;
+ struct lock_file lock;
setup_git_directory();
verbose = 1;
continue;
}
+ if (!strncmp("--depth=", arg, 8)) {
+ depth = strtol(arg + 8, NULL, 0);
+ if (stat(git_path("shallow"), &st))
+ st.st_mtime = 0;
+ continue;
+ }
usage(fetch_pack_usage);
}
dest = arg;
}
}
+ if (!ret && depth > 0) {
+ struct cache_time mtime;
+ char *shallow = git_path("shallow");
+ int fd;
+
+ mtime.sec = st.st_mtime;
+#ifdef USE_NSEC
+ mtime.usec = st.st_mtim.usec;
+#endif
+ if (stat(shallow, &st)) {
+ if (mtime.sec)
+ die("shallow file was removed during fetch");
+ } else if (st.st_mtime != mtime.sec
+#ifdef USE_NSEC
+ || st.st_mtim.usec != mtime.usec
+#endif
+ )
+ die("shallow file was changed during fetch");
+
+ fd = hold_lock_file_for_update(&lock, shallow, 1);
+ if (!write_shallow_commits(fd, 0)) {
+ unlink(shallow);
+ rollback_lock_file(&lock);
+ } else {
+ close(fd);
+ commit_lock_file(&lock);
+ }
+ }
+
return !!ret;
}
}
usage() {
- die "Usage: $0 [--template=<template_directory>] [--reference <reference-repo>] [--bare] [-l [-s]] [-q] [-u <upload-pack>] [--origin <name>] [-n] <repo> [<dir>]"
+ die "Usage: $0 [--template=<template_directory>] [--reference <reference-repo>] [--bare] [-l [-s]] [-q] [-u <upload-pack>] [--origin <name>] [--depth <n>] [-n] <repo> [<dir>]"
}
get_repo_base() {
origin=
origin_override=
use_separate_remote=t
+depth=
while
case "$#,$1" in
0,*) break ;;
*,-u|*,--upload-pack)
shift
upload_pack="--exec=$1" ;;
+ 1,--depth) usage;;
+ *,--depth)
+ shift
+ depth="--depth=$1";;
*,-*) usage ;;
*) break ;;
esac
*)
case "$repo" in
rsync://*)
+ case "$depth" in
+ "") ;;
+ *) die "shallow over rsync not supported" ;;
+ esac
rsync $quiet -av --ignore-existing \
--exclude info "$repo/objects/" "$GIT_DIR/objects/" ||
exit
git-ls-remote "$repo" >"$GIT_DIR/CLONE_HEAD" || exit 1
;;
https://*|http://*|ftp://*)
+ case "$depth" in
+ "") ;;
+ *) die "shallow over http or ftp not supported" ;;
+ esac
if test -z "@@NO_CURL@@"
then
clone_dumb_http "$repo" "$D"
;;
*)
case "$upload_pack" in
- '') git-fetch-pack --all -k $quiet "$repo" ;;
- *) git-fetch-pack --all -k $quiet "$upload_pack" "$repo" ;;
+ '') git-fetch-pack --all -k $quiet $depth "$repo" ;;
+ *) git-fetch-pack --all -k $quiet "$upload_pack" $depth "$repo" ;;
esac >"$GIT_DIR/CLONE_HEAD" ||
die "fetch-pack from '$repo' failed."
;;
exec=
upload_pack=
keep=
+shallow_depth=
while case "$#" in 0) break ;; esac
do
case "$1" in
--reflog-action=*)
rloga=`expr "z$1" : 'z-[^=]*=\(.*\)'`
;;
+ --depth=*)
+ shallow_depth="--depth=`expr "z$1" : 'z-[^=]*=\(.*\)'`"
+ ;;
+ --depth)
+ shift
+ shallow_depth="--depth=$1"
+ ;;
-*)
usage
;;
# There are transports that can fetch only one head at a time...
case "$remote" in
http://* | https://* | ftp://*)
+ test -n "$shallow_depth" &&
+ die "shallow clone with http not supported"
proto=`expr "$remote" : '\([^:]*\):'`
if [ -n "$GIT_SSL_NO_VERIFY" ]; then
curl_extra_args="-k"
git-http-fetch -v -a "$head" "$remote/" || exit
;;
rsync://*)
+ test -n "$shallow_depth" &&
+ die "shallow clone with rsync not supported"
TMP_HEAD="$GIT_DIR/TMP_HEAD"
rsync -L -q "$remote/$remote_name" "$TMP_HEAD" || exit 1
head=$(git-rev-parse --verify TMP_HEAD)
pack_lockfile=
IFS=" $LF"
(
- git-fetch-pack --thin $exec $keep "$remote" $rref || echo failed "$remote"
+ git-fetch-pack --thin $exec $keep $shallow_depth "$remote" $rref || echo failed "$remote"
) |
while read sha1 remote_name
do
case "$taglist" in
'') ;;
?*)
+ # do not deepen a shallow tree when following tags
+ shallow_depth=
fetch_main "$taglist" || exit ;;
esac
esac
--- /dev/null
+#include "cache.h"
+#include "commit.h"
+#include "tag.h"
+
+static int is_shallow = -1;
+
+int register_shallow(const unsigned char *sha1)
+{
+ struct commit_graft *graft =
+ xmalloc(sizeof(struct commit_graft));
+ struct commit *commit = lookup_commit(sha1);
+
+ hashcpy(graft->sha1, sha1);
+ graft->nr_parent = -1;
+ if (commit && commit->object.parsed)
+ commit->parents = NULL;
+ return register_commit_graft(graft, 0);
+}
+
+int is_repository_shallow()
+{
+ FILE *fp;
+ char buf[1024];
+
+ if (is_shallow >= 0)
+ return is_shallow;
+
+ fp = fopen(git_path("shallow"), "r");
+ if (!fp) {
+ is_shallow = 0;
+ return is_shallow;
+ }
+ is_shallow = 1;
+
+ while (fgets(buf, sizeof(buf), fp)) {
+ unsigned char sha1[20];
+ if (get_sha1_hex(buf, sha1))
+ die("bad shallow line: %s", buf);
+ register_shallow(sha1);
+ }
+ fclose(fp);
+ return is_shallow;
+}
+
+struct commit_list *get_shallow_commits(struct object_array *heads, int depth,
+ int shallow_flag, int not_shallow_flag)
+{
+ int i = 0, cur_depth = 0;
+ struct commit_list *result = NULL;
+ struct object_array stack = {0, 0, NULL};
+ struct commit *commit = NULL;
+
+ while (commit || i < heads->nr || stack.nr) {
+ struct commit_list *p;
+ if (!commit) {
+ if (i < heads->nr) {
+ commit = (struct commit *)
+ deref_tag(heads->objects[i++].item, NULL, 0);
+ if (commit->object.type != OBJ_COMMIT) {
+ commit = NULL;
+ continue;
+ }
+ if (!commit->util)
+ commit->util = xmalloc(sizeof(int));
+ *(int *)commit->util = 0;
+ cur_depth = 0;
+ } else {
+ commit = (struct commit *)
+ stack.objects[--stack.nr].item;
+ cur_depth = *(int *)commit->util;
+ }
+ }
+ parse_commit(commit);
+ commit->object.flags |= not_shallow_flag;
+ cur_depth++;
+ for (p = commit->parents, commit = NULL; p; p = p->next) {
+ if (!p->item->util) {
+ int *pointer = xmalloc(sizeof(int));
+ p->item->util = pointer;
+ *pointer = cur_depth;
+ } else {
+ int *pointer = p->item->util;
+ if (cur_depth >= *pointer)
+ continue;
+ *pointer = cur_depth;
+ }
+ if (cur_depth < depth) {
+ if (p->next)
+ add_object_array(&p->item->object,
+ NULL, &stack);
+ else {
+ commit = p->item;
+ cur_depth = *(int *)commit->util;
+ }
+ } else {
+ commit_list_insert(p->item, &result);
+ p->item->object.flags |= shallow_flag;
+ }
+ }
+ }
+
+ return result;
+}
+
pull_to_client 3rd "A" $((1*3)) # old fails
+test_expect_success "clone shallow" "git-clone --depth 2 . shallow"
+
+(cd shallow; git-count-objects -v) > count.shallow
+
+test_expect_success "clone shallow object count" \
+ "test \"in-pack: 18\" = \"$(grep in-pack count.shallow)\""
+
+count_output () {
+ sed -e '/^in-pack:/d' -e '/^packs:/d' -e '/: 0$/d' "$1"
+}
+
+test_expect_success "clone shallow object count (part 2)" '
+ test -z "$(count_output count.shallow)"
+'
+
+test_expect_success "fsck in shallow repo" \
+ "(cd shallow; git-fsck-objects --full)"
+
+#test_done; exit
+
+add B66 $B65
+add B67 $B66
+
+test_expect_success "pull in shallow repo" \
+ "(cd shallow; git pull .. B)"
+
+(cd shallow; git-count-objects -v) > count.shallow
+test_expect_success "clone shallow object count" \
+ "test \"count: 6\" = \"$(grep count count.shallow)\""
+
+add B68 $B67
+add B69 $B68
+
+test_expect_success "deepening pull in shallow repo" \
+ "(cd shallow; git pull --depth 4 .. B)"
+
+(cd shallow; git-count-objects -v) > count.shallow
+test_expect_success "clone shallow object count" \
+ "test \"count: 12\" = \"$(grep count count.shallow)\""
+
+test_expect_success "deepening fetch in shallow repo" \
+ "(cd shallow; git fetch --depth 4 .. A:A)"
+
+(cd shallow; git-count-objects -v) > count.shallow
+test_expect_success "clone shallow object count" \
+ "test \"count: 18\" = \"$(grep count count.shallow)\""
+
+test_expect_failure "pull in shallow repo with missing merge base" \
+ "(cd shallow; git pull --depth 4 .. A)"
+
test_done
#include "object.h"
#include "commit.h"
#include "exec_cmd.h"
+#include "diff.h"
+#include "revision.h"
+#include "list-objects.h"
static const char upload_pack_usage[] = "git-upload-pack [--strict] [--timeout=nn] <dir>";
#define COMMON_KNOWN (1u << 14)
#define REACHABLE (1u << 15)
+#define SHALLOW (1u << 16)
+#define NOT_SHALLOW (1u << 17)
+#define CLIENT_SHALLOW (1u << 18)
+
static unsigned long oldest_have;
static int multi_ack, nr_our_refs;
return safe_write(fd, data, sz);
}
+FILE *pack_pipe = NULL;
+static void show_commit(struct commit *commit)
+{
+ if (commit->object.flags & BOUNDARY)
+ fputc('-', pack_pipe);
+ if (fputs(sha1_to_hex(commit->object.sha1), pack_pipe) < 0)
+ die("broken output pipe");
+ fputc('\n', pack_pipe);
+ fflush(pack_pipe);
+ free(commit->buffer);
+ commit->buffer = NULL;
+}
+
+static void show_object(struct object_array_entry *p)
+{
+ /* An object with name "foo\n0000000..." can be used to
+ * confuse downstream git-pack-objects very badly.
+ */
+ const char *ep = strchr(p->name, '\n');
+ if (ep) {
+ fprintf(pack_pipe, "%s %.*s\n", sha1_to_hex(p->item->sha1),
+ (int) (ep - p->name),
+ p->name);
+ }
+ else
+ fprintf(pack_pipe, "%s %s\n",
+ sha1_to_hex(p->item->sha1), p->name);
+}
+
+static void show_edge(struct commit *commit)
+{
+ fprintf(pack_pipe, "-%s\n", sha1_to_hex(commit->object.sha1));
+}
+
static void create_pack_file(void)
{
/* Pipes between rev-list to pack-objects, pack-objects to us
if (!pid_rev_list) {
int i;
- int args;
- const char **argv;
- const char **p;
- char *buf;
+ struct rev_info revs;
- if (create_full_pack) {
- args = 10;
- use_thin_pack = 0; /* no point doing it */
- }
- else
- args = have_obj.nr + want_obj.nr + 5;
- p = xmalloc(args * sizeof(char *));
- argv = (const char **) p;
- buf = xmalloc(args * 45);
+ pack_pipe = fdopen(lp_pipe[1], "w");
- dup2(lp_pipe[1], 1);
- close(0);
- close(lp_pipe[0]);
- close(lp_pipe[1]);
- *p++ = "rev-list";
- *p++ = use_thin_pack ? "--objects-edge" : "--objects";
if (create_full_pack)
- *p++ = "--all";
- else {
+ use_thin_pack = 0; /* no point doing it */
+ init_revisions(&revs, NULL);
+ revs.tag_objects = 1;
+ revs.tree_objects = 1;
+ revs.blob_objects = 1;
+ if (use_thin_pack)
+ revs.edge_hint = 1;
+
+ if (create_full_pack) {
+ const char *args[] = {"rev-list", "--all", NULL};
+ setup_revisions(2, args, &revs, NULL);
+ } else {
for (i = 0; i < want_obj.nr; i++) {
struct object *o = want_obj.objects[i].item;
- *p++ = buf;
- memcpy(buf, sha1_to_hex(o->sha1), 41);
- buf += 41;
+ /* why??? */
+ o->flags &= ~UNINTERESTING;
+ add_pending_object(&revs, o, NULL);
}
- }
- if (!create_full_pack)
for (i = 0; i < have_obj.nr; i++) {
struct object *o = have_obj.objects[i].item;
- *p++ = buf;
- *buf++ = '^';
- memcpy(buf, sha1_to_hex(o->sha1), 41);
- buf += 41;
+ o->flags |= UNINTERESTING;
+ add_pending_object(&revs, o, NULL);
}
- *p++ = NULL;
- execv_git_cmd(argv);
- die("git-upload-pack: unable to exec git-rev-list");
+ setup_revisions(0, NULL, &revs, NULL);
+ }
+ prepare_revision_walk(&revs);
+ mark_edges_uninteresting(revs.commits, &revs, show_edge);
+ traverse_commit_list(&revs, show_commit, show_object);
+ exit(0);
}
if (pipe(pu_pipe) < 0)
static void receive_needs(void)
{
+ struct object_array shallows = {0, 0, NULL};
static char line[1000];
- int len;
+ int len, depth = 0;
for (;;) {
struct object *o;
len = packet_read_line(0, line, sizeof(line));
reset_timeout();
if (!len)
- return;
+ break;
+ if (!strncmp("shallow ", line, 8)) {
+ unsigned char sha1[20];
+ struct object *object;
+ use_thin_pack = 0;
+ if (get_sha1(line + 8, sha1))
+ die("invalid shallow line: %s", line);
+ object = parse_object(sha1);
+ if (!object)
+ die("did not find object for %s", line);
+ object->flags |= CLIENT_SHALLOW;
+ add_object_array(object, NULL, &shallows);
+ continue;
+ }
+ if (!strncmp("deepen ", line, 7)) {
+ char *end;
+ use_thin_pack = 0;
+ depth = strtol(line + 7, &end, 0);
+ if (end == line + 7 || depth <= 0)
+ die("Invalid deepen: %s", line);
+ continue;
+ }
if (strncmp("want ", line, 5) ||
get_sha1_hex(line+5, sha1_buf))
die("git-upload-pack: protocol error, "
add_object_array(o, NULL, &want_obj);
}
}
+ if (depth == 0 && shallows.nr == 0)
+ return;
+ if (depth > 0) {
+ struct commit_list *result, *backup;
+ int i;
+ backup = result = get_shallow_commits(&want_obj, depth,
+ SHALLOW, NOT_SHALLOW);
+ while (result) {
+ struct object *object = &result->item->object;
+ if (!(object->flags & (CLIENT_SHALLOW|NOT_SHALLOW))) {
+ packet_write(1, "shallow %s",
+ sha1_to_hex(object->sha1));
+ register_shallow(object->sha1);
+ }
+ result = result->next;
+ }
+ free_commit_list(backup);
+ for (i = 0; i < shallows.nr; i++) {
+ struct object *object = shallows.objects[i].item;
+ if (object->flags & NOT_SHALLOW) {
+ struct commit_list *parents;
+ packet_write(1, "unshallow %s",
+ sha1_to_hex(object->sha1));
+ object->flags &= ~CLIENT_SHALLOW;
+ /* make sure the real parents are parsed */
+ unregister_shallow(object->sha1);
+ object->parsed = 0;
+ parse_commit((struct commit *)object);
+ parents = ((struct commit *)object)->parents;
+ while (parents) {
+ add_object_array(&parents->item->object,
+ NULL, &want_obj);
+ parents = parents->next;
+ }
+ }
+ /* make sure commit traversal conforms to client */
+ register_shallow(object->sha1);
+ }
+ packet_flush(1);
+ } else
+ if (shallows.nr > 0) {
+ int i;
+ for (i = 0; i < shallows.nr; i++)
+ register_shallow(shallows.objects[i].item->sha1);
+ }
+ free(shallows.objects);
}
static int send_ref(const char *refname, const unsigned char *sha1, int flag, void *cb_data)
{
- static const char *capabilities = "multi_ack thin-pack side-band side-band-64k ofs-delta";
+ static const char *capabilities = "multi_ack thin-pack side-band"
+ " side-band-64k ofs-delta shallow";
struct object *o = parse_object(sha1);
if (!o)