remote-svn: add incremental import

Search for a note attached to the ref to update and read it's
'Revision-number:'-line. Start import from the next svn revision.

If there is no next revision in the svn repo, svnrdump terminates with
a message on stderr an non-zero return value. This looks a little
weird, but there is no other way to know whether there is a new
revision in the svn repo.

On the start of an incremental import, the parent of the first commit
in the fast-import stream is set to the branch name to update. All
following commits specify their parent by a mark number. Previous mark
files are currently not reused.

Signed-off-by: Florian Achleitner <florian.achleitner.2.6.31@gmail.com>
Acked-by: David Michael Barr <b@rr-dav.id.au>
Signed-off-by: Junio C Hamano <gitster@pobox.com>

diff --git a/contrib/svn-fe/svn-fe.c b/contrib/svn-fe/svn-fe.c
index c796cc0071ec07f48c8dd1e5913b689efec799df..f363505abb278f89a94413874099e0e80ee8192c 100644 (file)
--- a/contrib/svn-fe/svn-fe.c
+++ b/contrib/svn-fe/svn-fe.c
@@ -10,7 +10,8 @@ int main(int argc, char **argv)
 {
        if (svndump_init(NULL))
                return 1;
 {
        if (svndump_init(NULL))
                return 1;

-       svndump_read((argc > 1) ? argv[1] : NULL, "refs/heads/master");
+       svndump_read((argc > 1) ? argv[1] : NULL, "refs/heads/master",
+                       "refs/notes/svn/revs");

        svndump_deinit();
        svndump_reset();
        return 0;
        svndump_deinit();
        svndump_reset();
        return 0;

diff --git a/remote-testsvn.c b/remote-testsvn.c
index 45eba9f2d9d2ff51c08b7700da8edd02c1ca5f56..b741f6d9a79a405227be0fd5c445aad259a8938b 100644 (file)
--- a/remote-testsvn.c
+++ b/remote-testsvn.c
@@ -12,7 +12,8 @@ static const char *url;
 static int dump_from_file;
 static const char *private_ref;
 static const char *remote_ref = "refs/heads/master";
 static int dump_from_file;
 static const char *private_ref;
 static const char *remote_ref = "refs/heads/master";

-static const char *marksfilename;
+static const char *marksfilename, *notes_ref;
+struct rev_note { unsigned int rev_nr; };

 
 static int cmd_capabilities(const char *line);
 static int cmd_import(const char *line);
 
 static int cmd_capabilities(const char *line);
 static int cmd_import(const char *line);


@@ -48,14 +49,79 @@ static void terminate_batch(void)
        fflush(stdout);
 }
 
        fflush(stdout);
 }
 

+/* NOTE: 'ref' refers to a git reference, while 'rev' refers to a svn revision. */
+static char *read_ref_note(const unsigned char sha1[20])
+{
+       const unsigned char *note_sha1;
+       char *msg = NULL;
+       unsigned long msglen;
+       enum object_type type;
+
+       init_notes(NULL, notes_ref, NULL, 0);
+       if (!(note_sha1 = get_note(NULL, sha1)))
+               return NULL;    /* note tree not found */
+       if (!(msg = read_sha1_file(note_sha1, &type, &msglen)))
+               error("Empty notes tree. %s", notes_ref);
+       else if (!msglen || type != OBJ_BLOB) {
+               error("Note contains unusable content. "
+                       "Is something else using this notes tree? %s", notes_ref);
+               free(msg);
+               msg = NULL;
+       }
+       free_notes(NULL);
+       return msg;
+}
+
+static int parse_rev_note(const char *msg, struct rev_note *res)
+{
+       const char *key, *value, *end;
+       size_t len;
+
+       while (*msg) {
+               end = strchr(msg, '\n');
+               len = end ? end - msg : strlen(msg);
+
+               key = "Revision-number: ";
+               if (!prefixcmp(msg, key)) {
+                       long i;
+                       char *end;
+                       value = msg + strlen(key);
+                       i = strtol(value, &end, 0);
+                       if (end == value || i < 0 || i > UINT32_MAX)
+                               return -1;
+                       res->rev_nr = i;
+               }
+               msg += len + 1;
+       }
+       return 0;
+}
+

 static int cmd_import(const char *line)
 {
        int code;
        int dumpin_fd;
 static int cmd_import(const char *line)
 {
        int code;
        int dumpin_fd;

-       unsigned int startrev = 0;
+       char *note_msg;
+       unsigned char head_sha1[20];
+       unsigned int startrev;

        struct argv_array svndump_argv = ARGV_ARRAY_INIT;
        struct child_process svndump_proc;
 
        struct argv_array svndump_argv = ARGV_ARRAY_INIT;
        struct child_process svndump_proc;
 

+       if (read_ref(private_ref, head_sha1))
+               startrev = 0;
+       else {
+               note_msg = read_ref_note(head_sha1);
+               if(note_msg == NULL) {
+                       warning("No note found for %s.", private_ref);
+                       startrev = 0;
+               } else {
+                       struct rev_note note = { 0 };
+                       if (parse_rev_note(note_msg, &note))
+                               die("Revision number couldn't be parsed from note.");
+                       startrev = note.rev_nr + 1;
+                       free(note_msg);
+               }
+       }
+

        if (dump_from_file) {
                dumpin_fd = open(url, O_RDONLY);
                if(dumpin_fd < 0)
        if (dump_from_file) {
                dumpin_fd = open(url, O_RDONLY);
                if(dumpin_fd < 0)


@@ -79,7 +145,7 @@ static int cmd_import(const char *line)
                        "feature export-marks=%s\n", marksfilename, marksfilename);
 
        svndump_init_fd(dumpin_fd, STDIN_FILENO);
                        "feature export-marks=%s\n", marksfilename, marksfilename);
 
        svndump_init_fd(dumpin_fd, STDIN_FILENO);

-       svndump_read(url, private_ref);
+       svndump_read(url, private_ref, notes_ref);

        svndump_deinit();
        svndump_reset();
 
        svndump_deinit();
        svndump_reset();
 


@@ -150,7 +216,8 @@ static int do_command(struct strbuf *line)
 int main(int argc, const char **argv)
 {
        struct strbuf buf = STRBUF_INIT, url_sb = STRBUF_INIT,
 int main(int argc, const char **argv)
 {
        struct strbuf buf = STRBUF_INIT, url_sb = STRBUF_INIT,

-                       private_ref_sb = STRBUF_INIT, marksfilename_sb = STRBUF_INIT;
+                       private_ref_sb = STRBUF_INIT, marksfilename_sb = STRBUF_INIT,
+                       notes_ref_sb = STRBUF_INIT;

        static struct remote *remote;
        const char *url_in;
 
        static struct remote *remote;
        const char *url_in;
 


@@ -176,6 +243,9 @@ int main(int argc, const char **argv)
        strbuf_addf(&private_ref_sb, "refs/svn/%s/master", remote->name);
        private_ref = private_ref_sb.buf;
 
        strbuf_addf(&private_ref_sb, "refs/svn/%s/master", remote->name);
        private_ref = private_ref_sb.buf;
 

+       strbuf_addf(&notes_ref_sb, "refs/notes/%s/revs", remote->name);
+       notes_ref = notes_ref_sb.buf;
+

        strbuf_addf(&marksfilename_sb, "%s/info/fast-import/remote-svn/%s.marks",
                get_git_dir(), remote->name);
        marksfilename = marksfilename_sb.buf;
        strbuf_addf(&marksfilename_sb, "%s/info/fast-import/remote-svn/%s.marks",
                get_git_dir(), remote->name);
        marksfilename = marksfilename_sb.buf;


@@ -195,6 +265,7 @@ int main(int argc, const char **argv)
        strbuf_release(&buf);
        strbuf_release(&url_sb);
        strbuf_release(&private_ref_sb);
        strbuf_release(&buf);
        strbuf_release(&url_sb);
        strbuf_release(&private_ref_sb);

+       strbuf_release(&notes_ref_sb);

        strbuf_release(&marksfilename_sb);
        return 0;
 }
        strbuf_release(&marksfilename_sb);
        return 0;
 }

diff --git a/test-svn-fe.c b/test-svn-fe.c
index cb0d80f40c58b6f197977ba139f9c1647182c79f..0f2d9a4a3d8ea732c7c61e3818bd0bbe417c76b0 100644 (file)
--- a/test-svn-fe.c
+++ b/test-svn-fe.c
@@ -40,7 +40,7 @@ int main(int argc, char *argv[])
        if (argc == 2) {
                if (svndump_init(argv[1]))
                        return 1;
        if (argc == 2) {
                if (svndump_init(argv[1]))
                        return 1;

-               svndump_read(NULL, "refs/heads/master");
+               svndump_read(NULL, "refs/heads/master", "refs/notes/svn/revs");

                svndump_deinit();
                svndump_reset();
                return 0;
                svndump_deinit();
                svndump_reset();
                return 0;

diff --git a/vcs-svn/fast_export.c b/vcs-svn/fast_export.c
index df51c59ca5415387d593b5a726de9fcb1bffd7dc..f2b23c81de11a63c2c1c2ac925be9081ad5bea5c 100644 (file)
--- a/vcs-svn/fast_export.c
+++ b/vcs-svn/fast_export.c
@@ -68,13 +68,19 @@ void fast_export_modify(const char *path, uint32_t mode, const char *dataref)
 }
 
 void fast_export_begin_note(uint32_t revision, const char *author,
 }
 
 void fast_export_begin_note(uint32_t revision, const char *author,

-               const char *log, unsigned long timestamp)
+               const char *log, unsigned long timestamp, const char *note_ref)

 {
 {

+       static int firstnote = 1;

        size_t loglen = strlen(log);
        size_t loglen = strlen(log);

-       printf("commit refs/notes/svn/revs\n");
+       printf("commit %s\n", note_ref);

        printf("committer %s <%s@%s> %ld +0000\n", author, author, "local", timestamp);
        printf("data %"PRIuMAX"\n", (uintmax_t)loglen);
        fwrite(log, loglen, 1, stdout);
        printf("committer %s <%s@%s> %ld +0000\n", author, author, "local", timestamp);
        printf("data %"PRIuMAX"\n", (uintmax_t)loglen);
        fwrite(log, loglen, 1, stdout);

+       if (firstnote) {
+               if (revision > 1)
+                       printf("from %s^0", note_ref);
+               firstnote = 0;
+       }

        fputc('\n', stdout);
 }
 
        fputc('\n', stdout);
 }
 

diff --git a/vcs-svn/fast_export.h b/vcs-svn/fast_export.h
index c2f6f11b12ae2b2e588b6b181a0dd7d5a84f6f96..c8b5adb811c7ba5652b94a6cf6f9ea54d6d9335f 100644 (file)
--- a/vcs-svn/fast_export.h
+++ b/vcs-svn/fast_export.h
@@ -11,10 +11,10 @@ void fast_export_delete(const char *path);
 void fast_export_modify(const char *path, uint32_t mode, const char *dataref);
 void fast_export_note(const char *committish, const char *dataref);
 void fast_export_begin_note(uint32_t revision, const char *author,
 void fast_export_modify(const char *path, uint32_t mode, const char *dataref);
 void fast_export_note(const char *committish, const char *dataref);
 void fast_export_begin_note(uint32_t revision, const char *author,

-               const char *log, unsigned long timestamp);
+               const char *log, unsigned long timestamp, const char *note_ref);

 void fast_export_begin_commit(uint32_t revision, const char *author,
 void fast_export_begin_commit(uint32_t revision, const char *author,

-                       const struct strbuf *log, const char *uuid,
-                       const char *url, unsigned long timestamp, const char *local_ref);
+                       const struct strbuf *log, const char *uuid,const char *url,
+                       unsigned long timestamp, const char *local_ref);

 void fast_export_end_commit(uint32_t revision);
 void fast_export_data(uint32_t mode, off_t len, struct line_buffer *input);
 void fast_export_buf_to_data(const struct strbuf *data);
 void fast_export_end_commit(uint32_t revision);
 void fast_export_data(uint32_t mode, off_t len, struct line_buffer *input);
 void fast_export_buf_to_data(const struct strbuf *data);

diff --git a/vcs-svn/svndump.c b/vcs-svn/svndump.c
index 7ec1a5b99381f4de02b74bf33760db5a9251858d..31d1d83d451e0e78f19067799d14e555a62e2cc8 100644 (file)
--- a/vcs-svn/svndump.c
+++ b/vcs-svn/svndump.c
@@ -309,20 +309,20 @@ static void begin_revision(const char *remote_ref)
                rev_ctx.timestamp, remote_ref);
 }
 
                rev_ctx.timestamp, remote_ref);
 }
 

-static void end_revision(void)
+static void end_revision(const char *note_ref)

 {
        struct strbuf mark = STRBUF_INIT;
        if (rev_ctx.revision) {
                fast_export_end_commit(rev_ctx.revision);
                fast_export_begin_note(rev_ctx.revision, "remote-svn",
 {
        struct strbuf mark = STRBUF_INIT;
        if (rev_ctx.revision) {
                fast_export_end_commit(rev_ctx.revision);
                fast_export_begin_note(rev_ctx.revision, "remote-svn",

-                               "Note created by remote-svn.", rev_ctx.timestamp);
+                               "Note created by remote-svn.", rev_ctx.timestamp, note_ref);

                strbuf_addf(&mark, ":%"PRIu32, rev_ctx.revision);
                fast_export_note(mark.buf, "inline");
                fast_export_buf_to_data(&rev_ctx.note);
        }
 }
 
                strbuf_addf(&mark, ":%"PRIu32, rev_ctx.revision);
                fast_export_note(mark.buf, "inline");
                fast_export_buf_to_data(&rev_ctx.note);
        }
 }
 

-void svndump_read(const char *url, const char *local_ref)
+void svndump_read(const char *url, const char *local_ref, const char *notes_ref)

 {
        char *val;
        char *t;
 {
        char *val;
        char *t;


@@ -363,7 +363,7 @@ void svndump_read(const char *url, const char *local_ref)
                        if (active_ctx == REV_CTX)
                                begin_revision(local_ref);
                        if (active_ctx != DUMP_CTX)
                        if (active_ctx == REV_CTX)
                                begin_revision(local_ref);
                        if (active_ctx != DUMP_CTX)

-                               end_revision();
+                               end_revision(notes_ref);

                        active_ctx = REV_CTX;
                        reset_rev_ctx(atoi(val));
                        strbuf_addf(&rev_ctx.note, "%s\n", t);
                        active_ctx = REV_CTX;
                        reset_rev_ctx(atoi(val));
                        strbuf_addf(&rev_ctx.note, "%s\n", t);


@@ -479,7 +479,7 @@ void svndump_read(const char *url, const char *local_ref)
        if (active_ctx == REV_CTX)
                begin_revision(local_ref);
        if (active_ctx != DUMP_CTX)
        if (active_ctx == REV_CTX)
                begin_revision(local_ref);
        if (active_ctx != DUMP_CTX)

-               end_revision();
+               end_revision(notes_ref);

 }
 
 static void init(int report_fd)
 }
 
 static void init(int report_fd)

diff --git a/vcs-svn/svndump.h b/vcs-svn/svndump.h
index febeecbe9da82244fa6818f8cd611ea11b491e38..b8eb12954e816e0e1490f6ef7eefe64901c38ae1 100644 (file)
--- a/vcs-svn/svndump.h
+++ b/vcs-svn/svndump.h
@@ -3,7 +3,7 @@
 
 int svndump_init(const char *filename);
 int svndump_init_fd(int in_fd, int back_fd);
 
 int svndump_init(const char *filename);
 int svndump_init_fd(int in_fd, int back_fd);

-void svndump_read(const char *url, const char *local_ref);
+void svndump_read(const char *url, const char *local_ref, const char *notes_ref);

 void svndump_deinit(void);
 void svndump_reset(void);
 
 void svndump_deinit(void);
 void svndump_reset(void);