Allow frontends to bidirectionally communicate with fast-import

The existing checkpoint command is very useful to force fast-import
to dump the branches out to disk so that standard Git tools can
access them and the objects they refer to.  However there was not a
way to know when fast-import had finished executing the checkpoint
and it was safe to read those refs.

The progress command can be used to make fast-import output any
message of the frontend's choosing to standard out.  The frontend
can scan for these messages using select() or poll() to monitor a
pipe connected to the standard output of fast-import.

Signed-off-by: Shawn O. Pearce <spearce@spearce.org>

diff --git a/Documentation/git-fast-import.txt b/Documentation/git-fast-import.txt
index 1644b90ceafb3ca968f3fea3036e135cf17280d6..0a019dd2e5f9b20ae3169cea7dee16dedf892856 100644 (file)
--- a/Documentation/git-fast-import.txt
+++ b/Documentation/git-fast-import.txt
@@ -298,6 +298,11 @@ and control the current import process.  More detailed discussion
        This command is optional and is not needed to perform
        an import.
 
        This command is optional and is not needed to perform
        an import.
 

+`progress`::
+       Causes fast-import to echo the entire line to its own
+       standard output.  This command is optional and is not needed
+       to perform an import.
+

 `commit`
 ~~~~~~~~
 Create or update a branch with a new commit, recording one logical
 `commit`
 ~~~~~~~~
 Create or update a branch with a new commit, recording one logical


@@ -775,6 +780,31 @@ explicit checkpointing may not be necessary.
 
 The `LF` after the command is optional (it used to be required).
 
 
 The `LF` after the command is optional (it used to be required).
 

+`progress`
+~~~~~~~~~~
+Causes fast-import to print the entire `progress` line unmodified to
+its standard output channel (file descriptor 1) when the command is
+processed from the input stream.  The command otherwise has no impact
+on the current import, or on any of fast-import's internal state.
+
+....
+       'progress' SP <any> LF
+       LF?
+....
+
+The `<any>` part of the command may contain any sequence of bytes
+that does not contain `LF`.  The `LF` after the command is optional.
+Callers may wish to process the output through a tool such as sed to
+remove the leading part of the line, for example:
+
+====
+       frontend | git-fast-import | sed 's/^progress //'
+====
+
+Placing a `progress` command immediately after a `checkpoint` will
+inform the reader when the `checkpoint` has been completed and it
+can safely access the refs that fast-import updated.
+

 Tips and Tricks
 ---------------
 The following tips and tricks have been collected from various
 Tips and Tricks
 ---------------
 The following tips and tricks have been collected from various


@@ -867,6 +897,15 @@ This will take longer, but will also produce a smaller packfile.
 You only need to expend the effort once, and everyone using your
 project will benefit from the smaller repository.
 
 You only need to expend the effort once, and everyone using your
 project will benefit from the smaller repository.
 

+Include Some Progress Messages
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+Every once in a while have your frontend emit a `progress` message
+to fast-import.  The contents of the messages are entirely free-form,
+so one suggestion would be to output the current month and year
+each time the current commit date moves into the next month.
+Your users will feel better knowing how much of the data stream
+has been processed.
+

 
 Packfile Optimization
 ---------------------
 
 Packfile Optimization
 ---------------------

diff --git a/fast-import.c b/fast-import.c
index 4bc7f81bcb6e8d7dc927789caf0a36a7a383ae26..429ab491bd3dc6d084c7301ec4fc2bb5afa3ed04 100644 (file)
--- a/fast-import.c
+++ b/fast-import.c
@@ -8,6 +8,7 @@ Format of STDIN stream:
         | new_tag
         | reset_branch
         | checkpoint
         | new_tag
         | reset_branch
         | checkpoint

+        | progress

         ;
 
   new_blob ::= 'blob' lf
         ;
 
   new_blob ::= 'blob' lf


@@ -53,6 +54,9 @@ Format of STDIN stream:
   checkpoint ::= 'checkpoint' lf
     lf?;
 
   checkpoint ::= 'checkpoint' lf
     lf?;
 

+  progress ::= 'progress' sp not_lf* lf
+    lf?;
+

      # note: the first idnum in a stream should be 1 and subsequent
      # idnums should not have gaps between values as this will cause
      # the stream parser to reserve space for the gapped values.  An
      # note: the first idnum in a stream should be 1 and subsequent
      # idnums should not have gaps between values as this will cause
      # the stream parser to reserve space for the gapped values.  An


@@ -2125,6 +2129,14 @@ static void cmd_checkpoint(void)
        skip_optional_lf();
 }
 
        skip_optional_lf();
 }
 

+static void cmd_progress(void)
+{
+       fwrite(command_buf.buf, 1, command_buf.len - 1, stdout);
+       fputc('\n', stdout);
+       fflush(stdout);
+       skip_optional_lf();
+}
+

 static void import_marks(const char *input_file)
 {
        char line[512];
 static void import_marks(const char *input_file)
 {
        char line[512];


@@ -2235,6 +2247,8 @@ int main(int argc, const char **argv)
                        cmd_reset_branch();
                else if (!strcmp("checkpoint", command_buf.buf))
                        cmd_checkpoint();
                        cmd_reset_branch();
                else if (!strcmp("checkpoint", command_buf.buf))
                        cmd_checkpoint();

+               else if (!prefixcmp(command_buf.buf, "progress "))
+                       cmd_progress();

                else
                        die("Unsupported command: %s", command_buf.buf);
        }
                else
                        die("Unsupported command: %s", command_buf.buf);
        }

diff --git a/t/t9300-fast-import.sh b/t/t9300-fast-import.sh
index 5d82b0f1ce3c4873b0565c5b84bc3a2594f5e4b5..0595041af5d310f905306c6a289945cde26d88fc 100755 (executable)
--- a/t/t9300-fast-import.sh
+++ b/t/t9300-fast-import.sh
@@ -885,4 +885,35 @@ test_expect_success \
         git log --reverse --pretty=oneline O3 | sed s/^.*z// >actual &&
         git diff expect actual'
 
         git log --reverse --pretty=oneline O3 | sed s/^.*z// >actual &&
         git diff expect actual'
 

+cat >input <<INPUT_END
+commit refs/heads/O4
+committer $GIT_COMMITTER_NAME <$GIT_COMMITTER_EMAIL> $GIT_COMMITTER_DATE
+data <<COMMIT
+zstring
+COMMIT
+commit refs/heads/O4
+committer $GIT_COMMITTER_NAME <$GIT_COMMITTER_EMAIL> $GIT_COMMITTER_DATE
+data <<COMMIT
+zof
+COMMIT
+progress Two commits down, 2 to go!
+commit refs/heads/O4
+committer $GIT_COMMITTER_NAME <$GIT_COMMITTER_EMAIL> $GIT_COMMITTER_DATE
+data <<COMMIT
+zempty
+COMMIT
+progress Three commits down, 1 to go!
+commit refs/heads/O4
+committer $GIT_COMMITTER_NAME <$GIT_COMMITTER_EMAIL> $GIT_COMMITTER_DATE
+data <<COMMIT
+zcommits
+COMMIT
+progress I'm done!
+INPUT_END
+test_expect_success \
+       'O: progress outputs as requested by input' \
+       'git-fast-import <input >actual &&
+        grep "progress " <input >expect &&
+        git diff expect actual'
+

 test_done
 test_done