git-fetch: Avoid reading packed refs over and over again

When checking which tags to fetch, the old code used to call
git-show-ref --verify for each remote tag. Since reading even
packed refs is not a cheap operation when there are a lot of
local refs, the code became quite slow.

This fixes it by teaching git-show-ref to filter out existing
refs using a new mode of operation of git-show-ref.

Signed-off-by: Junio C Hamano <junkio@cox.net>

diff --git a/builtin-show-ref.c b/builtin-show-ref.c
index 073979855b30b72363ed8116288b7bbbb2f0ff74..b36f15eeaa1d8ab4416f4716eda9149ef60ca763 100644 (file)
--- a/builtin-show-ref.c
+++ b/builtin-show-ref.c
@@ -2,8 +2,9 @@
 #include "refs.h"
 #include "object.h"
 #include "tag.h"
 #include "refs.h"
 #include "object.h"
 #include "tag.h"

+#include "path-list.h"

 
 

-static const char show_ref_usage[] = "git show-ref [-q|--quiet] [--verify] [-h|--head] [-d|--dereference] [-s|--hash[=<length>]] [--abbrev[=<length>]] [--tags] [--heads] [--] [pattern*]";
+static const char show_ref_usage[] = "git show-ref [-q|--quiet] [--verify] [-h|--head] [-d|--dereference] [-s|--hash[=<length>]] [--abbrev[=<length>]] [--tags] [--heads] [--] [pattern*] | --filter-invalid < ref-list";

 
 static int deref_tags = 0, show_head = 0, tags_only = 0, heads_only = 0,
        found_match = 0, verify = 0, quiet = 0, hash_only = 0, abbrev = 0;
 
 static int deref_tags = 0, show_head = 0, tags_only = 0, heads_only = 0,
        found_match = 0, verify = 0, quiet = 0, hash_only = 0, abbrev = 0;


@@ -86,6 +87,59 @@ static int show_ref(const char *refname, const unsigned char *sha1, int flag, vo
        return 0;
 }
 
        return 0;
 }
 

+static int add_existing(const char *refname, const unsigned char *sha1, int flag, void *cbdata)
+{
+       struct path_list *list = (struct path_list *)cbdata;
+       path_list_insert(refname, list);
+       return 0;
+}
+
+/*
+ * read "^(?:<anything>\s)?<refname>(?:\^\{\})?$" from the standard input,
+ * and
+ * (1) strip "^{}" at the end of line if any;
+ * (2) ignore if match is provided and does not head-match refname;
+ * (3) warn if refname is not a well-formed refname and skip;
+ * (4) ignore if refname is a ref that exists in the local repository;
+ * (5) otherwise output the line.
+ */
+static int exclude_existing(const char *match)
+{
+       static struct path_list existing_refs = { NULL, 0, 0, 0 };
+       char buf[1024];
+       int matchlen = match ? strlen(match) : 0;
+
+       for_each_ref(add_existing, &existing_refs);
+       while (fgets(buf, sizeof(buf), stdin)) {
+               int len = strlen(buf);
+               char *ref;
+               if (len > 0 && buf[len - 1] == '\n')
+                       buf[--len] = '\0';
+               if (!strcmp(buf + len - 3, "^{}")) {
+                       len -= 3;
+                       buf[len] = '\0';
+               }
+               for (ref = buf + len; buf < ref; ref--)
+                       if (isspace(ref[-1]))
+                               break;
+               if (match) {
+                       int reflen = buf + len - ref;
+                       if (reflen < matchlen)
+                               continue;
+                       if (strncmp(ref, match, matchlen))
+                               continue;
+               }
+               if (check_ref_format(ref)) {
+                       fprintf(stderr, "warning: ref '%s' ignored\n", ref);
+                       continue;
+               }
+               if (!path_list_has_path(&existing_refs, ref)) {
+                       printf("%s\n", buf);
+               }
+       }
+       return 0;
+}
+

 int cmd_show_ref(int argc, const char **argv, const char *prefix)
 {
        int i;
 int cmd_show_ref(int argc, const char **argv, const char *prefix)
 {
        int i;


@@ -153,6 +207,10 @@ int cmd_show_ref(int argc, const char **argv, const char *prefix)
                        heads_only = 1;
                        continue;
                }
                        heads_only = 1;
                        continue;
                }

+               if (!strcmp(arg, "--exclude-existing"))
+                       return exclude_existing(NULL);
+               if (!strncmp(arg, "--exclude-existing=", 19))
+                       return exclude_existing(arg + 19);

                usage(show_ref_usage);
        }
        if (show_head)
                usage(show_ref_usage);
        }
        if (show_head)

diff --git a/git-fetch.sh b/git-fetch.sh
index fb35815a5fbf1074f6a4a2fb08c6c9656c6a1654..38101a6ace2331707bb7a424d32b32f3699d3bab 100755 (executable)
--- a/git-fetch.sh
+++ b/git-fetch.sh
@@ -242,7 +242,7 @@ esac
 reflist=$(get_remote_refs_for_fetch "$@")
 if test "$tags"
 then
 reflist=$(get_remote_refs_for_fetch "$@")
 if test "$tags"
 then

-       taglist=`IFS="  " &&
+       taglist=`IFS='  ' &&

                  echo "$ls_remote_result" |
                  while read sha1 name
                  do
                  echo "$ls_remote_result" |
                  while read sha1 name
                  do


@@ -438,17 +438,11 @@ case "$no_tags$tags" in
        *:refs/*)
                # effective only when we are following remote branch
                # using local tracking branch.
        *:refs/*)
                # effective only when we are following remote branch
                # using local tracking branch.

-               taglist=$(IFS=" " &&
+               taglist=$(IFS=' ' &&

                echo "$ls_remote_result" |
                echo "$ls_remote_result" |

-               sed -n  -e 's|^\('"$_x40"'\)    \(refs/tags/.*\)^{}$|\1 \2|p' \
-                       -e 's|^\('"$_x40"'\)    \(refs/tags/.*\)$|\1 \2|p' |
+               git-show-ref --exclude-existing=refs/tags/ |

                while read sha1 name
                do
                while read sha1 name
                do

-                       git-show-ref --verify --quiet -- "$name" && continue
-                       git-check-ref-format "$name" || {
-                               echo >&2 "warning: tag ${name} ignored"
-                               continue
-                       }

                        git-cat-file -t "$sha1" >/dev/null 2>&1 || continue
                        echo >&2 "Auto-following $name"
                        echo ".${name}:${name}"
                        git-cat-file -t "$sha1" >/dev/null 2>&1 || continue
                        echo >&2 "Auto-following $name"
                        echo ".${name}:${name}"