Merge branch 'jt/batch-fetch-blobs-in-diff'
authorJunio C Hamano <gitster@pobox.com>
Thu, 25 Apr 2019 07:41:19 +0000 (16:41 +0900)
committerJunio C Hamano <gitster@pobox.com>
Thu, 25 Apr 2019 07:41:19 +0000 (16:41 +0900)
While running "git diff" in a lazy clone, we can upfront know which
missing blobs we will need, instead of waiting for the on-demand
machinery to discover them one by one. Aim to achieve better
performance by batching the request for these promised blobs.

* jt/batch-fetch-blobs-in-diff:
diff: batch fetching of missing blobs
sha1-file: support OBJECT_INFO_FOR_PREFETCH

1  2 
diff.c
object-store.h
sha1-file.c
unpack-trees.c
diff --combined diff.c
index e30a645d9e01ecab1c12d8d14def7de5f6b8d2ed,c5fd9165f95e815fa2489671ec34bcfb6eb3c73c..4d3cf83a27e5785f5fd50dd2ce6155c94eb50840
--- 1/diff.c
--- 2/diff.c
+++ b/diff.c
@@@ -25,6 -25,7 +25,7 @@@
  #include "packfile.h"
  #include "parse-options.h"
  #include "help.h"
+ #include "fetch-object.h"
  
  #ifdef NO_FAST_WORKING_DIRECTORY
  #define FAST_WORKING_DIRECTORY 0
@@@ -4640,6 -4641,66 +4641,6 @@@ void diff_setup_done(struct diff_option
        FREE_AND_NULL(options->parseopts);
  }
  
 -static int opt_arg(const char *arg, int arg_short, const char *arg_long, int *val)
 -{
 -      char c, *eq;
 -      int len;
 -
 -      if (*arg != '-')
 -              return 0;
 -      c = *++arg;
 -      if (!c)
 -              return 0;
 -      if (c == arg_short) {
 -              c = *++arg;
 -              if (!c)
 -                      return 1;
 -              if (val && isdigit(c)) {
 -                      char *end;
 -                      int n = strtoul(arg, &end, 10);
 -                      if (*end)
 -                              return 0;
 -                      *val = n;
 -                      return 1;
 -              }
 -              return 0;
 -      }
 -      if (c != '-')
 -              return 0;
 -      arg++;
 -      eq = strchrnul(arg, '=');
 -      len = eq - arg;
 -      if (!len || strncmp(arg, arg_long, len))
 -              return 0;
 -      if (*eq) {
 -              int n;
 -              char *end;
 -              if (!isdigit(*++eq))
 -                      return 0;
 -              n = strtoul(eq, &end, 10);
 -              if (*end)
 -                      return 0;
 -              *val = n;
 -      }
 -      return 1;
 -}
 -
 -static inline int short_opt(char opt, const char **argv,
 -                          const char **optarg)
 -{
 -      const char *arg = argv[0];
 -      if (arg[0] != '-' || arg[1] != opt)
 -              return 0;
 -      if (arg[2] != '\0') {
 -              *optarg = arg + 2;
 -              return 1;
 -      }
 -      if (!argv[1])
 -              die("Option '%c' requires a value", opt);
 -      *optarg = argv[1];
 -      return 2;
 -}
 -
  int parse_long_opt(const char *opt, const char **argv,
                   const char **optarg)
  {
@@@ -4728,6 -4789,14 +4729,6 @@@ static int parse_dirstat_opt(struct dif
        return 1;
  }
  
 -static int parse_submodule_opt(struct diff_options *options, const char *value)
 -{
 -      if (parse_submodule_params(options, value))
 -              die(_("Failed to parse --submodule option parameter: '%s'"),
 -                      value);
 -      return 1;
 -}
 -
  static const char diff_status_letters[] = {
        DIFF_STATUS_ADDED,
        DIFF_STATUS_COPIED,
@@@ -4759,19 -4828,10 +4760,19 @@@ static unsigned filter_bit_tst(char sta
        return opt->filter & filter_bit[(int) status];
  }
  
 -static int parse_diff_filter_opt(const char *optarg, struct diff_options *opt)
 +unsigned diff_filter_bit(char status)
 +{
 +      prepare_filter_bits();
 +      return filter_bit[(int) status];
 +}
 +
 +static int diff_opt_diff_filter(const struct option *option,
 +                              const char *optarg, int unset)
  {
 +      struct diff_options *opt = option->value;
        int i, optch;
  
 +      BUG_ON_OPT_NEG(unset);
        prepare_filter_bits();
  
        /*
  
                bit = (0 <= optch && optch <= 'Z') ? filter_bit[optch] : 0;
                if (!bit)
 -                      return optarg[i];
 +                      return error(_("unknown change class '%c' in --diff-filter=%s"),
 +                                   optarg[i], optarg);
                if (negate)
                        opt->filter &= ~bit;
                else
@@@ -4818,29 -4877,25 +4819,29 @@@ static void enable_patch_output(int *fm
        *fmt |= DIFF_FORMAT_PATCH;
  }
  
 -static int parse_ws_error_highlight_opt(struct diff_options *opt, const char *arg)
 +static int diff_opt_ws_error_highlight(const struct option *option,
 +                                     const char *arg, int unset)
  {
 +      struct diff_options *opt = option->value;
        int val = parse_ws_error_highlight(arg);
  
 -      if (val < 0) {
 -              error("unknown value after ws-error-highlight=%.*s",
 -                    -1 - val, arg);
 -              return 0;
 -      }
 +      BUG_ON_OPT_NEG(unset);
 +      if (val < 0)
 +              return error(_("unknown value after ws-error-highlight=%.*s"),
 +                           -1 - val, arg);
        opt->ws_error_highlight = val;
 -      return 1;
 +      return 0;
  }
  
 -static int parse_objfind_opt(struct diff_options *opt, const char *arg)
 +static int diff_opt_find_object(const struct option *option,
 +                              const char *arg, int unset)
  {
 +      struct diff_options *opt = option->value;
        struct object_id oid;
  
 +      BUG_ON_OPT_NEG(unset);
        if (get_oid(arg, &oid))
 -              return error("unable to resolve '%s'", arg);
 +              return error(_("unable to resolve '%s'"), arg);
  
        if (!opt->objfind)
                opt->objfind = xcalloc(1, sizeof(*opt->objfind));
        opt->flags.recursive = 1;
        opt->flags.tree_in_recursive = 1;
        oidset_insert(opt->objfind, &oid);
 -      return 1;
 +      return 0;
 +}
 +
 +static int diff_opt_anchored(const struct option *opt,
 +                           const char *arg, int unset)
 +{
 +      struct diff_options *options = opt->value;
 +
 +      BUG_ON_OPT_NEG(unset);
 +      options->xdl_opts = DIFF_WITH_ALG(options, PATIENCE_DIFF);
 +      ALLOC_GROW(options->anchors, options->anchors_nr + 1,
 +                 options->anchors_alloc);
 +      options->anchors[options->anchors_nr++] = xstrdup(arg);
 +      return 0;
 +}
 +
 +static int diff_opt_binary(const struct option *opt,
 +                         const char *arg, int unset)
 +{
 +      struct diff_options *options = opt->value;
 +
 +      BUG_ON_OPT_NEG(unset);
 +      BUG_ON_OPT_ARG(arg);
 +      enable_patch_output(&options->output_format);
 +      options->flags.binary = 1;
 +      return 0;
  }
  
  static int diff_opt_break_rewrites(const struct option *opt,
@@@ -4914,57 -4944,6 +4915,57 @@@ static int diff_opt_char(const struct o
        return 0;
  }
  
 +static int diff_opt_color_moved(const struct option *opt,
 +                              const char *arg, int unset)
 +{
 +      struct diff_options *options = opt->value;
 +
 +      if (unset) {
 +              options->color_moved = COLOR_MOVED_NO;
 +      } else if (!arg) {
 +              if (diff_color_moved_default)
 +                      options->color_moved = diff_color_moved_default;
 +              if (options->color_moved == COLOR_MOVED_NO)
 +                      options->color_moved = COLOR_MOVED_DEFAULT;
 +      } else {
 +              int cm = parse_color_moved(arg);
 +              if (cm < 0)
 +                      return error(_("bad --color-moved argument: %s"), arg);
 +              options->color_moved = cm;
 +      }
 +      return 0;
 +}
 +
 +static int diff_opt_color_moved_ws(const struct option *opt,
 +                                 const char *arg, int unset)
 +{
 +      struct diff_options *options = opt->value;
 +      unsigned cm;
 +
 +      if (unset) {
 +              options->color_moved_ws_handling = 0;
 +              return 0;
 +      }
 +
 +      cm = parse_color_moved_ws(arg);
 +      if (cm & COLOR_MOVED_WS_ERROR)
 +              return error(_("invalid mode '%s' in --color-moved-ws"), arg);
 +      options->color_moved_ws_handling = cm;
 +      return 0;
 +}
 +
 +static int diff_opt_color_words(const struct option *opt,
 +                              const char *arg, int unset)
 +{
 +      struct diff_options *options = opt->value;
 +
 +      BUG_ON_OPT_NEG(unset);
 +      options->use_color = 1;
 +      options->word_diff = DIFF_WORDS_COLOR;
 +      options->word_regex = arg;
 +      return 0;
 +}
 +
  static int diff_opt_compact_summary(const struct option *opt,
                                    const char *arg, int unset)
  {
        return 0;
  }
  
 +static int diff_opt_diff_algorithm(const struct option *opt,
 +                                 const char *arg, int unset)
 +{
 +      struct diff_options *options = opt->value;
 +      long value = parse_algorithm_value(arg);
 +
 +      BUG_ON_OPT_NEG(unset);
 +      if (value < 0)
 +              return error(_("option diff-algorithm accepts \"myers\", "
 +                             "\"minimal\", \"patience\" and \"histogram\""));
 +
 +      /* clear out previous settings */
 +      DIFF_XDL_CLR(options, NEED_MINIMAL);
 +      options->xdl_opts &= ~XDF_DIFF_ALGORITHM_MASK;
 +      options->xdl_opts |= value;
 +      return 0;
 +}
 +
  static int diff_opt_dirstat(const struct option *opt,
                            const char *arg, int unset)
  {
@@@ -5050,58 -5011,6 +5051,58 @@@ static int diff_opt_find_renames(const 
        return 0;
  }
  
 +static int diff_opt_follow(const struct option *opt,
 +                         const char *arg, int unset)
 +{
 +      struct diff_options *options = opt->value;
 +
 +      BUG_ON_OPT_ARG(arg);
 +      if (unset) {
 +              options->flags.follow_renames = 0;
 +              options->flags.default_follow_renames = 0;
 +      } else {
 +              options->flags.follow_renames = 1;
 +      }
 +      return 0;
 +}
 +
 +static int diff_opt_ignore_submodules(const struct option *opt,
 +                                    const char *arg, int unset)
 +{
 +      struct diff_options *options = opt->value;
 +
 +      BUG_ON_OPT_NEG(unset);
 +      if (!arg)
 +              arg = "all";
 +      options->flags.override_submodule_config = 1;
 +      handle_ignore_submodules_arg(options, arg);
 +      return 0;
 +}
 +
 +static int diff_opt_line_prefix(const struct option *opt,
 +                              const char *optarg, int unset)
 +{
 +      struct diff_options *options = opt->value;
 +
 +      BUG_ON_OPT_NEG(unset);
 +      options->line_prefix = optarg;
 +      options->line_prefix_length = strlen(options->line_prefix);
 +      graph_setup_line_prefix(options);
 +      return 0;
 +}
 +
 +static int diff_opt_no_prefix(const struct option *opt,
 +                            const char *optarg, int unset)
 +{
 +      struct diff_options *options = opt->value;
 +
 +      BUG_ON_OPT_NEG(unset);
 +      BUG_ON_OPT_ARG(optarg);
 +      options->a_prefix = "";
 +      options->b_prefix = "";
 +      return 0;
 +}
 +
  static enum parse_opt_result diff_opt_output(struct parse_opt_ctx_t *ctx,
                                             const struct option *opt,
                                             const char *arg, int unset)
        return 0;
  }
  
 +static int diff_opt_patience(const struct option *opt,
 +                           const char *arg, int unset)
 +{
 +      struct diff_options *options = opt->value;
 +      int i;
 +
 +      BUG_ON_OPT_NEG(unset);
 +      BUG_ON_OPT_ARG(arg);
 +      options->xdl_opts = DIFF_WITH_ALG(options, PATIENCE_DIFF);
 +      /*
 +       * Both --patience and --anchored use PATIENCE_DIFF
 +       * internally, so remove any anchors previously
 +       * specified.
 +       */
 +      for (i = 0; i < options->anchors_nr; i++)
 +              free(options->anchors[i]);
 +      options->anchors_nr = 0;
 +      return 0;
 +}
 +
 +static int diff_opt_pickaxe_regex(const struct option *opt,
 +                                const char *arg, int unset)
 +{
 +      struct diff_options *options = opt->value;
 +
 +      BUG_ON_OPT_NEG(unset);
 +      options->pickaxe = arg;
 +      options->pickaxe_opts |= DIFF_PICKAXE_KIND_G;
 +      return 0;
 +}
 +
 +static int diff_opt_pickaxe_string(const struct option *opt,
 +                                 const char *arg, int unset)
 +{
 +      struct diff_options *options = opt->value;
 +
 +      BUG_ON_OPT_NEG(unset);
 +      options->pickaxe = arg;
 +      options->pickaxe_opts |= DIFF_PICKAXE_KIND_S;
 +      return 0;
 +}
 +
  static int diff_opt_relative(const struct option *opt,
                             const char *arg, int unset)
  {
        return 0;
  }
  
 +static int diff_opt_submodule(const struct option *opt,
 +                            const char *arg, int unset)
 +{
 +      struct diff_options *options = opt->value;
 +
 +      BUG_ON_OPT_NEG(unset);
 +      if (!arg)
 +              arg = "log";
 +      if (parse_submodule_params(options, arg))
 +              return error(_("failed to parse --submodule option parameter: '%s'"),
 +                           arg);
 +      return 0;
 +}
 +
 +static int diff_opt_textconv(const struct option *opt,
 +                           const char *arg, int unset)
 +{
 +      struct diff_options *options = opt->value;
 +
 +      BUG_ON_OPT_ARG(arg);
 +      if (unset) {
 +              options->flags.allow_textconv = 0;
 +      } else {
 +              options->flags.allow_textconv = 1;
 +              options->flags.textconv_set_via_cmdline = 1;
 +      }
 +      return 0;
 +}
 +
  static int diff_opt_unified(const struct option *opt,
                            const char *arg, int unset)
  {
        return 0;
  }
  
 +static int diff_opt_word_diff(const struct option *opt,
 +                            const char *arg, int unset)
 +{
 +      struct diff_options *options = opt->value;
 +
 +      BUG_ON_OPT_NEG(unset);
 +      if (arg) {
 +              if (!strcmp(arg, "plain"))
 +                      options->word_diff = DIFF_WORDS_PLAIN;
 +              else if (!strcmp(arg, "color")) {
 +                      options->use_color = 1;
 +                      options->word_diff = DIFF_WORDS_COLOR;
 +              }
 +              else if (!strcmp(arg, "porcelain"))
 +                      options->word_diff = DIFF_WORDS_PORCELAIN;
 +              else if (!strcmp(arg, "none"))
 +                      options->word_diff = DIFF_WORDS_NONE;
 +              else
 +                      return error(_("bad --word-diff argument: %s"), arg);
 +      } else {
 +              if (options->word_diff == DIFF_WORDS_NONE)
 +                      options->word_diff = DIFF_WORDS_PLAIN;
 +      }
 +      return 0;
 +}
 +
 +static int diff_opt_word_diff_regex(const struct option *opt,
 +                                  const char *arg, int unset)
 +{
 +      struct diff_options *options = opt->value;
 +
 +      BUG_ON_OPT_NEG(unset);
 +      if (options->word_diff == DIFF_WORDS_NONE)
 +              options->word_diff = DIFF_WORDS_PLAIN;
 +      options->word_regex = arg;
 +      return 0;
 +}
 +
  static void prep_parse_options(struct diff_options *options)
  {
        struct option parseopts[] = {
                OPT_CALLBACK_F(0, "compact-summary", options, NULL,
                               N_("generate compact summary in diffstat"),
                               PARSE_OPT_NOARG, diff_opt_compact_summary),
 +              OPT_CALLBACK_F(0, "binary", options, NULL,
 +                             N_("output a binary diff that can be applied"),
 +                             PARSE_OPT_NONEG | PARSE_OPT_NOARG, diff_opt_binary),
 +              OPT_BOOL(0, "full-index", &options->flags.full_index,
 +                       N_("show full pre- and post-image object names on the \"index\" lines")),
 +              OPT_COLOR_FLAG(0, "color", &options->use_color,
 +                             N_("show colored diff")),
 +              OPT_CALLBACK_F(0, "ws-error-highlight", options, N_("<kind>"),
 +                             N_("highlight whitespace errors in the 'context', 'old' or 'new' lines in the diff"),
 +                             PARSE_OPT_NONEG, diff_opt_ws_error_highlight),
 +              OPT_SET_INT('z', NULL, &options->line_termination,
 +                          N_("do not munge pathnames and use NULs as output field terminators in --raw or --numstat"),
 +                          0),
 +              OPT__ABBREV(&options->abbrev),
 +              OPT_STRING_F(0, "src-prefix", &options->a_prefix, N_("<prefix>"),
 +                           N_("show the given source prefix instead of \"a/\""),
 +                           PARSE_OPT_NONEG),
 +              OPT_STRING_F(0, "dst-prefix", &options->b_prefix, N_("<prefix>"),
 +                           N_("show the given source prefix instead of \"b/\""),
 +                           PARSE_OPT_NONEG),
 +              OPT_CALLBACK_F(0, "line-prefix", options, N_("<prefix>"),
 +                             N_("prepend an additional prefix to every line of output"),
 +                             PARSE_OPT_NONEG, diff_opt_line_prefix),
 +              OPT_CALLBACK_F(0, "no-prefix", options, NULL,
 +                             N_("do not show any source or destination prefix"),
 +                             PARSE_OPT_NONEG | PARSE_OPT_NOARG, diff_opt_no_prefix),
 +              OPT_INTEGER_F(0, "inter-hunk-context", &options->interhunkcontext,
 +                            N_("show context between diff hunks up to the specified number of lines"),
 +                            PARSE_OPT_NONEG),
                OPT_CALLBACK_F(0, "output-indicator-new",
                               &options->output_indicators[OUTPUT_INDICATOR_NEW],
                               N_("<char>"),
                              0, PARSE_OPT_NONEG),
                OPT_BOOL(0, "rename-empty", &options->flags.rename_empty,
                         N_("use empty blobs as rename source")),
 +              OPT_CALLBACK_F(0, "follow", options, NULL,
 +                             N_("continue listing the history of a file beyond renames"),
 +                             PARSE_OPT_NOARG, diff_opt_follow),
 +              OPT_INTEGER('l', NULL, &options->rename_limit,
 +                          N_("prevent rename/copy detection if the number of rename/copy targets exceeds given limit")),
  
                OPT_GROUP(N_("Diff algorithm options")),
                OPT_BIT(0, "minimal", &options->xdl_opts,
                OPT_BIT_F(0, "ignore-blank-lines", &options->xdl_opts,
                          N_("ignore changes whose lines are all blank"),
                          XDF_IGNORE_BLANK_LINES, PARSE_OPT_NONEG),
 +              OPT_BIT(0, "indent-heuristic", &options->xdl_opts,
 +                      N_("heuristic to shift diff hunk boundaries for easy reading"),
 +                      XDF_INDENT_HEURISTIC),
 +              OPT_CALLBACK_F(0, "patience", options, NULL,
 +                             N_("generate diff using the \"patience diff\" algorithm"),
 +                             PARSE_OPT_NONEG | PARSE_OPT_NOARG,
 +                             diff_opt_patience),
 +              OPT_BITOP(0, "histogram", &options->xdl_opts,
 +                        N_("generate diff using the \"histogram diff\" algorithm"),
 +                        XDF_HISTOGRAM_DIFF, XDF_DIFF_ALGORITHM_MASK),
 +              OPT_CALLBACK_F(0, "diff-algorithm", options, N_("<algorithm>"),
 +                             N_("choose a diff algorithm"),
 +                             PARSE_OPT_NONEG, diff_opt_diff_algorithm),
 +              OPT_CALLBACK_F(0, "anchored", options, N_("<text>"),
 +                             N_("generate diff using the \"anchored diff\" algorithm"),
 +                             PARSE_OPT_NONEG, diff_opt_anchored),
 +              OPT_CALLBACK_F(0, "word-diff", options, N_("<mode>"),
 +                             N_("show word diff, using <mode> to delimit changed words"),
 +                             PARSE_OPT_NONEG | PARSE_OPT_OPTARG, diff_opt_word_diff),
 +              OPT_CALLBACK_F(0, "word-diff-regex", options, N_("<regex>"),
 +                             N_("use <regex> to decide what a word is"),
 +                             PARSE_OPT_NONEG, diff_opt_word_diff_regex),
 +              OPT_CALLBACK_F(0, "color-words", options, N_("<regex>"),
 +                             N_("equivalent to --word-diff=color --word-diff-regex=<regex>"),
 +                             PARSE_OPT_NONEG | PARSE_OPT_OPTARG, diff_opt_color_words),
 +              OPT_CALLBACK_F(0, "color-moved", options, N_("<mode>"),
 +                             N_("move lines of code are colored differently"),
 +                             PARSE_OPT_OPTARG, diff_opt_color_moved),
 +              OPT_CALLBACK_F(0, "color-moved-ws", options, N_("<mode>"),
 +                             N_("how white spaces are ignored in --color-moved"),
 +                             0, diff_opt_color_moved_ws),
  
                OPT_GROUP(N_("Diff other options")),
                OPT_CALLBACK_F(0, "relative", options, N_("<prefix>"),
                               N_("when run from subdir, exclude changes outside and show relative paths"),
                               PARSE_OPT_NONEG | PARSE_OPT_OPTARG,
                               diff_opt_relative),
 +              OPT_BOOL('a', "text", &options->flags.text,
 +                       N_("treat all files as text")),
 +              OPT_BOOL('R', NULL, &options->flags.reverse_diff,
 +                       N_("swap two inputs, reverse the diff")),
 +              OPT_BOOL(0, "exit-code", &options->flags.exit_with_status,
 +                       N_("exit with 1 if there were differences, 0 otherwise")),
 +              OPT_BOOL(0, "quiet", &options->flags.quick,
 +                       N_("disable all output of the program")),
 +              OPT_BOOL(0, "ext-diff", &options->flags.allow_external,
 +                       N_("allow an external diff helper to be executed")),
 +              OPT_CALLBACK_F(0, "textconv", options, NULL,
 +                             N_("run external text conversion filters when comparing binary files"),
 +                             PARSE_OPT_NOARG, diff_opt_textconv),
 +              OPT_CALLBACK_F(0, "ignore-submodules", options, N_("<when>"),
 +                             N_("ignore changes to submodules in the diff generation"),
 +                             PARSE_OPT_NONEG | PARSE_OPT_OPTARG,
 +                             diff_opt_ignore_submodules),
 +              OPT_CALLBACK_F(0, "submodule", options, N_("<format>"),
 +                             N_("specify how differences in submodules are shown"),
 +                             PARSE_OPT_NONEG | PARSE_OPT_OPTARG,
 +                             diff_opt_submodule),
 +              OPT_SET_INT_F(0, "ita-invisible-in-index", &options->ita_invisible_in_index,
 +                            N_("hide 'git add -N' entries from the index"),
 +                            1, PARSE_OPT_NONEG),
 +              OPT_SET_INT_F(0, "ita-visible-in-index", &options->ita_invisible_in_index,
 +                            N_("treat 'git add -N' entries as real in the index"),
 +                            0, PARSE_OPT_NONEG),
 +              OPT_CALLBACK_F('S', NULL, options, N_("<string>"),
 +                             N_("look for differences that change the number of occurrences of the specified string"),
 +                             0, diff_opt_pickaxe_string),
 +              OPT_CALLBACK_F('G', NULL, options, N_("<regex>"),
 +                             N_("look for differences that change the number of occurrences of the specified regex"),
 +                             0, diff_opt_pickaxe_regex),
 +              OPT_BIT_F(0, "pickaxe-all", &options->pickaxe_opts,
 +                        N_("show all changes in the changeset with -S or -G"),
 +                        DIFF_PICKAXE_ALL, PARSE_OPT_NONEG),
 +              OPT_BIT_F(0, "pickaxe-regex", &options->pickaxe_opts,
 +                        N_("treat <string> in -S as extended POSIX regular expression"),
 +                        DIFF_PICKAXE_REGEX, PARSE_OPT_NONEG),
 +              OPT_FILENAME('O', NULL, &options->orderfile,
 +                           N_("control the order in which files appear in the output")),
 +              OPT_CALLBACK_F(0, "find-object", options, N_("<object-id>"),
 +                             N_("look for differences that change the number of occurrences of the specified object"),
 +                             PARSE_OPT_NONEG, diff_opt_find_object),
 +              OPT_CALLBACK_F(0, "diff-filter", options, N_("[(A|C|D|M|R|T|U|X|B)...[*]]"),
 +                             N_("select files by diff type"),
 +                             PARSE_OPT_NONEG, diff_opt_diff_filter),
                { OPTION_CALLBACK, 0, "output", options, N_("<file>"),
                  N_("Output to a specific file"),
                  PARSE_OPT_NONEG, NULL, 0, diff_opt_output },
  int diff_opt_parse(struct diff_options *options,
                   const char **av, int ac, const char *prefix)
  {
 -      const char *arg = av[0];
 -      const char *optarg;
 -      int argcount;
 -
        if (!prefix)
                prefix = "";
  
                           PARSE_OPT_ONE_SHOT |
                           PARSE_OPT_STOP_AT_NON_OPTION);
  
 -      if (ac)
 -              return ac;
 -
 -      /* xdiff options */
 -      if (!strcmp(arg, "--indent-heuristic"))
 -              DIFF_XDL_SET(options, INDENT_HEURISTIC);
 -      else if (!strcmp(arg, "--no-indent-heuristic"))
 -              DIFF_XDL_CLR(options, INDENT_HEURISTIC);
 -      else if (!strcmp(arg, "--patience")) {
 -              int i;
 -              options->xdl_opts = DIFF_WITH_ALG(options, PATIENCE_DIFF);
 -              /*
 -               * Both --patience and --anchored use PATIENCE_DIFF
 -               * internally, so remove any anchors previously
 -               * specified.
 -               */
 -              for (i = 0; i < options->anchors_nr; i++)
 -                      free(options->anchors[i]);
 -              options->anchors_nr = 0;
 -      } else if (!strcmp(arg, "--histogram"))
 -              options->xdl_opts = DIFF_WITH_ALG(options, HISTOGRAM_DIFF);
 -      else if ((argcount = parse_long_opt("diff-algorithm", av, &optarg))) {
 -              long value = parse_algorithm_value(optarg);
 -              if (value < 0)
 -                      return error("option diff-algorithm accepts \"myers\", "
 -                                   "\"minimal\", \"patience\" and \"histogram\"");
 -              /* clear out previous settings */
 -              DIFF_XDL_CLR(options, NEED_MINIMAL);
 -              options->xdl_opts &= ~XDF_DIFF_ALGORITHM_MASK;
 -              options->xdl_opts |= value;
 -              return argcount;
 -      } else if (skip_prefix(arg, "--anchored=", &arg)) {
 -              options->xdl_opts = DIFF_WITH_ALG(options, PATIENCE_DIFF);
 -              ALLOC_GROW(options->anchors, options->anchors_nr + 1,
 -                         options->anchors_alloc);
 -              options->anchors[options->anchors_nr++] = xstrdup(arg);
 -      }
 -
 -      /* flags options */
 -      else if (!strcmp(arg, "--binary")) {
 -              enable_patch_output(&options->output_format);
 -              options->flags.binary = 1;
 -      }
 -      else if (!strcmp(arg, "--full-index"))
 -              options->flags.full_index = 1;
 -      else if (!strcmp(arg, "-a") || !strcmp(arg, "--text"))
 -              options->flags.text = 1;
 -      else if (!strcmp(arg, "-R"))
 -              options->flags.reverse_diff = 1;
 -      else if (!strcmp(arg, "--follow"))
 -              options->flags.follow_renames = 1;
 -      else if (!strcmp(arg, "--no-follow")) {
 -              options->flags.follow_renames = 0;
 -              options->flags.default_follow_renames = 0;
 -      } else if (skip_to_optional_arg_default(arg, "--color", &arg, "always")) {
 -              int value = git_config_colorbool(NULL, arg);
 -              if (value < 0)
 -                      return error("option `color' expects \"always\", \"auto\", or \"never\"");
 -              options->use_color = value;
 -      }
 -      else if (!strcmp(arg, "--no-color"))
 -              options->use_color = 0;
 -      else if (!strcmp(arg, "--color-moved")) {
 -              if (diff_color_moved_default)
 -                      options->color_moved = diff_color_moved_default;
 -              if (options->color_moved == COLOR_MOVED_NO)
 -                      options->color_moved = COLOR_MOVED_DEFAULT;
 -      } else if (!strcmp(arg, "--no-color-moved"))
 -              options->color_moved = COLOR_MOVED_NO;
 -      else if (skip_prefix(arg, "--color-moved=", &arg)) {
 -              int cm = parse_color_moved(arg);
 -              if (cm < 0)
 -                      return error("bad --color-moved argument: %s", arg);
 -              options->color_moved = cm;
 -      } else if (!strcmp(arg, "--no-color-moved-ws")) {
 -              options->color_moved_ws_handling = 0;
 -      } else if (skip_prefix(arg, "--color-moved-ws=", &arg)) {
 -              unsigned cm = parse_color_moved_ws(arg);
 -              if (cm & COLOR_MOVED_WS_ERROR)
 -                      return -1;
 -              options->color_moved_ws_handling = cm;
 -      } else if (skip_to_optional_arg_default(arg, "--color-words", &options->word_regex, NULL)) {
 -              options->use_color = 1;
 -              options->word_diff = DIFF_WORDS_COLOR;
 -      }
 -      else if (!strcmp(arg, "--word-diff")) {
 -              if (options->word_diff == DIFF_WORDS_NONE)
 -                      options->word_diff = DIFF_WORDS_PLAIN;
 -      }
 -      else if (skip_prefix(arg, "--word-diff=", &arg)) {
 -              if (!strcmp(arg, "plain"))
 -                      options->word_diff = DIFF_WORDS_PLAIN;
 -              else if (!strcmp(arg, "color")) {
 -                      options->use_color = 1;
 -                      options->word_diff = DIFF_WORDS_COLOR;
 -              }
 -              else if (!strcmp(arg, "porcelain"))
 -                      options->word_diff = DIFF_WORDS_PORCELAIN;
 -              else if (!strcmp(arg, "none"))
 -                      options->word_diff = DIFF_WORDS_NONE;
 -              else
 -                      die("bad --word-diff argument: %s", arg);
 -      }
 -      else if ((argcount = parse_long_opt("word-diff-regex", av, &optarg))) {
 -              if (options->word_diff == DIFF_WORDS_NONE)
 -                      options->word_diff = DIFF_WORDS_PLAIN;
 -              options->word_regex = optarg;
 -              return argcount;
 -      }
 -      else if (!strcmp(arg, "--exit-code"))
 -              options->flags.exit_with_status = 1;
 -      else if (!strcmp(arg, "--quiet"))
 -              options->flags.quick = 1;
 -      else if (!strcmp(arg, "--ext-diff"))
 -              options->flags.allow_external = 1;
 -      else if (!strcmp(arg, "--no-ext-diff"))
 -              options->flags.allow_external = 0;
 -      else if (!strcmp(arg, "--textconv")) {
 -              options->flags.allow_textconv = 1;
 -              options->flags.textconv_set_via_cmdline = 1;
 -      } else if (!strcmp(arg, "--no-textconv"))
 -              options->flags.allow_textconv = 0;
 -      else if (skip_to_optional_arg_default(arg, "--ignore-submodules", &arg, "all")) {
 -              options->flags.override_submodule_config = 1;
 -              handle_ignore_submodules_arg(options, arg);
 -      } else if (skip_to_optional_arg_default(arg, "--submodule", &arg, "log"))
 -              return parse_submodule_opt(options, arg);
 -      else if (skip_prefix(arg, "--ws-error-highlight=", &arg))
 -              return parse_ws_error_highlight_opt(options, arg);
 -      else if (!strcmp(arg, "--ita-invisible-in-index"))
 -              options->ita_invisible_in_index = 1;
 -      else if (!strcmp(arg, "--ita-visible-in-index"))
 -              options->ita_invisible_in_index = 0;
 -
 -      /* misc options */
 -      else if (!strcmp(arg, "-z"))
 -              options->line_termination = 0;
 -      else if ((argcount = short_opt('l', av, &optarg))) {
 -              options->rename_limit = strtoul(optarg, NULL, 10);
 -              return argcount;
 -      }
 -      else if ((argcount = short_opt('S', av, &optarg))) {
 -              options->pickaxe = optarg;
 -              options->pickaxe_opts |= DIFF_PICKAXE_KIND_S;
 -              return argcount;
 -      } else if ((argcount = short_opt('G', av, &optarg))) {
 -              options->pickaxe = optarg;
 -              options->pickaxe_opts |= DIFF_PICKAXE_KIND_G;
 -              return argcount;
 -      }
 -      else if (!strcmp(arg, "--pickaxe-all"))
 -              options->pickaxe_opts |= DIFF_PICKAXE_ALL;
 -      else if (!strcmp(arg, "--pickaxe-regex"))
 -              options->pickaxe_opts |= DIFF_PICKAXE_REGEX;
 -      else if ((argcount = short_opt('O', av, &optarg))) {
 -              options->orderfile = prefix_filename(prefix, optarg);
 -              return argcount;
 -      } else if (skip_prefix(arg, "--find-object=", &arg))
 -              return parse_objfind_opt(options, arg);
 -      else if ((argcount = parse_long_opt("diff-filter", av, &optarg))) {
 -              int offending = parse_diff_filter_opt(optarg, options);
 -              if (offending)
 -                      die("unknown change class '%c' in --diff-filter=%s",
 -                          offending, optarg);
 -              return argcount;
 -      }
 -      else if (!strcmp(arg, "--no-abbrev"))
 -              options->abbrev = 0;
 -      else if (!strcmp(arg, "--abbrev"))
 -              options->abbrev = DEFAULT_ABBREV;
 -      else if (skip_prefix(arg, "--abbrev=", &arg)) {
 -              options->abbrev = strtoul(arg, NULL, 10);
 -              if (options->abbrev < MINIMUM_ABBREV)
 -                      options->abbrev = MINIMUM_ABBREV;
 -              else if (the_hash_algo->hexsz < options->abbrev)
 -                      options->abbrev = the_hash_algo->hexsz;
 -      }
 -      else if ((argcount = parse_long_opt("src-prefix", av, &optarg))) {
 -              options->a_prefix = optarg;
 -              return argcount;
 -      }
 -      else if ((argcount = parse_long_opt("line-prefix", av, &optarg))) {
 -              options->line_prefix = optarg;
 -              options->line_prefix_length = strlen(options->line_prefix);
 -              graph_setup_line_prefix(options);
 -              return argcount;
 -      }
 -      else if ((argcount = parse_long_opt("dst-prefix", av, &optarg))) {
 -              options->b_prefix = optarg;
 -              return argcount;
 -      }
 -      else if (!strcmp(arg, "--no-prefix"))
 -              options->a_prefix = options->b_prefix = "";
 -      else if (opt_arg(arg, '\0', "inter-hunk-context",
 -                       &options->interhunkcontext))
 -              ;
 -      else
 -              return 0;
 -      return 1;
 +      return ac;
  }
  
  int parse_rename_score(const char **cp_p)
@@@ -6477,8 -6367,41 +6478,41 @@@ void diffcore_fix_diff_index(void
        QSORT(q->queue, q->nr, diffnamecmp);
  }
  
+ static void add_if_missing(struct repository *r,
+                          struct oid_array *to_fetch,
+                          const struct diff_filespec *filespec)
+ {
+       if (filespec && filespec->oid_valid &&
+           oid_object_info_extended(r, &filespec->oid, NULL,
+                                    OBJECT_INFO_FOR_PREFETCH))
+               oid_array_append(to_fetch, &filespec->oid);
+ }
  void diffcore_std(struct diff_options *options)
  {
+       if (options->repo == the_repository &&
+           repository_format_partial_clone) {
+               /*
+                * Prefetch the diff pairs that are about to be flushed.
+                */
+               int i;
+               struct diff_queue_struct *q = &diff_queued_diff;
+               struct oid_array to_fetch = OID_ARRAY_INIT;
+               for (i = 0; i < q->nr; i++) {
+                       struct diff_filepair *p = q->queue[i];
+                       add_if_missing(options->repo, &to_fetch, p->one);
+                       add_if_missing(options->repo, &to_fetch, p->two);
+               }
+               if (to_fetch.nr)
+                       /*
+                        * NEEDSWORK: Consider deduplicating the OIDs sent.
+                        */
+                       fetch_objects(repository_format_partial_clone,
+                                     to_fetch.oid, to_fetch.nr);
+               oid_array_clear(&to_fetch);
+       }
        /* NOTE please keep the following in sync with diff_tree_combined() */
        if (options->skip_stat_unmatch)
                diffcore_skip_stat_unmatch(options);
diff --combined object-store.h
index 56f8aea1cc8df60f3664c50d81a17277a55a7458,dd3f9b75f04e6a23b235d415d15824aca8a43d20..b086f5ecdb82a105b9b3381de9893cb72bc5871c
@@@ -77,7 -77,7 +77,7 @@@ struct packed_git 
                 freshened:1,
                 do_not_close:1,
                 pack_promisor:1;
 -      unsigned char sha1[20];
 +      unsigned char hash[GIT_MAX_RAWSZ];
        struct revindex_entry *revindex;
        /* something like ".git/objects/pack/xxxxx.pack" */
        char pack_name[FLEX_ARRAY]; /* more */
@@@ -280,6 -280,12 +280,12 @@@ struct object_info 
  #define OBJECT_INFO_QUICK 8
  /* Do not check loose object */
  #define OBJECT_INFO_IGNORE_LOOSE 16
+ /*
+  * Do not attempt to fetch the object if missing (even if fetch_is_missing is
+  * nonzero). This is meant for bulk prefetching of missing blobs in a partial
+  * clone. Implies OBJECT_INFO_QUICK.
+  */
+ #define OBJECT_INFO_FOR_PREFETCH (32 + OBJECT_INFO_QUICK)
  
  int oid_object_info_extended(struct repository *r,
                             const struct object_id *,
diff --combined sha1-file.c
index bcd9470bce089012ab942973cd841262d35749eb,ad02649124bb817360d6cd1ab9b354fd3243cdd7..ed5c50dac427f2f3aa2b9b7b203bf2355049b163
@@@ -189,14 -189,6 +189,14 @@@ int hash_algo_by_id(uint32_t format_id
        return GIT_HASH_UNKNOWN;
  }
  
 +int hash_algo_by_length(int len)
 +{
 +      int i;
 +      for (i = 1; i < GIT_HASH_NALGOS; i++)
 +              if (len == hash_algos[i].rawsz)
 +                      return i;
 +      return GIT_HASH_UNKNOWN;
 +}
  
  /*
   * This is meant to hold a *small* number of objects that you would
@@@ -1378,7 -1370,8 +1378,8 @@@ int oid_object_info_extended(struct rep
  
                /* Check if it is a missing object */
                if (fetch_if_missing && repository_format_partial_clone &&
-                   !already_retried && r == the_repository) {
+                   !already_retried && r == the_repository &&
+                   !(flags & OBJECT_INFO_FOR_PREFETCH)) {
                        /*
                         * TODO Investigate having fetch_object() return
                         * TODO error/success and stopping the music here.
diff --combined unpack-trees.c
index afa4a5cea815a810334136fecf40c0bb39be716d,381b0cd65e3bb12412a8624bff83ff557ee71807..50189909b86d6ab48a0aa15893f5b66c2e9ce613
@@@ -219,9 -219,6 +219,9 @@@ static int add_rejected_path(struct unp
                             enum unpack_trees_error_types e,
                             const char *path)
  {
 +      if (o->quiet)
 +              return -1;
 +
        if (!o->show_all_errors)
                return error(ERRORMSG(o, e), super_prefixed(path));
  
@@@ -271,7 -268,8 +271,7 @@@ static int check_submodule_move_head(co
                flags |= SUBMODULE_MOVE_HEAD_FORCE;
  
        if (submodule_move_head(ce->name, old_id, new_id, flags))
 -              return o->gently ? -1 :
 -                                 add_rejected_path(o, ERROR_WOULD_LOSE_SUBMODULE, ce->name);
 +              return add_rejected_path(o, ERROR_WOULD_LOSE_SUBMODULE, ce->name);
        return 0;
  }
  
@@@ -406,20 -404,21 +406,21 @@@ static int check_updates(struct unpack_
                 * below.
                 */
                struct oid_array to_fetch = OID_ARRAY_INIT;
-               int fetch_if_missing_store = fetch_if_missing;
-               fetch_if_missing = 0;
                for (i = 0; i < index->cache_nr; i++) {
                        struct cache_entry *ce = index->cache[i];
-                       if ((ce->ce_flags & CE_UPDATE) &&
-                           !S_ISGITLINK(ce->ce_mode)) {
-                               if (!has_object_file(&ce->oid))
-                                       oid_array_append(&to_fetch, &ce->oid);
-                       }
+                       if (!(ce->ce_flags & CE_UPDATE) ||
+                           S_ISGITLINK(ce->ce_mode))
+                               continue;
+                       if (!oid_object_info_extended(the_repository, &ce->oid,
+                                                     NULL,
+                                                     OBJECT_INFO_FOR_PREFETCH))
+                               continue;
+                       oid_array_append(&to_fetch, &ce->oid);
                }
                if (to_fetch.nr)
                        fetch_objects(repository_format_partial_clone,
                                      to_fetch.oid, to_fetch.nr);
-               fetch_if_missing = fetch_if_missing_store;
                oid_array_clear(&to_fetch);
        }
        for (i = 0; i < index->cache_nr; i++) {
@@@ -709,6 -708,7 +710,6 @@@ static int index_pos_by_traverse_info(s
   * instead of ODB since we already know what these trees contain.
   */
  static int traverse_by_cache_tree(int pos, int nr_entries, int nr_names,
 -                                struct name_entry *names,
                                  struct traverse_info *info)
  {
        struct cache_entry *src[MAX_UNPACK_TREES + 1] = { NULL, };
@@@ -798,7 -798,7 +799,7 @@@ static int traverse_trees_recursive(in
                 * unprocessed entries before 'pos'.
                 */
                bottom = o->cache_bottom;
 -              ret = traverse_by_cache_tree(pos, nr_entries, n, names, info);
 +              ret = traverse_by_cache_tree(pos, nr_entries, n, info);
                o->cache_bottom = bottom;
                return ret;
        }
@@@ -1041,7 -1041,7 +1042,7 @@@ static int unpack_nondirectories(int n
  static int unpack_failed(struct unpack_trees_options *o, const char *message)
  {
        discard_index(&o->result);
 -      if (!o->gently && !o->exiting_early) {
 +      if (!o->quiet && !o->exiting_early) {
                if (message)
                        return error("%s", message);
                return -1;
@@@ -1619,8 -1619,6 +1620,8 @@@ int unpack_trees(unsigned len, struct t
                                                  WRITE_TREE_SILENT |
                                                  WRITE_TREE_REPAIR);
                }
 +
 +              o->result.updated_workdir = 1;
                discard_index(o->dst_index);
                *o->dst_index = o->result;
        } else {
@@@ -1648,7 -1646,8 +1649,7 @@@ return_failed
  static int reject_merge(const struct cache_entry *ce,
                        struct unpack_trees_options *o)
  {
 -      return o->gently ? -1 :
 -              add_rejected_path(o, ERROR_WOULD_OVERWRITE, ce->name);
 +      return add_rejected_path(o, ERROR_WOULD_OVERWRITE, ce->name);
  }
  
  static int same(const struct cache_entry *a, const struct cache_entry *b)
@@@ -1695,7 -1694,8 +1696,7 @@@ static int verify_uptodate_1(const stru
                        int r = check_submodule_move_head(ce,
                                "HEAD", oid_to_hex(&ce->oid), o);
                        if (r)
 -                              return o->gently ? -1 :
 -                                      add_rejected_path(o, error_type, ce->name);
 +                              return add_rejected_path(o, error_type, ce->name);
                        return 0;
                }
  
        }
        if (errno == ENOENT)
                return 0;
 -      return o->gently ? -1 :
 -              add_rejected_path(o, error_type, ce->name);
 +      return add_rejected_path(o, error_type, ce->name);
  }
  
  int verify_uptodate(const struct cache_entry *ce,
@@@ -1759,6 -1760,7 +1760,6 @@@ static void invalidate_ce_path(const st
   */
  static int verify_clean_submodule(const char *old_sha1,
                                  const struct cache_entry *ce,
 -                                enum unpack_trees_error_types error_type,
                                  struct unpack_trees_options *o)
  {
        if (!submodule_from_ce(ce))
  }
  
  static int verify_clean_subdirectory(const struct cache_entry *ce,
 -                                   enum unpack_trees_error_types error_type,
                                     struct unpack_trees_options *o)
  {
        /*
                if (!sub_head && oideq(&oid, &ce->oid))
                        return 0;
                return verify_clean_submodule(sub_head ? NULL : oid_to_hex(&oid),
 -                                            ce, error_type, o);
 +                                            ce, o);
        }
  
        /*
                d.exclude_per_dir = o->dir->exclude_per_dir;
        i = read_directory(&d, o->src_index, pathbuf, namelen+1, NULL);
        if (i)
 -              return o->gently ? -1 :
 -                      add_rejected_path(o, ERROR_NOT_UPTODATE_DIR, ce->name);
 +              return add_rejected_path(o, ERROR_NOT_UPTODATE_DIR, ce->name);
        free(pathbuf);
        return cnt;
  }
@@@ -1886,7 -1890,7 +1887,7 @@@ static int check_ok_to_remove(const cha
                 * files that are in "foo/" we would lose
                 * them.
                 */
 -              if (verify_clean_subdirectory(ce, error_type, o) < 0)
 +              if (verify_clean_subdirectory(ce, o) < 0)
                        return -1;
                return 0;
        }
                        return 0;
        }
  
 -      return o->gently ? -1 :
 -              add_rejected_path(o, error_type, name);
 +      return add_rejected_path(o, error_type, name);
  }
  
  /*
@@@ -2342,7 -2347,7 +2343,7 @@@ int bind_merge(const struct cache_entr
                return error("Cannot do a bind merge of %d trees",
                             o->merge_size);
        if (a && old)
 -              return o->gently ? -1 :
 +              return o->quiet ? -1 :
                        error(ERRORMSG(o, ERROR_BIND_OVERLAP),
                              super_prefixed(a->name),
                              super_prefixed(old->name));
@@@ -2382,7 -2387,7 +2383,7 @@@ int oneway_merge(const struct cache_ent
                if (o->update && S_ISGITLINK(old->ce_mode) &&
                    should_update_submodules() && !verify_uptodate(old, o))
                        update |= CE_UPDATE;
 -              add_entry(o, old, update, 0);
 +              add_entry(o, old, update, CE_STAGEMASK);
                return 0;
        }
        return merged_entry(a, old, o);