Merge branch 'en/rename-progress'

Historically, the diff machinery for rename detection had a
hardcoded limit of 32k paths; this is being lifted to allow users
trade cycles with a (possibly) easier to read result.

* en/rename-progress:
diffcore-rename: make diff-tree -l0 mean -l<large>
sequencer: show rename progress during cherry picks
diff: remove silent clamp of renameLimit
progress: fix progress meters when dealing with lots of work
sequencer: warn when internal merge may be suboptimal due to renameLimit

diff --combined diff.c
index 2ebe2227b467e53bbd759c21e463646701071999,40054070bdbeeafa72216fa193e4f0af4a87707e..b3ef25c3452c45227ac851dc730300d07c489bd0
--- 1/diff.c
--- 2/diff.c
+++ b/diff.c
@@@ -4129,7 -4129,9 +4129,7 @@@ void diff_setup_done(struct diff_option
         * inside contents.
         */
  
 -      if (DIFF_XDL_TST(options, IGNORE_WHITESPACE) ||
 -          DIFF_XDL_TST(options, IGNORE_WHITESPACE_CHANGE) ||
 -          DIFF_XDL_TST(options, IGNORE_WHITESPACE_AT_EOL))
 +      if ((options->xdl_opts & XDF_WHITESPACE_FLAGS))
                options->flags.diff_from_contents = 1;
        else
                options->flags.diff_from_contents = 0;
@@@ -4586,8 -4588,6 +4586,8 @@@ int diff_opt_parse(struct diff_options 
                DIFF_XDL_SET(options, IGNORE_WHITESPACE_CHANGE);
        else if (!strcmp(arg, "--ignore-space-at-eol"))
                DIFF_XDL_SET(options, IGNORE_WHITESPACE_AT_EOL);
 +      else if (!strcmp(arg, "--ignore-cr-at-eol"))
 +              DIFF_XDL_SET(options, IGNORE_CR_AT_EOL);
        else if (!strcmp(arg, "--ignore-blank-lines"))
                DIFF_XDL_SET(options, IGNORE_BLANK_LINES);
        else if (!strcmp(arg, "--indent-heuristic"))
@@@ -5454,7 -5454,7 +5454,7 @@@ void diff_warn_rename_limit(const char 
                warning(_(rename_limit_warning));
        else
                return;
-       if (0 < needed && needed < 32767)
+       if (0 < needed)
                warning(_(rename_limit_advice), varname, needed);
  }
  

diff --combined progress.c
index 5f87f4568f196a9d4ba531da1d0e3098629c7da2,edb97b1791eb5bcf3de46d638bceac1b88dce9e8..5a99c9fbf00bfb98d3a51cc7a7948b4e963b3517
--- 1/progress.c
--- 2/progress.c
+++ b/progress.c
@@@ -30,10 -30,11 +30,10 @@@ struct throughput 
  
  struct progress {
        const char *title;
-       int last_value;
-       unsigned total;
+       uint64_t last_value;
+       uint64_t total;
        unsigned last_percent;
        unsigned delay;
 -      unsigned delayed_percent_threshold;
        struct throughput *throughput;
        uint64_t start_ns;
  };
@@@ -78,12 -79,24 +78,12 @@@ static int is_foreground_fd(int fd
        return tpgrp < 0 || tpgrp == getpgid(0);
  }
  
- static int display(struct progress *progress, unsigned n, const char *done)
+ static int display(struct progress *progress, uint64_t n, const char *done)
  {
        const char *eol, *tp;
  
 -      if (progress->delay) {
 -              if (!progress_update || --progress->delay)
 -                      return 0;
 -              if (progress->total) {
 -                      unsigned percent = n * 100 / progress->total;
 -                      if (percent > progress->delayed_percent_threshold) {
 -                              /* inhibit this progress report entirely */
 -                              clear_progress_signal();
 -                              progress->delay = -1;
 -                              progress->total = 0;
 -                              return 0;
 -                      }
 -              }
 -      }
 +      if (progress->delay && (!progress_update || --progress->delay))
 +              return 0;
  
        progress->last_value = n;
        tp = (progress->throughput) ? progress->throughput->display.buf : "";
@@@ -93,9 -106,10 +93,10 @@@
                if (percent != progress->last_percent || progress_update) {
                        progress->last_percent = percent;
                        if (is_foreground_fd(fileno(stderr)) || done) {
-                               fprintf(stderr, "%s: %3u%% (%u/%u)%s%s",
-                                       progress->title, percent, n,
-                                       progress->total, tp, eol);
+                               fprintf(stderr, "%s: %3u%% (%"PRIuMAX"/%"PRIuMAX")%s%s",
+                                       progress->title, percent,
+                                       (uintmax_t)n, (uintmax_t)progress->total,
+                                       tp, eol);
                                fflush(stderr);
                        }
                        progress_update = 0;
@@@ -103,8 -117,8 +104,8 @@@
                }
        } else if (progress_update) {
                if (is_foreground_fd(fileno(stderr)) || done) {
-                       fprintf(stderr, "%s: %u%s%s",
-                               progress->title, n, tp, eol);
+                       fprintf(stderr, "%s: %"PRIuMAX"%s%s",
+                               progress->title, (uintmax_t)n, tp, eol);
                        fflush(stderr);
                }
                progress_update = 0;
@@@ -114,7 -128,7 +115,7 @@@
        return 0;
  }
  
- static void throughput_string(struct strbuf *buf, off_t total,
+ static void throughput_string(struct strbuf *buf, uint64_t total,
                              unsigned int rate)
  {
        strbuf_reset(buf);
@@@ -125,7 -139,7 +126,7 @@@
        strbuf_addstr(buf, "/s");
  }
  
- void display_throughput(struct progress *progress, off_t total)
+ void display_throughput(struct progress *progress, uint64_t total)
  {
        struct throughput *tp;
        uint64_t now_ns;
@@@ -187,13 -201,13 +188,13 @@@
                display(progress, progress->last_value, NULL);
  }
  
- int display_progress(struct progress *progress, unsigned n)
+ int display_progress(struct progress *progress, uint64_t n)
  {
        return progress ? display(progress, n, NULL) : 0;
  }
  
- static struct progress *start_progress_delay(const char *title, unsigned total,
+ static struct progress *start_progress_delay(const char *title, uint64_t total,
 -                                           unsigned percent_threshold, unsigned delay)
 +                                           unsigned delay)
  {
        struct progress *progress = malloc(sizeof(*progress));
        if (!progress) {
@@@ -206,6 -220,7 +207,6 @@@
        progress->total = total;
        progress->last_value = -1;
        progress->last_percent = -1;
 -      progress->delayed_percent_threshold = percent_threshold;
        progress->delay = delay;
        progress->throughput = NULL;
        progress->start_ns = getnanotime();
@@@ -213,14 -228,14 +214,14 @@@
        return progress;
  }
  
- struct progress *start_delayed_progress(const char *title, unsigned total)
+ struct progress *start_delayed_progress(const char *title, uint64_t total)
  {
 -      return start_progress_delay(title, total, 0, 2);
 +      return start_progress_delay(title, total, 2);
  }
  
- struct progress *start_progress(const char *title, unsigned total)
+ struct progress *start_progress(const char *title, uint64_t total)
  {
 -      return start_progress_delay(title, total, 0, 0);
 +      return start_progress_delay(title, total, 0);
  }
  
  void stop_progress(struct progress **p_progress)

diff --combined sequencer.c
index e90bc316bb7a232a40c332c6d1add255d3893869,c9e30776c20b3e3f420ed659de6fb258b7b3e7b0..c8cb20c1ebdd03f7897eb0ccecbb49d9ef9fd390
--- 1/sequencer.c
--- 2/sequencer.c
+++ b/sequencer.c
@@@ -347,7 -347,7 +347,7 @@@ static int read_oneliner(struct strbuf 
  
  static struct tree *empty_tree(void)
  {
 -      return lookup_tree(&empty_tree_oid);
 +      return lookup_tree(the_hash_algo->empty_tree);
  }
  
  static int error_dirty_index(struct replay_opts *opts)
@@@ -438,8 -438,7 +438,8 @@@ static int do_recursive_merge(struct co
        char **xopt;
        static struct lock_file index_lock;
  
 -      hold_locked_index(&index_lock, LOCK_DIE_ON_ERROR);
 +      if (hold_locked_index(&index_lock, LOCK_REPORT_ON_ERROR) < 0)
 +              return -1;
  
        read_cache();
  
@@@ -449,6 -448,7 +449,7 @@@
        o.branch2 = next ? next_label : "(empty tree)";
        if (is_rebase_i(opts))
                o.buffer_output = 2;
+       o.show_rename_progress = 1;
  
        head_tree = parse_tree_indirect(head);
        next_tree = next ? next->tree : empty_tree();
@@@ -463,6 -463,7 +464,7 @@@
        if (is_rebase_i(opts) && clean <= 0)
                fputs(o.obuf.buf, stdout);
        strbuf_release(&o.obuf);
+       diff_warn_rename_limit("merge.renamelimit", o.needed_rename_limit, 0);
        if (clean < 0)
                return clean;
  
@@@ -706,7 -707,7 +708,7 @@@ static int is_original_commit_empty(str
                                oid_to_hex(&parent->object.oid));
                ptree_oid = &parent->tree->object.oid;
        } else {
 -              ptree_oid = &empty_tree_oid; /* commit is root */
 +              ptree_oid = the_hash_algo->empty_tree; /* commit is root */
        }
  
        return !oidcmp(ptree_oid, &commit->tree->object.oid);
@@@ -959,7 -960,7 +961,7 @@@ static int do_pick_commit(enum todo_com
        } else {
                unborn = get_oid("HEAD", &head);
                if (unborn)
 -                      oidcpy(&head, &empty_tree_oid);
 +                      oidcpy(&head, the_hash_algo->empty_tree);
                if (index_differs_from(unborn ? EMPTY_TREE_SHA1_HEX : "HEAD",
                                       NULL, 0))
                        return error_dirty_index(opts);
@@@ -1118,11 -1119,11 +1120,11 @@@
         */
        if (command == TODO_PICK && !opts->no_commit && (res == 0 || res == 1) &&
            update_ref(NULL, "CHERRY_PICK_HEAD", &commit->object.oid, NULL,
 -                     REF_NODEREF, UPDATE_REFS_MSG_ON_ERR))
 +                     REF_NO_DEREF, UPDATE_REFS_MSG_ON_ERR))
                res = -1;
        if (command == TODO_REVERT && ((opts->no_commit && res == 0) || res == 1) &&
            update_ref(NULL, "REVERT_HEAD", &commit->object.oid, NULL,
 -                     REF_NODEREF, UPDATE_REFS_MSG_ON_ERR))
 +                     REF_NO_DEREF, UPDATE_REFS_MSG_ON_ERR))
                res = -1;
  
        if (res) {
@@@ -2131,7 -2132,7 +2133,7 @@@ cleanup_head_ref
                        msg = reflog_message(opts, "finish", "%s onto %s",
                                head_ref.buf, buf.buf);
                        if (update_ref(msg, head_ref.buf, &head, &orig,
 -                                     REF_NODEREF, UPDATE_REFS_MSG_ON_ERR)) {
 +                                     REF_NO_DEREF, UPDATE_REFS_MSG_ON_ERR)) {
                                res = error(_("could not update %s"),
                                        head_ref.buf);
                                goto cleanup_head_ref;
@@@ -2671,19 -2672,6 +2673,19 @@@ leave_check
        return res;
  }
  
 +static int rewrite_file(const char *path, const char *buf, size_t len)
 +{
 +      int rc = 0;
 +      int fd = open(path, O_WRONLY | O_TRUNC);
 +      if (fd < 0)
 +              return error_errno(_("could not open '%s' for writing"), path);
 +      if (write_in_full(fd, buf, len) < 0)
 +              rc = error_errno(_("could not write to '%s'"), path);
 +      if (close(fd) && !rc)
 +              rc = error_errno(_("could not close '%s'"), path);
 +      return rc;
 +}
 +
  /* skip picking commits whose parents are unchanged */
  int skip_unnecessary_picks(void)
  {
@@@ -2756,11 -2744,29 +2758,11 @@@
                }
                close(fd);
  
 -              fd = open(rebase_path_todo(), O_WRONLY, 0666);
 -              if (fd < 0) {
 -                      error_errno(_("could not open '%s' for writing"),
 -                                  rebase_path_todo());
 +              if (rewrite_file(rebase_path_todo(), todo_list.buf.buf + offset,
 +                               todo_list.buf.len - offset) < 0) {
                        todo_list_release(&todo_list);
                        return -1;
                }
 -              if (write_in_full(fd, todo_list.buf.buf + offset,
 -                              todo_list.buf.len - offset) < 0) {
 -                      error_errno(_("could not write to '%s'"),
 -                                  rebase_path_todo());
 -                      close(fd);
 -                      todo_list_release(&todo_list);
 -                      return -1;
 -              }
 -              if (ftruncate(fd, todo_list.buf.len - offset) < 0) {
 -                      error_errno(_("could not truncate '%s'"),
 -                                  rebase_path_todo());
 -                      todo_list_release(&todo_list);
 -                      close(fd);
 -                      return -1;
 -              }
 -              close(fd);
  
                todo_list.current = i;
                if (is_fixup(peek_command(&todo_list, 0)))
@@@ -2945,7 -2951,15 +2947,7 @@@ int rearrange_squash(void
                        }
                }
  
 -              fd = open(todo_file, O_WRONLY);
 -              if (fd < 0)
 -                      res = error_errno(_("could not open '%s'"), todo_file);
 -              else if (write(fd, buf.buf, buf.len) < 0)
 -                      res = error_errno(_("could not write to '%s'"), todo_file);
 -              else if (ftruncate(fd, buf.len) < 0)
 -                      res = error_errno(_("could not truncate '%s'"),
 -                                         todo_file);
 -              close(fd);
 +              res = rewrite_file(todo_file, buf.buf, buf.len);
                strbuf_release(&buf);
        }