Merge branch 'xz/send-email-batch-size'
authorJunio C Hamano <gitster@pobox.com>
Fri, 7 Jul 2017 01:14:46 +0000 (18:14 -0700)
committerJunio C Hamano <gitster@pobox.com>
Fri, 7 Jul 2017 01:14:46 +0000 (18:14 -0700)
"git send-email" learned to overcome some SMTP server limitation
that does not allow many pieces of e-mails to be sent over a single
session.

* xz/send-email-batch-size:
send-email: --batch-size to work around some SMTP server limit

1  2 
Documentation/config.txt
Documentation/git-send-email.txt
contrib/completion/git-completion.bash
git-send-email.perl
diff --combined Documentation/config.txt
index 06898a7498f47d2e935e330ffbd34467b6bd8f07,ee4a1118787ad91cc4639ba9274cda0793cceef9..d5c9c4cab60531d178d5c11d623a5c3f1036a0d0
@@@ -79,20 -79,14 +79,20 @@@ escape sequences) are invalid
  Includes
  ~~~~~~~~
  
 +The `include` and `includeIf` sections allow you to include config
 +directives from another source. These sections behave identically to
 +each other with the exception that `includeIf` sections may be ignored
 +if their condition does not evaluate to true; see "Conditional includes"
 +below.
 +
  You can include a config file from another by setting the special
 -`include.path` variable to the name of the file to be included. The
 -variable takes a pathname as its value, and is subject to tilde
 -expansion. `include.path` can be given multiple times.
 +`include.path` (or `includeIf.*.path`) variable to the name of the file
 +to be included. The variable takes a pathname as its value, and is
 +subject to tilde expansion. These variables can be given multiple times.
  
 -The included file is expanded immediately, as if its contents had been
 -found at the location of the include directive. If the value of the
 -`include.path` variable is a relative path, the path is considered to
 +The contents of the included file are inserted immediately, as if they
 +had been found at the location of the include directive. If the value of the
 +variable is a relative path, the path is considered to
  be relative to the configuration file in which the include directive
  was found.  See below for examples.
  
@@@ -101,7 -95,8 +101,7 @@@ Conditional include
  
  You can include a config file from another conditionally by setting a
  `includeIf.<condition>.path` variable to the name of the file to be
 -included. The variable's value is treated the same way as
 -`include.path`. `includeIf.<condition>.path` can be given multiple times.
 +included.
  
  The condition starts with a keyword followed by a colon and some data
  whose format and meaning depends on the keyword. Supported keywords
@@@ -145,16 -140,6 +145,16 @@@ A few more notes on matching via `gitdi
  
   * Symlinks in `$GIT_DIR` are not resolved before matching.
  
 + * Both the symlink & realpath versions of paths will be matched
 +   outside of `$GIT_DIR`. E.g. if ~/git is a symlink to
 +   /mnt/storage/git, both `gitdir:~/git` and `gitdir:/mnt/storage/git`
 +   will match.
 ++
 +This was not the case in the initial release of this feature in
 +v2.13.0, which only matched the realpath version. Configuration that
 +wants to be compatible with the initial release of this feature needs
 +to either specify only the realpath version, or both versions.
 +
   * Note that "../" is not special and will match literally, which is
     unlikely what you want.
  
@@@ -182,8 -167,8 +182,8 @@@ Exampl
  
        [include]
                path = /path/to/foo.inc ; include by absolute path
 -              path = foo ; expand "foo" relative to the current file
 -              path = ~/foo ; expand "foo" in your `$HOME` directory
 +              path = foo.inc ; find "foo.inc" relative to the current file
 +              path = ~/foo.inc ; find "foo.inc" in your `$HOME` directory
  
        ; include if $GIT_DIR is /path/to/foo/.git
        [includeIf "gitdir:/path/to/foo/.git"]
        [includeIf "gitdir:~/to/group/"]
                path = /path/to/foo.inc
  
 +      ; relative paths are always relative to the including
 +      ; file (if the condition is true); their location is not
 +      ; affected by the condition
 +      [includeIf "gitdir:/path/to/group/"]
 +              path = foo.inc
 +
  Values
  ~~~~~~
  
@@@ -348,9 -327,6 +348,9 @@@ advice.*:
        rmHints::
                In case of failure in the output of linkgit:git-rm[1],
                show directions on how to proceed from the current state.
 +      addEmbeddedRepo::
 +              Advice on what to do when you've accidentally added one
 +              git repo inside of another.
  --
  
  core.fileMode::
        is to be honored.
  +
  Some filesystems lose the executable bit when a file that is
 -marked as executable is checked out, or checks out an
 +marked as executable is checked out, or checks out a
  non-executable file with executable bit on.
  linkgit:git-clone[1] or linkgit:git-init[1] probe the filesystem
  to see if it handles the executable bit correctly
@@@ -686,8 -662,7 +686,8 @@@ core.packedGitLimit:
        bytes at once to complete an operation it will unmap existing
        regions to reclaim virtual address space within the process.
  +
 -Default is 256 MiB on 32 bit platforms and 8 GiB on 64 bit platforms.
 +Default is 256 MiB on 32 bit platforms and 32 TiB (effectively
 +unlimited) on 64 bit platforms.
  This should be reasonable for all users/operating systems, except on
  the largest projects.  You probably do not need to adjust this value.
  +
@@@ -887,7 -862,6 +887,7 @@@ core.abbrev:
        computed based on the approximate number of packed objects
        in your repository, which hopefully is enough for
        abbreviated object names to stay unique for some time.
 +      The minimum length is 4.
  
  add.ignoreErrors::
  add.ignore-errors (deprecated)::
@@@ -1163,10 -1137,7 +1163,10 @@@ color.status.<slot>:
        `untracked` (files which are not tracked by Git),
        `branch` (the current branch),
        `nobranch` (the color the 'no branch' warning is shown in, defaulting
 -      to red), or
 +      to red),
 +      `localBranch` or `remoteBranch` (the local and remote branch names,
 +      respectively, when branch and tracking information is displayed in the
 +      status short-format), or
        `unmerged` (files which have unmerged changes).
  
  color.ui::
@@@ -2169,10 -2140,6 +2169,10 @@@ log.showRoot:
        Tools like linkgit:git-log[1] or linkgit:git-whatchanged[1], which
        normally hide the root commit will now show it. True by default.
  
 +log.showSignature::
 +      If true, makes linkgit:git-log[1], linkgit:git-show[1], and
 +      linkgit:git-whatchanged[1] assume `--show-signature`.
 +
  log.mailmap::
        If true, makes linkgit:git-log[1], linkgit:git-show[1], and
        linkgit:git-whatchanged[1] assume `--use-mailmap`.
@@@ -2624,7 -2591,7 +2624,7 @@@ rebase.autoSquash:
        If set to true enable `--autosquash` option by default.
  
  rebase.autoStash::
 -      When set to true, automatically create a temporary stash
 +      When set to true, automatically create a temporary stash entry
        before the operation begins, and apply it after the operation
        ends.  This means that you can run rebase on a dirty worktree.
        However, use with care: the final stash application after a
@@@ -2653,8 -2620,9 +2653,8 @@@ receive.advertiseAtomic:
        capability, set this variable to false.
  
  receive.advertisePushOptions::
 -      By default, git-receive-pack will advertise the push options
 -      capability to its clients. If you don't want to advertise this
 -      capability, set this variable to false.
 +      When set to true, git-receive-pack will advertise the push options
 +      capability to its clients. False by default.
  
  receive.autogc::
        By default, git-receive-pack will run "git-gc --auto" after
@@@ -2946,6 -2914,16 +2946,16 @@@ sendemail.xmailer:
  sendemail.signedoffcc (deprecated)::
        Deprecated alias for `sendemail.signedoffbycc`.
  
+ sendemail.smtpBatchSize::
+       Number of messages to be sent per connection, after that a relogin
+       will happen.  If the value is 0 or undefined, send all messages in
+       one connection.
+       See also the `--batch-size` option of linkgit:git-send-email[1].
+ sendemail.smtpReloginDelay::
+       Seconds wait before reconnecting to smtp server.
+       See also the `--relogin-delay` option of linkgit:git-send-email[1].
  showbranch.default::
        The default set of branches for linkgit:git-show-branch[1].
        See linkgit:git-show-branch[1].
@@@ -2996,11 -2974,6 +3006,11 @@@ status.displayCommentPrefix:
        behavior of linkgit:git-status[1] in Git 1.8.4 and previous.
        Defaults to false.
  
 +status.showStash::
 +      If set to true, linkgit:git-status[1] will display the number of
 +      entries currently stashed away.
 +      Defaults to false.
 +
  status.showUntrackedFiles::
        By default, linkgit:git-status[1] and linkgit:git-commit[1] show
        files which are not currently tracked by Git. Directories which
@@@ -3038,12 -3011,12 +3048,12 @@@ status.submoduleSummary:
  
  stash.showPatch::
        If this is set to true, the `git stash show` command without an
 -      option will show the stash in patch form.  Defaults to false.
 +      option will show the stash entry in patch form.  Defaults to false.
        See description of 'show' command in linkgit:git-stash[1].
  
  stash.showStat::
        If this is set to true, the `git stash show` command without an
 -      option will show diffstat of the stash.  Defaults to true.
 +      option will show diffstat of the stash entry.  Defaults to true.
        See description of 'show' command in linkgit:git-stash[1].
  
  submodule.<name>.url::
@@@ -3100,11 -3073,6 +3110,11 @@@ submodule.active:
        submodule's path to determine if the submodule is of interest to git
        commands.
  
 +submodule.recurse::
 +      Specifies if commands recurse into submodules by default. This
 +      applies to all commands that have a `--recurse-submodules` option.
 +      Defaults to false.
 +
  submodule.fetchJobs::
        Specifies how many submodules are fetched/cloned at the same time.
        A positive integer allows up to that number of submodules fetched
@@@ -3250,13 -3218,6 +3260,13 @@@ url.<base>.insteadOf:
        the best alternative for the particular user, even for a
        never-before-seen repository on the site.  When more than one
        insteadOf strings match a given URL, the longest match is used.
 ++
 +Note that any protocol restrictions will be applied to the rewritten
 +URL. If the rewrite changes the URL to use a custom protocol or remote
 +helper, you may need to adjust the `protocol.*.allow` config to permit
 +the request.  In particular, protocols you expect to use for submodules
 +must be set to `always` rather than the default of `user`. See the
 +description of `protocol.allow` above.
  
  url.<base>.pushInsteadOf::
        Any URL that starts with this value will not be pushed to;
index bb23b02caf3ee50a32c86d97e0451cd4be6ab232,79b418bfa5a8a32bd9a91d7a45fa662de94a9340..bac9014ac71775e5116ad7686af697eb22286ee2
@@@ -248,6 -248,21 +248,21 @@@ must be used for each option
        commands and replies will be printed. Useful to debug TLS
        connection and authentication problems.
  
+ --batch-size=<num>::
+       Some email servers (e.g. smtp.163.com) limit the number emails to be
+       sent per session (connection) and this will lead to a faliure when
+       sending many messages.  With this option, send-email will disconnect after
+       sending $<num> messages and wait for a few seconds (see --relogin-delay)
+       and reconnect, to work around such a limit.  You may want to
+       use some form of credential helper to avoid having to retype
+       your password every time this happens.  Defaults to the
+       `sendemail.smtpBatchSize` configuration variable.
+ --relogin-delay=<int>::
+       Waiting $<int> seconds before reconnecting to SMTP server. Used together
+       with --batch-size option.  Defaults to the `sendemail.smtpReloginDelay`
+       configuration variable.
  Automating
  ~~~~~~~~~~
  
@@@ -377,7 -392,6 +392,7 @@@ have been specified, in which case defa
        Currently, validation means the following:
  +
  --
 +              *       Invoke the sendemail-validate hook if present (see linkgit:githooks[5]).
                *       Warn of patches that contain lines longer than 998 characters; this
                        is due to SMTP limits as described by http://www.ietf.org/rfc/rfc2821.txt.
  --
index 48a2f26622d992b89462edb3658b5cdccee806bc,29496353a462ba7e5f6b2de82d1a61bbe261729b..d93441747523ab2971e114cd6a02d56b79ba3382
@@@ -1319,7 -1319,6 +1319,7 @@@ _git_clone (
                        --template=
                        --depth
                        --single-branch
 +                      --no-tags
                        --branch
                        --recurse-submodules
                        --no-single-branch
@@@ -2336,23 -2335,14 +2336,23 @@@ _git_config (
        esac
        __gitcomp "
                add.ignoreErrors
 +              advice.amWorkDir
                advice.commitBeforeMerge
                advice.detachedHead
                advice.implicitIdentity
 -              advice.pushNonFastForward
 +              advice.pushAlreadyExists
 +              advice.pushFetchFirst
 +              advice.pushNeedsForce
 +              advice.pushNonFFCurrent
 +              advice.pushNonFFMatching
 +              advice.pushUpdateRejected
                advice.resolveConflict
 +              advice.rmHints
                advice.statusHints
 +              advice.statusUoption
                alias.
                am.keepcr
 +              am.threeWay
                apply.ignorewhitespace
                apply.whitespace
                branch.autosetupmerge
                color.status.added
                color.status.changed
                color.status.header
 +              color.status.localBranch
                color.status.nobranch
 +              color.status.remoteBranch
                color.status.unmerged
                color.status.untracked
                color.status.updated
                color.ui
 +              commit.cleanup
 +              commit.gpgSign
                commit.status
                commit.template
 +              commit.verbose
                core.abbrev
                core.askpass
                core.attributesfile
                core.autocrlf
                core.bare
                core.bigFileThreshold
 +              core.checkStat
 +              core.commentChar
                core.compression
                core.createObject
                core.deltaBaseCacheLimit
                core.fileMode
                core.fsyncobjectfiles
                core.gitProxy
 +              core.hideDotFiles
 +              core.hooksPath
                core.ignoreStat
                core.ignorecase
                core.logAllRefUpdates
                core.notesRef
                core.packedGitLimit
                core.packedGitWindowSize
 +              core.packedRefsTimeout
                core.pager
 +              core.precomposeUnicode
                core.preferSymlinkRefs
                core.preloadindex
 +              core.protectHFS
 +              core.protectNTFS
                core.quotepath
                core.repositoryFormatVersion
                core.safecrlf
                core.sharedRepository
                core.sparseCheckout
 +              core.splitIndex
 +              core.sshCommand
                core.symlinks
                core.trustctime
                core.untrackedCache
                core.warnAmbiguousRefs
                core.whitespace
                core.worktree
 +              credential.helper
 +              credential.useHttpPath
 +              credential.username
 +              credentialCache.ignoreSIGHUP
                diff.autorefreshindex
                diff.external
                diff.ignoreSubmodules
                format.thread
                format.to
                gc.
 +              gc.aggressiveDepth
                gc.aggressiveWindow
                gc.auto
 +              gc.autoDetach
                gc.autopacklimit
 +              gc.logExpiry
                gc.packrefs
                gc.pruneexpire
                gc.reflogexpire
                gc.reflogexpireunreachable
                gc.rerereresolved
                gc.rerereunresolved
 +              gc.worktreePruneExpire
                gitcvs.allbinary
                gitcvs.commitmsgannotation
                gitcvs.dbTableNamePrefix
                sendemail.thread
                sendemail.to
                sendemail.validate
+               sendemail.smtpbatchsize
+               sendemail.smtprelogindelay
                showbranch.default
                status.relativePaths
                status.showUntrackedFiles
@@@ -2843,7 -2812,7 +2845,7 @@@ _git_show_branch (
  _git_stash ()
  {
        local save_opts='--all --keep-index --no-keep-index --quiet --patch --include-untracked'
 -      local subcommands='save list show apply clear drop pop create branch'
 +      local subcommands='push save list show apply clear drop pop create branch'
        local subcommand="$(__git_find_on_cmdline "$subcommands")"
        if [ -z "$subcommand" ]; then
                case "$cur" in
                esac
        else
                case "$subcommand,$cur" in
 +              push,--*)
 +                      __gitcomp "$save_opts --message"
 +                      ;;
                save,--*)
                        __gitcomp "$save_opts"
                        ;;
diff --combined git-send-email.perl
index 7fd58744360a2bb14531ed3f73092b45449d5590,8a1ee0f0d43fdbc52b8257d973cb21063f318731..fa6526986ed40143a12ffcd6563fd52b49d209e7
@@@ -25,9 -25,8 +25,9 @@@ use Getopt::Long
  use Text::ParseWords;
  use Term::ANSIColor;
  use File::Temp qw/ tempdir tempfile /;
 -use File::Spec::Functions qw(catfile);
 +use File::Spec::Functions qw(catdir catfile);
  use Error qw(:try);
 +use Cwd qw(abs_path cwd);
  use Git;
  use Git::I18N;
  
@@@ -82,6 -81,10 +82,10 @@@ git send-email --dump-aliase
                                       This setting forces to use one of the listed mechanisms.
      --smtp-debug            <0|1>  * Disable, enable Net::SMTP debug.
  
+     --batch-size            <int>  * send max <int> message per connection.
+     --relogin-delay         <int>  * delay <int> seconds between two successive login.
+                                      This option can only be used with --batch-size
    Automating:
      --identity              <str>  * Use the sendemail.<id> options.
      --to-cmd                <str>  * Email To: via `<str> \$patch_path`
@@@ -154,6 -157,7 +158,7 @@@ my $have_email_valid = eval { require E
  my $have_mail_address = eval { require Mail::Address; 1 };
  my $smtp;
  my $auth;
+ my $num_sent = 0;
  
  # Regexes for RFC 2047 productions.
  my $re_token = qr/[^][()<>@,;:\\"\/?.= \000-\037\177-\377]+/;
@@@ -217,6 -221,7 +222,7 @@@ my ($cover_cc, $cover_to)
  my ($to_cmd, $cc_cmd);
  my ($smtp_server, $smtp_server_port, @smtp_server_options);
  my ($smtp_authuser, $smtp_encryption, $smtp_ssl_cert_path);
+ my ($batch_size, $relogin_delay);
  my ($identity, $aliasfiletype, @alias_files, $smtp_domain, $smtp_auth);
  my ($validate, $confirm);
  my (@suppress_cc);
@@@ -248,6 -253,8 +254,8 @@@ my %config_settings = 
      "smtppass" => \$smtp_authpass,
      "smtpdomain" => \$smtp_domain,
      "smtpauth" => \$smtp_auth,
+     "smtpbatchsize" => \$batch_size,
+     "smtprelogindelay" => \$relogin_delay,
      "to" => \@initial_to,
      "tocmd" => \$to_cmd,
      "cc" => \@initial_cc,
@@@ -359,6 -366,8 +367,8 @@@ $rc = GetOptions
                    "force" => \$force,
                    "xmailer!" => \$use_xmailer,
                    "no-xmailer" => sub {$use_xmailer = 0},
+                   "batch-size=i" => \$batch_size,
+                   "relogin-delay=i" => \$relogin_delay,
         );
  
  usage() if $help;
                        die __("The required SMTP server is not properly defined.")
                }
  
 +              require Net::SMTP;
 +              my $use_net_smtp_ssl = version->parse($Net::SMTP::VERSION) < version->parse("2.34");
 +              $smtp_domain ||= maildomain();
 +
                if ($smtp_encryption eq 'ssl') {
                        $smtp_server_port ||= 465; # ssmtp
 -                      require Net::SMTP::SSL;
 -                      $smtp_domain ||= maildomain();
                        require IO::Socket::SSL;
  
                        # Suppress "variable accessed once" warning.
                        # Net::SMTP::SSL->new() does not forward any SSL options
                        IO::Socket::SSL::set_client_defaults(
                                ssl_verify_params());
 -                      $smtp ||= Net::SMTP::SSL->new($smtp_server,
 -                                                    Hello => $smtp_domain,
 -                                                    Port => $smtp_server_port,
 -                                                    Debug => $debug_net_smtp);
 +
 +                      if ($use_net_smtp_ssl) {
 +                              require Net::SMTP::SSL;
 +                              $smtp ||= Net::SMTP::SSL->new($smtp_server,
 +                                                            Hello => $smtp_domain,
 +                                                            Port => $smtp_server_port,
 +                                                            Debug => $debug_net_smtp);
 +                      }
 +                      else {
 +                              $smtp ||= Net::SMTP->new($smtp_server,
 +                                                       Hello => $smtp_domain,
 +                                                       Port => $smtp_server_port,
 +                                                       Debug => $debug_net_smtp,
 +                                                       SSL => 1);
 +                      }
                }
                else {
 -                      require Net::SMTP;
 -                      $smtp_domain ||= maildomain();
                        $smtp_server_port ||= 25;
                        $smtp ||= Net::SMTP->new($smtp_server,
                                                 Hello => $smtp_domain,
                                                 Debug => $debug_net_smtp,
                                                 Port => $smtp_server_port);
                        if ($smtp_encryption eq 'tls' && $smtp) {
 -                              require Net::SMTP::SSL;
 -                              $smtp->command('STARTTLS');
 -                              $smtp->response();
 -                              if ($smtp->code == 220) {
 +                              if ($use_net_smtp_ssl) {
 +                                      $smtp->command('STARTTLS');
 +                                      $smtp->response();
 +                                      if ($smtp->code != 220) {
 +                                              die sprintf(__("Server does not support STARTTLS! %s"), $smtp->message);
 +                                      }
 +                                      require Net::SMTP::SSL;
                                        $smtp = Net::SMTP::SSL->start_SSL($smtp,
                                                                          ssl_verify_params())
 -                                              or die "STARTTLS failed! ".IO::Socket::SSL::errstr();
 -                                      $smtp_encryption = '';
 -                                      # Send EHLO again to receive fresh
 -                                      # supported commands
 -                                      $smtp->hello($smtp_domain);
 -                              } else {
 -                                      die sprintf(__("Server does not support STARTTLS! %s"), $smtp->message);
 +                                              or die sprintf(__("STARTTLS failed! %s"), IO::Socket::SSL::errstr());
 +                              }
 +                              else {
 +                                      $smtp->starttls(ssl_verify_params())
 +                                              or die sprintf(__("STARTTLS failed! %s"), IO::Socket::SSL::errstr());
                                }
 +                              $smtp_encryption = '';
 +                              # Send EHLO again to receive fresh
 +                              # supported commands
 +                              $smtp->hello($smtp_domain);
                        }
                }
  
@@@ -1681,6 -1674,14 +1691,14 @@@ foreach my $t (@files) 
                }
        }
        $message_id = undef;
+       $num_sent++;
+       if (defined $batch_size && $num_sent == $batch_size) {
+               $num_sent = 0;
+               $smtp->quit if defined $smtp;
+               undef $smtp;
+               undef $auth;
+               sleep($relogin_delay) if defined $relogin_delay;
+       }
  }
  
  # Execute a command (e.g. $to_cmd) to get a list of email addresses
@@@ -1754,25 -1755,6 +1772,25 @@@ sub unique_email_list 
  
  sub validate_patch {
        my $fn = shift;
 +
 +      if ($repo) {
 +              my $validate_hook = catfile(catdir($repo->repo_path(), 'hooks'),
 +                                          'sendemail-validate');
 +              my $hook_error;
 +              if (-x $validate_hook) {
 +                      my $target = abs_path($fn);
 +                      # The hook needs a correct cwd and GIT_DIR.
 +                      my $cwd_save = cwd();
 +                      chdir($repo->wc_path() or $repo->repo_path())
 +                              or die("chdir: $!");
 +                      local $ENV{"GIT_DIR"} = $repo->repo_path();
 +                      $hook_error = "rejected by sendemail-validate hook"
 +                              if system($validate_hook, $target);
 +                      chdir($cwd_save) or die("chdir: $!");
 +              }
 +              return $hook_error if $hook_error;
 +      }
 +
        open(my $fh, '<', $fn)
                or die sprintf(__("unable to open %s: %s\n"), $fn, $!);
        while (my $line = <$fh>) {