parse_object: try internal cache before reading object db
[gitweb.git] / git-send-email.perl
index cccbf4517aa46d4d217a1ce25105f7c413301d2d..d491db92c9379cf0eea63182dd704feb6582b6e7 100755 (executable)
@@ -1,4 +1,4 @@
-#!/usr/bin/perl -w
+#!/usr/bin/perl
 #
 # Copyright 2002,2005 Greg Kroah-Hartman <greg@kroah.com>
 # Copyright 2005 Ryan Anderson <ryan@michonline.com>
@@ -16,6 +16,7 @@
 #    and second line is the subject of the message.
 #
 
+use 5.008;
 use strict;
 use warnings;
 use Term::ReadLine;
@@ -24,6 +25,7 @@
 use Data::Dumper;
 use Term::ANSIColor;
 use File::Temp qw/ tempdir tempfile /;
+use File::Spec::Functions qw(catfile);
 use Error qw(:try);
 use Git;
 
@@ -47,31 +49,36 @@ sub usage {
 
   Composing:
     --from                  <str>  * Email From:
-    --to                    <str>  * Email To:
-    --cc                    <str>  * Email Cc:
-    --bcc                   <str>  * Email Bcc:
+    --[no-]to               <str>  * Email To:
+    --[no-]cc               <str>  * Email Cc:
+    --[no-]bcc              <str>  * Email Bcc:
     --subject               <str>  * Email "Subject:"
     --in-reply-to           <str>  * Email "In-Reply-To:"
     --annotate                     * Review each patch that will be sent in an editor.
     --compose                      * Open an editor for introduction.
+    --8bit-encoding         <str>  * Encoding to assume 8bit mails if undeclared
 
   Sending:
     --envelope-sender       <str>  * Email envelope sender.
     --smtp-server       <str:int>  * Outgoing SMTP server to use. The port
                                      is optional. Default 'localhost'.
+    --smtp-server-option    <str>  * Outgoing SMTP server option to use.
     --smtp-server-port      <int>  * Outgoing SMTP server port.
     --smtp-user             <str>  * Username for SMTP-AUTH.
     --smtp-pass             <str>  * Password for SMTP-AUTH; not necessary.
     --smtp-encryption       <str>  * tls or ssl; anything else disables.
     --smtp-ssl                     * Deprecated. Use '--smtp-encryption ssl'.
+    --smtp-domain           <str>  * The domain name sent to HELO/EHLO handshake
+    --smtp-debug            <0|1>  * Disable, enable Net::SMTP debug.
 
   Automating:
     --identity              <str>  * Use the sendemail.<id> options.
+    --to-cmd                <str>  * Email To: via `<str> \$patch_path`
     --cc-cmd                <str>  * Email Cc: via `<str> \$patch_path`
     --suppress-cc           <str>  * author, self, sob, cc, cccmd, body, bodycc, all.
     --[no-]signed-off-by-cc        * Send to Signed-off-by: addresses. Default on.
     --[no-]suppress-from           * Send to self. Default off.
-    --[no-]chain-reply-to          * Chain In-Reply-To: fields. Default on.
+    --[no-]chain-reply-to          * Chain In-Reply-To: fields. Default off.
     --[no-]thread                  * Use In-Reply-To: field. Default on.
 
   Administering:
@@ -82,6 +89,7 @@ sub usage {
     --[no-]validate                * Perform patch sanity checks. Default on.
     --[no-]format-patch            * understand any non optional arguments as
                                      `git format-patch` ones.
+    --force                        * Send even if safety checks would prevent it.
 
 EOT
        exit(1);
@@ -131,11 +139,8 @@ sub format_2822_time {
 my $smtp;
 my $auth;
 
-sub unique_email_list(@);
-sub cleanup_compose_files();
-
 # Variables we fill in automatically, or via prompting:
-my (@to,@cc,@initial_cc,@bcclist,@xh,
+my (@to,$no_to,@initial_to,@cc,$no_cc,@initial_cc,@bcclist,$no_bcc,@xh,
        $initial_reply_to,$initial_subject,@files,
        $author,$sender,$smtp_authpass,$annotate,$compose,$time);
 
@@ -159,11 +164,16 @@ sub format_2822_time {
 my ($quiet, $dry_run) = (0, 0);
 my $format_patch;
 my $compose_filename;
+my $force = 0;
 
 # Handle interactive edition of files.
 my $multiedit;
-my $editor = $ENV{GIT_EDITOR} || Git::config(@repo, "core.editor") || $ENV{VISUAL} || $ENV{EDITOR} || "vi";
+my $editor;
+
 sub do_edit {
+       if (!defined($editor)) {
+               $editor = Git::command_oneline('var', 'GIT_EDITOR');
+       }
        if (defined($multiedit) && !$multiedit) {
                map {
                        system('sh', '-c', $editor.' "$@"', $editor, $_);
@@ -180,15 +190,22 @@ sub do_edit {
 }
 
 # Variables with corresponding config settings
-my ($thread, $chain_reply_to, $suppress_from, $signed_off_by_cc, $cc_cmd);
-my ($smtp_server, $smtp_server_port, $smtp_authuser, $smtp_encryption);
-my ($identity, $aliasfiletype, @alias_files, @smtp_host_parts);
+my ($thread, $chain_reply_to, $suppress_from, $signed_off_by_cc);
+my ($to_cmd, $cc_cmd);
+my ($smtp_server, $smtp_server_port, @smtp_server_options);
+my ($smtp_authuser, $smtp_encryption);
+my ($identity, $aliasfiletype, @alias_files, $smtp_domain);
 my ($validate, $confirm);
 my (@suppress_cc);
+my ($auto_8bit_encoding);
+
+my ($debug_net_smtp) = 0;              # Net::SMTP, see send_message()
+
+my $not_set_by_user = "true but not set by the user";
 
 my %config_bool_settings = (
     "thread" => [\$thread, 1],
-    "chainreplyto" => [\$chain_reply_to, 1],
+    "chainreplyto" => [\$chain_reply_to, $not_set_by_user],
     "suppressfrom" => [\$suppress_from, undef],
     "signedoffbycc" => [\$signed_off_by_cc, undef],
     "signedoffcc" => [\$signed_off_by_cc, undef],      # Deprecated
@@ -198,20 +215,41 @@ sub do_edit {
 my %config_settings = (
     "smtpserver" => \$smtp_server,
     "smtpserverport" => \$smtp_server_port,
+    "smtpserveroption" => \@smtp_server_options,
     "smtpuser" => \$smtp_authuser,
     "smtppass" => \$smtp_authpass,
-    "to" => \@to,
+    "smtpdomain" => \$smtp_domain,
+    "to" => \@initial_to,
+    "tocmd" => \$to_cmd,
     "cc" => \@initial_cc,
     "cccmd" => \$cc_cmd,
     "aliasfiletype" => \$aliasfiletype,
     "bcc" => \@bcclist,
-    "aliasesfile" => \@alias_files,
     "suppresscc" => \@suppress_cc,
     "envelopesender" => \$envelope_sender,
     "multiedit" => \$multiedit,
     "confirm"   => \$confirm,
+    "from" => \$sender,
+    "assume8bitencoding" => \$auto_8bit_encoding,
 );
 
+my %config_path_settings = (
+    "aliasesfile" => \@alias_files,
+);
+
+# Help users prepare for 1.7.0
+sub chain_reply_to {
+       if (defined $chain_reply_to &&
+           $chain_reply_to eq $not_set_by_user) {
+               print STDERR
+                   "In git 1.7.0, the default has changed to --no-chain-reply-to\n" .
+                   "Set sendemail.chainreplyto configuration variable to true if\n" .
+                   "you want to keep --chain-reply-to as your default.\n";
+               $chain_reply_to = 0;
+       }
+       return $chain_reply_to;
+}
+
 # Handle Uncouth Termination
 sub signal_handler {
 
@@ -240,19 +278,28 @@ sub signal_handler {
 # Begin by accumulating all the variables (defined above), that we will end up
 # needing, first, from the command line:
 
-my $rc = GetOptions("sender|from=s" => \$sender,
+my $help;
+my $rc = GetOptions("h" => \$help,
+                   "sender|from=s" => \$sender,
                     "in-reply-to=s" => \$initial_reply_to,
                    "subject=s" => \$initial_subject,
-                   "to=s" => \@to,
+                   "to=s" => \@initial_to,
+                   "to-cmd=s" => \$to_cmd,
+                   "no-to" => \$no_to,
                    "cc=s" => \@initial_cc,
+                   "no-cc" => \$no_cc,
                    "bcc=s" => \@bcclist,
+                   "no-bcc" => \$no_bcc,
                    "chain-reply-to!" => \$chain_reply_to,
                    "smtp-server=s" => \$smtp_server,
+                   "smtp-server-option=s" => \@smtp_server_options,
                    "smtp-server-port=s" => \$smtp_server_port,
                    "smtp-user=s" => \$smtp_authuser,
                    "smtp-pass:s" => \$smtp_authpass,
                    "smtp-ssl" => sub { $smtp_encryption = 'ssl' },
                    "smtp-encryption=s" => \$smtp_encryption,
+                   "smtp-debug:i" => \$debug_net_smtp,
+                   "smtp-domain:s" => \$smtp_domain,
                    "identity=s" => \$identity,
                    "annotate" => \$annotate,
                    "compose" => \$compose,
@@ -267,8 +314,11 @@ sub signal_handler {
                    "thread!" => \$thread,
                    "validate!" => \$validate,
                    "format-patch!" => \$format_patch,
+                   "8bit-encoding=s" => \$auto_8bit_encoding,
+                   "force" => \$force,
         );
 
+usage() if $help;
 unless ($rc) {
     usage();
 }
@@ -286,8 +336,24 @@ sub read_config {
                $$target = Git::config_bool(@repo, "$prefix.$setting") unless (defined $$target);
        }
 
+       foreach my $setting (keys %config_path_settings) {
+               my $target = $config_path_settings{$setting};
+               if (ref($target) eq "ARRAY") {
+                       unless (@$target) {
+                               my @values = Git::config_path(@repo, "$prefix.$setting");
+                               @$target = @values if (@values && defined $values[0]);
+                       }
+               }
+               else {
+                       $$target = Git::config_path(@repo, "$prefix.$setting") unless (defined $$target);
+               }
+       }
+
        foreach my $setting (keys %config_settings) {
                my $target = $config_settings{$setting};
+               next if $setting eq "to" and defined $no_to;
+               next if $setting eq "cc" and defined $no_cc;
+               next if $setting eq "bcc" and defined $no_bcc;
                if (ref($target) eq "ARRAY") {
                        unless (@$target) {
                                my @values = Git::config(@repo, "$prefix.$setting");
@@ -327,13 +393,13 @@ sub read_config {
 if (@suppress_cc) {
        foreach my $entry (@suppress_cc) {
                die "Unknown --suppress-cc field: '$entry'\n"
-                       unless $entry =~ /^(all|cccmd|cc|author|self|sob|body|bodycc)$/;
+                       unless $entry =~ /^(?:all|cccmd|cc|author|self|sob|body|bodycc)$/;
                $suppress_cc{$entry} = 1;
        }
 }
 
 if ($suppress_cc{'all'}) {
-       foreach my $entry (qw (ccmd cc author self sob body bodycc)) {
+       foreach my $entry (qw (cccmd cc author self sob body bodycc)) {
                $suppress_cc{$entry} = 1;
        }
        delete $suppress_cc{'all'};
@@ -372,7 +438,7 @@ sub read_config {
 
 # Verify the user input
 
-foreach my $entry (@to) {
+foreach my $entry (@initial_to) {
        die "Comma in --to entry: $entry'\n" unless $entry !~ m/,/;
 }
 
@@ -400,7 +466,7 @@ sub split_addrs {
 my %parse_alias = (
        # multiline formats can be supported in the future
        mutt => sub { my $fh = shift; while (<$fh>) {
-               if (/^\s*alias\s+(\S+)\s+(.*)$/) {
+               if (/^\s*alias\s+(?:-group\s+\S+\s+)*(\S+)\s+(.*)$/) {
                        my ($alias, $addr) = ($1, $2);
                        $addr =~ s/#.*$//; # mutt allows # comments
                         # commas delimit multiple addresses
@@ -409,7 +475,7 @@ sub split_addrs {
        mailrc => sub { my $fh = shift; while (<$fh>) {
                if (/^alias\s+(\S+)\s+(.*)$/) {
                        # spaces delimit multiple addresses
-                       $aliases{$1} = [ split(/\s+/, $2) ];
+                       $aliases{$1} = [ quotewords('\s+', 0, $2) ];
                }}},
        pine => sub { my $fh = shift; my $f='\t[^\t]*';
                for (my $x = ''; defined($x); $x = $_) {
@@ -449,7 +515,6 @@ ($)
        try {
                $repo->command('rev-parse', '--verify', '--quiet', $f);
                if (defined($format_patch)) {
-                       print "foo\n";
                        return $format_patch;
                }
                die(<<EOF);
@@ -472,12 +537,12 @@ ($)
                push @rev_list_opts, "--", @ARGV;
                @ARGV = ();
        } elsif (-d $f and !check_file_rev_conflict($f)) {
-               opendir(DH,$f)
+               opendir my $dh, $f
                        or die "Failed to opendir $f: $!";
 
-               push @files, grep { -f $_ } map { +$f . "/" . $_ }
-                               sort readdir(DH);
-               closedir(DH);
+               push @files, grep { -f $_ } map { catfile($f, $_) }
+                               sort readdir $dh;
+               closedir $dh;
        } elsif ((-f $f or -p $f) and !check_file_rev_conflict($f)) {
                push @files, $f;
        } else {
@@ -509,7 +574,7 @@ ($)
        usage();
 }
 
-sub get_patch_subject($) {
+sub get_patch_subject {
        my $fn = shift;
        open (my $fh, '<', $fn);
        while (my $line = <$fh>) {
@@ -527,7 +592,7 @@ ($)
        $compose_filename = ($repo ?
                tempfile(".gitsendemail.msg.XXXXXX", DIR => $repo->repo_path()) :
                tempfile(".gitsendemail.msg.XXXXXX", DIR => "."))[1];
-       open(C,">",$compose_filename)
+       open my $c, ">", $compose_filename
                or die "Failed to open for writing $compose_filename: $!";
 
 
@@ -535,9 +600,9 @@ ($)
        my $tpl_subject = $initial_subject || '';
        my $tpl_reply_to = $initial_reply_to || '';
 
-       print C <<EOT;
+       print $c <<EOT;
 From $tpl_sender # This line is ignored.
-GIT: Lines beginning in "GIT: " will be removed.
+GIT: Lines beginning in "GIT:" will be removed.
 GIT: Consider including an overall diffstat or table of contents
 GIT: for the patch you are writing.
 GIT:
@@ -548,11 +613,9 @@ ($)
 
 EOT
        for my $f (@files) {
-               print C get_patch_subject($f);
+               print $c get_patch_subject($f);
        }
-       close(C);
-
-       my $editor = $ENV{GIT_EDITOR} || Git::config(@repo, "core.editor") || $ENV{VISUAL} || $ENV{EDITOR} || "vi";
+       close $c;
 
        if ($annotate) {
                do_edit($compose_filename, @files);
@@ -560,25 +623,25 @@ ($)
                do_edit($compose_filename);
        }
 
-       open(C2,">",$compose_filename . ".final")
+       open my $c2, ">", $compose_filename . ".final"
                or die "Failed to open $compose_filename.final : " . $!;
 
-       open(C,"<",$compose_filename)
+       open $c, "<", $compose_filename
                or die "Failed to open $compose_filename : " . $!;
 
        my $need_8bit_cte = file_has_nonascii($compose_filename);
        my $in_body = 0;
        my $summary_empty = 1;
-       while(<C>) {
-               next if m/^GIT: /;
+       while(<$c>) {
+               next if m/^GIT:/;
                if ($in_body) {
                        $summary_empty = 0 unless (/^\n$/);
                } elsif (/^\n$/) {
                        $in_body = 1;
                        if ($need_8bit_cte) {
-                               print C2 "MIME-Version: 1.0\n",
+                               print $c2 "MIME-Version: 1.0\n",
                                         "Content-Type: text/plain; ",
-                                          "charset=utf-8\n",
+                                          "charset=UTF-8\n",
                                         "Content-Transfer-Encoding: 8bit\n";
                        }
                } elsif (/^MIME-Version:/i) {
@@ -601,10 +664,10 @@ ($)
                        print "To/Cc/Bcc fields are not interpreted yet, they have been ignored\n";
                        next;
                }
-               print C2 $_;
+               print $c2 $_;
        }
-       close(C);
-       close(C2);
+       close $c;
+       close $c2;
 
        if ($summary_empty) {
                print "Summary email is empty, skipping it\n";
@@ -639,6 +702,45 @@ sub ask {
        return undef;
 }
 
+my %broken_encoding;
+
+sub file_declares_8bit_cte {
+       my $fn = shift;
+       open (my $fh, '<', $fn);
+       while (my $line = <$fh>) {
+               last if ($line =~ /^$/);
+               return 1 if ($line =~ /^Content-Transfer-Encoding: .*8bit.*$/);
+       }
+       close $fh;
+       return 0;
+}
+
+foreach my $f (@files) {
+       next unless (body_or_subject_has_nonascii($f)
+                    && !file_declares_8bit_cte($f));
+       $broken_encoding{$f} = 1;
+}
+
+if (!defined $auto_8bit_encoding && scalar %broken_encoding) {
+       print "The following files are 8bit, but do not declare " .
+               "a Content-Transfer-Encoding.\n";
+       foreach my $f (sort keys %broken_encoding) {
+               print "    $f\n";
+       }
+       $auto_8bit_encoding = ask("Which 8bit encoding should I declare [UTF-8]? ",
+                                 default => "UTF-8");
+}
+
+if (!$force) {
+       for my $f (@files) {
+               if (get_patch_subject($f) =~ /\Q*** SUBJECT HERE ***\E/) {
+                       die "Refusing to send because the patch\n\t$f\n"
+                               . "has the template subject '*** SUBJECT HERE ***'. "
+                               . "Pass --force if you really want to send.\n";
+               }
+       }
+}
+
 my $prompting = 0;
 if (!defined $sender) {
        $sender = $repoauthor || $repocommitter || '';
@@ -648,24 +750,28 @@ sub ask {
        $prompting++;
 }
 
-if (!@to) {
+if (!@initial_to && !defined $to_cmd) {
        my $to = ask("Who should the emails be sent to? ");
-       push @to, parse_address_line($to) if defined $to; # sanitized/validated later
+       push @initial_to, parse_address_line($to) if defined $to; # sanitized/validated later
        $prompting++;
 }
 
 sub expand_aliases {
-       my @cur = @_;
-       my @last;
-       do {
-               @last = @cur;
-               @cur = map { $aliases{$_} ? @{$aliases{$_}} : $_ } @last;
-       } while (join(',',@cur) ne join(',',@last));
-       return @cur;
+       return map { expand_one_alias($_) } @_;
+}
+
+my %EXPANDED_ALIASES;
+sub expand_one_alias {
+       my $alias = shift;
+       if ($EXPANDED_ALIASES{$alias}) {
+               die "fatal: alias '$alias' expands to itself\n";
+       }
+       local $EXPANDED_ALIASES{$alias} = 1;
+       return $aliases{$alias} ? expand_aliases(@{$aliases{$alias}}) : $alias;
 }
 
-@to = expand_aliases(@to);
-@to = (map { sanitize_address($_) } @to);
+@initial_to = expand_aliases(@initial_to);
+@initial_to = (map { sanitize_address($_) } @initial_to);
 @initial_cc = expand_aliases(@initial_cc);
 @bcclist = expand_aliases(@bcclist);
 
@@ -699,8 +805,8 @@ sub expand_aliases {
 
 sub extract_valid_address {
        my $address = shift;
-       my $local_part_regexp = '[^<>"\s@]+';
-       my $domain_regexp = '[^.<>"\s@]+(?:\.[^.<>"\s@]+)+';
+       my $local_part_regexp = qr/[^<>"\s@]+/;
+       my $domain_regexp = qr/[^.<>"\s@]+(?:\.[^.<>"\s@]+)+/;
 
        # check for a local address:
        return $address if ($address =~ /^($local_part_regexp)$/);
@@ -726,8 +832,7 @@ sub extract_valid_address {
 # We'll setup a template for the message id, using the "from" address:
 
 my ($message_id_stamp, $message_id_serial);
-sub make_message_id
-{
+sub make_message_id {
        my $uniq;
        if (!defined $message_id_stamp) {
                $message_id_stamp = sprintf("%s-%s", time, $$);
@@ -742,7 +847,7 @@ sub make_message_id
                last if (defined $du_part and $du_part ne '');
        }
        if (not defined $du_part or $du_part eq '') {
-               use Sys::Hostname qw();
+               require Sys::Hostname;
                $du_part = 'user@' . Sys::Hostname::hostname();
        }
        my $message_id_template = "<%s-git-send-email-%s>";
@@ -767,24 +872,31 @@ sub unquote_rfc2047 {
 
 sub quote_rfc2047 {
        local $_ = shift;
-       my $encoding = shift || 'utf-8';
+       my $encoding = shift || 'UTF-8';
        s/([^-a-zA-Z0-9!*+\/])/sprintf("=%02X", ord($1))/eg;
        s/(.*)/=\?$encoding\?q\?$1\?=/;
        return $_;
 }
 
+sub is_rfc2047_quoted {
+       my $s = shift;
+       my $token = qr/[^][()<>@,;:"\/?.= \000-\037\177-\377]+/;
+       my $encoded_text = qr/[!->@-~]+/;
+       length($s) <= 75 &&
+       $s =~ m/^(?:"[[:ascii:]]*"|=\?$token\?$token\?$encoded_text\?=)$/o;
+}
+
 # use the simplest quoting being able to handle the recipient
-sub sanitize_address
-{
+sub sanitize_address {
        my ($recipient) = @_;
        my ($recipient_name, $recipient_addr) = ($recipient =~ /^(.*?)\s*(<.*)/);
 
        if (not $recipient_name) {
-               return "$recipient";
+               return $recipient;
        }
 
        # if recipient_name is already quoted, do nothing
-       if ($recipient_name =~ /^("[[:ascii:]]*"|=\?utf-8\?q\?.*\?=)$/) {
+       if (is_rfc2047_quoted($recipient_name)) {
                return $recipient;
        }
 
@@ -797,18 +909,77 @@ sub sanitize_address
        # double quotes are needed if specials or CTLs are included
        elsif ($recipient_name =~ /[][()<>@,;:\\".\000-\037\177]/) {
                $recipient_name =~ s/(["\\\r])/\\$1/g;
-               $recipient_name = "\"$recipient_name\"";
+               $recipient_name = qq["$recipient_name"];
        }
 
        return "$recipient_name $recipient_addr";
 
 }
 
-sub send_message
-{
+# Returns the local Fully Qualified Domain Name (FQDN) if available.
+#
+# Tightly configured MTAa require that a caller sends a real DNS
+# domain name that corresponds the IP address in the HELO/EHLO
+# handshake. This is used to verify the connection and prevent
+# spammers from trying to hide their identity. If the DNS and IP don't
+# match, the receiveing MTA may deny the connection.
+#
+# Here is a deny example of Net::SMTP with the default "localhost.localdomain"
+#
+# Net::SMTP=GLOB(0x267ec28)>>> EHLO localhost.localdomain
+# Net::SMTP=GLOB(0x267ec28)<<< 550 EHLO argument does not match calling host
+#
+# This maildomain*() code is based on ideas in Perl library Test::Reporter
+# /usr/share/perl5/Test/Reporter/Mail/Util.pm ==> sub _maildomain ()
+
+sub valid_fqdn {
+       my $domain = shift;
+       return defined $domain && !($^O eq 'darwin' && $domain =~ /\.local$/) && $domain =~ /\./;
+}
+
+sub maildomain_net {
+       my $maildomain;
+
+       if (eval { require Net::Domain; 1 }) {
+               my $domain = Net::Domain::domainname();
+               $maildomain = $domain if valid_fqdn($domain);
+       }
+
+       return $maildomain;
+}
+
+sub maildomain_mta {
+       my $maildomain;
+
+       if (eval { require Net::SMTP; 1 }) {
+               for my $host (qw(mailhost localhost)) {
+                       my $smtp = Net::SMTP->new($host);
+                       if (defined $smtp) {
+                               my $domain = $smtp->domain;
+                               $smtp->quit;
+
+                               $maildomain = $domain if valid_fqdn($domain);
+
+                               last if $maildomain;
+                       }
+               }
+       }
+
+       return $maildomain;
+}
+
+sub maildomain {
+       return maildomain_net() || maildomain_mta() || 'localhost.localdomain';
+}
+
+# Returns 1 if the message was sent, and 0 otherwise.
+# In actuality, the whole program dies when there
+# is an error sending a message.
+
+sub send_message {
        my @recipients = unique_email_list(@to);
        @cc = (grep { my $cc = extract_valid_address($_);
-                     not grep { $cc eq $_ } @recipients
+                     not grep { $cc eq $_ || $_ =~ /<\Q${cc}\E>$/ } @recipients
                    }
               map { sanitize_address($_) }
               @cc);
@@ -821,7 +992,7 @@ sub send_message
            $gitversion = Git::version();
        }
 
-       my $cc = join(", ", unique_email_list(@cc));
+       my $cc = join(",\n\t", unique_email_list(@cc));
        my $ccline = "";
        if ($cc ne '') {
                $ccline = "\nCc: $cc";
@@ -847,7 +1018,9 @@ sub send_message
 
        my @sendmail_parameters = ('-i', @recipients);
        my $raw_from = $sanitized_sender;
-       $raw_from = $envelope_sender if (defined $envelope_sender);
+       if (defined $envelope_sender && $envelope_sender ne "auto") {
+               $raw_from = $envelope_sender;
+       }
        $raw_from = extract_valid_address($raw_from);
        unshift (@sendmail_parameters,
                        '-f', $raw_from) if(defined $envelope_sender);
@@ -872,7 +1045,7 @@ sub send_message
                         default => $ask_default);
                die "Send this email reply required" unless defined $_;
                if (/^n/i) {
-                       return;
+                       return 0;
                } elsif (/^q/i) {
                        cleanup_compose_files();
                        exit(0);
@@ -881,6 +1054,8 @@ sub send_message
                }
        }
 
+       unshift (@sendmail_parameters, @smtp_server_options);
+
        if ($dry_run) {
                # We don't want to send the email.
        } elsif ($smtp_server =~ m#^/#) {
@@ -890,7 +1065,7 @@ sub send_message
                        exec($smtp_server, @sendmail_parameters) or die $!;
                }
                print $sm "$header\n$message";
-               close $sm or die $?;
+               close $sm or die $!;
        } else {
 
                if (!defined $smtp_server) {
@@ -900,14 +1075,20 @@ sub send_message
                if ($smtp_encryption eq 'ssl') {
                        $smtp_server_port ||= 465; # ssmtp
                        require Net::SMTP::SSL;
-                       $smtp ||= Net::SMTP::SSL->new($smtp_server, Port => $smtp_server_port);
+                       $smtp_domain ||= maildomain();
+                       $smtp ||= Net::SMTP::SSL->new($smtp_server,
+                                                     Hello => $smtp_domain,
+                                                     Port => $smtp_server_port);
                }
                else {
                        require Net::SMTP;
+                       $smtp_domain ||= maildomain();
                        $smtp ||= Net::SMTP->new((defined $smtp_server_port)
                                                 ? "$smtp_server:$smtp_server_port"
-                                                : $smtp_server);
-                       if ($smtp_encryption eq 'tls') {
+                                                : $smtp_server,
+                                                Hello => $smtp_domain,
+                                                Debug => $debug_net_smtp);
+                       if ($smtp_encryption eq 'tls' && $smtp) {
                                require Net::SMTP::SSL;
                                $smtp->command('STARTTLS');
                                $smtp->response();
@@ -917,7 +1098,7 @@ sub send_message
                                        $smtp_encryption = '';
                                        # Send EHLO again to receive fresh
                                        # supported commands
-                                       $smtp->hello();
+                                       $smtp->hello($smtp_domain);
                                } else {
                                        die "Server does not support STARTTLS! ".$smtp->message;
                                }
@@ -925,10 +1106,20 @@ sub send_message
                }
 
                if (!$smtp) {
-                       die "Unable to initialize SMTP properly.  Is there something wrong with your config?";
+                       die "Unable to initialize SMTP properly. Check config and use --smtp-debug. ",
+                           "VALUES: server=$smtp_server ",
+                           "encryption=$smtp_encryption ",
+                           "hello=$smtp_domain",
+                           defined $smtp_server_port ? " port=$smtp_server_port" : "";
                }
 
                if (defined $smtp_authuser) {
+                       # Workaround AUTH PLAIN/LOGIN interaction defect
+                       # with Authen::SASL::Cyrus
+                       eval {
+                               require Authen::SASL;
+                               Authen::SASL->import(qw(Perl));
+                       };
 
                        if (!defined $smtp_authpass) {
 
@@ -953,7 +1144,7 @@ sub send_message
                $smtp->data or die $smtp->message;
                $smtp->datasend("$header\n$message") or die $smtp->message;
                $smtp->dataend() or die $smtp->message;
-               $smtp->ok or die "Failed to send $subject\n".$smtp->message;
+               $smtp->code =~ /250|200/ or die "Failed to send $subject\n".$smtp->message;
        }
        if ($quiet) {
                printf (($dry_run ? "Dry-" : "")."Sent %s\n", $subject);
@@ -962,7 +1153,9 @@ sub send_message
                if ($smtp_server !~ m#^/#) {
                        print "Server: $smtp_server\n";
                        print "MAIL FROM:<$raw_from>\n";
-                       print "RCPT TO:".join(',',(map { "<$_>" } @recipients))."\n";
+                       foreach my $entry (@recipients) {
+                           print "RCPT TO:<$entry>\n";
+                       }
                } else {
                        print "Sendmail: $smtp_server ".join(' ',@sendmail_parameters)."\n";
                }
@@ -974,6 +1167,8 @@ sub send_message
                        print "Result: OK\n";
                }
        }
+
+       return 1;
 }
 
 $reply_to = $initial_reply_to;
@@ -982,12 +1177,13 @@ sub send_message
 $message_num = 0;
 
 foreach my $t (@files) {
-       open(F,"<",$t) or die "can't open file $t";
+       open my $fh, "<", $t or die "can't open file $t";
 
        my $author = undef;
        my $author_encoding;
        my $has_content_type;
        my $body_encoding;
+       @to = ();
        @cc = ();
        @xh = ();
        my $input_format = undef;
@@ -995,7 +1191,7 @@ sub send_message
        $message = "";
        $message_num++;
        # First unfold multiline header fields
-       while(<F>) {
+       while(<$fh>) {
                last if /^\s*$/;
                if (/^\s+\S/ and @header) {
                        chomp($header[$#header]);
@@ -1028,6 +1224,13 @@ sub send_message
                                        $1, $_) unless $quiet;
                                push @cc, $1;
                        }
+                       elsif (/^To:\s+(.*)$/) {
+                               foreach my $addr (parse_address_line($1)) {
+                                       printf("(mbox) Adding to: %s from line '%s'\n",
+                                               $addr, $_) unless $quiet;
+                                       push @to, sanitize_address($addr);
+                               }
+                       }
                        elsif (/^Cc:\s+(.*)$/) {
                                foreach my $addr (parse_address_line($1)) {
                                        if (unquote_rfc2047($addr) eq $sender) {
@@ -1071,7 +1274,7 @@ sub send_message
                }
        }
        # Now parse the message body
-       while(<F>) {
+       while(<$fh>) {
                $message .=  $_;
                if (/^(Signed-off-by|Cc): (.*)$/i) {
                        chomp;
@@ -1088,22 +1291,23 @@ sub send_message
                                $c, $_) unless $quiet;
                }
        }
-       close F;
-
-       if (defined $cc_cmd && !$suppress_cc{'cccmd'}) {
-               open(F, "$cc_cmd $t |")
-                       or die "(cc-cmd) Could not execute '$cc_cmd'";
-               while(<F>) {
-                       my $c = $_;
-                       $c =~ s/^\s*//g;
-                       $c =~ s/\n$//g;
-                       next if ($c eq $sender and $suppress_from);
-                       push @cc, $c;
-                       printf("(cc-cmd) Adding cc: %s from: '%s'\n",
-                               $c, $cc_cmd) unless $quiet;
-               }
-               close F
-                       or die "(cc-cmd) failed to close pipe to '$cc_cmd'";
+       close $fh;
+
+       push @to, recipients_cmd("to-cmd", "to", $to_cmd, $t)
+               if defined $to_cmd;
+       push @cc, recipients_cmd("cc-cmd", "cc", $cc_cmd, $t)
+               if defined $cc_cmd && !$suppress_cc{'cccmd'};
+
+       if ($broken_encoding{$t} && !$has_content_type) {
+               $has_content_type = 1;
+               push @xh, "MIME-Version: 1.0",
+                       "Content-Type: text/plain; charset=$auto_8bit_encoding",
+                       "Content-Transfer-Encoding: 8bit";
+               $body_encoding = $auto_8bit_encoding;
+       }
+
+       if ($broken_encoding{$t} && !is_rfc2047_quoted($subject)) {
+               $subject = quote_rfc2047($subject, $auto_8bit_encoding);
        }
 
        if (defined $author and $author ne $sender) {
@@ -1118,6 +1322,7 @@ sub send_message
                                }
                        }
                        else {
+                               $has_content_type = 1;
                                push @xh,
                                  'MIME-Version: 1.0',
                                  "Content-Type: text/plain; charset=$author_encoding",
@@ -1132,12 +1337,15 @@ sub send_message
                ($confirm =~ /^(?:auto|compose)$/ && $compose && $message_num == 1));
        $needs_confirm = "inform" if ($needs_confirm && $confirm_unconfigured && @cc);
 
+       @to = (@initial_to, @to);
        @cc = (@initial_cc, @cc);
 
-       send_message();
+       my $message_was_sent = send_message();
 
        # set up for the next message
-       if ($chain_reply_to || !defined $reply_to || length($reply_to) == 0) {
+       if ($thread && $message_was_sent &&
+               (chain_reply_to() || !defined $reply_to || length($reply_to) == 0 ||
+               $message_num == 1)) {
                $reply_to = $message_id;
                if (length $references > 0) {
                        $references .= "\n $message_id";
@@ -1148,15 +1356,38 @@ sub send_message
        $message_id = undef;
 }
 
+# Execute a command (e.g. $to_cmd) to get a list of email addresses
+# and return a results array
+sub recipients_cmd {
+       my ($prefix, $what, $cmd, $file) = @_;
+
+       my $sanitized_sender = sanitize_address($sender);
+       my @addresses = ();
+       open my $fh, "$cmd \Q$file\E |"
+           or die "($prefix) Could not execute '$cmd'";
+       while (my $address = <$fh>) {
+               $address =~ s/^\s*//g;
+               $address =~ s/\s*$//g;
+               $address = sanitize_address($address);
+               next if ($address eq $sanitized_sender and $suppress_from);
+               push @addresses, $address;
+               printf("($prefix) Adding %s: %s from: '%s'\n",
+                      $what, $address, $cmd) unless $quiet;
+               }
+       close $fh
+           or die "($prefix) failed to close pipe to '$cmd'";
+       return @addresses;
+}
+
 cleanup_compose_files();
 
-sub cleanup_compose_files() {
+sub cleanup_compose_files {
        unlink($compose_filename, $compose_filename . ".final") if $compose;
 }
 
 $smtp->quit if $smtp;
 
-sub unique_email_list(@) {
+sub unique_email_list {
        my %seen;
        my @emails;
 
@@ -1194,3 +1425,17 @@ sub file_has_nonascii {
        }
        return 0;
 }
+
+sub body_or_subject_has_nonascii {
+       my $fn = shift;
+       open(my $fh, '<', $fn)
+               or die "unable to open $fn: $!\n";
+       while (my $line = <$fh>) {
+               last if $line =~ /^$/;
+               return 1 if $line =~ /^Subject.*[^[:ascii:]]/;
+       }
+       while (my $line = <$fh>) {
+               return 1 if $line =~ /[^[:ascii:]]/;
+       }
+       return 0;
+}