format-patch: RFC 2047 says multi-octet character may not be split

Even though an earlier attempt (bafc478..41dd00bad) cleaned
up RFC 2047 encoding, pretty.c::add_rfc2047() still decides
where to split the output line by going through the input
one byte at a time, and potentially splits a character in
the middle.  A subject line may end up showing like this:

".... fö?? bar".   (instead of  ".... föö bar".)

if split incorrectly.

RFC 2047, section 5 (3) explicitly forbids such beaviour

Each 'encoded-word' MUST represent an integral number of
characters.  A multi-octet character may not be split across
adjacent 'encoded- word's.

that means that e.g. for

Subject: .... föö bar

encoding

Subject: =?UTF-8?q?....=20f=C3=B6=C3=B6?=
=?UTF-8?q?=20bar?=

is correct, and

Subject: =?UTF-8?q?....=20f=C3=B6=C3?=      <-- NOTE ö is broken here
=?UTF-8?q?=B6=20bar?=

is not, because "ö" character UTF-8 encoding C3 B6 is split here across
adjacent encoded words.

To fix the problem, make the loop grab one _character_ at a time and
determine its output length to see where to break the output line.  Note
that this version only knows about UTF-8, but the logic to grab one
character is abstracted out in mbs_chrlen() function to make it possible
to extend it to other encodings with the help of iconv in the future.

Signed-off-by: Kirill Smelkov <kirr@mns.spb.ru>
Signed-off-by: Junio C Hamano <gitster@pobox.com>

diff --git a/pretty.c b/pretty.c
index dba682828c2e005b71c0ccbb325fb666915984bc..cccee87026253454e76d10e86d3f5609e187c8e0 100644 (file)
--- a/pretty.c
+++ b/pretty.c
@@ -345,7 +345,7 @@ static int needs_rfc2047_encoding(const char *line, int len,
        return 0;
 }
 
-static void add_rfc2047(struct strbuf *sb, const char *line, int len,
+static void add_rfc2047(struct strbuf *sb, const char *line, size_t len,
                       const char *encoding, enum rfc2047_type type)
 {
        static const int max_encoded_length = 76; /* per rfc2047 */
@@ -355,9 +355,22 @@ static void add_rfc2047(struct strbuf *sb, const char *line, int len,
        strbuf_grow(sb, len * 3 + strlen(encoding) + 100);
        strbuf_addf(sb, "=?%s?q?", encoding);
        line_len += strlen(encoding) + 5; /* 5 for =??q? */
-       for (i = 0; i < len; i++) {
-               unsigned ch = line[i] & 0xFF;
-               int is_special = is_rfc2047_special(ch, type);
+
+       while (len) {
+               /*
+                * RFC 2047, section 5 (3):
+                *
+                * Each 'encoded-word' MUST represent an integral number of
+                * characters.  A multi-octet character may not be split across
+                * adjacent 'encoded- word's.
+                */
+               const unsigned char *p = (const unsigned char *)line;
+               int chrlen = mbs_chrlen(&line, &len, encoding);
+               int is_special = (chrlen > 1) || is_rfc2047_special(*p, type);
+
+               /* "=%02X" * chrlen, or the byte itself */
+               const char *encoded_fmt = is_special ? "=%02X"    : "%c";
+               int         encoded_len = is_special ? 3 * chrlen : 1;
 
                /*
                 * According to RFC 2047, we could encode the special character
@@ -367,18 +380,15 @@ static void add_rfc2047(struct strbuf *sb, const char *line, int len,
                 * causes ' ' to be encoded as '=20', avoiding this problem.
                 */
 
-               if (line_len + 2 + (is_special ? 3 : 1) > max_encoded_length) {
+               if (line_len + encoded_len + 2 > max_encoded_length) {
+                       /* It won't fit with trailing "?=" --- break the line */
                        strbuf_addf(sb, "?=\n =?%s?q?", encoding);
                        line_len = strlen(encoding) + 5 + 1; /* =??q? plus SP */
                }
 
-               if (is_special) {
-                       strbuf_addf(sb, "=%02X", ch);
-                       line_len += 3;
-               } else {
-                       strbuf_addch(sb, ch);
-                       line_len++;
-               }
+               for (i = 0; i < chrlen; i++)
+                       strbuf_addf(sb, encoded_fmt, p[i]);
+               line_len += encoded_len;
        }
        strbuf_addstr(sb, "?=");
 }

diff --git a/t/t4014-format-patch.sh b/t/t4014-format-patch.sh
index ad9f69e6074311cc3828896bd8409822344d6f62..ba437379bc45255111a72c5a30ffd3b8b4f63543 100755 (executable)
--- a/t/t4014-format-patch.sh
+++ b/t/t4014-format-patch.sh
@@ -810,25 +810,26 @@ Subject: [PATCH] =?UTF-8?q?f=C3=B6=C3=B6=20bar=20f=C3=B6=C3=B6=20bar=20f?=
  =?UTF-8?q?=C3=B6=C3=B6=20bar=20f=C3=B6=C3=B6=20bar=20f=C3=B6=C3=B6=20bar?=
  =?UTF-8?q?=20f=C3=B6=C3=B6=20bar=20f=C3=B6=C3=B6=20bar=20f=C3=B6=C3=B6=20?=
  =?UTF-8?q?bar=20f=C3=B6=C3=B6=20bar=20f=C3=B6=C3=B6=20bar=20f=C3=B6=C3=B6?=
- =?UTF-8?q?=20bar=20f=C3=B6=C3=B6=20bar=20f=C3=B6=C3=B6=20bar=20f=C3=B6=C3?=
- =?UTF-8?q?=B6=20bar=20f=C3=B6=C3=B6=20bar=20f=C3=B6=C3=B6=20bar=20f=C3=B6?=
- =?UTF-8?q?=C3=B6=20bar=20f=C3=B6=C3=B6=20bar=20f=C3=B6=C3=B6=20bar=20f=C3?=
- =?UTF-8?q?=B6=C3=B6=20bar=20f=C3=B6=C3=B6=20bar=20f=C3=B6=C3=B6=20bar=20f?=
+ =?UTF-8?q?=20bar=20f=C3=B6=C3=B6=20bar=20f=C3=B6=C3=B6=20bar=20f=C3=B6?=
+ =?UTF-8?q?=C3=B6=20bar=20f=C3=B6=C3=B6=20bar=20f=C3=B6=C3=B6=20bar=20f?=
  =?UTF-8?q?=C3=B6=C3=B6=20bar=20f=C3=B6=C3=B6=20bar=20f=C3=B6=C3=B6=20bar?=
  =?UTF-8?q?=20f=C3=B6=C3=B6=20bar=20f=C3=B6=C3=B6=20bar=20f=C3=B6=C3=B6=20?=
  =?UTF-8?q?bar=20f=C3=B6=C3=B6=20bar=20f=C3=B6=C3=B6=20bar=20f=C3=B6=C3=B6?=
- =?UTF-8?q?=20bar=20f=C3=B6=C3=B6=20bar=20f=C3=B6=C3=B6=20bar=20f=C3=B6=C3?=
- =?UTF-8?q?=B6=20bar=20f=C3=B6=C3=B6=20bar=20f=C3=B6=C3=B6=20bar=20f=C3=B6?=
- =?UTF-8?q?=C3=B6=20bar=20f=C3=B6=C3=B6=20bar=20f=C3=B6=C3=B6=20bar=20f=C3?=
- =?UTF-8?q?=B6=C3=B6=20bar=20f=C3=B6=C3=B6=20bar=20f=C3=B6=C3=B6=20bar=20f?=
+ =?UTF-8?q?=20bar=20f=C3=B6=C3=B6=20bar=20f=C3=B6=C3=B6=20bar=20f=C3=B6?=
+ =?UTF-8?q?=C3=B6=20bar=20f=C3=B6=C3=B6=20bar=20f=C3=B6=C3=B6=20bar=20f?=
  =?UTF-8?q?=C3=B6=C3=B6=20bar=20f=C3=B6=C3=B6=20bar=20f=C3=B6=C3=B6=20bar?=
  =?UTF-8?q?=20f=C3=B6=C3=B6=20bar=20f=C3=B6=C3=B6=20bar=20f=C3=B6=C3=B6=20?=
  =?UTF-8?q?bar=20f=C3=B6=C3=B6=20bar=20f=C3=B6=C3=B6=20bar=20f=C3=B6=C3=B6?=
- =?UTF-8?q?=20bar=20f=C3=B6=C3=B6=20bar=20f=C3=B6=C3=B6=20bar=20f=C3=B6=C3?=
- =?UTF-8?q?=B6=20bar=20f=C3=B6=C3=B6=20bar=20f=C3=B6=C3=B6=20bar=20f=C3=B6?=
- =?UTF-8?q?=C3=B6=20bar=20f=C3=B6=C3=B6=20bar=20f=C3=B6=C3=B6=20bar=20f=C3?=
- =?UTF-8?q?=B6=C3=B6=20bar=20f=C3=B6=C3=B6=20bar=20f=C3=B6=C3=B6=20bar=20f?=
- =?UTF-8?q?=C3=B6=C3=B6=20bar=20f=C3=B6=C3=B6=20bar?=
+ =?UTF-8?q?=20bar=20f=C3=B6=C3=B6=20bar=20f=C3=B6=C3=B6=20bar=20f=C3=B6?=
+ =?UTF-8?q?=C3=B6=20bar=20f=C3=B6=C3=B6=20bar=20f=C3=B6=C3=B6=20bar=20f?=
+ =?UTF-8?q?=C3=B6=C3=B6=20bar=20f=C3=B6=C3=B6=20bar=20f=C3=B6=C3=B6=20bar?=
+ =?UTF-8?q?=20f=C3=B6=C3=B6=20bar=20f=C3=B6=C3=B6=20bar=20f=C3=B6=C3=B6=20?=
+ =?UTF-8?q?bar=20f=C3=B6=C3=B6=20bar=20f=C3=B6=C3=B6=20bar=20f=C3=B6=C3=B6?=
+ =?UTF-8?q?=20bar=20f=C3=B6=C3=B6=20bar=20f=C3=B6=C3=B6=20bar=20f=C3=B6?=
+ =?UTF-8?q?=C3=B6=20bar=20f=C3=B6=C3=B6=20bar=20f=C3=B6=C3=B6=20bar=20f?=
+ =?UTF-8?q?=C3=B6=C3=B6=20bar=20f=C3=B6=C3=B6=20bar=20f=C3=B6=C3=B6=20bar?=
+ =?UTF-8?q?=20f=C3=B6=C3=B6=20bar=20f=C3=B6=C3=B6=20bar=20f=C3=B6=C3=B6=20?=
+ =?UTF-8?q?bar?=
 EOF
 test_expect_success 'format-patch wraps extremely long subject (rfc2047)' '
        rm -rf patches/ &&

diff --git a/utf8.c b/utf8.c
index 5c61bbe1131e7bbdd939c8b815bd5222b872e3fb..0b30b54847127ec1e4af04b72ed5c6bd9ffef733 100644 (file)
--- a/utf8.c
+++ b/utf8.c
@@ -495,3 +495,42 @@ char *reencode_string(const char *in, const char *out_encoding, const char *in_e
        return out;
 }
 #endif
+
+/*
+ * Returns first character length in bytes for multi-byte `text` according to
+ * `encoding`.
+ *
+ * - The `text` pointer is updated to point at the next character.
+ * - When `remainder_p` is not NULL, on entry `*remainder_p` is how much bytes
+ *   we can consume from text, and on exit `*remainder_p` is reduced by returned
+ *   character length. Otherwise `text` is treated as limited by NUL.
+ */
+int mbs_chrlen(const char **text, size_t *remainder_p, const char *encoding)
+{
+       int chrlen;
+       const char *p = *text;
+       size_t r = (remainder_p ? *remainder_p : SIZE_MAX);
+
+       if (r < 1)
+               return 0;
+
+       if (is_encoding_utf8(encoding)) {
+               pick_one_utf8_char(&p, &r);
+
+               chrlen = p ? (p - *text)
+                          : 1 /* not valid UTF-8 -> raw byte sequence */;
+       }
+       else {
+               /*
+                * TODO use iconv to decode one char and obtain its chrlen
+                * for now, let's treat encodings != UTF-8 as one-byte
+                */
+               chrlen = 1;
+       }
+
+       *text += chrlen;
+       if (remainder_p)
+               *remainder_p -= chrlen;
+
+       return chrlen;
+}

diff --git a/utf8.h b/utf8.h
index 93ef60042c2fad93184b573f30e771ecddc842be..465c01d8c35f7c409ad71e5a15ca62e13af34f31 100644 (file)
--- a/utf8.h
+++ b/utf8.h
@@ -21,4 +21,6 @@ char *reencode_string(const char *in, const char *out_encoding, const char *in_e
 #define reencode_string(a,b,c) NULL
 #endif
 
+int mbs_chrlen(const char **text, size_t *remainder_p, const char *encoding);
+
 #endif