Merge branch 'tb/utf-16-le-with-explicit-bom'

A new encoding UTF-16LE-BOM has been invented to force encoding to
UTF-16 with BOM in little endian byte order, which cannot be directly
generated by using iconv.

* tb/utf-16-le-with-explicit-bom:
Support working-tree-encoding "UTF-16LE-BOM"

diff --git a/Documentation/gitattributes.txt b/Documentation/gitattributes.txt
index b8392fc3300cfa7d7c37e37dd069246f20ceadb1..a2310fb920ecf64cacf6617c97383bbc4443a03c 100644 (file)
--- a/Documentation/gitattributes.txt
+++ b/Documentation/gitattributes.txt
@@ -344,7 +344,9 @@ automatic line ending conversion based on your platform.
 
 Use the following attributes if your '*.ps1' files are UTF-16 little
 endian encoded without BOM and you want Git to use Windows line endings
-in the working directory. Please note, it is highly recommended to
+in the working directory (use `UTF-16-LE-BOM` instead of `UTF-16LE` if
+you want UTF-16 little endian with BOM).
+Please note, it is highly recommended to
 explicitly define the line endings with `eol` if the `working-tree-encoding`
 attribute is used to avoid ambiguity.
 

diff --git a/compat/precompose_utf8.c b/compat/precompose_utf8.c
index de61c15d3483fc4a9e8e163bd7cfa5280996c61a..136250fbf6c4f7e707c25dba936e2effebd847f2 100644 (file)
--- a/compat/precompose_utf8.c
+++ b/compat/precompose_utf8.c
@@ -79,7 +79,7 @@ void precompose_argv(int argc, const char **argv)
                size_t namelen;
                oldarg = argv[i];
                if (has_non_ascii(oldarg, (size_t)-1, &namelen)) {
-                       newarg = reencode_string_iconv(oldarg, namelen, ic_precompose, NULL);
+                       newarg = reencode_string_iconv(oldarg, namelen, ic_precompose, 0, NULL);
                        if (newarg)
                                argv[i] = newarg;
                }

diff --git a/t/t0028-working-tree-encoding.sh b/t/t0028-working-tree-encoding.sh
index 7e87b5a200a3f6483ea2e243fa0c497205ce4e61..e58ecbfc44037fa1d38707b30545e803e1401b4f 100755 (executable)
--- a/t/t0028-working-tree-encoding.sh
+++ b/t/t0028-working-tree-encoding.sh
@@ -11,9 +11,12 @@ test_expect_success 'setup test files' '
 
        text="hallo there!\ncan you read me?" &&
        echo "*.utf16 text working-tree-encoding=utf-16" >.gitattributes &&
+       echo "*.utf16lebom text working-tree-encoding=UTF-16LE-BOM" >>.gitattributes &&
        printf "$text" >test.utf8.raw &&
        printf "$text" | iconv -f UTF-8 -t UTF-16 >test.utf16.raw &&
        printf "$text" | iconv -f UTF-8 -t UTF-32 >test.utf32.raw &&
+       printf "\377\376"                         >test.utf16lebom.raw &&
+       printf "$text" | iconv -f UTF-8 -t UTF-32LE >>test.utf16lebom.raw &&
 
        # Line ending tests
        printf "one\ntwo\nthree\n" >lf.utf8.raw &&
@@ -32,7 +35,8 @@ test_expect_success 'setup test files' '
        # Add only UTF-16 file, we will add the UTF-32 file later
        cp test.utf16.raw test.utf16 &&
        cp test.utf32.raw test.utf32 &&
-       git add .gitattributes test.utf16 &&
+       cp test.utf16lebom.raw test.utf16lebom &&
+       git add .gitattributes test.utf16 test.utf16lebom &&
        git commit -m initial
 '
 
@@ -51,6 +55,12 @@ test_expect_success 're-encode to UTF-16 on checkout' '
        test_cmp_bin test.utf16.raw test.utf16
 '
 
+test_expect_success 're-encode to UTF-16-LE-BOM on checkout' '
+       rm test.utf16lebom &&
+       git checkout test.utf16lebom &&
+       test_cmp_bin test.utf16lebom.raw test.utf16lebom
+'
+
 test_expect_success 'check $GIT_DIR/info/attributes support' '
        test_when_finished "rm -f test.utf32.git" &&
        test_when_finished "git reset --hard HEAD" &&

diff --git a/utf8.c b/utf8.c
index eb785875042920b84e56abbd235ef9eb7ef22ac4..83824dc2f4ab151a19418c61c46e0c1ffbb0e42c 100644 (file)
--- a/utf8.c
+++ b/utf8.c
@@ -4,6 +4,11 @@
 
 /* This code is originally from http://www.cl.cam.ac.uk/~mgk25/ucs/ */
 
+static const char utf16_be_bom[] = {'\xFE', '\xFF'};
+static const char utf16_le_bom[] = {'\xFF', '\xFE'};
+static const char utf32_be_bom[] = {'\0', '\0', '\xFE', '\xFF'};
+static const char utf32_le_bom[] = {'\xFF', '\xFE', '\0', '\0'};
+
 struct interval {
        ucs_char_t first;
        ucs_char_t last;
@@ -470,16 +475,17 @@ int utf8_fprintf(FILE *stream, const char *format, ...)
 #else
        typedef char * iconv_ibp;
 #endif
-char *reencode_string_iconv(const char *in, size_t insz, iconv_t conv, size_t *outsz_p)
+char *reencode_string_iconv(const char *in, size_t insz, iconv_t conv,
+                           size_t bom_len, size_t *outsz_p)
 {
        size_t outsz, outalloc;
        char *out, *outpos;
        iconv_ibp cp;
 
        outsz = insz;
-       outalloc = st_add(outsz, 1); /* for terminating NUL */
+       outalloc = st_add(outsz, 1 + bom_len); /* for terminating NUL */
        out = xmalloc(outalloc);
-       outpos = out;
+       outpos = out + bom_len;
        cp = (iconv_ibp)in;
 
        while (1) {
@@ -540,10 +546,30 @@ char *reencode_string_len(const char *in, size_t insz,
 {
        iconv_t conv;
        char *out;
+       const char *bom_str = NULL;
+       size_t bom_len = 0;
 
        if (!in_encoding)
                return NULL;
 
+       /* UTF-16LE-BOM is the same as UTF-16 for reading */
+       if (same_utf_encoding("UTF-16LE-BOM", in_encoding))
+               in_encoding = "UTF-16";
+
+       /*
+        * For writing, UTF-16 iconv typically creates "UTF-16BE-BOM"
+        * Some users under Windows want the little endian version
+        */
+       if (same_utf_encoding("UTF-16LE-BOM", out_encoding)) {
+               bom_str = utf16_le_bom;
+               bom_len = sizeof(utf16_le_bom);
+               out_encoding = "UTF-16LE";
+       } else if (same_utf_encoding("UTF-16BE-BOM", out_encoding)) {
+               bom_str = utf16_be_bom;
+               bom_len = sizeof(utf16_be_bom);
+               out_encoding = "UTF-16BE";
+       }
+
        conv = iconv_open(out_encoding, in_encoding);
        if (conv == (iconv_t) -1) {
                in_encoding = fallback_encoding(in_encoding);
@@ -553,9 +579,10 @@ char *reencode_string_len(const char *in, size_t insz,
                if (conv == (iconv_t) -1)
                        return NULL;
        }
-
-       out = reencode_string_iconv(in, insz, conv, outsz);
+       out = reencode_string_iconv(in, insz, conv, bom_len, outsz);
        iconv_close(conv);
+       if (out && bom_str && bom_len)
+               memcpy(out, bom_str, bom_len);
        return out;
 }
 #endif
@@ -566,11 +593,6 @@ static int has_bom_prefix(const char *data, size_t len,
        return data && bom && (len >= bom_len) && !memcmp(data, bom, bom_len);
 }
 
-static const char utf16_be_bom[] = {'\xFE', '\xFF'};
-static const char utf16_le_bom[] = {'\xFF', '\xFE'};
-static const char utf32_be_bom[] = {'\0', '\0', '\xFE', '\xFF'};
-static const char utf32_le_bom[] = {'\xFF', '\xFE', '\0', '\0'};
-
 int has_prohibited_utf_bom(const char *enc, const char *data, size_t len)
 {
        return (

diff --git a/utf8.h b/utf8.h
index edea55e093a4713dcd0056da9a31b63f0cb51272..84efbfcb1fa163b5b799add81b4464ff6eb88611 100644 (file)
--- a/utf8.h
+++ b/utf8.h
@@ -27,7 +27,7 @@ void strbuf_utf8_replace(struct strbuf *sb, int pos, int width,
 
 #ifndef NO_ICONV
 char *reencode_string_iconv(const char *in, size_t insz,
-                           iconv_t conv, size_t *outsz);
+                           iconv_t conv, size_t bom_len, size_t *outsz);
 char *reencode_string_len(const char *in, size_t insz,
                          const char *out_encoding,
                          const char *in_encoding,