}
+static int validate_encoding(const char *path, const char *enc,
+ const char *data, size_t len, int die_on_error)
+{
+ /* We only check for UTF here as UTF?? can be an alias for UTF-?? */
+ if (istarts_with(enc, "UTF")) {
+ /*
+ * Check for detectable errors in UTF encodings
+ */
+ if (has_prohibited_utf_bom(enc, data, len)) {
+ const char *error_msg = _(
+ "BOM is prohibited in '%s' if encoded as %s");
+ /*
+ * This advice is shown for UTF-??BE and UTF-??LE encodings.
+ * We cut off the last two characters of the encoding name
+ * to generate the encoding name suitable for BOMs.
+ */
+ const char *advise_msg = _(
+ "The file '%s' contains a byte order "
+ "mark (BOM). Please use UTF-%s as "
+ "working-tree-encoding.");
+ const char *stripped = NULL;
+ char *upper = xstrdup_toupper(enc);
+ upper[strlen(upper)-2] = '\0';
+ if (!skip_prefix(upper, "UTF-", &stripped))
+ skip_prefix(stripped, "UTF", &stripped);
+ advise(advise_msg, path, stripped);
+ free(upper);
+ if (die_on_error)
+ die(error_msg, path, enc);
+ else {
+ return error(error_msg, path, enc);
+ }
+
+ } else if (is_missing_required_utf_bom(enc, data, len)) {
+ const char *error_msg = _(
+ "BOM is required in '%s' if encoded as %s");
+ const char *advise_msg = _(
+ "The file '%s' is missing a byte order "
+ "mark (BOM). Please use UTF-%sBE or UTF-%sLE "
+ "(depending on the byte order) as "
+ "working-tree-encoding.");
+ const char *stripped = NULL;
+ char *upper = xstrdup_toupper(enc);
+ if (!skip_prefix(upper, "UTF-", &stripped))
+ skip_prefix(stripped, "UTF", &stripped);
+ advise(advise_msg, path, stripped, stripped);
+ free(upper);
+ if (die_on_error)
+ die(error_msg, path, enc);
+ else {
+ return error(error_msg, path, enc);
+ }
+ }
+
+ }
+ return 0;
+}
+
+static void trace_encoding(const char *context, const char *path,
+ const char *encoding, const char *buf, size_t len)
+{
+ static struct trace_key coe = TRACE_KEY_INIT(WORKING_TREE_ENCODING);
+ struct strbuf trace = STRBUF_INIT;
+ int i;
+
+ strbuf_addf(&trace, "%s (%s, considered %s):\n", context, path, encoding);
+ for (i = 0; i < len && buf; ++i) {
+ strbuf_addf(
+ &trace,"| \e[2m%2i:\e[0m %2x \e[2m%c\e[0m%c",
+ i,
+ (unsigned char) buf[i],
+ (buf[i] > 32 && buf[i] < 127 ? buf[i] : ' '),
+ ((i+1) % 8 && (i+1) < len ? ' ' : '\n')
+ );
+ }
+ strbuf_addchars(&trace, '\n', 1);
+
+ trace_strbuf(&coe, &trace);
+ strbuf_release(&trace);
+}
+
+static int check_roundtrip(const char *enc_name)
+{
+ /*
+ * check_roundtrip_encoding contains a string of comma and/or
+ * space separated encodings (eg. "UTF-16, ASCII, CP1125").
+ * Search for the given encoding in that string.
+ */
+ const char *found = strcasestr(check_roundtrip_encoding, enc_name);
+ const char *next;
+ int len;
+ if (!found)
+ return 0;
+ next = found + strlen(enc_name);
+ len = strlen(check_roundtrip_encoding);
+ return (found && (
+ /*
+ * check that the found encoding is at the
+ * beginning of check_roundtrip_encoding or
+ * that it is prefixed with a space or comma
+ */
+ found == check_roundtrip_encoding || (
+ (isspace(found[-1]) || found[-1] == ',')
+ )
+ ) && (
+ /*
+ * check that the found encoding is at the
+ * end of check_roundtrip_encoding or
+ * that it is suffixed with a space or comma
+ */
+ next == check_roundtrip_encoding + len || (
+ next < check_roundtrip_encoding + len &&
+ (isspace(next[0]) || next[0] == ',')
+ )
+ ));
+}
+
static const char *default_encoding = "UTF-8";
static int encode_to_git(const char *path, const char *src, size_t src_len,
if (!buf && !src)
return 1;
+ if (validate_encoding(path, enc, src, src_len, die_on_error))
+ return 0;
+
+ trace_encoding("source", path, enc, src, src_len);
dst = reencode_string_len(src, src_len, default_encoding, enc,
&dst_len);
if (!dst) {
return 0;
}
}
+ trace_encoding("destination", path, default_encoding, dst, dst_len);
+
+ /*
+ * UTF supports lossless conversion round tripping [1] and conversions
+ * between UTF and other encodings are mostly round trip safe as
+ * Unicode aims to be a superset of all other character encodings.
+ * However, certain encodings (e.g. SHIFT-JIS) are known to have round
+ * trip issues [2]. Check the round trip conversion for all encodings
+ * listed in core.checkRoundtripEncoding.
+ *
+ * The round trip check is only performed if content is written to Git.
+ * This ensures that no information is lost during conversion to/from
+ * the internal UTF-8 representation.
+ *
+ * Please note, the code below is not tested because I was not able to
+ * generate a faulty round trip without an iconv error. Iconv errors
+ * are already caught above.
+ *
+ * [1] http://unicode.org/faq/utf_bom.html#gen2
+ * [2] https://support.microsoft.com/en-us/help/170559/prb-conversion-problem-between-shift-jis-and-unicode
+ */
+ if (die_on_error && check_roundtrip(enc)) {
+ char *re_src;
+ int re_src_len;
+
+ re_src = reencode_string_len(dst, dst_len,
+ enc, default_encoding,
+ &re_src_len);
+
+ trace_printf("Checking roundtrip encoding for %s...\n", enc);
+ trace_encoding("reencoded source", path, enc,
+ re_src, re_src_len);
+
+ if (!re_src || src_len != re_src_len ||
+ memcmp(src, re_src, src_len)) {
+ const char* msg = _("encoding '%s' from %s to %s and "
+ "back is not the same");
+ die(msg, path, enc, default_encoding);
+ }
+
+ free(re_src);
+ }
strbuf_attach(buf, dst, dst_len, dst_len + 1);
return 1;
static int ident_to_worktree(const char *path, const char *src, size_t len,
struct strbuf *buf, int ident)
{
- unsigned char sha1[20];
+ struct object_id oid;
char *to_free = NULL, *dollar, *spc;
int cnt;
/* are we "faking" in place editing ? */
if (src == buf->buf)
to_free = strbuf_detach(buf, NULL);
- hash_sha1_file(src, len, "blob", sha1);
+ hash_object_file(src, len, "blob", &oid);
- strbuf_grow(buf, len + cnt * 43);
+ strbuf_grow(buf, len + cnt * (the_hash_algo->hexsz + 3));
for (;;) {
/* step 1: run to the next '$' */
dollar = memchr(src, '$', len);
/* step 4: substitute */
strbuf_addstr(buf, "Id: ");
- strbuf_add(buf, sha1_to_hex(sha1), 40);
+ strbuf_addstr(buf, oid_to_hex(&oid));
strbuf_addstr(buf, " $");
}
strbuf_add(buf, src, len);
struct stream_filter filter;
struct strbuf left;
int state;
- char ident[45]; /* ": x40 $" */
+ char ident[GIT_MAX_HEXSZ + 5]; /* ": x40 $" */
};
static int is_foreign_ident(const char *str)
ident_free_fn,
};
-static struct stream_filter *ident_filter(const unsigned char *sha1)
+static struct stream_filter *ident_filter(const struct object_id *oid)
{
struct ident_filter *ident = xmalloc(sizeof(*ident));
xsnprintf(ident->ident, sizeof(ident->ident),
- ": %s $", sha1_to_hex(sha1));
+ ": %s $", oid_to_hex(oid));
strbuf_init(&ident->left, 0);
ident->filter.vtbl = &ident_vtbl;
ident->state = 0;
* Note that you would be crazy to set CRLF, smuge/clean or ident to a
* large binary blob you would want us not to slurp into the memory!
*/
-struct stream_filter *get_stream_filter(const char *path, const unsigned char *sha1)
+struct stream_filter *get_stream_filter(const char *path, const struct object_id *oid)
{
struct conv_attrs ca;
struct stream_filter *filter = NULL;
return NULL;
if (ca.ident)
- filter = ident_filter(sha1);
+ filter = ident_filter(oid);
if (output_eol(ca.crlf_action) == EOL_CRLF)
filter = cascade_filter(filter, lf_to_crlf_filter());