#define NO_THE_INDEX_COMPATIBILITY_MACROS
#include "cache.h"
#include "config.h"
+#include "object-store.h"
#include "attr.h"
#include "run-command.h"
#include "quote.h"
/* fall through */
return text_eol_is_crlf() ? EOL_CRLF : EOL_LF;
}
- warning("Illegal crlf_action %d\n", (int)crlf_action);
+ warning(_("illegal crlf_action %d"), (int)crlf_action);
return core_eol;
}
* CRLFs would not be restored by checkout
*/
if (conv_flags & CONV_EOL_RNDTRP_DIE)
- die(_("CRLF would be replaced by LF in %s."), path);
+ die(_("CRLF would be replaced by LF in %s"), path);
else if (conv_flags & CONV_EOL_RNDTRP_WARN)
warning(_("CRLF will be replaced by LF in %s.\n"
"The file will have its original line"
- " endings in your working directory."), path);
+ " endings in your working directory"), path);
} else if (old_stats->lonelf && !new_stats->lonelf ) {
/*
* CRLFs would be added by checkout
else if (conv_flags & CONV_EOL_RNDTRP_WARN)
warning(_("LF will be replaced by CRLF in %s.\n"
"The file will have its original line"
- " endings in your working directory."), path);
+ " endings in your working directory"), path);
}
}
strbuf_addf(&trace, "%s (%s, considered %s):\n", context, path, encoding);
for (i = 0; i < len && buf; ++i) {
strbuf_addf(
- &trace,"| \e[2m%2i:\e[0m %2x \e[2m%c\e[0m%c",
+ &trace, "| \033[2m%2i:\033[0m %2x \033[2m%c\033[0m%c",
i,
(unsigned char) buf[i],
(buf[i] > 32 && buf[i] < 127 ? buf[i] : ' '),
strbuf_release(&trace);
}
+static int check_roundtrip(const char *enc_name)
+{
+ /*
+ * check_roundtrip_encoding contains a string of comma and/or
+ * space separated encodings (eg. "UTF-16, ASCII, CP1125").
+ * Search for the given encoding in that string.
+ */
+ const char *found = strcasestr(check_roundtrip_encoding, enc_name);
+ const char *next;
+ int len;
+ if (!found)
+ return 0;
+ next = found + strlen(enc_name);
+ len = strlen(check_roundtrip_encoding);
+ return (found && (
+ /*
+ * check that the found encoding is at the
+ * beginning of check_roundtrip_encoding or
+ * that it is prefixed with a space or comma
+ */
+ found == check_roundtrip_encoding || (
+ (isspace(found[-1]) || found[-1] == ',')
+ )
+ ) && (
+ /*
+ * check that the found encoding is at the
+ * end of check_roundtrip_encoding or
+ * that it is suffixed with a space or comma
+ */
+ next == check_roundtrip_encoding + len || (
+ next < check_roundtrip_encoding + len &&
+ (isspace(next[0]) || next[0] == ',')
+ )
+ ));
+}
+
static const char *default_encoding = "UTF-8";
static int encode_to_git(const char *path, const char *src, size_t src_len,
struct strbuf *buf, const char *enc, int conv_flags)
{
char *dst;
- int dst_len;
+ size_t dst_len;
int die_on_error = conv_flags & CONV_WRITE_OBJECT;
/*
}
trace_encoding("destination", path, default_encoding, dst, dst_len);
+ /*
+ * UTF supports lossless conversion round tripping [1] and conversions
+ * between UTF and other encodings are mostly round trip safe as
+ * Unicode aims to be a superset of all other character encodings.
+ * However, certain encodings (e.g. SHIFT-JIS) are known to have round
+ * trip issues [2]. Check the round trip conversion for all encodings
+ * listed in core.checkRoundtripEncoding.
+ *
+ * The round trip check is only performed if content is written to Git.
+ * This ensures that no information is lost during conversion to/from
+ * the internal UTF-8 representation.
+ *
+ * Please note, the code below is not tested because I was not able to
+ * generate a faulty round trip without an iconv error. Iconv errors
+ * are already caught above.
+ *
+ * [1] http://unicode.org/faq/utf_bom.html#gen2
+ * [2] https://support.microsoft.com/en-us/help/170559/prb-conversion-problem-between-shift-jis-and-unicode
+ */
+ if (die_on_error && check_roundtrip(enc)) {
+ char *re_src;
+ size_t re_src_len;
+
+ re_src = reencode_string_len(dst, dst_len,
+ enc, default_encoding,
+ &re_src_len);
+
+ trace_printf("Checking roundtrip encoding for %s...\n", enc);
+ trace_encoding("reencoded source", path, enc,
+ re_src, re_src_len);
+
+ if (!re_src || src_len != re_src_len ||
+ memcmp(src, re_src, src_len)) {
+ const char* msg = _("encoding '%s' from %s to %s and "
+ "back is not the same");
+ die(msg, path, enc, default_encoding);
+ }
+
+ free(re_src);
+ }
+
strbuf_attach(buf, dst, dst_len, dst_len + 1);
return 1;
}
struct strbuf *buf, const char *enc)
{
char *dst;
- int dst_len;
+ size_t dst_len;
/*
* No encoding is specified or there is nothing to encode.
dst = reencode_string_len(src, src_len, enc, default_encoding,
&dst_len);
if (!dst) {
- error("failed to encode '%s' from %s to %s",
- path, default_encoding, enc);
+ error(_("failed to encode '%s' from %s to %s"),
+ path, default_encoding, enc);
return 0;
}
if (start_command(&child_process)) {
strbuf_release(&cmd);
- return error("cannot fork to run external filter '%s'", params->cmd);
+ return error(_("cannot fork to run external filter '%s'"),
+ params->cmd);
}
sigchain_push(SIGPIPE, SIG_IGN);
if (close(child_process.in))
write_err = 1;
if (write_err)
- error("cannot feed the input to external filter '%s'", params->cmd);
+ error(_("cannot feed the input to external filter '%s'"),
+ params->cmd);
sigchain_pop(SIGPIPE);
status = finish_command(&child_process);
if (status)
- error("external filter '%s' failed %d", params->cmd, status);
+ error(_("external filter '%s' failed %d"), params->cmd, status);
strbuf_release(&cmd);
return (write_err || status);
return 0; /* error was already reported */
if (strbuf_read(&nbuf, async.out, len) < 0) {
- err = error("read from external filter '%s' failed", cmd);
+ err = error(_("read from external filter '%s' failed"), cmd);
}
if (close(async.out)) {
- err = error("read from external filter '%s' failed", cmd);
+ err = error(_("read from external filter '%s' failed"), cmd);
}
if (finish_async(&async)) {
- err = error("external filter '%s' failed", cmd);
+ err = error(_("external filter '%s' failed"), cmd);
}
if (!err) {
* Something went wrong with the protocol filter.
* Force shutdown and restart if another blob requires filtering.
*/
- error("external filter '%s' failed", entry->subprocess.cmd);
+ error(_("external filter '%s' failed"), entry->subprocess.cmd);
subprocess_stop(&subprocess_map, &entry->subprocess);
free(entry);
}
else if (wanted_capability & CAP_SMUDGE)
filter_type = "smudge";
else
- die("unexpected filter type");
+ die(_("unexpected filter type"));
sigchain_push(SIGPIPE, SIG_IGN);
err = strlen(path) > LARGE_PACKET_DATA_MAX - strlen("pathname=\n");
if (err) {
- error("path name too long for external filter");
+ error(_("path name too long for external filter"));
goto done;
}
assert(subprocess_map_initialized);
entry = (struct cmd2process *)subprocess_find_entry(&subprocess_map, cmd);
if (!entry) {
- error("external filter '%s' is not available anymore although "
- "not all paths have been filtered", cmd);
+ error(_("external filter '%s' is not available anymore although "
+ "not all paths have been filtered"), cmd);
return 0;
}
process = &entry->subprocess.process;
static int ident_to_worktree(const char *path, const char *src, size_t len,
struct strbuf *buf, int ident)
{
- unsigned char sha1[20];
+ struct object_id oid;
char *to_free = NULL, *dollar, *spc;
int cnt;
/* are we "faking" in place editing ? */
if (src == buf->buf)
to_free = strbuf_detach(buf, NULL);
- hash_sha1_file(src, len, "blob", sha1);
+ hash_object_file(src, len, "blob", &oid);
- strbuf_grow(buf, len + cnt * 43);
+ strbuf_grow(buf, len + cnt * (the_hash_algo->hexsz + 3));
for (;;) {
/* step 1: run to the next '$' */
dollar = memchr(src, '$', len);
/* step 4: substitute */
strbuf_addstr(buf, "Id: ");
- strbuf_add(buf, sha1_to_hex(sha1), 40);
+ strbuf_addstr(buf, oid_to_hex(&oid));
strbuf_addstr(buf, " $");
}
strbuf_add(buf, src, len);
const char *working_tree_encoding; /* Supported encoding or default encoding if NULL */
};
-static void convert_attrs(struct conv_attrs *ca, const char *path)
+static void convert_attrs(const struct index_state *istate,
+ struct conv_attrs *ca, const char *path)
{
static struct attr_check *check;
+ struct attr_check_item *ccheck = NULL;
if (!check) {
check = attr_check_initl("crlf", "ident", "filter",
git_config(read_convert_config, NULL);
}
- if (!git_check_attr(path, check)) {
- struct attr_check_item *ccheck = check->items;
- ca->crlf_action = git_path_check_crlf(ccheck + 4);
- if (ca->crlf_action == CRLF_UNDEFINED)
- ca->crlf_action = git_path_check_crlf(ccheck + 0);
- ca->ident = git_path_check_ident(ccheck + 1);
- ca->drv = git_path_check_convert(ccheck + 2);
- if (ca->crlf_action != CRLF_BINARY) {
- enum eol eol_attr = git_path_check_eol(ccheck + 3);
- if (ca->crlf_action == CRLF_AUTO && eol_attr == EOL_LF)
- ca->crlf_action = CRLF_AUTO_INPUT;
- else if (ca->crlf_action == CRLF_AUTO && eol_attr == EOL_CRLF)
- ca->crlf_action = CRLF_AUTO_CRLF;
- else if (eol_attr == EOL_LF)
- ca->crlf_action = CRLF_TEXT_INPUT;
- else if (eol_attr == EOL_CRLF)
- ca->crlf_action = CRLF_TEXT_CRLF;
- }
- ca->working_tree_encoding = git_path_check_encoding(ccheck + 5);
- } else {
- ca->drv = NULL;
- ca->crlf_action = CRLF_UNDEFINED;
- ca->ident = 0;
+ git_check_attr(istate, path, check);
+ ccheck = check->items;
+ ca->crlf_action = git_path_check_crlf(ccheck + 4);
+ if (ca->crlf_action == CRLF_UNDEFINED)
+ ca->crlf_action = git_path_check_crlf(ccheck + 0);
+ ca->ident = git_path_check_ident(ccheck + 1);
+ ca->drv = git_path_check_convert(ccheck + 2);
+ if (ca->crlf_action != CRLF_BINARY) {
+ enum eol eol_attr = git_path_check_eol(ccheck + 3);
+ if (ca->crlf_action == CRLF_AUTO && eol_attr == EOL_LF)
+ ca->crlf_action = CRLF_AUTO_INPUT;
+ else if (ca->crlf_action == CRLF_AUTO && eol_attr == EOL_CRLF)
+ ca->crlf_action = CRLF_AUTO_CRLF;
+ else if (eol_attr == EOL_LF)
+ ca->crlf_action = CRLF_TEXT_INPUT;
+ else if (eol_attr == EOL_CRLF)
+ ca->crlf_action = CRLF_TEXT_CRLF;
}
+ ca->working_tree_encoding = git_path_check_encoding(ccheck + 5);
/* Save attr and make a decision for action */
ca->attr_action = ca->crlf_action;
ca->crlf_action = CRLF_AUTO_INPUT;
}
-int would_convert_to_git_filter_fd(const char *path)
+int would_convert_to_git_filter_fd(const struct index_state *istate, const char *path)
{
struct conv_attrs ca;
- convert_attrs(&ca, path);
+ convert_attrs(istate, &ca, path);
if (!ca.drv)
return 0;
return apply_filter(path, NULL, 0, -1, NULL, ca.drv, CAP_CLEAN, NULL);
}
-const char *get_convert_attr_ascii(const char *path)
+const char *get_convert_attr_ascii(const struct index_state *istate, const char *path)
{
struct conv_attrs ca;
- convert_attrs(&ca, path);
+ convert_attrs(istate, &ca, path);
switch (ca.attr_action) {
case CRLF_UNDEFINED:
return "";
int ret = 0;
struct conv_attrs ca;
- convert_attrs(&ca, path);
+ convert_attrs(istate, &ca, path);
ret |= apply_filter(path, src, len, -1, dst, ca.drv, CAP_CLEAN, NULL);
if (!ret && ca.drv && ca.drv->required)
- die("%s: clean filter '%s' failed", path, ca.drv->name);
+ die(_("%s: clean filter '%s' failed"), path, ca.drv->name);
if (ret && dst) {
src = dst->buf;
int conv_flags)
{
struct conv_attrs ca;
- convert_attrs(&ca, path);
+ convert_attrs(istate, &ca, path);
assert(ca.drv);
assert(ca.drv->clean || ca.drv->process);
if (!apply_filter(path, NULL, 0, fd, dst, ca.drv, CAP_CLEAN, NULL))
- die("%s: clean filter '%s' failed", path, ca.drv->name);
+ die(_("%s: clean filter '%s' failed"), path, ca.drv->name);
encode_to_git(path, dst->buf, dst->len, dst, ca.working_tree_encoding, conv_flags);
crlf_to_git(istate, path, dst->buf, dst->len, dst, ca.crlf_action, conv_flags);
ident_to_git(path, dst->buf, dst->len, dst, ca.ident);
}
-static int convert_to_working_tree_internal(const char *path, const char *src,
+static int convert_to_working_tree_internal(const struct index_state *istate,
+ const char *path, const char *src,
size_t len, struct strbuf *dst,
int normalizing, struct delayed_checkout *dco)
{
int ret = 0, ret_filter = 0;
struct conv_attrs ca;
- convert_attrs(&ca, path);
+ convert_attrs(istate, &ca, path);
ret |= ident_to_worktree(path, src, len, dst, ca.ident);
if (ret) {
ret_filter = apply_filter(
path, src, len, -1, dst, ca.drv, CAP_SMUDGE, dco);
if (!ret_filter && ca.drv && ca.drv->required)
- die("%s: smudge filter %s failed", path, ca.drv->name);
+ die(_("%s: smudge filter %s failed"), path, ca.drv->name);
return ret | ret_filter;
}
-int async_convert_to_working_tree(const char *path, const char *src,
+int async_convert_to_working_tree(const struct index_state *istate,
+ const char *path, const char *src,
size_t len, struct strbuf *dst,
void *dco)
{
- return convert_to_working_tree_internal(path, src, len, dst, 0, dco);
+ return convert_to_working_tree_internal(istate, path, src, len, dst, 0, dco);
}
-int convert_to_working_tree(const char *path, const char *src, size_t len, struct strbuf *dst)
+int convert_to_working_tree(const struct index_state *istate,
+ const char *path, const char *src,
+ size_t len, struct strbuf *dst)
{
- return convert_to_working_tree_internal(path, src, len, dst, 0, NULL);
+ return convert_to_working_tree_internal(istate, path, src, len, dst, 0, NULL);
}
int renormalize_buffer(const struct index_state *istate, const char *path,
const char *src, size_t len, struct strbuf *dst)
{
- int ret = convert_to_working_tree_internal(path, src, len, dst, 1, NULL);
+ int ret = convert_to_working_tree_internal(istate, path, src, len, dst, 1, NULL);
if (ret) {
src = dst->buf;
len = dst->len;
struct stream_filter filter;
struct strbuf left;
int state;
- char ident[45]; /* ": x40 $" */
+ char ident[GIT_MAX_HEXSZ + 5]; /* ": x40 $" */
};
static int is_foreign_ident(const char *str)
ident_free_fn,
};
-static struct stream_filter *ident_filter(const unsigned char *sha1)
+static struct stream_filter *ident_filter(const struct object_id *oid)
{
struct ident_filter *ident = xmalloc(sizeof(*ident));
xsnprintf(ident->ident, sizeof(ident->ident),
- ": %s $", sha1_to_hex(sha1));
+ ": %s $", oid_to_hex(oid));
strbuf_init(&ident->left, 0);
ident->filter.vtbl = &ident_vtbl;
ident->state = 0;
* Note that you would be crazy to set CRLF, smuge/clean or ident to a
* large binary blob you would want us not to slurp into the memory!
*/
-struct stream_filter *get_stream_filter(const char *path, const unsigned char *sha1)
+struct stream_filter *get_stream_filter(const struct index_state *istate,
+ const char *path,
+ const struct object_id *oid)
{
struct conv_attrs ca;
struct stream_filter *filter = NULL;
- convert_attrs(&ca, path);
+ convert_attrs(istate, &ca, path);
if (ca.drv && (ca.drv->process || ca.drv->smudge || ca.drv->clean))
return NULL;
return NULL;
if (ca.ident)
- filter = ident_filter(sha1);
+ filter = ident_filter(oid);
if (output_eol(ca.crlf_action) == EOL_CRLF)
filter = cascade_filter(filter, lf_to_crlf_filter());