static int find_invalid_utf8(const char *buf, int len)
{
int offset = 0;
+ static const unsigned int max_codepoint[] = {
+ 0x7f, 0x7ff, 0xffff, 0x10ffff
+ };
while (len) {
unsigned char c = *buf++;
int bytes, bad_offset;
+ unsigned int codepoint;
+ unsigned int min_val, max_val;
len--;
offset++;
bytes++;
}
- /* Must be between 1 and 5 more bytes */
- if (bytes < 1 || bytes > 5)
+ /*
+ * Must be between 1 and 3 more bytes. Longer sequences result in
+ * codepoints beyond U+10FFFF, which are guaranteed never to exist.
+ */
+ if (bytes < 1 || 3 < bytes)
return bad_offset;
/* Do we *have* that many bytes? */
if (len < bytes)
return bad_offset;
+ /*
+ * Place the encoded bits at the bottom of the value and compute the
+ * valid range.
+ */
+ codepoint = (c & 0x7f) >> bytes;
+ min_val = max_codepoint[bytes-1] + 1;
+ max_val = max_codepoint[bytes];
+
offset += bytes;
len -= bytes;
/* And verify that they are good continuation bytes */
do {
+ codepoint <<= 6;
+ codepoint |= *buf & 0x3f;
if ((*buf++ & 0xc0) != 0x80)
return bad_offset;
} while (--bytes);
- /* We could/should check the value and length here too */
+ /* Reject codepoints that are out of range for the sequence length. */
+ if (codepoint < min_val || codepoint > max_val)
+ return bad_offset;
+ /* Surrogates are only for UTF-16 and cannot be encoded in UTF-8. */
+ if ((codepoint & 0x1ff800) == 0xd800)
+ return bad_offset;
+ /* U+xxFFFE and U+xxFFFF are guaranteed non-characters. */
+ if ((codepoint & 0xffffe) == 0xfffe)
+ return bad_offset;
+ /* So are anything in the range U+FDD0..U+FDEF. */
+ if (codepoint >= 0xfdd0 && codepoint <= 0xfdef)
+ return bad_offset;
}
return -1;
}
*
* If it isn't, it assumes any non-utf8 characters are Latin1,
* and does the conversion.
- *
- * Fixme: we should probably also disallow overlong forms and
- * invalid characters. But we don't do that currently.
*/
static int verify_utf8(struct strbuf *buf)
{