Reject all overlong UTF8 sequences.

gen_utf8_tables.py

@@ -6,13 +6,12 @@ mask = lambda n: (1 << n) - 1
 
 
 def print_table(type, name, t):
-    assert len(t) == 256
     print("static const %s %s[] =" % (type, name))
     first = True
     for i in range(0,len(t),16):
         print ((" {" if i == 0 else "  ") +
                ", ".join("0x%02x"%n for n in t[i:i+16]) + 
-               ("," if i + 16 < 256 else "};"))
+               ("," if i + 16 < len(t) else "};"))
 
 
 def utf8info(c):
@@ -30,3 +29,4 @@ print("#define UTF8_CONTINUATION_BYTE ((unsigned char)255)")
 
 print_table("unsigned char", "utf8_coding_length", table(0))
 print_table("unsigned char", "utf8_coding_bits", table(1))
+print_table("int", "utf8_first_codepoint", [0, 0x0, 0x80, 0x800, 0x10000])

jv_unicode.c

@@ -33,6 +33,10 @@ const char* jvp_utf8_next(const char* in, const char* end, int* codepoint_ret) {
       }
       codepoint = (codepoint << 6) | (ch & 0x3f);
     }
+    if (codepoint < utf8_first_codepoint[length]) {
+      /* Overlong UTF8 sequence */
+      codepoint = -1;
+    }
     if (0xD800 <= codepoint && codepoint <= 0xDFFF) {
       /* Surrogate codepoints can't be encoded in UTF8 */
       codepoint = -1;

jv_utf8_tables.h

@@ -33,3 +33,5 @@ static const unsigned char utf8_coding_bits[] =
   0x1f, 0x1f, 0x1f, 0x1f, 0x1f, 0x1f, 0x1f, 0x1f, 0x1f, 0x1f, 0x1f, 0x1f, 0x1f, 0x1f, 0x1f, 0x1f,
   0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f,
   0x07, 0x07, 0x07, 0x07, 0x07, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00};
+static const int utf8_first_codepoint[] =
+ {0x00, 0x00, 0x80, 0x800, 0x10000};