Improve performance of UTF-8 validation (#3675)

jvp_utf8_next() had a fast path for ASCII, but it is not inlined into
its callers, so every byte of a string cost a function call.

Move the end check and the ASCII case into a static inline function in
jv_unicode.h and keep the rest out of line. This speeds up parsing of
string-heavy input by up to 1.17x and string functions such as length,
explode and @html, with the same output for every input.

src/jv_parse.c

@@ -784,9 +784,10 @@ jv jv_parser_next(struct jv_parser* p) {
   } else if (msg) {
     jv_free(value);
     if (ch != '\036' && (p->flags & JV_PARSE_SEQ)) {
-      // Skip to the next RS
       value = make_error(p, "%s at line %d, column %d (need RS to resync)", msg, p->line, p->column);
       parser_reset(p);
+      // Skip to the next RS.  This must come after parser_reset(), which
+      // sets the state back to JV_PARSER_NORMAL.
       p->st = JV_PARSER_WAITING_FOR_RS;
       return value;
     }

src/jv_unicode.c

@@ -27,19 +27,14 @@ const char* jvp_utf8_backtrack(const char* start, const char* min, int *missing_
   return start;
 }
 
-const char* jvp_utf8_next(const char* in, const char* end, int* codepoint_ret) {
-  assert(in <= end);
-  if (in == end) {
-    return 0;
-  }
+// The rest of jvp_utf8_next(), for a first byte that is not ASCII.
+const char* jvp_utf8_next_multibyte(const char* in, const char* end, int* codepoint_ret) {
+  assert(in < end);
   int codepoint = -1;
   unsigned char first = (unsigned char)in[0];
+  assert(first & 0x80);
   int length = utf8_coding_length[first];
-  if ((first & 0x80) == 0) {
-    /* Fast-path for ASCII */
-    codepoint = first;
-    length = 1;
-  } else if (length == 0 || length == UTF8_CONTINUATION_BYTE) {
+  if (length == 0 || length == UTF8_CONTINUATION_BYTE) {
     /* Bad single byte - either an invalid byte or an out-of-place continuation byte */
     length = 1;
   } else if (length > end - in) {

src/jv_unicode.h

@@ -1,10 +1,27 @@
 #ifndef JV_UNICODE_H
 #define JV_UNICODE_H
 
+#include <assert.h>
+
 const char* jvp_utf8_backtrack(const char* start, const char* min, int *missing_bytes);
-const char* jvp_utf8_next(const char* in, const char* end, int* codepoint);
+const char* jvp_utf8_next_multibyte(const char* in, const char* end, int* codepoint);
 int jvp_utf8_is_valid(const char* in, const char* end);
 
+// Decode one codepoint.  ASCII is handled here so that callers looping over
+// a string do not make a call per byte; everything else goes to
+// jvp_utf8_next_multibyte().
+static inline const char* jvp_utf8_next(const char* in, const char* end, int* codepoint) {
+  if (in >= end) {
+    assert(in == end);
+    return 0;
+  }
+  if (!(*in & 0x80)) {
+    *codepoint = (unsigned char)*in;
+    return in + 1;
+  }
+  return jvp_utf8_next_multibyte(in, end, codepoint);
+}
+
 int jvp_utf8_decode_length(char startchar);
 
 int jvp_utf8_encode_length(int codepoint);

tests/jq.test

@@ -84,6 +84,11 @@ null
 "Zm/Ds2Jhcgo="
 "foóbar\n"
 
+# Invalid UTF-8 next to ASCII runs is still replaced
+map(@base64d)
+["YWJjZGVmZ2hpav9rbG1ub3A=", "YWJjZGVmZ2hpasOpa2xtbm9w5pel", "/2FiY2RlZmdoaWo=", "YWJjZGVmZ2hpauaX", "YWJjZGVmZ2hpau2ggHh5eg=="]
+["abcdefghij�klmnop","abcdefghijéklmnop日","�abcdefghij","abcdefghij�","abcdefghij�xyz"]
+
 @uri
 "\u03bc"
 "%CE%BC"