Strip UTF-8 BOM at the start of each input file (#3657)

The UTF-8 BOM was stripped only at the start of the parser stream, but jq
feeds all input files into a single parser, so a BOM at the start of the
second and later files reached the parser as content and failed with a
confusing "Invalid numeric literal" error. Strip it in the file readers
instead, at the start of each input file including stdin, and in
jv_load_file.

This also strips the BOM for raw input, so -R and --rawfile no longer
yield a leading U+FEFF, and a BOM at the start of a -f program file or an
imported module is now accepted rather than reported as a syntax error.
In exchange, "\uFEFF1" | fromjson is now a parse error, since a U+FEFF
inside a string is content rather than an encoding marker, and the
dedicated "Malformed BOM" error is gone.

src/jv_file.c

@@ -43,6 +43,7 @@ jv jv_load_file(const char* filename, int raw) {
   // buffer, we need to be able to read the remainder of a sequence and add that
   // before appending.
   char buf[4096 + 4];
+  int at_file_start = 1;
   while (!feof(file) && !ferror(file)) {
     size_t n = fread(buf, 1, sizeof(buf) - 4, file);
     if (n == 0)
@@ -55,12 +56,22 @@ jv jv_load_file(const char* filename, int raw) {
       n += fread(end, 1, len, file);
     }
 
+    char *p = buf;
+    if (at_file_start) {
+      // Strip UTF-8 BOM at the start of the file
+      if (n >= 3 && !memcmp(buf, "\xEF\xBB\xBF", 3)) {
+        p += 3;
+        n -= 3;
+      }
+      at_file_start = 0;
+    }
+
     if (raw) {
-      data = jv_string_append_buf(data, buf, n);
+      data = jv_string_append_buf(data, p, n);
       if (!jv_is_valid(data))
         break;
     } else {
-      jv_parser_set_buf(parser, buf, n, !feof(file));
+      jv_parser_set_buf(parser, p, n, !feof(file));
       jv value;
       while (jv_is_valid((value = jv_parser_next(parser))))
         data = jv_array_append(data, value);

src/jv_parse.c

@@ -37,7 +37,6 @@ struct jv_parser {
   int curr_buf_pos;
   int curr_buf_is_partial;
   int eof;
-  unsigned bom_strip_position;
 
   int flags;
 
@@ -90,7 +89,6 @@ static void parser_init(struct jv_parser* p, int flags) {
   p->eof = 0;
   p->curr_buf = 0;
   p->curr_buf_length = p->curr_buf_pos = p->curr_buf_is_partial = 0;
-  p->bom_strip_position = 0;
   p->last_ch_was_ws = 0;
   p->line = 1;
   p->column = 0;
@@ -729,8 +727,6 @@ void jv_parser_free(struct jv_parser* p) {
   jv_mem_free(p);
 }
 
-static const unsigned char UTF8_BOM[] = {0xEF,0xBB,0xBF};
-
 int jv_parser_remaining(struct jv_parser* p) {
   if (p->curr_buf == 0)
     return 0;
@@ -740,22 +736,6 @@ int jv_parser_remaining(struct jv_parser* p) {
 void jv_parser_set_buf(struct jv_parser* p, const char* buf, int length, int is_partial) {
   assert((p->curr_buf == 0 || p->curr_buf_pos == p->curr_buf_length)
          && "previous buffer not exhausted");
-  while (length > 0 && p->bom_strip_position < sizeof(UTF8_BOM)) {
-    if ((unsigned char)*buf == UTF8_BOM[p->bom_strip_position]) {
-      // matched a BOM character
-      buf++;
-      length--;
-      p->bom_strip_position++;
-    } else {
-      if (p->bom_strip_position == 0) {
-        // no BOM in this document
-        p->bom_strip_position = sizeof(UTF8_BOM);
-      } else {
-        // malformed BOM (prefix present, rest missing)
-        p->bom_strip_position = 0xff;
-      }
-    }
-  }
   p->curr_buf = buf;
   p->curr_buf_length = length;
   p->curr_buf_pos = 0;
@@ -779,12 +759,6 @@ jv jv_parser_next(struct jv_parser* p) {
     return jv_invalid();
   if (!p->curr_buf)
     return jv_invalid(); // Need a buffer
-  if (p->bom_strip_position == 0xff) {
-    if (!(p->flags & JV_PARSE_SEQ))
-      return jv_invalid_with_msg(jv_string("Malformed BOM"));
-    p->st =JV_PARSER_WAITING_FOR_RS;
-    parser_reset(p);
-  }
   jv value = jv_invalid();
   if ((p->flags & JV_PARSE_STREAMING) && stream_check_done(p, &value))
     return value;

src/util.c

@@ -193,6 +193,7 @@ struct jq_util_input_state {
   size_t buf_valid_len;
   jv current_filename;
   size_t current_line;
+  int at_file_start;
 };
 
 static void fprinter(void *data, const char *fname) {
@@ -277,6 +278,7 @@ static int jq_util_input_read_more(jq_util_input_state *state) {
     if (f != NULL) {
       jv_free(state->current_filename);
       state->current_line = 0;
+      state->at_file_start = 1;
       if (!strcmp(f, "-")) {
         state->current_input = stdin;
         state->current_filename = jv_string("<stdin>");
@@ -343,6 +345,14 @@ static int jq_util_input_read_more(jq_util_input_state *state) {
         state->buf_valid_len = (p - state->buf) + 1;
       }
     }
+    if (state->at_file_start) {
+      // Strip UTF-8 BOM at the start of each file
+      if (state->buf_valid_len >= 3 && !memcmp(state->buf, "\xEF\xBB\xBF", 3)) {
+        state->buf_valid_len -= 3;
+        memmove(state->buf, state->buf + 3, state->buf_valid_len);
+      }
+      state->at_file_start = 0;
+    }
   }
   return state->curr_file == state->nfiles && !state->current_input;
 }

tests/jq.test

@@ -42,13 +42,6 @@ null
 {"x":-1}
 {"x":1}
 
-# The input line starts with a 0xFEFF (byte order mark) codepoint
-# No, there is no reason to have a byte order mark in UTF8 text.
-# But apparently people do, so jq shouldn't break on it.
-.
-"byte order mark"
-"byte order mark"
-
 # We test escapes by matching them against Unicode codepoints
 # FIXME: more tests needed for weird unicode stuff (e.g. utf16 pairs)
 "Aa\r\n\t\b\f\u03bc"

tests/shtest

@@ -57,6 +57,17 @@ for exp in '1+1' '1-1' '2*3' '9/3' '9%3' '9==3' '9!=3' \
   fi
 done
 
+## Test UTF-8 BOM is stripped at the start of each input file
+printf '\357\273\277[{"foo": 1}\n,{"foo": 2}\n]\n' > $d/bom1.json
+printf '\357\273\277[{"foo": 3}\n,{"foo": 4}\n]\n' > $d/bom2.json
+$VALGRIND $Q $JQ -e '.[0].foo == 1' $d/bom1.json
+$VALGRIND $Q $JQ -se '. == [[{"foo":1},{"foo":2}],[{"foo":3},{"foo":4}]]' $d/bom1.json $d/bom2.json
+$VALGRIND $Q $JQ -ne '[inputs] == [[{"foo":1},{"foo":2}],[{"foo":3},{"foo":4}]]' $d/bom1.json $d/bom2.json
+$VALGRIND $Q $JQ -ne '[inputs] == [[{"foo":1},{"foo":2}],[{"foo":3},{"foo":4}]]' $d/bom1.json - < $d/bom2.json
+$VALGRIND $Q $JQ -Rne '[inputs] == ["[{\"foo\": 1}", ",{\"foo\": 2}", "]", "[{\"foo\": 3}", ",{\"foo\": 4}", "]"]' $d/bom1.json $d/bom2.json
+$VALGRIND $Q $JQ -ne --slurpfile x $d/bom1.json '$x == [[{"foo":1},{"foo":2}]]'
+$VALGRIND $Q $JQ -ne --rawfile x $d/bom1.json '$x == "[{\"foo\": 1}\n,{\"foo\": 2}\n]\n"'
+
 ## Test JSON sequence support
 cat > $d/expected <<EOF
 jq: ignoring parse error: Truncated value at line 2, column 5