From e001deb4cfb735a6b14a802398a41bfa03df5a2a Mon Sep 17 00:00:00 2001 From: itchyny Date: Thu, 8 Oct 2026 23:01:36 +0900 Subject: [PATCH] Strip UTF-8 BOM at the start of each input file (#3657) The UTF-8 BOM was stripped only at the start of the parser stream, but jq feeds all input files into a single parser, so a BOM at the start of the second and later files reached the parser as content and failed with a confusing "Invalid numeric literal" error. Strip it in the file readers instead, at the start of each input file including stdin, and in jv_load_file. This also strips the BOM for raw input, so -R and --rawfile no longer yield a leading U+FEFF, and a BOM at the start of a -f program file or an imported module is now accepted rather than reported as a syntax error. In exchange, "\uFEFF1" | fromjson is now a parse error, since a U+FEFF inside a string is content rather than an encoding marker, and the dedicated "Malformed BOM" error is gone. --- src/jv_file.c | 15 +++++++++++++-- src/jv_parse.c | 26 -------------------------- src/util.c | 10 ++++++++++ tests/jq.test | 7 ------- tests/shtest | 11 +++++++++++ 5 files changed, 34 insertions(+), 35 deletions(-) diff --git a/src/jv_file.c b/src/jv_file.c index fbc1e4d6..b43d44a1 100644 --- a/src/jv_file.c +++ b/src/jv_file.c @@ -43,6 +43,7 @@ jv jv_load_file(const char* filename, int raw) { // buffer, we need to be able to read the remainder of a sequence and add that // before appending. char buf[4096 + 4]; + int at_file_start = 1; while (!feof(file) && !ferror(file)) { size_t n = fread(buf, 1, sizeof(buf) - 4, file); if (n == 0) @@ -55,12 +56,22 @@ jv jv_load_file(const char* filename, int raw) { n += fread(end, 1, len, file); } + char *p = buf; + if (at_file_start) { + // Strip UTF-8 BOM at the start of the file + if (n >= 3 && !memcmp(buf, "\xEF\xBB\xBF", 3)) { + p += 3; + n -= 3; + } + at_file_start = 0; + } + if (raw) { - data = jv_string_append_buf(data, buf, n); + data = jv_string_append_buf(data, p, n); if (!jv_is_valid(data)) break; } else { - jv_parser_set_buf(parser, buf, n, !feof(file)); + jv_parser_set_buf(parser, p, n, !feof(file)); jv value; while (jv_is_valid((value = jv_parser_next(parser)))) data = jv_array_append(data, value); diff --git a/src/jv_parse.c b/src/jv_parse.c index ceccc4d6..fe780540 100644 --- a/src/jv_parse.c +++ b/src/jv_parse.c @@ -37,7 +37,6 @@ struct jv_parser { int curr_buf_pos; int curr_buf_is_partial; int eof; - unsigned bom_strip_position; int flags; @@ -90,7 +89,6 @@ static void parser_init(struct jv_parser* p, int flags) { p->eof = 0; p->curr_buf = 0; p->curr_buf_length = p->curr_buf_pos = p->curr_buf_is_partial = 0; - p->bom_strip_position = 0; p->last_ch_was_ws = 0; p->line = 1; p->column = 0; @@ -729,8 +727,6 @@ void jv_parser_free(struct jv_parser* p) { jv_mem_free(p); } -static const unsigned char UTF8_BOM[] = {0xEF,0xBB,0xBF}; - int jv_parser_remaining(struct jv_parser* p) { if (p->curr_buf == 0) return 0; @@ -740,22 +736,6 @@ int jv_parser_remaining(struct jv_parser* p) { void jv_parser_set_buf(struct jv_parser* p, const char* buf, int length, int is_partial) { assert((p->curr_buf == 0 || p->curr_buf_pos == p->curr_buf_length) && "previous buffer not exhausted"); - while (length > 0 && p->bom_strip_position < sizeof(UTF8_BOM)) { - if ((unsigned char)*buf == UTF8_BOM[p->bom_strip_position]) { - // matched a BOM character - buf++; - length--; - p->bom_strip_position++; - } else { - if (p->bom_strip_position == 0) { - // no BOM in this document - p->bom_strip_position = sizeof(UTF8_BOM); - } else { - // malformed BOM (prefix present, rest missing) - p->bom_strip_position = 0xff; - } - } - } p->curr_buf = buf; p->curr_buf_length = length; p->curr_buf_pos = 0; @@ -779,12 +759,6 @@ jv jv_parser_next(struct jv_parser* p) { return jv_invalid(); if (!p->curr_buf) return jv_invalid(); // Need a buffer - if (p->bom_strip_position == 0xff) { - if (!(p->flags & JV_PARSE_SEQ)) - return jv_invalid_with_msg(jv_string("Malformed BOM")); - p->st =JV_PARSER_WAITING_FOR_RS; - parser_reset(p); - } jv value = jv_invalid(); if ((p->flags & JV_PARSE_STREAMING) && stream_check_done(p, &value)) return value; diff --git a/src/util.c b/src/util.c index 80d65fc8..25046c3d 100644 --- a/src/util.c +++ b/src/util.c @@ -193,6 +193,7 @@ struct jq_util_input_state { size_t buf_valid_len; jv current_filename; size_t current_line; + int at_file_start; }; static void fprinter(void *data, const char *fname) { @@ -277,6 +278,7 @@ static int jq_util_input_read_more(jq_util_input_state *state) { if (f != NULL) { jv_free(state->current_filename); state->current_line = 0; + state->at_file_start = 1; if (!strcmp(f, "-")) { state->current_input = stdin; state->current_filename = jv_string(""); @@ -343,6 +345,14 @@ static int jq_util_input_read_more(jq_util_input_state *state) { state->buf_valid_len = (p - state->buf) + 1; } } + if (state->at_file_start) { + // Strip UTF-8 BOM at the start of each file + if (state->buf_valid_len >= 3 && !memcmp(state->buf, "\xEF\xBB\xBF", 3)) { + state->buf_valid_len -= 3; + memmove(state->buf, state->buf + 3, state->buf_valid_len); + } + state->at_file_start = 0; + } } return state->curr_file == state->nfiles && !state->current_input; } diff --git a/tests/jq.test b/tests/jq.test index a7eaaee4..ca73c999 100644 --- a/tests/jq.test +++ b/tests/jq.test @@ -42,13 +42,6 @@ null {"x":-1} {"x":1} -# The input line starts with a 0xFEFF (byte order mark) codepoint -# No, there is no reason to have a byte order mark in UTF8 text. -# But apparently people do, so jq shouldn't break on it. -. -"byte order mark" -"byte order mark" - # We test escapes by matching them against Unicode codepoints # FIXME: more tests needed for weird unicode stuff (e.g. utf16 pairs) "Aa\r\n\t\b\f\u03bc" diff --git a/tests/shtest b/tests/shtest index 7e34dc52..511067fc 100755 --- a/tests/shtest +++ b/tests/shtest @@ -57,6 +57,17 @@ for exp in '1+1' '1-1' '2*3' '9/3' '9%3' '9==3' '9!=3' \ fi done +## Test UTF-8 BOM is stripped at the start of each input file +printf '\357\273\277[{"foo": 1}\n,{"foo": 2}\n]\n' > $d/bom1.json +printf '\357\273\277[{"foo": 3}\n,{"foo": 4}\n]\n' > $d/bom2.json +$VALGRIND $Q $JQ -e '.[0].foo == 1' $d/bom1.json +$VALGRIND $Q $JQ -se '. == [[{"foo":1},{"foo":2}],[{"foo":3},{"foo":4}]]' $d/bom1.json $d/bom2.json +$VALGRIND $Q $JQ -ne '[inputs] == [[{"foo":1},{"foo":2}],[{"foo":3},{"foo":4}]]' $d/bom1.json $d/bom2.json +$VALGRIND $Q $JQ -ne '[inputs] == [[{"foo":1},{"foo":2}],[{"foo":3},{"foo":4}]]' $d/bom1.json - < $d/bom2.json +$VALGRIND $Q $JQ -Rne '[inputs] == ["[{\"foo\": 1}", ",{\"foo\": 2}", "]", "[{\"foo\": 3}", ",{\"foo\": 4}", "]"]' $d/bom1.json $d/bom2.json +$VALGRIND $Q $JQ -ne --slurpfile x $d/bom1.json '$x == [[{"foo":1},{"foo":2}]]' +$VALGRIND $Q $JQ -ne --rawfile x $d/bom1.json '$x == "[{\"foo\": 1}\n,{\"foo\": 2}\n]\n"' + ## Test JSON sequence support cat > $d/expected <