diff --git a/Makefile.am b/Makefile.am index 1df41431..96d60381 100644 --- a/Makefile.am +++ b/Makefile.am @@ -231,7 +231,7 @@ EXTRA_DIST = $(DOC_FILES) $(man_MANS) $(TESTS) $(TEST_LOG_COMPILER) \ tests/setup tests/torture/input0.json \ tests/optional.test tests/man.test tests/manonig.test \ tests/jq.test tests/onig.test tests/base64.test tests/uri.test \ - tests/utf8-truncate.jq tests/jq-f-test.sh \ + tests/jq-f-test.sh \ tests/no-main-program.jq tests/yes-main-program.jq AM_DISTCHECK_CONFIGURE_FLAGS=--with-oniguruma=builtin diff --git a/src/jv_file.c b/src/jv_file.c index b10bcc0b..5976dafb 100644 --- a/src/jv_file.c +++ b/src/jv_file.c @@ -42,17 +42,17 @@ jv jv_load_file(const char* filename, int raw) { // To avoid mangling UTF-8 multi-byte sequences that cross the end of our read // buffer, we need to be able to read the remainder of a sequence and add that // before appending. - const int max_utf8_len = 4; - char buf[4096+max_utf8_len]; + char buf[4096 + 4]; while (!feof(file) && !ferror(file)) { - size_t n = fread(buf, 1, sizeof(buf)-max_utf8_len, file); - int len = 0; - + size_t n = fread(buf, 1, sizeof(buf) - 4, file); if (n == 0) continue; - if (jvp_utf8_backtrack(buf+(n-1), buf, &len) && len > 0 && + + char *end = buf + n; + int len = 0; + if (jvp_utf8_backtrack(end - 1, buf, &len) && len > 0 && !feof(file) && !ferror(file)) { - n += fread(buf+n, 1, len, file); + n += fread(end, 1, len, file); } if (raw) { diff --git a/src/util.c b/src/util.c index bcb86da8..fdfdb96d 100644 --- a/src/util.c +++ b/src/util.c @@ -65,6 +65,7 @@ void *alloca (size_t); #include "util.h" #include "jq.h" #include "jv_alloc.h" +#include "jv_unicode.h" #ifdef WIN32 FILE *fopen(const char *fname, const char *mode) { @@ -296,7 +297,9 @@ static int jq_util_input_read_more(jq_util_input_state *state) { char *res; memset(state->buf, 0xff, sizeof(state->buf)); - while (!(res = fgets(state->buf, sizeof(state->buf), state->current_input)) && + const int max_utf8_len = 4; + const int max_gets_len = sizeof(state->buf) - max_utf8_len; + while (!(res = fgets(state->buf, max_gets_len, state->current_input)) && ferror(state->current_input) && errno == EINTR) clearerr(state->current_input); if (res == NULL) { @@ -304,7 +307,7 @@ static int jq_util_input_read_more(jq_util_input_state *state) { if (ferror(state->current_input)) state->failures++; } else { - const char *p = memchr(state->buf, '\n', sizeof(state->buf)); + const char *p = memchr(state->buf, '\n', max_gets_len); if (p != NULL) state->current_line++; @@ -330,13 +333,18 @@ static int jq_util_input_read_more(jq_util_input_state *state) { * terminating '\0'. This only works because we previously memset our * buffer with something nonzero. */ - for (i = sizeof(state->buf) - 1; i > 0; i--) { + for (i = max_gets_len - 1; i > 0; i--) { if (state->buf[i] == '\0') break; } state->buf_valid_len = i; } else if (p == NULL) { - state->buf_valid_len = sizeof(state->buf) - 1; + state->buf_valid_len = max_gets_len - 1; + char *end = state->buf + state->buf_valid_len; + int len = 0; + if (jvp_utf8_backtrack(end - 1, state->buf, &len) && len > 0) { + state->buf_valid_len += fread(end, 1, len, state->current_input); + } } else { state->buf_valid_len = (p - state->buf) + 1; } diff --git a/tests/utf8-truncate.jq b/tests/utf8-truncate.jq deleted file mode 100644 index a6be8637..00000000 --- a/tests/utf8-truncate.jq +++ /dev/null @@ -1,3 +0,0 @@ -def fill:"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"; -def e:"日本語"; -e == "日本語" diff --git a/tests/utf8test b/tests/utf8test index a84b258f..37d8043f 100755 --- a/tests/utf8test +++ b/tests/utf8test @@ -1,10 +1,24 @@ -#!/bin/sh +#!/bin/sh -x . "${0%/*}/setup" "$@" -if [ "$($VALGRIND $Q $JQ -nf $JQTESTDIR/utf8-truncate.jq)" != "true" ]; then - echo "UTF-8 byte sequences that span the jv_load_file read buffer are mangled" - exit 1 -fi +# Reading jq file does not corrupt multi-byte characters -- #1311 +for l in 1 2 3 4; do + $VALGRIND $Q $JQ -rn "[(range($l) | 48), (range(3000) | 128515)] | implode | + \"\\(tojson) | explode | unique == [48, 128515]\"" > $d/input + if ! $VALGRIND $Q $JQ -nef $d/input > /dev/null; then + echo "Reading jq file corrupted multi-byte characters" + exit 1 + fi +done + +# Slurping raw input does not corrupt multi-byte characters -- #3389 +for l in 1 2 3 4; do + $VALGRIND $Q $JQ -jn "[(range($l) | 48), (range(3000) | 128515)] | implode" > $d/input + if ! $VALGRIND $Q $JQ -Rse 'explode | unique == [48, 128515]' $d/input > /dev/null; then + echo "Slurping raw input corrupted multi-byte characters" + exit 1 + fi +done exit 0