Refactor @uri and @urid and fix multi-byte UTF-8 corruption (#3495)
This commit improves performance by refactoring `@uri` and `@urid` to pre-alloacte buffers and build the `jv` string at the end. Also, this fixes `@urid` to prevent multi-byte UTF-8 corruption, and to preserve embedded null bytes.
This commit is contained in:
itchyny authored and GitHub committed
1 parent
36b6ae00b1
commit
bfcf82fde1
2 files changed
+118
-54
No files matched your search
+51
-41
@@ -515,6 +515,18 @@ static jv f_utf8bytelength(jq_state *jq, jv input) {
|
||||
return jv_number(jv_string_length_bytes(input));
|
||||
}
|
||||
|
||||
static const unsigned char URI_UNRESERVED[128] = {
|
||||
// 1 2 3 4 5 6 7 8 9 A B C D E F
|
||||
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x00
|
||||
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x10
|
||||
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, // 0x20: - .
|
||||
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, // 0x30: 0-9
|
||||
0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 0x40: A-O
|
||||
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 1, // 0x50: P-Z _
|
||||
0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 0x60: a-o
|
||||
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 1, 0, // 0x70: p-z ~
|
||||
};
|
||||
|
||||
#define CHARS_ALPHANUM "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"
|
||||
|
||||
static const unsigned char BASE64_ENCODE_TABLE[64 + 1] = CHARS_ALPHANUM "+/";
|
||||
@@ -635,64 +647,62 @@ static jv f_format(jq_state *jq, jv input, jv fmt) {
|
||||
} else if (!strcmp(fmt_s, "uri")) {
|
||||
jv_free(fmt);
|
||||
input = f_tostring(jq, input);
|
||||
|
||||
int unreserved[128] = {0};
|
||||
const char* p = CHARS_ALPHANUM "-_.~";
|
||||
while (*p) unreserved[(int)*p++] = 1;
|
||||
|
||||
jv line = jv_string("");
|
||||
const char* s = jv_string_value(input);
|
||||
for (int i=0; i<jv_string_length_bytes(jv_copy(input)); i++) {
|
||||
unsigned ch = (unsigned)(unsigned char)*s;
|
||||
if (ch < 128 && unreserved[ch]) {
|
||||
line = jv_string_append_buf(line, s, 1);
|
||||
int len = jv_string_length_bytes(jv_copy(input));
|
||||
char *result = jv_mem_alloc((size_t)len * 3 + 1);
|
||||
uint32_t ri = 0;
|
||||
for (int i = 0; i < len; i++) {
|
||||
unsigned c = (unsigned)(unsigned char)s[i];
|
||||
if (c < 128 && URI_UNRESERVED[c]) {
|
||||
result[ri++] = (char)c;
|
||||
} else {
|
||||
line = jv_string_concat(line, jv_string_fmt("%%%02X", ch));
|
||||
result[ri++] = '%';
|
||||
result[ri++] = "0123456789ABCDEF"[c >> 4];
|
||||
result[ri++] = "0123456789ABCDEF"[c & 0x0F];
|
||||
}
|
||||
s++;
|
||||
}
|
||||
jv line = jv_string_sized(result, ri);
|
||||
free(result);
|
||||
jv_free(input);
|
||||
return line;
|
||||
} else if (!strcmp(fmt_s, "urid")) {
|
||||
jv_free(fmt);
|
||||
input = f_tostring(jq, input);
|
||||
|
||||
jv line = jv_string("");
|
||||
const char *errmsg = "is not a valid uri encoding";
|
||||
const char *errmsg = "is not a valid uri encoding";
|
||||
const char *s = jv_string_value(input);
|
||||
while (*s) {
|
||||
if (*s != '%') {
|
||||
line = jv_string_append_buf(line, s++, 1);
|
||||
int len = jv_string_length_bytes(jv_copy(input));
|
||||
char *result = jv_mem_alloc((size_t)len + 1);
|
||||
uint32_t ri = 0;
|
||||
for (int i = 0; i < len; i++) {
|
||||
char c = s[i];
|
||||
if (c != '%') {
|
||||
result[ri++] = c;
|
||||
} else {
|
||||
unsigned char unicode[4] = {0};
|
||||
int b = 0;
|
||||
// check leading bits of first octet to determine length of unicode character
|
||||
// (https://datatracker.ietf.org/doc/html/rfc3629#section-3)
|
||||
while (b == 0 || (b < 4 && unicode[0] >> 7 & 1 && unicode[0] >> (7-b) & 1)) {
|
||||
if (*(s++) != '%') {
|
||||
jv_free(line);
|
||||
int hi = 0, lo = 0;
|
||||
for (int j = 0; j < 2; j++) {
|
||||
if (++i >= len) {
|
||||
free(result);
|
||||
return type_error(input, errmsg);
|
||||
}
|
||||
for (int i=0; i<2; i++) {
|
||||
unicode[b] <<= 4;
|
||||
char c = *(s++);
|
||||
if ('0' <= c && c <= '9') unicode[b] |= c - '0';
|
||||
else if ('a' <= c && c <= 'f') unicode[b] |= c - 'a' + 10;
|
||||
else if ('A' <= c && c <= 'F') unicode[b] |= c - 'A' + 10;
|
||||
else {
|
||||
jv_free(line);
|
||||
return type_error(input, errmsg);
|
||||
}
|
||||
int *d = j == 0 ? &hi : &lo;
|
||||
c = s[i];
|
||||
if ('0' <= c && c <= '9') *d = c - '0';
|
||||
else if ('a' <= c && c <= 'f') *d = c - 'a' + 10;
|
||||
else if ('A' <= c && c <= 'F') *d = c - 'A' + 10;
|
||||
else {
|
||||
free(result);
|
||||
return type_error(input, errmsg);
|
||||
}
|
||||
b++;
|
||||
}
|
||||
if (!jvp_utf8_is_valid((const char *)unicode, (const char *)unicode+b)) {
|
||||
jv_free(line);
|
||||
return type_error(input, errmsg);
|
||||
}
|
||||
line = jv_string_append_buf(line, (const char *)unicode, b);
|
||||
result[ri++] = (hi << 4) | lo;
|
||||
}
|
||||
}
|
||||
if (!jvp_utf8_is_valid(result, result + ri)) {
|
||||
free(result);
|
||||
return type_error(input, errmsg);
|
||||
}
|
||||
jv line = jv_string_sized(result, ri);
|
||||
free(result);
|
||||
jv_free(input);
|
||||
return line;
|
||||
} else if (!strcmp(fmt_s, "sh")) {
|
||||
|
||||
+67
-13
@@ -1,38 +1,92 @@
|
||||
# Tests are groups of three lines: program, input, expected output
|
||||
# Blank lines and lines starting with # are ignored
|
||||
|
||||
# @uri
|
||||
|
||||
@uri
|
||||
"<>&'\"\t"
|
||||
"%3C%3E%26%27%22%09"
|
||||
""
|
||||
""
|
||||
|
||||
# decoding encoded output results in same text
|
||||
(@uri|@urid)
|
||||
"<>&'\"\t"
|
||||
"<>&'\"\t"
|
||||
@uri
|
||||
"\b\t\n\f\r\"\\"
|
||||
"%08%09%0A%0C%0D%22%5C"
|
||||
|
||||
@uri
|
||||
",-./09:;<=>?@AZ[\\]^_`az{|}~\u007f"
|
||||
"%2C-.%2F09%3A%3B%3C%3D%3E%3F%40AZ%5B%5C%5D%5E_%60az%7B%7C%7D~%7F"
|
||||
|
||||
# testing variable length unicode characters
|
||||
@uri
|
||||
"a \u03bc \u2230 \ud83d\ude0e"
|
||||
"a%20%CE%BC%20%E2%88%B0%20%F0%9F%98%8E"
|
||||
|
||||
@uri
|
||||
"a\u0000b\u0000c"
|
||||
"a%00b%00c"
|
||||
|
||||
# @urid
|
||||
|
||||
@urid
|
||||
""
|
||||
""
|
||||
|
||||
@urid
|
||||
"%08%09%0A%0C%0D%22%5C"
|
||||
"\b\t\n\f\r\"\\"
|
||||
|
||||
@urid
|
||||
"%2C-.%2F09%3A%3B%3C%3D%3E%3F%40AZ%5B%5C%5D%5E_%60az%7B%7C%7D~%7F"
|
||||
",-./09:;<=>?@AZ[\\]^_`az{|}~\u007f"
|
||||
|
||||
@urid
|
||||
"a%20%CE%BC%20%E2%88%B0%20%F0%9F%98%8E"
|
||||
"a \u03bc \u2230 \ud83d\ude0e"
|
||||
|
||||
### invalid uri strings
|
||||
@urid
|
||||
"a%00b%00c"
|
||||
"a\u0000b\u0000c"
|
||||
|
||||
# unicode character should be length 4 (not 3)
|
||||
@urid
|
||||
"hello world"
|
||||
"hello world"
|
||||
|
||||
@urid
|
||||
"Knäckebröd"
|
||||
"Knäckebröd"
|
||||
|
||||
@urid
|
||||
"%c3%a4b%c3%a7d%c3%ab"
|
||||
"äbçdë"
|
||||
|
||||
@urid
|
||||
"a\u0000b\u0000c"
|
||||
"a\u0000b\u0000c"
|
||||
|
||||
# '%' at end of string
|
||||
. | try @urid catch .
|
||||
"%F0%93%81"
|
||||
"string (\"%F0%93%81\") is not a valid uri encoding"
|
||||
"abc%"
|
||||
"string (\"abc%\") is not a valid uri encoding"
|
||||
|
||||
# '%' followed by only one hex digit
|
||||
. | try @urid catch .
|
||||
"abc%f"
|
||||
"string (\"abc%f\") is not a valid uri encoding"
|
||||
|
||||
# '%' followed by non-hex character
|
||||
. | try @urid catch .
|
||||
"abc%g"
|
||||
"string (\"abc%g\") is not a valid uri encoding"
|
||||
|
||||
# invalid hex value ('FX')
|
||||
. | try @urid catch .
|
||||
"%FX%9F%98%8E"
|
||||
"string (\"%FX%9F%98%8E\") is not a valid uri encoding"
|
||||
|
||||
# trailing utf-8 octets must be formatted like 10xxxxxx
|
||||
# 'C0' = 11000000 invalid
|
||||
# incomplete 4-byte UTF-8 sequence (only 3 bytes)
|
||||
. | try @urid catch .
|
||||
"%F0%93%81"
|
||||
"string (\"%F0%93%81\") is not a valid uri encoding"
|
||||
|
||||
# invalid continuation byte (0xC0 is not 10xxxxxx)
|
||||
. | try @urid catch .
|
||||
"%F0%C0%81%8E"
|
||||
"string (\"%F0%C0%81%8E\") is not a valid uri encoding"
|
||||
Reference in new issue
Block a user