navigine/jq
1
0 Fork 0
mirror of https://github.com/jqlang/jq.git synced 2026-10-26 20:16:33 +00:00
Code Issues Packages Projects Releases Wiki Activity

merging upstream stedolan changes

Browse Source
This commit is contained in:
Lee Thompson committed 2013-02-02 20:39:23 -06:00
2 parents dd70eeb29d 925ec3751f
commit c7725a8d4d
40 files changed
+2426 -951
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

.gitignore
+4 -4
View File
@@ -1,10 +1,10 @@
*.o
*~
# Test binaries
jv_test
jv_parse
parsertest*~
jq_test
build
jq
jq.1
# Something delightfully recursive happens otherwise
docs/content/2.download/source/*
AUTHORS
+1 -1
View File
@@ -3,4 +3,4 @@ Stephen Dolan <mu@netsoc.tcd.ie>
Contributions by
Anthony Shortland <anthony@dtosolutions.com> - rpmbuild target
Lee Thompson <thompson@dtosolutions.com> - autoconf stuff
Lee Thompson <stagr.lee@gmail.com> - autoconf stuff
Makefile.am
+34 -2
View File
@@ -10,9 +10,41 @@ EXTRA_DIST = setup.sh config.h.in ChangeLog VERSION lexer.l lexer.h gen_utf8_tab
docdir = ${datadir}/doc/${PACKAGE}
dist_doc_DATA = README.md INSTALL COPYING AUTHORS README NEWS
JQ_INCS = jq_parser.h builtin.h bytecode.h compile.h execute.h forkable_stack.h frame_layout.h jv.h jv_dtoa.h jv_parse.h jv_unicode.h locfile.h opcode.h opcode_list.h parser.y jv_utf8_tables.gen.h
JQ_INCS = \
jq_parser.h \
builtin.h \
bytecode.h \
compile.h \
execute.h \
forkable_stack.h \
frame_layout.h \
jv.h \
jv_alloc.h \
jv_aux.h \
jv_dtoa.h \
jv_parse.h \
jv_unicode.h \
locfile.h \
opcode.h \
opcode_list.h \
parser.y \
jv_utf8_tables.gen.h
JQ_SRC = opcode.c bytecode.c compile.c execute.c builtin.c jv.c jv_parse.c jv_print.c jv_dtoa.c jv_unicode.c lexer.c $(JQ_INCS)
JQ_SRC = \
opcode.c \
bytecode.c \
compile.c \
execute.c \
builtin.c \
jv.c \
jv_parse.c \
jv_print.c \
jv_dtoa.c \
jv_unicode.c \
jv_aux.c \
jv_alloc.c \
lexer.c \
$(JQ_INCS)
if DEVCFLAGS_ENABLED
DEVCFLAGS = -Wextra -Wall -Wno-missing-field-initializers -Wno-unused-parameter -std=gnu99 -ggdb -Wno-unused-function
builtin.c
+465 -248
View File
@@ -1,51 +1,89 @@
#include <string.h>
#include "builtin.h"
#include "bytecode.h"
#include "compile.h"
#include "jq_parser.h"
#include "locfile.h"
#include "jv_aux.h"
#include "jv_unicode.h"
enum {
CMP_OP_LESS,
CMP_OP_GREATER,
CMP_OP_LESSEQ,
CMP_OP_GREATEREQ
} _cmp_op;
static void f_plus(jv input[], jv output[]) {
jv_free(input[0]);
jv a = input[2];
jv b = input[1];
if (jv_get_kind(a) == JV_KIND_NUMBER && jv_get_kind(b) == JV_KIND_NUMBER) {
output[0] = jv_number(jv_number_value(a) +
jv_number_value(b));
} else if (jv_get_kind(a) == JV_KIND_STRING && jv_get_kind(b) == JV_KIND_STRING) {
output[0] = jv_string_concat(a, b);
} else if (jv_get_kind(a) == JV_KIND_ARRAY && jv_get_kind(b) == JV_KIND_ARRAY) {
output[0] = jv_array_concat(a, b);
} else if (jv_get_kind(a) == JV_KIND_OBJECT && jv_get_kind(b) == JV_KIND_OBJECT) {
output[0] = jv_object_merge(a, b);
} else {
output[0] = jv_invalid_with_msg(jv_string_fmt("Attempted to add %s and %s",
jv_kind_name(jv_get_kind(a)),
jv_kind_name(jv_get_kind(b))));
jv_free(a);
jv_free(b);
typedef jv (*func_1)(jv);
typedef jv (*func_2)(jv,jv);
typedef jv (*func_3)(jv,jv,jv);
typedef jv (*func_4)(jv,jv,jv,jv);
typedef jv (*func_5)(jv,jv,jv,jv,jv);
jv cfunction_invoke(struct cfunction* function, jv input[]) {
switch (function->nargs) {
case 1: return ((func_1)function->fptr)(input[0]);
case 2: return ((func_2)function->fptr)(input[0], input[1]);
case 3: return ((func_3)function->fptr)(input[0], input[1], input[2]);
case 4: return ((func_4)function->fptr)(input[0], input[1], input[2], input[3]);
case 5: return ((func_5)function->fptr)(input[0], input[1], input[2], input[3], input[4]);
default: return jv_invalid_with_msg(jv_string("Function takes too many arguments"));
}
}
static void f_minus(jv input[], jv output[]) {
jv_free(input[0]);
jv a = input[2];
jv b = input[1];
static jv type_error(jv bad, const char* msg) {
jv err = jv_invalid_with_msg(jv_string_fmt("%s %s",
jv_kind_name(jv_get_kind(bad)),
msg));
jv_free(bad);
return err;
}
static jv type_error2(jv bad1, jv bad2, const char* msg) {
jv err = jv_invalid_with_msg(jv_string_fmt("%s and %s %s",
jv_kind_name(jv_get_kind(bad1)),
jv_kind_name(jv_get_kind(bad2)),
msg));
jv_free(bad1);
jv_free(bad2);
return err;
}
static jv f_plus(jv input, jv a, jv b) {
jv_free(input);
if (jv_get_kind(a) == JV_KIND_NULL) {
jv_free(a);
return b;
} else if (jv_get_kind(b) == JV_KIND_NULL) {
jv_free(b);
return a;
} else if (jv_get_kind(a) == JV_KIND_NUMBER && jv_get_kind(b) == JV_KIND_NUMBER) {
return jv_number(jv_number_value(a) +
jv_number_value(b));
} else if (jv_get_kind(a) == JV_KIND_STRING && jv_get_kind(b) == JV_KIND_STRING) {
return jv_string_concat(a, b);
} else if (jv_get_kind(a) == JV_KIND_ARRAY && jv_get_kind(b) == JV_KIND_ARRAY) {
return jv_array_concat(a, b);
} else if (jv_get_kind(a) == JV_KIND_OBJECT && jv_get_kind(b) == JV_KIND_OBJECT) {
return jv_object_merge(a, b);
} else {
return type_error2(a, b, "cannot be added");
}
}
static jv f_negate(jv input) {
if (jv_get_kind(input) != JV_KIND_NUMBER) {
return type_error(input, "cannot be negated");
}
jv ret = jv_number(-jv_number_value(input));
jv_free(input);
return ret;
}
static jv f_minus(jv input, jv a, jv b) {
jv_free(input);
if (jv_get_kind(a) == JV_KIND_NUMBER && jv_get_kind(b) == JV_KIND_NUMBER) {
output[0] = jv_number(jv_number_value(a) - jv_number_value(b));
return jv_number(jv_number_value(a) - jv_number_value(b));
} else if (jv_get_kind(a) == JV_KIND_ARRAY && jv_get_kind(b) == JV_KIND_ARRAY) {
jv out = jv_array();
for (int i=0; i<jv_array_length(jv_copy(a)); i++) {
jv x = jv_array_get(jv_copy(a), i);
jv_array_foreach(a, i, x) {
int include = 1;
for (int j=0; j<jv_array_length(jv_copy(b)); j++) {
if (jv_equal(jv_copy(x), jv_array_get(jv_copy(b), j))) {
jv_array_foreach(b, j, y) {
if (jv_equal(jv_copy(x), y)) {
include = 0;
break;
}
@@ -56,279 +94,462 @@ static void f_minus(jv input[], jv output[]) {
}
jv_free(a);
jv_free(b);
output[0] = out;
return out;
} else {
output[0] = jv_invalid_with_msg(jv_string_fmt("Attempted to subtract %s and %s",
jv_kind_name(jv_get_kind(a)),
jv_kind_name(jv_get_kind(b))));
jv_free(a);
jv_free(b);
return type_error2(a, b, "cannot be subtracted");
}
}
static void f_multiply(jv input[], jv output[]) {
jv_free(input[0]);
jv a = input[2];
jv b = input[1];
static jv f_multiply(jv input, jv a, jv b) {
jv_free(input);
if (jv_get_kind(a) == JV_KIND_NUMBER && jv_get_kind(b) == JV_KIND_NUMBER) {
output[0] = jv_number(jv_number_value(a) * jv_number_value(b));
return jv_number(jv_number_value(a) * jv_number_value(b));
} else {
output[0] = jv_invalid_with_msg(jv_string_fmt("Attempted to multiply %s and %s",
jv_kind_name(jv_get_kind(a)),
jv_kind_name(jv_get_kind(b))));
jv_free(a);
jv_free(b);
return type_error2(a, b, "cannot be multiplied");
}
}
static void f_divide(jv input[], jv output[]) {
jv_free(input[0]);
jv a = input[2];
jv b = input[1];
static jv f_divide(jv input, jv a, jv b) {
jv_free(input);
if (jv_get_kind(a) == JV_KIND_NUMBER && jv_get_kind(b) == JV_KIND_NUMBER) {
output[0] = jv_number(jv_number_value(a) / jv_number_value(b));
return jv_number(jv_number_value(a) / jv_number_value(b));
} else {
output[0] = jv_invalid_with_msg(jv_string_fmt("Attempted to divide %s by %s",
jv_kind_name(jv_get_kind(a)),
jv_kind_name(jv_get_kind(b))));
jv_free(a);
jv_free(b);
return type_error2(a, b, "cannot be divided");
}
}
static void f_add(jv input[], jv output[]) {
jv array = input[0];
if (jv_get_kind(array) != JV_KIND_ARRAY) {
output[0] = jv_invalid_with_msg(jv_string_fmt("Cannot add elements of an %s",
jv_kind_name(jv_get_kind(array))));
} else if (jv_array_length(jv_copy(array)) == 0) {
output[0] = jv_null();
static jv f_equal(jv input, jv a, jv b) {
jv_free(input);
return jv_bool(jv_equal(a, b));
}
static jv f_notequal(jv input, jv a, jv b) {
jv_free(input);
return jv_bool(!jv_equal(a, b));
}
enum cmp_op {
CMP_OP_LESS,
CMP_OP_GREATER,
CMP_OP_LESSEQ,
CMP_OP_GREATEREQ
};
static jv order_cmp(jv input, jv a, jv b, enum cmp_op op) {
jv_free(input);
int r = jv_cmp(a, b);
return jv_bool((op == CMP_OP_LESS && r < 0) ||
(op == CMP_OP_LESSEQ && r <= 0) ||
(op == CMP_OP_GREATEREQ && r >= 0) ||
(op == CMP_OP_GREATER && r > 0));
}
static jv f_less(jv input, jv a, jv b) {
return order_cmp(input, a, b, CMP_OP_LESS);
}
static jv f_greater(jv input, jv a, jv b) {
return order_cmp(input, a, b, CMP_OP_GREATER);
}
static jv f_lesseq(jv input, jv a, jv b) {
return order_cmp(input, a, b, CMP_OP_LESSEQ);
}
static jv f_greatereq(jv input, jv a, jv b) {
return order_cmp(input, a, b, CMP_OP_GREATEREQ);
}
static jv f_contains(jv a, jv b) {
if (jv_get_kind(a) == jv_get_kind(b)) {
return jv_bool(jv_contains(a, b));
} else {
jv sum = jv_array_get(jv_copy(array), 0);
for (int i = 1; i < jv_array_length(jv_copy(array)); i++) {
jv x = jv_array_get(jv_copy(array), i);
jv add_args[] = {jv_null(), x, sum};
f_plus(add_args, &sum);
return type_error2(a, b, "cannot have their containment checked");
}
}
static jv f_tonumber(jv input) {
if (jv_get_kind(input) == JV_KIND_NUMBER) {
return input;
}
if (jv_get_kind(input) == JV_KIND_STRING) {
jv parsed = jv_parse(jv_string_value(input));
if (!jv_is_valid(parsed) || jv_get_kind(parsed) == JV_KIND_NUMBER) {
jv_free(input);
return parsed;
}
output[0] = sum;
}
jv_free(array);
return type_error(input, "cannot be parsed as a number");
}
static void f_equal(jv input[], jv output[]) {
jv_free(input[0]);
output[0] = jv_bool(jv_equal(input[2], input[1]));
}
static void f_notequal(jv input[], jv output[]) {
jv_free(input[0]);
output[0] = jv_bool(jv_equal(input[2], input[1]) == 0);
}
static void order_cmp(jv input[], jv output[], int op) {
jv_free(input[0]);
jv a = input[2];
jv b = input[1];
if (jv_get_kind(a) == JV_KIND_NUMBER && jv_get_kind(b) == JV_KIND_NUMBER) {
double da = jv_number_value(a);
double db = jv_number_value(b);
output[0] = jv_bool((op == CMP_OP_LESS && da < db) ||
(op == CMP_OP_LESSEQ && da <= db) ||
(op == CMP_OP_GREATEREQ && da >= db) ||
(op == CMP_OP_GREATER && da > db));
static jv f_length(jv input) {
if (jv_get_kind(input) == JV_KIND_ARRAY) {
return jv_number(jv_array_length(input));
} else if (jv_get_kind(input) == JV_KIND_OBJECT) {
return jv_number(jv_object_length(input));
} else if (jv_get_kind(input) == JV_KIND_STRING) {
return jv_number(jv_string_length(input));
} else if (jv_get_kind(input) == JV_KIND_NULL) {
jv_free(input);
return jv_number(0);
} else {
output[0] = jv_invalid_with_msg(jv_string_fmt("Attempted to compare order of %s wrt %s",
jv_kind_name(jv_get_kind(a)),
jv_kind_name(jv_get_kind(b))));
jv_free(a);
jv_free(b);
return type_error(input, "has no length");
}
}
static void f_less(jv input[], jv output[]) {
order_cmp(input, output, CMP_OP_LESS);
}
static void f_greater(jv input[], jv output[]) {
order_cmp(input, output, CMP_OP_GREATER);
}
static void f_lesseq(jv input[], jv output[]) {
order_cmp(input, output, CMP_OP_LESSEQ);
}
static void f_greatereq(jv input[], jv output[]) {
order_cmp(input, output, CMP_OP_GREATEREQ);
}
static void f_contains(jv input[], jv output[]) {
jv_free(input[0]);
jv a = input[2];
jv b = input[1];
jv_kind akind = jv_get_kind(a);
if (akind == jv_get_kind(b)) {
output[0] = jv_bool(jv_contains(a, b));
} else {
output[0] = jv_invalid_with_msg(jv_string_fmt("Can only check containment of values of the same type."));
jv_free(a);
jv_free(b);
static jv f_tostring(jv input) {
if (jv_get_kind(input) == JV_KIND_STRING) {
return input;
} else {
return jv_dump_string(input, 0);
}
}
static void f_tonumber(jv input[], jv output[]) {
if (jv_get_kind(input[0]) == JV_KIND_NUMBER) {
output[0] = input[0];
} else if (jv_get_kind(input[0]) == JV_KIND_STRING) {
jv parsed = jv_parse(jv_string_value(input[0]));
if (!jv_is_valid(parsed)) {
jv_free(input[0]);
output[0] = parsed;
} else if (jv_get_kind(parsed) != JV_KIND_NUMBER) {
output[0] = jv_invalid_with_msg(jv_string_fmt("'%s' is not a number",
jv_string_value(input[0])));
jv_free(input[0]);
#define CHARS_ALPHANUM "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"
static jv escape_string(jv input, const char* escapings) {
assert(jv_get_kind(input) == JV_KIND_STRING);
const char* lookup[128] = {0};
const char* p = escapings;
while (*p) {
lookup[(int)*p] = p+1;
p++;
p += strlen(p);
p++;
}
jv ret = jv_string("");
const char* i = jv_string_value(input);
const char* end = i + jv_string_length(jv_copy(input));
const char* cstart;
int c = 0;
while ((i = jvp_utf8_next((cstart = i), end, &c))) {
assert(c != -1);
if (c < 128 && lookup[c]) {
ret = jv_string_append_str(ret, lookup[c]);
} else {
jv_free(input[0]);
output[0] = parsed;
ret = jv_string_append_buf(ret, cstart, i - cstart);
}
}
jv_free(input);
return ret;
}
static jv f_format(jv input, jv fmt) {
if (jv_get_kind(fmt) != JV_KIND_STRING) {
jv_free(input);
return type_error(fmt, "is not a valid format");
}
const char* fmt_s = jv_string_value(fmt);
if (!strcmp(fmt_s, "json")) {
jv_free(fmt);
return jv_dump_string(input, 0);
} else if (!strcmp(fmt_s, "text")) {
jv_free(fmt);
return f_tostring(input);
} else if (!strcmp(fmt_s, "csv")) {
jv_free(fmt);
if (jv_get_kind(input) != JV_KIND_ARRAY)
return type_error(input, "cannot be csv-formatted, only array");
jv line = jv_string("");
jv_array_foreach(input, i, x) {
if (i) line = jv_string_append_str(line, ",");
switch (jv_get_kind(x)) {
case JV_KIND_NULL:
/* null rendered as empty string */
jv_free(x);
break;
case JV_KIND_TRUE:
case JV_KIND_FALSE:
line = jv_string_concat(line, jv_dump_string(x, 0));
break;
case JV_KIND_NUMBER:
if (jv_number_value(x) != jv_number_value(x)) {
/* NaN, render as empty string */
jv_free(x);
} else {
line = jv_string_concat(line, jv_dump_string(x, 0));
}
break;
case JV_KIND_STRING: {
line = jv_string_append_str(line, "\"");
line = jv_string_concat(line, escape_string(x, "\"\"\"\0"));
line = jv_string_append_str(line, "\"");
break;
}
default:
jv_free(input);
jv_free(line);
return type_error(x, "is not valid in a csv row");
}
}
jv_free(input);
return line;
} else if (!strcmp(fmt_s, "html")) {
jv_free(fmt);
return escape_string(f_tostring(input), "&&amp;\0<&lt;\0>&gt;\0'&apos;\0\"&quot;\0");
} else if (!strcmp(fmt_s, "uri")) {
jv_free(fmt);
input = f_tostring(input);
int unreserved[128] = {0};
const char* p = CHARS_ALPHANUM "-_.!~*'()";
while (*p) unreserved[(int)*p++] = 1;
jv line = jv_string("");
const char* s = jv_string_value(input);
for (int i=0; i<jv_string_length(jv_copy(input)); i++) {
unsigned ch = (unsigned)(unsigned char)*s;
if (ch < 128 && unreserved[ch]) {
line = jv_string_append_buf(line, s, 1);
} else {
line = jv_string_concat(line, jv_string_fmt("%%%02x", ch));
}
s++;
}
jv_free(input);
return line;
} else if (!strcmp(fmt_s, "sh")) {
jv_free(fmt);
if (jv_get_kind(input) != JV_KIND_ARRAY)
input = jv_array_set(jv_array(), 0, input);
jv line = jv_string("");
jv_array_foreach(input, i, x) {
if (i) line = jv_string_append_str(line, " ");
switch (jv_get_kind(x)) {
case JV_KIND_NULL:
case JV_KIND_TRUE:
case JV_KIND_FALSE:
case JV_KIND_NUMBER:
line = jv_string_concat(line, jv_dump_string(x, 0));
break;
case JV_KIND_STRING: {
line = jv_string_append_str(line, "'");
line = jv_string_concat(line, escape_string(x, "''\\''\0"));
line = jv_string_append_str(line, "'");
break;
}
default:
jv_free(input);
jv_free(line);
return type_error(x, "can not be escaped for shell");
}
}
jv_free(input);
return line;
} else if (!strcmp(fmt_s, "base64")) {
jv_free(fmt);
input = f_tostring(input);
jv line = jv_string("");
const char b64[64 + 1] = CHARS_ALPHANUM "+/";
const char* data = jv_string_value(input);
int len = jv_string_length(jv_copy(input));
for (int i=0; i<len; i+=3) {
uint32_t code = 0;
int n = len - i >= 3 ? 3 : len-i;
for (int j=0; j<3; j++) {
code <<= 8;
code |= j < n ? (unsigned)data[i+j] : 0;
}
char buf[4];
for (int j=0; j<4; j++) {
buf[j] = b64[(code >> (18 - j*6)) & 0x3f];
}
if (n < 3) buf[3] = '=';
if (n < 2) buf[2] = '=';
line = jv_string_append_buf(line, buf, sizeof(buf));
}
jv_free(input);
return line;
} else {
output[0] = jv_invalid_with_msg(jv_string_fmt("Cannot parse %s as a number",
jv_kind_name(jv_get_kind(input[0]))));
jv_free(input[0]);
return;
jv_free(input);
return jv_invalid_with_msg(jv_string_concat(fmt, jv_string(" is not a valid format")));
}
}
static void f_length(jv input[], jv output[]) {
if (jv_get_kind(input[0]) == JV_KIND_ARRAY) {
output[0] = jv_number(jv_array_length(input[0]));
} else if (jv_get_kind(input[0]) == JV_KIND_OBJECT) {
output[0] = jv_number(jv_object_length(input[0]));
} else if (jv_get_kind(input[0]) == JV_KIND_STRING) {
output[0] = jv_number(jv_string_length(input[0]));
static jv f_keys(jv input) {
if (jv_get_kind(input) == JV_KIND_OBJECT || jv_get_kind(input) == JV_KIND_ARRAY) {
return jv_keys(input);
} else {
output[0] = jv_invalid_with_msg(jv_string_fmt("Cannot get the length of %s",
jv_kind_name(jv_get_kind(input[0]))));
jv_free(input[0]);
return type_error(input, "has no keys");
}
}
static void f_tostring(jv input[], jv output[]) {
if (jv_get_kind(input[0]) == JV_KIND_STRING) {
output[0] = input[0];
static jv f_sort(jv input){
if (jv_get_kind(input) == JV_KIND_ARRAY) {
return jv_sort(input, jv_copy(input));
} else {
output[0] = jv_dump_string(input[0], 0);
return type_error(input, "cannot be sorted, as it is not an array");
}
}
static int string_cmp(const void* pa, const void* pb){
const jv* a = pa;
const jv* b = pb;
int lena = jv_string_length(jv_copy(*a));
int lenb = jv_string_length(jv_copy(*b));
int minlen = lena < lenb ? lena : lenb;
int r = memcmp(jv_string_value(*a), jv_string_value(*b), minlen);
if (r == 0) r = lena - lenb;
return r;
}
static void f_keys(jv input[], jv output[]) {
if (jv_get_kind(input[0]) == JV_KIND_OBJECT) {
int nkeys = jv_object_length(jv_copy(input[0]));
jv* keys = malloc(sizeof(jv) * nkeys);
int kidx = 0;
jv_object_foreach(i, input[0]) {
keys[kidx++] = jv_object_iter_key(input[0], i);
}
qsort(keys, nkeys, sizeof(jv), string_cmp);
output[0] = jv_array_sized(nkeys);
for (int i = 0; i<nkeys; i++) {
output[0] = jv_array_append(output[0], keys[i]);
}
free(keys);
jv_free(input[0]);
} else if (jv_get_kind(input[0]) == JV_KIND_ARRAY) {
int n = jv_array_length(input[0]);
output[0] = jv_array();
for (int i=0; i<n; i++){
output[0] = jv_array_set(output[0], i, jv_number(i));
}
static jv f_sort_by_impl(jv input, jv keys) {
if (jv_get_kind(input) == JV_KIND_ARRAY &&
jv_get_kind(keys) == JV_KIND_ARRAY &&
jv_array_length(jv_copy(input)) == jv_array_length(jv_copy(keys))) {
return jv_sort(input, keys);
} else {
output[0] = jv_invalid_with_msg(jv_string_fmt("'keys' only supports object, not %s",
jv_kind_name(jv_get_kind(input[0]))));
jv_free(input[0]);
return type_error2(input, keys, "cannot be sorted, as they are not both arrays");
}
}
static void f_type(jv input[], jv output[]) {
output[0] = jv_string(jv_kind_name(jv_get_kind(input[0])));
jv_free(input[0]);
static jv f_group_by_impl(jv input, jv keys) {
if (jv_get_kind(input) == JV_KIND_ARRAY &&
jv_get_kind(keys) == JV_KIND_ARRAY &&
jv_array_length(jv_copy(input)) == jv_array_length(jv_copy(keys))) {
return jv_group(input, keys);
} else {
return type_error2(input, keys, "cannot be sorted, as they are not both arrays");
}
}
static block j_empty() {
return gen_op_block_defn(CLOSURE_CREATE, "empty", gen_op_simple(BACKTRACK));
static jv minmax_by(jv values, jv keys, int is_min) {
if (jv_get_kind(values) != JV_KIND_ARRAY)
return type_error2(values, keys, "cannot be iterated over");
if (jv_get_kind(keys) != JV_KIND_ARRAY)
return type_error2(values, keys, "cannot be iterated over");
if (jv_array_length(jv_copy(values)) != jv_array_length(jv_copy(keys)))
return type_error2(values, keys, "have wrong length");
if (jv_array_length(jv_copy(values)) == 0) {
jv_free(values);
jv_free(keys);
return jv_null();
}
jv ret = jv_array_get(jv_copy(values), 0);
jv retkey = jv_array_get(jv_copy(keys), 0);
for (int i=1; i<jv_array_length(jv_copy(values)); i++) {
jv item = jv_array_get(jv_copy(keys), i);
int cmp = jv_cmp(jv_copy(item), jv_copy(retkey));
if ((cmp < 0) == (is_min == 1)) {
jv_free(retkey);
retkey = item;
jv_free(ret);
ret = jv_array_get(jv_copy(values), i);
} else {
jv_free(item);
}
}
jv_free(values);
jv_free(keys);
jv_free(retkey);
return ret;
}
static block j_false() {
return gen_op_block_defn(CLOSURE_CREATE, "false",
block_join(gen_op_const(LOADK, jv_false()),
gen_op_simple(RET)));
static jv f_min(jv x) {
return minmax_by(x, jv_copy(x), 1);
}
static block j_true() {
return gen_op_block_defn(CLOSURE_CREATE, "true",
block_join(gen_op_const(LOADK, jv_true()),
gen_op_simple(RET)));
static jv f_max(jv x) {
return minmax_by(x, jv_copy(x), 0);
}
static block j_null() {
return gen_op_block_defn(CLOSURE_CREATE, "null",
block_join(gen_op_const(LOADK, jv_null()),
gen_op_simple(RET)));
static jv f_min_by_impl(jv x, jv y) {
return minmax_by(x, y, 1);
}
static block j_not() {
return gen_op_block_defn(CLOSURE_CREATE, "not",
block_join(gen_condbranch(gen_op_const(LOADK, jv_false()),
gen_op_const(LOADK, jv_true())),
gen_op_simple(RET)));
static jv f_max_by_impl(jv x, jv y) {
return minmax_by(x, y, 0);
}
static jv f_type(jv input) {
jv out = jv_string(jv_kind_name(jv_get_kind(input)));
jv_free(input);
return out;
}
static jv f_error(jv input, jv msg) {
jv_free(input);
msg = f_tostring(msg);
return jv_invalid_with_msg(msg);
}
static struct cfunction function_list[] = {
{f_plus, "_plus", CALL_BUILTIN_3_1},
{f_minus, "_minus", CALL_BUILTIN_3_1},
{f_multiply, "_multiply", CALL_BUILTIN_3_1},
{f_divide, "_divide", CALL_BUILTIN_3_1},
{f_tonumber, "tonumber", CALL_BUILTIN_1_1},
{f_tostring, "tostring", CALL_BUILTIN_1_1},
{f_keys, "keys", CALL_BUILTIN_1_1},
{f_equal, "_equal", CALL_BUILTIN_3_1},
{f_notequal, "_notequal", CALL_BUILTIN_3_1},
{f_less, "_less", CALL_BUILTIN_3_1},
{f_greater, "_greater", CALL_BUILTIN_3_1},
{f_lesseq, "_lesseq", CALL_BUILTIN_3_1},
{f_greatereq, "_greatereq", CALL_BUILTIN_3_1},
{f_contains, "_contains", CALL_BUILTIN_3_1},
{f_length, "length", CALL_BUILTIN_1_1},
{f_type, "type", CALL_BUILTIN_1_1},
{f_add, "add", CALL_BUILTIN_1_1},
{(cfunction_ptr)f_plus, "_plus", 3},
{(cfunction_ptr)f_negate, "_negate", 1},
{(cfunction_ptr)f_minus, "_minus", 3},
{(cfunction_ptr)f_multiply, "_multiply", 3},
{(cfunction_ptr)f_divide, "_divide", 3},
{(cfunction_ptr)f_tonumber, "tonumber", 1},
{(cfunction_ptr)f_tostring, "tostring", 1},
{(cfunction_ptr)f_keys, "keys", 1},
{(cfunction_ptr)jv_setpath, "setpath", 3}, // FIXME typechecking
{(cfunction_ptr)jv_getpath, "getpath", 2},
{(cfunction_ptr)jv_delpaths, "delpaths", 2},
{(cfunction_ptr)f_equal, "_equal", 3},
{(cfunction_ptr)f_notequal, "_notequal", 3},
{(cfunction_ptr)f_less, "_less", 3},
{(cfunction_ptr)f_greater, "_greater", 3},
{(cfunction_ptr)f_lesseq, "_lesseq", 3},
{(cfunction_ptr)f_greatereq, "_greatereq", 3},
{(cfunction_ptr)f_contains, "contains", 2},
{(cfunction_ptr)f_length, "length", 1},
{(cfunction_ptr)f_type, "type", 1},
{(cfunction_ptr)f_sort, "sort", 1},
{(cfunction_ptr)f_sort_by_impl, "_sort_by_impl", 2},
{(cfunction_ptr)f_group_by_impl, "_group_by_impl", 2},
{(cfunction_ptr)f_min, "min", 1},
{(cfunction_ptr)f_max, "max", 1},
{(cfunction_ptr)f_min_by_impl, "_min_by_impl", 2},
{(cfunction_ptr)f_max_by_impl, "_max_by_impl", 2},
{(cfunction_ptr)f_error, "error", 2},
{(cfunction_ptr)f_format, "format", 2},
};
static struct symbol_table cbuiltins = {function_list, sizeof(function_list)/sizeof(function_list[0])};
static struct symbol_table cbuiltins =
{function_list, sizeof(function_list)/sizeof(function_list[0])};
typedef block (*bytecoded_builtin)();
static bytecoded_builtin bytecoded_builtins[] = {
j_empty,
j_false,
j_true,
j_null,
j_not,
};
struct bytecoded_builtin { const char* name; block code; };
static block bind_bytecoded_builtins(block b) {
block builtins = gen_noop();
{
struct bytecoded_builtin builtin_defs[] = {
{"empty", gen_op_simple(BACKTRACK)},
{"false", gen_const(jv_false())},
{"true", gen_const(jv_true())},
{"null", gen_const(jv_null())},
{"not", gen_condbranch(gen_const(jv_false()),
gen_const(jv_true()))}
};
for (unsigned i=0; i<sizeof(builtin_defs)/sizeof(builtin_defs[0]); i++) {
builtins = BLOCK(builtins, gen_function(builtin_defs[i].name, gen_noop(),
builtin_defs[i].code));
}
}
{
struct bytecoded_builtin builtin_def_1arg[] = {
{"path", BLOCK(gen_op_simple(DUP),
gen_call("arg", gen_noop()),
gen_op_simple(GETPATH))},
};
for (unsigned i=0; i<sizeof(builtin_def_1arg)/sizeof(builtin_def_1arg[0]); i++) {
builtins = BLOCK(builtins, gen_function(builtin_def_1arg[i].name,
gen_op_block_unbound(CLOSURE_PARAM, "arg"),
builtin_def_1arg[i].code));
}
}
return block_bind(builtins, b, OP_IS_CALL_PSEUDO);
}
static const char* jq_builtins[] = {
"def map(f): [.[] | f];",
"def select(f): if f then . else empty end;",
"def sort_by(f): _sort_by_impl(map([f]));",
"def group_by(f): _group_by_impl(map([f]));",
"def unique: group_by(.) | map(.[0]);",
"def max_by(f): _max_by_impl(map([f]));",
"def min_by(f): _min_by_impl(map([f]));",
"def add: fold null as $sum (.[] | $sum + .);",
"def del(f): delpaths([path(f)]);",
"def _assign(paths; value): value as $v | fold . as $obj (path(paths) as $p | $obj | setpath($p; $v));",
"def _modify(paths; update): fold . as $obj (path(paths) as $p | $obj | setpath($p; getpath($p) | update));",
"def recurse(f): ., (f | select(. != null) | recurse(f));",
};
@@ -342,10 +563,6 @@ block builtins_bind(block b) {
b = block_bind(funcs, b, OP_IS_CALL_PSEUDO);
locfile_free(&src);
}
block builtins = gen_noop();
for (unsigned i=0; i<sizeof(bytecoded_builtins)/sizeof(bytecoded_builtins[0]); i++) {
block_append(&builtins, bytecoded_builtins[i]());
}
b = block_bind(builtins, b, OP_IS_CALL_PSEUDO);
b = bind_bytecoded_builtins(b);
return gen_cbinding(&cbuiltins, b);
}
builtin.h
+13 -1
View File
@@ -1,9 +1,21 @@
#ifndef BUILTIN_H
#define BUILTIN_H
#include "bytecode.h"
#include "compile.h"
block builtins_bind(block);
typedef void (*cfunction_ptr)(void);
struct cfunction {
cfunction_ptr fptr;
const char* name;
int nargs;
};
jv cfunction_invoke(struct cfunction* function, jv input[]);
#endif
bytecode.c
+12 -11
View File
@@ -4,13 +4,14 @@
#include "bytecode.h"
#include "opcode.h"
#include "jv_alloc.h"
static int bytecode_operation_length(uint16_t* codeptr) {
if (opcode_describe(*codeptr)->flags & OP_HAS_VARIABLE_LENGTH_ARGLIST) {
return 2 + codeptr[1] * 2;
} else {
return opcode_length(*codeptr);
int length = opcode_describe(*codeptr)->length;
if (*codeptr == CALL_JQ) {
length += codeptr[1] * 2;
}
return length;
}
void dump_disassembly(int indent, struct bytecode* bc) {
@@ -41,8 +42,8 @@ void dump_operation(struct bytecode* bc, uint16_t* codeptr) {
printf("%s", op->name);
if (op->length > 1) {
uint16_t imm = bc->code[pc++];
if (op->flags & OP_HAS_VARIABLE_LENGTH_ARGLIST) {
for (int i=0; i<imm; i++) {
if (op->op == CALL_JQ) {
for (int i=0; i<imm+1; i++) {
uint16_t level = bc->code[pc++];
uint16_t idx = bc->code[pc++];
if (idx & ARG_NEWCLOSURE) {
@@ -72,17 +73,17 @@ void dump_operation(struct bytecode* bc, uint16_t* codeptr) {
}
void symbol_table_free(struct symbol_table* syms) {
free(syms->cfunctions);
free(syms);
jv_mem_free(syms->cfunctions);
jv_mem_free(syms);
}
void bytecode_free(struct bytecode* bc) {
free(bc->code);
jv_mem_free(bc->code);
jv_free(bc->constants);
for (int i=0; i<bc->nsubfunctions; i++)
bytecode_free(bc->subfunctions[i]);
if (!bc->parent)
symbol_table_free(bc->globals);
free(bc->subfunctions);
free(bc);
jv_mem_free(bc->subfunctions);
jv_mem_free(bc);
}
bytecode.h
+1 -8
View File
@@ -4,14 +4,7 @@
#include "jv.h"
#include "opcode.h"
typedef void (*cfunction_ptr)(jv input[], jv output[]);
struct cfunction {
cfunction_ptr fptr;
const char* name;
opcode callop;
};
#include "builtin.h"
#define MAX_CFUNCTION_ARGS 10
struct symbol_table {
compile.c
+273 -246
View File
@@ -1,17 +1,32 @@
#define _GNU_SOURCE // for strdup
#include <assert.h>
#include <string.h>
#include <stdlib.h>
#include "opcode.h"
#include "compile.h"
#include "bytecode.h"
#include "locfile.h"
#include "jv_alloc.h"
/*
The intermediate representation for jq filters is as a sequence of
struct inst, which form a doubly-linked list via the next and prev
pointers.
A "block" represents a sequence of "struct inst", which may be
empty.
Blocks are generated by the parser bottom-up, so may have free
variables (refer to things not defined). See inst.bound_by and
inst.symbol.
*/
struct inst {
struct inst* next;
struct inst* prev;
opcode op;
union {
struct {
uint16_t intval;
struct inst* target;
jv constant;
@@ -21,16 +36,20 @@ struct inst {
location source;
// Binding
// An instruction requiring binding (for parameters/variables)
// An instruction requiring binding (for parameters/variables/functions)
// is in one of three states:
// bound_by = NULL - Unbound free variable
// bound_by = self - This instruction binds a variable
// bound_by = other - Uses variable bound by other instruction
// The immediate field is generally not meaningful until instructions
// are bound, and even then only for instructions which bind.
// inst->bound_by = NULL - Unbound free variable
// inst->bound_by = inst - This instruction binds a variable
// inst->bound_by = other - Uses variable bound by other instruction
// Unbound instructions (references to other things that may or may not
// exist) are created by "gen_foo_unbound", and bindings are created by
// block_bind(definition, body), which binds all instructions in
// body which are unboudn and refer to "definition" by name.
struct inst* bound_by;
char* symbol;
block subfn;
block subfn; // used by CLOSURE_CREATE (body of function)
block arglist; // used by CLOSURE_CREATE (formals) and CALL_JQ (arguments)
// This instruction is compiled as part of which function?
// (only used during block_compile)
@@ -40,26 +59,26 @@ struct inst {
};
static inst* inst_new(opcode op) {
inst* i = malloc(sizeof(inst));
inst* i = jv_mem_alloc(sizeof(inst));
i->next = i->prev = 0;
i->op = op;
i->bytecode_pos = -1;
i->bound_by = 0;
i->symbol = 0;
i->subfn = gen_noop();
i->arglist = gen_noop();
i->source = UNKNOWN_LOCATION;
return i;
}
static void inst_free(struct inst* i) {
free(i->symbol);
if (opcode_describe(i->op)->flags & OP_HAS_BLOCK) {
block_free(i->subfn);
}
jv_mem_free(i->symbol);
block_free(i->subfn);
block_free(i->arglist);
if (opcode_describe(i->op)->flags & OP_HAS_CONSTANT) {
jv_free(i->imm.constant);
}
free(i);
jv_mem_free(i);
}
static block inst_block(inst* i) {
@@ -106,9 +125,9 @@ block gen_op_simple(opcode op) {
}
block gen_op_const(opcode op, jv constant) {
assert(opcode_describe(op)->flags & OP_HAS_CONSTANT);
inst* i = inst_new(op);
block gen_const(jv constant) {
assert(opcode_describe(LOADK)->flags & OP_HAS_CONSTANT);
inst* i = inst_new(LOADK);
i->imm.constant = constant;
return inst_block(i);
}
@@ -148,29 +167,6 @@ block gen_op_var_bound(opcode op, block binder) {
return b;
}
block gen_op_symbol(opcode op, const char* sym) {
assert(opcode_describe(op)->flags & OP_HAS_SYMBOL);
inst* i = inst_new(op);
i->symbol = strdup(sym);
return inst_block(i);
}
block gen_op_block_defn(opcode op, const char* name, block block) {
assert(opcode_describe(op)->flags & OP_IS_CALL_PSEUDO);
assert(opcode_describe(op)->flags & OP_HAS_BLOCK);
inst* i = inst_new(op);
i->subfn = block;
i->symbol = strdup(name);
return inst_block(i);
}
static void block_bind_subblock(block binder, block body, int bindflags);
block gen_op_block_defn_rec(opcode op, const char* name, block blk) {
block b = gen_op_block_defn(op, name, blk);
block_bind_subblock(b, b, OP_IS_CALL_PSEUDO | OP_HAS_BINDING);
return b;
}
block gen_op_block_unbound(opcode op, const char* name) {
assert(opcode_describe(op)->flags & OP_IS_CALL_PSEUDO);
inst* i = inst_new(op);
@@ -185,33 +181,6 @@ block gen_op_block_bound(opcode op, block binder) {
return b;
}
block gen_op_call(opcode op, block arglist) {
assert(opcode_describe(op)->flags & OP_HAS_VARIABLE_LENGTH_ARGLIST);
inst* i = inst_new(op);
block prelude = gen_noop();
block call = inst_block(i);
int nargs = 0;
inst* curr = 0;
while ((curr = block_take(&arglist))) {
assert(opcode_describe(curr->op)->flags & OP_IS_CALL_PSEUDO);
block bcurr = inst_block(curr);
switch (curr->op) {
default: assert(0 && "Unknown type of parameter"); break;
case CLOSURE_REF:
block_append(&call, bcurr);
break;
case CLOSURE_CREATE:
block_append(&prelude, bcurr);
block_append(&call, gen_op_block_bound(CLOSURE_REF, bcurr));
break;
}
nargs++;
}
assert(nargs < 100); //FIXME
i->imm.intval = nargs;
return block_join(prelude, call);
}
static void inst_join(inst* a, inst* b) {
assert(a && b);
assert(!a->next);
@@ -262,168 +231,150 @@ static void block_bind_subblock(block binder, block body, int bindflags) {
// bind this instruction
i->bound_by = binder.first;
}
if (flags & OP_HAS_BLOCK) {
// binding recurses into closures
block_bind_subblock(binder, i->subfn, bindflags);
}
// binding recurses into closures
block_bind_subblock(binder, i->subfn, bindflags);
// binding recurses into argument list
block_bind_subblock(binder, i->arglist, bindflags);
}
}
block block_bind(block binder, block body, int bindflags) {
static void block_bind_each(block binder, block body, int bindflags) {
assert(block_has_only_binders(binder, bindflags));
bindflags |= OP_HAS_BINDING;
for (inst* curr = binder.first; curr; curr = curr->next) {
block_bind_subblock(inst_block(curr), body, bindflags);
}
}
block block_bind(block binder, block body, int bindflags) {
block_bind_each(binder, body, bindflags);
return block_join(binder, body);
}
block gen_function(const char* name, block formals, block body) {
block_bind_each(formals, body, OP_IS_CALL_PSEUDO);
inst* i = inst_new(CLOSURE_CREATE);
i->subfn = body;
i->symbol = strdup(name);
i->arglist = formals;
block b = inst_block(i);
block_bind_subblock(b, b, OP_IS_CALL_PSEUDO | OP_HAS_BINDING);
return b;
}
block gen_lambda(block body) {
return gen_function("@lambda", gen_noop(), body);
}
block gen_call(const char* name, block args) {
block b = gen_op_block_unbound(CALL_JQ, name);
b.first->arglist = args;
return b;
}
block gen_subexp(block a) {
block c = gen_noop();
block_append(&c, gen_op_simple(DUP));
block_append(&c, a);
block_append(&c, gen_op_simple(SWAP));
return c;
return BLOCK(gen_op_simple(DUP), a, gen_op_simple(SWAP));
}
block gen_both(block a, block b) {
block c = gen_noop();
block jump = gen_op_targetlater(JUMP);
block fork = gen_op_targetlater(FORK);
block_append(&c, fork);
block_append(&c, a);
block_append(&c, jump);
inst_set_target(fork, c);
block_append(&c, b);
block fork = gen_op_target(FORK, jump);
block c = BLOCK(fork, a, jump, b);
inst_set_target(jump, c);
return c;
}
block gen_collect(block expr) {
block c = gen_noop();
block_append(&c, gen_op_simple(DUP));
block_append(&c, gen_op_const(LOADK, jv_array()));
block array_var = block_bind(gen_op_var_unbound(STOREV, "collect"),
gen_noop(), OP_HAS_VARIABLE);
block_append(&c, array_var);
block c = BLOCK(gen_op_simple(DUP), gen_const(jv_array()), array_var);
block tail = {0};
block_append(&tail, gen_op_simple(DUP));
block_append(&tail, gen_op_var_bound(LOADV, array_var));
block_append(&tail, gen_op_simple(SWAP));
block_append(&tail, gen_op_simple(APPEND));
block_append(&tail, gen_op_var_bound(STOREV, array_var));
block_append(&tail, gen_op_simple(BACKTRACK));
block tail = BLOCK(gen_op_var_bound(APPEND, array_var),
gen_op_simple(BACKTRACK));
block_append(&c, gen_op_target(FORK, tail));
block_append(&c, expr);
block_append(&c, tail);
block_append(&c, gen_op_var_bound(LOADV, array_var));
return c;
return BLOCK(c,
gen_op_target(FORK, tail),
expr,
tail,
gen_op_var_bound(LOADV, array_var));
}
block gen_assign(block expr) {
block c = gen_noop();
block_append(&c, gen_op_simple(DUP));
block result_var = block_bind(gen_op_var_unbound(STOREV, "result"),
gen_noop(), OP_HAS_VARIABLE);
block_append(&c, result_var);
block loop = gen_noop();
block_append(&loop, gen_op_simple(DUP));
block_append(&loop, expr);
block_append(&loop, gen_op_var_bound(ASSIGN, result_var));
block_append(&loop, gen_op_simple(BACKTRACK));
block_append(&c, gen_op_target(FORK, loop));
block_append(&c, loop);
block_append(&c, gen_op_var_bound(LOADV, result_var));
return c;
block gen_fold(const char* varname, block init, block fold) {
block loop = BLOCK(fold, gen_op_var_unbound(STOREV, varname), gen_op_simple(BACKTRACK));
return BLOCK(gen_op_simple(DUP),
init,
block_bind(gen_op_var_unbound(STOREV, varname),
BLOCK(gen_op_target(FORK, loop),
loop,
gen_op_var_unbound(LOADV, varname)),
OP_HAS_VARIABLE));
}
block gen_definedor(block a, block b) {
// var found := false
block c = gen_op_simple(DUP);
block_append(&c, gen_op_const(LOADK, jv_false()));
block found_var = block_bind(gen_op_var_unbound(STOREV, "found"),
gen_noop(), OP_HAS_VARIABLE);
block_append(&c, found_var);
block init = BLOCK(gen_op_simple(DUP), gen_const(jv_false()), found_var);
// if found, backtrack. Otherwise execute b
block tail = gen_op_simple(DUP);
block_append(&tail, gen_op_var_bound(LOADV, found_var));
block backtrack = gen_op_simple(BACKTRACK);
block_append(&tail, gen_op_target(JUMP_F, backtrack));
block_append(&tail, backtrack);
block_append(&tail, gen_op_simple(POP));
block_append(&tail, b);
block tail = BLOCK(gen_op_simple(DUP),
gen_op_var_bound(LOADV, found_var),
gen_op_target(JUMP_F, backtrack),
backtrack,
gen_op_simple(POP),
b);
// try again
block if_notfound = gen_op_simple(BACKTRACK);
// found := true, produce result
block if_found = gen_op_simple(DUP);
block_append(&if_found, gen_op_const(LOADK, jv_true()));
block_append(&if_found, gen_op_var_bound(STOREV, found_var));
block_append(&if_found, gen_op_target(JUMP, tail));
block if_found = BLOCK(gen_op_simple(DUP),
gen_const(jv_true()),
gen_op_var_bound(STOREV, found_var),
gen_op_target(JUMP, tail));
block_append(&c, gen_op_target(FORK, if_notfound));
block_append(&c, a);
block_append(&c, gen_op_target(JUMP_F, if_found));
block_append(&c, if_found);
block_append(&c, if_notfound);
block_append(&c, tail);
return c;
return BLOCK(init,
gen_op_target(FORK, if_notfound),
a,
gen_op_target(JUMP_F, if_found),
if_found,
if_notfound,
tail);
}
block gen_condbranch(block iftrue, block iffalse) {
block b = gen_noop();
block_append(&iftrue, gen_op_target(JUMP, iffalse));
block_append(&b, gen_op_target(JUMP_F, iftrue));
block_append(&b, iftrue);
block_append(&b, iffalse);
return b;
iftrue = BLOCK(iftrue, gen_op_target(JUMP, iffalse));
return BLOCK(gen_op_target(JUMP_F, iftrue), iftrue, iffalse);
}
block gen_and(block a, block b) {
// a and b = if a then (if b then true else false) else false
block code = gen_op_simple(DUP);
block_append(&code, a);
block if_a_true = gen_op_simple(POP);
block_append(&if_a_true, b);
block_append(&if_a_true, gen_condbranch(gen_op_const(LOADK, jv_true()),
gen_op_const(LOADK, jv_false())));
block_append(&code, gen_condbranch(if_a_true,
block_join(gen_op_simple(POP), gen_op_const(LOADK, jv_false()))));
return code;
return BLOCK(gen_op_simple(DUP), a,
gen_condbranch(BLOCK(gen_op_simple(POP),
b,
gen_condbranch(gen_const(jv_true()),
gen_const(jv_false()))),
BLOCK(gen_op_simple(POP), gen_const(jv_false()))));
}
block gen_or(block a, block b) {
// a or b = if a then true else (if b then true else false)
block code = gen_op_simple(DUP);
block_append(&code, a);
block if_a_false = gen_op_simple(POP);
block_append(&if_a_false, b);
block_append(&if_a_false, gen_condbranch(gen_op_const(LOADK, jv_true()),
gen_op_const(LOADK, jv_false())));
block_append(&code, gen_condbranch(block_join(gen_op_simple(POP), gen_op_const(LOADK, jv_true())),
if_a_false));
return code;
return BLOCK(gen_op_simple(DUP), a,
gen_condbranch(BLOCK(gen_op_simple(POP), gen_const(jv_true())),
BLOCK(gen_op_simple(POP),
b,
gen_condbranch(gen_const(jv_true()),
gen_const(jv_false())))));
}
block gen_cond(block cond, block iftrue, block iffalse) {
block b = gen_op_simple(DUP);
block_append(&b, cond);
block_append(&b, gen_condbranch(block_join(gen_op_simple(POP), iftrue),
block_join(gen_op_simple(POP), iffalse)));
return b;
return BLOCK(gen_op_simple(DUP), cond,
gen_condbranch(BLOCK(gen_op_simple(POP), iftrue),
BLOCK(gen_op_simple(POP), iffalse)));
}
block gen_cbinding(struct symbol_table* t, block code) {
@@ -451,43 +402,135 @@ static int count_cfunctions(block b) {
int n = 0;
for (inst* i = b.first; i; i = i->next) {
if (i->op == CLOSURE_CREATE_C) n++;
if (opcode_describe(i->op)->flags & OP_HAS_BLOCK)
n += count_cfunctions(i->subfn);
n += count_cfunctions(i->subfn);
}
return n;
}
// Expands call instructions into a calling sequence
static int expand_call_arglist(struct locfile* locations, block* b) {
int errors = 0;
block ret = gen_noop();
for (inst* curr; (curr = block_take(b));) {
if (opcode_describe(curr->op)->flags & OP_HAS_BINDING) {
if (!curr->bound_by) {
locfile_locate(locations, curr->source, "error: %s is not defined", curr->symbol);
errors++;
// don't process this instruction if it's not well-defined
ret = BLOCK(ret, inst_block(curr));
continue;
}
}
block prelude = gen_noop();
if (curr->op == CALL_JQ) {
int actual_args = 0, desired_args = 0;
// We expand the argument list as a series of instructions
switch (curr->bound_by->op) {
default: assert(0 && "Unknown function type"); break;
case CLOSURE_CREATE:
case CLOSURE_PARAM: {
block callargs = gen_noop();
for (inst* i; (i = block_take(&curr->arglist));) {
assert(opcode_describe(i->op)->flags & OP_IS_CALL_PSEUDO);
block b = inst_block(i);
switch (i->op) {
default: assert(0 && "Unknown type of parameter"); break;
case CLOSURE_REF:
block_append(&callargs, b);
break;
case CLOSURE_CREATE:
block_append(&prelude, b);
block_append(&callargs, gen_op_block_bound(CLOSURE_REF, b));
break;
}
actual_args++;
}
curr->imm.intval = actual_args;
curr->arglist = callargs;
if (curr->bound_by->op == CLOSURE_CREATE) {
for (inst* i = curr->bound_by->arglist.first; i; i = i->next) {
assert(i->op == CLOSURE_PARAM);
desired_args++;
}
}
break;
}
case CLOSURE_CREATE_C: {
for (inst* i; (i = block_take(&curr->arglist)); ) {
assert(i->op == CLOSURE_CREATE); // FIXME
block body = i->subfn;
i->subfn = gen_noop();
inst_free(i);
// arguments should be pushed in reverse order, prepend them to prelude
errors += expand_call_arglist(locations, &body);
prelude = BLOCK(gen_subexp(body), prelude);
actual_args++;
}
assert(curr->op == CALL_JQ);
curr->op = CALL_BUILTIN;
curr->imm.intval = actual_args + 1 /* include the implicit input in arg count */;
assert(curr->bound_by->op == CLOSURE_CREATE_C);
desired_args = curr->bound_by->imm.cfunc->nargs - 1;
assert(!curr->arglist.first);
break;
}
}
if (actual_args != desired_args) {
locfile_locate(locations, curr->source,
"error: %s arguments to %s (expected %d but got %d)",
actual_args > desired_args ? "too many" : "too few",
curr->symbol, desired_args, actual_args);
errors++;
}
}
ret = BLOCK(ret, prelude, inst_block(curr));
}
*b = ret;
return errors;
}
static int compile(struct locfile* locations, struct bytecode* bc, block b) {
int errors = 0;
int pos = 0;
int var_frame_idx = 0;
bc->nsubfunctions = 0;
bc->nclosures = 0;
errors += expand_call_arglist(locations, &b);
if (bc->parent) {
// functions should end in a return
b = BLOCK(b, gen_op_simple(RET));
} else {
// the toplevel should YIELD;BACKTRACK; when it finds an answer
b = BLOCK(b, gen_op_simple(YIELD), gen_op_simple(BACKTRACK));
}
for (inst* curr = b.first; curr; curr = curr->next) {
if (!curr->next) assert(curr == b.last);
pos += opcode_length(curr->op);
int length = opcode_describe(curr->op)->length;
if (curr->op == CALL_JQ) {
for (inst* arg = curr->arglist.first; arg; arg = arg->next) {
length += 2;
}
}
pos += length;
curr->bytecode_pos = pos;
curr->compiled = bc;
int opflags = opcode_describe(curr->op)->flags;
if (opflags & OP_HAS_BINDING) {
if (!curr->bound_by) {
locfile_locate(locations, curr->source, "error: %s is not defined", curr->symbol);
errors++;
}
}
if ((opflags & OP_HAS_VARIABLE) &&
assert(curr->op != CLOSURE_REF && curr->op != CLOSURE_PARAM);
if ((opcode_describe(curr->op)->flags & OP_HAS_VARIABLE) &&
curr->bound_by == curr) {
curr->imm.intval = var_frame_idx++;
}
if (opflags & OP_HAS_BLOCK) {
if (curr->op == CLOSURE_CREATE) {
assert(curr->bound_by == curr);
curr->imm.intval = bc->nsubfunctions++;
}
if (curr->op == CLOSURE_PARAM) {
assert(curr->bound_by == curr);
curr->imm.intval = bc->nclosures++;
}
if (curr->op == CLOSURE_CREATE_C) {
assert(curr->bound_by == curr);
int idx = bc->globals->ncfunctions++;
@@ -496,21 +539,29 @@ static int compile(struct locfile* locations, struct bytecode* bc, block b) {
}
}
if (bc->nsubfunctions) {
bc->subfunctions = malloc(sizeof(struct bytecode*) * bc->nsubfunctions);
bc->subfunctions = jv_mem_alloc(sizeof(struct bytecode*) * bc->nsubfunctions);
for (inst* curr = b.first; curr; curr = curr->next) {
if (!(opcode_describe(curr->op)->flags & OP_HAS_BLOCK))
continue;
struct bytecode* subfn = malloc(sizeof(struct bytecode));
bc->subfunctions[curr->imm.intval] = subfn;
subfn->globals = bc->globals;
subfn->parent = bc;
errors += compile(locations, subfn, curr->subfn);
if (curr->op == CLOSURE_CREATE) {
struct bytecode* subfn = jv_mem_alloc(sizeof(struct bytecode));
bc->subfunctions[curr->imm.intval] = subfn;
subfn->globals = bc->globals;
subfn->parent = bc;
subfn->nclosures = 0;
for (inst* param = curr->arglist.first; param; param = param->next) {
assert(param->op == CLOSURE_PARAM);
assert(param->bound_by == param);
param->imm.intval = subfn->nclosures++;
param->compiled = subfn;
}
errors += compile(locations, subfn, curr->subfn);
curr->subfn = gen_noop();
}
}
} else {
bc->subfunctions = 0;
}
bc->codelen = pos;
uint16_t* code = malloc(sizeof(uint16_t) * bc->codelen);
uint16_t* code = jv_mem_alloc(sizeof(uint16_t) * bc->codelen);
bc->code = code;
pos = 0;
jv constant_pool = jv_array();
@@ -519,80 +570,56 @@ static int compile(struct locfile* locations, struct bytecode* bc, block b) {
const struct opcode_description* op = opcode_describe(curr->op);
if (op->length == 0)
continue;
uint16_t* opcode_rewrite = &code[pos];
code[pos++] = curr->op;
int opflags = op->flags;
assert(!(op->flags & OP_IS_CALL_PSEUDO));
if (opflags & OP_HAS_VARIABLE_LENGTH_ARGLIST) {
int nargs = curr->imm.intval;
assert(nargs > 0);
code[pos++] = (uint16_t)nargs;
int desired_params = 0;
for (int i=0; i<nargs; i++) {
curr = curr->next;
assert(curr && opcode_describe(curr->op)->flags & OP_IS_CALL_PSEUDO);
code[pos++] = nesting_level(bc, curr->bound_by);
switch (curr->bound_by->op) {
default: assert(0 && "Unknown type of argument");
case CLOSURE_CREATE:
code[pos++] = curr->bound_by->imm.intval | ARG_NEWCLOSURE;
if (i == 0) {
inst* i = curr->bound_by;
desired_params = i->compiled->subfunctions[i->imm.intval]->nclosures;
}
break;
case CLOSURE_PARAM:
code[pos++] = curr->bound_by->imm.intval;
if (i == 0) desired_params = 0;
break;
case CLOSURE_CREATE_C:
code[pos++] = curr->bound_by->imm.intval;
*opcode_rewrite = bc->globals->cfunctions[curr->bound_by->imm.intval].callop;
if (i == 0) desired_params = 0;
break;
}
assert(curr->op != CLOSURE_REF && curr->op != CLOSURE_PARAM);
if (curr->op == CALL_BUILTIN) {
assert(curr->bound_by->op == CLOSURE_CREATE_C);
assert(!curr->arglist.first);
code[pos++] = (uint16_t)curr->imm.intval;
code[pos++] = curr->bound_by->imm.intval;
} else if (curr->op == CALL_JQ) {
assert(curr->bound_by->op == CLOSURE_CREATE ||
curr->bound_by->op == CLOSURE_PARAM);
code[pos++] = (uint16_t)curr->imm.intval;
code[pos++] = nesting_level(bc, curr->bound_by);
code[pos++] = curr->bound_by->imm.intval |
(curr->bound_by->op == CLOSURE_CREATE ? ARG_NEWCLOSURE : 0);
for (inst* arg = curr->arglist.first; arg; arg = arg->next) {
assert(arg->op == CLOSURE_REF && arg->bound_by->op == CLOSURE_CREATE);
code[pos++] = nesting_level(bc, arg->bound_by);
code[pos++] = arg->bound_by->imm.intval | ARG_NEWCLOSURE;
}
assert(nargs - 1 == desired_params);
} else if (opflags & OP_HAS_CONSTANT) {
} else if (op->flags & OP_HAS_CONSTANT) {
code[pos++] = jv_array_length(jv_copy(constant_pool));
constant_pool = jv_array_append(constant_pool, jv_copy(curr->imm.constant));
} else if (opflags & OP_HAS_VARIABLE) {
} else if (op->flags & OP_HAS_VARIABLE) {
code[pos++] = nesting_level(bc, curr->bound_by);
uint16_t var = (uint16_t)curr->bound_by->imm.intval;
code[pos++] = var;
if (var > maxvar) maxvar = var;
} else if (opflags & OP_HAS_BRANCH) {
} else if (op->flags & OP_HAS_BRANCH) {
assert(curr->imm.target->bytecode_pos != -1);
assert(curr->imm.target->bytecode_pos > pos); // only forward branches
code[pos] = curr->imm.target->bytecode_pos - (pos + 1);
pos++;
} else if (opflags & OP_HAS_CFUNC) {
assert(curr->symbol);
int found = 0;
for (int i=0; i<bc->globals->ncfunctions; i++) {
if (!strcmp(curr->symbol, bc->globals->cfunctions[i].name)) {
code[pos++] = i;
found = 1;
break;
}
}
assert(found);
} else if (op->length > 1) {
assert(0 && "codegen not implemented for this operation");
}
}
bc->constants = constant_pool;
bc->nlocals = maxvar + 2; // FIXME: frames of size zero?
block_free(b);
return errors;
}
int block_compile(block b, struct locfile* locations, struct bytecode** out) {
struct bytecode* bc = malloc(sizeof(struct bytecode));
struct bytecode* bc = jv_mem_alloc(sizeof(struct bytecode));
bc->parent = 0;
bc->globals = malloc(sizeof(struct symbol_table));
bc->nclosures = 0;
bc->globals = jv_mem_alloc(sizeof(struct symbol_table));
int ncfunc = count_cfunctions(b);
bc->globals->ncfunctions = 0;
bc->globals->cfunctions = malloc(sizeof(struct cfunction) * ncfunc);
bc->globals->cfunctions = jv_mem_alloc(sizeof(struct cfunction) * ncfunc);
int nerrors = compile(locations, bc, b);
assert(bc->globals->ncfunctions == ncfunc);
if (nerrors > 0) {
compile.h
+25 -7
View File
@@ -1,10 +1,13 @@
#ifndef COMPILE_H
#define COMPILE_H
#include <stdint.h>
#include "bytecode.h"
#include "jv.h"
#include "opcode.h"
#include "locfile.h"
struct bytecode;
struct symbol_table;
struct inst;
typedef struct inst inst;
@@ -18,20 +21,19 @@ block gen_location(location, block);
block gen_noop();
block gen_op_simple(opcode op);
block gen_op_const(opcode op, jv constant);
block gen_const(jv constant);
block gen_op_target(opcode op, block target);
block gen_op_var_unbound(opcode op, const char* name);
block gen_op_var_bound(opcode op, block binder);
block gen_op_block_defn(opcode op, const char* name, block block);
block gen_op_block_defn_rec(opcode op, const char* name, block block);
block gen_op_block_unbound(opcode op, const char* name);
block gen_op_call(opcode op, block arglist);
block gen_op_symbol(opcode op, const char* name);
block gen_function(const char* name, block formals, block body);
block gen_lambda(block body);
block gen_call(const char* name, block body);
block gen_subexp(block a);
block gen_both(block a, block b);
block gen_collect(block expr);
block gen_assign(block expr);
block gen_fold(const char* varname, block init, block body);
block gen_definedor(block a, block b);
block gen_condbranch(block iftrue, block iffalse);
block gen_and(block a, block b);
@@ -52,4 +54,20 @@ void block_free(block);
// Here's some horrible preprocessor gunk so that code
// sequences can be contructed as BLOCK(block1, block2, block3)
#define BLOCK_1(b1) (b1)
#define BLOCK_2(b1,b2) (block_join((b1),(b2)))
#define BLOCK_3(b1,b2,b3) (block_join(BLOCK_2(b1,b2),(b3)))
#define BLOCK_4(b1,b2,b3,b4) (block_join(BLOCK_3(b1,b2,b3),(b4)))
#define BLOCK_5(b1,b2,b3,b4,b5) (block_join(BLOCK_4(b1,b2,b3,b4),(b5)))
#define BLOCK_6(b1,b2,b3,b4,b5,b6) (block_join(BLOCK_5(b1,b2,b3,b4,b5),(b6)))
#define BLOCK_7(b1,b2,b3,b4,b5,b6,b7) (block_join(BLOCK_6(b1,b2,b3,b4,b5,b6),(b7)))
#define BLOCK_IDX(_1,_2,_3,_4,_5,_6,_7,NAME,...) NAME
#define BLOCK(...) \
BLOCK_IDX(__VA_ARGS__, BLOCK_7, BLOCK_6, BLOCK_5, BLOCK_4, BLOCK_3, BLOCK_2, BLOCK_1)(__VA_ARGS__)
#endif
docs/Gemfile
+5
View File
@@ -0,0 +1,5 @@
source :rubygems
gem "bonsai"
gem "maruku"
gem "ronn"
docs/Gemfile.lock
+53
View File
@@ -0,0 +1,53 @@
GEM
remote: http://rubygems.org/
specs:
activesupport (3.2.9)
i18n (~> 0.6)
multi_json (~> 1.0)
addressable (2.3.2)
bonsai (1.4.7)
activesupport (>= 3.0.3)
builder (>= 3.0.0)
i18n (>= 0.5.0)
launchy (>= 0.3.7)
liquid (>= 2.2.2)
maruku (>= 0.6.0)
rack
sass
sinatra (>= 1.0)
tilt (>= 1.3)
watch (>= 0.1.0)
builder (3.1.4)
hpricot (0.8.6)
i18n (0.6.1)
launchy (2.1.2)
addressable (~> 2.3)
liquid (2.4.1)
maruku (0.6.1)
syntax (>= 1.0.0)
multi_json (1.5.0)
mustache (0.99.4)
rack (1.4.1)
rack-protection (1.3.2)
rack
rdiscount (1.6.8)
ronn (0.7.3)
hpricot (>= 0.8.2)
mustache (>= 0.7.0)
rdiscount (>= 1.5.8)
sass (3.2.3)
sinatra (1.3.3)
rack (~> 1.3, >= 1.3.6)
rack-protection (~> 1.2)
tilt (~> 1.3, >= 1.3.3)
syntax (1.0.0)
tilt (1.3.3)
watch (0.1.0)
PLATFORMS
ruby
DEPENDENCIES
bonsai
maruku
ronn
docs/Rakefile
+53 -2
View File
@@ -2,6 +2,8 @@ require 'bonsai'
require 'liquid'
require 'maruku'
require 'json'
require 'ronn'
require 'tempfile'
module ExtraFilters
def markdownify(input)
@@ -38,11 +40,13 @@ task :serve do
server = fork {
app = Rack::Builder.app {
use Bonsai::StaticPassThrough, :root => Bonsai.root_dir + "/output", :urls => ["/"]
map "/jq" do
use Bonsai::StaticPassThrough, :root => Bonsai.root_dir + "/output", :urls => ["/"]
end
run Bonsai::DevelopmentServer
}
Rack::Handler.default.run(app, :Port => 5000) do
Launchy.open("http://localhost:5000/")
Launchy.open("http://localhost:5000/jq")
end
}
Watch.new("{content,templates,public}/**/*") { Bonsai::Exporter.process! }
@@ -57,3 +61,50 @@ task :build do
Bonsai.root_dir = Dir.pwd
Bonsai::Exporter.publish!
end
def load_manual
YAML::ENGINE.yamler = 'syck'
YAML::load(File.open("content/3.manual/manual.yml"))
end
task :manpage do
Tempfile.open "manpage" do |f|
manual = load_manual
f.puts manual['manpage_intro']
f.puts manual['body']
manual['sections'].each do |section|
f.puts "## #{section['title'].upcase}\n"
f.puts section['body']
f.puts ""
(section['entries'] || []).each do |entry|
f.puts "### #{entry['title']}\n"
f.puts entry['body']
f.puts ""
(entry['examples'] || []).each do |example|
f.puts " jq '#{example['program']}'"
f.puts " #{example['input']}"
f.puts " => #{example['output'].join(", ")}"
f.puts
end
end
f.puts ""
end
f.puts manual['manpage_epilogue']
f.close
puts Ronn::Document.new(f.path).convert('roff').gsub(/<\/?code>/,"")
end
end
task :mantests do
load_manual['sections'].each do |section|
(section['entries'] || []).each do |entry|
(entry['examples'] || []).each do |example|
puts example['program'].gsub("\n", " ")
puts example['input']
example['output'].each do |s| puts s end
puts
end
end
end
end
docs/content/2.download/default.yml
+41 -33
View File
@@ -5,39 +5,46 @@ body:
jq is written in C and has no runtime dependencies, so it should be
possible to build it for nearly any platform. Prebuilt binaries are
available for Linux (64-bit x86) and OS X.
* [Download binary for 64-bit Linux](linux_x86_64/jq)
* [Download binary for OS X](osx_64/jq)
(or use [homebrew](http://mxcl.github.com/homebrew/): `brew install jq`)
* [Download source](source/jq.tgz)
available for Linux, OS X and Windows.
The binaries should just run, but you may need to make them
executable first using:
The binaries should just run, but on OS X and Linux you may need
to make them executable first using `chmod +x jq`.
chmod +x jq
jq is licensed under the MIT license. For all of the gory
details, read the file `COPYING` in the source distribution.
Arch Linux
----------
A PKGBUILD for jq-1.1 is in the
[AUR](https://aur.archlinux.org/packages.php?ID=63849), as well as a
PKGBUILD for the HEAD of master
([jq-git](https://aur.archlinux.org/packages.php?ID=63850)).
### Linux
* Binaries for [64-bit](linux64/jq) or [32-bit](linux32/jq) systems.
You can install jq using [Yaourt](https://wiki.archlinux.org/index.php/Yaourt)
or manually by following instructions on
[Arch Linux's Wiki](https://wiki.archlinux.org/index.php/Arch_User_Repository).
* For Arch users, a PKGBUILD for jq-1.1 is in the
[AUR](https://aur.archlinux.org/packages.php?ID=63849), as
well as a PKGBUILD for the HEAD of master
([jq-git](https://aur.archlinux.org/packages.php?ID=63850)).
You can install them using
[Yaourt](https://wiki.archlinux.org/index.php/Yaourt) or
manually by following instructions on [Arch Linux's
Wiki](https://wiki.archlinux.org/index.php/Arch_UseRepository).
Hacking on jq
=============
* Debian/Ubuntu packages are coming Real Soon Now.
If you want to work on jq, grab the source from
[https://github.com/stedolan/jq](https://github.com/stedolan/jq).
### OS X
* `brew install jq` using [homebrew](http://mxcl.github.com/homebrew/)
* Or, grab prebuilt [64-bit binaries](osx64/jq) or [32-bit
binaries](osx32/jq)
### Windows
* Executables for [64-bit](win64/jq.exe) or [32-bit](win32/jq.exe)
### From source on Linux or OS X
git clone https://github.com/stedolan/jq.git
cd jq
make && sudo make install
To build it from a git clone, you'll need to install a few
packages first:
@@ -48,23 +55,24 @@ body:
* [GCC](http://gcc.gnu.org)
* [Make](http://www.gnu.org/software/make)
Most of these aren't necessary if you're just trying to compile
jq from the released tarball - that version has the
non-platform-specific build steps already done, so you'll only
need a C compiler and `make` to finish it off.
For Linux systems, these will all be in your system's package
manager, and if you do development on the machine they're most
likely already installed. I have no idea how to get these
installed on OS X, you're on your own there.
likely already installed.
On OS X, these are all included in Apple's command line tools, which
can be installed from [Xcode](http://developer.apple.com/technologies/tools/).
However, you may find that you need a newer version of Bison than the one provided
by Apple. This can be found in [Homebrew](http://mxcl.github.com/homebrew/)
or [MacPorts](http://macports.org/).
`flex` and `bison` are used to generate the lexer and parser for
jq, and some python scripts generate the UTF8 encoding tables
needed for JSON parsing.
Building the documentation
--
#### Building the documentation
jq's documentation is compiled into static HTML using
[Bonsai](http://www.tinytree.info). To view the documentation
locally, run `rake serve` from the docs/ subdirectory.
docs/content/2.download/linux_x86_64/jq
BIN
View File
Binary file not shown.
docs/content/2.download/osx_64/jq
BIN
View File
Binary file not shown.
docs/content/3.manual/manual.yml
+344 -63
View File
@@ -26,6 +26,41 @@ body: |
But that's getting ahead of ourselves. :) Let's start with something
simpler:
manpage_intro: |
jq(1) -- Command-line JSON processor
====================================
## SYNOPSIS
`jq` [<options>...] <filter> [<files>...]
`jq` can transform JSON in various ways, by selecting, iterating,
reducing and otherwise mangling JSON documents. For instance,
running the command `jq 'map(.price) | add'` will take an array of
JSON objects as input and return the sum of their "price" fields.
By default, `jq` reads a stream of JSON objects (whitespace
separated) from `stdin`. One or more <files> may be specified, in
which case `jq` will read input from those instead.
The <options> are described in the [INVOKING JQ] section, they
mostly concern input and output formatting. The <filter> is written
in the jq language and specifies how to transform the input
document.
## FILTERS
manpage_epilogue: |
## BUGS
Presumably. Report them or discuss them at:
https://github.com/stedolan/jq/issues
## AUTHOR
Stephen Dolan `<mu@netsoc.tcd.ie>`
sections:
- title: Invoking jq
@@ -40,41 +75,47 @@ sections:
You can affect how jq reads and writes its input and output
using some command-line options:
* `--slurp`/`-s`
* `--slurp`/`-s`:
Instead of running the filter for each JSON object in the
input, read the entire input stream into a large array and run
the filter just once.
* `--raw-input`/`-R`
* `--raw-input`/`-R`:
Don't parse the input as JSON. Instead, each line of text is
passed to the filter as a string. If combined with `--slurp`,
then the entire input is passed to the filter as a single long
string.
* `--null-input`/`-n`
* `--null-input`/`-n`:
Don't read any input at all! Instead, the filter is run once
using `null` as the input. This is useful when using jq as a
simple calculator or to construct JSON data from scratch.
* `--compact-output` / `-c`
* `--compact-output` / `-c`:
By default, jq pretty-prints JSON output. Using this option
will result in more compact output by instead putting each
JSON object on a single line.
* `--ascii-output` / `-a`
* `--colour-output` / `-C` and `--monochrome-output` / `-M`:
By default, jq outputs colored JSON if writing to a
terminal. You can force it to produce color even if writing to
a pipe or a file using `-C`, and disable color with `-M`.
* `--ascii-output` / `-a`:
jq usually outputs non-ASCII Unicode codepoints as UTF-8, even
if the input specified them as escape sequences (like
"\u03bc"). Using this option, you can force jq to produce pure
ASCII output with every non-ASCII character replaced with the
equivalent escape sequence.
* `--raw-output` / `-r`
* `--raw-output` / `-r`:
With this option, if the filter's result is a string then it
will be written directly to standard output rather than being
formatted as a JSON string with quotes. This can be useful for
@@ -142,7 +183,7 @@ sections:
examples:
- program: '.[]'
input: '[{name":"JSON", "good":true}, {"name":"XML", "good":false}]'
input: '[{"name":"JSON", "good":true}, {"name":"XML", "good":false}]'
output:
- '{"name":"JSON", "good":true}'
- '{"name":"XML", "good":false}'
@@ -172,7 +213,7 @@ sections:
- program: '.[4,2]'
input: '["a","b","c","d","e"]'
output: ['"d"', '"c"']
output: ['"e"', '"c"']
- title: "`|`"
body: |
@@ -188,7 +229,7 @@ sections:
examples:
- program: '.[] | .name'
input: '[{name":"JSON", "good":true}, {"name":"XML", "good":false}]'
input: '[{"name":"JSON", "good":true}, {"name":"XML", "good":false}]'
output: ['"JSON"', '"XML"']
- title: Types and Values
@@ -315,14 +356,23 @@ sections:
the key-value pairs from both objects into a single
combined object. If both objects contain a value for the
same key, the object on the right of the `+` wins.
`null` can be added to any value, and returns the other
value unchanged.
examples:
- program: '.a + 1'
input: '{"a": 7}'
output: '{"a": 8}'
output: ['8']
- program: '.a + .b'
input: '{"a": [1,2], "b": [3,4]}'
output: ['[1,2,3,4]']
- program: '.a + null'
input: '{"a": 1}'
output: ['1']
- program: '.a + 1'
input: '{}'
output: ['1']
- program: '{a: 1} + {b: 2} + {c: 3} + {a: 42}'
input: 'null'
output: ['{"a": 42, "b": 2, "c": 3}']
@@ -403,11 +453,11 @@ sections:
`foo` returns true for that input, and produces no output
otherwise.
It's useful for filtering lists: `[1,2,3] | select(. >= 2)`
It's useful for filtering lists: `[1,2,3] | map(select(. >= 2))`
will give you `[3]`.
examples:
- program: 'select(. >= 2)'
- program: 'map(select(. >= 2))'
input: '[1,5,3,0,7]'
output: ['[5,3,7]']
@@ -421,7 +471,7 @@ sections:
examples:
- program: '1, empty, 2'
input: 'null'
output: [1,2]
output: [1, 2]
- program: '[1,2,empty,3]'
input: 'null'
output: ['[1,2,3]']
@@ -455,7 +505,7 @@ sections:
examples:
- program: add
input: '["a","b","c"]'
output: ["abc"]
output: ['"abc"']
- program: add
input: '[1, 2, 3]'
output: [6]
@@ -473,7 +523,7 @@ sections:
examples:
- program: '.[] | tonumber'
input: '[1, "1"]'
output: [1,1]
output: [1, 1]
- title: `tostring`
body: |
@@ -487,6 +537,144 @@ sections:
input: '[1, "1", [1]]'
output: ['"1"', '"1"', '"[1]"']
- title: `sort, sort_by`
body: |
The `sort` functions sorts its input, which must be an
array. Values are sorted in the following order:
* `null`
* `false`
* `true`
* numbers
* strings, in alphabetical order (by unicode codepoint value)
* arrays, in lexical order
* objects
The ordering for objects is a little complex: first they're
compared by comparing their sets of keys (as arrays in
sorted order), and if their keys are equal then the values
are compared key by key.
`sort_by` may be used to sort by a particular field of an
object, or by applying any jq filter. `sort_by(foo)`
compares two elements by comparing the result of `foo` on
each element.
examples:
- program: 'sort'
input: '[8,3,null,6]'
output: ['[null,3,6,8]']
- program: 'sort_by(.foo)'
input: '[{"foo":4, "bar":10}, {"foo":3, "bar":100}, {"foo":2, "bar":1}]'
output: ['[{"foo":2, "bar":1}, {"foo":3, "bar":100}, {"foo":4, "bar":10}]']
- title: `group_by`
body: |
`group_by(.foo)` takes as input an array, groups the
elements having the same `.foo` field into separate arrays,
and produces all of these arrays as elements of a larger
array, sorted by the value of the `.foo` field.
Any jq expression, not just a field access, may be used in
place of `.foo`. The sorting order is the same as described
in the `sort` function above.
examples:
- program: 'group_by(.foo)'
input: '[{"foo":1, "bar":10}, {"foo":3, "bar":100}, {"foo":1, "bar":1}]'
output: ['[[{"foo":1, "bar":10}, {"foo":1, "bar":1}], [{"foo":3, "bar":100}]]']
- title: `min`, `max`, `min_by`, `max_by`
body: |
Find the minimum or maximum element of the input array. The
`_by` versions allow you to specify a particular field or
property to examine, e.g. `min_by(.foo)` finds the object
with the smallest `foo` field.
examples:
- program: 'min'
input: '[5,4,2,7]'
output: ['2']
- program: 'max_by(.foo)'
input: '[{"foo":1, "bar":14}, {"foo":2, "bar":3}]'
output: ['{"foo":2, "bar":3}']
- title: `unique`
body: |
The `unique` function takes as input an array and produces
an array of the same elements, in sorted order, with
duplicates removed.
examples:
- program: 'unique'
input: '[1,2,5,3,5,3,1,3]'
output: ['[1,2,3,5]']
- title: `contains`
body: |
The filter `contains(b)` will produce true if b is
completely contained within the input. A string B is
contained in a string A if B is a substring of A. An array B
is contained in an array A is all elements in B are
contained in any element in A. An object B is contained in
object A if all of the values in B are contained in the
value in A with the same key. All other types are assumed to
be contained in each other if they are equal.
examples:
- program: 'contains("bar")'
input: '"foobar"'
output: ['true']
- program: 'contains(["baz", "bar"])'
input: '["foobar", "foobaz", "blarp"]'
output: ['true']
- program: 'contains(["bazzzzz", "bar"])'
input: '["foobar", "foobaz", "blarp"]'
output: ['false']
- program: 'contains({foo: 12, bar: [{barp: 12}]})'
input: '{"foo": 12, "bar":[1,2,{"barp":12, "blip":13}]}'
output: ['true']
- program: 'contains({foo: 12, bar: [{barp: 15}]})'
input: '{"foo": 12, "bar":[1,2,{"barp":12, "blip":13}]}'
output: ['false']
- title: `recurse`
body: |
The `recurse` function allows you to search through a
recursive structure, and extract interesting data from all
levels. Suppose your input represents a filesystem:
{"name": "/", "children": [
{"name": "/bin", "children": [
{"name": "/bin/ls", "children": []},
{"name": "/bin/sh", "children": []}]},
{"name": "/home", "children": [
{"name": "/home/stephen", "children": [
{"name": "/home/stephen/jq", "children": []}]}]}]}
Now suppose you want to extract all of the filenames
present. You need to retrieve `.name`, `.children[].name`,
`.children[].children[].name`, and so on. You can do this
with:
recurse(.children[]) | .name
examples:
- program: 'recurse(.foo[])'
input: '{"foo":[{"foo": []}, {"foo":[{"foo":[]}]}]}'
output:
- '{"foo":[{"foo":[]},{"foo":[{"foo":[]}]}]}'
- '{"foo":[]}'
- '{"foo":[{"foo":[]}]}'
- '{"foo":[]}'
- title: "String interpolation - `\(foo)`"
body: |
@@ -499,10 +687,77 @@ sections:
input: '42'
output: ['"The input was 42, which is one less than 43"']
- title: "Format strings and escaping"
body: |
The `@foo` syntax is used to format and escape strings,
which is useful for building URLs, documents in a language
like HTML or XML, and so forth. `@foo` can be used as a
filter on its own, the possible escapings are:
* `@text`:
Calls `tostring`, see that function for details.
* `@json`:
Serialises the input as JSON.
* `@html`:
Applies HTML/XML escaping, by mapping the characters
`<>&'"` to their entity equivalents `&lt;`, `&gt;`,
`&amp;`, `&apos;`, `&quot;`.
* `@uri`:
Applies percent-encoding, by mapping all reserved URI
characters to a `%xx` sequence.
* `@csv`:
The input must be an array, and it is rendered as CSV
with double quotes for strings, and quotes escaped by
repetition.
* `@sh`:
The input is escaped suitable for use in a command-line
for a POSIX shell. If the input is an array, the output
will be a series of space-separated strings.
* `@base64`:
The input is converted to base64 as specified by RFC 4648.
This syntax can be combined with string interpolation in a
useful way. You can follow a `@foo` token with a string
literal. The contents of the string literal will *not* be
escaped. However, all interpolations made inside that string
literal will be escaped. For instance,
@uri "http://www.google.com/search?q=\(.search)"
will produce the following output for the input
`{"search":"jq!"}`:
http://www.google.com/search?q=jq%21
Note that the slashes, question mark, etc. in the URL are
not escaped, as they were part of the string literal.
examples:
- program: '@html'
input: '"This works if x < y"'
output: ['"This works if x &lt; y"']
# - program: '@html "<span>Anonymous said: \(.)</span>"'
# input: '"<script>alert(\"lol hax\");</script>"'
# output: ["<span>Anonymous said: &lt;script&gt;alert(&quot;lol hax&quot;);&lt;/script&gt;</span>"]
- program: '@sh "echo \(.)"'
input: "\"O'Hara's Ale\""
output: ["\"echo 'O'\\''Hara'\\''s Ale\""]
- title: Conditionals and Comparisons
entries:
@@ -521,34 +776,7 @@ sections:
examples:
- program: '.[] == 1'
input: '[1, 1.0, "1", "banana"]'
output: ['[true, true, false, false]']
- title: `contains`
body: |
The expression 'a contains b' will produce true if b is completely
contained within a. A string B is contained in a string A if B is a
substring of A. An array B is contained in an array A is all elements
in B are contained in any element in A. An object B is contained in
object A if all of the values in B are contained in the value in A with
the same key. All other types are assumed to be contained in each other
if they are equal.
examples:
- program: '. == contains "bar"'
input: '"foobar"'
output: ['true']
- program: '. contains ["baz", "bar"]'
input: '["foobar", "foobaz", "blarp"]'
output: ['true']
- program: '. contains ["bazzzzz", "bar"]'
input: '["foobar", "foobaz", "blarp"]'
output: ['false']
- program: '. contains {foo: 12, bar: [{barp: 12}]}'
input: '{foo: 12, bar:[1,2,{barp:12, blip:13}]}'
output: ['true']
- program: '. contains {foo: 12, bar: [{barp: 15}]}'
input: '{foo: 12, bar:[1,2,{barp:12, blip:13}]}'
output: ['false']
output: ['true', 'true', 'false', 'false']
- title: if-then-else
body: |
@@ -561,8 +789,8 @@ sections:
means that you'll sometimes have to be more explicit about
the condition you want: you can't test whether, e.g. a
string is empty using `if .name then A else B end`, you'll
need something more like 'if (.name | count) > 0 then A else
B end' instead.
need something more like `if (.name | count) > 0 then A else
B end` instead.
If the condition A produces multiple results, it is
considered "true" if any of those results is not false or
@@ -581,6 +809,21 @@ sections:
end
input: 2
output: ['"many"']
- title: `>, >=, <=, <`
body: |
The comparison operators `>`, `>=`, `<=`, `<` return whether
their left argument is greater than, greater than or equal
to, less than or equal to or less than their right argument
(respectively).
The ordering is the same as that described for `sort`, above.
examples:
- program: '. < 5'
input: 2
output: ['true']
- title: and/or/not
body: |
@@ -639,7 +882,7 @@ sections:
input: '{}'
output: [42]
- title: Variables and Functions
- title: Advanced features
body: |
Variables are an absolute necessity in most programming languages, but
they're relegated to an "advanced feature" in jq.
@@ -656,6 +899,10 @@ sections:
(many jq functions such as `map` and `find` are in fact written
in jq).
Finally, jq has a `fold` operation, which is very powerful but a
bit tricky. Again, it's mostly used internally, to define some
useful bits of jq's standard library.
entries:
- title: Variables
body: |
@@ -670,8 +917,8 @@ sections:
For instance, calculating the average value of an array of numbers
requires a few variables in most languages - at least one to hold the
array, perhaps one for each element or for a loop counter. In jq, it's
simply `add / length` - the `sum` expression is given the array and
produces its sum, and the `count` expression is given the array and
simply `add / length` - the `add` expression is given the array and
produces its sum, and the `length` expression is given the array and
produces its length.
So, there's generally a cleaner way to solve most problems in jq that
@@ -754,12 +1001,45 @@ sections:
input's `.foo` field to each element of the array.
examples:
- program: 'def addvalue(f): map(. + [$value]); addvalue(.[0])'
- program: 'def addvalue(f): . + [f]; map(addvalue(.[0]))'
input: '[[1,2],[10,20]]'
output: ['[[1,2,1], [10,20,10]]']
- program: 'def addvalue(f): f as $x | map (. + $x); addvalue(.[0])'
- program: 'def addvalue(f): f as $x | map(. + $x); addvalue(.[0])'
input: '[[1,2],[10,20]]'
output: ['[[1,2,[1,2]], [10,20,[1,2]]]']
output: ['[[1,2,1,2], [10,20,1,2]]']
- title: Fold
body: |
The `fold` syntax in jq allows you to combine all of the
results of an expression by accumulating them into a single
answer. As an example, we'll pass `[3,2,1]` to this expression:
fold 0 as $sum (.[] | $sum + .)
The variable `$sum` is first given the value `0`. The body
of the fold (i.e. `.[] | $sum + .`) is evaluated. `.[]`
produces three results, `3`, `2`, and `1`. For the first
one, `$sum + .` gives `3`.
Having produced this answer, jq backtracks to find the next
result as per usual. However, this time, `$sum` is set to
the previous value of the body, so `$sum + .` gives
`5`. After the final backtracking, `$sum + .` gives
`6`. This final value is used as the value of the entire
`fold` expression, so the above filter returns `6`.
More formally, in order to evaluate `fold INIT as $VAR
(BODY)`, jq first sets `$VAR` to the value of `INIT`. It
then runs through `BODY`. Each time `BODY` produces a value,
`$VAR` is set to that value and jq backtracks to find the
next one. When `BODY` stops producing values, the final
value of `$VAR` is the result of the entire expression.
examples:
- program: 'fold 0 as $sum (.[] | $sum + .)'
input: '[10,2,5,3]'
output: ['20']
- title: Assignment
@@ -871,4 +1151,5 @@ sections:
"stedolan" wrote, and we can comment on each of them in the same way
that we did before:
(.posts[] | select(.author == "stedolan") | .comments) |= . + ["terrible."]
(.posts[] | select(.author == "stedolan") | .comments) |=
. + ["terrible."]
docs/site.yml
+1 -3
View File
@@ -1,10 +1,8 @@
# The key value pairs found below are available within the templates.
jq_version: 1.1
# Next line is needed for publishing to github pages
jq_version: 1.2
root: '/jq'
# root: ''
footer: |
This website is made with [Bonsai](http://www.tinytree.info) and
docs/templates/index.liquid
Vendored
+2 -2
View File
@@ -16,8 +16,8 @@
<span class="caret"></span>
</a>
<ul class="dropdown-menu">
<li><a href="{{root}}/download/linux_x86_64/jq">Linux (x86_64)</a></li>
<li><a href="{{root}}/download/osx_64/jq">OS X (64-bit)</a></li>
<li><a href="{{root}}/download/linux64/jq">Linux (64-bit)</a></li>
<li><a href="{{root}}/download/osx64/jq">OS X (64-bit)</a></li>
<li><a href="{{root}}/download">Other platforms and source</a></li>
</ul>
</div>
execute.c
+45 -61
View File
@@ -7,14 +7,15 @@
#include "opcode.h"
#include "bytecode.h"
#include "compile.h"
#include "forkable_stack.h"
#include "frame_layout.h"
#include "jv_alloc.h"
#include "locfile.h"
#include "jv.h"
#include "jq_parser.h"
#include "jv_aux.h"
#include "parser.h"
#include "builtin.h"
typedef struct {
@@ -33,7 +34,7 @@ int path_push(stackval sv, jv val) {
if (pos == pathsize) {
int oldpathsize = pathsize;
pathsize = oldpathsize ? oldpathsize * 2 : 100;
pathbuf = realloc(pathbuf, sizeof(pathbuf[0]) * pathsize);
pathbuf = jv_mem_realloc(pathbuf, sizeof(pathbuf[0]) * pathsize);
for (int i=oldpathsize; i<pathsize; i++) {
pathbuf[i] = jv_invalid();
}
@@ -149,7 +150,6 @@ void print_error(jv value) {
jv jq_next() {
jv cfunc_input[MAX_CFUNCTION_ARGS];
jv cfunc_output[MAX_CFUNCTION_ARGS];
assert(!forkable_stack_empty(&frame_stk));
uint16_t* pc = *frame_current_retaddr(&frame_stk);
@@ -166,7 +166,9 @@ jv jq_next() {
printf("\t");
const struct opcode_description* opdesc = opcode_describe(opcode);
data_stk_elem* param;
for (int i=0; i<opdesc->stack_in; i++) {
int stack_in = opdesc->stack_in;
if (stack_in == -1) stack_in = pc[1];
for (int i=0; i<stack_in; i++) {
if (i == 0) {
param = forkable_stack_peek(&data_stk);
} else {
@@ -207,6 +209,17 @@ jv jq_next() {
break;
}
case DUP2: {
stackval keep = stack_pop();
stackval v = stack_pop();
stackval v2 = v;
v2.value = jv_copy(v.value);
stack_push(v);
stack_push(keep);
stack_push(v2);
break;
}
case SWAP: {
stackval a = stack_pop();
stackval b = stack_pop();
@@ -223,9 +236,12 @@ jv jq_next() {
case APPEND: {
// FIXME paths
jv v = stack_pop().value;
jv array = stack_pop().value;
array = jv_array_append(array, v);
stack_push(stackval_root(array));
uint16_t level = *pc++;
uint16_t vidx = *pc++;
frame_ptr fp = frame_get_level(&frame_stk, frame_current(&frame_stk), level);
jv* var = frame_local_var(fp, vidx);
assert(jv_get_kind(*var) == JV_KIND_ARRAY);
*var = jv_array_append(*var, v);
break;
}
@@ -273,24 +289,15 @@ jv jq_next() {
break;
}
case ASSIGN: {
stackval replacement = stack_pop();
case GETPATH: {
stackval path_end = stack_pop();
stackval path_start = stack_pop();
jv_free(path_end.value);
jv_free(path_start.value);
uint16_t level = *pc++;
uint16_t v = *pc++;
frame_ptr fp = frame_get_level(&frame_stk, frame_current(&frame_stk), level);
jv* var = frame_local_var(fp, v);
jv result = jv_insert(*var, replacement.value, pathbuf + path_start.pathidx, path_end.pathidx - path_start.pathidx);
if (jv_is_valid(result)) {
*var = result;
} else {
print_error(result);
*var = jv_null();
jv path = jv_array();
for (int i=path_start.pathidx; i<path_end.pathidx; i++) {
path = jv_array_set(path, i, jv_copy(pathbuf[i]));
}
stack_push(stackval_replace(path_start, path));
break;
}
@@ -298,7 +305,7 @@ jv jq_next() {
stackval t = stack_pop();
jv k = stack_pop().value;
int pathidx = path_push(t, jv_copy(k));
jv v = jv_lookup(t.value, k);
jv v = jv_get(t.value, k);
if (jv_is_valid(v)) {
stackval sv;
sv.value = v;
@@ -409,15 +416,15 @@ jv jq_next() {
return value;
}
case CALL_BUILTIN_1_1: {
assert(*pc == 1); // no closure args allowed
pc++; // skip nclosures
pc++; // skip level
case CALL_BUILTIN: {
int nargs = *pc++;
stackval top = stack_pop();
cfunc_input[0] = top.value;
for (int i = 1; i < nargs; i++) {
cfunc_input[i] = stack_pop().value;
}
struct cfunction* func = &frame_current_bytecode(&frame_stk)->globals->cfunctions[*pc++];
func->fptr(cfunc_input, cfunc_output);
top.value = cfunc_output[0];
top.value = cfunction_invoke(func, cfunc_input);
if (jv_is_valid(top.value)) {
stack_push(top);
} else {
@@ -427,37 +434,16 @@ jv jq_next() {
break;
}
case CALL_BUILTIN_3_1: {
assert(*pc == 1); // no closure args allowed
pc++; // skip nclosures
pc++; // skip level
stackval top = stack_pop();
jv a = stack_pop().value;
jv b = stack_pop().value;
cfunc_input[0] = top.value;
cfunc_input[1] = a;
cfunc_input[2] = b;
struct cfunction* func = &frame_current_bytecode(&frame_stk)->globals->cfunctions[*pc++];
func->fptr(cfunc_input, cfunc_output);
top.value = cfunc_output[0];
if (jv_is_valid(top.value)) {
stack_push(top);
} else {
print_error(top.value);
goto do_backtrack;
}
break;
}
case CALL_1_1: {
case CALL_JQ: {
uint16_t nclosures = *pc++;
uint16_t* retaddr = pc + 2 + nclosures*2;
frame_ptr new_frame = frame_push(&frame_stk,
make_closure(&frame_stk, frame_current(&frame_stk), pc),
pc + nclosures * 2);
retaddr);
pc += 2;
frame_ptr old_frame = forkable_stack_peek_next(&frame_stk, new_frame);
assert(nclosures - 1 == frame_self(new_frame)->bc->nclosures);
for (int i=0; i<nclosures-1; i++) {
assert(nclosures == frame_self(new_frame)->bc->nclosures);
for (int i=0; i<nclosures; i++) {
*frame_closure_arg(new_frame, i) = make_closure(&frame_stk, old_frame, pc);
pc += 2;
}
@@ -477,9 +463,9 @@ jv jq_next() {
void jq_init(struct bytecode* bc, jv input) {
forkable_stack_init(&data_stk, sizeof(stackval) * 1000); // FIXME: lower this number, see if it breaks
forkable_stack_init(&frame_stk, 10240); // FIXME: lower this number, see if it breaks
forkable_stack_init(&fork_stk, 10240); // FIXME: lower this number, see if it breaks
forkable_stack_init(&data_stk, sizeof(stackval) * 100);
forkable_stack_init(&frame_stk, 1024);
forkable_stack_init(&fork_stk, 1024);
stack_push(stackval_root(input));
struct closure top = {bc, -1};
@@ -500,7 +486,7 @@ void jq_teardown() {
for (int i=0; i<pathsize; i++) {
jv_free(pathbuf[i]);
}
free(pathbuf);
jv_mem_free(pathbuf);
pathbuf = 0;
pathsize = 0;
}
@@ -512,10 +498,8 @@ struct bytecode* jq_compile(const char* str) {
struct bytecode* bc = 0;
int nerrors = jq_parse(&locations, &program);
if (nerrors == 0) {
block_append(&program, block_join(gen_op_simple(YIELD), gen_op_simple(BACKTRACK)));
program = builtins_bind(program);
nerrors = block_compile(program, &locations, &bc);
block_free(program);
}
if (nerrors) {
fprintf(stderr, "%d compile %s\n", nerrors, nerrors > 1 ? "errors" : "error");
forkable_stack.h
+28 -18
View File
@@ -1,9 +1,9 @@
#ifndef FORKABLE_STACK_H
#define FORKABLE_STACK_H
#include <stdlib.h>
#include <stddef.h>
#include <assert.h>
#include <string.h>
#include "jv_alloc.h"
struct forkable_stack_header {
/* distance in bytes between this header and the header
@@ -15,11 +15,10 @@ struct forkable_stack_header {
struct forkable_stack {
// Stack grows down from stk+length
char* stk;
// stk+length is just past end of allocated area
// stk = malloc(length)
// stk = jv_mem_alloc(length)
int length;
// stk+pos is header of top object, or stk+length if empty
@@ -41,7 +40,7 @@ static int forkable_stack_empty(struct forkable_stack* s) {
}
static void forkable_stack_init(struct forkable_stack* s, size_t sz) {
s->stk = malloc(sz);
s->stk = jv_mem_alloc(sz);
s->length = sz;
s->pos = s->length;
s->savedlimit = s->length;
@@ -51,7 +50,7 @@ static void forkable_stack_init(struct forkable_stack* s, size_t sz) {
static void forkable_stack_free(struct forkable_stack* s) {
assert(forkable_stack_empty(s));
assert(s->savedlimit == s->length);
free(s->stk);
jv_mem_free(s->stk);
s->stk = 0;
}
@@ -61,10 +60,9 @@ static void* forkable_stack_push(struct forkable_stack* s, size_t sz_size) {
forkable_stack_check(s);
int curr = s->pos < s->savedlimit ? s->pos : s->savedlimit;
if (curr - size < 0) {
//assert(0);
int oldlen = s->length;
s->length = (size + oldlen + 1024) * 2;
s->stk = realloc(s->stk, s->length);
s->stk = jv_mem_realloc(s->stk, s->length);
int shift = s->length - oldlen;
memmove(s->stk + shift, s->stk, oldlen);
s->pos += shift;
@@ -102,33 +100,45 @@ static void* forkable_stack_peek_next(struct forkable_stack* s, void* top) {
// Returns 1 if the next forkable_stack_pop will permanently remove an
// object from the stack (i.e. the top object was not saved with a fork)
static int forkable_stack_pop_will_free(struct forkable_stack* s) {
return s->pos < s->savedlimit ? 1 : 0;
return s->pos < s->savedlimit;
}
static void forkable_stack_pop(struct forkable_stack* s) {
forkable_stack_check(s);
struct forkable_stack_header* elem = forkable_stack_peek(s);
s->pos += elem->next_delta;
int reclaim_upto = s->pos + elem->next_delta < s->savedlimit ?
s->pos + elem->next_delta : s->savedlimit;
assert(reclaim_upto <= s->length);
int reclaimed = reclaim_upto > s->pos ? reclaim_upto - s->pos : 0;
assert(0 <= reclaimed && s->pos + reclaimed <= s->length);
// s->pos <= i < reclaim_upto means that position i is to be freed
assert((reclaimed > 0) == forkable_stack_pop_will_free(s));
int new_pos = s->pos + elem->next_delta;
jv_mem_invalidate(elem, reclaimed);
s->pos = new_pos;
}
struct forkable_stack_state {
int prevpos, prevlimit;
// We save the previous pos, savedlimit as
// length-pos, length-savedlimit since these
// values are stable across stack reallocations.
int prev_pos_delta, prev_limit_delta;
};
static void forkable_stack_save(struct forkable_stack* s, struct forkable_stack_state* state) {
forkable_stack_check(s);
state->prevpos = s->pos;
state->prevlimit = s->savedlimit;
state->prev_pos_delta = s->length - s->pos;
state->prev_limit_delta = s->length - s->savedlimit;
if (s->pos < s->savedlimit) s->savedlimit = s->pos;
}
static void forkable_stack_switch(struct forkable_stack* s, struct forkable_stack_state* state) {
forkable_stack_check(s);
int curr_pos = s->pos;
s->pos = state->prevpos;
state->prevpos = curr_pos;
s->pos = s->length - state->prev_pos_delta;
state->prev_pos_delta = s->length - curr_pos;
int curr_limit = s->savedlimit;
if (curr_pos < curr_limit) s->savedlimit = curr_pos;
@@ -138,10 +148,10 @@ static void forkable_stack_switch(struct forkable_stack* s, struct forkable_stac
static void forkable_stack_restore(struct forkable_stack* s, struct forkable_stack_state* state) {
forkable_stack_check(s);
assert(s->savedlimit <= state->prevpos);
assert(s->savedlimit <= state->prevlimit);
s->pos = state->prevpos;
s->savedlimit = state->prevlimit;
assert(s->savedlimit <= s->length - state->prev_pos_delta);
assert(s->savedlimit <= s->length - state->prev_limit_delta);
s->pos = s->length - state->prev_pos_delta;
s->savedlimit = s->length - state->prev_limit_delta;
forkable_stack_check(s);
}
jq_test.c
+29 -12
View File
@@ -8,9 +8,24 @@
static void jv_test();
static void run_jq_tests();
int main() {
FILE* testdata;
int main(int argc, char* argv[]) {
jv_test();
if (argc == 1) {
testdata = fopen("testdata", "r");
} else if (argc == 2) {
if (!strcmp(argv[1], "-")) {
testdata = stdin;
} else {
testdata = fopen(argv[1], "r");
}
} else {
printf("usage: %s OR cat testdata | %s - OR %s testdata\n", argv[0], argv[0], argv[0]);
return 127;
}
run_jq_tests();
if (testdata != stdin) fclose(testdata);
}
@@ -26,7 +41,7 @@ static int skipline(const char* buf) {
static void run_jq_tests() {
FILE* testdata = NULL;
char buf[4096];
int tests = 0, passed = 0;
int tests = 0, passed = 0, invalid = 0;
testdata = fopen("testdata","r");
if ( NULL == testdata )
@@ -42,37 +57,41 @@ static void run_jq_tests() {
while (1) {
if (!fgets(buf, sizeof(buf), testdata)) break;
if (skipline(buf)) continue;
if (buf[strlen(buf)-1] == '\n') buf[strlen(buf)-1] = 0;
printf("Testing %s\n", buf);
int pass = 1;
tests++;
struct bytecode* bc = jq_compile(buf);
assert(bc);
if (!bc) {invalid++; continue;}
#if JQ_DEBUG
printf("Disassembly:\n");
dump_disassembly(2, bc);
printf("\n");
#endif
fgets(buf, sizeof(buf), testdata);
jv input = jv_parse(buf);
assert(jv_is_valid(input));
if (!jv_is_valid(input)){ invalid++; continue; }
jq_init(bc, input);
while (fgets(buf, sizeof(buf), testdata)) {
if (skipline(buf)) break;
jv expected = jv_parse(buf);
assert(jv_is_valid(expected));
if (!jv_is_valid(expected)){ invalid++; continue; }
jv actual = jq_next();
if (!jv_is_valid(actual)) {
jv_free(actual);
printf("Insufficient results\n");
printf("*** Insufficient results\n");
pass = 0;
break;
} else if (!jv_equal(jv_copy(expected), jv_copy(actual))) {
printf("Expected ");
printf("*** Expected ");
jv_dump(jv_copy(expected), 0);
printf(", but got ");
jv_dump(jv_copy(actual), 0);
printf("\n");
pass = 0;
}
jv as_string = jv_dump_string(jv_copy(expected), rand());
jv as_string = jv_dump_string(jv_copy(expected), rand() & ~JV_PRINT_COLOUR);
jv reparsed = jv_parse_sized(jv_string_value(as_string), jv_string_length(jv_copy(as_string)));
assert(jv_equal(jv_copy(expected), jv_copy(reparsed)));
jv_free(as_string);
@@ -83,7 +102,7 @@ static void run_jq_tests() {
if (pass) {
jv extra = jq_next();
if (jv_is_valid(extra)) {
printf("Superfluous result: ");
printf("*** Superfluous result: ");
jv_dump(extra, 0);
printf("\n");
pass = 0;
@@ -93,11 +112,9 @@ static void run_jq_tests() {
}
jq_teardown();
bytecode_free(bc);
tests++;
passed+=pass;
}
fclose(testdata);
printf("%d of %d tests passed\n", passed,tests);
printf("%d of %d tests passed (%d malformed)\n", passed,tests,invalid);
if (passed != tests) exit(1);
}
jv.c
+26 -31
View File
@@ -6,6 +6,7 @@
#include <string.h>
#include <stdarg.h>
#include "jv_alloc.h"
#include "jv.h"
/*
@@ -86,7 +87,7 @@ jv jv_invalid_with_msg(jv err) {
jv x;
x.kind = JV_KIND_INVALID;
x.val.complex.i[0] = x.val.complex.i[1] = 0;
jvp_invalid* i = malloc(sizeof(jvp_invalid));
jvp_invalid* i = jv_mem_alloc(sizeof(jvp_invalid));
x.val.complex.ptr = &i->refcnt;
i->refcnt.count = 1;
i->errmsg = err;
@@ -113,7 +114,7 @@ int jv_invalid_has_msg(jv inv) {
static void jvp_invalid_free(jv_complex* x) {
if (jvp_refcnt_dec(x)) {
jv_free(((jvp_invalid*)x->ptr)->errmsg);
free(x->ptr);
jv_mem_free(x->ptr);
}
}
@@ -157,7 +158,7 @@ static jvp_array* jvp_array_ptr(jv_complex* a) {
}
static jvp_array* jvp_array_alloc(unsigned size) {
jvp_array* a = malloc(sizeof(jvp_array) + sizeof(jv) * size);
jvp_array* a = jv_mem_alloc(sizeof(jvp_array) + sizeof(jv) * size);
a->refcnt.count = 1;
a->length = 0;
a->alloc_length = size;
@@ -175,7 +176,7 @@ static void jvp_array_free(jv_complex* a) {
for (int i=0; i<array->length; i++) {
jv_free(array->elements[i]);
}
free(array);
jv_mem_free(array);
}
}
@@ -253,8 +254,7 @@ static int jvp_array_equal(jv_complex* a, jv_complex* b) {
static jv_complex jvp_array_slice(jv_complex* a, int start, int end) {
// FIXME: maybe slice should reallocate if the slice is small enough
assert(start <= end);
jvp_array* array = jvp_array_ptr(a);
assert(a->i[1] + end < array->length);
assert(a->i[0] + end <= a->i[1]);
jv_complex slice = *a;
slice.i[0] += start;
slice.i[1] = slice.i[0] + (end - start);
@@ -315,8 +315,8 @@ jv jv_array_concat(jv a, jv b) {
assert(jv_get_kind(b) == JV_KIND_ARRAY);
// FIXME: could be much faster
for (int i=0; i<jv_array_length(jv_copy(b)); i++) {
a = jv_array_append(a, jv_array_get(jv_copy(b), i));
jv_array_foreach(b, i, elem) {
a = jv_array_append(a, elem);
}
jv_free(b);
return a;
@@ -331,16 +331,15 @@ jv jv_array_slice(jv a, int start, int end) {
int jv_array_contains(jv a, jv b) {
int r = 1;
int a_length = jv_array_length(jv_copy(a));
int b_length = jv_array_length(jv_copy(b));
for (int bi = 0; bi < b_length; bi++) {
jv_array_foreach(b, bi, belem) {
int ri = 0;
for (int ai = 0; ai < a_length; ai++) {
if (jv_contains(jv_array_get(jv_copy(a), ai), jv_array_get(jv_copy(b), bi))) {
jv_array_foreach(a, ai, aelem) {
if (jv_contains(aelem, jv_copy(belem))) {
ri = 1;
break;
}
}
jv_free(belem);
if (!ri) {
r = 0;
break;
@@ -371,7 +370,7 @@ static jvp_string* jvp_string_ptr(jv_complex* a) {
}
static jvp_string* jvp_string_alloc(uint32_t size) {
jvp_string* s = malloc(sizeof(jvp_string) + size + 1);
jvp_string* s = jv_mem_alloc(sizeof(jvp_string) + size + 1);
s->refcnt.count = 1;
s->alloc_length = size;
return s;
@@ -389,7 +388,7 @@ static jv_complex jvp_string_new(const char* data, uint32_t length) {
static void jvp_string_free(jv_complex* s) {
if (jvp_refcnt_dec(s)) {
jvp_string* str = jvp_string_ptr(s);
free(str);
jv_mem_free(str);
}
}
@@ -569,17 +568,17 @@ jv jv_string_append_str(jv a, const char* str) {
jv jv_string_fmt(const char* fmt, ...) {
int size = 1024;
while (1) {
char* buf = malloc(size);
char* buf = jv_mem_alloc(size);
va_list args;
va_start(args, fmt);
int n = vsnprintf(buf, size, fmt, args);
va_end(args);
if (n < size) {
jv ret = jv_string_sized(buf, n);
free(buf);
jv_mem_free(buf);
return ret;
} else {
free(buf);
jv_mem_free(buf);
size = n * 2;
}
}
@@ -609,9 +608,9 @@ static jv_complex jvp_object_new(int size) {
// size must be a power of two
assert(size > 0 && (size & (size - 1)) == 0);
jvp_object* obj = malloc(sizeof(jvp_object) +
sizeof(struct object_slot) * size +
sizeof(int) * (size * 2));
jvp_object* obj = jv_mem_alloc(sizeof(jvp_object) +
sizeof(struct object_slot) * size +
sizeof(int) * (size * 2));
obj->refcnt.count = 1;
for (int i=0; i<size; i++) {
obj->elements[i].next = i - 1;
@@ -711,7 +710,7 @@ static void jvp_object_free(jv_complex* o) {
jv_free(slot->value);
}
}
free(jvp_object_ptr(o));
jv_mem_free(jvp_object_ptr(o));
}
}
@@ -730,7 +729,7 @@ static void jvp_object_rehash(jv_complex* object) {
new_slot->value = slot->value;
}
// references are transported, just drop the old table
free(jvp_object_ptr(object));
jv_mem_free(jvp_object_ptr(object));
*object = new_object;
}
@@ -876,10 +875,8 @@ int jv_object_length(jv object) {
jv jv_object_merge(jv a, jv b) {
assert(jv_get_kind(a) == JV_KIND_OBJECT);
jv_object_foreach(i, b) {
a = jv_object_set(a,
jv_object_iter_key(b, i),
jv_object_iter_value(b, i));
jv_object_foreach(b, k, v) {
a = jv_object_set(a, k, v);
}
jv_free(b);
return a;
@@ -890,10 +887,8 @@ int jv_object_contains(jv a, jv b) {
assert(jv_get_kind(b) == JV_KIND_OBJECT);
int r = 1;
jv_object_foreach(i, b) {
jv key = jv_object_iter_key(b, i);
jv_object_foreach(b, key, b_val) {
jv a_val = jv_object_get(jv_copy(a), jv_copy(key));
jv b_val = jv_object_get(jv_copy(b), jv_copy(key));
r = jv_contains(a_val, b_val);
jv_free(key);
@@ -1037,4 +1032,4 @@ int jv_contains(jv a, jv b) {
jv_free(a);
jv_free(b);
return r;
}
}
jv.h
+17 -67
View File
@@ -72,7 +72,13 @@ jv jv_array_set(jv, int, jv);
jv jv_array_append(jv, jv);
jv jv_array_concat(jv, jv);
jv jv_array_slice(jv, int, int);
#define jv_array_foreach(a, i, x) \
for (int jv_len__ = jv_array_length(jv_copy(a)), i=0, jv_j__ = 1; \
jv_j__; jv_j__ = 0) \
for (jv x; \
i < jv_len__ ? \
(x = jv_array_get(jv_copy(a), i), 1) : 0; \
i++)
jv jv_string(const char*);
jv jv_string_sized(const char*, int);
@@ -96,16 +102,21 @@ int jv_object_iter_next(jv, int);
int jv_object_iter_valid(jv, int);
jv jv_object_iter_key(jv, int);
jv jv_object_iter_value(jv, int);
#define jv_object_foreach(i,t) \
for (int i = jv_object_iter(t); \
jv_object_iter_valid(t, i); \
i = jv_object_iter_next(t, i)) \
#define jv_object_foreach(t, k, v) \
for (int jv_i__ = jv_object_iter(t), jv_j__ = 1; jv_j__; jv_j__ = 0) \
for (jv k, v; \
jv_object_iter_valid((t), jv_i__) ? \
(k = jv_object_iter_key(t, jv_i__), \
v = jv_object_iter_value(t, jv_i__), \
1) \
: 0; \
jv_i__ = jv_object_iter_next(t, jv_i__)) \
int jv_get_refcnt(jv);
enum { JV_PRINT_PRETTY = 1, JV_PRINT_ASCII = 2 };
enum { JV_PRINT_PRETTY = 1, JV_PRINT_ASCII = 2, JV_PRINT_COLOUR = 4 };
void jv_dump(jv, int flags);
jv jv_dump_string(jv, int flags);
@@ -113,67 +124,6 @@ jv jv_parse(const char* string);
jv jv_parse_sized(const char* string, int length);
static jv jv_lookup(jv t, jv k) {
jv v;
if (jv_get_kind(t) == JV_KIND_OBJECT && jv_get_kind(k) == JV_KIND_STRING) {
v = jv_object_get(t, k);
if (!jv_is_valid(v)) {
jv_free(v);
v = jv_null();
}
} else if (jv_get_kind(t) == JV_KIND_ARRAY && jv_get_kind(k) == JV_KIND_NUMBER) {
// FIXME: don't do lookup for noninteger index
v = jv_array_get(t, (int)jv_number_value(k));
if (!jv_is_valid(v)) {
jv_free(v);
v = jv_null();
}
} else if (jv_get_kind(t) == JV_KIND_NULL &&
(jv_get_kind(k) == JV_KIND_STRING || jv_get_kind(k) == JV_KIND_NUMBER)) {
jv_free(t);
jv_free(k);
v = jv_null();
} else {
v = jv_invalid_with_msg(jv_string_fmt("Cannot index %s with %s",
jv_kind_name(jv_get_kind(t)),
jv_kind_name(jv_get_kind(k))));
jv_free(t);
jv_free(k);
}
return v;
}
static jv jv_modify(jv t, jv k, jv v) {
int isnull = jv_get_kind(t) == JV_KIND_NULL;
if (jv_get_kind(k) == JV_KIND_STRING &&
(jv_get_kind(t) == JV_KIND_OBJECT || isnull)) {
if (isnull) t = jv_object();
t = jv_object_set(t, k, v);
} else if (jv_get_kind(k) == JV_KIND_NUMBER &&
(jv_get_kind(t) == JV_KIND_ARRAY || isnull)) {
if (isnull) t = jv_array();
t = jv_array_set(t, (int)jv_number_value(k), v);
} else {
jv err = jv_invalid_with_msg(jv_string_fmt("Cannot update field at %s index of %s",
jv_kind_name(jv_get_kind(t)),
jv_kind_name(jv_get_kind(v))));
jv_free(t);
jv_free(k);
jv_free(v);
t = err;
}
return t;
}
static jv jv_insert(jv root, jv value, jv* path, int pathlen) {
if (pathlen == 0) {
jv_free(root);
return value;
}
return jv_modify(root, jv_copy(*path),
jv_insert(jv_lookup(jv_copy(root), jv_copy(*path)), value, path+1, pathlen-1));
}
jv_alloc.c
+37
View File
@@ -0,0 +1,37 @@
#include <stdlib.h>
#include <stdio.h>
#include "jv_alloc.h"
static void memory_exhausted() {
fprintf(stderr, "error: cannot allocate memory\n");
abort();
}
void* jv_mem_alloc(size_t sz) {
void* p = malloc(sz);
if (!p) {
memory_exhausted();
}
return p;
}
void jv_mem_free(void* p) {
free(p);
}
void* jv_mem_realloc(void* p, size_t sz) {
p = realloc(p, sz);
if (!p) {
memory_exhausted();
}
return p;
}
#if JQ_DEBUG
volatile char jv_mem_uninitialised;
__attribute__((constructor)) void jv_mem_uninit_setup(){
char* p = malloc(1);
jv_mem_uninitialised = *p;
free(p);
}
#endif
jv_alloc.h
+21
View File
@@ -0,0 +1,21 @@
#ifndef JV_ALLOC_H
#define JV_ALLOC_H
#include <stddef.h>
#if JQ_DEBUG
extern volatile char jv_mem_uninitialised;
#endif
static void jv_mem_invalidate(void* mem, size_t n) {
#if JQ_DEBUG
char* m = mem;
while (n--) *m++ ^= jv_mem_uninitialised ^ jv_mem_uninitialised;
#endif
}
void* jv_mem_alloc(size_t);
void jv_mem_free(void*);
__attribute__((warn_unused_result)) void* jv_mem_realloc(void*, size_t);
#endif
jv_aux.c
+428
View File
@@ -0,0 +1,428 @@
#include "jv_aux.h"
#include <string.h>
#include <stdlib.h>
#include "jv_alloc.h"
jv jv_get(jv t, jv k) {
jv v;
if (jv_get_kind(t) == JV_KIND_OBJECT && jv_get_kind(k) == JV_KIND_STRING) {
v = jv_object_get(t, k);
if (!jv_is_valid(v)) {
jv_free(v);
v = jv_null();
}
} else if (jv_get_kind(t) == JV_KIND_ARRAY && jv_get_kind(k) == JV_KIND_NUMBER) {
// FIXME: don't do lookup for noninteger index
v = jv_array_get(t, (int)jv_number_value(k));
if (!jv_is_valid(v)) {
jv_free(v);
v = jv_null();
}
} else if (jv_get_kind(t) == JV_KIND_NULL &&
(jv_get_kind(k) == JV_KIND_STRING || jv_get_kind(k) == JV_KIND_NUMBER)) {
jv_free(t);
jv_free(k);
v = jv_null();
} else {
v = jv_invalid_with_msg(jv_string_fmt("Cannot index %s with %s",
jv_kind_name(jv_get_kind(t)),
jv_kind_name(jv_get_kind(k))));
jv_free(t);
jv_free(k);
}
return v;
}
jv jv_set(jv t, jv k, jv v) {
if (!jv_is_valid(v)) {
jv_free(t);
jv_free(k);
return v;
}
int isnull = jv_get_kind(t) == JV_KIND_NULL;
if (jv_get_kind(k) == JV_KIND_STRING &&
(jv_get_kind(t) == JV_KIND_OBJECT || isnull)) {
if (isnull) t = jv_object();
t = jv_object_set(t, k, v);
} else if (jv_get_kind(k) == JV_KIND_NUMBER &&
(jv_get_kind(t) == JV_KIND_ARRAY || isnull)) {
if (isnull) t = jv_array();
t = jv_array_set(t, (int)jv_number_value(k), v);
} else {
jv err = jv_invalid_with_msg(jv_string_fmt("Cannot update field at %s index of %s",
jv_kind_name(jv_get_kind(t)),
jv_kind_name(jv_get_kind(v))));
jv_free(t);
jv_free(k);
jv_free(v);
t = err;
}
return t;
}
// assumes keys is a sorted array
jv jv_dels(jv t, jv keys) {
assert(jv_get_kind(keys) == JV_KIND_ARRAY);
assert(jv_is_valid(t));
if (jv_get_kind(t) == JV_KIND_NULL || jv_array_length(jv_copy(keys)) == 0) {
// no change
} else if (jv_get_kind(t) == JV_KIND_ARRAY) {
jv new_array = jv_array();
int kidx = 0;
jv_array_foreach(t, i, elem) {
int del = 0;
while (kidx < jv_array_length(jv_copy(keys))) {
jv nextdel = jv_array_get(jv_copy(keys), kidx);
if (jv_get_kind(nextdel) != JV_KIND_NUMBER) {
jv err = jv_invalid_with_msg(jv_string_fmt("Cannot delete %s element of array",
jv_kind_name(jv_get_kind(nextdel))));
jv_free(nextdel);
jv_free(new_array);
jv_free(elem);
new_array = err;
goto arr_out; // break twice
}
int delidx = (int)jv_number_value(nextdel);
jv_free(nextdel);
if (i == delidx) {
del = 1;
}
if (i < delidx) {
break;
}
kidx++;
}
if (!del)
new_array = jv_array_append(new_array, elem);
else
jv_free(elem);
}
arr_out:
jv_free(t);
t = new_array;
} else if (jv_get_kind(t) == JV_KIND_OBJECT) {
jv_array_foreach(keys, i, k) {
if (jv_get_kind(k) != JV_KIND_STRING) {
jv_free(t);
t = jv_invalid_with_msg(jv_string_fmt("Cannot delete %s field of object",
jv_kind_name(jv_get_kind(k))));
jv_free(k);
break;
}
t = jv_object_delete(t, k);
}
} else {
jv err = jv_invalid_with_msg(jv_string_fmt("Cannot delete fields from %s",
jv_kind_name(jv_get_kind(t))));
jv_free(t);
t = err;
}
jv_free(keys);
return t;
}
jv jv_setpath(jv root, jv path, jv value) {
if (jv_get_kind(path) != JV_KIND_ARRAY) {
jv_free(value);
jv_free(root);
jv_free(path);
return jv_invalid_with_msg(jv_string("Path must be specified as an array"));
}
if (!jv_is_valid(root)){
jv_free(value);
jv_free(path);
return root;
}
if (jv_array_length(jv_copy(path)) == 0) {
jv_free(path);
jv_free(root);
return value;
}
jv pathcurr = jv_array_get(jv_copy(path), 0);
jv pathrest = jv_array_slice(path, 1, jv_array_length(jv_copy(path)));
return jv_set(root, pathcurr,
jv_setpath(jv_get(jv_copy(root), jv_copy(pathcurr)), pathrest, value));
}
jv jv_getpath(jv root, jv path) {
if (jv_get_kind(path) != JV_KIND_ARRAY) {
jv_free(root);
jv_free(path);
return jv_invalid_with_msg(jv_string("Path must be specified as an array"));
}
if (!jv_is_valid(root)) {
jv_free(path);
return root;
}
if (jv_array_length(jv_copy(path)) == 0) {
jv_free(path);
return root;
}
jv pathcurr = jv_array_get(jv_copy(path), 0);
jv pathrest = jv_array_slice(path, 1, jv_array_length(jv_copy(path)));
return jv_getpath(jv_get(root, pathcurr), pathrest);
}
// assumes paths is a sorted array of arrays
static jv delpaths_sorted(jv object, jv paths, int start) {
jv delkeys = jv_array();
for (int i=0; i<jv_array_length(jv_copy(paths));) {
int j = i;
assert(jv_array_length(jv_array_get(jv_copy(paths), i)) > start);
int delkey = jv_array_length(jv_array_get(jv_copy(paths), i)) == start + 1;
jv key = jv_array_get(jv_array_get(jv_copy(paths), i), start);
while (j < jv_array_length(jv_copy(paths)) &&
jv_equal(jv_copy(key), jv_array_get(jv_array_get(jv_copy(paths), j), start)))
j++;
// if i <= entry < j, then entry starts with key
if (delkey) {
// deleting this entire key, we don't care about any more specific deletions
delkeys = jv_array_append(delkeys, key);
} else {
// deleting certain sub-parts of this key
jv subobject = jv_get(jv_copy(object), jv_copy(key));
if (!jv_is_valid(subobject)) {
jv_free(key);
jv_free(object);
object = subobject;
break;
} else if (jv_get_kind(subobject) == JV_KIND_NULL) {
jv_free(key);
jv_free(subobject);
} else {
jv newsubobject = delpaths_sorted(subobject, jv_array_slice(jv_copy(paths), i, j), start+1);
if (!jv_is_valid(newsubobject)) {
jv_free(key);
jv_free(object);
object = newsubobject;
break;
}
object = jv_set(object, key, newsubobject);
}
if (!jv_is_valid(object)) break;
}
i = j;
}
jv_free(paths);
if (jv_is_valid(object))
object = jv_dels(object, delkeys);
else
jv_free(delkeys);
return object;
}
jv jv_delpaths(jv object, jv paths) {
paths = jv_sort(paths, jv_copy(paths));
jv_array_foreach(paths, i, elem) {
if (jv_get_kind(elem) != JV_KIND_ARRAY) {
jv_free(object);
jv_free(paths);
jv err = jv_invalid_with_msg(jv_string_fmt("Path must be specified as array, not %s",
jv_kind_name(jv_get_kind(elem))));
jv_free(elem);
return err;
}
jv_free(elem);
}
if (jv_array_length(jv_copy(paths)) == 0) {
// nothing is being deleted
jv_free(paths);
return object;
}
if (jv_array_length(jv_array_get(jv_copy(paths), 0)) == 0) {
// everything is being deleted
jv_free(paths);
jv_free(object);
return jv_null();
}
return delpaths_sorted(object, paths, 0);
}
static int string_cmp(const void* pa, const void* pb){
const jv* a = pa;
const jv* b = pb;
int lena = jv_string_length(jv_copy(*a));
int lenb = jv_string_length(jv_copy(*b));
int minlen = lena < lenb ? lena : lenb;
int r = memcmp(jv_string_value(*a), jv_string_value(*b), minlen);
if (r == 0) r = lena - lenb;
return r;
}
jv jv_keys(jv x) {
if (jv_get_kind(x) == JV_KIND_OBJECT) {
int nkeys = jv_object_length(jv_copy(x));
jv* keys = jv_mem_alloc(sizeof(jv) * nkeys);
int kidx = 0;
jv_object_foreach(x, key, value) {
keys[kidx++] = key;
jv_free(value);
}
qsort(keys, nkeys, sizeof(jv), string_cmp);
jv answer = jv_array_sized(nkeys);
for (int i = 0; i<nkeys; i++) {
answer = jv_array_append(answer, keys[i]);
}
jv_mem_free(keys);
jv_free(x);
return answer;
} else if (jv_get_kind(x) == JV_KIND_ARRAY) {
int n = jv_array_length(x);
jv answer = jv_array();
for (int i=0; i<n; i++){
answer = jv_array_set(answer, i, jv_number(i));
}
return answer;
} else {
assert(0 && "jv_keys passed something neither object nor array");
return jv_invalid();
}
}
int jv_cmp(jv a, jv b) {
if (jv_get_kind(a) != jv_get_kind(b)) {
int r = (int)jv_get_kind(a) - (int)jv_get_kind(b);
jv_free(a);
jv_free(b);
return r;
}
int r = 0;
switch (jv_get_kind(a)) {
default:
assert(0 && "invalid kind passed to jv_cmp");
case JV_KIND_NULL:
case JV_KIND_FALSE:
case JV_KIND_TRUE:
// there's only one of each of these values
r = 0;
break;
case JV_KIND_NUMBER: {
double da = jv_number_value(a), db = jv_number_value(b);
// handle NaN as though it were null
if (da != da) r = jv_cmp(jv_null(), jv_number(db));
else if (db != db) r = jv_cmp(jv_number(da), jv_null());
else if (da < db) r = -1;
else if (da == db) r = 0;
else r = 1;
break;
}
case JV_KIND_STRING: {
r = string_cmp(&a, &b);
break;
}
case JV_KIND_ARRAY: {
// Lexical ordering of arrays
int i = 0;
while (r == 0) {
int a_done = i >= jv_array_length(jv_copy(a));
int b_done = i >= jv_array_length(jv_copy(b));
if (a_done || b_done) {
r = b_done - a_done; //suddenly, logic
break;
}
jv xa = jv_array_get(jv_copy(a), i);
jv xb = jv_array_get(jv_copy(b), i);
r = jv_cmp(xa, xb);
i++;
}
break;
}
case JV_KIND_OBJECT: {
jv keys_a = jv_keys(jv_copy(a));
jv keys_b = jv_keys(jv_copy(b));
r = jv_cmp(jv_copy(keys_a), keys_b);
if (r == 0) {
jv_array_foreach(keys_a, i, key) {
jv xa = jv_object_get(jv_copy(a), jv_copy(key));
jv xb = jv_object_get(jv_copy(b), key);
r = jv_cmp(xa, xb);
if (r) break;
}
}
jv_free(keys_a);
break;
}
}
jv_free(a);
jv_free(b);
return r;
}
struct sort_entry {
jv object;
jv key;
};
static int sort_cmp(const void* pa, const void* pb) {
const struct sort_entry* a = pa;
const struct sort_entry* b = pb;
int r = jv_cmp(jv_copy(a->key), jv_copy(b->key));
// comparing by address if r == 0 makes the sort stable
return r ? r : (int)(a - b);
}
static struct sort_entry* sort_items(jv objects, jv keys) {
assert(jv_get_kind(objects) == JV_KIND_ARRAY);
assert(jv_get_kind(keys) == JV_KIND_ARRAY);
assert(jv_array_length(jv_copy(objects)) == jv_array_length(jv_copy(keys)));
int n = jv_array_length(jv_copy(objects));
struct sort_entry* entries = jv_mem_alloc(sizeof(struct sort_entry) * n);
for (int i=0; i<n; i++) {
entries[i].object = jv_array_get(jv_copy(objects), i);
entries[i].key = jv_array_get(jv_copy(keys), i);
}
jv_free(objects);
jv_free(keys);
qsort(entries, n, sizeof(struct sort_entry), sort_cmp);
return entries;
}
jv jv_sort(jv objects, jv keys) {
assert(jv_get_kind(objects) == JV_KIND_ARRAY);
assert(jv_get_kind(keys) == JV_KIND_ARRAY);
assert(jv_array_length(jv_copy(objects)) == jv_array_length(jv_copy(keys)));
int n = jv_array_length(jv_copy(objects));
struct sort_entry* entries = sort_items(objects, keys);
jv ret = jv_array();
for (int i=0; i<n; i++) {
jv_free(entries[i].key);
ret = jv_array_set(ret, i, entries[i].object);
}
jv_mem_free(entries);
return ret;
}
jv jv_group(jv objects, jv keys) {
assert(jv_get_kind(objects) == JV_KIND_ARRAY);
assert(jv_get_kind(keys) == JV_KIND_ARRAY);
assert(jv_array_length(jv_copy(objects)) == jv_array_length(jv_copy(keys)));
int n = jv_array_length(jv_copy(objects));
struct sort_entry* entries = sort_items(objects, keys);
jv ret = jv_array();
jv curr_key = entries[0].key;
jv group = jv_array_append(jv_array(), entries[0].object);
for (int i = 1; i < n; i++) {
if (jv_equal(jv_copy(curr_key), jv_copy(entries[i].key))) {
jv_free(entries[i].key);
} else {
jv_free(curr_key);
curr_key = entries[i].key;
ret = jv_array_append(ret, group);
group = jv_array();
}
group = jv_array_append(group, entries[i].object);
}
jv_free(curr_key);
ret = jv_array_append(ret, group);
jv_mem_free(entries);
return ret;
}
jv_aux.h
+19
View File
@@ -0,0 +1,19 @@
#ifndef JV_AUX_H
#define JV_AUX_H
#include "jv.h"
jv jv_get(jv t, jv k);
jv jv_set(jv t, jv k, jv v);
jv jv_setpath(jv root, jv path, jv value);
jv jv_getpath(jv root, jv path);
jv jv_delpaths(jv root, jv paths);
jv jv_keys(jv /*object or array*/);
int jv_cmp(jv, jv);
jv jv_group(jv objects, jv keys);
jv jv_sort(jv objects, jv keys);
#endif
jv_dtoa.c
+6 -1
View File
@@ -195,6 +195,11 @@
#include "jv_dtoa.h"
#include "jv_alloc.h"
#define MALLOC jv_mem_alloc
#define FREE jv_mem_free
#ifndef Long
#define Long long
@@ -572,7 +577,7 @@ void jvp_dtoa_context_free(struct dtoa_context* C) {
while (C->freelist[k]) {
Bigint* v = C->freelist[k];
C->freelist[k] = v->next;
free(v);
FREE(v);
}
}
}
jv_parse.c
+33 -5
View File
@@ -5,6 +5,7 @@
#include "jv_dtoa.h"
#include "jv_parse.h"
#include "jv_unicode.h"
#include "jv_alloc.h"
typedef const char* presult;
@@ -24,6 +25,9 @@ void jv_parser_init(struct jv_parser* p) {
p->st = JV_PARSER_NORMAL;
p->curr_buf = 0;
p->curr_buf_length = p->curr_buf_pos = p->curr_buf_is_partial = 0;
p->bom_strip_position = 0;
p->line = 1;
p->column = 0;
jvp_dtoa_context_init(&p->dtoa);
}
@@ -31,8 +35,8 @@ void jv_parser_free(struct jv_parser* p) {
jv_free(p->next);
for (int i=0; i<p->stackpos; i++)
jv_free(p->stack[i]);
free(p->stack);
free(p->tokenbuf);
jv_mem_free(p->stack);
jv_mem_free(p->tokenbuf);
jvp_dtoa_context_free(&p->dtoa);
}
@@ -47,7 +51,7 @@ static void push(struct jv_parser* p, jv v) {
assert(p->stackpos <= p->stacklen);
if (p->stackpos == p->stacklen) {
p->stacklen = p->stacklen * 2 + 10;
p->stack = realloc(p->stack, p->stacklen * sizeof(jv));
p->stack = jv_mem_realloc(p->stack, p->stacklen * sizeof(jv));
}
assert(p->stackpos < p->stacklen);
p->stack[p->stackpos++] = v;
@@ -141,7 +145,7 @@ static void tokenadd(struct jv_parser* p, char c) {
assert(p->tokenpos <= p->tokenlen);
if (p->tokenpos == p->tokenlen) {
p->tokenlen = p->tokenlen*2 + 256;
p->tokenbuf = realloc(p->tokenbuf, p->tokenlen);
p->tokenbuf = jv_mem_realloc(p->tokenbuf, p->tokenlen);
}
assert(p->tokenpos < p->tokenlen);
p->tokenbuf[p->tokenpos++] = c;
@@ -292,6 +296,11 @@ static int check_done(struct jv_parser* p, jv* out) {
}
static pfunc scan(struct jv_parser* p, char ch, jv* out) {
p->column++;
if (ch == '\n') {
p->line++;
p->column = 0;
}
presult answer = 0;
if (p->st == JV_PARSER_NORMAL) {
chclass cls = classify(ch);
@@ -332,9 +341,27 @@ static pfunc scan(struct jv_parser* p, char ch, jv* out) {
return answer;
}
static unsigned char UTF8_BOM[] = {0xEF,0xBB,0xBF};
void jv_parser_set_buf(struct jv_parser* p, const char* buf, int length, int is_partial) {
assert((p->curr_buf == 0 || p->curr_buf_pos == p->curr_buf_length)
&& "previous buffer not exhausted");
while (p->bom_strip_position < sizeof(UTF8_BOM)) {
if ((unsigned char)*buf == UTF8_BOM[p->bom_strip_position]) {
// matched a BOM character
buf++;
length--;
p->bom_strip_position++;
} else {
if (p->bom_strip_position == 0) {
// no BOM in this document
p->bom_strip_position = sizeof(UTF8_BOM);
} else {
// malformed BOM (prefix present, rest missing)
p->bom_strip_position = 0xff;
}
}
}
p->curr_buf = buf;
p->curr_buf_length = length;
p->curr_buf_pos = 0;
@@ -343,6 +370,7 @@ void jv_parser_set_buf(struct jv_parser* p, const char* buf, int length, int is_
jv jv_parser_next(struct jv_parser* p) {
assert(p->curr_buf && "a buffer must be provided");
if (p->bom_strip_position == 0xff) return jv_invalid_with_msg(jv_string("Malformed BOM"));
jv value;
presult msg = 0;
while (!msg && p->curr_buf_pos < p->curr_buf_length) {
@@ -352,7 +380,7 @@ jv jv_parser_next(struct jv_parser* p) {
if (msg == OK) {
return value;
} else if (msg) {
return jv_invalid_with_msg(jv_string(msg));
return jv_invalid_with_msg(jv_string_fmt("%s at line %d, column %d", msg, p->line, p->column));
} else if (p->curr_buf_is_partial) {
assert(p->curr_buf_pos == p->curr_buf_length);
// need another buffer
jv_parse.h
+3
View File
@@ -6,6 +6,7 @@ struct jv_parser {
int curr_buf_length;
int curr_buf_pos;
int curr_buf_is_partial;
unsigned bom_strip_position;
jv* stack;
int stackpos;
@@ -15,6 +16,8 @@ struct jv_parser {
char* tokenbuf;
int tokenpos;
int tokenlen;
int line, column;
struct dtoa_context dtoa;
jv_print.c
+50 -10
View File
@@ -6,6 +6,20 @@
#include "jv_dtoa.h"
#include "jv_unicode.h"
#define ESC "\033"
#define COL(c) (ESC "[" c "m")
#define COLRESET (ESC "[0m")
// Colour table. See http://en.wikipedia.org/wiki/ANSI_escape_code#Colors
// for how to choose these.
static jv_kind colour_kinds[] =
{JV_KIND_NULL, JV_KIND_FALSE, JV_KIND_TRUE, JV_KIND_NUMBER,
JV_KIND_STRING, JV_KIND_ARRAY, JV_KIND_OBJECT};
static const char* colours[] =
{COL("30;1"), COL("0"), COL("0"), COL("0"),
COL("32"), COL("37"), COL("37")};
#define FIELD_COLOUR COL("34;1")
static void put_buf(const char* s, int len, FILE* fout, jv* strout) {
if (strout) {
*strout = jv_string_append_buf(*strout, s, len);
@@ -35,6 +49,7 @@ static void jvp_dump_string(jv str, int ascii_only, FILE* F, jv* S) {
const char* cstart;
int c = 0;
char buf[32];
put_char('"', F, S);
while ((i = jvp_utf8_next((cstart = i), end, &c))) {
assert(c != -1);
int unicode_escape = 0;
@@ -91,12 +106,23 @@ static void jvp_dump_string(jv str, int ascii_only, FILE* F, jv* S) {
}
}
assert(c != -1);
put_char('"', F, S);
}
enum { INDENT = 2 };
static void jv_dump_term(struct dtoa_context* C, jv x, int flags, int indent, FILE* F, jv* S) {
char buf[JVP_DTOA_FMT_MAX_LEN];
const char* colour = 0;
if (flags & JV_PRINT_COLOUR) {
for (unsigned i=0; i<sizeof(colour_kinds)/sizeof(colour_kinds[0]); i++) {
if (jv_get_kind(x) == colour_kinds[i]) {
colour = colours[i];
put_str(colour, F, S);
break;
}
}
}
switch (jv_get_kind(x)) {
case JV_KIND_INVALID:
assert(0 && "Invalid value");
@@ -124,9 +150,7 @@ static void jv_dump_term(struct dtoa_context* C, jv x, int flags, int indent, FI
break;
}
case JV_KIND_STRING:
put_char('"', F, S);
jvp_dump_string(x, flags & JV_PRINT_ASCII, F, S);
put_char('"', F, S);
break;
case JV_KIND_ARRAY: {
if (jv_array_length(jv_copy(x)) == 0) {
@@ -138,21 +162,23 @@ static void jv_dump_term(struct dtoa_context* C, jv x, int flags, int indent, FI
put_char('\n', F, S);
put_space(indent+INDENT, F, S);
}
for (int i=0; i<jv_array_length(jv_copy(x)); i++) {
jv_array_foreach(x, i, elem) {
if (i!=0) {
if (flags & JV_PRINT_PRETTY) {
put_str(",\n", F, S);
put_space(indent+INDENT, F, S);
} else {
put_str(", ", F, S);
put_str(",", F, S);
}
}
jv_dump_term(C, jv_array_get(jv_copy(x), i), flags, indent + INDENT, F, S);
jv_dump_term(C, elem, flags, indent + INDENT, F, S);
if (colour) put_str(colour, F, S);
}
if (flags & JV_PRINT_PRETTY) {
put_char('\n', F, S);
put_space(indent, F, S);
}
if (colour) put_str(colour, F, S);
put_char(']', F, S);
break;
}
@@ -167,28 +193,42 @@ static void jv_dump_term(struct dtoa_context* C, jv x, int flags, int indent, FI
put_space(indent+INDENT, F, S);
}
int first = 1;
jv_object_foreach(i, x) {
jv_object_foreach(x, key, value) {
if (!first) {
if (flags & JV_PRINT_PRETTY){
put_str(",\n", F, S);
put_space(indent+INDENT, F, S);
} else {
put_str(", ", F, S);
put_str(",", F, S);
}
}
if (colour) put_str(COLRESET, F, S);
first = 0;
jv_dump_term(C, jv_object_iter_key(x, i), flags, indent + INDENT, F, S);
put_str(": ", F, S);
jv_dump_term(C, jv_object_iter_value(x, i), flags, indent + INDENT, F, S);
if (colour) put_str(FIELD_COLOUR, F, S);
jvp_dump_string(key, flags & JV_PRINT_ASCII, F, S);
jv_free(key);
if (colour) put_str(COLRESET, F, S);
if (colour) put_str(colour, F, S);
put_str((flags & JV_PRINT_PRETTY) ? ": " : ":", F, S);
if (colour) put_str(COLRESET, F, S);
jv_dump_term(C, value, flags, indent + INDENT, F, S);
if (colour) put_str(colour, F, S);
}
if (flags & JV_PRINT_PRETTY) {
put_char('\n', F, S);
put_space(indent, F, S);
}
if (colour) put_str(colour, F, S);
put_char('}', F, S);
}
}
jv_free(x);
if (colour) {
put_str(COLRESET, F, S);
}
}
void jv_dump(jv x, int flags) {
lexer.l
+21 -2
View File
@@ -1,4 +1,5 @@
%{
#include "jv_alloc.h"
#include "compile.h"
struct lexer_param;
@@ -25,6 +26,7 @@ struct lexer_param;
%}
%option noyywrap nounput noinput nodefault
%option noyyalloc noyyrealloc noyyfree
%option reentrant
%option extra-type="int"
%option bison-bridge bison-locations
@@ -47,6 +49,7 @@ struct lexer_param;
"and" { return AND; }
"or" { return OR; }
"end" { return END; }
"fold" { return FOLD; }
"//" { return DEFINEDOR; }
"|=" { return SETPIPE; }
"+=" { return SETPLUS; }
@@ -56,7 +59,6 @@ struct lexer_param;
"//=" { return SETDEFINEDOR; }
"<=" { return LESSEQ; }
">=" { return GREATEREQ; }
"contains" { return CONTAINS; }
"."|"="|";"|","|":"|"|"|"+"|"-"|"*"|"/"|"\$"|"<"|">" { return yytext[0];}
"["|"{"|"(" {
@@ -67,8 +69,11 @@ struct lexer_param;
return try_exit(yytext[0], YY_START, yyscanner);
}
"@"[a-zA-Z0-9_]+ {
yylval->literal = jv_string_sized(yytext + 1, yyleng - 1); return FORMAT;
}
-?[0-9.]+([eE][+-]?[0-9]+)? {
[0-9.]+([eE][+-]?[0-9]+)? {
yylval->literal = jv_parse_sized(yytext, yyleng); return LITERAL;
}
@@ -127,6 +132,10 @@ static int try_exit(int c, int state, yyscan_t yyscanner) {
match = ')';
ret = QQSTRING_INTERP_END;
break;
default:
// may not be the best error to give
return INVALID_CHARACTER;
}
assert(match);
if (match == c) {
@@ -150,3 +159,13 @@ static int enter(int c, int currstate, yyscan_t yyscanner) {
yy_push_state(state, yyscanner);
return c;
}
void* yyalloc(size_t sz, void* extra) {
return jv_mem_alloc(sz);
}
void* yyrealloc(void* p, size_t sz, void* extra) {
return jv_mem_realloc(p, sz);
}
void yyfree(void* p, void* extra) {
jv_mem_free(p);
}
locfile.h
+3 -2
View File
@@ -4,6 +4,7 @@
#include <stdio.h>
#include <assert.h>
#include <stdarg.h>
#include "jv_alloc.h"
typedef struct {
int start, end;
} location;
@@ -24,7 +25,7 @@ static void locfile_init(struct locfile* l, const char* data, int length) {
for (int i=0; i<length; i++) {
if (data[i] == '\n') l->nlines++;
}
l->linemap = malloc(sizeof(int) * (l->nlines + 1));
l->linemap = jv_mem_alloc(sizeof(int) * (l->nlines + 1));
l->linemap[0] = 0;
int line = 1;
for (int i=0; i<length; i++) {
@@ -37,7 +38,7 @@ static void locfile_init(struct locfile* l, const char* data, int length) {
}
static void locfile_free(struct locfile* l) {
free(l->linemap);
jv_mem_free(l->linemap);
}
static int locfile_get_line(struct locfile* l, int pos) {
main.c
+68 -11
View File
@@ -1,7 +1,9 @@
#define _POSIX_SOURCE
#include <stdio.h>
#include <errno.h>
#include <string.h>
#include <ctype.h>
#include <unistd.h>
#include "compile.h"
#include "builtin.h"
#include "jv.h"
@@ -10,15 +12,17 @@
#include "parser.h"
#include "execute.h"
#include "config.h" /* Autoconf generated header file */
#include "jv_alloc.h"
static const char* progname;
static void usage() {
fprintf(stderr, "\njq - commandline JSON processor [version %s]\n", PACKAGE_VERSION);
fprintf(stderr, "Usage: %s [options] <jq filter>\n\n", progname);
fprintf(stderr, "Usage: %s [options] <jq filter> [file...]\n\n", progname);
fprintf(stderr, "For a description of the command line options and\n");
fprintf(stderr, "how to write jq filters (and why you might want to)\n");
fprintf(stderr, "see the jq documentation at http://stedolan.github.com/jq\n\n");
fprintf(stderr, "see the jq manpage, or the online documentation at\n");
fprintf(stderr, "http://stedolan.github.com/jq\n\n");
exit(1);
}
@@ -50,8 +54,10 @@ enum {
RAW_OUTPUT = 8,
COMPACT_OUTPUT = 16,
ASCII_OUTPUT = 32,
COLOUR_OUTPUT = 64,
NO_COLOUR_OUTPUT = 128,
FROM_FILE = 64,
FROM_FILE = 256,
};
static int options = 0;
static struct bytecode* bc;
@@ -62,10 +68,18 @@ static void process(jv value) {
while (jv_is_valid(result = jq_next())) {
if ((options & RAW_OUTPUT) && jv_get_kind(result) == JV_KIND_STRING) {
fwrite(jv_string_value(result), 1, jv_string_length(jv_copy(result)), stdout);
jv_free(result);
} else {
int dumpopts = 0;
int dumpopts;
#ifdef JQ_DEFAULT_ENABLE_COLOR
dumpopts = JQ_DEFAULT_ENABLE_COLOR ? JV_PRINT_COLOUR : 0;
#else
dumpopts = isatty(fileno(stdout)) ? JV_PRINT_COLOUR : 0;
#endif
if (!(options & COMPACT_OUTPUT)) dumpopts |= JV_PRINT_PRETTY;
if (options & ASCII_OUTPUT) dumpopts |= JV_PRINT_ASCII;
if (options & COLOUR_OUTPUT) dumpopts |= JV_PRINT_COLOUR;
if (options & NO_COLOUR_OUTPUT) dumpopts &= ~JV_PRINT_COLOUR;
jv_dump(result, dumpopts);
}
printf("\n");
@@ -97,20 +111,63 @@ static jv slurp_file(const char* filename) {
return data;
}
FILE* current_input;
const char** input_filenames;
int ninput_files;
int next_input_idx;
static int read_more(char* buf, size_t size) {
while (!current_input || feof(current_input)) {
if (current_input) {
fclose(current_input);
current_input = 0;
}
if (next_input_idx == ninput_files) {
return 0;
}
if (!strcmp(input_filenames[next_input_idx], "-")) {
current_input = stdin;
} else {
current_input = fopen(input_filenames[next_input_idx], "r");
}
if (!current_input) {
fprintf(stderr, "%s: %s: %s\n", progname, input_filenames[next_input_idx], strerror(errno));
}
next_input_idx++;
}
if (!fgets(buf, size, current_input)) buf[0] = 0;
return 1;
}
int main(int argc, char* argv[]) {
if (argc) progname = argv[0];
const char* program = 0;
input_filenames = jv_mem_alloc(sizeof(const char*) * argc);
ninput_files = 0;
int further_args_are_files = 0;
for (int i=1; i<argc; i++) {
if (!isoptish(argv[i])) {
if (program) usage();
program = argv[i];
if (further_args_are_files) {
input_filenames[ninput_files++] = argv[i];
} else if (!strcmp(argv[i], "--")) {
if (!program) usage();
further_args_are_files = 1;
} else if (!isoptish(argv[i])) {
if (program) {
input_filenames[ninput_files++] = argv[i];
} else {
program = argv[i];
}
} else if (isoption(argv[i], 's', "slurp")) {
options |= SLURP;
} else if (isoption(argv[i], 'r', "raw-output")) {
options |= RAW_OUTPUT;
} else if (isoption(argv[i], 'c', "compact-output")) {
options |= COMPACT_OUTPUT;
} else if (isoption(argv[i], 'C', "color-output")) {
options |= COLOUR_OUTPUT;
} else if (isoption(argv[i], 'M', "monochrome-output")) {
options |= NO_COLOUR_OUTPUT;
} else if (isoption(argv[i], 'a', "ascii-output")) {
options |= ASCII_OUTPUT;
} else if (isoption(argv[i], 'R', "raw-input")) {
@@ -130,6 +187,7 @@ int main(int argc, char* argv[]) {
}
}
if (!program) usage();
if (ninput_files == 0) current_input = stdin;
if ((options & PROVIDE_NULL) && (options & (RAW_INPUT | SLURP))) {
fprintf(stderr, "%s: --null-input cannot be used with --raw-input or --slurp\n", progname);
@@ -169,9 +227,8 @@ int main(int argc, char* argv[]) {
}
struct jv_parser parser;
jv_parser_init(&parser);
while (!feof(stdin)) {
char buf[4096];
if (!fgets(buf, sizeof(buf), stdin)) buf[0] = 0;
char buf[4096];
while (read_more(buf, sizeof(buf))) {
if (options & RAW_INPUT) {
int len = strlen(buf);
if (len > 0) {
@@ -207,7 +264,7 @@ int main(int argc, char* argv[]) {
process(slurped);
}
}
jv_mem_free(input_filenames);
bytecode_free(bc);
return 0;
}
opcode.c
+3 -5
View File
@@ -5,11 +5,9 @@
#define CONSTANT OP_HAS_CONSTANT, 2
#define VARIABLE (OP_HAS_VARIABLE | OP_HAS_BINDING), 3
#define BRANCH OP_HAS_BRANCH, 2
#define CFUNC (OP_HAS_SYMBOL | OP_HAS_CFUNC | OP_HAS_VARIABLE_LENGTH_ARGLIST), 2
#define UFUNC (OP_HAS_UFUNC | OP_HAS_VARIABLE_LENGTH_ARGLIST), 2
#define CLOSURE_PARAM_IMM (OP_IS_CALL_PSEUDO | OP_HAS_BINDING), 0
#define CLOSURE_CREATE_IMM (OP_HAS_BLOCK | OP_IS_CALL_PSEUDO | OP_HAS_BINDING), 0
#define CLOSURE_CREATE_C_IMM (OP_IS_CALL_PSEUDO | OP_HAS_BINDING), 0
#define CFUNC (OP_HAS_CFUNC | OP_HAS_BINDING), 3
#define UFUNC (OP_HAS_UFUNC | OP_HAS_BINDING | OP_IS_CALL_PSEUDO), 4
#define DEFINITION (OP_IS_CALL_PSEUDO | OP_HAS_BINDING), 0
#define CLOSURE_REF_IMM (OP_IS_CALL_PSEUDO | OP_HAS_BINDING), 2
#define OP(name, imm, in, out) \
opcode.h
-7
View File
@@ -19,12 +19,9 @@ enum {
OP_HAS_CONSTANT = 2,
OP_HAS_VARIABLE = 4,
OP_HAS_BRANCH = 8,
OP_HAS_SYMBOL = 16,
OP_HAS_CFUNC = 32,
OP_HAS_UFUNC = 64,
OP_IS_CALL_PSEUDO = 128,
OP_HAS_VARIABLE_LENGTH_ARGLIST = 256,
OP_HAS_BLOCK = 512,
OP_HAS_BINDING = 1024,
};
struct opcode_description {
@@ -41,8 +38,4 @@ struct opcode_description {
const struct opcode_description* opcode_describe(opcode op);
static inline int opcode_length(opcode op) {
return opcode_describe(op)->length;
}
#endif
opcode_list.h
+8 -10
View File
@@ -1,30 +1,28 @@
OP(LOADK, CONSTANT, 1, 1)
OP(DUP, NONE, 1, 2)
OP(DUP2, NONE, 2, 3)
OP(SWAP, NONE, 2, 2)
OP(POP, NONE, 1, 0)
OP(LOADV, VARIABLE, 1, 1)
OP(STOREV, VARIABLE, 1, 0)
OP(INDEX, NONE, 2, 1)
//OP(DISPLAY, NONE, 1, 0)
OP(YIELD, NONE, 1, 0)
OP(EACH, NONE, 1, 1)
OP(FORK, BRANCH, 0, 0)
OP(JUMP, BRANCH, 0, 0)
OP(JUMP_F,BRANCH, 1, 0)
OP(BACKTRACK, NONE, 0, 0)
OP(APPEND, NONE, 2, 1)
OP(APPEND, VARIABLE,1, 0)
OP(INSERT, NONE, 4, 2)
OP(ASSIGN, VARIABLE, 3, 0)
OP(GETPATH, NONE, 2, 1)
OP(CALL_BUILTIN_1_1, CFUNC, 1, 1)
OP(CALL_BUILTIN_2_1, CFUNC, 2, 1)
OP(CALL_BUILTIN_3_1, CFUNC, 3, 1)
OP(CALL_BUILTIN, CFUNC, -1, 1)
OP(CALL_1_1, UFUNC, 1, 1)
OP(CALL_JQ, UFUNC, 1, 1)
OP(RET, NONE, 1, 1)
OP(CLOSURE_PARAM, CLOSURE_PARAM_IMM, 0, 0)
OP(CLOSURE_PARAM, DEFINITION, 0, 0)
OP(CLOSURE_REF, CLOSURE_REF_IMM, 0, 0)
OP(CLOSURE_CREATE, CLOSURE_CREATE_IMM, 0, 0)
OP(CLOSURE_CREATE_C, CLOSURE_CREATE_C_IMM, 0, 0)
OP(CLOSURE_CREATE, DEFINITION, 0, 0)
OP(CLOSURE_CREATE_C, DEFINITION, 0, 0)
parser.y
+88 -67
View File
@@ -2,6 +2,9 @@
#include <stdio.h>
#include <string.h>
#include "compile.h"
#include "jv_alloc.h"
#define YYMALLOC jv_mem_alloc
#define YYFREE jv_mem_free
struct lexer_param;
@@ -45,6 +48,7 @@ struct lexer_param;
%token INVALID_CHARACTER
%token <literal> IDENT
%token <literal> LITERAL
%token <literal> FORMAT
%token EQ "=="
%token NEQ "!="
%token DEFINEDOR "//"
@@ -54,6 +58,7 @@ struct lexer_param;
%token THEN "then"
%token ELSE "else"
%token ELSE_IF "elif"
%token FOLD "fold"
%token END "end"
%token AND "and"
%token OR "or"
@@ -65,7 +70,6 @@ struct lexer_param;
%token SETDEFINEDOR "//="
%token LESSEQ "<="
%token GREATEREQ ">="
%token CONTAINS "contains"
%token QQSTRING_START
%token <literal> QQSTRING_TEXT
@@ -75,13 +79,13 @@ struct lexer_param;
/* revolting hack */
%left ';'
%left '|'
%right '|'
%left ','
%right "//"
%nonassoc '=' SETPIPE SETPLUS SETMINUS SETMULT SETDIV SETDEFINEDOR
%left OR
%left AND
%nonassoc NEQ EQ '<' '>' LESSEQ GREATEREQ CONTAINS
%nonassoc NEQ EQ '<' '>' LESSEQ GREATEREQ
%left '+' '-'
%left '*' '/'
@@ -130,14 +134,11 @@ int yylex(YYSTYPE* yylval, YYLTYPE* yylloc, block* answer, int* errors,
}
static block gen_dictpair(block k, block v) {
block b = gen_subexp(k);
block_append(&b, gen_subexp(v));
block_append(&b, gen_op_simple(INSERT));
return b;
return BLOCK(gen_subexp(k), gen_subexp(v), gen_op_simple(INSERT));
}
static block gen_index(block obj, block key) {
return block_join(obj, block_join(gen_subexp(key), gen_op_simple(INDEX)));
return BLOCK(gen_subexp(key), obj, gen_op_simple(INDEX));
}
static block gen_binop(block a, block b, int op) {
@@ -153,29 +154,26 @@ static block gen_binop(block a, block b, int op) {
case '>': funcname = "_greater"; break;
case LESSEQ: funcname = "_lesseq"; break;
case GREATEREQ: funcname = "_greatereq"; break;
case CONTAINS: funcname = "_contains"; break;
}
assert(funcname);
block c = gen_noop();
block_append(&c, gen_subexp(a));
block_append(&c, gen_subexp(b));
block_append(&c, gen_op_call(CALL_1_1, gen_op_block_unbound(CLOSURE_REF, funcname)));
return c;
return gen_call(funcname, BLOCK(gen_lambda(a), gen_lambda(b)));
}
static block gen_format(block a) {
return block_join(a, gen_op_call(CALL_1_1, gen_op_block_unbound(CLOSURE_REF, "tostring")));
static block gen_format(block a, jv fmt) {
return BLOCK(a, gen_call("format", BLOCK(gen_lambda(gen_const(fmt)))));
}
static block gen_update(block a, block op, int optype) {
block assign = a;
block_append(&assign, gen_op_simple(DUP));
if (optype) {
op = gen_binop(gen_noop(), op, optype);
}
block_append(&assign, op);
return gen_assign(assign);
static block gen_update(block object, block val, int optype) {
block tmp = block_bind(gen_op_var_unbound(STOREV, "tmp"),
gen_noop(), OP_HAS_VARIABLE);
return BLOCK(gen_op_simple(DUP),
val,
tmp,
gen_call("_modify", BLOCK(gen_lambda(object),
gen_lambda(gen_binop(gen_noop(),
gen_op_var_bound(LOADV, tmp),
optype)))));
}
%}
@@ -203,12 +201,17 @@ FuncDef Exp %prec ';' {
} |
Term "as" '$' IDENT '|' Exp {
$$ = gen_op_simple(DUP);
block_append(&$$, $1);
block_append(&$$, block_bind(gen_op_var_unbound(STOREV, jv_string_value($4)), $6, OP_HAS_VARIABLE));
$$ = BLOCK(gen_op_simple(DUP), $1,
block_bind(gen_op_var_unbound(STOREV, jv_string_value($4)),
$6, OP_HAS_VARIABLE));
jv_free($4);
} |
"fold" Term "as" '$' IDENT '(' Exp ')' {
$$ = gen_fold(jv_string_value($5), $2, $7);
jv_free($5);
} |
"if" Exp "then" Exp ElseBody {
$$ = gen_cond($2, $4, $5);
} |
@@ -218,12 +221,7 @@ Term "as" '$' IDENT '|' Exp {
} |
Exp '=' Exp {
block assign = gen_op_simple(DUP);
block_append(&assign, $3);
block_append(&assign, gen_op_simple(SWAP));
block_append(&assign, $1);
block_append(&assign, gen_op_simple(SWAP));
$$ = gen_assign(assign);
$$ = gen_call("_assign", BLOCK(gen_lambda($1), gen_lambda($3)));
} |
Exp "or" Exp {
@@ -243,7 +241,7 @@ Exp "//=" Exp {
} |
Exp "|=" Exp {
$$ = gen_update($1, $3, 0);
$$ = gen_call("_modify", BLOCK(gen_lambda($1), gen_lambda($3)));
} |
Exp '|' Exp {
@@ -262,6 +260,10 @@ Exp "+=" Exp {
$$ = gen_update($1, $3, '+');
} |
'-' Exp {
$$ = BLOCK($2, gen_call("_negate", gen_noop()));
} |
Exp '-' Exp {
$$ = gen_binop($1, $3, '-');
} |
@@ -310,42 +312,56 @@ Exp ">=" Exp {
$$ = gen_binop($1, $3, GREATEREQ);
} |
Exp "contains" Exp {
$$ = gen_binop($1, $3, CONTAINS);
} |
Term {
$$ = $1;
}
String:
QQSTRING_START QQString QQSTRING_END {
$$ = $2;
}
FuncDef:
"def" IDENT ':' Exp ';' {
block body = block_join($4, gen_op_simple(RET));
$$ = gen_op_block_defn_rec(CLOSURE_CREATE, jv_string_value($2), body);
$$ = gen_function(jv_string_value($2), gen_noop(), $4);
jv_free($2);
} |
"def" IDENT '(' IDENT ')' ':' Exp ';' {
block body = block_bind(gen_op_block_unbound(CLOSURE_PARAM, jv_string_value($4)), block_join($7, gen_op_simple(RET)), OP_IS_CALL_PSEUDO);
$$ = gen_op_block_defn_rec(CLOSURE_CREATE, jv_string_value($2), body);
$$ = gen_function(jv_string_value($2),
gen_op_block_unbound(CLOSURE_PARAM, jv_string_value($4)),
$7);
jv_free($2);
jv_free($4);
} |
"def" IDENT '(' IDENT ';' IDENT ')' ':' Exp ';' {
$$ = gen_function(jv_string_value($2),
BLOCK(gen_op_block_unbound(CLOSURE_PARAM, jv_string_value($4)),
gen_op_block_unbound(CLOSURE_PARAM, jv_string_value($6))),
$9);
jv_free($2);
jv_free($4);
jv_free($6);
}
String:
QQSTRING_START { $<literal>$ = jv_string("text"); } QQString QQSTRING_END {
$$ = $3;
jv_free($<literal>2);
} |
FORMAT QQSTRING_START { $<literal>$ = $1; } QQString QQSTRING_END {
$$ = $4;
jv_free($<literal>3);
}
QQString:
/* empty */ {
$$ = gen_op_const(LOADK, jv_string(""));
$$ = gen_const(jv_string(""));
} |
QQString QQSTRING_TEXT {
$$ = gen_binop($1, gen_op_const(LOADK, $2), '+');
$$ = gen_binop($1, gen_const($2), '+');
} |
QQString QQSTRING_INTERP_START Exp QQSTRING_INTERP_END {
$$ = gen_binop($1, gen_format($3), '+');
$$ = gen_binop($1, gen_format($3, jv_copy($<literal>0)), '+');
}
@@ -372,10 +388,10 @@ Term:
$$ = gen_noop();
} |
Term '.' IDENT {
$$ = gen_index($1, gen_op_const(LOADK, $3));
$$ = gen_index($1, gen_const($3));
} |
'.' IDENT {
$$ = gen_index(gen_noop(), gen_op_const(LOADK, $2));
$$ = gen_index(gen_noop(), gen_const($2));
} |
/* FIXME: string literals */
Term '[' Exp ']' {
@@ -385,11 +401,14 @@ Term '[' ']' {
$$ = block_join($1, gen_op_simple(EACH));
} |
LITERAL {
$$ = gen_op_const(LOADK, $1);
$$ = gen_const($1);
} |
String {
$$ = $1;
} |
FORMAT {
$$ = gen_format(gen_noop(), $1);
} |
'(' Exp ')' {
$$ = $2;
} |
@@ -397,28 +416,26 @@ String {
$$ = gen_collect($2);
} |
'[' ']' {
$$ = gen_op_const(LOADK, jv_array());
$$ = gen_const(jv_array());
} |
'{' MkDict '}' {
$$ = gen_subexp(gen_op_const(LOADK, jv_object()));
block_append(&$$, $2);
block_append(&$$, gen_op_simple(POP));
$$ = BLOCK(gen_subexp(gen_const(jv_object())), $2, gen_op_simple(POP));
} |
'$' IDENT {
$$ = gen_location(@$, gen_op_var_unbound(LOADV, jv_string_value($2)));
jv_free($2);
} |
IDENT {
$$ = gen_location(@$, gen_op_call(CALL_1_1, gen_op_block_unbound(CLOSURE_REF, jv_string_value($1))));
$$ = gen_location(@$, gen_call(jv_string_value($1), gen_noop()));
jv_free($1);
} |
IDENT '(' Exp ')' {
$$ = gen_op_call(CALL_1_1,
block_join(gen_op_block_unbound(CLOSURE_REF, jv_string_value($1)),
block_bind(gen_op_block_defn(CLOSURE_CREATE,
"lambda",
block_join($3, gen_op_simple(RET))),
gen_noop(), OP_IS_CALL_PSEUDO)));
$$ = gen_call(jv_string_value($1), gen_lambda($3));
$$ = gen_location(@1, $$);
jv_free($1);
} |
IDENT '(' Exp ';' Exp ')' {
$$ = gen_call(jv_string_value($1), BLOCK(gen_lambda($3), gen_lambda($5)));
$$ = gen_location(@1, $$);
jv_free($1);
} |
@@ -437,14 +454,18 @@ MkDict:
MkDictPair
: IDENT ':' ExpD {
$$ = gen_dictpair(gen_op_const(LOADK, $1), $3);
$$ = gen_dictpair(gen_const($1), $3);
}
| String ':' ExpD {
$$ = gen_dictpair($1, $3);
}
| String {
$$ = gen_dictpair($1, BLOCK(gen_op_simple(POP), gen_op_simple(DUP2),
gen_op_simple(DUP2), gen_op_simple(INDEX)));
}
| IDENT {
$$ = gen_dictpair(gen_op_const(LOADK, jv_copy($1)),
gen_index(gen_noop(), gen_op_const(LOADK, $1)));
$$ = gen_dictpair(gen_const(jv_copy($1)),
gen_index(gen_noop(), gen_const($1)));
}
| '(' Exp ')' ':' ExpD {
$$ = gen_dictpair($2, $5);
setup.sh
+1
View File
@@ -18,6 +18,7 @@ elif [ "superclean" == "$1" ]; then
else
autoreconf --install
./configure --prefix=/opt/junk
make check
mkdir tmp
make DESTDIR=./tmp install
make distcheck
testdata
+165 -11
View File
@@ -21,6 +21,11 @@ null
null
1
-1
null
-1
# FIXME: much more number testing needed
{}
@@ -31,6 +36,13 @@ null
null
[]
# The input line starts with a 0xFEFF (byte order mark) codepoint
# No, there is no reason to have a byte order mark in UTF8 text.
# But apparently people do, so jq shouldn't break on it.
.
"byte order mark"
"byte order mark"
# We test escapes by matching them against Unicode codepoints
# FIXME: more tests needed for weird unicode stuff (e.g. utf16 pairs)
"Aa\r\n\t\b\f\u03bc"
@@ -45,6 +57,24 @@ null
null
"interpolation"
@text,@json,([1,.] | @csv),@html,@uri,@sh,@base64
"<>&'\""
"<>&'\""
"\"<>&'\\\"\""
"1,\"<>&'\"\"\""
"&lt;&gt;&amp;&apos;&quot;"
"%3c%3e%26'%22"
"'<>&'\\''\"'"
"PD4mJyI="
@uri
"\u03bc"
"%ce%bc"
@html "<b>\(.)</b>"
"<script>hax</script>"
"<b>&lt;script&gt;hax&lt;/script&gt;</b>"
#
# Dictionary construction syntax
#
@@ -57,6 +87,10 @@ null
{"a":1, "b":2, "c":3, "d":"c"}
{"a":1, "b":2, "c":1, "e":2}
{"a",b,"a$\(1+1)"}
{"a":1, "b":2, "c":3, "a$2":4}
{"a":1, "b":2, "a$2":4}
#
# Field access, piping
#
@@ -97,6 +131,11 @@ null
1
1
1,.
[]
1
[]
[.]
[2]
[[2]]
@@ -141,6 +180,10 @@ null
[6]
[7]
42 as $x | . | . | . + 432 | $x + 1
34324
43
1 as $x | [$x,$x,$x as $x | $x]
null
[1,1,1]
@@ -159,6 +202,14 @@ null
"wtasdf"
2.0
2-1
null
1
2-(-1)
null
3
1e+0+0.001e3
"I wonder what this will be?"
20e-1
@@ -167,6 +218,18 @@ null
15
19.0
.+null
{"a":42}
{"a":42}
null+.
null
null
.a+.b
{"a":42}
42
[1,2,3] + [.]
null
[1,2,3,null]
@@ -291,6 +354,65 @@ def fac: if . == 1 then 1 else . * (. - 1 | fac) end; [.[] | fac]
[1,2,3,4]
[1,2,6,24]
# test stack overflow and reallocation
# this test is disabled for now, it takes a realllllly long time.
# def f: if length > 1000 then . else .+[1]|f end; f | length
# []
# 1001
fold 0 as $s (.[] | $s + .)
[1,2,4]
7
#
# Paths
#
path(.foo[0,1])
null
["foo", 0]
["foo", 1]
path(.[] | select(.>3))
[1,5,3]
[1]
path(.)
42
[]
["foo",1] as $p | getpath($p), setpath($p; 20), delpaths([$p])
{"bar": 42, "foo": ["a", "b", "c", "d"]}
"b"
{"bar": 42, "foo": ["a", 20, "c", "d"]}
{"bar": 42, "foo": ["a", "c", "d"]}
map(getpath([2])), map(setpath([2]; 42)), map(delpaths([[2]]))
[[0], [0,1], [0,1,2]]
[null, null, 2]
[[0,null,42], [0,1,42], [0,1,42]]
[[0], [0,1], [0,1]]
map(delpaths([[0,"foo"]]))
[[{"foo":2, "x":1}], [{"bar":2}]]
[[{"x":1}], [{"bar":2}]]
["foo",1] as $p | getpath($p), setpath($p; 20), delpaths([$p])
{"bar":false}
null
{"bar":false, "foo": [null, 20]}
{"bar":false}
delpaths([[-200]])
[1,2,3]
[1,2,3]
del(.), del(empty), del((.foo,.bar,.baz) | .[2,3,0]), del(.foo[0], .bar[0], .foo, .baz.bar[0].x)
{"foo": [0,1,2,3,4], "bar": [0,1]}
null
{"foo": [0,1,2,3,4], "bar": [0,1]}
{"foo": [1,4], "bar": [1]}
{"bar": [1]}
#
# Assignment
@@ -314,6 +436,10 @@ def fac: if . == 1 then 1 else . * (. - 1 | fac) end; [.[] | fac]
[-1,1,3]
[0.5, 1.5, 2.5]
.foo += .foo
{"foo":2}
{"foo":4}
.[0].a |= {"old":., "new":(.+1)}
[{"a":1,"b":2}]
[{"a":{"old":1, "new":2},"b":2}]
@@ -397,26 +523,54 @@ def inc(x): x |= .+1; inc(.[].a)
[true,false]
# containment operator
["foo" contains "foo", "foobar" contains "foo", "foo" contains "foobar"]
[("foo" | contains("foo")), ("foobar" | contains("foo")), ("foo" | contains("foobar"))]
{}
[true, true, false]
[[] contains [], [1,2,3] contains [1,2], [1,2,3] contains [3,1], [1,2,3] contains [4], [1,2,3] contains [1,4]]
{}
map(.[1] as $needle | .[0] | contains($needle))
[[[],[]], [[1,2,3], [1,2]], [[1,2,3], [3,1]], [[1,2,3], [4]], [[1,2,3], [1,4]]]
[true, true, true, false, false]
[["foobar", "foobaz"] contains ["baz", "bar"], ["foobar", "foobaz"] contains ["foo"], ["foobar", "foobaz"] contains ["blap"]]
map(.[1] as $needle | .[0] | contains($needle))
[[["foobar", "foobaz"], ["baz", "bar"]], [["foobar", "foobaz"], ["foo"]], [["foobar", "foobaz"], ["blap"]]]
[true, true, false]
[({foo: 12, bar:13} | contains({foo: 12})), ({foo: 12} | contains({})), ({foo: 12, bar:13} | contains({baz:14}))]
{}
[true, true, false]
[{foo: 12, bar:13} contains {foo: 12}, {foo: 12} contains {}, {foo: 12, bar:13} contains {baz:14}]
{}
[true, true, false]
{foo: {baz: 12, blap: {bar: 13}}, bar: 14} contains {bar: 14, foo: {blap: {}}}
{foo: {baz: 12, blap: {bar: 13}}, bar: 14} | contains({bar: 14, foo: {blap: {}}})
{}
true
{foo: {baz: 12, blap: {bar: 13}}, bar: 14} contains {bar: 14, foo: {blap: {bar: 14}}}
{foo: {baz: 12, blap: {bar: 13}}, bar: 14} | contains({bar: 14, foo: {blap: {bar: 14}}})
{}
false
false
sort
[42,[2,5,3,11],10,{"a":42,"b":2},{"a":42},true,2,[2,6],"hello",null,[2,5,6],{"a":[],"b":1},"abc","ab",[3,10],{},false,"abcd",null]
[null,null,false,true,2,10,42,"ab","abc","abcd","hello",[2,5,3,11],[2,5,6],[2,6],[3,10],{},{"a":42},{"a":42,"b":2},{"a":[],"b":1}]
(sort_by(.b) | sort_by(.a)), sort_by(.a, .b), sort_by(.b, .c), group_by(.b), group_by(.a + .b - .c == 2)
[{"a": 1, "b": 4, "c": 14}, {"a": 4, "b": 1, "c": 3}, {"a": 1, "b": 4, "c": 3}, {"a": 0, "b": 2, "c": 43}]
[{"a": 0, "b": 2, "c": 43}, {"a": 1, "b": 4, "c": 14}, {"a": 1, "b": 4, "c": 3}, {"a": 4, "b": 1, "c": 3}]
[{"a": 0, "b": 2, "c": 43}, {"a": 1, "b": 4, "c": 14}, {"a": 1, "b": 4, "c": 3}, {"a": 4, "b": 1, "c": 3}]
[{"a": 4, "b": 1, "c": 3}, {"a": 0, "b": 2, "c": 43}, {"a": 1, "b": 4, "c": 3}, {"a": 1, "b": 4, "c": 14}]
[[{"a": 4, "b": 1, "c": 3}], [{"a": 0, "b": 2, "c": 43}], [{"a": 1, "b": 4, "c": 14}, {"a": 1, "b": 4, "c": 3}]]
[[{"a": 1, "b": 4, "c": 14}, {"a": 0, "b": 2, "c": 43}], [{"a": 4, "b": 1, "c": 3}, {"a": 1, "b": 4, "c": 3}]]
unique
[1,2,5,3,5,3,1,3]
[1,2,3,5]
[min, max, min_by(.[1]), max_by(.[1]), min_by(.[2]), max_by(.[2])]
[[4,2,"a"],[3,1,"a"],[2,4,"a"],[1,3,"a"]]
[[1,3,"a"],[4,2,"a"],[3,1,"a"],[2,4,"a"],[4,2,"a"],[1,3,"a"]]
[min,max,min_by(.),max_by(.)]
[]
[null,null,null,null]
.foo[.baz]
{"foo":{"bar":4},"baz":"bar"}
4