diff --git a/.gitignore b/.gitignore index 56c1a5ca..f76b15d8 100644 --- a/.gitignore +++ b/.gitignore @@ -1,10 +1,10 @@ *.o *~ -# Test binaries -jv_test -jv_parse -parsertest*~ +jq_test +build +jq +jq.1 # Something delightfully recursive happens otherwise docs/content/2.download/source/* diff --git a/AUTHORS b/AUTHORS index 6d78ef76..1cf1441b 100644 --- a/AUTHORS +++ b/AUTHORS @@ -3,4 +3,4 @@ Stephen Dolan Contributions by Anthony Shortland - rpmbuild target -Lee Thompson - autoconf stuff +Lee Thompson - autoconf stuff diff --git a/Makefile.am b/Makefile.am index 06d9c55a..7bdc53ef 100644 --- a/Makefile.am +++ b/Makefile.am @@ -10,9 +10,41 @@ EXTRA_DIST = setup.sh config.h.in ChangeLog VERSION lexer.l lexer.h gen_utf8_tab docdir = ${datadir}/doc/${PACKAGE} dist_doc_DATA = README.md INSTALL COPYING AUTHORS README NEWS -JQ_INCS = jq_parser.h builtin.h bytecode.h compile.h execute.h forkable_stack.h frame_layout.h jv.h jv_dtoa.h jv_parse.h jv_unicode.h locfile.h opcode.h opcode_list.h parser.y jv_utf8_tables.gen.h +JQ_INCS = \ + jq_parser.h \ + builtin.h \ + bytecode.h \ + compile.h \ + execute.h \ + forkable_stack.h \ + frame_layout.h \ + jv.h \ + jv_alloc.h \ + jv_aux.h \ + jv_dtoa.h \ + jv_parse.h \ + jv_unicode.h \ + locfile.h \ + opcode.h \ + opcode_list.h \ + parser.y \ + jv_utf8_tables.gen.h -JQ_SRC = opcode.c bytecode.c compile.c execute.c builtin.c jv.c jv_parse.c jv_print.c jv_dtoa.c jv_unicode.c lexer.c $(JQ_INCS) +JQ_SRC = \ + opcode.c \ + bytecode.c \ + compile.c \ + execute.c \ + builtin.c \ + jv.c \ + jv_parse.c \ + jv_print.c \ + jv_dtoa.c \ + jv_unicode.c \ + jv_aux.c \ + jv_alloc.c \ + lexer.c \ + $(JQ_INCS) if DEVCFLAGS_ENABLED DEVCFLAGS = -Wextra -Wall -Wno-missing-field-initializers -Wno-unused-parameter -std=gnu99 -ggdb -Wno-unused-function diff --git a/builtin.c b/builtin.c index bbda554d..b09bd961 100644 --- a/builtin.c +++ b/builtin.c @@ -1,51 +1,89 @@ #include #include "builtin.h" +#include "bytecode.h" #include "compile.h" #include "jq_parser.h" #include "locfile.h" +#include "jv_aux.h" +#include "jv_unicode.h" -enum { - CMP_OP_LESS, - CMP_OP_GREATER, - CMP_OP_LESSEQ, - CMP_OP_GREATEREQ -} _cmp_op; -static void f_plus(jv input[], jv output[]) { - jv_free(input[0]); - jv a = input[2]; - jv b = input[1]; - if (jv_get_kind(a) == JV_KIND_NUMBER && jv_get_kind(b) == JV_KIND_NUMBER) { - output[0] = jv_number(jv_number_value(a) + - jv_number_value(b)); - } else if (jv_get_kind(a) == JV_KIND_STRING && jv_get_kind(b) == JV_KIND_STRING) { - output[0] = jv_string_concat(a, b); - } else if (jv_get_kind(a) == JV_KIND_ARRAY && jv_get_kind(b) == JV_KIND_ARRAY) { - output[0] = jv_array_concat(a, b); - } else if (jv_get_kind(a) == JV_KIND_OBJECT && jv_get_kind(b) == JV_KIND_OBJECT) { - output[0] = jv_object_merge(a, b); - } else { - output[0] = jv_invalid_with_msg(jv_string_fmt("Attempted to add %s and %s", - jv_kind_name(jv_get_kind(a)), - jv_kind_name(jv_get_kind(b)))); - jv_free(a); - jv_free(b); + +typedef jv (*func_1)(jv); +typedef jv (*func_2)(jv,jv); +typedef jv (*func_3)(jv,jv,jv); +typedef jv (*func_4)(jv,jv,jv,jv); +typedef jv (*func_5)(jv,jv,jv,jv,jv); +jv cfunction_invoke(struct cfunction* function, jv input[]) { + switch (function->nargs) { + case 1: return ((func_1)function->fptr)(input[0]); + case 2: return ((func_2)function->fptr)(input[0], input[1]); + case 3: return ((func_3)function->fptr)(input[0], input[1], input[2]); + case 4: return ((func_4)function->fptr)(input[0], input[1], input[2], input[3]); + case 5: return ((func_5)function->fptr)(input[0], input[1], input[2], input[3], input[4]); + default: return jv_invalid_with_msg(jv_string("Function takes too many arguments")); } } -static void f_minus(jv input[], jv output[]) { - jv_free(input[0]); - jv a = input[2]; - jv b = input[1]; +static jv type_error(jv bad, const char* msg) { + jv err = jv_invalid_with_msg(jv_string_fmt("%s %s", + jv_kind_name(jv_get_kind(bad)), + msg)); + jv_free(bad); + return err; +} + +static jv type_error2(jv bad1, jv bad2, const char* msg) { + jv err = jv_invalid_with_msg(jv_string_fmt("%s and %s %s", + jv_kind_name(jv_get_kind(bad1)), + jv_kind_name(jv_get_kind(bad2)), + msg)); + jv_free(bad1); + jv_free(bad2); + return err; +} + +static jv f_plus(jv input, jv a, jv b) { + jv_free(input); + if (jv_get_kind(a) == JV_KIND_NULL) { + jv_free(a); + return b; + } else if (jv_get_kind(b) == JV_KIND_NULL) { + jv_free(b); + return a; + } else if (jv_get_kind(a) == JV_KIND_NUMBER && jv_get_kind(b) == JV_KIND_NUMBER) { + return jv_number(jv_number_value(a) + + jv_number_value(b)); + } else if (jv_get_kind(a) == JV_KIND_STRING && jv_get_kind(b) == JV_KIND_STRING) { + return jv_string_concat(a, b); + } else if (jv_get_kind(a) == JV_KIND_ARRAY && jv_get_kind(b) == JV_KIND_ARRAY) { + return jv_array_concat(a, b); + } else if (jv_get_kind(a) == JV_KIND_OBJECT && jv_get_kind(b) == JV_KIND_OBJECT) { + return jv_object_merge(a, b); + } else { + return type_error2(a, b, "cannot be added"); + } +} + +static jv f_negate(jv input) { + if (jv_get_kind(input) != JV_KIND_NUMBER) { + return type_error(input, "cannot be negated"); + } + jv ret = jv_number(-jv_number_value(input)); + jv_free(input); + return ret; +} + +static jv f_minus(jv input, jv a, jv b) { + jv_free(input); if (jv_get_kind(a) == JV_KIND_NUMBER && jv_get_kind(b) == JV_KIND_NUMBER) { - output[0] = jv_number(jv_number_value(a) - jv_number_value(b)); + return jv_number(jv_number_value(a) - jv_number_value(b)); } else if (jv_get_kind(a) == JV_KIND_ARRAY && jv_get_kind(b) == JV_KIND_ARRAY) { jv out = jv_array(); - for (int i=0; i= 0) || + (op == CMP_OP_GREATER && r > 0)); +} + +static jv f_less(jv input, jv a, jv b) { + return order_cmp(input, a, b, CMP_OP_LESS); +} + +static jv f_greater(jv input, jv a, jv b) { + return order_cmp(input, a, b, CMP_OP_GREATER); +} + +static jv f_lesseq(jv input, jv a, jv b) { + return order_cmp(input, a, b, CMP_OP_LESSEQ); +} + +static jv f_greatereq(jv input, jv a, jv b) { + return order_cmp(input, a, b, CMP_OP_GREATEREQ); +} + +static jv f_contains(jv a, jv b) { + if (jv_get_kind(a) == jv_get_kind(b)) { + return jv_bool(jv_contains(a, b)); } else { - jv sum = jv_array_get(jv_copy(array), 0); - for (int i = 1; i < jv_array_length(jv_copy(array)); i++) { - jv x = jv_array_get(jv_copy(array), i); - jv add_args[] = {jv_null(), x, sum}; - f_plus(add_args, &sum); + return type_error2(a, b, "cannot have their containment checked"); + } +} + +static jv f_tonumber(jv input) { + if (jv_get_kind(input) == JV_KIND_NUMBER) { + return input; + } + if (jv_get_kind(input) == JV_KIND_STRING) { + jv parsed = jv_parse(jv_string_value(input)); + if (!jv_is_valid(parsed) || jv_get_kind(parsed) == JV_KIND_NUMBER) { + jv_free(input); + return parsed; } - output[0] = sum; } - jv_free(array); + return type_error(input, "cannot be parsed as a number"); } -static void f_equal(jv input[], jv output[]) { - jv_free(input[0]); - output[0] = jv_bool(jv_equal(input[2], input[1])); -} - -static void f_notequal(jv input[], jv output[]) { - jv_free(input[0]); - output[0] = jv_bool(jv_equal(input[2], input[1]) == 0); -} - -static void order_cmp(jv input[], jv output[], int op) { - jv_free(input[0]); - jv a = input[2]; - jv b = input[1]; - if (jv_get_kind(a) == JV_KIND_NUMBER && jv_get_kind(b) == JV_KIND_NUMBER) { - double da = jv_number_value(a); - double db = jv_number_value(b); - output[0] = jv_bool((op == CMP_OP_LESS && da < db) || - (op == CMP_OP_LESSEQ && da <= db) || - (op == CMP_OP_GREATEREQ && da >= db) || - (op == CMP_OP_GREATER && da > db)); +static jv f_length(jv input) { + if (jv_get_kind(input) == JV_KIND_ARRAY) { + return jv_number(jv_array_length(input)); + } else if (jv_get_kind(input) == JV_KIND_OBJECT) { + return jv_number(jv_object_length(input)); + } else if (jv_get_kind(input) == JV_KIND_STRING) { + return jv_number(jv_string_length(input)); + } else if (jv_get_kind(input) == JV_KIND_NULL) { + jv_free(input); + return jv_number(0); } else { - output[0] = jv_invalid_with_msg(jv_string_fmt("Attempted to compare order of %s wrt %s", - jv_kind_name(jv_get_kind(a)), - jv_kind_name(jv_get_kind(b)))); - jv_free(a); - jv_free(b); + return type_error(input, "has no length"); } } -static void f_less(jv input[], jv output[]) { - order_cmp(input, output, CMP_OP_LESS); -} - -static void f_greater(jv input[], jv output[]) { - order_cmp(input, output, CMP_OP_GREATER); -} - -static void f_lesseq(jv input[], jv output[]) { - order_cmp(input, output, CMP_OP_LESSEQ); -} - -static void f_greatereq(jv input[], jv output[]) { - order_cmp(input, output, CMP_OP_GREATEREQ); -} - -static void f_contains(jv input[], jv output[]) { - jv_free(input[0]); - jv a = input[2]; - jv b = input[1]; - jv_kind akind = jv_get_kind(a); - - if (akind == jv_get_kind(b)) { - output[0] = jv_bool(jv_contains(a, b)); - } else { - output[0] = jv_invalid_with_msg(jv_string_fmt("Can only check containment of values of the same type.")); - jv_free(a); - jv_free(b); +static jv f_tostring(jv input) { + if (jv_get_kind(input) == JV_KIND_STRING) { + return input; + } else { + return jv_dump_string(input, 0); } } -static void f_tonumber(jv input[], jv output[]) { - if (jv_get_kind(input[0]) == JV_KIND_NUMBER) { - output[0] = input[0]; - } else if (jv_get_kind(input[0]) == JV_KIND_STRING) { - jv parsed = jv_parse(jv_string_value(input[0])); - if (!jv_is_valid(parsed)) { - jv_free(input[0]); - output[0] = parsed; - } else if (jv_get_kind(parsed) != JV_KIND_NUMBER) { - output[0] = jv_invalid_with_msg(jv_string_fmt("'%s' is not a number", - jv_string_value(input[0]))); - jv_free(input[0]); +#define CHARS_ALPHANUM "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789" + +static jv escape_string(jv input, const char* escapings) { + + assert(jv_get_kind(input) == JV_KIND_STRING); + const char* lookup[128] = {0}; + const char* p = escapings; + while (*p) { + lookup[(int)*p] = p+1; + p++; + p += strlen(p); + p++; + } + + jv ret = jv_string(""); + const char* i = jv_string_value(input); + const char* end = i + jv_string_length(jv_copy(input)); + const char* cstart; + int c = 0; + while ((i = jvp_utf8_next((cstart = i), end, &c))) { + assert(c != -1); + if (c < 128 && lookup[c]) { + ret = jv_string_append_str(ret, lookup[c]); } else { - jv_free(input[0]); - output[0] = parsed; + ret = jv_string_append_buf(ret, cstart, i - cstart); } + } + jv_free(input); + return ret; + +} + +static jv f_format(jv input, jv fmt) { + if (jv_get_kind(fmt) != JV_KIND_STRING) { + jv_free(input); + return type_error(fmt, "is not a valid format"); + } + const char* fmt_s = jv_string_value(fmt); + if (!strcmp(fmt_s, "json")) { + jv_free(fmt); + return jv_dump_string(input, 0); + } else if (!strcmp(fmt_s, "text")) { + jv_free(fmt); + return f_tostring(input); + } else if (!strcmp(fmt_s, "csv")) { + jv_free(fmt); + if (jv_get_kind(input) != JV_KIND_ARRAY) + return type_error(input, "cannot be csv-formatted, only array"); + jv line = jv_string(""); + jv_array_foreach(input, i, x) { + if (i) line = jv_string_append_str(line, ","); + switch (jv_get_kind(x)) { + case JV_KIND_NULL: + /* null rendered as empty string */ + jv_free(x); + break; + case JV_KIND_TRUE: + case JV_KIND_FALSE: + line = jv_string_concat(line, jv_dump_string(x, 0)); + break; + case JV_KIND_NUMBER: + if (jv_number_value(x) != jv_number_value(x)) { + /* NaN, render as empty string */ + jv_free(x); + } else { + line = jv_string_concat(line, jv_dump_string(x, 0)); + } + break; + case JV_KIND_STRING: { + line = jv_string_append_str(line, "\""); + line = jv_string_concat(line, escape_string(x, "\"\"\"\0")); + line = jv_string_append_str(line, "\""); + break; + } + default: + jv_free(input); + jv_free(line); + return type_error(x, "is not valid in a csv row"); + } + } + jv_free(input); + return line; + } else if (!strcmp(fmt_s, "html")) { + jv_free(fmt); + return escape_string(f_tostring(input), "&&\0<<\0>>\0''\0\""\0"); + } else if (!strcmp(fmt_s, "uri")) { + jv_free(fmt); + input = f_tostring(input); + + int unreserved[128] = {0}; + const char* p = CHARS_ALPHANUM "-_.!~*'()"; + while (*p) unreserved[(int)*p++] = 1; + + jv line = jv_string(""); + const char* s = jv_string_value(input); + for (int i=0; i= 3 ? 3 : len-i; + for (int j=0; j<3; j++) { + code <<= 8; + code |= j < n ? (unsigned)data[i+j] : 0; + } + char buf[4]; + for (int j=0; j<4; j++) { + buf[j] = b64[(code >> (18 - j*6)) & 0x3f]; + } + if (n < 3) buf[3] = '='; + if (n < 2) buf[2] = '='; + line = jv_string_append_buf(line, buf, sizeof(buf)); + } + jv_free(input); + return line; } else { - output[0] = jv_invalid_with_msg(jv_string_fmt("Cannot parse %s as a number", - jv_kind_name(jv_get_kind(input[0])))); - jv_free(input[0]); - return; + jv_free(input); + return jv_invalid_with_msg(jv_string_concat(fmt, jv_string(" is not a valid format"))); } } -static void f_length(jv input[], jv output[]) { - if (jv_get_kind(input[0]) == JV_KIND_ARRAY) { - output[0] = jv_number(jv_array_length(input[0])); - } else if (jv_get_kind(input[0]) == JV_KIND_OBJECT) { - output[0] = jv_number(jv_object_length(input[0])); - } else if (jv_get_kind(input[0]) == JV_KIND_STRING) { - output[0] = jv_number(jv_string_length(input[0])); +static jv f_keys(jv input) { + if (jv_get_kind(input) == JV_KIND_OBJECT || jv_get_kind(input) == JV_KIND_ARRAY) { + return jv_keys(input); } else { - output[0] = jv_invalid_with_msg(jv_string_fmt("Cannot get the length of %s", - jv_kind_name(jv_get_kind(input[0])))); - jv_free(input[0]); + return type_error(input, "has no keys"); } } -static void f_tostring(jv input[], jv output[]) { - if (jv_get_kind(input[0]) == JV_KIND_STRING) { - output[0] = input[0]; +static jv f_sort(jv input){ + if (jv_get_kind(input) == JV_KIND_ARRAY) { + return jv_sort(input, jv_copy(input)); } else { - output[0] = jv_dump_string(input[0], 0); + return type_error(input, "cannot be sorted, as it is not an array"); } } -static int string_cmp(const void* pa, const void* pb){ - const jv* a = pa; - const jv* b = pb; - int lena = jv_string_length(jv_copy(*a)); - int lenb = jv_string_length(jv_copy(*b)); - int minlen = lena < lenb ? lena : lenb; - int r = memcmp(jv_string_value(*a), jv_string_value(*b), minlen); - if (r == 0) r = lena - lenb; - return r; -} - -static void f_keys(jv input[], jv output[]) { - if (jv_get_kind(input[0]) == JV_KIND_OBJECT) { - int nkeys = jv_object_length(jv_copy(input[0])); - jv* keys = malloc(sizeof(jv) * nkeys); - int kidx = 0; - jv_object_foreach(i, input[0]) { - keys[kidx++] = jv_object_iter_key(input[0], i); - } - qsort(keys, nkeys, sizeof(jv), string_cmp); - output[0] = jv_array_sized(nkeys); - for (int i = 0; iflags & OP_HAS_VARIABLE_LENGTH_ARGLIST) { - return 2 + codeptr[1] * 2; - } else { - return opcode_length(*codeptr); + int length = opcode_describe(*codeptr)->length; + if (*codeptr == CALL_JQ) { + length += codeptr[1] * 2; } + return length; } void dump_disassembly(int indent, struct bytecode* bc) { @@ -41,8 +42,8 @@ void dump_operation(struct bytecode* bc, uint16_t* codeptr) { printf("%s", op->name); if (op->length > 1) { uint16_t imm = bc->code[pc++]; - if (op->flags & OP_HAS_VARIABLE_LENGTH_ARGLIST) { - for (int i=0; iop == CALL_JQ) { + for (int i=0; icode[pc++]; uint16_t idx = bc->code[pc++]; if (idx & ARG_NEWCLOSURE) { @@ -72,17 +73,17 @@ void dump_operation(struct bytecode* bc, uint16_t* codeptr) { } void symbol_table_free(struct symbol_table* syms) { - free(syms->cfunctions); - free(syms); + jv_mem_free(syms->cfunctions); + jv_mem_free(syms); } void bytecode_free(struct bytecode* bc) { - free(bc->code); + jv_mem_free(bc->code); jv_free(bc->constants); for (int i=0; insubfunctions; i++) bytecode_free(bc->subfunctions[i]); if (!bc->parent) symbol_table_free(bc->globals); - free(bc->subfunctions); - free(bc); + jv_mem_free(bc->subfunctions); + jv_mem_free(bc); } diff --git a/bytecode.h b/bytecode.h index 5ffe80a6..e84f1387 100644 --- a/bytecode.h +++ b/bytecode.h @@ -4,14 +4,7 @@ #include "jv.h" #include "opcode.h" - -typedef void (*cfunction_ptr)(jv input[], jv output[]); - -struct cfunction { - cfunction_ptr fptr; - const char* name; - opcode callop; -}; +#include "builtin.h" #define MAX_CFUNCTION_ARGS 10 struct symbol_table { diff --git a/compile.c b/compile.c index fe50c70a..b06e4f75 100644 --- a/compile.c +++ b/compile.c @@ -1,17 +1,32 @@ +#define _GNU_SOURCE // for strdup #include #include #include #include "opcode.h" #include "compile.h" +#include "bytecode.h" #include "locfile.h" +#include "jv_alloc.h" +/* + The intermediate representation for jq filters is as a sequence of + struct inst, which form a doubly-linked list via the next and prev + pointers. + + A "block" represents a sequence of "struct inst", which may be + empty. + + Blocks are generated by the parser bottom-up, so may have free + variables (refer to things not defined). See inst.bound_by and + inst.symbol. + */ struct inst { struct inst* next; struct inst* prev; opcode op; - union { + struct { uint16_t intval; struct inst* target; jv constant; @@ -21,16 +36,20 @@ struct inst { location source; // Binding - // An instruction requiring binding (for parameters/variables) + // An instruction requiring binding (for parameters/variables/functions) // is in one of three states: - // bound_by = NULL - Unbound free variable - // bound_by = self - This instruction binds a variable - // bound_by = other - Uses variable bound by other instruction - // The immediate field is generally not meaningful until instructions - // are bound, and even then only for instructions which bind. + // inst->bound_by = NULL - Unbound free variable + // inst->bound_by = inst - This instruction binds a variable + // inst->bound_by = other - Uses variable bound by other instruction + // Unbound instructions (references to other things that may or may not + // exist) are created by "gen_foo_unbound", and bindings are created by + // block_bind(definition, body), which binds all instructions in + // body which are unboudn and refer to "definition" by name. struct inst* bound_by; char* symbol; - block subfn; + + block subfn; // used by CLOSURE_CREATE (body of function) + block arglist; // used by CLOSURE_CREATE (formals) and CALL_JQ (arguments) // This instruction is compiled as part of which function? // (only used during block_compile) @@ -40,26 +59,26 @@ struct inst { }; static inst* inst_new(opcode op) { - inst* i = malloc(sizeof(inst)); + inst* i = jv_mem_alloc(sizeof(inst)); i->next = i->prev = 0; i->op = op; i->bytecode_pos = -1; i->bound_by = 0; i->symbol = 0; i->subfn = gen_noop(); + i->arglist = gen_noop(); i->source = UNKNOWN_LOCATION; return i; } static void inst_free(struct inst* i) { - free(i->symbol); - if (opcode_describe(i->op)->flags & OP_HAS_BLOCK) { - block_free(i->subfn); - } + jv_mem_free(i->symbol); + block_free(i->subfn); + block_free(i->arglist); if (opcode_describe(i->op)->flags & OP_HAS_CONSTANT) { jv_free(i->imm.constant); } - free(i); + jv_mem_free(i); } static block inst_block(inst* i) { @@ -106,9 +125,9 @@ block gen_op_simple(opcode op) { } -block gen_op_const(opcode op, jv constant) { - assert(opcode_describe(op)->flags & OP_HAS_CONSTANT); - inst* i = inst_new(op); +block gen_const(jv constant) { + assert(opcode_describe(LOADK)->flags & OP_HAS_CONSTANT); + inst* i = inst_new(LOADK); i->imm.constant = constant; return inst_block(i); } @@ -148,29 +167,6 @@ block gen_op_var_bound(opcode op, block binder) { return b; } -block gen_op_symbol(opcode op, const char* sym) { - assert(opcode_describe(op)->flags & OP_HAS_SYMBOL); - inst* i = inst_new(op); - i->symbol = strdup(sym); - return inst_block(i); -} - -block gen_op_block_defn(opcode op, const char* name, block block) { - assert(opcode_describe(op)->flags & OP_IS_CALL_PSEUDO); - assert(opcode_describe(op)->flags & OP_HAS_BLOCK); - inst* i = inst_new(op); - i->subfn = block; - i->symbol = strdup(name); - return inst_block(i); -} - -static void block_bind_subblock(block binder, block body, int bindflags); -block gen_op_block_defn_rec(opcode op, const char* name, block blk) { - block b = gen_op_block_defn(op, name, blk); - block_bind_subblock(b, b, OP_IS_CALL_PSEUDO | OP_HAS_BINDING); - return b; -} - block gen_op_block_unbound(opcode op, const char* name) { assert(opcode_describe(op)->flags & OP_IS_CALL_PSEUDO); inst* i = inst_new(op); @@ -185,33 +181,6 @@ block gen_op_block_bound(opcode op, block binder) { return b; } -block gen_op_call(opcode op, block arglist) { - assert(opcode_describe(op)->flags & OP_HAS_VARIABLE_LENGTH_ARGLIST); - inst* i = inst_new(op); - block prelude = gen_noop(); - block call = inst_block(i); - int nargs = 0; - inst* curr = 0; - while ((curr = block_take(&arglist))) { - assert(opcode_describe(curr->op)->flags & OP_IS_CALL_PSEUDO); - block bcurr = inst_block(curr); - switch (curr->op) { - default: assert(0 && "Unknown type of parameter"); break; - case CLOSURE_REF: - block_append(&call, bcurr); - break; - case CLOSURE_CREATE: - block_append(&prelude, bcurr); - block_append(&call, gen_op_block_bound(CLOSURE_REF, bcurr)); - break; - } - nargs++; - } - assert(nargs < 100); //FIXME - i->imm.intval = nargs; - return block_join(prelude, call); -} - static void inst_join(inst* a, inst* b) { assert(a && b); assert(!a->next); @@ -262,168 +231,150 @@ static void block_bind_subblock(block binder, block body, int bindflags) { // bind this instruction i->bound_by = binder.first; } - if (flags & OP_HAS_BLOCK) { - // binding recurses into closures - block_bind_subblock(binder, i->subfn, bindflags); - } + // binding recurses into closures + block_bind_subblock(binder, i->subfn, bindflags); + // binding recurses into argument list + block_bind_subblock(binder, i->arglist, bindflags); } } -block block_bind(block binder, block body, int bindflags) { +static void block_bind_each(block binder, block body, int bindflags) { assert(block_has_only_binders(binder, bindflags)); bindflags |= OP_HAS_BINDING; for (inst* curr = binder.first; curr; curr = curr->next) { block_bind_subblock(inst_block(curr), body, bindflags); } +} + +block block_bind(block binder, block body, int bindflags) { + block_bind_each(binder, body, bindflags); return block_join(binder, body); } +block gen_function(const char* name, block formals, block body) { + block_bind_each(formals, body, OP_IS_CALL_PSEUDO); + inst* i = inst_new(CLOSURE_CREATE); + i->subfn = body; + i->symbol = strdup(name); + i->arglist = formals; + block b = inst_block(i); + block_bind_subblock(b, b, OP_IS_CALL_PSEUDO | OP_HAS_BINDING); + return b; +} + +block gen_lambda(block body) { + return gen_function("@lambda", gen_noop(), body); +} + +block gen_call(const char* name, block args) { + block b = gen_op_block_unbound(CALL_JQ, name); + b.first->arglist = args; + return b; +} + + block gen_subexp(block a) { - block c = gen_noop(); - block_append(&c, gen_op_simple(DUP)); - block_append(&c, a); - block_append(&c, gen_op_simple(SWAP)); - return c; + return BLOCK(gen_op_simple(DUP), a, gen_op_simple(SWAP)); } block gen_both(block a, block b) { - block c = gen_noop(); block jump = gen_op_targetlater(JUMP); - block fork = gen_op_targetlater(FORK); - block_append(&c, fork); - block_append(&c, a); - block_append(&c, jump); - inst_set_target(fork, c); - block_append(&c, b); + block fork = gen_op_target(FORK, jump); + block c = BLOCK(fork, a, jump, b); inst_set_target(jump, c); return c; } block gen_collect(block expr) { - block c = gen_noop(); - block_append(&c, gen_op_simple(DUP)); - block_append(&c, gen_op_const(LOADK, jv_array())); block array_var = block_bind(gen_op_var_unbound(STOREV, "collect"), gen_noop(), OP_HAS_VARIABLE); - block_append(&c, array_var); + block c = BLOCK(gen_op_simple(DUP), gen_const(jv_array()), array_var); - block tail = {0}; - block_append(&tail, gen_op_simple(DUP)); - block_append(&tail, gen_op_var_bound(LOADV, array_var)); - block_append(&tail, gen_op_simple(SWAP)); - block_append(&tail, gen_op_simple(APPEND)); - block_append(&tail, gen_op_var_bound(STOREV, array_var)); - block_append(&tail, gen_op_simple(BACKTRACK)); + block tail = BLOCK(gen_op_var_bound(APPEND, array_var), + gen_op_simple(BACKTRACK)); - block_append(&c, gen_op_target(FORK, tail)); - block_append(&c, expr); - block_append(&c, tail); - - block_append(&c, gen_op_var_bound(LOADV, array_var)); - - return c; + return BLOCK(c, + gen_op_target(FORK, tail), + expr, + tail, + gen_op_var_bound(LOADV, array_var)); } -block gen_assign(block expr) { - block c = gen_noop(); - block_append(&c, gen_op_simple(DUP)); - block result_var = block_bind(gen_op_var_unbound(STOREV, "result"), - gen_noop(), OP_HAS_VARIABLE); - block_append(&c, result_var); - - block loop = gen_noop(); - block_append(&loop, gen_op_simple(DUP)); - block_append(&loop, expr); - block_append(&loop, gen_op_var_bound(ASSIGN, result_var)); - block_append(&loop, gen_op_simple(BACKTRACK)); - - block_append(&c, gen_op_target(FORK, loop)); - block_append(&c, loop); - block_append(&c, gen_op_var_bound(LOADV, result_var)); - return c; +block gen_fold(const char* varname, block init, block fold) { + block loop = BLOCK(fold, gen_op_var_unbound(STOREV, varname), gen_op_simple(BACKTRACK)); + return BLOCK(gen_op_simple(DUP), + init, + block_bind(gen_op_var_unbound(STOREV, varname), + BLOCK(gen_op_target(FORK, loop), + loop, + gen_op_var_unbound(LOADV, varname)), + OP_HAS_VARIABLE)); } block gen_definedor(block a, block b) { // var found := false - block c = gen_op_simple(DUP); - block_append(&c, gen_op_const(LOADK, jv_false())); block found_var = block_bind(gen_op_var_unbound(STOREV, "found"), gen_noop(), OP_HAS_VARIABLE); - block_append(&c, found_var); + block init = BLOCK(gen_op_simple(DUP), gen_const(jv_false()), found_var); // if found, backtrack. Otherwise execute b - block tail = gen_op_simple(DUP); - block_append(&tail, gen_op_var_bound(LOADV, found_var)); block backtrack = gen_op_simple(BACKTRACK); - block_append(&tail, gen_op_target(JUMP_F, backtrack)); - block_append(&tail, backtrack); - block_append(&tail, gen_op_simple(POP)); - block_append(&tail, b); + block tail = BLOCK(gen_op_simple(DUP), + gen_op_var_bound(LOADV, found_var), + gen_op_target(JUMP_F, backtrack), + backtrack, + gen_op_simple(POP), + b); // try again block if_notfound = gen_op_simple(BACKTRACK); // found := true, produce result - block if_found = gen_op_simple(DUP); - block_append(&if_found, gen_op_const(LOADK, jv_true())); - block_append(&if_found, gen_op_var_bound(STOREV, found_var)); - block_append(&if_found, gen_op_target(JUMP, tail)); + block if_found = BLOCK(gen_op_simple(DUP), + gen_const(jv_true()), + gen_op_var_bound(STOREV, found_var), + gen_op_target(JUMP, tail)); - block_append(&c, gen_op_target(FORK, if_notfound)); - block_append(&c, a); - block_append(&c, gen_op_target(JUMP_F, if_found)); - block_append(&c, if_found); - block_append(&c, if_notfound); - block_append(&c, tail); - - return c; + return BLOCK(init, + gen_op_target(FORK, if_notfound), + a, + gen_op_target(JUMP_F, if_found), + if_found, + if_notfound, + tail); } block gen_condbranch(block iftrue, block iffalse) { - block b = gen_noop(); - block_append(&iftrue, gen_op_target(JUMP, iffalse)); - block_append(&b, gen_op_target(JUMP_F, iftrue)); - block_append(&b, iftrue); - block_append(&b, iffalse); - return b; + iftrue = BLOCK(iftrue, gen_op_target(JUMP, iffalse)); + return BLOCK(gen_op_target(JUMP_F, iftrue), iftrue, iffalse); } block gen_and(block a, block b) { // a and b = if a then (if b then true else false) else false - block code = gen_op_simple(DUP); - block_append(&code, a); - - block if_a_true = gen_op_simple(POP); - block_append(&if_a_true, b); - block_append(&if_a_true, gen_condbranch(gen_op_const(LOADK, jv_true()), - gen_op_const(LOADK, jv_false()))); - block_append(&code, gen_condbranch(if_a_true, - block_join(gen_op_simple(POP), gen_op_const(LOADK, jv_false())))); - return code; + return BLOCK(gen_op_simple(DUP), a, + gen_condbranch(BLOCK(gen_op_simple(POP), + b, + gen_condbranch(gen_const(jv_true()), + gen_const(jv_false()))), + BLOCK(gen_op_simple(POP), gen_const(jv_false())))); } block gen_or(block a, block b) { // a or b = if a then true else (if b then true else false) - block code = gen_op_simple(DUP); - block_append(&code, a); - - block if_a_false = gen_op_simple(POP); - block_append(&if_a_false, b); - block_append(&if_a_false, gen_condbranch(gen_op_const(LOADK, jv_true()), - gen_op_const(LOADK, jv_false()))); - block_append(&code, gen_condbranch(block_join(gen_op_simple(POP), gen_op_const(LOADK, jv_true())), - if_a_false)); - return code; + return BLOCK(gen_op_simple(DUP), a, + gen_condbranch(BLOCK(gen_op_simple(POP), gen_const(jv_true())), + BLOCK(gen_op_simple(POP), + b, + gen_condbranch(gen_const(jv_true()), + gen_const(jv_false()))))); } block gen_cond(block cond, block iftrue, block iffalse) { - block b = gen_op_simple(DUP); - block_append(&b, cond); - block_append(&b, gen_condbranch(block_join(gen_op_simple(POP), iftrue), - block_join(gen_op_simple(POP), iffalse))); - return b; + return BLOCK(gen_op_simple(DUP), cond, + gen_condbranch(BLOCK(gen_op_simple(POP), iftrue), + BLOCK(gen_op_simple(POP), iffalse))); } block gen_cbinding(struct symbol_table* t, block code) { @@ -451,43 +402,135 @@ static int count_cfunctions(block b) { int n = 0; for (inst* i = b.first; i; i = i->next) { if (i->op == CLOSURE_CREATE_C) n++; - if (opcode_describe(i->op)->flags & OP_HAS_BLOCK) - n += count_cfunctions(i->subfn); + n += count_cfunctions(i->subfn); } return n; } + +// Expands call instructions into a calling sequence +static int expand_call_arglist(struct locfile* locations, block* b) { + int errors = 0; + block ret = gen_noop(); + for (inst* curr; (curr = block_take(b));) { + if (opcode_describe(curr->op)->flags & OP_HAS_BINDING) { + if (!curr->bound_by) { + locfile_locate(locations, curr->source, "error: %s is not defined", curr->symbol); + errors++; + // don't process this instruction if it's not well-defined + ret = BLOCK(ret, inst_block(curr)); + continue; + } + } + + block prelude = gen_noop(); + if (curr->op == CALL_JQ) { + int actual_args = 0, desired_args = 0; + // We expand the argument list as a series of instructions + switch (curr->bound_by->op) { + default: assert(0 && "Unknown function type"); break; + case CLOSURE_CREATE: + case CLOSURE_PARAM: { + block callargs = gen_noop(); + for (inst* i; (i = block_take(&curr->arglist));) { + assert(opcode_describe(i->op)->flags & OP_IS_CALL_PSEUDO); + block b = inst_block(i); + switch (i->op) { + default: assert(0 && "Unknown type of parameter"); break; + case CLOSURE_REF: + block_append(&callargs, b); + break; + case CLOSURE_CREATE: + block_append(&prelude, b); + block_append(&callargs, gen_op_block_bound(CLOSURE_REF, b)); + break; + } + actual_args++; + } + curr->imm.intval = actual_args; + curr->arglist = callargs; + + if (curr->bound_by->op == CLOSURE_CREATE) { + for (inst* i = curr->bound_by->arglist.first; i; i = i->next) { + assert(i->op == CLOSURE_PARAM); + desired_args++; + } + } + break; + } + + case CLOSURE_CREATE_C: { + for (inst* i; (i = block_take(&curr->arglist)); ) { + assert(i->op == CLOSURE_CREATE); // FIXME + block body = i->subfn; + i->subfn = gen_noop(); + inst_free(i); + // arguments should be pushed in reverse order, prepend them to prelude + errors += expand_call_arglist(locations, &body); + prelude = BLOCK(gen_subexp(body), prelude); + actual_args++; + } + assert(curr->op == CALL_JQ); + curr->op = CALL_BUILTIN; + curr->imm.intval = actual_args + 1 /* include the implicit input in arg count */; + assert(curr->bound_by->op == CLOSURE_CREATE_C); + desired_args = curr->bound_by->imm.cfunc->nargs - 1; + assert(!curr->arglist.first); + break; + } + } + + if (actual_args != desired_args) { + locfile_locate(locations, curr->source, + "error: %s arguments to %s (expected %d but got %d)", + actual_args > desired_args ? "too many" : "too few", + curr->symbol, desired_args, actual_args); + errors++; + } + + } + ret = BLOCK(ret, prelude, inst_block(curr)); + } + *b = ret; + return errors; +} + static int compile(struct locfile* locations, struct bytecode* bc, block b) { int errors = 0; int pos = 0; int var_frame_idx = 0; bc->nsubfunctions = 0; - bc->nclosures = 0; + errors += expand_call_arglist(locations, &b); + if (bc->parent) { + // functions should end in a return + b = BLOCK(b, gen_op_simple(RET)); + } else { + // the toplevel should YIELD;BACKTRACK; when it finds an answer + b = BLOCK(b, gen_op_simple(YIELD), gen_op_simple(BACKTRACK)); + } for (inst* curr = b.first; curr; curr = curr->next) { if (!curr->next) assert(curr == b.last); - pos += opcode_length(curr->op); + int length = opcode_describe(curr->op)->length; + if (curr->op == CALL_JQ) { + for (inst* arg = curr->arglist.first; arg; arg = arg->next) { + length += 2; + } + } + pos += length; curr->bytecode_pos = pos; curr->compiled = bc; - int opflags = opcode_describe(curr->op)->flags; - if (opflags & OP_HAS_BINDING) { - if (!curr->bound_by) { - locfile_locate(locations, curr->source, "error: %s is not defined", curr->symbol); - errors++; - } - } - if ((opflags & OP_HAS_VARIABLE) && + assert(curr->op != CLOSURE_REF && curr->op != CLOSURE_PARAM); + + if ((opcode_describe(curr->op)->flags & OP_HAS_VARIABLE) && curr->bound_by == curr) { curr->imm.intval = var_frame_idx++; } - if (opflags & OP_HAS_BLOCK) { + + if (curr->op == CLOSURE_CREATE) { assert(curr->bound_by == curr); curr->imm.intval = bc->nsubfunctions++; } - if (curr->op == CLOSURE_PARAM) { - assert(curr->bound_by == curr); - curr->imm.intval = bc->nclosures++; - } if (curr->op == CLOSURE_CREATE_C) { assert(curr->bound_by == curr); int idx = bc->globals->ncfunctions++; @@ -496,21 +539,29 @@ static int compile(struct locfile* locations, struct bytecode* bc, block b) { } } if (bc->nsubfunctions) { - bc->subfunctions = malloc(sizeof(struct bytecode*) * bc->nsubfunctions); + bc->subfunctions = jv_mem_alloc(sizeof(struct bytecode*) * bc->nsubfunctions); for (inst* curr = b.first; curr; curr = curr->next) { - if (!(opcode_describe(curr->op)->flags & OP_HAS_BLOCK)) - continue; - struct bytecode* subfn = malloc(sizeof(struct bytecode)); - bc->subfunctions[curr->imm.intval] = subfn; - subfn->globals = bc->globals; - subfn->parent = bc; - errors += compile(locations, subfn, curr->subfn); + if (curr->op == CLOSURE_CREATE) { + struct bytecode* subfn = jv_mem_alloc(sizeof(struct bytecode)); + bc->subfunctions[curr->imm.intval] = subfn; + subfn->globals = bc->globals; + subfn->parent = bc; + subfn->nclosures = 0; + for (inst* param = curr->arglist.first; param; param = param->next) { + assert(param->op == CLOSURE_PARAM); + assert(param->bound_by == param); + param->imm.intval = subfn->nclosures++; + param->compiled = subfn; + } + errors += compile(locations, subfn, curr->subfn); + curr->subfn = gen_noop(); + } } } else { bc->subfunctions = 0; } bc->codelen = pos; - uint16_t* code = malloc(sizeof(uint16_t) * bc->codelen); + uint16_t* code = jv_mem_alloc(sizeof(uint16_t) * bc->codelen); bc->code = code; pos = 0; jv constant_pool = jv_array(); @@ -519,80 +570,56 @@ static int compile(struct locfile* locations, struct bytecode* bc, block b) { const struct opcode_description* op = opcode_describe(curr->op); if (op->length == 0) continue; - uint16_t* opcode_rewrite = &code[pos]; code[pos++] = curr->op; - int opflags = op->flags; - assert(!(op->flags & OP_IS_CALL_PSEUDO)); - if (opflags & OP_HAS_VARIABLE_LENGTH_ARGLIST) { - int nargs = curr->imm.intval; - assert(nargs > 0); - code[pos++] = (uint16_t)nargs; - int desired_params = 0; - for (int i=0; inext; - assert(curr && opcode_describe(curr->op)->flags & OP_IS_CALL_PSEUDO); - code[pos++] = nesting_level(bc, curr->bound_by); - switch (curr->bound_by->op) { - default: assert(0 && "Unknown type of argument"); - case CLOSURE_CREATE: - code[pos++] = curr->bound_by->imm.intval | ARG_NEWCLOSURE; - if (i == 0) { - inst* i = curr->bound_by; - desired_params = i->compiled->subfunctions[i->imm.intval]->nclosures; - } - break; - case CLOSURE_PARAM: - code[pos++] = curr->bound_by->imm.intval; - if (i == 0) desired_params = 0; - break; - case CLOSURE_CREATE_C: - code[pos++] = curr->bound_by->imm.intval; - *opcode_rewrite = bc->globals->cfunctions[curr->bound_by->imm.intval].callop; - if (i == 0) desired_params = 0; - break; - } + assert(curr->op != CLOSURE_REF && curr->op != CLOSURE_PARAM); + if (curr->op == CALL_BUILTIN) { + assert(curr->bound_by->op == CLOSURE_CREATE_C); + assert(!curr->arglist.first); + code[pos++] = (uint16_t)curr->imm.intval; + code[pos++] = curr->bound_by->imm.intval; + } else if (curr->op == CALL_JQ) { + assert(curr->bound_by->op == CLOSURE_CREATE || + curr->bound_by->op == CLOSURE_PARAM); + code[pos++] = (uint16_t)curr->imm.intval; + code[pos++] = nesting_level(bc, curr->bound_by); + code[pos++] = curr->bound_by->imm.intval | + (curr->bound_by->op == CLOSURE_CREATE ? ARG_NEWCLOSURE : 0); + for (inst* arg = curr->arglist.first; arg; arg = arg->next) { + assert(arg->op == CLOSURE_REF && arg->bound_by->op == CLOSURE_CREATE); + code[pos++] = nesting_level(bc, arg->bound_by); + code[pos++] = arg->bound_by->imm.intval | ARG_NEWCLOSURE; } - assert(nargs - 1 == desired_params); - } else if (opflags & OP_HAS_CONSTANT) { + } else if (op->flags & OP_HAS_CONSTANT) { code[pos++] = jv_array_length(jv_copy(constant_pool)); constant_pool = jv_array_append(constant_pool, jv_copy(curr->imm.constant)); - } else if (opflags & OP_HAS_VARIABLE) { + } else if (op->flags & OP_HAS_VARIABLE) { code[pos++] = nesting_level(bc, curr->bound_by); uint16_t var = (uint16_t)curr->bound_by->imm.intval; code[pos++] = var; if (var > maxvar) maxvar = var; - } else if (opflags & OP_HAS_BRANCH) { + } else if (op->flags & OP_HAS_BRANCH) { assert(curr->imm.target->bytecode_pos != -1); assert(curr->imm.target->bytecode_pos > pos); // only forward branches code[pos] = curr->imm.target->bytecode_pos - (pos + 1); pos++; - } else if (opflags & OP_HAS_CFUNC) { - assert(curr->symbol); - int found = 0; - for (int i=0; iglobals->ncfunctions; i++) { - if (!strcmp(curr->symbol, bc->globals->cfunctions[i].name)) { - code[pos++] = i; - found = 1; - break; - } - } - assert(found); } else if (op->length > 1) { assert(0 && "codegen not implemented for this operation"); } } bc->constants = constant_pool; bc->nlocals = maxvar + 2; // FIXME: frames of size zero? + block_free(b); return errors; } int block_compile(block b, struct locfile* locations, struct bytecode** out) { - struct bytecode* bc = malloc(sizeof(struct bytecode)); + struct bytecode* bc = jv_mem_alloc(sizeof(struct bytecode)); bc->parent = 0; - bc->globals = malloc(sizeof(struct symbol_table)); + bc->nclosures = 0; + bc->globals = jv_mem_alloc(sizeof(struct symbol_table)); int ncfunc = count_cfunctions(b); bc->globals->ncfunctions = 0; - bc->globals->cfunctions = malloc(sizeof(struct cfunction) * ncfunc); + bc->globals->cfunctions = jv_mem_alloc(sizeof(struct cfunction) * ncfunc); int nerrors = compile(locations, bc, b); assert(bc->globals->ncfunctions == ncfunc); if (nerrors > 0) { diff --git a/compile.h b/compile.h index 5390420b..150eef28 100644 --- a/compile.h +++ b/compile.h @@ -1,10 +1,13 @@ #ifndef COMPILE_H #define COMPILE_H #include -#include "bytecode.h" +#include "jv.h" #include "opcode.h" #include "locfile.h" +struct bytecode; +struct symbol_table; + struct inst; typedef struct inst inst; @@ -18,20 +21,19 @@ block gen_location(location, block); block gen_noop(); block gen_op_simple(opcode op); -block gen_op_const(opcode op, jv constant); +block gen_const(jv constant); block gen_op_target(opcode op, block target); block gen_op_var_unbound(opcode op, const char* name); block gen_op_var_bound(opcode op, block binder); -block gen_op_block_defn(opcode op, const char* name, block block); -block gen_op_block_defn_rec(opcode op, const char* name, block block); block gen_op_block_unbound(opcode op, const char* name); -block gen_op_call(opcode op, block arglist); -block gen_op_symbol(opcode op, const char* name); +block gen_function(const char* name, block formals, block body); +block gen_lambda(block body); +block gen_call(const char* name, block body); block gen_subexp(block a); block gen_both(block a, block b); block gen_collect(block expr); -block gen_assign(block expr); +block gen_fold(const char* varname, block init, block body); block gen_definedor(block a, block b); block gen_condbranch(block iftrue, block iffalse); block gen_and(block a, block b); @@ -52,4 +54,20 @@ void block_free(block); +// Here's some horrible preprocessor gunk so that code +// sequences can be contructed as BLOCK(block1, block2, block3) + +#define BLOCK_1(b1) (b1) +#define BLOCK_2(b1,b2) (block_join((b1),(b2))) +#define BLOCK_3(b1,b2,b3) (block_join(BLOCK_2(b1,b2),(b3))) +#define BLOCK_4(b1,b2,b3,b4) (block_join(BLOCK_3(b1,b2,b3),(b4))) +#define BLOCK_5(b1,b2,b3,b4,b5) (block_join(BLOCK_4(b1,b2,b3,b4),(b5))) +#define BLOCK_6(b1,b2,b3,b4,b5,b6) (block_join(BLOCK_5(b1,b2,b3,b4,b5),(b6))) +#define BLOCK_7(b1,b2,b3,b4,b5,b6,b7) (block_join(BLOCK_6(b1,b2,b3,b4,b5,b6),(b7))) + +#define BLOCK_IDX(_1,_2,_3,_4,_5,_6,_7,NAME,...) NAME +#define BLOCK(...) \ + BLOCK_IDX(__VA_ARGS__, BLOCK_7, BLOCK_6, BLOCK_5, BLOCK_4, BLOCK_3, BLOCK_2, BLOCK_1)(__VA_ARGS__) + + #endif diff --git a/docs/Gemfile b/docs/Gemfile new file mode 100644 index 00000000..f06950f6 --- /dev/null +++ b/docs/Gemfile @@ -0,0 +1,5 @@ +source :rubygems + +gem "bonsai" +gem "maruku" +gem "ronn" diff --git a/docs/Gemfile.lock b/docs/Gemfile.lock new file mode 100644 index 00000000..ccbc84fe --- /dev/null +++ b/docs/Gemfile.lock @@ -0,0 +1,53 @@ +GEM + remote: http://rubygems.org/ + specs: + activesupport (3.2.9) + i18n (~> 0.6) + multi_json (~> 1.0) + addressable (2.3.2) + bonsai (1.4.7) + activesupport (>= 3.0.3) + builder (>= 3.0.0) + i18n (>= 0.5.0) + launchy (>= 0.3.7) + liquid (>= 2.2.2) + maruku (>= 0.6.0) + rack + sass + sinatra (>= 1.0) + tilt (>= 1.3) + watch (>= 0.1.0) + builder (3.1.4) + hpricot (0.8.6) + i18n (0.6.1) + launchy (2.1.2) + addressable (~> 2.3) + liquid (2.4.1) + maruku (0.6.1) + syntax (>= 1.0.0) + multi_json (1.5.0) + mustache (0.99.4) + rack (1.4.1) + rack-protection (1.3.2) + rack + rdiscount (1.6.8) + ronn (0.7.3) + hpricot (>= 0.8.2) + mustache (>= 0.7.0) + rdiscount (>= 1.5.8) + sass (3.2.3) + sinatra (1.3.3) + rack (~> 1.3, >= 1.3.6) + rack-protection (~> 1.2) + tilt (~> 1.3, >= 1.3.3) + syntax (1.0.0) + tilt (1.3.3) + watch (0.1.0) + +PLATFORMS + ruby + +DEPENDENCIES + bonsai + maruku + ronn diff --git a/docs/Rakefile b/docs/Rakefile index 62b0139e..a95c97fa 100644 --- a/docs/Rakefile +++ b/docs/Rakefile @@ -2,6 +2,8 @@ require 'bonsai' require 'liquid' require 'maruku' require 'json' +require 'ronn' +require 'tempfile' module ExtraFilters def markdownify(input) @@ -38,11 +40,13 @@ task :serve do server = fork { app = Rack::Builder.app { - use Bonsai::StaticPassThrough, :root => Bonsai.root_dir + "/output", :urls => ["/"] + map "/jq" do + use Bonsai::StaticPassThrough, :root => Bonsai.root_dir + "/output", :urls => ["/"] + end run Bonsai::DevelopmentServer } Rack::Handler.default.run(app, :Port => 5000) do - Launchy.open("http://localhost:5000/") + Launchy.open("http://localhost:5000/jq") end } Watch.new("{content,templates,public}/**/*") { Bonsai::Exporter.process! } @@ -57,3 +61,50 @@ task :build do Bonsai.root_dir = Dir.pwd Bonsai::Exporter.publish! end + +def load_manual + YAML::ENGINE.yamler = 'syck' + YAML::load(File.open("content/3.manual/manual.yml")) +end + +task :manpage do + Tempfile.open "manpage" do |f| + manual = load_manual + f.puts manual['manpage_intro'] + f.puts manual['body'] + manual['sections'].each do |section| + + f.puts "## #{section['title'].upcase}\n" + f.puts section['body'] + f.puts "" + (section['entries'] || []).each do |entry| + f.puts "### #{entry['title']}\n" + f.puts entry['body'] + f.puts "" + (entry['examples'] || []).each do |example| + f.puts " jq '#{example['program']}'" + f.puts " #{example['input']}" + f.puts " => #{example['output'].join(", ")}" + f.puts + end + end + f.puts "" + end + f.puts manual['manpage_epilogue'] + f.close + puts Ronn::Document.new(f.path).convert('roff').gsub(/<\/?code>/,"") + end +end + +task :mantests do + load_manual['sections'].each do |section| + (section['entries'] || []).each do |entry| + (entry['examples'] || []).each do |example| + puts example['program'].gsub("\n", " ") + puts example['input'] + example['output'].each do |s| puts s end + puts + end + end + end +end diff --git a/docs/content/2.download/default.yml b/docs/content/2.download/default.yml index 7834ce72..502ef0b4 100644 --- a/docs/content/2.download/default.yml +++ b/docs/content/2.download/default.yml @@ -5,39 +5,46 @@ body: jq is written in C and has no runtime dependencies, so it should be possible to build it for nearly any platform. Prebuilt binaries are - available for Linux (64-bit x86) and OS X. - - * [Download binary for 64-bit Linux](linux_x86_64/jq) - * [Download binary for OS X](osx_64/jq) - (or use [homebrew](http://mxcl.github.com/homebrew/): `brew install jq`) - * [Download source](source/jq.tgz) + available for Linux, OS X and Windows. - The binaries should just run, but you may need to make them - executable first using: + The binaries should just run, but on OS X and Linux you may need + to make them executable first using `chmod +x jq`. - chmod +x jq - jq is licensed under the MIT license. For all of the gory details, read the file `COPYING` in the source distribution. - Arch Linux - ---------- - A PKGBUILD for jq-1.1 is in the - [AUR](https://aur.archlinux.org/packages.php?ID=63849), as well as a - PKGBUILD for the HEAD of master - ([jq-git](https://aur.archlinux.org/packages.php?ID=63850)). + ### Linux + + * Binaries for [64-bit](linux64/jq) or [32-bit](linux32/jq) systems. - You can install jq using [Yaourt](https://wiki.archlinux.org/index.php/Yaourt) - or manually by following instructions on - [Arch Linux's Wiki](https://wiki.archlinux.org/index.php/Arch_User_Repository). + * For Arch users, a PKGBUILD for jq-1.1 is in the + [AUR](https://aur.archlinux.org/packages.php?ID=63849), as + well as a PKGBUILD for the HEAD of master + ([jq-git](https://aur.archlinux.org/packages.php?ID=63850)). + You can install them using + [Yaourt](https://wiki.archlinux.org/index.php/Yaourt) or + manually by following instructions on [Arch Linux's + Wiki](https://wiki.archlinux.org/index.php/Arch_UseRepository). - Hacking on jq - ============= + * Debian/Ubuntu packages are coming Real Soon Now. - If you want to work on jq, grab the source from - [https://github.com/stedolan/jq](https://github.com/stedolan/jq). - + ### OS X + + * `brew install jq` using [homebrew](http://mxcl.github.com/homebrew/) + + * Or, grab prebuilt [64-bit binaries](osx64/jq) or [32-bit + binaries](osx32/jq) + + ### Windows + + * Executables for [64-bit](win64/jq.exe) or [32-bit](win32/jq.exe) + + ### From source on Linux or OS X + + git clone https://github.com/stedolan/jq.git + cd jq + make && sudo make install To build it from a git clone, you'll need to install a few packages first: @@ -48,23 +55,24 @@ body: * [GCC](http://gcc.gnu.org) * [Make](http://www.gnu.org/software/make) - Most of these aren't necessary if you're just trying to compile - jq from the released tarball - that version has the - non-platform-specific build steps already done, so you'll only - need a C compiler and `make` to finish it off. - For Linux systems, these will all be in your system's package manager, and if you do development on the machine they're most - likely already installed. I have no idea how to get these - installed on OS X, you're on your own there. + likely already installed. + + On OS X, these are all included in Apple's command line tools, which + can be installed from [Xcode](http://developer.apple.com/technologies/tools/). + However, you may find that you need a newer version of Bison than the one provided + by Apple. This can be found in [Homebrew](http://mxcl.github.com/homebrew/) + or [MacPorts](http://macports.org/). `flex` and `bison` are used to generate the lexer and parser for jq, and some python scripts generate the UTF8 encoding tables needed for JSON parsing. - Building the documentation - -- + #### Building the documentation jq's documentation is compiled into static HTML using [Bonsai](http://www.tinytree.info). To view the documentation locally, run `rake serve` from the docs/ subdirectory. + + diff --git a/docs/content/2.download/linux_x86_64/jq b/docs/content/2.download/linux_x86_64/jq deleted file mode 100755 index 4395acc1..00000000 Binary files a/docs/content/2.download/linux_x86_64/jq and /dev/null differ diff --git a/docs/content/2.download/osx_64/jq b/docs/content/2.download/osx_64/jq deleted file mode 100755 index c7b521ec..00000000 Binary files a/docs/content/2.download/osx_64/jq and /dev/null differ diff --git a/docs/content/3.manual/manual.yml b/docs/content/3.manual/manual.yml index 437afdff..de608210 100644 --- a/docs/content/3.manual/manual.yml +++ b/docs/content/3.manual/manual.yml @@ -26,6 +26,41 @@ body: | But that's getting ahead of ourselves. :) Let's start with something simpler: + +manpage_intro: | + jq(1) -- Command-line JSON processor + ==================================== + + ## SYNOPSIS + + `jq` [...] [...] + + `jq` can transform JSON in various ways, by selecting, iterating, + reducing and otherwise mangling JSON documents. For instance, + running the command `jq 'map(.price) | add'` will take an array of + JSON objects as input and return the sum of their "price" fields. + + By default, `jq` reads a stream of JSON objects (whitespace + separated) from `stdin`. One or more may be specified, in + which case `jq` will read input from those instead. + + The are described in the [INVOKING JQ] section, they + mostly concern input and output formatting. The is written + in the jq language and specifies how to transform the input + document. + + ## FILTERS + +manpage_epilogue: | + ## BUGS + + Presumably. Report them or discuss them at: + + https://github.com/stedolan/jq/issues + + ## AUTHOR + + Stephen Dolan `` sections: - title: Invoking jq @@ -40,41 +75,47 @@ sections: You can affect how jq reads and writes its input and output using some command-line options: - * `--slurp`/`-s` - + * `--slurp`/`-s`: + Instead of running the filter for each JSON object in the input, read the entire input stream into a large array and run the filter just once. - * `--raw-input`/`-R` + * `--raw-input`/`-R`: Don't parse the input as JSON. Instead, each line of text is passed to the filter as a string. If combined with `--slurp`, then the entire input is passed to the filter as a single long string. - - * `--null-input`/`-n` - + + * `--null-input`/`-n`: + Don't read any input at all! Instead, the filter is run once using `null` as the input. This is useful when using jq as a simple calculator or to construct JSON data from scratch. - - * `--compact-output` / `-c` - + + * `--compact-output` / `-c`: + By default, jq pretty-prints JSON output. Using this option will result in more compact output by instead putting each JSON object on a single line. - - * `--ascii-output` / `-a` - + + * `--colour-output` / `-C` and `--monochrome-output` / `-M`: + + By default, jq outputs colored JSON if writing to a + terminal. You can force it to produce color even if writing to + a pipe or a file using `-C`, and disable color with `-M`. + + * `--ascii-output` / `-a`: + jq usually outputs non-ASCII Unicode codepoints as UTF-8, even if the input specified them as escape sequences (like "\u03bc"). Using this option, you can force jq to produce pure ASCII output with every non-ASCII character replaced with the equivalent escape sequence. - - * `--raw-output` / `-r` - + + * `--raw-output` / `-r`: + With this option, if the filter's result is a string then it will be written directly to standard output rather than being formatted as a JSON string with quotes. This can be useful for @@ -142,7 +183,7 @@ sections: examples: - program: '.[]' - input: '[{name":"JSON", "good":true}, {"name":"XML", "good":false}]' + input: '[{"name":"JSON", "good":true}, {"name":"XML", "good":false}]' output: - '{"name":"JSON", "good":true}' - '{"name":"XML", "good":false}' @@ -172,7 +213,7 @@ sections: - program: '.[4,2]' input: '["a","b","c","d","e"]' - output: ['"d"', '"c"'] + output: ['"e"', '"c"'] - title: "`|`" body: | @@ -188,7 +229,7 @@ sections: examples: - program: '.[] | .name' - input: '[{name":"JSON", "good":true}, {"name":"XML", "good":false}]' + input: '[{"name":"JSON", "good":true}, {"name":"XML", "good":false}]' output: ['"JSON"', '"XML"'] - title: Types and Values @@ -315,14 +356,23 @@ sections: the key-value pairs from both objects into a single combined object. If both objects contain a value for the same key, the object on the right of the `+` wins. + + `null` can be added to any value, and returns the other + value unchanged. examples: - program: '.a + 1' input: '{"a": 7}' - output: '{"a": 8}' + output: ['8'] - program: '.a + .b' input: '{"a": [1,2], "b": [3,4]}' output: ['[1,2,3,4]'] + - program: '.a + null' + input: '{"a": 1}' + output: ['1'] + - program: '.a + 1' + input: '{}' + output: ['1'] - program: '{a: 1} + {b: 2} + {c: 3} + {a: 42}' input: 'null' output: ['{"a": 42, "b": 2, "c": 3}'] @@ -403,11 +453,11 @@ sections: `foo` returns true for that input, and produces no output otherwise. - It's useful for filtering lists: `[1,2,3] | select(. >= 2)` + It's useful for filtering lists: `[1,2,3] | map(select(. >= 2))` will give you `[3]`. examples: - - program: 'select(. >= 2)' + - program: 'map(select(. >= 2))' input: '[1,5,3,0,7]' output: ['[5,3,7]'] @@ -421,7 +471,7 @@ sections: examples: - program: '1, empty, 2' input: 'null' - output: [1,2] + output: [1, 2] - program: '[1,2,empty,3]' input: 'null' output: ['[1,2,3]'] @@ -455,7 +505,7 @@ sections: examples: - program: add input: '["a","b","c"]' - output: ["abc"] + output: ['"abc"'] - program: add input: '[1, 2, 3]' output: [6] @@ -473,7 +523,7 @@ sections: examples: - program: '.[] | tonumber' input: '[1, "1"]' - output: [1,1] + output: [1, 1] - title: `tostring` body: | @@ -487,6 +537,144 @@ sections: input: '[1, "1", [1]]' output: ['"1"', '"1"', '"[1]"'] + - title: `sort, sort_by` + body: | + + The `sort` functions sorts its input, which must be an + array. Values are sorted in the following order: + + * `null` + * `false` + * `true` + * numbers + * strings, in alphabetical order (by unicode codepoint value) + * arrays, in lexical order + * objects + + The ordering for objects is a little complex: first they're + compared by comparing their sets of keys (as arrays in + sorted order), and if their keys are equal then the values + are compared key by key. + + `sort_by` may be used to sort by a particular field of an + object, or by applying any jq filter. `sort_by(foo)` + compares two elements by comparing the result of `foo` on + each element. + + examples: + - program: 'sort' + input: '[8,3,null,6]' + output: ['[null,3,6,8]'] + - program: 'sort_by(.foo)' + input: '[{"foo":4, "bar":10}, {"foo":3, "bar":100}, {"foo":2, "bar":1}]' + output: ['[{"foo":2, "bar":1}, {"foo":3, "bar":100}, {"foo":4, "bar":10}]'] + + - title: `group_by` + body: | + + `group_by(.foo)` takes as input an array, groups the + elements having the same `.foo` field into separate arrays, + and produces all of these arrays as elements of a larger + array, sorted by the value of the `.foo` field. + + Any jq expression, not just a field access, may be used in + place of `.foo`. The sorting order is the same as described + in the `sort` function above. + + examples: + - program: 'group_by(.foo)' + input: '[{"foo":1, "bar":10}, {"foo":3, "bar":100}, {"foo":1, "bar":1}]' + output: ['[[{"foo":1, "bar":10}, {"foo":1, "bar":1}], [{"foo":3, "bar":100}]]'] + + - title: `min`, `max`, `min_by`, `max_by` + body: | + + Find the minimum or maximum element of the input array. The + `_by` versions allow you to specify a particular field or + property to examine, e.g. `min_by(.foo)` finds the object + with the smallest `foo` field. + + examples: + - program: 'min' + input: '[5,4,2,7]' + output: ['2'] + - program: 'max_by(.foo)' + input: '[{"foo":1, "bar":14}, {"foo":2, "bar":3}]' + output: ['{"foo":2, "bar":3}'] + + - title: `unique` + body: | + + The `unique` function takes as input an array and produces + an array of the same elements, in sorted order, with + duplicates removed. + + examples: + - program: 'unique' + input: '[1,2,5,3,5,3,1,3]' + output: ['[1,2,3,5]'] + + - title: `contains` + body: | + + The filter `contains(b)` will produce true if b is + completely contained within the input. A string B is + contained in a string A if B is a substring of A. An array B + is contained in an array A is all elements in B are + contained in any element in A. An object B is contained in + object A if all of the values in B are contained in the + value in A with the same key. All other types are assumed to + be contained in each other if they are equal. + + examples: + - program: 'contains("bar")' + input: '"foobar"' + output: ['true'] + - program: 'contains(["baz", "bar"])' + input: '["foobar", "foobaz", "blarp"]' + output: ['true'] + - program: 'contains(["bazzzzz", "bar"])' + input: '["foobar", "foobaz", "blarp"]' + output: ['false'] + - program: 'contains({foo: 12, bar: [{barp: 12}]})' + input: '{"foo": 12, "bar":[1,2,{"barp":12, "blip":13}]}' + output: ['true'] + - program: 'contains({foo: 12, bar: [{barp: 15}]})' + input: '{"foo": 12, "bar":[1,2,{"barp":12, "blip":13}]}' + output: ['false'] + + - title: `recurse` + body: | + + The `recurse` function allows you to search through a + recursive structure, and extract interesting data from all + levels. Suppose your input represents a filesystem: + + {"name": "/", "children": [ + {"name": "/bin", "children": [ + {"name": "/bin/ls", "children": []}, + {"name": "/bin/sh", "children": []}]}, + {"name": "/home", "children": [ + {"name": "/home/stephen", "children": [ + {"name": "/home/stephen/jq", "children": []}]}]}]} + + Now suppose you want to extract all of the filenames + present. You need to retrieve `.name`, `.children[].name`, + `.children[].children[].name`, and so on. You can do this + with: + + recurse(.children[]) | .name + + examples: + - program: 'recurse(.foo[])' + input: '{"foo":[{"foo": []}, {"foo":[{"foo":[]}]}]}' + output: + - '{"foo":[{"foo":[]},{"foo":[{"foo":[]}]}]}' + - '{"foo":[]}' + - '{"foo":[{"foo":[]}]}' + - '{"foo":[]}' + + - title: "String interpolation - `\(foo)`" body: | @@ -499,10 +687,77 @@ sections: input: '42' output: ['"The input was 42, which is one less than 43"'] - + - title: "Format strings and escaping" + body: | - + The `@foo` syntax is used to format and escape strings, + which is useful for building URLs, documents in a language + like HTML or XML, and so forth. `@foo` can be used as a + filter on its own, the possible escapings are: + * `@text`: + + Calls `tostring`, see that function for details. + + * `@json`: + + Serialises the input as JSON. + + * `@html`: + + Applies HTML/XML escaping, by mapping the characters + `<>&'"` to their entity equivalents `<`, `>`, + `&`, `'`, `"`. + + * `@uri`: + + Applies percent-encoding, by mapping all reserved URI + characters to a `%xx` sequence. + + * `@csv`: + + The input must be an array, and it is rendered as CSV + with double quotes for strings, and quotes escaped by + repetition. + + * `@sh`: + + The input is escaped suitable for use in a command-line + for a POSIX shell. If the input is an array, the output + will be a series of space-separated strings. + + * `@base64`: + + The input is converted to base64 as specified by RFC 4648. + + This syntax can be combined with string interpolation in a + useful way. You can follow a `@foo` token with a string + literal. The contents of the string literal will *not* be + escaped. However, all interpolations made inside that string + literal will be escaped. For instance, + + @uri "http://www.google.com/search?q=\(.search)" + + will produce the following output for the input + `{"search":"jq!"}`: + + http://www.google.com/search?q=jq%21 + + Note that the slashes, question mark, etc. in the URL are + not escaped, as they were part of the string literal. + + examples: + - program: '@html' + input: '"This works if x < y"' + output: ['"This works if x < y"'] + +# - program: '@html "Anonymous said: \(.)"' +# input: '""' +# output: ["Anonymous said: <script>alert("lol hax");</script>"] + + - program: '@sh "echo \(.)"' + input: "\"O'Hara's Ale\"" + output: ["\"echo 'O'\\''Hara'\\''s Ale\""] - title: Conditionals and Comparisons entries: @@ -521,34 +776,7 @@ sections: examples: - program: '.[] == 1' input: '[1, 1.0, "1", "banana"]' - output: ['[true, true, false, false]'] - - title: `contains` - body: | - - The expression 'a contains b' will produce true if b is completely - contained within a. A string B is contained in a string A if B is a - substring of A. An array B is contained in an array A is all elements - in B are contained in any element in A. An object B is contained in - object A if all of the values in B are contained in the value in A with - the same key. All other types are assumed to be contained in each other - if they are equal. - - examples: - - program: '. == contains "bar"' - input: '"foobar"' - output: ['true'] - - program: '. contains ["baz", "bar"]' - input: '["foobar", "foobaz", "blarp"]' - output: ['true'] - - program: '. contains ["bazzzzz", "bar"]' - input: '["foobar", "foobaz", "blarp"]' - output: ['false'] - - program: '. contains {foo: 12, bar: [{barp: 12}]}' - input: '{foo: 12, bar:[1,2,{barp:12, blip:13}]}' - output: ['true'] - - program: '. contains {foo: 12, bar: [{barp: 15}]}' - input: '{foo: 12, bar:[1,2,{barp:12, blip:13}]}' - output: ['false'] + output: ['true', 'true', 'false', 'false'] - title: if-then-else body: | @@ -561,8 +789,8 @@ sections: means that you'll sometimes have to be more explicit about the condition you want: you can't test whether, e.g. a string is empty using `if .name then A else B end`, you'll - need something more like 'if (.name | count) > 0 then A else - B end' instead. + need something more like `if (.name | count) > 0 then A else + B end` instead. If the condition A produces multiple results, it is considered "true" if any of those results is not false or @@ -581,6 +809,21 @@ sections: end input: 2 output: ['"many"'] + + - title: `>, >=, <=, <` + body: | + + The comparison operators `>`, `>=`, `<=`, `<` return whether + their left argument is greater than, greater than or equal + to, less than or equal to or less than their right argument + (respectively). + + The ordering is the same as that described for `sort`, above. + + examples: + - program: '. < 5' + input: 2 + output: ['true'] - title: and/or/not body: | @@ -639,7 +882,7 @@ sections: input: '{}' output: [42] - - title: Variables and Functions + - title: Advanced features body: | Variables are an absolute necessity in most programming languages, but they're relegated to an "advanced feature" in jq. @@ -656,6 +899,10 @@ sections: (many jq functions such as `map` and `find` are in fact written in jq). + Finally, jq has a `fold` operation, which is very powerful but a + bit tricky. Again, it's mostly used internally, to define some + useful bits of jq's standard library. + entries: - title: Variables body: | @@ -670,8 +917,8 @@ sections: For instance, calculating the average value of an array of numbers requires a few variables in most languages - at least one to hold the array, perhaps one for each element or for a loop counter. In jq, it's - simply `add / length` - the `sum` expression is given the array and - produces its sum, and the `count` expression is given the array and + simply `add / length` - the `add` expression is given the array and + produces its sum, and the `length` expression is given the array and produces its length. So, there's generally a cleaner way to solve most problems in jq that @@ -754,12 +1001,45 @@ sections: input's `.foo` field to each element of the array. examples: - - program: 'def addvalue(f): map(. + [$value]); addvalue(.[0])' + - program: 'def addvalue(f): . + [f]; map(addvalue(.[0]))' input: '[[1,2],[10,20]]' output: ['[[1,2,1], [10,20,10]]'] - - program: 'def addvalue(f): f as $x | map (. + $x); addvalue(.[0])' + - program: 'def addvalue(f): f as $x | map(. + $x); addvalue(.[0])' input: '[[1,2],[10,20]]' - output: ['[[1,2,[1,2]], [10,20,[1,2]]]'] + output: ['[[1,2,1,2], [10,20,1,2]]'] + + - title: Fold + body: | + + The `fold` syntax in jq allows you to combine all of the + results of an expression by accumulating them into a single + answer. As an example, we'll pass `[3,2,1]` to this expression: + + fold 0 as $sum (.[] | $sum + .) + + The variable `$sum` is first given the value `0`. The body + of the fold (i.e. `.[] | $sum + .`) is evaluated. `.[]` + produces three results, `3`, `2`, and `1`. For the first + one, `$sum + .` gives `3`. + + Having produced this answer, jq backtracks to find the next + result as per usual. However, this time, `$sum` is set to + the previous value of the body, so `$sum + .` gives + `5`. After the final backtracking, `$sum + .` gives + `6`. This final value is used as the value of the entire + `fold` expression, so the above filter returns `6`. + + More formally, in order to evaluate `fold INIT as $VAR + (BODY)`, jq first sets `$VAR` to the value of `INIT`. It + then runs through `BODY`. Each time `BODY` produces a value, + `$VAR` is set to that value and jq backtracks to find the + next one. When `BODY` stops producing values, the final + value of `$VAR` is the result of the entire expression. + + examples: + - program: 'fold 0 as $sum (.[] | $sum + .)' + input: '[10,2,5,3]' + output: ['20'] - title: Assignment @@ -871,4 +1151,5 @@ sections: "stedolan" wrote, and we can comment on each of them in the same way that we did before: - (.posts[] | select(.author == "stedolan") | .comments) |= . + ["terrible."] + (.posts[] | select(.author == "stedolan") | .comments) |= + . + ["terrible."] diff --git a/docs/site.yml b/docs/site.yml index 92a690d1..863d578d 100644 --- a/docs/site.yml +++ b/docs/site.yml @@ -1,10 +1,8 @@ # The key value pairs found below are available within the templates. -jq_version: 1.1 -# Next line is needed for publishing to github pages +jq_version: 1.2 root: '/jq' -# root: '' footer: | This website is made with [Bonsai](http://www.tinytree.info) and diff --git a/docs/templates/index.liquid b/docs/templates/index.liquid index aaff1c9f..7785526c 100644 --- a/docs/templates/index.liquid +++ b/docs/templates/index.liquid @@ -16,8 +16,8 @@ diff --git a/execute.c b/execute.c index a596e28b..fb4a3895 100644 --- a/execute.c +++ b/execute.c @@ -7,14 +7,15 @@ #include "opcode.h" #include "bytecode.h" -#include "compile.h" #include "forkable_stack.h" #include "frame_layout.h" +#include "jv_alloc.h" #include "locfile.h" #include "jv.h" -#include "jq_parser.h" +#include "jv_aux.h" +#include "parser.h" #include "builtin.h" typedef struct { @@ -33,7 +34,7 @@ int path_push(stackval sv, jv val) { if (pos == pathsize) { int oldpathsize = pathsize; pathsize = oldpathsize ? oldpathsize * 2 : 100; - pathbuf = realloc(pathbuf, sizeof(pathbuf[0]) * pathsize); + pathbuf = jv_mem_realloc(pathbuf, sizeof(pathbuf[0]) * pathsize); for (int i=oldpathsize; istack_in; i++) { + int stack_in = opdesc->stack_in; + if (stack_in == -1) stack_in = pc[1]; + for (int i=0; iglobals->cfunctions[*pc++]; - func->fptr(cfunc_input, cfunc_output); - top.value = cfunc_output[0]; + top.value = cfunction_invoke(func, cfunc_input); if (jv_is_valid(top.value)) { stack_push(top); } else { @@ -427,37 +434,16 @@ jv jq_next() { break; } - case CALL_BUILTIN_3_1: { - assert(*pc == 1); // no closure args allowed - pc++; // skip nclosures - pc++; // skip level - stackval top = stack_pop(); - jv a = stack_pop().value; - jv b = stack_pop().value; - cfunc_input[0] = top.value; - cfunc_input[1] = a; - cfunc_input[2] = b; - struct cfunction* func = &frame_current_bytecode(&frame_stk)->globals->cfunctions[*pc++]; - func->fptr(cfunc_input, cfunc_output); - top.value = cfunc_output[0]; - if (jv_is_valid(top.value)) { - stack_push(top); - } else { - print_error(top.value); - goto do_backtrack; - } - break; - } - - case CALL_1_1: { + case CALL_JQ: { uint16_t nclosures = *pc++; + uint16_t* retaddr = pc + 2 + nclosures*2; frame_ptr new_frame = frame_push(&frame_stk, make_closure(&frame_stk, frame_current(&frame_stk), pc), - pc + nclosures * 2); + retaddr); pc += 2; frame_ptr old_frame = forkable_stack_peek_next(&frame_stk, new_frame); - assert(nclosures - 1 == frame_self(new_frame)->bc->nclosures); - for (int i=0; ibc->nclosures); + for (int i=0; i 1 ? "errors" : "error"); diff --git a/forkable_stack.h b/forkable_stack.h index 5bfad8a9..74fcf433 100644 --- a/forkable_stack.h +++ b/forkable_stack.h @@ -1,9 +1,9 @@ #ifndef FORKABLE_STACK_H #define FORKABLE_STACK_H -#include #include #include #include +#include "jv_alloc.h" struct forkable_stack_header { /* distance in bytes between this header and the header @@ -15,11 +15,10 @@ struct forkable_stack_header { struct forkable_stack { // Stack grows down from stk+length - char* stk; // stk+length is just past end of allocated area - // stk = malloc(length) + // stk = jv_mem_alloc(length) int length; // stk+pos is header of top object, or stk+length if empty @@ -41,7 +40,7 @@ static int forkable_stack_empty(struct forkable_stack* s) { } static void forkable_stack_init(struct forkable_stack* s, size_t sz) { - s->stk = malloc(sz); + s->stk = jv_mem_alloc(sz); s->length = sz; s->pos = s->length; s->savedlimit = s->length; @@ -51,7 +50,7 @@ static void forkable_stack_init(struct forkable_stack* s, size_t sz) { static void forkable_stack_free(struct forkable_stack* s) { assert(forkable_stack_empty(s)); assert(s->savedlimit == s->length); - free(s->stk); + jv_mem_free(s->stk); s->stk = 0; } @@ -61,10 +60,9 @@ static void* forkable_stack_push(struct forkable_stack* s, size_t sz_size) { forkable_stack_check(s); int curr = s->pos < s->savedlimit ? s->pos : s->savedlimit; if (curr - size < 0) { - //assert(0); int oldlen = s->length; s->length = (size + oldlen + 1024) * 2; - s->stk = realloc(s->stk, s->length); + s->stk = jv_mem_realloc(s->stk, s->length); int shift = s->length - oldlen; memmove(s->stk + shift, s->stk, oldlen); s->pos += shift; @@ -102,33 +100,45 @@ static void* forkable_stack_peek_next(struct forkable_stack* s, void* top) { // Returns 1 if the next forkable_stack_pop will permanently remove an // object from the stack (i.e. the top object was not saved with a fork) static int forkable_stack_pop_will_free(struct forkable_stack* s) { - return s->pos < s->savedlimit ? 1 : 0; + return s->pos < s->savedlimit; } static void forkable_stack_pop(struct forkable_stack* s) { forkable_stack_check(s); struct forkable_stack_header* elem = forkable_stack_peek(s); - s->pos += elem->next_delta; + int reclaim_upto = s->pos + elem->next_delta < s->savedlimit ? + s->pos + elem->next_delta : s->savedlimit; + assert(reclaim_upto <= s->length); + int reclaimed = reclaim_upto > s->pos ? reclaim_upto - s->pos : 0; + assert(0 <= reclaimed && s->pos + reclaimed <= s->length); + // s->pos <= i < reclaim_upto means that position i is to be freed + assert((reclaimed > 0) == forkable_stack_pop_will_free(s)); + int new_pos = s->pos + elem->next_delta; + jv_mem_invalidate(elem, reclaimed); + s->pos = new_pos; } struct forkable_stack_state { - int prevpos, prevlimit; + // We save the previous pos, savedlimit as + // length-pos, length-savedlimit since these + // values are stable across stack reallocations. + int prev_pos_delta, prev_limit_delta; }; static void forkable_stack_save(struct forkable_stack* s, struct forkable_stack_state* state) { forkable_stack_check(s); - state->prevpos = s->pos; - state->prevlimit = s->savedlimit; + state->prev_pos_delta = s->length - s->pos; + state->prev_limit_delta = s->length - s->savedlimit; if (s->pos < s->savedlimit) s->savedlimit = s->pos; } static void forkable_stack_switch(struct forkable_stack* s, struct forkable_stack_state* state) { forkable_stack_check(s); int curr_pos = s->pos; - s->pos = state->prevpos; - state->prevpos = curr_pos; + s->pos = s->length - state->prev_pos_delta; + state->prev_pos_delta = s->length - curr_pos; int curr_limit = s->savedlimit; if (curr_pos < curr_limit) s->savedlimit = curr_pos; @@ -138,10 +148,10 @@ static void forkable_stack_switch(struct forkable_stack* s, struct forkable_stac static void forkable_stack_restore(struct forkable_stack* s, struct forkable_stack_state* state) { forkable_stack_check(s); - assert(s->savedlimit <= state->prevpos); - assert(s->savedlimit <= state->prevlimit); - s->pos = state->prevpos; - s->savedlimit = state->prevlimit; + assert(s->savedlimit <= s->length - state->prev_pos_delta); + assert(s->savedlimit <= s->length - state->prev_limit_delta); + s->pos = s->length - state->prev_pos_delta; + s->savedlimit = s->length - state->prev_limit_delta; forkable_stack_check(s); } diff --git a/jq_test.c b/jq_test.c index dd41a2f9..77e2860f 100644 --- a/jq_test.c +++ b/jq_test.c @@ -8,9 +8,24 @@ static void jv_test(); static void run_jq_tests(); -int main() { +FILE* testdata; + +int main(int argc, char* argv[]) { jv_test(); + if (argc == 1) { + testdata = fopen("testdata", "r"); + } else if (argc == 2) { + if (!strcmp(argv[1], "-")) { + testdata = stdin; + } else { + testdata = fopen(argv[1], "r"); + } + } else { + printf("usage: %s OR cat testdata | %s - OR %s testdata\n", argv[0], argv[0], argv[0]); + return 127; + } run_jq_tests(); + if (testdata != stdin) fclose(testdata); } @@ -26,7 +41,7 @@ static int skipline(const char* buf) { static void run_jq_tests() { FILE* testdata = NULL; char buf[4096]; - int tests = 0, passed = 0; + int tests = 0, passed = 0, invalid = 0; testdata = fopen("testdata","r"); if ( NULL == testdata ) @@ -42,37 +57,41 @@ static void run_jq_tests() { while (1) { if (!fgets(buf, sizeof(buf), testdata)) break; if (skipline(buf)) continue; + if (buf[strlen(buf)-1] == '\n') buf[strlen(buf)-1] = 0; printf("Testing %s\n", buf); int pass = 1; + tests++; struct bytecode* bc = jq_compile(buf); - assert(bc); + if (!bc) {invalid++; continue;} +#if JQ_DEBUG printf("Disassembly:\n"); dump_disassembly(2, bc); printf("\n"); +#endif fgets(buf, sizeof(buf), testdata); jv input = jv_parse(buf); - assert(jv_is_valid(input)); + if (!jv_is_valid(input)){ invalid++; continue; } jq_init(bc, input); while (fgets(buf, sizeof(buf), testdata)) { if (skipline(buf)) break; jv expected = jv_parse(buf); - assert(jv_is_valid(expected)); + if (!jv_is_valid(expected)){ invalid++; continue; } jv actual = jq_next(); if (!jv_is_valid(actual)) { jv_free(actual); - printf("Insufficient results\n"); + printf("*** Insufficient results\n"); pass = 0; break; } else if (!jv_equal(jv_copy(expected), jv_copy(actual))) { - printf("Expected "); + printf("*** Expected "); jv_dump(jv_copy(expected), 0); printf(", but got "); jv_dump(jv_copy(actual), 0); printf("\n"); pass = 0; } - jv as_string = jv_dump_string(jv_copy(expected), rand()); + jv as_string = jv_dump_string(jv_copy(expected), rand() & ~JV_PRINT_COLOUR); jv reparsed = jv_parse_sized(jv_string_value(as_string), jv_string_length(jv_copy(as_string))); assert(jv_equal(jv_copy(expected), jv_copy(reparsed))); jv_free(as_string); @@ -83,7 +102,7 @@ static void run_jq_tests() { if (pass) { jv extra = jq_next(); if (jv_is_valid(extra)) { - printf("Superfluous result: "); + printf("*** Superfluous result: "); jv_dump(extra, 0); printf("\n"); pass = 0; @@ -93,11 +112,9 @@ static void run_jq_tests() { } jq_teardown(); bytecode_free(bc); - tests++; passed+=pass; } - fclose(testdata); - printf("%d of %d tests passed\n", passed,tests); + printf("%d of %d tests passed (%d malformed)\n", passed,tests,invalid); if (passed != tests) exit(1); } diff --git a/jv.c b/jv.c index 77937a8d..fb4d2103 100644 --- a/jv.c +++ b/jv.c @@ -6,6 +6,7 @@ #include #include +#include "jv_alloc.h" #include "jv.h" /* @@ -86,7 +87,7 @@ jv jv_invalid_with_msg(jv err) { jv x; x.kind = JV_KIND_INVALID; x.val.complex.i[0] = x.val.complex.i[1] = 0; - jvp_invalid* i = malloc(sizeof(jvp_invalid)); + jvp_invalid* i = jv_mem_alloc(sizeof(jvp_invalid)); x.val.complex.ptr = &i->refcnt; i->refcnt.count = 1; i->errmsg = err; @@ -113,7 +114,7 @@ int jv_invalid_has_msg(jv inv) { static void jvp_invalid_free(jv_complex* x) { if (jvp_refcnt_dec(x)) { jv_free(((jvp_invalid*)x->ptr)->errmsg); - free(x->ptr); + jv_mem_free(x->ptr); } } @@ -157,7 +158,7 @@ static jvp_array* jvp_array_ptr(jv_complex* a) { } static jvp_array* jvp_array_alloc(unsigned size) { - jvp_array* a = malloc(sizeof(jvp_array) + sizeof(jv) * size); + jvp_array* a = jv_mem_alloc(sizeof(jvp_array) + sizeof(jv) * size); a->refcnt.count = 1; a->length = 0; a->alloc_length = size; @@ -175,7 +176,7 @@ static void jvp_array_free(jv_complex* a) { for (int i=0; ilength; i++) { jv_free(array->elements[i]); } - free(array); + jv_mem_free(array); } } @@ -253,8 +254,7 @@ static int jvp_array_equal(jv_complex* a, jv_complex* b) { static jv_complex jvp_array_slice(jv_complex* a, int start, int end) { // FIXME: maybe slice should reallocate if the slice is small enough assert(start <= end); - jvp_array* array = jvp_array_ptr(a); - assert(a->i[1] + end < array->length); + assert(a->i[0] + end <= a->i[1]); jv_complex slice = *a; slice.i[0] += start; slice.i[1] = slice.i[0] + (end - start); @@ -315,8 +315,8 @@ jv jv_array_concat(jv a, jv b) { assert(jv_get_kind(b) == JV_KIND_ARRAY); // FIXME: could be much faster - for (int i=0; irefcnt.count = 1; s->alloc_length = size; return s; @@ -389,7 +388,7 @@ static jv_complex jvp_string_new(const char* data, uint32_t length) { static void jvp_string_free(jv_complex* s) { if (jvp_refcnt_dec(s)) { jvp_string* str = jvp_string_ptr(s); - free(str); + jv_mem_free(str); } } @@ -569,17 +568,17 @@ jv jv_string_append_str(jv a, const char* str) { jv jv_string_fmt(const char* fmt, ...) { int size = 1024; while (1) { - char* buf = malloc(size); + char* buf = jv_mem_alloc(size); va_list args; va_start(args, fmt); int n = vsnprintf(buf, size, fmt, args); va_end(args); if (n < size) { jv ret = jv_string_sized(buf, n); - free(buf); + jv_mem_free(buf); return ret; } else { - free(buf); + jv_mem_free(buf); size = n * 2; } } @@ -609,9 +608,9 @@ static jv_complex jvp_object_new(int size) { // size must be a power of two assert(size > 0 && (size & (size - 1)) == 0); - jvp_object* obj = malloc(sizeof(jvp_object) + - sizeof(struct object_slot) * size + - sizeof(int) * (size * 2)); + jvp_object* obj = jv_mem_alloc(sizeof(jvp_object) + + sizeof(struct object_slot) * size + + sizeof(int) * (size * 2)); obj->refcnt.count = 1; for (int i=0; ielements[i].next = i - 1; @@ -711,7 +710,7 @@ static void jvp_object_free(jv_complex* o) { jv_free(slot->value); } } - free(jvp_object_ptr(o)); + jv_mem_free(jvp_object_ptr(o)); } } @@ -730,7 +729,7 @@ static void jvp_object_rehash(jv_complex* object) { new_slot->value = slot->value; } // references are transported, just drop the old table - free(jvp_object_ptr(object)); + jv_mem_free(jvp_object_ptr(object)); *object = new_object; } @@ -876,10 +875,8 @@ int jv_object_length(jv object) { jv jv_object_merge(jv a, jv b) { assert(jv_get_kind(a) == JV_KIND_OBJECT); - jv_object_foreach(i, b) { - a = jv_object_set(a, - jv_object_iter_key(b, i), - jv_object_iter_value(b, i)); + jv_object_foreach(b, k, v) { + a = jv_object_set(a, k, v); } jv_free(b); return a; @@ -890,10 +887,8 @@ int jv_object_contains(jv a, jv b) { assert(jv_get_kind(b) == JV_KIND_OBJECT); int r = 1; - jv_object_foreach(i, b) { - jv key = jv_object_iter_key(b, i); + jv_object_foreach(b, key, b_val) { jv a_val = jv_object_get(jv_copy(a), jv_copy(key)); - jv b_val = jv_object_get(jv_copy(b), jv_copy(key)); r = jv_contains(a_val, b_val); jv_free(key); @@ -1037,4 +1032,4 @@ int jv_contains(jv a, jv b) { jv_free(a); jv_free(b); return r; -} \ No newline at end of file +} diff --git a/jv.h b/jv.h index a77346c3..c3bff367 100644 --- a/jv.h +++ b/jv.h @@ -72,7 +72,13 @@ jv jv_array_set(jv, int, jv); jv jv_array_append(jv, jv); jv jv_array_concat(jv, jv); jv jv_array_slice(jv, int, int); - +#define jv_array_foreach(a, i, x) \ + for (int jv_len__ = jv_array_length(jv_copy(a)), i=0, jv_j__ = 1; \ + jv_j__; jv_j__ = 0) \ + for (jv x; \ + i < jv_len__ ? \ + (x = jv_array_get(jv_copy(a), i), 1) : 0; \ + i++) jv jv_string(const char*); jv jv_string_sized(const char*, int); @@ -96,16 +102,21 @@ int jv_object_iter_next(jv, int); int jv_object_iter_valid(jv, int); jv jv_object_iter_key(jv, int); jv jv_object_iter_value(jv, int); -#define jv_object_foreach(i,t) \ - for (int i = jv_object_iter(t); \ - jv_object_iter_valid(t, i); \ - i = jv_object_iter_next(t, i)) \ +#define jv_object_foreach(t, k, v) \ + for (int jv_i__ = jv_object_iter(t), jv_j__ = 1; jv_j__; jv_j__ = 0) \ + for (jv k, v; \ + jv_object_iter_valid((t), jv_i__) ? \ + (k = jv_object_iter_key(t, jv_i__), \ + v = jv_object_iter_value(t, jv_i__), \ + 1) \ + : 0; \ + jv_i__ = jv_object_iter_next(t, jv_i__)) \ int jv_get_refcnt(jv); -enum { JV_PRINT_PRETTY = 1, JV_PRINT_ASCII = 2 }; +enum { JV_PRINT_PRETTY = 1, JV_PRINT_ASCII = 2, JV_PRINT_COLOUR = 4 }; void jv_dump(jv, int flags); jv jv_dump_string(jv, int flags); @@ -113,67 +124,6 @@ jv jv_parse(const char* string); jv jv_parse_sized(const char* string, int length); -static jv jv_lookup(jv t, jv k) { - jv v; - if (jv_get_kind(t) == JV_KIND_OBJECT && jv_get_kind(k) == JV_KIND_STRING) { - v = jv_object_get(t, k); - if (!jv_is_valid(v)) { - jv_free(v); - v = jv_null(); - } - } else if (jv_get_kind(t) == JV_KIND_ARRAY && jv_get_kind(k) == JV_KIND_NUMBER) { - // FIXME: don't do lookup for noninteger index - v = jv_array_get(t, (int)jv_number_value(k)); - if (!jv_is_valid(v)) { - jv_free(v); - v = jv_null(); - } - } else if (jv_get_kind(t) == JV_KIND_NULL && - (jv_get_kind(k) == JV_KIND_STRING || jv_get_kind(k) == JV_KIND_NUMBER)) { - jv_free(t); - jv_free(k); - v = jv_null(); - } else { - v = jv_invalid_with_msg(jv_string_fmt("Cannot index %s with %s", - jv_kind_name(jv_get_kind(t)), - jv_kind_name(jv_get_kind(k)))); - jv_free(t); - jv_free(k); - } - return v; -} - -static jv jv_modify(jv t, jv k, jv v) { - int isnull = jv_get_kind(t) == JV_KIND_NULL; - if (jv_get_kind(k) == JV_KIND_STRING && - (jv_get_kind(t) == JV_KIND_OBJECT || isnull)) { - if (isnull) t = jv_object(); - t = jv_object_set(t, k, v); - } else if (jv_get_kind(k) == JV_KIND_NUMBER && - (jv_get_kind(t) == JV_KIND_ARRAY || isnull)) { - if (isnull) t = jv_array(); - t = jv_array_set(t, (int)jv_number_value(k), v); - } else { - jv err = jv_invalid_with_msg(jv_string_fmt("Cannot update field at %s index of %s", - jv_kind_name(jv_get_kind(t)), - jv_kind_name(jv_get_kind(v)))); - jv_free(t); - jv_free(k); - jv_free(v); - t = err; - } - return t; -} - -static jv jv_insert(jv root, jv value, jv* path, int pathlen) { - if (pathlen == 0) { - jv_free(root); - return value; - } - return jv_modify(root, jv_copy(*path), - jv_insert(jv_lookup(jv_copy(root), jv_copy(*path)), value, path+1, pathlen-1)); -} - diff --git a/jv_alloc.c b/jv_alloc.c new file mode 100644 index 00000000..e127029d --- /dev/null +++ b/jv_alloc.c @@ -0,0 +1,37 @@ +#include +#include +#include "jv_alloc.h" + +static void memory_exhausted() { + fprintf(stderr, "error: cannot allocate memory\n"); + abort(); +} + +void* jv_mem_alloc(size_t sz) { + void* p = malloc(sz); + if (!p) { + memory_exhausted(); + } + return p; +} + +void jv_mem_free(void* p) { + free(p); +} + +void* jv_mem_realloc(void* p, size_t sz) { + p = realloc(p, sz); + if (!p) { + memory_exhausted(); + } + return p; +} + +#if JQ_DEBUG +volatile char jv_mem_uninitialised; +__attribute__((constructor)) void jv_mem_uninit_setup(){ + char* p = malloc(1); + jv_mem_uninitialised = *p; + free(p); +} +#endif diff --git a/jv_alloc.h b/jv_alloc.h new file mode 100644 index 00000000..b4e72c0d --- /dev/null +++ b/jv_alloc.h @@ -0,0 +1,21 @@ +#ifndef JV_ALLOC_H +#define JV_ALLOC_H + +#include + +#if JQ_DEBUG +extern volatile char jv_mem_uninitialised; +#endif + +static void jv_mem_invalidate(void* mem, size_t n) { +#if JQ_DEBUG + char* m = mem; + while (n--) *m++ ^= jv_mem_uninitialised ^ jv_mem_uninitialised; +#endif +} + +void* jv_mem_alloc(size_t); +void jv_mem_free(void*); +__attribute__((warn_unused_result)) void* jv_mem_realloc(void*, size_t); + +#endif diff --git a/jv_aux.c b/jv_aux.c new file mode 100644 index 00000000..2c33491d --- /dev/null +++ b/jv_aux.c @@ -0,0 +1,428 @@ +#include "jv_aux.h" +#include +#include +#include "jv_alloc.h" + +jv jv_get(jv t, jv k) { + jv v; + if (jv_get_kind(t) == JV_KIND_OBJECT && jv_get_kind(k) == JV_KIND_STRING) { + v = jv_object_get(t, k); + if (!jv_is_valid(v)) { + jv_free(v); + v = jv_null(); + } + } else if (jv_get_kind(t) == JV_KIND_ARRAY && jv_get_kind(k) == JV_KIND_NUMBER) { + // FIXME: don't do lookup for noninteger index + v = jv_array_get(t, (int)jv_number_value(k)); + if (!jv_is_valid(v)) { + jv_free(v); + v = jv_null(); + } + } else if (jv_get_kind(t) == JV_KIND_NULL && + (jv_get_kind(k) == JV_KIND_STRING || jv_get_kind(k) == JV_KIND_NUMBER)) { + jv_free(t); + jv_free(k); + v = jv_null(); + } else { + v = jv_invalid_with_msg(jv_string_fmt("Cannot index %s with %s", + jv_kind_name(jv_get_kind(t)), + jv_kind_name(jv_get_kind(k)))); + jv_free(t); + jv_free(k); + } + return v; +} + +jv jv_set(jv t, jv k, jv v) { + if (!jv_is_valid(v)) { + jv_free(t); + jv_free(k); + return v; + } + int isnull = jv_get_kind(t) == JV_KIND_NULL; + if (jv_get_kind(k) == JV_KIND_STRING && + (jv_get_kind(t) == JV_KIND_OBJECT || isnull)) { + if (isnull) t = jv_object(); + t = jv_object_set(t, k, v); + } else if (jv_get_kind(k) == JV_KIND_NUMBER && + (jv_get_kind(t) == JV_KIND_ARRAY || isnull)) { + if (isnull) t = jv_array(); + t = jv_array_set(t, (int)jv_number_value(k), v); + } else { + jv err = jv_invalid_with_msg(jv_string_fmt("Cannot update field at %s index of %s", + jv_kind_name(jv_get_kind(t)), + jv_kind_name(jv_get_kind(v)))); + jv_free(t); + jv_free(k); + jv_free(v); + t = err; + } + return t; +} + +// assumes keys is a sorted array +jv jv_dels(jv t, jv keys) { + assert(jv_get_kind(keys) == JV_KIND_ARRAY); + assert(jv_is_valid(t)); + + if (jv_get_kind(t) == JV_KIND_NULL || jv_array_length(jv_copy(keys)) == 0) { + // no change + } else if (jv_get_kind(t) == JV_KIND_ARRAY) { + jv new_array = jv_array(); + int kidx = 0; + jv_array_foreach(t, i, elem) { + int del = 0; + while (kidx < jv_array_length(jv_copy(keys))) { + jv nextdel = jv_array_get(jv_copy(keys), kidx); + if (jv_get_kind(nextdel) != JV_KIND_NUMBER) { + jv err = jv_invalid_with_msg(jv_string_fmt("Cannot delete %s element of array", + jv_kind_name(jv_get_kind(nextdel)))); + jv_free(nextdel); + jv_free(new_array); + jv_free(elem); + new_array = err; + goto arr_out; // break twice + } + int delidx = (int)jv_number_value(nextdel); + jv_free(nextdel); + if (i == delidx) { + del = 1; + } + if (i < delidx) { + break; + } + kidx++; + } + if (!del) + new_array = jv_array_append(new_array, elem); + else + jv_free(elem); + } + arr_out: + jv_free(t); + t = new_array; + } else if (jv_get_kind(t) == JV_KIND_OBJECT) { + jv_array_foreach(keys, i, k) { + if (jv_get_kind(k) != JV_KIND_STRING) { + jv_free(t); + t = jv_invalid_with_msg(jv_string_fmt("Cannot delete %s field of object", + jv_kind_name(jv_get_kind(k)))); + jv_free(k); + break; + } + t = jv_object_delete(t, k); + } + } else { + jv err = jv_invalid_with_msg(jv_string_fmt("Cannot delete fields from %s", + jv_kind_name(jv_get_kind(t)))); + jv_free(t); + t = err; + } + jv_free(keys); + return t; +} + +jv jv_setpath(jv root, jv path, jv value) { + if (jv_get_kind(path) != JV_KIND_ARRAY) { + jv_free(value); + jv_free(root); + jv_free(path); + return jv_invalid_with_msg(jv_string("Path must be specified as an array")); + } + if (!jv_is_valid(root)){ + jv_free(value); + jv_free(path); + return root; + } + if (jv_array_length(jv_copy(path)) == 0) { + jv_free(path); + jv_free(root); + return value; + } + jv pathcurr = jv_array_get(jv_copy(path), 0); + jv pathrest = jv_array_slice(path, 1, jv_array_length(jv_copy(path))); + return jv_set(root, pathcurr, + jv_setpath(jv_get(jv_copy(root), jv_copy(pathcurr)), pathrest, value)); +} + +jv jv_getpath(jv root, jv path) { + if (jv_get_kind(path) != JV_KIND_ARRAY) { + jv_free(root); + jv_free(path); + return jv_invalid_with_msg(jv_string("Path must be specified as an array")); + } + if (!jv_is_valid(root)) { + jv_free(path); + return root; + } + if (jv_array_length(jv_copy(path)) == 0) { + jv_free(path); + return root; + } + jv pathcurr = jv_array_get(jv_copy(path), 0); + jv pathrest = jv_array_slice(path, 1, jv_array_length(jv_copy(path))); + return jv_getpath(jv_get(root, pathcurr), pathrest); +} + +// assumes paths is a sorted array of arrays +static jv delpaths_sorted(jv object, jv paths, int start) { + jv delkeys = jv_array(); + for (int i=0; i start); + int delkey = jv_array_length(jv_array_get(jv_copy(paths), i)) == start + 1; + jv key = jv_array_get(jv_array_get(jv_copy(paths), i), start); + while (j < jv_array_length(jv_copy(paths)) && + jv_equal(jv_copy(key), jv_array_get(jv_array_get(jv_copy(paths), j), start))) + j++; + // if i <= entry < j, then entry starts with key + if (delkey) { + // deleting this entire key, we don't care about any more specific deletions + delkeys = jv_array_append(delkeys, key); + } else { + // deleting certain sub-parts of this key + jv subobject = jv_get(jv_copy(object), jv_copy(key)); + if (!jv_is_valid(subobject)) { + jv_free(key); + jv_free(object); + object = subobject; + break; + } else if (jv_get_kind(subobject) == JV_KIND_NULL) { + jv_free(key); + jv_free(subobject); + } else { + jv newsubobject = delpaths_sorted(subobject, jv_array_slice(jv_copy(paths), i, j), start+1); + if (!jv_is_valid(newsubobject)) { + jv_free(key); + jv_free(object); + object = newsubobject; + break; + } + object = jv_set(object, key, newsubobject); + } + if (!jv_is_valid(object)) break; + } + i = j; + } + jv_free(paths); + if (jv_is_valid(object)) + object = jv_dels(object, delkeys); + else + jv_free(delkeys); + return object; +} + +jv jv_delpaths(jv object, jv paths) { + paths = jv_sort(paths, jv_copy(paths)); + jv_array_foreach(paths, i, elem) { + if (jv_get_kind(elem) != JV_KIND_ARRAY) { + jv_free(object); + jv_free(paths); + jv err = jv_invalid_with_msg(jv_string_fmt("Path must be specified as array, not %s", + jv_kind_name(jv_get_kind(elem)))); + jv_free(elem); + return err; + } + jv_free(elem); + } + if (jv_array_length(jv_copy(paths)) == 0) { + // nothing is being deleted + jv_free(paths); + return object; + } + if (jv_array_length(jv_array_get(jv_copy(paths), 0)) == 0) { + // everything is being deleted + jv_free(paths); + jv_free(object); + return jv_null(); + } + return delpaths_sorted(object, paths, 0); +} + + +static int string_cmp(const void* pa, const void* pb){ + const jv* a = pa; + const jv* b = pb; + int lena = jv_string_length(jv_copy(*a)); + int lenb = jv_string_length(jv_copy(*b)); + int minlen = lena < lenb ? lena : lenb; + int r = memcmp(jv_string_value(*a), jv_string_value(*b), minlen); + if (r == 0) r = lena - lenb; + return r; +} + +jv jv_keys(jv x) { + if (jv_get_kind(x) == JV_KIND_OBJECT) { + int nkeys = jv_object_length(jv_copy(x)); + jv* keys = jv_mem_alloc(sizeof(jv) * nkeys); + int kidx = 0; + jv_object_foreach(x, key, value) { + keys[kidx++] = key; + jv_free(value); + } + qsort(keys, nkeys, sizeof(jv), string_cmp); + jv answer = jv_array_sized(nkeys); + for (int i = 0; i= jv_array_length(jv_copy(a)); + int b_done = i >= jv_array_length(jv_copy(b)); + if (a_done || b_done) { + r = b_done - a_done; //suddenly, logic + break; + } + jv xa = jv_array_get(jv_copy(a), i); + jv xb = jv_array_get(jv_copy(b), i); + r = jv_cmp(xa, xb); + i++; + } + break; + } + + case JV_KIND_OBJECT: { + jv keys_a = jv_keys(jv_copy(a)); + jv keys_b = jv_keys(jv_copy(b)); + r = jv_cmp(jv_copy(keys_a), keys_b); + if (r == 0) { + jv_array_foreach(keys_a, i, key) { + jv xa = jv_object_get(jv_copy(a), jv_copy(key)); + jv xb = jv_object_get(jv_copy(b), key); + r = jv_cmp(xa, xb); + if (r) break; + } + } + jv_free(keys_a); + break; + } + } + + jv_free(a); + jv_free(b); + return r; +} + + +struct sort_entry { + jv object; + jv key; +}; + +static int sort_cmp(const void* pa, const void* pb) { + const struct sort_entry* a = pa; + const struct sort_entry* b = pb; + int r = jv_cmp(jv_copy(a->key), jv_copy(b->key)); + // comparing by address if r == 0 makes the sort stable + return r ? r : (int)(a - b); +} + +static struct sort_entry* sort_items(jv objects, jv keys) { + assert(jv_get_kind(objects) == JV_KIND_ARRAY); + assert(jv_get_kind(keys) == JV_KIND_ARRAY); + assert(jv_array_length(jv_copy(objects)) == jv_array_length(jv_copy(keys))); + int n = jv_array_length(jv_copy(objects)); + struct sort_entry* entries = jv_mem_alloc(sizeof(struct sort_entry) * n); + for (int i=0; ifreelist[k]) { Bigint* v = C->freelist[k]; C->freelist[k] = v->next; - free(v); + FREE(v); } } } diff --git a/jv_parse.c b/jv_parse.c index 63cdf935..f55c17e5 100644 --- a/jv_parse.c +++ b/jv_parse.c @@ -5,6 +5,7 @@ #include "jv_dtoa.h" #include "jv_parse.h" #include "jv_unicode.h" +#include "jv_alloc.h" typedef const char* presult; @@ -24,6 +25,9 @@ void jv_parser_init(struct jv_parser* p) { p->st = JV_PARSER_NORMAL; p->curr_buf = 0; p->curr_buf_length = p->curr_buf_pos = p->curr_buf_is_partial = 0; + p->bom_strip_position = 0; + p->line = 1; + p->column = 0; jvp_dtoa_context_init(&p->dtoa); } @@ -31,8 +35,8 @@ void jv_parser_free(struct jv_parser* p) { jv_free(p->next); for (int i=0; istackpos; i++) jv_free(p->stack[i]); - free(p->stack); - free(p->tokenbuf); + jv_mem_free(p->stack); + jv_mem_free(p->tokenbuf); jvp_dtoa_context_free(&p->dtoa); } @@ -47,7 +51,7 @@ static void push(struct jv_parser* p, jv v) { assert(p->stackpos <= p->stacklen); if (p->stackpos == p->stacklen) { p->stacklen = p->stacklen * 2 + 10; - p->stack = realloc(p->stack, p->stacklen * sizeof(jv)); + p->stack = jv_mem_realloc(p->stack, p->stacklen * sizeof(jv)); } assert(p->stackpos < p->stacklen); p->stack[p->stackpos++] = v; @@ -141,7 +145,7 @@ static void tokenadd(struct jv_parser* p, char c) { assert(p->tokenpos <= p->tokenlen); if (p->tokenpos == p->tokenlen) { p->tokenlen = p->tokenlen*2 + 256; - p->tokenbuf = realloc(p->tokenbuf, p->tokenlen); + p->tokenbuf = jv_mem_realloc(p->tokenbuf, p->tokenlen); } assert(p->tokenpos < p->tokenlen); p->tokenbuf[p->tokenpos++] = c; @@ -292,6 +296,11 @@ static int check_done(struct jv_parser* p, jv* out) { } static pfunc scan(struct jv_parser* p, char ch, jv* out) { + p->column++; + if (ch == '\n') { + p->line++; + p->column = 0; + } presult answer = 0; if (p->st == JV_PARSER_NORMAL) { chclass cls = classify(ch); @@ -332,9 +341,27 @@ static pfunc scan(struct jv_parser* p, char ch, jv* out) { return answer; } +static unsigned char UTF8_BOM[] = {0xEF,0xBB,0xBF}; + void jv_parser_set_buf(struct jv_parser* p, const char* buf, int length, int is_partial) { assert((p->curr_buf == 0 || p->curr_buf_pos == p->curr_buf_length) && "previous buffer not exhausted"); + while (p->bom_strip_position < sizeof(UTF8_BOM)) { + if ((unsigned char)*buf == UTF8_BOM[p->bom_strip_position]) { + // matched a BOM character + buf++; + length--; + p->bom_strip_position++; + } else { + if (p->bom_strip_position == 0) { + // no BOM in this document + p->bom_strip_position = sizeof(UTF8_BOM); + } else { + // malformed BOM (prefix present, rest missing) + p->bom_strip_position = 0xff; + } + } + } p->curr_buf = buf; p->curr_buf_length = length; p->curr_buf_pos = 0; @@ -343,6 +370,7 @@ void jv_parser_set_buf(struct jv_parser* p, const char* buf, int length, int is_ jv jv_parser_next(struct jv_parser* p) { assert(p->curr_buf && "a buffer must be provided"); + if (p->bom_strip_position == 0xff) return jv_invalid_with_msg(jv_string("Malformed BOM")); jv value; presult msg = 0; while (!msg && p->curr_buf_pos < p->curr_buf_length) { @@ -352,7 +380,7 @@ jv jv_parser_next(struct jv_parser* p) { if (msg == OK) { return value; } else if (msg) { - return jv_invalid_with_msg(jv_string(msg)); + return jv_invalid_with_msg(jv_string_fmt("%s at line %d, column %d", msg, p->line, p->column)); } else if (p->curr_buf_is_partial) { assert(p->curr_buf_pos == p->curr_buf_length); // need another buffer diff --git a/jv_parse.h b/jv_parse.h index 5b8e7cdf..1d538c12 100644 --- a/jv_parse.h +++ b/jv_parse.h @@ -6,6 +6,7 @@ struct jv_parser { int curr_buf_length; int curr_buf_pos; int curr_buf_is_partial; + unsigned bom_strip_position; jv* stack; int stackpos; @@ -15,6 +16,8 @@ struct jv_parser { char* tokenbuf; int tokenpos; int tokenlen; + + int line, column; struct dtoa_context dtoa; diff --git a/jv_print.c b/jv_print.c index 9e0a8812..388bf663 100644 --- a/jv_print.c +++ b/jv_print.c @@ -6,6 +6,20 @@ #include "jv_dtoa.h" #include "jv_unicode.h" +#define ESC "\033" +#define COL(c) (ESC "[" c "m") +#define COLRESET (ESC "[0m") + +// Colour table. See http://en.wikipedia.org/wiki/ANSI_escape_code#Colors +// for how to choose these. +static jv_kind colour_kinds[] = + {JV_KIND_NULL, JV_KIND_FALSE, JV_KIND_TRUE, JV_KIND_NUMBER, + JV_KIND_STRING, JV_KIND_ARRAY, JV_KIND_OBJECT}; +static const char* colours[] = + {COL("30;1"), COL("0"), COL("0"), COL("0"), + COL("32"), COL("37"), COL("37")}; +#define FIELD_COLOUR COL("34;1") + static void put_buf(const char* s, int len, FILE* fout, jv* strout) { if (strout) { *strout = jv_string_append_buf(*strout, s, len); @@ -35,6 +49,7 @@ static void jvp_dump_string(jv str, int ascii_only, FILE* F, jv* S) { const char* cstart; int c = 0; char buf[32]; + put_char('"', F, S); while ((i = jvp_utf8_next((cstart = i), end, &c))) { assert(c != -1); int unicode_escape = 0; @@ -91,12 +106,23 @@ static void jvp_dump_string(jv str, int ascii_only, FILE* F, jv* S) { } } assert(c != -1); + put_char('"', F, S); } enum { INDENT = 2 }; static void jv_dump_term(struct dtoa_context* C, jv x, int flags, int indent, FILE* F, jv* S) { char buf[JVP_DTOA_FMT_MAX_LEN]; + const char* colour = 0; + if (flags & JV_PRINT_COLOUR) { + for (unsigned i=0; i=" { return GREATEREQ; } -"contains" { return CONTAINS; } "."|"="|";"|","|":"|"|"|"+"|"-"|"*"|"/"|"\$"|"<"|">" { return yytext[0];} "["|"{"|"(" { @@ -67,8 +69,11 @@ struct lexer_param; return try_exit(yytext[0], YY_START, yyscanner); } +"@"[a-zA-Z0-9_]+ { + yylval->literal = jv_string_sized(yytext + 1, yyleng - 1); return FORMAT; +} --?[0-9.]+([eE][+-]?[0-9]+)? { +[0-9.]+([eE][+-]?[0-9]+)? { yylval->literal = jv_parse_sized(yytext, yyleng); return LITERAL; } @@ -127,6 +132,10 @@ static int try_exit(int c, int state, yyscan_t yyscanner) { match = ')'; ret = QQSTRING_INTERP_END; break; + + default: + // may not be the best error to give + return INVALID_CHARACTER; } assert(match); if (match == c) { @@ -150,3 +159,13 @@ static int enter(int c, int currstate, yyscan_t yyscanner) { yy_push_state(state, yyscanner); return c; } + +void* yyalloc(size_t sz, void* extra) { + return jv_mem_alloc(sz); +} +void* yyrealloc(void* p, size_t sz, void* extra) { + return jv_mem_realloc(p, sz); +} +void yyfree(void* p, void* extra) { + jv_mem_free(p); +} diff --git a/locfile.h b/locfile.h index 77b1aef1..710cfbd1 100644 --- a/locfile.h +++ b/locfile.h @@ -4,6 +4,7 @@ #include #include #include +#include "jv_alloc.h" typedef struct { int start, end; } location; @@ -24,7 +25,7 @@ static void locfile_init(struct locfile* l, const char* data, int length) { for (int i=0; inlines++; } - l->linemap = malloc(sizeof(int) * (l->nlines + 1)); + l->linemap = jv_mem_alloc(sizeof(int) * (l->nlines + 1)); l->linemap[0] = 0; int line = 1; for (int i=0; ilinemap); + jv_mem_free(l->linemap); } static int locfile_get_line(struct locfile* l, int pos) { diff --git a/main.c b/main.c index 06a7e91e..17019a8d 100644 --- a/main.c +++ b/main.c @@ -1,7 +1,9 @@ +#define _POSIX_SOURCE #include #include #include #include +#include #include "compile.h" #include "builtin.h" #include "jv.h" @@ -10,15 +12,17 @@ #include "parser.h" #include "execute.h" #include "config.h" /* Autoconf generated header file */ +#include "jv_alloc.h" static const char* progname; static void usage() { fprintf(stderr, "\njq - commandline JSON processor [version %s]\n", PACKAGE_VERSION); - fprintf(stderr, "Usage: %s [options] \n\n", progname); + fprintf(stderr, "Usage: %s [options] [file...]\n\n", progname); fprintf(stderr, "For a description of the command line options and\n"); fprintf(stderr, "how to write jq filters (and why you might want to)\n"); - fprintf(stderr, "see the jq documentation at http://stedolan.github.com/jq\n\n"); + fprintf(stderr, "see the jq manpage, or the online documentation at\n"); + fprintf(stderr, "http://stedolan.github.com/jq\n\n"); exit(1); } @@ -50,8 +54,10 @@ enum { RAW_OUTPUT = 8, COMPACT_OUTPUT = 16, ASCII_OUTPUT = 32, + COLOUR_OUTPUT = 64, + NO_COLOUR_OUTPUT = 128, - FROM_FILE = 64, + FROM_FILE = 256, }; static int options = 0; static struct bytecode* bc; @@ -62,10 +68,18 @@ static void process(jv value) { while (jv_is_valid(result = jq_next())) { if ((options & RAW_OUTPUT) && jv_get_kind(result) == JV_KIND_STRING) { fwrite(jv_string_value(result), 1, jv_string_length(jv_copy(result)), stdout); + jv_free(result); } else { - int dumpopts = 0; + int dumpopts; +#ifdef JQ_DEFAULT_ENABLE_COLOR + dumpopts = JQ_DEFAULT_ENABLE_COLOR ? JV_PRINT_COLOUR : 0; +#else + dumpopts = isatty(fileno(stdout)) ? JV_PRINT_COLOUR : 0; +#endif if (!(options & COMPACT_OUTPUT)) dumpopts |= JV_PRINT_PRETTY; if (options & ASCII_OUTPUT) dumpopts |= JV_PRINT_ASCII; + if (options & COLOUR_OUTPUT) dumpopts |= JV_PRINT_COLOUR; + if (options & NO_COLOUR_OUTPUT) dumpopts &= ~JV_PRINT_COLOUR; jv_dump(result, dumpopts); } printf("\n"); @@ -97,20 +111,63 @@ static jv slurp_file(const char* filename) { return data; } +FILE* current_input; +const char** input_filenames; +int ninput_files; +int next_input_idx; +static int read_more(char* buf, size_t size) { + while (!current_input || feof(current_input)) { + if (current_input) { + fclose(current_input); + current_input = 0; + } + if (next_input_idx == ninput_files) { + return 0; + } + if (!strcmp(input_filenames[next_input_idx], "-")) { + current_input = stdin; + } else { + current_input = fopen(input_filenames[next_input_idx], "r"); + } + if (!current_input) { + fprintf(stderr, "%s: %s: %s\n", progname, input_filenames[next_input_idx], strerror(errno)); + } + next_input_idx++; + } + + if (!fgets(buf, size, current_input)) buf[0] = 0; + return 1; +} + int main(int argc, char* argv[]) { if (argc) progname = argv[0]; const char* program = 0; + input_filenames = jv_mem_alloc(sizeof(const char*) * argc); + ninput_files = 0; + int further_args_are_files = 0; for (int i=1; i 0) { @@ -207,7 +264,7 @@ int main(int argc, char* argv[]) { process(slurped); } } - + jv_mem_free(input_filenames); bytecode_free(bc); return 0; } diff --git a/opcode.c b/opcode.c index bc259c22..4e785b8f 100644 --- a/opcode.c +++ b/opcode.c @@ -5,11 +5,9 @@ #define CONSTANT OP_HAS_CONSTANT, 2 #define VARIABLE (OP_HAS_VARIABLE | OP_HAS_BINDING), 3 #define BRANCH OP_HAS_BRANCH, 2 -#define CFUNC (OP_HAS_SYMBOL | OP_HAS_CFUNC | OP_HAS_VARIABLE_LENGTH_ARGLIST), 2 -#define UFUNC (OP_HAS_UFUNC | OP_HAS_VARIABLE_LENGTH_ARGLIST), 2 -#define CLOSURE_PARAM_IMM (OP_IS_CALL_PSEUDO | OP_HAS_BINDING), 0 -#define CLOSURE_CREATE_IMM (OP_HAS_BLOCK | OP_IS_CALL_PSEUDO | OP_HAS_BINDING), 0 -#define CLOSURE_CREATE_C_IMM (OP_IS_CALL_PSEUDO | OP_HAS_BINDING), 0 +#define CFUNC (OP_HAS_CFUNC | OP_HAS_BINDING), 3 +#define UFUNC (OP_HAS_UFUNC | OP_HAS_BINDING | OP_IS_CALL_PSEUDO), 4 +#define DEFINITION (OP_IS_CALL_PSEUDO | OP_HAS_BINDING), 0 #define CLOSURE_REF_IMM (OP_IS_CALL_PSEUDO | OP_HAS_BINDING), 2 #define OP(name, imm, in, out) \ diff --git a/opcode.h b/opcode.h index 73948cd5..ee956e27 100644 --- a/opcode.h +++ b/opcode.h @@ -19,12 +19,9 @@ enum { OP_HAS_CONSTANT = 2, OP_HAS_VARIABLE = 4, OP_HAS_BRANCH = 8, - OP_HAS_SYMBOL = 16, OP_HAS_CFUNC = 32, OP_HAS_UFUNC = 64, OP_IS_CALL_PSEUDO = 128, - OP_HAS_VARIABLE_LENGTH_ARGLIST = 256, - OP_HAS_BLOCK = 512, OP_HAS_BINDING = 1024, }; struct opcode_description { @@ -41,8 +38,4 @@ struct opcode_description { const struct opcode_description* opcode_describe(opcode op); -static inline int opcode_length(opcode op) { - return opcode_describe(op)->length; -} - #endif diff --git a/opcode_list.h b/opcode_list.h index 306373fb..f384d715 100644 --- a/opcode_list.h +++ b/opcode_list.h @@ -1,30 +1,28 @@ OP(LOADK, CONSTANT, 1, 1) OP(DUP, NONE, 1, 2) +OP(DUP2, NONE, 2, 3) OP(SWAP, NONE, 2, 2) OP(POP, NONE, 1, 0) OP(LOADV, VARIABLE, 1, 1) OP(STOREV, VARIABLE, 1, 0) OP(INDEX, NONE, 2, 1) -//OP(DISPLAY, NONE, 1, 0) OP(YIELD, NONE, 1, 0) OP(EACH, NONE, 1, 1) OP(FORK, BRANCH, 0, 0) OP(JUMP, BRANCH, 0, 0) OP(JUMP_F,BRANCH, 1, 0) OP(BACKTRACK, NONE, 0, 0) -OP(APPEND, NONE, 2, 1) +OP(APPEND, VARIABLE,1, 0) OP(INSERT, NONE, 4, 2) -OP(ASSIGN, VARIABLE, 3, 0) +OP(GETPATH, NONE, 2, 1) -OP(CALL_BUILTIN_1_1, CFUNC, 1, 1) -OP(CALL_BUILTIN_2_1, CFUNC, 2, 1) -OP(CALL_BUILTIN_3_1, CFUNC, 3, 1) +OP(CALL_BUILTIN, CFUNC, -1, 1) -OP(CALL_1_1, UFUNC, 1, 1) +OP(CALL_JQ, UFUNC, 1, 1) OP(RET, NONE, 1, 1) -OP(CLOSURE_PARAM, CLOSURE_PARAM_IMM, 0, 0) +OP(CLOSURE_PARAM, DEFINITION, 0, 0) OP(CLOSURE_REF, CLOSURE_REF_IMM, 0, 0) -OP(CLOSURE_CREATE, CLOSURE_CREATE_IMM, 0, 0) -OP(CLOSURE_CREATE_C, CLOSURE_CREATE_C_IMM, 0, 0) +OP(CLOSURE_CREATE, DEFINITION, 0, 0) +OP(CLOSURE_CREATE_C, DEFINITION, 0, 0) diff --git a/parser.y b/parser.y index 515f5a40..dd2e3a12 100644 --- a/parser.y +++ b/parser.y @@ -2,6 +2,9 @@ #include #include #include "compile.h" +#include "jv_alloc.h" +#define YYMALLOC jv_mem_alloc +#define YYFREE jv_mem_free struct lexer_param; @@ -45,6 +48,7 @@ struct lexer_param; %token INVALID_CHARACTER %token IDENT %token LITERAL +%token FORMAT %token EQ "==" %token NEQ "!=" %token DEFINEDOR "//" @@ -54,6 +58,7 @@ struct lexer_param; %token THEN "then" %token ELSE "else" %token ELSE_IF "elif" +%token FOLD "fold" %token END "end" %token AND "and" %token OR "or" @@ -65,7 +70,6 @@ struct lexer_param; %token SETDEFINEDOR "//=" %token LESSEQ "<=" %token GREATEREQ ">=" -%token CONTAINS "contains" %token QQSTRING_START %token QQSTRING_TEXT @@ -75,13 +79,13 @@ struct lexer_param; /* revolting hack */ %left ';' -%left '|' +%right '|' %left ',' %right "//" %nonassoc '=' SETPIPE SETPLUS SETMINUS SETMULT SETDIV SETDEFINEDOR %left OR %left AND -%nonassoc NEQ EQ '<' '>' LESSEQ GREATEREQ CONTAINS +%nonassoc NEQ EQ '<' '>' LESSEQ GREATEREQ %left '+' '-' %left '*' '/' @@ -130,14 +134,11 @@ int yylex(YYSTYPE* yylval, YYLTYPE* yylloc, block* answer, int* errors, } static block gen_dictpair(block k, block v) { - block b = gen_subexp(k); - block_append(&b, gen_subexp(v)); - block_append(&b, gen_op_simple(INSERT)); - return b; + return BLOCK(gen_subexp(k), gen_subexp(v), gen_op_simple(INSERT)); } static block gen_index(block obj, block key) { - return block_join(obj, block_join(gen_subexp(key), gen_op_simple(INDEX))); + return BLOCK(gen_subexp(key), obj, gen_op_simple(INDEX)); } static block gen_binop(block a, block b, int op) { @@ -153,29 +154,26 @@ static block gen_binop(block a, block b, int op) { case '>': funcname = "_greater"; break; case LESSEQ: funcname = "_lesseq"; break; case GREATEREQ: funcname = "_greatereq"; break; - case CONTAINS: funcname = "_contains"; break; } assert(funcname); - block c = gen_noop(); - block_append(&c, gen_subexp(a)); - block_append(&c, gen_subexp(b)); - block_append(&c, gen_op_call(CALL_1_1, gen_op_block_unbound(CLOSURE_REF, funcname))); - return c; + return gen_call(funcname, BLOCK(gen_lambda(a), gen_lambda(b))); } -static block gen_format(block a) { - return block_join(a, gen_op_call(CALL_1_1, gen_op_block_unbound(CLOSURE_REF, "tostring"))); +static block gen_format(block a, jv fmt) { + return BLOCK(a, gen_call("format", BLOCK(gen_lambda(gen_const(fmt))))); } -static block gen_update(block a, block op, int optype) { - block assign = a; - block_append(&assign, gen_op_simple(DUP)); - if (optype) { - op = gen_binop(gen_noop(), op, optype); - } - block_append(&assign, op); - return gen_assign(assign); +static block gen_update(block object, block val, int optype) { + block tmp = block_bind(gen_op_var_unbound(STOREV, "tmp"), + gen_noop(), OP_HAS_VARIABLE); + return BLOCK(gen_op_simple(DUP), + val, + tmp, + gen_call("_modify", BLOCK(gen_lambda(object), + gen_lambda(gen_binop(gen_noop(), + gen_op_var_bound(LOADV, tmp), + optype))))); } %} @@ -203,12 +201,17 @@ FuncDef Exp %prec ';' { } | Term "as" '$' IDENT '|' Exp { - $$ = gen_op_simple(DUP); - block_append(&$$, $1); - block_append(&$$, block_bind(gen_op_var_unbound(STOREV, jv_string_value($4)), $6, OP_HAS_VARIABLE)); + $$ = BLOCK(gen_op_simple(DUP), $1, + block_bind(gen_op_var_unbound(STOREV, jv_string_value($4)), + $6, OP_HAS_VARIABLE)); jv_free($4); } | +"fold" Term "as" '$' IDENT '(' Exp ')' { + $$ = gen_fold(jv_string_value($5), $2, $7); + jv_free($5); +} | + "if" Exp "then" Exp ElseBody { $$ = gen_cond($2, $4, $5); } | @@ -218,12 +221,7 @@ Term "as" '$' IDENT '|' Exp { } | Exp '=' Exp { - block assign = gen_op_simple(DUP); - block_append(&assign, $3); - block_append(&assign, gen_op_simple(SWAP)); - block_append(&assign, $1); - block_append(&assign, gen_op_simple(SWAP)); - $$ = gen_assign(assign); + $$ = gen_call("_assign", BLOCK(gen_lambda($1), gen_lambda($3))); } | Exp "or" Exp { @@ -243,7 +241,7 @@ Exp "//=" Exp { } | Exp "|=" Exp { - $$ = gen_update($1, $3, 0); + $$ = gen_call("_modify", BLOCK(gen_lambda($1), gen_lambda($3))); } | Exp '|' Exp { @@ -262,6 +260,10 @@ Exp "+=" Exp { $$ = gen_update($1, $3, '+'); } | +'-' Exp { + $$ = BLOCK($2, gen_call("_negate", gen_noop())); +} | + Exp '-' Exp { $$ = gen_binop($1, $3, '-'); } | @@ -310,42 +312,56 @@ Exp ">=" Exp { $$ = gen_binop($1, $3, GREATEREQ); } | -Exp "contains" Exp { - $$ = gen_binop($1, $3, CONTAINS); -} | - Term { $$ = $1; } -String: -QQSTRING_START QQString QQSTRING_END { - $$ = $2; -} - FuncDef: "def" IDENT ':' Exp ';' { - block body = block_join($4, gen_op_simple(RET)); - $$ = gen_op_block_defn_rec(CLOSURE_CREATE, jv_string_value($2), body); + $$ = gen_function(jv_string_value($2), gen_noop(), $4); jv_free($2); } | "def" IDENT '(' IDENT ')' ':' Exp ';' { - block body = block_bind(gen_op_block_unbound(CLOSURE_PARAM, jv_string_value($4)), block_join($7, gen_op_simple(RET)), OP_IS_CALL_PSEUDO); - $$ = gen_op_block_defn_rec(CLOSURE_CREATE, jv_string_value($2), body); + $$ = gen_function(jv_string_value($2), + gen_op_block_unbound(CLOSURE_PARAM, jv_string_value($4)), + $7); jv_free($2); jv_free($4); +} | + +"def" IDENT '(' IDENT ';' IDENT ')' ':' Exp ';' { + $$ = gen_function(jv_string_value($2), + BLOCK(gen_op_block_unbound(CLOSURE_PARAM, jv_string_value($4)), + gen_op_block_unbound(CLOSURE_PARAM, jv_string_value($6))), + $9); + jv_free($2); + jv_free($4); + jv_free($6); } + + +String: +QQSTRING_START { $$ = jv_string("text"); } QQString QQSTRING_END { + $$ = $3; + jv_free($2); +} | +FORMAT QQSTRING_START { $$ = $1; } QQString QQSTRING_END { + $$ = $4; + jv_free($3); +} + + QQString: /* empty */ { - $$ = gen_op_const(LOADK, jv_string("")); + $$ = gen_const(jv_string("")); } | QQString QQSTRING_TEXT { - $$ = gen_binop($1, gen_op_const(LOADK, $2), '+'); + $$ = gen_binop($1, gen_const($2), '+'); } | QQString QQSTRING_INTERP_START Exp QQSTRING_INTERP_END { - $$ = gen_binop($1, gen_format($3), '+'); + $$ = gen_binop($1, gen_format($3, jv_copy($0)), '+'); } @@ -372,10 +388,10 @@ Term: $$ = gen_noop(); } | Term '.' IDENT { - $$ = gen_index($1, gen_op_const(LOADK, $3)); + $$ = gen_index($1, gen_const($3)); } | '.' IDENT { - $$ = gen_index(gen_noop(), gen_op_const(LOADK, $2)); + $$ = gen_index(gen_noop(), gen_const($2)); } | /* FIXME: string literals */ Term '[' Exp ']' { @@ -385,11 +401,14 @@ Term '[' ']' { $$ = block_join($1, gen_op_simple(EACH)); } | LITERAL { - $$ = gen_op_const(LOADK, $1); + $$ = gen_const($1); } | String { $$ = $1; } | +FORMAT { + $$ = gen_format(gen_noop(), $1); +} | '(' Exp ')' { $$ = $2; } | @@ -397,28 +416,26 @@ String { $$ = gen_collect($2); } | '[' ']' { - $$ = gen_op_const(LOADK, jv_array()); + $$ = gen_const(jv_array()); } | '{' MkDict '}' { - $$ = gen_subexp(gen_op_const(LOADK, jv_object())); - block_append(&$$, $2); - block_append(&$$, gen_op_simple(POP)); + $$ = BLOCK(gen_subexp(gen_const(jv_object())), $2, gen_op_simple(POP)); } | '$' IDENT { $$ = gen_location(@$, gen_op_var_unbound(LOADV, jv_string_value($2))); jv_free($2); } | IDENT { - $$ = gen_location(@$, gen_op_call(CALL_1_1, gen_op_block_unbound(CLOSURE_REF, jv_string_value($1)))); + $$ = gen_location(@$, gen_call(jv_string_value($1), gen_noop())); jv_free($1); } | IDENT '(' Exp ')' { - $$ = gen_op_call(CALL_1_1, - block_join(gen_op_block_unbound(CLOSURE_REF, jv_string_value($1)), - block_bind(gen_op_block_defn(CLOSURE_CREATE, - "lambda", - block_join($3, gen_op_simple(RET))), - gen_noop(), OP_IS_CALL_PSEUDO))); + $$ = gen_call(jv_string_value($1), gen_lambda($3)); + $$ = gen_location(@1, $$); + jv_free($1); +} | +IDENT '(' Exp ';' Exp ')' { + $$ = gen_call(jv_string_value($1), BLOCK(gen_lambda($3), gen_lambda($5))); $$ = gen_location(@1, $$); jv_free($1); } | @@ -437,14 +454,18 @@ MkDict: MkDictPair : IDENT ':' ExpD { - $$ = gen_dictpair(gen_op_const(LOADK, $1), $3); + $$ = gen_dictpair(gen_const($1), $3); } | String ':' ExpD { $$ = gen_dictpair($1, $3); } +| String { + $$ = gen_dictpair($1, BLOCK(gen_op_simple(POP), gen_op_simple(DUP2), + gen_op_simple(DUP2), gen_op_simple(INDEX))); + } | IDENT { - $$ = gen_dictpair(gen_op_const(LOADK, jv_copy($1)), - gen_index(gen_noop(), gen_op_const(LOADK, $1))); + $$ = gen_dictpair(gen_const(jv_copy($1)), + gen_index(gen_noop(), gen_const($1))); } | '(' Exp ')' ':' ExpD { $$ = gen_dictpair($2, $5); diff --git a/setup.sh b/setup.sh index cf360e7d..763e78f4 100755 --- a/setup.sh +++ b/setup.sh @@ -18,6 +18,7 @@ elif [ "superclean" == "$1" ]; then else autoreconf --install ./configure --prefix=/opt/junk + make check mkdir tmp make DESTDIR=./tmp install make distcheck diff --git a/testdata b/testdata index ce330112..83f91dae 100644 --- a/testdata +++ b/testdata @@ -21,6 +21,11 @@ null null 1 + +-1 +null +-1 + # FIXME: much more number testing needed {} @@ -31,6 +36,13 @@ null null [] +# The input line starts with a 0xFEFF (byte order mark) codepoint +# No, there is no reason to have a byte order mark in UTF8 text. +# But apparently people do, so jq shouldn't break on it. +. +"byte order mark" +"byte order mark" + # We test escapes by matching them against Unicode codepoints # FIXME: more tests needed for weird unicode stuff (e.g. utf16 pairs) "Aa\r\n\t\b\f\u03bc" @@ -45,6 +57,24 @@ null null "interpolation" +@text,@json,([1,.] | @csv),@html,@uri,@sh,@base64 +"<>&'\"" +"<>&'\"" +"\"<>&'\\\"\"" +"1,\"<>&'\"\"\"" +"<>&'"" +"%3c%3e%26'%22" +"'<>&'\\''\"'" +"PD4mJyI=" + +@uri +"\u03bc" +"%ce%bc" + +@html "\(.)" +"" +"<script>hax</script>" + # # Dictionary construction syntax # @@ -57,6 +87,10 @@ null {"a":1, "b":2, "c":3, "d":"c"} {"a":1, "b":2, "c":1, "e":2} +{"a",b,"a$\(1+1)"} +{"a":1, "b":2, "c":3, "a$2":4} +{"a":1, "b":2, "a$2":4} + # # Field access, piping # @@ -97,6 +131,11 @@ null 1 1 +1,. +[] +1 +[] + [.] [2] [[2]] @@ -141,6 +180,10 @@ null [6] [7] +42 as $x | . | . | . + 432 | $x + 1 +34324 +43 + 1 as $x | [$x,$x,$x as $x | $x] null [1,1,1] @@ -159,6 +202,14 @@ null "wtasdf" 2.0 +2-1 +null +1 + +2-(-1) +null +3 + 1e+0+0.001e3 "I wonder what this will be?" 20e-1 @@ -167,6 +218,18 @@ null 15 19.0 +.+null +{"a":42} +{"a":42} + +null+. +null +null + +.a+.b +{"a":42} +42 + [1,2,3] + [.] null [1,2,3,null] @@ -291,6 +354,65 @@ def fac: if . == 1 then 1 else . * (. - 1 | fac) end; [.[] | fac] [1,2,3,4] [1,2,6,24] +# test stack overflow and reallocation +# this test is disabled for now, it takes a realllllly long time. +# def f: if length > 1000 then . else .+[1]|f end; f | length +# [] +# 1001 + +fold 0 as $s (.[] | $s + .) +[1,2,4] +7 + +# +# Paths +# + +path(.foo[0,1]) +null +["foo", 0] +["foo", 1] + +path(.[] | select(.>3)) +[1,5,3] +[1] + +path(.) +42 +[] + +["foo",1] as $p | getpath($p), setpath($p; 20), delpaths([$p]) +{"bar": 42, "foo": ["a", "b", "c", "d"]} +"b" +{"bar": 42, "foo": ["a", 20, "c", "d"]} +{"bar": 42, "foo": ["a", "c", "d"]} + +map(getpath([2])), map(setpath([2]; 42)), map(delpaths([[2]])) +[[0], [0,1], [0,1,2]] +[null, null, 2] +[[0,null,42], [0,1,42], [0,1,42]] +[[0], [0,1], [0,1]] + +map(delpaths([[0,"foo"]])) +[[{"foo":2, "x":1}], [{"bar":2}]] +[[{"x":1}], [{"bar":2}]] + +["foo",1] as $p | getpath($p), setpath($p; 20), delpaths([$p]) +{"bar":false} +null +{"bar":false, "foo": [null, 20]} +{"bar":false} + +delpaths([[-200]]) +[1,2,3] +[1,2,3] + +del(.), del(empty), del((.foo,.bar,.baz) | .[2,3,0]), del(.foo[0], .bar[0], .foo, .baz.bar[0].x) +{"foo": [0,1,2,3,4], "bar": [0,1]} +null +{"foo": [0,1,2,3,4], "bar": [0,1]} +{"foo": [1,4], "bar": [1]} +{"bar": [1]} # # Assignment @@ -314,6 +436,10 @@ def fac: if . == 1 then 1 else . * (. - 1 | fac) end; [.[] | fac] [-1,1,3] [0.5, 1.5, 2.5] +.foo += .foo +{"foo":2} +{"foo":4} + .[0].a |= {"old":., "new":(.+1)} [{"a":1,"b":2}] [{"a":{"old":1, "new":2},"b":2}] @@ -397,26 +523,54 @@ def inc(x): x |= .+1; inc(.[].a) [true,false] # containment operator -["foo" contains "foo", "foobar" contains "foo", "foo" contains "foobar"] +[("foo" | contains("foo")), ("foobar" | contains("foo")), ("foo" | contains("foobar"))] {} [true, true, false] -[[] contains [], [1,2,3] contains [1,2], [1,2,3] contains [3,1], [1,2,3] contains [4], [1,2,3] contains [1,4]] -{} +map(.[1] as $needle | .[0] | contains($needle)) +[[[],[]], [[1,2,3], [1,2]], [[1,2,3], [3,1]], [[1,2,3], [4]], [[1,2,3], [1,4]]] [true, true, true, false, false] -[["foobar", "foobaz"] contains ["baz", "bar"], ["foobar", "foobaz"] contains ["foo"], ["foobar", "foobaz"] contains ["blap"]] +map(.[1] as $needle | .[0] | contains($needle)) +[[["foobar", "foobaz"], ["baz", "bar"]], [["foobar", "foobaz"], ["foo"]], [["foobar", "foobaz"], ["blap"]]] +[true, true, false] + +[({foo: 12, bar:13} | contains({foo: 12})), ({foo: 12} | contains({})), ({foo: 12, bar:13} | contains({baz:14}))] {} [true, true, false] -[{foo: 12, bar:13} contains {foo: 12}, {foo: 12} contains {}, {foo: 12, bar:13} contains {baz:14}] -{} -[true, true, false] - -{foo: {baz: 12, blap: {bar: 13}}, bar: 14} contains {bar: 14, foo: {blap: {}}} +{foo: {baz: 12, blap: {bar: 13}}, bar: 14} | contains({bar: 14, foo: {blap: {}}}) {} true -{foo: {baz: 12, blap: {bar: 13}}, bar: 14} contains {bar: 14, foo: {blap: {bar: 14}}} +{foo: {baz: 12, blap: {bar: 13}}, bar: 14} | contains({bar: 14, foo: {blap: {bar: 14}}}) {} -false \ No newline at end of file +false + +sort +[42,[2,5,3,11],10,{"a":42,"b":2},{"a":42},true,2,[2,6],"hello",null,[2,5,6],{"a":[],"b":1},"abc","ab",[3,10],{},false,"abcd",null] +[null,null,false,true,2,10,42,"ab","abc","abcd","hello",[2,5,3,11],[2,5,6],[2,6],[3,10],{},{"a":42},{"a":42,"b":2},{"a":[],"b":1}] + +(sort_by(.b) | sort_by(.a)), sort_by(.a, .b), sort_by(.b, .c), group_by(.b), group_by(.a + .b - .c == 2) +[{"a": 1, "b": 4, "c": 14}, {"a": 4, "b": 1, "c": 3}, {"a": 1, "b": 4, "c": 3}, {"a": 0, "b": 2, "c": 43}] +[{"a": 0, "b": 2, "c": 43}, {"a": 1, "b": 4, "c": 14}, {"a": 1, "b": 4, "c": 3}, {"a": 4, "b": 1, "c": 3}] +[{"a": 0, "b": 2, "c": 43}, {"a": 1, "b": 4, "c": 14}, {"a": 1, "b": 4, "c": 3}, {"a": 4, "b": 1, "c": 3}] +[{"a": 4, "b": 1, "c": 3}, {"a": 0, "b": 2, "c": 43}, {"a": 1, "b": 4, "c": 3}, {"a": 1, "b": 4, "c": 14}] +[[{"a": 4, "b": 1, "c": 3}], [{"a": 0, "b": 2, "c": 43}], [{"a": 1, "b": 4, "c": 14}, {"a": 1, "b": 4, "c": 3}]] +[[{"a": 1, "b": 4, "c": 14}, {"a": 0, "b": 2, "c": 43}], [{"a": 4, "b": 1, "c": 3}, {"a": 1, "b": 4, "c": 3}]] + +unique +[1,2,5,3,5,3,1,3] +[1,2,3,5] + +[min, max, min_by(.[1]), max_by(.[1]), min_by(.[2]), max_by(.[2])] +[[4,2,"a"],[3,1,"a"],[2,4,"a"],[1,3,"a"]] +[[1,3,"a"],[4,2,"a"],[3,1,"a"],[2,4,"a"],[4,2,"a"],[1,3,"a"]] + +[min,max,min_by(.),max_by(.)] +[] +[null,null,null,null] + +.foo[.baz] +{"foo":{"bar":4},"baz":"bar"} +4