From 56ae88d9d55582e8d0036601e3186d88861bf105 Mon Sep 17 00:00:00 2001 From: Nicolas Williams Date: Mon, 24 Nov 2014 17:58:34 -0600 Subject: [PATCH] Module search revamp for pkg managers The search path listed in an import directive can now be an array. The top-level search path is appended. Null and empty strings in the path terminate any search. The "." in "." and "./*" is replaced with the directory containing the file doing the import (for command-line programs this is the current directory, though that may be a bad idea). No version numbers or anything of the sort are gratuitously added to the search paths. All this makes external package managers possible by allowing dependencies to be installed local to dependents. --- .gitignore | 1 + docs/content/3.manual/manual.yml | 68 ++++++++++++-------- execute.c | 10 ++- jq.h | 3 +- linker.c | 107 +++++++++++++++++++------------ main.c | 28 ++++++-- tests/modules/c/c.jq | 13 +++- tests/modules/lib/jq/e/e.jq | 1 + tests/modules/lib/jq/f.jq | 1 + tests/run | 15 ++++- 10 files changed, 167 insertions(+), 80 deletions(-) create mode 100644 tests/modules/lib/jq/e/e.jq create mode 100644 tests/modules/lib/jq/f.jq diff --git a/.gitignore b/.gitignore index 52ce0f2f..f5689661 100644 --- a/.gitignore +++ b/.gitignore @@ -10,6 +10,7 @@ tags jq +!tests/modules/lib/jq/ jq.1 # Something delightfully recursive happens otherwise diff --git a/docs/content/3.manual/manual.yml b/docs/content/3.manual/manual.yml index 42c6c98c..cf78c7cd 100644 --- a/docs/content/3.manual/manual.yml +++ b/docs/content/3.manual/manual.yml @@ -2286,45 +2286,58 @@ sections: body: | jq has a library/module system. Modules are files whose names end - in `.jq`. Modules should start with a `module` directive. - Programs and modules must `import` their dependencies, if any. + in `.jq`. Modules should (but don't have to) start with a + `module` directive. Programs and modules must `import` their + dependencies, if any. - A search path is used to search for modules. Each directory in - the search path is tested twice: once with a directory named "any" - added on the end, and once with the jq version number as a - directory added on the end. + Modules imported by a program are searched for in a default search + path (see below). The `import` directive allows the importer to + alter this path. + + Paths in the a search path are subject to various substitutions. For paths starting with "~/", the user's home directory is substituted for "~". - For paths starting with "$ORIGIN/", the path of the directory of - the dependent is used. For main programs that's the path of the - jq executable's containing directory. + For paths starting with "$ORIGIN/", the path of the jq executable + is substituted for "$ORIGIN". + + For paths starting with "./" or paths that are ".", the path of + the including file is substituted for ".". For top-level programs + given on the command-line, the current directory is used. + + Import directives can optionally specify a search path to which + the default is appended. + + The default search path is the search path given to the `-L` + command-line option, else the "$JQ_LIBRARY_PATH", if set in the + environment, else `["~/.jq", "$ORIGIN/../lib/jq"]` (on Unix) or + `["~/.jq", "$ORIGIN/lib"]` (on Windows). + + The `-L` argument's and "$JQ_LIBRARY_PATH" environment variable's + string values are split on ":" on Unix, or ";" on Windows. + + Null and empty string path elements terminate search path + processing. Modules may be organized into a hierarchy, with name components separated by double-colons ("::"). For example: "foo::bar". Each component in the name corresponds to a directory on the - filesystem. Consecutive components with the same name are not - allowed to avoid ambiguities (e.g., "foo::foo"). + filesystem. - For example, a module named "foo::bar" would be searched for in - "foo/bar.jq" and "foo/bar/bar.jq" in the given search path. This - is intended to allow modules to be placed in a directory along - with, for example, version control files, README files, and so on, - but also to allow for single-file modules. + A module named "foo::bar" would be searched for in "foo/bar.jq" + and "foo/bar/bar.jq" in the given search path. This is intended to + allow modules to be placed in a directory along with, for example, + version control files, README files, and so on, but also to allow + for single-file modules. - The search path used is as follows: + Consecutive components with the same name are not allowed to avoid + ambiguities (e.g., "foo::foo"). - - the directory indicated in the search key of the `import` - directive metadata (see below) - - otherwise any directory(ies) listed in the `-L` command-line - argument - - otherwise any directory(ies) listed in the `$JQ_LIBRARY_PATH` - environment variable + For example, with `-L$HOME/.jq` a module `foo` can be found in + `$HOME/.jq/foo.jq` and `$HOME/.jq/foo/foo.jq`. - For example, with `-L$HOME/.jq` A module `foo` can be found in - `$HOME/.jq/any/foo.jq`, `$HOME/.jq/1.4/foo.jq`, - `$HOME/.jq/any/foo/foo.jq`, and in `$HOME/.jq/foo/foo.jq`. + If "$HOME/.jq" is a file, it is sourced into the main program. entries: - title: "`module NAME {};`" @@ -2349,7 +2362,8 @@ sections: an object with keys like "version", "repo", "URI", and so on. The "search" key in the metadata, if present, should have a - string value consisting of a single directory path to search. + string or array value (array of strings); this is the search + path to be prefixed to the top-level search path. - title: "`modulemeta`" body: | diff --git a/execute.c b/execute.c index 2b8a0572..e8fba19d 100644 --- a/execute.c +++ b/execute.c @@ -1004,12 +1004,16 @@ int jq_compile(jq_state *jq, const char* str) { return jq_compile_args(jq, str, jv_array()); } -jv jq_get_lib_origin(jq_state *jq) { - return jq_get_attr(jq, jv_string("ORIGIN")); +jv jq_get_jq_origin(jq_state *jq) { + return jq_get_attr(jq, jv_string("JQ_ORIGIN")); +} + +jv jq_get_prog_origin(jq_state *jq) { + return jq_get_attr(jq, jv_string("PROGRAM_ORIGIN")); } jv jq_get_lib_dirs(jq_state *jq) { - return jq_get_attr(jq, jv_string("LIB_DIRS")); + return jq_get_attr(jq, jv_string("JQ_LIBRARY_PATH")); } void jq_set_attrs(jq_state *jq, jv attrs) { diff --git a/jq.h b/jq.h index 3c9c7e31..e3730cc6 100644 --- a/jq.h +++ b/jq.h @@ -24,7 +24,8 @@ void jq_teardown(jq_state **); void jq_set_attrs(jq_state *, jv); jv jq_get_attrs(jq_state *); -jv jq_get_lib_origin(jq_state *); +jv jq_get_jq_origin(jq_state *); +jv jq_get_prog_origin(jq_state *); jv jq_get_lib_dirs(jq_state *); void jq_set_attr(jq_state *, jv, jv); jv jq_get_attr(jq_state *, jv); diff --git a/linker.c b/linker.c index bea1c30b..e6a70569 100644 --- a/linker.c +++ b/linker.c @@ -28,31 +28,19 @@ static int load_library(jq_state *jq, jv lib_path, block *out_block, struct lib_ // 1. lib_path // 2. -L paths passed in on the command line (from jq_state*) // 3. JQ_LIBRARY_PATH environment variable -jv build_lib_search_chain(jq_state *jq, jv lib_path) { - assert(jv_get_kind(lib_path) == JV_KIND_STRING); - - jv out_paths = jv_array(); - if (jv_string_length_bytes(jv_copy(lib_path))) - out_paths = jv_array_append(out_paths, lib_path); - else - jv_free(lib_path); - jv lib_dirs = jq_get_lib_dirs(jq); - jv_array_foreach(lib_dirs, i, path) { - if (jv_string_length_bytes(jv_copy(path)) == 0) { - jv_free(path); - continue; - } +jv build_lib_search_chain(jq_state *jq, jv search_path) { + assert(jv_get_kind(search_path) == JV_KIND_ARRAY); + jv expanded = jv_array(); + jv_array_foreach(search_path, i, path) { path = expand_path(path); - if (jv_is_valid(path)) { - out_paths = jv_array_append(out_paths, path); - } else { - jv emsg = jv_invalid_get_msg(path); - jq_report_error(jq, jv_string_fmt("jq: warning: skipping search path: %s\n",jv_string_value(emsg))); - jv_free(emsg); - } + if (jv_is_valid(path)) + expanded = jv_array_append(expanded, expand_path(path)); + else + jv_free(path); } - jv_free(lib_dirs); - return out_paths; + jv_free(search_path); + expanded = jv_array_concat(expanded, jq_get_lib_dirs(jq)); + return expanded; } static jv name2relpath(jv name) { @@ -77,7 +65,7 @@ static jv name2relpath(jv name) { } static jv find_lib(jq_state *jq, jv lib_name, jv lib_search_path) { - assert(jv_get_kind(lib_search_path) == JV_KIND_STRING); + assert(jv_get_kind(lib_search_path) == JV_KIND_ARRAY); assert(jv_get_kind(lib_name) == JV_KIND_STRING); jv rel_path = name2relpath(jv_copy(lib_name)); @@ -89,9 +77,18 @@ static jv find_lib(jq_state *jq, jv lib_name, jv lib_search_path) { struct stat st; int ret; - jv lib_search_paths = build_lib_search_chain(jq, expand_path(lib_search_path)); + jv lib_search_paths = build_lib_search_chain(jq, lib_search_path); jv_array_foreach(lib_search_paths, i, spath) { + if (jv_get_kind(spath) == JV_KIND_NULL) { + jv_free(spath); + break; + } + if (jv_get_kind(spath) != JV_KIND_STRING || + strcmp(jv_string_value(spath), "") == 0) { + jv_free(spath); + continue; /* XXX report non-strings in search path?? */ + } jv testpath = jq_realpath(jv_string_fmt("%s/%s.jq", jv_string_value(spath), jv_string_value(rel_path))); @@ -125,7 +122,18 @@ static int version_matches(jq_state *jq, block importer, block module) { return 1; } -static int process_dependencies(jq_state *jq, jv lib_origin, block *src_block, struct lib_loading_state *lib_state) { +static jv default_search(jv value) { + if (!jv_is_valid(value)) { + jv_free(value); + return JV_ARRAY(jv_string("."), jv_string("$ORIGIN")); + } + if (jv_get_kind(value) != JV_KIND_ARRAY) + return JV_ARRAY(value); + return value; +} + +// XXX Split this into a util that takes a callback, and then... +static int process_dependencies(jq_state *jq, jv jq_origin, jv lib_origin, block *src_block, struct lib_loading_state *lib_state) { jv deps = block_take_imports(src_block); block bk = *src_block; int nerrors = 0; @@ -137,26 +145,38 @@ static int process_dependencies(jq_state *jq, jv lib_origin, block *src_block, s jv_free(as); as = jv_string(""); } - jv search = jv_object_get(dep, jv_string("search")); - if (!jv_is_valid(search)) { - jv_free(search); - search = jv_string(""); - } - if (strncmp("$ORIGIN/",jv_string_value(search),8) == 0) { - jv tsearch = jv_string_fmt("%s/%s", - jv_string_value(lib_origin), - jv_string_value(search) + sizeof ("$ORIGIN/") - 1); - jv_free(search); - search = tsearch; + jv search = default_search(jv_object_get(dep, jv_string("search"))); + jv_array_foreach(search, k, search_elt) { + if (strcmp(".",jv_string_value(search_elt)) == 0) { + jv tsearch = jv_copy(lib_origin); + jv_free(search_elt); + search = jv_array_set(search, k, tsearch); + } else if (strncmp("./",jv_string_value(search_elt),sizeof("./")-1) == 0) { + jv tsearch = jv_string_fmt("%s/%s", + jv_string_value(lib_origin), + jv_string_value(search_elt) + sizeof ("./") - 1); + jv_free(search_elt); + search = jv_array_set(search, k, tsearch); + } else if (strncmp("$ORIGIN/",jv_string_value(search_elt),sizeof("$ORIGIN/")-1) == 0) { + jv tsearch = jv_string_fmt("%s/%s", + jv_string_value(jq_origin), + jv_string_value(search_elt) + sizeof ("$ORIGIN/") - 1); + jv_free(search_elt); + search = jv_array_set(search, k, tsearch); + } else { + jv_free(search_elt); + } } jv lib_path = find_lib(jq, name, search); + // XXX ...move the rest of this into a callback. if (!jv_is_valid(lib_path)) { jv emsg = jv_invalid_get_msg(lib_path); jq_report_error(jq, jv_string_fmt("jq: error: %s\n",jv_string_value(emsg))); jv_free(emsg); - jv_free(lib_origin); jv_free(as); jv_free(deps); + jv_free(jq_origin); + jv_free(lib_origin); return 1; } uint64_t state_idx = 0; @@ -186,6 +206,7 @@ static int process_dependencies(jq_state *jq, jv lib_origin, block *src_block, s jv_free(as); } jv_free(lib_origin); + jv_free(jq_origin); jv_free(deps); return nerrors; } @@ -208,7 +229,9 @@ static int load_library(jq_state *jq, jv lib_path, block *out_block, struct lib_ lib_state->names[state_idx] = strdup(jv_string_value(lib_path)); lib_state->defs[state_idx] = program; char *lib_origin = strdup(jv_string_value(lib_path)); - nerrors += process_dependencies(jq, jv_string(dirname(lib_origin)), &lib_state->defs[state_idx], lib_state); + nerrors += process_dependencies(jq, jq_get_jq_origin(jq), + jv_string(dirname(lib_origin)), + &lib_state->defs[state_idx], lib_state); free(lib_origin); *out_block = lib_state->defs[state_idx]; } @@ -219,8 +242,10 @@ static int load_library(jq_state *jq, jv lib_path, block *out_block, struct lib_ return nerrors; } +// FIXME It'd be nice to have an option to search the same search path +// as we do in process_dependencies. jv load_module_meta(jq_state *jq, jv modname) { - jv lib_path = find_lib(jq, modname, jv_string("")); + jv lib_path = find_lib(jq, modname, jv_array()); jv meta = jv_null(); jv data = jv_load_file(jv_string_value(lib_path), 1); if (jv_is_valid(data)) { @@ -249,7 +274,7 @@ int load_program(jq_state *jq, struct locfile* src, block *out_block) { if (nerrors) return nerrors; - nerrors = process_dependencies(jq, jq_get_lib_origin(jq), &program, &lib_state); + nerrors = process_dependencies(jq, jq_get_jq_origin(jq), jq_get_prog_origin(jq), &program, &lib_state); block libs = gen_noop(); for (uint64_t i = 0; i < lib_state.ct; ++i) { free(lib_state.names[i]); diff --git a/main.c b/main.c index 1c9b7fe2..a43f8156 100644 --- a/main.c +++ b/main.c @@ -10,6 +10,7 @@ #include "jv.h" #include "jq.h" #include "jv_alloc.h" +#include "util.h" #include "version.h" int jq_testsuite(int argc, char* argv[]); @@ -210,7 +211,7 @@ int main(int argc, char* argv[]) { int jq_flags = 0; size_t short_opts = 0; jv program_arguments = jv_array(); - jv lib_search_paths = jv_array(); + jv lib_search_paths = jv_null(); for (int i=1; i= argc - 1) { @@ -376,7 +379,8 @@ int main(int argc, char* argv[]) { } char *penv = getenv("JQ_LIBRARY_PATH"); - if (penv) { + if (penv && jv_get_kind(lib_search_paths) == JV_KIND_NULL) { + // Use $JQ_LIBRARY_PATH #ifdef WIN32 #define PATH_ENV_SEPARATOR ";" #else @@ -384,15 +388,22 @@ int main(int argc, char* argv[]) { #endif lib_search_paths = jv_array_concat(lib_search_paths,jv_string_split(jv_string(penv),jv_string(PATH_ENV_SEPARATOR))); #undef PATH_ENV_SEPARATOR + } else if (jv_get_kind(lib_search_paths) == JV_KIND_NULL) { + // Use compiled-in default JQ_LIBRARY_PATH +#ifdef WIN32 + lib_search_paths = JV_ARRAY(jv_string("~/.jq"), jv_string("$ORIGIN/lib")); +#else + lib_search_paths = JV_ARRAY(jv_string("~/.jq"), jv_string("$ORIGIN/../lib/jq")); +#endif } - jq_set_attr(jq, jv_string("LIB_DIRS"), lib_search_paths); + jq_set_attr(jq, jv_string("JQ_LIBRARY_PATH"), lib_search_paths); char *origin = strdup(argv[0]); if (origin == NULL) { fprintf(stderr, "Error: out of memory\n"); exit(1); } - jq_set_attr(jq, jv_string("ORIGIN"), jv_string(dirname(origin))); + jq_set_attr(jq, jv_string("JQ_ORIGIN"), jv_string(dirname(origin))); free(origin); if (strchr(JQ_VERSION, '-') == NULL) @@ -436,6 +447,12 @@ int main(int argc, char* argv[]) { } if (options & FROM_FILE) { + char *program_origin = strdup(program); + if (program_origin == NULL) { + perror("malloc"); + exit(2); + } + jv data = jv_load_file(program, 1); if (!jv_is_valid(data)) { data = jv_invalid_get_msg(data); @@ -444,9 +461,12 @@ int main(int argc, char* argv[]) { ret = 2; goto out; } + jq_set_attr(jq, jv_string("PROGRAM_ORIGIN"), jq_realpath(jv_string(dirname(program_origin)))); compiled = jq_compile_args(jq, jv_string_value(data), program_arguments); + free(program_origin); jv_free(data); } else { + jq_set_attr(jq, jv_string("PROGRAM_ORIGIN"), jq_realpath(jv_string("."))); // XXX is this good? compiled = jq_compile_args(jq, program, program_arguments); } if (!compiled){ diff --git a/tests/modules/c/c.jq b/tests/modules/c/c.jq index 2d355ed3..d5000067 100644 --- a/tests/modules/c/c.jq +++ b/tests/modules/c/c.jq @@ -1,5 +1,14 @@ module c {whatever:null}; import a as foo; -import d as d {search:"$ORIGIN/"}; +import d as d {search:"./"}; +import d {search:"./"}; +import e as e {search:"./../lib/jq"}; +import f as f {search:"./../lib/jq"}; def a: 0; -def c: foo::a + "c" + d::meh; +def c: + if meh != d::meh then error("import into namespace doesn't work") + elif foo::a != "a" then error("foo::a didn't work as expected") + elif d::meh != "meh" then error("d::meh didn't work as expected") + elif e::bah != "bah" then error("e::bah didn't work as expected") + elif f::f != "f is here" then error("f::f didn't work as expected") + else foo::a + "c" + d::meh + e::bah end; diff --git a/tests/modules/lib/jq/e/e.jq b/tests/modules/lib/jq/e/e.jq new file mode 100644 index 00000000..84b6a8cc --- /dev/null +++ b/tests/modules/lib/jq/e/e.jq @@ -0,0 +1 @@ +def bah: "bah"; diff --git a/tests/modules/lib/jq/f.jq b/tests/modules/lib/jq/f.jq new file mode 100644 index 00000000..1018b198 --- /dev/null +++ b/tests/modules/lib/jq/f.jq @@ -0,0 +1 @@ +def f: "f is here"; diff --git a/tests/run b/tests/run index 252ad732..c39e7fa7 100755 --- a/tests/run +++ b/tests/run @@ -137,12 +137,12 @@ if ! $VALGRIND $Q ./jq -ner -L $mods 'import a as foo; import b as bar; import a exit 1 fi -if ! $VALGRIND $Q ./jq -ner -L $mods 'import c as foo; [foo::a, foo::c] | . == [0,"acmeh"]' > /dev/null; then +if ! $VALGRIND $Q ./jq -ner -L $mods 'import c as foo; [foo::a, foo::c] | . == [0,"acmehbah"]' > /dev/null; then echo "Module system appears to be broken" 1>&2 exit 1 fi -if [ "`$VALGRIND $Q ./jq -cner -L $mods '\"c\" | modulemeta'`" != '{"whatever":null,"name":"c","deps":[{"as":"foo","name":"a"},{"search":"$ORIGIN/","as":"d","name":"d"}]}' ]; then +if [ "`$VALGRIND $Q ./jq -cner -L $mods '\"c\" | modulemeta'`" != '{"whatever":null,"name":"c","deps":[{"as":"foo","name":"a"},{"search":"./","as":"d","name":"d"},{"search":"./","name":"d"},{"search":"./../lib/jq","as":"e","name":"e"},{"search":"./../lib/jq","as":"f","name":"f"}]}' ]; then echo "modulemeta builtin appears to be broken" 1>&2 exit 1 fi @@ -171,3 +171,14 @@ if [ -n "$VALGRIND" ] && ! grep 'ERROR SUMMARY: 0 errors from 0 contexts' $d/out cat $d/out exit 1 fi + +if ! $VALGRIND ./jq -ner -L $mods -f $mods/test_bind_order.jq > $d/out 2>&1; then + echo "Import bind order is broken" 1>&2 + exit 1 +fi + +if [ -n "$VALGRIND" ] && ! grep 'ERROR SUMMARY: 0 errors from 0 contexts' $d/out > /dev/null; then + echo "Memory errors when programs have syntax errors" 1>&2 + cat $d/out + exit 1 +fi