Module search revamp for pkg managers

The search path listed in an import directive can now be an array.  The
top-level search path is appended.  Null and empty strings in the path
terminate any search.  The "." in "." and "./*" is replaced with the
directory containing the file doing the import (for command-line
programs this is the current directory, though that may be a bad idea).

No version numbers or anything of the sort are gratuitously added to the
search paths.

All this makes external package managers possible by allowing
dependencies to be installed local to dependents.

.gitignore

@@ -10,6 +10,7 @@
 tags
 
 jq
+!tests/modules/lib/jq/
 jq.1
 
 # Something delightfully recursive happens otherwise

docs/content/3.manual/manual.yml

@@ -2286,45 +2286,58 @@ sections:
     body: |
 
       jq has a library/module system.  Modules are files whose names end
-      in `.jq`.  Modules should start with a `module` directive.
-      Programs and modules must `import` their dependencies, if any.
+      in `.jq`.  Modules should (but don't have to) start with a
+      `module` directive.  Programs and modules must `import` their
+      dependencies, if any.
 
-      A search path is used to search for modules.  Each directory in
-      the search path is tested twice: once with a directory named "any"
-      added on the end, and once with the jq version number as a
-      directory added on the end.
+      Modules imported by a program are searched for in a default search
+      path (see below).  The `import` directive allows the importer to
+      alter this path.
+
+      Paths in the a search path are subject to various substitutions.
 
       For paths starting with "~/", the user's home directory is
       substituted for "~".
 
-      For paths starting with "$ORIGIN/", the path of the directory of
-      the dependent is used.  For main programs that's the path of the
-      jq executable's containing directory.
+      For paths starting with "$ORIGIN/", the path of the jq executable
+      is substituted for "$ORIGIN".
+
+      For paths starting with "./" or paths that are ".", the path of
+      the including file is substituted for ".".  For top-level programs
+      given on the command-line, the current directory is used.
+
+      Import directives can optionally specify a search path to which
+      the default is appended.
+
+      The default search path is the search path given to the `-L`
+      command-line option, else the "$JQ_LIBRARY_PATH", if set in the
+      environment, else `["~/.jq", "$ORIGIN/../lib/jq"]` (on Unix) or
+      `["~/.jq", "$ORIGIN/lib"]` (on Windows).
+
+      The `-L` argument's and "$JQ_LIBRARY_PATH" environment variable's
+      string values are split on ":" on Unix, or ";" on Windows.
+
+      Null and empty string path elements terminate search path
+      processing.
 
       Modules may be organized into a hierarchy, with name components
       separated by double-colons ("::").  For example: "foo::bar".  Each
       component in the name corresponds to a directory on the
-      filesystem.  Consecutive components with the same name are not
-      allowed to avoid ambiguities (e.g., "foo::foo").
+      filesystem.
 
-      For example, a module named "foo::bar" would be searched for in
-      "foo/bar.jq" and "foo/bar/bar.jq" in the given search path. This
-      is intended to allow modules to be placed in a directory along
-      with, for example, version control files, README files, and so on,
-      but also to allow for single-file modules.
+      A module named "foo::bar" would be searched for in "foo/bar.jq"
+      and "foo/bar/bar.jq" in the given search path. This is intended to
+      allow modules to be placed in a directory along with, for example,
+      version control files, README files, and so on, but also to allow
+      for single-file modules.
 
-      The search path used is as follows:
+      Consecutive components with the same name are not allowed to avoid
+      ambiguities (e.g., "foo::foo").
 
-      - the directory indicated in the search key of the `import`
-        directive metadata (see below)
-      - otherwise any directory(ies) listed in the `-L` command-line
-        argument
-      - otherwise any directory(ies) listed in the `$JQ_LIBRARY_PATH`
-        environment variable
+      For example, with `-L$HOME/.jq` a module `foo` can be found in
+      `$HOME/.jq/foo.jq` and `$HOME/.jq/foo/foo.jq`.
 
-      For example, with `-L$HOME/.jq` A module `foo` can be found in
-      `$HOME/.jq/any/foo.jq`, `$HOME/.jq/1.4/foo.jq`,
-      `$HOME/.jq/any/foo/foo.jq`, and in `$HOME/.jq/foo/foo.jq`.
+      If "$HOME/.jq" is a file, it is sourced into the main program.
 
     entries:
       - title: "`module NAME {<metadata>};`"
@@ -2349,7 +2362,8 @@ sections:
           an object with keys like "version", "repo", "URI", and so on.
           
           The "search" key in the metadata, if present, should have a
-          string value consisting of a single directory path to search.
+          string or array value (array of strings); this is the search
+          path to be prefixed to the top-level search path.
 
       - title: "`modulemeta`"
         body: |

execute.c

@@ -1004,12 +1004,16 @@ int jq_compile(jq_state *jq, const char* str) {
   return jq_compile_args(jq, str, jv_array());
 }
 
-jv jq_get_lib_origin(jq_state *jq) {
-  return jq_get_attr(jq, jv_string("ORIGIN"));
+jv jq_get_jq_origin(jq_state *jq) {
+  return jq_get_attr(jq, jv_string("JQ_ORIGIN"));
+}
+
+jv jq_get_prog_origin(jq_state *jq) {
+  return jq_get_attr(jq, jv_string("PROGRAM_ORIGIN"));
 }
 
 jv jq_get_lib_dirs(jq_state *jq) {
-  return jq_get_attr(jq, jv_string("LIB_DIRS"));
+  return jq_get_attr(jq, jv_string("JQ_LIBRARY_PATH"));
 }
 
 void jq_set_attrs(jq_state *jq, jv attrs) {

jq.h

@@ -24,7 +24,8 @@ void jq_teardown(jq_state **);
 
 void jq_set_attrs(jq_state *, jv);
 jv jq_get_attrs(jq_state *);
-jv jq_get_lib_origin(jq_state *);
+jv jq_get_jq_origin(jq_state *);
+jv jq_get_prog_origin(jq_state *);
 jv jq_get_lib_dirs(jq_state *);
 void jq_set_attr(jq_state *, jv, jv);
 jv jq_get_attr(jq_state *, jv);

linker.c

@@ -28,31 +28,19 @@ static int load_library(jq_state *jq, jv lib_path, block *out_block, struct lib_
 // 1. lib_path
 // 2. -L paths passed in on the command line (from jq_state*)
 // 3. JQ_LIBRARY_PATH environment variable
-jv build_lib_search_chain(jq_state *jq, jv lib_path) {
-  assert(jv_get_kind(lib_path) == JV_KIND_STRING);
-
-  jv out_paths = jv_array();
-  if (jv_string_length_bytes(jv_copy(lib_path)))
-    out_paths = jv_array_append(out_paths, lib_path);
-  else
-    jv_free(lib_path);
-  jv lib_dirs = jq_get_lib_dirs(jq);
-  jv_array_foreach(lib_dirs, i, path) {
-    if (jv_string_length_bytes(jv_copy(path)) == 0)  {
-      jv_free(path);
-      continue;
-    }
+jv build_lib_search_chain(jq_state *jq, jv search_path) {
+  assert(jv_get_kind(search_path) == JV_KIND_ARRAY);
+  jv expanded = jv_array();
+  jv_array_foreach(search_path, i, path) {
     path = expand_path(path);
-    if (jv_is_valid(path)) {
-      out_paths = jv_array_append(out_paths, path);
-    } else {
-      jv emsg = jv_invalid_get_msg(path);
-      jq_report_error(jq, jv_string_fmt("jq: warning: skipping search path: %s\n",jv_string_value(emsg)));
-      jv_free(emsg);
-    } 
+    if (jv_is_valid(path))
+      expanded = jv_array_append(expanded, expand_path(path));
+    else
+      jv_free(path);
   }
-  jv_free(lib_dirs);
-  return out_paths;
+  jv_free(search_path);
+  expanded = jv_array_concat(expanded, jq_get_lib_dirs(jq));
+  return expanded;
 }
 
 static jv name2relpath(jv name) {
@@ -77,7 +65,7 @@ static jv name2relpath(jv name) {
 }
 
 static jv find_lib(jq_state *jq, jv lib_name, jv lib_search_path) {
-  assert(jv_get_kind(lib_search_path) == JV_KIND_STRING);
+  assert(jv_get_kind(lib_search_path) == JV_KIND_ARRAY);
   assert(jv_get_kind(lib_name) == JV_KIND_STRING);
 
   jv rel_path = name2relpath(jv_copy(lib_name));
@@ -89,9 +77,18 @@ static jv find_lib(jq_state *jq, jv lib_name, jv lib_search_path) {
   struct stat st;
   int ret;
 
-  jv lib_search_paths = build_lib_search_chain(jq, expand_path(lib_search_path));
+  jv lib_search_paths = build_lib_search_chain(jq, lib_search_path);
 
   jv_array_foreach(lib_search_paths, i, spath) {
+    if (jv_get_kind(spath) == JV_KIND_NULL) {
+      jv_free(spath);
+      break;
+    }
+    if (jv_get_kind(spath) != JV_KIND_STRING ||
+        strcmp(jv_string_value(spath), "") == 0) {
+      jv_free(spath);
+      continue; /* XXX report non-strings in search path?? */
+    }
     jv testpath = jq_realpath(jv_string_fmt("%s/%s.jq",
                                             jv_string_value(spath),
                                             jv_string_value(rel_path)));
@@ -125,7 +122,18 @@ static int version_matches(jq_state *jq, block importer, block module) {
   return 1;
 }
 
-static int process_dependencies(jq_state *jq, jv lib_origin, block *src_block, struct lib_loading_state *lib_state) {
+static jv default_search(jv value) {
+  if (!jv_is_valid(value)) {
+    jv_free(value);
+    return JV_ARRAY(jv_string("."), jv_string("$ORIGIN"));
+  }
+  if (jv_get_kind(value) != JV_KIND_ARRAY)
+    return JV_ARRAY(value);
+  return value;
+}
+
+// XXX Split this into a util that takes a callback, and then...
+static int process_dependencies(jq_state *jq, jv jq_origin, jv lib_origin, block *src_block, struct lib_loading_state *lib_state) {
   jv deps = block_take_imports(src_block);
   block bk = *src_block;
   int nerrors = 0;
@@ -137,26 +145,38 @@ static int process_dependencies(jq_state *jq, jv lib_origin, block *src_block, s
       jv_free(as);
       as = jv_string("");
     }
-    jv search = jv_object_get(dep, jv_string("search"));
-    if (!jv_is_valid(search)) {
-      jv_free(search);
-      search = jv_string("");
-    }
-    if (strncmp("$ORIGIN/",jv_string_value(search),8) == 0) {
-      jv tsearch = jv_string_fmt("%s/%s",
-                                 jv_string_value(lib_origin),
-                                 jv_string_value(search) + sizeof ("$ORIGIN/") - 1);
-      jv_free(search);
-      search = tsearch;
+    jv search = default_search(jv_object_get(dep, jv_string("search")));
+    jv_array_foreach(search, k, search_elt) {
+      if (strcmp(".",jv_string_value(search_elt)) == 0) {
+        jv tsearch = jv_copy(lib_origin);
+        jv_free(search_elt);
+        search = jv_array_set(search, k, tsearch);
+      } else if (strncmp("./",jv_string_value(search_elt),sizeof("./")-1) == 0) {
+        jv tsearch = jv_string_fmt("%s/%s",
+                                   jv_string_value(lib_origin),
+                                   jv_string_value(search_elt) + sizeof ("./") - 1);
+        jv_free(search_elt);
+        search = jv_array_set(search, k, tsearch);
+      } else if (strncmp("$ORIGIN/",jv_string_value(search_elt),sizeof("$ORIGIN/")-1) == 0) {
+        jv tsearch = jv_string_fmt("%s/%s",
+                                   jv_string_value(jq_origin),
+                                   jv_string_value(search_elt) + sizeof ("$ORIGIN/") - 1);
+        jv_free(search_elt);
+        search = jv_array_set(search, k, tsearch);
+      } else {
+        jv_free(search_elt);
+      }
     }
     jv lib_path = find_lib(jq, name, search);
+    // XXX ...move the rest of this into a callback.
     if (!jv_is_valid(lib_path)) {
       jv emsg = jv_invalid_get_msg(lib_path);
       jq_report_error(jq, jv_string_fmt("jq: error: %s\n",jv_string_value(emsg)));
       jv_free(emsg);
-      jv_free(lib_origin);
       jv_free(as);
       jv_free(deps);
+      jv_free(jq_origin);
+      jv_free(lib_origin);
       return 1;
     }
     uint64_t state_idx = 0;
@@ -186,6 +206,7 @@ static int process_dependencies(jq_state *jq, jv lib_origin, block *src_block, s
     jv_free(as);
   }
   jv_free(lib_origin);
+  jv_free(jq_origin);
   jv_free(deps);
   return nerrors;
 }
@@ -208,7 +229,9 @@ static int load_library(jq_state *jq, jv lib_path, block *out_block, struct lib_
       lib_state->names[state_idx] = strdup(jv_string_value(lib_path));
       lib_state->defs[state_idx] = program;
       char *lib_origin = strdup(jv_string_value(lib_path));
-      nerrors += process_dependencies(jq, jv_string(dirname(lib_origin)), &lib_state->defs[state_idx], lib_state);
+      nerrors += process_dependencies(jq, jq_get_jq_origin(jq),
+                                      jv_string(dirname(lib_origin)),
+                                      &lib_state->defs[state_idx], lib_state);
       free(lib_origin);
       *out_block = lib_state->defs[state_idx];
     }
@@ -219,8 +242,10 @@ static int load_library(jq_state *jq, jv lib_path, block *out_block, struct lib_
   return nerrors;
 }
 
+// FIXME It'd be nice to have an option to search the same search path
+// as we do in process_dependencies.
 jv load_module_meta(jq_state *jq, jv modname) {
-  jv lib_path = find_lib(jq, modname, jv_string(""));
+  jv lib_path = find_lib(jq, modname, jv_array());
   jv meta = jv_null();
   jv data = jv_load_file(jv_string_value(lib_path), 1);
   if (jv_is_valid(data)) {
@@ -249,7 +274,7 @@ int load_program(jq_state *jq, struct locfile* src, block *out_block) {
   if (nerrors)
     return nerrors;
 
-  nerrors = process_dependencies(jq, jq_get_lib_origin(jq), &program, &lib_state);
+  nerrors = process_dependencies(jq, jq_get_jq_origin(jq), jq_get_prog_origin(jq), &program, &lib_state);
   block libs = gen_noop();
   for (uint64_t i = 0; i < lib_state.ct; ++i) {
     free(lib_state.names[i]);

main.c

@@ -10,6 +10,7 @@
 #include "jv.h"
 #include "jq.h"
 #include "jv_alloc.h"
+#include "util.h"
 #include "version.h"
 
 int jq_testsuite(int argc, char* argv[]);
@@ -210,7 +211,7 @@ int main(int argc, char* argv[]) {
   int jq_flags = 0;
   size_t short_opts = 0;
   jv program_arguments = jv_array();
-  jv lib_search_paths = jv_array();
+  jv lib_search_paths = jv_null();
   for (int i=1; i<argc; i++, short_opts = 0) {
     if (further_args_are_files) {
       input_filenames[ninput_files++] = argv[i];
@@ -225,6 +226,8 @@ int main(int argc, char* argv[]) {
       }
     } else {
       if (argv[i][1] == 'L') {
+        if (jv_get_kind(lib_search_paths) == JV_KIND_NULL)
+          lib_search_paths = jv_array();
         if (argv[i][2] != 0) { // -Lname (faster check than strlen)
             lib_search_paths = jv_array_append(lib_search_paths, jv_string(argv[i]+2));
         } else if (i >= argc - 1) {
@@ -376,7 +379,8 @@ int main(int argc, char* argv[]) {
   }
 
   char *penv = getenv("JQ_LIBRARY_PATH");
-  if (penv) {
+  if (penv && jv_get_kind(lib_search_paths) == JV_KIND_NULL) {
+    // Use $JQ_LIBRARY_PATH
 #ifdef WIN32
 #define PATH_ENV_SEPARATOR ";"
 #else
@@ -384,15 +388,22 @@ int main(int argc, char* argv[]) {
 #endif
     lib_search_paths = jv_array_concat(lib_search_paths,jv_string_split(jv_string(penv),jv_string(PATH_ENV_SEPARATOR)));
 #undef PATH_ENV_SEPARATOR
+  } else if (jv_get_kind(lib_search_paths) == JV_KIND_NULL) {
+    // Use compiled-in default JQ_LIBRARY_PATH
+#ifdef WIN32
+    lib_search_paths = JV_ARRAY(jv_string("~/.jq"), jv_string("$ORIGIN/lib"));
+#else
+    lib_search_paths = JV_ARRAY(jv_string("~/.jq"), jv_string("$ORIGIN/../lib/jq"));
+#endif
   }
-  jq_set_attr(jq, jv_string("LIB_DIRS"), lib_search_paths);
+  jq_set_attr(jq, jv_string("JQ_LIBRARY_PATH"), lib_search_paths);
 
   char *origin = strdup(argv[0]);
   if (origin == NULL) {
     fprintf(stderr, "Error: out of memory\n");
     exit(1);
   }
-  jq_set_attr(jq, jv_string("ORIGIN"), jv_string(dirname(origin)));
+  jq_set_attr(jq, jv_string("JQ_ORIGIN"), jv_string(dirname(origin)));
   free(origin);
 
   if (strchr(JQ_VERSION, '-') == NULL)
@@ -436,6 +447,12 @@ int main(int argc, char* argv[]) {
   }
   
   if (options & FROM_FILE) {
+    char *program_origin = strdup(program);
+    if (program_origin == NULL) {
+      perror("malloc");
+      exit(2);
+    }
+
     jv data = jv_load_file(program, 1);
     if (!jv_is_valid(data)) {
       data = jv_invalid_get_msg(data);
@@ -444,9 +461,12 @@ int main(int argc, char* argv[]) {
       ret = 2;
       goto out;
     }
+    jq_set_attr(jq, jv_string("PROGRAM_ORIGIN"), jq_realpath(jv_string(dirname(program_origin))));
     compiled = jq_compile_args(jq, jv_string_value(data), program_arguments);
+    free(program_origin);
     jv_free(data);
   } else {
+    jq_set_attr(jq, jv_string("PROGRAM_ORIGIN"), jq_realpath(jv_string("."))); // XXX is this good?
     compiled = jq_compile_args(jq, program, program_arguments);
   }
   if (!compiled){

tests/modules/c/c.jq

@@ -1,5 +1,14 @@
 module c {whatever:null};
 import a as foo;
-import d as d {search:"$ORIGIN/"};
+import d as d {search:"./"};
+import d {search:"./"};
+import e as e {search:"./../lib/jq"};
+import f as f {search:"./../lib/jq"};
 def a: 0;
-def c: foo::a + "c" + d::meh;
+def c:
+  if meh != d::meh then error("import into namespace doesn't work")
+  elif foo::a != "a" then error("foo::a didn't work as expected")
+  elif d::meh != "meh" then error("d::meh didn't work as expected")
+  elif e::bah != "bah" then error("e::bah didn't work as expected")
+  elif f::f != "f is here" then error("f::f didn't work as expected")
+  else foo::a + "c" + d::meh + e::bah end;

tests/modules/lib/jq/e/e.jq

@@ -0,0 +1 @@
+def bah: "bah";

tests/modules/lib/jq/f.jq

@@ -0,0 +1 @@
+def f: "f is here";

tests/run

@@ -137,12 +137,12 @@ if ! $VALGRIND $Q ./jq -ner -L $mods 'import a as foo; import b as bar; import a
     exit 1
 fi
 
-if ! $VALGRIND $Q ./jq -ner -L $mods 'import c as foo; [foo::a, foo::c] | . == [0,"acmeh"]' > /dev/null; then
+if ! $VALGRIND $Q ./jq -ner -L $mods 'import c as foo; [foo::a, foo::c] | . == [0,"acmehbah"]' > /dev/null; then
     echo "Module system appears to be broken" 1>&2
     exit 1
 fi
 
-if [ "`$VALGRIND $Q ./jq -cner -L $mods '\"c\" | modulemeta'`" != '{"whatever":null,"name":"c","deps":[{"as":"foo","name":"a"},{"search":"$ORIGIN/","as":"d","name":"d"}]}' ]; then
+if [ "`$VALGRIND $Q ./jq -cner -L $mods '\"c\" | modulemeta'`" != '{"whatever":null,"name":"c","deps":[{"as":"foo","name":"a"},{"search":"./","as":"d","name":"d"},{"search":"./","name":"d"},{"search":"./../lib/jq","as":"e","name":"e"},{"search":"./../lib/jq","as":"f","name":"f"}]}' ]; then
     echo "modulemeta builtin appears to be broken" 1>&2
     exit 1
 fi
@@ -171,3 +171,14 @@ if [ -n "$VALGRIND" ] && ! grep 'ERROR SUMMARY: 0 errors from 0 contexts' $d/out
     cat $d/out
     exit 1
 fi
+
+if ! $VALGRIND ./jq -ner -L $mods -f $mods/test_bind_order.jq > $d/out 2>&1; then
+    echo "Import bind order is broken" 1>&2
+    exit 1
+fi
+
+if [ -n "$VALGRIND" ] && ! grep 'ERROR SUMMARY: 0 errors from 0 contexts' $d/out > /dev/null; then
+    echo "Memory errors when programs have syntax errors" 1>&2
+    cat $d/out
+    exit 1
+fi