Lots of build system and docs improvements, including a manpage.

- Build binaries for multiple platforms
 - Make a manpage out of the manual (see #19)
 - Extract more tests from the documentation
 - Fix a few documentation bugs uncovered by above.

.gitignore

@@ -9,5 +9,7 @@ jv_test
 jv_parse
 parsertest*~
 
+build
+
 # Something delightfully recursive happens otherwise
 docs/content/2.download/source/*

Makefile

@@ -1,10 +1,13 @@
-CC=gcc -Wextra -Wall -Wno-missing-field-initializers -Wno-unused-parameter -std=gnu99 -ggdb -Wno-unused-function
+CC=gcc
+CFLAGS=-Wextra -Wall -Wno-missing-field-initializers -Wno-unused-parameter -std=gnu99 -ggdb -Wno-unused-function
+
 prefix=/usr/local
+mandir=$(prefix)/share/man
+
 
 .PHONY: all clean releasedep tarball install uninstall test releasetag
 all: jq
 
-
 lexer.gen.c: lexer.l
 	flex -o lexer.gen.c --header-file=lexer.gen.h lexer.l
 lexer.gen.h: lexer.gen.c
@@ -23,16 +26,45 @@ main.c: version.gen.h
 
 JQ_SRC=parser.gen.c lexer.gen.c opcode.c bytecode.c compile.c execute.c builtin.c jv.c jv_parse.c jv_print.c jv_dtoa.c jv_unicode.c jv_aux.c
 
-
+jq_test: CFLAGS += -DJQ_DEBUG=1
 jq_test: $(JQ_SRC) jq_test.c
-	$(CC) -DJQ_DEBUG=1 -o $@ $^
+	$(CC) $(CFLAGS) $(CFLAGS_DEBUG) -o $@ $^
 
+jq: CFLAGS += -O -DJQ_DEBUG=0
 jq: $(JQ_SRC) main.c
-	$(CC) -O -DJQ_DEBUG=0 -o $@ $^
+	$(CC) $(CFLAGS) $(CFLAGS_OPT) -o $@ $^
 
 test: jq_test
 	valgrind --error-exitcode=1 -q --leak-check=full ./jq_test >/dev/null
 
+BINARIES=jq jq_test
+PLATFORMS=linux32 linux64 osx32 osx64 win32 win64
+
+build/linux32%: CC='x86_64-linux-gnu-gcc -m32'
+build/linux64%: CC='x86_64-linux-gnu-gcc -m64'
+
+# OS X cross compilers can be gotten from
+# https://launchpad.net/~flosoft/+archive/cross-apple
+build/osx32%:   CC='i686-apple-darwin10-gcc -m32'
+build/osx64%:   CC='i686-apple-darwin10-gcc -m64'
+
+# On Debian, you can get windows compilers in the
+# gcc-mingw-w64-i686 and gcc-mingw-w64-x86-64 packages.
+build/win32%:   CC='i686-w64-mingw32-gcc -m32'
+build/win64%:   CC='x86_64-w64-mingw32-gcc -m64'
+
+ALL_BINARIES=$(foreach platform, $(PLATFORMS), $(foreach binary, $(BINARIES), build/$(platform)/$(binary)))
+
+$(ALL_BINARIES): build/%:
+	mkdir -p $(@D)
+	make -B $(BINARIES) CC=$(CC)
+	cp $(BINARIES) $(@D)
+
+binaries: $(ALL_BINARIES)
+
+clean:
+	rm -rf build
+	rm -f $(BINARIES) *.gen.*
 
 releasedep: lexer.gen.c parser.gen.c jv_utf8_tables.gen.h
 
@@ -45,12 +77,16 @@ docs/content/2.download/source/jq.tgz: jq
 
 tarball: docs/content/2.download/source/jq.tgz
 
-install: jq
+jq.1: docs/content/3.manual/manual.yml
+	( cd docs; rake manpage; ) > $@
+
+install: jq jq.1
 	install -d -m 0755 $(prefix)/bin
 	install -m 0755 jq $(prefix)/bin
+	install -d -m 0755 $(mandir)/man1
+	install -m 0755 jq.1 $(mandir)/man1
 
 uninstall:
-	test -d $(prefix)/bin && \
-	cd $(prefix)/bin && \
-	rm -f jq
+	rm -vf $(prefix)/bin/jq
+	rm -vf $(mandir)/man1/jq.1
 

docs/Gemfile

@@ -2,3 +2,4 @@ source :rubygems
 
 gem "bonsai"
 gem "maruku"
+gem "ronn"

docs/Gemfile.lock

@@ -18,6 +18,7 @@ GEM
       tilt (>= 1.3)
       watch (>= 0.1.0)
     builder (3.1.4)
+    hpricot (0.8.6)
     i18n (0.6.1)
     launchy (2.1.2)
       addressable (~> 2.3)
@@ -25,9 +26,15 @@ GEM
     maruku (0.6.1)
       syntax (>= 1.0.0)
     multi_json (1.5.0)
+    mustache (0.99.4)
     rack (1.4.1)
     rack-protection (1.3.2)
       rack
+    rdiscount (1.6.8)
+    ronn (0.7.3)
+      hpricot (>= 0.8.2)
+      mustache (>= 0.7.0)
+      rdiscount (>= 1.5.8)
     sass (3.2.3)
     sinatra (1.3.3)
       rack (~> 1.3, >= 1.3.6)
@@ -43,3 +50,4 @@ PLATFORMS
 DEPENDENCIES
   bonsai
   maruku
+  ronn

docs/Rakefile

@@ -2,6 +2,8 @@ require 'bonsai'
 require 'liquid'
 require 'maruku'
 require 'json'
+require 'ronn'
+require 'tempfile'
 
 module ExtraFilters
   def markdownify(input)
@@ -59,3 +61,43 @@ task :build do
   Bonsai.root_dir = Dir.pwd
   Bonsai::Exporter.publish!
 end
+
+def load_manual
+  YAML::ENGINE.yamler = 'syck'
+  YAML::load(File.open("content/3.manual/manual.yml"))
+end
+
+task :manpage do
+  Tempfile.open "manpage" do |f|
+    manual = load_manual
+    f.puts manual['manpage_intro']
+    f.puts manual['body']
+    manual['sections'].each do |section|
+      
+      f.puts "## #{section['title'].upcase}\n"
+      f.puts section['body']
+      f.puts ""
+      (section['entries'] || []).each do |entry|
+        f.puts "### #{entry['title']}\n"
+        f.puts entry['body']
+        f.puts ""
+      end
+      f.puts ""
+    end
+    f.close
+    puts Ronn::Document.new(f.path).convert('roff').gsub(/<\/?code>/,"")
+  end  
+end
+
+task :mantests do
+  load_manual['sections'].each do |section|
+    (section['entries'] || []).each do |entry|
+      (entry['examples'] || []).each do |example|
+        puts example['program'].gsub("\n", " ")
+        puts example['input']
+        example['output'].each do |s| puts s end
+        puts
+      end
+    end
+  end
+end

docs/content/3.manual/manual.yml

@@ -26,6 +26,15 @@ body: |
 
   But that's getting ahead of ourselves. :) Let's start with something
   simpler:
+
+manpage_intro: |
+  jq(1) -- Command-line JSON processor
+  ====================================
+
+  ## SYNOPSIS
+
+  `jq` [<options>...] <filter>
+
   
 sections:
   - title: Invoking jq
@@ -40,51 +49,51 @@ sections:
       You can affect how jq reads and writes its input and output
       using some command-line options:
 
-      * `--slurp`/`-s` 
-
-        Instead of running the filter for each JSON object in the
-        input, read the entire input stream into a large array and run
-        the filter just once.
-      
-      * `--raw-input`/`-R` 
-      
-        Don't parse the input as JSON. Instead, each line of text is
-        passed to the filter as a string. If combined with `--slurp`,
-        then the entire input is passed to the filter as a single long
-        string.
-
-      * `--null-input`/`-n`
-
-        Don't read any input at all! Instead, the filter is run once
-        using `null` as the input. This is useful when using jq as a
-        simple calculator or to construct JSON data from scratch.
-
-      * `--compact-output` / `-c`
-
-        By default, jq pretty-prints JSON output. Using this option
-        will result in more compact output by instead putting each
-        JSON object on a single line.
-
-      * `--colour-output` / `-C` and `--monochrome-output` / `-M`
-      
-        By default, jq outputs colored JSON if writing to a
-        terminal. You can force it to produce color even if writing to
-        a pipe or a file using `-C`, and disable color with `-M`.
-
-      * `--ascii-output` / `-a`
-
-        jq usually outputs non-ASCII Unicode codepoints as UTF-8, even
-        if the input specified them as escape sequences (like
-        "\u03bc"). Using this option, you can force jq to produce pure
-        ASCII output with every non-ASCII character replaced with the
-        equivalent escape sequence.
-
-      * `--raw-output` / `-r`
-
-        With this option, if the filter's result is a string then it
-        will be written directly to standard output rather than being
-        formatted as a JSON string with quotes. This can be useful for
-        making jq filters talk to non-JSON-based systems.
+        * `--slurp`/`-s`:
+  
+          Instead of running the filter for each JSON object in the
+          input, read the entire input stream into a large array and run
+          the filter just once.
+        
+        * `--raw-input`/`-R`:
+        
+          Don't parse the input as JSON. Instead, each line of text is
+          passed to the filter as a string. If combined with `--slurp`,
+          then the entire input is passed to the filter as a single long
+          string.
+  
+        * `--null-input`/`-n`:
+  
+          Don't read any input at all! Instead, the filter is run once
+          using `null` as the input. This is useful when using jq as a
+          simple calculator or to construct JSON data from scratch.
+  
+        * `--compact-output` / `-c`:
+  
+          By default, jq pretty-prints JSON output. Using this option
+          will result in more compact output by instead putting each
+          JSON object on a single line.
+  
+        * `--colour-output` / `-C` and `--monochrome-output` / `-M`:
+        
+          By default, jq outputs colored JSON if writing to a
+          terminal. You can force it to produce color even if writing to
+          a pipe or a file using `-C`, and disable color with `-M`.
+  
+        * `--ascii-output` / `-a`:
+  
+          jq usually outputs non-ASCII Unicode codepoints as UTF-8, even
+          if the input specified them as escape sequences (like
+          "\u03bc"). Using this option, you can force jq to produce pure
+          ASCII output with every non-ASCII character replaced with the
+          equivalent escape sequence.
+  
+        * `--raw-output` / `-r`:
+  
+          With this option, if the filter's result is a string then it
+          will be written directly to standard output rather than being
+          formatted as a JSON string with quotes. This can be useful for
+          making jq filters talk to non-JSON-based systems.
 
   - title: Basic filters
     entries:
@@ -148,7 +157,7 @@ sections:
 
         examples:
           - program: '.[]'
-            input: '[{name":"JSON", "good":true}, {"name":"XML", "good":false}]'
+            input: '[{"name":"JSON", "good":true}, {"name":"XML", "good":false}]'
             output:
               - '{"name":"JSON", "good":true}'
               - '{"name":"XML", "good":false}'
@@ -178,7 +187,7 @@ sections:
             
           - program: '.[4,2]'
             input: '["a","b","c","d","e"]'
-            output: ['"d"', '"c"']
+            output: ['"e"', '"c"']
           
       - title: "`|`"
         body: |
@@ -194,7 +203,7 @@ sections:
 
         examples:
           - program: '.[] | .name'
-            input: '[{name":"JSON", "good":true}, {"name":"XML", "good":false}]'
+            input: '[{"name":"JSON", "good":true}, {"name":"XML", "good":false}]'
             output: ['"JSON"', '"XML"']
 
   - title: Types and Values
@@ -325,7 +334,7 @@ sections:
         examples:
           - program: '.a + 1'
             input: '{"a": 7}'
-            output: '{"a": 8}'
+            output: ['8']
           - program: '.a + .b'
             input: '{"a": [1,2], "b": [3,4]}'
             output: ['[1,2,3,4]']
@@ -409,11 +418,11 @@ sections:
           `foo` returns true for that input, and produces no output
           otherwise.
 
-          It's useful for filtering lists: `[1,2,3] | select(. >= 2)`
+          It's useful for filtering lists: `[1,2,3] | map(select(. >= 2))`
           will give you `[3]`.
 
         examples:
-          - program: 'select(. >= 2)'
+          - program: 'map(select(. >= 2))'
             input: '[1,5,3,0,7]'
             output: ['[5,3,7]']
 
@@ -427,7 +436,7 @@ sections:
         examples:
           - program: '1, empty, 2'
             input: 'null'
-            output: [1,2]
+            output: [1, 2]
           - program: '[1,2,empty,3]'
             input: 'null'
             output: ['[1,2,3]']
@@ -461,7 +470,7 @@ sections:
         examples:
           - program: add
             input: '["a","b","c"]'
-            output: ["abc"]
+            output: ['"abc"']
           - program: add
             input: '[1, 2, 3]'
             output: [6]
@@ -479,7 +488,7 @@ sections:
         examples:
           - program: '.[] | tonumber'
             input: '[1, "1"]'
-            output: [1,1]
+            output: [1, 1]
 
       - title: `tostring`
         body: |
@@ -593,10 +602,10 @@ sections:
             input: '["foobar", "foobaz", "blarp"]'
             output: ['false']
           - program: 'contains({foo: 12, bar: [{barp: 12}]})'
-            input: '{foo: 12, bar:[1,2,{barp:12, blip:13}]}'
+            input: '{"foo": 12, "bar":[1,2,{"barp":12, "blip":13}]}'
             output: ['true']
           - program: 'contains({foo: 12, bar: [{barp: 15}]})'
-            input: '{foo: 12, bar:[1,2,{barp:12, blip:13}]}'
+            input: '{"foo": 12, "bar":[1,2,{"barp":12, "blip":13}]}'
             output: ['false']
 
       - title: "String interpolation - `\(foo)`"
@@ -633,7 +642,7 @@ sections:
         examples:
           - program: '.[] == 1'
             input: '[1, 1.0, "1", "banana"]'
-            output: ['[true, true, false, false]']
+            output: ['true', 'true', 'false', 'false']
       - title: if-then-else
         body: |
 
@@ -854,12 +863,12 @@ sections:
           input's `.foo` field to each element of the array.
 
         examples:
-          - program: 'def addvalue(f): map(. + [$value]); addvalue(.[0])'
+          - program: 'def addvalue(f): . + [f]; map(addvalue(.[0]))'
             input: '[[1,2],[10,20]]'
             output: ['[[1,2,1], [10,20,10]]']
-          - program: 'def addvalue(f): f as $x | map (. + $x); addvalue(.[0])'
+          - program: 'def addvalue(f): f as $x | map(. + $x); addvalue(.[0])'
             input: '[[1,2],[10,20]]'
-            output: ['[[1,2,[1,2]], [10,20,[1,2]]]']
+            output: ['[[1,2,1,2], [10,20,1,2]]']
       
 
   - title: Assignment
@@ -971,4 +980,5 @@ sections:
           "stedolan" wrote, and we can comment on each of them in the same way
           that we did before:
           
-              (.posts[] | select(.author == "stedolan") | .comments) |= . + ["terrible."]
+              (.posts[] | select(.author == "stedolan") | .comments) |=
+                  . + ["terrible."]

jq_test.c

@@ -8,9 +8,24 @@
 static void jv_test();
 static void run_jq_tests();
 
-int main() {
+FILE* testdata;
+
+int main(int argc, char* argv[]) {
   jv_test();
+  if (argc == 1) {
+    testdata = fopen("testdata", "r");
+  } else if (argc == 2) {
+    if (!strcmp(argv[1], "-")) {
+      testdata = stdin;
+    } else {
+      testdata = fopen(argv[1], "r");
+    }
+  } else {
+    printf("usage: %s OR cat testdata | %s - OR %s testdata\n", argv[0], argv[0], argv[0]);
+    return 127;
+  }
   run_jq_tests();
+  if (testdata != stdin) fclose(testdata);
 }
 
 
@@ -24,37 +39,40 @@ static int skipline(const char* buf) {
 }
 
 static void run_jq_tests() {
-  FILE* testdata = fopen("testdata","r");
   char buf[4096];
-  int tests = 0, passed = 0;
+  int tests = 0, passed = 0, invalid = 0;
 
   while (1) {
     if (!fgets(buf, sizeof(buf), testdata)) break;
     if (skipline(buf)) continue;
+    if (buf[strlen(buf)-1] == '\n') buf[strlen(buf)-1] = 0;
     printf("Testing %s\n", buf);
     int pass = 1;
+    tests++;
     struct bytecode* bc = jq_compile(buf);
-    assert(bc);
+    if (!bc) {invalid++; continue;}
+#if JQ_DEBUG
     printf("Disassembly:\n");
     dump_disassembly(2, bc);
     printf("\n");
+#endif
     fgets(buf, sizeof(buf), testdata);
     jv input = jv_parse(buf);
-    assert(jv_is_valid(input));
+    if (!jv_is_valid(input)){ invalid++; continue; }
     jq_init(bc, input);
 
     while (fgets(buf, sizeof(buf), testdata)) {
       if (skipline(buf)) break;
       jv expected = jv_parse(buf);
-      assert(jv_is_valid(expected));
+      if (!jv_is_valid(expected)){ invalid++; continue; }
       jv actual = jq_next();
       if (!jv_is_valid(actual)) {
         jv_free(actual);
-        printf("Insufficient results\n");
+        printf("*** Insufficient results\n");
         pass = 0;
         break;
       } else if (!jv_equal(jv_copy(expected), jv_copy(actual))) {
-        printf("Expected ");
+        printf("*** Expected ");
         jv_dump(jv_copy(expected), 0);
         printf(", but got ");
         jv_dump(jv_copy(actual), 0);
@@ -72,7 +90,7 @@ static void run_jq_tests() {
     if (pass) {
       jv extra = jq_next();
       if (jv_is_valid(extra)) {
-        printf("Superfluous result: ");
+        printf("*** Superfluous result: ");
         jv_dump(extra, 0);
         printf("\n");
         pass = 0;
@@ -82,11 +100,9 @@ static void run_jq_tests() {
     }
     jq_teardown();
     bytecode_free(bc);
-    tests++;
     passed+=pass;
   }
-  fclose(testdata);
-  printf("%d of %d tests passed\n", passed,tests);
+  printf("%d of %d tests passed (%d malformed)\n", passed,tests,invalid);
   if (passed != tests) exit(1);
 }