#!/usr/bin/env python3 from datetime import date from io import StringIO from lxml import etree import markdown from markdown.extensions import Extension import re import os import sys from time import time import yaml # Prevent our markdown parser from trying to help by interpreting things in angle brackets as HTML tags. class EscapeHtml(Extension): def extendMarkdown(self, md): md.preprocessors.deregister('html_block') md.inlinePatterns.deregister('html') class RoffWalker(object): def __init__(self, tree, output=sys.stdout): self.tree = tree self.target = output self.f = StringIO() def walk(self): self._walk(self.tree, parent_tag=None) # We don't want to start lines with \. because that can confuse man # For lines that start with \., we need to prefix them with \& so it # knows not to treat that line as a directive data = re.sub(r'^\\\.', r'\&.', self.f.getvalue(), flags=re.MULTILINE) self.target.write(data) def _ul_is_special(self, root): if len(root) != 1: return False child = root[0] if child.tag != 'li': return False msg = ''.join(child.itertext()).strip() return msg.endswith(':') def _walk_child(self, root): if len(root) > 0: self._walk(root[0], parent_tag=root.tag) def _write_element(self, root, ensure_newline=True): if root.text is not None: text = self._sanitize(root.text) self.__write_raw(text) self._walk_child(root) self._write_tail(root, ensure_newline=ensure_newline) def _write_tail(self, root, ensure_newline=False, inline=False): if root.tail is not None: if inline or root.tail != '\n': text = self._sanitize(root.tail) if text.endswith('\n'): ensure_newline = False self.__write_raw(text) if ensure_newline: self.__write_raw('\n') def _walk(self, root, parent_tag=None): last_tag = None while root is not None: if root.tag == 'h1': epoch = int(os.environ.get('SOURCE_DATE_EPOCH', time())) self.__write_cmd('.TH "JQ" "1" "{}" "" ""'.format( date.fromtimestamp(epoch).strftime('%B %Y'))) self.__write_cmd('.SH "NAME"') # TODO: properly parse this self.__write_raw(r'\fBjq\fR \- Command\-line JSON processor' + "\n") elif root.tag == 'h2': self.__write_cmd('.SH "{}"'.format(''.join( root.itertext()).strip())) elif root.tag == 'h3': text = ''.join(root.itertext()).strip() self.__write_cmd('.SS "{}"'.format(self._h3_sanitize(text))) elif root.tag == 'p': if last_tag not in ['h2', 'h3'] and parent_tag not in ['li']: self.__write_cmd('.P') self._write_element(root, ensure_newline=(parent_tag != 'li')) elif root.tag == 'a': self._write_element(root, ensure_newline=(parent_tag != 'li')) elif root.tag == 'ul': if self._ul_is_special(root): li = root[0] self.__write_cmd('.TP') self._write_element(li) next = root.getnext() while next is not None and next.tag == 'p': if next.getnext() is not None and next.getnext( ).tag == 'pre': # we don't want to .IP these, because it'll look funny with the code indent break self.__write_cmd('.IP') self._write_element(next) root = next next = root.getnext() else: self._walk_child(root) self._write_tail(root) # A pre tag after the end of a list doesn't want two of the indentation commands if root.getnext() is None or root.getnext().tag != 'pre': self.__write_cmd('.IP "" 0') elif root.tag == 'li': self.__write_cmd(r'.IP "\(bu" 4') if root.text is not None and root.text.strip() != '': text = self._sanitize(root.text) self.__write_raw(text) self._walk_child(root) self._write_tail(root, ensure_newline=True) elif root.tag == 'strong': if root.text is not None: text = self._sanitize(root.text) self.__write_raw('\\fB{}\\fR'.format(text)) self._write_tail(root, inline=True) elif root.tag == 'em': if root.text is not None: text = self._sanitize(root.text) self.__write_raw('\\fI{}\\fR'.format(text)) self._write_tail(root, inline=True) elif root.tag == 'code': if root.text is not None: text = self._code_sanitize(root.text) self.__write_raw('\\fB{}\\fR'.format(text)) self._write_tail(root, inline=True) elif root.tag == 'pre': self.__write_cmd('.IP "" 4') self.__write_cmd('.nf\n') # extra newline for spacing reasons next = root first = True while next is not None and next.tag == 'pre': if not first: self.__write_raw('\n') text = ''.join(next.itertext(with_tail=False)) self.__write_raw(self._pre_sanitize(text)) first = False root = next next = next.getnext() self.__write_cmd('.fi') self.__write_cmd('.IP "" 0') else: self._walk_child(root) last_tag = root.tag root = root.getnext() def _base_sanitize(self, text): text = re.sub(r'\\', r'\\e', text) text = re.sub(r'\.', r'\\.', text) text = re.sub("'", r"\'", text) text = re.sub('-', r'\-', text) return text def _pre_sanitize(self, text): return self._base_sanitize(text) def _code_sanitize(self, text): text = self._base_sanitize(text) text = re.sub(r'\s', ' ', text) return text def _h3_sanitize(self, text): text = self._base_sanitize(text) text = re.sub(' \n|\n ', ' ', text) text = re.sub('\n', ' ', text) return text def _sanitize(self, text): text = self._base_sanitize(text) text = re.sub(r'<([^>]+)>', r'\\fI\1\\fR', text) text = re.sub(r' +', ' ', text) text = re.sub('\n', ' ', text) return text def __write_cmd(self, dat): print('.', dat, sep='\n', file=self.f) pass def __write_raw(self, dat): print(dat, sep='', end='', file=self.f) pass def load_yml_file(fn): with open(fn) as f: return yaml.safe_load(f) def dedent_body(body): lines = [re.sub(r'^ (\S)', r'\1', l) for l in body.split('\n')] return '\n'.join(lines) def convert_manual_to_markdown(): f = StringIO() manual = load_yml_file("content/manual/dev/manual.yml") f.write(manual.get('manpage_intro', '\n')) f.write(dedent_body(manual.get('body', '\n'))) for section in manual.get('sections', []): f.write('## {}\n'.format(section.get('title', '').upper())) f.write(dedent_body(section.get('body', '\n'))) f.write('\n') for entry in section.get('entries', []): f.write('### {}\n'.format(entry.get('title', ''))) f.write(dedent_body(entry.get('body', '\n'))) f.write('\n') if entry.get('examples') is not None: f.write("~~~~\n") first = True for example in entry.get('examples'): if not first: f.write('\n') f.write("jq '{}'\n".format(example.get('program', ''))) f.write(" {}\n".format(example.get('input', ''))) output = [str(x) for x in example.get('output', [])] f.write("=> {}\n".format(', '.join(output))) first = False f.write("~~~~\n") f.write('\n') f.write(manual.get('manpage_epilogue', '')) return f.getvalue() # Convert manual.yml to our special markdown format markdown_data = convert_manual_to_markdown() # Convert markdown to html html_data = markdown.markdown(markdown_data, extensions=[EscapeHtml(), 'fenced_code']) # Parse the html into a tree so we can walk it tr = etree.HTML(html_data, etree.HTMLParser()) # Convert the markdown to ROFF RoffWalker(tr).walk()