255 lines
8.9 KiB
Python
Executable File
255 lines
8.9 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
from datetime import date
|
|
from io import StringIO
|
|
from lxml import etree
|
|
import markdown
|
|
from markdown.extensions import Extension
|
|
import re
|
|
import os
|
|
import sys
|
|
from time import time
|
|
import yaml
|
|
|
|
|
|
# Prevent our markdown parser from trying to help by interpreting things in angle brackets as HTML tags.
|
|
class EscapeHtml(Extension):
|
|
|
|
def extendMarkdown(self, md):
|
|
md.preprocessors.deregister('html_block')
|
|
md.inlinePatterns.deregister('html')
|
|
|
|
|
|
class RoffWalker(object):
|
|
|
|
def __init__(self, tree, output=sys.stdout):
|
|
self.tree = tree
|
|
self.target = output
|
|
self.f = StringIO()
|
|
|
|
def walk(self):
|
|
self._walk(self.tree, parent_tag=None)
|
|
# We don't want to start lines with \. because that can confuse man
|
|
# For lines that start with \., we need to prefix them with \& so it
|
|
# knows not to treat that line as a directive
|
|
data = re.sub(r'^\\\.', r'\&.', self.f.getvalue(), flags=re.MULTILINE)
|
|
self.target.write(data)
|
|
|
|
def _ul_is_special(self, root):
|
|
if len(root) != 1:
|
|
return False
|
|
child = root[0]
|
|
if child.tag != 'li':
|
|
return False
|
|
msg = ''.join(child.itertext()).strip()
|
|
return msg.endswith(':')
|
|
|
|
def _walk_child(self, root):
|
|
if len(root) > 0:
|
|
self._walk(root[0], parent_tag=root.tag)
|
|
|
|
def _write_element(self, root, ensure_newline=True):
|
|
if root.text is not None:
|
|
text = self._sanitize(root.text)
|
|
self.__write_raw(text)
|
|
self._walk_child(root)
|
|
self._write_tail(root, ensure_newline=ensure_newline)
|
|
|
|
def _write_tail(self, root, ensure_newline=False, inline=False):
|
|
if root.tail is not None:
|
|
if inline or root.tail != '\n':
|
|
text = self._sanitize(root.tail)
|
|
if text.endswith('\n'):
|
|
ensure_newline = False
|
|
self.__write_raw(text)
|
|
if ensure_newline:
|
|
self.__write_raw('\n')
|
|
|
|
def _walk(self, root, parent_tag=None):
|
|
last_tag = None
|
|
while root is not None:
|
|
if root.tag == 'h1':
|
|
epoch = int(os.environ.get('SOURCE_DATE_EPOCH', time()))
|
|
self.__write_cmd('.TH "JQ" "1" "{}" "" ""'.format(
|
|
date.fromtimestamp(epoch).strftime('%B %Y')))
|
|
self.__write_cmd('.SH "NAME"')
|
|
# TODO: properly parse this
|
|
self.__write_raw(r'\fBjq\fR \- Command\-line JSON processor' +
|
|
"\n")
|
|
|
|
elif root.tag == 'h2':
|
|
self.__write_cmd('.SH "{}"'.format(''.join(
|
|
root.itertext()).strip()))
|
|
|
|
elif root.tag == 'h3':
|
|
text = ''.join(root.itertext()).strip()
|
|
self.__write_cmd('.SS "{}"'.format(self._h3_sanitize(text)))
|
|
|
|
elif root.tag == 'p':
|
|
if last_tag not in ['h2', 'h3'] and parent_tag not in ['li']:
|
|
self.__write_cmd('.P')
|
|
self._write_element(root, ensure_newline=(parent_tag != 'li'))
|
|
|
|
elif root.tag == 'a':
|
|
self._write_element(root, ensure_newline=(parent_tag != 'li'))
|
|
|
|
elif root.tag == 'ul':
|
|
if self._ul_is_special(root):
|
|
li = root[0]
|
|
self.__write_cmd('.TP')
|
|
self._write_element(li)
|
|
next = root.getnext()
|
|
while next is not None and next.tag == 'p':
|
|
if next.getnext() is not None and next.getnext(
|
|
).tag == 'pre':
|
|
# we don't want to .IP these, because it'll look funny with the code indent
|
|
break
|
|
self.__write_cmd('.IP')
|
|
self._write_element(next)
|
|
root = next
|
|
next = root.getnext()
|
|
else:
|
|
self._walk_child(root)
|
|
self._write_tail(root)
|
|
# A pre tag after the end of a list doesn't want two of the indentation commands
|
|
if root.getnext() is None or root.getnext().tag != 'pre':
|
|
self.__write_cmd('.IP "" 0')
|
|
|
|
elif root.tag == 'li':
|
|
self.__write_cmd(r'.IP "\(bu" 4')
|
|
if root.text is not None and root.text.strip() != '':
|
|
text = self._sanitize(root.text)
|
|
self.__write_raw(text)
|
|
self._walk_child(root)
|
|
self._write_tail(root, ensure_newline=True)
|
|
|
|
elif root.tag == 'strong':
|
|
if root.text is not None:
|
|
text = self._sanitize(root.text)
|
|
self.__write_raw('\\fB{}\\fR'.format(text))
|
|
|
|
self._write_tail(root, inline=True)
|
|
|
|
elif root.tag == 'em':
|
|
if root.text is not None:
|
|
text = self._sanitize(root.text)
|
|
self.__write_raw('\\fI{}\\fR'.format(text))
|
|
self._write_tail(root, inline=True)
|
|
|
|
elif root.tag == 'code':
|
|
if root.text is not None:
|
|
text = self._code_sanitize(root.text)
|
|
self.__write_raw('\\fB{}\\fR'.format(text))
|
|
self._write_tail(root, inline=True)
|
|
|
|
elif root.tag == 'pre':
|
|
self.__write_cmd('.IP "" 4')
|
|
self.__write_cmd('.nf\n') # extra newline for spacing reasons
|
|
next = root
|
|
first = True
|
|
while next is not None and next.tag == 'pre':
|
|
if not first:
|
|
self.__write_raw('\n')
|
|
text = ''.join(next.itertext(with_tail=False))
|
|
self.__write_raw(self._pre_sanitize(text))
|
|
first = False
|
|
root = next
|
|
next = next.getnext()
|
|
self.__write_cmd('.fi')
|
|
self.__write_cmd('.IP "" 0')
|
|
|
|
else:
|
|
self._walk_child(root)
|
|
|
|
last_tag = root.tag
|
|
root = root.getnext()
|
|
|
|
def _base_sanitize(self, text):
|
|
text = re.sub(r'\\', r'\\e', text)
|
|
text = re.sub(r'\.', r'\\.', text)
|
|
text = re.sub("'", r"\'", text)
|
|
text = re.sub('-', r'\-', text)
|
|
return text
|
|
|
|
def _pre_sanitize(self, text):
|
|
return self._base_sanitize(text)
|
|
|
|
def _code_sanitize(self, text):
|
|
text = self._base_sanitize(text)
|
|
text = re.sub(r'\s', ' ', text)
|
|
return text
|
|
|
|
def _h3_sanitize(self, text):
|
|
text = self._base_sanitize(text)
|
|
text = re.sub(' \n|\n ', ' ', text)
|
|
text = re.sub('\n', ' ', text)
|
|
return text
|
|
|
|
def _sanitize(self, text):
|
|
text = self._base_sanitize(text)
|
|
text = re.sub(r'<([^>]+)>', r'\\fI\1\\fR', text)
|
|
text = re.sub(r' +', ' ', text)
|
|
text = re.sub('\n', ' ', text)
|
|
return text
|
|
|
|
def __write_cmd(self, dat):
|
|
print('.', dat, sep='\n', file=self.f)
|
|
pass
|
|
|
|
def __write_raw(self, dat):
|
|
print(dat, sep='', end='', file=self.f)
|
|
pass
|
|
|
|
|
|
def load_yml_file(fn):
|
|
with open(fn) as f:
|
|
return yaml.safe_load(f)
|
|
|
|
|
|
def dedent_body(body):
|
|
lines = [re.sub(r'^ (\S)', r'\1', l) for l in body.split('\n')]
|
|
return '\n'.join(lines)
|
|
|
|
|
|
def convert_manual_to_markdown():
|
|
f = StringIO()
|
|
manual = load_yml_file("content/manual/dev/manual.yml")
|
|
f.write(manual.get('manpage_intro', '\n'))
|
|
f.write(dedent_body(manual.get('body', '\n')))
|
|
for section in manual.get('sections', []):
|
|
f.write('## {}\n'.format(section.get('title', '').upper()))
|
|
f.write(dedent_body(section.get('body', '\n')))
|
|
f.write('\n')
|
|
for entry in section.get('entries', []):
|
|
f.write('### {}\n'.format(entry.get('title', '')))
|
|
f.write(dedent_body(entry.get('body', '\n')))
|
|
f.write('\n')
|
|
if entry.get('examples') is not None:
|
|
f.write("~~~~\n")
|
|
first = True
|
|
for example in entry.get('examples'):
|
|
if not first:
|
|
f.write('\n')
|
|
f.write("jq '{}'\n".format(example.get('program', '')))
|
|
f.write(" {}\n".format(example.get('input', '')))
|
|
output = [str(x) for x in example.get('output', [])]
|
|
f.write("=> {}\n".format(', '.join(output)))
|
|
first = False
|
|
f.write("~~~~\n")
|
|
f.write('\n')
|
|
f.write(manual.get('manpage_epilogue', ''))
|
|
return f.getvalue()
|
|
|
|
|
|
# Convert manual.yml to our special markdown format
|
|
markdown_data = convert_manual_to_markdown()
|
|
|
|
# Convert markdown to html
|
|
html_data = markdown.markdown(markdown_data,
|
|
extensions=[EscapeHtml(), 'fenced_code'])
|
|
|
|
# Parse the html into a tree so we can walk it
|
|
tr = etree.HTML(html_data, etree.HTMLParser())
|
|
|
|
# Convert the markdown to ROFF
|
|
RoffWalker(tr).walk()
|