usfmtc 0.2__py3-none-any.whl

usfmtc/rnc2rng/rnctree.py

@@ -0,0 +1,17 @@
+#!/usr/bin/env python
+# Compatibility API for rnc2rng 1.0
+from . import parser, serializer
+
+class Tree(object):
+
+    def __init__(self, root):
+        self.root = root
+
+    def toxml(self):
+        return serializer.XMLSerializer().toxml(self.root)
+
+def token_list(src):
+    return parser.lex(src)
+
+def make_nodetree(tokens):
+    return Tree(parser.parser.parse(tokens, parser.State()))

usfmtc/rnc2rng/serializer.py

@@ -0,0 +1,255 @@
+# Convert an RELAX NG compact syntax schema to a Node tree
+# This file released to the Public Domain by David Mertz
+from . import parser
+from usfmtc.rnc2rng.parser import (
+    ANNO_ATTR, ANNOTATION, ANY, ASSIGN, ATTR, CHOICE, DATATAG, DATATYPES,
+    DEFAULT_NS, DEFINE, DIV, DOCUMENTATION, ELEM, EMPTY, EXCEPT, GRAMMAR,
+    GROUP, INTERLEAVE, LIST, LITERAL, MAYBE, MIXED, NAME, NOT_ALLOWED, NS,
+    PARAM, PARENT, REF, ROOT, SEQ, SOME, TEXT,
+)
+
+import html
+
+QUANTS = {SOME: 'oneOrMore', MAYBE: 'optional', ANY: 'zeroOrMore'}
+TYPELIB_NS = 'http://www.w3.org/2001/XMLSchema-datatypes'
+NAMESPACES = {
+    'a': 'http://relaxng.org/ns/compatibility/annotations/1.0',
+    'xml': 'http://www.w3.org/XML/1998/namespace',
+}
+
+class XMLSerializer(object):
+
+    def __init__(self, indent=None):
+        self.indent = indent or '  '
+        self.reset()
+
+    def reset(self):
+        self.buf = []
+        self.needs = {}
+        self.types = None
+        self.ns = {}
+        self.default = ''
+        self.level = 0
+
+    def write(self, s):
+        self.buf.append(self.indent * self.level + s)
+
+    def namespace(self, ns):
+        assert ns in self.ns or ns in NAMESPACES, ns
+        if ns not in self.ns:
+            self.ns[ns] = NAMESPACES[ns]
+        return self.ns[ns]
+
+    def toxml(self, node):
+
+        self.reset()
+        types = None
+        for n in node.value:
+            if n.type == DATATYPES:
+                types = n.value[0]
+                self.types = types
+            elif n.type == DEFAULT_NS:
+                self.default = n.value[0]
+                if n.name is not None:
+                    self.ns[n.name] = n.value[0]
+            elif n.type == NS:
+                self.ns[n.name] = n.value[0]
+
+        prelude = ['<?xml version="1.0" encoding="UTF-8"?>']
+        prelude.append('<grammar xmlns="http://relaxng.org/ns/structure/1.0"')
+        if self.default:
+            prelude.append('         ns="%s"' % self.default)
+
+        self.visit(node.value)
+        for ns, url in sorted(self.ns.items()):
+            prelude.append('         xmlns:%s="%s"' % (ns, url))
+        if types is not None or self.needs.get('types'):
+            url = types if types is not None else TYPELIB_NS
+            prelude.append('         datatypeLibrary="%s"' % url)
+
+        prelude[-1] = prelude[-1] + '>'
+        self.write('</grammar>')
+        return '\n'.join(prelude + self.buf)
+
+    def anno_attrs(self, nodes):
+        select = lambda n: isinstance(n, parser.Node) and n.type == ANNO_ATTR
+        pairs = [(n.name, html.escape(n.value[0])) for n in nodes if select(n)]
+        if not pairs:
+            return ''
+        return ' ' + ' '.join('%s="%s"' % attr for attr in pairs)
+
+    def visit(self, nodes, ctx=None, indent=True):
+        '''Visiting a list of nodes, writes out the XML content to the internal
+        line-based buffer. By default, adds one level of indentation to the
+        output compared to the caller's level; passing False as the second
+        argument will prevent this from happening.'''
+        if indent:
+            self.level += 1
+        for x in nodes:
+
+            if not isinstance(x, parser.Node):
+                raise TypeError("Not a Node: " + repr(x))
+            elif x.type in set([ANNO_ATTR, DATATYPES, DEFAULT_NS, NS]):
+                continue
+
+            attribs = self.anno_attrs(x.value)
+            if x.type == DEFINE:
+
+                op, attrib = x.value[0].name, ''
+                if op in set(['|=', '&=']):
+                    modes = {'|': 'choice', '&': 'interleave'}
+                    attrib = ' combine="%s"' % modes[op[0]]
+
+                if x.name == 'start':
+                    self.write('<start%s%s>' % (attrib, attribs))
+                else:
+                    bits = x.name, attrib, attribs
+                    self.write('<define name="%s"%s%s>' % bits)
+
+                self.visit(x.value)
+                if x.name == 'start':
+                    self.write('</start>')
+                else:
+                    self.write('</define>')
+
+            elif x.type == ASSIGN:
+                self.visit(x.value, indent=False)
+            elif x.type == GRAMMAR:
+                self.write('<grammar>')
+                self.visit(x.value)
+                self.write('</grammar>')
+            elif x.type in set([MAYBE, SOME, ANY]):
+                self.write('<%s>' % QUANTS[x.type])
+                self.visit(x.value)
+                self.write('</%s>' % QUANTS[x.type])
+            elif x.type in set([INTERLEAVE, CHOICE, MIXED, LIST, DIV]):
+                self.write('<%s>' % x.type.lower())
+                self.visit(x.value)
+                self.write('</%s>' % x.type.lower())
+            elif x.type == EXCEPT:
+                self.write('<except>')
+                self.visit(x.value, ctx=ctx)
+                self.write('</except>')
+            elif x.type == NAME:
+                if not x.value and '*' in x.name:
+                    if x.name == '*':
+                        self.write('<anyName/>')
+                    else:
+                        uri = self.ns[x.name.split(':', 1)[0]]
+                        self.write('<nsName ns="%s"/>' % uri)
+                elif x.value:
+                    if x.name == '*':
+                        self.write('<anyName>')
+                    else:
+                        uri = self.ns[x.name.split(':', 1)[0]]
+                        self.write('<nsName ns="%s">' % uri)
+                    self.visit(x.value, ctx=ctx)
+                    if x.name == '*':
+                        self.write('</anyName>')
+                    else:
+                        self.write('</nsName>')
+                else:
+                    ns = '' if ctx == 'ATTR' else self.default
+                    name = x.name
+                    if ':' in name:
+                        parts = x.name.split(':', 1)
+                        ns = self.namespace(parts[0])
+                        name = parts[1]
+                    self.write('<name ns="%s">%s</name>' % (ns, name))
+            elif x.type in set([REF, PARENT]):
+                bits = x.type.lower(), x.name, attribs
+                self.write('<%s name="%s"%s/>' % bits)
+            elif x.type == LITERAL:
+                bits = attribs, html.escape(x.name)
+                self.write('<value%s>%s</value>' % bits)
+                self.visit(x.value, indent=False)
+            elif x.type == ANNOTATION:
+
+                literals, rest = [], []
+                for n in x.value:
+                    if n.type == LITERAL:
+                        literals.append(n.name)
+                    elif n.type != ANNO_ATTR:
+                        rest.append(n)
+
+                end = '/' if not (literals or rest) else ''
+                tail = ''
+                if literals and not rest:
+                    tail = html.escape(''.join(literals)) + '</%s>' % x.name
+
+                bits = x.name, attribs, end, tail
+                self.write('<%s%s%s>%s' % bits)
+                if not rest:
+                    continue
+
+                for n in x.value:
+                    if n.type == ANNO_ATTR:
+                        continue
+                    elif n.type == LITERAL:
+                        self.level += 1
+                        self.write(html.escape(n.name))
+                        self.level -= 1
+                    else:
+                        self.visit([n])
+
+                self.write('</%s>' % x.name)
+
+            elif x.type == DOCUMENTATION:
+                self.namespace('a')
+                fmt = '<a:documentation>%s</a:documentation>'
+                self.write(fmt % html.escape('\n'.join(x.value)))
+            elif x.type == GROUP:
+                if len(x.value) == 1 and x.value[0].type != SEQ:
+                    self.visit(x.value, indent=False)
+                else:
+                    self.write('<%s%s>' % (x.type.lower(), attribs))
+                    self.visit(x.value)
+                    self.write('</%s>' % x.type.lower())
+            elif x.type == NOT_ALLOWED:
+                self.write('<notAllowed/>')
+            elif x.type in set([TEXT, EMPTY]):
+                if not len(x.value):
+                    self.write('<%s%s/>' % (x.type.lower(), attribs))
+                else:
+                    self.write('<%s%s>' % (x.type.lower(), attribs))
+                    self.visit(x.value)
+                    self.write('</%s>' % x.type.lower())
+            elif x.type == SEQ:
+                self.visit(x.value, indent=False)
+            elif x.type == DATATAG:
+                self.needs['types'] = True
+                if not x.value: # no parameters
+                    self.write('<data type="%s"/>' % x.name)
+                else:
+                    name = x.name
+                    if name not in ('string', 'token'):
+                        name = x.name.split(':', 1)[1]
+                    self.write('<data type="%s">' % name)
+                    self.visit(x.value)
+                    self.write('</data>')
+            elif x.type == PARAM:
+                bits = x.name, html.escape(x.value[0])
+                self.write('<param name="%s">%s</param>' % bits)
+            elif x.type == ELEM:
+                self.write('<element%s>' % attribs)
+                self.visit(x.value)
+                self.write('</element>')
+            elif x.type == ATTR:
+                self.write('<attribute%s>' % attribs)
+                self.visit(x.value, ctx=x.type)
+                self.write('</attribute>')
+            elif x.type == ROOT:
+                # Verify the included document has the same metadata
+                for n in x.value:
+                    if n.type == DATATYPES:
+                        assert self.types == n.value[0]
+                    elif n.type == DEFAULT_NS:
+                        assert self.default == n.value[0]
+                    elif n.type == NS:
+                        assert n.name in self.ns
+                        assert self.ns[n.name] == n.value[0]
+                self.visit(x.value, indent=False)
+            else:
+                assert False, x
+        if indent:
+            self.level -= 1

usfmtc/sfmparser.py

@@ -0,0 +1,345 @@
+#!/usr/bin/python3
+
+import re, logging
+from dataclasses import dataclass
+import xml.etree.ElementTree as et
+import regex
+import usfmtc.parser as usfmp
+from usfmtc.xmlutils import ParentElement
+
+logger = logging.getLogger("sfmparser")
+
+class GlobalState(usfmp.GlobalState):
+    ''' text based global state '''
+    def __init__(self, txt, timeout=1e7):
+        super().__init__(timeout=timeout)
+        self.str = txt
+        self.captures = []
+        self.refs = {}
+
+    def __getitem__(self, key):
+        return self.str[key]
+
+    def __call__(self):
+        return self.str
+
+    def _ensure(self, index):
+        if index >= len(self.captures):
+            #self.captures.extend([""] * (index - len(self.captures) + 1))
+            self.captures += [[[]] for i in range(index - len(self.captures) + 1)]
+
+    def capture(self, index, txt):
+        self._ensure(index)
+        if isinstance(txt, (list, tuple)):
+            txt = "".join(txt)
+        logger.debug(f"Capture[{index}][{len(self.captures[index])-1}] = '{txt}'")
+        self.captures[index][-1] = txt
+
+    def init(self, index, txt):
+        self._ensure(index)
+        logger.debug(f"Init capture[{index}][{len(self.captures[index])}]")
+        self.captures[index].append("")
+
+    def release(self, index):
+        self.captures[index].pop()
+        logger.debug(f"Pop capture[{index}][{len(self.captures[index])}]")
+
+    def getcapture(self, index):
+        logger.debug(f"Get capture[{index}][{len(self.captures[index])}] = '{self.captures[index][-1]}'")
+        return self.captures[index][-1]
+
+    def pop(self):
+        return self.captures.pop()
+
+    def getref(self, name):
+        return self.refs.get(name)
+
+
+class Group(usfmp.Group):
+
+    def _returnRes(self, res, s):
+        res = super()._returnRes(res, s)
+        if res is None:
+            return None
+        while res is not None and len(res) == 1:
+            if isinstance(res, (str, Attribute, Element)):
+                break
+            res = res[0]
+        final = self.result(res, self) if self.result is not None else res
+        if self.capture and final is not None:
+            s.gs.capture(self.capture, final)
+        return final
+
+
+class String(usfmp.Parser):
+    def __init__(self, reg, dump=False, **kw):
+        self.keep = not dump
+        def text(s):
+            m = regex.match(self.re, s.gs(), pos=s.pos)
+            if not m:
+                raise usfmp.NoParseError(f'String ({self.re}) not found', s)
+            return (m.group(1) if self.keep else None, s.extend(m.end()-s.pos))
+        super().__init__(text, **kw)
+        reg = re.sub(r"\\u([0-9a-fA-F]{4})", lambda m:chr(int(m.group(1), 16)), reg)
+        self.re = reg if dump else "(" + reg + ")"
+        self.name = "/" + reg + "/"
+
+    def __repr__(self):
+        return self.asstr()
+
+    def asstr(self, context=None):
+        return '/{}/{}'.format(self.re, "!" if self.keep else "")
+
+    def merge(self, other, mode=None):
+        if not isinstance(other, String):
+            return super().merge(other, mode=mode)
+        if mode and "|" in mode:
+            ore = other.re[1:-1] if other.re.startswith("(") and other.re.endswith(")") else other.re
+            if self.re.startswith("(") and self.re.endswith(")"):
+                self.re = self.re[:-1] + "|" + ore + ")"
+            else:
+                self.re = "(" + self.re + "|" + ore + ")"
+        elif self.keep and other.keep:
+            return super().merge(other, mode=mode)
+        else:
+            self.re += other.re
+        self.name = "/" + self.re + "/"
+        self.keep |= other.keep
+        return (self,)
+
+    def empty(self):
+        return len(self.name) == 0
+
+
+class Reference(usfmp.Parser):
+    def __init__(self, index, **kw):
+        self.backref = index
+        def test(s):
+            v = s.gs.getcapture(self.backref)
+            if s.gs().startswith(v, s.pos):
+                return(v if not kw.get('dump', False) else None, s.extend(len(v)))
+            else:
+                raise usfmp.NoParseError(f'String from backref (v) not found', s)
+        super().__init__(test, **kw)
+
+    def __repr__(self):
+        return "\u21D0{}".format(self.backref)
+
+    def asstr(self, context=None):
+        return "\u21D0{}".format(self.backref)
+
+
+@dataclass
+class Attribute:
+    name: str
+    value: str
+    override: str
+
+    def __init__(self, name, value, override, **kw):
+        self.override = override
+        if isinstance(value, list):
+            if len(value) > 1:
+                self.name, self.value = value
+                return
+            else:
+                value = value[0]
+        self.name = name
+        self.value = value
+
+    def __len__(self):
+        return 1
+
+
+class Element(list):
+    def __init__(self, *a, name=None, propmap=None, **kw):
+        self.name = name
+        self.attributes = {}
+        self.propmap = propmap
+        self.kw = kw
+        for e in a:
+            self._append(e)
+        if usfmp.Parser.debug:
+            logger.debug(f"Create Element({self})")
+
+    def _append(self, e):
+        if isinstance(e, Attribute):
+            if self.propmap is not None and e.override is not None and e.name == "_default":
+                self.attributes[self.propmap.get(self.attributes[e.override], e.name)] = e.value
+            else:
+                self.attributes[e.name] = e.value
+        elif isinstance(e, list) and not isinstance(e, Element):
+            for c in e:
+                self._append(c)
+        else:
+            self.append(e)
+
+    def asEt(self, parent=None, elfactory=None):
+        if elfactory is None:
+            elfactory = ParentElement
+        res = elfactory(self.name, attrib=self.attributes, parent=parent)
+        if parent is not None:
+            parent.append(res)
+        def asetchild(c, res):
+            if isinstance(c, str):
+                if len(res):
+                    res[-1].tail = c if res[-1].tail is None else res[-1].tail + c
+                else:
+                    res.text = c if res.text is None else res.text + c
+            elif isinstance(c, Element):
+                c.asEt(res, elfactory=elfactory)
+            elif isinstance(c, list):
+                for e in c:
+                    asetchild(e, res)
+        j = self[:]
+        self.clear()
+        for c in j:
+            self._append(c)
+        for c in self:
+            asetchild(c, res)
+        return res
+
+    def __str__(self):
+        return "{}<{}>{}".format(self.name, self.attributes, [str(x) for x in self])
+
+    def __repr__(self):
+        return str(self)
+
+
+def parseusfm(infilename, parser, timeout=1e7, isdata=True):
+    if isdata:
+        dat = infilename
+    elif hasattr(infilename, 'read'):
+        dat = infilename.read().decode("utf-8")
+    else:
+        with open(infilename, encoding="utf-8") as inf:
+            dat = inf.read()
+    gs = GlobalState(dat, timeout=timeout)
+    return parser.parse(usfmp.State(gs))
+
+escapes = {
+    '\\' : '\\',
+    'n': '\n'
+}
+
+def expandescape(s):
+    return re.sub(r'\\(.)', lambda m: escapes.get(m.group(1), "\\"+m.group(1)), s)
+
+class UsfmParserBackend:
+    _terminals = {
+        'text': lambda **kw : Text(**kw),
+        'ws': lambda **kw: String(r"\s", **kw),
+        'word': lambda **kw: Text(**kw)
+    }
+    _grouptypes = {
+        'sequence': "&",
+        'choice': "|",
+        'optional': "?",
+        'zeroOrMore': "*",
+        'oneOrMore': "+"
+    }
+
+    def __init__(self):
+        self.references = {}
+        self.elements = []
+        self.nodes = []
+        self.defines = {}
+        self.defstack = []
+        pass
+
+    def _trimTo(self, curr):
+        i = self.nodes.index(curr)
+        self.nodes[:] = self.nodes[:i]
+
+    def get_nodename(self):
+        res = {}
+        if len(self.nodes):
+            res['to'] = self.nodes[-1].name
+        return res
+
+# --- Parser backend API
+
+    def attrib_start(self, parser, e, context, name, **kw):
+        if name == "*":
+            name = "_default"
+        group = Group(name=name, parent=context, mode="&",
+                        result=(lambda r,s: Attribute(name, r, kw.get('fallback-from', None))),
+                        **self.get_nodename())
+        self.nodes.append(group)
+        return group
+
+    def attrib_end(self, parser, e, context, **kw):
+        self._trimTo(context)
+        return context
+
+    def elem_start(self, parser, e, context, name, **kw):
+        group = Group(name=name, parent=context, mode="&",
+                    result=(lambda r,s: Element(r, name=name, propmap=s.propmap.copy())),
+                    **self.get_nodename())
+        self.nodes.append(group)
+        return group
+
+    def elem_end(self, parser, e, context, **kw):
+        self._trimTo(context)
+        return context
+
+    def append_seq(self, context=None, to=None, **kw):
+        return Group(parent=context, mode="&", **self.get_nodename(), **kw)
+
+    def append_opt(self, context=None, to=None, **kw):
+        return Group(parent=context, mode="?", **self.get_nodename(), **kw)
+
+    def append_or(self, context=None, to=None, **kw):
+        return Group(parent=context, mode="|", **self.get_nodename(), **kw)
+
+    def append_plus(self, context=None, to=None, **kw):
+        bet = context.properties.get("between", None)
+        # handle "between"
+        return Group(parent=context, mode="+", **self.get_nodename(), **kw)
+
+    def append_star(self, context=None, to=None, **kw):
+        return Group(parent=context, mode="*", **self.get_nodename(), **kw)
+
+    def append_group(self, index, context=None, to=None, **kw):
+        group = Group(parent=context, mode="&", **self.get_nodename(), **kw)
+        return group
+
+    def append_interleave(self, context=None, to=None, **kw):
+        return Group(parent=context, mode="|+", **self.get_nodename(), **kw)
+
+    def match(self, name, context, dump=False, to=None, **kw):
+        if not len(name):
+            return context
+        if name[0] in "\"'":
+            res = String(regex.escape(expandescape(name[1:-1])), parent=context, dump=dump, **self.get_nodename(), **kw)
+        elif name.startswith("/"):
+            res = String(name[1:-1], parent=context, dump=dump, **self.get_nodename(), **kw)
+        else:
+            res = self._terminals.get(name.lower(), lambda **kw: usfmp.Skip(**kw))(parent=context, dump=dump, **self.get_nodename(), **kw)
+        context.append(res)
+        return context
+
+    def terminal(self, name, context, to=None, **kw):
+        return self.match(name, context, dump=True, **self.get_nodename(), **kw)
+
+    def backref(self, index, context=None, to=None, **kw):
+        res = Reference(index, parent=context, **self.get_nodename(), **kw)
+        context.append(res)
+        return context
+
+    def add_terminal(self, name, value, context, **kw):
+        if value[0] in "\"'":
+            res = lambda **kw: String(regex.escape(value[1:-1]), **kw)
+        elif value[0].startswith("/"):
+            res = lambda **kw: String(value[1:-1], **kw)
+        self._terminals[name.lower()] = res
+
+    def add_define(self, name, e, context=None, **kw):
+        res = self.append_seq(context, inref=name)
+        self.defines[name] = res
+        return res
+
+    def ref(self, name, context, **kw):
+        res = self.defines[name]
+        context.append(res)
+        return res
+