usfmtc 0.2__py3-none-any.whl

usfmtc/__init__.py

@@ -0,0 +1,379 @@
+#!/usr/bin/env python3
+
+# nuitka configuration
+# nuitka-project: --onefile
+# nuitka-project: --include-data-files={MAIN_DIRECTORY}/usx.rng=usx.rng
+# nuitka-project-if: {OS} in ("Windows",):
+#     nuitka-project: --output-filename=usfmconv.exe
+# nuitka-project-else:
+#     nuitka-project: --output-filename=usfmconv.bin
+
+import os
+from usfmtc.sfmparser import parseusfm, UsfmParserBackend
+from usfmtc.parser import NoParseError
+from usfmtc.extension import Extensions
+from usfmtc.xmlutils import ParentElement, prettyxml, writexml
+from usfmtc.usxparser import USXConverter
+from usfmtc.grammar import UsfmGrammarParser
+from usfmtc.diagrams import UsfmRailRoad
+from usfmtc.usxmodel import addesids, cleanup, messup, canonicalise
+from usfmtc.usjproc import usxtousj, usjtousx
+from usfmtc.usfmparser import USFMParser, Grammar
+from usfmtc.usfmgenerate import usx2usfm
+import xml.etree.ElementTree as et
+
+def _readsrc(src):
+    if hasattr(src, "read"):
+        return src.read()
+    elif not isinstance(src, str):      # we're a parsed xml doc
+        return src
+    elif len(src) < 128 and os.path.exists(src):
+        with open(src, encoding="utf-8") as inf:
+            data = inf.read()
+        return data
+    elif "\n" in src or len(src) > 127:
+        return src
+    else:
+        raise FileNotFoundError(src)
+
+def _grammarDoc(gsrc, extensions=[], factory=et):
+    data = _readsrc(gsrc)
+    if isinstance(data, (str, bytes)):
+        rdoc = factory.ElementTree(factory.fromstring(data))
+    else:
+        rdoc = data
+    for ef in extensions:
+        e = Extensions(ef)
+        dirty = e.applyto(rdoc, factory=factory)
+    return rdoc
+
+def _usfmGrammar(rdoc, backend=None, start=None):
+    if backend is None:
+        backend = UsfmParserBackend()
+    sfmproc = UsfmGrammarParser(rdoc, backend)
+    if start is None:
+        start = "Scripture"
+    parser = sfmproc.parseRef(start)
+    return parser
+
+def usfmGrammar(gsrc, extensions=[], altparser=True, backend=None, start=None):
+    """ Create UsfmGrammarParser from gsrc as used by USX.fromUsfm """
+    if len(extensions) or not altparser:
+        rdoc = _grammarDoc(gsrc, extensions)
+    if altparser:
+        res = Grammar()
+        if len(extensions):
+            for e in extensions:
+                res.readmrkrs(e)
+        return res
+    return _usfmGrammar(rdoc, backend, start)
+
+
+_filetypes = {".xml": "usx", ".usx": "usx", ".usfm": "usfm", ".sfm": "usfm3.0", ".json": "usj"}
+
+def readFile(infpath, informat=None, gramfile=None, grammar=None, extfiles=[], altparser=True):
+    """ Reads a USFM file of a given type or inferred from the filename
+        extension. extfiles allows for extra markers.ext files to extend the grammar"""
+    if informat is None:
+        inroot, ext = os.path.splitext(infpath)
+        intype = _filetypes.get(ext.lower(), informat)
+    else:
+        intype = informat
+    if intype is None:
+        return None
+
+    if intype == "usx":
+        usxdoc = USX.fromUsx(infpath)
+    elif intype == "usj":
+        usxdoc = USX.fromUsj(infpath)
+    elif intype.startswith("usfm"):
+        if grammar is None:
+            if gramfile is None:
+                for a in ([], ['..', '..', '..', 'grammar']):
+                    gramfile = os.path.join(os.path.dirname(__file__), *a, "usx.rng")
+                    if os.path.exists(gramfile):
+                        break
+            fname = getattr(infpath, 'name', infpath)
+            extfiles.append(os.path.join(os.path.dirname(fname), "markers.ext"))
+            exts = [x for x in extfiles if os.path.exists(x)]
+            grammar = usfmGrammar(gramfile, extensions=exts, altparser=altparser)
+        usxdoc = USX.fromUsfm(infpath, grammar, altparser=altparser)
+    return usxdoc
+
+
+class USX:
+    @classmethod
+    def fromUsx(cls, src, elfactory=None):
+        """ Loads USX and creates USX object to hold it """
+        if elfactory is None:
+            elfactory = ParentElement
+        tb = et.TreeBuilder(element_factory=elfactory)
+        parser = et.XMLParser(target=tb)
+        if os.path.exists(src):
+            inf = open(src, encoding="utf_8_sig")
+        else:
+            inf = src
+        if hasattr(inf, "read"):
+            anet = et.ElementTree()
+            anet.parse(inf, parser=parser)
+            res = anet.getroot()
+            if src != inf:
+                inf.close()
+        else:
+            try:
+                res = et.fromstring(src, parser=parser)
+            except et.ParseError:
+                return None
+        return cls(res)
+
+    @classmethod
+    def fromUsfm(cls, src, grammar, altparser=True, elfactory=None, timeout=1e7):
+        """ Parses USFM using UsfmGrammarParser grammar and creates USX object.
+            Raise usfmtc.parser.NoParseError on error. """
+        data = _readsrc(src)
+
+        if altparser:
+            p = USFMParser(data, factory=elfactory or ParentElement, grammar=grammar)
+            xml = p.parse()
+        else:
+        # This can raise usfmtc.parser.NoParseError
+            result = parseusfm(data, grammar, timeout=timeout, isdata=True)
+            xml = result.asEt(elfactory=elfactory)
+
+        cleanup(xml)            # normalize space, de-escape chars, cell aligns, etc.
+        res = cls(xml)
+        return res
+
+    @classmethod
+    def fromUsj(cls, src, elfactory=None):
+        data = _readsrc(src)
+        djson = json.loads(data)
+        xml = usjtousx(djson, elfactory=elfactory)
+        return cls(xml)
+
+    def __init__(self, xml):
+        self.xml = xml      # an Element, not an ElementTree
+
+    def _outwrite(self, file, dat, fn=None):
+        if fn is None:
+            fn = lambda f, d: f.write(d)
+        if file is None:
+            fh = io.StringIO()
+            fn(fh, dat)
+            res = fh.getvalue()
+            fh.close()
+            return res
+        if not hasattr(file, "read"):
+            fh = open(file, "w", encoding="utf-8")
+            fn(fh, dat)
+            fh.close()
+        else:
+            fn(file, dat)
+        return True
+
+    def outUsx(self, file=None):
+        """ Output pretty XML USX. If file is None returns string """
+        if self.xml is None:
+            return None
+        prettyxml(self.xml)
+        self._outwrite(file, self.xml, fn=writexml)
+
+    def outUsfm(self, grammar, file=None, altparser=True, **kw):
+        """ Output USFM from USX object. grammar is et doc. If file is None returns string """
+        el = messup(self.xml)
+        if altparser:
+            return self._outwrite(file, el, fn=usx2usfm)
+        parser = USXConverter(grammar.getroot(), **kw)
+        res = parser.parse(el)
+        if res:
+            dat = "".join(res.results)
+            return self._outwrite(file, dat)
+        return False
+
+    def outUsj(self, file=None):
+        """ Output USJ from USX object. If file is None returns dict """
+        res, _ = usxtousj(self.xml.getroot())
+        if file is None:
+            return res
+        else:
+            dat = json.dumps(res, indent=2)
+            self._outwrite(file, dat)
+
+    def saveAs(self, outfpath, outformat=None, addesids=False, grammar=None, gramfile=None, version=None, altparser=True):
+        """ Saves the document to a file in the appropriate format, either given
+            or inferred from the filename extension. """
+        if outformat is None:
+            outroot, ext = os.path.splitext(outfpath)
+            outtype = _filetypes.get(ext.lower(), outformat)
+        else:
+            outtype = outformat
+        if outtype is None:
+            return
+
+        if outtype == "usx":
+            if addesids:
+                usxdoc.addesids()
+            self.outUsx(outfpath)
+        elif outtype == "usj":
+            self.outUsj(outfpath)
+        elif outtype == "usfm":
+            if grammar is None and not altparser:
+                if gramfile is None:
+                    for a in ([], ['..', '..', '..', 'grammar']):
+                        gramfile = os.path.join(os.path.dirname(__file__), *a, "usx.rng")
+                        if os.path.exists(gramfile):
+                            break
+                grammar = _grammarDoc(gramfile)
+            if outtype == "usfm3.0":
+                outtype = "usfm"
+                if version is None:
+                    version = "3.0"
+            self.outUsfm(grammar, outfpath, outversion=version, altparser=altparser)
+
+    def canonicalise(self):
+        canonicalise(self.getroot())
+
+    def getroot(self):
+        """ Returns root XML element """
+        return self.xml
+
+    def addesids(self):
+        """ Add esids to USX object (eid, sids, vids) """
+        addesids(self.xml)
+
+    @property
+    def version(self):
+        return self.getroot().get('version', None)
+
+    @version.setter
+    def version(self, version):
+        self.getroot().set('version', str(version))
+
+def main(hookcli=None, hookusx=None):
+
+    import argparse, logging, sys
+    from glob import glob
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument("infile",nargs="+",help="Input file")
+    parser.add_argument("-o", "--outfile",help="Output file, with inferred format")
+    parser.add_argument("-F","--outformat",help="Output format [usfm, usx, usj, usfm3.0]")
+    parser.add_argument("-I","--informat",help="Input format [usfm, usx, usj]")
+    parser.add_argument("-g","--grammar",help="Grammar file to use, if needed")
+    parser.add_argument("-e","--esids",action="store_true",help="Add esids, vids, sids, etc. to USX output")
+    parser.add_argument("-v","--version",default=None,help="Set USFM version [3.1]")
+    parser.add_argument("-x","--extfiles",action="append",default=[],help="markers.ext files to include")
+    parser.add_argument("--qusfm",action="store_true",help="Use quick nonvalidating USFM parser")
+    parser.add_argument("-C","--canonical",action="store_true",help="Do not canonicalise")
+    parser.add_argument("-l","--logging",help="Set logging level to usfmxtest.log")
+    parser.add_argument("-q","--quiet",action="store_true",help="Don't say much")
+    parser.add_argument("--nooutput",action="store_true",help="Don't output any data")
+    if hookcli is not None:
+        hookcli(parser)
+    args = parser.parse_args()
+
+    if args.logging:
+        try:
+            loglevel = int(args.logging)
+        except ValueError:
+            loglevel = getattr(logging, args.logging.upper(), None)
+        if isinstance(loglevel, int):
+            parms = {'level':  loglevel, 'datefmt': '%d/%b/%Y %H:%M:%S',
+                     'format': '%(asctime)s.%(msecs)03d %(levelname)s:%(module)s(%(lineno)d) %(message)s'}
+            logfh = open("usfmconv.log", "w", encoding="utf-8")
+            parms.update(stream=logfh, filemode="w") #, encoding="utf-8")
+            try:
+                logging.basicConfig(**parms)
+            except FileNotFoundError as e:      # no write access to the log
+                print("Exception", e)
+        log = logging.getLogger('usfmconv')
+    else:
+        log = None
+
+    def doerror(msg, doexit=True):
+        if log:
+            log.error(msg)
+        if not args.quiet:
+            print(msg)
+        if doexit:
+            sys.exit(1)
+    
+    fileexts = {"usx": ".xml", "usfm": ".usfm", "usj": ".json"}
+
+    def _makeoutfile(infile, oformat):
+        outext = fileexts.get(oformat, None)
+        inroot, ext = os.path.splitext(infile)
+        return inroot + outext if outext else None
+    
+    infiles = sum((glob(x) for x in args.infile), [])
+    if not len(infiles):
+        doerror("No files found in {args.infile}")
+
+    root, ext = os.path.splitext(infiles[0])
+    args.informat = args.informat or _filetypes.get(ext.lower(), args.informat)
+    if args.outformat is None:
+        if args.outfile is not None and not os.path.isdir(args.outfile):
+            root, ext = os.path.splitext(args.outfile)
+            args.outformat = _filetypes.get(ext.lower(), None)
+    ingrammar = None
+    outgrammar = None
+    if args.informat.startswith("usfm") or args.outformat.startswith("usfm"):
+        if args.grammar is None:
+            for a in ([], ['..', '..', '..', 'grammar']):
+                args.grammar = os.path.join(os.path.dirname(__file__), *a, "usx.rng")
+                if os.path.exists(args.grammar):
+                    break
+        if args.informat.startswith("usfm"):
+            args.extfiles.append(os.path.join(os.path.dirname(infiles[0]), "markers.ext"))
+            exts = [x for x in args.extfiles if os.path.exists(x)]
+            ingrammar = usfmGrammar(args.grammar, altparser=args.qusfm, extensions=exts)
+        if args.outformat and args.outformat.startswith("usfm"):
+            outgrammar = _grammarDoc(args.grammar)
+
+    for infile in infiles:
+        outfile = None
+        if args.outfile is None:
+            outfile = _makeoutfile(infile, args.format)
+        elif os.path.isdir(args.outfile):
+            outf = _makeoutfile(infile, args.format)
+            if outf is None:
+                doerror(f"invalid output format {args.format} in {args.outfile}")
+            outfile = os.path.join(args.outfile, os.path.basename(outf))
+        elif len(infiles) == 1:
+            outfile = args.outfile
+
+        if infile == "-":
+            infile = sys.stdin
+        if outfile == "-":
+            outfile = sys.stdout
+
+        usxdoc = None
+        if not args.quiet:
+            print(f"{infile} -> {outfile}" if outfile else f"{infile}")
+        try:
+            usxdoc = readFile(infile, informat=args.informat, grammar=ingrammar, altparser=args.qusfm)
+        except NoParseError as e:
+            doerror(f"Failed to parse {infile}: {e}", False)
+
+        if len(infiles) == 1 and usxdoc is None:
+            doerror(f"Unable to read in {args.infile}")
+
+        if hookusx is not None:
+            hookusx(usxdoc, args)
+
+        if args.nooutput or outfile is None or usxdoc is None:
+            continue
+
+        version = usxdoc.version
+        if version is None:
+            usxdoc.version = args.version or "3.1"
+        elif args.version is not None:
+            usxdoc.version = args.version
+
+        if not args.canonical:
+            usxdoc.canonicalise()
+
+        usxdoc.saveAs(outfile, outformat=args.outformat, addesids=args.esids, grammar=outgrammar, altparser=args.qusfm)
+
+if __name__ == "__main__":
+    main()

usfmtc/diagrams.py

@@ -0,0 +1,247 @@
+import usfmtc.railroad as rr
+
+usfmns = "{http://usfm.bible/parse/2023}"
+
+CSS_STYLE = '''
+    svg.railroad-diagram {{
+        background-color: hsl(30,20%,95%);
+    }}
+    svg.railroad-diagram path {{
+        stroke-width: {line-width};
+        stroke: black;
+        fill: none;
+    }}
+    svg.railroad-diagram text {{
+        font-size: {text-size};
+        text-anchor: middle;
+        font-family: {font};
+    }}
+    svg.railroad-diagram text.label {{
+        text-anchor:start;
+    }}
+    svg.railroad-diagram text.comment {{
+        font-family: {font};
+        font-size: {comment-size};
+    }}
+    svg.railroad-diagram rect{{
+        stroke-width: {line-width};
+        stroke: black;
+        fill: {color};
+    }}
+    svg.railroad-diagram rect.group-box {{
+        stroke: gray;
+        stroke-dasharray: 10 5;
+        fill: none;
+    }}
+    .terminal {{
+        font-family: {font};
+    }}
+    svg.railroad-diagram g.altterminal rect {{
+        fill: {altcolor}
+    }}
+'''
+
+class UsfmRailRoad:
+    def __init__(self):
+        self.terminals = {}
+        self.defines = {}
+
+    class WrappedRail:
+        def __init__(self, content, parent):
+            self.rtype = content
+            self.parent = parent
+            self.properties = {}
+        def asRail(self):
+            return self.rtype
+        def getProperty(self, k, default):
+            return self.properties.get(k, default)
+        def __str__(self):
+            return f"WrappedRail({self.rtype})"
+
+    class WrappedRailList(list):
+        def __init__(self, rtype, parent, *args, swap=False, forced=False, **kw):
+            self.properties = {}
+            self.rtype = rtype
+            self.parent = parent
+            self.args = list(args)
+            self.swap = swap
+            self.forced = forced
+            self.kw = kw
+            self.propmap = {}
+
+        def asRail(self):
+            content = []
+            for x in self:
+                y = x.asRail()
+                if y is not None:
+                    content.append(y)
+            if self.rtype == rr.Choice and self.kw.get('groupby', 0) > 0:
+                s = []
+                for i in range(0, len(content), self.kw['groupby']):
+                    s.append(self.rtype(3 if len(content) - i > 3 else 0,
+                                *content[i:i+self.kw['groupby']], **self.kw))
+                return rr.HorizontalChoice(*s, **self.kw)
+            if self.swap:
+                return self.rtype(*(content + list(self.args)), **self.kw) if len(content) else None
+            else:
+                return self.rtype(*(list(self.args) + content), **self.kw) if len(content) else None
+        def getProperty(self, k, default):
+            return self.properties.get(k, default)
+        def __str__(self):
+            return f"WrappedRailList({self.rtype}{self.args})[" + ", ".join(str(x) for x in self) + "]"
+
+    def output(self, top, **kw):
+        defaults = {
+            'font': 'DejaVu Sans',
+            'text-size': '12px',
+            'comment-size': '14px',
+            'color': 'rgb(210, 255, 210)',
+            'altcolor': 'rgb(251, 252, 207)',
+            'line-width': '2'}
+        defaults.update(kw)
+        return rr.Diagram(top.asRail(), type='complex', css=CSS_STYLE.format(**defaults))
+
+    def attrib_start(self, parser, e, context, name, **kw):
+        res = self.append_seq(context, stacked=e.get("{}stacked".format(usfmns), "false") in ("true", "1"))
+        res.name = name
+        parser.push_element(res)
+        return res
+
+    def attrib_end(self, parser, e, context, **kw):
+        parser.pop_element()
+        return context
+
+    def elem_start(self, parser, e, context, name, **kw):
+        res = self.append_seq(context, stacked=e.get("{}stacked".format(usfmns), "false") in ("true", "1"))
+        res.name = name
+        parser.push_element(res)
+        return res
+
+    def elem_end(self, parser, e, context, **kw):
+        parser.pop_element()
+        return context
+
+    def append_type(self, context, tclass, *a, swap=False, forced=False, **kw):
+        if context is not None and not forced \
+                and (context.rtype == tclass or context.rtype == rr.Stack and tclass == rr.Sequence):
+            return context
+        else:
+            res = self.WrappedRailList(tclass, context, *a, swap=swap, forced=forced, **kw)
+            if context is not None:
+                context.append(res)
+            return res
+
+    def append_seq(self, context=None, stacked=False, forced=False, **kw):
+        return self.append_type(context, rr.Stack if stacked else rr.Sequence, forced=forced)
+
+    def append_opt(self, context=None, **kw):
+        return self.append_type(context, rr.Choice, 0, rr.Skip())
+
+    def append_or(self, context=None, groupby=0, **kw):
+        return self.append_type(context, rr.Choice, 0, groupby=groupby)
+
+    def append_plus(self, context=None, **kw):
+        bet = context.getProperty("between", None)
+        if bet is not None:
+            rep = rr.Terminal(bet)
+            res = self.append_type(context, rr.OneOrMore, rep, swap=True)
+        else:
+            res = self.append_type(context, rr.OneOrMore, swap=True)
+        return res
+
+    def append_star(self, context=None, **kw):
+        res = self.WrappedRailList(rr.ZeroOrMore, context)   # returns Choice(OneOrMore)
+        context.append(res)
+        return res
+
+    def append_group(self, index, context=None, **kw):
+        label = chr(0x278A + index)
+        res = self.WrappedRailList(rr.Group, context, label, swap=True)
+        context.append(res)
+        return res
+
+    def append_interleave(self, context=None, **kw):
+        def inter(repeat, *items):
+            return rr.OneOrMore(rr.MultipleChoice(0, "any", *items), repeat)
+        bet = context.getProperty("interleave_between", None)
+        rep = rr.Terminal(bet) if bet is not None else None
+        res = self.WrappedRailList(inter, context, bet)
+        context.append(res)
+        return res
+
+    def match(self, txt, context, alt=None, **kw):
+        cls = {k: kw.get(k, None) for k in ('href',)}
+        if alt is None or not len(alt):
+            lcontext = context
+        else:
+            lcontext = self.append_or(context)
+            lcontext.append(self.WrappedRail(rr.Terminal(alt, **cls), lcontext))
+            cls["cls"] = "altterminal"
+        lcontext.append(self.WrappedRail(rr.Terminal(txt, **cls), lcontext))
+        return context
+
+    def terminal(self, name,  context, **kw):
+        context.append(self.WrappedRail(rr.NonTerminal(name), context))
+        return context
+
+    def backref(self, index, context=None, **kw):
+        context.append(self.WrappedRail(rr.Terminal("\u21D0 {}".format(chr(0x278A + index))), context))
+        return context
+
+    def add_terminal(self, name, value, context, **kw):
+        self.terminals[name] = value
+
+    def add_define(self, name, e, context=None, box=False, **kw):
+        if box:
+            bres = self.WrappedRailList(rr.Group, context, name, swap=True)
+            context.append(bres)
+            res = self.append_seq(context=bres)
+        else:
+            res = self.append_seq(context=context)
+        self.defines[name] = res
+        return res
+
+    def ref(self, name, res, **kw):
+        return self.terminal(name, res)
+
+    def end_ref(self, e, name, **kw):
+        pass
+
+
+class XmlRailRoad(UsfmRailRoad):
+
+    def elem_start(self, parser, e, context, name, **kw):
+        seq = self.append_type(context, rr.Sequence, forced=True)
+        context = self.match(f"<{name}>", seq)
+        res = self.append_type(seq, rr.Choice, 0, forced=True, noclose=1)
+        res.attribs = {}
+        return res
+
+    def elem_end(self, parser, e, context, **kw):
+        if context.args[0] == len(context):
+            context.append(self.WrappedRail(rr.Skip(), context))
+        elif context.args[0] > len(context):
+            print(f"{context} has index of {context.args[0]} but length of {len(context)}")
+            context.args[0] = len(context) - 1
+        return context
+
+    def attrib_start(self, parser, e, context, name, **kw):
+        parent = context
+        while parent is not None and parent.kw.get('noclose', 0) == 0:
+            parent = parent.parent
+        s = self.WrappedRailList(rr.Sequence, parent)
+        n = self.WrappedRail(rr.Terminal(f"@{name}"), s)
+        s.append(n)
+        if parent is None:
+            context.append(s)
+        elif getattr(parent, 'attribs', {}).get(name, None) is None:
+            parent.insert(0, s)
+            parent.args[0] += 1
+            parent.attribs[name] = 1
+        return s
+
+    def attrib_end(self, parser, e, context, **kw):
+        if context[-1].rtype in (rr.Choice, rr.HorizontalChoice):
+            context[-1].kw['noclose'] = 7
+        return context
+

usfmtc/extension.py

@@ -0,0 +1,79 @@
+#!/usr/bin/env python3
+
+import re
+import xml.etree.ElementTree as et
+
+categories = {
+    "char":         "Char.char",
+    "crossreference":   "CrossReference",
+    "crossreferencechar":   "CrossReferenceChar.char",
+    "footnote":     "Footnote",
+    "footnotechar": "FootnoteChar.char",
+    "header":       "Header.para",
+    "internal":     "Internal",
+    "introchar":    "IntroChar.char",
+    "introduction": "Introduction.para",
+    "list":         "List.para",
+    "listchar":     "ListChar.char",
+    "milestone":    "Milestone",
+    "otherpara":    "OtherPara.para",
+    "sectionpara":  "SectionPara.para",
+    "title":        "Title.para",
+    "versepara":    "VersePara.para",
+}
+
+
+class SFMFile:
+    def __init__(self, fname):
+        self.fname = fname
+        self.markers = {}
+        self.parse()
+
+    def parse(self):
+        with open(self.fname, encoding="utf-8") as inf:
+            for l in inf.readlines():
+                s = l.strip()
+                s = re.sub(r'#.*$', '', s)
+                if not s.startswith("\\"):
+                    continue
+                b = s[1:].split(None, 1)
+                b = self.preproc(b)
+                mk = b[0].lower()
+                if mk == "marker":
+                    curr = {}
+                    self.markers[b[1]] = curr
+                else:
+                    curr[mk] = b[1]
+
+    def preproc(self, b):
+        return b
+
+relaxns = "{http://relaxng.org/ns/structure/1.0}"
+
+class Extensions(SFMFile):
+    def preproc(self, b):
+        if b[0].lower() == "category":
+            b[1] = b[1].lower()
+        return b
+
+    def applyto(self, rdoc, factory=et):
+        res = False     # nothing added
+        allcats = set([x.get('category', None) for x in self.markers.values()])
+        for a in allcats:
+            if a not in categories:
+                continue
+            ms = [k for k, v in self.markers.items() if v.get('category', None) == a]
+            enum = categories[a]+".style.enum"
+            e = rdoc.find('./{0}define[@name="{1}"]/{0}choice'.format(relaxns, enum))
+            allmks = set([v.text for v in e.findall(f'./{relaxns}value')])
+            added = False
+            for m in ms:
+                if m in allmks:
+                    continue
+                else:
+                    v = factory.Element(f'{relaxns}value')
+                    v.text = m
+                    e.insert(0, v)
+                    added = True
+            res = res or added
+        return res