usfmtc 0.2.2__tar.gz → 0.2.4__tar.gz

{usfmtc-0.2.2/python/lib/usfmtc.egg-info → usfmtc-0.2.4}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.2
 Name: usfmtc
-Version: 0.2.2
+Version: 0.2.4
 Summary: USFM reference parser
 Author-email: USFM Technical Committee <fonts@sil.org>
 License: Copyright © 2022-2023 by the members of the USFM/USX technical committee, listed below:

{usfmtc-0.2.2 → usfmtc-0.2.4}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "usfmtc"
-version = "0.2.2"
+version = "0.2.4"
 authors = [{name = "USFM Technical Committee", email = "fonts@sil.org"}]
 readme = "python/README.md"
 license = {file = "LICENSE.md"}

{usfmtc-0.2.2 → usfmtc-0.2.4}/python/lib/usfmtc/__init__.py

@@ -58,15 +58,15 @@ def _usfmGrammar(rdoc, backend=None, start=None):
 
 def usfmGrammar(gsrc, extensions=[], altparser=True, backend=None, start=None):
     """ Create UsfmGrammarParser from gsrc as used by USX.fromUsfm """
-    if len(extensions) or not altparser:
-        rdoc = _grammarDoc(gsrc, extensions)
     if altparser:
+        rdoc = _grammarDoc(gsrc, extensions)
+        return _usfmGrammar(rdoc, backend, start)
+    else:
         res = Grammar()
         if len(extensions):
             for e in extensions:
                 res.readmrkrs(e)
         return res
-    return _usfmGrammar(rdoc, backend, start)
 
 
 _filetypes = {".xml": "usx", ".usx": "usx", ".usfm": "usfm", ".sfm": "usfm3.0", ".json": "usj"}
@@ -129,10 +129,12 @@ class USX:
     @classmethod
     def fromUsfm(cls, src, grammar, altparser=True, elfactory=None, timeout=1e7):
         """ Parses USFM using UsfmGrammarParser grammar and creates USX object.
-            Raise usfmtc.parser.NoParseError on error. """
+            Raise usfmtc.parser.NoParseError on error.
+            elfactory must take parent and pos named parameters not as attributes
+        """
         data = _readsrc(src)
 
-        if altparser:
+        if not altparser:
             p = USFMParser(data, factory=elfactory or ParentElement, grammar=grammar)
             xml = p.parse()
         else:
@@ -171,7 +173,7 @@ class USX:
             fn(file, dat)
         return True
 
-    def outUsx(self, file=None):
+    def outUsx(self, file=None, **kw):
         """ Output pretty XML USX. If file is None returns string """
         if self.xml is None:
             return None
@@ -181,7 +183,7 @@ class USX:
     def outUsfm(self, grammar, file=None, altparser=True, **kw):
         """ Output USFM from USX object. grammar is et doc. If file is None returns string """
         el = messup(self.xml)
-        if altparser:
+        if not altparser:
             return self._outwrite(file, el, fn=usx2usfm)
         parser = USXConverter(grammar.getroot(), **kw)
         res = parser.parse(el)
@@ -190,16 +192,17 @@ class USX:
             return self._outwrite(file, dat)
         return False
 
-    def outUsj(self, file=None):
+    def outUsj(self, file=None, ensure_ascii=False, **kw):
         """ Output USJ from USX object. If file is None returns dict """
         res = usxtousj(self.xml)
         if file is None:
             return res
         else:
-            dat = json.dumps(res, indent=2)
+            dat = json.dumps(res, indent=2, ensure_ascii=ensure_ascii)
             self._outwrite(file, dat)
 
-    def saveAs(self, outfpath, outformat=None, addesids=False, grammar=None, gramfile=None, version=None, altparser=True):
+    def saveAs(self, outfpath, outformat=None, addesids=False, grammar=None,
+                gramfile=None, version=None, altparser=True, **kw):
         """ Saves the document to a file in the appropriate format, either given
             or inferred from the filename extension. """
         if outformat is None:
@@ -213,11 +216,11 @@ class USX:
         if outtype == "usx":
             if addesids:
                 usxdoc.addesids()
-            self.outUsx(outfpath)
+            self.outUsx(outfpath, **kw)
         elif outtype == "usj":
-            self.outUsj(outfpath)
+            self.outUsj(outfpath, **kw)
         elif outtype == "usfm":
-            if grammar is None and not altparser:
+            if grammar is None and altparser:
                 if gramfile is None:
                     for a in ([], ['..', '..', '..', 'grammar']):
                         gramfile = os.path.join(os.path.dirname(__file__), *a, "usx.rng")
@@ -228,7 +231,7 @@ class USX:
                 outtype = "usfm"
                 if version is None:
                     version = "3.0"
-            self.outUsfm(grammar, outfpath, outversion=version, altparser=altparser)
+            self.outUsfm(grammar, outfpath, outversion=version, altparser=altparser, **kw)
 
     def canonicalise(self):
         canonicalise(self.getroot())
@@ -260,11 +263,14 @@ def main(hookcli=None, hookusx=None):
     parser.add_argument("-F","--outformat",help="Output format [usfm, usx, usj, usfm3.0]")
     parser.add_argument("-I","--informat",help="Input format [usfm, usx, usj]")
     parser.add_argument("-g","--grammar",help="Grammar file to use, if needed")
-    parser.add_argument("-e","--esids",action="store_true",help="Add esids, vids, sids, etc. to USX output")
+    parser.add_argument("-e","--esids",action="store_true",
+                        help="Add esids, vids, sids, etc. to USX output")
     parser.add_argument("-v","--version",default=None,help="Set USFM version [3.1]")
-    parser.add_argument("-x","--extfiles",action="append",default=[],help="markers.ext files to include")
-    parser.add_argument("--qusfm",action="store_true",help="Use quick nonvalidating USFM parser")
+    parser.add_argument("-x","--extfiles",action="append",default=[],
+                        help="markers.ext files to include")
+    parser.add_argument("-V","--validate",action="store_true",help="Use validating parsers for USFM")
     parser.add_argument("-C","--canonical",action="store_true",help="Do not canonicalise")
+    parser.add_argument("-A","--ascii",action="store_true",help="Output as ASCII only in json")
     parser.add_argument("-l","--logging",help="Set logging level to usfmxtest.log")
     parser.add_argument("-q","--quiet",action="store_true",help="Don't say much")
     parser.add_argument("--nooutput",action="store_true",help="Don't output any data")
@@ -326,16 +332,16 @@ def main(hookcli=None, hookusx=None):
         if args.informat.startswith("usfm"):
             args.extfiles.append(os.path.join(os.path.dirname(infiles[0]), "markers.ext"))
             exts = [x for x in args.extfiles if os.path.exists(x)]
-            ingrammar = usfmGrammar(args.grammar, altparser=args.qusfm, extensions=exts)
+            ingrammar = usfmGrammar(args.grammar, altparser=args.validate, extensions=exts)
         if args.outformat and args.outformat.startswith("usfm"):
             outgrammar = _grammarDoc(args.grammar)
 
     for infile in infiles:
         outfile = None
         if args.outfile is None:
-            outfile = _makeoutfile(infile, args.format)
+            outfile = _makeoutfile(infile, args.outformat)
         elif os.path.isdir(args.outfile):
-            outf = _makeoutfile(infile, args.format)
+            outf = _makeoutfile(infile, args.outformat)
             if outf is None:
                 doerror(f"invalid output format {args.format} in {args.outfile}")
             outfile = os.path.join(args.outfile, os.path.basename(outf))
@@ -351,7 +357,8 @@ def main(hookcli=None, hookusx=None):
         if not args.quiet:
             print(f"{infile} -> {outfile}" if outfile else f"{infile}")
         try:
-            usxdoc = readFile(infile, informat=args.informat, grammar=ingrammar, altparser=args.qusfm)
+            usxdoc = readFile(infile, informat=args.informat, grammar=ingrammar,
+                              altparser=args.validate)
         except NoParseError as e:
             doerror(f"Failed to parse {infile}: {e}", False)
 
@@ -373,7 +380,8 @@ def main(hookcli=None, hookusx=None):
         if not args.canonical:
             usxdoc.canonicalise()
 
-        usxdoc.saveAs(outfile, outformat=args.outformat, addesids=args.esids, grammar=outgrammar, altparser=args.qusfm)
+        usxdoc.saveAs(outfile, outformat=args.outformat, addesids=args.esids,
+                      grammar=outgrammar, altparser=args.validate, ensure_ascii=args.ascii)
 
 if __name__ == "__main__":
     main()

{usfmtc-0.2.2 → usfmtc-0.2.4}/python/lib/usfmtc/extension.py

@@ -58,11 +58,11 @@ class Extensions(SFMFile):
 
     def applyto(self, rdoc, factory=et):
         res = False     # nothing added
-        allcats = set([x.get('category', None) for x in self.markers.values()])
+        allcats = set([x.get('category', "").split()[0] for x in self.markers.values()])
         for a in allcats:
             if a not in categories:
                 continue
-            ms = [k for k, v in self.markers.items() if v.get('category', None) == a]
+            ms = [k for k, v in self.markers.items() if a in v.get('category', "").split()]
             enum = categories[a]+".style.enum"
             e = rdoc.find('./{0}define[@name="{1}"]/{0}choice'.format(relaxns, enum))
             allmks = set([v.text for v in e.findall(f'./{relaxns}value')])

{usfmtc-0.2.2 → usfmtc-0.2.4}/python/lib/usfmtc/usfmparser.py

@@ -5,8 +5,15 @@ import xml.etree.ElementTree as et
 from usfmtc.extension import SFMFile
 from collections import UserDict, UserString
 
+
+class Pos:
+    def __init__(self, l, c, **kw):
+        self.l = l
+        self.c = c
+        self.kw = kw
+
 class Tag(str):
-    def __new__(cls, s):
+    def __new__(cls, s, l=0, c=0, **kw):
         isend = False
         isplus = False
         if s.startswith("+"):
@@ -16,6 +23,7 @@ class Tag(str):
             isend = True
             s = s[:-1]
         res = super().__new__(cls, s)
+        res.kw = kw
         res.isplus = isplus
         res.isend = isend
         return res
@@ -23,7 +31,7 @@ class Tag(str):
     def __init__(self, s, l=0, c=0):
         self.isplus = getattr(self, 'isplus', False)
         self.isend = getattr(self, 'isend', False)
-        self.pos = (l, c)
+        self.pos = Pos(l, c)
 
     def __repr__(self):
         return "Tag("+str(self)+")"
@@ -41,14 +49,16 @@ class Tag(str):
         self.pos = pos
 
 class AttribText(UserString):
-    def __init__(self, s, l=0, c=0):
+    def __init__(self, s, l=0, c=0, **kw):
         super().__init__(s)
-        self.pos = (l, c)
+        self.pos = Pos(l, c)
+        self.kw = kw
     pass
 
 class OptBreak:
-    def __init__(self, l=0, c=0):
-        self.pos = (l, c)
+    def __init__(self, l=0, c=0, **kw):
+        self.pos = Pos(l, c)
+        self.kw = kw
 
     def __repr__(self):
         return "OptBreak(" + str(self) + ")"
@@ -56,9 +66,10 @@ class OptBreak:
         return "//"
 
 class String(UserString):
-    def __init__(self, s, l=0, c=0):
+    def __init__(self, s, l=0, c=0, **kw):
         super().__init__(s)
-        self.pos = (l, c)
+        self.pos = Pos(l, c)
+        self.kw = kw
 
     def __add__(self, s):
         return String(str(self) + s)
@@ -66,25 +77,28 @@ class String(UserString):
     pass
 
 class Attribs(UserDict):
-    def __init__(self, l=0, c=0):
+    def __init__(self, l=0, c=0, **kw):
         super().__init__()
-        self.pos = (l, c)
+        self.pos = Pos(l, c)
+        self.kw = kw
 
 class Lexer:
 
     tokenre = regex.compile(r'((?:[^\\|/]|\\[\\|\n]|/[^/])+)|([\\|\n]|//)')
-    tagre = regex.compile(r'(?:\+?[a-zA-Z_][a-zA-Z_0-9-]*)?\*?')
+    tagre = regex.compile(r'(?:\+?[a-zA-Z_][a-zA-Z_0-9^-]*)?\*?')
     attribsre = regex.compile(r'\s*([a-zA-Z_][a-zA-Z0-9_-]*)\s*=\s*"((?:\\"|[^"])*)"')
     textrunre = regex.compile(r'(?:[^\\]|\\[^a-zA-Z_*+])*')
 
-    def __init__(self, txt):
+    def __init__(self, txt, expanded=False):
         self.txt = txt
+        self.expanded = expanded
 
     def __iter__(self):
         self.nexts = []
         self.cindex = 0
         self.lindex = 0
         self.lpos = 0
+        self.currxpand = None
         return self
 
     def __next__(self):
@@ -113,7 +127,9 @@ class Lexer:
                     curri = m.end() + 1
                     continue
                 else:
-                    res = Tag(t.group(0))
+                    tagname = self.processtag(t.group(0))
+                    extras = {"xp": self.currxpand} if self.expanded else {}
+                    res = Tag(tagname, l=self.lindex, c=curri-self.lpos, **extras)
                     curri += t.end()
                     break
             elif n == '|':
@@ -155,6 +171,13 @@ class Lexer:
             self.cindex = m.end()
             self.nexts.append(String(m.group(1)))
 
+    def processtag(self, t):
+        i = t.find("^")
+        if self.expanded and i >= 0:
+            self.currxpand = t[i+1:]
+            return t[:i]
+        return t
+
 class Grammar:
     category_markers = {
         "attrib": "cp vp usfm ca va cat",
@@ -178,8 +201,14 @@ class Grammar:
         "title": "mt1 mt2 mt3 mt4 mt",
         "versepara": "cls nb pc pi1 pi2 pi3 pi po pr pmo pmc pmr pm ph1 ph2 ph3 ph p q1 q2 q3 q4 qc qr qm1 qm2 qm3 qm qd q b d mi1 mi2 mi3 mi4 mi m",
     }
+    category_tags = {
+        "char": ("char", "crossreferencechar", "footnotechar", "introchar", "listchar"),
+        "para": ("header", "introduction", "list", "otherpara", "sectionpara", "title", "versepara"),
+        "ms": ("milestone", )
+    }
 
-    marker_categories = {t:k for k, v in category_markers.items() for t in v.split(' ')}
+    marker_categories = {t:k for k, v in category_markers.items() for t in v.split()}
+    marker_tags = {t:k for k, v in category_tags.items() for t in v}
 
     attribmap = { 'jmp' : 'href', 'k' : 'key', 'qt-s': 'who', 'qt1-s': 'who', 'qt2-s': 'who',
         'qt3-s': 'who', 'qt4-s': 'who', 'qt5-s': 'who', 'rb': 'gloss', 't-s': 'sid', 'ts-s': 'sid',
@@ -211,13 +240,15 @@ def isfirstText(e):
     return True
 
 class Node:
-    def __init__(self, parser, usxtag, tag, ispara=False, notag=False):
+    def __init__(self, parser, usxtag, tag, ispara=False, notag=False, pos=None, **kw):
         self.parser = parser
         self.tag = tag
         self.ispara = ispara
         parent = parser.stack[-1] if len(parser.stack) else None
-        self.element = parser.factory(usxtag, {} if notag or tag is None else {"style": tag},
-                    parent=getattr(parent, 'element', None))
+        attribs = kw
+        if not notag and tag is not None:
+            attribs['style'] = tag
+        self.element = parser.factory(usxtag, attribs, parent=getattr(parent, 'element', None), pos=pos)
         if parent:
             parent.addNodeElement(self.element)
         self.attribnodes = []
@@ -227,7 +258,7 @@ class Node:
 
     def appendElement(self, child):
         if isinstance(child, OptBreak):
-            c = self.parser.factory("optbreak", {}, parent=self.element)
+            c = self.parser.factory("optbreak", {}, parent=self.element, pos=child.pos)
             self.element.append(c)
         self.clearAttribNodes()
 
@@ -297,11 +328,12 @@ class USXNode(Node):
         self.element.set('version', str(t).strip())
 
 class AttribNode(Node):
-    def __init__(self, parser, parent, tag):
+    def __init__(self, parser, parent, tag, pos=None, **kw):
         self.parser = parser
         self.parent = parent
         self.parent.addAttribNode(self)
         self.tag = tag
+        self.pos = pos
         self.attribnodes = []
 
     def addNodeElement(self, e):
@@ -313,8 +345,8 @@ class AttribNode(Node):
         self.parent.element.set(attrib, str(t).strip())
 
 class NumberNode(Node):
-    def __init__(self, parser, usxtag, tag, ispara=False):
-        super().__init__(parser, usxtag, tag, ispara=ispara)
+    def __init__(self, parser, usxtag, tag, ispara=False, pos=None, **kw):
+        super().__init__(parser, usxtag, tag, ispara=ispara, pos=pos, **kw)
         self.hasarg = False
 
     def appendText(self, t):
@@ -327,7 +359,7 @@ class NumberNode(Node):
         if len(b) > 1 and b[1].strip():
             self.parser.removeTag(self.tag)
             if self.ispara:
-                self.parser.addNode(Node(self.parser, 'para', 'p'))
+                self.parser.addNode(Node(self.parser, 'para', 'p', pos=self.element.pos))
             self.parser.stack[-1].appendText(b[1])
 
     def addNodeElement(self, e):
@@ -335,8 +367,8 @@ class NumberNode(Node):
         self.parser.stack[-1].addNodeElement(e)
 
 class NoteNode(Node):
-    def __init__(self, parser, usxtag, tag):
-        super().__init__(parser, usxtag, tag)
+    def __init__(self, parser, usxtag, tag, pos=None, **kw):
+        super().__init__(parser, usxtag, tag, pos=pos, **kw)
         self.hascaller = False
 
     def appendText(self, t):
@@ -358,38 +390,51 @@ class PeriphNode(Node):
         if t:
             self.element.set('alt', str(t).strip())
 
+class UnknownNode(Node):
+    pass
+
 
 paratypes = ('header', 'introduction', 'list', 'otherpara', 'sectionpara', 'versepara', 'title', 'chapter', 'ident')
 paratags = ('rem', ' table')
-def setupParser(cls):
-    # tag closed types
-    for a in (('introchar', 'char'), ('listchar', 'char'), ('char', 'char'), ('_fig', 'figure')):
-        def maketype(c, t):
-            def dotype(self, tag):
-                if tag.isend:
-                    return self.removeTag(str(tag))
-                return self.addNode(Node(self, t, tag.basestr()))
-            return dotype
-        setattr(cls, a[0], maketype(*a))
-    # implicit closed paras
-    for a in paratypes:
-        def dotype(self, tag):
-            self.removeType(paratypes, paratags)
-            return self.addNode(Node(self, 'para', str(tag), ispara=True))
-        if not hasattr(cls, a):
-            setattr(cls, a, dotype)
-
 
 class USFMParser:
 
-    def __init__(self, txt, factory=None, grammar=None):
+    def __init__(self, txt, factory=None, grammar=None, expanded=False, strict=False):
         if factory is None:
-            factory = et.Element
+            def makeel(tag, attrib, **extras):
+                attrib.update({" "+k:v for k, v in extras.items()})
+                return et.Element(tag, attrib)
+            factory = makeel
         if grammar is None:
             grammar = Grammar()
         self.factory = factory
         self.grammar = grammar
-        self.lexer = Lexer(txt)
+        self._setup(expanded=expanded)
+        self.lexer = Lexer(txt, expanded=expanded)
+
+    def _setup(s, expanded=False):
+        clsself = s.__class__
+        # tag closed types
+        for a in (('introchar', 'char'), ('listchar', 'char'), ('char', 'char'), ('_fig', 'figure')):
+            def maketype(c, t):
+                def dotype(self, tag):
+                    if tag.isend:
+                        return self.removeTag(str(tag))
+                    return self.addNode(Node(self, t, tag.basestr()))
+                return dotype
+            setattr(clsself, a[-1], maketype(*a))
+        # implicit closed paras
+        for a in paratypes:
+            if expanded:
+                def dotype(self, tag):
+                    self.removeType(paratypes, paratags)
+                    return self.addNode(Node(self, 'para', str(tag), ispara=True, xpand=tag.xp))
+            else:
+                def dotype(self, tag):
+                    self.removeType(paratypes, paratags)
+                    return self.addNode(Node(self, 'para', str(tag), ispara=True))
+            if not hasattr(clsself, a):
+                setattr(clsself, a, dotype)
 
     def parse(self):
         self.result = []
@@ -439,6 +484,8 @@ class USFMParser:
         while len(self.stack):
             curr = self.stack.pop()
             if curr.tag == tag:
+                if curr.element.tag == "unk":
+                    curr.element.tag = "char"
                 break
         else:
             self.stack = oldstack
@@ -450,12 +497,31 @@ class USFMParser:
         oldstack = self.stack[:]
         while len(self.stack):
             curr = self.stack.pop()
+            e = curr.element
+            if e.tag == "usx":
+                self.stack.append(curr)
+                break
             curr.close()
-            if curr is not None and (curr.tag in tags or self.grammar.marker_categories.get(curr.tag, None) in t):
+            cat = self.grammar.marker_categories.get(e.tag, None)
+            if e.tag == "unk":
+                e.tag = self.grammar.marker_tags.get(t[0], "para")
+                if e.tag == 'para' and e.parent is not None:
+                    parent = e.parent
+                    while parent is not None and parent.tag not in ("usx", "para"):
+                        parent = parent.parent
+                    if parent.tag != "usx":
+                        i, p = parent._getindex()
+                        if p is not None:
+                            e.parent.remove(e)
+                            p.insert(i+1, e)
+                            e.parent = p
+                            self.removeType(t, tags=tags)
+                break
+            if curr.tag in tags or cat in t:
                 break
         else:
             self.stack = oldstack
-        return self.stack[-1]
+        return self.stack[-1] if len(self.stack) else None
 
 #### Event methods
 
@@ -467,31 +533,31 @@ class USFMParser:
         if tag.isend:
             return self.removeTag(str(tag))
         elif self.stack[-1].tag == "c":
-            parent = AttribNode(self, self.stack[-1], str(tag))
+            parent = AttribNode(self, self.stack[-1], str(tag), pos=tag.pos)
         else:
-            parent = Node(self, 'para', str(tag), ispara=True)
+            parent = Node(self, 'para', str(tag), ispara=True, pos=tag.pos)
         self.stack.append(parent)
         return parent
 
     def _esb(self, tag):
         self.removeType(paratypes, paratags)
-        return self.addNode(Node(self, "sidebar", str(tag)))
+        return self.addNode(Node(self, "sidebar", str(tag), pos=tag.pos))
 
     def _esbe(self, tag):
         return self.removeTag("esb")
 
     def _periph(self, tag):
         self.removeTag(str(tag))
-        return self.addNode(PeriphNode(self, "periph", str(tag), notag=True))
+        return self.addNode(PeriphNode(self, "periph", str(tag), notag=True, pos=tag.pos))
 
     def _ref(self, tag):
         if tag.isend:
             return self.removeTag(str(tag))
-        return self.addNode(Node(self, 'ref', tag.basestr(), notag=True))
+        return self.addNode(Node(self, 'ref', tag.basestr(), notag=True, pos=tag.pos))
 
     def _rem(self, tag):
         self.removeType(paratypes, paratags)
-        res = self.addNode(Node(self, 'para', str(tag), ispara=True))
+        res = self.addNode(Node(self, 'para', str(tag), ispara=True, pos=tag.pos))
         self.lexer.readLine()
         return res
 
@@ -499,23 +565,23 @@ class USFMParser:
         self.removeTag('tr')
         if not len(self.stack) or self.stack[-1].element.tag != "table":
             self.removeType(paratypes,)
-            self.addNode(Node(self, 'table', ' table', notag=True))
-        return self.addNode(Node(self, 'row', 'tr'))
+            self.addNode(Node(self, 'table', ' table', notag=True, pos=tag.pos))
+        return self.addNode(Node(self, 'row', 'tr', pos=tag.pos))
 
     def _v(self, tag):
         self.removeTag('v')
         if len(self.stack) and self.stack[-1].tag == "c":
             self.removeTag('c')
-            self.addNode(Node(self, 'para', 'p'))
-        return self.addNode(NumberNode(self, "verse", str(tag)))
+            self.addNode(Node(self, 'para', 'p', pos=tag.pos))
+        return self.addNode(NumberNode(self, "verse", str(tag), pos=tag.pos))
 
     def _vp(self, tag):
         if tag.isend:
             return self.removeTag(str(tag))
         elif self.stack[-1].tag == "v":
-            parent = AttribNode(self, self.stack[-1], str(tag))
+            parent = AttribNode(self, self.stack[-1], str(tag), pos=tag.pos)
         else:
-            parent = Node(self, 'para', tag.basestr())
+            parent = Node(self, 'para', tag.basestr(), pos=tag.pos)
         self.stack.append(parent)
         return parent
 
@@ -523,43 +589,43 @@ class USFMParser:
         if self.grammar.marker_categories.get(self.stack[-1].tag, "") == "crossreferencechar":
             res = self.removeType('crossreferencechar')
         if not tag.isend:
-            res = self.addNode(Node(self, 'char', tag.basestr()))
+            res = self.addNode(Node(self, 'char', tag.basestr(), pos=tag.pos))
         return res
 
     def attrib(self, tag):
         if tag.isend:
             return self.removeTag(str(tag))
         else:
-            parent = AttribNode(self, self.stack[-1], str(tag))
+            parent = AttribNode(self, self.stack[-1], str(tag), pos=tag.pos)
         self.stack.append(parent)
         return parent
 
     def cell(self, tag):
         self.removeType('cell')
         if not tag.isend:
-            return self.addNode(Node(self, 'cell', str(tag)))
+            return self.addNode(Node(self, 'cell', str(tag), pos=tag.pos))
         return self.parent
 
     def crossreference(self, tag):
         if tag.isend:
             return self.removeTag(str(tag))
-        return self.addNode(NoteNode(self, 'note', str(tag)))
+        return self.addNode(NoteNode(self, 'note', str(tag), pos=tag.pos))
 
     def crossreferencechar(self, tag):
         res = self.removeType('crossreferencechar')
         if not tag.isend:
-            res = self.addNode(Node(self, 'char', tag.basestr()))
+            res = self.addNode(Node(self, 'char', tag.basestr(), pos=tag.pos))
         return res
 
     def footnote(self, tag):
         if tag.isend:
             return self.removeTag(str(tag))
-        return self.addNode(NoteNode(self, 'note', str(tag)))
+        return self.addNode(NoteNode(self, 'note', str(tag), pos=tag.pos))
 
     def footnotechar(self, tag):
         res = self.removeType('footnotechar')
         if not tag.isend:
-            res = self.addNode(Node(self, 'char', tag.basestr()))
+            res = self.addNode(Node(self, 'char', tag.basestr(), pos=tag.pos))
         return res
 
     def ident(self, tag):
@@ -570,15 +636,22 @@ class USFMParser:
     def milestone(self, tag):
         if tag.isend:
             return self.removeType('milestone')
-        return self.addNode(Node(self, 'ms', tag.basestr()))
+        return self.addNode(Node(self, 'ms', tag.basestr(), pos=tag.pos))
 
     def unknown(self, tag):
-        self.addNode(Node(self, 'ms', str(tag)))
-        res = self.removeTag(tag)
+        if strict:
+            if len(self.stack):
+                raise SyntaxError(f"Unknown tag {tag} in {self.stack[-1]}")
+            else:
+                raise SyntaxError(f"Unknown tag {tag}")
+            return None
+        if not tag.isend:
+            res = self.addNode(UnknownNode(self, 'unk', str(tag), pos=tag.pos))
+        else:
+            res = self.removeTag(str(tag))
+            res.tag = "char"
         return res
 
-setupParser(USFMParser)
-
 def main():
     import sys
     p = USFMParser(sys.argv[1])

{usfmtc-0.2.2 → usfmtc-0.2.4}/python/lib/usfmtc/xmlutils.py

@@ -3,12 +3,13 @@ import re
 import xml.etree.ElementTree as et
 
 class ParentElement(et.Element):
-    def __init__(self, tag, attrib=None, parent=None):
+    def __init__(self, tag, attrib=None, parent=None, pos=None):
         et.Element.__init__(self, tag, attrib)
         self.parent = parent
+        self.pos = pos
 
-    def makeelement(self, tag, attrib):
-        return self.__class__(tag, attrib, parent=self)
+    def makeelement(self, tag, attrib, pos=None):
+        return self.__class__(tag, attrib, parent=self, pos=pos or self.pos)
 
     def __str__(self):
         return "{}[{}]".format(self.tag, " ".join('@{}="{}"'.format(k, v) for k, v in self.attrib.items()))
@@ -18,19 +19,24 @@ class ParentElement(et.Element):
         return "{}/{}".format(p, str(self))
 
     def _getindex(self):
+        ''' Finds my index in my parent, returning both '''
         if self.parent is None:
             return -1, None
         return list(self.parent).index(self), self.parent
 
     def getprevious(self):
+        ''' Returns the previous element with the same parent, if any '''
         i, parent = self._getindex()
         return parent[i-1] if parent is not None and i > 0 else None
 
     def getnext(self):
+        ''' Returns the next element with the same parent, if any '''
         i, parent = self._getindex()
         return parent[i+1] if parent is not None and i < len(parent) - 1 else None
         
     def getnext_sibling(self):
+        ''' Returns the next element, while going up and down the hierarchy.
+            Notice that any text at the start of an element stops any hierarchy descent '''
         i, parent = self._getindex()
         if i < len(parent) - 1:
             return parent[i+1]
@@ -41,6 +47,8 @@ class ParentElement(et.Element):
         return parent
 
     def getprevious_sibling(self):
+        ''' Returns the previous element, while going up and down the hierarchy
+            Notice that any text after an element stops any hierarchy descent '''
         i, parent = self._getindex()
         if i > 0:
             return parent[i-1]
@@ -51,11 +59,13 @@ class ParentElement(et.Element):
         return parent
         
     def addprevious(self, el):
+        ''' Inserts el before the current element in the same parent '''
         i, parent = self._getindex()
         if parent is not None:
             parent.insert(i, el)
 
     def getparent(self):
+        ''' Returns the parent '''
         return self.parent
 
 

{usfmtc-0.2.2 → usfmtc-0.2.4/python/lib/usfmtc.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.2
 Name: usfmtc
-Version: 0.2.2
+Version: 0.2.4
 Summary: USFM reference parser
 Author-email: USFM Technical Committee <fonts@sil.org>
 License: Copyright © 2022-2023 by the members of the USFM/USX technical committee, listed below: