usfmtc 0.2.2__tar.gz → 0.2.3__tar.gz

{usfmtc-0.2.2/python/lib/usfmtc.egg-info → usfmtc-0.2.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: usfmtc
-Version: 0.2.2
+Version: 0.2.3
 Summary: USFM reference parser
 Author-email: USFM Technical Committee <fonts@sil.org>
 License: Copyright © 2022-2023 by the members of the USFM/USX technical committee, listed below:

{usfmtc-0.2.2 → usfmtc-0.2.3}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "usfmtc"
-version = "0.2.2"
+version = "0.2.3"
 authors = [{name = "USFM Technical Committee", email = "fonts@sil.org"}]
 readme = "python/README.md"
 license = {file = "LICENSE.md"}

{usfmtc-0.2.2 → usfmtc-0.2.3}/python/lib/usfmtc/__init__.py

@@ -58,15 +58,15 @@ def _usfmGrammar(rdoc, backend=None, start=None):
 
 def usfmGrammar(gsrc, extensions=[], altparser=True, backend=None, start=None):
     """ Create UsfmGrammarParser from gsrc as used by USX.fromUsfm """
-    if len(extensions) or not altparser:
-        rdoc = _grammarDoc(gsrc, extensions)
     if altparser:
+        rdoc = _grammarDoc(gsrc, extensions)
+        return _usfmGrammar(rdoc, backend, start)
+    else:
         res = Grammar()
         if len(extensions):
             for e in extensions:
                 res.readmrkrs(e)
         return res
-    return _usfmGrammar(rdoc, backend, start)
 
 
 _filetypes = {".xml": "usx", ".usx": "usx", ".usfm": "usfm", ".sfm": "usfm3.0", ".json": "usj"}
@@ -129,10 +129,12 @@ class USX:
     @classmethod
     def fromUsfm(cls, src, grammar, altparser=True, elfactory=None, timeout=1e7):
         """ Parses USFM using UsfmGrammarParser grammar and creates USX object.
-            Raise usfmtc.parser.NoParseError on error. """
+            Raise usfmtc.parser.NoParseError on error.
+            elfactory must take parent and pos named parameters not as attributes
+        """
         data = _readsrc(src)
 
-        if altparser:
+        if not altparser:
             p = USFMParser(data, factory=elfactory or ParentElement, grammar=grammar)
             xml = p.parse()
         else:
@@ -181,7 +183,7 @@ class USX:
     def outUsfm(self, grammar, file=None, altparser=True, **kw):
         """ Output USFM from USX object. grammar is et doc. If file is None returns string """
         el = messup(self.xml)
-        if altparser:
+        if not altparser:
             return self._outwrite(file, el, fn=usx2usfm)
         parser = USXConverter(grammar.getroot(), **kw)
         res = parser.parse(el)
@@ -217,7 +219,7 @@ class USX:
         elif outtype == "usj":
             self.outUsj(outfpath)
         elif outtype == "usfm":
-            if grammar is None and not altparser:
+            if grammar is None and altparser:
                 if gramfile is None:
                     for a in ([], ['..', '..', '..', 'grammar']):
                         gramfile = os.path.join(os.path.dirname(__file__), *a, "usx.rng")
@@ -263,7 +265,7 @@ def main(hookcli=None, hookusx=None):
     parser.add_argument("-e","--esids",action="store_true",help="Add esids, vids, sids, etc. to USX output")
     parser.add_argument("-v","--version",default=None,help="Set USFM version [3.1]")
     parser.add_argument("-x","--extfiles",action="append",default=[],help="markers.ext files to include")
-    parser.add_argument("--qusfm",action="store_true",help="Use quick nonvalidating USFM parser")
+    parser.add_argument("-V","--validate",action="store_true",help="Use validating parsers for USFM")
     parser.add_argument("-C","--canonical",action="store_true",help="Do not canonicalise")
     parser.add_argument("-l","--logging",help="Set logging level to usfmxtest.log")
     parser.add_argument("-q","--quiet",action="store_true",help="Don't say much")
@@ -326,16 +328,16 @@ def main(hookcli=None, hookusx=None):
         if args.informat.startswith("usfm"):
             args.extfiles.append(os.path.join(os.path.dirname(infiles[0]), "markers.ext"))
             exts = [x for x in args.extfiles if os.path.exists(x)]
-            ingrammar = usfmGrammar(args.grammar, altparser=args.qusfm, extensions=exts)
+            ingrammar = usfmGrammar(args.grammar, altparser=args.validate, extensions=exts)
         if args.outformat and args.outformat.startswith("usfm"):
             outgrammar = _grammarDoc(args.grammar)
 
     for infile in infiles:
         outfile = None
         if args.outfile is None:
-            outfile = _makeoutfile(infile, args.format)
+            outfile = _makeoutfile(infile, args.outformat)
         elif os.path.isdir(args.outfile):
-            outf = _makeoutfile(infile, args.format)
+            outf = _makeoutfile(infile, args.outformat)
             if outf is None:
                 doerror(f"invalid output format {args.format} in {args.outfile}")
             outfile = os.path.join(args.outfile, os.path.basename(outf))
@@ -351,7 +353,7 @@ def main(hookcli=None, hookusx=None):
         if not args.quiet:
             print(f"{infile} -> {outfile}" if outfile else f"{infile}")
         try:
-            usxdoc = readFile(infile, informat=args.informat, grammar=ingrammar, altparser=args.qusfm)
+            usxdoc = readFile(infile, informat=args.informat, grammar=ingrammar, altparser=args.validate)
         except NoParseError as e:
             doerror(f"Failed to parse {infile}: {e}", False)
 
@@ -373,7 +375,7 @@ def main(hookcli=None, hookusx=None):
         if not args.canonical:
             usxdoc.canonicalise()
 
-        usxdoc.saveAs(outfile, outformat=args.outformat, addesids=args.esids, grammar=outgrammar, altparser=args.qusfm)
+        usxdoc.saveAs(outfile, outformat=args.outformat, addesids=args.esids, grammar=outgrammar, altparser=args.validate)
 
 if __name__ == "__main__":
     main()

{usfmtc-0.2.2 → usfmtc-0.2.3}/python/lib/usfmtc/extension.py

@@ -58,11 +58,11 @@ class Extensions(SFMFile):
 
     def applyto(self, rdoc, factory=et):
         res = False     # nothing added
-        allcats = set([x.get('category', None) for x in self.markers.values()])
+        allcats = set([x.get('category', "").split()[0] for x in self.markers.values()])
         for a in allcats:
             if a not in categories:
                 continue
-            ms = [k for k, v in self.markers.items() if v.get('category', None) == a]
+            ms = [k for k, v in self.markers.items() if a in v.get('category', "").split()]
             enum = categories[a]+".style.enum"
             e = rdoc.find('./{0}define[@name="{1}"]/{0}choice'.format(relaxns, enum))
             allmks = set([v.text for v in e.findall(f'./{relaxns}value')])

{usfmtc-0.2.2 → usfmtc-0.2.3}/python/lib/usfmtc/usfmparser.py

@@ -5,8 +5,14 @@ import xml.etree.ElementTree as et
 from usfmtc.extension import SFMFile
 from collections import UserDict, UserString
 
+
+class Pos:
+    def __init__(self, l, c):
+        self.l = l
+        self.c = c
+
 class Tag(str):
-    def __new__(cls, s):
+    def __new__(cls, s, l=0, c=0):
         isend = False
         isplus = False
         if s.startswith("+"):
@@ -23,7 +29,7 @@ class Tag(str):
     def __init__(self, s, l=0, c=0):
         self.isplus = getattr(self, 'isplus', False)
         self.isend = getattr(self, 'isend', False)
-        self.pos = (l, c)
+        self.pos = Pos(l, c)
 
     def __repr__(self):
         return "Tag("+str(self)+")"
@@ -43,12 +49,12 @@ class Tag(str):
 class AttribText(UserString):
     def __init__(self, s, l=0, c=0):
         super().__init__(s)
-        self.pos = (l, c)
+        self.pos = Pos(l, c)
     pass
 
 class OptBreak:
     def __init__(self, l=0, c=0):
-        self.pos = (l, c)
+        self.pos = Pos(l, c)
 
     def __repr__(self):
         return "OptBreak(" + str(self) + ")"
@@ -58,7 +64,7 @@ class OptBreak:
 class String(UserString):
     def __init__(self, s, l=0, c=0):
         super().__init__(s)
-        self.pos = (l, c)
+        self.pos = Pos(l, c)
 
     def __add__(self, s):
         return String(str(self) + s)
@@ -68,7 +74,7 @@ class String(UserString):
 class Attribs(UserDict):
     def __init__(self, l=0, c=0):
         super().__init__()
-        self.pos = (l, c)
+        self.pos = Pos(l, c)
 
 class Lexer:
 
@@ -113,7 +119,7 @@ class Lexer:
                     curri = m.end() + 1
                     continue
                 else:
-                    res = Tag(t.group(0))
+                    res = Tag(t.group(0), l=self.lindex, c=curri-self.lpos)
                     curri += t.end()
                     break
             elif n == '|':
@@ -179,7 +185,7 @@ class Grammar:
         "versepara": "cls nb pc pi1 pi2 pi3 pi po pr pmo pmc pmr pm ph1 ph2 ph3 ph p q1 q2 q3 q4 qc qr qm1 qm2 qm3 qm qd q b d mi1 mi2 mi3 mi4 mi m",
     }
 
-    marker_categories = {t:k for k, v in category_markers.items() for t in v.split(' ')}
+    marker_categories = {t:k for k, v in category_markers.items() for t in v.split()}
 
     attribmap = { 'jmp' : 'href', 'k' : 'key', 'qt-s': 'who', 'qt1-s': 'who', 'qt2-s': 'who',
         'qt3-s': 'who', 'qt4-s': 'who', 'qt5-s': 'who', 'rb': 'gloss', 't-s': 'sid', 'ts-s': 'sid',
@@ -211,13 +217,13 @@ def isfirstText(e):
     return True
 
 class Node:
-    def __init__(self, parser, usxtag, tag, ispara=False, notag=False):
+    def __init__(self, parser, usxtag, tag, ispara=False, notag=False, pos=None):
         self.parser = parser
         self.tag = tag
         self.ispara = ispara
         parent = parser.stack[-1] if len(parser.stack) else None
         self.element = parser.factory(usxtag, {} if notag or tag is None else {"style": tag},
-                    parent=getattr(parent, 'element', None))
+                    parent=getattr(parent, 'element', None), pos=pos)
         if parent:
             parent.addNodeElement(self.element)
         self.attribnodes = []
@@ -227,7 +233,7 @@ class Node:
 
     def appendElement(self, child):
         if isinstance(child, OptBreak):
-            c = self.parser.factory("optbreak", {}, parent=self.element)
+            c = self.parser.factory("optbreak", {}, parent=self.element, pos=child.pos)
             self.element.append(c)
         self.clearAttribNodes()
 
@@ -297,11 +303,12 @@ class USXNode(Node):
         self.element.set('version', str(t).strip())
 
 class AttribNode(Node):
-    def __init__(self, parser, parent, tag):
+    def __init__(self, parser, parent, tag, pos=None):
         self.parser = parser
         self.parent = parent
         self.parent.addAttribNode(self)
         self.tag = tag
+        self.pos = pos
         self.attribnodes = []
 
     def addNodeElement(self, e):
@@ -313,8 +320,8 @@ class AttribNode(Node):
         self.parent.element.set(attrib, str(t).strip())
 
 class NumberNode(Node):
-    def __init__(self, parser, usxtag, tag, ispara=False):
-        super().__init__(parser, usxtag, tag, ispara=ispara)
+    def __init__(self, parser, usxtag, tag, ispara=False, pos=None):
+        super().__init__(parser, usxtag, tag, ispara=ispara, pos=pos)
         self.hasarg = False
 
     def appendText(self, t):
@@ -327,7 +334,7 @@ class NumberNode(Node):
         if len(b) > 1 and b[1].strip():
             self.parser.removeTag(self.tag)
             if self.ispara:
-                self.parser.addNode(Node(self.parser, 'para', 'p'))
+                self.parser.addNode(Node(self.parser, 'para', 'p', pos=self.element.pos))
             self.parser.stack[-1].appendText(b[1])
 
     def addNodeElement(self, e):
@@ -335,8 +342,8 @@ class NumberNode(Node):
         self.parser.stack[-1].addNodeElement(e)
 
 class NoteNode(Node):
-    def __init__(self, parser, usxtag, tag):
-        super().__init__(parser, usxtag, tag)
+    def __init__(self, parser, usxtag, tag, pos=None):
+        super().__init__(parser, usxtag, tag, pos=pos)
         self.hascaller = False
 
     def appendText(self, t):
@@ -384,7 +391,10 @@ class USFMParser:
 
     def __init__(self, txt, factory=None, grammar=None):
         if factory is None:
-            factory = et.Element
+            def makeel(tag, attrib, **extras):
+                attrib.update({" "+k:v for k, v in extras.items()})
+                return et.Element(tag, attrib)
+            factory = makeel
         if grammar is None:
             grammar = Grammar()
         self.factory = factory
@@ -467,31 +477,31 @@ class USFMParser:
         if tag.isend:
             return self.removeTag(str(tag))
         elif self.stack[-1].tag == "c":
-            parent = AttribNode(self, self.stack[-1], str(tag))
+            parent = AttribNode(self, self.stack[-1], str(tag), pos=tag.pos)
         else:
-            parent = Node(self, 'para', str(tag), ispara=True)
+            parent = Node(self, 'para', str(tag), ispara=True, pos=tag.pos)
         self.stack.append(parent)
         return parent
 
     def _esb(self, tag):
         self.removeType(paratypes, paratags)
-        return self.addNode(Node(self, "sidebar", str(tag)))
+        return self.addNode(Node(self, "sidebar", str(tag), pos=tag.pos))
 
     def _esbe(self, tag):
         return self.removeTag("esb")
 
     def _periph(self, tag):
         self.removeTag(str(tag))
-        return self.addNode(PeriphNode(self, "periph", str(tag), notag=True))
+        return self.addNode(PeriphNode(self, "periph", str(tag), notag=True, pos=tag.pos))
 
     def _ref(self, tag):
         if tag.isend:
             return self.removeTag(str(tag))
-        return self.addNode(Node(self, 'ref', tag.basestr(), notag=True))
+        return self.addNode(Node(self, 'ref', tag.basestr(), notag=True, pos=tag.pos))
 
     def _rem(self, tag):
         self.removeType(paratypes, paratags)
-        res = self.addNode(Node(self, 'para', str(tag), ispara=True))
+        res = self.addNode(Node(self, 'para', str(tag), ispara=True, pos=tag.pos))
         self.lexer.readLine()
         return res
 
@@ -499,23 +509,23 @@ class USFMParser:
         self.removeTag('tr')
         if not len(self.stack) or self.stack[-1].element.tag != "table":
             self.removeType(paratypes,)
-            self.addNode(Node(self, 'table', ' table', notag=True))
-        return self.addNode(Node(self, 'row', 'tr'))
+            self.addNode(Node(self, 'table', ' table', notag=True, pos=tag.pos))
+        return self.addNode(Node(self, 'row', 'tr', pos=tag.pos))
 
     def _v(self, tag):
         self.removeTag('v')
         if len(self.stack) and self.stack[-1].tag == "c":
             self.removeTag('c')
-            self.addNode(Node(self, 'para', 'p'))
-        return self.addNode(NumberNode(self, "verse", str(tag)))
+            self.addNode(Node(self, 'para', 'p', pos=tag.pos))
+        return self.addNode(NumberNode(self, "verse", str(tag), pos=tag.pos))
 
     def _vp(self, tag):
         if tag.isend:
             return self.removeTag(str(tag))
         elif self.stack[-1].tag == "v":
-            parent = AttribNode(self, self.stack[-1], str(tag))
+            parent = AttribNode(self, self.stack[-1], str(tag), pos=tag.pos)
         else:
-            parent = Node(self, 'para', tag.basestr())
+            parent = Node(self, 'para', tag.basestr(), pos=tag.pos)
         self.stack.append(parent)
         return parent
 
@@ -523,43 +533,43 @@ class USFMParser:
         if self.grammar.marker_categories.get(self.stack[-1].tag, "") == "crossreferencechar":
             res = self.removeType('crossreferencechar')
         if not tag.isend:
-            res = self.addNode(Node(self, 'char', tag.basestr()))
+            res = self.addNode(Node(self, 'char', tag.basestr(), pos=tag.pos))
         return res
 
     def attrib(self, tag):
         if tag.isend:
             return self.removeTag(str(tag))
         else:
-            parent = AttribNode(self, self.stack[-1], str(tag))
+            parent = AttribNode(self, self.stack[-1], str(tag), pos=tag.pos)
         self.stack.append(parent)
         return parent
 
     def cell(self, tag):
         self.removeType('cell')
         if not tag.isend:
-            return self.addNode(Node(self, 'cell', str(tag)))
+            return self.addNode(Node(self, 'cell', str(tag), pos=tag.pos))
         return self.parent
 
     def crossreference(self, tag):
         if tag.isend:
             return self.removeTag(str(tag))
-        return self.addNode(NoteNode(self, 'note', str(tag)))
+        return self.addNode(NoteNode(self, 'note', str(tag), pos=tag.pos))
 
     def crossreferencechar(self, tag):
         res = self.removeType('crossreferencechar')
         if not tag.isend:
-            res = self.addNode(Node(self, 'char', tag.basestr()))
+            res = self.addNode(Node(self, 'char', tag.basestr(), pos=tag.pos))
         return res
 
     def footnote(self, tag):
         if tag.isend:
             return self.removeTag(str(tag))
-        return self.addNode(NoteNode(self, 'note', str(tag)))
+        return self.addNode(NoteNode(self, 'note', str(tag), pos=tag.pos))
 
     def footnotechar(self, tag):
         res = self.removeType('footnotechar')
         if not tag.isend:
-            res = self.addNode(Node(self, 'char', tag.basestr()))
+            res = self.addNode(Node(self, 'char', tag.basestr(), pos=tag.pos))
         return res
 
     def ident(self, tag):
@@ -570,10 +580,10 @@ class USFMParser:
     def milestone(self, tag):
         if tag.isend:
             return self.removeType('milestone')
-        return self.addNode(Node(self, 'ms', tag.basestr()))
+        return self.addNode(Node(self, 'ms', tag.basestr(), pos=tag.pos))
 
     def unknown(self, tag):
-        self.addNode(Node(self, 'ms', str(tag)))
+        self.addNode(Node(self, 'ms', str(tag), pos=tag.pos))
         res = self.removeTag(tag)
         return res
 

{usfmtc-0.2.2 → usfmtc-0.2.3}/python/lib/usfmtc/xmlutils.py

@@ -3,12 +3,13 @@ import re
 import xml.etree.ElementTree as et
 
 class ParentElement(et.Element):
-    def __init__(self, tag, attrib=None, parent=None):
+    def __init__(self, tag, attrib=None, parent=None, pos=None):
         et.Element.__init__(self, tag, attrib)
         self.parent = parent
+        self.pos = pos
 
-    def makeelement(self, tag, attrib):
-        return self.__class__(tag, attrib, parent=self)
+    def makeelement(self, tag, attrib, pos=None):
+        return self.__class__(tag, attrib, parent=self, pos=pos or self.pos)
 
     def __str__(self):
         return "{}[{}]".format(self.tag, " ".join('@{}="{}"'.format(k, v) for k, v in self.attrib.items()))
@@ -18,19 +19,24 @@ class ParentElement(et.Element):
         return "{}/{}".format(p, str(self))
 
     def _getindex(self):
+        ''' Finds my index in my parent, returning both '''
         if self.parent is None:
             return -1, None
         return list(self.parent).index(self), self.parent
 
     def getprevious(self):
+        ''' Returns the previous element with the same parent, if any '''
         i, parent = self._getindex()
         return parent[i-1] if parent is not None and i > 0 else None
 
     def getnext(self):
+        ''' Returns the next element with the same parent, if any '''
         i, parent = self._getindex()
         return parent[i+1] if parent is not None and i < len(parent) - 1 else None
         
     def getnext_sibling(self):
+        ''' Returns the next element, while going up and down the hierarchy.
+            Notice that any text at the start of an element stops any hierarchy descent '''
         i, parent = self._getindex()
         if i < len(parent) - 1:
             return parent[i+1]
@@ -41,6 +47,8 @@ class ParentElement(et.Element):
         return parent
 
     def getprevious_sibling(self):
+        ''' Returns the previous element, while going up and down the hierarchy
+            Notice that any text after an element stops any hierarchy descent '''
         i, parent = self._getindex()
         if i > 0:
             return parent[i-1]
@@ -51,11 +59,13 @@ class ParentElement(et.Element):
         return parent
         
     def addprevious(self, el):
+        ''' Inserts el before the current element in the same parent '''
         i, parent = self._getindex()
         if parent is not None:
             parent.insert(i, el)
 
     def getparent(self):
+        ''' Returns the parent '''
         return self.parent
 
 

{usfmtc-0.2.2 → usfmtc-0.2.3/python/lib/usfmtc.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: usfmtc
-Version: 0.2.2
+Version: 0.2.3
 Summary: USFM reference parser
 Author-email: USFM Technical Committee <fonts@sil.org>
 License: Copyright © 2022-2023 by the members of the USFM/USX technical committee, listed below: