usfmtc 0.4__tar.gz → 0.4.2__tar.gz

{usfmtc-0.4/src/usfmtc.egg-info → usfmtc-0.4.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: usfmtc
-Version: 0.4
+Version: 0.4.2
 Summary: USFM reference parser
 Author-email: USFM Technical Committee <fonts@sil.org>
 License: MIT

{usfmtc-0.4 → usfmtc-0.4.2}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "usfmtc"
-version = "0.4"
+version = "0.4.2"
 authors = [{name = "USFM Technical Committee", email = "fonts@sil.org"}]
 readme = "README.md"
 license = { text = "MIT" }

{usfmtc-0.4 → usfmtc-0.4.2}/src/usfmtc/__init__.py

@@ -9,7 +9,7 @@
 #     nuitka-project: --output-filename=usfmconv.bin
 
 import os, json, io
-from usfmtc.utils import readsrc
+from usfmtc.utils import readsrc, getSrcName
 from usfmtc.validating.usfmparser import parseusfm, UsfmParserBackend
 from usfmtc.validating.rngparser import NoParseError
 from usfmtc.extension import Extensions
@@ -25,7 +25,7 @@ from usfmtc.usfmgenerate import usx2usfm
 from usfmtc.reference import RefList
 import xml.etree.ElementTree as et
 
-version = "0.4"
+version = "0.4.2"
 
 def _grammarDoc(gsrc, extensions=[], factory=et):
     data = readsrc(gsrc)
@@ -126,7 +126,12 @@ class USX:
             Raise usfmtc.parser.NoParseError on error.
             elfactory must take parent and pos named parameters not as attributes
         """
-        data = readsrc(src)
+        readerr = None
+        try:
+            data = readsrc(src, errors='strict')
+        except UnicodeError as e:
+            readerr = f"In file {getSrcName(src)}: {e}"
+            data = readsrc(src)
 
         if not altparser:
             p = USFMParser(data, factory=elfactory or ParentElement, grammar=grammar, strict=strict, **kw)
@@ -137,6 +142,8 @@ class USX:
             result = parseusfm(data, grammar, timeout=timeout, isdata=True, **kw)
             xml = result.asEt(elfactory=elfactory)
 
+        if p and readerr is not None:
+            p.errors.insert(0, (readerr,))
         cleanup(xml)            # normalize space, de-escape chars, cell aligns, etc.
         res = cls(xml, grammar, errors=p.errors if p else None)
         if keepparser:
@@ -145,7 +152,7 @@ class USX:
 
     @classmethod
     def fromUsj(cls, src, elfactory=None, grammar=None, **kw):
-        data = readsrc(src)
+        data = readsrc(src, errors=kw.get("errors", "replace"))
         djson = json.loads(data)
         xml = usjtousx(djson, elfactory=elfactory)
         return cls(xml, grammar)
@@ -291,7 +298,7 @@ class USX:
             if outtype == "usfm3.0":
                 outtype = "usfm"
                 if version is None:
-                    version = "0.4"
+                    version = "0.4.2"
             self.outUsfm(grammar=grammar, file=outfpath, outversion=version, altparser=altparser, **kw)
 
     def canonicalise(self, version=None):
@@ -337,7 +344,7 @@ class USX:
     @version.setter
     def version(self, version):
         if isinstance(version, (list, tuple)):
-            version = "0.4".join([str(x) for x in version])
+            version = "0.4.2".join([str(x) for x in version])
         if version is not None:
             self.getroot().set('version', str(version))
 
@@ -348,6 +355,12 @@ class USX:
             return bke.get("code", None)
         return None
 
+    @book.setter
+    def book(self, code):
+        bke = self.getroot().find(".//book")
+        if bke is not None:
+            bke.set('code', code)
+
 def main(hookcli=None, hookusx=None):
 
     import argparse, logging, sys
@@ -470,6 +483,10 @@ def main(hookcli=None, hookusx=None):
         if len(infiles) == 1 and usxdoc is None:
             doerror(f"Unable to read in {args.infile}")
 
+        if usxdoc.errors is not None and len(usxdoc.errors):
+            for m, p, r in usxdoc.errors:
+                doerror(f"{r} ({p}): {m}", False)
+
         if reflist is not None:
             book = usxdoc.book
             bkrefs = [r for r in reflist if r.first.book == book]

{usfmtc-0.4 → usfmtc-0.4.2}/src/usfmtc/reference.py

@@ -696,7 +696,7 @@ class RefRange:
             return first
         return super().__new__(cls)
 
-    def __init__(self, first: Optional[Ref]=None, last: Optional[Ref]=None, strict: bool=False, **kw):
+    def __init__(self, first: Optional[Ref]=None, last: Optional[Ref]=None, strict: bool=False, test: bool=True,  **kw):
         self.strict = strict
         self.first = first.first
         self.last = last.last
@@ -705,7 +705,7 @@ class RefRange:
             self.sep = kw['sep']
         if getattr(self.last, 'book', None) is None:
             self.last.book = self.first.book
-        if self.last < self.first:
+        if test and self.last < self.first:
             raise ValueError(f"{first=} is after {last=}")
         if isinstance(self.first, RefRange):
             raise ValueError(f"Nested RefRange({self.first})")

{usfmtc-0.4 → usfmtc-0.4.2}/src/usfmtc/usfmgenerate.py

@@ -10,13 +10,14 @@ def usvout(m):
     else:
         return "\\u{:04X}".format(u)
 
-_reesc = re.compile(r'([\\|]|//)')
+_reesc = re.compile(r'([\\|~]|//)')
 _reatt = re.compile(r'([\\|"])')
 def attribescaped(s, escapes):
     return escaped(s, escapes, reg=_reatt)
 
 def escaped(s, escapes, reg=None):
     res = (reg or _reesc).sub(r'\\\1', s)
+    res = res.replace("\u00A0", "~")
     if escapes:
         res = re.sub(r'([{}])'.format(escapes), usvout, res)
     return res
@@ -94,6 +95,8 @@ def iterels(el, events):
 def _isnextref(cref, ref, e):
     if ref.chapter == cref.chapter:
         return ref.verse == cref.verse or ref.verse == cref.verse + 1
+    elif cref.chapter is None:
+        return False
     else:
         return ref.chapter == cref.chapter + 1 and ref.verse == 1
 

{usfmtc-0.4 → usfmtc-0.4.2}/src/usfmtc/usfmparser.py

@@ -4,26 +4,32 @@ import regex
 import xml.etree.ElementTree as et
 from usfmtc.extension import SFMFile
 from collections import UserDict, UserString
+from typing import Optional, Dict, Any, List, Type, Union
 
 WS = "\t\n\r "   # \v\f\u001C\u001D\u001E\u001F "  full python ASCII WS
 
 class Pos:
-    def __init__(self, l, c, **kw):
+    def __init__(self, l: int, c: int, **kw) -> None:
         self.l = l
         self.c = c
         self.kw = kw
 
-    def __str__(self):
+    def __str__(self) ->str:
         return f"{self.l}:{self.c}"
 
     def __repr__(self):
         return f"Pos({self.l}:{self.c})"
 
-    def copy(self):
+    def copy(self) -> 'Pos':
         return self.__class__(self.l, self.c, **self.kw)
 
 class Tag(str):
-    def __new__(cls, s, l=0, c=0, **kw):
+    pos: Pos
+    isplus: bool
+    isend: bool
+    attribs: Optional[Dict[str, str]]
+
+    def __new__(cls, s: str, l: int = 0, c: int = 0, **kw: Any) -> "Tag":
         isend = False
         isplus = False
         if s.startswith("+"):
@@ -55,7 +61,7 @@ class Tag(str):
         res += "*" if self.isend else ""
         return res
 
-    def basestr(self):
+    def basestr(self) -> str:
         """ Returns marker with no final *, etc. """
         return super().__str__()
 
@@ -80,7 +86,10 @@ class OptBreak:
         return "//"
 
 class String(UserString):
-    def __init__(self, s, l=0, c=0, **kw):
+    pos: Pos
+    kw: Dict[str, Any]
+
+    def __init__(self, s: str, l: int=0, c: int=0, **kw: Any) -> None:
         super().__init__(s)
         self.pos = Pos(l, c)
         self.kw = kw
@@ -91,7 +100,7 @@ class String(UserString):
     def be(self, s):
         return String(str(s), l=self.pos.l, c=self.pos.c, **self.kw)
 
-    def addToNode(self, node, position, lstrip=False):
+    def addToNode(self, node: et.Element, position: str, lstrip: bool=False) -> None:
         cp = self.pos.c
         s = str(self)
         if lstrip:
@@ -114,14 +123,18 @@ class Attribs(UserDict):
         self.kw = kw
 
 class Lexer:
+    txt: str
+    cindex: int
+    lindex: int
+    nexts: List[Union[Tag, String, 'Attribs', 'OptBreak']]
 
     tokenre = regex.compile(r'''    # the next token in text
         ((?: [^\\|/\r\n]            # string component consists of: normal characters
-           | \\ (?:[\\|] | \r?\n)           # escaped specials
+           | \\ (?:[\\|~] | \r?\n)           # escaped specials
            | / (?!/)                        # / sequence, not //
           )+                            # one or more of these
         )
-        | ( [\\|] | \r?\n | // )    # then specials: tag start or attributes start, newline, //   
+        | ( [\\|~] | \r?\n | // )    # then specials: tag start or attributes start, newline, //   
     ''', regex.X)
     tagre = regex.compile(r'''
         (?: \+? [a-zA-Z_][a-zA-Z_0-9^-]* )?  # optional + then the usfm tag spec
@@ -139,14 +152,15 @@ class Lexer:
     afterattribs = regex.compile(r'\s*\\')
     usvre = regex.compile(r'(?:\\u([0-9a-fA-F]{4})|\\U([0-9a-fA-F]{8}))')
 
-    def __init__(self, txt, parser, expanded=False, strict=False, tagger=Tag):
+    def __init__(self, txt: str, parser: "USFMParser", expanded: bool=False,
+                        strict: bool=False, tagger: Type[Tag]=Tag):
         self.txt = txt
         self.expanded = expanded
         self.strict = strict
         self.parser = parser
         self.tagger = tagger
 
-    def __iter__(self):
+    def __iter__(self) -> "Lexer":
         self.nexts = []
         self.cindex = 0
         self.lindex = 0
@@ -155,7 +169,7 @@ class Lexer:
         self.currxpand = None
         return self
 
-    def __next__(self):
+    def __next__(self) -> Union[Tag, String, "Attribs", "AttribText", "OptBreak"]:
         if len(self.nexts):
             return self.nexts.pop(0)
         curri = self.cindex
@@ -181,7 +195,7 @@ class Lexer:
                     u = self.usvre.match(self.txt[m.end():])
                     if u:
                         c = chr(int((m.group(1) or m.group(2)), 16))
-                        s += c
+                        res += c
                         curri += u.end()
                         continue
                 t = self.tagre.match(self.txt[m.end():])
@@ -207,9 +221,9 @@ class Lexer:
             elif n == '|':
                 t, curri = self.readAttrib(curri)
                 if not len(t):
-                    res = String("", l=self.lindex, c=curri-self.lpos)
-                    continue
-                res = t
+                    res = AttribText("", l=self.lindex, c=curri-self.lpos)
+                else:
+                    res = t
                 break
             elif n == '//':
                 res = OptBreak(l=self.lindex, c=curri-self.lpos)
@@ -299,6 +313,9 @@ class Grammar:
         "note": ("footnote", "crossreference"),
         "intro": ("introduction", "introchar")
     }
+    node_depths = {"usx": 1, "book": 2, "periph": 3, "sidebar": 4, "chapter": 5, "para": 5, "table": 5,
+        "row": 6, "cell": 7, "char": 8, "optbreak": 8, "figure": 8, "ref": 8, "link": 8, "verse": 8,
+        "note": 9, "ms": 10} 
 
     marker_categories = {t:k for k, v in category_markers.items() for t in v.split()}
     marker_tags = {t:k for k, v in category_tags.items() for t in v}
@@ -362,7 +379,7 @@ class Grammar:
     def copy(self):
         res = self.__class__(quick=True)
         for a in ('marker_categories', 'attribmap', 'attributes'):
-            setattr(res, a, getattr(self, a))
+            setattr(res, a, getattr(self, a).copy())
         return res
 
     def readmrkrs(self, fname):
@@ -393,7 +410,7 @@ def isfirstText(e):
     return True
 
 class Node:
-    def __init__(self, parser, usxtag, tag, ispara=False, notag=False, pos=None, **kw):
+    def __init__(self, parser, usxtag, tag, *, ispara=False, notag=False, pos=None, **kw):
         self.parser = parser
         self.tag = tag
         self.ispara = ispara
@@ -495,7 +512,7 @@ class USXNode(Node):
         self.element.set('version', str(t).strip(WS))
 
 class AttribNode(Node):
-    def __init__(self, parser, parent, tag, pos=None, only=[], **kw):
+    def __init__(self, parser, parent, tag, *, pos=None, only=[], **kw):
         self.parser = parser
         self.parent = parent
         self.tag = tag
@@ -550,7 +567,7 @@ class FwdAttribNode(Node):
 
 
 class NumberNode(Node):
-    def __init__(self, parser, usxtag, tag, ispara=False, pos=None, **kw):
+    def __init__(self, parser, usxtag, tag, *, ispara=False, pos=None, **kw):
         super().__init__(parser, usxtag, tag, ispara=ispara, pos=pos, **kw)
         self.hasarg = False
 
@@ -631,9 +648,10 @@ class FallBackError(Exception):
 paratypes = ('header', 'introduction', 'list', 'otherpara', 'sectionpara', 'versepara', 'title', 'chapter', 'ident')
 paratags = ('rem', ' table', 'sidebar')
 
+
 class USFMParser:
 
-    def __init__(self, txt, factory=None, grammar=None, expanded=False, strict=False, version=[99], index=True, **kw):
+    def __init__(self, txt, *, factory=None, grammar=None, expanded=False, strict=False, version=[99], index=True, **kw):
         if factory is None:
             def makeel(tag, attrib, **extras):
                 attrib.update({" "+k:v for k, v in extras.items()})
@@ -753,7 +771,7 @@ class USFMParser:
                 self.error(SyntaxError, f"Closing '{tag}' with no corresponding opening", self.lexer.currpos())
         return self.stack[-1]
 
-    def removeType(self, t, tags=[]):
+    def removeType(self, t, tags=[], node=None):
         if isinstance(t, str):
             t = [t]
         oldstack = self.stack[:]
@@ -761,7 +779,8 @@ class USFMParser:
             curr = self.stack.pop()
             if hasattr(curr, 'element'):
                 e = curr.element
-                if e.tag in ("usx", "periph") or e.tag in tags:
+                if self.grammar.node_depths.get(e.tag, 10) < self.grammar.node_depths.get(node, 4) \
+                            or e.tag in tags:
                     self.stack.append(curr)
                     break
                 curr.close()
@@ -786,7 +805,7 @@ class USFMParser:
             elif e.tag in tags or cat in t:
                 break
         else:
-            self.stack = oldstack
+            self.stack = oldstack       # what about everything we closed on getting here?
         return self.stack[-1] if len(self.stack) else None
 
     def cvref(self):
@@ -795,7 +814,7 @@ class USFMParser:
 
 #### Event methods
     def _c(self, tag):
-        self.removeType(paratypes, paratags)
+        self.removeType(paratypes, paratags, node="chapter")
         return self.addNode(NumberNode(self, "chapter", tag, ispara=True, pos=tag.pos))
 
     def _cp(self, tag):
@@ -809,7 +828,7 @@ class USFMParser:
         return parent
 
     def _esb(self, tag):
-        self.removeType(paratypes, paratags)
+        self.removeType(paratypes, paratags, node="sidebar")
         return self.addNode(Node(self, "sidebar", tag, pos=tag.pos))
 
     def _esbe(self, tag):
@@ -831,7 +850,7 @@ class USFMParser:
             return self.otherpara(tag)
         if tag.isend:
             return self.removeTag(str(tag))
-        self.removeType(paratypes, paratags)
+        self.removeType(paratypes, paratags, node="para")
         res = self.addNode(Node(self, 'para', tag, ispara=True, pos=tag.pos))
         self.lexer.readLine()
         self.lexer.appendtag("rem*")
@@ -840,7 +859,7 @@ class USFMParser:
     def _tr(self, tag):
         self.removeTag('tr', absentok=True)
         if not len(self.stack) or self.stack[-1].element.tag != "table":
-            self.removeType(paratypes,)
+            self.removeType(paratypes, node="table")
             self.addNode(Node(self, 'table', ' table', notag=True, pos=tag.pos))
         return self.addNode(Node(self, 'row', 'tr', pos=tag.pos))
 
@@ -865,7 +884,7 @@ class USFMParser:
     def _xt(self, tag):
         removed = False
         if self.grammar.marker_categories.get(self.stack[-1].tag, "") == "crossreferencechar":
-            res = self.removeType('crossreferencechar')
+            res = self.removeType('crossreferencechar', node="char")
             removed = True
         if not tag.isend:
             res = self.addNode(Node(self, 'char', tag, pos=tag.pos))
@@ -894,7 +913,7 @@ class USFMParser:
         return self.addNode(FwdAttribNode(self, 'ms', tag, pos=tag.pos))
 
     def cell(self, tag):
-        self.removeType('cell', tags=["row"])
+        self.removeType('cell', tags=["row"], node="cell")
         if not tag.isend:
             return self.addNode(Node(self, 'cell', tag, pos=tag.pos))
         return self.parent
@@ -916,7 +935,7 @@ class USFMParser:
         return self.addNode(NoteNode(self, 'note', tag, pos=tag.pos))
 
     def crossreferencechar(self, tag):
-        res = self.removeType('crossreferencechar', tags=["note"])
+        res = self.removeType('crossreferencechar', tags=["note"], node="char")
         if not tag.isend:
             res = self.addNode(Node(self, 'char', tag, pos=tag.pos))
         return res
@@ -927,7 +946,7 @@ class USFMParser:
         return self.addNode(NoteNode(self, 'note', tag, pos=tag.pos))
 
     def footnotechar(self, tag):
-        res = self.removeType('footnotechar', tags=["note"])
+        res = self.removeType('footnotechar', tags=["note"], node="char")
         if not tag.isend:
             res = self.addNode(Node(self, 'char', tag, pos=tag.pos))
         return res
@@ -963,7 +982,7 @@ class USFMParser:
 
 def main():
     import sys
-    p = USFMParser(sys.argv[1])
+    p = USFMParser(open(sys.argv[1], encoding="utf-8").read())
     e = p.parse()
     et.indent(e)
     et.dump(e)

{usfmtc-0.4 → usfmtc-0.4.2}/src/usfmtc/usxcursor.py

@@ -117,9 +117,8 @@ def _findcvel(ref, usx, atend=False, parindex=0):
         the end of the elment, the next element. parindex speeds up the hunt for the
         chapter.'''
     resm = 0
-    if ref.book:
-        if ref.book is not None and usx.book != ref.book:
-            raise ValueError("Reference book {} != text book {}".format(ref, usx.book))
+    if ref.book is not None and usx.book != ref.book:
+        raise ValueError("Reference book {} != text book {}".format(ref, usx.book))
     root = usx.getroot()
     c = ref.chapter
     foundend = False
@@ -331,11 +330,12 @@ class USXCursor(ETCursor):
                 p = p.parent
         else:
             p = a.el
-        i = list(root).index(p)
+        #i = list(root).index(p)
         allnew = {}
         res = factory(root.tag, attrib=root.attrib)
         currp = res
         curr = root
+        start = 0
         if addintro or titles:
             categories = ["versepara", "sectionpara"] + ["title", "introduction"] if not addintro else []
             def isendintro(e):
@@ -344,21 +344,22 @@ class USXCursor(ETCursor):
                 s = e.get("style", "")
                 return grammar.marker_categories.get(s, "") in categories
             # copy the tree up to the first chapter or verse text
-            for eloc in root:
+            for i, eloc in enumerate(root):
                 if isendintro(eloc):
+                    start = i
                     break
                 newp = eloc.copy(deep=True, parent=currp, factory=factory)
                 currp.append(newp)
             curr = root
             currp = res
 
-        for eloc, isin in iterusx(root, parindex=i, start=a.el, until=b.el, untilafter=bool(b.attrib)):
+        for eloc, isin in iterusx(root, parindex=start, start=a.el, until=b.el, untilafter=bool(b.attrib)):
             if isin and eloc == b.el:
                 break
             # at the start
-            elif isin and eloc == a.el and eloc.tag not in ("para", "book", "sidebar"):
+            elif isin and eloc == a.el and eloc.tag not in ("para", "book", "sidebar", "chapter"):
                 # if we are at the start of the parent (para, since a verse is always only in a para) check for subheadings
-                if headers and grammar and isempty(eloc.parent.text) and eloc.parent.index(eloc) == 0:  # are we first?
+                if headers and grammar and isempty(eloc.parent.text) and eloc.parent.index(eloc) == start:  # are we first?
                     r = eloc.parent.parent  # should be root
                     i = r.index(eloc.parent) - 1
                     # scan back over section heads

{usfmtc-0.4 → usfmtc-0.4.2}/src/usfmtc/usxmodel.py

@@ -5,7 +5,7 @@ from usfmtc.xmlutils import isempty, ParentElement
 from usfmtc.usfmparser import Grammar, WS
 from usfmtc.reference import Ref, RefRange, MarkerRef
 import xml.etree.ElementTree as et
-from typing import Optional
+from typing import Optional, Dict, List, Any, Tuple, Type, Union
 
 # This should be read from usx.rng
 allpartypes = {
@@ -226,16 +226,18 @@ def addindexes(root):
     return chapters, ids
 
 escapes = {
-    '\\' : '\\',
-    'n': '\n'
+    r'\n': '\n',
+    r'\~': '~',
+    r'\|': '|',
+    r'\\' : '\\',
+    '~': '\u00A0'
 }
 
+escapere = re.compile(r"(\\[n~\\|]|~)")
 def add_specials(t, node, parent, istext=False):
     if t is None:
         return t
-    t = re.sub(r'\\(.)', lambda m: escapes.get(m.group(1), "\\"+m.group(1)), t)
-    if "~" in t:
-        t = t.replace("~", "\u00A0")
+    t = escapere.sub(lambda m: escapes.get(m.group(1), m.group(1)[1:]), t)
     return t
 
 alignments = {
@@ -249,6 +251,7 @@ def _stripws(s, atend=False):
         return None
     return _wsere.sub("", s) if atend else _wssre.sub("", s)
 
+backre = re.compile(r"\\([\\~|])")
 def cleanup(node, parent=None):
     if node.tag == 'para':
         # cleanup spaces at start and end of para
@@ -305,7 +308,11 @@ def cleanup(node, parent=None):
             node = refnode
     # strip character escapes
     for k, v in node.attrib.items():
-        node.attrib[k] = re.sub(r"\\(.)", r"\1", v)
+        node.attrib[k] = backre.sub(r"\1", v)
+    #if node.text is not None:
+    #    node.text = backre.sub(r"\1", node.text)
+    #if node.tail is not None:
+    #    node.tail = backre.sub(r"\1", node.tail)
     for c in node:
         cleanup(c, parent=node)
 
@@ -382,18 +389,42 @@ def canonicalise(node, endofpara=False, factory=et, version=None):
                 inserted += 1
                 ft.text = c.tail.lstrip()
                 c.tail = c.tail[:len(c.tail)-len(ft.text)]
-        for i, c in enumerate(list(node)):
+        nodes = list(node)
+        i = 0
+        while i < len(nodes):
+            c = nodes[i]
             if c.get('style', '') not in notechars[0 if replace == "ft" else 1]:
-                if len(node) == 1:
+                if i > 0 and c.getprevious().get('style', '') == replace:
+                    prev = c.getprevious()
+                    prev.append(c)
+                    c.parent = prev
+                    if len(prev) > 1:
+                        prev[-2].tail = (prev[-2].tail or "") + (prev.tail or "")
+                    elif prev.tail is not None:
+                        prev.text += prev.tail
+                    prev.tail = c.tail
+                    c.tail = None
+                    if i < len(nodes) - 1 and nodes[i+1].get('style', '') == replace:
+                        # merge the nodes
+                        c.tail = prev.tail + nodes[i+1].text
+                        for n in nodes[i+1]:
+                            prev.append(n)
+                            n.parent = prev
+                        prev.tail = nodes[i+1].tail
+                        node.remove(nodes[i+1])
+                        i += 1
+                elif i < len(nodes) - 1 and nodes[i+1].get('style', '') == replace:
+                    nodes[i+1].insert(0, c)
+                    c.tail = nodes[i+1].text
+                    nodes[i+1].text = None
+                    c.parent = nodes[i+1]
+                else:
                     ft = node.makeelement("char", {"style": replace})
                     ft.append(c)
                     c.parent = ft
-                elif i == 0:
-                    node[1].insert(0, c)
-                else:
-                    node[i-1].append(c)
+                    node.insert(node.find(c), ft)
                 node.remove(c)
-                break
+            i += 1
     if [int(x) for x in version.split(".")] >= [3, 1] and node.tag == "char" and node.get("style", "") == "xt" \
                 and len(node) == 1 and isempty(node.text) and isempty(node[0].tail) and node[0].tag == "ref":
         i = node.parent.index(node)
@@ -856,7 +887,7 @@ def _getref(e, book, lastchap=0):
         ref = Ref(refstr)
     return ref
 
-def _list_insert(arr, k, v, default=0):
+def _list_insert(arr: List[int], k: int, v: int, default: int=0):
     if k >= len(arr):
         arr += [default] * (k - len(arr) + 1)
     arr[k] = v

{usfmtc-0.4 → usfmtc-0.4.2}/src/usfmtc/utils.py

@@ -5,13 +5,13 @@ import traceback
 
 logger = logging.getLogger(__name__)
 
-def readsrc(src):
+def readsrc(src, errors="replace"):
     if hasattr(src, "read"):
         return src.read()
     elif not isinstance(src, str):      # we're a parsed xml doc
         return src
-    elif len(src) < 128 and os.path.exists(src):
-        with open(src, encoding="utf-8") as inf:
+    elif os.path.exists(src):
+        with open(src, encoding="utf-8", errors=errors) as inf:
             data = inf.read()
         return data
     elif "\n" in src or len(src) > 127:
@@ -19,5 +19,15 @@ def readsrc(src):
     else:
         raise FileNotFoundError
 
+def getSrcName(src):
+    if hasattr(src, "name"):
+        return src.name
+    elif not isinstance(src, str):
+        return ""
+    elif os.path.isfile(src):
+        return src
+    else:
+        return ""
+
 def get_trace():
     return traceback.format_stack()

{usfmtc-0.4 → usfmtc-0.4.2/src/usfmtc.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: usfmtc
-Version: 0.4
+Version: 0.4.2
 Summary: USFM reference parser
 Author-email: USFM Technical Committee <fonts@sil.org>
 License: MIT

{usfmtc-0.4 → usfmtc-0.4.2}/tests/test_parser.py

@@ -36,7 +36,7 @@ def test_hardspaces():
     t = e.text
     if not t.endswith("\u00A0"):
         fail("No hard space after fr in usx")
-    if not re.search("\u00A0\\\\ft", f):
+    if not re.search("~\\\\ft", f):
         fail("No hard space after fr in usfm")
 
 def test_glossary():
@@ -246,3 +246,42 @@ def test_regularise():
     if not doc.getroot()[2][0].tail.endswith(" "):
         fail(f"Missing inserted space before v2 in {f}")
 
+def test_canonicalxt():
+    usfm = r"""\id JHN SLU
+\c 6
+\p
+\v 53 Lemadendye Yesuske tyanuk ti sir ma byohe, \wj “Nam kutanuk ti e ne mlay: kolnye ei neke lema mi Irkye Ananke tenanke,\wj*\f + \fr 6:53 \ft Ti ayat ne Yesuske byohe, \fq Irkye Ananke\fq*, neke Yesuske yal ma lyosu I a myesan bo. Kola ne ti ayat \xt 6:27\xt*ke ode \xt 6:62\xt*ke dakun. (Matos ti \xt Dan. 7:13-14\xt*.)\f* \wj ode lema myenw a larake, desikeo ana mormyorif manal nini namke lema kimin a e.\wj*
+\v 54 \wj Klala esei desike nya tenakkwe ode yenw a larakkwe, desikeo i desike mormyorif manal nini namke kimin a i ti kiba knaru ne, ode ana kaorif a i ti sew a nuske ktem ne ana kilodur ne.\wj*
+"""
+    doc, f = _dousfm(usfm)
+    if len(doc.getroot()[2][2]) != 4 or len(doc.getroot()[2][2][3]) != 3:
+        fail(f"Bad footnote canonicalising in {f}")
+
+def test_specialchars():
+    usfm = r"""\id JHN test
+\c 4
+\p
+\v 1 This tests \| pipe, \~ tilde and \\ backslash. Notice the hard~space
+"""
+    print(usfm)
+    doc, f = _dousfm(usfm)
+    if '\\\\\\' in f or '~' not in f:
+        fail(f"Bad character escaping in {f}")
+
+def test_emptyattrib():
+    usfm = r"""\id JHN empty attributes
+\p
+\v 1 \rb normal|good\rb* \rb test|\rb* \rb |empty\rb*
+"""
+    doc, f = _dousfm(usfm)
+    if 'test' not in f or 'empty' not in f:
+        fail("{f} is missing data")
+
+def test_tilde():
+    usfm = r"""\id JHN with tildes
+\p
+\v 1 This is a~test
+"""
+    doc, f = _dousfm(usfm)
+    if "~" not in f or "\u00A0" not in doc.getroot()[1][0].tail:
+        fail("{f} has bad hard space")

{usfmtc-0.4 → usfmtc-0.4.2}/tests/test_refs.py

@@ -434,7 +434,6 @@ def test_isvalid():
 
 def test_subdoc1():
     res = jon_usfm.getrefs(*RefList("JON 1:4-8; 4:9-11"), headers=True, titles=False)
-    et.dump(res.xml)
     f = res.outUsfm(None, forcevid=True)
     print(f)
     if "Acts 27:13-26" not in f and "vid|JON 4:9" not in f:
@@ -442,6 +441,14 @@ def test_subdoc1():
     if f.count('vid') != 2:
         fail(f"{f} wrong number of \\vid in the text")
 
+def test_subdoc2():
+    res = jon_usfm.getrefs(*RefList("JON 1:1-3"))
+    et.dump(res.getroot())
+    f = res.outUsfm(None, forcevid=True)
+    print(f)
+    if len(res.getroot()) != 7:
+        fail(f"{f} has {len(res.getroot())} children and not 7")
+
 def test_finalv():
     res = jon_usfm.getrefs(Ref("JON 1:17"), titles=False)
     et.dump(res.xml)
@@ -471,3 +478,28 @@ def test_rangeseq():
     print(r.str())
     if "19-2" in str(r):
         fail(f"{r} has {len(r)} refs: {r[0]}, {r[1]}")
+
+def test_subsetemptyfinal():
+    usfm = r"""\id ISA test
+\c 4
+\q1 \v 4 Ka wakava some other text
+\q2 This is a secondary line
+\q1 Now another line of poetry:
+\q2 with its secondary.
+\q1 More lines of poetry:
+\q2 with its secondary again.
+\b
+\q1
+\v 5 The start of another verse
+\q2 And the secondary line for this verse
+\s A subheading leading forward
+\q1
+\v 6 Another verse
+\q2 with its secondary
+"""
+    doc = readFile(usfm, informat="usfm")
+    doc.canonicalise()
+    subdoc = doc.getrefs(Ref("ISA 4:4-5"))
+    f = subdoc.outUsfm(None)
+    if '\\s' in f:
+        fail(f"{f} has a final section")