usfmtc 0.3.2__tar.gz → 0.3.3__tar.gz

{usfmtc-0.3.2/src/usfmtc.egg-info → usfmtc-0.3.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: usfmtc
-Version: 0.3.2
+Version: 0.3.3
 Summary: USFM reference parser
 Author-email: USFM Technical Committee <fonts@sil.org>
 License-Expression: MIT

{usfmtc-0.3.2 → usfmtc-0.3.3}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "usfmtc"
-version = "0.3.2"
+version = "0.3.3"
 authors = [{name = "USFM Technical Committee", email = "fonts@sil.org"}]
 readme = "python/README.md"
 license = "MIT"

{usfmtc-0.3.2 → usfmtc-0.3.3}/src/usfmtc/reference.py

@@ -97,35 +97,6 @@ class MarkerRef:
             res.append(str(self.char))
         return "".join(res)
 
-
-_regexes = {
-    "book": r"""(?P<transid>(?:[a-z0-9_-]*[+])*)
-                    (?P<book>(?:[A-Z][A-Z0-9][A-Z0-9]|[0-9](?:[A-Z][A-Z]|[0-9][A-Z]|[A-Z][0-9])))
-                    \s*{chap}""",
-    "id": r"(?:[a-z][a-z0-9_-]*[a-z0-9]?)",
-    "charref": r"(?:(?:[0-9]|end)[+]?)",
-    "wordrefanon": r"",
-    "mrkref": r"(?:\!{id}(?:\[[0-9]\])?(?:\!(?:[0-9|end)(?:[+]{charref})?)?)",
-    "wordrefonly": r"(?P<word>[0-9]|end)(?P<char>[+]{charref})?",
-    "wordref1": r"""(?:(?: \!(?P<word1>[0-9]|end)(?:[+](?P<char1>{charref}))?
-                          |(?P<mrk1>{mrkref}))
-                       (?P<mrkn1>{mrkref}*))""",
-    "wordref2": r"""(?:(?: \!(?P<word2>[0-9]|end)(?:[+](?P<char2>{charref}))?
-                          |(?P<mrk2>{mrkref}))
-                       (?P<mrkn2>{mrkref}*))""",
-    "verse1": r"(?P<verse1>[0-9]+)(?P<subv1>[a-z]?|end)",
-    "verse2": r"(?P<verse2>[0-9]+)(?P<subv2>[a-z]?|end)",
-    "chapter": r"(?:(?P<chap>[0-9]+))",
-    "chap": r"{chapter}(?:[:.]{verse1})?{wordref1}?",
-    "context": r"""(?:{chap}
-                    | {verse2}{wordref2}? | {wordrefonly}(?P<mrkn3>{mrkref}*)
-                    | (?P<char3>{charref})(?P<mrkn4>{mrkref}*))(?=[-;,\s]|$)"""
-    }
-
-regexes = _regexes
-for i in range(3):
-    regexes = {k: v.format(**regexes) for k, v in regexes.items()}
-
 def intend(s: str) -> Optional[int]:
     if not s:
         return None
@@ -198,6 +169,35 @@ def readvrs(fname):
     return res
 
 
+_regexes = {
+    "book": r"""(?P<transid>(?:[a-z0-9_-]*[+])*)
+                    (?P<book>(?:[A-Z][A-Z0-9][A-Z0-9]|[0-9](?:[A-Z][A-Z]|[0-9][A-Z]|[A-Z][0-9])))
+                    \s*{chap}""",
+    "id": r"(?:[a-z][a-z0-9_-]*[a-z0-9]?)",
+    "charref": r"(?:(?:[0-9]|end)[+]?)",
+    "wordrefanon": r"",
+    "mrkref": r"(?:\!{id}(?:\[[0-9]\])?(?:\!(?:[0-9]|end)(?:[+]{charref})?)?)",
+    "wordrefonly": r"(?P<word>[0-9]|end)(?P<char>[+]{charref})?",
+    "wordref1": r"""(?:(?: \!(?P<word1>[0-9]|end)(?:[+](?P<char1>{charref}))?
+                          |(?P<mrk1>{mrkref}))
+                       (?P<mrkn1>{mrkref}*))""",
+    "wordref2": r"""(?:(?: \!(?P<word2>[0-9]|end)(?:[+](?P<char2>{charref}))?
+                          |(?P<mrk2>{mrkref}))
+                       (?P<mrkn2>{mrkref}*))""",
+    "verse1": r"(?P<verse1>[0-9]+|end)(?P<subv1>[a-z]?)",
+    "verse2": r"(?P<verse2>[0-9]+|end)(?P<subv2>[a-z]?)",
+    "chapter": r"(?:(?P<chap>[0-9]+|end))",
+    "chap": r"{chapter}(?:[:.]{verse1})?{wordref1}?",
+    "context": r"""(?:{chap}
+                    | {verse2}{wordref2}? | {wordrefonly}(?P<mrkn3>{mrkref}*)
+                    | (?P<char3>{charref})(?P<mrkn4>{mrkref}*))(?=[-;,\s]|$)"""
+    }
+
+regexes = _regexes
+for i in range(3):
+    regexes = {k: v.format(**regexes) for k, v in regexes.items()}
+
+
 class Ref:
     product: Optional[str]
     book: Optional[str]
@@ -232,20 +232,21 @@ class Ref:
     def __init__(self, string: Optional[str] = None,
                     context: Optional['Ref'] = None, start: int = 0, **kw):
         if string is not None:
-            self.parse(string, context=context, start=start)
+            s = string.strip()
+            self.parse(s, context=(context.last if context is not None else None), start=start)
+            if self.strend < len(s):
+                raise SyntaxError(f"Extra content after reference {s[0:self.strend]} | {s[self.strend:]}")
             return
 
         if context is not None:
-            isempty = all(not v for k, v in kw.items() if k != "word")
-            rel = None
-            if isempty:
-                rel = kw["word"]
-                kw["word"] = None
-                idx = [k for k in self._parmlist if getattr(context.last, k, None)][-1]
+            hitlimit = False
             for a in self._parmlist:
-                setattr(self, a, kw.get(a, getattr(context.last, a, None)))
-            if rel is not None:
-                setattr(self, idx, rel)
+                v = kw.get(a, None)
+                if v is not None:
+                    hitlimit = True
+                elif not hitlimit:
+                    v = getattr(context.last, a, None)
+                setattr(self, a, v)
         else:
             for a in self._parmlist:
                 setattr(self, a, kw.get(a, None))
@@ -254,54 +255,31 @@ class Ref:
         if "-" in s:
             return 
         p = {}
+        s = s.strip()
         if m := self._rebook.match(s, pos=start):
             p['product'] = m.group('transid') or None
             p['book'] = m.group('book')
-            p['chapter'] = int(m.group('chap')) if m.group('chap') else None
-            p['verse'] = intend(m.group('verse1'))
-            p['subverse'] = m.group('subv1') or None
-            p['word'] = intend(m.group('word1'))
-            p['char'] = intend(m.group('char1'))
-            p['mrkrs'] = asmarkers(m.group("mrk1"), m.group("mrkn1"))
         elif not (m:= self._recontext.match(s, pos=start)):
             raise SyntaxError("Cannot parse {}".format(s))
-        elif m.group('verse1'):       # we know we have a chapter and verse
-            p['chapter'] = int(m.group('chap'))
-            p['verse'] = intend(m.group('verse1'))
-            p['subverse'] = m.group('subv1') or None
-            p['word'] = intend(m.group('word1'))
-            p['char'] = intend(m.group('char1'))
-            p['mrkrs'] = asmarkers(m.group("mrk1"), m.group("mrkn1"))
-        elif m.group('subv2'):
-            p['verse'] = intend(m.group('verse2'))
-            p['subverse'] = m.group('subv2') or None
-            p['word'] = intend(m.group('word2'))
-            p['char'] = intend(m.group('char2'))
-            p['mrkrs'] = asmarkers(m.group("mrk2"), m.group("mrkn2"))
-        elif m.group('word'):
-            p['word'] = intend(m.group('word'))
-            p['char'] = intend(m.group('char'))
-            p['mrkrs'] = asmarkers("", m.group('mrkn3'))
-        elif m.group('char3'):
-            p['char'] = intend(m.group('char3'))
-            p['mrkrs'] = asmarkers("", m.group('mrkn4'))
-        elif not m.group('chap') and m.group('word1'):
-            p['word'] = intend(m.group('word1'))
-            p['char'] = intend(m.group('char1'))
-            p['mrkrs'] = asmarkers(m.group('mrk1'), m.group('mrkn1'))
-        elif context is not None and context.char:
-            p['char'] = intend(m.group('chap'))
-        elif context is not None and context.word:
-            p['word'] = intend(m.group('chap'))
-            p['char'] = intend(m.group('char1'))
-        elif context is not None and context.verse:
-            p['verse'] = intend(m.group('chap'))
-            p['word'] = intend(m.group('word1'))
-            p['char'] = intend(m.group('char1'))
-        else:
-            p['chapter'] = int(m.group('chap'))
-            p['word'] = intend(m.group('word1'))
-            p['char'] = intend(m.group('char1'))
+        gs = m.groupdict()
+        p['chapter'] = intend(gs.get('chap', None))
+        p['verse'] = intend(gs.get('verse1', gs.get('verse2', None)))
+        p['subverse'] = gs.get('subv1', gs.get('subv2', None)) or None
+        p['word'] = intend(gs.get('word1', gs.get('word2', gs.get('word', None))))
+        p['char'] = intend(gs.get('char', gs.get('char1', gs.get('char2', None))))
+        mnum = "2" if 'mrk2' in gs else "1"
+        p['mrkrs'] = asmarkers(gs.get('mrk'+mnum, None), gs.get('mrkn'+mnum, None))
+        if 'book' not in p and context is not None and p['chapter'] is not None and p['verse'] is None and p['word'] is None:
+            rep = None
+            if context.char is not None:
+                rep = 'char'
+            elif context.word is not None:
+                rep = 'word'
+            elif context.verse is not None:
+                rep = 'verse'
+            if rep is not None:
+                p[rep] = p['chapter']
+                p['chapter'] = None 
         self.strend = m.end(0)
         if p.get('mrkrs', None) == []:
             p['mrkrs'] = None
@@ -319,7 +297,7 @@ class Ref:
     def __eq__(self, o):
         if not isinstance(o, Ref):
             return False
-        res = all(getattr(self, a) == getattr(o, a) for a in self._parmlist)
+        res = all(getattr(self, a) is None or getattr(o, a) is None or getattr(self, a) == getattr(o, a) for a in self._parmlist)
         return res
 
     def __contains__(self, o):
@@ -409,7 +387,7 @@ class Ref:
         if self.mrkrs is not None and len(self.mrkrs):
             forcemkr = False
             for i, m in enumerate(self.mrkrs):
-                if not forcemkr and context is not None and i < len(context.mrkrs):
+                if not forcemkr and context is not None and context.mrkrs is not None and i < len(context.mrkrs):
                     if context.mrkrs[i] != m:
                         res.append(m.str(context=contex.mrkrs[i], force=force or len(res)))
                         forcemkr = True
@@ -429,27 +407,57 @@ class Ref:
         kw = {k: getattr(self, k) for k in self._parmlist}
         return self.__class__(**kw)
 
-    def end(self):
+    def _setall(self, val):
         res = self.copy()
-        if self.chapter is None:
-            res.chapter = -1
-        elif self.verse is None:
-            res.verse = -1
-        elif self.word is None:
-            res.word = -1
-        elif self.char is None:
-            res.char = -1
+        for a in ('chapter', 'verse', 'word', 'char'):
+            if getattr(self, a, None) is None:
+                setattr(res, a, val)
+        return res
+
+    def end(self):
+        if self.last != self:
+            self.last = end(self.last)
+            return self.last
+        res = self._setall(-1)
+        if res.chapter == -1:
+            vrs = self.versification or Ref.loadversification()
+            book = books.get(self.book, None)
+            if book is not None:
+                res.chapter = len(vrs[book])
+        if res.verse == -1:
+            res.verse = self._getmaxvrs(self.book, self.chapter)
         return res
 
+    def start(self):
+        if self.first != self:
+            self.first = start(self.first)
+            return self.first
+        return self._setall(1)
+
+    def expand(self):
+        return RefRange(self.start(), self.end())
+
     def _getmaxvrs(self, bk, chap):
         vrs = self.first.versification or Ref.loadversification()
         book = books.get(bk, None)
         if book is None or len(vrs[book]) <= chap:
-            maxvrs = 200
+            maxvrs = -1
         else:
             maxvrs = vrs[book][chap] - (vrs[book][chap-1] if chap > 1 else 0)
         return maxvrs
 
+    def isvalid(self):
+        vrs = self.first.versification or Ref.loadversification()
+        if self.book not in books:
+            return False
+        book = vrs[books.get(self.book)]
+        if len(book) < self.chapter:
+            return False
+        if book[self.chapter] < self.verse:
+            return False
+        # checking subverse, word and char involves having access to the document
+        return True
+
     def nextverse(self):
         r = self.first.copy()
         maxvrs = self._getmaxvrs(r.book, r.chapter)
@@ -458,7 +466,7 @@ class Ref:
         else:
             r.verse += 1
         if r.verse > maxvrs:
-            r.chapter += 1
+            r.chapter = (r.chapter + 1) if r.chapter is not None else 1
             r.verse = 1
             if r.book not in books:
                 r.book = "GEN"
@@ -483,6 +491,8 @@ class RefRange:
     def __init__(self, first: Optional[Ref]=None, last: Optional[Ref]=None):
         self.first = first
         self.last = last
+        if self.last < self.first:
+            raise ValueError(f"{first=} is after {last=}")
 
     def str(self, context: Optional[Ref] = None, force: bool = False):
         res = [self.first.str(context, force=force)]
@@ -523,6 +533,14 @@ class RefRange:
     def strend(self):
         return self.last.strend
 
+    def expand(self):
+        self.first = self.first.start()
+        self.last = self.last.end()
+        return self
+
+    def isvalid(self):
+        return self.first.isvalid() and self.last.isvalid()
+
     def allverses(self):
         r = self.first
         while r is not None and r <= self.last:

{usfmtc-0.3.2 → usfmtc-0.3.3}/src/usfmtc/usxmodel.py

@@ -3,6 +3,7 @@ import re
 from dataclasses import dataclass
 from usfmtc.xmlutils import isempty
 from usfmtc.usfmparser import Grammar
+from usfmtc.reference import Ref
 import xml.etree.ElementTree as et
 from typing import Optional
 
@@ -259,11 +260,11 @@ def strnormal(s, t, mode=0):
     '''
     if s is None:
         return ""
-    res = re.sub("[\n\\s]+", " ", s) if t in ('para', 'char') else s
+    res = re.sub("[\n\t\r ]+", " ", s) if t in ('para', 'char') else s
     if mode & 1 == 1:
-        res = res.lstrip()
+        res = res.lstrip(" \t\n\r")
     if mode & 2 == 2:
-        res = res.rstrip()
+        res = res.rstrip(" \t\n\r")
     res = re.sub(r"[ \n]*\n[ ]*", "\n", res)
     for k, v in unescapes.items():
         if k in res:
@@ -442,6 +443,8 @@ def iterusx(root, parindex=0, start=None, blocks=[], filt=[], unblocks=False, gr
     yield from runiter(root, parindex=parindex, started=start is None)
 
 def copy_range(root, a, b):
+    ''' Returns a usx document containing paragraphs containing the content
+        between a and b '''
     factory = root.__class__
     if a.el not in root:
         p = a.el.parent
@@ -482,6 +485,52 @@ def copy_range(root, a, b):
             res.remove(last)
     return res
 
+def copy_text(root, a, b):
+    ''' Returns a text string of all the main text between a and b '''
+    res = []
+    if a.el not in root:
+        p = a.el.parent
+        while p not in root:
+            p = p.parent
+    else:
+        p = a.el
+    i = list(root).index(p)
+    for eloc in iterusx(root, parindex=i, start=a.el, until=b.el):
+        if eloc.head is None:
+            if not isempty(eloc.parent.text):
+                start = 0; end = len(eloc.parent.text)
+                if eloc.parent == a.el:
+                    if a.attrib == " text":
+                        start = a.char
+                if eloc.parent == b.el:
+                    if b.attrib == " text":
+                        end = b.char
+                res.append(eloc.parent.text[start:end])
+        elif eloc.head == a.el and a.attrib == " tail":
+            res.append(eloc.head.tail[a.char:])
+        elif eloc.head == b.el and b.attrib == " text":
+            pass
+        elif not isempty(eloc.head.tail):
+            res.append(eloc.head.tail)
+    if not len(res) and a.attrib == " tail":
+        start = a.char
+    else:
+        start = 0
+    if b.attrib == " tail":
+        res.append(b.el.tail[start:b.char])
+    return "".join(res)
+    
+
+def vnumin(s, v, base):
+    b = base.copy()
+    b.verse = None
+    b.subverse = None
+    b.word = None
+    b.char = None
+    b.mrkrs = None
+    r = Ref(str(b)+":"+s)
+    if v >= r.first.verse and v <= r.last.verse:
+        return True
 
 @dataclass
 class USXLoc:
@@ -504,6 +553,8 @@ def _findel(node, tag, attrib, limits=[]):
 
 def _findtext(node, limits):
     """ Iterator returning text strings until an element in limits is encountered """
+    if not isempty(node.text):
+        yield(node.text, node, " text")
     for eloc in iterusx(node, until=lambda e: e.tag in limits):
         if eloc.head is None:
             if not isempty(eloc.parent.text):
@@ -513,7 +564,7 @@ def _findtext(node, limits):
     if not isempty(node.tail):  # but what if we hit the limits?
         yield(node.tail, node, " tail")
 
-def _findcvel(ref, root, atend=False, parindex=0):
+def _findcvel(ref, root, grammar, atend=False, parindex=0):
     if ref.book:
         bkel = root.find("./book")
         bk = bkel.get("code", "")
@@ -539,31 +590,74 @@ def _findcvel(ref, root, atend=False, parindex=0):
     v = ref.verse
     if v is not None and v > 0:
         if atend:
-            parindex = startparindex
-        if parindex < len(root) - 1:
-            for n in root[parindex+1:]:
-                el = _findel(n,  "verse", {"number": str(ref.verse + (1 if atend and not ref.word else 0))}, limits=("chapter",))
-                if el is not None:
-                    break
+            parindex = startparindex + 1
+            if not ref.mrkrs and not ref.word and not ref.char:
+                v += 1
+        else:
+            parindex += 1
+        while parindex < len(root):
+            n = root[parindex]
+            el = _findel(n,  "verse", {"number": str(v)}, limits=("chapter",))
+            if el is not None:
+                break
+            parindex += 1
+        else:
+            raise ValueError("Reference verse {} out of range".format(ref))
+    if ref.mrkrs:
+        if not ref.word and not ref.char:
+            oparindex = parindex
+            t = ref.mrkrs[0].mrkr   # look forward or backwards for a para
+            if grammar.marker_categories.get(t, "") in ("sectionpara",):
+                while parindex > 0:
+                    p = root[parindex]
+                    if p.get("style", "") == t:
+                        if ref.mrkrs[0].index is None or ref.mrkrs[0].index - 1 == count:
+                            foundme = True
+                            break
+                    parindex -= 1
+                    endme = False
+                    for e in root[parindex]:
+                        if e.tag == "verse" and not vnumin(e.get("number", ""), v, ref.last if atend else ref.first):
+                            endme = True
+                            break
+                    if endme:
+                        parindex = oparindex
+                        raise SyntaxError("Cannot find preceding section paragraph {t} in {ref}")
+                    else:
+                        el = root[parindex]
             else:
-                raise ValueError("Reference verse {} out of range".format(ref))
+                for eloc in iterusx(root, parindex=parindex, start=lambda e:e.tag == "verse" and vnumin(e.get("number", ""), v, ref.last if atend else ref.first), until=lambda e:e.tag == "verse" and not vnumin(e.get("number", ""), v, ref.last if atend else ref.first)):
+                    if eloc.head is not None:
+                        continue
+                    if eloc.parent.get("style", "") == t:
+                        el = eloc.parent
+                        break
     return el
 
-def _findtextref(ref, el, atend=False):
+def _findtextref(ref, el, atend=False, mrkri=0):
+    if mrkri > 0:
+        r = ref.mrkrs[mrkri-1]
+    else:
+        r = ref
     a = ""
     w = -1
     c = -1
     clen = -1
     t = ""
-    if ref.word is None or ref.word == 0:
+    if r.word is None or r.word == 0:
         t = el.get('number', "")
         a = "number"
         c = 0
         clen = len(t)
     else:
-        word = ref.word - 1
+        word = r.word - 1
         for t, el, a in _findtext(el, limits=("chapter", "verse")):
             b = re.split("([\u0020\u00A0\u1680\u2000-\u200B\u202F\u205F\u3000]+)", t)
+            if not len(b[-1]):
+                b.pop()
+            if not len(b[0]):
+                b.pop(0)
+                b.pop(0)
             l = (len(b) + 1) // 2
             if not len(b[0]):
                 l -= 1
@@ -572,24 +666,38 @@ def _findtextref(ref, el, atend=False):
                 continue
             c = sum(len(s) for s in b[:word*2])
             clen = len(b[word*2])
+            if ref.mrkrs and len(ref.mrkrs) > mrkri and c + clen == len(t.rstrip()):
+                en = el.getnext()
+                while en is not None:
+                    if en.get("style", "") == ref.mrkrs[mrkri].mrkr:
+                        break
+                else:
+                    raise SyntaxError(f"Can't find {ref.mrkrs[mrkri+1].mrkr}")
+                return _findtext(ref, en, atend=atend, mrkri=mrkri+1)
             break
         else:
             raise ValueError("Reference word {} out of range".format(ref))
-    if ref.char is not None:
-        if ref.char > clen:
+    if r.char is not None:
+        if r.char > clen:
             raise ValueError("Reference char {} out of range for word length {}".format(ref, clen))
-        c += ref.char
+        c += r.char
     elif atend:
         c += clen
     return USXLoc(el, a, c)
 
-def findref(ref, root, atend=False, parindex=0):
+def findref(ref, root, atend=False, parindex=0, grammar=None):
     """ From a reference, return a USXLoc (element, attribute and char index) """
-    el = _findcvel(ref, root, atend=atend, parindex=parindex)
+    if grammar is None:
+        grammar = Grammar()
+    el = _findcvel(ref, root, grammar, atend=atend, parindex=parindex)
     if ref.word is not None or ref.char is not None:
         res = _findtextref(ref, el, atend=atend)
+    elif ref.mrkrs:
+        res = _findtextref(ref, el, atend=atend, mrkri=1)
+    elif atend and len(el):
+        res = USXLoc(el[-1], " tail", -1)
     else:
-        res = USXLoc(el, "", -1)
+        res = USXLoc(el, " text", -1 if atend else 0)
     # What about markers?
     return res
 

{usfmtc-0.3.2 → usfmtc-0.3.3/src/usfmtc.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: usfmtc
-Version: 0.3.2
+Version: 0.3.3
 Summary: USFM reference parser
 Author-email: USFM Technical Committee <fonts@sil.org>
 License-Expression: MIT

{usfmtc-0.3.2 → usfmtc-0.3.3}/src/usfmtc.egg-info/SOURCES.txt

@@ -21,4 +21,5 @@ src/usfmtc/validating/rngparser.py
 src/usfmtc/validating/usfmgrammar.py
 src/usfmtc/validating/usfmparser.py
 src/usfmtc/validating/usxparser.py
+tests/test_parser.py
 tests/test_refs.py

usfmtc-0.3.3/tests/test_parser.py

@@ -0,0 +1,40 @@
+import pytest
+from pytest import fail
+import usfmtc
+import xml.etree.ElementTree as et
+import re, io
+
+def asusfm(root):
+    grammar = usfmtc.Grammar()
+    with io.StringIO() as fh:
+        usfmtc.usx2usfm(fh, root, grammar)
+        res = fh.getvalue()
+    return res
+
+def _dousfm(s):
+    doc = usfmtc.readFile(s, informat="usfm")
+    doc.canonicalise()
+    r = doc.getroot()
+    et.dump(r)
+    f = asusfm(r)
+    print(f)
+    return (doc, f)
+
+def test_hardspaces():
+    usfm = r"""\id TST testing hardspaces
+\c 1 \p \v 1 This{0}has a hard{0}space\f + \fr 1:1{0}\ft And here\f*""".format("\u00A0")
+    doc, f = _dousfm(usfm)
+    r = doc.getroot()
+    e = r.find('.//char[@style="fr"]')
+    t = e.text
+    if not t.endswith("\u00A0"):
+        fail("No hard space after fr in usx")
+    if not re.search("\u00A0\\\\ft", f):
+        fail("No hard space after fr in usfm")
+
+def test_glossary():
+    usfm = r"""\id TST glossary in text
+\c 1 \p \v 1 We have \w glossary\w* words to deal with"""
+    doc, f = _dousfm(usfm)
+    if not re.search(" \\w", f):
+        fail("No space before glossary word")

usfmtc-0.3.3/tests/test_refs.py

@@ -0,0 +1,365 @@
+import pytest
+from pytest import fail
+from usfmtc import readFile, usx2usfm
+from usfmtc.usxmodel import findref, copy_range, copy_text
+from usfmtc.reference import Ref, RefList, RefRange
+import os, io
+import xml.etree.ElementTree as et
+
+# To see the print() output of passing tests, run pytest -s
+
+def check_ref(ref, product=None, book=None, chapter=None, verse=None, subverse=None,
+                   word=None, char=None, mrkrs=None):
+    r = Ref(product=product, book=book, chapter=chapter, verse=verse, subverse=subverse,
+                    word=word, char=char, mrkrs=mrkrs)
+    return r == ref
+
+def asusfm(root):
+    with io.StringIO() as fh:
+        usx2usfm(fh, root)
+        res = fh.getvalue()
+    return res
+
+jon_usfm = readFile(os.path.join(os.path.dirname(__file__), "32JONBSB.usfm"))
+
+def _get_textref(s):
+    r = Ref(s)
+    root = jon_usfm.getroot()
+    start = findref(r.first, root)
+    end = findref(r.last, root, atend=True)
+    res = copy_text(root, start, end)
+    return res
+
+def _r(bk, chap, verse, subv=None):
+    return Ref(book=bk, chapter=chap, verse=verse, subverse=subv)
+
+def _t(s, *r):
+    res = RefList(s)
+    if len(res) != len(r):
+        fail("Failed '{}' resulted in {} references, {}, rather than {}".format(s, len(res), res, len(r)))
+    for z in zip(res, r):
+        if z[0] != z[1]:
+            fail("Reference list failed '{}', {} != {}".format(s, z[0], z[1]))
+    if str(res) != s:
+        fail("{} != canonical string of {}".format(s, str(res)))
+    print("{} = {}".format(s, str(res)))
+
+def _listtest(s, a):
+    res = RefList(s)
+    res.simplify()
+    base = RefList(a)
+    if res != base:
+        fail("{} simplified to {} instead of {}".format(s, res, base))
+    print("{} -> {}".format(s, res))
+
+def _o(a, b, res):
+    x = RefList(a)[0]
+    y = RefList(b)[0]
+    if (isinstance(res,tuple)):
+      restup=res
+      restupd=f"x<y {restup[0]}, y<x {restup[1]}, x in y {restup[2]}"
+    else:
+      restup=(res,res,res)
+      restupd=res
+    a = (y<x)
+    b = (x<y)
+    #print(f"y<x :{a} x<y:{b}")
+    if restup[0] != b:
+        raise fail(f"{x} should be < {y} [expect: {restupd}]")
+    if restup[1] != a:
+        raise fail(f"{y} should not be < {x} [expect: {restupd}]")
+    if (x in y) != restup[2]:
+        raise fail(f"{x} in {y} should be {res}")
+    print("{} <-> {} ({})".format(x, y,restupd))
+
+def _do_findreftest(first, last):
+    root = jon_usfm.getroot()
+    start = findref(first, root)
+    end = findref(last, root, atend=True)
+    res = copy_range(root, start, end)
+
+    print(f"{first=} {start}; {last=} {end}")
+    et.dump(res)
+    print(asusfm(res))
+    if len(res) != 7 or len(res[2]) != 1 or "vomited" not in res[-1][-1].tail:
+        fail(f"{len(res)=}, {res[-1][-1].tail=}")
+
+def test_basic():
+    r = Ref("JHN 3:16")
+    rr = Ref("12", context=r)
+
+    if not check_ref(r, None, "JHN", 3, 16):
+        fail(f"{r} is not JHN 3:16")
+    if not check_ref(rr, None, "JHN", 3, 12):
+        fail(f"{rr} is not JHN 3:12")
+
+def test_range():
+    r = RefList("JHN 3:16-18")
+
+    if not check_ref(r[0].first, None, "JHN", 3, 16):
+        fail(f"{r[0].first} is not JHN 3:16")
+    if not check_ref(r[0].last, None, "JHN", 3, 18):
+        fail(f"{r[0].last} is not JHN 3:18")
+
+def test_findreftext():
+    r = Ref("JON 3:6!3")
+    loc = findref(r, jon_usfm.getroot())
+
+    if loc.el.tag != "verse" and loc.attrib != " tail" and loc.char != 18:
+        fail(f"{r} resulted in {loc}")
+    end = Ref("JON 3:6!4")
+    eloc = findref(end, jon_usfm.getroot(), atend=True)
+    if eloc.el.tag != "verse" and eloc.attrib != " tail" and eloc.char !=24:
+        fail(f"{end} resulted in {eloc}")
+    res = eloc.el.tail[loc.char:eloc.char]
+    if res != "reached the":
+        fail(f"{res} != 'reached the'")
+
+def test_findref1():
+    first = Ref("JON 2:8")
+    last = Ref("JON 2")
+    _do_findreftest(first, last)
+
+def test_findref2():
+    ref = Ref("JON 2:8-end")
+    _do_findreftest(ref.first, ref.last)
+
+def test_getrefs():
+    refs = RefList("JON 1:3; 2:4-5; 3:9-end")
+    res = jon_usfm.getrefs(*refs)
+    root = res.getroot()
+    print(res.outUsfm())
+    if len(root) != 14 or root[0].get("vid", "") != "JON 1:3":
+        fail(f"{len(root)=}")
+
+def test_textref1():
+    r = Ref("JON 2:8!2-4")
+    root = jon_usfm.getroot()
+    start = findref(r.first, root)
+    end = findref(r.last, root, atend=True)
+    print(start, end)
+    res = copy_text(jon_usfm.getroot(), start, end)
+    print(res)
+    if res != "who cling to":
+        fail(f"{res}")
+
+def test_gen1():
+    _t("GEN 1:1", _r("GEN", 1, 1))
+
+def test_jhn3():
+    _t("JHN 3", _r("JHN", 3, None))
+
+def test_3jn():
+    _t("3JN 3", _r("3JN", 1, 3))
+
+def test_1co():
+    _t("1CO 6:5a", _r("1CO", 6, 5, "a"))
+
+def test_mat5():
+    _t("MAT 5:1-7", RefRange(_r("MAT", 5, 1), _r("MAT", 5, 7)))
+
+def test_mat7():
+    _t("MAT 7:1,2; 8:6b-9:4", _r("MAT", 7, 1), _r("MAT", 7, 2), RefRange(_r("MAT", 8, 6, "b"), _r("MAT", 9, 4)))
+
+def test_luk3():
+    _t("LUK 3:35-end", RefRange(_r("LUK", 3, 35), _r("LUK", 3, -1)))
+
+def test_rom1():
+    _listtest("ROM 1; MAT 3:4-11; ROM 1:3-2:7", "MAT 3:4-11; ROM 1-2:7")
+
+def test_gen1_1():
+    _t("GEN 1:1-3; 3:2-11; LUK 4:5", RefRange(_r("GEN", 1, 1), _r("GEN", 1, 3)), RefRange(_r("GEN", 3, 2), _r("GEN", 3, 11)), _r("LUK", 4, 5))
+
+def test_jud():
+    _t("JUD 1,2,4", _r("JUD", 1, 1), _r("JUD", 1, 2), _r("JUD", 1, 4))
+
+def test_gen1_1a():
+    _o("GEN 1:1", "EXO 2:3", (True,False,False))
+
+def test_exo2():
+    _o("EXO 2:4", "EXO 2", (False,True,True))
+
+def test_exo2a():
+    _o("EXO 2:4-5", "EXO 2", (False,True,True))
+
+def test_exo2b():
+    _o("EXO 2:4-5", "EXO 3", (True,False,False))
+
+def text_exo2c():
+    _o("EXO 2:1-2", "EXO 2:1-5", (False,False,True))
+
+def test_exo2d():
+    _o("EXO 2:1-2", "EXO 2:2-7", (True,False,True))
+
+def test_exo2e():
+    _o("EXO 2:1-2", "EXO 2:3-6", (True,False,False))
+
+def test_gen2():
+    _o("GEN 2:1-2", "EXO 2:3-6", (True,False,False))
+
+def test_deu2():
+    _o("DEU 2:1-2", "EXO 2:3-6", (False,True,False))
+
+def test_exo2f():
+    _o("EXO 2:2-3", "EXO 2:1-5", (False,False,True))
+
+
+# ChatGPT tests:
+def test_invalid_ref():
+    try:
+        r = Ref("INVALID 1:1")
+        fail("Invalid reference did not raise an error")
+    except SyntaxError:
+        pass
+
+def test_cross_book_range():
+    r = Ref("JON 3:10 - MIC 1:2")
+    if not check_ref(r.first, None, "JON", 3, 10):
+        fail(f"{r.first} is not JON 3:10")
+    if not check_ref(r.last, None, "MIC", 1, 2):
+        fail(f"{r.last} is not MIC 1:2")
+
+def test_partial_chapter():
+    r = Ref("JON 2")
+    if not check_ref(r.first, None, "JON", 2, None):
+        fail(f"{r.first} is not JON 2")
+
+def test_textref_boundary():
+    res = _get_textref("JON 3:10!1-2")
+    print(res)
+    if res != "When God":
+        fail(f"Unexpected result: {res}")
+
+def test_multiple_ranges():
+    refs = RefList("JON 1:1-3; 2:4; 3:8-end")
+    res = jon_usfm.getrefs(*refs)
+    root = res.getroot()
+    et.dump(root)
+
+    if len(root) != 11 or root[0].get("vid", "") != "JON 1:1":
+        fail(f"Incorrect range extraction: {len(root)=}")
+
+def test_chapter_only_range():
+    r = Ref("JON 3").expand()
+    if not check_ref(r.first, None, "JON", 3, 1):
+        fail(f"{r.first} is not JON 3:1")
+
+def test_single_verse():
+    r = Ref("JON 1:1")
+    if not check_ref(r.first, None, "JON", 1, 1):
+        fail(f"{r.first} is not JON 1:1")
+
+def test_multi_book_list():
+    refs = RefList("JON 1:1; MIC 2:3; NAH 1:7")
+    if len(refs) != 3:
+        fail("Expected three references")
+
+def test_edge_case_verse():
+    r = Ref("JON 4:11")
+    if not check_ref(r.first, None, "JON", 4, 11):
+        fail(f"{r.first} is not JON 4:11")
+
+def test_reverse_range():
+    try:
+        r = RefList("JON 3:5 - JON 2:1")
+        fail("Reverse range did not raise an error")
+    except ValueError:
+        pass
+
+def test_reference_with_suffix():
+    r = Ref("JON 3:10a")
+    if not check_ref(r.first, None, "JON", 3, 10):
+        fail(f"{r.first} is not JON 3:10a")
+    if not check_ref(r.first, None, "JON", 3, 10, "a"):
+        fail(f"{r.first} is not JON 3:10")
+
+def test_overlapping_ranges():
+    refs = RefList("JON 1:3-5; 1:4-6")
+    if len(refs) != 2:
+        fail("Expected two overlapping references")
+
+def test_whitespace_in_ref():
+    try:
+        r = Ref("  JON   2 :  3 ")
+        fail(f"{r.first} has whitespace in it and should fail")
+    except SyntaxError:
+        pass
+
+def test_missing_chapter():
+    try:
+        r = Ref("JON :5")
+        fail("Missing chapter did not raise an error")
+    except SyntaxError:
+        pass
+
+def test_long_reference():
+    r = RefList("JON 1:1-4:11")
+    if not (check_ref(r[0].first, None, "JON", 1, 1) and check_ref(r[0].last, None, "JON", 4, 11)):
+        fail("Long reference not parsed correctly")
+
+def test_empty_reference():
+    try:
+        r = Ref("")
+        fail("Empty reference did not raise an error")
+    except SyntaxError:
+        pass
+
+def test_invalid_book_code():
+    r = Ref("XYZ 1:1")
+    if r.isvalid():
+        fail("Invalid book code did not raise an error")
+
+def test_multiple_chapters():
+    refs = RefList("JON 2:1-3:5")
+    if not (check_ref(refs[0].first, None, "JON", 2, 1) and check_ref(refs[0].last, None, "JON", 3, 5)):
+        fail("Multiple chapters not handled correctly")
+
+def test_partial_verse():
+    r = Ref("JON 3:10!1-4")
+    root = jon_usfm.getroot()
+    start = findref(r.first, root)
+    end = findref(r.last, root, atend=True)
+    res = copy_text(root, start, end)
+
+    if "God" not in res:
+        fail("Partial verse not extracted correctly")
+
+def test_ref_with_extra_text():
+    try:
+        r = Ref("JON 3:10abc")
+        fail("Reference with extra text did not raise an error")
+    except SyntaxError:
+        pass
+
+def test_range_with_negative_chapter():
+    try:
+        r = Ref("JON -1:5")
+        fail("Negative chapter did not raise an error")
+    except SyntaxError:
+        pass
+
+def test_incomplete_reference():
+    try:
+        r = Ref("JON")
+        fail("Incomplete reference did not raise an error")
+    except SyntaxError:
+        pass
+
+def test_large_chapter_number():
+    r = Ref("JON 999:1")
+    if r.isvalid():
+        fail("Large chapter number did not raise an error")
+
+### Markers
+
+def test_parsemarker():
+    res = _get_textref("JON 1:4!s1!2")
+    if res != "Great":
+        fail(f"{r}: [{start}, {end}] {res}")
+
+def test_footnote():
+    res = _get_textref("JON 1:13!f!5")
+    if res != "dug":
+        fail(f"{r}: [{start}, {end}] {res}")
+
+

usfmtc-0.3.2/tests/test_refs.py

@@ -1,199 +0,0 @@
-import pytest
-from pytest import fail
-from usfmtc import readFile, usx2usfm
-from usfmtc.usxmodel import findref, copy_range
-from usfmtc.reference import Ref, RefList, RefRange
-import os, io
-import xml.etree.ElementTree as et
-
-# To see the print() output of passing tests, run pytest -s
-
-def check_ref(ref, product=None, book=None, chapter=None, verse=None, subverse=None,
-                   word=None, char=None, mrkrs=None):
-    if product != ref.product:
-        return False
-    if book != ref.book:
-        return False
-    if chapter != ref.chapter:
-        return False
-    if verse != ref.verse:
-        return False
-    if subverse != ref.subverse:
-        return False
-    if word != ref.word:
-        return False
-    if char != ref.char:
-        return False
-    return True
-
-def asusfm(root):
-    with io.StringIO() as fh:
-        usx2usfm(fh, root)
-        res = fh.getvalue()
-    return res
-
-jon_usfm = readFile(os.path.join(os.path.dirname(__file__), "32JONBSB.usfm"))
-
-def test_basic():
-    r = Ref("JHN 3:16")
-    rr = Ref("12", context=r)
-
-    if not check_ref(r, None, "JHN", 3, 16):
-        fail(f"{r} is not JHN 3:16")
-    if not check_ref(rr, None, "JHN", 3, 12):
-        fail(f"{r} is not JHN 3:12")
-
-def test_range():
-    r = RefList("JHN 3:16-18")
-
-    if not check_ref(r[0].first, None, "JHN", 3, 16):
-        fail(f"{r[0].first} is not JHN 3:16")
-    if not check_ref(r[0].last, None, "JHN", 3, 18):
-        fail(f"{r[0].last} is not JHN 3:18")
-
-def test_findreftext():
-    r = Ref("JON 3:6!3")
-    loc = findref(r, jon_usfm.getroot())
-
-    if loc.el.tag != "verse" and loc.attrib != " tail" and loc.char != 18:
-        fail(f"{r} resulted in {loc}")
-    end = Ref("JON 3:6!4")
-    eloc = findref(end, jon_usfm.getroot(), atend=True)
-    if eloc.el.tag != "verse" and eloc.attrib != " tail" and eloc.char !=24:
-        fail(f"{end} resulted in {eloc}")
-    res = eloc.el.tail[loc.char:eloc.char]
-    if res != "reached the":
-        fail(f"{res} != 'reached the'")
-
-def test_findref1():
-    first = Ref("JON 2:8")
-    last = Ref("JON 2")
-    _do_findreftest(first, last)
-
-def test_findref2():
-    ref = Ref("JON 2:8-end")
-    _do_findreftest(ref.first, ref.last)
-
-def test_getrefs():
-    refs = RefList("JON 1:3; 2:4-5; 3:9-end")
-    res = jon_usfm.getrefs(*refs)
-    root = res.getroot()
-    print(res.outUsfm())
-    if len(root) != 14 or root[0].get("vid", "") != "JON 1:3":
-        fail(f"{len(root)=}")
-
-def _do_findreftest(first, last):
-    root = jon_usfm.getroot()
-    start = findref(first, root)
-    end = findref(last, root, atend=True)
-    res = copy_range(root, start, end)
-
-    print(f"{first=} {start}; {last=} {end}")
-    et.dump(res)
-    print(asusfm(res))
-    if len(res) != 7 or len(res[2]) != 1 or "vomited" not in res[-1][-1].tail:
-        fail(f"{len(res)=}, {res[-1][-1].tail=}")
-
-def r(bk, chap, verse, subv=None):
-    return Ref(book=bk, chapter=chap, verse=verse, subverse=subv)
-
-def t(s, *r):
-    res = RefList(s)
-    if len(res) != len(r):
-        fail("Failed '{}' resulted in {} references, {}, rather than {}".format(s, len(res), res, len(r)))
-    for z in zip(res, r):
-        if z[0] != z[1]:
-            fail("Reference list failed '{}', {} != {}".format(s, z[0], z[1]))
-    if str(res) != s:
-        fail("{} != canonical string of {}".format(s, str(res)))
-    print("{} = {}".format(s, str(res)))
-
-def listtest(s, a):
-    res = RefList(s)
-    res.simplify()
-    base = RefList(a)
-    if res != base:
-        fail("{} simplified to {} instead of {}".format(s, res, base))
-    print("{} -> {}".format(s, res))
-
-def o(a, b, res):
-    x = RefList(a)[0]
-    y = RefList(b)[0]
-    if (isinstance(res,tuple)):
-      restup=res
-      restupd=f"x<y {restup[0]}, y<x {restup[1]}, x in y {restup[2]}"
-    else:
-      restup=(res,res,res)
-      restupd=res
-    a = (y<x)
-    b = (x<y)
-    #print(f"y<x :{a} x<y:{b}")
-    if restup[0] != b:
-        raise fail(f"{x} should be < {y} [expect: {restupd}]")
-    if restup[1] != a:
-        raise fail(f"{y} should not be < {x} [expect: {restupd}]")
-    if (x in y) != restup[2]:
-        raise fail(f"{x} in {y} should be {res}")
-    print("{} <-> {} ({})".format(x, y,restupd))
-
-def test_gen1():
-    t("GEN 1:1", r("GEN", 1, 1))
-
-def test_jhn3():
-    t("JHN 3", r("JHN", 3, None))
-
-def test_3jn():
-    t("3JN 3", r("3JN", 1, 3))
-
-def test_1co():
-    t("1CO 6:5a", r("1CO", 6, 5, "a"))
-
-def test_mat5():
-    t("MAT 5:1-7", RefRange(r("MAT", 5, 1), r("MAT", 5, 7)))
-
-def test_mat7():
-    t("MAT 7:1,2; 8:6b-9:4", r("MAT", 7, 1), r("MAT", 7, 2), RefRange(r("MAT", 8, 6, "b"), r("MAT", 9, 4)))
-
-def test_luk3():
-    t("LUK 3:35-end", RefRange(r("LUK", 3, 35), r("LUK", 3, -1)))
-
-def test_rom1():
-    listtest("ROM 1; MAT 3:4-11; ROM 1:3-2:7", "MAT 3:4-11; ROM 1-2:7")
-
-def test_gen1_1():
-    t("GEN 1:1-3; 3:2-11; LUK 4:5", RefRange(r("GEN", 1, 1), r("GEN", 1, 3)), RefRange(r("GEN", 3, 2), r("GEN", 3, 11)), r("LUK", 4, 5))
-
-def test_jud():
-    t("JUD 1,2,4", r("JUD", 1, 1), r("JUD", 1, 2), r("JUD", 1, 4))
-
-def test_gen1_1a():
-    o("GEN 1:1", "EXO 2:3", (True,False,False))
-
-def test_exo2():
-    o("EXO 2:4", "EXO 2", (False,True,True))
-
-def test_exo2a():
-    o("EXO 2:4-5", "EXO 2", (False,True,True))
-
-def test_exo2b():
-    o("EXO 2:4-5", "EXO 3", (True,False,False))
-
-def text_exo2c():
-    o("EXO 2:1-2", "EXO 2:1-5", (False,False,True))
-
-def test_exo2d():
-    o("EXO 2:1-2", "EXO 2:2-7", (True,False,True))
-
-def test_exo2e():
-    o("EXO 2:1-2", "EXO 2:3-6", (True,False,False))
-
-def test_gen2():
-    o("GEN 2:1-2", "EXO 2:3-6", (True,False,False))
-
-def test_deu2():
-    o("DEU 2:1-2", "EXO 2:3-6", (False,True,False))
-
-def test_exo2f():
-    o("EXO 2:2-3", "EXO 2:1-5", (False,False,True))
-
-