lexgrep 0.2.0__py3-none-any.whl

lexgrep/__init__.py

@@ -0,0 +1,3 @@
+from .lexgrep import lexgrep
+
+__all__ = ["lexgrep"]

lexgrep/__main__.py

@@ -0,0 +1,4 @@
+from .cli import main
+
+if __name__ == "__main__":
+    raise SystemExit(main())

lexgrep/bench.py

@@ -0,0 +1,113 @@
+from time import perf_counter
+from io import StringIO
+import re
+
+
+def related_words(word):
+    """
+    Generate words related to a supplied word
+
+    This is intended to be an over estimate. not all returned result need to
+    be proper words.
+
+    >>> 'compute' in related_words("computer")
+    True
+
+    >>> 'computes' in related_words("computer")
+    True
+
+    >>> 'computes' in related_words("computing")
+    True
+
+    >>> 'computation' in related_words("computing")
+    True
+
+    >>> 'running' in related_words("run")
+    True
+    """
+    word = word.strip().lower()
+
+    # Guess the verb behind agent nouns: computer -> compute.
+    for suffix in ["er", "or", "ing", "ers", "ors", "ed"]:
+        if word.endswith(suffix):
+            base = word[: -len(suffix)]
+            if base and not base.endswith(("a", "e", "i", "o", "u")):
+                base += "e"
+            break
+    else:
+        base = word
+
+    if base.endswith("e"):
+        # Separate the stem for common -ed and -ing forms.
+        stem = base[:-1]
+    elif re.search(r"[^aeiou][aeiou][^aeiou]$", base):
+        # Apply double consonant rule
+        stem = base + base[-1]
+    else:
+        stem = base
+
+    if stem.endswith("y") and len(stem) > 1:
+        plural = stem[:-1] + "ies"
+    else:
+        plural = base + "s"
+
+    return {
+        word,
+        base,
+        plural,
+        stem + "er",
+        stem + "ers",
+        stem + "ed",
+        stem + "ing",
+        stem + "ings",
+        stem + "ation",
+        stem + "ion",
+    }
+
+
+def splitter(text, query):
+    keywords = [f" {kw} " for kw in query.split()]
+
+    matches = []
+
+    for line in text.splitlines():
+        count = 0
+
+        line = " " + re.sub(r"[^A-Za-z]", " ", line).lower()
+
+        for kw in keywords:
+            count += line.count(kw)
+
+        if count > 0:
+            matches.append((count / max(200, len(line)), line))
+
+    matches.sort()
+
+    return matches
+
+
+def regex(text, query):
+    rx = re.compile(r"\b" + r"\b|\b".join(query.split()) + r"\b", flags=re.I)
+
+    matches = []
+
+    for line in text.splitlines():
+        count = sum(1 for _ in rx.finditer(line))
+
+        if count > 0:
+            matches.append((count / max(200, len(line)), line))
+
+    matches.sort()
+
+    return matches
+
+
+if __name__ == "__main__":
+    text = open("artofwar.txt").read()
+
+    for fn in [splitter, regex]:
+        start = perf_counter()
+
+        print(fn(text, "war info")[-1])
+
+        print(fn, perf_counter() - start)

lexgrep/cli.py

@@ -0,0 +1,20 @@
+import argparse
+
+from .lexgrep import lexgrep
+
+
+def main(argv=None):
+    parser = argparse.ArgumentParser(
+        prog="lexgrep",
+        description="",
+    )
+    parser.add_argument("files", nargs="*")
+    args = parser.parse_args(argv)
+
+    lexgrep(args.files)
+
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

lexgrep/epub.py

@@ -0,0 +1,118 @@
+from concurrent.futures import ProcessPoolExecutor
+from zipfile import ZipFile
+import re
+import unicodedata
+
+ascii_map = str.maketrans(
+    {
+        "\u2018": "'",
+        "\u2019": "'",
+        "\u201a": "'",
+        "\u201b": "'",
+        "\u201c": '"',
+        "\u201d": '"',
+        "\u201e": '"',
+        "\u201f": '"',
+        "\u2013": "-",
+        "\u2014": "-",
+        "\u2212": "-",
+        "\u2026": "...",
+        "\u2022": "*",
+        "\u00a9": "(c)",
+        "\u00ae": "(R)",
+        "\u2122": "TM",
+        "\u00df": "ss",
+        "\u00c6": "AE",
+        "\u00e6": "ae",
+        "\u0152": "OE",
+        "\u0153": "oe",
+        "\u00d8": "O",
+        "\u00f8": "o",
+        "\u0141": "L",
+        "\u0142": "l",
+        "\u0300": "",
+        "\u0301": "",
+        "\u0302": "",
+        "\u0303": "",
+        "\u0304": "",
+        "\u0306": "",
+        "\u0307": "",
+        "\u0308": "",
+        "\u030a": "",
+        "\u030b": "",
+        "\u030c": "",
+        "\u0327": "",
+        "\u0328": "",
+        "\n": " ",
+        "\t": " ",
+        "\r": " ",
+    }
+)
+
+
+def to_ascii(text):
+    """Transliterate common punctuation and accented letters to ASCII.
+
+
+    >>> to_ascii("Café déjà vu")
+    'Cafe deja vu'
+    """
+    text = unicodedata.normalize("NFKD", text)
+    text = text.translate(ascii_map)
+    return text.encode("ascii", "replace").decode("ascii")
+    return text
+
+
+addbreaks = re.compile(r"<[ph][^>]*>")
+rmtags = re.compile(r" *<[^>]*> *")
+normbreaks = re.compile(r"\s*\n\s*\n\s*")
+normspaces = re.compile(r"  +")
+archive = None
+
+
+def init_worker(path):
+    global archive
+    archive = ZipFile(path)
+
+
+def process_chapter(name):
+    with archive.open(name) as file:
+        chapter = file.read().decode()
+
+    chapter = to_ascii(chapter)
+    chapter = addbreaks.sub("\n\n", chapter)
+    chapter = rmtags.sub(" ", chapter)
+    chapter = normbreaks.sub("\n\n", chapter)
+    chapter = normspaces.sub(" ", chapter)
+    return name, chapter.strip()
+
+
+def read_epub_text(filename):
+    chapters = []
+
+    with ZipFile(filename) as source:
+        names = sorted(
+            (
+                item.filename
+                for item in source.infolist()
+                if not item.is_dir() and item.filename.endswith("xhtml")
+            )
+        )
+
+    with ProcessPoolExecutor(
+        max_workers=4,
+        initializer=init_worker,
+        initargs=(filename,),
+    ) as executor:
+        for name, chapter in executor.map(process_chapter, names, chunksize=4):
+            chapters.append(chapter)
+
+    return "\n\n".join(c for c in chapters if c)
+
+
+def main():
+    print(read_epub_text("test.epub"))
+
+
+if __name__ == "__main__":
+    main()