lexgrep 0.2.0__tar.gz

lexgrep-0.2.0/.github/workflows/lint.yml

@@ -0,0 +1,15 @@
+name: Lint
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    branches:
+      - main
+jobs:
+  Lint:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v5
+      - uses: astral-sh/setup-uv@v5
+      - run: make lint

lexgrep-0.2.0/.github/workflows/pypi.yml

@@ -0,0 +1,30 @@
+name: PyPI
+
+on:
+  push:
+    tags:
+      - 'v*.*.*'
+
+jobs:
+  PyPI:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+      id-token: write
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v5
+
+      - name: Setup uv
+        uses: astral-sh/setup-uv@v5
+
+      - name: Build
+        run: uv build
+
+      - name: Publish
+        run: uv publish
+
+      - name: Release
+        uses: softprops/action-gh-release@v2
+        with:
+          files: |

lexgrep-0.2.0/.github/workflows/test.yml

@@ -0,0 +1,15 @@
+name: Test
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    branches:
+      - main
+jobs:
+  Test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v5
+      - uses: astral-sh/setup-uv@v5
+      - run: make test

lexgrep-0.2.0/PKG-INFO

@@ -0,0 +1,9 @@
+Metadata-Version: 2.5
+Name: lexgrep
+Version: 0.2.0
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+
+https://github.com/clips/pattern/blob/master/pattern/text/en/inflect.py
+
+https://github.com/nizarhabash1/catvar

lexgrep-0.2.0/lexgrep/__init__.py

@@ -0,0 +1,3 @@
+from .lexgrep import lexgrep
+
+__all__ = ["lexgrep"]

lexgrep-0.2.0/lexgrep/__main__.py

@@ -0,0 +1,4 @@
+from .cli import main
+
+if __name__ == "__main__":
+    raise SystemExit(main())

lexgrep-0.2.0/lexgrep/bench.py

@@ -0,0 +1,113 @@
+from time import perf_counter
+from io import StringIO
+import re
+
+
+def related_words(word):
+    """
+    Generate words related to a supplied word
+
+    This is intended to be an over estimate. not all returned result need to
+    be proper words.
+
+    >>> 'compute' in related_words("computer")
+    True
+
+    >>> 'computes' in related_words("computer")
+    True
+
+    >>> 'computes' in related_words("computing")
+    True
+
+    >>> 'computation' in related_words("computing")
+    True
+
+    >>> 'running' in related_words("run")
+    True
+    """
+    word = word.strip().lower()
+
+    # Guess the verb behind agent nouns: computer -> compute.
+    for suffix in ["er", "or", "ing", "ers", "ors", "ed"]:
+        if word.endswith(suffix):
+            base = word[: -len(suffix)]
+            if base and not base.endswith(("a", "e", "i", "o", "u")):
+                base += "e"
+            break
+    else:
+        base = word
+
+    if base.endswith("e"):
+        # Separate the stem for common -ed and -ing forms.
+        stem = base[:-1]
+    elif re.search(r"[^aeiou][aeiou][^aeiou]$", base):
+        # Apply double consonant rule
+        stem = base + base[-1]
+    else:
+        stem = base
+
+    if stem.endswith("y") and len(stem) > 1:
+        plural = stem[:-1] + "ies"
+    else:
+        plural = base + "s"
+
+    return {
+        word,
+        base,
+        plural,
+        stem + "er",
+        stem + "ers",
+        stem + "ed",
+        stem + "ing",
+        stem + "ings",
+        stem + "ation",
+        stem + "ion",
+    }
+
+
+def splitter(text, query):
+    keywords = [f" {kw} " for kw in query.split()]
+
+    matches = []
+
+    for line in text.splitlines():
+        count = 0
+
+        line = " " + re.sub(r"[^A-Za-z]", " ", line).lower()
+
+        for kw in keywords:
+            count += line.count(kw)
+
+        if count > 0:
+            matches.append((count / max(200, len(line)), line))
+
+    matches.sort()
+
+    return matches
+
+
+def regex(text, query):
+    rx = re.compile(r"\b" + r"\b|\b".join(query.split()) + r"\b", flags=re.I)
+
+    matches = []
+
+    for line in text.splitlines():
+        count = sum(1 for _ in rx.finditer(line))
+
+        if count > 0:
+            matches.append((count / max(200, len(line)), line))
+
+    matches.sort()
+
+    return matches
+
+
+if __name__ == "__main__":
+    text = open("artofwar.txt").read()
+
+    for fn in [splitter, regex]:
+        start = perf_counter()
+
+        print(fn(text, "war info")[-1])
+
+        print(fn, perf_counter() - start)

lexgrep-0.2.0/lexgrep/cli.py

@@ -0,0 +1,20 @@
+import argparse
+
+from .lexgrep import lexgrep
+
+
+def main(argv=None):
+    parser = argparse.ArgumentParser(
+        prog="lexgrep",
+        description="",
+    )
+    parser.add_argument("files", nargs="*")
+    args = parser.parse_args(argv)
+
+    lexgrep(args.files)
+
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

lexgrep-0.2.0/lexgrep/epub.py

@@ -0,0 +1,118 @@
+from concurrent.futures import ProcessPoolExecutor
+from zipfile import ZipFile
+import re
+import unicodedata
+
+ascii_map = str.maketrans(
+    {
+        "\u2018": "'",
+        "\u2019": "'",
+        "\u201a": "'",
+        "\u201b": "'",
+        "\u201c": '"',
+        "\u201d": '"',
+        "\u201e": '"',
+        "\u201f": '"',
+        "\u2013": "-",
+        "\u2014": "-",
+        "\u2212": "-",
+        "\u2026": "...",
+        "\u2022": "*",
+        "\u00a9": "(c)",
+        "\u00ae": "(R)",
+        "\u2122": "TM",
+        "\u00df": "ss",
+        "\u00c6": "AE",
+        "\u00e6": "ae",
+        "\u0152": "OE",
+        "\u0153": "oe",
+        "\u00d8": "O",
+        "\u00f8": "o",
+        "\u0141": "L",
+        "\u0142": "l",
+        "\u0300": "",
+        "\u0301": "",
+        "\u0302": "",
+        "\u0303": "",
+        "\u0304": "",
+        "\u0306": "",
+        "\u0307": "",
+        "\u0308": "",
+        "\u030a": "",
+        "\u030b": "",
+        "\u030c": "",
+        "\u0327": "",
+        "\u0328": "",
+        "\n": " ",
+        "\t": " ",
+        "\r": " ",
+    }
+)
+
+
+def to_ascii(text):
+    """Transliterate common punctuation and accented letters to ASCII.
+
+
+    >>> to_ascii("Café déjà vu")
+    'Cafe deja vu'
+    """
+    text = unicodedata.normalize("NFKD", text)
+    text = text.translate(ascii_map)
+    return text.encode("ascii", "replace").decode("ascii")
+    return text
+
+
+addbreaks = re.compile(r"<[ph][^>]*>")
+rmtags = re.compile(r" *<[^>]*> *")
+normbreaks = re.compile(r"\s*\n\s*\n\s*")
+normspaces = re.compile(r"  +")
+archive = None
+
+
+def init_worker(path):
+    global archive
+    archive = ZipFile(path)
+
+
+def process_chapter(name):
+    with archive.open(name) as file:
+        chapter = file.read().decode()
+
+    chapter = to_ascii(chapter)
+    chapter = addbreaks.sub("\n\n", chapter)
+    chapter = rmtags.sub(" ", chapter)
+    chapter = normbreaks.sub("\n\n", chapter)
+    chapter = normspaces.sub(" ", chapter)
+    return name, chapter.strip()
+
+
+def read_epub_text(filename):
+    chapters = []
+
+    with ZipFile(filename) as source:
+        names = sorted(
+            (
+                item.filename
+                for item in source.infolist()
+                if not item.is_dir() and item.filename.endswith("xhtml")
+            )
+        )
+
+    with ProcessPoolExecutor(
+        max_workers=4,
+        initializer=init_worker,
+        initargs=(filename,),
+    ) as executor:
+        for name, chapter in executor.map(process_chapter, names, chunksize=4):
+            chapters.append(chapter)
+
+    return "\n\n".join(c for c in chapters if c)
+
+
+def main():
+    print(read_epub_text("test.epub"))
+
+
+if __name__ == "__main__":
+    main()