quran-processing-toolkit 0.4.0__py3-none-any.whl

quran_processing_toolkit/similarity.py

@@ -0,0 +1,98 @@
+"""Pairwise surface-word comparison; corpus-wide research searches live in client scripts."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+from .collections import QuerySet
+from .models import Verse
+
+
+@dataclass(frozen=True, slots=True)
+class VerseMatch:
+    """One verse pair and a deterministic one-word edit explanation.
+
+    Positions refer to the 1-based surface-word sequence, NOT QAC word addresses.
+    For addition A is shorter and position_a/word_a are None. Substitution uses
+    earlier-addressed A. Shift is oriented so A's word moves later in B; among
+    ambiguous explanations, earliest A, B, and positions win. Identical texts do
+    not match, even at different addresses. Original Verse objects are retained.
+    """
+
+    kind: str
+    verse_a: Verse
+    verse_b: Verse
+    position_a: int | None
+    position_b: int
+    word_a: str | None
+    word_b: str
+
+    @property
+    def verses(self) -> tuple[Verse, Verse]:
+        """Return the original (verse A, verse B) objects."""
+        return self.verse_a, self.verse_b
+
+
+def _address(verse: Verse) -> tuple[int, int]:
+    return verse.chapter, verse.verse
+
+
+def compare_verses(
+    verse_a: Verse,
+    verse_b: Verse,
+    *,
+    ignore_pause_marks: bool = False,
+) -> QuerySet[VerseMatch]:
+    """Explain a one-word insertion, substitution, or later movement between two verses.
+
+    Only these two inputs are compared; this is not a corpus-wide search. Original
+    display text is preserved. Addition puts the shorter verse first; substitution
+    uses address order; shifts choose the earliest valid A/B addresses and positions.
+    """
+    if _address(verse_a) == _address(verse_b):
+        return QuerySet()
+    first, second = sorted((verse_a, verse_b), key=_address)
+    a = first.surface_words(ignore_pause_marks=ignore_pause_marks)
+    b = second.surface_words(ignore_pause_marks=ignore_pause_marks)
+    if a == b:
+        return QuerySet()
+    if abs(len(a) - len(b)) == 1:
+        shorter, longer, short_text, long_text = (
+            (first, second, a, b) if len(a) < len(b) else (second, first, b, a)
+        )
+        for pos, word in enumerate(long_text):
+            if long_text[:pos] + long_text[pos + 1 :] == short_text:
+                return QuerySet(
+                    (VerseMatch("addition", shorter, longer, None, pos + 1, None, word),)
+                )
+        return QuerySet()
+    if len(a) != len(b):
+        return QuerySet()
+    changed = [pos for pos, (left, right) in enumerate(zip(a, b, strict=True)) if left != right]
+    if len(changed) == 1:
+        pos = changed[0]
+        return QuerySet(
+            (VerseMatch("substitution", first, second, pos + 1, pos + 1, a[pos], b[pos]),)
+        )
+    candidates = []
+    for source, target, left, right in ((first, second, a, b), (second, first, b, a)):
+        for start, word in enumerate(left):
+            remainder = left[:start] + left[start + 1 :]
+            for end in range(start + 1, len(right)):
+                if right[end] == word and right[:end] + right[end + 1 :] == remainder:
+                    candidates.append(
+                        VerseMatch("shift", source, target, start + 1, end + 1, word, word)
+                    )
+    if not candidates:
+        return QuerySet()
+    chosen = min(
+        candidates,
+        key=lambda m: (
+            _address(m.verse_a),
+            _address(m.verse_b),
+            m.position_a or 0,
+            m.position_b,
+            m.word_b,
+        ),
+    )
+    return QuerySet((chosen,))

quran_processing_toolkit/source_data.py

@@ -0,0 +1,201 @@
+"""Discovery, integrity verification, and local caching for source data."""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import os
+import tempfile
+from contextlib import suppress
+from pathlib import Path
+from typing import Any
+
+SUPPORTED_SOURCE_SHA256 = {
+    "quranic-corpus-morphology-0.4.txt": (
+        "a1d12923815341face765083805d2148ed2d9f5cc3f7d6665219d887675d8c46"
+    ),
+    "uthmani-numbered.txt": ("7919b8a04d63b164273a743e0d2538a9704582dd72c2f49e8d557b77dd819e40"),
+    "juzz-breakdown.tsv": ("ee3e1e72be6bebe5e0dea0571b2a03a16ebc1fc3ae33397576341a529fb087f2"),
+}
+
+
+class DataError(RuntimeError):
+    """Base class for source discovery, integrity, and corpus-loading failures."""
+
+
+class DataPackageNotInstalledError(DataError):
+    """Raised when no override, installed provider, or checkout data is available."""
+
+
+class DataIntegrityError(DataError):
+    """Raised when source bytes or deterministic derivation fail verification."""
+
+
+def default_data_directory() -> Path:
+    """Find an override, installed data provider, or monorepo source package."""
+    override = os.environ.get("QURAN_PROCESSING_TOOLKIT_DATA")
+    if override:
+        path = Path(override).expanduser().resolve()
+        if not path.is_dir():
+            raise DataError(f"Configured source-data directory does not exist: {path}")
+        return path
+
+    try:
+        from quran_processing_toolkit_qac_data import data_directory
+
+        return data_directory()
+    except ImportError:
+        pass
+
+    checkout = (
+        Path(__file__).resolve().parents[1]
+        / "packages"
+        / "quran_processing_toolkit_qac_data"
+        / "src"
+        / "quran_processing_toolkit_qac_data"
+        / "data"
+    )
+    if (checkout / "manifest.json").is_file():
+        return checkout
+    raise DataPackageNotInstalledError(
+        "Quran source data is not installed. Install "
+        "quran-processing-toolkit-qac-data or pass data_dir=... to load_quran()."
+    )
+
+
+def _sha256(path: Path) -> str:
+    digest = hashlib.sha256()
+    with path.open("rb") as handle:
+        for chunk in iter(lambda: handle.read(1024 * 1024), b""):
+            digest.update(chunk)
+    return digest.hexdigest()
+
+
+def load_manifest(data_dir: Path) -> dict[str, Any]:
+    """Load and minimally validate the source-data manifest schema."""
+    path = data_dir / "manifest.json"
+    try:
+        with path.open(encoding="utf-8") as handle:
+            manifest: dict[str, Any] = json.load(handle)
+    except FileNotFoundError as error:
+        raise DataIntegrityError(f"Source-data manifest is missing: {path}") from error
+    if manifest.get("schema_version") != 1 or not isinstance(manifest.get("sources"), dict):
+        raise DataIntegrityError(f"Unsupported source-data manifest: {path}")
+    return manifest
+
+
+def verify_data_directory(data_dir: Path) -> dict[str, Any]:
+    """Verify every source file against its declared byte size and SHA-256."""
+    data_dir = data_dir.expanduser().resolve()
+    manifest = load_manifest(data_dir)
+    for name, source in manifest["sources"].items():
+        path = data_dir / source["filename"]
+        expected_digest = SUPPORTED_SOURCE_SHA256.get(path.name)
+        if expected_digest is None or source["sha256"] != expected_digest:
+            raise DataIntegrityError(
+                f"Unsupported declared SHA-256 for {path.name}: {source['sha256']}"
+            )
+        if not path.is_file():
+            raise DataIntegrityError(f"Missing {name} source file: {path}")
+        actual_size = path.stat().st_size
+        if actual_size != source["bytes"]:
+            raise DataIntegrityError(
+                f"Size mismatch for {path.name}: {actual_size} != {source['bytes']}"
+            )
+        actual_digest = _sha256(path)
+        if actual_digest != source["sha256"]:
+            raise DataIntegrityError(
+                f"SHA-256 mismatch for {path.name}: {actual_digest} != {source['sha256']}"
+            )
+    return manifest
+
+
+def default_cache_directory() -> Path:
+    """Return the environment override or platform-style user cache directory."""
+    override = os.environ.get("QURAN_PROCESSING_TOOLKIT_CACHE")
+    if override:
+        return Path(override).expanduser().resolve()
+    base = Path(os.environ.get("XDG_CACHE_HOME", Path.home() / ".cache"))
+    return base / "quran-processing-toolkit"
+
+
+def _read_cached_records(path: Path, expected_digest: str) -> list[dict[str, Any]] | None:
+    try:
+        payload = path.read_bytes()
+        if hashlib.sha256(payload).hexdigest() != expected_digest:
+            return None
+        try:
+            import orjson  # type: ignore[import-not-found]
+
+            records: list[dict[str, Any]] = orjson.loads(payload)
+        except ImportError:
+            records = json.loads(payload)
+        return records
+    except (FileNotFoundError, OSError, ValueError, TypeError):
+        return None
+
+
+def _write_cache(path: Path, payload: bytes) -> None:
+    try:
+        path.parent.mkdir(parents=True, exist_ok=True)
+        descriptor, temporary_name = tempfile.mkstemp(
+            prefix=path.name + ".", suffix=".tmp", dir=path.parent
+        )
+        try:
+            with os.fdopen(descriptor, "wb") as handle:
+                handle.write(payload)
+            os.replace(temporary_name, path)
+        except BaseException:
+            with suppress(FileNotFoundError):
+                os.unlink(temporary_name)
+            raise
+    except OSError:
+        # Read-only homes and restricted execution environments should still load.
+        return
+
+
+def load_normalized_records(
+    data_dir: Path,
+    *,
+    disk_cache: bool = True,
+    cache_dir: Path | None = None,
+) -> tuple[list[dict[str, Any]], dict[str, Any]]:
+    """Verify source bytes, then load or deterministically derive normalized records."""
+    from .build import (
+        SUPPORTED_DERIVED_DIGESTS,
+        TRANSFORMATION_VERSION,
+        canonical_records_bytes,
+        load_source_records,
+        rebuild_records,
+    )
+
+    manifest = verify_data_directory(data_dir)
+    source = manifest["sources"]["morphology"]
+    try:
+        expected_digest = SUPPORTED_DERIVED_DIGESTS[source["sha256"]]
+    except KeyError as error:
+        raise DataIntegrityError(
+            f"Unsupported morphology source SHA-256: {source['sha256']}"
+        ) from error
+    cache_path = (cache_dir or default_cache_directory()) / (
+        f"morphology-v{TRANSFORMATION_VERSION}-{source['sha256'][:16]}.json"
+    )
+    if disk_cache:
+        cached = _read_cached_records(cache_path, expected_digest)
+        if cached is not None and len(cached) == source["records"]:
+            return cached, manifest
+
+    records = rebuild_records(load_source_records(data_dir / source["filename"]))
+    if len(records) != source["records"]:
+        raise DataIntegrityError(
+            f"Derived {len(records)} morphology records; expected {source['records']}"
+        )
+    payload = canonical_records_bytes(records)
+    actual_digest = hashlib.sha256(payload).hexdigest()
+    if actual_digest != expected_digest:
+        raise DataIntegrityError(
+            f"Derived morphology digest mismatch: {actual_digest} != {expected_digest}"
+        )
+    if disk_cache:
+        _write_cache(cache_path, payload)
+    return records, manifest

quran_processing_toolkit/text.py

@@ -0,0 +1,259 @@
+"""Unicode-aware normalization, representation selection, and text predicates.
+
+The toolkit never normalizes stored text in place. Each comparison starts from the
+preserved Arabic or Buckwalter form and applies only options explicitly requested by
+the caller. With ``representation='auto'``, wholly ASCII input means Buckwalter and
+non-ASCII input means Arabic.
+"""
+
+from __future__ import annotations
+
+import re
+import unicodedata
+from collections.abc import Collection
+from typing import Any
+
+QURANIC_PAUSE_MARKS = frozenset("ۛۖۗۚۙۘ۩ۜ")
+QURANIC_PAUSE_TRANSLATION = str.maketrans("", "", "".join(QURANIC_PAUSE_MARKS))
+# Extended Buckwalter vowel and recitation marks. Long-vowel letters A, w, and Y/y
+# are deliberately not included.
+BUCKWALTER_DIACRITICS = frozenset("aiuoFNK~`^#@")
+_ARABIC_DIACRITIC_RE = re.compile("[\u0610-\u061a\u064b-\u065f\u0670\u06d6-\u06ed]")
+
+
+class TranslationNotAvailableError(ValueError):
+    """Raised when English output is requested without a translation for an item."""
+
+
+def split_surface_words(text: str, *, ignore_pause_marks: bool = False) -> tuple[str, ...]:
+    """Split at literal spaces, optionally deleting only known Quranic pause marks.
+
+    Empty fragments are omitted. No Unicode normalization, vowel removal, spelling
+    changes, or morphology-based contractions are applied. Tabs are not delimiters.
+    The original input is not modified; these are not QAC morphology word indices.
+    """
+    if ignore_pause_marks:
+        text = text.translate(QURANIC_PAUSE_TRANSLATION)
+    return tuple(part for part in text.split(" ") if part)
+
+
+def normalize_arabic(
+    text: str,
+    *,
+    strip_diacritics: bool = False,
+    strip_quranic_marks: bool = False,
+    normalize_alif: bool = False,
+    normalize_ya: bool = False,
+    remove_spaces: bool = False,
+) -> str:
+    """Return NFC Arabic text with only the requested optional transformations.
+
+    The conservative defaults are intentional: alif variants, alif maqsura, Quranic
+    marks, and vowel marks can all be linguistically significant for a particular
+    analysis. Callers choose which distinctions to collapse.
+    """
+    result = unicodedata.normalize("NFC", text)
+    if strip_quranic_marks:
+        result = "".join(character for character in result if character not in QURANIC_PAUSE_MARKS)
+    if strip_diacritics:
+        result = _ARABIC_DIACRITIC_RE.sub("", result)
+    if normalize_alif:
+        result = result.translate(str.maketrans("أإآٱ", "اااا"))
+    if normalize_ya:
+        result = result.translate(str.maketrans("ىی", "يي"))
+    if remove_spaces:
+        result = "".join(result.split())
+    return result
+
+
+def normalize_buckwalter(
+    text: str,
+    *,
+    strip_diacritics: bool = False,
+    normalize_alif: bool = False,
+    normalize_ya: bool = False,
+    remove_spaces: bool = False,
+    **_: Any,
+) -> str:
+    """Normalize Buckwalter with options parallel to Arabic normalization."""
+    result = text
+    if strip_diacritics:
+        result = "".join(
+            character for character in result if character not in BUCKWALTER_DIACRITICS
+        )
+    if normalize_alif:
+        result = result.translate(str.maketrans("><|{", "AAAA"))
+    if normalize_ya:
+        result = result.replace("Y", "y")
+    if remove_spaces:
+        result = "".join(result.split())
+    return result
+
+
+def canonical_representation(representation: str) -> str:
+    """Accept concise names for the three user-facing text representations."""
+    aliases = {
+        "arabic": "arabic",
+        "transliteration": "transliteration",
+        "buckwalter": "transliteration",
+        "translation": "translation",
+        "english": "translation",
+    }
+    try:
+        return aliases[representation.lower()]
+    except KeyError as error:
+        raise ValueError(
+            f"Unknown representation {representation!r}; choose Arabic, "
+            "transliteration/Buckwalter, or translation/English"
+        ) from error
+
+
+def infer_representation(text: str, representation: str = "auto") -> str:
+    """Treat ASCII input as Buckwalter and non-ASCII input as Arabic."""
+    if representation == "auto":
+        return "transliteration" if text.isascii() else "arabic"
+    return canonical_representation(representation)
+
+
+class TextMixin:
+    """Display and comparison operations shared by tokens, words, and verses."""
+
+    def get_text(self, representation: str = "arabic") -> str:
+        """Return Arabic, Buckwalter, or supplied translation text."""
+        representation = canonical_representation(representation)
+        candidates = {
+            "arabic": "arabic_text",
+            "translation": "translation_text",
+            "transliteration": "transliteration",
+        }
+        attribute = candidates[representation]
+        value = getattr(self, attribute, None)
+        if value is None:
+            if representation == "translation":
+                raise TranslationNotAvailableError(
+                    f"{type(self).__name__} has no translation representation; "
+                    "pass translation=... to load_quran()"
+                )
+            raise ValueError(f"{type(self).__name__} has no {representation} representation")
+        return value
+
+    def normalized_text(self, representation: str = "arabic", **options: Any) -> str:
+        """Return one representation after applying explicit normalization options."""
+        representation = canonical_representation(representation)
+        text = self.get_text(representation)
+        if representation == "arabic":
+            return normalize_arabic(text, **options)
+        if representation == "transliteration":
+            return normalize_buckwalter(text, **options)
+        return text
+
+    def display(self, representation: str = "arabic") -> str:
+        """Return Arabic, English translation, or Buckwalter for display."""
+        return self.get_text(representation)
+
+    def get_arabic(self) -> str:
+        """Return preserved Arabic surface text."""
+        return self.get_text("arabic")
+
+    def get_transliteration(self) -> str:
+        """Return Buckwalter transliteration assembled from morphology tokens."""
+        return self.get_text("transliteration")
+
+    def get_translation(self) -> str:
+        """Return user-supplied translation text or raise a clear error."""
+        return self.get_text("translation")
+
+    @staticmethod
+    def _normalize_input(text: str, representation: str, options: dict[str, Any]) -> str:
+        if representation == "arabic":
+            return normalize_arabic(text, **options)
+        if representation == "transliteration":
+            return normalize_buckwalter(text, **options)
+        return text
+
+    def starts_with(
+        self, prefix: str, *, representation: str = "auto", **normalization: Any
+    ) -> bool:
+        """Test a normalized prefix; ASCII input automatically means Buckwalter."""
+        representation = infer_representation(prefix, representation)
+        text = self.normalized_text(representation, **normalization)
+        expected = self._normalize_input(prefix, representation, normalization)
+        return text.startswith(expected)
+
+    def ends_with(self, suffix: str, *, representation: str = "auto", **normalization: Any) -> bool:
+        """Test a normalized suffix; ASCII input automatically means Buckwalter."""
+        representation = infer_representation(suffix, representation)
+        text = self.normalized_text(representation, **normalization)
+        expected = self._normalize_input(suffix, representation, normalization)
+        return text.endswith(expected)
+
+    def contains(
+        self, fragment: str, *, representation: str = "auto", **normalization: Any
+    ) -> bool:
+        """Test a normalized substring; ASCII input automatically means Buckwalter."""
+        representation = infer_representation(fragment, representation)
+        text = self.normalized_text(representation, **normalization)
+        expected = self._normalize_input(fragment, representation, normalization)
+        return expected in text
+
+    def equals(
+        self, expected_text: str, *, representation: str = "auto", **normalization: Any
+    ) -> bool:
+        """Test normalized whole-text equality."""
+        representation = infer_representation(expected_text, representation)
+        text = self.normalized_text(representation, **normalization)
+        expected = self._normalize_input(expected_text, representation, normalization)
+        return text == expected
+
+    def contains_letter(
+        self, letter: str, *, representation: str = "auto", **normalization: Any
+    ) -> bool:
+        """Return whether a letter occurs; ASCII letters mean Buckwalter."""
+        if len(letter) != 1:
+            raise ValueError("contains_letter() expects exactly one character")
+        return self.contains(letter, representation=representation, **normalization)
+
+    def contains_any_letter(
+        self, letters: str, *, representation: str = "auto", **normalization: Any
+    ) -> bool:
+        """Return whether at least one requested character occurs."""
+        return any(
+            self.contains_letter(letter, representation=representation, **normalization)
+            for letter in letters
+        )
+
+    def contains_all_letters(
+        self, letters: str, *, representation: str = "auto", **normalization: Any
+    ) -> bool:
+        """Return whether every requested character occurs at least once."""
+        return all(
+            self.contains_letter(letter, representation=representation, **normalization)
+            for letter in letters
+        )
+
+    def contains_no_letters(
+        self, letters: str, *, representation: str = "auto", **normalization: Any
+    ) -> bool:
+        """Return whether none of the requested characters occur."""
+        return not self.contains_any_letter(letters, representation=representation, **normalization)
+
+    def count_characters(
+        self,
+        characters: str | Collection[str],
+        *,
+        representation: str = "auto",
+        **normalization: Any,
+    ) -> int:
+        """Count occurrences belonging to a set of requested characters."""
+        sample = characters if isinstance(characters, str) else "".join(characters)
+        representation = infer_representation(sample, representation)
+        wanted = set(self._normalize_input(sample, representation, normalization))
+        return sum(
+            character in wanted
+            for character in self.normalized_text(representation, **normalization)
+        )
+
+    def letter_count(self, *, representation: str = "arabic", **normalization: Any) -> int:
+        """Count alphabetic characters after optional normalization."""
+        text = self.normalized_text(representation, **normalization)
+        return sum(character.isalpha() for character in text)