quran-processing-toolkit 0.4.0__py3-none-any.whl

quran_processing_toolkit/corpus.py

@@ -0,0 +1,611 @@
+"""Load verified source data into the public, queryable Quran object graph.
+
+This module is the boundary between byte-preserved third-party data and convenient
+Python objects. Loading verifies all source hashes, derives normalized morphology,
+aligns it with Uthmani surface text, and constructs semantic indexes. It never writes
+to the installed data package; the optional disk cache contains only reproducible
+derived JSON in the user's cache directory.
+"""
+
+from __future__ import annotations
+
+import copy
+import os
+from collections import defaultdict
+from collections.abc import Callable, Iterable
+from pathlib import Path
+from threading import Lock
+from typing import Any
+
+from .collections import IndexedCollection, OneBasedCollection, QuerySet, resolve_value
+from .models import (
+    Chapter,
+    Juz,
+    Token,
+    Verse,
+    VerseAddress,
+    Word,
+    WordAddress,
+)
+from .source_data import (
+    DataError,
+    default_cache_directory,
+    default_data_directory,
+    load_normalized_records,
+)
+from .text import QURANIC_PAUSE_TRANSLATION
+from .translations import Translation, TranslationCoverage, load_translation
+
+_CACHE: dict[tuple[Any, ...], QuranCorpus] = {}
+_CACHE_LOCK = Lock()
+
+# QAC's nominal/particle families, with DET (the segmented article) included
+# among particles. INL (disconnected letters) is deliberately neither family.
+# https://corpus.quran.com/documentation/tagset.jsp
+_NOMINAL_TAGS = ("N", "PN", "ADJ", "IMPN", "PRON", "DEM", "REL", "T", "LOC")
+_PARTICLE_TAGS = (
+    "P",
+    "EMPH",
+    "IMPV",
+    "PRP",
+    "CONJ",
+    "SUB",
+    "ACC",
+    "AMD",
+    "ANS",
+    "AVR",
+    "CAUS",
+    "CERT",
+    "CIRC",
+    "COM",
+    "COND",
+    "EQ",
+    "EXH",
+    "EXL",
+    "EXP",
+    "FUT",
+    "INC",
+    "INT",
+    "INTG",
+    "NEG",
+    "PREV",
+    "PRO",
+    "REM",
+    "RES",
+    "RET",
+    "RSLT",
+    "SUP",
+    "SUR",
+    "VOC",
+    "DET",
+)
+
+CorpusError = DataError
+
+
+class CorpusAlignmentError(DataError):
+    """Raised when morphology word boundaries and Uthmani text cannot be aligned."""
+
+
+def _resolve_translation_path(
+    translation: str | os.PathLike[str] | Translation | None,
+) -> Path | None:
+    if isinstance(translation, Translation):
+        return translation.path
+    configured = translation or os.environ.get("QURAN_PROCESSING_TOOLKIT_TRANSLATION")
+    if configured is None:
+        return None
+    path = Path(configured).expanduser().resolve()
+    if not path.is_file():
+        raise CorpusError(f"Configured translation file does not exist: {path}")
+    return path
+
+
+def _read_numbered_text(path: Path, delimiter: str = "|") -> dict[VerseAddress, str]:
+    result: dict[VerseAddress, str] = {}
+    with path.open(encoding="utf-8") as handle:
+        for line_number, raw_line in enumerate(handle, 1):
+            line = raw_line.rstrip("\n")
+            if not line or line.startswith("#"):
+                continue
+            parts = line.split(delimiter, 2)
+            if len(parts) != 3:
+                raise CorpusError(f"Malformed line {line_number} in {path}")
+            try:
+                address = VerseAddress(int(parts[0]), int(parts[1]))
+            except ValueError as error:
+                raise CorpusError(f"Malformed address on line {line_number} in {path}") from error
+            if address in result:
+                raise CorpusError(f"Duplicate verse {address} in {path}")
+            result[address] = parts[2]
+    return result
+
+
+_BASMALA_FORMS = (
+    "بِسْمِ ٱللَّهِ ٱلرَّحْمَـٰنِ ٱلرَّحِيمِ ",
+    "بِّسْمِ ٱللَّهِ ٱلرَّحْمَـٰنِ ٱلرَّحِيمِ ",
+)
+
+
+def _without_non_verse_basmala(address: VerseAddress, text: str) -> str:
+    if address.chapter == 1 or address.verse != 1:
+        return text
+    for basmala in _BASMALA_FORMS:
+        if text.startswith(basmala):
+            return text[len(basmala) :]
+    return text
+
+
+def _arabic_words(address: VerseAddress, text: str) -> list[str]:
+    # These are orthographic contractions represented as one corpus word.
+    text = text.replace("بَعْدَ مَا", "بَعْد###مَا")
+    text = text.replace("إِلْ يَاسِينَ", "إِلْ###يَاسِينَ")
+    text = text.translate(QURANIC_PAUSE_TRANSLATION)
+    return [word.replace("###", " ") for word in text.strip().split()]
+
+
+class QuranCorpus:
+    """The complete Quran corpus with queryable, semantic 1-based indexes.
+
+    Constructing this class directly always builds a new object graph. Most programs
+    should call :func:`load_quran`, which reuses an in-process instance by default.
+    """
+
+    __slots__ = (
+        "data_dir",
+        "translation_path",
+        "translation",
+        "chapters",
+        "verses",
+        "words",
+        "tokens",
+        "juzs",
+        "_word_indexes",
+        "_verbs",
+        "_nouns",
+        "_token_selections",
+        "_manifest",
+        "_disk_cache",
+        "_cache_dir",
+    )
+
+    def __init__(
+        self,
+        data_dir: str | os.PathLike[str] | None = None,
+        *,
+        translation: str | os.PathLike[str] | Translation | None = None,
+        disk_cache: bool = True,
+        cache_dir: str | os.PathLike[str] | None = None,
+    ) -> None:
+        self.data_dir = (
+            Path(data_dir).expanduser().resolve() if data_dir else default_data_directory()
+        )
+        self.translation_path = _resolve_translation_path(translation)
+        self.translation = (
+            translation
+            if isinstance(translation, Translation)
+            else load_translation(self.translation_path)
+            if self.translation_path
+            else None
+        )
+        self._disk_cache = disk_cache
+        self._cache_dir = Path(cache_dir).expanduser().resolve() if cache_dir else None
+        self._word_indexes: dict[str, dict[Any, tuple[Word, ...]]] = {}
+        self._verbs: QuerySet[Word] | None = None
+        self._nouns: QuerySet[Word] | None = None
+        self._token_selections: dict[tuple[str, ...], QuerySet[Token]] = {}
+        self._build()
+
+    @classmethod
+    def load(
+        cls,
+        data_dir: str | os.PathLike[str] | None = None,
+        *,
+        translation: str | os.PathLike[str] | Translation | None = None,
+        cache: bool = True,
+        disk_cache: bool = True,
+        cache_dir: str | os.PathLike[str] | None = None,
+    ) -> QuranCorpus:
+        """Load a corpus, reusing an equivalent in-process instance by default.
+
+        ``cache`` controls whole-object reuse within this Python process.
+        ``disk_cache`` controls the verified derived-record cache between processes.
+        They are independent: disabling one does not disable the other.
+        """
+        path = Path(data_dir).expanduser().resolve() if data_dir else default_data_directory()
+        translation_path = _resolve_translation_path(translation)
+        resolved_cache_dir = (
+            Path(cache_dir).expanduser().resolve() if cache_dir else default_cache_directory()
+        )
+        translation_key: tuple[Any, ...] | None = None
+        if translation_path:
+            stat = translation_path.stat()
+            translation_key = (translation_path, stat.st_size, stat.st_mtime_ns)
+            if isinstance(translation, Translation):
+                # Layer content AND attribution affect corpus identity. A raw path
+                # must not reuse a previously labeled downloaded Translation object.
+                translation_key += (translation.fingerprint,)
+        # Loading options belong in the cache key. Otherwise a previous default load
+        # could silently defeat a later request for a specific cache directory.
+        key = (path, translation_key, disk_cache, resolved_cache_dir)
+        if not cache:
+            return cls(
+                path,
+                translation=translation
+                if isinstance(translation, Translation)
+                else translation_path,
+                disk_cache=disk_cache,
+                cache_dir=resolved_cache_dir,
+            )
+        with _CACHE_LOCK:
+            corpus = _CACHE.get(key)
+            if corpus is None:
+                corpus = cls(
+                    path,
+                    translation=translation
+                    if isinstance(translation, Translation)
+                    else translation_path,
+                    disk_cache=disk_cache,
+                    cache_dir=resolved_cache_dir,
+                )
+                _CACHE[key] = corpus
+            return corpus
+
+    def _build(self) -> None:
+        records, self._manifest = load_normalized_records(
+            self.data_dir,
+            disk_cache=self._disk_cache,
+            cache_dir=self._cache_dir,
+        )
+        sources = self._manifest["sources"]
+        arabic = _read_numbered_text(self.data_dir / sources["arabic_text"]["filename"])
+        translations = self.translation.texts if self.translation else {}
+
+        token_groups: defaultdict[tuple[int, int, int], list[Token]] = defaultdict(list)
+        verse_addresses: defaultdict[tuple[int, int], list[tuple[int, int, int]]] = defaultdict(
+            list
+        )
+        all_tokens: list[Token] = []
+        last_address: tuple[int, int, int, int] | None = None
+        for record in records:
+            record_address = (
+                int(record["CHAPTER"]),
+                int(record["VERSE"]),
+                int(record["WORD"]),
+                int(record["WORD_PART"]),
+            )
+            if last_address is not None and record_address <= last_address:
+                raise CorpusError(
+                    f"Morphology records are not strictly ordered at {record_address}"
+                )
+            last_address = record_address
+            token = Token(record)
+            word_address = record_address[:3]
+            if word_address not in token_groups:
+                verse_addresses[word_address[:2]].append(word_address)
+            token_groups[word_address].append(token)
+            all_tokens.append(token)
+
+        verse_word_groups: defaultdict[VerseAddress, list[Word]] = defaultdict(list)
+        all_words: list[Word] = []
+        for verse_key in sorted(verse_addresses):
+            verse_address = VerseAddress(*verse_key)
+            word_addresses = verse_addresses[verse_key]
+            if verse_address not in arabic:
+                raise CorpusAlignmentError(f"Missing Arabic text for verse {verse_address}")
+            verse_arabic = _without_non_verse_basmala(verse_address, arabic[verse_address])
+            surface_words = _arabic_words(verse_address, verse_arabic)
+            if len(surface_words) != len(word_addresses):
+                raise CorpusAlignmentError(
+                    f"Verse {verse_address} has {len(surface_words)} Arabic words but "
+                    f"{len(word_addresses)} morphology words"
+                )
+            for word_address, surface in zip(word_addresses, surface_words, strict=True):
+                word = Word(WordAddress(*word_address), token_groups[word_address], surface)
+                verse_word_groups[verse_address].append(word)
+                all_words.append(word)
+
+        chapter_verse_groups: defaultdict[int, list[Verse]] = defaultdict(list)
+        all_verses: list[Verse] = []
+        for verse_address in sorted(verse_word_groups):
+            verse_arabic = _without_non_verse_basmala(verse_address, arabic[verse_address])
+            verse = Verse(
+                verse_address,
+                verse_word_groups[verse_address],
+                verse_arabic,
+                translations.get((verse_address.chapter, verse_address.verse)),
+            )
+            chapter_verse_groups[verse_address.chapter].append(verse)
+            all_verses.append(verse)
+
+        all_chapters = [
+            Chapter(number, chapter_verse_groups[number]) for number in sorted(chapter_verse_groups)
+        ]
+        self.chapters = OneBasedCollection(all_chapters)
+        self.verses = IndexedCollection(
+            all_verses,
+            key=lambda verse: (verse.address.chapter, verse.address.verse),
+        )
+        self.words = IndexedCollection(all_words, key=lambda word: word.address.as_tuple())
+        self.tokens = IndexedCollection(all_tokens, key=lambda token: token.address_tuple)
+        self.juzs = self._build_juzs()
+
+    def _build_juzs(self) -> OneBasedCollection[Juz]:
+        starts: list[VerseAddress] = []
+        filename = self._manifest["sources"]["juz_boundaries"]["filename"]
+        with (self.data_dir / filename).open(encoding="utf-8") as handle:
+            for line_number, line in enumerate(handle, 1):
+                parts = line.rstrip("\n").split("\t")
+                if len(parts) != 3 or int(parts[0]) != line_number:
+                    raise CorpusError(f"Malformed juz boundary on line {line_number}")
+                starts.append(VerseAddress(int(parts[1]), int(parts[2])))
+        verse_positions = {verse.address: index for index, verse in enumerate(self.verses)}
+        juzs: list[Juz] = []
+        for index, start in enumerate(starts):
+            start_position = verse_positions[start]
+            end_position = (
+                verse_positions[starts[index + 1]] if index + 1 < len(starts) else len(self.verses)
+            )
+            juzs.append(
+                Juz(index + 1, start, QuerySet(self.verses.values()[start_position:end_position]))
+            )
+        return OneBasedCollection(juzs)
+
+    @property
+    def verb_words(self) -> QuerySet[Word]:
+        """Return all words with at least one verb stem, computed lazily once."""
+        if self._verbs is None:
+            self._verbs = self.words.filter(Word.is_verb)
+        return self._verbs
+
+    @property
+    def noun_words(self) -> QuerySet[Word]:
+        """Return all words with at least one noun stem, computed lazily once."""
+        if self._nouns is None:
+            self._nouns = self.words.filter(Word.is_noun)
+        return self._nouns
+
+    def _tokens_with_tags(self, *tags: str) -> QuerySet[Token]:
+        """Cache each token selection lazily while preserving corpus order."""
+        if tags not in self._token_selections:
+            wanted = frozenset(tags)
+            self._token_selections[tags] = self.tokens.filter(lambda token: token.tag in wanted)
+        return self._token_selections[tags]
+
+    def verbs(self) -> QuerySet[Token]:
+        """Return verb tokens (V), equivalent to ``tokens.with_tag('V')``."""
+        return self._tokens_with_tags("V")
+
+    def nouns(self) -> QuerySet[Token]:
+        """Return noun tokens (N); use proper_nouns() for PN or nominals() for all."""
+        return self._tokens_with_tags("N")
+
+    def proper_nouns(self) -> QuerySet[Token]:
+        """Return proper-noun tokens (PN)."""
+        return self._tokens_with_tags("PN")
+
+    def adjectives(self) -> QuerySet[Token]:
+        """Return adjective tokens (ADJ)."""
+        return self._tokens_with_tags("ADJ")
+
+    def pronouns(self) -> QuerySet[Token]:
+        """Return personal, demonstrative, and relative pronouns (PRON, DEM, REL)."""
+        return self._tokens_with_tags("PRON", "DEM", "REL")
+
+    def adverbs(self) -> QuerySet[Token]:
+        """Return time and location adverb tokens (T, LOC)."""
+        return self._tokens_with_tags("T", "LOC")
+
+    def prepositions(self) -> QuerySet[Token]:
+        """Return preposition tokens (P), including attached prefixes."""
+        return self._tokens_with_tags("P")
+
+    def conjunctions(self) -> QuerySet[Token]:
+        """Return coordinating and subordinating conjunctions (CONJ, SUB)."""
+        return self._tokens_with_tags("CONJ", "SUB")
+
+    def nominals(self) -> QuerySet[Token]:
+        """Return the nominal family: N, PN, ADJ, IMPN, PRON, DEM, REL, T, LOC."""
+        return self._tokens_with_tags(*_NOMINAL_TAGS)
+
+    def particles(self) -> QuerySet[Token]:
+        """Return the QAC particle family plus DET; exclude disconnected letters.
+
+        Includes prepositions, conjunctions, lam prefixes, and emphatic suffixes,
+        not just independent words. See the API reference for the exact tag set.
+        """
+        return self._tokens_with_tags(*_PARTICLE_TAGS)
+
+    def translation_metadata(self) -> dict[str, str] | None:
+        """Return a copy of the supplied translation's provenance, or None."""
+        return dict(self.translation.metadata) if self.translation else None
+
+    def translation_coverage(self, verses: Iterable[Verse] | None = None) -> TranslationCoverage:
+        """Check translation coverage of all verses, or any iterable of Verse objects."""
+        addresses = {(v.chapter, v.verse) for v in (self.verses if verses is None else verses)}
+        available = self.translation.texts.keys() if self.translation else set()
+        missing = tuple(sorted(addresses - available))
+        return TranslationCoverage(len(addresses), len(addresses) - len(missing), missing)
+
+    def require_translation(self, verses: Iterable[Verse] | None = None) -> None:
+        """Raise TranslationNotAvailableError if any requested verse is untranslated."""
+        from .text import TranslationNotAvailableError
+
+        coverage = self.translation_coverage(verses)
+        if coverage.missing:
+            sample = ", ".join(f"{c}:{v}" for c, v in coverage.missing[:8])
+            raise TranslationNotAvailableError(
+                f"Translation missing for {len(coverage.missing)} verses: {sample}"
+            )
+
+    def provenance(self) -> dict[str, Any]:
+        """Return source versions, hashes, origins, and license identifiers."""
+        return copy.deepcopy(self._manifest)
+
+    def licenses(self) -> tuple[dict[str, str], ...]:
+        """Summarize the terms attached to each installed source file."""
+        return tuple(
+            {
+                "source": source["title"],
+                "license": source["license"],
+                "url": source.get("official_url", ""),
+            }
+            for source in self._manifest["sources"].values()
+        )
+
+    def chapter(self, number: int) -> Chapter:
+        """Return chapter ``number`` using Quranic 1-based numbering."""
+        return self.chapters[number]
+
+    get_chapter = chapter
+
+    def verse(self, chapter: int, verse: int) -> Verse:
+        """Return one verse by its 1-based chapter and verse numbers."""
+        return self.verses[chapter, verse]
+
+    get_verse = verse
+
+    def word(self, chapter: int, verse: int, word: int) -> Word:
+        """Return one morphology word by its complete 1-based address."""
+        return self.words[chapter, verse, word]
+
+    get_word = word
+
+    def token(self, chapter: int, verse: int, word: int, part: int) -> Token:
+        """Return one morphology segment by its complete 1-based address."""
+        return self.tokens[chapter, verse, word, part]
+
+    get_token = token
+
+    def index_words(self, field: str) -> dict[Any, tuple[Word, ...]]:
+        """Build and cache an inverted word index for a field or property."""
+        if field not in self._word_indexes:
+            index: defaultdict[Any, list[Word]] = defaultdict(list)
+            for word in self.words:
+                value = resolve_value(word, field)
+                values = value if isinstance(value, tuple) else (value,)
+                for member in values:
+                    index[member].append(word)
+            self._word_indexes[field] = {key: tuple(words) for key, words in index.items()}
+        return self._word_indexes[field]
+
+    def find_words(self, **criteria: Any) -> QuerySet[Word]:
+        """Find words by one or more exact attributes.
+
+        The first equality criterion uses a lazily built inverted index; remaining
+        criteria filter only those candidates. Tuple-valued word fields such as
+        ``roots`` use membership semantics through :meth:`QuerySet.where`.
+        """
+        if not criteria:
+            return QuerySet(self.words)
+        field, expected = next(iter(criteria.items()))
+        if callable(expected):
+            return self.words.where(**criteria)
+        candidates = QuerySet(self.index_words(field).get(expected, ()))
+        return candidates.where(**criteria)
+
+    def search(
+        self,
+        text: str,
+        *,
+        level: str = "verse",
+        representation: str = "auto",
+        **normalization: Any,
+    ) -> QuerySet[Verse] | QuerySet[Word]:
+        """Search verse or word text without requiring a predicate.
+
+        ASCII ``text`` is interpreted as Buckwalter when ``representation='auto'``;
+        non-ASCII text is interpreted as Arabic. ``level`` accepts singular or plural
+        ``'verse'`` and ``'word'``. Normalization options are the same as
+        :meth:`~quran_processing_toolkit.text.TextMixin.contains`.
+        """
+        level = level.lower().rstrip("s")
+        if level == "verse":
+            return self.verses.contains(text, representation=representation, **normalization)
+        if level == "word":
+            return self.words.contains(text, representation=representation, **normalization)
+        raise ValueError("search() level must be 'verse' or 'word'")
+
+    def stats(self) -> dict[str, int]:
+        """Return canonical counts; legacy verbs/nouns entries count whole words."""
+        return {
+            "chapters": len(self.chapters),
+            "verses": len(self.verses),
+            "words": len(self.words),
+            "tokens": len(self.tokens),
+            "juzs": len(self.juzs),
+            "verbs": len(self.verb_words),
+            "nouns": len(self.noun_words),
+        }
+
+    def longest_word_sequence(
+        self,
+        predicate: Callable[[Word], bool],
+        *,
+        cross_verse_boundaries: bool = False,
+    ) -> QuerySet[Word]:
+        """Find the longest contiguous sequence of words matching a predicate."""
+        if cross_verse_boundaries:
+            return QuerySet(self.words).longest_run(predicate)
+        best = QuerySet[Word]()
+        for verse in self.verses:
+            candidate = QuerySet(verse.words).longest_run(predicate)
+            if len(candidate) > len(best):
+                best = candidate
+        return best
+
+    def longest_word_sequence_without_letters(
+        self,
+        letters: str,
+        *,
+        representation: str = "auto",
+        cross_verse_boundaries: bool = False,
+        **normalization: Any,
+    ) -> QuerySet[Word]:
+        """Find the longest word run avoiding letters, with ASCII meaning Buckwalter."""
+        return self.longest_word_sequence(
+            lambda word: word.contains_no_letters(
+                letters, representation=representation, **normalization
+            ),
+            cross_verse_boundaries=cross_verse_boundaries,
+        )
+
+
+def load_quran(
+    data_dir: str | os.PathLike[str] | None = None,
+    *,
+    translation: str | os.PathLike[str] | Translation | None = None,
+    cache: bool = True,
+    disk_cache: bool = True,
+    cache_dir: str | os.PathLike[str] | None = None,
+) -> QuranCorpus:
+    """Load the Quran corpus with optional user data and two levels of caching.
+
+    Args:
+        data_dir: Directory containing a supported source-data manifest and files.
+            When omitted, discovery checks the environment override, installed data
+            distribution, and finally the monorepo checkout.
+        translation: Optional numbered UTF-8/structured JSON file, or a Translation
+            returned by load_translation() or download_translation(). The
+            ``QURAN_PROCESSING_TOOLKIT_TRANSLATION`` environment variable is used
+            when this argument is omitted.
+        cache: Reuse an equivalent :class:`QuranCorpus` in this process.
+        disk_cache: Read and write the verified derived morphology cache.
+        cache_dir: Override the derived cache location for this load.
+
+    Returns:
+        A fully validated, immutable-by-convention corpus object graph.
+    """
+    return QuranCorpus.load(
+        data_dir,
+        translation=translation,
+        cache=cache,
+        disk_cache=disk_cache,
+        cache_dir=cache_dir,
+    )
+
+
+def clear_memory_cache() -> None:
+    """Forget all in-process corpus instances without deleting the disk cache."""
+    with _CACHE_LOCK:
+        _CACHE.clear()