hadith-detector 0.1.0__py3-none-any.whl

hadith_detector/__init__.py

@@ -0,0 +1,33 @@
+"""Offline Arabic Hadith detection with bundled or explicitly supplied resources."""
+
+from importlib.metadata import version
+
+from ._convenience import annotate, clear_cache, detect, get_hadith
+from .config import DetectorConfig
+from .detector import HadithDetector
+from .models import (
+    Detection,
+    HadithRecord,
+    HadithReference,
+    OmittedSourceToken,
+    SourceOccurrence,
+    WordAlignment,
+)
+
+__version__ = version("hadith-detector")
+
+__all__ = [
+    "__version__",
+    "HadithDetector",
+    "DetectorConfig",
+    "detect",
+    "annotate",
+    "get_hadith",
+    "clear_cache",
+    "HadithRecord",
+    "SourceOccurrence",
+    "HadithReference",
+    "Detection",
+    "WordAlignment",
+    "OmittedSourceToken",
+]

hadith_detector/_annotation.py

@@ -0,0 +1,49 @@
+"""Lossless plain-text escaping and deterministic HTML fragments."""
+
+from collections.abc import Sequence
+from html import escape
+
+from .models import Detection
+
+
+def validate(text: str, format: str) -> None:
+    """Reject unsupported formats and characters HTML cannot preserve losslessly."""
+    if not isinstance(text, str):
+        raise TypeError("text must be a string")
+    if not isinstance(format, str):
+        raise TypeError("format must be a string")
+    if format != "html":
+        raise ValueError("Only format='html' is supported")
+    if any(char == "\x00" or 0xD800 <= ord(char) <= 0xDFFF for char in text):
+        raise ValueError("HTML annotation cannot preserve NUL or lone surrogate characters")
+
+
+def _escape(text: str) -> str:
+    # Character references keep CR intact through HTML's newline preprocessing.
+    return escape(text, quote=True).replace("\r", "&#13;")
+
+
+def render(text: str, detections: Sequence[Detection]) -> str:
+    """Render detector-selected spans without repeating matching or overlap selection."""
+    parts = []
+    cursor = 0
+    for index, match in enumerate(detections):
+        for value in (match.start, match.end, match.errors, match.missing_words):
+            if isinstance(value, bool) or not isinstance(value, int) or value < 0:
+                raise ValueError("Annotation offsets and costs must be nonnegative integers")
+        if not cursor <= match.start < match.end <= len(text):
+            raise ValueError("Annotation spans must be in bounds, ordered and non-overlapping")
+        if text[match.start : match.end] != match.text:
+            raise ValueError("Annotation span differs from original input text")
+        parts.append(_escape(text[cursor : match.start]))
+        parts.append(
+            f'<mark class="hadith-match" data-match-index="{index}" '
+            f'data-start="{match.start}" data-end="{match.end}" '
+            f'data-errors="{match.errors}" data-missing-words="{match.missing_words}" '
+            f'data-reference-count="{len(match.references)}">'
+        )
+        parts.append(_escape(text[match.start : match.end]))
+        parts.append("</mark>")
+        cursor = match.end
+    parts.append(_escape(text[cursor:]))
+    return "".join(parts)

hadith_detector/_bundle.json

@@ -0,0 +1,45 @@
+{
+  "artifacts": {
+    "record-offsets.bin": "5ff1228d1d7cc82df5baa6853b510dca8d0897e165217d4df9bf3a26f16f32d1",
+    "records.jsonl": "9ec8e19aaab80ffbc01aa162122060099bc0a5de3841b544b4afa1c381414246",
+    "tokens.bin": "3488e5e47e5a91d42c2f84ba14560cf673e134081aea52a6ab28f08fdda2d317",
+    "trigram-offsets.bin": "c7edf613d74f00c806eafea7f0a6fd3f7dcd916c094f2c210d88b8380d6cbf6b",
+    "trigram-postings.bin": "0a246cc840d7fad97945ca2164545c72ad59a35245d87cb6aa7359fd6bea21cf",
+    "trigrams.marisa": "9bb9eae9fc858f7a5ed9a6675035f407b77f9c561e9cfd441cbcf0b14b49518e",
+    "unigram-offsets.bin": "3e8f5dc282137fc6f418f6bc33e2b0dbe987d860b4ffc5761944432a5f3f5d01",
+    "unigram-postings.bin": "752a095ec3f92dddbc90a751c6ed295f5b363035f76ba6479098d13750381b8a",
+    "vocabulary.marisa": "faf3d4b7d5d9952e5ea3823d305d29e46d4fcc1a6f52751cdada81a529c655cc"
+  },
+  "counts": {
+    "records": 14940,
+    "tokens": 1039600,
+    "trigram_keys": 382866,
+    "trigram_postings": 1009649,
+    "vocabulary": 40949
+  },
+  "notices": {
+    "LICENSES.md": "211ab9b99234f8b4c99f4e295cd29f0524d1a8508b10916c2ce84aa28ee4fde3",
+    "PROVENANCE.md": "f565245adb555fbe5b4d4a838369fd3c50e37de3f0302c5a0291c6a36c9517c7",
+    "README.md": "fc207de3c37457f4cd02075cdd1f76562905f7b5efe7cdaf3c0e2ae0a51bc9fc",
+    "SOURCES.md": "a395c79251f54b1b780a475c32480f968d277b71a04017af18227df3e9545e53",
+    "UPSTREAM-UNLICENSE.txt": "6b0382b16279f26ff69014300541967a356a666eb0b91b422f6862f6b7dad17e",
+    "collections.csv": "806aef24f330464c3b9bb0f2526ed1f962d12c6edecc8d0047633c324189f11b",
+    "configs.csv": "5c397a92a7449a568104ff2e6d6e60bdea68d7d0f19556591871be38693c210c"
+  },
+  "source_dataset": {
+    "configurations": [
+      "bukhari-ar",
+      "muslim-ar"
+    ],
+    "repository": "quranlab/hadith",
+    "revision": "cc6972dae1f46f5f5e2b65e877d24c4539bc4391"
+  },
+  "version": 1,
+  "versions": {
+    "index": 2,
+    "manifest": 1,
+    "normalization": 1,
+    "records": 1,
+    "tokens": 1
+  }
+}

hadith_detector/_bundled.py

@@ -0,0 +1,16 @@
+"""Resolve preinstalled resources without extraction, downloads or construction."""
+
+from pathlib import Path
+
+
+def directory(index_dir: str | Path | None) -> Path:
+    if index_dir is not None:
+        return Path(index_dir).resolve()
+    bundled = Path(__file__).resolve().parent / "_data"
+    if not (bundled / "manifest.json").is_file():
+        raise FileNotFoundError(
+            "Bundled Hadith resources are missing. Reinstall the complete hadith-detector "
+            "distribution, or pass index_dir explicitly. Development checkouts must run "
+            "python -m tools.stage_bundle before using bundled resources."
+        )
+    return bundled

hadith_detector/_convenience.py

@@ -0,0 +1,84 @@
+"""One synchronized, lazily populated convenience detector cache."""
+
+from pathlib import Path
+from threading import RLock
+
+from ._annotation import validate
+from ._bundled import directory
+from .config import DetectorConfig
+from .detector import HadithDetector
+from .models import Detection, HadithRecord
+
+_lock = RLock()
+_entry: tuple[Path, HadithDetector] | None = None
+
+
+def _detector(index_dir: str | Path | None) -> HadithDetector:
+    global _entry
+    resolved = directory(index_dir)
+    with _lock:
+        if _entry is None or _entry[0] != resolved:
+            detector = HadithDetector(index_dir)
+            _entry = resolved, detector
+        return _entry[1]
+
+
+def clear_cache() -> None:
+    """Forget the cached detector without interrupting any in-flight call."""
+    global _entry
+    with _lock:
+        _entry = None
+
+
+def detect(
+    text: str,
+    *,
+    index_dir: str | Path | None = None,
+    config: DetectorConfig | None = None,
+    min_match_words: int | None = None,
+    max_errors: int | None = None,
+    max_missing_words: int | None = None,
+) -> list[Detection]:
+    """Detect using cached bundled/local resources and independent per-call defaults."""
+    if config is not None and not isinstance(config, DetectorConfig):
+        raise TypeError("config must be a DetectorConfig or None")
+    effective = (config if config is not None else DetectorConfig())._override(
+        min_match_words, max_errors, max_missing_words
+    )
+    return _detector(index_dir).detect(
+        text,
+        min_match_words=effective.min_match_words,
+        max_errors=effective.max_errors,
+        max_missing_words=effective.max_missing_words,
+    )
+
+
+def annotate(
+    text: str,
+    *,
+    index_dir: str | Path | None = None,
+    format: str = "html",
+    config: DetectorConfig | None = None,
+    min_match_words: int | None = None,
+    max_errors: int | None = None,
+    max_missing_words: int | None = None,
+) -> str:
+    """Annotate plain text using the single cached detector and per-call settings."""
+    validate(text, format)
+    if config is not None and not isinstance(config, DetectorConfig):
+        raise TypeError("config must be a DetectorConfig or None")
+    effective = (config if config is not None else DetectorConfig())._override(
+        min_match_words, max_errors, max_missing_words
+    )
+    return _detector(index_dir).annotate(
+        text,
+        format=format,
+        min_match_words=effective.min_match_words,
+        max_errors=effective.max_errors,
+        max_missing_words=effective.max_missing_words,
+    )
+
+
+def get_hadith(hadith_key: str, *, index_dir: str | Path | None = None) -> HadithRecord:
+    """Retrieve an isolated source snapshot using the single cached detector."""
+    return _detector(index_dir).get_hadith(hadith_key)

hadith_detector/_data/bundle.json

@@ -0,0 +1 @@
+{"files":{"manifest.json":"a34ebdd099389594b039a4ac55e667081110c2c192f43eee08095fe1f7d5dcbd","notices/LICENSES.md":"211ab9b99234f8b4c99f4e295cd29f0524d1a8508b10916c2ce84aa28ee4fde3","notices/PROVENANCE.md":"f565245adb555fbe5b4d4a838369fd3c50e37de3f0302c5a0291c6a36c9517c7","notices/README.md":"fc207de3c37457f4cd02075cdd1f76562905f7b5efe7cdaf3c0e2ae0a51bc9fc","notices/SOURCES.md":"a395c79251f54b1b780a475c32480f968d277b71a04017af18227df3e9545e53","notices/UPSTREAM-UNLICENSE.txt":"6b0382b16279f26ff69014300541967a356a666eb0b91b422f6862f6b7dad17e","notices/collections.csv":"806aef24f330464c3b9bb0f2526ed1f962d12c6edecc8d0047633c324189f11b","notices/configs.csv":"5c397a92a7449a568104ff2e6d6e60bdea68d7d0f19556591871be38693c210c","record-offsets.bin":"5ff1228d1d7cc82df5baa6853b510dca8d0897e165217d4df9bf3a26f16f32d1","records.jsonl":"9ec8e19aaab80ffbc01aa162122060099bc0a5de3841b544b4afa1c381414246","tokens.bin":"3488e5e47e5a91d42c2f84ba14560cf673e134081aea52a6ab28f08fdda2d317","trigram-offsets.bin":"c7edf613d74f00c806eafea7f0a6fd3f7dcd916c094f2c210d88b8380d6cbf6b","trigram-postings.bin":"0a246cc840d7fad97945ca2164545c72ad59a35245d87cb6aa7359fd6bea21cf","trigrams.marisa":"9bb9eae9fc858f7a5ed9a6675035f407b77f9c561e9cfd441cbcf0b14b49518e","unigram-offsets.bin":"3e8f5dc282137fc6f418f6bc33e2b0dbe987d860b4ffc5761944432a5f3f5d01","unigram-postings.bin":"752a095ec3f92dddbc90a751c6ed295f5b363035f76ba6479098d13750381b8a","vocabulary.marisa":"faf3d4b7d5d9952e5ea3823d305d29e46d4fcc1a6f52751cdada81a529c655cc"},"lock_sha256":"1f4711afa95bda7cd5ce69d54ca804553e53d0b51ef64611dd796f2b219f593d","version":1}

hadith_detector/_data/manifest.json

@@ -0,0 +1 @@
+{"algorithm":"trigram+unigram","artifacts":{"record-offsets.bin":"5ff1228d1d7cc82df5baa6853b510dca8d0897e165217d4df9bf3a26f16f32d1","records.jsonl":"9ec8e19aaab80ffbc01aa162122060099bc0a5de3841b544b4afa1c381414246","tokens.bin":"3488e5e47e5a91d42c2f84ba14560cf673e134081aea52a6ab28f08fdda2d317","trigram-offsets.bin":"c7edf613d74f00c806eafea7f0a6fd3f7dcd916c094f2c210d88b8380d6cbf6b","trigram-postings.bin":"0a246cc840d7fad97945ca2164545c72ad59a35245d87cb6aa7359fd6bea21cf","trigrams.marisa":"9bb9eae9fc858f7a5ed9a6675035f407b77f9c561e9cfd441cbcf0b14b49518e","unigram-offsets.bin":"3e8f5dc282137fc6f418f6bc33e2b0dbe987d860b4ffc5761944432a5f3f5d01","unigram-postings.bin":"752a095ec3f92dddbc90a751c6ed295f5b363035f76ba6479098d13750381b8a","vocabulary.marisa":"faf3d4b7d5d9952e5ea3823d305d29e46d4fcc1a6f52751cdada81a529c655cc"},"build_fingerprint":"e92ba58375816eb02ebb26c9f66a9cb15add35e990037e1b6444864ba7ce344b","builder":{"dependency_lock_sha256":"8ef4b3b8de07c227551b845eb52ef46d47ff7b618f153c3cc9ebd157aa1666ce","marisa_trie":"1.4.1","native_settings":{"binary":false,"cache_size":512,"num_tries":3,"order":131072},"python":"3.14.6","sources":{"src/hadith_detector/_marisa.py":"a3b7800da1923af9aef7a16127ae92a8aa4a6506bfc89540a9cc8b93f5a9a6b1","src/hadith_detector/_resources.py":"f7e9c2e124ba1fec9c255fbf89fe1f0590e82f827237f939d3bb854e0ca92a59","src/hadith_detector/_text.py":"825013da3b680c286c824d9915668618a9fa2aab3afab93f9eea85ae813a1571","tools/build_index.py":"3752db36fcd43a107edec828e253ffd76925b282a103d077ec638b2f5b6dfec5","tools/exact_benchmark/corpus.py":"5ccd73de6eb35a41e2df0f95ddc37550f9a7649282a928b5b5489902a472bc09","tools/exact_benchmark/indexes.py":"38266cdffc92f383b6d728cabb6c38699fae5b0177a1b2e9f60f4abb9d9c2a1c"},"unicode":"16.0.0"},"byte_order":"little","counts":{"records":14940,"tokens":1039600,"trigram_keys":382866,"trigram_postings":1009649,"vocabulary":40949},"format":"hadith-detector-index","prepared_manifest_sha256":"f0ee8509b911b7835771de94fe65a97a17098ebba53de57f9c274dd077a5a0e2","source_artifacts":{"records.jsonl":"9ec8e19aaab80ffbc01aa162122060099bc0a5de3841b544b4afa1c381414246","tokens.jsonl":"ee07fce76d7862854e069ef81d72255cf3cb616daad9d0c57977e287a4e11f33","validation.json":"0e2187d3b4e7abb19de4648de54ff5ec79c595893fc5a8ef659dd8ed5baadbb1"},"source_dataset":{"configurations":["bukhari-ar","muslim-ar"],"repository":"quranlab/hadith","revision":"cc6972dae1f46f5f5e2b65e877d24c4539bc4391"},"source_fingerprint":"87fd7b7824fe60063026a582a845082f7fc6fe88587cb1d0805dbae372ea01ed","versions":{"index":2,"manifest":1,"normalization":1,"records":1,"tokens":1}}

hadith_detector/_data/notices/LICENSES.md

@@ -0,0 +1,17 @@
+# Licenses, Attribution & Removal (NOTICE) — quranlab/hadith
+
+Binding terms are **per row** (`license`, `source`, `attribution`). Summary below.
+
+## Theology gate
+Strictly **Ahl al-Sunnah wa'l-Jamāʿah**. Excluded by design: the Shia Four Books (al-Kāfī, Man lā yaḥḍuruhu al-Faqīh, Tahdhīb al-Aḥkām, al-Istibṣār) and other Shia/Ismāʿīlī collections; Ibāḍī (Musnad al-Rabīʿ b. Ḥabīb); Ahmadiyya/Qādiānī; Quranist/hadith-rejecter sources; hostile-orientalist grade framings.
+
+## Sources & tiers
+- **Arabic matn (65,834 hadith across 13 collections):** public domain; digitisation from `mhashim6/Open-Hadith-Data`, ODbL-1.0 + DbCL-1.0. Attribution required.
+- **Graded multilingual layer (72 languages):** HadeethEnc.com (IslamHouse / Saudi MoIA) — verbatim, version-pinned, with grade + attribution; reproduced under their open-with-attribution terms; removal on request.
+- **Copyrighted full-collection translations (roadmap):** by reference + full attribution under **notice-and-takedown** (removal on rights-holder request via the HF Community tab).
+
+## Grade integrity
+Grades are **grader-attributed, never asserted by QuranLab**; the matn layer is ungraded; collection-level status lives in the `collections` config, never auto-stamped per row.
+
+## Removal
+Open the dataset's **Community (Discussions)** tab to request removal of any row.

hadith_detector/_data/notices/PROVENANCE.md

@@ -0,0 +1,46 @@
+# Bundled corpus provenance and redistribution record
+
+This package includes the Arabic `bukhari-ar` and `muslim-ar` configurations from
+QuranLab Hadith revision `cc6972dae1f46f5f5e2b65e877d24c4539bc4391`: 7,580 and
+7,360 records respectively. Original fields, identities, duplicate narrations,
+nullable URLs and source text are preserved. Tokens and the prebuilt MARISA v2
+index are derived resources; normalization version is 1.
+
+Credit **QuranLab** (<https://huggingface.co/datasets/quranlab/hadith>) and
+**fawazahmed0/hadith-api** (<https://github.com/fawazahmed0/hadith-api>).
+The pinned `configs.csv` attributes both selected configurations to hadith-api,
+describes classical Arabic text as public domain, and marks commercial use as
+allowed. Upstream's Unlicense permits copying, modification and redistribution.
+Its preserved text was retrieved at commit
+`df57907be35291c91ad6a6691180e22ca9920784`; this pins the licensing evidence, not
+the unknown upstream data revision used by QuranLab.
+
+## Distinguishing the available evidence
+
+The broad QuranLab `LICENSES.md` summary attributes Arabic material to
+Open-Hadith-Data and mentions ODbL/DbCL. The selected configuration rows instead
+identify hadith-api. `SOURCES.md` identifies Open-Hadith-Data with Ahmad/Darimi,
+which are not bundled. Both notices are preserved verbatim, alongside the full
+pinned configuration and collection metadata. This package follows the selected
+configurations' attribution and the maintainer's approved Phase 2 provenance
+decision; the broad summary is not rewritten or presented as consistent.
+
+Only the selected Arabic records are redistributed: no translations, HadeethEnc
+text, Ahmad/Darimi records, or text scraped from sunnah.com. Source links and
+numbering are retained as supplied; links do not establish rights to linked pages.
+No independent authentication or grader endorsement is asserted.
+
+The project code is MIT licensed. This does not relicense third-party data.
+`UPSTREAM-UNLICENSE.txt`, QuranLab `README.md`, `LICENSES.md`, `SOURCES.md`,
+`configs.csv` and `collections.csv` accompany the data. Preserve these notices
+when redistributing this bundle. Consult the original notices for attribution,
+correction and removal contact information.
+
+## Integrity and build provenance
+
+`_bundle.json` records approved resource payload hashes, counts, versions and
+source revision. `_data/manifest.json` records source preparation and builder
+fingerprints; `_data/bundle.json` records every staged file and its checksum.
+The builder manifest may differ when code or environment provenance changes,
+while approved resource payloads must remain byte-identical. No runtime resource
+construction or download is performed.