corpora-py 0.1.3__py3-none-any.whl

admin/__init__.py

@@ -0,0 +1,22 @@
+"""admin — admin-only / full-featured tooling.
+
+Houses conversion pipelines (EPUB/HTML → Text-Fabric, TF → .exg packaging)
+and other heavy or privileged operations. These typically require the
+[full] extra (text-fabric) and are not needed in the slim client runtime.
+"""
+
+from . import converters, parsers, services
+
+__all__ = ["converters", "parsers", "services"]
+
+try:
+    # admin.ingest imports docling-core (pandas, pillow) at module import
+    # time. Slim serverless bundles (Vercel — see vercel.json) exclude that
+    # chain to stay under the function size limit, so its absence must not
+    # break `import admin`; /ingest degrades to 503 there (see
+    # services/ingest_api.py).
+    from . import ingest  # noqa: F401
+
+    __all__.append("ingest")
+except ModuleNotFoundError:  # pragma: no cover - exercised only in slim deploys
+    pass

admin/converters/CLAUDE.md

@@ -0,0 +1,62 @@
+# CLAUDE.md — `admin.converters`
+
+`Document`/`Unit` tree → Text-Fabric dataset → `.cfm` cache → `.corpus` archive. See
+`packages/admin/CLAUDE.md` for the shared-schema/shared-walker architecture that makes this package
+possible; this file covers the Text-Fabric, Context-Fabric, and `.corpus` details specific to
+`converters/`.
+
+Read `_walker.py` before touching any feature/node-creation logic — it's the one TF walk reused by
+every `_{format}_to_tf.py`.
+
+## Text-Fabric walker gotchas (all handled in `_walker.py`)
+
+- **Every feature name must have metadata**, or `cv.walk()` fails validation with `"node feature has no metadata"`.
+  Feature names here vary per document (HTML attributes, TEI `@type`, ...) so they can't be declared upfront in
+  `featureMeta=`; `set_features()` registers each one dynamically via `cv.meta(name, valueType="str")` right before
+  setting it. If you add a
+  `cv.feature()` call anywhere, route it through `set_features()`, not
+  `cv.feature()` directly, or you'll reintroduce this failure.
+- **Dynamically-registered features need an explicit `valueType`** or the exporter warns `"Missing @valueType"`
+  (non-fatal, but avoidable — that's why `set_features()` always passes `valueType="str"`).
+- **A node covering zero slots gets silently deleted** by Text-Fabric's
+  "remove unlinked nodes" pass. A leaf `Unit` with no tokens and no children (a blank PDF page, an `<img>`, an `<hr>`)
+  would otherwise vanish along with its attributes — `_walk_unit()` gives genuinely empty leaves one placeholder
+  empty-text slot so they survive.
+- **`otext.sectionTypes`/`sectionFeatures` can't be empty**, but also don't need to be elaborate: every converter uses a
+  single section level (the root `book`/`document`/`text` node, with `title` as its section feature). Finer structure
+  (chapters, pages, divs) is still expressed as ordinary node types via `otype_for` — it just isn't declared as TF
+  "sections", which would require strict, consistent nesting we can't guarantee across arbitrary source documents.
+- **`SKIP_TAGS` in `parsers/_html.py` is scoped to tags that only make sense to drop when nested inside `<body>`**
+  (script/style/noscript/svg/math). It used to include `"head"` for HTML's metadata tag, which silently ate TEI's
+  `<head>` (a heading element, reused by the shared walker) — don't add HTML-specific tag names back to that set without
+  checking what they mean in TEI/XML first.
+
+## Context-Fabric (`cfabric`) notes
+
+- There is **no separate compile API**. `.cfm` compilation happens automatically the first time a dataset is loaded via
+  `cfabric.Fabric(locations=...).loadAll()`. `convert_to_cfm()` exists only to trigger that load on purpose and hand
+  back the resulting `.cfm` path.
+- `Fabric(...).loadAll()` returns `Api | bool` (`False` on failure) — always narrow with `isinstance(result, bool)`
+  before touching `.F`/`.T`/`.Fall()`; those are dynamically populated at load time so mypy can't see their attributes
+  either (hence the `type: ignore[attr-defined]` in
+  `convert_to_corpus.py`).
+
+## `.corpus` archive
+
+The archive format (`manifest.yml`, `toc.yml`, `assets/`, `.git/`,
+`corpora/{*.tf, .cfm/}`) is the contract both the Corpora and Exegia apps parse. The canonical spec is maintained in an
+external vault by the Corpora team. Before changing manifest/toc shape in `convert_to_corpus.py`, consult the current
+schema definition with the team or check the app's schema loader to understand the expected format.
+
+## Known gaps (converter-side)
+
+Service-side gaps (progress reporting, job registry, archive cleanup) live in
+`src/admin/services/CLAUDE.md`.
+
+- **No `_xml_to_tf.py`.** `XmlParser` exists (`admin.parsers`) but there's no matching Text-Fabric converter — generic
+  XML has no fixed node-type vocabulary to map onto, unlike TEI's `<div>`/`<p>` convention. Add one the same way as the
+  others (pick an `otype_for`, wire it into
+  `converters/__init__.py`'s `CONVERTERS`) if a concrete need shows up; don't add it speculatively.
+- `dataset_id`/`project_id`/`publisher_id`/`author_ids` in
+  `convert_to_corpus()` are caller-supplied and default to `""` — this package has no way to know them; they're assigned
+  by whatever backend calls it (the Corpora/Exegia app, not this converter).

admin/converters/__init__.py

@@ -0,0 +1,44 @@
+"""admin.converters — conversion and packaging tools (admin / full runtime).
+
+These modules depend on optional heavy deps (text-fabric, context-fabric,
+under the `[full]` extra) and are intended for dataset ingestion / admin
+tooling only.
+
+Pipeline: a `_{format}_to_tf.py` converter turns a source document into a
+Text-Fabric dataset; `convert_to_cfm` compiles that into Context-Fabric's
+`.cfm` cache; `convert_to_corpus` packages both into a `.corpus` archive.
+"""
+
+from ..parsers import SourceFormat
+from ._epub_to_tf import convert_epub_to_tf
+from ._html_to_tf import convert_html_to_tf
+from ._pdf_to_tf import convert_pdf_to_tf
+from ._tei_to_tf import convert_tei_to_tf
+from ._tei_zip_to_tf import convert_tei_zip_to_tf
+from ._text_to_tf import convert_text_to_tf
+from ._tf_zip_to_tf import convert_tf_zip_to_tf
+from .convert_to_cfm import convert_to_cfm
+from .convert_to_corpus import convert_to_corpus
+
+CONVERTERS = {
+    SourceFormat.EPUB: convert_epub_to_tf,
+    SourceFormat.HTML: convert_html_to_tf,
+    SourceFormat.PDF: convert_pdf_to_tf,
+    SourceFormat.TEI: convert_tei_to_tf,
+    SourceFormat.TEI_ZIP: convert_tei_zip_to_tf,
+    SourceFormat.PLAIN: convert_text_to_tf,
+    SourceFormat.TF_ZIP: convert_tf_zip_to_tf,
+}
+
+__all__ = [
+    "CONVERTERS",
+    "convert_epub_to_tf",
+    "convert_html_to_tf",
+    "convert_pdf_to_tf",
+    "convert_tei_to_tf",
+    "convert_tei_zip_to_tf",
+    "convert_text_to_tf",
+    "convert_tf_zip_to_tf",
+    "convert_to_cfm",
+    "convert_to_corpus",
+]

admin/converters/_epub_to_tf.py

@@ -0,0 +1,51 @@
+"""
+    EPUB to Text-Fabric Converter
+
+    Converts EPUB ebook files into Text-Fabric datasets using the epub service
+    for parsing and the tf.convert.walker library for TF generation.
+
+    Features:
+    - Extracts EPUB metadata (title, author, publisher, etc.)
+    - Preserves book structure (chapters/pages)
+    - Converts HTML content to queryable nodes
+    - Creates semantic nodes from clean HTML
+    - Tracks conversion progress
+
+    Node Types:
+    - book: Root node for the entire EPUB
+    - chapter: Individual pages/chapters from the EPUB
+    - element: HTML elements from page content
+    - paragraph: Paragraph-like elements
+    - link: Link elements with href
+    - word: Individual words (slots)
+"""
+
+from pathlib import Path
+
+from ..parsers import EpubParser
+from ..parsers.schema import Unit
+from ._walker import convert_document
+
+_PARAGRAPH_TAGS = {"p", "blockquote"}
+_LINK_TAGS = {"a"}
+
+
+def _otype_for(unit: Unit) -> str:
+    if unit.type == "chapter":
+        return "chapter"
+    if unit.type in _PARAGRAPH_TAGS:
+        return "paragraph"
+    if unit.type in _LINK_TAGS:
+        return "link"
+    return "element"
+
+
+def convert_epub_to_tf(source: str, output_dir: str | Path) -> Path:
+    """Convert an EPUB at `source` (path or URL) into a Text-Fabric dataset."""
+    return convert_document(
+        EpubParser(),
+        source,
+        output_dir,
+        root_type="book",
+        otype_for=_otype_for,
+    )

admin/converters/_html_to_tf.py

@@ -0,0 +1,43 @@
+"""
+HTML to Text-Fabric Converter
+
+Converts HTML documents into Text-Fabric (TF) datasets using the tf.convert.walker library.
+This allows HTML content to be queried using Context-Fabric's powerful graph query API.
+
+Features:
+- Preserves HTML structure as a node hierarchy
+- Creates slots for text content (words/tokens)
+- Stores HTML attributes as node features
+- Supports nested HTML elements
+- Generates valid Text-Fabric datasets
+
+Node Types:
+- document: Root node for each HTML document
+- element: HTML tags (div, p, span, etc.)
+- word: Individual words (slots)
+
+Features:
+- tag: HTML tag name (div, p, span, etc.)
+- class: CSS class names
+- id: HTML id attribute
+- href: Link URLs (for <a> tags)
+- src: Source URLs (for <img>, <script> tags)
+- text: Raw text content
+- * (any HTML attribute preserved as feature)
+"""
+
+from pathlib import Path
+
+from ..parsers import HtmlParser
+from ._walker import convert_document
+
+
+def convert_html_to_tf(source: str, output_dir: str | Path) -> Path:
+    """Convert an HTML document at `source` (path or URL) into a Text-Fabric dataset."""
+    return convert_document(
+        HtmlParser(),
+        source,
+        output_dir,
+        root_type="document",
+        otype_for=lambda unit: "element",
+    )

admin/converters/_pdf_to_tf.py

@@ -0,0 +1,37 @@
+"""
+PDF to Text-Fabric Converter
+
+Converts PDF files into Text-Fabric datasets using the pdf parser for
+extraction and the tf.convert.walker library for TF generation.
+
+Features:
+- Extracts PDF metadata (title, author, producer, etc.)
+- One node per page, so a large PDF converts one page at a time
+- Tracks conversion progress
+
+Node Types:
+- book: Root node for the entire PDF
+- page: Individual pages
+- word: Individual words (slots)
+
+Features:
+- title, creators, date, subjects, ...: document metadata
+- page_number: 1-based page number
+- text, after: word text and its trailing whitespace
+"""
+
+from pathlib import Path
+
+from ..parsers import PdfParser
+from ._walker import convert_document
+
+
+def convert_pdf_to_tf(source: str, output_dir: str | Path) -> Path:
+    """Convert a PDF at `source` (path or URL) into a Text-Fabric dataset."""
+    return convert_document(
+        PdfParser(),
+        source,
+        output_dir,
+        root_type="book",
+        otype_for=lambda unit: "page",
+    )

admin/converters/_tei_to_tf.py

@@ -0,0 +1,53 @@
+"""
+TEI to Text-Fabric Converter
+
+Converts TEI (Text Encoding Initiative) XML documents into Text-Fabric
+datasets using the tei parser for extraction and the tf.convert.walker
+library for TF generation.
+
+Features:
+- Extracts teiHeader metadata (title, author, publisher, date, ...)
+- Preserves the TEI division hierarchy (<div> nesting)
+- Lifts each division's <head> into a `label` feature
+- One node per top-level division, so a large edition (a multi-book bible,
+  a multi-volume critical edition) converts one division at a time
+
+Node Types:
+- text: Root node for the entire TEI document
+- div: A TEI division (chapter, book, ...); its `type` attribute survives
+  as an ordinary feature (e.g. `type="chapter"`)
+- paragraph: <p> elements
+- element: Any other TEI element (l, seg, note, ...)
+- word: Individual words (slots)
+
+Features:
+- title, creators, language, publisher, date, identifier, rights: document metadata
+- label: division heading, lifted from its <head> child
+- type: TEI `@type` attribute (e.g. div type="chapter")
+- text, after: word text and its trailing whitespace
+"""
+
+from pathlib import Path
+
+from ..parsers import TeiParser
+from ..parsers.schema import Unit
+from ._walker import convert_document
+
+
+def _otype_for(unit: Unit) -> str:
+    if unit.type == "div":
+        return "div"
+    if unit.type == "p":
+        return "paragraph"
+    return "element"
+
+
+def convert_tei_to_tf(source: str, output_dir: str | Path) -> Path:
+    """Convert a TEI document at `source` (path or URL) into a Text-Fabric dataset."""
+    return convert_document(
+        TeiParser(),
+        source,
+        output_dir,
+        root_type="text",
+        otype_for=_otype_for,
+    )

admin/converters/_tei_zip_to_tf.py

@@ -0,0 +1,94 @@
+"""
+TEI ZIP to Text-Fabric Converter
+
+Converts a ZIP archive of TEI (or TEI-in-plain-`.xml`) documents into a
+single Text-Fabric dataset: each member document is parsed with the same
+`TeiParser` the single-file `tei` converter uses, and all of them are walked
+into one dataset via `convert_documents` -- one "text" root node per member
+document, carrying that document's `<teiHeader>` metadata as features.
+Members convert in archive-path order (sorted), so a corpus zipped as
+`01-genesis.xml`, `02-exodus.xml`, ... keeps its intended sequence.
+
+Node Types / Features: identical to `_tei_to_tf.py` (same parser, same
+`otype_for`) -- the only difference is that the dataset can contain several
+"text" roots instead of exactly one.
+
+Archive safety mirrors `_tf_zip_to_tf.py`: path traversal, symlinks,
+encrypted members, member count, and expanded size are all validated before
+any bytes are written. macOS `__MACOSX/` resource forks and hidden dotfiles
+are ignored.
+"""
+
+from __future__ import annotations
+
+import shutil
+import tempfile
+from pathlib import Path, PurePosixPath
+from zipfile import BadZipFile, ZipFile, ZipInfo
+
+from ..parsers import TeiParser
+from ..parsers.schema import Document
+from ._tei_to_tf import _otype_for
+from ._tf_zip_to_tf import _MAX_FILES, _MAX_UNCOMPRESSED_BYTES, _safe_path
+from ._walker import convert_documents
+
+_TEI_SUFFIXES = frozenset({".tei", ".xml"})
+
+
+def _is_noise(path: PurePosixPath) -> bool:
+    """Housekeeping entries that say nothing about the corpus itself."""
+    return path.parts[0] == "__MACOSX" or path.name.startswith(".")
+
+
+def _tei_members(archive: ZipFile) -> list[ZipInfo]:
+    infos = [info for info in archive.infolist() if not info.is_dir()]
+    if len(infos) > _MAX_FILES:
+        raise ValueError(f"TEI ZIP contains more than {_MAX_FILES:,} files")
+    if sum(info.file_size for info in infos) > _MAX_UNCOMPRESSED_BYTES:
+        raise ValueError("Expanded TEI ZIP exceeds the 2 GiB limit")
+
+    members = [
+        info
+        for info in infos
+        for path in (_safe_path(info),)
+        if not _is_noise(path) and path.suffix.lower() in _TEI_SUFFIXES
+    ]
+    if not members:
+        raise ValueError(
+            "ZIP does not contain any TEI documents (.tei or .xml files are required)"
+        )
+    return sorted(members, key=lambda info: info.filename)
+
+
+def convert_tei_zip_to_tf(source: str, output_dir: str | Path) -> Path:
+    """Convert every TEI/XML document inside ``source`` into one Text-Fabric
+    dataset at ``output_dir``."""
+    parser = TeiParser()
+    documents: list[Document] = []
+
+    try:
+        with (
+            ZipFile(source) as archive,
+            tempfile.TemporaryDirectory(prefix="tei-zip-") as scratch,
+        ):
+            for info in _tei_members(archive):
+                # Flattened to the basename: members were validated against
+                # traversal above, and the parser only needs a readable file.
+                extracted = Path(scratch) / f"{len(documents)}-{PurePosixPath(info.filename).name}"
+                with (
+                    archive.open(info) as member,
+                    extracted.open("wb") as out,
+                ):
+                    shutil.copyfileobj(member, out)
+                documents.append(parser.parse(str(extracted)))
+    except BadZipFile as exc:
+        raise ValueError("Uploaded file is not a valid ZIP archive") from exc
+
+    return convert_documents(
+        documents,
+        output_dir,
+        root_type="text",
+        otype_for=_otype_for,
+        format_value=parser.format.value,
+        source_label=source,
+    )

admin/converters/_text_to_tf.py

@@ -0,0 +1,36 @@
+"""
+Plain Text to Text-Fabric Converter
+
+Converts raw text files into Text-Fabric datasets using the plain-text
+parser for extraction and the tf.convert.walker library for TF generation.
+
+Features:
+- One node per paragraph (blank-line-separated), so a large plain-text
+  corpus converts one paragraph at a time
+- Minimal metadata (a title derived from the file name)
+
+Node Types:
+- book: Root node for the entire text file
+- paragraph: A blank-line-separated block of text
+- word: Individual words (slots)
+
+Features:
+- title: derived from the source file name
+- text, after: word text and its trailing whitespace
+"""
+
+from pathlib import Path
+
+from ..parsers import PlainTextParser
+from ._walker import convert_document
+
+
+def convert_text_to_tf(source: str, output_dir: str | Path) -> Path:
+    """Convert a plain-text file at `source` (path or URL) into a Text-Fabric dataset."""
+    return convert_document(
+        PlainTextParser(),
+        source,
+        output_dir,
+        root_type="book",
+        otype_for=lambda unit: "paragraph",
+    )

admin/converters/_tf_zip_to_tf.py

@@ -0,0 +1,87 @@
+"""Import an existing Text-Fabric dataset from a ZIP archive."""
+
+from __future__ import annotations
+
+import shutil
+import stat
+from pathlib import Path, PurePosixPath
+from zipfile import BadZipFile, ZipFile, ZipInfo
+
+_MAX_FILES = 10_000
+_MAX_UNCOMPRESSED_BYTES = 2 * 1024 * 1024 * 1024
+_REQUIRED_FILES = frozenset({"otype.tf", "oslots.tf"})
+
+
+def _safe_path(info: ZipInfo) -> PurePosixPath:
+    path = PurePosixPath(info.filename)
+    mode = info.external_attr >> 16
+    if path.is_absolute() or ".." in path.parts:
+        raise ValueError(f"Unsafe path in Text-Fabric ZIP: {info.filename!r}")
+    if stat.S_ISLNK(mode):
+        raise ValueError(
+            f"Symbolic links are not allowed in Text-Fabric ZIPs: {info.filename!r}"
+        )
+    if info.flag_bits & 0x1:
+        raise ValueError("Encrypted Text-Fabric ZIPs are not supported")
+    return path
+
+
+def _find_dataset_root(files: dict[PurePosixPath, ZipInfo]) -> PurePosixPath:
+    candidates: list[PurePosixPath] = []
+    for path in files:
+        if path.name != "otype.tf":
+            continue
+        if all(path.parent / required in files for required in _REQUIRED_FILES):
+            candidates.append(path.parent)
+
+    if not candidates:
+        raise ValueError(
+            "ZIP does not contain a Text-Fabric dataset (otype.tf and oslots.tf are required)"
+        )
+    if len(candidates) > 1:
+        roots = ", ".join(str(path) for path in sorted(candidates, key=str))
+        raise ValueError(f"ZIP contains multiple Text-Fabric datasets: {roots}")
+    return candidates[0]
+
+
+def convert_tf_zip_to_tf(source: str, output_dir: str | Path) -> Path:
+    """Extract the single Text-Fabric dataset in ``source`` to ``output_dir``.
+
+    Archive paths, symlinks, member count, and expanded size are validated
+    before any bytes are written so a malformed upload cannot escape or fill
+    the conversion work directory.
+    """
+    output_dir = Path(output_dir)
+
+    try:
+        with ZipFile(source) as archive:
+            infos = [info for info in archive.infolist() if not info.is_dir()]
+            if len(infos) > _MAX_FILES:
+                raise ValueError(
+                    f"Text-Fabric ZIP contains more than {_MAX_FILES:,} files"
+                )
+            if sum(info.file_size for info in infos) > _MAX_UNCOMPRESSED_BYTES:
+                raise ValueError("Expanded Text-Fabric ZIP exceeds the 2 GiB limit")
+
+            files = {_safe_path(info): info for info in infos}
+            dataset_root = _find_dataset_root(files)
+            dataset_files = {
+                path: info
+                for path, info in files.items()
+                if path.is_relative_to(dataset_root) and path.suffix == ".tf"
+            }
+
+            output_dir.mkdir(parents=True, exist_ok=False)
+            for path, info in dataset_files.items():
+                relative = path.relative_to(dataset_root)
+                destination = output_dir.joinpath(*relative.parts)
+                destination.parent.mkdir(parents=True, exist_ok=True)
+                with (
+                    archive.open(info) as source_file,
+                    destination.open("wb") as output_file,
+                ):
+                    shutil.copyfileobj(source_file, output_file)
+    except BadZipFile as exc:
+        raise ValueError("Uploaded file is not a valid ZIP archive") from exc
+
+    return output_dir