biblicus 0.1.1__py3-none-any.whl → 0.2.0__py3-none-any.whl

biblicus/ignore.py

@@ -0,0 +1,67 @@
+"""
+Corpus ignore rules for bulk import and crawling.
+"""
+
+from __future__ import annotations
+
+import fnmatch
+from pathlib import Path
+from typing import List
+
+from pydantic import BaseModel, ConfigDict, Field
+
+
+class CorpusIgnoreSpec(BaseModel):
+    """
+    Parsed corpus ignore patterns.
+
+    Patterns are matched against a forward-slash relative path string.
+
+    :ivar patterns: Glob patterns to ignore.
+    :vartype patterns: list[str]
+    """
+
+    model_config = ConfigDict(extra="forbid")
+
+    patterns: List[str] = Field(default_factory=list)
+
+    def matches(self, relpath: str) -> bool:
+        """
+        Return True if the relative path matches any ignore pattern.
+
+        :param relpath: Forward-slash relative path.
+        :type relpath: str
+        :return: True if the path should be ignored.
+        :rtype: bool
+        """
+
+        normalized = relpath.replace("\\", "/").lstrip("/")
+        return any(fnmatch.fnmatch(normalized, pattern) for pattern in self.patterns)
+
+
+def load_corpus_ignore_spec(corpus_root: Path) -> CorpusIgnoreSpec:
+    """
+    Load ignore patterns from the corpus ignore file, if present.
+
+    The ignore file is stored at the corpus root as `.biblicusignore`.
+
+    :param corpus_root: Corpus root directory.
+    :type corpus_root: Path
+    :return: Parsed ignore specification.
+    :rtype: CorpusIgnoreSpec
+    """
+
+    ignore_path = corpus_root / ".biblicusignore"
+    if not ignore_path.is_file():
+        return CorpusIgnoreSpec(patterns=[])
+
+    patterns: List[str] = []
+    for raw_line in ignore_path.read_text(encoding="utf-8").splitlines():
+        line = raw_line.strip()
+        if not line:
+            continue
+        if line.startswith("#"):
+            continue
+        patterns.append(line)
+    return CorpusIgnoreSpec(patterns=patterns)
+

biblicus/models.py

@@ -9,6 +9,7 @@ from typing import Any, Dict, List, Optional
 from pydantic import BaseModel, ConfigDict, Field, model_validator
 
 from .constants import SCHEMA_VERSION
+from .hooks import HookSpec
 
 
 class CorpusConfig(BaseModel):
@@ -25,6 +26,8 @@ class CorpusConfig(BaseModel):
     :vartype raw_dir: str
     :ivar notes: Optional free-form notes for operators.
     :vartype notes: dict[str, Any] or None
+    :ivar hooks: Optional hook specifications for corpus lifecycle events.
+    :vartype hooks: list[HookSpec] or None
     """
 
     model_config = ConfigDict(extra="forbid")
@@ -34,6 +37,7 @@ class CorpusConfig(BaseModel):
     corpus_uri: str
     raw_dir: str = "raw"
     notes: Optional[Dict[str, Any]] = None
+    hooks: Optional[List[HookSpec]] = None
 
     @model_validator(mode="after")
     def _enforce_schema_version(self) -> "CorpusConfig":
@@ -305,3 +309,19 @@ class RetrievalResult(BaseModel):
     generated_at: str
     evidence: List[Evidence] = Field(default_factory=list)
     stats: Dict[str, Any] = Field(default_factory=dict)
+
+
+class ExtractedText(BaseModel):
+    """
+    Text payload produced by an extractor plugin.
+
+    :ivar text: Extracted text content.
+    :vartype text: str
+    :ivar producer_extractor_id: Extractor identifier that produced this text.
+    :vartype producer_extractor_id: str
+    """
+
+    model_config = ConfigDict(extra="forbid")
+
+    text: str
+    producer_extractor_id: str = Field(min_length=1)

biblicus/sources.py

@@ -53,6 +53,46 @@ def _media_type_from_filename(name: str) -> str:
     return media_type or "application/octet-stream"
 
 
+def _sniff_media_type_from_bytes(data: bytes) -> Optional[str]:
+    """
+    Sniff a media type from leading bytes for a small set of common formats.
+
+    :param data: Raw bytes to inspect.
+    :type data: bytes
+    :return: Detected media type or None.
+    :rtype: str or None
+    """
+
+    prefix = data[:32]
+    if prefix.startswith(b"%PDF-"):
+        return "application/pdf"
+    if prefix.startswith(b"\x89PNG\r\n\x1a\n"):
+        return "image/png"
+    if prefix[:3] == b"\xff\xd8\xff":
+        return "image/jpeg"
+    if prefix.lstrip().lower().startswith(b"<!doctype html") or prefix.lstrip().lower().startswith(b"<html"):
+        return "text/html"
+    return None
+
+
+def _ensure_extension_for_media_type(filename: str, media_type: str) -> str:
+    """
+    Ensure the filename has a usable extension for the media type.
+
+    :param filename: Filename candidate.
+    :type filename: str
+    :param media_type: Media type to target.
+    :type media_type: str
+    :return: Filename with extension.
+    :rtype: str
+    """
+
+    if Path(filename).suffix:
+        return filename
+    ext = mimetypes.guess_extension(media_type) or ""
+    return filename + ext if ext else filename
+
+
 @dataclass(frozen=True)
 class SourcePayload:
     """
@@ -115,6 +155,11 @@ def load_source(source: str | Path, *, source_uri: Optional[str] = None) -> Sour
                 content_type = response.headers.get("Content-Type", "").split(";", 1)[0].strip()
                 filename = _filename_from_url_path(parsed.path)
                 media_type = content_type or _media_type_from_filename(filename)
+                if media_type == "application/octet-stream":
+                    sniffed = _sniff_media_type_from_bytes(response_bytes)
+                    if sniffed:
+                        media_type = sniffed
+                        filename = _ensure_extension_for_media_type(filename, media_type)
                 if Path(filename).suffix.lower() in {".md", ".markdown"}:
                     media_type = "text/markdown"
                 return SourcePayload(

{biblicus-0.1.1.dist-info → biblicus-0.2.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: biblicus
-Version: 0.1.1
+Version: 0.2.0
 Summary: Command line interface and Python library for corpus ingestion, retrieval, and evaluation.
 License: MIT
 Requires-Python: >=3.9
@@ -20,6 +20,9 @@ Dynamic: license-file
 
 # Biblicus
 
+![Continuous integration][continuous-integration-badge]
+![Coverage][coverage-badge]
+
 Make your documents usable by your assistant, then decide later how you will search and retrieve them.
 
 If you are building an assistant in Python, you probably have material you want it to use: notes, documents, web pages, and reference files. A common approach is retrieval augmented generation, where a system retrieves relevant material and uses it as evidence when generating a response.
@@ -45,6 +48,84 @@ The framework is a small, explicit vocabulary that appears in code, specificatio
 - Recipe is a named configuration for a backend.
 - Pipeline stage is a distinct retrieval step such as retrieve, rerank, and filter.
 
+## Diagram
+
+This diagram shows how a corpus becomes evidence for an assistant.
+The legend shows what the border styles and fill styles mean.
+The your code region is where you decide how to turn evidence into context and how to call a model.
+
+```mermaid
+%%{init: {"flowchart": {"useMaxWidth": true, "nodeSpacing": 18, "rankSpacing": 22}}}%%
+flowchart LR
+  subgraph Legend[Legend]
+    direction LR
+    LegendArtifact[Stored artifact or evidence]
+    LegendStep[Step]
+    LegendArtifact --- LegendStep
+  end
+
+  subgraph Main[" "]
+    direction TB
+
+    subgraph StableCore[Stable core]
+      direction TB
+      Source[Source items] --> Ingest[Ingest]
+      Ingest --> Raw[Raw item files]
+      Raw --> Catalog[Catalog file]
+    end
+
+    subgraph PluggableRetrievalBackend[Pluggable retrieval backend]
+      direction LR
+
+      subgraph BackendIngestionIndexing[Ingestion and indexing]
+        direction TB
+        Catalog --> Build[Build run]
+        Build --> BackendIndex[Backend index]
+        BackendIndex --> Run[Run manifest]
+      end
+
+      subgraph BackendRetrievalGeneration[Retrieval and generation]
+        direction TB
+        Run --> Query[Query]
+        Query --> Evidence[Evidence]
+      end
+    end
+
+    Evidence --> Context
+
+    subgraph YourCode[Your code]
+      direction TB
+      Context[Assistant context] --> Model[Large language model call]
+      Model --> Answer[Answer]
+    end
+
+    style StableCore fill:#ffffff,stroke:#8e24aa,stroke-width:2px,color:#111111
+    style PluggableRetrievalBackend fill:#ffffff,stroke:#1e88e5,stroke-dasharray:6 3,stroke-width:2px,color:#111111
+    style YourCode fill:#ffffff,stroke:#d81b60,stroke-width:2px,color:#111111
+    style BackendIngestionIndexing fill:#ffffff,stroke:#cfd8dc,color:#111111
+    style BackendRetrievalGeneration fill:#ffffff,stroke:#cfd8dc,color:#111111
+
+    style Raw fill:#f3e5f5,stroke:#8e24aa,color:#111111
+    style Catalog fill:#f3e5f5,stroke:#8e24aa,color:#111111
+    style BackendIndex fill:#f3e5f5,stroke:#8e24aa,color:#111111
+    style Run fill:#f3e5f5,stroke:#8e24aa,color:#111111
+    style Evidence fill:#f3e5f5,stroke:#8e24aa,color:#111111
+    style Context fill:#f3e5f5,stroke:#8e24aa,color:#111111
+    style Answer fill:#f3e5f5,stroke:#8e24aa,color:#111111
+    style Source fill:#f3e5f5,stroke:#8e24aa,color:#111111
+
+    style Ingest fill:#eceff1,stroke:#90a4ae,color:#111111
+    style Build fill:#eceff1,stroke:#90a4ae,color:#111111
+    style Query fill:#eceff1,stroke:#90a4ae,color:#111111
+    style Model fill:#eceff1,stroke:#90a4ae,color:#111111
+  end
+
+  style Legend fill:#ffffff,stroke:#ffffff,color:#111111
+  style Main fill:#ffffff,stroke:#ffffff,color:#111111
+  style LegendArtifact fill:#f3e5f5,stroke:#8e24aa,color:#111111
+  style LegendStep fill:#eceff1,stroke:#90a4ae,color:#111111
+```
+
 ## Practical value
 
 - You can ingest raw material once, then try many retrieval approaches over time.
@@ -110,7 +191,11 @@ In an assistant system, retrieval usually produces context for a model call. Thi
 The documents below are written to be read in order.
 
 - [Architecture][architecture]
+- [Corpus][corpus]
+- [Text extraction][text-extraction]
 - [Backends][backends]
+- [Next steps][next-steps]
+- [Testing][testing]
 
 ## Metadata and catalog
 
@@ -143,12 +228,20 @@ Use `scripts/download_wikipedia.py` to download a small integration corpus from
 
 The dataset file `datasets/wikipedia_mini.json` provides a small evaluation set that matches the integration corpus.
 
+Use `scripts/download_pdf_samples.py` to download a small Portable Document Format integration corpus when running tests or demos. The repository does not include that content.
+
 ## Tests and coverage
 
 ```
 python3 scripts/test.py
 ```
 
+To include integration scenarios that download public test data at runtime, run this command.
+
+```
+python3 scripts/test.py --integration
+```
+
 ## Releases
 
 Releases are automated from the main branch using semantic versioning and conventional commit messages.
@@ -171,4 +264,11 @@ License terms are in `LICENSE`.
 
 [retrieval augmented generation overview]: https://en.wikipedia.org/wiki/Retrieval-augmented_generation
 [architecture]: docs/ARCHITECTURE.md
+[corpus]: docs/CORPUS.md
+[text-extraction]: docs/EXTRACTION.md
 [backends]: docs/BACKENDS.md
+[next-steps]: docs/NEXT_STEPS.md
+[testing]: docs/TESTING.md
+
+[continuous-integration-badge]: https://github.com/AnthusAI/Biblicus/actions/workflows/ci.yml/badge.svg?branch=main
+[coverage-badge]: https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/AnthusAI/Biblicus/main/coverage_badge.json

biblicus-0.2.0.dist-info/RECORD

@@ -0,0 +1,32 @@
+biblicus/__init__.py,sha256=3IXdbt-q80_BlKDwTsZw7MScRW4hBgQ-Vn6xHbgNwE8,432
+biblicus/__main__.py,sha256=ipfkUoTlocVnrQDM69C7TeBqQxmHVeiWMRaT3G9rtnk,117
+biblicus/cli.py,sha256=zDD6juQGTDmrRE2DHUku-G3wV3AtXjwYTNDFACwpdC0,19501
+biblicus/constants.py,sha256=R6fZDoLVMCwgKvTaxEx7G0CstwHGaUTlW9MsmNLDZ44,269
+biblicus/corpus.py,sha256=5naoFi0GSKBg4RFd6wOU-U30NMbG6bfs_RM90JcvDGA,47460
+biblicus/evaluation.py,sha256=H_W35vF5_L4B2JCfLu19VRu402tZ2pFkN2BbBP69lVY,8119
+biblicus/extraction.py,sha256=WX1LRsKrsyHI4Wido6gMwukzRGf5cfPWvRASgu_MRN4,10614
+biblicus/frontmatter.py,sha256=8Tqlpd3bVzZrGRB9Rdj2IwHMSJLvd2ABxMNOi3L5br4,2466
+biblicus/hook_logging.py,sha256=8Rl3BpkfTexSJ7rFi94kl6DMRDD-8eu2N7zv18wXyUM,5371
+biblicus/hook_manager.py,sha256=ucDZoVM-9fg1gQAhUxi-PECaNlHoegAxb-kYCx-OMZs,6987
+biblicus/hooks.py,sha256=OfG3VsCDWQVVZnOTQHnN9GQ0AIws9SK6-85WYTrKkzk,7847
+biblicus/ignore.py,sha256=Di37CTlg6Mg3SKJc2qxZcZdYX00IcTORB2hb0g-Jins,1803
+biblicus/models.py,sha256=6cgJX7Jmm5rBVrXWH46fQf3v__jSyDy73MnKaUQMSHQ,11099
+biblicus/retrieval.py,sha256=T7HELWCNAxZ26yj7dPH8IBUaxV_gx8Ql9iwwGz0teyI,4184
+biblicus/sources.py,sha256=C4P8oM6d50tLXr4z9Shsv4z-hDiQuylXfkT3Bx03dEM,5844
+biblicus/time.py,sha256=rvp2fJXSLVmyA76GCfNKtZoifASodemJTOWN8smPt0s,486
+biblicus/uris.py,sha256=sRDyGmoHr_H4XR4qv_lSbQJXylYD0fNEr02H5wjomnQ,1986
+biblicus/backends/__init__.py,sha256=5OXKSzsn7THhwh9T5StOvEqojx_85XXuYSGdTpMK11U,1214
+biblicus/backends/base.py,sha256=699TKygGgL72Ifkhz1V890nOK6BslwO0-OY7xeqZl-I,1764
+biblicus/backends/scan.py,sha256=DZ-CgZ0jy6_928hu4dASJ8_JH7BTfF8gwVkVhd38W1U,12421
+biblicus/backends/sqlite_full_text_search.py,sha256=FMpASLeK5diK-Uyhr4pqtpDpb_Qyk5_XRaXAKUHDzjs,16502
+biblicus/extractors/__init__.py,sha256=_6Z_JkLoDYwmay76y1fy11lCSqDDizMDPX3Vke_l8x4,1008
+biblicus/extractors/base.py,sha256=yvp709uUCnPEbK-bx6u5WKNPPH3SBWhbSaewoyUIgvA,1870
+biblicus/extractors/cascade.py,sha256=ExojAYsARtF99zVg78wY_wifVfDaJFa6wiRIaT-cpRo,3209
+biblicus/extractors/metadata_text.py,sha256=C0i8fcEC9aLmwhSdK9IlZVZ9ugOocIe0y522pSjvaCA,3203
+biblicus/extractors/pass_through_text.py,sha256=ngDyI13RpCldP-OzV4q9lBTGPxDL6MDxp7OCo1rORyQ,2421
+biblicus-0.2.0.dist-info/licenses/LICENSE,sha256=lw44GXFG_Q0fS8m5VoEvv_xtdBXK26pBcbSPUCXee_Q,1078
+biblicus-0.2.0.dist-info/METADATA,sha256=nTB344GRVrKuT6oPOrWBpFA_BiG3UAIgq3wCoHEVDgw,10307
+biblicus-0.2.0.dist-info/WHEEL,sha256=wUyA8OaulRlbfwMtmQsvNngGrxQHAvkKcvRmdizlJi0,92
+biblicus-0.2.0.dist-info/entry_points.txt,sha256=BZmO4H8Uz00fyi1RAFryOCGfZgX7eHWkY2NE-G54U5A,47
+biblicus-0.2.0.dist-info/top_level.txt,sha256=sUD_XVZwDxZ29-FBv1MknTGh4mgDXznGuP28KJY_WKc,9
+biblicus-0.2.0.dist-info/RECORD,,

biblicus-0.1.1.dist-info/RECORD

@@ -1,22 +0,0 @@
-biblicus/__init__.py,sha256=o_1kQ7q9DCcjH7zm5MAvPx49hArnSvbr88kHKzBFMvM,432
-biblicus/__main__.py,sha256=ipfkUoTlocVnrQDM69C7TeBqQxmHVeiWMRaT3G9rtnk,117
-biblicus/cli.py,sha256=DwnvcDmjelzUq_9VMo_U_-FoBs3Si3QONVJdWGonXs4,15116
-biblicus/constants.py,sha256=t8p0yStpJAYPxsFlM0u5zJcQr_ARKEqEnIgNckjyF5Y,196
-biblicus/corpus.py,sha256=953gzT77HvYeTs2pcBXyixYRTxh65nm1JtlHVfKvCzg,30921
-biblicus/evaluation.py,sha256=H_W35vF5_L4B2JCfLu19VRu402tZ2pFkN2BbBP69lVY,8119
-biblicus/frontmatter.py,sha256=8Tqlpd3bVzZrGRB9Rdj2IwHMSJLvd2ABxMNOi3L5br4,2466
-biblicus/models.py,sha256=ZDb7-t9pycPpgZWVs5CcrpyeA_8OZLoQk-aflKjU7M4,10512
-biblicus/retrieval.py,sha256=T7HELWCNAxZ26yj7dPH8IBUaxV_gx8Ql9iwwGz0teyI,4184
-biblicus/sources.py,sha256=XFF75kqMyYdeYy6k8NtDnOmCxAmroW7DH6mdzWMPMuY,4358
-biblicus/time.py,sha256=rvp2fJXSLVmyA76GCfNKtZoifASodemJTOWN8smPt0s,486
-biblicus/uris.py,sha256=sRDyGmoHr_H4XR4qv_lSbQJXylYD0fNEr02H5wjomnQ,1986
-biblicus/backends/__init__.py,sha256=5OXKSzsn7THhwh9T5StOvEqojx_85XXuYSGdTpMK11U,1214
-biblicus/backends/base.py,sha256=699TKygGgL72Ifkhz1V890nOK6BslwO0-OY7xeqZl-I,1764
-biblicus/backends/scan.py,sha256=qvktqHIB0459sjzEO4EnS1PCXwwM19LjOx8oaDoU7DQ,9245
-biblicus/backends/sqlite_full_text_search.py,sha256=s_3gsEcdlxSFuluWcug4XEklwEoY42_Dgd7luY-BqqI,14152
-biblicus-0.1.1.dist-info/licenses/LICENSE,sha256=lw44GXFG_Q0fS8m5VoEvv_xtdBXK26pBcbSPUCXee_Q,1078
-biblicus-0.1.1.dist-info/METADATA,sha256=lgvWJUgESiwWTCZ6_uUzgZeM3SkvnwjIzcsb8OE53BA,6635
-biblicus-0.1.1.dist-info/WHEEL,sha256=wUyA8OaulRlbfwMtmQsvNngGrxQHAvkKcvRmdizlJi0,92
-biblicus-0.1.1.dist-info/entry_points.txt,sha256=BZmO4H8Uz00fyi1RAFryOCGfZgX7eHWkY2NE-G54U5A,47
-biblicus-0.1.1.dist-info/top_level.txt,sha256=sUD_XVZwDxZ29-FBv1MknTGh4mgDXznGuP28KJY_WKc,9
-biblicus-0.1.1.dist-info/RECORD,,