biblicus 0.3.0__py3-none-any.whl → 0.5.0__py3-none-any.whl

biblicus/crawl.py

@@ -0,0 +1,186 @@
+"""
+Website crawl utilities for Biblicus corpora.
+"""
+
+from __future__ import annotations
+
+from collections import deque
+from html.parser import HTMLParser
+from typing import Deque, List, Optional, Set
+from urllib.parse import urldefrag, urljoin
+
+from pydantic import BaseModel, ConfigDict, Field
+
+from .ignore import load_corpus_ignore_spec
+from .sources import load_source
+
+
+class CrawlRequest(BaseModel):
+    """
+    Request describing a website crawl into a corpus.
+
+    :ivar root_url: Initial uniform resource locator to fetch.
+    :vartype root_url: str
+    :ivar allowed_prefix: Uniform resource locator prefix that limits which links are eligible for crawl.
+    :vartype allowed_prefix: str
+    :ivar max_items: Maximum number of items to store during the crawl.
+    :vartype max_items: int
+    :ivar tags: Tags to apply to stored items.
+    :vartype tags: list[str]
+    """
+
+    model_config = ConfigDict(extra="forbid")
+
+    root_url: str = Field(min_length=1)
+    allowed_prefix: str = Field(min_length=1)
+    max_items: int = Field(default=50, ge=1)
+    tags: List[str] = Field(default_factory=list)
+
+
+class CrawlResult(BaseModel):
+    """
+    Summary result for a crawl execution.
+
+    :ivar crawl_id: Crawl identifier used in the corpus raw import namespace.
+    :vartype crawl_id: str
+    :ivar discovered_items: Total number of distinct uniform resource locators discovered.
+    :vartype discovered_items: int
+    :ivar fetched_items: Number of eligible items fetched over hypertext transfer protocol.
+    :vartype fetched_items: int
+    :ivar stored_items: Number of items stored into the corpus.
+    :vartype stored_items: int
+    :ivar skipped_outside_prefix_items: Number of discovered items outside the allowed prefix.
+    :vartype skipped_outside_prefix_items: int
+    :ivar skipped_ignored_items: Number of eligible items skipped due to corpus ignore rules.
+    :vartype skipped_ignored_items: int
+    :ivar errored_items: Number of eligible items that failed to fetch or store.
+    :vartype errored_items: int
+    """
+
+    model_config = ConfigDict(extra="forbid")
+
+    crawl_id: str
+    discovered_items: int = Field(default=0, ge=0)
+    fetched_items: int = Field(default=0, ge=0)
+    stored_items: int = Field(default=0, ge=0)
+    skipped_outside_prefix_items: int = Field(default=0, ge=0)
+    skipped_ignored_items: int = Field(default=0, ge=0)
+    errored_items: int = Field(default=0, ge=0)
+
+
+class _LinkExtractor(HTMLParser):
+    def __init__(self) -> None:
+        super().__init__()
+        self.links: List[str] = []
+
+    def handle_starttag(self, tag: str, attrs):  # type: ignore[no-untyped-def]
+        _ = tag
+        for key, value in attrs:
+            if key in {"href", "src"} and isinstance(value, str) and value.strip():
+                self.links.append(value.strip())
+
+
+def _normalize_crawl_url(candidate: str, *, base_url: str) -> Optional[str]:
+    joined = urljoin(base_url, candidate)
+    joined, _fragment = urldefrag(joined)
+    joined = joined.strip()
+    if joined.startswith(("mailto:", "javascript:")):
+        return None
+    return joined
+
+
+def _crawl_relative_path(url: str, *, allowed_prefix: str) -> str:
+    relative = url[len(allowed_prefix) :].lstrip("/")
+    if not relative or relative.endswith("/"):
+        relative = relative.rstrip("/") + "/index.html" if relative else "index.html"
+    return relative
+
+
+def _should_parse_links(media_type: str) -> bool:
+    return media_type.startswith("text/html")
+
+
+def _discover_links(html_text: str, *, base_url: str) -> List[str]:
+    parser = _LinkExtractor()
+    parser.feed(html_text)
+    discovered: List[str] = []
+    for raw in parser.links:
+        normalized = _normalize_crawl_url(raw, base_url=base_url)
+        if normalized is not None:
+            discovered.append(normalized)
+    return discovered
+
+
+def crawl_into_corpus(*, corpus, request: CrawlRequest) -> CrawlResult:  # type: ignore[no-untyped-def]
+    """
+    Crawl a website prefix into a corpus.
+
+    :param corpus: Target corpus to receive crawled items.
+    :type corpus: biblicus.corpus.Corpus
+    :param request: Crawl request describing limits and allowed prefix.
+    :type request: CrawlRequest
+    :return: Crawl result summary.
+    :rtype: CrawlResult
+    """
+    ignore_spec = load_corpus_ignore_spec(corpus.root)
+    allowed_prefix = request.allowed_prefix
+    root_url = request.root_url
+
+    crawl_id = corpus.create_crawl_id()
+
+    queue: Deque[str] = deque([root_url])
+    seen: Set[str] = set()
+    stored_count = 0
+    fetched_count = 0
+    skipped_outside_prefix_count = 0
+    skipped_ignored_count = 0
+    errored_count = 0
+    discovered_urls: Set[str] = set()
+
+    while queue and stored_count < request.max_items:
+        url = queue.popleft()
+        if url in seen:
+            continue
+        seen.add(url)
+        discovered_urls.add(url)
+
+        if not url.startswith(allowed_prefix):
+            skipped_outside_prefix_count += 1
+            continue
+
+        relative_path = _crawl_relative_path(url, allowed_prefix=allowed_prefix)
+        if ignore_spec.matches(relative_path):
+            skipped_ignored_count += 1
+            continue
+
+        try:
+            payload = load_source(url)
+            fetched_count += 1
+            corpus.ingest_crawled_payload(
+                crawl_id=crawl_id,
+                relative_path=relative_path,
+                data=payload.data,
+                filename=payload.filename,
+                media_type=payload.media_type,
+                source_uri=payload.source_uri,
+                tags=request.tags,
+            )
+            stored_count += 1
+        except Exception:
+            errored_count += 1
+            continue
+
+        if _should_parse_links(payload.media_type):
+            text = payload.data.decode("utf-8", errors="replace")
+            for discovered in _discover_links(text, base_url=url):
+                queue.append(discovered)
+
+    return CrawlResult(
+        crawl_id=crawl_id,
+        discovered_items=len(discovered_urls),
+        fetched_items=fetched_count,
+        stored_items=stored_count,
+        skipped_outside_prefix_items=skipped_outside_prefix_count,
+        skipped_ignored_items=skipped_ignored_count,
+        errored_items=errored_count,
+    )

biblicus/evidence_processing.py

@@ -0,0 +1,201 @@
+"""
+Evidence processing stages for Biblicus.
+
+Retrieval backends return ranked evidence. Additional stages can be applied without changing the
+backend implementation:
+
+- Rerank: reorder evidence.
+- Filter: remove evidence.
+
+These stages are explicit so they can be configured, tested, and evaluated independently from the
+retrieval backend.
+"""
+
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from typing import Any, Dict, List
+
+from pydantic import BaseModel, ConfigDict, Field
+
+from .models import Evidence
+
+
+class EvidenceReranker(ABC):
+    """
+    Evidence reranker interface.
+
+    :param reranker_id: Stable identifier for this reranker implementation.
+    :type reranker_id: str
+    """
+
+    reranker_id: str
+
+    @abstractmethod
+    def rerank(self, *, query_text: str, evidence: List[Evidence]) -> List[Evidence]:
+        """
+        Reorder evidence for the given query.
+
+        :param query_text: Query text associated with the evidence.
+        :type query_text: str
+        :param evidence: Evidence objects to rerank.
+        :type evidence: list[Evidence]
+        :return: Reranked evidence list.
+        :rtype: list[Evidence]
+        """
+
+
+class EvidenceFilter(ABC):
+    """
+    Evidence filter interface.
+
+    :param filter_id: Stable identifier for this filter implementation.
+    :type filter_id: str
+    """
+
+    filter_id: str
+
+    @abstractmethod
+    def filter(
+        self, *, query_text: str, evidence: List[Evidence], config: Dict[str, Any]
+    ) -> List[Evidence]:
+        """
+        Filter evidence for the given query.
+
+        :param query_text: Query text associated with the evidence.
+        :type query_text: str
+        :param evidence: Evidence objects to filter.
+        :type evidence: list[Evidence]
+        :param config: Filter-specific configuration values.
+        :type config: dict[str, Any]
+        :return: Filtered evidence list.
+        :rtype: list[Evidence]
+        """
+
+
+class EvidenceRerankLongestText(EvidenceReranker):
+    """
+    Reranker that prioritizes evidence with longer text.
+
+    This is a deterministic policy that is useful when a downstream context pack is limited by a
+    character or token budget and longer evidence is preferred.
+
+    :ivar reranker_id: Stable reranker identifier.
+    :vartype reranker_id: str
+    """
+
+    reranker_id = "rerank-longest-text"
+
+    def rerank(self, *, query_text: str, evidence: List[Evidence]) -> List[Evidence]:
+        """
+        Reorder evidence by descending text length.
+
+        :param query_text: Query text associated with the evidence.
+        :type query_text: str
+        :param evidence: Evidence objects to rerank.
+        :type evidence: list[Evidence]
+        :return: Evidence list ordered by text length.
+        :rtype: list[Evidence]
+        """
+        return sorted(
+            evidence,
+            key=lambda evidence_item: (-len((evidence_item.text or "").strip()), evidence_item.item_id),
+        )
+
+
+class EvidenceFilterMinimumScoreConfig(BaseModel):
+    """
+    Configuration for the minimum score evidence filter.
+
+    :ivar minimum_score: Evidence with score below this threshold is removed.
+    :vartype minimum_score: float
+    """
+
+    model_config = ConfigDict(extra="forbid")
+
+    minimum_score: float = Field(ge=0.0)
+
+
+class EvidenceFilterMinimumScore(EvidenceFilter):
+    """
+    Filter that removes evidence below a minimum score threshold.
+
+    :ivar filter_id: Stable filter identifier.
+    :vartype filter_id: str
+    """
+
+    filter_id = "filter-minimum-score"
+
+    def filter(
+        self, *, query_text: str, evidence: List[Evidence], config: Dict[str, Any]
+    ) -> List[Evidence]:
+        """
+        Filter evidence by score threshold.
+
+        :param query_text: Query text associated with the evidence.
+        :type query_text: str
+        :param evidence: Evidence objects to filter.
+        :type evidence: list[Evidence]
+        :param config: Filter configuration values.
+        :type config: dict[str, Any]
+        :return: Evidence list with low-score items removed.
+        :rtype: list[Evidence]
+        """
+        parsed_config = EvidenceFilterMinimumScoreConfig.model_validate(config)
+        return [
+            evidence_item
+            for evidence_item in evidence
+            if float(evidence_item.score) >= parsed_config.minimum_score
+        ]
+
+
+_EVIDENCE_RERANKERS: Dict[str, EvidenceReranker] = {
+    EvidenceRerankLongestText.reranker_id: EvidenceRerankLongestText(),
+}
+
+_EVIDENCE_FILTERS: Dict[str, EvidenceFilter] = {
+    EvidenceFilterMinimumScore.filter_id: EvidenceFilterMinimumScore(),
+}
+
+
+def apply_evidence_reranker(
+    *, reranker_id: str, query_text: str, evidence: List[Evidence]
+) -> List[Evidence]:
+    """
+    Apply a reranker to evidence by identifier.
+
+    :param reranker_id: Reranker identifier.
+    :type reranker_id: str
+    :param query_text: Query text associated with the evidence.
+    :type query_text: str
+    :param evidence: Evidence objects to rerank.
+    :type evidence: list[Evidence]
+    :return: Reranked evidence list.
+    :rtype: list[Evidence]
+    :raises KeyError: If the reranker identifier is unknown.
+    """
+    reranker = _EVIDENCE_RERANKERS[reranker_id]
+    return reranker.rerank(query_text=query_text, evidence=evidence)
+
+
+def apply_evidence_filter(
+    *, filter_id: str, query_text: str, evidence: List[Evidence], config: Dict[str, Any]
+) -> List[Evidence]:
+    """
+    Apply a filter to evidence by identifier.
+
+    :param filter_id: Filter identifier.
+    :type filter_id: str
+    :param query_text: Query text associated with the evidence.
+    :type query_text: str
+    :param evidence: Evidence objects to filter.
+    :type evidence: list[Evidence]
+    :param config: Filter-specific configuration values.
+    :type config: dict[str, Any]
+    :return: Filtered evidence list.
+    :rtype: list[Evidence]
+    :raises KeyError: If the filter identifier is unknown.
+    """
+    evidence_filter = _EVIDENCE_FILTERS[filter_id]
+    return evidence_filter.filter(query_text=query_text, evidence=evidence, config=config)
+

biblicus/extraction.py

@@ -7,7 +7,6 @@ from __future__ import annotations
 import json
 from pathlib import Path
 from typing import Any, Dict, List, Optional, Tuple
-from uuid import uuid4
 
 from pydantic import BaseModel, ConfigDict, Field
 
@@ -196,8 +195,9 @@ def create_extraction_run_manifest(
     :rtype: ExtractionRunManifest
     """
     catalog = corpus.load_catalog()
+    run_id = hash_text(f"{recipe.recipe_id}:{catalog.generated_at}")
     return ExtractionRunManifest(
-        run_id=str(uuid4()),
+        run_id=run_id,
         recipe=recipe,
         corpus_uri=corpus.uri,
         catalog_generated_at=catalog.generated_at,
@@ -341,6 +341,8 @@ def build_extraction_run(
     )
     manifest = create_extraction_run_manifest(corpus, recipe=recipe)
     run_dir = corpus.extraction_run_dir(extractor_id=extractor_id, run_id=manifest.run_id)
+    if run_dir.exists():
+        return corpus.load_extraction_run_manifest(extractor_id=extractor_id, run_id=manifest.run_id)
     run_dir.mkdir(parents=True, exist_ok=False)
 
     catalog = corpus.load_catalog()

biblicus/models.py

@@ -189,6 +189,37 @@ def parse_extraction_run_reference(value: str) -> ExtractionRunReference:
     return ExtractionRunReference(extractor_id=extractor_id, run_id=run_id)
 
 
+class ExtractionRunListEntry(BaseModel):
+    """
+    Summary entry for an extraction run stored in a corpus.
+
+    :ivar extractor_id: Extractor plugin identifier.
+    :vartype extractor_id: str
+    :ivar run_id: Extraction run identifier.
+    :vartype run_id: str
+    :ivar recipe_id: Deterministic recipe identifier.
+    :vartype recipe_id: str
+    :ivar recipe_name: Human-readable recipe name.
+    :vartype recipe_name: str
+    :ivar catalog_generated_at: Catalog timestamp used for the run.
+    :vartype catalog_generated_at: str
+    :ivar created_at: International Organization for Standardization 8601 timestamp for run creation.
+    :vartype created_at: str
+    :ivar stats: Run statistics.
+    :vartype stats: dict[str, object]
+    """
+
+    model_config = ConfigDict(extra="forbid")
+
+    extractor_id: str = Field(min_length=1)
+    run_id: str = Field(min_length=1)
+    recipe_id: str = Field(min_length=1)
+    recipe_name: str = Field(min_length=1)
+    catalog_generated_at: str = Field(min_length=1)
+    created_at: str = Field(min_length=1)
+    stats: Dict[str, object] = Field(default_factory=dict)
+
+
 class QueryBudget(BaseModel):
     """
     Evidence selection budget for retrieval.

biblicus/time.py

@@ -14,4 +14,4 @@ def utc_now_iso() -> str:
     :return: Current Coordinated Universal Time timestamp in International Organization for Standardization 8601 format.
     :rtype: str
     """
-    return datetime.now(timezone.utc).replace(microsecond=0).isoformat()
+    return datetime.now(timezone.utc).isoformat(timespec="microseconds")