biblicus 0.6.0__py3-none-any.whl

biblicus/crawl.py

@@ -0,0 +1,186 @@
+"""
+Website crawl utilities for Biblicus corpora.
+"""
+
+from __future__ import annotations
+
+from collections import deque
+from html.parser import HTMLParser
+from typing import Deque, List, Optional, Set
+from urllib.parse import urldefrag, urljoin
+
+from pydantic import BaseModel, ConfigDict, Field
+
+from .ignore import load_corpus_ignore_spec
+from .sources import load_source
+
+
+class CrawlRequest(BaseModel):
+    """
+    Request describing a website crawl into a corpus.
+
+    :ivar root_url: Initial uniform resource locator to fetch.
+    :vartype root_url: str
+    :ivar allowed_prefix: Uniform resource locator prefix that limits which links are eligible for crawl.
+    :vartype allowed_prefix: str
+    :ivar max_items: Maximum number of items to store during the crawl.
+    :vartype max_items: int
+    :ivar tags: Tags to apply to stored items.
+    :vartype tags: list[str]
+    """
+
+    model_config = ConfigDict(extra="forbid")
+
+    root_url: str = Field(min_length=1)
+    allowed_prefix: str = Field(min_length=1)
+    max_items: int = Field(default=50, ge=1)
+    tags: List[str] = Field(default_factory=list)
+
+
+class CrawlResult(BaseModel):
+    """
+    Summary result for a crawl execution.
+
+    :ivar crawl_id: Crawl identifier used in the corpus raw import namespace.
+    :vartype crawl_id: str
+    :ivar discovered_items: Total number of distinct uniform resource locators discovered.
+    :vartype discovered_items: int
+    :ivar fetched_items: Number of eligible items fetched over hypertext transfer protocol.
+    :vartype fetched_items: int
+    :ivar stored_items: Number of items stored into the corpus.
+    :vartype stored_items: int
+    :ivar skipped_outside_prefix_items: Number of discovered items outside the allowed prefix.
+    :vartype skipped_outside_prefix_items: int
+    :ivar skipped_ignored_items: Number of eligible items skipped due to corpus ignore rules.
+    :vartype skipped_ignored_items: int
+    :ivar errored_items: Number of eligible items that failed to fetch or store.
+    :vartype errored_items: int
+    """
+
+    model_config = ConfigDict(extra="forbid")
+
+    crawl_id: str
+    discovered_items: int = Field(default=0, ge=0)
+    fetched_items: int = Field(default=0, ge=0)
+    stored_items: int = Field(default=0, ge=0)
+    skipped_outside_prefix_items: int = Field(default=0, ge=0)
+    skipped_ignored_items: int = Field(default=0, ge=0)
+    errored_items: int = Field(default=0, ge=0)
+
+
+class _LinkExtractor(HTMLParser):
+    def __init__(self) -> None:
+        super().__init__()
+        self.links: List[str] = []
+
+    def handle_starttag(self, tag: str, attrs):  # type: ignore[no-untyped-def]
+        _ = tag
+        for key, value in attrs:
+            if key in {"href", "src"} and isinstance(value, str) and value.strip():
+                self.links.append(value.strip())
+
+
+def _normalize_crawl_url(candidate: str, *, base_url: str) -> Optional[str]:
+    joined = urljoin(base_url, candidate)
+    joined, _fragment = urldefrag(joined)
+    joined = joined.strip()
+    if joined.startswith(("mailto:", "javascript:")):
+        return None
+    return joined
+
+
+def _crawl_relative_path(url: str, *, allowed_prefix: str) -> str:
+    relative = url[len(allowed_prefix) :].lstrip("/")
+    if not relative or relative.endswith("/"):
+        relative = relative.rstrip("/") + "/index.html" if relative else "index.html"
+    return relative
+
+
+def _should_parse_links(media_type: str) -> bool:
+    return media_type.startswith("text/html")
+
+
+def _discover_links(html_text: str, *, base_url: str) -> List[str]:
+    parser = _LinkExtractor()
+    parser.feed(html_text)
+    discovered: List[str] = []
+    for raw in parser.links:
+        normalized = _normalize_crawl_url(raw, base_url=base_url)
+        if normalized is not None:
+            discovered.append(normalized)
+    return discovered
+
+
+def crawl_into_corpus(*, corpus, request: CrawlRequest) -> CrawlResult:  # type: ignore[no-untyped-def]
+    """
+    Crawl a website prefix into a corpus.
+
+    :param corpus: Target corpus to receive crawled items.
+    :type corpus: biblicus.corpus.Corpus
+    :param request: Crawl request describing limits and allowed prefix.
+    :type request: CrawlRequest
+    :return: Crawl result summary.
+    :rtype: CrawlResult
+    """
+    ignore_spec = load_corpus_ignore_spec(corpus.root)
+    allowed_prefix = request.allowed_prefix
+    root_url = request.root_url
+
+    crawl_id = corpus.create_crawl_id()
+
+    queue: Deque[str] = deque([root_url])
+    seen: Set[str] = set()
+    stored_count = 0
+    fetched_count = 0
+    skipped_outside_prefix_count = 0
+    skipped_ignored_count = 0
+    errored_count = 0
+    discovered_urls: Set[str] = set()
+
+    while queue and stored_count < request.max_items:
+        url = queue.popleft()
+        if url in seen:
+            continue
+        seen.add(url)
+        discovered_urls.add(url)
+
+        if not url.startswith(allowed_prefix):
+            skipped_outside_prefix_count += 1
+            continue
+
+        relative_path = _crawl_relative_path(url, allowed_prefix=allowed_prefix)
+        if ignore_spec.matches(relative_path):
+            skipped_ignored_count += 1
+            continue
+
+        try:
+            payload = load_source(url)
+            fetched_count += 1
+            corpus.ingest_crawled_payload(
+                crawl_id=crawl_id,
+                relative_path=relative_path,
+                data=payload.data,
+                filename=payload.filename,
+                media_type=payload.media_type,
+                source_uri=payload.source_uri,
+                tags=request.tags,
+            )
+            stored_count += 1
+        except Exception:
+            errored_count += 1
+            continue
+
+        if _should_parse_links(payload.media_type):
+            text = payload.data.decode("utf-8", errors="replace")
+            for discovered in _discover_links(text, base_url=url):
+                queue.append(discovered)
+
+    return CrawlResult(
+        crawl_id=crawl_id,
+        discovered_items=len(discovered_urls),
+        fetched_items=fetched_count,
+        stored_items=stored_count,
+        skipped_outside_prefix_items=skipped_outside_prefix_count,
+        skipped_ignored_items=skipped_ignored_count,
+        errored_items=errored_count,
+    )

biblicus/errors.py

@@ -0,0 +1,15 @@
+"""
+Error types for Biblicus.
+"""
+
+from __future__ import annotations
+
+
+class ExtractionRunFatalError(RuntimeError):
+    """
+    Fatal extraction run error that should abort the entire run.
+
+    This exception is used for conditions that indicate a configuration or environment problem
+    rather than a per-item extraction failure. For example, a selection extractor that depends
+    on referenced extraction run manifests treats missing manifests as fatal.
+    """

biblicus/evaluation.py

@@ -0,0 +1,257 @@
+"""
+Evaluation utilities for Biblicus retrieval runs.
+"""
+
+from __future__ import annotations
+
+import json
+import time
+from pathlib import Path
+from typing import Dict, List, Optional
+
+from pydantic import BaseModel, ConfigDict, Field, model_validator
+
+from .backends import get_backend
+from .constants import DATASET_SCHEMA_VERSION
+from .corpus import Corpus
+from .models import QueryBudget, RetrievalResult, RetrievalRun
+from .time import utc_now_iso
+
+
+class EvaluationQuery(BaseModel):
+    """
+    Query record for retrieval evaluation.
+
+    :ivar query_id: Unique identifier for the query.
+    :vartype query_id: str
+    :ivar query_text: Natural language query to execute.
+    :vartype query_text: str
+    :ivar expected_item_id: Optional expected item identifier.
+    :vartype expected_item_id: str or None
+    :ivar expected_source_uri: Optional expected source uniform resource identifier.
+    :vartype expected_source_uri: str or None
+    :ivar kind: Query kind (gold or synthetic).
+    :vartype kind: str
+    """
+
+    model_config = ConfigDict(extra="forbid")
+
+    query_id: str
+    query_text: str
+    expected_item_id: Optional[str] = None
+    expected_source_uri: Optional[str] = None
+    kind: str = Field(default="gold")
+
+    @model_validator(mode="after")
+    def _require_expectation(self) -> "EvaluationQuery":
+        if not self.expected_item_id and not self.expected_source_uri:
+            raise ValueError(
+                "Evaluation queries must include expected_item_id or expected_source_uri"
+            )
+        return self
+
+
+class EvaluationDataset(BaseModel):
+    """
+    Dataset for retrieval evaluation.
+
+    :ivar schema_version: Dataset schema version.
+    :vartype schema_version: int
+    :ivar name: Dataset name.
+    :vartype name: str
+    :ivar description: Optional description.
+    :vartype description: str or None
+    :ivar queries: List of evaluation queries.
+    :vartype queries: list[EvaluationQuery]
+    """
+
+    model_config = ConfigDict(extra="forbid")
+
+    schema_version: int = Field(ge=1)
+    name: str
+    description: Optional[str] = None
+    queries: List[EvaluationQuery] = Field(default_factory=list)
+
+    @model_validator(mode="after")
+    def _enforce_schema_version(self) -> "EvaluationDataset":
+        if self.schema_version != DATASET_SCHEMA_VERSION:
+            raise ValueError(f"Unsupported dataset schema version: {self.schema_version}")
+        return self
+
+
+class EvaluationResult(BaseModel):
+    """
+    Result bundle for a retrieval evaluation.
+
+    :ivar dataset: Dataset metadata.
+    :vartype dataset: dict[str, object]
+    :ivar backend_id: Backend identifier.
+    :vartype backend_id: str
+    :ivar run_id: Retrieval run identifier.
+    :vartype run_id: str
+    :ivar evaluated_at: International Organization for Standardization 8601 evaluation timestamp.
+    :vartype evaluated_at: str
+    :ivar metrics: Quality metrics for retrieval.
+    :vartype metrics: dict[str, float]
+    :ivar system: System metrics for retrieval.
+    :vartype system: dict[str, float]
+    """
+
+    model_config = ConfigDict(extra="forbid")
+
+    dataset: Dict[str, object]
+    backend_id: str
+    run_id: str
+    evaluated_at: str
+    metrics: Dict[str, float]
+    system: Dict[str, float]
+
+
+def load_dataset(path: Path) -> EvaluationDataset:
+    """
+    Load an evaluation dataset from JavaScript Object Notation.
+
+    :param path: Path to the dataset JavaScript Object Notation file.
+    :type path: Path
+    :return: Parsed evaluation dataset.
+    :rtype: EvaluationDataset
+    """
+    data = json.loads(path.read_text(encoding="utf-8"))
+    return EvaluationDataset.model_validate(data)
+
+
+def evaluate_run(
+    *,
+    corpus: Corpus,
+    run: RetrievalRun,
+    dataset: EvaluationDataset,
+    budget: QueryBudget,
+) -> EvaluationResult:
+    """
+    Evaluate a retrieval run against a dataset.
+
+    :param corpus: Corpus associated with the run.
+    :type corpus: Corpus
+    :param run: Retrieval run manifest.
+    :type run: RetrievalRun
+    :param dataset: Evaluation dataset.
+    :type dataset: EvaluationDataset
+    :param budget: Evidence selection budget.
+    :type budget: QueryBudget
+    :return: Evaluation result bundle.
+    :rtype: EvaluationResult
+    """
+    backend = get_backend(run.recipe.backend_id)
+    latency_seconds: List[float] = []
+    hit_count = 0
+    reciprocal_ranks: List[float] = []
+
+    for query in dataset.queries:
+        timer_start = time.perf_counter()
+        result = backend.query(corpus, run=run, query_text=query.query_text, budget=budget)
+        elapsed_seconds = time.perf_counter() - timer_start
+        latency_seconds.append(elapsed_seconds)
+        expected_rank = _expected_rank(result, query)
+        if expected_rank is not None:
+            hit_count += 1
+            reciprocal_ranks.append(1.0 / expected_rank)
+        else:
+            reciprocal_ranks.append(0.0)
+
+    total_queries = max(len(dataset.queries), 1)
+    max_total_items = float(budget.max_total_items)
+    hit_rate = hit_count / total_queries
+    precision_at_max_total_items = hit_count / (total_queries * max_total_items)
+    mean_reciprocal_rank = sum(reciprocal_ranks) / total_queries
+
+    metrics = {
+        "hit_rate": hit_rate,
+        "precision_at_max_total_items": precision_at_max_total_items,
+        "mean_reciprocal_rank": mean_reciprocal_rank,
+    }
+    system = {
+        "average_latency_milliseconds": _average_latency_milliseconds(latency_seconds),
+        "percentile_95_latency_milliseconds": _percentile_95_latency_milliseconds(latency_seconds),
+        "index_bytes": float(_run_artifact_bytes(corpus, run)),
+    }
+    dataset_meta = {
+        "name": dataset.name,
+        "description": dataset.description,
+        "queries": len(dataset.queries),
+    }
+    return EvaluationResult(
+        dataset=dataset_meta,
+        backend_id=run.recipe.backend_id,
+        run_id=run.run_id,
+        evaluated_at=utc_now_iso(),
+        metrics=metrics,
+        system=system,
+    )
+
+
+def _expected_rank(result: RetrievalResult, query: EvaluationQuery) -> Optional[int]:
+    """
+    Locate the first evidence rank that matches the expected item or source.
+
+    :param result: Retrieval result for a query.
+    :type result: RetrievalResult
+    :param query: Evaluation query definition.
+    :type query: EvaluationQuery
+    :return: Rank of the first matching evidence item, or None.
+    :rtype: int or None
+    """
+    for evidence in result.evidence:
+        if query.expected_item_id and evidence.item_id == query.expected_item_id:
+            return evidence.rank
+        if query.expected_source_uri and evidence.source_uri == query.expected_source_uri:
+            return evidence.rank
+    return None
+
+
+def _average_latency_milliseconds(latencies: List[float]) -> float:
+    """
+    Compute average latency in milliseconds.
+
+    :param latencies: Latency samples in seconds.
+    :type latencies: list[float]
+    :return: Average latency in milliseconds.
+    :rtype: float
+    """
+    if not latencies:
+        return 0.0
+    return sum(latencies) / len(latencies) * 1000.0
+
+
+def _percentile_95_latency_milliseconds(latencies: List[float]) -> float:
+    """
+    Compute the percentile 95 latency in milliseconds.
+
+    :param latencies: Latency samples in seconds.
+    :type latencies: list[float]
+    :return: Percentile 95 latency in milliseconds.
+    :rtype: float
+    """
+    if not latencies:
+        return 0.0
+    sorted_latencies = sorted(latencies)
+    percentile_index = int(round(0.95 * (len(sorted_latencies) - 1)))
+    return sorted_latencies[percentile_index] * 1000.0
+
+
+def _run_artifact_bytes(corpus: Corpus, run: RetrievalRun) -> int:
+    """
+    Sum artifact sizes for a retrieval run.
+
+    :param corpus: Corpus that owns the artifacts.
+    :type corpus: Corpus
+    :param run: Retrieval run manifest.
+    :type run: RetrievalRun
+    :return: Total artifact bytes.
+    :rtype: int
+    """
+    total_bytes = 0
+    for artifact_relpath in run.artifact_paths:
+        artifact_path = corpus.root / artifact_relpath
+        if artifact_path.exists():
+            total_bytes += artifact_path.stat().st_size
+    return total_bytes

biblicus/evidence_processing.py

@@ -0,0 +1,201 @@
+"""
+Evidence processing stages for Biblicus.
+
+Retrieval backends return ranked evidence. Additional stages can be applied without changing the
+backend implementation:
+
+- Rerank: reorder evidence.
+- Filter: remove evidence.
+
+These stages are explicit so they can be configured, tested, and evaluated independently from the
+retrieval backend.
+"""
+
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from typing import Any, Dict, List
+
+from pydantic import BaseModel, ConfigDict, Field
+
+from .models import Evidence
+
+
+class EvidenceReranker(ABC):
+    """
+    Evidence reranker interface.
+
+    :param reranker_id: Stable identifier for this reranker implementation.
+    :type reranker_id: str
+    """
+
+    reranker_id: str
+
+    @abstractmethod
+    def rerank(self, *, query_text: str, evidence: List[Evidence]) -> List[Evidence]:
+        """
+        Reorder evidence for the given query.
+
+        :param query_text: Query text associated with the evidence.
+        :type query_text: str
+        :param evidence: Evidence objects to rerank.
+        :type evidence: list[Evidence]
+        :return: Reranked evidence list.
+        :rtype: list[Evidence]
+        """
+
+
+class EvidenceFilter(ABC):
+    """
+    Evidence filter interface.
+
+    :param filter_id: Stable identifier for this filter implementation.
+    :type filter_id: str
+    """
+
+    filter_id: str
+
+    @abstractmethod
+    def filter(
+        self, *, query_text: str, evidence: List[Evidence], config: Dict[str, Any]
+    ) -> List[Evidence]:
+        """
+        Filter evidence for the given query.
+
+        :param query_text: Query text associated with the evidence.
+        :type query_text: str
+        :param evidence: Evidence objects to filter.
+        :type evidence: list[Evidence]
+        :param config: Filter-specific configuration values.
+        :type config: dict[str, Any]
+        :return: Filtered evidence list.
+        :rtype: list[Evidence]
+        """
+
+
+class EvidenceRerankLongestText(EvidenceReranker):
+    """
+    Reranker that prioritizes evidence with longer text.
+
+    This is a deterministic policy that is useful when a downstream context pack is limited by a
+    character or token budget and longer evidence is preferred.
+
+    :ivar reranker_id: Stable reranker identifier.
+    :vartype reranker_id: str
+    """
+
+    reranker_id = "rerank-longest-text"
+
+    def rerank(self, *, query_text: str, evidence: List[Evidence]) -> List[Evidence]:
+        """
+        Reorder evidence by descending text length.
+
+        :param query_text: Query text associated with the evidence.
+        :type query_text: str
+        :param evidence: Evidence objects to rerank.
+        :type evidence: list[Evidence]
+        :return: Evidence list ordered by text length.
+        :rtype: list[Evidence]
+        """
+        return sorted(
+            evidence,
+            key=lambda evidence_item: (-len((evidence_item.text or "").strip()), evidence_item.item_id),
+        )
+
+
+class EvidenceFilterMinimumScoreConfig(BaseModel):
+    """
+    Configuration for the minimum score evidence filter.
+
+    :ivar minimum_score: Evidence with score below this threshold is removed.
+    :vartype minimum_score: float
+    """
+
+    model_config = ConfigDict(extra="forbid")
+
+    minimum_score: float = Field(ge=0.0)
+
+
+class EvidenceFilterMinimumScore(EvidenceFilter):
+    """
+    Filter that removes evidence below a minimum score threshold.
+
+    :ivar filter_id: Stable filter identifier.
+    :vartype filter_id: str
+    """
+
+    filter_id = "filter-minimum-score"
+
+    def filter(
+        self, *, query_text: str, evidence: List[Evidence], config: Dict[str, Any]
+    ) -> List[Evidence]:
+        """
+        Filter evidence by score threshold.
+
+        :param query_text: Query text associated with the evidence.
+        :type query_text: str
+        :param evidence: Evidence objects to filter.
+        :type evidence: list[Evidence]
+        :param config: Filter configuration values.
+        :type config: dict[str, Any]
+        :return: Evidence list with low-score items removed.
+        :rtype: list[Evidence]
+        """
+        parsed_config = EvidenceFilterMinimumScoreConfig.model_validate(config)
+        return [
+            evidence_item
+            for evidence_item in evidence
+            if float(evidence_item.score) >= parsed_config.minimum_score
+        ]
+
+
+_EVIDENCE_RERANKERS: Dict[str, EvidenceReranker] = {
+    EvidenceRerankLongestText.reranker_id: EvidenceRerankLongestText(),
+}
+
+_EVIDENCE_FILTERS: Dict[str, EvidenceFilter] = {
+    EvidenceFilterMinimumScore.filter_id: EvidenceFilterMinimumScore(),
+}
+
+
+def apply_evidence_reranker(
+    *, reranker_id: str, query_text: str, evidence: List[Evidence]
+) -> List[Evidence]:
+    """
+    Apply a reranker to evidence by identifier.
+
+    :param reranker_id: Reranker identifier.
+    :type reranker_id: str
+    :param query_text: Query text associated with the evidence.
+    :type query_text: str
+    :param evidence: Evidence objects to rerank.
+    :type evidence: list[Evidence]
+    :return: Reranked evidence list.
+    :rtype: list[Evidence]
+    :raises KeyError: If the reranker identifier is unknown.
+    """
+    reranker = _EVIDENCE_RERANKERS[reranker_id]
+    return reranker.rerank(query_text=query_text, evidence=evidence)
+
+
+def apply_evidence_filter(
+    *, filter_id: str, query_text: str, evidence: List[Evidence], config: Dict[str, Any]
+) -> List[Evidence]:
+    """
+    Apply a filter to evidence by identifier.
+
+    :param filter_id: Filter identifier.
+    :type filter_id: str
+    :param query_text: Query text associated with the evidence.
+    :type query_text: str
+    :param evidence: Evidence objects to filter.
+    :type evidence: list[Evidence]
+    :param config: Filter-specific configuration values.
+    :type config: dict[str, Any]
+    :return: Filtered evidence list.
+    :rtype: list[Evidence]
+    :raises KeyError: If the filter identifier is unknown.
+    """
+    evidence_filter = _EVIDENCE_FILTERS[filter_id]
+    return evidence_filter.filter(query_text=query_text, evidence=evidence, config=config)
+