biblicus 0.6.0__py3-none-any.whl

biblicus/hook_manager.py

@@ -0,0 +1,203 @@
+"""
+Hook manager for executing configured lifecycle hooks.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Any, Dict, Iterable, List, Optional
+
+from pydantic import BaseModel
+
+from .constants import CORPUS_DIR_NAME, HOOK_LOGS_DIR_NAME
+from .hook_logging import HookLogger, new_operation_id
+from .hooks import (
+    HookContext,
+    HookPoint,
+    HookSpec,
+    IngestHookContext,
+    IngestMutation,
+    LifecycleHook,
+    build_builtin_hook,
+)
+from .time import utc_now_iso
+
+
+class HookManager:
+    """
+    Hook manager that executes configured hooks and records execution.
+
+    :ivar corpus_uri: Canonical uniform resource identifier for the corpus.
+    :vartype corpus_uri: str
+    :ivar log_dir: Directory where hook logs are recorded.
+    :vartype log_dir: object
+    :ivar operation_id: Identifier for this hook execution session.
+    :vartype operation_id: str
+    """
+
+    def __init__(
+        self,
+        *,
+        corpus_uri: str,
+        log_dir: Path,
+        hooks: Iterable[LifecycleHook],
+        operation_id: Optional[str] = None,
+    ):
+        """
+        Initialize a hook manager.
+
+        :param corpus_uri: Canonical uniform resource identifier for the corpus.
+        :type corpus_uri: str
+        :param log_dir: Directory where hook logs are written.
+        :type log_dir: object
+        :param hooks: Hook instances to execute.
+        :type hooks: Iterable[LifecycleHook]
+        :param operation_id: Optional operation identifier override.
+        :type operation_id: str or None
+        """
+        self.corpus_uri = corpus_uri
+        self.log_dir = log_dir
+        self.operation_id = operation_id or new_operation_id()
+        self._hooks = list(hooks)
+        self._logger = HookLogger(log_dir=self.log_dir, operation_id=self.operation_id)
+
+    @classmethod
+    def from_config(
+        cls, *, corpus_root: Path, corpus_uri: str, hook_specs: Iterable[HookSpec]
+    ) -> "HookManager":
+        """
+        Build a hook manager from config data.
+
+        :param corpus_root: Corpus root directory.
+        :type corpus_root: Path
+        :param corpus_uri: Canonical uniform resource identifier for the corpus.
+        :type corpus_uri: str
+        :param hook_specs: Hook specifications loaded from config.
+        :type hook_specs: Iterable[HookSpec]
+        :return: Hook manager.
+        :rtype: HookManager
+        :raises KeyError: If a hook identifier is unknown.
+        """
+        log_dir = corpus_root / CORPUS_DIR_NAME / HOOK_LOGS_DIR_NAME
+        hooks: List[LifecycleHook] = []
+
+        for spec in hook_specs:
+            hooks.append(build_builtin_hook(spec))
+
+        return cls(corpus_uri=corpus_uri, log_dir=log_dir, hooks=hooks)
+
+    def run_ingest_hooks(
+        self,
+        *,
+        hook_point: HookPoint,
+        filename: Optional[str],
+        media_type: str,
+        title: Optional[str],
+        tags: List[str],
+        metadata: Dict[str, Any],
+        source_uri: str,
+        item_id: Optional[str] = None,
+        relpath: Optional[str] = None,
+    ) -> IngestMutation:
+        """
+        Run ingestion hooks for a hook point.
+
+        :param hook_point: Hook point to execute.
+        :type hook_point: HookPoint
+        :param filename: Suggested filename.
+        :type filename: str or None
+        :param media_type: Media type for the item.
+        :type media_type: str
+        :param title: Optional title.
+        :type title: str or None
+        :param tags: Tags associated with the item.
+        :type tags: list[str]
+        :param metadata: Metadata mapping.
+        :type metadata: dict[str, Any]
+        :param source_uri: Source uniform resource identifier.
+        :type source_uri: str
+        :param item_id: Optional item identifier.
+        :type item_id: str or None
+        :param relpath: Optional relative path.
+        :type relpath: str or None
+        :return: Combined ingestion mutation result.
+        :rtype: IngestMutation
+        :raises ValueError: If ingestion is denied by a hook.
+        """
+        context = IngestHookContext(
+            hook_point=hook_point,
+            operation_id=self.operation_id,
+            corpus_uri=self.corpus_uri,
+            created_at=utc_now_iso(),
+            filename=filename,
+            media_type=media_type,
+            title=title,
+            tags=list(tags),
+            metadata=dict(metadata),
+            source_uri=source_uri,
+            item_id=item_id,
+            relpath=relpath,
+        )
+
+        combined = IngestMutation()
+        for hook in self._hooks_for_point(hook_point):
+            result_dict = self._run_single(hook=hook, context=context)
+            mutation = IngestMutation.model_validate(result_dict)
+            if mutation.deny:
+                self._logger.record(
+                    hook_point=hook_point,
+                    hook_id=hook.hook_id,
+                    status="denied",
+                    message=mutation.deny_reason or mutation.message,
+                    item_id=item_id,
+                    relpath=relpath,
+                    source_uri=source_uri,
+                    details={"add_tags": mutation.add_tags},
+                )
+                raise ValueError(mutation.deny_reason or "Ingest denied")
+            if mutation.add_tags:
+                combined.add_tags.extend(mutation.add_tags)
+            self._logger.record(
+                hook_point=hook_point,
+                hook_id=hook.hook_id,
+                status="ok",
+                message=mutation.message,
+                item_id=item_id,
+                relpath=relpath,
+                source_uri=source_uri,
+                details={"add_tags": mutation.add_tags},
+            )
+
+        deduplicated_tags: List[str] = []
+        for tag in combined.add_tags:
+            if tag not in deduplicated_tags:
+                deduplicated_tags.append(tag)
+        combined.add_tags = deduplicated_tags
+        return combined
+
+    def _hooks_for_point(self, hook_point: HookPoint) -> List[LifecycleHook]:
+        eligible: List[LifecycleHook] = []
+        for hook in self._hooks:
+            if hook_point in list(getattr(hook, "hook_points", [])):
+                eligible.append(hook)
+        return eligible
+
+    def _run_single(self, *, hook: LifecycleHook, context: HookContext) -> Dict[str, Any]:
+        """
+        Run a single hook with error capture.
+
+        :param hook: Hook to execute.
+        :type hook: LifecycleHook
+        :param context: Hook context.
+        :type context: HookContext
+        :return: Hook result mapping.
+        :rtype: dict[str, Any]
+        :raises ValueError: If a hook raises an exception.
+        """
+        try:
+            result = hook.run(context)
+        except Exception as exc:
+            raise ValueError(f"Hook {hook.hook_id!r} failed: {exc}") from exc
+        if isinstance(result, BaseModel):
+            return result.model_dump()
+        raise ValueError(f"Hook {hook.hook_id!r} returned a non-Pydantic result")

biblicus/hooks.py

@@ -0,0 +1,261 @@
+"""
+Lifecycle hook interfaces and built-in hook implementations.
+"""
+
+from __future__ import annotations
+
+from enum import Enum
+from typing import Any, Dict, List, Optional, Sequence
+
+from pydantic import BaseModel, ConfigDict, Field
+
+
+class HookPoint(str, Enum):
+    """
+    Canonical lifecycle hook points for corpus operations.
+
+    :cvar before_ingest: Called before an item is ingested.
+    :cvar after_ingest: Called after an item is ingested and indexed.
+    :cvar before_reindex: Called before a catalog rebuild starts.
+    :cvar after_reindex: Called after a catalog rebuild completes.
+    :cvar before_build_run: Called before a backend run build starts.
+    :cvar after_build_run: Called after a backend run build completes.
+    :cvar before_query: Called before a query is executed.
+    :cvar after_query: Called after a query completes.
+    :cvar before_evaluate_run: Called before an evaluation starts.
+    :cvar after_evaluate_run: Called after an evaluation completes.
+    """
+
+    before_ingest = "before_ingest"
+    after_ingest = "after_ingest"
+    before_reindex = "before_reindex"
+    after_reindex = "after_reindex"
+    before_build_run = "before_build_run"
+    after_build_run = "after_build_run"
+    before_query = "before_query"
+    after_query = "after_query"
+    before_evaluate_run = "before_evaluate_run"
+    after_evaluate_run = "after_evaluate_run"
+
+
+class HookSpec(BaseModel):
+    """
+    On-disk hook specification stored in a corpus config.
+
+    :ivar hook_id: Identifier used to locate a hook implementation.
+    :vartype hook_id: str
+    :ivar hook_points: Hook points where the hook executes.
+    :vartype hook_points: list[HookPoint]
+    :ivar config: Hook-specific configuration values.
+    :vartype config: dict[str, Any]
+    """
+
+    model_config = ConfigDict(extra="forbid")
+
+    hook_id: str = Field(min_length=1)
+    hook_points: List[HookPoint] = Field(default_factory=list)
+    config: Dict[str, Any] = Field(default_factory=dict)
+
+
+class HookContext(BaseModel):
+    """
+    Base context passed to hooks.
+
+    :ivar hook_point: Hook point currently executing.
+    :vartype hook_point: HookPoint
+    :ivar operation_id: Identifier for the enclosing command or call.
+    :vartype operation_id: str
+    :ivar corpus_uri: Canonical uniform resource identifier for the corpus.
+    :vartype corpus_uri: str
+    :ivar created_at: International Organization for Standardization 8601 timestamp when the context was created.
+    :vartype created_at: str
+    """
+
+    model_config = ConfigDict(extra="forbid")
+
+    hook_point: HookPoint
+    operation_id: str
+    corpus_uri: str
+    created_at: str
+
+
+class IngestHookContext(HookContext):
+    """
+    Hook context for ingestion hooks.
+
+    :ivar filename: Suggested filename for the item.
+    :vartype filename: str or None
+    :ivar media_type: Media type for the item.
+    :vartype media_type: str
+    :ivar title: Optional title associated with the item.
+    :vartype title: str or None
+    :ivar tags: Tags associated with the item.
+    :vartype tags: list[str]
+    :ivar metadata: Metadata mapping associated with the item.
+    :vartype metadata: dict[str, Any]
+    :ivar source_uri: Source uniform resource identifier.
+    :vartype source_uri: str
+    :ivar item_id: Item identifier when available.
+    :vartype item_id: str or None
+    :ivar relpath: Relative path to stored raw bytes when available.
+    :vartype relpath: str or None
+    """
+
+    filename: Optional[str] = None
+    media_type: str
+    title: Optional[str] = None
+    tags: List[str] = Field(default_factory=list)
+    metadata: Dict[str, Any] = Field(default_factory=dict)
+    source_uri: str
+    item_id: Optional[str] = None
+    relpath: Optional[str] = None
+
+
+class HookResult(BaseModel):
+    """
+    Base hook result with optional message fields.
+
+    :ivar message: Optional human-readable message.
+    :vartype message: str or None
+    """
+
+    model_config = ConfigDict(extra="forbid")
+
+    message: Optional[str] = None
+
+
+class IngestMutation(HookResult):
+    """
+    Hook result describing ingestion mutations.
+
+    :ivar deny: Whether ingest should be denied.
+    :vartype deny: bool
+    :ivar deny_reason: Optional reason for denial.
+    :vartype deny_reason: str or None
+    :ivar add_tags: Tags to add.
+    :vartype add_tags: list[str]
+    """
+
+    deny: bool = False
+    deny_reason: Optional[str] = None
+    add_tags: List[str] = Field(default_factory=list)
+
+
+class LifecycleHook:
+    """
+    Base class for a lifecycle hook implementation.
+
+    :param context: Validated hook context.
+    :type context: HookContext
+    :return: Hook result. Concrete hook points may require a more specific result type.
+    :rtype: HookResult
+    """
+
+    hook_id: str
+    hook_points: Sequence[HookPoint]
+
+    def run(self, context: HookContext) -> HookResult:
+        """
+        Execute the hook.
+
+        :param context: Hook context.
+        :type context: HookContext
+        :return: Hook result.
+        :rtype: HookResult
+        :raises NotImplementedError: If the hook does not implement run.
+        """
+        _ = context
+        raise NotImplementedError("LifecycleHook.run must be implemented by concrete hooks")
+
+
+class AddTagsHook:
+    """
+    Built-in hook that adds tags during ingestion.
+
+    :ivar hook_id: Hook identifier.
+    :vartype hook_id: str
+    :ivar hook_points: Hook points where the hook applies.
+    :vartype hook_points: list[HookPoint]
+    :ivar tags: Tags to add.
+    :vartype tags: list[str]
+    """
+
+    hook_id = "add-tags"
+
+    def __init__(self, *, hook_points: Sequence[HookPoint], tags: Sequence[str]):
+        """
+        Initialize the add-tags hook.
+
+        :param hook_points: Hook points where the hook runs.
+        :type hook_points: Sequence[HookPoint]
+        :param tags: Tags to add.
+        :type tags: Sequence[str]
+        """
+        self.hook_points = list(hook_points)
+        self.tags = [t.strip() for t in tags if isinstance(t, str) and t.strip()]
+
+    def run(self, context: HookContext) -> HookResult:
+        """
+        Run the hook.
+
+        :param context: Hook context.
+        :type context: HookContext
+        :return: Ingest mutation result.
+        :rtype: HookResult
+        """
+        _ = context
+        return IngestMutation(add_tags=list(self.tags))
+
+
+class DenyAllHook:
+    """
+    Built-in hook that denies every ingest.
+
+    :ivar hook_id: Hook identifier.
+    :vartype hook_id: str
+    :ivar hook_points: Hook points where the hook applies.
+    :vartype hook_points: list[HookPoint]
+    """
+
+    hook_id = "deny-all"
+
+    def __init__(self, *, hook_points: Sequence[HookPoint]):
+        """
+        Initialize the deny-all hook.
+
+        :param hook_points: Hook points where the hook runs.
+        :type hook_points: Sequence[HookPoint]
+        """
+        self.hook_points = list(hook_points)
+
+    def run(self, context: HookContext) -> HookResult:
+        """
+        Run the hook.
+
+        :param context: Hook context.
+        :type context: HookContext
+        :return: Ingest denial result.
+        :rtype: HookResult
+        """
+        _ = context
+        return IngestMutation(deny=True, deny_reason="Ingest denied by deny-all hook")
+
+
+def build_builtin_hook(spec: HookSpec) -> LifecycleHook:
+    """
+    Build a built-in hook from a hook specification.
+
+    :param spec: Hook specification.
+    :type spec: HookSpec
+    :return: Hook instance.
+    :rtype: LifecycleHook
+    :raises KeyError: If the hook identifier is unknown.
+    """
+    if spec.hook_id == AddTagsHook.hook_id:
+        tags = spec.config.get("tags") or []
+        return AddTagsHook(
+            hook_points=spec.hook_points, tags=tags if isinstance(tags, list) else []
+        )
+    if spec.hook_id == DenyAllHook.hook_id:
+        return DenyAllHook(hook_points=spec.hook_points)
+    raise KeyError(f"Unknown hook_id {spec.hook_id!r}")

biblicus/ignore.py

@@ -0,0 +1,64 @@
+"""
+Corpus ignore rules for bulk import and crawling.
+"""
+
+from __future__ import annotations
+
+import fnmatch
+from pathlib import Path
+from typing import List
+
+from pydantic import BaseModel, ConfigDict, Field
+
+
+class CorpusIgnoreSpec(BaseModel):
+    """
+    Parsed corpus ignore patterns.
+
+    Patterns are matched against a forward-slash relative path string.
+
+    :ivar patterns: Glob patterns to ignore.
+    :vartype patterns: list[str]
+    """
+
+    model_config = ConfigDict(extra="forbid")
+
+    patterns: List[str] = Field(default_factory=list)
+
+    def matches(self, relpath: str) -> bool:
+        """
+        Return True if the relative path matches any ignore pattern.
+
+        :param relpath: Forward-slash relative path.
+        :type relpath: str
+        :return: True if the path should be ignored.
+        :rtype: bool
+        """
+        normalized = relpath.replace("\\", "/").lstrip("/")
+        return any(fnmatch.fnmatch(normalized, pattern) for pattern in self.patterns)
+
+
+def load_corpus_ignore_spec(corpus_root: Path) -> CorpusIgnoreSpec:
+    """
+    Load ignore patterns from the corpus ignore file, if present.
+
+    The ignore file is stored at the corpus root as `.biblicusignore`.
+
+    :param corpus_root: Corpus root directory.
+    :type corpus_root: Path
+    :return: Parsed ignore specification.
+    :rtype: CorpusIgnoreSpec
+    """
+    ignore_path = corpus_root / ".biblicusignore"
+    if not ignore_path.is_file():
+        return CorpusIgnoreSpec(patterns=[])
+
+    patterns: List[str] = []
+    for raw_line in ignore_path.read_text(encoding="utf-8").splitlines():
+        line = raw_line.strip()
+        if not line:
+            continue
+        if line.startswith("#"):
+            continue
+        patterns.append(line)
+    return CorpusIgnoreSpec(patterns=patterns)