biblicus 1.0.0__py3-none-any.whl → 1.1.0__py3-none-any.whl

biblicus/{backends → retrievers}/tf_vector.py

@@ -1,5 +1,5 @@
 """
-Deterministic term-frequency vector retrieval backend.
+Deterministic term-frequency vector retriever.
 """
 
 from __future__ import annotations
@@ -14,87 +14,97 @@ from ..corpus import Corpus
 from ..frontmatter import parse_front_matter
 from ..models import (
     Evidence,
-    ExtractionRunReference,
+    ExtractionSnapshotReference,
     QueryBudget,
     RetrievalResult,
-    RetrievalRun,
-    parse_extraction_run_reference,
+    RetrievalSnapshot,
+    parse_extraction_snapshot_reference,
+)
+from ..retrieval import (
+    apply_budget,
+    create_configuration_manifest,
+    create_snapshot_manifest,
+    hash_text,
 )
-from ..retrieval import apply_budget, create_recipe_manifest, create_run_manifest, hash_text
 from ..time import utc_now_iso
 
 
-class TfVectorRecipeConfig(BaseModel):
+class TfVectorConfiguration(BaseModel):
     """
-    Configuration for the term-frequency vector retrieval backend.
+    Configuration for the term-frequency vector retriever.
 
-    :ivar extraction_run: Optional extraction run reference in the form extractor_id:run_id.
-    :vartype extraction_run: str or None
+    :ivar extraction_snapshot: Optional extraction snapshot reference in the form extractor_id:snapshot_id.
+    :vartype extraction_snapshot: str or None
     :ivar snippet_characters: Optional maximum character count for returned evidence text.
     :vartype snippet_characters: int or None
     """
 
     model_config = ConfigDict(extra="forbid")
 
-    extraction_run: Optional[str] = None
+    extraction_snapshot: Optional[str] = None
     snippet_characters: Optional[int] = None
 
 
-class TfVectorBackend:
+class TfVectorRetriever:
     """
-    Deterministic vector backend using term-frequency cosine similarity.
+    Deterministic vector retriever using term-frequency cosine similarity.
 
-    :ivar backend_id: Backend identifier.
-    :vartype backend_id: str
+    :ivar retriever_id: Retriever identifier.
+    :vartype retriever_id: str
     """
 
-    backend_id = "tf-vector"
+    retriever_id = "tf-vector"
 
-    def build_run(
-        self, corpus: Corpus, *, recipe_name: str, config: Dict[str, object]
-    ) -> RetrievalRun:
+    def build_snapshot(
+        self, corpus: Corpus, *, configuration_name: str, configuration: Dict[str, object]
+    ) -> RetrievalSnapshot:
         """
-        Register a vector backend run (no materialization).
+        Register a vector retriever snapshot (no snapshot artifacts).
 
         :param corpus: Corpus to build against.
         :type corpus: Corpus
-        :param recipe_name: Human-readable recipe name.
-        :type recipe_name: str
-        :param config: Backend-specific configuration values.
-        :type config: dict[str, object]
-        :return: Run manifest describing the build.
-        :rtype: RetrievalRun
+        :param configuration_name: Human-readable configuration name.
+        :type configuration_name: str
+        :param configuration: Retriever-specific configuration values.
+        :type configuration: dict[str, object]
+        :return: Snapshot manifest describing the build.
+        :rtype: RetrievalSnapshot
         """
-        recipe_config = TfVectorRecipeConfig.model_validate(config)
+        parsed_config = TfVectorConfiguration.model_validate(configuration)
         catalog = corpus.load_catalog()
-        recipe = create_recipe_manifest(
-            backend_id=self.backend_id,
-            name=recipe_name,
-            config=recipe_config.model_dump(),
+        configuration_manifest = create_configuration_manifest(
+            retriever_id=self.retriever_id,
+            name=configuration_name,
+            configuration=parsed_config.model_dump(),
         )
         stats = {
             "items": len(catalog.items),
-            "text_items": _count_text_items(corpus, catalog.items.values(), recipe_config),
+            "text_items": _count_text_items(corpus, catalog.items.values(), parsed_config),
         }
-        run = create_run_manifest(corpus, recipe=recipe, stats=stats, artifact_paths=[])
-        corpus.write_run(run)
-        return run
+        snapshot = create_snapshot_manifest(
+            corpus,
+            configuration=configuration_manifest,
+            stats=stats,
+            snapshot_artifacts=[],
+        )
+        corpus.write_snapshot(snapshot)
+        return snapshot
 
     def query(
         self,
         corpus: Corpus,
         *,
-        run: RetrievalRun,
+        snapshot: RetrievalSnapshot,
         query_text: str,
         budget: QueryBudget,
     ) -> RetrievalResult:
         """
         Query the corpus using term-frequency cosine similarity.
 
-        :param corpus: Corpus associated with the run.
+        :param corpus: Corpus associated with the snapshot.
         :type corpus: Corpus
-        :param run: Run manifest to use for querying.
-        :type run: RetrievalRun
+        :param snapshot: Snapshot manifest to use for querying.
+        :type snapshot: RetrievalSnapshot
         :param query_text: Query text to execute.
         :type query_text: str
         :param budget: Evidence selection budget.
@@ -102,15 +112,15 @@ class TfVectorBackend:
         :return: Retrieval results containing evidence.
         :rtype: RetrievalResult
         """
-        recipe_config = TfVectorRecipeConfig.model_validate(run.recipe.config)
+        parsed_config = TfVectorConfiguration.model_validate(snapshot.configuration.configuration)
         query_tokens = _tokenize_text(query_text)
         if not query_tokens:
             return RetrievalResult(
                 query_text=query_text,
                 budget=budget,
-                run_id=run.run_id,
-                recipe_id=run.recipe.recipe_id,
-                backend_id=self.backend_id,
+                snapshot_id=snapshot.snapshot_id,
+                configuration_id=snapshot.configuration.configuration_id,
+                retriever_id=snapshot.configuration.retriever_id,
                 generated_at=utc_now_iso(),
                 evidence=[],
                 stats={"candidates": 0, "returned": 0},
@@ -118,7 +128,7 @@ class TfVectorBackend:
         query_vector = _term_frequencies(query_tokens)
         query_norm = _vector_norm(query_vector)
         catalog = corpus.load_catalog()
-        extraction_reference = _resolve_extraction_reference(corpus, recipe_config)
+        extraction_reference = _resolve_extraction_reference(corpus, parsed_config)
         scored_candidates = _score_items(
             corpus,
             catalog.items.values(),
@@ -126,7 +136,7 @@ class TfVectorBackend:
             query_vector=query_vector,
             query_norm=query_norm,
             extraction_reference=extraction_reference,
-            snippet_characters=recipe_config.snippet_characters,
+            snippet_characters=parsed_config.snippet_characters,
         )
         sorted_candidates = sorted(
             scored_candidates,
@@ -136,8 +146,8 @@ class TfVectorBackend:
             evidence_item.model_copy(
                 update={
                     "rank": index,
-                    "recipe_id": run.recipe.recipe_id,
-                    "run_id": run.run_id,
+                    "configuration_id": snapshot.configuration.configuration_id,
+                    "snapshot_id": snapshot.snapshot_id,
                 }
             )
             for index, evidence_item in enumerate(sorted_candidates, start=1)
@@ -147,9 +157,9 @@ class TfVectorBackend:
         return RetrievalResult(
             query_text=query_text,
             budget=budget,
-            run_id=run.run_id,
-            recipe_id=run.recipe.recipe_id,
-            backend_id=self.backend_id,
+            snapshot_id=snapshot.snapshot_id,
+            configuration_id=snapshot.configuration.configuration_id,
+            retriever_id=snapshot.configuration.retriever_id,
             generated_at=utc_now_iso(),
             evidence=evidence,
             stats=stats,
@@ -157,33 +167,33 @@ class TfVectorBackend:
 
 
 def _resolve_extraction_reference(
-    corpus: Corpus, recipe_config: TfVectorRecipeConfig
-) -> Optional[ExtractionRunReference]:
+    corpus: Corpus, configuration: TfVectorConfiguration
+) -> Optional[ExtractionSnapshotReference]:
     """
-    Resolve an extraction run reference from a recipe config.
+    Resolve an extraction snapshot reference from a configuration.
 
-    :param corpus: Corpus associated with the recipe.
+    :param corpus: Corpus associated with the configuration.
     :type corpus: Corpus
-    :param recipe_config: Parsed vector recipe configuration.
-    :type recipe_config: TfVectorRecipeConfig
+    :param configuration: Parsed vector configuration.
+    :type configuration: TfVectorConfiguration
     :return: Parsed extraction reference or None.
-    :rtype: ExtractionRunReference or None
-    :raises FileNotFoundError: If an extraction run is referenced but not present.
+    :rtype: ExtractionSnapshotReference or None
+    :raises FileNotFoundError: If an extraction snapshot is referenced but not present.
     """
-    if not recipe_config.extraction_run:
+    if not configuration.extraction_snapshot:
         return None
-    extraction_reference = parse_extraction_run_reference(recipe_config.extraction_run)
-    run_dir = corpus.extraction_run_dir(
+    extraction_reference = parse_extraction_snapshot_reference(configuration.extraction_snapshot)
+    snapshot_dir = corpus.extraction_snapshot_dir(
         extractor_id=extraction_reference.extractor_id,
-        run_id=extraction_reference.run_id,
+        snapshot_id=extraction_reference.snapshot_id,
     )
-    if not run_dir.is_dir():
-        raise FileNotFoundError(f"Missing extraction run: {extraction_reference.as_string()}")
+    if not snapshot_dir.is_dir():
+        raise FileNotFoundError(f"Missing extraction snapshot: {extraction_reference.as_string()}")
     return extraction_reference
 
 
 def _count_text_items(
-    corpus: Corpus, items: Iterable[object], recipe_config: TfVectorRecipeConfig
+    corpus: Corpus, items: Iterable[object], configuration: TfVectorConfiguration
 ) -> int:
     """
     Count catalog items that represent text content.
@@ -192,19 +202,19 @@ def _count_text_items(
     :type corpus: Corpus
     :param items: Catalog items to inspect.
     :type items: Iterable[object]
-    :param recipe_config: Parsed vector recipe configuration.
-    :type recipe_config: TfVectorRecipeConfig
+    :param configuration: Parsed vector configuration.
+    :type configuration: TfVectorConfiguration
     :return: Number of text items.
     :rtype: int
     """
     text_item_count = 0
-    extraction_reference = _resolve_extraction_reference(corpus, recipe_config)
+    extraction_reference = _resolve_extraction_reference(corpus, configuration)
     for catalog_item in items:
         item_id = str(getattr(catalog_item, "id", ""))
         if extraction_reference and item_id:
             extracted_text = corpus.read_extracted_text(
                 extractor_id=extraction_reference.extractor_id,
-                run_id=extraction_reference.run_id,
+                snapshot_id=extraction_reference.snapshot_id,
                 item_id=item_id,
             )
             if isinstance(extracted_text, str) and extracted_text.strip():
@@ -292,7 +302,7 @@ def _load_text_from_item(
     item_id: str,
     relpath: str,
     media_type: str,
-    extraction_reference: Optional[ExtractionRunReference],
+    extraction_reference: Optional[ExtractionSnapshotReference],
 ) -> Optional[str]:
     """
     Load a text payload from a catalog item.
@@ -305,15 +315,15 @@ def _load_text_from_item(
     :type relpath: str
     :param media_type: Media type for the stored content.
     :type media_type: str
-    :param extraction_reference: Optional extraction run reference.
-    :type extraction_reference: ExtractionRunReference or None
+    :param extraction_reference: Optional extraction snapshot reference.
+    :type extraction_reference: ExtractionSnapshotReference or None
     :return: Text payload or None if not decodable as text.
     :rtype: str or None
     """
     if extraction_reference:
         extracted_text = corpus.read_extracted_text(
             extractor_id=extraction_reference.extractor_id,
-            run_id=extraction_reference.run_id,
+            snapshot_id=extraction_reference.snapshot_id,
             item_id=item_id,
         )
         if isinstance(extracted_text, str) and extracted_text.strip():
@@ -382,7 +392,7 @@ def _score_items(
     query_tokens: List[str],
     query_vector: Dict[str, float],
     query_norm: float,
-    extraction_reference: Optional[ExtractionRunReference],
+    extraction_reference: Optional[ExtractionSnapshotReference],
     snippet_characters: Optional[int],
 ) -> List[Evidence]:
     """
@@ -398,8 +408,8 @@ def _score_items(
     :type query_vector: dict[str, float]
     :param query_norm: Query vector norm.
     :type query_norm: float
-    :param extraction_reference: Optional extraction run reference.
-    :type extraction_reference: ExtractionRunReference or None
+    :param extraction_reference: Optional extraction snapshot reference.
+    :type extraction_reference: ExtractionSnapshotReference or None
     :param snippet_characters: Optional maximum character count for returned evidence text.
     :type snippet_characters: int or None
     :return: Evidence candidates with provisional ranks.
@@ -444,8 +454,8 @@ def _score_items(
                 span_start=span_start,
                 span_end=span_end,
                 stage="tf-vector",
-                recipe_id="",
-                run_id="",
+                configuration_id="",
+                snapshot_id="",
                 metadata=getattr(catalog_item, "metadata", {}) or {},
                 hash=hash_text(evidence_text or ""),
             )

biblicus/text/prompts.py

@@ -11,14 +11,16 @@ DEFAULT_EXTRACT_SYSTEM_PROMPT = (
     "Interpret the word 'return' in the user's request as: wrap the returned text with "
     "<span>...</span> in-place in the current text.\n\n"
     "Use the str_replace tool to insert <span>...</span> tags and the done tool when finished.\n"
+    "For long spans, insert <span> and </span> using separate str_replace calls. "
+    "For short spans (a few words), it is acceptable to insert both tags in one call.\n"
     "When finished, call done. Do NOT return JSON in the assistant message.\n\n"
     "Rules:\n"
     "- Use str_replace only.\n"
     "- old_str must match exactly once in the current text.\n"
     "- When choosing old_str, copy the exact substring (including punctuation/case) from the current text.\n"
     "- old_str and new_str must be non-empty strings.\n"
-    "- new_str must be identical to old_str with only <span> and </span> inserted.\n"
-    "- Do not include <span> or </span> inside old_str or new_str.\n"
+    "- new_str must be identical to old_str with only <span> and/or </span> inserted.\n"
+    "- Do not include <span> or </span> inside old_str.\n"
     "- Do not insert nested spans.\n"
     "- If a tool call fails due to non-unique old_str, retry with a longer unique old_str.\n"
     "- If a tool call fails, read the error and keep editing. Do not call done until spans are inserted.\n"
@@ -49,13 +51,15 @@ DEFAULT_ANNOTATE_SYSTEM_PROMPT = (
     '<span ATTRIBUTE="VALUE">...</span> in-place in the current text.\n'
     "Each span must include exactly one attribute from: {{ allowed_attributes }}.\n\n"
     "Use the str_replace tool to insert span tags and the done tool when finished.\n"
+    "For long spans, insert the opening and closing tags using separate str_replace calls. "
+    "For short spans (a few words), it is acceptable to insert both tags in one call.\n"
     "When finished, call done. Do NOT return JSON in the assistant message.\n\n"
     "Rules:\n"
     "- Use str_replace only.\n"
     "- old_str must match exactly once in the current text.\n"
     "- old_str and new_str must be non-empty strings.\n"
-    "- new_str must be identical to old_str with only <span ...> and </span> inserted.\n"
-    "- Do not include <span or </span> inside old_str or new_str.\n"
+    "- new_str must be identical to old_str with only <span ...> and/or </span> inserted.\n"
+    "- Do not include <span or </span> inside old_str.\n"
     "- Do not insert nested spans.\n"
     "- Do not wrap text that is already inside a span; spans must never overlap.\n"
     "- If a name appears inside an existing span, leave it alone and wrap only bare text.\n"
@@ -80,13 +84,15 @@ DEFAULT_LINK_SYSTEM_PROMPT = (
     "- Do not call done until every repeated name or entity in the text is wrapped.\n"
     "- If a name appears multiple times, there must be one id and refs for every later occurrence.\n\n"
     "Use the str_replace tool to insert span tags and the done tool when finished.\n"
+    "For long spans, insert the opening and closing tags using separate str_replace calls. "
+    "For short spans (a few words), it is acceptable to insert both tags in one call.\n"
     "When finished, call done. Do NOT return JSON in the assistant message.\n\n"
     "Rules:\n"
     "- Use str_replace only.\n"
     "- old_str must match exactly once in the current text.\n"
     "- old_str and new_str must be non-empty strings.\n"
-    "- new_str must be identical to old_str with only <span ...> and </span> inserted.\n"
-    "- Do not include <span or </span> inside old_str or new_str.\n"
+    "- new_str must be identical to old_str with only <span ...> and/or </span> inserted.\n"
+    "- Do not include <span or </span> inside old_str.\n"
     "- Do not insert nested spans.\n"
     "- If a tool call fails due to non-unique old_str, retry with a longer unique old_str.\n"
     "- If a tool call fails, read the error and keep editing. Do not call done until spans are inserted.\n"
@@ -100,13 +106,15 @@ DEFAULT_REDACT_SYSTEM_PROMPT = (
     "<span>...</span> in-place in the current text.\n"
     "If redaction types are provided, use a redact attribute with one of: {{ redaction_types }}.\n\n"
     "Use the str_replace tool to insert span tags and the done tool when finished.\n"
+    "For long spans, insert the opening and closing tags using separate str_replace calls. "
+    "For short spans (a few words), it is acceptable to insert both tags in one call.\n"
     "When finished, call done. Do NOT return JSON in the assistant message.\n\n"
     "Rules:\n"
     "- Use str_replace only.\n"
     "- old_str must match exactly once in the current text.\n"
     "- old_str and new_str must be non-empty strings.\n"
-    "- new_str must be identical to old_str with only <span ...> and </span> inserted.\n"
-    "- Do not include <span or </span> inside old_str or new_str.\n"
+    "- new_str must be identical to old_str with only <span ...> and/or </span> inserted.\n"
+    "- Do not include <span or </span> inside old_str.\n"
     "- Do not insert nested spans.\n"
     "- If a tool call fails due to non-unique old_str, retry with a longer unique old_str.\n"
     "- If a tool call fails, read the error and keep editing. Do not call done until spans are inserted.\n"

biblicus/text/tool_loop.py

@@ -5,6 +5,7 @@ Shared tool loop for virtual file edit workflows.
 from __future__ import annotations
 
 import json
+import re
 from dataclasses import dataclass
 from typing import Any, Callable, Dict, List, Optional, Sequence
 
@@ -182,6 +183,18 @@ def run_tool_loop(
                         last_error = "Tool loop requires non-empty old_str and new_str"
                         tool_result = f"Error: {last_error}"
                     else:
+                        if old_str == new_str:
+                            last_error = "Tool loop requires str_replace to make a change"
+                            tool_result = f"Error: {last_error}"
+                            had_tool_error = True
+                            messages.append(
+                                {
+                                    "role": "tool",
+                                    "tool_call_id": tool_call.get("id", ""),
+                                    "content": tool_result,
+                                }
+                            )
+                            continue
                         try:
                             current_text = apply_str_replace(current_text, old_str, new_str)
                             tool_result = (
@@ -214,6 +227,7 @@ def run_tool_loop(
                     "content": _build_tool_error_message(
                         error_message=last_error,
                         current_text=current_text,
+                        old_str=old_str if "old_str" in locals() else "",
                     ),
                 }
             )
@@ -260,19 +274,26 @@ def _build_retry_message(
     )
 
 
-def _build_tool_error_message(*, error_message: str, current_text: str) -> str:
-    if "not unique" in error_message:
+def _build_tool_error_message(*, error_message: str, current_text: str, old_str: str) -> str:
+    if "found 0 matches" in error_message or "not found" in error_message:
+        guidance = (
+            "Copy the exact old_str from the current text (including punctuation/case) "
+            "or call view to inspect the latest text."
+        )
+    elif "found " in error_message and "matches" in error_message:
         guidance = (
             "Use a longer unique old_str by including surrounding words or punctuation "
             "so it matches exactly once."
         )
-    elif "not found" in error_message:
+    elif "not unique" in error_message:
         guidance = (
-            "Copy the exact old_str from the current text (including punctuation/case) "
-            "or call view to inspect the latest text."
+            "Use a longer unique old_str by including surrounding words or punctuation "
+            "so it matches exactly once."
         )
     else:
         guidance = "Fix the tool call and try again."
+    if old_str and len(old_str) <= 3:
+        guidance = f"{guidance} If unsure, call view to pick a longer unique substring."
     return (
         "Your last tool call failed.\n"
         f"Error: {error_message}\n"
@@ -282,6 +303,43 @@ def _build_tool_error_message(*, error_message: str, current_text: str) -> str:
     )
 
 
+_SPAN_OPEN_PATTERN = re.compile(r"<span\b[^>]*>")
+_SPAN_CLOSE_PATTERN = re.compile(r"</span>")
+_SLICE_PATTERN = re.compile(r"<slice\s*/>")
+
+
+def _strip_markup(text: str) -> str:
+    without_spans = _SPAN_CLOSE_PATTERN.sub("", _SPAN_OPEN_PATTERN.sub("", text))
+    return _SLICE_PATTERN.sub("", without_spans)
+
+
+def apply_unique_str_replace(text: str, old_str: str, new_str: str) -> str:
+    """
+    Apply a single replacement only when old_str matches exactly once.
+
+    :param text: Current text content.
+    :type text: str
+    :param old_str: Substring to replace.
+    :type old_str: str
+    :param new_str: Replacement string.
+    :type new_str: str
+    :return: Updated text.
+    :rtype: str
+    :raises ValueError: If old_str matches zero or multiple times.
+    """
+    matches = text.count(old_str)
+    if matches != 1:
+        raise ValueError(
+            "Tool loop requires old_str to match exactly once " f"(found {matches} matches)"
+        )
+    if _strip_markup(old_str) != _strip_markup(new_str):
+        raise ValueError(
+            "Tool loop replacements may only insert markup tags; "
+            "the underlying text must stay the same"
+        )
+    return text.replace(old_str, new_str, 1)
+
+
 def _build_no_tool_calls_message(*, assistant_message: str, current_text: str) -> str:
     guidance = (
         "Use the tools to edit the text. "