biblicus 0.13.0__py3-none-any.whl → 0.15.0__py3-none-any.whl

biblicus/text/markup.py

@@ -0,0 +1,200 @@
+"""
+Shared span markup parsing utilities.
+"""
+
+from __future__ import annotations
+
+import re
+from typing import Dict, List, Sequence
+
+from pydantic import BaseModel, ConfigDict, Field
+
+
+class TextAnnotatedSpan(BaseModel):
+    """
+    Span annotated with arbitrary attributes.
+
+    :param index: One-based index of the span in the output order.
+    :type index: int
+    :param start_char: Start character offset in the original text.
+    :type start_char: int
+    :param end_char: End character offset in the original text.
+    :type end_char: int
+    :param text: Span text.
+    :type text: str
+    :param attributes: Attribute mapping extracted from the span tag.
+    :type attributes: dict[str, str]
+    """
+
+    model_config = ConfigDict(extra="forbid")
+
+    index: int = Field(ge=1)
+    start_char: int = Field(ge=0)
+    end_char: int = Field(ge=0)
+    text: str
+    attributes: Dict[str, str] = Field(default_factory=dict)
+
+
+_TAG_PATTERN = re.compile(r"<span\b[^>]*>|</span>")
+_OPEN_TAG_PATTERN = re.compile(r"<span\b([^>]*)>")
+_ATTRIBUTE_PATTERN = re.compile(r'([A-Za-z_][A-Za-z0-9_-]*)="([^"]*)"')
+_SPAN_INDEX_PATTERN = re.compile(r"Span (\d+)")
+
+
+def strip_span_tags(text: str) -> str:
+    """
+    Remove span tags from text.
+
+    :param text: Text with span tags.
+    :type text: str
+    :return: Text with span tags removed.
+    :rtype: str
+    """
+    return re.sub(r"</?span\b[^>]*>", "", text)
+
+
+def parse_span_markup(marked_up_text: str) -> List[TextAnnotatedSpan]:
+    """
+    Parse span tags with attributes into annotated spans.
+
+    :param marked_up_text: Text containing span tags.
+    :type marked_up_text: str
+    :return: Parsed spans with attributes.
+    :rtype: list[TextAnnotatedSpan]
+    :raises ValueError: If tags are malformed or nested.
+    """
+    spans: List[TextAnnotatedSpan] = []
+    cursor = 0
+    original_index = 0
+    span_start = None
+    span_text = ""
+    span_attributes: Dict[str, str] = {}
+
+    for match in _TAG_PATTERN.finditer(marked_up_text):
+        chunk = marked_up_text[cursor : match.start()]
+        if chunk:
+            if span_start is not None:
+                span_text += chunk
+            original_index += len(chunk)
+        tag = match.group(0)
+        if tag.startswith("<span"):
+            if span_start is not None:
+                raise ValueError("Text markup contains nested spans")
+            span_start = original_index
+            span_text = ""
+            span_attributes = _parse_span_attributes(tag)
+        else:
+            if span_start is None:
+                raise ValueError("Text markup contains an unmatched closing tag")
+            span_end = original_index
+            spans.append(
+                TextAnnotatedSpan(
+                    index=len(spans) + 1,
+                    start_char=span_start,
+                    end_char=span_end,
+                    text=span_text,
+                    attributes=span_attributes,
+                )
+            )
+            span_start = None
+            span_text = ""
+            span_attributes = {}
+        cursor = match.end()
+
+    tail = marked_up_text[cursor:]
+    if tail:
+        if span_start is not None:
+            span_text += tail
+        original_index += len(tail)
+
+    if span_start is not None:
+        raise ValueError("Text markup contains an unclosed span")
+
+    return spans
+
+
+def extract_span_indices(errors: Sequence[str]) -> List[int]:
+    """
+    Extract span indices referenced in error messages.
+
+    :param errors: Validation error messages.
+    :type errors: Sequence[str]
+    :return: Sorted list of referenced span indices.
+    :rtype: list[int]
+    """
+    indices: List[int] = []
+    for error in errors:
+        match = _SPAN_INDEX_PATTERN.search(error)
+        if match is None:
+            continue
+        indices.append(int(match.group(1)))
+    return sorted(set(indices))
+
+
+def summarize_span_context(marked_up_text: str, span_indices: Sequence[int]) -> List[str]:
+    """
+    Summarize span context for the requested indices.
+
+    :param marked_up_text: Text containing span tags.
+    :type marked_up_text: str
+    :param span_indices: Span indices to summarize.
+    :type span_indices: Sequence[int]
+    :return: Human-readable span summaries.
+    :rtype: list[str]
+    :raises ValueError: If the markup is invalid.
+    """
+    spans = parse_span_markup(marked_up_text)
+    span_by_index = {span.index: span for span in spans}
+    summaries: List[str] = []
+    for index in span_indices:
+        span = span_by_index.get(index)
+        if span is None:
+            continue
+        cleaned_text = " ".join(span.text.split())
+        if cleaned_text:
+            summaries.append(f"Span {index}: {cleaned_text}")
+    return summaries
+
+
+def build_span_context_section(marked_up_text: str, errors: Sequence[str]) -> str:
+    """
+    Build a formatted span context section for retry messages.
+
+    :param marked_up_text: Text containing span tags.
+    :type marked_up_text: str
+    :param errors: Validation error messages.
+    :type errors: Sequence[str]
+    :return: Formatted span context block or empty string.
+    :rtype: str
+    """
+    indices = extract_span_indices(errors)
+    if not indices:
+        return ""
+    try:
+        summaries = summarize_span_context(marked_up_text, indices)
+    except ValueError:
+        return ""
+    if not summaries:
+        return ""
+    summary_lines = "\n".join(f"- {summary}" for summary in summaries)
+    return f"Relevant spans:\n{summary_lines}\n\n"
+
+
+def _parse_span_attributes(tag_text: str) -> Dict[str, str]:
+    match = _OPEN_TAG_PATTERN.fullmatch(tag_text)
+    if match is None:
+        raise ValueError("Text markup contains an invalid span tag")
+    attr_text = match.group(1).strip().replace('\\"', '"')
+    if not attr_text:
+        return {}
+    attributes: Dict[str, str] = {}
+    for attr_match in _ATTRIBUTE_PATTERN.finditer(attr_text):
+        name = attr_match.group(1)
+        value = attr_match.group(2)
+        if name in attributes:
+            raise ValueError("Text markup contains duplicate span attributes")
+        attributes[name] = value
+    cleaned = _ATTRIBUTE_PATTERN.sub("", attr_text).strip()
+    if cleaned:
+        raise ValueError("Text markup contains unsupported span attributes")
+    return attributes

biblicus/text/models.py

@@ -0,0 +1,319 @@
+"""
+Pydantic models for agentic text utilities.
+"""
+
+from __future__ import annotations
+
+from typing import List, Optional
+
+from pydantic import BaseModel, ConfigDict, Field, model_validator
+
+from ..ai.models import LlmClientConfig
+from .markup import TextAnnotatedSpan
+from .prompts import (
+    DEFAULT_ANNOTATE_SYSTEM_PROMPT,
+    DEFAULT_EXTRACT_SYSTEM_PROMPT,
+    DEFAULT_LINK_SYSTEM_PROMPT,
+    DEFAULT_PROMPT_TEMPLATE,
+    DEFAULT_REDACT_SYSTEM_PROMPT,
+    DEFAULT_SLICE_SYSTEM_PROMPT,
+)
+
+
+class TextToolLoopRequest(BaseModel):
+    """
+    Request to apply a tool-loop text operation using a language model.
+
+    :param text: Input text to process.
+    :type text: str
+    :param client: LLM client configuration.
+    :type client: biblicus.ai.models.LlmClientConfig
+    :param prompt_template: Prompt template describing what to return (must not include ``{text}``).
+    :type prompt_template: str
+    :param system_prompt: System prompt template containing ``{text}``. The base request requires
+        callers to supply this. Specific utility requests provide built-in defaults so callers
+        can typically omit it.
+    :type system_prompt: str
+    :param max_rounds: Maximum number of edit rounds.
+    :type max_rounds: int
+    :param max_edits_per_round: Maximum edits per round.
+    :type max_edits_per_round: int
+    :param mock_marked_up_text: Optional pre-rendered markup for deterministic tests.
+    :type mock_marked_up_text: str or None
+    """
+
+    model_config = ConfigDict(extra="forbid")
+
+    text: str = Field(min_length=1)
+    client: LlmClientConfig
+    prompt_template: str = Field(default=DEFAULT_PROMPT_TEMPLATE, min_length=1)
+    system_prompt: str = Field(min_length=1)
+    max_rounds: int = Field(default=6, ge=1)
+    max_edits_per_round: int = Field(default=500, ge=1)
+    mock_marked_up_text: Optional[str] = Field(default=None, min_length=1)
+
+    @model_validator(mode="after")
+    def _validate_prompts(self) -> "TextToolLoopRequest":
+        if "{text}" not in self.system_prompt:
+            raise ValueError("system_prompt must include {text}")
+        if "{text}" in self.prompt_template:
+            raise ValueError("prompt_template must not include {text}")
+        return self
+
+
+class TextExtractRequest(TextToolLoopRequest):
+    """
+    Request to apply text extract using a language model.
+
+    :param text: Input text to annotate with XML span tags.
+    :type text: str
+    :param client: LLM client configuration.
+    :type client: biblicus.ai.models.LlmClientConfig
+    :param prompt_template: Prompt template describing what to return (must not include ``{text}``).
+    :type prompt_template: str
+    :param system_prompt: System prompt template containing ``{text}``. Defaults to the built-in
+        text extract system prompt.
+    :type system_prompt: str
+    :param max_rounds: Maximum number of edit rounds.
+    :type max_rounds: int
+    :param max_edits_per_round: Maximum edits per round.
+    :type max_edits_per_round: int
+    """
+
+    system_prompt: str = Field(default=DEFAULT_EXTRACT_SYSTEM_PROMPT, min_length=1)
+
+
+class TextSliceRequest(TextToolLoopRequest):
+    """
+    Request to apply text slice using a language model.
+
+    :param text: Input text to mark with slice markers.
+    :type text: str
+    :param client: LLM client configuration.
+    :type client: biblicus.ai.models.LlmClientConfig
+    :param prompt_template: Prompt template describing what to return (must not include ``{text}``).
+    :type prompt_template: str
+    :param system_prompt: System prompt template containing ``{text}``. Defaults to the built-in
+        text slice system prompt.
+    :type system_prompt: str
+    :param max_rounds: Maximum number of edit rounds.
+    :type max_rounds: int
+    :param max_edits_per_round: Maximum edits per round.
+    :type max_edits_per_round: int
+    """
+
+    system_prompt: str = Field(default=DEFAULT_SLICE_SYSTEM_PROMPT, min_length=1)
+
+
+class TextAnnotateRequest(TextToolLoopRequest):
+    """
+    Request to apply text annotation using span attributes.
+
+    :param text: Input text to annotate with XML span tags.
+    :type text: str
+    :param client: LLM client configuration.
+    :type client: biblicus.ai.models.LlmClientConfig
+    :param prompt_template: Prompt template describing what to return (must not include ``{text}``).
+    :type prompt_template: str
+    :param system_prompt: System prompt containing ``{text}``. Defaults to the built-in
+        text annotate system prompt.
+    :type system_prompt: str
+    :param allowed_attributes: Optional list of allowed span attribute names.
+    :type allowed_attributes: list[str] or None
+    :param max_rounds: Maximum number of edit rounds.
+    :type max_rounds: int
+    :param max_edits_per_round: Maximum edits per round.
+    :type max_edits_per_round: int
+    """
+
+    system_prompt: str = Field(default=DEFAULT_ANNOTATE_SYSTEM_PROMPT, min_length=1)
+    allowed_attributes: Optional[List[str]] = None
+
+
+class TextRedactRequest(TextToolLoopRequest):
+    """
+    Request to apply text redaction using span markers.
+
+    :param text: Input text to annotate with XML span tags.
+    :type text: str
+    :param client: LLM client configuration.
+    :type client: biblicus.ai.models.LlmClientConfig
+    :param prompt_template: Prompt template describing what to return (must not include ``{text}``).
+    :type prompt_template: str
+    :param system_prompt: System prompt containing ``{text}``. Defaults to the built-in
+        text redact system prompt.
+    :type system_prompt: str
+    :param redaction_types: Optional list of allowed redaction types. When omitted, no attributes are allowed.
+    :type redaction_types: list[str] or None
+    :param max_rounds: Maximum number of edit rounds.
+    :type max_rounds: int
+    :param max_edits_per_round: Maximum edits per round.
+    :type max_edits_per_round: int
+    """
+
+    system_prompt: str = Field(default=DEFAULT_REDACT_SYSTEM_PROMPT, min_length=1)
+    redaction_types: Optional[List[str]] = None
+
+
+class TextLinkRequest(TextToolLoopRequest):
+    """
+    Request to apply text linking using id/ref span attributes.
+
+    :param text: Input text to annotate with XML span tags.
+    :type text: str
+    :param client: LLM client configuration.
+    :type client: biblicus.ai.models.LlmClientConfig
+    :param prompt_template: Prompt template describing what to return (must not include ``{text}``).
+    :type prompt_template: str
+    :param system_prompt: System prompt containing ``{text}``. Defaults to the built-in
+        text link system prompt.
+    :type system_prompt: str
+    :param id_prefix: Prefix required for id attributes.
+    :type id_prefix: str
+    :param max_rounds: Maximum number of edit rounds.
+    :type max_rounds: int
+    :param max_edits_per_round: Maximum edits per round.
+    :type max_edits_per_round: int
+    """
+
+    system_prompt: str = Field(default=DEFAULT_LINK_SYSTEM_PROMPT, min_length=1)
+    id_prefix: str = Field(default="link_", min_length=1)
+
+
+class TextExtractSpan(BaseModel):
+    """
+    Extracted span of text.
+
+    :param index: One-based index of the span in the output order.
+    :type index: int
+    :param start_char: Start character offset in the original text.
+    :type start_char: int
+    :param end_char: End character offset in the original text.
+    :type end_char: int
+    :param text: Span text.
+    :type text: str
+    """
+
+    model_config = ConfigDict(extra="forbid")
+
+    index: int = Field(ge=1)
+    start_char: int = Field(ge=0)
+    end_char: int = Field(ge=0)
+    text: str
+
+
+class TextSliceSegment(BaseModel):
+    """
+    Extracted text slice.
+
+    :param index: One-based index of the slice in the output order.
+    :type index: int
+    :param start_char: Start character offset in the original text.
+    :type start_char: int
+    :param end_char: End character offset in the original text.
+    :type end_char: int
+    :param text: Slice text.
+    :type text: str
+    """
+
+    model_config = ConfigDict(extra="forbid")
+
+    index: int = Field(ge=1)
+    start_char: int = Field(ge=0)
+    end_char: int = Field(ge=0)
+    text: str
+
+
+class TextExtractResult(BaseModel):
+    """
+    Text extract output bundle.
+
+    :param marked_up_text: Original text with XML span tags inserted.
+    :type marked_up_text: str
+    :param spans: Extracted spans in document order.
+    :type spans: list[TextExtractSpan]
+    :param warnings: Warning messages for the caller.
+    :type warnings: list[str]
+    """
+
+    model_config = ConfigDict(extra="forbid")
+
+    marked_up_text: str
+    spans: List[TextExtractSpan] = Field(default_factory=list)
+    warnings: List[str] = Field(default_factory=list)
+
+
+class TextSliceResult(BaseModel):
+    """
+    Text slice output bundle.
+
+    :param marked_up_text: Original text with slice markers inserted.
+    :type marked_up_text: str
+    :param slices: Extracted slices in document order.
+    :type slices: list[TextSliceSegment]
+    :param warnings: Warning messages for the caller.
+    :type warnings: list[str]
+    """
+
+    model_config = ConfigDict(extra="forbid")
+
+    marked_up_text: str
+    slices: List[TextSliceSegment] = Field(default_factory=list)
+    warnings: List[str] = Field(default_factory=list)
+
+
+class TextAnnotateResult(BaseModel):
+    """
+    Text annotation output bundle.
+
+    :param marked_up_text: Original text with XML span tags inserted.
+    :type marked_up_text: str
+    :param spans: Extracted spans in document order.
+    :type spans: list[TextAnnotatedSpan]
+    :param warnings: Warning messages for the caller.
+    :type warnings: list[str]
+    """
+
+    model_config = ConfigDict(extra="forbid")
+
+    marked_up_text: str
+    spans: List[TextAnnotatedSpan] = Field(default_factory=list)
+    warnings: List[str] = Field(default_factory=list)
+
+
+class TextRedactResult(BaseModel):
+    """
+    Text redaction output bundle.
+
+    :param marked_up_text: Original text with XML span tags inserted.
+    :type marked_up_text: str
+    :param spans: Redacted spans in document order.
+    :type spans: list[TextAnnotatedSpan]
+    :param warnings: Warning messages for the caller.
+    :type warnings: list[str]
+    """
+
+    model_config = ConfigDict(extra="forbid")
+
+    marked_up_text: str
+    spans: List[TextAnnotatedSpan] = Field(default_factory=list)
+    warnings: List[str] = Field(default_factory=list)
+
+
+class TextLinkResult(BaseModel):
+    """
+    Text linking output bundle.
+
+    :param marked_up_text: Original text with XML span tags inserted.
+    :type marked_up_text: str
+    :param spans: Linked spans in document order.
+    :type spans: list[TextAnnotatedSpan]
+    :param warnings: Warning messages for the caller.
+    :type warnings: list[str]
+    """
+
+    model_config = ConfigDict(extra="forbid")
+
+    marked_up_text: str
+    spans: List[TextAnnotatedSpan] = Field(default_factory=list)
+    warnings: List[str] = Field(default_factory=list)

biblicus/text/prompts.py

@@ -0,0 +1,113 @@
+"""
+Default prompts for Biblicus text utilities.
+"""
+
+from __future__ import annotations
+
+DEFAULT_PROMPT_TEMPLATE = "Return the requested text."
+
+DEFAULT_EXTRACT_SYSTEM_PROMPT = (
+    "You are a virtual file editor. Use the available tools to edit the text.\n"
+    "Interpret the word 'return' in the user's request as: wrap the returned text with "
+    "<span>...</span> in-place in the current text.\n\n"
+    "Use the str_replace tool to insert <span>...</span> tags and the done tool when finished.\n"
+    "When finished, call done. Do NOT return JSON in the assistant message.\n\n"
+    "Rules:\n"
+    "- Use str_replace only.\n"
+    "- old_str must match exactly once in the current text.\n"
+    "- When choosing old_str, copy the exact substring (including punctuation/case) from the current text.\n"
+    "- old_str and new_str must be non-empty strings.\n"
+    "- new_str must be identical to old_str with only <span> and </span> inserted.\n"
+    "- Do not include <span> or </span> inside old_str or new_str.\n"
+    "- Do not insert nested spans.\n"
+    "- If a tool call fails due to non-unique old_str, retry with a longer unique old_str.\n"
+    "- If a tool call fails, read the error and keep editing. Do not call done until spans are inserted.\n"
+    "- Do not delete, reorder, paraphrase, or label text.\n\n"
+    "Current text:\n---\n{text}\n---\n"
+)
+
+DEFAULT_SLICE_SYSTEM_PROMPT = (
+    "You are a virtual file editor. Use the available tools to edit the text.\n"
+    "Interpret the word 'return' in the user's request as: insert <slice/> markers in-place in the current text.\n\n"
+    "Use the str_replace tool to insert <slice/> markers and the done tool when finished.\n"
+    "When finished, call done. Do NOT return JSON in the assistant message.\n\n"
+    "Rules:\n"
+    "- Use str_replace only.\n"
+    "- old_str must match exactly once in the current text.\n"
+    "- old_str and new_str must be non-empty strings.\n"
+    "- new_str must be identical to old_str with only <slice/> inserted.\n"
+    "- Do not include <slice/> inside old_str or new_str.\n"
+    "- If a tool call fails due to non-unique old_str, retry with a longer unique old_str.\n"
+    "- If a tool call fails, read the error and keep editing. Do not call done until markers are inserted.\n"
+    "- Do not delete, reorder, or paraphrase text.\n\n"
+    "Current text:\n---\n{text}\n---\n"
+)
+
+DEFAULT_ANNOTATE_SYSTEM_PROMPT = (
+    "You are a virtual file editor. Use the available tools to edit the text.\n"
+    "Interpret the user's request as: wrap the requested text with "
+    '<span ATTRIBUTE="VALUE">...</span> in-place in the current text.\n'
+    "Each span must include exactly one attribute from: {{ allowed_attributes }}.\n\n"
+    "Use the str_replace tool to insert span tags and the done tool when finished.\n"
+    "When finished, call done. Do NOT return JSON in the assistant message.\n\n"
+    "Rules:\n"
+    "- Use str_replace only.\n"
+    "- old_str must match exactly once in the current text.\n"
+    "- old_str and new_str must be non-empty strings.\n"
+    "- new_str must be identical to old_str with only <span ...> and </span> inserted.\n"
+    "- Do not include <span or </span> inside old_str or new_str.\n"
+    "- Do not insert nested spans.\n"
+    "- If a tool call fails due to non-unique old_str, retry with a longer unique old_str.\n"
+    "- If a tool call fails, read the error and keep editing. Do not call done until spans are inserted.\n"
+    "- Do not delete, reorder, paraphrase, or label text beyond the span attributes.\n\n"
+    "Current text:\n---\n{text}\n---\n"
+)
+
+DEFAULT_LINK_SYSTEM_PROMPT = (
+    "You are a virtual file editor. Use the available tools to edit the text.\n"
+    "Interpret the word 'return' in the user's request as: wrap the returned text with "
+    '<span ATTRIBUTE="VALUE">...</span> in-place in the current text.\n'
+    "Each span must include exactly one attribute: id for first mentions and ref for repeats.\n"
+    "Id values must start with '{{ id_prefix }}'.\n\n"
+    "Linking rules:\n"
+    "- For each distinct name or entity, assign exactly one id on its first occurrence.\n"
+    "- Use ref on every subsequent occurrence of the same name or entity.\n"
+    "- Wrap only the repeated name or entity text itself (no extra surrounding words).\n"
+    "- Reuse the same id/ref value for identical names; do not create multiple ids for the same name.\n"
+    "- Use ids in order of first appearance ({{ id_prefix }}1, {{ id_prefix }}2, ...).\n"
+    "- Do not call done until every repeated name or entity in the text is wrapped.\n"
+    "- If a name appears multiple times, there must be one id and refs for every later occurrence.\n\n"
+    "Use the str_replace tool to insert span tags and the done tool when finished.\n"
+    "When finished, call done. Do NOT return JSON in the assistant message.\n\n"
+    "Rules:\n"
+    "- Use str_replace only.\n"
+    "- old_str must match exactly once in the current text.\n"
+    "- old_str and new_str must be non-empty strings.\n"
+    "- new_str must be identical to old_str with only <span ...> and </span> inserted.\n"
+    "- Do not include <span or </span> inside old_str or new_str.\n"
+    "- Do not insert nested spans.\n"
+    "- If a tool call fails due to non-unique old_str, retry with a longer unique old_str.\n"
+    "- If a tool call fails, read the error and keep editing. Do not call done until spans are inserted.\n"
+    "- Do not delete, reorder, or paraphrase text.\n\n"
+    "Current text:\n---\n{text}\n---\n"
+)
+
+DEFAULT_REDACT_SYSTEM_PROMPT = (
+    "You are a virtual file editor. Use the available tools to edit the text.\n"
+    "Interpret the word 'return' in the user's request as: wrap the returned text with "
+    "<span>...</span> in-place in the current text.\n"
+    "If redaction types are provided, use a redact attribute with one of: {{ redaction_types }}.\n\n"
+    "Use the str_replace tool to insert span tags and the done tool when finished.\n"
+    "When finished, call done. Do NOT return JSON in the assistant message.\n\n"
+    "Rules:\n"
+    "- Use str_replace only.\n"
+    "- old_str must match exactly once in the current text.\n"
+    "- old_str and new_str must be non-empty strings.\n"
+    "- new_str must be identical to old_str with only <span ...> and </span> inserted.\n"
+    "- Do not include <span or </span> inside old_str or new_str.\n"
+    "- Do not insert nested spans.\n"
+    "- If a tool call fails due to non-unique old_str, retry with a longer unique old_str.\n"
+    "- If a tool call fails, read the error and keep editing. Do not call done until spans are inserted.\n"
+    "- Do not delete, reorder, or paraphrase text.\n\n"
+    "Current text:\n---\n{text}\n---\n"
+)