biblicus 0.14.0__py3-none-any.whl → 0.15.1__py3-none-any.whl

biblicus/text/tool_loop.py

@@ -0,0 +1,334 @@
+"""
+Shared tool loop for virtual file edit workflows.
+"""
+
+from __future__ import annotations
+
+import json
+from dataclasses import dataclass
+from typing import Any, Callable, Dict, List, Optional, Sequence
+
+from ..ai.llm import chat_completion
+from ..ai.models import LlmClientConfig
+
+
+@dataclass
+class ToolLoopResult:
+    """
+    Tool loop result payload.
+
+    :param text: Final text after tool edits.
+    :type text: str
+    :param done: Whether the model called done.
+    :type done: bool
+    :param last_error: Last error message, if any.
+    :type last_error: str or None
+    :param messages: Conversation history including tool calls/results.
+    :type messages: list[dict[str, Any]]
+    """
+
+    text: str
+    done: bool
+    last_error: Optional[str]
+    messages: List[Dict[str, Any]]
+
+
+def run_tool_loop(
+    *,
+    text: str,
+    client: LlmClientConfig,
+    system_prompt: str,
+    prompt_template: str,
+    max_rounds: int,
+    max_edits_per_round: int,
+    apply_str_replace: Callable[[str, str, str], str],
+    validate_text: Optional[Callable[[str], Sequence[str]]] = None,
+    build_retry_message: Optional[Callable[[Sequence[str], str], str]] = None,
+    messages: Optional[List[Dict[str, Any]]] = None,
+) -> ToolLoopResult:
+    """
+    Run a tool-driven virtual file edit loop.
+
+    :param text: Input text to edit.
+    :type text: str
+    :param client: LLM client configuration.
+    :type client: biblicus.ai.models.LlmClientConfig
+    :param system_prompt: System prompt containing the text placeholder.
+    :type system_prompt: str
+    :param prompt_template: User prompt describing what to return.
+    :type prompt_template: str
+    :param max_rounds: Maximum number of rounds.
+    :type max_rounds: int
+    :param max_edits_per_round: Maximum edits per round.
+    :type max_edits_per_round: int
+    :param apply_str_replace: Replacement function for str_replace edits.
+    :type apply_str_replace: Callable[[str, str, str], str]
+    :param validate_text: Optional validation callback returning error messages.
+    :type validate_text: Callable[[str], Sequence[str]] or None
+    :param build_retry_message: Optional retry message builder.
+    :type build_retry_message: Callable[[Sequence[str], str], str] or None
+    :param messages: Optional conversation history to continue (system prompt should already be included).
+    :type messages: list[dict[str, Any]] or None
+    :return: Tool loop result.
+    :rtype: ToolLoopResult
+    :raises ValueError: If the provider backend is unavailable.
+
+    Validation errors trigger a retry by appending a user feedback message to the
+    conversation history (including all prior tool calls and tool results).
+    """
+    tools = [
+        {
+            "type": "function",
+            "function": {
+                "name": "str_replace",
+                "description": "Replace an exact substring with a new string.",
+                "parameters": {
+                    "type": "object",
+                    "properties": {
+                        "old_str": {"type": "string"},
+                        "new_str": {"type": "string"},
+                    },
+                    "required": ["old_str", "new_str"],
+                    "additionalProperties": False,
+                },
+            },
+        },
+        {
+            "type": "function",
+            "function": {
+                "name": "view",
+                "description": "Return the current text.",
+                "parameters": {"type": "object", "properties": {}, "additionalProperties": False},
+            },
+        },
+        {
+            "type": "function",
+            "function": {
+                "name": "done",
+                "description": "Finish editing.",
+                "parameters": {"type": "object", "properties": {}, "additionalProperties": False},
+            },
+        },
+    ]
+
+    if messages is None:
+        rendered_prompt = _render_template(
+            prompt_template,
+            text=text,
+            text_length=len(text),
+            error="",
+        )
+        rendered_system = _render_template(
+            system_prompt,
+            text=text,
+            text_length=len(text),
+            error="",
+        )
+        messages = [
+            {"role": "system", "content": rendered_system},
+            {"role": "user", "content": rendered_prompt},
+        ]
+    else:
+        messages = list(messages)
+
+    done = False
+    last_error: Optional[str] = None
+    current_text = text
+
+    for _ in range(max_rounds):
+        had_tool_error = False
+        response = chat_completion(
+            client=client,
+            messages=messages,
+            tools=tools,
+            tool_choice="auto",
+        )
+        tool_calls = response.tool_calls
+        if not tool_calls:
+            content = response.text or ""
+            last_error = "Tool loop requires tool calls (str_replace/view/done)"
+            messages.append({"role": "assistant", "content": content})
+            messages.append(
+                {
+                    "role": "user",
+                    "content": _build_no_tool_calls_message(
+                        assistant_message=content,
+                        current_text=current_text,
+                    ),
+                }
+            )
+            continue
+        messages.append(
+            {
+                "role": "assistant",
+                "content": response.text or "",
+                "tool_calls": list(tool_calls),
+            }
+        )
+        edit_count = 0
+        for tool_call in tool_calls:
+            function = tool_call.get("function", {})
+            name = str(function.get("name") or "")
+            args = json.loads(str(function.get("arguments") or "{}"))
+            if name == "str_replace":
+                edit_count += 1
+                if edit_count > max_edits_per_round:
+                    last_error = "Tool loop exceeded max edits per round"
+                    tool_result = f"Error: {last_error}"
+                else:
+                    old_str = str(args.get("old_str", ""))
+                    new_str = str(args.get("new_str", ""))
+                    if not old_str or not new_str:
+                        last_error = "Tool loop requires non-empty old_str and new_str"
+                        tool_result = f"Error: {last_error}"
+                    else:
+                        try:
+                            current_text = apply_str_replace(current_text, old_str, new_str)
+                            tool_result = (
+                                "Applied str_replace.\nCurrent text:\n---\n" f"{current_text}\n---"
+                            )
+                            last_error = None
+                        except ValueError as exc:
+                            last_error = str(exc)
+                            tool_result = f"Error: {last_error}"
+                            had_tool_error = True
+            elif name == "view":
+                tool_result = f"Current text:\n---\n{current_text}\n---"
+            elif name == "done":
+                done = True
+                tool_result = "Done"
+            else:
+                raise ValueError(f"Tool loop received unknown tool: {name}")
+            messages.append(
+                {
+                    "role": "tool",
+                    "tool_call_id": tool_call.get("id", ""),
+                    "content": tool_result,
+                }
+            )
+        if had_tool_error and last_error is not None:
+            done = False
+            messages.append(
+                {
+                    "role": "user",
+                    "content": _build_tool_error_message(
+                        error_message=last_error,
+                        current_text=current_text,
+                    ),
+                }
+            )
+            continue
+        if validate_text is not None:
+            validation_errors = list(validate_text(current_text))
+            if validation_errors:
+                last_error = "; ".join(validation_errors)
+                done = False
+                retry_message = _build_retry_message(
+                    validation_errors=validation_errors,
+                    current_text=current_text,
+                    build_retry_message=build_retry_message,
+                )
+                messages.append({"role": "user", "content": retry_message})
+                continue
+        if done:
+            break
+
+    return ToolLoopResult(
+        text=current_text,
+        done=done,
+        last_error=last_error,
+        messages=messages,
+    )
+
+
+def _build_retry_message(
+    *,
+    validation_errors: Sequence[str],
+    current_text: str,
+    build_retry_message: Optional[Callable[[Sequence[str], str], str]],
+) -> str:
+    if build_retry_message is not None:
+        return build_retry_message(validation_errors, current_text)
+    error_lines = "\n".join(f"- {error}" for error in validation_errors)
+    return (
+        "Your last edit did not validate.\n"
+        "Issues:\n"
+        f"{error_lines}\n\n"
+        "Please fix the markup using str_replace and keep the source text unchanged.\n"
+        "Current text:\n"
+        f"---\n{current_text}\n---"
+    )
+
+
+def _build_tool_error_message(*, error_message: str, current_text: str) -> str:
+    if "not unique" in error_message:
+        guidance = (
+            "Use a longer unique old_str by including surrounding words or punctuation "
+            "so it matches exactly once."
+        )
+    elif "not found" in error_message:
+        guidance = (
+            "Copy the exact old_str from the current text (including punctuation/case) "
+            "or call view to inspect the latest text."
+        )
+    else:
+        guidance = "Fix the tool call and try again."
+    return (
+        "Your last tool call failed.\n"
+        f"Error: {error_message}\n"
+        f"{guidance}\n"
+        "Current text:\n"
+        f"---\n{current_text}\n---"
+    )
+
+
+def _build_no_tool_calls_message(*, assistant_message: str, current_text: str) -> str:
+    guidance = (
+        "Use the tools to edit the text. "
+        "Call str_replace to insert markup, view to inspect, and done when finished."
+    )
+    message = "Your last response did not include any tool calls."
+    if assistant_message.strip():
+        message = f"{message}\nAssistant message: {assistant_message}"
+    return f"{message}\n" f"{guidance}\n" "Current text:\n" f"---\n{current_text}\n---"
+
+
+def request_confirmation(
+    *,
+    result: ToolLoopResult,
+    text: str,
+    client: LlmClientConfig,
+    system_prompt: str,
+    prompt_template: str,
+    max_rounds: int,
+    max_edits_per_round: int,
+    apply_str_replace: Callable[[str, str, str], str],
+    confirmation_message: str,
+    validate_text: Optional[Callable[[str], Sequence[str]]] = None,
+    build_retry_message: Optional[Callable[[Sequence[str], str], str]] = None,
+) -> ToolLoopResult:
+    """
+    Continue a tool loop with a confirmation message appended to the conversation history.
+
+    This preserves the model's prior tool calls and the current text state while giving it
+    a chance to confirm an empty/ambiguous result.
+    """
+    messages = list(result.messages)
+    messages.append({"role": "user", "content": confirmation_message})
+    return run_tool_loop(
+        text=text,
+        client=client,
+        system_prompt=system_prompt,
+        prompt_template=prompt_template,
+        max_rounds=max_rounds,
+        max_edits_per_round=max_edits_per_round,
+        apply_str_replace=apply_str_replace,
+        validate_text=validate_text,
+        build_retry_message=build_retry_message,
+        messages=messages,
+    )
+
+
+def _render_template(template: str, *, text: str, text_length: int, error: str) -> str:
+    rendered = template.replace("{text_length}", str(text_length)).replace("{error}", error)
+    return rendered.replace("{text}", text)

{biblicus-0.14.0.dist-info → biblicus-0.15.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: biblicus
-Version: 0.14.0
+Version: 0.15.1
 Summary: Command line interface and Python library for corpus ingestion, retrieval, and evaluation.
 License: MIT
 Requires-Python: >=3.9
@@ -9,6 +9,8 @@ License-File: LICENSE
 Requires-Dist: pydantic>=2.0
 Requires-Dist: PyYAML>=6.0
 Requires-Dist: pypdf>=4.0
+Requires-Dist: Jinja2>=3.1
+Requires-Dist: dotyaml>=0.1.3
 Provides-Extra: dev
 Requires-Dist: behave>=1.2.6; extra == "dev"
 Requires-Dist: coverage[toml]>=7.0; extra == "dev"
@@ -18,6 +20,9 @@ Requires-Dist: sphinx_rtd_theme>=2.0; extra == "dev"
 Requires-Dist: ruff>=0.4.0; extra == "dev"
 Requires-Dist: black>=24.0; extra == "dev"
 Requires-Dist: python-semantic-release>=9.0.0; extra == "dev"
+Provides-Extra: dspy
+Requires-Dist: dspy>=2.5; extra == "dspy"
+Requires-Dist: litellm>=1.0; extra == "dspy"
 Provides-Extra: openai
 Requires-Dist: openai>=1.0; extra == "openai"
 Provides-Extra: unstructured
@@ -40,6 +45,8 @@ Provides-Extra: docling-mlx
 Requires-Dist: docling[mlx-vlm]>=2.0.0; extra == "docling-mlx"
 Provides-Extra: topic-modeling
 Requires-Dist: bertopic>=0.15.0; extra == "topic-modeling"
+Provides-Extra: markov-analysis
+Requires-Dist: hmmlearn>=0.3.0; extra == "markov-analysis"
 Provides-Extra: datasets
 Requires-Dist: datasets>=2.18.0; extra == "datasets"
 Dynamic: license-file
@@ -50,18 +57,33 @@ Dynamic: license-file
 ![Coverage][coverage-badge]
 ![Documentation][documentation-badge]
 
-Make your documents usable by your assistant, then decide later how you will search and retrieve them.
-
+<p>
+  <img
+    src="docs/_static/Biblicus-logo.png"
+    alt="Biblicus logo"
+    align="right"
+    width="216"
+  />
+  Make your documents usable by your assistant, then decide later how you will search and retrieve them.
+</p>
 If you are building an assistant in Python, you probably have material you want it to use: notes, documents, web pages, and reference files. A common approach is retrieval augmented generation, where a system retrieves relevant material and uses it as evidence when generating a response.
 
 The first practical problem is not retrieval. It is collection and care. You need a stable place to put raw items, you need a small amount of metadata so you can find them again, and you need a way to evolve your retrieval approach over time without rewriting ingestion.
 
-This library gives you a corpus, which is a normal folder on disk. It stores each ingested item as a file, with optional metadata stored next to it. You can open and inspect the raw files directly. Any derived catalog or index can be rebuilt from the raw corpus.
+Biblicus gives you a normal folder on disk to manage. In Biblicus documentation, that managed folder is called a *corpus* (plural: *corpora*). It stores each ingested item as a file, with optional metadata stored next to it. You can open and inspect the raw files directly. Any derived catalog or index can be rebuilt from the raw files.
 
 It can be used alongside LangGraph, Tactus, Pydantic AI, any agent framework, or your own setup. Use it from Python or from the command line interface.
 
 See [retrieval augmented generation overview] for a short introduction to the idea.
 
+## Analysis highlights
+
+- `biblicus analyze markov` learns a directed, weighted state transition graph over segmented text.
+- YAML recipes support cascading composition plus dotted `--config key=value` overrides.
+- Text extract splits long texts with an LLM by inserting XML tags in-place for structured spans.
+- See `docs/MARKOV_ANALYSIS.md` for Markov analysis details and runnable demos.
+- See `docs/TEXT_EXTRACT.md` for the text extract utility and examples.
+
 ## Start with a knowledge base
 
 If you just want to hand a folder to your assistant and move on, use the high-level knowledge base interface. The folder can be nothing more than a handful of plain text files. You are not choosing a retrieval strategy yet. You are just collecting.
@@ -106,7 +128,7 @@ Think in three stages.
 
 If you learn a few project words, the rest of the system becomes predictable.
 
-- Corpus is the folder that holds raw items and their metadata.
+- Corpus is the managed folder that holds raw items and their metadata.
 - Item is the raw bytes plus optional metadata and source information.
 - Catalog is the rebuildable index of the corpus.
 - Extraction run is a recorded extraction build that produces text artifacts.
@@ -161,28 +183,28 @@ sequenceDiagram
 This repository is a working Python package. Install it into a virtual environment from the repository root.
 
 ```
-python3 -m pip install -e .
+python -m pip install -e .
 ```
 
 After the first release, you can install it from Python Package Index.
 
 ```
-python3 -m pip install biblicus
+python -m pip install biblicus
 ```
 
 ### Optional extras
 
 Some extractors are optional so the base install stays small.
 
-- Optical character recognition for images: `python3 -m pip install "biblicus[ocr]"`
-- Advanced optical character recognition with PaddleOCR: `python3 -m pip install "biblicus[paddleocr]"`
-- Document understanding with Docling VLM: `python3 -m pip install "biblicus[docling]"`
-- Document understanding with Docling VLM and MLX acceleration: `python3 -m pip install "biblicus[docling-mlx]"`
-- Speech to text transcription with OpenAI: `python3 -m pip install "biblicus[openai]"` (requires an OpenAI API key in `~/.biblicus/config.yml` or `./.biblicus/config.yml`)
-- Speech to text transcription with Deepgram: `python3 -m pip install "biblicus[deepgram]"` (requires a Deepgram API key in `~/.biblicus/config.yml` or `./.biblicus/config.yml`)
-- Broad document parsing fallback: `python3 -m pip install "biblicus[unstructured]"`
-- MarkItDown document conversion (requires Python 3.10 or higher): `python3 -m pip install "biblicus[markitdown]"`
-- Topic modeling analysis with BERTopic: `python3 -m pip install "biblicus[topic-modeling]"`
+- Optical character recognition for images: `python -m pip install "biblicus[ocr]"`
+- Advanced optical character recognition with PaddleOCR: `python -m pip install "biblicus[paddleocr]"`
+- Document understanding with Docling VLM: `python -m pip install "biblicus[docling]"`
+- Document understanding with Docling VLM and MLX acceleration: `python -m pip install "biblicus[docling-mlx]"`
+- Speech to text transcription with OpenAI: `python -m pip install "biblicus[openai]"` (requires an OpenAI API key in `~/.biblicus/config.yml` or `./.biblicus/config.yml`)
+- Speech to text transcription with Deepgram: `python -m pip install "biblicus[deepgram]"` (requires a Deepgram API key in `~/.biblicus/config.yml` or `./.biblicus/config.yml`)
+- Broad document parsing fallback: `python -m pip install "biblicus[unstructured]"`
+- MarkItDown document conversion (requires Python 3.10 or higher): `python -m pip install "biblicus[markitdown]"`
+- Topic modeling analysis with BERTopic: `python -m pip install "biblicus[topic-modeling]"`
 
 ## Quick start
 
@@ -200,16 +222,49 @@ biblicus build --corpus corpora/example --backend scan
 biblicus query --corpus corpora/example --query "note"
 ```
 
-If you want to turn a website section into corpus items, crawl a root web address while restricting the crawl to an allowed prefix:
+## Web Ingestion
+
+Biblicus supports ingesting content directly from the web using two approaches.
+
+### Ingest from URLs
 
+Ingest individual documents or web pages from URLs. The `ingest` command automatically detects content types including PDF, HTML, Markdown, images, and audio:
+
+```bash
+# Ingest a document from a URL
+biblicus ingest https://example.com/document.pdf --tags "research"
+
+# Ingest a web page
+biblicus ingest https://example.com/article.html --tags "article"
+
+# Ingest with a corpus path specified
+biblicus ingest --corpus corpora/example https://docs.example.com/guide.md --tags "documentation"
 ```
-biblicus crawl --corpus corpora/example \\
-  --root-url https://example.com/docs/index.html \\
-  --allowed-prefix https://example.com/docs/ \\
-  --max-items 50 \\
-  --tag crawled
+
+### Crawl Websites
+
+Crawl entire website sections with automatic link discovery. The crawler follows links within the allowed prefix and stores discovered content:
+
+```bash
+# Crawl a documentation site
+biblicus crawl \
+  --corpus corpora/example \
+  --root-url https://docs.example.com/ \
+  --allowed-prefix https://docs.example.com/ \
+  --max-items 100 \
+  --tags "documentation"
+
+# Crawl a specific blog category
+biblicus crawl \
+  --corpus corpora/example \
+  --root-url https://blog.example.com/category/tutorials/ \
+  --allowed-prefix https://blog.example.com/category/tutorials/ \
+  --max-items 50 \
+  --tags "tutorials,blog"
 ```
 
+The `--allowed-prefix` parameter restricts the crawler to only follow links that start with the specified URL prefix, preventing it from crawling outside the intended scope. The crawler respects `.biblicusignore` rules and stores items under `raw/imports/crawl/` in your corpus.
+
 ## End-to-end example: lower-level control
 
 The command-line interface returns JavaScript Object Notation by default. This makes it easy to use Biblicus in scripts and to treat retrieval as a deterministic, testable step.
@@ -490,7 +545,7 @@ Three backends are included.
 
 - `scan` is a minimal baseline that scans raw items directly.
 - `sqlite-full-text-search` is a practical baseline that builds a full text search index in SQLite.
-- `vector` is a deterministic term-frequency vector baseline with cosine similarity scoring.
+- `tf-vector` is a deterministic term-frequency vector baseline with cosine similarity scoring.
 
 For detailed documentation including configuration options, performance characteristics, and usage examples, see the [Backend Reference][backend-reference].
 
@@ -540,6 +595,21 @@ For detailed documentation on all extractors, see the [Extractor Reference][extr
 For extraction evaluation workflows, dataset formats, and report interpretation, see
 `docs/EXTRACTION_EVALUATION.md`.
 
+## Text extract utility
+
+Text extract is a reusable analysis utility that lets a model insert XML tags into a long text without re-emitting the
+entire document. It returns structured spans and the marked-up text, and it is used as a segmentation option in Markov
+analysis.
+
+See `docs/TEXT_EXTRACT.md` for the utility API and examples, and `docs/MARKOV_ANALYSIS.md` for the Markov integration.
+
+## Text slice utility
+
+Text slice is a reusable analysis utility that lets a model insert `<slice/>` markers into a long text without
+re-emitting the entire document. It returns ordered slices and the marked-up text for auditing and reuse.
+
+See `docs/TEXT_SLICE.md` for the utility API and examples.
+
 ## Topic modeling analysis
 
 Biblicus can run analysis pipelines on extracted text without changing the raw corpus. Profiling and topic modeling
@@ -594,7 +664,7 @@ AG News integration runs require `biblicus[datasets]` in addition to `biblicus[t
 For a repeatable, real-world integration run that downloads AG News and executes topic modeling, use:
 
 ```
-python3 scripts/topic_modeling_integration.py --corpus corpora/ag_news_demo --force
+python scripts/topic_modeling_integration.py --corpus corpora/ag_news_demo --force
 ```
 
 See `docs/TOPIC_MODELING.md` for parameter examples and per-topic output behavior.
@@ -608,13 +678,13 @@ Use `scripts/download_pdf_samples.py` to download a small Portable Document Form
 ## Tests and coverage
 
 ```
-python3 scripts/test.py
+python scripts/test.py
 ```
 
 To include integration scenarios that download public test data at runtime, run this command.
 
 ```
-python3 scripts/test.py --integration
+python scripts/test.py --integration
 ```
 
 ## Releases
@@ -632,13 +702,13 @@ Reference documentation is generated from Sphinx style docstrings.
 Install development dependencies:
 
 ```
-python3 -m pip install -e ".[dev]"
+python -m pip install -e ".[dev]"
 ```
 
 Build the documentation:
 
 ```
-python3 -m sphinx -b html docs docs/_build/html
+python -m sphinx -b html docs docs/_build/html
 ```
 
 ## License

{biblicus-0.14.0.dist-info → biblicus-0.15.1.dist-info}/RECORD

@@ -1,6 +1,6 @@
-biblicus/__init__.py,sha256=S14cBWrRwFcNLNmhl2viTvy2CXIfK8-vcBixsh5y49U,496
+biblicus/__init__.py,sha256=X0FEFK03CEs4FUqCxWrZmSV1tPd6p5IEd3GgelqqXqk,496
 biblicus/__main__.py,sha256=ipfkUoTlocVnrQDM69C7TeBqQxmHVeiWMRaT3G9rtnk,117
-biblicus/cli.py,sha256=cMoirLFPhTwftNuqaadajCcRUEz_FBaLkupjVxpAxO8,38403
+biblicus/cli.py,sha256=APxBNLztfGTVGgxt5qgCofYti2AsGGeZQ7SB9tkneXQ,41760
 biblicus/constants.py,sha256=gAlEVJhxdFj-eWWJrlYbP7H1X3c5gwhrIBq9NQ1Vq_E,371
 biblicus/context.py,sha256=iXRFGpf_5YDPsDsm_iTK6nCvtUWDoYVI7op-l2QU3uA,10189
 biblicus/corpus.py,sha256=qSDnYJXhWlF2p_BbFLl6xtI53lIIPxwyKLLGLC432Sg,55612
@@ -18,27 +18,32 @@ biblicus/ignore.py,sha256=fyjt34E6tWNNrm1FseOhgH2MgryyVBQVzxhKL5s4aio,1800
 biblicus/inference.py,sha256=_k00AIPoXD2lruiTB-JUagtY4f_WKcdzA3axwiq1tck,3512
 biblicus/knowledge_base.py,sha256=JmlJw8WD_fgstuq1PyWVzU9kzvVzyv7_xOvhS70xwUw,6654
 biblicus/models.py,sha256=r28O6cg3d1bjJnKqpLieVLTgtXTfzb_60wMORvVuDN0,15846
+biblicus/recipes.py,sha256=rqU66QnjOup6O8Y9Yq7XszmpoM0Pyrjw3RrfdnlVqgE,4210
 biblicus/retrieval.py,sha256=A1SI4WK5cX-WbtN6FJ0QQxqlEOtQhddLrL0LZIuoTC4,4180
 biblicus/sources.py,sha256=EFy8-rQNLsyzz-98mH-z8gEHMYbqigcNFKLaR92KfDE,7241
 biblicus/time.py,sha256=3BSKOSo7R10K-0Dzrbdtl3fh5_yShTYqfdlKvvdkx7M,485
 biblicus/uris.py,sha256=xXD77lqsT9NxbyzI1spX9Y5a3-U6sLYMnpeSAV7g-nM,2013
 biblicus/user_config.py,sha256=UXUYBNUN4FR37ggZGJG1wv3K8XzsMR8pXW1T18lrivw,6495
-biblicus/_vendor/dotyaml/__init__.py,sha256=WAWdbFNFqO5cJPthxA8Kx-L76Bh07sKMosUxC_3o9qA,375
+biblicus/_vendor/dotyaml/__init__.py,sha256=OVv6IsuCvsjaUznLzuit4UbSLVg4TiTVm9cOPY1Y2Cs,409
 biblicus/_vendor/dotyaml/interpolation.py,sha256=FVUkdQr_KbXjoFPvGTv6I5v0X5iZkJe5yhZtYKRbYzI,1991
-biblicus/_vendor/dotyaml/loader.py,sha256=zy_zinR5fiatmRyZSiELHv1vVz1Y2eRSboSf_x3kfi4,5623
+biblicus/_vendor/dotyaml/loader.py,sha256=vFfnhbvHYYyOKzl5iq2FH97GSHH2GvEHmGiPnE0g0kA,6954
 biblicus/_vendor/dotyaml/transformer.py,sha256=RWNrm_KAsanG409HEIWquTH9i_jz-ZFK9fM86emXeF4,3724
-biblicus/analysis/__init__.py,sha256=Z4Wb4d-EoUuGHkcfRm9ILuZ8vr9FBqRxC0u1i6Fp_0w,1288
+biblicus/ai/__init__.py,sha256=HY8PKhqRLIDYJYlL9A2JjqKxQaujITNLYgIytNUhnrU,1161
+biblicus/ai/embeddings.py,sha256=n2xlonZOHcmDrP1XMhGcja5Hzr8r87PF-IecH-Yhu98,3703
+biblicus/ai/llm.py,sha256=g724_UAxmicB_W-Z7Uu9SRsI9-aVNZUlYIjvnlE17VE,4712
+biblicus/ai/models.py,sha256=6newnT0NJf3uf9FvWXVC-9Gkk5xRB-PjXDZpeBHA04Y,7857
+biblicus/analysis/__init__.py,sha256=I4LqxfKPKF3DEVmAyagQ8J1RN-ia3fyfKJ9frCllZQE,1385
 biblicus/analysis/base.py,sha256=gB4ilvyMpiWU1m_ydy2dIHGP96ZFIFvVUL9iVDZKPJM,1265
-biblicus/analysis/llm.py,sha256=VjkZDKauHCDfj-TP-bTbI6a9WAXEIDe8bEiwErPx9xc,3309
-biblicus/analysis/models.py,sha256=LuR52w27JRzV-Mr-WAOduZrBOCTrp5uYkMc46QHTRrI,27300
+biblicus/analysis/markov.py,sha256=XiNbfg8lmKWXCzQJhZtZxghT7gBPh2o6xA3D1-tMQqs,61762
+biblicus/analysis/models.py,sha256=FnUAO6n1yjagYlhe2ocRtfb3IZT3W03rrEZ9LdCx7Kc,56214
 biblicus/analysis/profiling.py,sha256=v2B4Tn9WiXRRP_wIADBPRQVKkMc92KXCas7OBa7n0LU,10670
 biblicus/analysis/schema.py,sha256=MCiAQJmijVk8iM8rOUYbzyaDwsMR-Oo86iZU5NCbDMM,435
-biblicus/analysis/topic_modeling.py,sha256=ZGXvm2MyU6plxz2FE1RQU-3bra6QZ-t8EJj8kG1TW0M,19438
+biblicus/analysis/topic_modeling.py,sha256=mNBiRMpY5Jtyz8Aj-WXYY8guEghx9jozTfgveinJLoc,22135
 biblicus/backends/__init__.py,sha256=3HJY0oMm8pFFVGC4Z-dlPRHhIPVDdUzsa4IMjKP_9dI,1378
 biblicus/backends/base.py,sha256=Erfj9dXg0nkRKnEcNjHR9_0Ddb2B1NvbmRksVm_g1dU,1776
 biblicus/backends/hybrid.py,sha256=FjCzrXdicbM-qVDM-_AHYTT8Gzvem1eZJaQUVKk_tMg,10753
 biblicus/backends/scan.py,sha256=hdNnQWqi5IH6j95w30BZHxLJ0W9PTaOkqfWJuxCCEMI,12478
-biblicus/backends/sqlite_full_text_search.py,sha256=VAn4fDdfiaS1Rn6zHlYz3E10_3vMU9P94QU8cL0l8Mk,24466
+biblicus/backends/sqlite_full_text_search.py,sha256=z0WsURTj4w8Y6SPa6k4bV9wdze-HwNWgxj5qcab0TUE,24480
 biblicus/backends/vector.py,sha256=3RdxSBPb1kOX4Sfd4d1qXFW9ecuiRvGpOHadLCbeh1g,15183
 biblicus/extractors/__init__.py,sha256=ci3oldbdQZ8meAfHccM48CqQtZsPSRg3HkPrBSZF15M,2673
 biblicus/extractors/base.py,sha256=ka-nz_1zHPr4TS9sU4JfOoY-PJh7lbHPBOEBrbQFGSc,2171
@@ -58,9 +63,19 @@ biblicus/extractors/select_override.py,sha256=gSpffFmn1ux9pGtFvHD5Uu_LO8TmmJC4L_
 biblicus/extractors/select_smart_override.py,sha256=-sLMnNoeXbCB3dO9zflQq324eHuLbd6hpveSwduXP-U,6763
 biblicus/extractors/select_text.py,sha256=w0ATmDy3tWWbOObzW87jGZuHbgXllUhotX5XyySLs-o,3395
 biblicus/extractors/unstructured_text.py,sha256=l2S_wD_htu7ZHoJQNQtP-kGlEgOeKV_w2IzAC93lePE,3564
-biblicus-0.14.0.dist-info/licenses/LICENSE,sha256=lw44GXFG_Q0fS8m5VoEvv_xtdBXK26pBcbSPUCXee_Q,1078
-biblicus-0.14.0.dist-info/METADATA,sha256=caVN-IcUlwaRRdid40XN6FBcdJmFkMBldyW-ySf-6pk,28088
-biblicus-0.14.0.dist-info/WHEEL,sha256=wUyA8OaulRlbfwMtmQsvNngGrxQHAvkKcvRmdizlJi0,92
-biblicus-0.14.0.dist-info/entry_points.txt,sha256=BZmO4H8Uz00fyi1RAFryOCGfZgX7eHWkY2NE-G54U5A,47
-biblicus-0.14.0.dist-info/top_level.txt,sha256=sUD_XVZwDxZ29-FBv1MknTGh4mgDXznGuP28KJY_WKc,9
-biblicus-0.14.0.dist-info/RECORD,,
+biblicus/text/__init__.py,sha256=MiaGAY7xWlUCeBzDzNz6pJnSMiU_Ge5EmlSiEzhqTRo,947
+biblicus/text/annotate.py,sha256=asmpj3_s_t8hl6stEg99apmqxAhDTkoPzHhZNggYE3Y,8355
+biblicus/text/extract.py,sha256=pdnUiZWtfCUj7kZK5zhd-tjqokgmhYYheWhyN3iShRU,7669
+biblicus/text/link.py,sha256=Xl0yxD1rvbRJRVdWG_ZP6zgmbpgWSJYcUcNM06-OUWU,20077
+biblicus/text/markup.py,sha256=8jj9aX03HiZTOWdPs_VC4JLpQ7TlPHgGuXj_QUQIHVw,6265
+biblicus/text/models.py,sha256=REp6RowUWFdV-6y437JENP7XtGKt57BOvVtF91KmUqI,10853
+biblicus/text/prompts.py,sha256=Z5fSsy1Xzr0rCI0WZ3djiEQlbRDncyNBQ7_ZoWVPL4g,6704
+biblicus/text/redact.py,sha256=tkDRmA0VvOZwMryEmBPLEHf3Z6VHJkkaWjBaNIMyGZ0,8415
+biblicus/text/slice.py,sha256=dlHxGO8c5P8BszXGwlNQoQ-cyWjJf6PfS1LUBJXXGEE,5762
+biblicus/text/tool_loop.py,sha256=w1PGLBvIemOdi6l0ArdYDVL7zgx-RC76bBOO0PKqpt0,11831
+biblicus-0.15.1.dist-info/licenses/LICENSE,sha256=lw44GXFG_Q0fS8m5VoEvv_xtdBXK26pBcbSPUCXee_Q,1078
+biblicus-0.15.1.dist-info/METADATA,sha256=TujS_sOnPhf7nGhulkFJJh50nskTWtjcNLv-E5LLles,30906
+biblicus-0.15.1.dist-info/WHEEL,sha256=wUyA8OaulRlbfwMtmQsvNngGrxQHAvkKcvRmdizlJi0,92
+biblicus-0.15.1.dist-info/entry_points.txt,sha256=BZmO4H8Uz00fyi1RAFryOCGfZgX7eHWkY2NE-G54U5A,47
+biblicus-0.15.1.dist-info/top_level.txt,sha256=sUD_XVZwDxZ29-FBv1MknTGh4mgDXznGuP28KJY_WKc,9
+biblicus-0.15.1.dist-info/RECORD,,