docpull 4.2.0__tar.gz → 4.3.1__tar.gz

{docpull-4.2.0/src/docpull.egg-info → docpull-4.3.1}/PKG-INFO

@@ -1,17 +1,21 @@
 Metadata-Version: 2.4
 Name: docpull
-Version: 4.2.0
-Summary: Pull documentation from the web and convert to clean markdown
+Version: 4.3.1
+Summary: Convert public web pages into clean Markdown for AI agents and RAG
 Author-email: Zachary Roth <support@raintree.technology>
 Maintainer-email: Raintree Technology <support@raintree.technology>
 License-Expression: MIT
-Project-URL: Homepage, https://github.com/raintree-technology/docpull
-Project-URL: Documentation, https://github.com/raintree-technology/docpull#readme
+Project-URL: Homepage, https://docpull.raintree.technology
+Project-URL: Documentation, https://docpull.raintree.technology
 Project-URL: Repository, https://github.com/raintree-technology/docpull
 Project-URL: Source Code, https://github.com/raintree-technology/docpull
 Project-URL: Bug Tracker, https://github.com/raintree-technology/docpull/issues
 Project-URL: Releases, https://github.com/raintree-technology/docpull/releases
-Keywords: python,markdown,documentation,web-scraping,developer-tools,claude,ai-training-data
+Project-URL: MCP Plugin, https://github.com/raintree-technology/docpull/tree/main/plugin
+Project-URL: Comparison Guide, https://github.com/raintree-technology/docpull/blob/main/docs/alternatives.md
+Project-URL: Download Stats, https://pepy.tech/project/docpull
+Project-URL: Changelog, https://github.com/raintree-technology/docpull/blob/main/docs/CHANGELOG.md
+Keywords: python,markdown,documentation,web-scraping,web-crawler,crawler,developer-tools,cli,rag,llm,mcp,ai-agents,claude,ai-training-data
 Classifier: Development Status :: 5 - Production/Stable
 Classifier: Intended Audience :: Developers
 Classifier: Intended Audience :: Information Technology
@@ -96,11 +100,12 @@ Dynamic: license-file
 
 # docpull
 
-**Security-hardened, browser-free web scraper and crawler that turns server-rendered pages into clean, AI-ready Markdown — fast.**
+**Public web to agent-ready Markdown. Fast, local, browser-free.**
 
 [![Python 3.10+](https://img.shields.io/badge/python-3.10+-blue.svg)](https://www.python.org/downloads/)
 [![PyPI version](https://badge.fury.io/py/docpull.svg)](https://badge.fury.io/py/docpull)
-[![Downloads](https://pepy.tech/badge/docpull)](https://pepy.tech/project/docpull)
+[![PyPI downloads](https://img.shields.io/pepy/dt/docpull?label=downloads)](https://pepy.tech/project/docpull)
+[![GitHub stars](https://img.shields.io/github/stars/raintree-technology/docpull?style=social)](https://github.com/raintree-technology/docpull/stargazers)
 [![License: MIT](https://img.shields.io/github/license/raintree-technology/docpull)](https://github.com/raintree-technology/docpull/blob/main/LICENSE)
 
 <p align="center">
@@ -109,16 +114,35 @@ Dynamic: license-file
   </a>
 </p>
 
-docpull is a web scraper for static and server-rendered sites, with
-documentation crawling as its sharpest default workflow. It uses async HTTP (not
-Playwright) to fetch pages, discover links, extract main content, and write clean
-Markdown with source-URL frontmatter — in seconds, with a small install
-footprint. It won't render JavaScript, but for the large class of sites that
-don't need it (API references, vendor docs, Python/Go stdlib, blogs, OpenAPI
-specs, Next.js and Docusaurus builds), it is a fast, auditable,
-sandbox-friendly way to pipe web content into an LLM context, a RAG index, or an
-offline archive. SSRF, XXE, DNS-rebinding, and CRLF-injection protections are on
-by default — a necessity when an AI agent is choosing the URLs.
+docpull is a Python CLI, SDK, and MCP server for pulling public static and
+server-rendered web pages into clean Markdown, NDJSON, OKF bundles, SQLite, or
+local archives. Documentation crawling is its sharpest default workflow, but
+the product is broader than docs: blogs, API references, vendor pages, OpenAPI
+specs, and other server-rendered web content all fit. It uses async HTTP instead
+of Playwright, preserves source metadata, and is built for agent-selected URLs
+with SSRF, XXE, DNS-rebinding, and CRLF-injection protections enabled by
+default.
+
+Use docpull when you need to:
+
+- Load public web pages, docs, API references, or blogs into an LLM context.
+- Build inspectable source corpora for RAG/search pipelines.
+- Give a coding agent a local MCP tool for fetching, caching, grepping, and
+  reading docs.
+- Mirror public web content for offline work while keeping source attribution.
+
+docpull intentionally does not render JavaScript. For JS-only apps, browser
+automation, paid extraction APIs, or general web crawling, see
+[`docs/scraping-boundary.md`](docs/scraping-boundary.md) and
+[`docs/alternatives.md`](docs/alternatives.md).
+
+## Try it in 60 seconds
+
+```bash
+pip install docpull
+docpull https://docs.python.org/3/library/asyncio.html --single
+docpull https://docs.python.org/3/library/asyncio.html --profile llm --stream | jq .
+```
 
 ## Install
 
@@ -146,10 +170,27 @@ docpull https://docs.example.com/guide --single
 # LLM-ready NDJSON with 4k-token chunks streamed to stdout
 docpull https://docs.example.com --profile llm --stream | jq .
 
+# Open Knowledge Format bundle for agent/wiki interoperability
+docpull https://example.com --format okf -o ./site-okf
+
 # Mirror scraped content for offline use
 docpull https://docs.example.com --profile mirror --cache
 ```
 
+## Project traction
+
+<a href="https://pepy.tech/project/docpull">
+  <img alt="Cumulative PyPI download history chart for docpull" src="docs/cumulative-downloads-history.svg" />
+</a>
+
+<a href="https://star-history.com/#raintree-technology/docpull&Date">
+  <picture>
+    <source media="(prefers-color-scheme: dark)" srcset="https://api.star-history.com/svg?repos=raintree-technology/docpull&type=Date&theme=dark" />
+    <source media="(prefers-color-scheme: light)" srcset="https://api.star-history.com/svg?repos=raintree-technology/docpull&type=Date" />
+    <img alt="Star history chart for raintree-technology/docpull" src="https://api.star-history.com/svg?repos=raintree-technology/docpull&type=Date" />
+  </picture>
+</a>
+
 ## Framework-aware extraction
 
 docpull inspects each page before running the generic extractor and can pull
@@ -160,8 +201,13 @@ content directly from framework data feeds:
 | Next.js   | Parses `__NEXT_DATA__` JSON |
 | Mintlify  | `__NEXT_DATA__` with Mintlify tagging |
 | OpenAPI   | Renders `openapi.json` / `swagger.json` into Markdown |
-| Docusaurus| Detected and tagged; generic extractor produces Markdown |
-| Sphinx    | Detected from generator metadata / Read the Docs hosts and tagged; generic extractor produces Markdown |
+| Docusaurus| Extracts static article regions and tags framework metadata |
+| Sphinx    | Extracts static body/document regions and tags framework metadata |
+| MkDocs / Material | Extracts static content regions and tags framework metadata |
+| VitePress / VuePress | Extracts static doc regions and tags framework metadata |
+| Astro Starlight | Extracts static markdown content regions and tags framework metadata |
+| GitBook / ReadMe.io | Extracts static article/content regions and tags framework metadata |
+| Redoc / Scalar | Extracts static API reference regions; OpenAPI JSON is still preferred when available |
 
 JS-only SPAs with no server-rendered content are detected and skipped with a
 clear reason (or, with `--strict-js-required`, reported as an error so agents
@@ -181,6 +227,30 @@ can route elsewhere).
 
 ## Python API
 
+Scraper-facing API:
+
+```python
+import asyncio
+from docpull import scrape_one, Scraper
+
+async def main():
+    page = await scrape_one("https://docs.python.org/3/library/asyncio.html")
+    print(page.title, page.source_type)
+    print(page.text[:500])
+
+    scraper = Scraper()
+    run = await scraper.scrape_site(
+        "https://docs.example.com",
+        output_format="ndjson",
+        max_pages=100,
+    )
+    print(run.stats.pages_fetched, run.manifest_path)
+
+asyncio.run(main())
+```
+
+Core fetcher API:
+
 ```python
 from docpull import fetch_one
 
@@ -225,6 +295,7 @@ async def tool_call(url: str) -> str:
 ```bash
 docpull https://site.com --profile rag      # Default. Dedup, rich metadata.
 docpull https://site.com --profile llm      # NDJSON + chunks + metadata; JS-only pages skip unless --strict-js-required is passed.
+docpull https://site.com --profile okf      # OKF bundle with generated index.md files.
 docpull https://site.com --profile mirror   # Full archive, polite, cached, hierarchical paths.
 docpull https://site.com --profile quick    # Sampling: 50 pages, depth 2.
 ```
@@ -506,10 +577,57 @@ NDJSON (one record per page or chunk):
 {"document_id": "doc_...", "chunk_id": "chunk_...", "url": "...", "title": "...", "content": "...", "hash": "...", "token_count": 842, "chunk_index": 0}
 ```
 
+Open Knowledge Format:
+
+```bash
+docpull https://example.com --format okf -o ./site-okf
+
+# Equivalent profile form
+docpull https://example.com --profile okf -o ./site-okf
+```
+
+A generated OKF bundle has a normal Markdown tree:
+
+```text
+site-okf/
+  index.md
+  corpus.manifest.json
+  _root.md
+  docs/
+    index.md
+    getting-started.md
+```
+
+OKF output is an opt-in Markdown bundle. Scraped pages are written as concept
+documents with OKF frontmatter (`type`, `title`, `description`, `resource`,
+`tags`, `timestamp` when source metadata exists) and docpull keeps `source` as a
+compatibility extension. Generated `index.md` files are reserved for directory
+listings, so URL landing pages use safe concept filenames such as `_root.md` or
+`_page.md` instead of occupying `index.md`. The root `index.md` declares
+`okf_version: "0.1"`; nested indexes stay plain listing files.
+
+Because OKF treats every non-reserved `.md` file in the tree as a concept, write
+OKF output to a clean or dedicated directory rather than mixing it with unrelated
+Markdown files.
+
 Every output format also writes `corpus.manifest.json` next to the generated
 documents. The manifest records the run identity, output format, stable
-`document_id` / `chunk_id` values, content hashes, relative output paths, and
-chunk counts so regenerated corpora can be diffed and cited by agents.
+`document_id` / `chunk_id` values, content hashes, file-backed relative output
+paths (or the stdout marker for streamed NDJSON), and chunk counts so
+regenerated corpora can be diffed and cited by agents. See
+[`docs/corpus-manifest.md`](docs/corpus-manifest.md) for the schema and
+stability contract.
+
+SQLite output includes an FTS5 index for local retrieval:
+
+```python
+from pathlib import Path
+from docpull import search_sqlite_documents
+
+hits = search_sqlite_documents(Path("docs/documents.db"), "rate limit")
+for hit in hits:
+    print(hit.url, hit.snippet)
+```
 
 ## Security
 
@@ -530,9 +648,9 @@ Run `docpull --help` for the full list. Highlights:
 
 ```
 Core:
-  --profile {rag,mirror,quick,llm}
+  --profile {rag,mirror,quick,llm,okf}
   --single                Fetch one URL (no crawl)
-  --format {markdown,json,ndjson,sqlite}
+  --format {markdown,json,ndjson,sqlite,okf}
   --stream                Stream NDJSON to stdout
 
 LLM / chunking:

{docpull-4.2.0 → docpull-4.3.1}/README.md

@@ -1,10 +1,11 @@
 # docpull
 
-**Security-hardened, browser-free web scraper and crawler that turns server-rendered pages into clean, AI-ready Markdown — fast.**
+**Public web to agent-ready Markdown. Fast, local, browser-free.**
 
 [![Python 3.10+](https://img.shields.io/badge/python-3.10+-blue.svg)](https://www.python.org/downloads/)
 [![PyPI version](https://badge.fury.io/py/docpull.svg)](https://badge.fury.io/py/docpull)
-[![Downloads](https://pepy.tech/badge/docpull)](https://pepy.tech/project/docpull)
+[![PyPI downloads](https://img.shields.io/pepy/dt/docpull?label=downloads)](https://pepy.tech/project/docpull)
+[![GitHub stars](https://img.shields.io/github/stars/raintree-technology/docpull?style=social)](https://github.com/raintree-technology/docpull/stargazers)
 [![License: MIT](https://img.shields.io/github/license/raintree-technology/docpull)](https://github.com/raintree-technology/docpull/blob/main/LICENSE)
 
 <p align="center">
@@ -13,16 +14,35 @@
   </a>
 </p>
 
-docpull is a web scraper for static and server-rendered sites, with
-documentation crawling as its sharpest default workflow. It uses async HTTP (not
-Playwright) to fetch pages, discover links, extract main content, and write clean
-Markdown with source-URL frontmatter — in seconds, with a small install
-footprint. It won't render JavaScript, but for the large class of sites that
-don't need it (API references, vendor docs, Python/Go stdlib, blogs, OpenAPI
-specs, Next.js and Docusaurus builds), it is a fast, auditable,
-sandbox-friendly way to pipe web content into an LLM context, a RAG index, or an
-offline archive. SSRF, XXE, DNS-rebinding, and CRLF-injection protections are on
-by default — a necessity when an AI agent is choosing the URLs.
+docpull is a Python CLI, SDK, and MCP server for pulling public static and
+server-rendered web pages into clean Markdown, NDJSON, OKF bundles, SQLite, or
+local archives. Documentation crawling is its sharpest default workflow, but
+the product is broader than docs: blogs, API references, vendor pages, OpenAPI
+specs, and other server-rendered web content all fit. It uses async HTTP instead
+of Playwright, preserves source metadata, and is built for agent-selected URLs
+with SSRF, XXE, DNS-rebinding, and CRLF-injection protections enabled by
+default.
+
+Use docpull when you need to:
+
+- Load public web pages, docs, API references, or blogs into an LLM context.
+- Build inspectable source corpora for RAG/search pipelines.
+- Give a coding agent a local MCP tool for fetching, caching, grepping, and
+  reading docs.
+- Mirror public web content for offline work while keeping source attribution.
+
+docpull intentionally does not render JavaScript. For JS-only apps, browser
+automation, paid extraction APIs, or general web crawling, see
+[`docs/scraping-boundary.md`](docs/scraping-boundary.md) and
+[`docs/alternatives.md`](docs/alternatives.md).
+
+## Try it in 60 seconds
+
+```bash
+pip install docpull
+docpull https://docs.python.org/3/library/asyncio.html --single
+docpull https://docs.python.org/3/library/asyncio.html --profile llm --stream | jq .
+```
 
 ## Install
 
@@ -50,10 +70,27 @@ docpull https://docs.example.com/guide --single
 # LLM-ready NDJSON with 4k-token chunks streamed to stdout
 docpull https://docs.example.com --profile llm --stream | jq .
 
+# Open Knowledge Format bundle for agent/wiki interoperability
+docpull https://example.com --format okf -o ./site-okf
+
 # Mirror scraped content for offline use
 docpull https://docs.example.com --profile mirror --cache
 ```
 
+## Project traction
+
+<a href="https://pepy.tech/project/docpull">
+  <img alt="Cumulative PyPI download history chart for docpull" src="docs/cumulative-downloads-history.svg" />
+</a>
+
+<a href="https://star-history.com/#raintree-technology/docpull&Date">
+  <picture>
+    <source media="(prefers-color-scheme: dark)" srcset="https://api.star-history.com/svg?repos=raintree-technology/docpull&type=Date&theme=dark" />
+    <source media="(prefers-color-scheme: light)" srcset="https://api.star-history.com/svg?repos=raintree-technology/docpull&type=Date" />
+    <img alt="Star history chart for raintree-technology/docpull" src="https://api.star-history.com/svg?repos=raintree-technology/docpull&type=Date" />
+  </picture>
+</a>
+
 ## Framework-aware extraction
 
 docpull inspects each page before running the generic extractor and can pull
@@ -64,8 +101,13 @@ content directly from framework data feeds:
 | Next.js   | Parses `__NEXT_DATA__` JSON |
 | Mintlify  | `__NEXT_DATA__` with Mintlify tagging |
 | OpenAPI   | Renders `openapi.json` / `swagger.json` into Markdown |
-| Docusaurus| Detected and tagged; generic extractor produces Markdown |
-| Sphinx    | Detected from generator metadata / Read the Docs hosts and tagged; generic extractor produces Markdown |
+| Docusaurus| Extracts static article regions and tags framework metadata |
+| Sphinx    | Extracts static body/document regions and tags framework metadata |
+| MkDocs / Material | Extracts static content regions and tags framework metadata |
+| VitePress / VuePress | Extracts static doc regions and tags framework metadata |
+| Astro Starlight | Extracts static markdown content regions and tags framework metadata |
+| GitBook / ReadMe.io | Extracts static article/content regions and tags framework metadata |
+| Redoc / Scalar | Extracts static API reference regions; OpenAPI JSON is still preferred when available |
 
 JS-only SPAs with no server-rendered content are detected and skipped with a
 clear reason (or, with `--strict-js-required`, reported as an error so agents
@@ -85,6 +127,30 @@ can route elsewhere).
 
 ## Python API
 
+Scraper-facing API:
+
+```python
+import asyncio
+from docpull import scrape_one, Scraper
+
+async def main():
+    page = await scrape_one("https://docs.python.org/3/library/asyncio.html")
+    print(page.title, page.source_type)
+    print(page.text[:500])
+
+    scraper = Scraper()
+    run = await scraper.scrape_site(
+        "https://docs.example.com",
+        output_format="ndjson",
+        max_pages=100,
+    )
+    print(run.stats.pages_fetched, run.manifest_path)
+
+asyncio.run(main())
+```
+
+Core fetcher API:
+
 ```python
 from docpull import fetch_one
 
@@ -129,6 +195,7 @@ async def tool_call(url: str) -> str:
 ```bash
 docpull https://site.com --profile rag      # Default. Dedup, rich metadata.
 docpull https://site.com --profile llm      # NDJSON + chunks + metadata; JS-only pages skip unless --strict-js-required is passed.
+docpull https://site.com --profile okf      # OKF bundle with generated index.md files.
 docpull https://site.com --profile mirror   # Full archive, polite, cached, hierarchical paths.
 docpull https://site.com --profile quick    # Sampling: 50 pages, depth 2.
 ```
@@ -410,10 +477,57 @@ NDJSON (one record per page or chunk):
 {"document_id": "doc_...", "chunk_id": "chunk_...", "url": "...", "title": "...", "content": "...", "hash": "...", "token_count": 842, "chunk_index": 0}
 ```
 
+Open Knowledge Format:
+
+```bash
+docpull https://example.com --format okf -o ./site-okf
+
+# Equivalent profile form
+docpull https://example.com --profile okf -o ./site-okf
+```
+
+A generated OKF bundle has a normal Markdown tree:
+
+```text
+site-okf/
+  index.md
+  corpus.manifest.json
+  _root.md
+  docs/
+    index.md
+    getting-started.md
+```
+
+OKF output is an opt-in Markdown bundle. Scraped pages are written as concept
+documents with OKF frontmatter (`type`, `title`, `description`, `resource`,
+`tags`, `timestamp` when source metadata exists) and docpull keeps `source` as a
+compatibility extension. Generated `index.md` files are reserved for directory
+listings, so URL landing pages use safe concept filenames such as `_root.md` or
+`_page.md` instead of occupying `index.md`. The root `index.md` declares
+`okf_version: "0.1"`; nested indexes stay plain listing files.
+
+Because OKF treats every non-reserved `.md` file in the tree as a concept, write
+OKF output to a clean or dedicated directory rather than mixing it with unrelated
+Markdown files.
+
 Every output format also writes `corpus.manifest.json` next to the generated
 documents. The manifest records the run identity, output format, stable
-`document_id` / `chunk_id` values, content hashes, relative output paths, and
-chunk counts so regenerated corpora can be diffed and cited by agents.
+`document_id` / `chunk_id` values, content hashes, file-backed relative output
+paths (or the stdout marker for streamed NDJSON), and chunk counts so
+regenerated corpora can be diffed and cited by agents. See
+[`docs/corpus-manifest.md`](docs/corpus-manifest.md) for the schema and
+stability contract.
+
+SQLite output includes an FTS5 index for local retrieval:
+
+```python
+from pathlib import Path
+from docpull import search_sqlite_documents
+
+hits = search_sqlite_documents(Path("docs/documents.db"), "rate limit")
+for hit in hits:
+    print(hit.url, hit.snippet)
+```
 
 ## Security
 
@@ -434,9 +548,9 @@ Run `docpull --help` for the full list. Highlights:
 
 ```
 Core:
-  --profile {rag,mirror,quick,llm}
+  --profile {rag,mirror,quick,llm,okf}
   --single                Fetch one URL (no crawl)
-  --format {markdown,json,ndjson,sqlite}
+  --format {markdown,json,ndjson,sqlite,okf}
   --stream                Stream NDJSON to stdout
 
 LLM / chunking:

{docpull-4.2.0 → docpull-4.3.1}/pyproject.toml

@@ -4,9 +4,9 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "docpull"
-version = "4.2.0"
+version = "4.3.1"
 dynamic = []
-description = "Pull documentation from the web and convert to clean markdown"
+description = "Convert public web pages into clean Markdown for AI agents and RAG"
 readme = {file = "README.md", content-type = "text/markdown"}
 requires-python = ">=3.10"
 license = "MIT"
@@ -18,8 +18,20 @@ maintainers = [
     {name = "Raintree Technology", email = "support@raintree.technology"}
 ]
 keywords = [
-    "python", "markdown", "documentation", "web-scraping", "developer-tools",
-    "claude", "ai-training-data"
+    "python",
+    "markdown",
+    "documentation",
+    "web-scraping",
+    "web-crawler",
+    "crawler",
+    "developer-tools",
+    "cli",
+    "rag",
+    "llm",
+    "mcp",
+    "ai-agents",
+    "claude",
+    "ai-training-data",
 ]
 classifiers = [
     # Development Status
@@ -134,12 +146,16 @@ dev = [
 docpull = "docpull.cli:main"
 
 [project.urls]
-Homepage = "https://github.com/raintree-technology/docpull"
-Documentation = "https://github.com/raintree-technology/docpull#readme"
+Homepage = "https://docpull.raintree.technology"
+Documentation = "https://docpull.raintree.technology"
 Repository = "https://github.com/raintree-technology/docpull"
 "Source Code" = "https://github.com/raintree-technology/docpull"
 "Bug Tracker" = "https://github.com/raintree-technology/docpull/issues"
 "Releases" = "https://github.com/raintree-technology/docpull/releases"
+"MCP Plugin" = "https://github.com/raintree-technology/docpull/tree/main/plugin"
+"Comparison Guide" = "https://github.com/raintree-technology/docpull/blob/main/docs/alternatives.md"
+"Download Stats" = "https://pepy.tech/project/docpull"
+"Changelog" = "https://github.com/raintree-technology/docpull/blob/main/docs/CHANGELOG.md"
 
 [tool.setuptools.packages.find]
 where = ["src"]

{docpull-4.2.0 → docpull-4.3.1}/src/docpull/__init__.py

@@ -1,5 +1,5 @@
 """
-docpull - Fetch and convert documentation from any URL to markdown.
+docpull - Fetch and convert static/server-rendered documentation to markdown.
 
 Usage:
     from docpull import Fetcher, DocpullConfig, ProfileName
@@ -14,7 +14,7 @@ Usage:
             print(event)
 """
 
-__version__ = "4.2.0"
+__version__ = "4.3.1"
 
 from .cache import CacheManager, StreamingDeduplicator
 from .conversion.chunking import Chunk, TokenCounter, chunk_markdown
@@ -31,12 +31,27 @@ from .models.config import (
 )
 from .models.events import EventType, FetchEvent, FetchStats
 from .pipeline.base import PageContext
+from .pipeline.steps import SqliteSearchResult, search_sqlite_documents
+from .scraper import (
+    Scraper,
+    ScrapeResult,
+    ScrapeRunResult,
+    scrape_one,
+    scrape_one_blocking,
+    scrape_site,
+)
 
 __all__ = [
     "__version__",
     "Fetcher",
     "fetch_blocking",
     "fetch_one",
+    "ScrapeResult",
+    "ScrapeRunResult",
+    "Scraper",
+    "scrape_one",
+    "scrape_one_blocking",
+    "scrape_site",
     "PageContext",
     "DocpullConfig",
     "ProfileName",
@@ -49,6 +64,8 @@ __all__ = [
     "EventType",
     "FetchEvent",
     "FetchStats",
+    "SqliteSearchResult",
+    "search_sqlite_documents",
     "CacheManager",
     "StreamingDeduplicator",
     "Chunk",