docpull 4.2.0__tar.gz → 4.3.0__tar.gz

{docpull-4.2.0/src/docpull.egg-info → docpull-4.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: docpull
-Version: 4.2.0
+Version: 4.3.0
 Summary: Pull documentation from the web and convert to clean markdown
 Author-email: Zachary Roth <support@raintree.technology>
 Maintainer-email: Raintree Technology <support@raintree.technology>
@@ -100,7 +100,8 @@ Dynamic: license-file
 
 [![Python 3.10+](https://img.shields.io/badge/python-3.10+-blue.svg)](https://www.python.org/downloads/)
 [![PyPI version](https://badge.fury.io/py/docpull.svg)](https://badge.fury.io/py/docpull)
-[![Downloads](https://pepy.tech/badge/docpull)](https://pepy.tech/project/docpull)
+[![PyPI downloads](https://img.shields.io/pepy/dt/docpull?label=downloads)](https://pepy.tech/project/docpull)
+[![GitHub stars](https://img.shields.io/github/stars/raintree-technology/docpull?style=social)](https://github.com/raintree-technology/docpull/stargazers)
 [![License: MIT](https://img.shields.io/github/license/raintree-technology/docpull)](https://github.com/raintree-technology/docpull/blob/main/LICENSE)
 
 <p align="center">
@@ -109,6 +110,22 @@ Dynamic: license-file
   </a>
 </p>
 
+## Download History
+
+<a href="https://pepy.tech/project/docpull">
+  <img alt="Cumulative PyPI download history chart for docpull" src="docs/cumulative-downloads-history.svg" />
+</a>
+
+## Star History
+
+<a href="https://star-history.com/#raintree-technology/docpull&Date">
+  <picture>
+    <source media="(prefers-color-scheme: dark)" srcset="https://api.star-history.com/svg?repos=raintree-technology/docpull&type=Date&theme=dark" />
+    <source media="(prefers-color-scheme: light)" srcset="https://api.star-history.com/svg?repos=raintree-technology/docpull&type=Date" />
+    <img alt="Star history chart for raintree-technology/docpull" src="https://api.star-history.com/svg?repos=raintree-technology/docpull&type=Date" />
+  </picture>
+</a>
+
 docpull is a web scraper for static and server-rendered sites, with
 documentation crawling as its sharpest default workflow. It uses async HTTP (not
 Playwright) to fetch pages, discover links, extract main content, and write clean
@@ -120,6 +137,11 @@ sandbox-friendly way to pipe web content into an LLM context, a RAG index, or an
 offline archive. SSRF, XXE, DNS-rebinding, and CRLF-injection protections are on
 by default — a necessity when an AI agent is choosing the URLs.
 
+docpull is intentionally not a general browser-automation scraper. See
+[`docs/scraping-boundary.md`](docs/scraping-boundary.md) for the exact product
+boundary and when to use Scrapy, Crawlee, hosted extraction APIs, or trafilatura
+directly.
+
 ## Install
 
 ```bash
@@ -146,6 +168,9 @@ docpull https://docs.example.com/guide --single
 # LLM-ready NDJSON with 4k-token chunks streamed to stdout
 docpull https://docs.example.com --profile llm --stream | jq .
 
+# Open Knowledge Format bundle for agent/wiki interoperability
+docpull https://docs.example.com --format okf
+
 # Mirror scraped content for offline use
 docpull https://docs.example.com --profile mirror --cache
 ```
@@ -160,8 +185,13 @@ content directly from framework data feeds:
 | Next.js   | Parses `__NEXT_DATA__` JSON |
 | Mintlify  | `__NEXT_DATA__` with Mintlify tagging |
 | OpenAPI   | Renders `openapi.json` / `swagger.json` into Markdown |
-| Docusaurus| Detected and tagged; generic extractor produces Markdown |
-| Sphinx    | Detected from generator metadata / Read the Docs hosts and tagged; generic extractor produces Markdown |
+| Docusaurus| Extracts static article regions and tags framework metadata |
+| Sphinx    | Extracts static body/document regions and tags framework metadata |
+| MkDocs / Material | Extracts static content regions and tags framework metadata |
+| VitePress / VuePress | Extracts static doc regions and tags framework metadata |
+| Astro Starlight | Extracts static markdown content regions and tags framework metadata |
+| GitBook / ReadMe.io | Extracts static article/content regions and tags framework metadata |
+| Redoc / Scalar | Extracts static API reference regions; OpenAPI JSON is still preferred when available |
 
 JS-only SPAs with no server-rendered content are detected and skipped with a
 clear reason (or, with `--strict-js-required`, reported as an error so agents
@@ -181,6 +211,30 @@ can route elsewhere).
 
 ## Python API
 
+Scraper-facing API:
+
+```python
+import asyncio
+from docpull import scrape_one, Scraper
+
+async def main():
+    page = await scrape_one("https://docs.python.org/3/library/asyncio.html")
+    print(page.title, page.source_type)
+    print(page.text[:500])
+
+    scraper = Scraper()
+    run = await scraper.scrape_site(
+        "https://docs.example.com",
+        output_format="ndjson",
+        max_pages=100,
+    )
+    print(run.stats.pages_fetched, run.manifest_path)
+
+asyncio.run(main())
+```
+
+Core fetcher API:
+
 ```python
 from docpull import fetch_one
 
@@ -225,6 +279,7 @@ async def tool_call(url: str) -> str:
 ```bash
 docpull https://site.com --profile rag      # Default. Dedup, rich metadata.
 docpull https://site.com --profile llm      # NDJSON + chunks + metadata; JS-only pages skip unless --strict-js-required is passed.
+docpull https://site.com --profile okf      # OKF bundle with generated index.md files.
 docpull https://site.com --profile mirror   # Full archive, polite, cached, hierarchical paths.
 docpull https://site.com --profile quick    # Sampling: 50 pages, depth 2.
 ```
@@ -506,10 +561,42 @@ NDJSON (one record per page or chunk):
 {"document_id": "doc_...", "chunk_id": "chunk_...", "url": "...", "title": "...", "content": "...", "hash": "...", "token_count": 842, "chunk_index": 0}
 ```
 
+Open Knowledge Format:
+
+```bash
+docpull https://docs.example.com --format okf
+```
+
+OKF output is an opt-in Markdown bundle. Scraped pages are written as concept
+documents with OKF frontmatter (`type`, `title`, `description`, `resource`,
+`tags`, `timestamp` when source metadata exists) and docpull keeps `source` as a
+compatibility extension. Generated `index.md` files are reserved for directory
+listings, so URL landing pages use safe concept filenames such as `_root.md` or
+`_page.md` instead of occupying `index.md`. The root `index.md` declares
+`okf_version: "0.1"`; nested indexes stay plain listing files.
+
+Because OKF treats every non-reserved `.md` file in the tree as a concept, write
+OKF output to a clean or dedicated directory rather than mixing it with unrelated
+Markdown files.
+
 Every output format also writes `corpus.manifest.json` next to the generated
 documents. The manifest records the run identity, output format, stable
-`document_id` / `chunk_id` values, content hashes, relative output paths, and
-chunk counts so regenerated corpora can be diffed and cited by agents.
+`document_id` / `chunk_id` values, content hashes, file-backed relative output
+paths (or the stdout marker for streamed NDJSON), and chunk counts so
+regenerated corpora can be diffed and cited by agents. See
+[`docs/corpus-manifest.md`](docs/corpus-manifest.md) for the schema and
+stability contract.
+
+SQLite output includes an FTS5 index for local retrieval:
+
+```python
+from pathlib import Path
+from docpull import search_sqlite_documents
+
+hits = search_sqlite_documents(Path("docs/documents.db"), "rate limit")
+for hit in hits:
+    print(hit.url, hit.snippet)
+```
 
 ## Security
 
@@ -530,9 +617,9 @@ Run `docpull --help` for the full list. Highlights:
 
 ```
 Core:
-  --profile {rag,mirror,quick,llm}
+  --profile {rag,mirror,quick,llm,okf}
   --single                Fetch one URL (no crawl)
-  --format {markdown,json,ndjson,sqlite}
+  --format {markdown,json,ndjson,sqlite,okf}
   --stream                Stream NDJSON to stdout
 
 LLM / chunking:

{docpull-4.2.0 → docpull-4.3.0}/README.md

@@ -4,7 +4,8 @@
 
 [![Python 3.10+](https://img.shields.io/badge/python-3.10+-blue.svg)](https://www.python.org/downloads/)
 [![PyPI version](https://badge.fury.io/py/docpull.svg)](https://badge.fury.io/py/docpull)
-[![Downloads](https://pepy.tech/badge/docpull)](https://pepy.tech/project/docpull)
+[![PyPI downloads](https://img.shields.io/pepy/dt/docpull?label=downloads)](https://pepy.tech/project/docpull)
+[![GitHub stars](https://img.shields.io/github/stars/raintree-technology/docpull?style=social)](https://github.com/raintree-technology/docpull/stargazers)
 [![License: MIT](https://img.shields.io/github/license/raintree-technology/docpull)](https://github.com/raintree-technology/docpull/blob/main/LICENSE)
 
 <p align="center">
@@ -13,6 +14,22 @@
   </a>
 </p>
 
+## Download History
+
+<a href="https://pepy.tech/project/docpull">
+  <img alt="Cumulative PyPI download history chart for docpull" src="docs/cumulative-downloads-history.svg" />
+</a>
+
+## Star History
+
+<a href="https://star-history.com/#raintree-technology/docpull&Date">
+  <picture>
+    <source media="(prefers-color-scheme: dark)" srcset="https://api.star-history.com/svg?repos=raintree-technology/docpull&type=Date&theme=dark" />
+    <source media="(prefers-color-scheme: light)" srcset="https://api.star-history.com/svg?repos=raintree-technology/docpull&type=Date" />
+    <img alt="Star history chart for raintree-technology/docpull" src="https://api.star-history.com/svg?repos=raintree-technology/docpull&type=Date" />
+  </picture>
+</a>
+
 docpull is a web scraper for static and server-rendered sites, with
 documentation crawling as its sharpest default workflow. It uses async HTTP (not
 Playwright) to fetch pages, discover links, extract main content, and write clean
@@ -24,6 +41,11 @@ sandbox-friendly way to pipe web content into an LLM context, a RAG index, or an
 offline archive. SSRF, XXE, DNS-rebinding, and CRLF-injection protections are on
 by default — a necessity when an AI agent is choosing the URLs.
 
+docpull is intentionally not a general browser-automation scraper. See
+[`docs/scraping-boundary.md`](docs/scraping-boundary.md) for the exact product
+boundary and when to use Scrapy, Crawlee, hosted extraction APIs, or trafilatura
+directly.
+
 ## Install
 
 ```bash
@@ -50,6 +72,9 @@ docpull https://docs.example.com/guide --single
 # LLM-ready NDJSON with 4k-token chunks streamed to stdout
 docpull https://docs.example.com --profile llm --stream | jq .
 
+# Open Knowledge Format bundle for agent/wiki interoperability
+docpull https://docs.example.com --format okf
+
 # Mirror scraped content for offline use
 docpull https://docs.example.com --profile mirror --cache
 ```
@@ -64,8 +89,13 @@ content directly from framework data feeds:
 | Next.js   | Parses `__NEXT_DATA__` JSON |
 | Mintlify  | `__NEXT_DATA__` with Mintlify tagging |
 | OpenAPI   | Renders `openapi.json` / `swagger.json` into Markdown |
-| Docusaurus| Detected and tagged; generic extractor produces Markdown |
-| Sphinx    | Detected from generator metadata / Read the Docs hosts and tagged; generic extractor produces Markdown |
+| Docusaurus| Extracts static article regions and tags framework metadata |
+| Sphinx    | Extracts static body/document regions and tags framework metadata |
+| MkDocs / Material | Extracts static content regions and tags framework metadata |
+| VitePress / VuePress | Extracts static doc regions and tags framework metadata |
+| Astro Starlight | Extracts static markdown content regions and tags framework metadata |
+| GitBook / ReadMe.io | Extracts static article/content regions and tags framework metadata |
+| Redoc / Scalar | Extracts static API reference regions; OpenAPI JSON is still preferred when available |
 
 JS-only SPAs with no server-rendered content are detected and skipped with a
 clear reason (or, with `--strict-js-required`, reported as an error so agents
@@ -85,6 +115,30 @@ can route elsewhere).
 
 ## Python API
 
+Scraper-facing API:
+
+```python
+import asyncio
+from docpull import scrape_one, Scraper
+
+async def main():
+    page = await scrape_one("https://docs.python.org/3/library/asyncio.html")
+    print(page.title, page.source_type)
+    print(page.text[:500])
+
+    scraper = Scraper()
+    run = await scraper.scrape_site(
+        "https://docs.example.com",
+        output_format="ndjson",
+        max_pages=100,
+    )
+    print(run.stats.pages_fetched, run.manifest_path)
+
+asyncio.run(main())
+```
+
+Core fetcher API:
+
 ```python
 from docpull import fetch_one
 
@@ -129,6 +183,7 @@ async def tool_call(url: str) -> str:
 ```bash
 docpull https://site.com --profile rag      # Default. Dedup, rich metadata.
 docpull https://site.com --profile llm      # NDJSON + chunks + metadata; JS-only pages skip unless --strict-js-required is passed.
+docpull https://site.com --profile okf      # OKF bundle with generated index.md files.
 docpull https://site.com --profile mirror   # Full archive, polite, cached, hierarchical paths.
 docpull https://site.com --profile quick    # Sampling: 50 pages, depth 2.
 ```
@@ -410,10 +465,42 @@ NDJSON (one record per page or chunk):
 {"document_id": "doc_...", "chunk_id": "chunk_...", "url": "...", "title": "...", "content": "...", "hash": "...", "token_count": 842, "chunk_index": 0}
 ```
 
+Open Knowledge Format:
+
+```bash
+docpull https://docs.example.com --format okf
+```
+
+OKF output is an opt-in Markdown bundle. Scraped pages are written as concept
+documents with OKF frontmatter (`type`, `title`, `description`, `resource`,
+`tags`, `timestamp` when source metadata exists) and docpull keeps `source` as a
+compatibility extension. Generated `index.md` files are reserved for directory
+listings, so URL landing pages use safe concept filenames such as `_root.md` or
+`_page.md` instead of occupying `index.md`. The root `index.md` declares
+`okf_version: "0.1"`; nested indexes stay plain listing files.
+
+Because OKF treats every non-reserved `.md` file in the tree as a concept, write
+OKF output to a clean or dedicated directory rather than mixing it with unrelated
+Markdown files.
+
 Every output format also writes `corpus.manifest.json` next to the generated
 documents. The manifest records the run identity, output format, stable
-`document_id` / `chunk_id` values, content hashes, relative output paths, and
-chunk counts so regenerated corpora can be diffed and cited by agents.
+`document_id` / `chunk_id` values, content hashes, file-backed relative output
+paths (or the stdout marker for streamed NDJSON), and chunk counts so
+regenerated corpora can be diffed and cited by agents. See
+[`docs/corpus-manifest.md`](docs/corpus-manifest.md) for the schema and
+stability contract.
+
+SQLite output includes an FTS5 index for local retrieval:
+
+```python
+from pathlib import Path
+from docpull import search_sqlite_documents
+
+hits = search_sqlite_documents(Path("docs/documents.db"), "rate limit")
+for hit in hits:
+    print(hit.url, hit.snippet)
+```
 
 ## Security
 
@@ -434,9 +521,9 @@ Run `docpull --help` for the full list. Highlights:
 
 ```
 Core:
-  --profile {rag,mirror,quick,llm}
+  --profile {rag,mirror,quick,llm,okf}
   --single                Fetch one URL (no crawl)
-  --format {markdown,json,ndjson,sqlite}
+  --format {markdown,json,ndjson,sqlite,okf}
   --stream                Stream NDJSON to stdout
 
 LLM / chunking:

{docpull-4.2.0 → docpull-4.3.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "docpull"
-version = "4.2.0"
+version = "4.3.0"
 dynamic = []
 description = "Pull documentation from the web and convert to clean markdown"
 readme = {file = "README.md", content-type = "text/markdown"}

{docpull-4.2.0 → docpull-4.3.0}/src/docpull/__init__.py

@@ -1,5 +1,5 @@
 """
-docpull - Fetch and convert documentation from any URL to markdown.
+docpull - Fetch and convert static/server-rendered documentation to markdown.
 
 Usage:
     from docpull import Fetcher, DocpullConfig, ProfileName
@@ -14,7 +14,7 @@ Usage:
             print(event)
 """
 
-__version__ = "4.2.0"
+__version__ = "4.3.0"
 
 from .cache import CacheManager, StreamingDeduplicator
 from .conversion.chunking import Chunk, TokenCounter, chunk_markdown
@@ -31,12 +31,27 @@ from .models.config import (
 )
 from .models.events import EventType, FetchEvent, FetchStats
 from .pipeline.base import PageContext
+from .pipeline.steps import SqliteSearchResult, search_sqlite_documents
+from .scraper import (
+    Scraper,
+    ScrapeResult,
+    ScrapeRunResult,
+    scrape_one,
+    scrape_one_blocking,
+    scrape_site,
+)
 
 __all__ = [
     "__version__",
     "Fetcher",
     "fetch_blocking",
     "fetch_one",
+    "ScrapeResult",
+    "ScrapeRunResult",
+    "Scraper",
+    "scrape_one",
+    "scrape_one_blocking",
+    "scrape_site",
     "PageContext",
     "DocpullConfig",
     "ProfileName",
@@ -49,6 +64,8 @@ __all__ = [
     "EventType",
     "FetchEvent",
     "FetchStats",
+    "SqliteSearchResult",
+    "search_sqlite_documents",
     "CacheManager",
     "StreamingDeduplicator",
     "Chunk",