docpull 6.0.1__tar.gz → 6.1.0__tar.gz

{docpull-6.0.1/src/docpull.egg-info → docpull-6.1.0}/PKG-INFO

@@ -1,12 +1,12 @@
 Metadata-Version: 2.4
 Name: docpull
-Version: 6.0.1
+Version: 6.1.0
 Summary: Declare, sync, diff, and lock context dependencies for AI agents
 Author-email: Zachary Roth <support@raintree.technology>
 Maintainer-email: Raintree Technology <support@raintree.technology>
 License-Expression: MIT
-Project-URL: Homepage, https://docpull.raintree.technology
-Project-URL: Documentation, https://docpull.raintree.technology
+Project-URL: Homepage, https://github.com/raintree-technology/docpull
+Project-URL: Documentation, https://github.com/raintree-technology/docpull#readme
 Project-URL: Repository, https://github.com/raintree-technology/docpull
 Project-URL: Source Code, https://github.com/raintree-technology/docpull
 Project-URL: Bug Tracker, https://github.com/raintree-technology/docpull/issues
@@ -71,11 +71,16 @@ Requires-Dist: uvicorn>=0.30.0; extra == "serve"
 Provides-Extra: parquet
 Requires-Dist: pyarrow>=16.0.0; extra == "parquet"
 Provides-Extra: markitdown
-Requires-Dist: markitdown>=0.1.0; extra == "markitdown"
+Requires-Dist: pypdf>=6.14.2; extra == "markitdown"
+Requires-Dist: markitdown[docx,pdf]>=0.1.0; extra == "markitdown"
 Provides-Extra: unstructured
+Requires-Dist: pypdf>=6.14.2; extra == "unstructured"
 Requires-Dist: unstructured[all-docs]>=0.16.0; extra == "unstructured"
+Provides-Extra: pdf
+Requires-Dist: pypdf>=6.14.2; extra == "pdf"
 Provides-Extra: parse
-Requires-Dist: markitdown>=0.1.0; extra == "parse"
+Requires-Dist: pypdf>=6.14.2; extra == "parse"
+Requires-Dist: markitdown[docx,pdf]>=0.1.0; extra == "parse"
 Requires-Dist: unstructured[all-docs]>=0.16.0; extra == "parse"
 Provides-Extra: presidio
 Requires-Dist: presidio-analyzer>=2.2.0; extra == "presidio"
@@ -96,7 +101,8 @@ Requires-Dist: python-multipart>=0.0.27; extra == "all"
 Requires-Dist: starlette>=1.3.1; extra == "all"
 Requires-Dist: uvicorn>=0.30.0; extra == "all"
 Requires-Dist: pyarrow>=16.0.0; extra == "all"
-Requires-Dist: markitdown>=0.1.0; extra == "all"
+Requires-Dist: pypdf>=6.14.2; extra == "all"
+Requires-Dist: markitdown[docx,pdf]>=0.1.0; extra == "all"
 Requires-Dist: unstructured[all-docs]>=0.16.0; extra == "all"
 Requires-Dist: presidio-analyzer>=2.2.0; extra == "all"
 Requires-Dist: e2b>=2.0.0; extra == "all"
@@ -181,6 +187,12 @@ pages such as blogs, API references, OpenAPI specs, changelogs, vendor pages,
 product pages, filings, docs sites, and other pages where the useful content is
 available in HTML or embedded page data.
 
+Direct fetches also recognize standards-style plain text and common textual
+formats from their response media type, including extensionless URLs. Remote
+documents remain denied by default; explicitly opt into local PDF parsing with
+`docpull URL --remote-documents pdf` after installing `docpull[markitdown]` or
+`docpull[parse]`. This does not enable a browser, cloud parser, or paid route.
+
 DocPull is browser-free by default. JS-only pages are skipped with a clear
 reason unless you explicitly opt into a local renderer. See
 [Web Source Boundary](docs/scraping-boundary.md) and
@@ -592,6 +604,18 @@ MarkItDown or Unstructured parsers for complex office/PDF files when installed.
 Install `docpull[markitdown]`, `docpull[unstructured]`, or `docpull[parse]`
 for those backends.
 
+An HTTPS PDF can enter the ordinary fetch contract only with explicit consent:
+
+```bash
+docpull https://example.com/handbook.pdf --remote-documents pdf \
+  --remote-document-backend auto -o ./packs/handbook
+```
+
+The response must identify as PDF and contain a PDF signature. DocPull parses
+it locally through a mode-`0600` temporary file, records the source hash and
+parser provenance, and removes the temporary source. Other remote download
+types remain blocked.
+
 Every file-backed run writes `corpus.manifest.json` with stable document IDs,
 chunk IDs, hashes, output paths, and chunk counts. See
 [Corpus Manifest](docs/corpus-manifest.md).
@@ -754,7 +778,7 @@ docpull URL --preview-urls
 
 ## Links
 
-- [Website](https://docpull.raintree.technology)
+- [Documentation](https://github.com/raintree-technology/docpull#readme)
 - [PyPI](https://pypi.org/project/docpull/)
 - [GitHub](https://github.com/raintree-technology/docpull)
 - [Metrics](METRICS.md)

{docpull-6.0.1 → docpull-6.1.0}/README.md

@@ -63,6 +63,12 @@ pages such as blogs, API references, OpenAPI specs, changelogs, vendor pages,
 product pages, filings, docs sites, and other pages where the useful content is
 available in HTML or embedded page data.
 
+Direct fetches also recognize standards-style plain text and common textual
+formats from their response media type, including extensionless URLs. Remote
+documents remain denied by default; explicitly opt into local PDF parsing with
+`docpull URL --remote-documents pdf` after installing `docpull[markitdown]` or
+`docpull[parse]`. This does not enable a browser, cloud parser, or paid route.
+
 DocPull is browser-free by default. JS-only pages are skipped with a clear
 reason unless you explicitly opt into a local renderer. See
 [Web Source Boundary](docs/scraping-boundary.md) and
@@ -474,6 +480,18 @@ MarkItDown or Unstructured parsers for complex office/PDF files when installed.
 Install `docpull[markitdown]`, `docpull[unstructured]`, or `docpull[parse]`
 for those backends.
 
+An HTTPS PDF can enter the ordinary fetch contract only with explicit consent:
+
+```bash
+docpull https://example.com/handbook.pdf --remote-documents pdf \
+  --remote-document-backend auto -o ./packs/handbook
+```
+
+The response must identify as PDF and contain a PDF signature. DocPull parses
+it locally through a mode-`0600` temporary file, records the source hash and
+parser provenance, and removes the temporary source. Other remote download
+types remain blocked.
+
 Every file-backed run writes `corpus.manifest.json` with stable document IDs,
 chunk IDs, hashes, output paths, and chunk counts. See
 [Corpus Manifest](docs/corpus-manifest.md).
@@ -636,7 +654,7 @@ docpull URL --preview-urls
 
 ## Links
 
-- [Website](https://docpull.raintree.technology)
+- [Documentation](https://github.com/raintree-technology/docpull#readme)
 - [PyPI](https://pypi.org/project/docpull/)
 - [GitHub](https://github.com/raintree-technology/docpull)
 - [Metrics](METRICS.md)

{docpull-6.0.1 → docpull-6.1.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "docpull"
-version = "6.0.1"
+version = "6.1.0"
 dynamic = []
 description = "Declare, sync, diff, and lock context dependencies for AI agents"
 readme = {file = "README.md", content-type = "text/markdown"}
@@ -123,13 +123,19 @@ parquet = [
     "pyarrow>=16.0.0",
 ]
 markitdown = [
-    "markitdown>=0.1.0",
+    "pypdf>=6.14.2",
+    "markitdown[docx,pdf]>=0.1.0",
 ]
 unstructured = [
+    "pypdf>=6.14.2",
     "unstructured[all-docs]>=0.16.0",
 ]
+pdf = [
+    "pypdf>=6.14.2",
+]
 parse = [
-    "markitdown>=0.1.0",
+    "pypdf>=6.14.2",
+    "markitdown[docx,pdf]>=0.1.0",
     "unstructured[all-docs]>=0.16.0",
 ]
 presidio = [
@@ -154,7 +160,8 @@ all = [
     "starlette>=1.3.1",
     "uvicorn>=0.30.0",
     "pyarrow>=16.0.0",
-    "markitdown>=0.1.0",
+    "pypdf>=6.14.2",
+    "markitdown[docx,pdf]>=0.1.0",
     "unstructured[all-docs]>=0.16.0",
     "presidio-analyzer>=2.2.0",
     "e2b>=2.0.0",
@@ -179,8 +186,8 @@ dev = [
 docpull = "docpull.cli:main"
 
 [project.urls]
-Homepage = "https://docpull.raintree.technology"
-Documentation = "https://docpull.raintree.technology"
+Homepage = "https://github.com/raintree-technology/docpull"
+Documentation = "https://github.com/raintree-technology/docpull#readme"
 Repository = "https://github.com/raintree-technology/docpull"
 "Source Code" = "https://github.com/raintree-technology/docpull"
 "Bug Tracker" = "https://github.com/raintree-technology/docpull/issues"

{docpull-6.0.1 → docpull-6.1.0}/src/docpull/__init__.py

@@ -14,7 +14,7 @@ Usage:
             print(event)
 """
 
-__version__ = "6.0.1"
+__version__ = "6.1.0"
 
 from .cache import CacheManager, StreamingDeduplicator
 from .context_ci import CIThresholds, ContextCIError, run_context_ci

{docpull-6.0.1 → docpull-6.1.0}/src/docpull/benchmark.py

@@ -200,8 +200,8 @@ TOOL_DOC_TARGETS: tuple[_BenchmarkTarget, ...] = (
     _BenchmarkTarget(
         id="docpull_docs",
         label="DocPull docs",
-        url="https://docpull.raintree.technology",
-        include_domains=("docpull.raintree.technology",),
+        url="https://github.com/raintree-technology/docpull",
+        include_domains=("github.com",),
         objective="Build an agent context pack for DocPull documentation",
         queries=("DocPull documentation CLI provider benchmark docs",),
         freshness_terms=("changelog", "release", "benchmark"),

{docpull-6.0.1 → docpull-6.1.0}/src/docpull/cli.py

@@ -335,6 +335,32 @@ Examples:
         action="store_true",
         help="Stream NDJSON records to stdout as each page completes (implies --format ndjson)",
     )
+    parser.add_argument(
+        "--remote-documents",
+        choices=["off", "pdf"],
+        default=None,
+        help="Explicitly download and locally parse selected remote document types (default: off)",
+    )
+    parser.add_argument(
+        "--remote-document-backend",
+        choices=["auto", "pypdf", "markitdown", "unstructured"],
+        default=None,
+        help="Local parser backend for --remote-documents (default: auto)",
+    )
+    parser.add_argument(
+        "--remote-document-timeout-seconds",
+        type=int,
+        default=None,
+        metavar="SECONDS",
+        help="Wall-time limit for each isolated remote-document parser (default: 60)",
+    )
+    parser.add_argument(
+        "--remote-document-memory-mib",
+        type=int,
+        default=None,
+        metavar="MIB",
+        help="Address-space limit for each isolated remote-document parser (default: 1024)",
+    )
 
     # Crawl settings
     crawl_group = parser.add_argument_group("crawl settings")
@@ -677,6 +703,14 @@ def run_fetcher(args: argparse.Namespace) -> int:
         filter_kwargs["enable_special_cases"] = False
     if args.strict_js_required:
         filter_kwargs["strict_js_required"] = True
+    if args.remote_documents:
+        filter_kwargs["remote_documents"] = args.remote_documents
+    if args.remote_document_backend:
+        filter_kwargs["remote_document_backend"] = args.remote_document_backend
+    if args.remote_document_timeout_seconds is not None:
+        filter_kwargs["remote_document_timeout_seconds"] = args.remote_document_timeout_seconds
+    if args.remote_document_memory_mib is not None:
+        filter_kwargs["remote_document_memory_mib"] = args.remote_document_memory_mib
     if filter_kwargs:
         config_kwargs["content_filter"] = filter_kwargs
 
@@ -1164,7 +1198,7 @@ def run_render_cli(argv: list[str]) -> int:
         args.cloud_max_estimated_cost if backend in {"vercel-sandbox", "e2b-sandbox"} else None,
     )
     cloud_estimated_cost = (
-        estimate_cloud_render_cost_usd(cast(Literal["vercel-sandbox", "e2b-sandbox"], backend), config)
+        estimate_cloud_render_cost_usd(backend, config)
         if backend in {"vercel-sandbox", "e2b-sandbox"}
         else 0.0
     )

{docpull-6.0.1 → docpull-6.1.0}/src/docpull/conversion/special_cases.py

@@ -16,6 +16,7 @@ import json
 import logging
 import re
 from dataclasses import dataclass
+from email.message import Message
 from typing import Any, Protocol
 from urllib.parse import urlparse
 
@@ -710,33 +711,143 @@ class RawTextExtractor:
 
     name = "raw_text"
 
-    _TEXT_SUFFIXES = (".md", ".markdown", ".mdx", ".txt")
+    _TEXT_SUFFIXES = (
+        ".csv",
+        ".json",
+        ".jsonl",
+        ".markdown",
+        ".md",
+        ".mdx",
+        ".ndjson",
+        ".rst",
+        ".text",
+        ".tsv",
+        ".txt",
+        ".xml",
+        ".yaml",
+        ".yml",
+    )
+    _TEXT_CONTENT_TYPES = {
+        "application/json",
+        "application/ld+json",
+        "application/x-ndjson",
+        "application/xml",
+        "application/yaml",
+        "application/x-yaml",
+        "text/csv",
+        "text/markdown",
+        "text/plain",
+        "text/tab-separated-values",
+        "text/x-markdown",
+        "text/x-rst",
+        "text/xml",
+        "text/yaml",
+    }
     _HTML_MARKERS = ("<html", "<body", "<!doctype")
-
-    def try_extract(self, html: bytes, url: str) -> SpecialCaseResult | None:
+    _STRUCTURED_LANGUAGES = {
+        ".csv": "csv",
+        ".json": "json",
+        ".jsonl": "ndjson",
+        ".ndjson": "ndjson",
+        ".rst": "rst",
+        ".tsv": "tsv",
+        ".xml": "xml",
+        ".yaml": "yaml",
+        ".yml": "yaml",
+    }
+    _MEDIA_LANGUAGES = {
+        "application/json": "json",
+        "application/ld+json": "json",
+        "application/x-ndjson": "ndjson",
+        "application/xml": "xml",
+        "application/yaml": "yaml",
+        "application/x-yaml": "yaml",
+        "text/csv": "csv",
+        "text/tab-separated-values": "tsv",
+        "text/x-rst": "rst",
+        "text/xml": "xml",
+        "text/yaml": "yaml",
+    }
+
+    def try_extract(
+        self,
+        html: bytes,
+        url: str,
+        *,
+        content_type: str | None = None,
+    ) -> SpecialCaseResult | None:
         parsed = urlparse(url)
         path = parsed.path.lower()
-        if not path.endswith(self._TEXT_SUFFIXES):
+        media_type = self._media_type(content_type)
+        identified_by_type = media_type in self._TEXT_CONTENT_TYPES
+        if not identified_by_type and not path.endswith(self._TEXT_SUFFIXES):
             return None
 
-        text = _decode_html(html).strip()
+        text = self._decode_text(html, content_type).strip()
         if not text:
             return None
+        language = self._structured_language(path, media_type)
         lower_head = text[:500].lower()
-        if any(marker in lower_head for marker in self._HTML_MARKERS):
+        if language is None and any(marker in lower_head for marker in self._HTML_MARKERS):
             return None
-        if path.endswith(".txt") and not self._looks_like_docs_text(text, path):
+        if (
+            path.endswith((".txt", ".text"))
+            and not identified_by_type
+            and not self._looks_like_docs_text(text, path)
+        ):
             return None
 
         title = self._extract_title(text) or parsed.path.rsplit("/", 1)[-1] or "Document"
         source_type = "llms_txt" if path.endswith(("/llms.txt", "/llms-full.txt")) else self.name
+        markdown = self._fenced(text, language) if language else self._safe_direct_markdown(text)
         return SpecialCaseResult(
-            markdown=text.rstrip() + "\n",
+            markdown=markdown,
             title=title,
             source_type=source_type,
             extra={"framework": source_type},
         )
 
+    @classmethod
+    def _structured_language(cls, path: str, media_type: str) -> str | None:
+        if media_type in {"text/markdown", "text/x-markdown"} or path.endswith((".markdown", ".md", ".mdx")):
+            return None
+        media_language = cls._MEDIA_LANGUAGES.get(media_type)
+        if media_language:
+            return media_language
+        return next(
+            (language for suffix, language in cls._STRUCTURED_LANGUAGES.items() if path.endswith(suffix)),
+            None,
+        )
+
+    @staticmethod
+    def _fenced(text: str, language: str) -> str:
+        longest_run = max((len(match.group(0)) for match in re.finditer(r"`+", text)), default=0)
+        fence = "`" * max(3, longest_run + 1)
+        return f"{fence}{language}\n{text.rstrip()}\n{fence}\n"
+
+    @staticmethod
+    def _safe_direct_markdown(text: str) -> str:
+        inert = re.sub(r"<(/?)script\b", r"&lt;\1script", text, flags=re.IGNORECASE)
+        return inert.rstrip() + "\n"
+
+    @staticmethod
+    def _media_type(content_type: str | None) -> str:
+        return (content_type or "").split(";", 1)[0].strip().casefold()
+
+    @staticmethod
+    def _decode_text(body: bytes, content_type: str | None) -> str:
+        if body.startswith(b"\xef\xbb\xbf"):
+            return body.decode("utf-8-sig", errors="replace")
+        encoding = "utf-8"
+        if content_type:
+            message = Message()
+            message["content-type"] = content_type
+            encoding = message.get_content_charset() or encoding
+        try:
+            return body.decode(encoding)
+        except (LookupError, UnicodeDecodeError):
+            return body.decode("utf-8", errors="replace")
+
     @staticmethod
     def _looks_like_docs_text(text: str, path: str) -> bool:
         if path.endswith(("/llms.txt", "/llms-full.txt")):
@@ -802,6 +913,33 @@ class SphinxObjectsInvExtractor:
         )
 
 
+class RfcEditorExtractor:
+    """Preserve complete RFC Editor HTML documents instead of one scored section."""
+
+    name = "rfc_editor"
+
+    def try_extract(self, html: bytes, url: str) -> SpecialCaseResult | None:
+        parsed = urlparse(url)
+        if not _hostname_matches_domain(parsed.hostname or "", "rfc-editor.org"):
+            return None
+        if re.fullmatch(r"/rfc/rfc\d+\.html", parsed.path.casefold()) is None:
+            return None
+
+        from .extractor import MainContentExtractor
+        from .markdown import HtmlToMarkdown
+
+        extracted_html = MainContentExtractor(content_selectors=["body"]).extract(html, url)
+        markdown = HtmlToMarkdown().convert(extracted_html, url).strip()
+        if len(markdown) < 80:
+            return None
+        return SpecialCaseResult(
+            markdown=markdown + "\n",
+            title=_title_from_soup(_soup(html)),
+            source_type=self.name,
+            extra={"framework": "rfc_editor"},
+        )
+
+
 class MdxSourceExtractor:
     """Rewrite ``edit-this-page`` GitHub links to raw MDX source URLs.
 
@@ -842,6 +980,7 @@ class MdxSourceExtractor:
 DEFAULT_CHAIN: list[SpecialCaseExtractor] = [
     OpenApiExtractor(),
     RawTextExtractor(),
+    RfcEditorExtractor(),
     MintlifyExtractor(),
     NextDataExtractor(),
     MkDocsMaterialExtractor(),
@@ -969,6 +1108,7 @@ __all__ = [
     "RawTextExtractor",
     "ReadMeExtractor",
     "RedocScalarExtractor",
+    "RfcEditorExtractor",
     "SpecialCaseExtractor",
     "SpecialCaseResult",
     "SphinxObjectsInvExtractor",

{docpull-6.0.1 → docpull-6.1.0}/src/docpull/conversion/trafilatura_extractor.py

@@ -63,7 +63,7 @@ class TrafilaturaExtractor:
         if result is None:
             logger.debug("trafilatura returned no content for %s", url)
             return ""
-        return result.strip() + "\n"
+        return str(result).strip() + "\n"
 
 
 __all__ = ["TrafilaturaExtractor"]

{docpull-6.0.1 → docpull-6.1.0}/src/docpull/core/fetcher.py

@@ -34,6 +34,7 @@ from ..pipeline.steps import (
     SqliteSaveStep,
     ValidateStep,
 )
+from ..security.download_policy import SafeDownloadPolicy
 from ..security.robots import RobotsChecker
 from ..security.url_validator import UrlValidator
 
@@ -331,6 +332,9 @@ class Fetcher:
             if self.config.content_filter.max_file_size is not None
             else AsyncHttpClient.MAX_CONTENT_SIZE
         )
+        allowed_remote_document_types = (
+            {"application/pdf"} if self.config.content_filter.remote_documents == "pdf" else set()
+        )
         self._http_client = AsyncHttpClient(
             rate_limiter=self._rate_limiter,
             max_retries=self.config.network.max_retries,
@@ -344,6 +348,9 @@ class Fetcher:
             auth_scope_hosts=auth_scope_hosts,
             require_pinned_dns=self.config.network.require_pinned_dns,
             log_retry_warnings=self.config.network.log_retry_warnings,
+            download_policy=SafeDownloadPolicy(
+                allowed_remote_document_types=allowed_remote_document_types,
+            ),
         )
         await self._http_client.__aenter__()
 
@@ -396,6 +403,7 @@ class Fetcher:
                     http_client=self._http_client,
                     cache_manager=self._cache_manager,
                     skip_unchanged=self.config.cache.skip_unchanged,
+                    allowed_remote_document_types=allowed_remote_document_types,
                 )
             )
             if render_mode == "fallback":
@@ -417,6 +425,10 @@ class Fetcher:
                 use_ensemble=self.config.content_filter.extractor == "ensemble",
                 strict_js_required=self.config.content_filter.strict_js_required,
                 clean_inline_xbrl=self.config.content_filter.clean_inline_xbrl,
+                remote_documents=self.config.content_filter.remote_documents,
+                remote_document_backend=self.config.content_filter.remote_document_backend,
+                remote_document_timeout_seconds=(self.config.content_filter.remote_document_timeout_seconds),
+                remote_document_memory_mib=self.config.content_filter.remote_document_memory_mib,
             )
         )