docpull 6.0.0__tar.gz → 6.1.0__tar.gz

{docpull-6.0.0/src/docpull.egg-info → docpull-6.1.0}/PKG-INFO

@@ -1,12 +1,12 @@
 Metadata-Version: 2.4
 Name: docpull
-Version: 6.0.0
+Version: 6.1.0
 Summary: Declare, sync, diff, and lock context dependencies for AI agents
 Author-email: Zachary Roth <support@raintree.technology>
 Maintainer-email: Raintree Technology <support@raintree.technology>
 License-Expression: MIT
-Project-URL: Homepage, https://docpull.raintree.technology
-Project-URL: Documentation, https://docpull.raintree.technology
+Project-URL: Homepage, https://github.com/raintree-technology/docpull
+Project-URL: Documentation, https://github.com/raintree-technology/docpull#readme
 Project-URL: Repository, https://github.com/raintree-technology/docpull
 Project-URL: Source Code, https://github.com/raintree-technology/docpull
 Project-URL: Bug Tracker, https://github.com/raintree-technology/docpull/issues
@@ -71,11 +71,16 @@ Requires-Dist: uvicorn>=0.30.0; extra == "serve"
 Provides-Extra: parquet
 Requires-Dist: pyarrow>=16.0.0; extra == "parquet"
 Provides-Extra: markitdown
-Requires-Dist: markitdown>=0.1.0; extra == "markitdown"
+Requires-Dist: pypdf>=6.14.2; extra == "markitdown"
+Requires-Dist: markitdown[docx,pdf]>=0.1.0; extra == "markitdown"
 Provides-Extra: unstructured
+Requires-Dist: pypdf>=6.14.2; extra == "unstructured"
 Requires-Dist: unstructured[all-docs]>=0.16.0; extra == "unstructured"
+Provides-Extra: pdf
+Requires-Dist: pypdf>=6.14.2; extra == "pdf"
 Provides-Extra: parse
-Requires-Dist: markitdown>=0.1.0; extra == "parse"
+Requires-Dist: pypdf>=6.14.2; extra == "parse"
+Requires-Dist: markitdown[docx,pdf]>=0.1.0; extra == "parse"
 Requires-Dist: unstructured[all-docs]>=0.16.0; extra == "parse"
 Provides-Extra: presidio
 Requires-Dist: presidio-analyzer>=2.2.0; extra == "presidio"
@@ -96,7 +101,8 @@ Requires-Dist: python-multipart>=0.0.27; extra == "all"
 Requires-Dist: starlette>=1.3.1; extra == "all"
 Requires-Dist: uvicorn>=0.30.0; extra == "all"
 Requires-Dist: pyarrow>=16.0.0; extra == "all"
-Requires-Dist: markitdown>=0.1.0; extra == "all"
+Requires-Dist: pypdf>=6.14.2; extra == "all"
+Requires-Dist: markitdown[docx,pdf]>=0.1.0; extra == "all"
 Requires-Dist: unstructured[all-docs]>=0.16.0; extra == "all"
 Requires-Dist: presidio-analyzer>=2.2.0; extra == "all"
 Requires-Dist: e2b>=2.0.0; extra == "all"
@@ -126,10 +132,12 @@ Dynamic: license-file
 [![GitHub stars](https://img.shields.io/github/stars/raintree-technology/docpull?style=social)](https://github.com/raintree-technology/docpull/stargazers)
 [![License: MIT](https://img.shields.io/github/license/raintree-technology/docpull)](https://github.com/raintree-technology/docpull/blob/main/LICENSE)
 
+<!-- mcp-name: io.github.raintree-technology/docpull -->
+
 DocPull is a local-first dependency manager for AI context. Define the public
-docs and web sources an agent depends on, sync them into cited context packs,
-diff what changed, and export reproducible context for Cursor, Claude, Codex,
-OpenAI, LlamaIndex, LangChain, MCP clients, and RAG pipelines.
+web sources an agent depends on, sync them into cited context packs, diff what
+changed, and export reproducible context for Cursor, Claude, OpenAI,
+LlamaIndex, LangChain, MCP clients, and RAG pipelines.
 
 The core workflow is a `docpull.yaml` plus a `.docpull/context.lock.json`,
 similar in spirit to code dependency manifests and lockfiles:
@@ -141,7 +149,7 @@ docpull install
 docpull deps
 docpull sync
 docpull diff
-docpull export context-pack --target codex
+docpull export context-pack --target openai
 ```
 
 Bundled aliases such as `stripe`, `react`, `postgres`, `openai`, and
@@ -179,6 +187,12 @@ pages such as blogs, API references, OpenAPI specs, changelogs, vendor pages,
 product pages, filings, docs sites, and other pages where the useful content is
 available in HTML or embedded page data.
 
+Direct fetches also recognize standards-style plain text and common textual
+formats from their response media type, including extensionless URLs. Remote
+documents remain denied by default; explicitly opt into local PDF parsing with
+`docpull URL --remote-documents pdf` after installing `docpull[markitdown]` or
+`docpull[parse]`. This does not enable a browser, cloud parser, or paid route.
+
 DocPull is browser-free by default. JS-only pages are skipped with a clear
 reason unless you explicitly opt into a local renderer. See
 [Web Source Boundary](docs/scraping-boundary.md) and
@@ -504,12 +518,11 @@ More examples live in [CLI Recipes](docs/examples/README.md).
 With an explicit `--skill-agent`, docpull stores the fetched corpus under
 `.docpull/skills/<name>/references` and creates agent-specific wrappers that
 point at that corpus. `--skill-agent claude` writes a Claude Code skill under
-`.claude/skills/<name>/`, `--skill-agent codex` writes a Codex skill under
-`.agents/skills/<name>/` with `agents/openai.yaml`, and `--skill-agent cursor`
-writes a Cursor project rule at `.cursor/rules/<name>.mdc`. Use
-`--skill-agent all` to create all three. If you pass `--output-dir`, docpull
-stages the generated corpus there; explicit `--skill-agent` targets still write
-their active agent wrappers.
+`.claude/skills/<name>/`, and `--skill-agent cursor` writes a Cursor project
+rule at `.cursor/rules/<name>.mdc`. Use `--skill-agent all` to create every
+supported wrapper. If you pass `--output-dir`, docpull stages the generated
+corpus there; explicit `--skill-agent` targets still write their active agent
+wrappers.
 
 Use docpull when you need to:
 
@@ -591,6 +604,18 @@ MarkItDown or Unstructured parsers for complex office/PDF files when installed.
 Install `docpull[markitdown]`, `docpull[unstructured]`, or `docpull[parse]`
 for those backends.
 
+An HTTPS PDF can enter the ordinary fetch contract only with explicit consent:
+
+```bash
+docpull https://example.com/handbook.pdf --remote-documents pdf \
+  --remote-document-backend auto -o ./packs/handbook
+```
+
+The response must identify as PDF and contain a PDF signature. DocPull parses
+it locally through a mode-`0600` temporary file, records the source hash and
+parser provenance, and removes the temporary source. Other remote download
+types remain blocked.
+
 Every file-backed run writes `corpus.manifest.json` with stable document IDs,
 chunk IDs, hashes, output paths, and chunk counts. See
 [Corpus Manifest](docs/corpus-manifest.md).
@@ -719,7 +744,7 @@ part of the package release contract.
 - `docpull export` writes local files for OpenAI vector JSONL, LangChain,
   LlamaIndex, DSPy, Sheets CSV/TSV, n8n workflow JSON, Vercel AI SDK JSON,
   CrewAI JSON, warehouse NDJSON, optional Parquet via `docpull[parquet]`, and
-  Codex/Claude/Cursor agent references.
+  agent reference bundles.
 ## Security Defaults
 
 - HTTPS-only fetching with robots.txt compliance.
@@ -753,7 +778,7 @@ docpull URL --preview-urls
 
 ## Links
 
-- [Website](https://docpull.raintree.technology)
+- [Documentation](https://github.com/raintree-technology/docpull#readme)
 - [PyPI](https://pypi.org/project/docpull/)
 - [GitHub](https://github.com/raintree-technology/docpull)
 - [Metrics](METRICS.md)

{docpull-6.0.0 → docpull-6.1.0}/README.md

@@ -8,10 +8,12 @@
 [![GitHub stars](https://img.shields.io/github/stars/raintree-technology/docpull?style=social)](https://github.com/raintree-technology/docpull/stargazers)
 [![License: MIT](https://img.shields.io/github/license/raintree-technology/docpull)](https://github.com/raintree-technology/docpull/blob/main/LICENSE)
 
+<!-- mcp-name: io.github.raintree-technology/docpull -->
+
 DocPull is a local-first dependency manager for AI context. Define the public
-docs and web sources an agent depends on, sync them into cited context packs,
-diff what changed, and export reproducible context for Cursor, Claude, Codex,
-OpenAI, LlamaIndex, LangChain, MCP clients, and RAG pipelines.
+web sources an agent depends on, sync them into cited context packs, diff what
+changed, and export reproducible context for Cursor, Claude, OpenAI,
+LlamaIndex, LangChain, MCP clients, and RAG pipelines.
 
 The core workflow is a `docpull.yaml` plus a `.docpull/context.lock.json`,
 similar in spirit to code dependency manifests and lockfiles:
@@ -23,7 +25,7 @@ docpull install
 docpull deps
 docpull sync
 docpull diff
-docpull export context-pack --target codex
+docpull export context-pack --target openai
 ```
 
 Bundled aliases such as `stripe`, `react`, `postgres`, `openai`, and
@@ -61,6 +63,12 @@ pages such as blogs, API references, OpenAPI specs, changelogs, vendor pages,
 product pages, filings, docs sites, and other pages where the useful content is
 available in HTML or embedded page data.
 
+Direct fetches also recognize standards-style plain text and common textual
+formats from their response media type, including extensionless URLs. Remote
+documents remain denied by default; explicitly opt into local PDF parsing with
+`docpull URL --remote-documents pdf` after installing `docpull[markitdown]` or
+`docpull[parse]`. This does not enable a browser, cloud parser, or paid route.
+
 DocPull is browser-free by default. JS-only pages are skipped with a clear
 reason unless you explicitly opt into a local renderer. See
 [Web Source Boundary](docs/scraping-boundary.md) and
@@ -386,12 +394,11 @@ More examples live in [CLI Recipes](docs/examples/README.md).
 With an explicit `--skill-agent`, docpull stores the fetched corpus under
 `.docpull/skills/<name>/references` and creates agent-specific wrappers that
 point at that corpus. `--skill-agent claude` writes a Claude Code skill under
-`.claude/skills/<name>/`, `--skill-agent codex` writes a Codex skill under
-`.agents/skills/<name>/` with `agents/openai.yaml`, and `--skill-agent cursor`
-writes a Cursor project rule at `.cursor/rules/<name>.mdc`. Use
-`--skill-agent all` to create all three. If you pass `--output-dir`, docpull
-stages the generated corpus there; explicit `--skill-agent` targets still write
-their active agent wrappers.
+`.claude/skills/<name>/`, and `--skill-agent cursor` writes a Cursor project
+rule at `.cursor/rules/<name>.mdc`. Use `--skill-agent all` to create every
+supported wrapper. If you pass `--output-dir`, docpull stages the generated
+corpus there; explicit `--skill-agent` targets still write their active agent
+wrappers.
 
 Use docpull when you need to:
 
@@ -473,6 +480,18 @@ MarkItDown or Unstructured parsers for complex office/PDF files when installed.
 Install `docpull[markitdown]`, `docpull[unstructured]`, or `docpull[parse]`
 for those backends.
 
+An HTTPS PDF can enter the ordinary fetch contract only with explicit consent:
+
+```bash
+docpull https://example.com/handbook.pdf --remote-documents pdf \
+  --remote-document-backend auto -o ./packs/handbook
+```
+
+The response must identify as PDF and contain a PDF signature. DocPull parses
+it locally through a mode-`0600` temporary file, records the source hash and
+parser provenance, and removes the temporary source. Other remote download
+types remain blocked.
+
 Every file-backed run writes `corpus.manifest.json` with stable document IDs,
 chunk IDs, hashes, output paths, and chunk counts. See
 [Corpus Manifest](docs/corpus-manifest.md).
@@ -601,7 +620,7 @@ part of the package release contract.
 - `docpull export` writes local files for OpenAI vector JSONL, LangChain,
   LlamaIndex, DSPy, Sheets CSV/TSV, n8n workflow JSON, Vercel AI SDK JSON,
   CrewAI JSON, warehouse NDJSON, optional Parquet via `docpull[parquet]`, and
-  Codex/Claude/Cursor agent references.
+  agent reference bundles.
 ## Security Defaults
 
 - HTTPS-only fetching with robots.txt compliance.
@@ -635,7 +654,7 @@ docpull URL --preview-urls
 
 ## Links
 
-- [Website](https://docpull.raintree.technology)
+- [Documentation](https://github.com/raintree-technology/docpull#readme)
 - [PyPI](https://pypi.org/project/docpull/)
 - [GitHub](https://github.com/raintree-technology/docpull)
 - [Metrics](METRICS.md)

{docpull-6.0.0 → docpull-6.1.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "docpull"
-version = "6.0.0"
+version = "6.1.0"
 dynamic = []
 description = "Declare, sync, diff, and lock context dependencies for AI agents"
 readme = {file = "README.md", content-type = "text/markdown"}
@@ -123,13 +123,19 @@ parquet = [
     "pyarrow>=16.0.0",
 ]
 markitdown = [
-    "markitdown>=0.1.0",
+    "pypdf>=6.14.2",
+    "markitdown[docx,pdf]>=0.1.0",
 ]
 unstructured = [
+    "pypdf>=6.14.2",
     "unstructured[all-docs]>=0.16.0",
 ]
+pdf = [
+    "pypdf>=6.14.2",
+]
 parse = [
-    "markitdown>=0.1.0",
+    "pypdf>=6.14.2",
+    "markitdown[docx,pdf]>=0.1.0",
     "unstructured[all-docs]>=0.16.0",
 ]
 presidio = [
@@ -154,7 +160,8 @@ all = [
     "starlette>=1.3.1",
     "uvicorn>=0.30.0",
     "pyarrow>=16.0.0",
-    "markitdown>=0.1.0",
+    "pypdf>=6.14.2",
+    "markitdown[docx,pdf]>=0.1.0",
     "unstructured[all-docs]>=0.16.0",
     "presidio-analyzer>=2.2.0",
     "e2b>=2.0.0",
@@ -179,8 +186,8 @@ dev = [
 docpull = "docpull.cli:main"
 
 [project.urls]
-Homepage = "https://docpull.raintree.technology"
-Documentation = "https://docpull.raintree.technology"
+Homepage = "https://github.com/raintree-technology/docpull"
+Documentation = "https://github.com/raintree-technology/docpull#readme"
 Repository = "https://github.com/raintree-technology/docpull"
 "Source Code" = "https://github.com/raintree-technology/docpull"
 "Bug Tracker" = "https://github.com/raintree-technology/docpull/issues"

{docpull-6.0.0 → docpull-6.1.0}/src/docpull/__init__.py

@@ -1,5 +1,5 @@
 """
-docpull - Fetch and convert static/server-rendered web content to markdown.
+docpull - Context dependencies for AI agents.
 
 Usage:
     from docpull import Fetcher, DocpullConfig, ProfileName
@@ -14,7 +14,7 @@ Usage:
             print(event)
 """
 
-__version__ = "6.0.0"
+__version__ = "6.1.0"
 
 from .cache import CacheManager, StreamingDeduplicator
 from .context_ci import CIThresholds, ContextCIError, run_context_ci

{docpull-6.0.0 → docpull-6.1.0}/src/docpull/benchmark.py

@@ -200,8 +200,8 @@ TOOL_DOC_TARGETS: tuple[_BenchmarkTarget, ...] = (
     _BenchmarkTarget(
         id="docpull_docs",
         label="DocPull docs",
-        url="https://docpull.raintree.technology",
-        include_domains=("docpull.raintree.technology",),
+        url="https://github.com/raintree-technology/docpull",
+        include_domains=("github.com",),
         objective="Build an agent context pack for DocPull documentation",
         queries=("DocPull documentation CLI provider benchmark docs",),
         freshness_terms=("changelog", "release", "benchmark"),
@@ -963,13 +963,45 @@ def run_quick_benchmark(
 
     report_path = run_dir / "benchmark.report.json"
     markdown_path = run_dir / "benchmark.summary.md"
+    config_path = run_dir / "benchmark.config.json"
+    generated_at = utc_now_iso()
+    safe_provider_status = _benchmark_provider_statuses(provider_status)
     artifacts: dict[str, str] = {
+        "config": str(config_path),
         "json": str(report_path),
         "markdown": str(markdown_path),
     }
+    benchmark_config = {
+        "schema_version": BENCHMARK_SCHEMA_VERSION,
+        "generated_at": generated_at,
+        "run_dir": str(run_dir),
+        "target_url": targets[0].url,
+        "target_set": target_set,
+        "targets": [target.report_dict() for target in targets],
+        "requested_providers": requested_providers,
+        "enabled_providers": ["core", *providers],
+        "skipped_providers": skipped_providers,
+        "provider_status": safe_provider_status,
+        "runs_per_case": runs,
+        "budget_limit_usd": budget_limit,
+        "zero_dollar": zero_dollar,
+        "settings": {
+            "max_pages": max_pages,
+            "max_depth": max_depth,
+            "max_concurrent": max_concurrent,
+            "per_host_concurrent": per_host_concurrent,
+            "cache_enabled": cache_enabled,
+            "cached_pass": cached_pass,
+            "mode": mode,
+            "max_search_results": max_search_results,
+            "extract_limit": extract_limit,
+            "max_estimated_cost": max_estimated_cost,
+        },
+        "cost_normalization": _cost_normalization_metadata(tavily_credit_usd),
+    }
     report = {
         "schema_version": BENCHMARK_SCHEMA_VERSION,
-        "generated_at": utc_now_iso(),
+        "generated_at": generated_at,
         "run_dir": str(run_dir),
         "target_url": targets[0].url,
         "target_set": target_set,
@@ -981,7 +1013,7 @@ def run_quick_benchmark(
         "skipped_providers": skipped_providers,
         "budget_limit_usd": budget_limit,
         "zero_dollar": zero_dollar,
-        "provider_status": _benchmark_provider_statuses(provider_status),
+        "provider_status": safe_provider_status,
         "cost_normalization": _cost_normalization_metadata(tavily_credit_usd),
         "runs_per_case": runs,
         "trace": trace.metadata(),
@@ -1011,6 +1043,10 @@ def run_quick_benchmark(
         report["zero_dollar_completion"] = _zero_dollar_completion(cases)
     trace.finish(report)
     report["trace"] = trace.metadata()
+    config_path.write_text(
+        json.dumps(benchmark_config, indent=2, ensure_ascii=False) + "\n",
+        encoding="utf-8",
+    )
     report_path.write_text(json.dumps(report, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
     markdown_path.write_text(_markdown_report(report), encoding="utf-8")
     return report
@@ -3848,6 +3884,7 @@ def _article_markdown(report: dict[str, Any], *, title: str) -> str:
             "",
             "## Artifacts",
             "",
+            f"- Config: `{artifacts.get('config')}`",
             f"- JSON report: `{artifacts.get('json')}`",
             f"- Summary: `{artifacts.get('markdown')}`",
             "",
@@ -3977,6 +4014,7 @@ def _legacy_article_markdown(report: dict[str, Any], *, title: str) -> str:
             "",
             "## Artifacts",
             "",
+            f"- Config: `{artifacts.get('config')}`",
             f"- JSON report: `{artifacts.get('json')}`",
             f"- Summary: `{artifacts.get('markdown')}`",
             "",

{docpull-6.0.0 → docpull-6.1.0}/src/docpull/cli.py

@@ -335,6 +335,32 @@ Examples:
         action="store_true",
         help="Stream NDJSON records to stdout as each page completes (implies --format ndjson)",
     )
+    parser.add_argument(
+        "--remote-documents",
+        choices=["off", "pdf"],
+        default=None,
+        help="Explicitly download and locally parse selected remote document types (default: off)",
+    )
+    parser.add_argument(
+        "--remote-document-backend",
+        choices=["auto", "pypdf", "markitdown", "unstructured"],
+        default=None,
+        help="Local parser backend for --remote-documents (default: auto)",
+    )
+    parser.add_argument(
+        "--remote-document-timeout-seconds",
+        type=int,
+        default=None,
+        metavar="SECONDS",
+        help="Wall-time limit for each isolated remote-document parser (default: 60)",
+    )
+    parser.add_argument(
+        "--remote-document-memory-mib",
+        type=int,
+        default=None,
+        metavar="MIB",
+        help="Address-space limit for each isolated remote-document parser (default: 1024)",
+    )
 
     # Crawl settings
     crawl_group = parser.add_argument_group("crawl settings")
@@ -677,6 +703,14 @@ def run_fetcher(args: argparse.Namespace) -> int:
         filter_kwargs["enable_special_cases"] = False
     if args.strict_js_required:
         filter_kwargs["strict_js_required"] = True
+    if args.remote_documents:
+        filter_kwargs["remote_documents"] = args.remote_documents
+    if args.remote_document_backend:
+        filter_kwargs["remote_document_backend"] = args.remote_document_backend
+    if args.remote_document_timeout_seconds is not None:
+        filter_kwargs["remote_document_timeout_seconds"] = args.remote_document_timeout_seconds
+    if args.remote_document_memory_mib is not None:
+        filter_kwargs["remote_document_memory_mib"] = args.remote_document_memory_mib
     if filter_kwargs:
         config_kwargs["content_filter"] = filter_kwargs
 
@@ -1164,7 +1198,7 @@ def run_render_cli(argv: list[str]) -> int:
         args.cloud_max_estimated_cost if backend in {"vercel-sandbox", "e2b-sandbox"} else None,
     )
     cloud_estimated_cost = (
-        estimate_cloud_render_cost_usd(cast(Literal["vercel-sandbox", "e2b-sandbox"], backend), config)
+        estimate_cloud_render_cost_usd(backend, config)
         if backend in {"vercel-sandbox", "e2b-sandbox"}
         else 0.0
     )

{docpull-6.0.0 → docpull-6.1.0}/src/docpull/conversion/special_cases.py

@@ -16,6 +16,7 @@ import json
 import logging
 import re
 from dataclasses import dataclass
+from email.message import Message
 from typing import Any, Protocol
 from urllib.parse import urlparse
 
@@ -710,33 +711,143 @@ class RawTextExtractor:
 
     name = "raw_text"
 
-    _TEXT_SUFFIXES = (".md", ".markdown", ".mdx", ".txt")
+    _TEXT_SUFFIXES = (
+        ".csv",
+        ".json",
+        ".jsonl",
+        ".markdown",
+        ".md",
+        ".mdx",
+        ".ndjson",
+        ".rst",
+        ".text",
+        ".tsv",
+        ".txt",
+        ".xml",
+        ".yaml",
+        ".yml",
+    )
+    _TEXT_CONTENT_TYPES = {
+        "application/json",
+        "application/ld+json",
+        "application/x-ndjson",
+        "application/xml",
+        "application/yaml",
+        "application/x-yaml",
+        "text/csv",
+        "text/markdown",
+        "text/plain",
+        "text/tab-separated-values",
+        "text/x-markdown",
+        "text/x-rst",
+        "text/xml",
+        "text/yaml",
+    }
     _HTML_MARKERS = ("<html", "<body", "<!doctype")
-
-    def try_extract(self, html: bytes, url: str) -> SpecialCaseResult | None:
+    _STRUCTURED_LANGUAGES = {
+        ".csv": "csv",
+        ".json": "json",
+        ".jsonl": "ndjson",
+        ".ndjson": "ndjson",
+        ".rst": "rst",
+        ".tsv": "tsv",
+        ".xml": "xml",
+        ".yaml": "yaml",
+        ".yml": "yaml",
+    }
+    _MEDIA_LANGUAGES = {
+        "application/json": "json",
+        "application/ld+json": "json",
+        "application/x-ndjson": "ndjson",
+        "application/xml": "xml",
+        "application/yaml": "yaml",
+        "application/x-yaml": "yaml",
+        "text/csv": "csv",
+        "text/tab-separated-values": "tsv",
+        "text/x-rst": "rst",
+        "text/xml": "xml",
+        "text/yaml": "yaml",
+    }
+
+    def try_extract(
+        self,
+        html: bytes,
+        url: str,
+        *,
+        content_type: str | None = None,
+    ) -> SpecialCaseResult | None:
         parsed = urlparse(url)
         path = parsed.path.lower()
-        if not path.endswith(self._TEXT_SUFFIXES):
+        media_type = self._media_type(content_type)
+        identified_by_type = media_type in self._TEXT_CONTENT_TYPES
+        if not identified_by_type and not path.endswith(self._TEXT_SUFFIXES):
             return None
 
-        text = _decode_html(html).strip()
+        text = self._decode_text(html, content_type).strip()
         if not text:
             return None
+        language = self._structured_language(path, media_type)
         lower_head = text[:500].lower()
-        if any(marker in lower_head for marker in self._HTML_MARKERS):
+        if language is None and any(marker in lower_head for marker in self._HTML_MARKERS):
             return None
-        if path.endswith(".txt") and not self._looks_like_docs_text(text, path):
+        if (
+            path.endswith((".txt", ".text"))
+            and not identified_by_type
+            and not self._looks_like_docs_text(text, path)
+        ):
             return None
 
         title = self._extract_title(text) or parsed.path.rsplit("/", 1)[-1] or "Document"
         source_type = "llms_txt" if path.endswith(("/llms.txt", "/llms-full.txt")) else self.name
+        markdown = self._fenced(text, language) if language else self._safe_direct_markdown(text)
         return SpecialCaseResult(
-            markdown=text.rstrip() + "\n",
+            markdown=markdown,
             title=title,
             source_type=source_type,
             extra={"framework": source_type},
         )
 
+    @classmethod
+    def _structured_language(cls, path: str, media_type: str) -> str | None:
+        if media_type in {"text/markdown", "text/x-markdown"} or path.endswith((".markdown", ".md", ".mdx")):
+            return None
+        media_language = cls._MEDIA_LANGUAGES.get(media_type)
+        if media_language:
+            return media_language
+        return next(
+            (language for suffix, language in cls._STRUCTURED_LANGUAGES.items() if path.endswith(suffix)),
+            None,
+        )
+
+    @staticmethod
+    def _fenced(text: str, language: str) -> str:
+        longest_run = max((len(match.group(0)) for match in re.finditer(r"`+", text)), default=0)
+        fence = "`" * max(3, longest_run + 1)
+        return f"{fence}{language}\n{text.rstrip()}\n{fence}\n"
+
+    @staticmethod
+    def _safe_direct_markdown(text: str) -> str:
+        inert = re.sub(r"<(/?)script\b", r"&lt;\1script", text, flags=re.IGNORECASE)
+        return inert.rstrip() + "\n"
+
+    @staticmethod
+    def _media_type(content_type: str | None) -> str:
+        return (content_type or "").split(";", 1)[0].strip().casefold()
+
+    @staticmethod
+    def _decode_text(body: bytes, content_type: str | None) -> str:
+        if body.startswith(b"\xef\xbb\xbf"):
+            return body.decode("utf-8-sig", errors="replace")
+        encoding = "utf-8"
+        if content_type:
+            message = Message()
+            message["content-type"] = content_type
+            encoding = message.get_content_charset() or encoding
+        try:
+            return body.decode(encoding)
+        except (LookupError, UnicodeDecodeError):
+            return body.decode("utf-8", errors="replace")
+
     @staticmethod
     def _looks_like_docs_text(text: str, path: str) -> bool:
         if path.endswith(("/llms.txt", "/llms-full.txt")):
@@ -802,6 +913,33 @@ class SphinxObjectsInvExtractor:
         )
 
 
+class RfcEditorExtractor:
+    """Preserve complete RFC Editor HTML documents instead of one scored section."""
+
+    name = "rfc_editor"
+
+    def try_extract(self, html: bytes, url: str) -> SpecialCaseResult | None:
+        parsed = urlparse(url)
+        if not _hostname_matches_domain(parsed.hostname or "", "rfc-editor.org"):
+            return None
+        if re.fullmatch(r"/rfc/rfc\d+\.html", parsed.path.casefold()) is None:
+            return None
+
+        from .extractor import MainContentExtractor
+        from .markdown import HtmlToMarkdown
+
+        extracted_html = MainContentExtractor(content_selectors=["body"]).extract(html, url)
+        markdown = HtmlToMarkdown().convert(extracted_html, url).strip()
+        if len(markdown) < 80:
+            return None
+        return SpecialCaseResult(
+            markdown=markdown + "\n",
+            title=_title_from_soup(_soup(html)),
+            source_type=self.name,
+            extra={"framework": "rfc_editor"},
+        )
+
+
 class MdxSourceExtractor:
     """Rewrite ``edit-this-page`` GitHub links to raw MDX source URLs.
 
@@ -842,6 +980,7 @@ class MdxSourceExtractor:
 DEFAULT_CHAIN: list[SpecialCaseExtractor] = [
     OpenApiExtractor(),
     RawTextExtractor(),
+    RfcEditorExtractor(),
     MintlifyExtractor(),
     NextDataExtractor(),
     MkDocsMaterialExtractor(),
@@ -969,6 +1108,7 @@ __all__ = [
     "RawTextExtractor",
     "ReadMeExtractor",
     "RedocScalarExtractor",
+    "RfcEditorExtractor",
     "SpecialCaseExtractor",
     "SpecialCaseResult",
     "SphinxObjectsInvExtractor",

{docpull-6.0.0 → docpull-6.1.0}/src/docpull/conversion/trafilatura_extractor.py

@@ -63,7 +63,7 @@ class TrafilaturaExtractor:
         if result is None:
             logger.debug("trafilatura returned no content for %s", url)
             return ""
-        return result.strip() + "\n"
+        return str(result).strip() + "\n"
 
 
 __all__ = ["TrafilaturaExtractor"]