docpull 4.3.1__tar.gz → 4.4.0__tar.gz

{docpull-4.3.1 → docpull-4.4.0}/LICENSE

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2025 Raintree Technology
+Copyright (c) 2025-2026 Raintree Technology
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

{docpull-4.3.1/src/docpull.egg-info → docpull-4.4.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: docpull
-Version: 4.3.1
+Version: 4.4.0
 Summary: Convert public web pages into clean Markdown for AI agents and RAG
 Author-email: Zachary Roth <support@raintree.technology>
 Maintainer-email: Raintree Technology <support@raintree.technology>
@@ -15,16 +15,14 @@ Project-URL: MCP Plugin, https://github.com/raintree-technology/docpull/tree/mai
 Project-URL: Comparison Guide, https://github.com/raintree-technology/docpull/blob/main/docs/alternatives.md
 Project-URL: Download Stats, https://pepy.tech/project/docpull
 Project-URL: Changelog, https://github.com/raintree-technology/docpull/blob/main/docs/CHANGELOG.md
-Keywords: python,markdown,documentation,web-scraping,web-crawler,crawler,developer-tools,cli,rag,llm,mcp,ai-agents,claude,ai-training-data
+Keywords: python,markdown,documentation,web-extraction,web-scraping,web-crawler,crawler,developer-tools,cli,rag,llm,mcp,ai-agents,context-engineering,claude,ai-training-data
 Classifier: Development Status :: 5 - Production/Stable
 Classifier: Intended Audience :: Developers
 Classifier: Intended Audience :: Information Technology
 Classifier: Intended Audience :: Science/Research
 Classifier: Intended Audience :: Education
 Classifier: Environment :: Console
-Classifier: Topic :: Documentation
 Classifier: Topic :: Internet :: WWW/HTTP :: Indexing/Search
-Classifier: Topic :: Software Development :: Documentation
 Classifier: Topic :: Text Processing :: Markup :: HTML
 Classifier: Topic :: Text Processing :: Markup :: Markdown
 Classifier: Topic :: Utilities
@@ -54,7 +52,7 @@ Requires-Dist: pyyaml>=6.0
 Requires-Dist: pydantic>=2.0
 Requires-Dist: urllib3>=2.7.0
 Provides-Extra: proxy
-Requires-Dist: aiohttp-socks>=0.8.0; extra == "proxy"
+Requires-Dist: aiohttp-socks>=0.11.0; extra == "proxy"
 Provides-Extra: normalize
 Requires-Dist: url-normalize>=1.4.0; extra == "normalize"
 Provides-Extra: trafilatura
@@ -73,7 +71,7 @@ Requires-Dist: raindrop-ai>=0.0.50; extra == "observability"
 Provides-Extra: llm
 Requires-Dist: tiktoken>=0.7.0; extra == "llm"
 Provides-Extra: all
-Requires-Dist: aiohttp-socks>=0.8.0; extra == "all"
+Requires-Dist: aiohttp-socks>=0.11.0; extra == "all"
 Requires-Dist: url-normalize>=1.4.0; extra == "all"
 Requires-Dist: trafilatura>=1.12.0; extra == "all"
 Requires-Dist: tiktoken>=0.7.0; extra == "all"
@@ -116,9 +114,9 @@ Dynamic: license-file
 
 docpull is a Python CLI, SDK, and MCP server for pulling public static and
 server-rendered web pages into clean Markdown, NDJSON, OKF bundles, SQLite, or
-local archives. Documentation crawling is its sharpest default workflow, but
-the product is broader than docs: blogs, API references, vendor pages, OpenAPI
-specs, and other server-rendered web content all fit. It uses async HTTP instead
+local archives. Web-source extraction is the core workflow; docs are one
+high-value lane alongside blogs, API references, vendor pages, OpenAPI specs,
+and other server-rendered web content. It uses async HTTP instead
 of Playwright, preserves source metadata, and is built for agent-selected URLs
 with SSRF, XXE, DNS-rebinding, and CRLF-injection protections enabled by
 default.
@@ -173,6 +171,13 @@ docpull https://docs.example.com --profile llm --stream | jq .
 # Open Knowledge Format bundle for agent/wiki interoperability
 docpull https://example.com --format okf -o ./site-okf
 
+# SEC filing evidence pack with citation anchors
+docpull evidence-pack filings.ndjson \
+  --profile sec-filing \
+  --rules docs/examples/vendor-dependency-rules.yml \
+  --sec-user-agent "YourOrg your-email@example.com" \
+  --output-dir packs/dla-vendors
+
 # Mirror scraped content for offline use
 docpull https://docs.example.com --profile mirror --cache
 ```
@@ -296,15 +301,37 @@ async def tool_call(url: str) -> str:
 docpull https://site.com --profile rag      # Default. Dedup, rich metadata.
 docpull https://site.com --profile llm      # NDJSON + chunks + metadata; JS-only pages skip unless --strict-js-required is passed.
 docpull https://site.com --profile okf      # OKF bundle with generated index.md files.
+docpull https://site.com --profile sec-filing # EDGAR-friendly NDJSON chunks, trafilatura, Inline XBRL cleanup.
 docpull https://site.com --profile mirror   # Full archive, polite, cached, hierarchical paths.
 docpull https://site.com --profile quick    # Sampling: 50 pages, depth 2.
 ```
 
+## SEC filing evidence packs
+
+`docpull evidence-pack` turns an NDJSON list of filings into a source pack for
+government/vendor-dependency research. Each input row needs a filing URL under
+`primary_document_url` or `url`; filing fields such as `cik`,
+`accession_number`, `form`, `filing_date`, and `issuer_name` are carried into
+the emitted records.
+
+Use `docs/examples/vendor-dependency-rules.yml` as a starting rule profile for
+government customer, customer concentration, segment revenue, and related-party
+signals. Rules can use literal strings or regex entries with optional
+confidence values.
+
+The pack writes `evidence.pack.json`, `documents.ndjson`, `evidence.ndjson`,
+`diagnostics.ndjson`, `sources.md`, `corpus.manifest.json`,
+`EVIDENCE_CONTEXT.md`, and `AGENT_CONTEXT.md`. Evidence hits include source URL,
+chunk id, section heading when available, quote, surrounding context, source
+document hash, confidence, and extraction method. Diagnostics flag no readable
+content, high XBRL noise, degraded tables, missing evidence categories, and
+source hash changes since the last run.
+
 ## Parallel context packs
 
 `docpull[parallel]` adds an optional source-discovery and research layer on top
 of Parallel web intelligence APIs. Use the core crawler when you already know
-the docs URL. Use Parallel packs when an agent needs current web sources found,
+the target URL. Use Parallel packs when an agent needs current web sources found,
 extracted, researched, organized, and loaded as durable local context.
 
 Parallel handles live Search, Extract, Task, Entity Search, FindAll, TaskGroup,
@@ -527,7 +554,7 @@ Write:
 - `parallel_context_pack(...)` — build or dry-run a Parallel Search + Extract pack without shelling out.
 - `parallel_api_pack(source, kind?, output_dir?)` — build a pack from `llms.txt` or OpenAPI.
 - `add_source(name, url, description?, category?, max_pages?, force?)` — register a user alias (HTTPS-only, atomic write to `sources.yaml`).
-- `remove_source(name, delete_cache?)` — drop a user alias and (optionally) its cached docs.
+- `remove_source(name, delete_cache?)` — drop a user alias and (optionally) its cached Markdown.
 
 All schema-backed tools return `structuredContent` validated against an
 `outputSchema` for clients that prefer typed output. `fetch_url` intentionally
@@ -641,6 +668,8 @@ for hit in hits:
 When running with `--proxy`, DNS pinning is delegated to the proxy. Pass
 `--require-pinned-dns` to refuse this configuration and keep the connector-
 level SSRF guarantees in effect.
+HTTP and HTTPS proxies work with the base install; SOCKS proxies require
+`pip install "docpull[proxy]"`.
 
 ## Options
 
@@ -648,7 +677,7 @@ Run `docpull --help` for the full list. Highlights:
 
 ```
 Core:
-  --profile {rag,mirror,quick,llm,okf}
+  --profile {rag,mirror,quick,llm,okf,sec-filing}
   --single                Fetch one URL (no crawl)
   --format {markdown,json,ndjson,sqlite,okf}
   --stream                Stream NDJSON to stdout

{docpull-4.3.1 → docpull-4.4.0}/README.md

@@ -16,9 +16,9 @@
 
 docpull is a Python CLI, SDK, and MCP server for pulling public static and
 server-rendered web pages into clean Markdown, NDJSON, OKF bundles, SQLite, or
-local archives. Documentation crawling is its sharpest default workflow, but
-the product is broader than docs: blogs, API references, vendor pages, OpenAPI
-specs, and other server-rendered web content all fit. It uses async HTTP instead
+local archives. Web-source extraction is the core workflow; docs are one
+high-value lane alongside blogs, API references, vendor pages, OpenAPI specs,
+and other server-rendered web content. It uses async HTTP instead
 of Playwright, preserves source metadata, and is built for agent-selected URLs
 with SSRF, XXE, DNS-rebinding, and CRLF-injection protections enabled by
 default.
@@ -73,6 +73,13 @@ docpull https://docs.example.com --profile llm --stream | jq .
 # Open Knowledge Format bundle for agent/wiki interoperability
 docpull https://example.com --format okf -o ./site-okf
 
+# SEC filing evidence pack with citation anchors
+docpull evidence-pack filings.ndjson \
+  --profile sec-filing \
+  --rules docs/examples/vendor-dependency-rules.yml \
+  --sec-user-agent "YourOrg your-email@example.com" \
+  --output-dir packs/dla-vendors
+
 # Mirror scraped content for offline use
 docpull https://docs.example.com --profile mirror --cache
 ```
@@ -196,15 +203,37 @@ async def tool_call(url: str) -> str:
 docpull https://site.com --profile rag      # Default. Dedup, rich metadata.
 docpull https://site.com --profile llm      # NDJSON + chunks + metadata; JS-only pages skip unless --strict-js-required is passed.
 docpull https://site.com --profile okf      # OKF bundle with generated index.md files.
+docpull https://site.com --profile sec-filing # EDGAR-friendly NDJSON chunks, trafilatura, Inline XBRL cleanup.
 docpull https://site.com --profile mirror   # Full archive, polite, cached, hierarchical paths.
 docpull https://site.com --profile quick    # Sampling: 50 pages, depth 2.
 ```
 
+## SEC filing evidence packs
+
+`docpull evidence-pack` turns an NDJSON list of filings into a source pack for
+government/vendor-dependency research. Each input row needs a filing URL under
+`primary_document_url` or `url`; filing fields such as `cik`,
+`accession_number`, `form`, `filing_date`, and `issuer_name` are carried into
+the emitted records.
+
+Use `docs/examples/vendor-dependency-rules.yml` as a starting rule profile for
+government customer, customer concentration, segment revenue, and related-party
+signals. Rules can use literal strings or regex entries with optional
+confidence values.
+
+The pack writes `evidence.pack.json`, `documents.ndjson`, `evidence.ndjson`,
+`diagnostics.ndjson`, `sources.md`, `corpus.manifest.json`,
+`EVIDENCE_CONTEXT.md`, and `AGENT_CONTEXT.md`. Evidence hits include source URL,
+chunk id, section heading when available, quote, surrounding context, source
+document hash, confidence, and extraction method. Diagnostics flag no readable
+content, high XBRL noise, degraded tables, missing evidence categories, and
+source hash changes since the last run.
+
 ## Parallel context packs
 
 `docpull[parallel]` adds an optional source-discovery and research layer on top
 of Parallel web intelligence APIs. Use the core crawler when you already know
-the docs URL. Use Parallel packs when an agent needs current web sources found,
+the target URL. Use Parallel packs when an agent needs current web sources found,
 extracted, researched, organized, and loaded as durable local context.
 
 Parallel handles live Search, Extract, Task, Entity Search, FindAll, TaskGroup,
@@ -427,7 +456,7 @@ Write:
 - `parallel_context_pack(...)` — build or dry-run a Parallel Search + Extract pack without shelling out.
 - `parallel_api_pack(source, kind?, output_dir?)` — build a pack from `llms.txt` or OpenAPI.
 - `add_source(name, url, description?, category?, max_pages?, force?)` — register a user alias (HTTPS-only, atomic write to `sources.yaml`).
-- `remove_source(name, delete_cache?)` — drop a user alias and (optionally) its cached docs.
+- `remove_source(name, delete_cache?)` — drop a user alias and (optionally) its cached Markdown.
 
 All schema-backed tools return `structuredContent` validated against an
 `outputSchema` for clients that prefer typed output. `fetch_url` intentionally
@@ -541,6 +570,8 @@ for hit in hits:
 When running with `--proxy`, DNS pinning is delegated to the proxy. Pass
 `--require-pinned-dns` to refuse this configuration and keep the connector-
 level SSRF guarantees in effect.
+HTTP and HTTPS proxies work with the base install; SOCKS proxies require
+`pip install "docpull[proxy]"`.
 
 ## Options
 
@@ -548,7 +579,7 @@ Run `docpull --help` for the full list. Highlights:
 
 ```
 Core:
-  --profile {rag,mirror,quick,llm,okf}
+  --profile {rag,mirror,quick,llm,okf,sec-filing}
   --single                Fetch one URL (no crawl)
   --format {markdown,json,ndjson,sqlite,okf}
   --stream                Stream NDJSON to stdout

{docpull-4.3.1 → docpull-4.4.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "docpull"
-version = "4.3.1"
+version = "4.4.0"
 dynamic = []
 description = "Convert public web pages into clean Markdown for AI agents and RAG"
 readme = {file = "README.md", content-type = "text/markdown"}
@@ -21,6 +21,7 @@ keywords = [
     "python",
     "markdown",
     "documentation",
+    "web-extraction",
     "web-scraping",
     "web-crawler",
     "crawler",
@@ -30,6 +31,7 @@ keywords = [
     "llm",
     "mcp",
     "ai-agents",
+    "context-engineering",
     "claude",
     "ai-training-data",
 ]
@@ -47,9 +49,7 @@ classifiers = [
     "Environment :: Console",
 
     # Topic
-    "Topic :: Documentation",
     "Topic :: Internet :: WWW/HTTP :: Indexing/Search",
-    "Topic :: Software Development :: Documentation",
     "Topic :: Text Processing :: Markup :: HTML",
     "Topic :: Text Processing :: Markup :: Markdown",
     "Topic :: Utilities",
@@ -79,17 +79,17 @@ dependencies = [
     "defusedxml>=0.7.1",
     "extruct>=0.15.0",
     "aiohttp>=3.14.0",  # 3.14.0 fixes CVE-2026-34993 and CVE-2026-47265
-    "idna>=3.15",
+    "idna>=3.15",  # Security floor for transitive URL/IDNA handling; Dependabot raises as needed
     "regex>=2024.11.6",
     "rich>=13.0.0",
     "pyyaml>=6.0",
     "pydantic>=2.0",
-    "urllib3>=2.7.0",
+    "urllib3>=2.7.0",  # Security floor for transitive HTTP clients; Dependabot raises as needed
 ]
 
 [project.optional-dependencies]
 proxy = [
-    "aiohttp-socks>=0.8.0",
+    "aiohttp-socks>=0.11.0",
 ]
 normalize = [
     "url-normalize>=1.4.0",
@@ -116,7 +116,7 @@ llm = [
     "tiktoken>=0.7.0",
 ]
 all = [
-    "aiohttp-socks>=0.8.0",
+    "aiohttp-socks>=0.11.0",
     "url-normalize>=1.4.0",
     "trafilatura>=1.12.0",
     "tiktoken>=0.7.0",

{docpull-4.3.1 → docpull-4.4.0}/src/docpull/__init__.py

@@ -1,5 +1,5 @@
 """
-docpull - Fetch and convert static/server-rendered documentation to markdown.
+docpull - Fetch and convert static/server-rendered web content to markdown.
 
 Usage:
     from docpull import Fetcher, DocpullConfig, ProfileName
@@ -14,7 +14,7 @@ Usage:
             print(event)
 """
 
-__version__ = "4.3.1"
+__version__ = "4.4.0"
 
 from .cache import CacheManager, StreamingDeduplicator
 from .conversion.chunking import Chunk, TokenCounter, chunk_markdown

{docpull-4.3.1 → docpull-4.4.0}/src/docpull/benchmark.py

@@ -237,7 +237,7 @@ def create_benchmark_parser() -> argparse.ArgumentParser:
         "quick",
         help="Run a small real-site benchmark and write JSON/Markdown reports",
     )
-    quick.add_argument("--target-url", default=DEFAULT_TARGET_URL, help="Docs URL to crawl")
+    quick.add_argument("--target-url", default=DEFAULT_TARGET_URL, help="Target URL to crawl")
     quick.add_argument(
         "--target-set",
         choices=TARGET_SET_CHOICES,

{docpull-4.3.1 → docpull-4.4.0}/src/docpull/cli.py

@@ -48,7 +48,7 @@ def create_parser() -> argparse.ArgumentParser:
     """Create argument parser for CLI."""
     parser = argparse.ArgumentParser(
         prog="docpull",
-        description="Fetch and convert static/server-rendered documentation to markdown",
+        description="Fetch and convert static/server-rendered web content to markdown",
         formatter_class=argparse.RawDescriptionHelpFormatter,
         epilog="""
 Examples:
@@ -69,7 +69,7 @@ Examples:
     parser.add_argument(
         "url",
         nargs="?",
-        help="URL to fetch documentation from",
+        help="URL to fetch content from",
     )
 
     parser.add_argument(
@@ -87,9 +87,12 @@ Examples:
     parser.add_argument(
         "--profile",
         "-p",
-        choices=["rag", "mirror", "quick", "llm", "okf"],
+        choices=["rag", "mirror", "quick", "llm", "okf", "sec-filing"],
         default="rag",
-        help="Preset profile (default: rag). 'llm' streams chunked NDJSON; 'okf' writes an OKF bundle.",
+        help=(
+            "Preset profile (default: rag). 'llm' streams chunked NDJSON; "
+            "'okf' writes an OKF bundle; 'sec-filing' tunes extraction for EDGAR filings."
+        ),
     )
 
     parser.add_argument(
@@ -257,7 +260,7 @@ Examples:
         "--proxy",
         type=str,
         metavar="URL",
-        help="Proxy URL",
+        help="HTTP, HTTPS, or SOCKS proxy URL (SOCKS requires docpull[proxy])",
     )
     network_group.add_argument(
         "--user-agent",
@@ -384,6 +387,7 @@ def run_fetcher(args: argparse.Namespace) -> int:
         "quick": ProfileName.QUICK,
         "llm": ProfileName.LLM,
         "okf": ProfileName.OKF,
+        "sec-filing": ProfileName.SEC_FILING,
     }
     profile = profile_map.get(args.profile, ProfileName.RAG)
 
@@ -683,6 +687,10 @@ def main(argv: list[str] | None = None) -> int:
         from .pack_tools import run_pack_cli
 
         return run_pack_cli(raw_argv[1:])
+    if raw_argv and raw_argv[0] == "evidence-pack":
+        from .evidence_pack import run_evidence_pack_cli
+
+        return run_evidence_pack_cli(raw_argv[1:])
     if raw_argv and raw_argv[0] == "benchmark":
         from .benchmark import run_benchmark_cli
 

{docpull-4.3.1 → docpull-4.4.0}/src/docpull/conversion/extractor.py

@@ -196,12 +196,10 @@ class MainContentExtractor:
         keep_attrs = {"href", "src", "alt", "title", "class", "id"}
 
         for tag in element.find_all(True):
-            # Get list of attrs to remove (can't modify during iteration)
             attrs_to_remove = [attr for attr in tag.attrs if attr not in keep_attrs]
             for attr in attrs_to_remove:
                 del tag[attr]
 
-            # Remove empty class/id
             if tag.get("class") == []:
                 del tag["class"]
             if tag.get("id") == "":
@@ -211,7 +209,6 @@ class MainContentExtractor:
         """Convert relative URLs to absolute URLs."""
         urlparse(base_url)
 
-        # Resolve href attributes
         for tag in element.find_all("a", href=True):
             href = tag["href"]
             if href.startswith("#"):
@@ -219,7 +216,6 @@ class MainContentExtractor:
             if not href.startswith(("http://", "https://", "//")):
                 tag["href"] = urljoin(base_url, href)
 
-        # Resolve src attributes
         for tag in element.find_all(src=True):
             src = tag["src"]
             if not src.startswith(("http://", "https://", "//", "data:")):
@@ -227,11 +223,8 @@ class MainContentExtractor:
 
     def _clean_whitespace(self, text: str) -> str:
         """Clean up excessive whitespace."""
-        # Normalize line endings
         text = text.replace("\r\n", "\n").replace("\r", "\n")
-        # Remove excessive blank lines (more than 2)
         text = re.sub(r"\n{3,}", "\n\n", text)
-        # Remove trailing whitespace on lines
         text = "\n".join(line.rstrip() for line in text.split("\n"))
         return text.strip()
 
@@ -248,7 +241,6 @@ class MainContentExtractor:
         """
         soup = self._parse_html(html)
 
-        # Find main content
         main_content = self._find_main_content(soup)
         if main_content is None:
             # Try the entire document as fallback
@@ -261,7 +253,6 @@ class MainContentExtractor:
         # Make a copy to avoid modifying original
         content = BeautifulSoup(str(main_content), "html.parser")
 
-        # Clean up
         # Math-rendering libraries often hide the source TeX in script,
         # MathML annotation, or aria-hidden wrappers that the generic cleanup
         # removes. Normalize those to Markdown text before deleting chrome.

docpull-4.4.0/src/docpull/conversion/filings.py

@@ -0,0 +1,57 @@
+"""SEC filing-oriented HTML cleanup helpers."""
+
+from __future__ import annotations
+
+from bs4 import BeautifulSoup, Tag
+
+_HIDDEN_INLINE_XBRL_TAGS = {"ix:hidden", "ix:header", "ix:references", "ix:resources"}
+
+
+def clean_inline_xbrl_html(html: bytes) -> tuple[bytes, dict[str, int]]:
+    """Remove hidden Inline XBRL scaffolding while preserving visible facts.
+
+    SEC filings often include machine-readable Inline XBRL tags around visible
+    text plus hidden header/resources blocks. The visible ``ix:*`` tags are
+    unwrapped so their text remains available, while hidden sections and
+    browser-hidden elements are removed before extraction.
+    """
+    text = html.decode("utf-8", errors="replace")
+    soup = BeautifulSoup(text, "html.parser")
+    stats = {
+        "hidden_inline_xbrl_removed": 0,
+        "hidden_elements_removed": 0,
+        "inline_xbrl_tags_unwrapped": 0,
+    }
+
+    for tag in list(soup.find_all(True)):
+        if not isinstance(tag, Tag):
+            continue
+        name = (tag.name or "").lower()
+        if not name:
+            continue
+        if name in _HIDDEN_INLINE_XBRL_TAGS:
+            tag.decompose()
+            stats["hidden_inline_xbrl_removed"] += 1
+            continue
+        if _is_hidden_element(tag):
+            tag.decompose()
+            stats["hidden_elements_removed"] += 1
+            continue
+        if name.startswith("ix:"):
+            tag.unwrap()
+            stats["inline_xbrl_tags_unwrapped"] += 1
+
+    return str(soup).encode("utf-8"), stats
+
+
+def _is_hidden_element(tag: Tag) -> bool:
+    if tag.has_attr("hidden") or tag.get("aria-hidden") == "true":
+        return True
+    style = str(tag.get("style") or "").replace(" ", "").lower()
+    if "display:none" in style or "visibility:hidden" in style:
+        return True
+    classes = tag.get("class")
+    return isinstance(classes, list) and any(str(item).lower() == "hidden" for item in classes)
+
+
+__all__ = ["clean_inline_xbrl_html"]

{docpull-4.3.1 → docpull-4.4.0}/src/docpull/conversion/markdown.py

@@ -384,13 +384,10 @@ class HtmlToMarkdown:
         markdown = _restore_latex_math_delimiters(markdown)
         markdown = _rewrite_markdown_links(markdown, _normalize_protected_absolute_destination)
 
-        # Remove excessive blank lines
         markdown = re.sub(r"\n{3,}", "\n\n", markdown)
 
-        # Remove trailing whitespace on each line
         markdown = "\n".join(line.rstrip() for line in markdown.split("\n"))
 
-        # Ensure single newline at end
         return markdown.strip() + "\n"
 
     def _fix_relative_links(self, markdown: str, base_url: str) -> str:
@@ -412,13 +409,10 @@ class HtmlToMarkdown:
             Markdown string
         """
         try:
-            # Set base URL for link resolution
             self._converter.baseurl = url
 
-            # Convert to markdown
             markdown = self._converter.handle(html)
 
-            # Clean up output
             markdown = self._clean_output(markdown)
 
             # Fix any remaining relative links
@@ -518,7 +512,7 @@ class FrontmatterBuilder:
     def build_okf(
         self,
         *,
-        concept_type: str = "Documentation Page",
+        concept_type: str = "Web Page",
         title: str | None = None,
         resource: str | None = None,
         description: str | None = None,

{docpull-4.3.1 → docpull-4.4.0}/src/docpull/core/fetcher.py

@@ -54,23 +54,18 @@ def _url_to_filename(url: str, base_url: str | None = None) -> str:
     parsed = urlparse(url)
     path = parsed.path.strip("/")
 
-    # Remove base URL prefix if provided
     if base_url:
         base_path = urlparse(base_url).path.strip("/")
         if path.startswith(base_path):
             path = path[len(base_path) :].strip("/")
 
-    # Convert path to filename
     if not path or path == "/":
         filename = "index"
     else:
-        # Replace path separators with underscores
         filename = path.replace("/", "_")
-        # Remove file extension if present
         if filename.endswith(".html") or filename.endswith(".htm"):
             filename = filename.rsplit(".", 1)[0]
 
-    # Clean up filename
     filename = re.sub(r"[^\w\-]", "_", filename)
     filename = re.sub(r"_+", "_", filename)
     filename = filename.strip("_")
@@ -402,6 +397,7 @@ class Fetcher:
                 enable_special_cases=self.config.content_filter.enable_special_cases,
                 use_trafilatura=self.config.content_filter.extractor == "trafilatura",
                 strict_js_required=self.config.content_filter.strict_js_required,
+                clean_inline_xbrl=self.config.content_filter.clean_inline_xbrl,
             )
         )
 

{docpull-4.3.1 → docpull-4.4.0}/src/docpull/discovery/link_extractors/enhanced.py

@@ -274,16 +274,13 @@ class EnhancedLinkExtractor:
             logger.debug("Could not resolve href %r against %s: %s", href, base_url, err)
             return None
 
-        # Validate it's a proper URL
         parsed = urlparse(absolute_url)
         if not parsed.scheme or not parsed.netloc:
             return None
 
-        # Only allow http/https
         if parsed.scheme not in ("http", "https"):
             return None
 
-        # Remove fragment
         clean_url = f"{parsed.scheme}://{parsed.netloc}{parsed.path}"
         if parsed.query:
             clean_url += f"?{parsed.query}"

{docpull-4.3.1 → docpull-4.4.0}/src/docpull/discovery/link_extractors/static.py

@@ -126,7 +126,6 @@ class StaticLinkExtractor:
             logger.debug("Could not resolve href %r against %s: %s", href, base_url, err)
             return None
 
-        # Remove fragment
         parsed = urlparse(absolute_url)
         clean_url = f"{parsed.scheme}://{parsed.netloc}{parsed.path}"
         if parsed.query:

{docpull-4.3.1 → docpull-4.4.0}/src/docpull/discovery/sitemap.py

@@ -170,12 +170,10 @@ class SitemapDiscoverer:
             ns = SITEMAP_NS if use_ns else {}
             prefix = "ns:" if use_ns else ""
 
-            # Find page URLs
             for url_elem in root.findall(f".//{prefix}url/{prefix}loc", ns):
                 if url_elem.text:
                     page_urls.append(url_elem.text.strip())
 
-            # Find nested sitemap URLs
             for sitemap_elem in root.findall(f".//{prefix}sitemap/{prefix}loc", ns):
                 if sitemap_elem.text:
                     sitemap_urls.append(sitemap_elem.text.strip())
@@ -229,14 +227,12 @@ class SitemapDiscoverer:
             if self._filter and not self._filter.should_include(url):
                 continue
 
-            # Skip duplicates
             if not self._seen.add(url):
                 continue
 
             yield url
             count += 1
 
-        # Process nested sitemaps
         remaining = max_urls - count if max_urls is not None else None
         for nested_url in nested_sitemaps:
             if remaining is not None and remaining <= 0: