docpull 1.0.2__tar.gz → 1.1.0__tar.gz

{docpull-1.0.2 → docpull-1.1.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: docpull
-Version: 1.0.2
+Version: 1.1.0
 Summary: Pull documentation from the web and convert to clean markdown
 Author-email: Zachary Roth <support@raintree.technology>
 Maintainer-email: Raintree Technology <support@raintree.technology>
@@ -62,7 +62,7 @@ Requires-Dist: bandit>=1.7.0; extra == "dev"
 Requires-Dist: pip-audit>=2.0.0; extra == "dev"
 Requires-Dist: types-requests>=2.31.0; extra == "dev"
 Requires-Dist: types-beautifulsoup4>=4.12.0; extra == "dev"
-Requires-Dist: types-aiohttp>=3.9.0; extra == "dev"
+Requires-Dist: types-defusedxml>=0.7.0; extra == "dev"
 Dynamic: license-file
 
 # docpull
@@ -97,6 +97,7 @@ Unlike tools like wget or httrack, docpull extracts only the main content, remov
 
 ```bash
 pip install docpull
+docpull --doctor         # verify installation
 docpull https://aptos.dev
 docpull stripe           # use a built-in profile
 docpull https://site.com/docs --max-pages 100 --max-concurrent 20
@@ -126,6 +127,7 @@ fetcher.fetch()
 
 ## Common Options
 
+- `--doctor` – verify installation and dependencies
 - `--max-pages N` – limit crawl size
 - `--max-depth N` – restrict link depth
 - `--max-concurrent N` – control parallel fetches
@@ -200,10 +202,14 @@ MY_PROFILE = SiteProfile(
 
 ## Troubleshooting
 
+- **Installation issues**: Run `docpull --doctor` to diagnose problems
+- **Missing dependencies**: See [TROUBLESHOOTING.md](TROUBLESHOOTING.md) for common fixes
 - **Site requires JS**: install Playwright + `--js`
 - **Slow or rate limited**: lower concurrency or raise `--rate-limit`
 - **Large sites**: set `--max-pages`
 
+For detailed troubleshooting, see [TROUBLESHOOTING.md](TROUBLESHOOTING.md).
+
 ## Links
 
 - [PyPI](https://pypi.org/project/docpull/)

{docpull-1.0.2 → docpull-1.1.0}/README.md

@@ -30,6 +30,7 @@ Unlike tools like wget or httrack, docpull extracts only the main content, remov
 
 ```bash
 pip install docpull
+docpull --doctor         # verify installation
 docpull https://aptos.dev
 docpull stripe           # use a built-in profile
 docpull https://site.com/docs --max-pages 100 --max-concurrent 20
@@ -59,6 +60,7 @@ fetcher.fetch()
 
 ## Common Options
 
+- `--doctor` – verify installation and dependencies
 - `--max-pages N` – limit crawl size
 - `--max-depth N` – restrict link depth
 - `--max-concurrent N` – control parallel fetches
@@ -133,10 +135,14 @@ MY_PROFILE = SiteProfile(
 
 ## Troubleshooting
 
+- **Installation issues**: Run `docpull --doctor` to diagnose problems
+- **Missing dependencies**: See [TROUBLESHOOTING.md](TROUBLESHOOTING.md) for common fixes
 - **Site requires JS**: install Playwright + `--js`
 - **Slow or rate limited**: lower concurrency or raise `--rate-limit`
 - **Large sites**: set `--max-pages`
 
+For detailed troubleshooting, see [TROUBLESHOOTING.md](TROUBLESHOOTING.md).
+
 ## Links
 
 - [PyPI](https://pypi.org/project/docpull/)

{docpull-1.0.2 → docpull-1.1.0}/docpull/__init__.py

@@ -1,4 +1,4 @@
-__version__ = "1.0.2"
+__version__ = "1.1.0"
 
 from .fetchers.base import BaseFetcher
 from .fetchers.bun import BunFetcher

{docpull-1.0.2 → docpull-1.1.0}/docpull/cli.py

@@ -3,6 +3,40 @@ import sys
 from pathlib import Path
 from typing import Optional
 
+# Check if --doctor flag is present before checking dependencies
+# This allows users to diagnose issues even when dependencies are missing
+if "--doctor" in sys.argv:
+    from .doctor import run_doctor
+
+    # Parse output dir if provided
+    output_dir = None
+    if "--output-dir" in sys.argv or "-o" in sys.argv:
+        try:
+            flag_idx = sys.argv.index("--output-dir") if "--output-dir" in sys.argv else sys.argv.index("-o")
+            if flag_idx + 1 < len(sys.argv):
+                output_dir = Path(sys.argv[flag_idx + 1])
+        except (ValueError, IndexError):
+            pass
+    sys.exit(run_doctor(output_dir=output_dir))
+
+# Verify core dependencies are available
+try:
+    import aiohttp  # noqa: F401
+    import bs4  # noqa: F401
+    import defusedxml  # noqa: F401
+    import html2text  # noqa: F401
+    import requests  # noqa: F401
+    import rich  # noqa: F401
+except ImportError as e:
+    print(f"\nERROR: Missing required dependency: {e.name}", file=sys.stderr)
+    print("\nDocpull requires all core dependencies to be installed.", file=sys.stderr)
+    print("\nRecommended fixes:", file=sys.stderr)
+    print("  1. For pipx users: pipx reinstall docpull --force", file=sys.stderr)
+    print("  2. For pip users: pip install --upgrade --force-reinstall docpull", file=sys.stderr)
+    print("  3. For development: pip install -e .[dev]", file=sys.stderr)
+    print("\nTo diagnose issues, run: docpull --doctor", file=sys.stderr)
+    sys.exit(1)
+
 from . import __version__
 from .config import FetcherConfig
 from .fetchers import (
@@ -185,6 +219,12 @@ Examples:
         version=f"%(prog)s {__version__}",
     )
 
+    parser.add_argument(
+        "--doctor",
+        action="store_true",
+        help="Run diagnostic checks to verify installation",
+    )
+
     return parser
 
 
@@ -200,17 +240,31 @@ def generate_sample_config(output_path: Path) -> None:
     # Determine format from extension
     suffix = output_path.suffix.lower()
 
-    if suffix in [".yaml", ".yml"]:
-        config.save_yaml(output_path)
-        print(f"Sample YAML config generated: {output_path}")
-    elif suffix == ".json":
-        config.save_json(output_path)
-        print(f"Sample JSON config generated: {output_path}")
-    else:
-        print(f"Warning: Unknown extension {suffix}, generating YAML")
-        output_path = output_path.with_suffix(".yaml")
-        config.save_yaml(output_path)
-        print(f"Sample YAML config generated: {output_path}")
+    try:
+        if suffix in [".yaml", ".yml"]:
+            config.save_yaml(output_path)
+            print(f"Sample YAML config generated: {output_path}")
+        elif suffix == ".json":
+            config.save_json(output_path)
+            print(f"Sample JSON config generated: {output_path}")
+        else:
+            # Try YAML first, fall back to JSON if PyYAML not available
+            try:
+                print(f"Warning: Unknown extension {suffix}, generating YAML")
+                output_path = output_path.with_suffix(".yaml")
+                config.save_yaml(output_path)
+                print(f"Sample YAML config generated: {output_path}")
+            except ImportError:
+                print("PyYAML not installed, generating JSON instead")
+                output_path = output_path.with_suffix(".json")
+                config.save_json(output_path)
+                print(f"Sample JSON config generated: {output_path}")
+    except ImportError:
+        print("\nERROR: PyYAML is required for YAML config files")
+        print("Install it with: pip install docpull[yaml]")
+        print("\nAlternatively, use JSON format:")
+        print(f"  docpull --generate-config {output_path.with_suffix('.json')}")
+        raise
 
 
 def get_config(args: argparse.Namespace) -> FetcherConfig:
@@ -224,7 +278,17 @@ def get_config(args: argparse.Namespace) -> FetcherConfig:
         FetcherConfig instance
     """
     # Load from config file if provided
-    config = FetcherConfig.from_file(args.config) if args.config else FetcherConfig()
+    if args.config:
+        try:
+            config = FetcherConfig.from_file(args.config)
+        except ImportError as e:
+            print(f"\nERROR: Error loading config file: {e}")
+            if "yaml" in str(e).lower() or "pyyaml" in str(e).lower():
+                print("Install PyYAML with: pip install docpull[yaml]")
+                print("\nAlternatively, convert your config to JSON format")
+            raise
+    else:
+        config = FetcherConfig()
 
     # Override with command-line arguments
     if args.output_dir is not None:
@@ -411,6 +475,13 @@ def main(argv: Optional[list[str]] = None) -> int:
     parser = create_parser()
     args = parser.parse_args(argv)
 
+    # Handle --doctor
+    if args.doctor:
+        from .doctor import run_doctor
+
+        output_dir = Path(args.output_dir) if args.output_dir else None
+        return run_doctor(output_dir=output_dir)
+
     # Handle --generate-config
     if args.generate_config:
         try:

docpull-1.1.0/docpull/doctor.py

@@ -0,0 +1,188 @@
+"""Diagnostic tool for verifying docpull installation and dependencies."""
+
+import sys
+from importlib import import_module
+from pathlib import Path
+from typing import Optional
+
+try:
+    from rich.console import Console
+    from rich.table import Table
+
+    RICH_AVAILABLE = True
+except ImportError:
+    RICH_AVAILABLE = False
+    Console = None  # type: ignore
+    Table = None  # type: ignore
+
+
+def check_dependency(
+    module_name: str, package_name: Optional[str] = None, optional: bool = False
+) -> tuple[bool, str]:
+    """
+    Check if a Python module is importable.
+
+    Args:
+        module_name: Name of the module to import
+        package_name: Display name of the package (defaults to module_name)
+        optional: Whether this is an optional dependency
+
+    Returns:
+        Tuple of (success: bool, message: str)
+    """
+    display_name = package_name or module_name
+
+    try:
+        import_module(module_name)
+        return True, f"[OK] {display_name}"
+    except ImportError:
+        if optional:
+            return False, f"[WARN] {display_name} (optional - not installed)"
+        else:
+            return False, f"[MISSING] {display_name}"
+
+
+def check_network() -> tuple[bool, str]:
+    """
+    Check basic network connectivity.
+
+    Returns:
+        Tuple of (success: bool, message: str)
+    """
+    try:
+        import socket
+
+        # Try to resolve a common DNS name
+        socket.gethostbyname("www.google.com")
+        return True, "[OK] Network connectivity"
+    except socket.gaierror:
+        return False, "[FAIL] Network connectivity - DNS resolution failed"
+    except Exception as e:
+        return False, f"[WARN] Network connectivity - {str(e)}"
+
+
+def check_output_dir(output_dir: Optional[Path] = None) -> tuple[bool, str]:
+    """
+    Check if output directory is writable.
+
+    Args:
+        output_dir: Directory to check (defaults to ./docs)
+
+    Returns:
+        Tuple of (success: bool, message: str)
+    """
+    test_dir = output_dir or Path("./docs")
+
+    try:
+        # Create directory if it doesn't exist
+        test_dir.mkdir(parents=True, exist_ok=True)
+
+        # Try to write a test file
+        test_file = test_dir / ".docpull_test"
+        test_file.write_text("test")
+        test_file.unlink()
+
+        return True, f"[OK] Output directory writable ({test_dir})"
+    except PermissionError:
+        return False, f"[FAIL] Output directory - permission denied ({test_dir})"
+    except Exception as e:
+        return False, f"[FAIL] Output directory - {str(e)} ({test_dir})"
+
+
+def run_doctor(output_dir: Optional[Path] = None, use_rich: bool = True) -> int:
+    """
+    Run diagnostic checks and display results.
+
+    Args:
+        output_dir: Output directory to check for writability
+        use_rich: Whether to use rich formatting (if available)
+
+    Returns:
+        Exit code (0 if all core dependencies OK, 1 if any core dependency missing)
+    """
+    # Determine if we can use rich formatting
+    use_rich = use_rich and RICH_AVAILABLE
+
+    print("Running docpull diagnostics...\n")
+
+    # Core dependencies
+    core_checks = [
+        ("requests", "requests"),
+        ("bs4", "beautifulsoup4"),
+        ("html2text", "html2text"),
+        ("defusedxml", "defusedxml"),
+        ("aiohttp", "aiohttp"),
+        ("rich", "rich"),
+    ]
+
+    # Optional dependencies
+    optional_checks = [
+        ("yaml", "pyyaml", True),
+        ("playwright.async_api", "playwright", True),
+    ]
+
+    # Other checks
+    system_checks = [
+        check_network(),
+        check_output_dir(output_dir),
+    ]
+
+    # Run core dependency checks
+    core_results = [check_dependency(mod, pkg) for mod, pkg in core_checks]
+    optional_results = [check_dependency(mod, pkg, opt) for mod, pkg, opt in optional_checks]
+
+    all_checks = {
+        "Core Dependencies": core_results,
+        "Optional Dependencies": optional_results,
+        "System": system_checks,
+    }
+
+    # Display results
+    if use_rich:
+        console = Console()
+
+        for category, results in all_checks.items():
+            table = Table(title=category, show_header=False, box=None)
+            table.add_column("Status", style="bold")
+
+            for success, message in results:
+                style = "green" if success else ("yellow" if "optional" in message else "red")
+                table.add_row(message, style=style)
+
+            console.print(table)
+            console.print()
+    else:
+        # Fallback to plain text
+        for category, results in all_checks.items():
+            print(f"{category}:")
+            for _success, message in results:
+                print(f"  {message}")
+            print()
+
+    # Check if any core dependencies failed
+    core_failed = any(not success for success, _ in core_results)
+
+    # Print summary
+    if core_failed:
+        print("\nWARNING: Some core dependencies are missing!")
+        print("\nRecommended fixes:")
+        print("  1. For pipx users: pipx reinstall docpull --force")
+        print("  2. For pip users: pip install --upgrade --force-reinstall docpull")
+        print("  3. For development: pip install -e .[dev]")
+        return 1
+    else:
+        print("\nAll core dependencies installed correctly!")
+
+        # Check if optional dependencies are missing
+        optional_missing = [msg for success, msg in optional_results if not success]
+        if optional_missing:
+            print("\nOptional features available:")
+            print("  - YAML config support: pip install docpull[yaml]")
+            print("  - JavaScript rendering: pip install docpull[js]")
+            print("  - All optional features: pip install docpull[all]")
+
+        return 0
+
+
+if __name__ == "__main__":
+    sys.exit(run_doctor())

{docpull-1.0.2 → docpull-1.1.0}/docpull/fetchers/async_fetcher.py

@@ -3,7 +3,7 @@
 import asyncio
 import time
 from pathlib import Path
-from typing import TYPE_CHECKING, Any, List, Optional, Tuple
+from typing import Any, Optional
 
 import aiohttp
 from bs4 import BeautifulSoup
@@ -12,15 +12,15 @@ from ..utils.file_utils import ensure_dir, validate_output_path
 from .base import BaseFetcher
 
 # Optional Playwright support
-if TYPE_CHECKING:
-    from playwright.async_api import Browser, Page, Playwright
-
 try:
-    from playwright.async_api import async_playwright
+    from playwright.async_api import Browser, Playwright, async_playwright
 
     PLAYWRIGHT_AVAILABLE = True
 except ImportError:
     PLAYWRIGHT_AVAILABLE = False
+    # Fallback types for when playwright is not installed
+    Browser = Any  # type: ignore[misc,assignment]
+    Playwright = Any  # type: ignore[misc,assignment]
 
 
 class AsyncFetcher:
@@ -68,8 +68,8 @@ class AsyncFetcher:
         self.rate_limit_delay = base_fetcher.rate_limit
 
         # Browser instance (if using JS)
-        self.browser: Optional["Browser"] = None
-        self.playwright: Optional["Playwright"] = None
+        self.browser: Optional[Browser] = None  # type: ignore[no-any-unimported]
+        self.playwright: Optional[Playwright] = None  # type: ignore[no-any-unimported]
 
         if use_js and not PLAYWRIGHT_AVAILABLE:
             self.logger.warning("Playwright not installed. Install with: pip install docpull[js]")
@@ -152,7 +152,7 @@ class AsyncFetcher:
             )
 
             # Get rendered HTML
-            content = await page.content()
+            content: str = await page.content()
 
             return content
 

{docpull-1.0.2 → docpull-1.1.0}/docpull/fetchers/base.py

@@ -84,9 +84,7 @@ class BaseFetcher(ABC):
                 self.validator_func = validator_func
                 super().__init__(*args, **kwargs)
 
-            def send(  # type: ignore[override]
-                self, request: PreparedRequest, **kwargs: Any
-            ) -> Response:
+            def send(self, request: PreparedRequest, **kwargs: Any) -> Response:  # type: ignore[override]
                 if request.url is None:
                     raise ValueError("Request URL is None")
                 if not self.validator_func(request.url):

{docpull-1.0.2 → docpull-1.1.0}/docpull/fetchers/plaid.py

@@ -57,9 +57,9 @@ class PlaidFetcher(BaseFetcher):
         sitemap_urls = self.fetch_sitemap(self.sitemap_url)
 
         for url in sitemap_urls:
-            if (
-                "/docs/" in url or "/api/" in url
-            ) and not any(x in url for x in ["/blog/", "/resources/", "/company/", "/customers/"]):
+            if ("/docs/" in url or "/api/" in url) and not any(
+                x in url for x in ["/blog/", "/resources/", "/company/", "/customers/"]
+            ):
                 doc_urls.add(url.split("#")[0].split("?")[0])
 
         doc_urls_list = sorted(doc_urls)

{docpull-1.0.2 → docpull-1.1.0}/docpull.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: docpull
-Version: 1.0.2
+Version: 1.1.0
 Summary: Pull documentation from the web and convert to clean markdown
 Author-email: Zachary Roth <support@raintree.technology>
 Maintainer-email: Raintree Technology <support@raintree.technology>
@@ -62,7 +62,7 @@ Requires-Dist: bandit>=1.7.0; extra == "dev"
 Requires-Dist: pip-audit>=2.0.0; extra == "dev"
 Requires-Dist: types-requests>=2.31.0; extra == "dev"
 Requires-Dist: types-beautifulsoup4>=4.12.0; extra == "dev"
-Requires-Dist: types-aiohttp>=3.9.0; extra == "dev"
+Requires-Dist: types-defusedxml>=0.7.0; extra == "dev"
 Dynamic: license-file
 
 # docpull
@@ -97,6 +97,7 @@ Unlike tools like wget or httrack, docpull extracts only the main content, remov
 
 ```bash
 pip install docpull
+docpull --doctor         # verify installation
 docpull https://aptos.dev
 docpull stripe           # use a built-in profile
 docpull https://site.com/docs --max-pages 100 --max-concurrent 20
@@ -126,6 +127,7 @@ fetcher.fetch()
 
 ## Common Options
 
+- `--doctor` – verify installation and dependencies
 - `--max-pages N` – limit crawl size
 - `--max-depth N` – restrict link depth
 - `--max-concurrent N` – control parallel fetches
@@ -200,10 +202,14 @@ MY_PROFILE = SiteProfile(
 
 ## Troubleshooting
 
+- **Installation issues**: Run `docpull --doctor` to diagnose problems
+- **Missing dependencies**: See [TROUBLESHOOTING.md](TROUBLESHOOTING.md) for common fixes
 - **Site requires JS**: install Playwright + `--js`
 - **Slow or rate limited**: lower concurrency or raise `--rate-limit`
 - **Large sites**: set `--max-pages`
 
+For detailed troubleshooting, see [TROUBLESHOOTING.md](TROUBLESHOOTING.md).
+
 ## Links
 
 - [PyPI](https://pypi.org/project/docpull/)

{docpull-1.0.2 → docpull-1.1.0}/docpull.egg-info/SOURCES.txt

@@ -5,6 +5,7 @@ docpull/__init__.py
 docpull/__main__.py
 docpull/cli.py
 docpull/config.py
+docpull/doctor.py
 docpull/py.typed
 docpull.egg-info/PKG-INFO
 docpull.egg-info/SOURCES.txt

{docpull-1.0.2 → docpull-1.1.0}/docpull.egg-info/requires.txt

@@ -20,7 +20,7 @@ bandit>=1.7.0
 pip-audit>=2.0.0
 types-requests>=2.31.0
 types-beautifulsoup4>=4.12.0
-types-aiohttp>=3.9.0
+types-defusedxml>=0.7.0
 
 [js]
 playwright>=1.40.0

{docpull-1.0.2 → docpull-1.1.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "docpull"
-version = "1.0.2"
+version = "1.1.0"
 description = "Pull documentation from the web and convert to clean markdown"
 readme = {file = "README.md", content-type = "text/markdown"}
 requires-python = ">=3.9"
@@ -89,7 +89,7 @@ dev = [
     "pip-audit>=2.0.0",
     "types-requests>=2.31.0",
     "types-beautifulsoup4>=4.12.0",
-    "types-aiohttp>=3.9.0",
+    "types-defusedxml>=0.7.0",
 ]
 
 [project.scripts]
@@ -132,6 +132,10 @@ no_implicit_optional = true
 strict_equality = true
 warn_redundant_casts = true
 
+[[tool.mypy.overrides]]
+module = "playwright.*"
+ignore_missing_imports = true
+
 [tool.pytest.ini_options]
 minversion = "7.0"
 testpaths = ["tests"]