docpull 1.0.1__py3-none-any.whl

docpull/__init__.py

@@ -0,0 +1,29 @@
+__version__ = "1.0.1"
+
+from .fetchers.base import BaseFetcher
+from .fetchers.bun import BunFetcher
+from .fetchers.d3 import D3DevDocsFetcher
+from .fetchers.generic import GenericFetcher
+from .fetchers.generic_async import GenericAsyncFetcher
+from .fetchers.nextjs import NextJSFetcher
+from .fetchers.parallel_base import ParallelFetcher
+from .fetchers.plaid import PlaidFetcher
+from .fetchers.react import ReactFetcher
+from .fetchers.stripe import StripeFetcher
+from .fetchers.tailwind import TailwindFetcher
+from .fetchers.turborepo import TurborepoFetcher
+
+__all__ = [
+    "BaseFetcher",
+    "BunFetcher",
+    "D3DevDocsFetcher",
+    "GenericFetcher",
+    "GenericAsyncFetcher",
+    "NextJSFetcher",
+    "ParallelFetcher",
+    "PlaidFetcher",
+    "ReactFetcher",
+    "StripeFetcher",
+    "TailwindFetcher",
+    "TurborepoFetcher",
+]

docpull/__main__.py

@@ -0,0 +1,6 @@
+import sys
+
+from .cli import main
+
+if __name__ == "__main__":
+    sys.exit(main())

docpull/cli.py

@@ -0,0 +1,440 @@
+import argparse
+import sys
+from pathlib import Path
+from typing import Optional
+
+from . import __version__
+from .config import FetcherConfig
+from .fetchers import (
+    BunFetcher,
+    D3DevDocsFetcher,
+    NextJSFetcher,
+    PlaidFetcher,
+    ReactFetcher,
+    StripeFetcher,
+    TailwindFetcher,
+    TurborepoFetcher,
+)
+from .fetchers.generic_async import GenericAsyncFetcher
+from .utils.logging_config import setup_logging
+
+
+def create_parser() -> argparse.ArgumentParser:
+    """Create argument parser for CLI."""
+    parser = argparse.ArgumentParser(
+        prog="docpull",
+        description="Fetch and convert documentation from any URL or known sources to markdown",
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+        epilog="""
+Examples:
+  # Fetch from any documentation URL
+  docpull https://aptos.dev
+  docpull https://docs.anthropic.com
+
+  # Fetch using profile names (shortcuts)
+  docpull stripe
+  docpull nextjs plaid
+
+  # Mix URLs and profiles
+  docpull stripe https://newsite.com/docs
+
+  # Control scraping depth and pages
+  docpull https://example.com/docs --max-pages 100 --max-depth 3
+
+  # Legacy syntax still works
+  docpull --source stripe --source nextjs
+
+  # Use a config file
+  docpull --config config.yaml
+
+  # Generate a sample config file
+  docpull --generate-config config.yaml
+        """,
+    )
+
+    # Positional arguments for URLs or profile names
+    parser.add_argument(
+        "targets",
+        nargs="*",
+        help="URLs or profile names to fetch (e.g., 'https://docs.site.com', 'stripe', 'nextjs')",
+    )
+
+    parser.add_argument(
+        "--config",
+        "-c",
+        type=Path,
+        help="Path to config file (YAML or JSON)",
+    )
+
+    parser.add_argument(
+        "--output-dir",
+        "-o",
+        type=Path,
+        default=None,
+        help="Directory to save documentation (default: ./docs)",
+    )
+
+    parser.add_argument(
+        "--source",
+        "-s",
+        nargs="+",
+        choices=["all", "bun", "d3", "nextjs", "plaid", "react", "stripe", "tailwind", "turborepo"],
+        default=None,
+        dest="sources",
+        help="Documentation source(s) to fetch. Use 'all' for everything. (default: all)",
+    )
+
+    parser.add_argument(
+        "--rate-limit",
+        "-r",
+        type=float,
+        default=None,
+        help="Seconds to wait between requests (default: 0.5)",
+    )
+
+    parser.add_argument(
+        "--no-skip-existing",
+        action="store_true",
+        help="Re-fetch files that already exist",
+    )
+
+    parser.add_argument(
+        "--log-level",
+        "-l",
+        choices=["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"],
+        default=None,
+        help="Logging level (default: INFO)",
+    )
+
+    parser.add_argument(
+        "--verbose",
+        "-v",
+        action="store_const",
+        const="DEBUG",
+        dest="log_level_override",
+        help="Enable verbose output (equivalent to --log-level DEBUG)",
+    )
+
+    parser.add_argument(
+        "--quiet",
+        "-q",
+        action="store_const",
+        const="ERROR",
+        dest="log_level_override",
+        help="Suppress informational output (equivalent to --log-level ERROR)",
+    )
+
+    parser.add_argument(
+        "--dry-run",
+        action="store_true",
+        help="Show what would be fetched without actually downloading",
+    )
+
+    parser.add_argument(
+        "--max-pages",
+        type=int,
+        default=None,
+        help="Maximum number of pages to fetch (default: unlimited)",
+    )
+
+    parser.add_argument(
+        "--max-depth",
+        type=int,
+        default=5,
+        help="Maximum crawl depth when following links (default: 5)",
+    )
+
+    parser.add_argument(
+        "--max-concurrent",
+        type=int,
+        default=10,
+        help="Maximum concurrent requests for async fetching (default: 10)",
+    )
+
+    parser.add_argument(
+        "--js",
+        "--javascript",
+        action="store_true",
+        dest="use_js",
+        help="Enable JavaScript rendering with Playwright (slower but handles JS-heavy sites)",
+    )
+
+    parser.add_argument(
+        "--no-progress",
+        action="store_true",
+        help="Disable progress bars",
+    )
+
+    parser.add_argument(
+        "--log-file",
+        type=Path,
+        default=None,
+        help="Path to log file (default: console only)",
+    )
+
+    parser.add_argument(
+        "--generate-config",
+        type=Path,
+        metavar="PATH",
+        help="Generate a sample config file and exit",
+    )
+
+    parser.add_argument(
+        "--version",
+        action="version",
+        version=f"%(prog)s {__version__}",
+    )
+
+    return parser
+
+
+def generate_sample_config(output_path: Path) -> None:
+    """
+    Generate a sample configuration file.
+
+    Args:
+        output_path: Path to save the config file
+    """
+    config = FetcherConfig()
+
+    # Determine format from extension
+    suffix = output_path.suffix.lower()
+
+    if suffix in [".yaml", ".yml"]:
+        config.save_yaml(output_path)
+        print(f"Sample YAML config generated: {output_path}")
+    elif suffix == ".json":
+        config.save_json(output_path)
+        print(f"Sample JSON config generated: {output_path}")
+    else:
+        print(f"Warning: Unknown extension {suffix}, generating YAML")
+        output_path = output_path.with_suffix(".yaml")
+        config.save_yaml(output_path)
+        print(f"Sample YAML config generated: {output_path}")
+
+
+def get_config(args: argparse.Namespace) -> FetcherConfig:
+    """
+    Get configuration from args and config file.
+
+    Args:
+        args: Parsed command-line arguments
+
+    Returns:
+        FetcherConfig instance
+    """
+    # Load from config file if provided
+    config = FetcherConfig.from_file(args.config) if args.config else FetcherConfig()
+
+    # Override with command-line arguments
+    if args.output_dir is not None:
+        config.output_dir = args.output_dir
+
+    if args.sources is not None:
+        # Handle "all" keyword
+        if "all" in args.sources:
+            config.sources = ["bun", "d3", "nextjs", "plaid", "react", "stripe", "tailwind", "turborepo"]
+        else:
+            config.sources = args.sources
+
+    if args.rate_limit is not None:
+        config.rate_limit = args.rate_limit
+
+    if args.no_skip_existing:
+        config.skip_existing = False
+
+    # Handle log level (verbose/quiet shortcuts override --log-level)
+    if args.log_level_override is not None:
+        config.log_level = args.log_level_override
+    elif args.log_level is not None:
+        config.log_level = args.log_level
+
+    if args.log_file is not None:
+        config.log_file = str(args.log_file)
+
+    # Store dry-run flag in config
+    config.dry_run = args.dry_run
+
+    return config
+
+
+def run_fetchers(config: FetcherConfig) -> int:
+    """
+    Run the fetchers based on configuration.
+
+    Args:
+        config: FetcherConfig instance
+
+    Returns:
+        Exit code (0 for success, 1 for error)
+    """
+    # Setup logging
+    logger = setup_logging(
+        level=config.log_level,
+        log_file=config.log_file,
+    )
+
+    logger.info("docpull - Documentation Fetcher")
+    logger.info(f"Mode: {'DRY RUN' if config.dry_run else 'FETCH'}")
+    logger.info(f"Output directory: {config.output_dir}")
+    logger.info(f"Rate limit: {config.rate_limit}s between requests")
+    logger.info(f"Skip existing: {config.skip_existing}")
+    logger.info(f"Sources: {', '.join(config.sources)}")
+    logger.info("")
+
+    if config.dry_run:
+        logger.info("DRY RUN MODE: No files will be downloaded")
+        logger.info("")
+
+    # Map source names to fetcher classes
+    fetcher_map = {
+        "bun": BunFetcher,
+        "d3": D3DevDocsFetcher,
+        "nextjs": NextJSFetcher,
+        "plaid": PlaidFetcher,
+        "react": ReactFetcher,
+        "stripe": StripeFetcher,
+        "tailwind": TailwindFetcher,
+        "turborepo": TurborepoFetcher,
+    }
+
+    # Run fetchers
+    errors = 0
+    for source in config.sources:
+        if source not in fetcher_map:
+            logger.error(f"Unknown source: {source}")
+            errors += 1
+            continue
+
+        try:
+            fetcher_class = fetcher_map[source]
+            fetcher = fetcher_class(
+                output_dir=config.output_dir,
+                rate_limit=config.rate_limit,
+                skip_existing=config.skip_existing,
+                logger=logger,
+            )
+            fetcher.fetch()
+        except Exception as e:
+            logger.error(f"Error fetching {source}: {e}", exc_info=True)
+            errors += 1
+
+    logger.info("")
+    if errors > 0:
+        logger.error(f"Completed with {errors} error(s)")
+        return 1
+    else:
+        logger.info("All documentation fetched successfully")
+        return 0
+
+
+def run_generic_fetchers(args: argparse.Namespace) -> int:
+    """
+    Run generic fetchers for URLs or profile names.
+
+    Args:
+        args: Parsed command-line arguments
+
+    Returns:
+        Exit code (0 for success, 1 for error)
+    """
+    # Setup logging
+    log_level = args.log_level_override or args.log_level or "INFO"
+    logger = setup_logging(
+        level=log_level,
+        log_file=args.log_file,
+    )
+
+    output_dir = Path(args.output_dir) if args.output_dir else Path("./docs")
+    rate_limit = args.rate_limit if args.rate_limit is not None else 0.5
+    skip_existing = not args.no_skip_existing
+    max_pages = args.max_pages
+    max_depth = args.max_depth
+    max_concurrent = args.max_concurrent
+    use_js = args.use_js
+    show_progress = not args.no_progress
+
+    logger.info("docpull - Universal Documentation Fetcher")
+    logger.info(f"Targets: {', '.join(args.targets)}")
+    logger.info(f"Output directory: {output_dir}")
+    logger.info(f"Rate limit: {rate_limit}s between requests")
+    logger.info(f"Skip existing: {skip_existing}")
+    logger.info(f"Max concurrent: {max_concurrent}")
+    if max_pages:
+        logger.info(f"Max pages: {max_pages}")
+    logger.info(f"Max depth: {max_depth}")
+    if use_js:
+        logger.info("JavaScript rendering: ENABLED (slower but handles JS sites)")
+    logger.info("")
+
+    # Run async generic fetcher for each target
+    errors = 0
+    for target in args.targets:
+        try:
+            logger.info(f"Fetching: {target}")
+            fetcher = GenericAsyncFetcher(
+                url_or_profile=target,
+                output_dir=output_dir,
+                rate_limit=rate_limit,
+                skip_existing=skip_existing,
+                logger=logger,
+                max_pages=max_pages,
+                max_depth=max_depth,
+                max_concurrent=max_concurrent,
+                use_js=use_js,
+                show_progress=show_progress,
+            )
+            fetcher.fetch()  # This calls asyncio.run() internally
+        except Exception as e:
+            logger.error(f"Error fetching {target}: {e}", exc_info=True)
+            errors += 1
+
+    logger.info("")
+    if errors > 0:
+        logger.error(f"Completed with {errors} error(s)")
+        return 1
+    else:
+        logger.info("All documentation fetched successfully")
+        return 0
+
+
+def main(argv: Optional[list[str]] = None) -> int:
+    """
+    Main entry point for CLI.
+
+    Args:
+        argv: Command-line arguments (defaults to sys.argv)
+
+    Returns:
+        Exit code
+    """
+    parser = create_parser()
+    args = parser.parse_args(argv)
+
+    # Handle --generate-config
+    if args.generate_config:
+        try:
+            generate_sample_config(args.generate_config)
+            return 0
+        except Exception as e:
+            print(f"Error generating config: {e}", file=sys.stderr)
+            return 1
+
+    # Determine if using new URL-based interface or legacy source-based
+    use_generic = bool(args.targets)
+
+    if use_generic:
+        # New URL-based interface
+        return run_generic_fetchers(args)
+    else:
+        # Legacy source-based interface
+        try:
+            config = get_config(args)
+        except Exception as e:
+            print(f"Error loading configuration: {e}", file=sys.stderr)
+            return 1
+        return run_fetchers(config)
+
+
+if __name__ == "__main__":
+    sys.exit(main())

docpull/config.py

@@ -0,0 +1,199 @@
+import json
+from pathlib import Path
+from typing import Any, Optional
+
+try:
+    import yaml  # type: ignore
+except ImportError:
+    yaml = None  # type: ignore
+
+
+class FetcherConfig:
+    """Configuration for documentation fetchers."""
+
+    def __init__(
+        self,
+        output_dir: str = "./docs",
+        rate_limit: float = 0.5,
+        skip_existing: bool = True,
+        log_level: str = "INFO",
+        log_file: Optional[str] = None,
+        sources: Optional[list[str]] = None,
+        dry_run: bool = False,
+    ):
+        """
+        Initialize configuration.
+
+        Args:
+            output_dir: Directory to save documentation
+            rate_limit: Seconds between requests
+            skip_existing: Skip existing files
+            log_level: Logging level
+            log_file: Optional log file path
+            sources: List of sources to fetch (e.g., ['stripe', 'plaid'])
+            dry_run: Dry run mode (don't download files)
+        """
+        self.output_dir = Path(output_dir)
+        self.rate_limit = rate_limit
+        self.skip_existing = skip_existing
+        self.log_level = log_level
+        self.log_file = log_file
+        self.sources = sources or ["plaid", "stripe"]
+        self.dry_run = dry_run
+
+    @classmethod
+    def from_dict(cls, config_dict: dict[str, Any]) -> "FetcherConfig":
+        """
+        Create configuration from dictionary.
+
+        Args:
+            config_dict: Configuration dictionary
+
+        Returns:
+            FetcherConfig instance
+
+        Raises:
+            ValueError: If configuration values are invalid
+        """
+        # Validate output_dir doesn't contain path traversal
+        output_dir = str(config_dict.get("output_dir", "./docs"))
+        if ".." in output_dir or output_dir.startswith("/etc") or output_dir.startswith("/sys"):
+            raise ValueError("Invalid output directory path")
+
+        # Validate rate_limit is reasonable
+        rate_limit = config_dict.get("rate_limit", 0.5)
+        if not isinstance(rate_limit, (int, float)) or rate_limit < 0 or rate_limit > 60:
+            raise ValueError("rate_limit must be between 0 and 60")
+
+        # Validate sources
+        valid_sources = {"bun", "d3", "nextjs", "plaid", "react", "stripe", "tailwind", "turborepo"}
+        sources = config_dict.get("sources", ["plaid", "stripe"])
+        if not all(s in valid_sources for s in sources):
+            raise ValueError(f"Invalid sources. Must be from: {valid_sources}")
+
+        # Validate log_level
+        log_level = config_dict.get("log_level", "INFO")
+        valid_log_levels = {"DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"}
+        if log_level.upper() not in valid_log_levels:
+            raise ValueError(f"Invalid log_level. Must be one of: {valid_log_levels}")
+
+        return cls(
+            output_dir=output_dir,
+            rate_limit=rate_limit,
+            skip_existing=config_dict.get("skip_existing", True),
+            log_level=log_level,
+            log_file=config_dict.get("log_file"),
+            sources=sources,
+            dry_run=config_dict.get("dry_run", False),
+        )
+
+    @classmethod
+    def from_yaml(cls, yaml_path: Path) -> "FetcherConfig":
+        """
+        Load configuration from YAML file.
+
+        Args:
+            yaml_path: Path to YAML config file
+
+        Returns:
+            FetcherConfig instance
+
+        Raises:
+            ImportError: If pyyaml is not installed
+            FileNotFoundError: If config file doesn't exist
+        """
+        if yaml is None:
+            raise ImportError("PyYAML is required for YAML config. Install with: pip install pyyaml")
+
+        if not yaml_path.exists():
+            raise FileNotFoundError(f"Config file not found: {yaml_path}")
+
+        with open(yaml_path) as f:
+            config_dict = yaml.safe_load(f)
+
+        return cls.from_dict(config_dict)
+
+    @classmethod
+    def from_json(cls, json_path: Path) -> "FetcherConfig":
+        """
+        Load configuration from JSON file.
+
+        Args:
+            json_path: Path to JSON config file
+
+        Returns:
+            FetcherConfig instance
+
+        Raises:
+            FileNotFoundError: If config file doesn't exist
+        """
+        if not json_path.exists():
+            raise FileNotFoundError(f"Config file not found: {json_path}")
+
+        with open(json_path) as f:
+            config_dict = json.load(f)
+
+        return cls.from_dict(config_dict)
+
+    @classmethod
+    def from_file(cls, config_path: Path) -> "FetcherConfig":
+        """
+        Load configuration from file (auto-detect format).
+
+        Args:
+            config_path: Path to config file
+
+        Returns:
+            FetcherConfig instance
+        """
+        suffix = config_path.suffix.lower()
+
+        if suffix in [".yaml", ".yml"]:
+            return cls.from_yaml(config_path)
+        elif suffix == ".json":
+            return cls.from_json(config_path)
+        else:
+            raise ValueError(f"Unsupported config file format: {suffix}")
+
+    def to_dict(self) -> dict[str, Any]:
+        """
+        Convert configuration to dictionary.
+
+        Returns:
+            Configuration as dictionary
+        """
+        return {
+            "output_dir": str(self.output_dir),
+            "rate_limit": self.rate_limit,
+            "skip_existing": self.skip_existing,
+            "log_level": self.log_level,
+            "log_file": self.log_file,
+            "sources": self.sources,
+            "dry_run": self.dry_run,
+        }
+
+    def save_yaml(self, yaml_path: Path) -> None:
+        """
+        Save configuration to YAML file.
+
+        Args:
+            yaml_path: Path to save YAML config
+
+        Raises:
+            ImportError: If pyyaml is not installed
+        """
+        if yaml is None:
+            raise ImportError("PyYAML is required for YAML config. Install with: pip install pyyaml")
+
+        with open(yaml_path, "w") as f:
+            yaml.dump(self.to_dict(), f, default_flow_style=False, sort_keys=False)
+
+    def save_json(self, json_path: Path) -> None:
+        """
+        Save configuration to JSON file.
+
+        Args:
+            json_path: Path to save JSON config
+        """
+        with open(json_path, "w") as f:
+            json.dump(self.to_dict(), f, indent=2)

docpull/fetchers/__init__.py

@@ -0,0 +1,23 @@
+from .base import BaseFetcher
+from .bun import BunFetcher
+from .d3 import D3DevDocsFetcher
+from .nextjs import NextJSFetcher
+from .parallel_base import ParallelFetcher
+from .plaid import PlaidFetcher
+from .react import ReactFetcher
+from .stripe import StripeFetcher
+from .tailwind import TailwindFetcher
+from .turborepo import TurborepoFetcher
+
+__all__ = [
+    "BaseFetcher",
+    "BunFetcher",
+    "D3DevDocsFetcher",
+    "NextJSFetcher",
+    "ParallelFetcher",
+    "PlaidFetcher",
+    "ReactFetcher",
+    "StripeFetcher",
+    "TailwindFetcher",
+    "TurborepoFetcher",
+]