docpull 1.0.1__py3-none-any.whl

docpull/fetchers/d3.py

@@ -0,0 +1,211 @@
+"""D3.js documentation fetcher from DevDocs.io."""
+
+import logging
+import re
+import time
+from pathlib import Path
+from typing import Any, Optional, cast
+
+from .parallel_base import ParallelFetcher
+
+
+class D3DevDocsFetcher(ParallelFetcher):
+    """Fetcher for D3.js documentation from DevDocs.io."""
+
+    def __init__(
+        self,
+        output_dir: Path,
+        rate_limit: float = 0.05,  # DevDocs can handle faster requests
+        skip_existing: bool = True,
+        logger: Optional[logging.Logger] = None,
+        max_workers: int = 20,  # More workers for DevDocs
+        version: str = "7",  # D3 version
+    ) -> None:
+        """
+        Initialize D3 DevDocs fetcher.
+
+        Args:
+            output_dir: Directory to save documentation
+            rate_limit: Seconds between requests (per worker)
+            skip_existing: Skip existing files
+            logger: Logger instance
+            max_workers: Number of concurrent workers
+            version: D3 version (default: 7 for latest)
+        """
+        super().__init__(output_dir, rate_limit, skip_existing, logger, max_workers)
+        self.version = version
+        self.doc_slug = f"d3~{version}"
+        self.base_url = "https://devdocs.io/"
+        self.api_base = "https://documents.devdocs.io/"
+        self.index_url = f"{self.api_base}{self.doc_slug}/index.json"
+
+    def fetch_index(self) -> list[dict[Any, Any]]:
+        """
+        Fetch the documentation index from DevDocs API.
+
+        Returns:
+            List of entry dictionaries
+        """
+        self.logger.info(f"Fetching D3 v{self.version} index from DevDocs")
+
+        try:
+            response = self.session.get(self.index_url, timeout=30)
+            response.raise_for_status()
+
+            data = response.json()
+            entries = cast(list[dict[Any, Any]], data.get("entries", []))
+
+            self.logger.info(f"Found {len(entries)} D3 documentation entries")
+            return entries
+
+        except Exception as e:
+            self.logger.error(f"Error fetching index: {e}")
+            return []
+
+    def fetch_entry_content(self, entry: dict) -> str:
+        """
+        Fetch content for a single entry from DevDocs.
+
+        Args:
+            entry: Entry dictionary with 'path' key
+
+        Returns:
+            Markdown content with frontmatter
+        """
+        path = entry["path"]
+        url = f"{self.api_base}{self.doc_slug}/{path}.html"
+
+        try:
+            response = self.session.get(url, timeout=30)
+            response.raise_for_status()
+
+            # DevDocs returns HTML, convert to markdown
+            from bs4 import BeautifulSoup
+
+            soup = BeautifulSoup(response.content, "html.parser")
+
+            # Convert to markdown
+            markdown = self.h2t.handle(str(soup))
+
+            # Add frontmatter
+            frontmatter = f"""---
+name: {entry.get('name', '')}
+type: {entry.get('type', '')}
+path: {path}
+source: DevDocs.io - D3.js v{self.version}
+url: {self.base_url}d3~{self.version}/{path}
+fetched: {time.strftime('%Y-%m-%d')}
+---
+
+"""
+            return frontmatter + markdown.strip()
+
+        except Exception as e:
+            self.logger.error(f"Error fetching {path}: {e}")
+            self.stats["errors"] += 1
+            return f"# Error\n\nFailed to fetch {path}\n\nError: {str(e)}"
+
+    def process_entry(self, entry_data: tuple[dict, Path]) -> tuple[bool, str]:
+        """
+        Process a single entry.
+
+        Args:
+            entry_data: Tuple of (entry, output_path)
+
+        Returns:
+            Tuple of (success, entry_name)
+        """
+        entry, output_path = entry_data
+
+        # Skip if exists
+        if self.skip_existing and output_path.exists():
+            self.logger.debug(f"Skipping (already exists): {entry['name']}")
+            self.stats["skipped"] += 1
+            return (True, entry["name"])
+
+        # Fetch content
+        content = self.fetch_entry_content(entry)
+
+        # Save
+        self.save_content(content, output_path)
+        self.stats["fetched"] += 1
+
+        # Rate limiting
+        time.sleep(self.rate_limit)
+
+        return (True, entry["name"])
+
+    def fetch(self) -> None:
+        """Fetch all D3.js documentation from DevDocs."""
+        self.logger.info(f"Fetching D3.js v{self.version} documentation from DevDocs")
+
+        entries = self.fetch_index()
+
+        if not entries:
+            self.logger.error("No entries found")
+            return
+
+        # Group by type for organization
+        by_type: dict[str, list[dict[Any, Any]]] = {}
+        for entry in entries:
+            entry_type = entry.get("type", "Other")
+            if entry_type not in by_type:
+                by_type[entry_type] = []
+            by_type[entry_type].append(entry)
+
+        self.logger.info(f"Found {len(by_type)} categories:")
+        for cat, cat_entries in sorted(by_type.items(), key=lambda x: len(x[1]), reverse=True):
+            self.logger.info(f"  {cat}: {len(cat_entries)} entries")
+
+        # Prepare URL paths
+        entry_paths = []
+        for entry in entries:
+            # Organize by type
+            entry_type = entry.get("type", "Other")
+            # Clean type name for directory - sanitize to prevent path traversal
+            type_dir = entry_type.lower().replace(" ", "-")
+            type_dir = re.sub(r"[^\w\-]", "-", type_dir)
+            type_dir = type_dir.strip("-").strip(".")
+
+            # Generate filename from path
+            path = entry["path"]
+            filename = path.replace("/", "-") + ".md"
+
+            output_path = self.output_dir / "d3" / type_dir / filename
+
+            # Validate output path to prevent path traversal
+            try:
+                from ..utils.file_utils import validate_output_path
+
+                output_path = validate_output_path(output_path, self.output_dir)
+            except ValueError as e:
+                self.logger.error(f"Invalid output path for entry {entry.get('name', 'unknown')}: {e}")
+                continue
+
+            entry_paths.append(((entry, output_path), entry))
+
+        self.logger.info(f"Fetching with {self.max_workers} concurrent workers")
+        start_time = time.time()
+
+        from concurrent.futures import ThreadPoolExecutor, as_completed
+
+        with ThreadPoolExecutor(max_workers=self.max_workers) as executor:
+            futures = {executor.submit(self.process_entry, ep[0]): ep[1] for ep in entry_paths}
+
+            total = len(futures)
+
+            for completed, future in enumerate(as_completed(futures), start=1):
+                success, name = future.result()
+
+                if completed % 50 == 0 or completed == total:
+                    elapsed = time.time() - start_time
+                    rate = completed / elapsed if elapsed > 0 else 0
+                    self.logger.info(
+                        f"[{completed}/{total}] " f"({completed*100//total}%) " f"- {rate:.1f} docs/sec"
+                    )
+
+        elapsed = time.time() - start_time
+        self.logger.info(f"Completed in {elapsed:.1f}s ({total/elapsed:.1f} docs/sec)")
+
+        self.logger.info("D3.js documentation fetch complete")
+        self.print_stats()

docpull/fetchers/generic.py

@@ -0,0 +1,255 @@
+"""Generic documentation fetcher that works with any URL."""
+
+import logging
+from pathlib import Path
+from typing import Optional
+from urllib.parse import urljoin, urlparse
+
+from bs4 import BeautifulSoup
+
+from ..profiles import SiteProfile, get_profile_by_name, get_profile_for_url
+from .base import BaseFetcher
+
+
+class GenericFetcher(BaseFetcher):
+    """
+    Generic fetcher that can scrape documentation from any URL.
+
+    Supports both profile-based (optimized) and generic scraping modes.
+    """
+
+    def __init__(
+        self,
+        url_or_profile: str,
+        output_dir: Path,
+        profile: Optional[SiteProfile] = None,
+        rate_limit: float = 0.5,
+        skip_existing: bool = True,
+        logger: Optional[logging.Logger] = None,
+        max_pages: Optional[int] = None,
+        max_depth: int = 5,
+    ) -> None:
+        """
+        Initialize generic fetcher.
+
+        Args:
+            url_or_profile: URL to scrape or profile name (e.g., 'stripe')
+            output_dir: Directory to save documentation
+            profile: Optional SiteProfile to use (overrides auto-detection)
+            rate_limit: Seconds between requests
+            skip_existing: Skip existing files
+            logger: Logger instance
+            max_pages: Maximum pages to fetch
+            max_depth: Maximum crawl depth
+        """
+        super().__init__(output_dir, rate_limit, skip_existing=skip_existing, logger=logger)
+
+        # Determine if input is a URL or profile name
+        if url_or_profile.startswith(("http://", "https://")):
+            self.start_url = url_or_profile
+            # Try to auto-detect profile
+            if profile is None:
+                profile = get_profile_for_url(url_or_profile)
+                if profile:
+                    self.logger.info(f"Auto-detected profile: {profile.name}")
+        else:
+            # Treat as profile name
+            profile = get_profile_by_name(url_or_profile)
+            if profile is None:
+                raise ValueError(f"Unknown profile: {url_or_profile}")
+            start_url_candidate = profile.base_url or (profile.start_urls[0] if profile.start_urls else None)
+            if not start_url_candidate:
+                raise ValueError(f"Profile {url_or_profile} has no start URL")
+            self.start_url = start_url_candidate
+            self.logger.info(f"Using profile: {profile.name}")
+
+        self.profile = profile
+        self.max_pages = max_pages
+        self.max_depth = max_depth
+
+        # Set defaults from profile if available
+        if profile:
+            self.rate_limit = profile.rate_limit
+            self.sitemap_url = profile.sitemap_url
+            self.base_url = profile.base_url or self._extract_base_url(self.start_url)
+            self.include_patterns = profile.include_patterns
+            self.exclude_patterns = profile.exclude_patterns
+            self.output_subdir = profile.output_subdir or urlparse(self.start_url).netloc.replace(".", "_")
+            self.strip_prefix = profile.strip_prefix
+            self.follow_links = profile.follow_links
+        else:
+            # Generic mode - infer from URL
+            self.sitemap_url = self._guess_sitemap_url(self.start_url)
+            self.base_url = self._extract_base_url(self.start_url)
+            self.include_patterns = [self.base_url]
+            self.exclude_patterns = []
+            self.output_subdir = urlparse(self.start_url).netloc.replace(".", "_")
+            self.strip_prefix = None
+            self.follow_links = False
+
+    def _extract_base_url(self, url: str) -> str:
+        """Extract base URL from a full URL."""
+        parsed = urlparse(url)
+        return f"{parsed.scheme}://{parsed.netloc}/"
+
+    def _guess_sitemap_url(self, url: str) -> Optional[str]:
+        """
+        Guess sitemap URL for a given domain.
+
+        Args:
+            url: URL to guess sitemap for
+
+        Returns:
+            Guessed sitemap URL or None
+        """
+        base = self._extract_base_url(url)
+        common_paths = ["sitemap.xml", "sitemap_index.xml", "docs/sitemap.xml"]
+
+        for path in common_paths:
+            sitemap_url = urljoin(base, path)
+            try:
+                self.logger.debug(f"Trying sitemap: {sitemap_url}")
+                response = self.session.head(sitemap_url, timeout=10)
+                if response.status_code == 200:
+                    self.logger.info(f"Found sitemap: {sitemap_url}")
+                    return sitemap_url
+            except Exception:
+                continue
+
+        return None
+
+    def _crawl_links(self, start_urls: set[str], max_depth: int = 5) -> set[str]:
+        """
+        Crawl links from start URLs.
+
+        Args:
+            start_urls: URLs to start crawling from
+            max_depth: Maximum depth to crawl
+
+        Returns:
+            Set of discovered URLs
+        """
+        discovered: set[str] = set()
+        to_visit: set[tuple[str, int]] = {(url, 0) for url in start_urls}
+        visited: set[str] = set()
+
+        while to_visit:
+            url, depth = to_visit.pop()
+
+            if url in visited or depth > max_depth:
+                continue
+
+            if not self.validate_url(url):
+                continue
+
+            visited.add(url)
+            discovered.add(url)
+
+            if depth >= max_depth:
+                continue
+
+            try:
+                self.logger.debug(f"Crawling: {url} (depth {depth})")
+                response = self.session.get(url, timeout=30)
+                response.raise_for_status()
+
+                soup = BeautifulSoup(response.content, "html.parser")
+
+                for link in soup.find_all("a", href=True):
+                    href = link["href"]
+                    if not isinstance(href, str):
+                        continue
+
+                    # Resolve relative URLs
+                    absolute_url = urljoin(url, href)
+
+                    # Remove fragments and query params
+                    absolute_url = absolute_url.split("#")[0].split("?")[0]
+
+                    # Check if URL matches patterns
+                    if not any(pattern in absolute_url for pattern in self.include_patterns):
+                        continue
+
+                    if any(pattern in absolute_url for pattern in self.exclude_patterns):
+                        continue
+
+                    if absolute_url not in visited:
+                        to_visit.add((absolute_url, depth + 1))
+
+            except Exception as e:
+                self.logger.debug(f"Error crawling {url}: {e}")
+                continue
+
+        return discovered
+
+    def fetch(self) -> None:
+        """Fetch documentation using profile or generic scraping."""
+        self.logger.info(f"Fetching documentation from {self.start_url}")
+
+        urls: set[str] = set()
+
+        # Try sitemap first
+        if self.sitemap_url:
+            sitemap_urls = self.fetch_sitemap(self.sitemap_url)
+            if sitemap_urls:
+                urls.update(sitemap_urls)
+                self.logger.info(f"Found {len(sitemap_urls)} URLs in sitemap")
+
+        # Add start URLs if using profile
+        if self.profile and self.profile.start_urls:
+            urls.update(self.profile.start_urls)
+
+        # Crawl links if needed
+        if self.follow_links or (not urls and not self.sitemap_url):
+            start_urls = {self.start_url}
+            if self.profile and self.profile.start_urls:
+                start_urls.update(self.profile.start_urls)
+
+            self.logger.info(f"Crawling links from {len(start_urls)} start URL(s)")
+            crawled_urls = self._crawl_links(start_urls, self.max_depth)
+            urls.update(crawled_urls)
+            self.logger.info(f"Discovered {len(crawled_urls)} URLs via crawling")
+
+        if not urls:
+            self.logger.error("No URLs found to fetch")
+            return
+
+        # Apply filters
+        if self.include_patterns or self.exclude_patterns:
+            filtered_urls = []
+            for url in urls:
+                if self.include_patterns and not any(pattern in url for pattern in self.include_patterns):
+                    continue
+                if self.exclude_patterns and any(pattern in url for pattern in self.exclude_patterns):
+                    continue
+                filtered_urls.append(url)
+            urls = set(filtered_urls)
+
+        urls_list = sorted(urls)
+
+        # Apply max_pages limit
+        if self.max_pages:
+            urls_list = urls_list[: self.max_pages]
+            self.logger.info(f"Limited to {self.max_pages} pages")
+
+        self.logger.info(f"Processing {len(urls_list)} URLs")
+
+        # Process each URL
+        total = len(urls_list)
+        for idx, url in enumerate(urls_list, 1):
+            self.logger.info(f"[{idx}/{total}] Processing: {url}")
+
+            if self.profile and self.base_url:
+                filepath = self.create_output_path(url, self.base_url, self.output_subdir, self.strip_prefix)
+            else:
+                # Generic path creation
+                parsed = urlparse(url)
+                path = parsed.path.strip("/")
+                if not path:
+                    path = "index"
+                filepath = self.output_dir / self.output_subdir / f"{path.replace('/', '_')}.md"
+
+            self.process_url(url, filepath)
+
+        self.logger.info("Fetch complete")
+        self.print_stats()