sitewalker 0.2.0__tar.gz

sitewalker-0.2.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 Neil Johnson
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

sitewalker-0.2.0/PKG-INFO

@@ -0,0 +1,100 @@
+Metadata-Version: 2.4
+Name: sitewalker
+Version: 0.2.0
+Summary: Crawl a website and create a structured map of its pages
+License: MIT
+License-File: LICENSE
+Author: Neil Johnson
+Author-email: neil@cadent.net
+Requires-Python: >=3.9,<4.0
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
+Requires-Dist: beautifulsoup4 (>=4.12.3,<5.0.0)
+Requires-Dist: requests (>=2.32.3,<3.0.0)
+Requires-Dist: urllib3 (>=2.2.3,<3.0.0)
+Description-Content-Type: text/markdown
+
+# sitewalker
+
+Crawl a website and create a structured map of its pages.
+
+## Installation
+
+```bash
+pipx install sitewalker
+```
+
+## Usage
+
+```bash
+# Map all pages on a site (single-level crawl)
+sitewalker example.com
+
+# Recursive crawl of all internal pages
+sitewalker example.com -r
+
+# Collect external links
+sitewalker example.com -e
+
+# Recursive crawl with external link collection
+sitewalker example.com -r -e
+
+# Only crawl web pages (skip images, PDFs, etc.)
+sitewalker example.com -r -p
+
+# Crawl an HTTP-only site (e.g., LAN staging server)
+sitewalker http://staging.lan --allow-private
+
+# Verbose output for debugging
+sitewalker example.com -r -v
+```
+
+The target accepts a bare domain (`example.com`) or a full URL (`http://example.com`). Bare domains default to HTTPS — if the connection fails, sitewalker exits with a message to provide the full URL.
+
+## Options
+
+| Flag | Description | Default |
+|------|-------------|---------|
+| `-r`, `--recursive` | Recursively crawl internal links | Off |
+| `-e`, `--external-links` | Collect external links | Off |
+| `-p`, `--pages` | Only crawl web pages (HTML, PHP, etc.) | Off |
+| `-v`, `--verbose` | Enable verbose/debug output | Off |
+| `-t`, `--timeout` | Request timeout in seconds | 30 |
+| `--max-pages` | Maximum number of pages to crawl | 1000 |
+| `--max-depth` | Maximum crawl depth for recursive mode | 10 |
+| `--allow-private` | Allow crawling domains that resolve to private IPs | Off |
+| `--ignore-robots` | Ignore robots.txt rules | Off |
+
+## Output
+
+Results are saved to a CSV file named `{domain}_{timestamp}.csv` with columns:
+
+- **URL** — the page URL
+- **Title** — the page's `<title>` tag content
+- **Status Code** — HTTP response status
+
+When using `-e`, external links are saved to a separate `{domain}_{timestamp}_external_links.csv`.
+
+## Security
+
+- **SSRF protection**: Domains that resolve to private/reserved IP addresses are blocked by default. Use `--allow-private` to override for legitimate internal use.
+- **robots.txt**: Respected by default. Use `--ignore-robots` to override.
+- **CSV injection**: Output values are sanitized to prevent spreadsheet formula injection.
+- **Crawl limits**: Recursive crawls are bounded by `--max-pages` and `--max-depth` to prevent resource exhaustion.
+
+## Roadmap
+
+- `--format json` — JSON output format
+- `--check-links` — broken link detection
+- `--images --check-alt` — image inventory with alt text auditing
+
+## License
+
+MIT
+

sitewalker-0.2.0/README.md

@@ -0,0 +1,77 @@
+# sitewalker
+
+Crawl a website and create a structured map of its pages.
+
+## Installation
+
+```bash
+pipx install sitewalker
+```
+
+## Usage
+
+```bash
+# Map all pages on a site (single-level crawl)
+sitewalker example.com
+
+# Recursive crawl of all internal pages
+sitewalker example.com -r
+
+# Collect external links
+sitewalker example.com -e
+
+# Recursive crawl with external link collection
+sitewalker example.com -r -e
+
+# Only crawl web pages (skip images, PDFs, etc.)
+sitewalker example.com -r -p
+
+# Crawl an HTTP-only site (e.g., LAN staging server)
+sitewalker http://staging.lan --allow-private
+
+# Verbose output for debugging
+sitewalker example.com -r -v
+```
+
+The target accepts a bare domain (`example.com`) or a full URL (`http://example.com`). Bare domains default to HTTPS — if the connection fails, sitewalker exits with a message to provide the full URL.
+
+## Options
+
+| Flag | Description | Default |
+|------|-------------|---------|
+| `-r`, `--recursive` | Recursively crawl internal links | Off |
+| `-e`, `--external-links` | Collect external links | Off |
+| `-p`, `--pages` | Only crawl web pages (HTML, PHP, etc.) | Off |
+| `-v`, `--verbose` | Enable verbose/debug output | Off |
+| `-t`, `--timeout` | Request timeout in seconds | 30 |
+| `--max-pages` | Maximum number of pages to crawl | 1000 |
+| `--max-depth` | Maximum crawl depth for recursive mode | 10 |
+| `--allow-private` | Allow crawling domains that resolve to private IPs | Off |
+| `--ignore-robots` | Ignore robots.txt rules | Off |
+
+## Output
+
+Results are saved to a CSV file named `{domain}_{timestamp}.csv` with columns:
+
+- **URL** — the page URL
+- **Title** — the page's `<title>` tag content
+- **Status Code** — HTTP response status
+
+When using `-e`, external links are saved to a separate `{domain}_{timestamp}_external_links.csv`.
+
+## Security
+
+- **SSRF protection**: Domains that resolve to private/reserved IP addresses are blocked by default. Use `--allow-private` to override for legitimate internal use.
+- **robots.txt**: Respected by default. Use `--ignore-robots` to override.
+- **CSV injection**: Output values are sanitized to prevent spreadsheet formula injection.
+- **Crawl limits**: Recursive crawls are bounded by `--max-pages` and `--max-depth` to prevent resource exhaustion.
+
+## Roadmap
+
+- `--format json` — JSON output format
+- `--check-links` — broken link detection
+- `--images --check-alt` — image inventory with alt text auditing
+
+## License
+
+MIT

sitewalker-0.2.0/pyproject.toml

@@ -0,0 +1,26 @@
+[tool.poetry]
+name = "sitewalker"
+version = "0.2.0"
+description = "Crawl a website and create a structured map of its pages"
+authors = ["Neil Johnson <neil@cadent.net>"]
+readme = "README.md"
+license = "MIT"
+packages = [{include = "sitewalker", from = "src"}]
+
+[tool.poetry.scripts]
+sitewalker = "sitewalker.cli:main"
+
+[tool.poetry.dependencies]
+python = "^3.9"
+requests = "^2.32.3"
+beautifulsoup4 = "^4.12.3"
+urllib3 = "^2.2.3"
+
+[tool.poetry.group.dev.dependencies]
+pytest = "^8.3.4"
+pytest-cov = "^6.0.0"
+responses = "^0.25.3"
+
+[build-system]
+requires = ["poetry-core"]
+build-backend = "poetry.core.masonry.api"

sitewalker-0.2.0/src/sitewalker/__init__.py

@@ -0,0 +1 @@
+"""sitewalker — Crawl a website and create a structured map of its pages."""

sitewalker-0.2.0/src/sitewalker/cli.py

@@ -0,0 +1,145 @@
+#!/usr/bin/env python3
+
+import sys
+import argparse
+import logging
+from datetime import datetime
+from urllib.parse import urlparse
+import requests
+from sitewalker.crawler import WebsiteCrawler
+
+
+def setup_logging(verbose: bool):
+    """Configure logging based on verbosity level."""
+    root_logger = logging.getLogger()
+    handler = logging.StreamHandler()
+    formatter = logging.Formatter(
+        '%(asctime)s - %(levelname)s - %(message)s',
+        '%Y-%m-%d %H:%M:%S'
+    )
+    handler.setFormatter(formatter)
+    root_logger.addHandler(handler)
+
+    if verbose:
+        root_logger.setLevel(logging.DEBUG)
+    else:
+        root_logger.setLevel(logging.INFO)
+
+
+def main():
+    """Main function to run the crawler."""
+    parser = argparse.ArgumentParser(
+        description="Crawl a website and create a structured map of its pages"
+    )
+    parser.add_argument(
+        "target",
+        help="Domain or URL to crawl (e.g., example.com or http://example.com)"
+    )
+    parser.add_argument(
+        "-e", "--external-links",
+        action="store_true",
+        help="Check for external links on the domain"
+    )
+    parser.add_argument(
+        "-v", "--verbose",
+        action="store_true",
+        help="Enable verbose output"
+    )
+    parser.add_argument(
+        "-r", "--recursive",
+        action="store_true",
+        help="Recursively crawl internal links"
+    )
+    parser.add_argument(
+        "-p", "--pages",
+        action="store_true",
+        help="Only crawl web pages (HTML, PHP, etc.) and skip other file types"
+    )
+    parser.add_argument(
+        "-t", "--timeout",
+        type=int,
+        default=30,
+        help="Request timeout in seconds (default: 30)"
+    )
+    parser.add_argument(
+        "--max-pages",
+        type=int,
+        default=1000,
+        help="Maximum number of pages to crawl (default: 1000)"
+    )
+    parser.add_argument(
+        "--max-depth",
+        type=int,
+        default=10,
+        help="Maximum crawl depth for recursive mode (default: 10)"
+    )
+    parser.add_argument(
+        "--allow-private",
+        action="store_true",
+        help="Allow crawling domains that resolve to private/reserved IPs"
+    )
+    parser.add_argument(
+        "--ignore-robots",
+        action="store_true",
+        help="Ignore robots.txt rules when crawling"
+    )
+
+    args = parser.parse_args()
+    setup_logging(args.verbose)
+
+    try:
+        target = args.target
+        parsed = urlparse(target)
+
+        # If bare domain (no scheme), probe HTTPS first
+        if parsed.scheme not in ('http', 'https'):
+            probe_url = f"https://{target}"
+            try:
+                requests.head(probe_url, timeout=5, allow_redirects=True)
+                target = probe_url
+            except requests.ConnectionError:
+                logging.error(
+                    f"Could not connect to {probe_url}\n"
+                    f"If this site uses HTTP, provide the full URL:\n"
+                    f"  sitewalker http://{target}"
+                )
+                sys.exit(1)
+
+        # Extract domain for safe filename
+        parsed = urlparse(target)
+        safe_domain = parsed.netloc.replace('/', '_').replace('\\', '_').replace('..', '_')
+
+        crawler = WebsiteCrawler(target, timeout=args.timeout,
+                                  allow_private=args.allow_private,
+                                  ignore_robots=args.ignore_robots)
+        timestamp = datetime.now().strftime("%Y-%m-%dT%H%M")
+
+        if args.external_links:
+            crawler.crawl(
+                collect_external=True,
+                recursive=args.recursive,
+                pages_only=args.pages,
+                max_pages=args.max_pages,
+                max_depth=args.max_depth
+            )
+            ext_links_root = "_external_links.csv"
+            external_links_file = f"{safe_domain}_{timestamp}{ext_links_root}"
+            crawler.save_external_links_results(external_links_file)
+            logging.info(f"External links saved to {external_links_file}")
+        else:
+            crawler.crawl(recursive=args.recursive, pages_only=args.pages,
+                         max_pages=args.max_pages, max_depth=args.max_depth)
+            output_file = f"{safe_domain}_{timestamp}.csv"
+            crawler.save_results(output_file)
+            logging.info(f"Crawling complete! Results saved to {output_file}")
+
+        logging.info(f"Total pages crawled: {len(crawler.visited_urls)}")
+
+    except Exception as e:
+        logging.error(f"An error occurred: {str(e)}")
+        sys.exit(1)
+
+
+# pragma: no cover
+if __name__ == "__main__":
+    main()

sitewalker-0.2.0/src/sitewalker/crawler.py

@@ -0,0 +1,295 @@
+#!/usr/bin/env python3
+
+import csv
+import ipaddress
+import socket
+import requests
+from bs4 import BeautifulSoup
+from urllib.parse import urljoin, urlparse
+from urllib.robotparser import RobotFileParser
+from typing import Set, List, Tuple
+import time
+import logging
+
+logger = logging.getLogger(__name__)
+
+# List of file extensions that are considered web pages
+PAGE_EXTENSIONS = {
+    '', # for URLs ending in '/'
+    'html',
+    'htm',
+    'php',
+    'asp',
+    'aspx',
+    'jsp',
+    'shtml',
+    'phtml',
+    'xhtml',
+    'jspx',
+    'do',
+    'cfm',
+    'cgi'
+}
+
+class URLProcessingError(Exception):
+    """Custom exception for URL processing errors"""
+    pass
+
+class CrawlingError(Exception):
+    """Custom exception for crawling errors"""
+    pass
+
+class SSRFProtectionError(Exception):
+    """Raised when a domain resolves to a private/reserved IP address."""
+    pass
+
+
+def validate_domain_ssrf(domain: str) -> None:
+    """Check that a domain does not resolve to a private or reserved IP.
+
+    Raises SSRFProtectionError if the domain resolves to loopback,
+    private, link-local, or reserved address ranges.
+    """
+    try:
+        results = socket.getaddrinfo(domain, None)
+        for family, _, _, _, sockaddr in results:
+            ip = ipaddress.ip_address(sockaddr[0])
+            if ip.is_private or ip.is_loopback or ip.is_link_local or ip.is_reserved:
+                raise SSRFProtectionError(
+                    f"Domain '{domain}' resolves to private/reserved IP {ip}. "
+                    f"Use --allow-private to override."
+                )
+    except socket.gaierror as e:
+        raise CrawlingError(f"Cannot resolve domain '{domain}': {e}")
+
+class WebsiteCrawler:
+    USER_AGENT = 'Mozilla/5.0 (compatible; sitewalker/0.2.0; +https://github.com/cadentdev/sitewalker)'
+
+    def __init__(self, target: str, timeout: int = 30, allow_private: bool = False,
+                 ignore_robots: bool = False):
+        # Parse target: accept full URL (http://example.com) or bare domain (example.com)
+        parsed = urlparse(target)
+        if parsed.scheme in ('http', 'https'):
+            self.domain = parsed.netloc
+            self.base_url = f"{parsed.scheme}://{parsed.netloc}"
+        else:
+            # Bare domain — assume HTTPS
+            self.domain = target
+            self.base_url = f"https://{target}"
+
+        if not allow_private:
+            validate_domain_ssrf(self.domain)
+
+        # Normalize the base URL
+        self.base_url, _ = self.process_url(self.base_url)
+        self.visited_urls: Set[str] = set()
+        self.results: List[Tuple[str, str, int]] = []
+        self.external_links: Set[str] = set()
+        self.pages_only: bool = False
+        self.timeout = timeout
+        self.ignore_robots = ignore_robots
+        self.robot_parser: RobotFileParser | None = None
+        self.session = requests.Session()
+        self.session.headers.update({'User-Agent': self.USER_AGENT})
+
+    def process_url(self, url: str) -> Tuple[str, bool]:
+        """
+        Process and validate a URL.
+        Returns: (cleaned_url, is_internal)
+        Raises: URLProcessingError if URL is invalid
+        """
+        if not url:
+            raise URLProcessingError("Empty URL")
+
+        try:
+            parsed_url = urlparse(url)
+            if not parsed_url.scheme or not parsed_url.netloc:
+                raise URLProcessingError("Invalid URL format")
+
+            # Clean URL by removing fragments, query parameters, and trailing slashes
+            path = parsed_url.path
+            if not path or path == '/':
+                path = ''
+            else:
+                path = path.rstrip('/')
+
+            clean_url = f"{parsed_url.scheme}://{parsed_url.netloc}{path}"
+            is_internal = self.domain in parsed_url.netloc
+
+            if parsed_url.scheme not in ('http', 'https'):
+                raise URLProcessingError("Unsupported protocol")
+
+            return clean_url, is_internal
+
+        except Exception as e:
+            raise URLProcessingError(f"URL processing error: {str(e)}")
+
+    def is_page(self, url: str) -> bool:
+        """
+        Check if a URL points to a web page based on its extension or path.
+        """
+        try:
+            parsed = urlparse(url)
+            if not parsed.scheme or not parsed.netloc:
+                return False
+
+            path = parsed.path.rstrip('/')
+
+            # URLs ending with '/' are considered pages (directory index)
+            if not path or path.endswith('/'):
+                return True
+
+            # Check if the file extension (if any) is in our list of page extensions
+            if '.' in path:
+                ext = path.split('.')[-1].lower()
+                return ext in PAGE_EXTENSIONS
+
+            # URLs without extensions are considered pages
+            return True
+
+        except Exception as e:
+            logger.debug(f"Error checking if URL is page: {str(e)}")
+            return False
+
+    def _load_robots_txt(self) -> None:
+        """Fetch and parse robots.txt for the target domain."""
+        if self.ignore_robots:
+            return
+        robots_url = f"{self.base_url}/robots.txt"
+        try:
+            resp = self.session.get(robots_url, timeout=self.timeout)
+            if resp.status_code == 200:
+                rp = RobotFileParser()
+                rp.set_url(robots_url)
+                rp.parse(resp.text.splitlines())
+                self.robot_parser = rp
+                logger.info(f"Loaded robots.txt from {robots_url}")
+            else:
+                logger.info(f"No robots.txt found at {robots_url} (HTTP {resp.status_code})")
+        except Exception as e:
+            logger.warning(f"Could not load robots.txt from {robots_url}: {e}")
+            self.robot_parser = None
+
+    def _is_allowed_by_robots(self, url: str) -> bool:
+        """Check if a URL is allowed by robots.txt rules."""
+        if self.ignore_robots or self.robot_parser is None:
+            return True
+        return self.robot_parser.can_fetch(self.USER_AGENT, url)
+
+    def crawl(self, collect_external: bool = False, recursive: bool = False,
+              pages_only: bool = False, max_pages: int = 1000, max_depth: int = 10) -> None:
+        """
+        Crawl the website starting from the base URL.
+
+        In non-recursive mode:
+        - Crawls the base URL and follows internal links found on that page
+        - Does not follow links found on subsequent pages
+
+        In recursive mode:
+        - Crawls the base URL and follows all internal links recursively
+        - Continues until all reachable internal pages are visited
+        """
+        self.pages_only = pages_only
+        self.max_pages = max_pages
+        self.max_depth = max_depth
+        self._load_robots_txt()
+        logger.info(f"Starting crawl of {self.base_url}")
+        logger.info(f"Mode: {'Recursive' if recursive else 'Single-level'} crawl, "
+                   f"{'collecting' if collect_external else 'ignoring'} external links, "
+                   f"{'pages only' if pages_only else 'all files'}, "
+                   f"max_pages={max_pages}, max_depth={max_depth}")
+        self._crawl_page(self.base_url, collect_external, recursive, depth=0)
+        logger.info(f"Crawl complete. Visited {len(self.visited_urls)} pages")
+        if collect_external:
+            logger.info(f"Found {len(self.external_links)} unique external links")
+
+    def _crawl_page(self, url: str, collect_external: bool, recursive: bool, depth: int = 0) -> None:
+        """Internal method to crawl a single page and process its links."""
+        if len(self.visited_urls) >= self.max_pages:
+            logger.info(f"Reached max_pages limit ({self.max_pages})")
+            return
+        if depth > self.max_depth:
+            logger.debug(f"Reached max_depth limit ({self.max_depth}) at {url}")
+            return
+        try:
+            clean_url, is_internal = self.process_url(url)
+            if not is_internal or clean_url in self.visited_urls:
+                return
+
+            # Check robots.txt rules
+            if not self._is_allowed_by_robots(clean_url):
+                logger.debug(f"Blocked by robots.txt: {clean_url}")
+                return
+
+            # Skip non-page URLs if pages_only is True
+            if self.pages_only and not self.is_page(clean_url):
+                logger.debug(f"Skipping non-page URL: {clean_url}")
+                return
+
+            self.visited_urls.add(clean_url)
+            logger.debug(f"Crawling {clean_url}")
+
+            response = self.session.get(clean_url, timeout=self.timeout)
+            response.raise_for_status()
+            soup = BeautifulSoup(response.text, 'html.parser')
+
+            # Process page title
+            title = soup.title.string.strip() if soup.title else "No title"
+            self.results.append((clean_url, title, response.status_code))
+
+            # Process links
+            for link in soup.find_all('a', href=True):
+                next_url = urljoin(url, link['href'])
+                try:
+                    next_clean_url, next_is_internal = self.process_url(next_url)
+
+                    if next_is_internal and recursive:
+                        if next_clean_url not in self.visited_urls:
+                            self._crawl_page(next_clean_url, collect_external, recursive, depth + 1)
+                    elif not next_is_internal and collect_external:
+                        self.external_links.add(next_clean_url)
+
+                except URLProcessingError:
+                    continue
+
+        except requests.HTTPError as e:
+            logger.error(f"HTTP Error crawling {url}: {str(e)}")
+            self.results.append((url, "Error", e.response.status_code))
+        except Exception as e:
+            logger.error(f"Error crawling {url}: {str(e)}")
+            self.results.append((url, "Error", 0))
+
+        time.sleep(1)  # Be polite
+
+    @staticmethod
+    def _sanitize_csv_value(value: str) -> str:
+        """Sanitize a value for safe CSV output.
+
+        Prevents CSV injection by prefixing dangerous characters that
+        spreadsheet applications interpret as formulas.
+        """
+        if isinstance(value, str) and value and value[0] in ('=', '+', '-', '@', '\t', '\r'):
+            return "'" + value
+        return value
+
+    def save_results(self, output_file: str) -> None:
+        """Save results to a CSV file."""
+        with open(output_file, 'w', newline='', encoding='utf-8') as f:
+            writer = csv.writer(f)
+            writer.writerow(['URL', 'Title', 'Status Code'])
+            for url, title, status in self.results:
+                writer.writerow([
+                    self._sanitize_csv_value(url),
+                    self._sanitize_csv_value(title),
+                    status
+                ])
+        logger.info(f"Results saved to {output_file}")
+
+    def save_external_links_results(self, filename: str) -> None:
+        """Save external links to a CSV file."""
+        with open(filename, 'w', newline='', encoding='utf-8') as csvfile:
+            writer = csv.writer(csvfile)
+            writer.writerow(['External URL'])
+            for url in sorted(self.external_links):
+                writer.writerow([self._sanitize_csv_value(url)])
+        logger.info(f"External links saved to {filename}")