ref-agents 1.0.0__py3-none-any.whl

ref_agents/security/url_scan.py

@@ -0,0 +1,240 @@
+"""URL/Endpoint Scanner.
+
+Scans files for URLs and API endpoints that could indicate
+external data transmission or cloud service connections.
+"""
+
+import re
+import subprocess
+from dataclasses import dataclass, field
+from pathlib import Path
+
+import structlog
+
+logger = structlog.get_logger(__name__)
+
+# Directories to skip during scanning
+IGNORED_DIRS: set[str] = {
+    ".venv",
+    "venv",
+    ".env",
+    "node_modules",
+    ".git",
+    "dist",
+    "build",
+    ".tox",
+}
+
+
+def _should_skip(path: Path) -> bool:
+    """Check if path should be skipped."""
+    for part in path.parts:
+        if part in IGNORED_DIRS:
+            return True
+        if part.endswith("_cache") or part.endswith("cache__"):
+            return True
+    return False
+
+
+# Patterns to detect URLs and endpoints
+URL_PATTERNS: list[tuple[str, re.Pattern[str]]] = [
+    ("HTTP/HTTPS URL", re.compile(r'https?://[^\s\'"<>)}\]]+', re.IGNORECASE)),
+    ("WebSocket URL", re.compile(r'wss?://[^\s\'"<>)}\]]+', re.IGNORECASE)),
+    ("FTP URL", re.compile(r'ftp://[^\s\'"<>)}\]]+', re.IGNORECASE)),
+]
+
+# URLs that are safe and expected (documentation, localhost, etc.)
+WHITELIST_PATTERNS: list[re.Pattern[str]] = [
+    re.compile(r"https?://localhost", re.IGNORECASE),
+    re.compile(r"https?://127\.0\.0\.1", re.IGNORECASE),
+    re.compile(r"https?://0\.0\.0\.0", re.IGNORECASE),
+    re.compile(r"https?://example\.com", re.IGNORECASE),
+    re.compile(r"https?://example\.org", re.IGNORECASE),
+    # Documentation and reference links
+    re.compile(r"https?://modelcontextprotocol\.io", re.IGNORECASE),
+    re.compile(r"https?://github\.com", re.IGNORECASE),
+    re.compile(r"https?://ghcr\.io", re.IGNORECASE),
+    re.compile(r"https?://files\.pythonhosted\.org", re.IGNORECASE),
+    re.compile(r"https?://pypi\.org", re.IGNORECASE),
+    re.compile(r"https?://docs\.pytest\.org", re.IGNORECASE),
+    # Schema references (not actual URLs called at runtime)
+    re.compile(r"https?://json-schema\.org", re.IGNORECASE),
+    re.compile(r"https?://www\.w3\.org", re.IGNORECASE),
+    re.compile(r"https?://docs\.astral\.sh", re.IGNORECASE),
+    re.compile(r"https?://blog\.crewai\.com", re.IGNORECASE),
+    re.compile(r"https?://api\.example\.com", re.IGNORECASE),
+    re.compile(r"https?://astral\.sh", re.IGNORECASE),
+    re.compile(r"https?://api\.stripe\.com", re.IGNORECASE),
+    re.compile(r"https?://stripe\.com", re.IGNORECASE),
+    re.compile(r"https?://boto3\.amazonaws\.com", re.IGNORECASE),
+    # Common Badges and Libraries
+    re.compile(r"https?://img\.shields\.io", re.IGNORECASE),
+    re.compile(r"https?://www\.structlog\.org", re.IGNORECASE),
+    # vis.js CDN — used in context_graph.to_html() for interactive graph viewer
+    re.compile(r"https?://cdn\.jsdelivr\.net", re.IGNORECASE),
+    # FastMCP Cloud schema references (not actual URLs called at runtime)
+    re.compile(r"https?://gofastmcp\.com", re.IGNORECASE),
+    # Microsoft XML schema references in .csproj / docs (not called at runtime)
+    re.compile(r"https?://schemas\.microsoft\.com", re.IGNORECASE),
+]
+
+
+@dataclass
+class URLScanResult:
+    """Result of URL scan."""
+
+    passed: bool
+    files_scanned: int
+    external_urls: list[dict[str, str]] = field(default_factory=list)
+    whitelisted_urls: list[dict[str, str]] = field(default_factory=list)
+
+    @property
+    def summary(self) -> str:
+        """One-line summary for CLI output."""
+        if self.passed:
+            return f"0 external URLs found in {self.files_scanned} files"
+        return f"{len(self.external_urls)} external URLs found"
+
+
+def _is_whitelisted(url: str) -> bool:
+    """Check if URL matches any whitelist pattern."""
+    # Skip regex pattern fragments (contain backslashes or regex metacharacters)
+    if "\\" in url or "[^" in url or "\\s" in url:
+        return True
+    return any(pattern.search(url) for pattern in WHITELIST_PATTERNS)
+
+
+def _should_scan_file(file_path: Path, include_extensions: set[str]) -> bool:
+    """Check if file should be scanned."""
+    # Skip directories and non-matching extensions
+    if file_path.is_dir():
+        return False
+    if file_path.suffix not in include_extensions:
+        return False
+    if _should_skip(file_path):
+        return False
+    # Skip test files (they contain test data)
+    if "/tests/" in str(file_path) or "test_" in file_path.name:
+        return False
+    # Skip generated reports
+    if "ref-reports" in str(file_path):
+        return False
+    # Skip lock files (they contain many package URLs)
+    if file_path.name in {"uv.lock", "poetry.lock", "package-lock.json"}:
+        return False
+    # Skip generated JSON reports (bandit, pip-audit)
+    if file_path.name in {"bandit-report.json", "pip-audit-report.json"}:
+        return False
+    # User defined ignores
+    if "GAP_ANALYSIS.md" in file_path.name:
+        return False
+    if "docs/misc" in str(file_path):
+        return False
+    return True
+
+
+def _process_file_urls(
+    file_path: Path,
+    directory: Path,
+    external_urls: list[dict[str, str]],
+    whitelisted_urls: list[dict[str, str]],
+) -> None:
+    """Process a single file for URLs."""
+    try:
+        content = file_path.read_text(encoding="utf-8")
+    except UnicodeDecodeError:
+        return
+
+    relative_path = str(file_path.relative_to(directory))
+
+    for pattern_name, pattern in URL_PATTERNS:
+        for match in pattern.finditer(content):
+            url = match.group(0)
+            # Clean up trailing punctuation
+            url = url.rstrip(".,;:\"')")
+
+            url_info = {
+                "file": relative_path,
+                "url": url,
+                "type": pattern_name,
+            }
+
+            if _is_whitelisted(url):
+                whitelisted_urls.append(url_info)
+            else:
+                external_urls.append(url_info)
+                logger.info(
+                    "external_url_found",
+                    file=relative_path,
+                    url=url,
+                )
+
+
+def _iter_scannable_files(directory: Path, include_extensions: set[str]) -> list[Path]:
+    """Return files to scan, preferring git-tracked files when inside a repo.
+
+    Falls back to rglob when not in a git repository (e.g., CI without full
+    checkout).
+    """
+    try:
+        result = subprocess.run(
+            ["git", "ls-files", "--cached", "--others", "--exclude-standard"],
+            cwd=directory,
+            capture_output=True,
+            text=True,
+            timeout=10,
+        )
+        if result.returncode == 0:
+            paths = []
+            for line in result.stdout.splitlines():
+                p = directory / line.strip()
+                if p.suffix in include_extensions and p.exists():
+                    paths.append(p)
+            return paths
+    except (subprocess.TimeoutExpired, FileNotFoundError):
+        pass
+    return list(directory.rglob("*"))
+
+
+def scan_urls(
+    directory: Path,
+    include_extensions: set[str] | None = None,
+) -> URLScanResult:
+    """
+    Scan files for external URLs and endpoints.
+
+    Args:
+        directory: Root directory to scan
+        include_extensions: File extensions to scan (default: .py, .md, .txt, .json)
+
+    Returns:
+        URLScanResult with pass/fail status and found URLs
+    """
+    include_extensions = include_extensions or {".py", ".md", ".txt", ".json", ".toml"}
+    external_urls: list[dict[str, str]] = []
+    whitelisted_urls: list[dict[str, str]] = []
+    files_scanned = 0
+
+    for file_path in _iter_scannable_files(directory, include_extensions):
+        if not _should_scan_file(file_path, include_extensions):
+            continue
+
+        files_scanned += 1
+        _process_file_urls(file_path, directory, external_urls, whitelisted_urls)
+
+    passed = len(external_urls) == 0
+
+    logger.info(
+        "url_scan_complete",
+        passed=passed,
+        files_scanned=files_scanned,
+        external_count=len(external_urls),
+        whitelisted_count=len(whitelisted_urls),
+    )
+
+    return URLScanResult(
+        passed=passed,
+        files_scanned=files_scanned,
+        external_urls=external_urls,
+        whitelisted_urls=whitelisted_urls,
+    )

ref_agents/security_scan.py

@@ -0,0 +1,236 @@
+"""REF Agents Security Scanner CLI.
+
+A single command that runs all security checks and produces a professional
+security audit report for enterprise customers.
+
+Usage:
+    uv run python -m ref_agents.security_scan
+    uv run python -m ref_agents.security_scan --verbose
+    uv run python -m ref_agents.security_scan --format json
+"""
+
+import argparse
+import json
+import sys
+from datetime import datetime
+from pathlib import Path
+
+import structlog
+
+from ref_agents.constants import Icons, Status
+from ref_agents.security.dependency_audit import audit_dependencies
+from ref_agents.security.file_audit import audit_file_operations
+from ref_agents.security.network_scan import scan_network_imports
+from ref_agents.security.report_generator import SecurityReport, save_report
+from ref_agents.security.secret_scan import scan_secrets
+from ref_agents.security.url_scan import scan_urls
+
+# Configure structlog for CLI output
+structlog.configure(
+    processors=[
+        structlog.processors.TimeStamper(fmt="iso"),
+        structlog.processors.JSONRenderer(),
+    ]
+)
+logger = structlog.get_logger(__name__)
+
+
+def echo(message: str = "") -> None:
+    """Write message to stdout for CLI output.
+
+    Args:
+        message: Text to display. Empty string prints blank line.
+    """
+    sys.stdout.write(message + "\n")
+    sys.stdout.flush()
+
+
+def print_status(check_num: int, total: int, name: str, passed: bool) -> None:
+    """Print formatted status line to stdout."""
+    status = Status.PASS if passed else Status.FAIL
+    dots = "." * (30 - len(name))
+    echo(f"[{check_num}/{total}] {name}{dots} {status}")
+
+
+def run_security_scan(
+    directory: Path,
+    verbose: bool = False,
+    output_format: str = "text",
+    no_report: bool = False,
+) -> bool:
+    """
+    Run all security scans and generate report.
+
+    Args:
+        directory: Root directory to scan
+        verbose: Print detailed output
+        output_format: Output format (text, json, markdown)
+        no_report: If True, skip saving report to disk
+
+    Returns:
+        True if all scans passed, False otherwise
+    """
+    echo(Icons.HEAVY_LINE * 60)
+    echo("          REF-AGENTS SECURITY AUDIT")
+    echo(Icons.HEAVY_LINE * 60)
+    echo()
+    echo(f"Scanning: {directory}")
+    echo(f"Started:  {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}")
+    echo()
+
+    # Run all scans
+    src_dir = directory / "src"
+    if not src_dir.exists():
+        src_dir = directory
+
+    # 1. Network Import Scan
+    if verbose:
+        echo("Running network import scan...")
+    network_result = scan_network_imports(src_dir)
+    print_status(1, 6, "Network Import Scan", network_result.passed)
+
+    # 2. URL Scan
+    if verbose:
+        echo("Running URL scan...")
+    url_result = scan_urls(directory)
+    print_status(2, 6, "Outbound URL Scan", url_result.passed)
+
+    # 3. Secret Scan
+    if verbose:
+        echo("Running secret scan...")
+    secret_result = scan_secrets(directory)
+    print_status(3, 6, "Secret Detection", secret_result.passed)
+
+    # 4. Dependency Audit
+    if verbose:
+        echo("Running dependency audit...")
+    dependency_result = audit_dependencies(directory)
+    print_status(4, 6, "Dependency Audit", dependency_result.passed)
+
+    # 5. License Check (part of dependency audit)
+    license_passed = len(dependency_result.unsafe_licenses) == 0
+    print_status(5, 6, "License Check", license_passed)
+
+    # 6. File Operation Audit
+    if verbose:
+        echo("Running file operation audit...")
+    file_result = audit_file_operations(src_dir)
+    print_status(6, 6, "File Operation Audit", file_result.passed)
+
+    echo()
+
+    # Generate report
+    report = SecurityReport(
+        network_result=network_result,
+        url_result=url_result,
+        secret_result=secret_result,
+        dependency_result=dependency_result,
+        file_result=file_result,
+        generated_at=datetime.now(),
+    )
+
+    if output_format == "json":
+        # JSON output for CI
+        result = {
+            "passed": report.overall_passed,
+            "timestamp": report.generated_at.isoformat(),
+            "checks": {
+                "network_scan": {
+                    "passed": network_result.passed,
+                    "violations": len(network_result.violations),
+                },
+                "url_scan": {
+                    "passed": url_result.passed,
+                    "external_urls": len(url_result.external_urls),
+                },
+                "secret_scan": {
+                    "passed": secret_result.passed,
+                    "secrets_found": len(secret_result.secrets_found),
+                },
+                "dependency_audit": {
+                    "passed": dependency_result.passed,
+                    "dependencies": len(dependency_result.dependencies),
+                },
+                "file_audit": {
+                    "passed": file_result.passed,
+                    "dangerous_ops": len(file_result.dangerous_operations),
+                },
+            },
+        }
+        echo(json.dumps(result, indent=2))
+    else:
+        # Save markdown report (unless --no-report)
+        if not no_report:
+            report_path = save_report(report)
+            echo(f"Report saved: {report_path}")
+        echo()
+
+        # Overall status
+        if report.overall_passed:
+            echo(Icons.HEAVY_LINE * 60)
+            echo(f"               {Status.SUCCESS} ALL CHECKS PASSED")
+            echo(Icons.HEAVY_LINE * 60)
+        else:
+            echo(Icons.HEAVY_LINE * 60)
+            echo(f"               {Status.ERROR} SOME CHECKS FAILED")
+            echo(Icons.HEAVY_LINE * 60)
+            echo()
+            echo("Review the report for details on failures.")
+
+    return report.overall_passed
+
+
+def main() -> int:
+    """CLI entry point."""
+    parser = argparse.ArgumentParser(
+        description="REF Agents Security Scanner - Enterprise Security Audit Tool"
+    )
+    parser.add_argument(
+        "--directory",
+        "-d",
+        type=Path,
+        default=Path.cwd(),
+        help="Directory to scan (default: current directory)",
+    )
+    parser.add_argument(
+        "--verbose",
+        "-v",
+        action="store_true",
+        help="Print detailed output",
+    )
+    parser.add_argument(
+        "--format",
+        "-f",
+        choices=["text", "json"],
+        default="text",
+        help="Output format (default: text)",
+    )
+    parser.add_argument(
+        "--no-report",
+        action="store_true",
+        help="Skip saving report to disk (for pre-commit hooks)",
+    )
+
+    args = parser.parse_args()
+
+    # Find project root (look for pyproject.toml)
+    directory = args.directory.resolve()
+    if not (directory / "pyproject.toml").exists():
+        # Try parent directories
+        for parent in directory.parents:
+            if (parent / "pyproject.toml").exists():
+                directory = parent
+                break
+
+    passed = run_security_scan(
+        directory=directory,
+        verbose=args.verbose,
+        output_format=args.format,
+        no_report=args.no_report,
+    )
+
+    return 0 if passed else 1
+
+
+if __name__ == "__main__":
+    sys.exit(main())