complio 0.1.1__py3-none-any.whl

complio/utils/history.py

@@ -0,0 +1,243 @@
+"""
+Scan history tracking for Complio.
+
+This module manages local storage of scan results to allow users to:
+- View recent scans
+- Compare scan results over time
+- Track compliance improvements
+
+Example:
+    >>> from complio.utils.history import save_scan_to_history, get_scan_history
+    >>> save_scan_to_history(scan_id, results)
+    >>> history = get_scan_history(limit=10)
+"""
+
+import json
+import os
+from datetime import datetime
+from pathlib import Path
+from typing import Any, Dict, List, Optional
+
+from complio.core.runner import ScanResults
+
+
+# History directory location
+SCAN_HISTORY_DIR = Path.home() / ".complio" / "history"
+
+
+def ensure_history_dir() -> Path:
+    """Ensure scan history directory exists with proper permissions.
+
+    Returns:
+        Path to history directory
+
+    Example:
+        >>> history_dir = ensure_history_dir()
+        >>> print(history_dir)
+        PosixPath('/home/user/.complio/history')
+    """
+    SCAN_HISTORY_DIR.mkdir(parents=True, exist_ok=True)
+    # Set directory permissions to 700 (owner only)
+    SCAN_HISTORY_DIR.chmod(0o700)
+    return SCAN_HISTORY_DIR
+
+
+def save_scan_to_history(scan_id: str, results: ScanResults, region: str) -> Path:
+    """Save scan results to local history.
+
+    Args:
+        scan_id: Unique scan identifier
+        results: Scan results to save
+        region: AWS region that was scanned
+
+    Returns:
+        Path to saved history file
+
+    Example:
+        >>> save_scan_to_history("scan_20260107_162335_abc123", results, "eu-west-3")
+        PosixPath('/home/user/.complio/history/scan_20260107_162335_abc123.json')
+    """
+    history_dir = ensure_history_dir()
+    filepath = history_dir / f"{scan_id}.json"
+
+    # Prepare history data
+    history_data = {
+        "scan_id": scan_id,
+        "timestamp": datetime.fromtimestamp(results.timestamp).isoformat(),
+        "region": region,
+        "summary": {
+            "total_tests": results.total_tests,
+            "passed_tests": results.passed_tests,
+            "failed_tests": results.failed_tests,
+            "error_tests": results.error_tests,
+            "overall_score": results.overall_score,
+            "compliance_status": "COMPLIANT" if results.overall_score >= 90 else "NON_COMPLIANT",
+            "execution_time_seconds": round(results.execution_time, 2),
+        },
+        "test_results": [
+            {
+                "test_id": tr.test_id,
+                "test_name": tr.test_name,
+                "status": tr.status,
+                "passed": tr.passed,
+                "score": tr.score,
+                "findings_count": len(tr.findings),
+            }
+            for tr in results.test_results
+        ],
+    }
+
+    # Save to file
+    with open(filepath, 'w') as f:
+        json.dump(history_data, f, indent=2)
+
+    # Set file permissions to 600 (owner read/write only)
+    filepath.chmod(0o600)
+
+    return filepath
+
+
+def get_scan_history(limit: int = 10) -> List[Dict[str, Any]]:
+    """Get list of recent scans from history.
+
+    Args:
+        limit: Maximum number of scans to return (default: 10)
+
+    Returns:
+        List of scan history entries, most recent first
+
+    Example:
+        >>> history = get_scan_history(limit=5)
+        >>> for scan in history:
+        ...     print(f"{scan['timestamp']}: {scan['summary']['overall_score']}%")
+    """
+    if not SCAN_HISTORY_DIR.exists():
+        return []
+
+    # Get all scan files
+    scan_files = sorted(
+        SCAN_HISTORY_DIR.glob("scan_*.json"),
+        key=lambda p: p.stat().st_mtime,
+        reverse=True  # Most recent first
+    )
+
+    history = []
+    for scan_file in scan_files[:limit]:
+        try:
+            with open(scan_file, 'r') as f:
+                data = json.load(f)
+                history.append(data)
+        except (json.JSONDecodeError, KeyError):
+            # Skip corrupted files
+            continue
+
+    return history
+
+
+def get_scan_by_id(scan_id: str) -> Optional[Dict[str, Any]]:
+    """Get a specific scan by its ID.
+
+    Args:
+        scan_id: Scan identifier to retrieve
+
+    Returns:
+        Scan data dictionary, or None if not found
+
+    Example:
+        >>> scan = get_scan_by_id("scan_20260107_162335_abc123")
+        >>> if scan:
+        ...     print(scan['summary']['overall_score'])
+    """
+    filepath = SCAN_HISTORY_DIR / f"{scan_id}.json"
+
+    if not filepath.exists():
+        return None
+
+    try:
+        with open(filepath, 'r') as f:
+            return json.load(f)
+    except (json.JSONDecodeError, IOError):
+        return None
+
+
+def compare_scans(scan_id_1: str, scan_id_2: str) -> Dict[str, Any]:
+    """Compare two scans and return the differences.
+
+    Args:
+        scan_id_1: First scan ID
+        scan_id_2: Second scan ID
+
+    Returns:
+        Dictionary containing comparison data
+
+    Example:
+        >>> diff = compare_scans("scan_20260107_...", "scan_20260106_...")
+        >>> print(f"Score change: {diff['score_change']}%")
+    """
+    scan1 = get_scan_by_id(scan_id_1)
+    scan2 = get_scan_by_id(scan_id_2)
+
+    if not scan1 or not scan2:
+        raise ValueError(f"One or both scans not found in history")
+
+    # Calculate differences
+    score1 = scan1['summary']['overall_score']
+    score2 = scan2['summary']['overall_score']
+    score_change = score1 - score2
+
+    passed1 = scan1['summary']['passed_tests']
+    passed2 = scan2['summary']['passed_tests']
+
+    failed1 = scan1['summary']['failed_tests']
+    failed2 = scan2['summary']['failed_tests']
+
+    return {
+        "scan1": {
+            "scan_id": scan1['scan_id'],
+            "timestamp": scan1['timestamp'],
+            "score": score1,
+            "passed": passed1,
+            "failed": failed1,
+        },
+        "scan2": {
+            "scan_id": scan2['scan_id'],
+            "timestamp": scan2['timestamp'],
+            "score": score2,
+            "passed": passed2,
+            "failed": failed2,
+        },
+        "differences": {
+            "score_change": score_change,
+            "score_change_direction": "improved" if score_change > 0 else "declined" if score_change < 0 else "unchanged",
+            "passed_change": passed1 - passed2,
+            "failed_change": failed1 - failed2,
+        }
+    }
+
+
+def clear_old_history(keep_days: int = 30) -> int:
+    """Clear scan history older than specified days.
+
+    Args:
+        keep_days: Number of days to keep (default: 30)
+
+    Returns:
+        Number of files deleted
+
+    Example:
+        >>> deleted = clear_old_history(keep_days=30)
+        >>> print(f"Deleted {deleted} old scans")
+    """
+    if not SCAN_HISTORY_DIR.exists():
+        return 0
+
+    from datetime import timedelta
+    cutoff_time = datetime.now().timestamp() - (keep_days * 86400)
+
+    deleted_count = 0
+    for scan_file in SCAN_HISTORY_DIR.glob("scan_*.json"):
+        if scan_file.stat().st_mtime < cutoff_time:
+            scan_file.unlink()
+            deleted_count += 1
+
+    return deleted_count

complio/utils/logger.py

@@ -0,0 +1,391 @@
+"""
+Structured logging with credential filtering.
+
+This module provides secure, structured logging for Complio with automatic
+credential redaction. All sensitive data is filtered before being written to logs.
+
+Security Features:
+    - Automatic credential redaction (AWS keys, secrets, passwords)
+    - Structured JSON logging for machine parsing
+    - Log rotation to prevent disk fill
+    - Separate console and file logging
+    - Configurable log levels
+    - Context binding for request tracking
+
+Example:
+    >>> from complio.utils.logger import get_logger
+    >>> logger = get_logger(__name__)
+    >>> logger.info("test_started", test_name="s3_encryption", region="us-east-1")
+    >>> # Credentials automatically redacted:
+    >>> logger.info("aws_response", access_key="AKIAIOSFODNN7EXAMPLE")
+    >>> # Logged as: access_key="[REDACTED_AWS_ACCESS_KEY]"
+"""
+
+import logging
+import re
+from logging.handlers import RotatingFileHandler
+from pathlib import Path
+from typing import Any, Dict, Optional
+
+import structlog
+from structlog.processors import JSONRenderer
+from structlog.stdlib import add_log_level, filter_by_level
+
+from complio.config.settings import get_settings
+
+# ============================================================================
+# CREDENTIAL PATTERNS (for redaction)
+# ============================================================================
+
+# AWS Access Key patterns
+AWS_ACCESS_KEY_PATTERN = re.compile(r"(AKIA[0-9A-Z]{16})", re.IGNORECASE)
+AWS_SECRET_KEY_PATTERN = re.compile(r"([A-Za-z0-9/+=]{40})", re.IGNORECASE)
+AWS_SESSION_TOKEN_PATTERN = re.compile(r"(FwoGZXIvYXdzE[A-Za-z0-9/+=]{100,})", re.IGNORECASE)
+
+# Generic password patterns
+PASSWORD_PATTERN = re.compile(
+    r'(password["\s:=]+)([^"\s,}]+)',
+    re.IGNORECASE
+)
+
+# API keys and tokens
+API_KEY_PATTERN = re.compile(
+    r'(["\s](api[_-]?key|token|secret)["\s:=]+)([^"\s,}]+)',
+    re.IGNORECASE
+)
+
+# Private IP addresses (optional - may want to keep these)
+PRIVATE_IP_PATTERN = re.compile(
+    r'\b(?:10|172\.(?:1[6-9]|2[0-9]|3[01])|192\.168)\.\d{1,3}\.\d{1,3}\b'
+)
+
+# Email addresses (optional - may want to keep these)
+EMAIL_PATTERN = re.compile(r'\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,}\b')
+
+
+# ============================================================================
+# CREDENTIAL FILTERING PROCESSOR
+# ============================================================================
+
+
+def filter_credentials(logger: Any, method_name: str, event_dict: Dict[str, Any]) -> Dict[str, Any]:
+    """Filter sensitive credentials from log events.
+
+    This processor scans all log event data and redacts any credentials,
+    passwords, API keys, or other sensitive information.
+
+    Redaction Rules:
+        - AWS Access Keys (AKIA...) → [REDACTED_AWS_ACCESS_KEY]
+        - AWS Secret Keys (40 chars) → [REDACTED_AWS_SECRET_KEY]
+        - AWS Session Tokens → [REDACTED_AWS_SESSION_TOKEN]
+        - AWS Account IDs (12 digits) → ****1234 (last 4 digits only)
+        - User IDs → ****W53X (last 4 chars only)
+        - Passwords → [REDACTED_PASSWORD]
+        - API Keys/Tokens → [REDACTED_API_KEY]
+
+    Args:
+        logger: Logger instance
+        method_name: Log method name (info, error, etc.)
+        event_dict: Log event dictionary
+
+    Returns:
+        Event dictionary with credentials redacted
+
+    Example:
+        >>> event = {"msg": "Connected", "access_key": "AKIAIOSFODNN7EXAMPLE"}
+        >>> filtered = filter_credentials(None, "info", event)
+        >>> print(filtered["access_key"])
+        '[REDACTED_AWS_ACCESS_KEY]'
+    """
+    # Process all values in the event dict
+    for key, value in event_dict.items():
+        if isinstance(value, str):
+            # Redact AWS Access Keys
+            value = AWS_ACCESS_KEY_PATTERN.sub("[REDACTED_AWS_ACCESS_KEY]", value)
+
+            # Redact AWS Secret Keys (be careful with this - 40 chars is common)
+            # Only redact if key name suggests it's a secret
+            if any(secret_key in key.lower() for secret_key in ["secret", "password", "token"]):
+                if len(value) >= 40:
+                    value = "[REDACTED_AWS_SECRET_KEY]"
+
+            # Redact AWS Session Tokens
+            value = AWS_SESSION_TOKEN_PATTERN.sub("[REDACTED_AWS_SESSION_TOKEN]", value)
+
+            # Mask AWS Account IDs (show last 4 digits only)
+            if any(acct_key in key.lower() for acct_key in ["account_id", "account"]):
+                if value.isdigit() and len(value) == 12:
+                    value = f"****{value[-4:]}"
+
+            # Mask User IDs (show last 4 chars only)
+            if any(user_key in key.lower() for user_key in ["user_id", "userid"]):
+                if len(value) > 4:
+                    value = f"****{value[-4:]}"
+
+            # Redact passwords
+            value = PASSWORD_PATTERN.sub(r'\1[REDACTED_PASSWORD]', value)
+
+            # Redact API keys
+            value = API_KEY_PATTERN.sub(r'\1[REDACTED_API_KEY]', value)
+
+            # Update the value
+            event_dict[key] = value
+
+        elif isinstance(value, dict):
+            # Recursively filter nested dicts
+            event_dict[key] = _filter_dict(value)
+
+        elif isinstance(value, list):
+            # Filter lists
+            event_dict[key] = [_filter_value(item) for item in value]
+
+    return event_dict
+
+
+def _filter_dict(data: Dict[str, Any]) -> Dict[str, Any]:
+    """Recursively filter credentials from dictionary.
+
+    Args:
+        data: Dictionary to filter
+
+    Returns:
+        Filtered dictionary
+    """
+    filtered = {}
+    for key, value in data.items():
+        if isinstance(value, str):
+            # Check for sensitive keys
+            if any(sensitive in key.lower() for sensitive in [
+                "password", "secret", "token", "key", "credential"
+            ]):
+                filtered[key] = "[REDACTED]"
+            else:
+                # Still scan for patterns
+                filtered[key] = AWS_ACCESS_KEY_PATTERN.sub("[REDACTED_AWS_ACCESS_KEY]", value)
+                filtered[key] = AWS_SESSION_TOKEN_PATTERN.sub("[REDACTED_AWS_SESSION_TOKEN]", filtered[key])
+        elif isinstance(value, dict):
+            filtered[key] = _filter_dict(value)
+        elif isinstance(value, list):
+            filtered[key] = [_filter_value(item) for item in value]
+        else:
+            filtered[key] = value
+    return filtered
+
+
+def _filter_value(value: Any) -> Any:
+    """Filter a single value.
+
+    Args:
+        value: Value to filter
+
+    Returns:
+        Filtered value
+    """
+    if isinstance(value, str):
+        value = AWS_ACCESS_KEY_PATTERN.sub("[REDACTED_AWS_ACCESS_KEY]", value)
+        value = AWS_SESSION_TOKEN_PATTERN.sub("[REDACTED_AWS_SESSION_TOKEN]", value)
+        return value
+    elif isinstance(value, dict):
+        return _filter_dict(value)
+    elif isinstance(value, list):
+        return [_filter_value(item) for item in value]
+    else:
+        return value
+
+
+# ============================================================================
+# LOGGER SETUP
+# ============================================================================
+
+
+def setup_logging() -> None:
+    """Configure structured logging for the application.
+
+    Sets up:
+    - Console handler with colored output (if enabled)
+    - File handler with rotation (if enabled)
+    - Credential filtering processor
+    - JSON formatting for file logs
+    - Structured event dict format
+
+    Called automatically when getting a logger.
+
+    Example:
+        >>> setup_logging()
+        >>> logger = logging.getLogger("complio")
+        >>> logger.info("Application started")
+    """
+    settings = get_settings()
+
+    # Get log level
+    log_level = getattr(logging, settings.log_level)
+
+    # Configure stdlib logging
+    logging.basicConfig(
+        format="%(message)s",
+        level=log_level,
+    )
+
+    # Silence noisy libraries
+    logging.getLogger("boto3").setLevel(logging.WARNING)
+    logging.getLogger("botocore").setLevel(logging.WARNING)
+    logging.getLogger("urllib3").setLevel(logging.WARNING)
+
+    # Build processor chain
+    shared_processors = [
+        structlog.stdlib.add_logger_name,
+        add_log_level,
+        structlog.processors.TimeStamper(fmt="iso"),
+        filter_credentials,  # CRITICAL: Filter credentials before any output
+    ]
+
+    # Configure structlog
+    structlog.configure(
+        processors=shared_processors + [
+            filter_by_level,
+            structlog.stdlib.ProcessorFormatter.wrap_for_formatter,
+        ],
+        context_class=dict,
+        logger_factory=structlog.stdlib.LoggerFactory(),
+        cache_logger_on_first_use=True,
+    )
+
+    # Set up file logging if enabled
+    if settings.log_to_file:
+        # Ensure log directory exists
+        settings.log_dir.mkdir(parents=True, exist_ok=True)
+        settings.log_dir.chmod(0o700)
+
+        log_file = settings.log_dir / "complio.log"
+
+        # Create rotating file handler
+        file_handler = RotatingFileHandler(
+            filename=str(log_file),
+            maxBytes=settings.log_max_bytes,
+            backupCount=settings.log_backup_count,
+            encoding="utf-8",
+        )
+        file_handler.setLevel(log_level)
+
+        # Use JSON formatter for file logs
+        file_handler.setFormatter(
+            structlog.stdlib.ProcessorFormatter(
+                processor=JSONRenderer(),
+                foreign_pre_chain=shared_processors,
+            )
+        )
+
+        # Add to root logger
+        root_logger = logging.getLogger()
+        root_logger.addHandler(file_handler)
+
+        # Set file permissions to 600
+        log_file.chmod(0o600)
+
+
+# Track if logging has been set up
+_logging_configured = False
+
+
+def get_logger(name: str) -> structlog.stdlib.BoundLogger:
+    """Get a structured logger instance.
+
+    Args:
+        name: Logger name (usually __name__)
+
+    Returns:
+        Structured logger with credential filtering
+
+    Example:
+        >>> logger = get_logger(__name__)
+        >>> logger.info("user_login", user_id=123, ip="10.0.0.1")
+        >>> logger.error("auth_failed", error="Invalid password")
+    """
+    global _logging_configured
+    if not _logging_configured:
+        setup_logging()
+        _logging_configured = True
+
+    return structlog.get_logger(name)
+
+
+# ============================================================================
+# CONVENIENCE FUNCTIONS
+# ============================================================================
+
+
+def log_function_call(logger: structlog.stdlib.BoundLogger, function_name: str, **kwargs: Any) -> None:
+    """Log a function call with parameters.
+
+    Automatically filters sensitive parameters.
+
+    Args:
+        logger: Logger instance
+        function_name: Name of function being called
+        **kwargs: Function parameters
+
+    Example:
+        >>> logger = get_logger(__name__)
+        >>> log_function_call(logger, "connect_to_aws", region="us-east-1", profile="prod")
+    """
+    logger.debug(
+        "function_call",
+        function=function_name,
+        **kwargs
+    )
+
+
+def log_aws_api_call(
+    logger: structlog.stdlib.BoundLogger,
+    service: str,
+    operation: str,
+    region: str,
+    **kwargs: Any
+) -> None:
+    """Log an AWS API call.
+
+    Args:
+        logger: Logger instance
+        service: AWS service name (e.g., 's3', 'ec2')
+        operation: API operation (e.g., 'list_buckets')
+        region: AWS region
+        **kwargs: Additional context
+
+    Example:
+        >>> logger = get_logger(__name__)
+        >>> log_aws_api_call(logger, "s3", "list_buckets", "us-east-1")
+    """
+    logger.debug(
+        "aws_api_call",
+        service=service,
+        operation=operation,
+        region=region,
+        **kwargs
+    )
+
+
+def log_test_result(
+    logger: structlog.stdlib.BoundLogger,
+    test_name: str,
+    passed: bool,
+    **kwargs: Any
+) -> None:
+    """Log a compliance test result.
+
+    Args:
+        logger: Logger instance
+        test_name: Name of compliance test
+        passed: Whether test passed
+        **kwargs: Additional context (findings, resources, etc.)
+
+    Example:
+        >>> logger = get_logger(__name__)
+        >>> log_test_result(logger, "s3_encryption", True, bucket_count=5)
+    """
+    log_level = "info" if passed else "warning"
+    getattr(logger, log_level)(
+        "compliance_test_result",
+        test_name=test_name,
+        passed=passed,
+        **kwargs
+    )