souleyez 2.27.0__py3-none-any.whl → 2.32.0__py3-none-any.whl

souleyez/storage/database.py

@@ -31,7 +31,7 @@ class Database:
                 os.makedirs(db_dir, exist_ok=True)
 
             conn = sqlite3.connect(self.db_path, timeout=30.0)
-            
+
             # Set secure permissions (owner read/write only)
             os.chmod(self.db_path, 0o600)
             conn.row_factory = sqlite3.Row
@@ -46,7 +46,44 @@ class Database:
                 "foreign_keys": True
             })
 
+            # Check if this is an existing database (has tables)
+            cursor = conn.execute(
+                "SELECT name FROM sqlite_master WHERE type='table' AND name='engagements'"
+            )
+            is_existing_db = cursor.fetchone() is not None
+            conn.close()
+
+            # For EXISTING databases: Run migrations FIRST
+            # This ensures new columns exist before schema.sql tries to create indexes on them
+            if is_existing_db:
+                try:
+                    from .migrations.migration_manager import MigrationManager
+                    manager = MigrationManager(self.db_path)
+                    pending = manager.get_pending_migrations()
+                    if pending:
+                        logger.info("Running pending migrations for existing database", extra={
+                            "pending_count": len(pending)
+                        })
+                        manager.migrate()
+                        logger.info("Migrations completed successfully", extra={
+                            "count": len(pending)
+                        })
+                except Exception as migration_error:
+                    logger.error("Failed to run migrations on existing database", extra={
+                        "error": str(migration_error),
+                        "error_type": type(migration_error).__name__,
+                        "traceback": traceback.format_exc()
+                    })
+                    raise  # Don't continue if migrations fail for existing DB
+
+            # Reconnect for schema loading
+            conn = sqlite3.connect(self.db_path, timeout=30.0)
+            conn.row_factory = sqlite3.Row
+            conn.execute("PRAGMA foreign_keys = ON")
+
             # Load and execute schema from the same directory as this file
+            # For fresh DBs: Creates all tables with current schema
+            # For existing DBs: CREATE TABLE IF NOT EXISTS is no-op, but ensures new tables/indexes
             schema_path = Path(__file__).parent / "schema.sql"
 
             if schema_path.exists():
@@ -228,26 +265,28 @@ class Database:
 
             conn.commit()
             conn.close()
-            
-            # Run pending migrations after schema.sql loads
-            try:
-                from .migrations.migration_manager import MigrationManager
-                manager = MigrationManager(self.db_path)
-                pending = manager.get_pending_migrations()
-                if pending:
-                    logger.info("Running pending migrations for fresh database", extra={
-                        "pending_count": len(pending)
-                    })
-                    manager.migrate()
-                    logger.info("Migrations completed successfully", extra={
-                        "count": len(pending)
+
+            # For FRESH databases: Run migrations after schema.sql loads
+            # (Existing DBs already had migrations run before schema.sql)
+            if not is_existing_db:
+                try:
+                    from .migrations.migration_manager import MigrationManager
+                    manager = MigrationManager(self.db_path)
+                    pending = manager.get_pending_migrations()
+                    if pending:
+                        logger.info("Running pending migrations for fresh database", extra={
+                            "pending_count": len(pending)
+                        })
+                        manager.migrate()
+                        logger.info("Migrations completed successfully", extra={
+                            "count": len(pending)
+                        })
+                except Exception as migration_error:
+                    logger.error("Failed to run migrations", extra={
+                        "error": str(migration_error),
+                        "error_type": type(migration_error).__name__,
+                        "traceback": traceback.format_exc()
                     })
-            except Exception as migration_error:
-                logger.error("Failed to run migrations", extra={
-                    "error": str(migration_error),
-                    "error_type": type(migration_error).__name__,
-                    "traceback": traceback.format_exc()
-                })
 
         except Exception as e:
             logger.error("Database initialization failed", extra={

souleyez/storage/hosts.py

@@ -28,9 +28,12 @@ class HostManager:
         if not ip:
             raise ValueError("Host must have an IP address")
 
+        # Determine scope status for this host
+        scope_status = self._determine_scope_status(engagement_id, ip)
+
         # Check if host already exists
         existing = self.db.execute_one(
-            "SELECT id FROM hosts WHERE engagement_id = ? AND ip_address = ?",
+            "SELECT id, scope_status FROM hosts WHERE engagement_id = ? AND ip_address = ?",
             (engagement_id, ip)
         )
 
@@ -43,6 +46,10 @@ class HostManager:
             # Always update status
             update_data['status'] = host_data.get('status', 'up')
 
+            # Update scope_status if it was unknown and we now have a determination
+            if existing.get('scope_status') == 'unknown' and scope_status != 'unknown':
+                update_data['scope_status'] = scope_status
+
             # Only update these fields if they have values
             if host_data.get('hostname'):
                 update_data['hostname'] = host_data['hostname']
@@ -71,11 +78,89 @@ class HostManager:
                 'os_name': host_data.get('os'),
                 'mac_address': host_data.get('mac_address'),
                 'os_accuracy': host_data.get('os_accuracy'),
-                'status': host_data.get('status', 'up')
+                'status': host_data.get('status', 'up'),
+                'scope_status': scope_status
             })
 
             return host_id
 
+    def _determine_scope_status(self, engagement_id: int, ip: str) -> str:
+        """
+        Determine scope status for a host based on engagement scope.
+
+        Args:
+            engagement_id: Engagement ID
+            ip: IP address to check
+
+        Returns:
+            'in_scope', 'out_of_scope', or 'unknown'
+        """
+        try:
+            from souleyez.security.scope_validator import ScopeValidator
+            validator = ScopeValidator(engagement_id)
+            if validator.has_scope_defined():
+                result = validator.validate_ip(ip)
+                return 'in_scope' if result.is_in_scope else 'out_of_scope'
+            return 'unknown'  # No scope defined
+        except Exception as e:
+            logger.warning(f"Failed to determine scope status for {ip}: {e}")
+            return 'unknown'
+
+    def update_scope_status(self, host_id: int, scope_status: str) -> bool:
+        """
+        Update scope status for a host.
+
+        Args:
+            host_id: Host ID
+            scope_status: 'in_scope', 'out_of_scope', or 'unknown'
+
+        Returns:
+            True if successful
+        """
+        valid_statuses = ['in_scope', 'out_of_scope', 'unknown']
+        if scope_status not in valid_statuses:
+            raise ValueError(f"Invalid scope_status: {scope_status}. Must be one of: {valid_statuses}")
+
+        try:
+            self.db.execute(
+                "UPDATE hosts SET scope_status = ? WHERE id = ?",
+                (scope_status, host_id)
+            )
+            return True
+        except Exception:
+            return False
+
+    def revalidate_scope_status(self, engagement_id: int) -> Dict[str, int]:
+        """
+        Revalidate scope status for all hosts in an engagement.
+
+        Call this after scope entries are added/modified to update all hosts.
+
+        Args:
+            engagement_id: Engagement ID
+
+        Returns:
+            {'updated': N, 'in_scope': X, 'out_of_scope': Y}
+        """
+        hosts = self.list_hosts(engagement_id)
+        updated = 0
+        in_scope = 0
+        out_of_scope = 0
+
+        for host in hosts:
+            ip = host.get('ip') or host.get('ip_address')
+            new_status = self._determine_scope_status(engagement_id, ip)
+            if new_status != host.get('scope_status'):
+                self.update_scope_status(host['id'], new_status)
+                updated += 1
+
+            if new_status == 'in_scope':
+                in_scope += 1
+            elif new_status == 'out_of_scope':
+                out_of_scope += 1
+
+        return {'updated': updated, 'in_scope': in_scope, 'out_of_scope': out_of_scope}
+
     def add_service(self, host_id: int, service_data: Dict[str, Any]) -> int:
         """
         Add or update a service for a host.

souleyez/storage/migrations/_026_add_engagement_scope.py

@@ -0,0 +1,87 @@
+"""
+Migration 026: Add Engagement Scope Validation
+
+Tables created:
+- engagement_scope: Structured scope definitions (CIDR, domains, URLs)
+- scope_validation_log: Audit trail of scope validation decisions
+
+Columns added:
+- engagements.scope_enforcement: Enforcement mode (off, warn, block)
+- hosts.scope_status: Host scope status (in_scope, out_of_scope, unknown)
+"""
+import os
+
+
+def upgrade(conn):
+    """Add scope validation tables and columns."""
+
+    # Engagement scope definitions table
+    conn.execute("""
+        CREATE TABLE IF NOT EXISTS engagement_scope (
+            id INTEGER PRIMARY KEY AUTOINCREMENT,
+            engagement_id INTEGER NOT NULL,
+            scope_type TEXT NOT NULL,
+            value TEXT NOT NULL,
+            is_excluded BOOLEAN DEFAULT 0,
+            description TEXT,
+            added_by TEXT,
+            created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+            FOREIGN KEY (engagement_id) REFERENCES engagements(id) ON DELETE CASCADE,
+            UNIQUE(engagement_id, scope_type, value)
+        )
+    """)
+
+    # Scope validation audit log
+    conn.execute("""
+        CREATE TABLE IF NOT EXISTS scope_validation_log (
+            id INTEGER PRIMARY KEY AUTOINCREMENT,
+            engagement_id INTEGER NOT NULL,
+            job_id INTEGER,
+            target TEXT NOT NULL,
+            validation_result TEXT NOT NULL,
+            action_taken TEXT NOT NULL,
+            matched_scope_id INTEGER,
+            user_id TEXT,
+            created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+            FOREIGN KEY (engagement_id) REFERENCES engagements(id) ON DELETE CASCADE
+        )
+    """)
+
+    # Add scope_enforcement column to engagements
+    try:
+        conn.execute("ALTER TABLE engagements ADD COLUMN scope_enforcement TEXT DEFAULT 'off'")
+    except Exception:
+        pass  # Column may already exist
+
+    # Add scope_status column to hosts
+    try:
+        conn.execute("ALTER TABLE hosts ADD COLUMN scope_status TEXT DEFAULT 'unknown'")
+    except Exception:
+        pass  # Column may already exist
+
+    # Indexes for performance
+    conn.execute("CREATE INDEX IF NOT EXISTS idx_scope_engagement ON engagement_scope(engagement_id)")
+    conn.execute("CREATE INDEX IF NOT EXISTS idx_scope_type ON engagement_scope(scope_type)")
+    conn.execute("CREATE INDEX IF NOT EXISTS idx_scope_log_engagement ON scope_validation_log(engagement_id)")
+    conn.execute("CREATE INDEX IF NOT EXISTS idx_scope_log_result ON scope_validation_log(validation_result)")
+    conn.execute("CREATE INDEX IF NOT EXISTS idx_scope_log_timestamp ON scope_validation_log(created_at DESC)")
+    conn.execute("CREATE INDEX IF NOT EXISTS idx_hosts_scope_status ON hosts(scope_status)")
+
+    conn.commit()
+
+    if not os.environ.get('SOULEYEZ_MIGRATION_SILENT'):
+        print("Migration 026: Engagement scope validation tables created")
+
+
+def downgrade(conn):
+    """Remove scope validation tables."""
+    conn.execute("DROP TABLE IF EXISTS scope_validation_log")
+    conn.execute("DROP TABLE IF EXISTS engagement_scope")
+    conn.execute("DROP INDEX IF EXISTS idx_scope_engagement")
+    conn.execute("DROP INDEX IF EXISTS idx_scope_type")
+    conn.execute("DROP INDEX IF EXISTS idx_scope_log_engagement")
+    conn.execute("DROP INDEX IF EXISTS idx_scope_log_result")
+    conn.execute("DROP INDEX IF EXISTS idx_scope_log_timestamp")
+    conn.execute("DROP INDEX IF EXISTS idx_hosts_scope_status")
+    # Note: Cannot easily drop columns in SQLite, they remain but unused
+    print("Migration 026: Engagement scope validation tables dropped")

souleyez/storage/migrations/_027_multi_siem_persistence.py

@@ -0,0 +1,119 @@
+"""
+Migration 027: Multi-SIEM Persistence
+
+Changes wazuh_config table to support multiple SIEM configs per engagement.
+- Removes UNIQUE constraint on engagement_id
+- Adds UNIQUE constraint on (engagement_id, siem_type)
+- Allows each engagement to have separate configs for Wazuh, Splunk, etc.
+"""
+import os
+
+
+def upgrade(conn):
+    """Migrate to multi-SIEM persistence."""
+    cursor = conn.cursor()
+
+    # Check if siem_type column exists (from migration 025)
+    cursor.execute("PRAGMA table_info(wazuh_config)")
+    columns = [col[1] for col in cursor.fetchall()]
+
+    if 'siem_type' not in columns:
+        cursor.execute("ALTER TABLE wazuh_config ADD COLUMN siem_type TEXT DEFAULT 'wazuh'")
+    if 'config_json' not in columns:
+        cursor.execute("ALTER TABLE wazuh_config ADD COLUMN config_json TEXT")
+
+    # Create new table with correct constraint
+    cursor.execute("""
+        CREATE TABLE IF NOT EXISTS siem_config_new (
+            id INTEGER PRIMARY KEY AUTOINCREMENT,
+            engagement_id INTEGER NOT NULL,
+            siem_type TEXT NOT NULL DEFAULT 'wazuh',
+            api_url TEXT,
+            api_user TEXT,
+            api_password TEXT,
+            indexer_url TEXT,
+            indexer_user TEXT DEFAULT 'admin',
+            indexer_password TEXT,
+            verify_ssl BOOLEAN DEFAULT 0,
+            enabled BOOLEAN DEFAULT 1,
+            config_json TEXT,
+            created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+            updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+            FOREIGN KEY (engagement_id) REFERENCES engagements(id) ON DELETE CASCADE,
+            UNIQUE(engagement_id, siem_type)
+        )
+    """)
+
+    # Copy existing data
+    cursor.execute("""
+        INSERT OR IGNORE INTO siem_config_new (
+            id, engagement_id, siem_type, api_url, api_user, api_password,
+            indexer_url, indexer_user, indexer_password, verify_ssl, enabled,
+            config_json, created_at, updated_at
+        )
+        SELECT
+            id, engagement_id, COALESCE(siem_type, 'wazuh'), api_url, api_user, api_password,
+            indexer_url, indexer_user, indexer_password, verify_ssl, enabled,
+            config_json, created_at, updated_at
+        FROM wazuh_config
+    """)
+
+    # Drop old table and rename new one
+    cursor.execute("DROP TABLE wazuh_config")
+    cursor.execute("ALTER TABLE siem_config_new RENAME TO wazuh_config")
+
+    # Recreate index
+    cursor.execute("CREATE INDEX IF NOT EXISTS idx_wazuh_config_engagement ON wazuh_config(engagement_id)")
+    cursor.execute("CREATE INDEX IF NOT EXISTS idx_wazuh_config_siem_type ON wazuh_config(siem_type)")
+
+    conn.commit()
+
+    if not os.environ.get('SOULEYEZ_MIGRATION_SILENT'):
+        print("Migration 027: Multi-SIEM persistence enabled")
+
+
+def downgrade(conn):
+    """Revert to single SIEM per engagement (lossy - keeps only first config per engagement)."""
+    cursor = conn.cursor()
+
+    cursor.execute("""
+        CREATE TABLE IF NOT EXISTS wazuh_config_old (
+            id INTEGER PRIMARY KEY AUTOINCREMENT,
+            engagement_id INTEGER NOT NULL UNIQUE,
+            api_url TEXT NOT NULL,
+            api_user TEXT NOT NULL,
+            api_password TEXT,
+            indexer_url TEXT,
+            indexer_user TEXT DEFAULT 'admin',
+            indexer_password TEXT,
+            verify_ssl BOOLEAN DEFAULT 0,
+            enabled BOOLEAN DEFAULT 1,
+            siem_type TEXT DEFAULT 'wazuh',
+            config_json TEXT,
+            created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+            updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+            FOREIGN KEY (engagement_id) REFERENCES engagements(id) ON DELETE CASCADE
+        )
+    """)
+
+    # Copy only first config per engagement
+    cursor.execute("""
+        INSERT OR IGNORE INTO wazuh_config_old (
+            engagement_id, api_url, api_user, api_password,
+            indexer_url, indexer_user, indexer_password, verify_ssl, enabled,
+            siem_type, config_json, created_at, updated_at
+        )
+        SELECT
+            engagement_id, api_url, api_user, api_password,
+            indexer_url, indexer_user, indexer_password, verify_ssl, enabled,
+            siem_type, config_json, created_at, updated_at
+        FROM wazuh_config
+        GROUP BY engagement_id
+    """)
+
+    cursor.execute("DROP TABLE wazuh_config")
+    cursor.execute("ALTER TABLE wazuh_config_old RENAME TO wazuh_config")
+    cursor.execute("CREATE INDEX IF NOT EXISTS idx_wazuh_config_engagement ON wazuh_config(engagement_id)")
+
+    conn.commit()
+    print("Migration 027: Reverted to single SIEM per engagement")

souleyez/storage/migrations/__init__.py

@@ -29,6 +29,9 @@ from . import (
     _022_wazuh_indexer_columns,
     _023_fix_detection_results_fk,
     _024_wazuh_vulnerabilities,
+    _025_multi_siem_support,
+    _026_add_engagement_scope,
+    _027_multi_siem_persistence,
 )
 
 # Migration registry - maps version to module
@@ -56,6 +59,9 @@ MIGRATIONS_REGISTRY = {
     '022': _022_wazuh_indexer_columns,
     '023': _023_fix_detection_results_fk,
     '024': _024_wazuh_vulnerabilities,
+    '025': _025_multi_siem_support,
+    '026': _026_add_engagement_scope,
+    '027': _027_multi_siem_persistence,
 }
 
 

souleyez/storage/schema.sql

@@ -6,6 +6,7 @@ CREATE TABLE IF NOT EXISTS engagements (
     owner_id INTEGER,
     estimated_hours FLOAT DEFAULT 0,
     actual_hours FLOAT DEFAULT 0,
+    scope_enforcement TEXT DEFAULT 'off',
     created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
     updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
 );
@@ -21,6 +22,7 @@ CREATE TABLE IF NOT EXISTS hosts (
     mac_address TEXT,
     status TEXT DEFAULT 'up',
     access_level TEXT DEFAULT 'none',
+    scope_status TEXT DEFAULT 'unknown',
     notes TEXT,
     tags TEXT,
     created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
@@ -502,18 +504,21 @@ CREATE INDEX IF NOT EXISTS idx_msf_sessions_active ON msf_sessions(is_active);
 -- Wazuh SIEM Integration (Detection Validation)
 CREATE TABLE IF NOT EXISTS wazuh_config (
     id INTEGER PRIMARY KEY AUTOINCREMENT,
-    engagement_id INTEGER NOT NULL UNIQUE,
-    api_url TEXT NOT NULL,
-    api_user TEXT NOT NULL,
+    engagement_id INTEGER NOT NULL,
+    siem_type TEXT NOT NULL DEFAULT 'wazuh',
+    api_url TEXT,
+    api_user TEXT,
     api_password TEXT,
     indexer_url TEXT,
     indexer_user TEXT DEFAULT 'admin',
     indexer_password TEXT,
     verify_ssl BOOLEAN DEFAULT 0,
     enabled BOOLEAN DEFAULT 1,
+    config_json TEXT,
     created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
     updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
-    FOREIGN KEY (engagement_id) REFERENCES engagements(id) ON DELETE CASCADE
+    FOREIGN KEY (engagement_id) REFERENCES engagements(id) ON DELETE CASCADE,
+    UNIQUE(engagement_id, siem_type)
 );
 
 -- Detection validation results per job
@@ -537,6 +542,41 @@ CREATE TABLE IF NOT EXISTS detection_results (
 );
 
 CREATE INDEX IF NOT EXISTS idx_wazuh_config_engagement ON wazuh_config(engagement_id);
+CREATE INDEX IF NOT EXISTS idx_wazuh_config_siem_type ON wazuh_config(siem_type);
 CREATE INDEX IF NOT EXISTS idx_detection_results_job ON detection_results(job_id);
 CREATE INDEX IF NOT EXISTS idx_detection_results_engagement ON detection_results(engagement_id);
 CREATE INDEX IF NOT EXISTS idx_detection_results_status ON detection_results(detection_status);
+
+-- Engagement Scope Validation (from migration 026)
+CREATE TABLE IF NOT EXISTS engagement_scope (
+    id INTEGER PRIMARY KEY AUTOINCREMENT,
+    engagement_id INTEGER NOT NULL,
+    scope_type TEXT NOT NULL,
+    value TEXT NOT NULL,
+    is_excluded BOOLEAN DEFAULT 0,
+    description TEXT,
+    added_by TEXT,
+    created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+    FOREIGN KEY (engagement_id) REFERENCES engagements(id) ON DELETE CASCADE,
+    UNIQUE(engagement_id, scope_type, value)
+);
+
+CREATE TABLE IF NOT EXISTS scope_validation_log (
+    id INTEGER PRIMARY KEY AUTOINCREMENT,
+    engagement_id INTEGER NOT NULL,
+    job_id INTEGER,
+    target TEXT NOT NULL,
+    validation_result TEXT NOT NULL,
+    action_taken TEXT NOT NULL,
+    matched_scope_id INTEGER,
+    user_id TEXT,
+    created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+    FOREIGN KEY (engagement_id) REFERENCES engagements(id) ON DELETE CASCADE
+);
+
+CREATE INDEX IF NOT EXISTS idx_scope_engagement ON engagement_scope(engagement_id);
+CREATE INDEX IF NOT EXISTS idx_scope_type ON engagement_scope(scope_type);
+CREATE INDEX IF NOT EXISTS idx_scope_log_engagement ON scope_validation_log(engagement_id);
+CREATE INDEX IF NOT EXISTS idx_scope_log_result ON scope_validation_log(validation_result);
+CREATE INDEX IF NOT EXISTS idx_scope_log_timestamp ON scope_validation_log(created_at DESC);
+CREATE INDEX IF NOT EXISTS idx_hosts_scope_status ON hosts(scope_status);