voidaccess 1.3.0__py3-none-any.whl

db/session.py

@@ -0,0 +1,270 @@
+"""
+SQLAlchemy engine and session factory.
+
+Usage (application code)
+------------------------
+    from db.session import get_session
+
+    with get_session() as session:
+        session.add(some_object)
+        # commits on exit, rolls back on exception
+
+For async code, prefer get_async_session() with async with:
+------------------------
+    from db.session import get_async_session
+
+    async with get_async_session() as session:
+        await session.add(some_object)
+        await session.commit()
+
+For short-lived async operations, use async_session_scope():
+------------------------
+    from db.session import async_session_scope
+
+    async with async_session_scope() as session:
+        # session is auto-committed on exit, rolled back on exception
+        await session.execute(...)
+
+Usage (testing — pass an explicit URL to avoid needing DATABASE_URL in env)
+---------------------------------------------------------------------------
+    from db.session import get_engine, get_session_factory
+    from db.models import Base
+
+    engine = get_engine("sqlite:///:memory:")
+    Base.metadata.create_all(engine)
+    Session = get_session_factory("sqlite:///:memory:")
+"""
+
+from __future__ import annotations
+
+from functools import lru_cache
+
+from contextlib import asynccontextmanager, contextmanager
+from typing import AsyncGenerator, Generator, Optional
+
+from sqlalchemy import create_engine, Engine
+from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine
+from sqlalchemy.orm import sessionmaker, Session
+
+import config
+
+_async_engine_cache: dict[str, "AsyncEngine"] = {}
+
+
+@lru_cache(maxsize=8)
+def _get_engine_cached(target_url: str) -> Engine:
+    is_sqlite = target_url.startswith("sqlite")
+
+    if is_sqlite:
+        engine = create_engine(
+            target_url,
+            pool_pre_ping=True,
+            connect_args={"check_same_thread": False},
+        )
+    else:
+        engine = create_engine(
+            target_url,
+            pool_pre_ping=True,
+            pool_size=20,
+            max_overflow=40,
+            pool_timeout=30,
+            pool_recycle=1800,
+        )
+
+    return engine
+
+
+def get_engine(url: Optional[str] = None) -> Engine:
+    """
+    Return a SQLAlchemy Engine for *url* (defaults to DATABASE_URL env var).
+
+    Uses lru_cache with maxsize=8 to bound the cache and prevent unbounded
+    growth during test suites. Least-recently-used engines are evicted
+    automatically when the limit is reached.
+
+    PostgreSQL gets a connection pool tuned for the scraping workload.
+    SQLite skips pool parameters that only apply to QueuePool.
+    """
+    target_url = url or config.DATABASE_URL
+    if not target_url:
+        raise RuntimeError(
+            "DATABASE_URL is not configured.\n"
+            "Add it to your .env file, e.g.:\n"
+            "  DATABASE_URL=postgresql://voidaccess:voidaccess@localhost:5433/voidaccess"
+        )
+
+    return _get_engine_cached(target_url)
+
+
+def release_engine(url: Optional[str] = None) -> None:
+    """
+    Explicitly release and remove an engine from the cache.
+
+    Calls engine.dispose() to release connection pool resources and file handles,
+    then clears the cache. Use this in test teardown to prevent leaks.
+    """
+    target_url = url or config.DATABASE_URL
+    if target_url:
+        try:
+            engine = get_engine(target_url)
+            engine.dispose()
+        except Exception:
+            pass
+        _get_engine_cached.cache_clear()
+
+
+def get_async_engine(url: Optional[str] = None) -> "AsyncEngine":
+    """
+    Return an async SQLAlchemy AsyncEngine for *url*.
+
+    Converts postgresql:// to postgresql+asyncpg:// and sqlite:// to sqlite+aiosqlite://.
+    """
+    from sqlalchemy.ext.asyncio import AsyncEngine
+
+    target_url = url or config.DATABASE_URL
+    if not target_url:
+        raise RuntimeError(
+            "DATABASE_URL is not configured.\n"
+            "Add it to your .env file, e.g.:\n"
+            "  DATABASE_URL=postgresql://voidaccess:voidaccess@localhost:5433/voidaccess"
+        )
+
+    if target_url in _async_engine_cache:
+        return _async_engine_cache[target_url]
+
+    if target_url.startswith("postgresql://"):
+        async_url = target_url.replace("postgresql://", "postgresql+asyncpg://", 1)
+    elif target_url.startswith("sqlite://"):
+        async_url = target_url.replace("sqlite://", "sqlite+aiosqlite://", 1)
+    else:
+        async_url = target_url
+
+    is_sqlite = "sqlite" in async_url
+
+    if is_sqlite:
+        engine = create_async_engine(
+            async_url,
+            pool_pre_ping=True,
+            connect_args={"check_same_thread": False},
+        )
+    else:
+        engine = create_async_engine(
+            async_url,
+            pool_pre_ping=True,
+            pool_size=20,
+            max_overflow=40,
+            pool_timeout=30,
+            pool_recycle=1800,
+        )
+
+    _async_engine_cache[target_url] = engine
+    return engine
+
+
+def release_async_engine(url: Optional[str] = None) -> None:
+    """
+    Explicitly release and remove an async engine from the cache.
+
+    Calls engine.dispose() to release connection pool resources and file handles.
+    """
+    target_url = url or config.DATABASE_URL
+    if target_url in _async_engine_cache:
+        _async_engine_cache[target_url].dispose()
+        del _async_engine_cache[target_url]
+
+
+def get_session_factory(url: Optional[str] = None) -> sessionmaker:
+    """Return a sessionmaker bound to an engine for *url*."""
+    engine = get_engine(url)
+    return sessionmaker(bind=engine, autoflush=False, autocommit=False)
+
+
+def get_async_session_factory(url: Optional[str] = None) -> async_sessionmaker:
+    """Return an async_sessionmaker bound to an async engine for *url*."""
+    engine = get_async_engine(url)
+    return async_sessionmaker(bind=engine, autoflush=False, expire_on_commit=False)
+
+
+@contextmanager
+def get_session(url: Optional[str] = None) -> Generator[Session, None, None]:
+    """
+    Context manager that yields a sync Session, commits on clean exit,
+    rolls back on any exception, and always closes.
+
+    Example::
+
+        with get_session() as session:
+            session.add(entity)
+        # committed here
+    """
+    factory = get_session_factory(url)
+    session: Session = factory()
+    try:
+        yield session
+        session.commit()
+    except Exception:
+        session.rollback()
+        raise
+    finally:
+        session.close()
+
+
+def get_db(url: Optional[str] = None) -> Generator[Session, None, None]:
+    """
+    FastAPI dependency that yields a database session.
+    The session is closed automatically after the request.
+    Usage: db: Session = Depends(get_db)
+    """
+    factory = get_session_factory(url)
+    db = factory()
+    try:
+        yield db
+    finally:
+        db.close()
+
+
+@asynccontextmanager
+async def get_async_session(url: Optional[str] = None) -> AsyncGenerator[AsyncSession, None]:
+    """
+    Async generator that yields an AsyncSession.
+
+    Usage::
+
+        async with get_async_session() as session:
+            await session.add(entity)
+            await session.commit()
+
+    The session is automatically closed on exit.
+    """
+    factory = get_async_session_factory(url)
+    async with factory() as session:
+        yield session
+
+
+@asynccontextmanager
+async def async_session_scope(
+    url: Optional[str] = None,
+) -> AsyncGenerator[AsyncSession, None]:
+    """
+    Async context manager for short-lived sessions.
+
+    Automatically commits on clean exit, rolls back on exception,
+    and always closes the session. Use this for targeted DB operations.
+
+    Example::
+
+        async with async_session_scope() as session:
+            result = await session.execute(select(Investigation))
+            await session.commit()
+
+    This is the preferred pattern for the investigation pipeline —
+    each step gets its own session that commits and closes immediately.
+    """
+    factory = get_async_session_factory(url)
+    async with factory() as session:
+        try:
+            yield session
+            await session.commit()
+        except Exception:
+            await session.rollback()
+            raise

export/__init__.py

@@ -0,0 +1,34 @@
+"""
+export — Phase 5 intelligence export module.
+
+Re-exports the public API from stix, misp, and sigma sub-modules.
+"""
+
+from export.stix import (
+    bundle_to_dict,
+    bundle_to_json,
+    investigation_to_stix_bundle,
+)
+from export.misp import (
+    investigation_to_misp_event,
+    misp_event_to_json,
+)
+from export.sigma import (
+    entities_to_sigma_rules,
+    export_sigma_rules,
+    sigma_rule_to_yaml,
+)
+
+__all__ = [
+    # stix
+    "investigation_to_stix_bundle",
+    "bundle_to_json",
+    "bundle_to_dict",
+    # misp
+    "investigation_to_misp_event",
+    "misp_event_to_json",
+    # sigma
+    "entities_to_sigma_rules",
+    "sigma_rule_to_yaml",
+    "export_sigma_rules",
+]

export/misp.py

@@ -0,0 +1,257 @@
+"""
+export/misp.py — Generates MISP event JSON from a VoidAccess investigation.
+
+MISP format is constructed directly as a dict — no MISP library required.
+The format follows the MISP standard event structure as documented at
+https://www.misp-standard.org/rfc/misp-core-format.html
+
+Public interface
+----------------
+investigation_to_misp_event(investigation_id) → dict
+misp_event_to_json(event)                     → str
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+import os
+from datetime import datetime, timezone
+from typing import Any, Optional
+
+logger = logging.getLogger(__name__)
+
+# ---------------------------------------------------------------------------
+# Entity type → MISP attribute mapping
+# ---------------------------------------------------------------------------
+
+_MISP_ATTR_MAP: dict[str, dict] = {
+    "BITCOIN_ADDRESS": {
+        "type": "btc",
+        "category": "Financial fraud",
+        "to_ids": True,
+    },
+    "ETHEREUM_ADDRESS": {
+        "type": "other",
+        "category": "Financial fraud",
+        "to_ids": True,
+    },
+    "MONERO_ADDRESS": {
+        "type": "other",
+        "category": "Financial fraud",
+        "to_ids": True,
+    },
+    "EMAIL_ADDRESS": {
+        "type": "email-src",
+        "category": "Network activity",
+        "to_ids": False,
+    },
+    "ONION_URL": {
+        "type": "url",
+        "category": "Network activity",
+        "to_ids": True,
+    },
+    "IP_ADDRESS": {
+        "type": "ip-dst",
+        "category": "Network activity",
+        "to_ids": True,
+    },
+    "CVE_NUMBER": {
+        "type": "vulnerability",
+        "category": "External analysis",
+        "to_ids": False,
+    },
+    "MALWARE_FAMILY": {
+        "type": "malware-type",
+        "category": "Antivirus detection",
+        "to_ids": False,
+    },
+    "RANSOMWARE_GROUP": {
+        "type": "malware-type",
+        "category": "Antivirus detection",
+        "to_ids": False,
+    },
+    "THREAT_ACTOR_HANDLE": {
+        "type": "threat-actor",
+        "category": "Attribution",
+        "to_ids": False,
+    },
+}
+
+
+# ---------------------------------------------------------------------------
+# Public interface
+# ---------------------------------------------------------------------------
+
+
+def investigation_to_misp_event(
+    investigation_id: Any,
+    entity_ids: Optional[list[str]] = None,
+) -> dict:
+    """
+    Build a MISP-compatible event dict for the given investigation.
+
+    Returns a valid (but empty-attribute) event if the investigation is not found.
+    Never raises.
+    """
+    investigation, entities = _load_investigation_and_entities(
+        investigation_id, entity_ids=entity_ids
+    )
+
+    if investigation is None:
+        return {
+            "Event": {
+                "info": "Not found",
+                "Attribute": [],
+            }
+        }
+
+    date_str = _utc_date_str(investigation.created_at)
+    query = getattr(investigation, "query", "") or ""
+
+    attributes: list[dict] = []
+    for entity in entities:
+        mapping = _MISP_ATTR_MAP.get(entity.entity_type)
+        if mapping is None:
+            continue
+        attr = {
+            "type": mapping["type"],
+            "category": mapping["category"],
+            "value": entity.value,
+            "comment": f"Source: {entity.source_url}" if entity.source_url else "Source: unknown",
+            "to_ids": mapping["to_ids"],
+        }
+        attributes.append(attr)
+
+    return {
+        "Event": {
+            "info": f"VoidAccess Investigation: {query}",
+            "date": date_str,
+            "threat_level_id": "2",   # Medium
+            "analysis": "2",           # Completed
+            "distribution": "0",       # Your organisation only
+            "Attribute": attributes,
+        }
+    }
+
+
+def misp_event_to_json(event: dict) -> str:
+    """
+    Return JSON string of a MISP event dict (pretty-printed, 2-space indent).
+    """
+    try:
+        return json.dumps(event, indent=2, default=str)
+    except Exception as exc:
+        logger.warning("misp_event_to_json failed: %s", exc)
+        return json.dumps({"Event": {"info": "Not found", "Attribute": []}}, indent=2)
+
+
+# ---------------------------------------------------------------------------
+# Internal helpers
+# ---------------------------------------------------------------------------
+
+
+def _load_investigation_and_entities(
+    investigation_id: Any,
+    entity_ids: Optional[list[str]] = None,
+):
+    """
+    Load the investigation record and its entities from DB.
+
+    Includes entities owned directly by the investigation AND entities linked
+    via InvestigationEntityLink (canonical dedup junction table).
+
+    Returns (investigation, entities) or (None, []) on error / not found.
+    """
+    import uuid as _uuid
+
+    if not os.getenv("DATABASE_URL"):
+        return None, []
+
+    try:
+        from db.session import get_session  # noqa: PLC0415
+        from db.queries import get_investigation_by_id_or_run  # noqa: PLC0415
+        from db.models import Entity, InvestigationEntityLink  # noqa: PLC0415
+        from extractor.normalizer import NormalizedEntity  # noqa: PLC0415
+
+        inv_uuid = _coerce_uuid(investigation_id)
+        if inv_uuid is None:
+            return None, []
+
+        filter_uuids: Optional[list[_uuid.UUID]] = None
+        if entity_ids:
+            filter_uuids = []
+            for raw in entity_ids:
+                try:
+                    filter_uuids.append(_uuid.UUID(str(raw)))
+                except (ValueError, AttributeError):
+                    continue
+
+        with get_session() as session:
+            investigation = get_investigation_by_id_or_run(session, inv_uuid)
+            if investigation is None:
+                return None, []
+
+            linked_ids_subq = (
+                session.query(InvestigationEntityLink.entity_id)
+                .filter(InvestigationEntityLink.investigation_id == investigation.id)
+                .subquery()
+            )
+            db_entities = (
+                session.query(Entity)
+                .filter(
+                    (Entity.investigation_id == investigation.id)
+                    | Entity.id.in_(linked_ids_subq)
+                )
+                .all()
+            )
+
+            if filter_uuids is not None:
+                want = frozenset(filter_uuids)
+                db_entities = [e for e in db_entities if e.id in want]
+
+            normalized: list[NormalizedEntity] = []
+            for e in db_entities:
+                source_url = ""
+                try:
+                    if e.page:
+                        source_url = e.page.url or ""
+                except Exception:
+                    pass
+                ne = NormalizedEntity(
+                    entity_type=e.entity_type,
+                    value=e.canonical_value or e.value,
+                    confidence=e.confidence,
+                    source_url=source_url,
+                    page_id=e.page_id,
+                    context_snippet=e.context_snippet or "",
+                )
+                normalized.append(ne)
+
+            session.expunge_all()
+            return investigation, normalized
+
+    except Exception as exc:
+        logger.warning("_load_investigation_and_entities failed: %s", exc)
+        return None, []
+
+
+def _coerce_uuid(value: Any):
+    """Coerce value to uuid.UUID. Returns None on failure."""
+    import uuid as _uuid
+    if isinstance(value, _uuid.UUID):
+        return value
+    try:
+        return _uuid.UUID(str(value))
+    except (ValueError, AttributeError):
+        return None
+
+
+def _utc_date_str(dt: Optional[Any]) -> str:
+    """Format a datetime as YYYY-MM-DD string. Defaults to today on None."""
+    if dt is None:
+        return datetime.now(timezone.utc).strftime("%Y-%m-%d")
+    try:
+        return dt.strftime("%Y-%m-%d")
+    except Exception:
+        return datetime.now(timezone.utc).strftime("%Y-%m-%d")