ventra 1.0.1__py3-none-any.whl

app/__init__.py

@@ -0,0 +1,8 @@
+"""Ventra analyst console backend.
+
+A thin FastAPI layer over the case store the ingester produces. It never calls out to the
+cloud; it only reads ``cases/<id>/`` (events.parquet via DuckDB + the JSON sidecars). RBAC is
+enforced here, server-side. No telemetry.
+"""
+
+__version__ = "0.1.0"

app/config.py

@@ -0,0 +1,88 @@
+"""Backend configuration. Everything is local and offline by design."""
+
+from __future__ import annotations
+
+import os
+from pathlib import Path
+
+
+def _default_data_root() -> Path:
+    env = os.environ.get("VENTRA_HOME", "").strip()
+    if env:
+        return Path(env).expanduser().resolve()
+    # Packaged / pipx installs: keep state under the user data dir, not cwd. Shared with
+    # the CLI (`ventra import`, collect auto-ingest) so both use the same case store.
+    try:
+        from collector.paths import default_data_root
+
+        return default_data_root()
+    except Exception:
+        pass
+    return Path.cwd().resolve()
+
+
+def _default_artifacts_root() -> Path:
+    """Locate the artifact YAML catalog regardless of the backend's working directory."""
+    env = os.environ.get("VENTRA_ARTIFACTS_ROOT")
+    if env:
+        return Path(env).resolve()
+    try:
+        from collector.paths import default_artifacts_root
+
+        return default_artifacts_root()
+    except Exception:
+        pass
+    return Path("artifacts").resolve()
+
+
+_DATA = _default_data_root()
+
+
+class Settings:
+    # Root of the case store the ingester writes to.
+    case_store: Path = Path(os.environ.get("VENTRA_CASE_STORE", str(_DATA / "cases"))).resolve()
+    # Root of the artifact YAML catalog the Acquire tab and kit builder read.
+    artifacts_root: Path = _default_artifacts_root()
+    # Where uploaded packages are staged before ingest.
+    upload_dir: Path = Path(os.environ.get("VENTRA_UPLOAD_DIR", str(_DATA / ".ventra-uploads"))).resolve()
+    # Hard cap on an uploaded evidence package, in MB. Streamed to disk; the request is
+    # rejected once the limit is exceeded so a large upload can't exhaust memory or disk.
+    max_upload_mb: int = int(os.environ.get("VENTRA_MAX_UPLOAD_MB", "20480"))
+    # CORS origins for the frontend dev server.
+    cors_origins: list[str] = os.environ.get(
+        "VENTRA_CORS", "http://localhost:3000,http://localhost:8080"
+    ).split(",")
+    # Refuse requests whose Host isn't this machine (DNS rebinding; see app.host_check). Add
+    # names with VENTRA_ALLOWED_HOSTS rather than turning this off.
+    host_check: bool = os.environ.get("VENTRA_HOST_CHECK", "on").strip().lower() not in (
+        "0",
+        "off",
+        "false",
+        "no",
+    )
+    # Telemetry is OFF and not configurable to on. Stated explicitly for auditors.
+    telemetry: bool = False
+    # S3 prefix polled by Import from S3 in the console (s3://bucket/prefix/).
+    ingest_s3_prefix: str = os.environ.get("VENTRA_INGEST_S3_PREFIX", "").strip()
+    ingest_download_dir: Path = Path(
+        os.environ.get("VENTRA_INGEST_DOWNLOAD_DIR", str(_DATA / ".ventra-ingest-watch"))
+    ).resolve()
+    _ingest_state = os.environ.get("VENTRA_INGEST_STATE_FILE", "").strip()
+    ingest_state_file: Path | None = Path(_ingest_state).resolve() if _ingest_state else None
+    # Saved connections and collection profiles for the Configuration section.
+    config_dir: Path = Path(os.environ.get("VENTRA_CONFIG_DIR", str(_DATA / ".ventra-config"))).resolve()
+    # File-backed collection run state (matrix + SSE events).
+    runs_dir: Path = Path(os.environ.get("VENTRA_RUNS_DIR", str(_DATA / ".ventra-runs"))).resolve()
+    # Optional SIEM drop zone: export writes NDJSON here for Logstash/Filebeat/forwarders.
+    # Unset = download-only. Never holds SIEM credentials — shippers watch this path.
+    _export_drop = os.environ.get("VENTRA_EXPORT_DROP_DIR", "").strip()
+    export_drop_dir: Path | None = Path(_export_drop).resolve() if _export_drop else None
+
+
+settings = Settings()
+settings.upload_dir.mkdir(parents=True, exist_ok=True)
+settings.config_dir.mkdir(parents=True, exist_ok=True)
+settings.runs_dir.mkdir(parents=True, exist_ok=True)
+settings.case_store.mkdir(parents=True, exist_ok=True)
+if settings.export_drop_dir is not None:
+    settings.export_drop_dir.mkdir(parents=True, exist_ok=True)

app/config_store.py

@@ -0,0 +1,147 @@
+"""File-backed configuration store for connections and collection profiles."""
+
+from __future__ import annotations
+
+import json
+import os
+import uuid
+from pathlib import Path
+from typing import Any
+
+# Connection fields that hold credentials. They stay on disk (the collectors need them) but
+# never leave the backend: API responses carry only which ones are set.
+SECRET_FIELDS = (
+    "aws_secret_access_key",
+    "aws_session_token",
+    "azure_client_secret",
+    "azure_client_certificate_content",
+    "gcp_service_account_json",
+    "kubeconfig_content",
+)
+
+
+def public_connection(conn: dict[str, Any]) -> dict[str, Any]:
+    """A connection as the API returns it: secrets removed, ``stored_secrets`` lists what is set."""
+    out = {k: v for k, v in conn.items() if k not in SECRET_FIELDS}
+    out["stored_secrets"] = [k for k in SECRET_FIELDS if conn.get(k)]
+    return out
+
+
+class ConfigNotFound(Exception):
+    pass
+
+
+class ConfigStore:
+    def __init__(self, root: Path) -> None:
+        self.root = root
+        self.root.mkdir(mode=0o700, parents=True, exist_ok=True)
+        self.connections_path = self.root / "connections.json"
+        self.profiles_path = self.root / "profiles.json"
+        self._restrict_permissions()
+
+    def _restrict_permissions(self) -> None:
+        """Owner-only access for the store (and files written by older versions)."""
+        if os.name != "posix":
+            return
+        for path, mode in (
+            (self.root, 0o700),
+            (self.connections_path, 0o600),
+            (self.profiles_path, 0o600),
+        ):
+            try:
+                if path.exists() and (path.stat().st_mode & 0o777) != mode:
+                    path.chmod(mode)
+            except OSError:
+                pass  # not ours to change (e.g. read-only mount); nothing more we can do
+
+    def _read(self, path: Path) -> list[dict[str, Any]]:
+        if not path.is_file():
+            return []
+        return json.loads(path.read_text(encoding="utf-8"))
+
+    def _write(self, path: Path, items: list[dict[str, Any]]) -> None:
+        """Write atomically, owner read/write only: the file holds cloud credentials."""
+        tmp = path.with_name(f".{path.name}.{uuid.uuid4().hex}.tmp")
+        fd = os.open(tmp, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)
+        try:
+            with os.fdopen(fd, "w", encoding="utf-8") as fh:
+                json.dump(items, fh, indent=2)
+            os.replace(tmp, path)
+        except BaseException:
+            tmp.unlink(missing_ok=True)
+            raise
+
+    def list_connections(self) -> list[dict[str, Any]]:
+        return self._read(self.connections_path)
+
+    def get_connection(self, conn_id: str) -> dict[str, Any]:
+        item = next((c for c in self.list_connections() if c.get("id") == conn_id), None)
+        if item is None:
+            raise ConfigNotFound(f"Connection not found: {conn_id}")
+        return item
+
+    def create_connection(self, data: dict[str, Any]) -> dict[str, Any]:
+        from datetime import datetime, timezone
+
+        items = self.list_connections()
+        entry = {
+            "id": str(uuid.uuid4()),
+            "created_at": datetime.now(timezone.utc).isoformat(),
+            **data,
+        }
+        items.append(entry)
+        self._write(self.connections_path, items)
+        return entry
+
+    def update_connection(self, conn_id: str, patch: dict[str, Any]) -> dict[str, Any]:
+        items = self.list_connections()
+        for i, item in enumerate(items):
+            if item.get("id") == conn_id:
+                updated = {**item, **patch, "id": conn_id}
+                items[i] = updated
+                self._write(self.connections_path, items)
+                return updated
+        raise ConfigNotFound(f"Connection not found: {conn_id}")
+
+    def delete_connection(self, conn_id: str) -> None:
+        items = [c for c in self.list_connections() if c.get("id") != conn_id]
+        if len(items) == len(self._read(self.connections_path)):
+            raise ConfigNotFound(f"Connection not found: {conn_id}")
+        self._write(self.connections_path, items)
+
+    def list_profiles(self) -> list[dict[str, Any]]:
+        return self._read(self.profiles_path)
+
+    def get_profile(self, profile_id: str) -> dict[str, Any]:
+        item = next((p for p in self.list_profiles() if p.get("id") == profile_id), None)
+        if item is None:
+            raise ConfigNotFound(f"Profile not found: {profile_id}")
+        return item
+
+    def create_profile(self, data: dict[str, Any]) -> dict[str, Any]:
+        items = self.list_profiles()
+        entry = {"id": str(uuid.uuid4()), **data}
+        items.append(entry)
+        self._write(self.profiles_path, items)
+        return entry
+
+    def update_profile(self, profile_id: str, patch: dict[str, Any]) -> dict[str, Any]:
+        items = self.list_profiles()
+        for i, item in enumerate(items):
+            if item.get("id") == profile_id:
+                updated = {**item, **patch, "id": profile_id}
+                items[i] = updated
+                self._write(self.profiles_path, items)
+                return updated
+        raise ConfigNotFound(f"Profile not found: {profile_id}")
+
+    def delete_profile(self, profile_id: str) -> None:
+        items = [p for p in self.list_profiles() if p.get("id") != profile_id]
+        if len(items) == len(self._read(self.profiles_path)):
+            raise ConfigNotFound(f"Profile not found: {profile_id}")
+        self._write(self.profiles_path, items)
+
+
+from .config import settings
+
+config_store = ConfigStore(settings.config_dir)

app/evidence.py

@@ -0,0 +1,243 @@
+"""Browse raw evidence files extracted from the sealed package into ``cases/<id>/evidence/``."""
+
+from __future__ import annotations
+
+import gzip
+import json
+import re
+from pathlib import Path
+from typing import Any
+
+from .config import settings
+
+_PATH_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._/-]{0,512}$")
+
+
+class EvidenceError(ValueError):
+    pass
+
+
+class EvidenceNotFound(EvidenceError):
+    pass
+
+
+def _evidence_root(case_dir: Path) -> Path:
+    return case_dir / "evidence"
+
+
+def _resolve_evidence_path(case_dir: Path, rel_path: str) -> Path:
+    rel = rel_path.strip().lstrip("/").replace("\\", "/")
+    if not rel or ".." in rel.split("/") or not _PATH_RE.fullmatch(rel):
+        raise EvidenceError(f"Invalid evidence path: {rel_path!r}")
+    root = _evidence_root(case_dir).resolve()
+    target = (root / rel).resolve()
+    if target != root and root not in target.parents:
+        raise EvidenceError(f"Path escapes evidence root: {rel_path!r}")
+    return target
+
+
+def _manifest_source_meta(manifest: dict[str, Any]) -> dict[str, dict[str, Any]]:
+    out: dict[str, dict[str, Any]] = {}
+    for entry in manifest.get("sources") or []:
+        if not isinstance(entry, dict):
+            continue
+        path = str(entry.get("path") or "").strip()
+        if path:
+            out[path] = entry
+        name = str(entry.get("name") or "").strip()
+        if name and name not in out:
+            out[name] = entry
+    return out
+
+
+def _classify_file(rel: str) -> str:
+    name = rel.rsplit("/", 1)[-1]
+    if name.startswith("events"):
+        return "events"
+    if name == "config.json":
+        return "config"
+    if name == "snapshot.json":
+        return "snapshot"
+    if name == "_meta.json":
+        return "meta"
+    if "credential_report" in name:
+        return "credential_report"
+    if rel.startswith("errors/"):
+        return "error"
+    if rel == "collection.log":
+        return "collection_log"
+    if rel == "manifest.json":
+        return "manifest"
+    if rel.endswith(".sig"):
+        return "signature"
+    return "other"
+
+
+# Sidecar / packaging kinds must not display a source-level inherited record_count.
+_NON_DATA_KINDS = frozenset({"config", "meta", "manifest", "signature", "collection_log", "error"})
+
+
+def _record_count_for_file(kind: str, manifest_entry: dict[str, Any] | None) -> int | None:
+    if not manifest_entry:
+        return None
+    if kind in _NON_DATA_KINDS:
+        return None
+    raw = manifest_entry.get("record_count")
+    if raw is None:
+        return None
+    try:
+        return int(raw)
+    except (TypeError, ValueError):
+        return None
+
+
+def ensure_evidence_extracted(case_dir: Path, case_id: str) -> Path:
+    """Return the evidence directory, extracting from a retained package when missing."""
+    root = _evidence_root(case_dir)
+    if root.is_dir() and any(root.rglob("*")):
+        return root
+
+    from ventra_ingester.evidence_extract import extract_package, find_package_for_case
+
+    pkg_dir = case_dir / "package"
+    package = find_package_for_case(case_id, pkg_dir, settings.upload_dir)
+    if package is None:
+        raise EvidenceNotFound(f"No raw evidence files for case {case_id!r}. Re-import the sealed package.")
+    extract_package(package, root)
+    return root
+
+
+def list_evidence_files(case_dir: Path, manifest: dict[str, Any]) -> dict[str, Any]:
+    case_id = str(manifest.get("case_id") or case_dir.name)
+    root = ensure_evidence_extracted(case_dir, case_id)
+    meta_by_path = _manifest_source_meta(manifest)
+    files: list[dict[str, Any]] = []
+    total_bytes = 0
+    for path in sorted(root.rglob("*")):
+        if not path.is_file():
+            continue
+        rel = path.relative_to(root).as_posix()
+        size = path.stat().st_size
+        total_bytes += size
+        source_name = ""
+        if rel.startswith("sources/"):
+            parts = rel.split("/")
+            if len(parts) >= 2:
+                source_name = parts[1]
+        # Prefer exact path, then source name (collector-level entry), then first path segment.
+        manifest_entry = (
+            meta_by_path.get(rel)
+            or (meta_by_path.get(source_name) if source_name else None)
+            or meta_by_path.get(rel.split("/", 1)[0])
+        )
+        kind = _classify_file(rel)
+        files.append(
+            {
+                "path": rel,
+                "size": size,
+                "kind": kind,
+                "source": source_name or None,
+                "sha256": (manifest_entry or {}).get("sha256"),
+                "record_count": _record_count_for_file(kind, manifest_entry),
+                "status": (manifest_entry or {}).get("status"),
+                "notes": (manifest_entry or {}).get("notes"),
+            }
+        )
+    return {
+        "case_id": case_id,
+        "root": "evidence",
+        "files": files,
+        "total_files": len(files),
+        "total_bytes": total_bytes,
+    }
+
+
+def read_evidence_text(case_dir: Path, rel_path: str, *, max_bytes: int | None = None) -> dict[str, Any]:
+    target = _resolve_evidence_path(case_dir, rel_path)
+    if not target.is_file():
+        raise EvidenceNotFound(f"No evidence file at {rel_path!r}")
+    size = target.stat().st_size
+    cap = max_bytes if max_bytes is not None else size
+    raw = target.read_bytes()[:cap]
+    truncated = len(raw) < size
+    if rel_path.endswith(".gz") and not rel_path.endswith(".tar.gz"):
+        try:
+            raw = gzip.decompress(raw)
+            truncated = truncated or len(raw) >= cap
+        except OSError:
+            pass
+    text = raw.decode("utf-8", errors="replace")
+    content_type = "json" if rel_path.endswith(".json") else "text"
+    if content_type == "json":
+        try:
+            parsed = json.loads(text)
+            return {
+                "path": rel_path,
+                "size": size,
+                "truncated": truncated,
+                "content_type": "json",
+                "json": parsed,
+            }
+        except json.JSONDecodeError:
+            content_type = "text"
+    return {
+        "path": rel_path,
+        "size": size,
+        "truncated": truncated,
+        "content_type": content_type,
+        "text": text,
+    }
+
+
+def read_evidence_lines(
+    case_dir: Path,
+    rel_path: str,
+    *,
+    offset: int = 0,
+    limit: int | None = None,
+) -> dict[str, Any]:
+    """Return JSON-lines records from an events file with optional pagination."""
+    target = _resolve_evidence_path(case_dir, rel_path)
+    if not target.is_file():
+        raise EvidenceNotFound(f"No evidence file at {rel_path!r}")
+
+    if limit is None:
+        limit = 10_000_000
+
+    data = target.read_bytes()
+    if rel_path.endswith(".gz"):
+        data = gzip.decompress(data)
+    lines = data.decode("utf-8", errors="replace").splitlines()
+    total_lines = len(lines)
+    slice_lines = lines[offset : offset + limit]
+    records: list[Any] = []
+    for line in slice_lines:
+        line = line.strip()
+        if not line:
+            continue
+        try:
+            records.append(json.loads(line))
+        except json.JSONDecodeError:
+            records.append({"_raw": line})
+    return {
+        "path": rel_path,
+        "total_lines": total_lines,
+        "offset": offset,
+        "count": len(records),
+        "has_more": offset + len(slice_lines) < total_lines,
+        "records": records,
+    }
+
+
+def read_evidence_bytes(case_dir: Path, rel_path: str) -> tuple[Path, str]:
+    target = _resolve_evidence_path(case_dir, rel_path)
+    if not target.is_file():
+        raise EvidenceNotFound(f"No evidence file at {rel_path!r}")
+    media = "application/octet-stream"
+    if rel_path.endswith(".json"):
+        media = "application/json"
+    elif rel_path.endswith(".log"):
+        media = "text/plain"
+    elif rel_path.endswith(".gz"):
+        media = "application/gzip"
+    return target, media