ventra 0.0.0.post26__py3-none-any.whl

collector/__init__.py

@@ -0,0 +1,23 @@
+"""Ventra collector — read-only cloud forensic triage acquisition.
+
+The collector runs in the client's cloud shell, gathers exactly the logs and artifacts
+incident responders need, and seals them into a signed evidence package described by the
+Ventra Evidence Package Format (EPF).
+
+Forensic invariant: nothing in this package may call a mutating cloud API. See
+``collector.tools.verify_readonly`` and the ``readonly-guard`` CI check.
+"""
+
+from importlib.metadata import PackageNotFoundError
+from importlib.metadata import version as _pkg_version
+
+try:
+    # Resolved from the installed distribution's metadata, which setuptools-scm derives from
+    # the git tag at build/install time. Recorded in every manifest as ``tool_version``, so an
+    # evidence package always shows exactly which build collected it (a tagged release like
+    # ``0.2.0``, or a dev build like ``0.2.0.dev3+g1a2b3c4`` when run from a working tree).
+    __version__ = _pkg_version("ventra")
+except PackageNotFoundError:  # running from a source tree that was never installed
+    __version__ = "0.0.0+unknown"
+
+del PackageNotFoundError, _pkg_version

collector/__main__.py

@@ -0,0 +1,48 @@
+"""Allow ``python -m collector dev`` from a fresh clone before ``pip install``."""
+
+from __future__ import annotations
+
+import os
+import shutil
+import sys
+
+_MIN_PYTHON = (3, 11)
+
+
+def _reexec_with_newer_python() -> None:
+    """Re-run this module with Python 3.11+ when the default ``python3`` is too old."""
+    if sys.version_info >= _MIN_PYTHON:
+        return
+    for cmd in ("python3.12", "python3.11", "python3"):
+        path = shutil.which(cmd)
+        if not path or os.path.realpath(path) == os.path.realpath(sys.executable):
+            continue
+        import subprocess
+
+        ok = subprocess.run(
+            [
+                path,
+                "-c",
+                f"import sys; raise SystemExit(0 if sys.version_info >= {_MIN_PYTHON!r} else 1)",
+            ],
+            capture_output=True,
+        ).returncode == 0
+        if not ok:
+            continue
+        os.execv(path, [path, "-m", "collector", *sys.argv[1:]])
+
+    print(
+        "error: Ventra requires Python 3.11 or newer.\n"
+        f"  Current: {sys.executable} ({sys.version.split()[0]})\n"
+        "  Install:   brew install python@3.11\n"
+        "  Then run:  python3.11 -m collector dev",
+        file=sys.stderr,
+    )
+    raise SystemExit(1)
+
+
+if __name__ == "__main__":
+    _reexec_with_newer_python()
+    from .cli import main
+
+    raise SystemExit(main())

collector/aws/__init__.py

@@ -0,0 +1,5 @@
+"""AWS orchestration — registry, runner, and boto3 client factory.
+
+Collector modules (identity, control_plane, network, …) live in this package.
+Import ``collector.aws.registry`` to populate and access the registry.
+"""

collector/aws/client_factory.py

@@ -0,0 +1,183 @@
+"""AWS client management used by every collector.
+
+Wraps boto3 so collectors don't each reinvent region handling, pagination, and the
+all-important AccessDenied detection (an AccessDenied is a *gap*, recorded as evidence, not a
+crash). All clients are created from a single session so credentials are resolved once.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Iterator
+from dataclasses import dataclass
+from typing import Any
+
+import boto3
+from botocore.config import Config
+from botocore.exceptions import ClientError, EndpointConnectionError, NoCredentialsError
+
+# Error codes that mean "you can't see this" rather than "something broke".
+ACCESS_DENIED_CODES = frozenset(
+    {
+        "AccessDenied",
+        "AccessDeniedException",
+        "UnauthorizedOperation",
+        "AuthorizationError",
+        "AuthFailure",
+        "ForbiddenException",
+    }
+)
+
+# Error codes that mean "this service/feature isn't enabled here".
+NOT_ENABLED_CODES = frozenset(
+    {
+        "ResourceNotFoundException",
+        "BadRequestException",
+        "InvalidInputException",
+        "SubscriptionRequiredException",
+        "OptInRequired",
+        # Security Hub raises this when the account is not subscribed to the hub.
+        "InvalidAccessException",
+        # WAFv2 raises this when e.g. a Web ACL has no logging configuration.
+        "WAFNonexistentItemException",
+        # Standalone accounts that are not part of an AWS Organization.
+        "AWSOrganizationsNotInUseException",
+        # CloudTrail trails without Insights enabled.
+        "InsightNotEnabledException",
+        # IAM raises this when e.g. no custom password policy exists on the account.
+        "NoSuchEntity",
+    }
+)
+
+
+class AccessDenied(Exception):
+    """Raised by helpers when an API returns an access-denied style error."""
+
+    def __init__(self, action: str, message: str) -> None:
+        super().__init__(f"{action}: {message}")
+        self.action = action
+        self.message = message
+
+
+class ServiceNotEnabled(Exception):
+    def __init__(self, service: str, message: str) -> None:
+        super().__init__(f"{service}: {message}")
+        self.service = service
+        self.message = message
+
+
+@dataclass
+class CallerIdentity:
+    account_id: str
+    arn: str
+    user_id: str
+    partition: str
+
+
+class AwsClientFactory:
+    """Creates per-service, per-region boto3 clients from one session."""
+
+    def __init__(self, session: boto3.Session | None = None) -> None:
+        self._session = session or boto3.Session()
+        self._cfg = Config(retries={"max_attempts": 5, "mode": "adaptive"}, user_agent_extra="ventra")
+        self._cache: dict[tuple[str, str | None], Any] = {}
+
+    def client(self, service: str, region: str | None = None) -> Any:
+        key = (service, region)
+        if key not in self._cache:
+            self._cache[key] = self._session.client(service, region_name=region, config=self._cfg)
+        return self._cache[key]
+
+    # -- identity / region discovery -----------------------------------------------------
+
+    def caller_identity(self) -> CallerIdentity:
+        try:
+            ident = self.client("sts").get_caller_identity()
+        except NoCredentialsError as exc:  # pragma: no cover
+            raise RuntimeError(
+                "No AWS credentials found. Run inside CloudShell or configure a profile."
+            ) from exc
+        arn = ident["Arn"]
+        partition = arn.split(":")[1] if arn.startswith("arn:") else "aws"
+        return CallerIdentity(
+            account_id=ident["Account"],
+            arn=arn,
+            user_id=ident.get("UserId", ""),
+            partition=partition,
+        )
+
+    def enabled_regions(self) -> list[str]:
+        """Regions enabled for this account (opt-in regions included if active)."""
+        try:
+            resp = self.client("ec2", "us-east-1").describe_regions(
+                Filters=[{"Name": "opt-in-status", "Values": ["opt-in-not-required", "opted-in"]}]
+            )
+            return sorted(r["RegionName"] for r in resp["Regions"])
+        except ClientError:
+            # Fall back to the SDK's static partition list.
+            return sorted(self._session.get_available_regions("ec2"))
+
+    # -- safe call helpers ---------------------------------------------------------------
+
+    def paginate(
+        self, service: str, region: str | None, operation: str, result_key: str, **kwargs: Any
+    ) -> Iterator[dict[str, Any]]:
+        """Yield items across pages, translating access/enablement errors into typed gaps."""
+        client = self.client(service, region)
+        try:
+            paginator = client.get_paginator(operation)
+            for page in paginator.paginate(**kwargs):
+                yield from page.get(result_key, [])
+        except ClientError as exc:
+            _raise_typed(exc, f"{service}:{operation}")
+        except EndpointConnectionError:
+            return
+
+    def call(self, service: str, region: str | None, operation: str, **kwargs: Any) -> dict[str, Any]:
+        client = self.client(service, region)
+        try:
+            return getattr(client, operation)(**kwargs)
+        except ClientError as exc:
+            _raise_typed(exc, f"{service}:{operation}")
+            raise  # unreachable, keeps type-checkers happy
+        except EndpointConnectionError as exc:
+            # The service has no endpoint in this region — same gap as "not enabled".
+            raise ServiceNotEnabled(f"{service}:{operation}", str(exc)) from exc
+
+    def paginate_manual(
+        self,
+        service: str,
+        region: str | None,
+        operation: str,
+        result_key: str,
+        *,
+        token_request_key: str = "NextToken",
+        token_response_key: str = "NextToken",
+        max_pages: int = 500,
+        **kwargs: Any,
+    ) -> Iterator[dict[str, Any]]:
+        """Token-loop pagination for operations botocore has no paginator for
+        (e.g. wafv2 ListWebACLs / detective ListInvestigations)."""
+        token: str | None = None
+        for _ in range(max_pages):
+            params = dict(kwargs)
+            if token:
+                params[token_request_key] = token
+            page = self.call(service, region, operation, **params)
+            items = page.get(result_key) or []
+            yield from items
+            new_token = page.get(token_response_key)
+            # Stop on a missing, repeated, or itemless marker so a quirky
+            # implementation can never loop us forever.
+            if not new_token or new_token == token or not items:
+                return
+            token = new_token
+
+
+def _raise_typed(exc: ClientError, action: str) -> None:
+    code = exc.response.get("Error", {}).get("Code", "")
+    msg = exc.response.get("Error", {}).get("Message", str(exc))
+    if code in ACCESS_DENIED_CODES:
+        raise AccessDenied(action, msg)
+    if code in NOT_ENABLED_CODES:
+        raise ServiceNotEnabled(action, msg)
+    raise exc

collector/aws/common/__init__.py

@@ -0,0 +1 @@
+"""Shared acquisition transports used by multiple log collectors."""

collector/aws/common/cw_logs.py

@@ -0,0 +1,70 @@
+"""Bounded reader for CloudWatch Logs-delivered logs (EKS audit, Route53 Resolver to CW).
+
+One transport, many consumers: time-windowed ``FilterLogEvents`` with an optional stream
+prefix, hard record caps, and typed-gap translation so a missing/denied log group is
+recorded as evidence rather than crashing the run.
+"""
+
+from __future__ import annotations
+
+from datetime import datetime
+from typing import Any
+
+from ...lib.models import GapReason
+from ..client_factory import AccessDenied, ServiceNotEnabled
+
+MAX_CW_RECORDS = 200_000
+
+
+def collect_cw_log_events(
+    cf,
+    region: str,
+    log_group: str,
+    start: datetime,
+    end: datetime,
+    gaps: list[tuple[str, GapReason, str]],
+    gap_name: str,
+    *,
+    stream_prefix: str | None = None,
+    max_records: int = MAX_CW_RECORDS,
+) -> tuple[list[dict[str, Any]], dict[str, Any]]:
+    """Pull events from one log group in the window; returns (events, stats)."""
+    stats: dict[str, Any] = {
+        "log_group": log_group,
+        "region": region,
+        "records": 0,
+        "truncated": False,
+    }
+    events: list[dict[str, Any]] = []
+    kwargs: dict[str, Any] = {
+        "logGroupName": log_group,
+        "startTime": int(start.timestamp() * 1000),
+        "endTime": int(end.timestamp() * 1000),
+    }
+    if stream_prefix:
+        kwargs["logStreamNamePrefix"] = stream_prefix
+
+    try:
+        for ev in cf.paginate("logs", region, "filter_log_events", "events", **kwargs):
+            if len(events) >= max_records:
+                stats["truncated"] = True
+                gaps.append(
+                    (
+                        gap_name,
+                        GapReason.COLLECTOR_ERROR,
+                        f"{log_group}: truncated at {max_records} records; "
+                        "narrow the window (--since/--until) for full coverage.",
+                    )
+                )
+                break
+            ev["_ventra_region"] = region
+            ev["_ventra_log_group"] = log_group
+            events.append(ev)
+            stats["records"] += 1
+    except AccessDenied as exc:
+        gaps.append((gap_name, GapReason.ACCESS_DENIED, f"{log_group}: {exc.message}"))
+    except ServiceNotEnabled as exc:
+        gaps.append(
+            (gap_name, GapReason.NOT_PRESENT, f"{log_group}: log group not found ({exc.message})")
+        )
+    return events, stats

collector/aws/common/s3_logs.py

@@ -0,0 +1,130 @@
+"""Bounded reader for S3-delivered line-format service logs (ELB, CloudFront, S3 access).
+
+Mirrors the CloudTrail S3 path: list day-scoped prefixes, read objects (gzip or plain),
+yield records, count everything, and translate access errors into manifest gaps instead of
+crashes. Collectors ship raw lines; the ingester owns versioned parsing.
+"""
+
+from __future__ import annotations
+
+import gzip
+import io
+from collections.abc import Callable, Iterator
+from datetime import datetime, timedelta
+from typing import Any
+
+from ...lib.models import GapReason
+from ..client_factory import AccessDenied
+
+# Keep CloudShell runs bounded.
+MAX_LOG_OBJECTS = 2000
+MAX_RECORDS = 200_000
+
+# (key, line) -> record dict, or None to skip (comment line / out of window).
+LineToRecord = Callable[[str, str], dict[str, Any] | None]
+
+
+def iter_days(start: datetime, end: datetime) -> Iterator[datetime]:
+    day = start.replace(hour=0, minute=0, second=0, microsecond=0)
+    last = end.replace(hour=0, minute=0, second=0, microsecond=0)
+    while day <= last:
+        yield day
+        day += timedelta(days=1)
+
+
+def slash_day_prefixes(base: str, start: datetime, end: datetime) -> list[str]:
+    """``<base>YYYY/MM/DD/`` layout (ELB, Route53 Resolver to S3)."""
+    return [f"{base}{d.year:04d}/{d.month:02d}/{d.day:02d}/" for d in iter_days(start, end)]
+
+
+def dash_day_prefixes(base: str, start: datetime, end: datetime) -> list[str]:
+    """``<base>YYYY-MM-DD`` flat layout (CloudFront, S3 server access logs)."""
+    return [f"{base}{d.year:04d}-{d.month:02d}-{d.day:02d}" for d in iter_days(start, end)]
+
+
+def bucket_region(cf, bucket: str, default: str = "us-east-1") -> str:
+    """Best-effort bucket region so cross-region log buckets still list correctly."""
+    try:
+        loc = cf.call("s3", default, "get_bucket_location", Bucket=bucket)
+        return loc.get("LocationConstraint") or "us-east-1"
+    except Exception:  # noqa: BLE001 - region resolution is an optimization, not a requirement
+        return default
+
+
+def _object_lines(body: bytes, key: str) -> Iterator[str]:
+    if key.endswith(".gz"):
+        with gzip.GzipFile(fileobj=io.BytesIO(body)) as gz:
+            text = gz.read().decode("utf-8", errors="replace")
+    else:
+        text = body.decode("utf-8", errors="replace")
+    for line in text.splitlines():
+        if line.strip():
+            yield line
+
+
+def collect_s3_line_records(
+    cf,
+    region: str,
+    bucket: str,
+    prefixes: list[str],
+    line_to_record: LineToRecord,
+    gaps: list[tuple[str, GapReason, str]],
+    gap_name: str,
+    *,
+    max_objects: int = MAX_LOG_OBJECTS,
+    max_records: int = MAX_RECORDS,
+) -> tuple[list[dict[str, Any]], dict[str, Any]]:
+    """Read line logs under ``prefixes`` in ``bucket``; returns (records, stats)."""
+    stats: dict[str, Any] = {
+        "bucket": bucket,
+        "objects_scanned": 0,
+        "objects_read": 0,
+        "records": 0,
+        "truncated": False,
+    }
+    records: list[dict[str, Any]] = []
+    s3 = cf.client("s3", region)
+
+    for prefix in prefixes:
+        if stats["truncated"]:
+            break
+        try:
+            for obj in cf.paginate(
+                "s3", region, "list_objects_v2", "Contents", Bucket=bucket, Prefix=prefix
+            ):
+                stats["objects_scanned"] += 1
+                if stats["objects_scanned"] > max_objects:
+                    stats["truncated"] = True
+                    break
+                key = obj.get("Key", "")
+                if key.endswith("/"):
+                    continue
+                try:
+                    body = s3.get_object(Bucket=bucket, Key=key)["Body"].read()
+                except Exception as exc:  # noqa: BLE001 - one unreadable object is a gap, not a crash
+                    gaps.append((gap_name, GapReason.COLLECTOR_ERROR, f"{bucket}/{key}: {exc}"))
+                    continue
+                stats["objects_read"] += 1
+                for line in _object_lines(body, key):
+                    if len(records) >= max_records:
+                        stats["truncated"] = True
+                        break
+                    rec = line_to_record(key, line)
+                    if rec is not None:
+                        records.append(rec)
+                        stats["records"] += 1
+                if stats["truncated"]:
+                    break
+        except AccessDenied as exc:
+            gaps.append((gap_name, GapReason.ACCESS_DENIED, f"{bucket}/{prefix}: {exc.message}"))
+
+    if stats["truncated"]:
+        gaps.append(
+            (
+                gap_name,
+                GapReason.COLLECTOR_ERROR,
+                f"{bucket}: truncated at {max_objects} objects / {max_records} records; "
+                "narrow the window (--since/--until) for full coverage.",
+            )
+        )
+    return records, stats