shellbrain 0.1.0__py3-none-any.whl

app/migrations/versions/20260320_0008_instance_metadata_and_backup_safety.py

@@ -0,0 +1,31 @@
+"""Create instance metadata used for destructive guardrails and backup safety."""
+
+from alembic import op
+
+
+revision = "20260320_0008"
+down_revision = "20260319_0007"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    """Create the instance metadata table when missing."""
+
+    op.execute(
+        """
+        CREATE TABLE IF NOT EXISTS instance_metadata (
+          instance_id TEXT PRIMARY KEY,
+          instance_mode TEXT NOT NULL,
+          created_at TIMESTAMPTZ NOT NULL,
+          created_by TEXT NOT NULL,
+          notes TEXT NULL
+        );
+        """
+    )
+
+
+def downgrade() -> None:
+    """Drop the instance metadata table for downgrade compatibility."""
+
+    op.execute("DROP TABLE IF EXISTS instance_metadata;")

app/migrations/versions/__init__.py

@@ -0,0 +1 @@
+"""This package contains Alembic migration revision modules."""

app/periphery/__init__.py

@@ -0,0 +1 @@
+"""This package contains infrastructure adapters used by the core."""

app/periphery/admin/__init__.py

@@ -0,0 +1 @@
+"""Administrative safety, backup, and recovery helpers."""

app/periphery/admin/backup.py

@@ -0,0 +1,360 @@
+"""Logical backup helpers for Shellbrain databases."""
+
+from __future__ import annotations
+
+from dataclasses import asdict, dataclass
+from datetime import datetime, timezone
+import gzip
+import hashlib
+import json
+from pathlib import Path
+import re
+import shutil
+import subprocess
+from uuid import uuid4
+
+import psycopg
+
+from app.periphery.admin.instance_guard import PROTECTED_DB_NAMES, fetch_instance_metadata, fingerprint_summary
+
+
+_UNSUPPORTED_RESTORE_PARAMETERS = ("transaction_timeout",)
+_UNSUPPORTED_SET_LINE_RE = re.compile(r"^SET\s+([a-zA-Z_][a-zA-Z0-9_]*)\s*=")
+_UNSUPPORTED_SET_CONFIG_RE = re.compile(r"^SELECT\s+pg_catalog\.set_config\('([a-zA-Z_][a-zA-Z0-9_]*)'")
+
+
+@dataclass(frozen=True)
+class BackupManifest:
+    """Portable metadata stored next to one logical backup artifact."""
+
+    backup_id: str
+    instance_id: str
+    instance_mode: str
+    source: dict[str, str]
+    schema_revision: str
+    created_at: str
+    artifact_filename: str
+    artifact_sha256: str
+    artifact_size_bytes: int
+    compression: str
+
+
+@dataclass(frozen=True)
+class _ResolvedInstanceMetadata:
+    """Minimal metadata used to bucket and label backup artifacts."""
+
+    instance_id: str
+    instance_mode: str
+
+
+def create_backup(
+    *,
+    admin_dsn: str,
+    backup_root: Path,
+    mirror_root: Path | None = None,
+    container_name: str | None = None,
+    container_db_name: str | None = None,
+    container_admin_user: str | None = None,
+    container_admin_password: str | None = None,
+) -> BackupManifest:
+    """Create one compressed logical backup and return its manifest."""
+
+    if container_name is None and shutil.which("pg_dump") is None:
+        raise RuntimeError("pg_dump is required to create Shellbrain backups.")
+    metadata = _resolve_instance_metadata(admin_dsn)
+    schema_revision = _fetch_schema_revision(admin_dsn)
+    created_at = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ")
+    backup_id = uuid4().hex
+    instance_dir = backup_root / metadata.instance_id
+    instance_dir.mkdir(parents=True, exist_ok=True)
+    artifact_path = instance_dir / f"{created_at}-{backup_id}.sql.gz"
+    manifest_path = instance_dir / f"{created_at}-{backup_id}.manifest.json"
+    raw_dsn = admin_dsn.replace("+psycopg", "")
+
+    command = _backup_command(
+        admin_dsn=admin_dsn,
+        container_name=container_name,
+        container_db_name=container_db_name,
+        container_admin_user=container_admin_user,
+    )
+    env = _backup_env(
+        container_admin_password=container_admin_password,
+    )
+    popen_kwargs = {
+        "stdout": subprocess.PIPE,
+        "stderr": subprocess.PIPE,
+        "text": False,
+    }
+    if env is not None:
+        popen_kwargs["env"] = env
+    with subprocess.Popen(command, **popen_kwargs) as process:
+        stdout, stderr = process.communicate()
+        if process.returncode != 0:
+            raise RuntimeError(f"pg_dump failed: {stderr.decode('utf-8', errors='replace').strip()}")
+    with gzip.open(artifact_path, "wb") as handle:
+        handle.write(stdout)
+
+    manifest = BackupManifest(
+        backup_id=backup_id,
+        instance_id=metadata.instance_id,
+        instance_mode=metadata.instance_mode,
+        source=fingerprint_summary(admin_dsn),
+        schema_revision=schema_revision,
+        created_at=datetime.now(timezone.utc).isoformat(),
+        artifact_filename=artifact_path.name,
+        artifact_sha256=_sha256(artifact_path),
+        artifact_size_bytes=artifact_path.stat().st_size,
+        compression="gzip",
+    )
+    manifest_path.write_text(json.dumps(asdict(manifest), indent=2, sort_keys=True), encoding="utf-8")
+
+    if mirror_root is not None:
+        mirror_dir = mirror_root / metadata.instance_id
+        mirror_dir.mkdir(parents=True, exist_ok=True)
+        shutil.copy2(artifact_path, mirror_dir / artifact_path.name)
+        shutil.copy2(manifest_path, mirror_dir / manifest_path.name)
+    return manifest
+
+
+def list_backups(*, backup_root: Path) -> list[BackupManifest]:
+    """Return every parseable backup manifest under the configured backup root."""
+
+    if not backup_root.exists():
+        return []
+    manifests: list[BackupManifest] = []
+    for path in sorted(backup_root.rglob("*.manifest.json")):
+        try:
+            payload = json.loads(path.read_text(encoding="utf-8"))
+        except (FileNotFoundError, json.JSONDecodeError):
+            continue
+        manifests.append(BackupManifest(**payload))
+    return sorted(manifests, key=lambda item: item.created_at, reverse=True)
+
+
+def verify_backup(*, backup_root: Path, backup_id: str | None = None) -> BackupManifest:
+    """Verify one backup manifest and artifact hash, defaulting to the newest artifact."""
+
+    manifest = resolve_backup(backup_root=backup_root, backup_id=backup_id)
+    artifact_path = backup_root / manifest.instance_id / manifest.artifact_filename
+    if not artifact_path.exists():
+        raise RuntimeError(f"Backup artifact is missing: {artifact_path}")
+    actual_sha256 = _sha256(artifact_path)
+    if actual_sha256 != manifest.artifact_sha256:
+        raise RuntimeError("Backup artifact hash mismatch.")
+    return manifest
+
+
+def resolve_backup(*, backup_root: Path, backup_id: str | None = None) -> BackupManifest:
+    """Resolve one manifest by id, defaulting to the newest available backup."""
+
+    manifests = list_backups(backup_root=backup_root)
+    if not manifests:
+        raise RuntimeError("No Shellbrain backups are available.")
+    if backup_id is None:
+        return manifests[0]
+    for manifest in manifests:
+        if manifest.backup_id == backup_id:
+            return manifest
+    raise RuntimeError(f"Backup id not found: {backup_id}")
+
+
+def restore_backup(
+    *,
+    admin_dsn: str,
+    backup_root: Path,
+    target_db: str,
+    app_dsn: str | None = None,
+    backup_id: str | None = None,
+    container_name: str | None = None,
+    container_admin_user: str | None = None,
+    container_admin_password: str | None = None,
+) -> BackupManifest:
+    """Restore one verified backup into a new scratch database."""
+
+    if container_name is None and shutil.which("psql") is None:
+        raise RuntimeError("psql is required to restore Shellbrain backups.")
+    if target_db.lower() in PROTECTED_DB_NAMES:
+        raise RuntimeError(
+            f"Refusing to restore into protected database name '{target_db}'. Use a fresh scratch database name."
+        )
+    manifest = verify_backup(backup_root=backup_root, backup_id=backup_id)
+    artifact_path = backup_root / manifest.instance_id / manifest.artifact_filename
+    raw_admin_dsn = admin_dsn.replace("+psycopg", "")
+    _create_empty_database(admin_dsn=admin_dsn, target_db=target_db)
+    target_dsn = _replace_database(raw_admin_dsn, target_db)
+    with gzip.open(artifact_path, "rb") as handle:
+        restored_sql = _sanitize_restore_sql(handle.read())
+    run_kwargs = {
+        "input": restored_sql,
+        "capture_output": True,
+        "check": False,
+    }
+    env = _backup_env(container_admin_password=container_admin_password)
+    if env is not None:
+        run_kwargs["env"] = env
+    process = subprocess.run(
+        _restore_command(
+            target_dsn=target_dsn,
+            target_db=target_db,
+            container_name=container_name,
+            container_admin_user=container_admin_user,
+        ),
+        **run_kwargs,
+    )
+    if process.returncode != 0:
+        raise RuntimeError(process.stderr.decode("utf-8", errors="replace").strip() or "psql restore failed")
+    from app.periphery.admin.instance_guard import ensure_instance_metadata
+
+    target_admin_dsn = _replace_database(admin_dsn, target_db)
+    ensure_instance_metadata(
+        target_admin_dsn,
+        instance_mode="scratch",
+        created_by="app.admin.restore",
+        notes=f"Restored from backup {manifest.backup_id}",
+    )
+    if app_dsn:
+        from app.periphery.admin.privileges import reconcile_app_role_privileges
+
+        target_app_dsn = _replace_database(app_dsn, target_db)
+        reconcile_app_role_privileges(admin_dsn=target_admin_dsn, app_dsn=target_app_dsn)
+    return manifest
+
+
+def _resolve_instance_metadata(admin_dsn: str) -> _ResolvedInstanceMetadata:
+    """Resolve stored instance metadata, or synthesize a stable fallback label."""
+
+    metadata = fetch_instance_metadata(admin_dsn)
+    if metadata is not None:
+        return _ResolvedInstanceMetadata(
+            instance_id=metadata.instance_id,
+            instance_mode=metadata.instance_mode,
+        )
+    source = fingerprint_summary(admin_dsn)
+    return _ResolvedInstanceMetadata(
+        instance_id=source["fingerprint"],
+        instance_mode="unknown",
+    )
+
+
+def _backup_command(
+    *,
+    admin_dsn: str,
+    container_name: str | None,
+    container_db_name: str | None,
+    container_admin_user: str | None,
+) -> list[str]:
+    """Return the pg_dump command for host or managed-container execution."""
+
+    if container_name is None:
+        raw_dsn = admin_dsn.replace("+psycopg", "")
+        return ["pg_dump", "--no-owner", "--no-privileges", f"--dbname={raw_dsn}"]
+    if not container_db_name or not container_admin_user:
+        raise RuntimeError("Managed container backups require container DB name and admin user.")
+    return [
+        "docker",
+        "exec",
+        "-e",
+        "PGPASSWORD",
+        container_name,
+        "pg_dump",
+        "--no-owner",
+        "--no-privileges",
+        "--username",
+        container_admin_user,
+        "--dbname",
+        container_db_name,
+    ]
+
+
+def _restore_command(
+    *,
+    target_dsn: str,
+    target_db: str,
+    container_name: str | None,
+    container_admin_user: str | None,
+) -> list[str]:
+    """Return the psql command for host or managed-container restore execution."""
+
+    if container_name is None:
+        return ["psql", "--set", "ON_ERROR_STOP=1", f"--dbname={target_dsn}"]
+    if not container_admin_user:
+        raise RuntimeError("Managed container restore requires the container admin user.")
+    return [
+        "docker",
+        "exec",
+        "-i",
+        "-e",
+        "PGPASSWORD",
+        container_name,
+        "psql",
+        "--set",
+        "ON_ERROR_STOP=1",
+        "--username",
+        container_admin_user,
+        "--dbname",
+        target_db,
+    ]
+
+
+def _backup_env(*, container_admin_password: str | None) -> dict[str, str] | None:
+    """Return subprocess environment overrides for managed-container operations."""
+
+    if container_admin_password is None:
+        return None
+    return {"PGPASSWORD": container_admin_password}
+
+
+def _create_empty_database(*, admin_dsn: str, target_db: str) -> None:
+    """Create one empty restore target database, failing if it already exists."""
+
+    raw_admin_dsn = admin_dsn.replace("+psycopg", "")
+    postgres_dsn = _replace_database(raw_admin_dsn, "postgres")
+    with psycopg.connect(postgres_dsn, autocommit=True) as conn:
+        with conn.cursor() as cur:
+            cur.execute("SELECT 1 FROM pg_database WHERE datname = %s", (target_db,))
+            if cur.fetchone() is not None:
+                raise RuntimeError(f"Target database already exists: {target_db}")
+            cur.execute(f'CREATE DATABASE "{target_db}"')
+
+
+def _fetch_schema_revision(admin_dsn: str) -> str:
+    """Read the current alembic revision for manifest metadata."""
+
+    raw_dsn = admin_dsn.replace("+psycopg", "")
+    with psycopg.connect(raw_dsn) as conn:
+        with conn.cursor() as cur:
+            cur.execute("SELECT version_num FROM alembic_version")
+            return str(cur.fetchone()[0])
+
+
+def _replace_database(dsn: str, db_name: str) -> str:
+    """Replace the database path portion of one DSN string."""
+
+    prefix, _, _ = dsn.rpartition("/")
+    return f"{prefix}/{db_name}"
+
+
+def _sha256(path: Path) -> str:
+    """Compute one stable SHA-256 hash for a file."""
+
+    digest = hashlib.sha256()
+    with path.open("rb") as handle:
+        for chunk in iter(lambda: handle.read(1024 * 1024), b""):
+            digest.update(chunk)
+    return digest.hexdigest()
+
+
+def _sanitize_restore_sql(dump_bytes: bytes) -> bytes:
+    """Strip session settings that newer pg_dump versions emit but older servers reject."""
+
+    filtered: list[bytes] = []
+    for line in dump_bytes.splitlines(keepends=True):
+        decoded = line.decode("utf-8", errors="replace").strip()
+        set_match = _UNSUPPORTED_SET_LINE_RE.match(decoded)
+        if set_match and set_match.group(1) in _UNSUPPORTED_RESTORE_PARAMETERS:
+            continue
+        set_config_match = _UNSUPPORTED_SET_CONFIG_RE.match(decoded)
+        if set_config_match and set_config_match.group(1) in _UNSUPPORTED_RESTORE_PARAMETERS:
+            continue
+        filtered.append(line)
+    return b"".join(filtered)

app/periphery/admin/destructive_guard.py

@@ -0,0 +1,32 @@
+"""Shared guardrails for official destructive Shellbrain admin operations."""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+from app.periphery.admin.backup import BackupManifest, create_backup, verify_backup
+
+
+def backup_and_verify_before_destructive_action(
+    *,
+    admin_dsn: str,
+    backup_root: Path,
+    mirror_root: Path | None = None,
+    container_name: str | None = None,
+    container_db_name: str | None = None,
+    container_admin_user: str | None = None,
+    container_admin_password: str | None = None,
+) -> BackupManifest:
+    """Create and verify one backup before an official destructive admin mutation."""
+
+    manifest = create_backup(
+        admin_dsn=admin_dsn,
+        backup_root=backup_root,
+        mirror_root=mirror_root,
+        container_name=container_name,
+        container_db_name=container_db_name,
+        container_admin_user=container_admin_user,
+        container_admin_password=container_admin_password,
+    )
+    verify_backup(backup_root=backup_root, backup_id=manifest.backup_id)
+    return manifest

app/periphery/admin/doctor.py

@@ -0,0 +1,192 @@
+"""Operational diagnostics for Shellbrain safety posture."""
+
+from __future__ import annotations
+
+from datetime import datetime, timezone
+import json
+from pathlib import Path
+import shutil
+from typing import Any
+
+import psycopg
+
+from app.boot.home import get_shellbrain_home
+from app.periphery.admin.backup import list_backups
+from app.periphery.admin.instance_guard import fetch_instance_metadata, inspect_role_safety
+from app.periphery.admin.machine_state import MachineConfig, try_load_machine_config
+from app.periphery.admin.repo_state import IDENTITY_STRENGTH_WEAK_LOCAL, load_repo_registration, resolve_git_root
+
+_LOW_DISK_WARNING_BYTES = 2 * 1024 * 1024 * 1024
+
+
+def build_doctor_report(
+    *,
+    app_dsn: str | None,
+    admin_dsn: str | None,
+    backup_root: Path,
+    repo_root: Path | None = None,
+) -> dict[str, Any]:
+    """Return one structured safety report for the current Shellbrain environment."""
+
+    machine_config, machine_error = try_load_machine_config()
+    instance = None
+    if app_dsn:
+        instance = fetch_instance_metadata(app_dsn)
+    if instance is None and admin_dsn:
+        instance = fetch_instance_metadata(admin_dsn)
+    backups = list_backups(backup_root=backup_root)
+    app_role_warnings = [] if not app_dsn else inspect_role_safety(app_dsn)
+    admin_role_warnings = [] if not admin_dsn else inspect_role_safety(admin_dsn)
+    checked_at = datetime.now(timezone.utc)
+    latest_backup = None if not backups else json.loads(json.dumps(backups[0].__dict__))
+    backup_age_seconds = None
+    if backups:
+        created_at = datetime.fromisoformat(backups[0].created_at.replace("Z", "+00:00"))
+        backup_age_seconds = int((checked_at - created_at.astimezone(timezone.utc)).total_seconds())
+    home_root = get_shellbrain_home()
+    disk = shutil.disk_usage(home_root if home_root.exists() else home_root.parent)
+    repo_report = _build_repo_report(repo_root=repo_root)
+
+    report: dict[str, Any] = {
+        "checked_at": checked_at.isoformat(),
+        "shellbrain_home": str(home_root),
+        "config_status": _config_status(machine_config=machine_config, machine_error=machine_error),
+        "config_error": machine_error,
+        "runtime_mode": None if machine_config is None else machine_config.runtime_mode,
+        "bootstrap_state": None if machine_config is None else machine_config.bootstrap_state,
+        "current_step": None if machine_config is None else machine_config.current_step,
+        "last_error": None if machine_config is None else machine_config.last_error,
+        "config_version": None if machine_config is None else machine_config.config_version,
+        "bootstrap_version": None if machine_config is None else machine_config.bootstrap_version,
+        "machine_instance": _machine_instance_report(machine_config),
+        "effective_config": _effective_config_summary(machine_config=machine_config, app_dsn=app_dsn, admin_dsn=admin_dsn),
+        "app_dsn_configured": bool(app_dsn),
+        "admin_dsn_configured": bool(admin_dsn),
+        "instance": None if instance is None else instance.__dict__,
+        "app_role_warnings": app_role_warnings,
+        "admin_role_warnings": admin_role_warnings,
+        "schema_revision": None if not app_dsn else _fetch_schema_revision(app_dsn),
+        "backup_root": str(backup_root),
+        "backup_count": len(backups),
+        "latest_backup": latest_backup,
+        "backup_age_seconds": backup_age_seconds,
+        "disk_free_bytes": disk.free,
+        "disk_warning": None if disk.free >= _LOW_DISK_WARNING_BYTES else "Low free disk space under Shellbrain home.",
+        "repo": repo_report,
+    }
+    if repo_report and repo_report.get("identity_strength") == IDENTITY_STRENGTH_WEAK_LOCAL:
+        report["repo_warning"] = "Repo identity is weak-local and will change if this directory moves."
+    return report
+
+
+def _build_repo_report(*, repo_root: Path | None) -> dict[str, Any] | None:
+    """Return repo-local registration status when the target looks repo-shaped."""
+
+    if repo_root is None:
+        return None
+    target = Path(repo_root).expanduser().resolve()
+    git_root = resolve_git_root(target)
+    registration_root = git_root or target
+    registration = load_repo_registration(registration_root)
+    if registration is None and git_root is None and not (target / ".shellbrain").exists():
+        return None
+    return {
+        "repo_root": str(target),
+        "git_root": str(git_root) if git_root is not None else None,
+        "registered": registration is not None,
+        "repo_id": None if registration is None else registration.repo_id,
+        "identity_strength": None if registration is None else registration.identity_strength,
+        "source_remote": None if registration is None else registration.source_remote,
+        "machine_instance_id": None if registration is None else registration.machine_instance_id,
+        "registered_at": None if registration is None else registration.registered_at,
+        "claude_status": None if registration is None else registration.claude_status,
+        "claude_settings_path": None if registration is None else registration.claude_settings_path,
+        "claude_note": None if registration is None else registration.claude_note,
+    }
+
+
+def _config_status(*, machine_config: MachineConfig | None, machine_error: str | None) -> str:
+    """Return one short machine config health label."""
+
+    if machine_error:
+        return "corrupt"
+    if machine_config is not None:
+        return "ok"
+    return "absent"
+
+
+def _machine_instance_report(machine_config: MachineConfig | None) -> dict[str, Any] | None:
+    """Return managed-instance details when machine config exists."""
+
+    if machine_config is None:
+        return None
+    return {
+        "instance_id": machine_config.machine_instance_id,
+        "container_name": machine_config.managed.container_name,
+        "image": machine_config.managed.image,
+        "host": machine_config.managed.host,
+        "port": machine_config.managed.port,
+        "db_name": machine_config.managed.db_name,
+        "data_dir": machine_config.managed.data_dir,
+        "backup_root": machine_config.backups.root,
+        "embeddings": {
+            "provider": machine_config.embeddings.provider,
+            "model": machine_config.embeddings.model,
+            "model_revision": machine_config.embeddings.model_revision,
+            "backend_version": machine_config.embeddings.backend_version,
+            "cache_path": machine_config.embeddings.cache_path,
+            "readiness_state": machine_config.embeddings.readiness_state,
+            "last_error": machine_config.embeddings.last_error,
+        },
+    }
+
+
+def _effective_config_summary(
+    *,
+    machine_config: MachineConfig | None,
+    app_dsn: str | None,
+    admin_dsn: str | None,
+) -> dict[str, Any]:
+    """Return a redacted summary of the effective runtime config."""
+
+    if machine_config is not None:
+        return {
+            "source": "machine_config",
+            "app_dsn": _redact_dsn(machine_config.database.app_dsn),
+            "admin_dsn": _redact_dsn(machine_config.database.admin_dsn),
+            "runtime_mode": machine_config.runtime_mode,
+            "backup_root": machine_config.backups.root,
+        }
+    return {
+        "source": "legacy_env",
+        "app_dsn": _redact_dsn(app_dsn),
+        "admin_dsn": _redact_dsn(admin_dsn),
+        "runtime_mode": None,
+        "backup_root": None,
+    }
+
+
+def _redact_dsn(dsn: str | None) -> str | None:
+    """Return a redacted DSN for diagnostics."""
+
+    if not dsn:
+        return None
+    raw = dsn.replace("+psycopg", "")
+    prefix, at_sign, host_part = raw.rpartition("@")
+    if not at_sign:
+        return "<redacted>"
+    user_prefix, _, _ = prefix.partition("://")
+    scheme = prefix.split("://", 1)[0]
+    return f"{scheme}://<redacted>@{host_part}"
+
+
+def _fetch_schema_revision(dsn: str) -> str | None:
+    """Best-effort read of the current alembic revision."""
+
+    try:
+        with psycopg.connect(dsn.replace("+psycopg", "")) as conn:
+            with conn.cursor() as cur:
+                cur.execute("SELECT version_num FROM alembic_version")
+                return str(cur.fetchone()[0])
+    except psycopg.Error:
+        return None