engineering-platform 2.2.0__py3-none-any.whl

engineering_platform/central_data_transfer.py

@@ -0,0 +1,195 @@
+"""Validated export and deferred import of durable platform data.
+
+An export intentionally contains *state*, not the installed Python runtime or
+live process files. This lets a clean installation on another Mac retain its
+own qualified runtime while receiving the complete CENTRAL state.
+"""
+from __future__ import annotations
+
+from datetime import datetime, timezone
+import hashlib
+import json
+import os
+from pathlib import Path, PurePosixPath
+import shutil
+import tempfile
+from uuid import uuid4
+import zipfile
+
+from . import central_database
+
+
+FORMAT_VERSION = 1
+MANIFEST_NAME = "central-data-manifest.json"
+EXCLUDED_TOP_LEVEL = frozenset({"runtime"})
+MAX_ARCHIVE_BYTES = 2 * 1024 * 1024 * 1024
+MAX_ARCHIVE_MEMBERS = 100_000
+
+
+class CentralDataTransferError(ValueError):
+    """An archive is malformed, unsafe, or unavailable for transfer."""
+
+
+def _sha256(content: bytes) -> str:
+    return hashlib.sha256(content).hexdigest()
+
+
+def _safe_member(name: str) -> PurePosixPath:
+    path = PurePosixPath(name)
+    if not name or path.is_absolute() or ".." in path.parts or path.parts[0] in EXCLUDED_TOP_LEVEL:
+        raise CentralDataTransferError("CENTRAL_ARCHIVE_MEMBER_INVALID")
+    return path
+
+
+def _durable_files(data_root: Path) -> list[tuple[PurePosixPath, Path]]:
+    """List durable files by logical path, dereferencing legacy local links."""
+    root = data_root.resolve()
+    files: list[tuple[PurePosixPath, Path]] = []
+    for top_level in sorted(root.iterdir(), key=lambda item: item.name):
+        if top_level.name in EXCLUDED_TOP_LEVEL:
+            continue
+        logical = PurePosixPath(top_level.name)
+        resolved = top_level.resolve()
+        if resolved.is_file():
+            files.append((logical, resolved))
+        elif resolved.is_dir():
+            for item in sorted(resolved.rglob("*")):
+                if item.is_file():
+                    files.append((logical / item.relative_to(resolved).as_posix(), item.resolve()))
+    return files
+
+
+def export_snapshot(data_root: Path) -> tuple[str, bytes]:
+    """Create a portable immutable archive while Server writers are quiesced."""
+    root = data_root.resolve()
+    database = central_database.snapshot(root)
+    if database is None:
+        raise CentralDataTransferError("CENTRAL_DATABASE_UNAVAILABLE")
+    entries: list[dict[str, object]] = []
+    with tempfile.SpooledTemporaryFile(max_size=32 * 1024 * 1024, mode="w+b") as output:
+        with zipfile.ZipFile(output, "w", compression=zipfile.ZIP_DEFLATED, compresslevel=9) as archive:
+            for logical, source in _durable_files(root):
+                content = database if logical.as_posix() == "engineering.db" else source.read_bytes()
+                archive.writestr(logical.as_posix(), content)
+                entries.append({"path": logical.as_posix(), "sha256": _sha256(content), "size": len(content)})
+            manifest = {
+                "format_version": FORMAT_VERSION,
+                "created_at": datetime.now(timezone.utc).isoformat(),
+                "kind": "engineering-platform-central-data",
+                "entries": entries,
+            }
+            archive.writestr(MANIFEST_NAME, json.dumps(manifest, sort_keys=True, separators=(",", ":")))
+        output.seek(0)
+        content = output.read()
+    filename = f"engineering-platform-central-data-{datetime.now(timezone.utc).strftime('%Y%m%dT%H%M%SZ')}.zip"
+    return filename, content
+
+
+def inspect_archive(path: Path) -> dict[str, object]:
+    """Validate an archive completely before it can become a pending import."""
+    if not path.is_file() or path.stat().st_size > MAX_ARCHIVE_BYTES:
+        raise CentralDataTransferError("CENTRAL_ARCHIVE_SIZE_INVALID")
+    try:
+        with zipfile.ZipFile(path) as archive:
+            infos = archive.infolist()
+            if len(infos) > MAX_ARCHIVE_MEMBERS or any(item.is_dir() for item in infos):
+                raise CentralDataTransferError("CENTRAL_ARCHIVE_MEMBER_INVALID")
+            names = {item.filename for item in infos}
+            if MANIFEST_NAME not in names or len(names) != len(infos):
+                raise CentralDataTransferError("CENTRAL_ARCHIVE_MANIFEST_INVALID")
+            manifest = json.loads(archive.read(MANIFEST_NAME).decode("utf-8"))
+            if not isinstance(manifest, dict) or manifest.get("format_version") != FORMAT_VERSION or manifest.get("kind") != "engineering-platform-central-data":
+                raise CentralDataTransferError("CENTRAL_ARCHIVE_MANIFEST_INVALID")
+            entries = manifest.get("entries")
+            if not isinstance(entries, list) or not entries:
+                raise CentralDataTransferError("CENTRAL_ARCHIVE_MANIFEST_INVALID")
+            expected: dict[str, dict[str, object]] = {}
+            for entry in entries:
+                if not isinstance(entry, dict) or not isinstance(entry.get("path"), str) or not isinstance(entry.get("sha256"), str) or not isinstance(entry.get("size"), int):
+                    raise CentralDataTransferError("CENTRAL_ARCHIVE_MANIFEST_INVALID")
+                name = _safe_member(entry["path"]).as_posix()
+                expected[name] = entry
+            if set(expected) != names - {MANIFEST_NAME} or "engineering.db" not in expected:
+                raise CentralDataTransferError("CENTRAL_ARCHIVE_MANIFEST_INVALID")
+            for name, entry in expected.items():
+                content = archive.read(name)
+                if len(content) != entry["size"] or _sha256(content) != entry["sha256"]:
+                    raise CentralDataTransferError("CENTRAL_ARCHIVE_INTEGRITY_INVALID")
+    except (OSError, zipfile.BadZipFile, UnicodeDecodeError, json.JSONDecodeError) as error:
+        raise CentralDataTransferError("CENTRAL_ARCHIVE_INVALID") from error
+    return {"entries": len(expected), "size_bytes": path.stat().st_size}
+
+
+def stage_import(data_root: Path, source: Path) -> dict[str, object]:
+    """Copy a validated upload into runtime-owned staging for a clean restart."""
+    root = data_root.resolve()
+    imports = root / "runtime" / "central-data-imports"
+    imports.mkdir(mode=0o700, parents=True, exist_ok=True)
+    target = imports / f"{uuid4().hex}.zip"
+    shutil.copyfile(source, target)
+    target.chmod(0o600)
+    details = inspect_archive(target)
+    pending = root / "runtime" / "pending-central-data-import.json"
+    if pending.exists():
+        target.unlink(missing_ok=True)
+        raise CentralDataTransferError("CENTRAL_IMPORT_ALREADY_PENDING")
+    temporary = pending.with_name(f".{pending.name}.{uuid4().hex}")
+    try:
+        temporary.write_text(json.dumps({"archive": str(target)}, sort_keys=True), encoding="utf-8")
+        temporary.chmod(0o600)
+        os.replace(temporary, pending)
+    finally:
+        temporary.unlink(missing_ok=True)
+    return details
+
+
+def apply_pending_import(data_root: Path) -> dict[str, object] | None:
+    """Replace durable state before initialization starts any writer.
+
+    The current runtime stays in place, which is essential when restoring on a
+    clean host with its own compatible EP installation.
+    """
+    root = data_root.resolve()
+    pending = root / "runtime" / "pending-central-data-import.json"
+    if not pending.exists():
+        return None
+    try:
+        payload = json.loads(pending.read_text(encoding="utf-8"))
+        archive_path = Path(payload["archive"])
+        imports = (root / "runtime" / "central-data-imports").resolve()
+        if imports not in archive_path.resolve().parents:
+            raise CentralDataTransferError("CENTRAL_IMPORT_REQUEST_INVALID")
+        details = inspect_archive(archive_path)
+        staging = Path(tempfile.mkdtemp(prefix=".central-data-import-", dir=root.parent))
+        retired = root.with_name(f".{root.name}.pre-import-{uuid4().hex}")
+        try:
+            with zipfile.ZipFile(archive_path) as archive:
+                manifest = json.loads(archive.read(MANIFEST_NAME).decode("utf-8"))
+                for entry in manifest["entries"]:
+                    logical = _safe_member(entry["path"])
+                    target = staging.joinpath(*logical.parts)
+                    target.parent.mkdir(mode=0o700, parents=True, exist_ok=True)
+                    target.write_bytes(archive.read(logical.as_posix()))
+                    target.chmod(0o600)
+            # Switch a fully prepared replacement directory, then transplant
+            # only the target host's installed runtime. A failure restores the
+            # original root rather than leaving a partially copied state.
+            os.replace(root, retired)
+            try:
+                os.replace(staging, root)
+                os.replace(retired / "runtime", root / "runtime")
+            except Exception:
+                if root.exists():
+                    if (root / "runtime").exists() and not (retired / "runtime").exists():
+                        os.replace(root / "runtime", retired / "runtime")
+                    shutil.rmtree(root)
+                os.replace(retired, root)
+                raise
+            shutil.rmtree(retired)
+        finally:
+            shutil.rmtree(staging, ignore_errors=True)
+    except (KeyError, TypeError, json.JSONDecodeError) as error:
+        raise CentralDataTransferError("CENTRAL_IMPORT_REQUEST_INVALID") from error
+    pending.unlink(missing_ok=True)
+    archive_path.unlink(missing_ok=True)
+    return details

engineering_platform/central_database.py

@@ -0,0 +1,245 @@
+"""Installation-owned CENTRAL database inspection, backup, and maintenance."""
+
+from __future__ import annotations
+
+from datetime import datetime, timedelta, timezone
+import json
+import os
+from pathlib import Path
+import sqlite3
+import tempfile
+
+
+DATABASE_FILENAME = "engineering.db"
+MAINTENANCE_INTERVAL_KEY = "central_database.maintenance_interval_seconds"
+MAINTENANCE_LAST_ATTEMPT_KEY = "central_database.maintenance_last_attempt_at"
+PROVIDER_CAPACITY_HISTORY_KEY = "ep.provider_capacity_history.v1"
+CODEX_CAPACITY_RESERVE_KEY = "ep.codex_capacity_reserve_percent"
+DEFAULT_MAINTENANCE_INTERVAL_SECONDS = 60 * 60
+MAINTENANCE_INTERVAL_OPTIONS = frozenset({60, 60 * 60, 24 * 60 * 60, 7 * 24 * 60 * 60})
+CODEX_CAPACITY_RESERVE_OPTIONS = frozenset({0, 5, 10, 15, 20, 25, 50, 75})
+CONSOLE_CONFIGURATION_OPTIONS = {
+    "log_retention_days": frozenset({30, 60, 90, 120, 180, 360}), "telemetry_retention_days": frozenset({30, 60, 90, 120, 180, 360}),
+    "log_level": frozenset({"INFO", "DEBUG"}), "inbox_scan_interval_seconds": frozenset({5, 15, 30, 60}),
+    "open_pr_check_interval_seconds": frozenset({30, 60}), "dashboard_stream_interval_seconds": frozenset(range(1, 11)),
+    "provider_readiness_refresh_seconds": frozenset({60, 300, 600}), "platform_health_refresh_seconds": frozenset({5, 15, 30, 60}),
+    "component_details_refresh_seconds": frozenset({5, 15, 30, 60}),
+}
+CONSOLE_CONFIGURATION_DEFAULTS = {"log_retention_days":30,"telemetry_retention_days":90,"log_level":"INFO","inbox_scan_interval_seconds":15,"open_pr_check_interval_seconds":30,"dashboard_stream_interval_seconds":1,"provider_readiness_refresh_seconds":300,"platform_health_refresh_seconds":15,"component_details_refresh_seconds":5}
+
+
+def path(data_root: Path) -> Path:
+    return data_root.resolve() / DATABASE_FILENAME
+
+
+def _schema_version(connection: sqlite3.Connection) -> int:
+    row = connection.execute("SELECT MAX(version) FROM engineering_schema_migrations").fetchone()
+    return int(row[0]) if row and row[0] is not None else 0
+
+
+def details(data_root: Path) -> dict[str, object]:
+    """Read CENTRAL identity facts without creating or mutating it."""
+    database = path(data_root)
+    result: dict[str, object] = {"path": str(database), "size_bytes": 0, "schema_version": 0, "integrity": "UNAVAILABLE"}
+    try:
+        result["size_bytes"] = database.stat().st_size
+        with sqlite3.connect(f"file:{database}?mode=ro", uri=True) as connection:
+            result["schema_version"] = _schema_version(connection)
+            result["integrity"] = "PASS" if [str(row[0]) for row in connection.execute("PRAGMA integrity_check")] == ["ok"] else "FAILED"
+    except (OSError, sqlite3.DatabaseError):
+        pass
+    return result
+
+
+def snapshot(data_root: Path) -> bytes | None:
+    """Return a consistent, read-only backup of the one CENTRAL database."""
+    database = path(data_root)
+    if not database.is_file():
+        return None
+    temporary_path: Path | None = None
+    try:
+        with tempfile.NamedTemporaryFile(prefix="ep-central-backup-", suffix=".db", delete=False) as temporary:
+            temporary_path = Path(temporary.name)
+        with sqlite3.connect(f"file:{database}?mode=ro", uri=True) as source, sqlite3.connect(temporary_path) as backup:
+            source.backup(backup)
+        return temporary_path.read_bytes()
+    except (OSError, sqlite3.DatabaseError):
+        return None
+    finally:
+        if temporary_path is not None:
+            temporary_path.unlink(missing_ok=True)
+
+
+def maintenance_configuration(data_root: Path) -> dict[str, int]:
+    try:
+        with sqlite3.connect(f"file:{path(data_root)}?mode=ro", uri=True) as connection:
+            row = connection.execute("SELECT value FROM engineering_metadata WHERE key=?", (MAINTENANCE_INTERVAL_KEY,)).fetchone()
+    except (OSError, sqlite3.DatabaseError):
+        return {"interval_seconds": DEFAULT_MAINTENANCE_INTERVAL_SECONDS}
+    try:
+        value = int(json.loads(row[0])) if row else DEFAULT_MAINTENANCE_INTERVAL_SECONDS
+    except (TypeError, ValueError, json.JSONDecodeError):
+        value = DEFAULT_MAINTENANCE_INTERVAL_SECONDS
+    return {"interval_seconds": value if value in MAINTENANCE_INTERVAL_OPTIONS else DEFAULT_MAINTENANCE_INTERVAL_SECONDS}
+
+
+def update_maintenance_configuration(data_root: Path, interval_seconds: object) -> dict[str, int]:
+    if not isinstance(interval_seconds, int) or isinstance(interval_seconds, bool) or interval_seconds not in MAINTENANCE_INTERVAL_OPTIONS:
+        raise ValueError("CENTRAL_DATABASE_MAINTENANCE_INTERVAL_INVALID")
+    with sqlite3.connect(path(data_root)) as connection:
+        previous = maintenance_configuration(data_root)["interval_seconds"]
+        connection.execute("INSERT INTO engineering_metadata(key,value) VALUES(?,?) ON CONFLICT(key) DO UPDATE SET value=excluded.value", (MAINTENANCE_INTERVAL_KEY, json.dumps(interval_seconds)))
+    return {"previous": previous, "interval_seconds": interval_seconds}
+
+
+def capacity_configuration(data_root: Path) -> dict[str, int]:
+    """Return the one installation-wide admission reserve for Codex capacity."""
+    try:
+        with sqlite3.connect(f"file:{path(data_root)}?mode=ro", uri=True) as connection:
+            row = connection.execute("SELECT value FROM engineering_metadata WHERE key=?", (CODEX_CAPACITY_RESERVE_KEY,)).fetchone()
+        value = int(json.loads(row[0])) if row else 0
+    except (OSError, sqlite3.DatabaseError, TypeError, ValueError, json.JSONDecodeError):
+        value = 0
+    return {"codex_capacity_reserve_percent": value if value in CODEX_CAPACITY_RESERVE_OPTIONS else 0}
+
+
+def capacity_reserve_from_environment() -> int:
+    """Resolve the active Server's platform policy for worker-side admission."""
+    configured_root = os.environ.get("EP_SERVER_DATA_ROOT")
+    if not configured_root:
+        return 0
+    return capacity_configuration(Path(configured_root))["codex_capacity_reserve_percent"]
+
+
+def update_capacity_configuration(data_root: Path, reserve_percent: object) -> dict[str, int]:
+    """Persist an EP-owned reserve; projects cannot choose different limits."""
+    if not isinstance(reserve_percent, int) or isinstance(reserve_percent, bool) or reserve_percent not in CODEX_CAPACITY_RESERVE_OPTIONS:
+        raise ValueError("CODEX_CAPACITY_RESERVE_INVALID")
+    previous = capacity_configuration(data_root)["codex_capacity_reserve_percent"]
+    with sqlite3.connect(path(data_root)) as connection:
+        connection.execute(
+            "INSERT INTO engineering_metadata(key,value) VALUES(?,?) ON CONFLICT(key) DO UPDATE SET value=excluded.value",
+            (CODEX_CAPACITY_RESERVE_KEY, json.dumps(reserve_percent)),
+        )
+    return {"previous": previous, "codex_capacity_reserve_percent": reserve_percent}
+
+
+def console_interval_configuration(data_root: Path) -> dict[str, object]:
+    result = dict(CONSOLE_CONFIGURATION_DEFAULTS)
+    try:
+        with sqlite3.connect(f"file:{path(data_root)}?mode=ro", uri=True) as connection:
+            for key, value in connection.execute("SELECT key,value FROM engineering_metadata WHERE key LIKE 'console.%'"):
+                name = str(key).removeprefix("console.")
+                parsed = json.loads(value)
+                if name in result and parsed in CONSOLE_CONFIGURATION_OPTIONS[name]: result[name] = parsed
+    except (OSError, sqlite3.DatabaseError, TypeError, ValueError, json.JSONDecodeError):
+        pass
+    return result
+
+
+def update_console_interval_configuration(data_root: Path, key: object, value: object) -> dict[str, object]:
+    if not isinstance(key, str) or key not in CONSOLE_CONFIGURATION_OPTIONS or value not in CONSOLE_CONFIGURATION_OPTIONS[key]:
+        raise ValueError("CONSOLE_CONFIGURATION_INVALID")
+    previous = console_interval_configuration(data_root)[key]
+    with sqlite3.connect(path(data_root)) as connection:
+        connection.execute("INSERT INTO engineering_metadata(key,value) VALUES(?,?) ON CONFLICT(key) DO UPDATE SET value=excluded.value", ("console." + key, json.dumps(value)))
+    return {"key": key, "previous": previous, "value": value}
+
+
+def record_provider_capacity(
+    data_root: Path, *, provider: str, remaining_percent: float, observed_at: datetime | None = None,
+) -> list[dict[str, object]]:
+    """Store a bounded, account-wide two-hour capacity series in CENTRAL."""
+    provider = provider.strip()[:120]
+    try:
+        remaining = float(remaining_percent)
+    except (TypeError, ValueError):
+        return []
+    if not provider or not 0 <= remaining <= 100:
+        return []
+    timestamp = (observed_at or datetime.now(timezone.utc)).astimezone(timezone.utc)
+    bucket = timestamp.replace(hour=timestamp.hour - timestamp.hour % 2, minute=0, second=0, microsecond=0)
+    cutoff = bucket - timedelta(days=7)
+    try:
+        with sqlite3.connect(path(data_root)) as connection:
+            row = connection.execute("SELECT value FROM engineering_metadata WHERE key=?", (PROVIDER_CAPACITY_HISTORY_KEY,)).fetchone()
+            try:
+                payload = json.loads(row[0]) if row else {}
+            except (TypeError, ValueError, json.JSONDecodeError):
+                payload = {}
+            providers = payload.get("providers") if isinstance(payload, dict) else None
+            providers = providers if isinstance(providers, dict) else {}
+            samples = providers.get(provider)
+            samples = samples if isinstance(samples, dict) else {}
+            key = bucket.isoformat()
+            current = samples.get(key)
+            if isinstance(current, (int, float)) and not isinstance(current, bool):
+                remaining = min(remaining, float(current))
+            samples[key] = remaining
+            filtered: dict[str, float] = {}
+            for sample_at, sample_value in samples.items():
+                try:
+                    parsed = datetime.fromisoformat(str(sample_at)).astimezone(timezone.utc)
+                except ValueError:
+                    continue
+                if parsed >= cutoff and isinstance(sample_value, (int, float)) and not isinstance(sample_value, bool) and 0 <= float(sample_value) <= 100:
+                    filtered[str(sample_at)] = float(sample_value)
+            providers[provider] = filtered
+            connection.execute(
+                "INSERT INTO engineering_metadata(key,value) VALUES(?,?) ON CONFLICT(key) DO UPDATE SET value=excluded.value",
+                (PROVIDER_CAPACITY_HISTORY_KEY, json.dumps({"providers": providers}, separators=(",", ":"))),
+            )
+    except (OSError, sqlite3.DatabaseError):
+        return []
+    return provider_capacity_history(data_root, provider=provider)
+
+
+def provider_capacity_history(data_root: Path, *, provider: str, hours: int = 168) -> list[dict[str, object]]:
+    """Read CENTRAL-only provider capacity evidence; it is never project data."""
+    provider = provider.strip()[:120]
+    if not provider or hours < 1:
+        return []
+    try:
+        with sqlite3.connect(f"file:{path(data_root)}?mode=ro", uri=True) as connection:
+            row = connection.execute("SELECT value FROM engineering_metadata WHERE key=?", (PROVIDER_CAPACITY_HISTORY_KEY,)).fetchone()
+        payload = json.loads(row[0]) if row else {}
+    except (OSError, sqlite3.DatabaseError, TypeError, ValueError, json.JSONDecodeError):
+        return []
+    samples = payload.get("providers", {}).get(provider, {}) if isinstance(payload, dict) and isinstance(payload.get("providers"), dict) else {}
+    cutoff = datetime.now(timezone.utc) - timedelta(hours=hours)
+    result: list[dict[str, object]] = []
+    if not isinstance(samples, dict):
+        return result
+    for observed_at, remaining in sorted(samples.items()):
+        try:
+            parsed = datetime.fromisoformat(str(observed_at)).astimezone(timezone.utc)
+        except ValueError:
+            continue
+        if parsed >= cutoff and isinstance(remaining, (int, float)) and not isinstance(remaining, bool) and 0 <= float(remaining) <= 100:
+            result.append({"at": str(observed_at), "remaining_percent": float(remaining)})
+    return result
+
+
+def run_periodic_maintenance(data_root: Path, *, now: datetime | None = None) -> dict[str, object]:
+    """Compact CENTRAL only while no lifecycle is active; never touch project stores."""
+    moment = (now or datetime.now(timezone.utc)).astimezone(timezone.utc)
+    interval = maintenance_configuration(data_root)["interval_seconds"]
+    try:
+        with sqlite3.connect(path(data_root)) as connection:
+            row = connection.execute("SELECT value FROM engineering_metadata WHERE key=?", (MAINTENANCE_LAST_ATTEMPT_KEY,)).fetchone()
+            try:
+                previous = datetime.fromisoformat(json.loads(row[0]).replace("Z", "+00:00")) if row else None
+            except (TypeError, ValueError, json.JSONDecodeError):
+                previous = None
+            if previous is not None and moment - previous.astimezone(timezone.utc) < timedelta(seconds=interval):
+                return {"state": "NOT_DUE"}
+            active = connection.execute("SELECT 1 FROM ep_parity_lifecycle_dispatches WHERE state IN ('CLAIMED','RUNNING') LIMIT 1").fetchone()
+            if active:
+                return {"state": "SKIPPED_ACTIVE_RUN"}
+            connection.execute("PRAGMA busy_timeout=1000")
+            connection.execute("PRAGMA optimize")
+            connection.execute("VACUUM")
+            connection.execute("INSERT INTO engineering_metadata(key,value) VALUES(?,?) ON CONFLICT(key) DO UPDATE SET value=excluded.value", (MAINTENANCE_LAST_ATTEMPT_KEY, json.dumps(moment.isoformat())))
+        return {"state": "COMPACTED"}
+    except (OSError, sqlite3.DatabaseError):
+        return {"state": "DEFERRED"}