imagen-cli 0.1.1__py3-none-any.whl

imagen_cli/__main__.py

@@ -0,0 +1,3 @@
+from imagen_cli.cli.entrypoint import main
+
+raise SystemExit(main())

imagen_cli/application/approval_service.py

@@ -0,0 +1,36 @@
+"""Approval lifecycle facade: pending, approved, consumed, expired, invalidated."""
+
+from typing import Any
+
+from imagen_cli.application.planning_service import PlanningService
+from imagen_cli.storage.repositories import Repositories
+
+__all__ = ["ApprovalService"]
+
+
+class ApprovalService:
+    def __init__(self, repositories: Repositories) -> None:
+        self.repositories = repositories
+        self.planning = PlanningService(repositories)
+
+    def approve(self, plan_id: str, digest: str) -> dict[str, Any]:
+        return self.planning.approve(plan_id, digest).model_dump()
+
+    def state(self, plan_id: str) -> str:
+        rows = self.repositories.list("approvals", plan_id=plan_id)
+        if not rows:
+            raise KeyError(plan_id)
+        return str(rows[0]["state"])
+
+    def invalidate(self, plan_id: str, reason: str) -> str:
+        """Only an unconsumed approval may be invalidated; execution rights never return."""
+        rows = self.repositories.list("approvals", plan_id=plan_id)
+        if not rows:
+            raise KeyError(plan_id)
+        approval = rows[0]
+        if approval["state"] == "consumed":
+            raise ValueError("consumed approval is immutable")
+        self.repositories.update(
+            "approvals", approval["id"], {"state": "invalidated"},
+        )
+        return "invalidated"

imagen_cli/application/artifact_service.py

@@ -0,0 +1,97 @@
+"""Save existing provider output through the shared durable artifact store."""
+
+from collections.abc import Callable
+from typing import Any
+
+from imagen_cli.domain.primitive import new_id
+from imagen_cli.providers.base import ArtifactDescriptor
+from imagen_cli.storage.artifacts import ArtifactStore
+from imagen_cli.storage.repositories import Repositories
+
+
+class ArtifactService:
+    def __init__(self, repositories: Repositories, artifact_store: ArtifactStore) -> None:
+        self.repositories = repositories
+        self.artifact_store = artifact_store
+
+    def _job_turn(self, job_id: str) -> tuple[dict[str, Any], dict[str, Any]]:
+        job = self.repositories.get("jobs", job_id)
+        if job is None:
+            raise KeyError(job_id)
+        turn = self.repositories.get("turns", job["turn_id"])
+        if turn is None or turn.get("job_id") != job_id:
+            raise ValueError("job must belong to its producer turn")
+        return job, turn
+
+    def _reference(self, row: dict[str, Any]) -> None:
+        _, turn = self._job_turn(row["producer_job_id"])
+        index = row["origin_manifest"]["output_index"]
+        refs = self.repositories.list(
+            "artifact_references",
+            turn_id=turn["id"],
+            role="output",
+            index=index,
+        )
+        if refs:
+            if refs[0]["artifact_id"] != row["id"]:
+                raise ValueError("output slot already occupied")
+            return
+        self.repositories.insert(
+            "artifact_references",
+            {
+                "session_id": turn["session_id"],
+                "turn_id": turn["id"],
+                "artifact_id": row["id"],
+                "role": "output",
+                "index": index,
+            },
+        )
+
+    def save(
+        self,
+        job_id: str,
+        data: bytes,
+        *,
+        artifact_id: str | None = None,
+        index: int | None = None,
+    ) -> dict[str, Any]:
+        _, turn = self._job_turn(job_id)
+        with self.repositories.transaction():
+            refs = self.repositories.list("artifact_references", turn_id=turn["id"], role="output")
+            if index is None:
+                index = max((ref["index"] for ref in refs), default=-1) + 1
+            if type(index) is not int or index < 0:
+                raise ValueError("invalid output index")
+            if any(ref["index"] == index for ref in refs):
+                raise ValueError("output slot already occupied")
+        row = self.artifact_store.stage(
+            artifact_id or new_id("art"),
+            data,
+            producer_job_id=job_id,
+            producer_turn_id=turn["id"],
+            origin_manifest={"kind": "provider_output", "output_index": index},
+        )
+        self._reference(row)
+        return dict(self.artifact_store.publish(row["id"]))
+
+    def fetch(
+        self,
+        job_id: str,
+        descriptor: ArtifactDescriptor,
+        fetch: Callable[[ArtifactDescriptor], bytes],
+    ) -> dict[str, Any]:
+        # Authorization and bounded retrieval belong to the injected provider/download boundary.
+        self._job_turn(job_id)
+        return self.save(
+            job_id,
+            fetch(descriptor),
+            artifact_id=descriptor.id,
+            index=descriptor.index,
+        )
+
+    def reconcile(self, artifact_id: str) -> dict[str, Any]:
+        row = self.repositories.get("artifacts", artifact_id)
+        if row is None:
+            raise KeyError(artifact_id)
+        self._reference(row)
+        return dict(self.artifact_store.reconcile(artifact_id))

imagen_cli/application/batch_service.py

@@ -0,0 +1,190 @@
+"""Bounded batch orchestration over ordinary child jobs; children are never rewritten."""
+
+from collections.abc import Callable, Mapping, Sequence
+from concurrent.futures import ThreadPoolExecutor
+from datetime import UTC, datetime
+from typing import Any
+
+from imagen_cli.application.batch_summary import summarize
+from imagen_cli.domain.contracts import CreationInput
+from imagen_cli.domain.primitive import new_id
+from imagen_cli.storage.repositories import Repositories
+
+DEFAULT_CONCURRENCY = 2
+MAX_CONCURRENCY = 8
+MAX_REQUESTS = 100
+
+Prepare = Callable[[CreationInput], Any]
+RunChild = Callable[[Any, int], str]
+ProviderLimit = Callable[[Any], int | None]
+
+
+def effective_concurrency(
+    requested: int | None, provider_limits: Sequence[int | None]
+) -> int:
+    """Requested concurrency capped at 8 and never above a declared provider limit."""
+    value = DEFAULT_CONCURRENCY if requested is None else requested
+    if type(value) is not int or not 1 <= value <= MAX_CONCURRENCY:
+        raise ValueError(f"concurrency must be 1-{MAX_CONCURRENCY}")
+    limits: list[int] = []
+    for limit in provider_limits:
+        if limit is None:
+            continue
+        if type(limit) is not int or limit < 1:
+            raise ValueError("provider limit must be positive")
+        limits.append(limit)
+    return min([value, *limits])
+
+
+def _now() -> str:
+    return datetime.now(UTC).isoformat().replace("+00:00", "Z")
+
+
+def _rows(repositories: Repositories, table: str, **filters: Any) -> list[dict[str, Any]]:
+    rows: list[dict[str, Any]] = []
+    while True:
+        page = repositories.list(table, limit=1000, offset=len(rows), **filters)
+        rows.extend(page)
+        if len(page) < 1000:
+            return rows
+
+
+class BatchService:
+    def __init__(
+        self,
+        repositories: Repositories,
+        *,
+        prepare: Prepare | None = None,
+        run: RunChild | None = None,
+        provider_limit: ProviderLimit | None = None,
+    ) -> None:
+        self.repositories = repositories
+        self.prepare = prepare or (lambda creation: creation)
+        self.run = run
+        self.provider_limit = provider_limit
+
+    def submit(
+        self,
+        requests: Sequence[CreationInput | Mapping[str, Any]],
+        concurrency: int | None = None,
+    ) -> tuple[str, dict[str, Any]]:
+        if isinstance(requests, (str, bytes)) or not 1 <= len(requests) <= MAX_REQUESTS:
+            raise ValueError(f"batch requires 1-{MAX_REQUESTS} requests")
+        creations = [
+            request
+            if isinstance(request, CreationInput)
+            else CreationInput.model_validate(request)
+            for request in requests
+        ]
+        # Every child is validated and planned offline before any provider call.
+        prepared = [self.prepare(creation) for creation in creations]
+        limits = [
+            self.provider_limit(item) if self.provider_limit is not None else None
+            for item in prepared
+        ]
+        effective = effective_concurrency(concurrency, limits)
+        if self.run is None:
+            raise ValueError("batch execution is unavailable")
+        with ThreadPoolExecutor(max_workers=effective) as pool:
+            futures = [
+                pool.submit(self.run, item, index) for index, item in enumerate(prepared)
+            ]
+            # Read in request order so child identities stay ordered regardless of finish order.
+            job_ids = [future.result() for future in futures]
+        if len(set(job_ids)) != len(job_ids):
+            raise ValueError("batch child jobs must be unique")
+        identity = new_id("bat")
+        with self.repositories.transaction():
+            summary = summarize(self._facts(job_ids))
+            self.repositories.insert(
+                "batches",
+                {
+                    "id": identity,
+                    "child_job_ids": job_ids,
+                    "concurrency": effective,
+                    "summary_state": summary["state"],
+                    "created_at": _now(),
+                },
+            )
+        return self._status(summary["state"]), self._data(identity)
+
+    def facts(self, batch_id: str) -> tuple[dict[str, Any], list[dict[str, Any]]]:
+        batch = self.repositories.get("batches", batch_id)
+        if batch is None:
+            raise KeyError(batch_id)
+        return batch, self._facts(batch.get("child_job_ids", []))
+
+    def get(self, batch_id: str) -> tuple[str, dict[str, Any]]:
+        if self.repositories.get("batches", batch_id) is None:
+            return "not_found", {}
+        return "succeeded", self._data(batch_id)
+
+    def wait(self, batch_id: str, *, timeout: int = 300) -> tuple[str, dict[str, Any]]:
+        if type(timeout) is not int or not 1 <= timeout <= 86400:
+            raise ValueError("invalid timeout")
+        if self.repositories.get("batches", batch_id) is None:
+            return "not_found", {}
+        import time
+
+        deadline = time.monotonic() + timeout
+        while True:
+            data = self._data(batch_id)
+            state = data["summary"]["state"]
+            if state != "running" or time.monotonic() >= deadline:
+                break
+            time.sleep(min(0.05, max(0.0, deadline - time.monotonic())))
+        state = data["summary"]["state"]
+        if state == "running":
+            return "wait_timeout", data
+        return self._status(state), data
+
+    def _child(self, job_id: str) -> dict[str, Any]:
+        job = self.repositories.get("jobs", job_id)
+        if job is None:
+            raise ValueError("batch child job is missing")
+        saved = sum(
+            1
+            for row in _rows(self.repositories, "artifacts", producer_job_id=job_id)
+            if row.get("save_state") == "published"
+        )
+        return {
+            "job_id": job_id,
+            "state": job.get("state"),
+            "remote_state": job.get("remote_state"),
+            "local_save_state": job.get("local_save_state"),
+            "actual_count": job.get("actual_count", 0),
+            "requested_count": job.get("requested_count", 0),
+            "saved_artifact_count": saved,
+        }
+
+    def _facts(self, job_ids: Sequence[str]) -> list[dict[str, Any]]:
+        return [self._child(job_id) for job_id in job_ids]
+
+    def _data(self, batch_id: str) -> dict[str, Any]:
+        batch, children = self.facts(batch_id)
+        summary = summarize(children)
+        self.repositories.update("batches", batch_id, {"summary_state": summary["state"]})
+        events: list[dict[str, Any]] = []
+        for child in children:
+            for row in _rows(self.repositories, "events", job_id=child["job_id"]):
+                events.append(
+                    {
+                        "job_id": row["job_id"],
+                        "job_seq": row["job_seq"],
+                        "event_type": row["event_type"],
+                        "timestamp": row["timestamp"],
+                        "safe_payload": row.get("safe_payload", {}),
+                    }
+                )
+        return {
+            "batch": self.repositories.get("batches", batch_id),
+            "summary": summary,
+            "children": children,
+            "events": events,
+        }
+
+    @staticmethod
+    def _status(state: str) -> str:
+        if state == "running":
+            return "wait_timeout"
+        return state

imagen_cli/application/batch_summary.py

@@ -0,0 +1,66 @@
+"""Truthful batch aggregates; child facts stay untouched and unknown is not terminal."""
+
+from collections.abc import Mapping, Sequence
+from typing import Any
+
+_ACTIVE = {"queued", "running"}
+
+
+def _complete(child: Mapping[str, Any]) -> bool:
+    saved = child.get("saved_artifact_count", 0)
+    actual = child.get("actual_count", 0)
+    return (
+        child.get("state") == "succeeded"
+        and child.get("remote_state") == "succeeded"
+        and child.get("local_save_state") == "saved"
+        and type(saved) is int
+        and type(actual) is int
+        and actual > 0
+        and saved >= actual
+    )
+
+
+def _partial(child: Mapping[str, Any]) -> bool:
+    saved = child.get("saved_artifact_count", 0)
+    return (
+        not _complete(child)
+        and (child.get("local_save_state") == "partial" or (type(saved) is int and saved > 0))
+    )
+
+
+def summarize(children: Sequence[Mapping[str, Any]]) -> dict[str, Any]:
+    """Apply the six-step priority; success counts jobs, saved count counts files."""
+    complete_count = sum(1 for child in children if _complete(child))
+    saved_count = sum(
+        child.get("saved_artifact_count", 0)
+        for child in children
+        if type(child.get("saved_artifact_count", 0)) is int
+    )
+    unknown = any(
+        child.get("state") == "unknown" or child.get("remote_state") == "unknown"
+        for child in children
+    )
+    partial = any(_partial(child) for child in children)
+    if any(child.get("state") in _ACTIVE for child in children):
+        state = "running"
+    elif children and complete_count == len(children):
+        state = "succeeded"
+    elif complete_count:
+        # Any non-complete child after the active check leaves a mixed outcome.
+        state = "partial_success"
+    elif partial:
+        state = "unknown" if unknown else "partial_success"
+    elif unknown:
+        state = "unknown"
+    elif any(child.get("state") == "failed" for child in children):
+        state = "failed"
+    elif children and all(child.get("state") == "canceled" for child in children):
+        state = "canceled"
+    else:
+        state = "failed"
+    return {
+        "state": state,
+        "child_count": len(children),
+        "complete_success_count": complete_count,
+        "saved_artifact_count": saved_count,
+    }

imagen_cli/application/cache_service.py

@@ -0,0 +1,262 @@
+"""Cache audit and confirmed reclamation; reports file bytes only."""
+
+from __future__ import annotations
+
+import hashlib
+import json
+from collections.abc import Callable
+from dataclasses import dataclass
+from datetime import UTC, datetime, timedelta
+from pathlib import Path
+from typing import Any
+
+from imagen_cli.storage import queries
+from imagen_cli.storage.artifacts import ArtifactStore
+from imagen_cli.storage.private_state import PrivateStateStore
+from imagen_cli.storage.repositories import Repositories
+from imagen_cli.storage.safety import safe_path
+
+TERMINAL_JOB_STATES = {"succeeded", "failed", "canceled"}
+STAGING_STATES = {"staging", "save_failed"}
+ROW_ONLY_KINDS = {"expired_approval", "expired_plan"}
+
+
+@dataclass(frozen=True)
+class Entry:
+    kind: str
+    identity: str
+    bytes: int
+    detail: dict[str, Any]
+
+    def as_json(self) -> dict[str, Any]:
+        return {"kind": self.kind, "id": self.identity, "bytes": self.bytes, **self.detail}
+
+
+def _utc(value: str) -> datetime:
+    return datetime.fromisoformat(value.replace("Z", "+00:00")).astimezone(UTC)
+
+
+def _parse(value: Any) -> datetime | None:
+    if not isinstance(value, str):
+        return None
+    try:
+        return _utc(value)
+    except ValueError:
+        return None
+
+
+def _size(path: Path) -> int:
+    try:
+        return path.stat().st_size
+    except OSError:
+        return 0
+
+
+def _paths(row: dict[str, Any]) -> list[str]:
+    names = {row.get("staging_relpath"), row.get("managed_relpath")}
+    return sorted(name for name in names if isinstance(name, str))
+
+
+def audit(
+    repositories: Repositories,
+    store: ArtifactStore,
+    *,
+    now: datetime | None = None,
+    staging_days: int = 7,
+    deleted_days: int = 30,
+    private_store: PrivateStateStore | None = None,
+) -> list[Entry]:
+    """Classify reclaimable rows; kinds are mutually exclusive by save_state."""
+    moment = (now or datetime.now(UTC)).astimezone(UTC)
+    if staging_days < 0 or deleted_days < 0:
+        raise ValueError("retention days must be non-negative")
+    sessions = {row["id"]: row for row in queries.iter_all(repositories, "sessions")}
+    holders = queries.reference_sessions(repositories)
+    jobs = {row["id"]: row for row in queries.iter_all(repositories, "jobs")}
+    entries: list[Entry] = []
+
+    for row in queries.iter_all(repositories, "artifacts", include_private=True):
+        identity = row["id"]
+        state = row["save_state"]
+        if state in STAGING_STATES:
+            created = _parse(row.get("created_at"))
+            producer = row.get("producer_job_id")
+            job = jobs.get(producer) if isinstance(producer, str) else None
+            if created is None or moment - created < timedelta(days=staging_days):
+                continue
+            if job is not None and job.get("state") not in TERMINAL_JOB_STATES:
+                continue
+            size = sum(_size(safe_path(store.root, name)) for name in _paths(row))
+            entries.append(Entry("staging_artifact", identity, size, {"save_state": state}))
+            continue
+        if state != "published":
+            continue
+        holding = holders.get(identity, set())
+        if not holding:
+            size = sum(_size(safe_path(store.root, name)) for name in _paths(row))
+            entries.append(Entry("unreferenced_artifact", identity, size, {}))
+            continue
+        deleted = [sessions[s]["deleted_at"] for s in holding if s in sessions]
+        if len(deleted) != len(holding) or not all(deleted):
+            continue
+        stamps = [_parse(value) for value in deleted]
+        if any(stamp is None for stamp in stamps):
+            continue
+        oldest = max(stamp for stamp in stamps if stamp is not None)
+        if moment - oldest < timedelta(days=deleted_days):
+            continue
+        size = sum(_size(safe_path(store.root, name)) for name in _paths(row))
+        entries.append(Entry("deleted_session_artifact", identity, size, {}))
+
+    approvals = {row["plan_id"]: row for row in queries.iter_all(repositories, "approvals")}
+    jobs_by_plan = {
+        row["plan_id"] for row in jobs.values() if isinstance(row.get("plan_id"), str)
+    }
+    for row in queries.iter_all(repositories, "approvals"):
+        expires = _parse(row.get("expires_at"))
+        if expires is not None and expires < moment and row.get("state") != "consumed":
+            entries.append(Entry("expired_approval", row["id"], 0, {"state": row["state"]}))
+    for row in queries.iter_all(repositories, "plans"):
+        expires = _parse(row.get("expires_at"))
+        if expires is None or expires >= moment:
+            continue
+        if row["id"] in jobs_by_plan or row["id"] in approvals:
+            continue
+        entries.append(Entry("expired_plan", row["id"], 0, {}))
+
+    if private_store is not None:
+        turns = {
+            row["id"]: row for row in queries.iter_all(repositories, "turns")
+        }
+        # A profile key is private state too; it must never be reclaimed as a job payload.
+        credentials = {
+            row["credential_ref"][8:] for row in queries.iter_all(repositories, "profiles")
+            if isinstance(row.get("credential_ref"), str)
+            and row["credential_ref"].startswith("private:")
+        }
+        claimed: set[str] = set()
+        for row in queries.iter_all(repositories, "native_contexts", include_private=True):
+            relpath = row.get("private_relpath")
+            if not isinstance(relpath, str):
+                continue
+            claimed.add(relpath)
+            expires = _parse(row.get("expires_at"))
+            source = row.get("source_turn_id")
+            turn = turns.get(source) if isinstance(source, str) else None
+            session_id = turn.get("session_id") if turn else None
+            orphaned = bool(
+                session_id and sessions.get(session_id, {}).get("deleted_at")
+            )
+            if not ((expires is not None and expires < moment) or (expires is None and orphaned)):
+                continue
+            size = _size(safe_path(private_store.root, relpath))
+            entries.append(Entry("expired_native_context", row["id"], size, {}))
+        for identity in private_store.identities():
+            if identity in jobs or identity in credentials or identity + ".bin" in claimed:
+                continue
+            size = _size(safe_path(private_store.root, identity + ".bin"))
+            entries.append(Entry("orphan_job_payload", identity, size, {}))
+
+    return sorted(entries, key=lambda item: (item.kind, item.identity))
+
+
+def digest(entries: list[Entry]) -> str:
+    payload = [[item.kind, item.identity, item.bytes] for item in entries]
+    encoded = json.dumps(payload, ensure_ascii=False, separators=(",", ":")).encode("utf-8")
+    return hashlib.sha256(encoded).hexdigest()
+
+
+def report(entries: list[Entry]) -> dict[str, Any]:
+    total = sum(item.bytes for item in entries)
+    return {
+        "entries": [item.as_json() for item in entries],
+        "plan_digest": digest(entries),
+        "planned_bytes": total,
+        "count": len(entries),
+    }
+
+
+def purge(
+    repositories: Repositories,
+    store: ArtifactStore,
+    plan_digest: str,
+    *,
+    clock: Callable[[], datetime] | None = None,
+    staging_days: int = 7,
+    deleted_days: int = 30,
+    private_store: PrivateStateStore | None = None,
+    max_entries: int = 2000,
+) -> dict[str, Any]:
+    """Delete exactly the confirmed plan; any audit drift refuses the whole run."""
+    now = (clock or (lambda: datetime.now(UTC)))().astimezone(UTC)
+    with repositories.transaction():
+        entries = audit(
+            repositories, store, now=now,
+            staging_days=staging_days, deleted_days=deleted_days,
+            private_store=private_store,
+        )
+        if len(entries) > max_entries:
+            raise ValueError("cache plan exceeds the per-run entry limit")
+        if digest(entries) != plan_digest:
+            raise ValueError("cache plan digest mismatch")
+        results: list[dict[str, Any]] = []
+        planned = sum(item.bytes for item in entries)
+        freed = 0
+        for entry in sorted(entries, key=lambda item: (item.kind, item.identity)):
+            if entry.kind in ROW_ONLY_KINDS:
+                table = "approvals" if entry.kind == "expired_approval" else "plans"
+                repositories.delete(table, entry.identity)
+                results.append({"id": entry.identity, "kind": entry.kind, "result": "deleted"})
+                continue
+            if entry.kind == "expired_native_context":
+                row = repositories.get("native_contexts", entry.identity, include_private=True)
+                if row is None or private_store is None:
+                    results.append({"id": entry.identity, "kind": entry.kind,
+                                    "result": "already_missing"})
+                    continue
+                private_store.delete(row["private_relpath"])
+                repositories.delete("native_contexts", entry.identity)
+                freed += entry.bytes
+                results.append({"id": entry.identity, "kind": entry.kind, "result": "deleted"})
+                continue
+            if entry.kind == "orphan_job_payload":
+                if private_store is not None:
+                    private_store.delete(entry.identity + ".bin")
+                freed += entry.bytes
+                results.append({"id": entry.identity, "kind": entry.kind, "result": "deleted"})
+                continue
+            row = repositories.get("artifacts", entry.identity, include_private=True)
+            if row is None:
+                results.append({
+                    "id": entry.identity, "kind": entry.kind, "result": "already_missing",
+                })
+                continue
+            failed: OSError | None = None
+            released = 0
+            for name in _paths(row):
+                path = safe_path(store.root, name)
+                try:
+                    size = path.stat().st_size
+                    path.unlink()
+                    released += size
+                except FileNotFoundError:
+                    continue
+                except OSError as error:
+                    failed = error
+                    break
+            if failed is not None:
+                results.append({
+                    "id": entry.identity, "kind": entry.kind,
+                    "result": "failed", "error": str(failed),
+                })
+                continue
+            queries.delete_references(repositories, entry.identity)
+            repositories.delete("artifacts", entry.identity)
+            freed += released
+            results.append({"id": entry.identity, "kind": entry.kind, "result": "deleted"})
+        return {
+            "planned_bytes": planned,
+            "deleted_bytes": freed,
+            "results": results,
+            "remaining": sum(1 for item in results if item["result"] == "failed"),
+        }