buildai-cli 0.3.151__tar.gz → 0.3.152__tar.gz

{buildai_cli-0.3.151 → buildai_cli-0.3.152}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: buildai-cli
-Version: 0.3.151
+Version: 0.3.152
 Summary: Build AI CLI (Typer)
 Requires-Python: >=3.11
 Requires-Dist: google-crc32c>=1.7.1

{buildai_cli-0.3.151 → buildai_cli-0.3.152}/cli/commands/egoexo.py

@@ -11,11 +11,13 @@ import typer
 
 from cli.commands.api_proxy import _build_url
 from cli.commands.egoexo_config import app as config_app
+from cli.commands.egoexo_status import status
 from cli.config import resolve_api_url, resolve_credential
 from cli.console import error
 
 app = typer.Typer(help="Inspect EgoExo verdict-plane outcomes.", no_args_is_help=True)
 app.add_typer(config_app, name="config")
+app.command("status")(status)
 
 
 def _request(path: str, *, query_params: list[str] | None = None) -> None:

{buildai_cli-0.3.151 → buildai_cli-0.3.152}/cli/commands/egoexo_config.py

@@ -425,6 +425,152 @@ def clear(
     )
 
 
+def _effective_documents(store, deployment: Path | None):
+    """Return the recipe and trusted assets the API would use, and where the recipe came from.
+
+    Returns ``(recipe, assets, recipe_source, generation, problem)``. Store values
+    win; the checked-in deployment files are the fallback. Both are deep copies.
+    """
+    document, generation, problem = _read_base(store)
+    env, deployment_dir = _deployment_env(_deployment_dir(deployment))
+    if document is not None and document.recipe is not None:
+        recipe = json.loads(json.dumps(document.recipe))
+        recipe_source = "store"
+    elif env.recipe_json:
+        recipe = json.loads(env.recipe_json)
+        recipe_source = f"env ({deployment_dir / 'egoexo-recipe.json'})"
+    else:
+        error("No recipe in the store and no --deployment recipe to start from.")
+        raise typer.Exit(1)
+    if document is not None and document.trusted_assets is not None:
+        assets = json.loads(json.dumps(document.trusted_assets))
+    else:
+        assets = json.loads(env.trusted_assets_json)
+    return recipe, assets, recipe_source, generation, problem
+
+
+IDENTITY_FIELDS = ("generation", "size_bytes", "crc32c_base64")
+RECIPE_ASSET_KEYS = ("config", "board", "crown_observations_config", "crown_model_config")
+
+
+def _recipe_refs(recipe: dict[str, Any]) -> list[dict[str, Any]]:
+    """Collect every artifact reference the recipe hands to a stage or the planner."""
+    refs = []
+    for stage, spec in sorted(recipe.get("preset", {}).items()):
+        for key in RECIPE_ASSET_KEYS:
+            if key in spec:
+                refs.append({"source": f"preset.{stage}.{key}", **spec[key]})
+        for index, checkpoint in enumerate(spec.get("checkpoints", [])):
+            refs.append({"source": f"preset.{stage}.checkpoints[{index}]", **checkpoint})
+    for key in ("keypoint_schema", "episode_selection"):
+        if recipe.get(key) is not None:
+            refs.append({"source": key, **recipe[key]})
+    return refs
+
+
+def _live_identity(storage, asset: dict[str, Any]) -> dict[str, Any]:
+    """Read the pinned generation's metadata, and its bytes when small enough to hash."""
+    from egoexo.adapters.gcs import publication
+
+    try:
+        actual = {"present": True, **storage.metadata(asset["uri"], asset["generation"])}
+    except Exception as exc:  # noqa: BLE001 - the storage SDK reports a miss as 404 or 412
+        if getattr(exc, "code", None) in {404, 412} or type(exc).__name__ in {
+            "NotFound",
+            "PreconditionFailed",
+        }:
+            return {"present": False}
+        raise
+    size = actual["size_bytes"]
+    if size is not None and size <= LOCAL_JSON_MAX_BYTES:
+        with tempfile.TemporaryDirectory() as scratch:
+            path = Path(scratch) / "asset"
+            storage.download(asset["uri"], asset["generation"], path, size)
+            _, sha, crc = publication.identity(path)
+        actual |= {"sha256": sha, "downloaded_crc32c_base64": crc}
+    return actual
+
+
+def _asset_failures(asset: dict[str, Any], actual: dict[str, Any]) -> list[str]:
+    """Return every way the live object differs from the pinned identity."""
+    if not actual.get("present"):
+        return ["object_missing_at_pinned_generation"]
+    failures = [
+        f"{field}: pinned {asset[field]!r} live {actual[field]!r}"
+        for field in IDENTITY_FIELDS
+        if str(asset[field]) != str(actual[field])
+    ]
+    if "sha256" in actual and actual["sha256"] != asset["sha256"]:
+        failures.append(f"sha256: pinned {asset['sha256']} live {actual['sha256']}")
+    if "downloaded_crc32c_base64" in actual and (
+        actual["downloaded_crc32c_base64"] != actual["crc32c_base64"]
+    ):
+        failures.append("crc32c does not match the downloaded bytes")
+    return failures
+
+
+@app.command("verify-assets")
+def verify_assets(
+    uri: str | None = URI_OPTION,
+    deployment: Path | None = DEPLOYMENT_OPTION,
+    json_path: Path | None = typer.Option(
+        None, "--json", help="Also write the full report to this path."
+    ),
+) -> None:
+    """Check every recipe asset and trusted asset against the live object it pins.
+
+    Compares generation, size and CRC32C for each pinned object, and SHA256 for
+    objects up to 1 MiB, which are downloaded. Larger checkpoints are checked by
+    metadata only, the same check the API makes before planning. Reads only.
+    """
+    _runtime_config()
+    store = _store(uri)
+    recipe, assets, recipe_source, generation, problem = _effective_documents(store, deployment)
+    trusted = {asset["uri"]: asset for asset in assets}
+    problems = []
+    for stage, spec in sorted(recipe.get("preset", {}).items()):
+        config = spec.get("config")
+        if config is not None and spec["method"]["config_sha256"] != config["sha256"]:
+            problems.append(f"preset.{stage}: method.config_sha256 != config.sha256")
+    refs = _recipe_refs(recipe)
+    for ref in refs:
+        asset = trusted.get(ref["uri"])
+        if asset is None:
+            problems.append(f"{ref['source']}: {ref['uri']} is not a trusted asset")
+            continue
+        problems.extend(
+            f"{ref['source']}: {field} differs from the trusted asset entry"
+            for field in (*IDENTITY_FIELDS, "sha256")
+            if asset[field] != ref[field]
+        )
+    used = {ref["uri"] for ref in refs}
+    objects = {}
+    for asset in sorted(trusted.values(), key=lambda a: a["uri"]):
+        actual = _live_identity(store.storage, asset)
+        failures = _asset_failures(asset, actual)
+        objects[asset["uri"]] = {
+            "used_by_recipe": asset["uri"] in used,
+            "actual": actual,
+            "failures": failures,
+        }
+        problems.extend(f"{asset['uri']}: {failure}" for failure in failures)
+    report = {
+        "uri": store.uri,
+        "generation": None if generation == "0" else generation,
+        "current_problem": problem,
+        "recipe_source": recipe_source,
+        "recipe_references": len(refs),
+        "trusted_assets": len(assets),
+        "problems": problems,
+        "objects": objects,
+    }
+    if json_path is not None:
+        json_path.write_text(json.dumps(report, indent=2, sort_keys=True, default=str) + "\n")
+    _emit(report)
+    if problems:
+        raise typer.Exit(1)
+
+
 def _asset_group(uri: str) -> str | None:
     """Return the <group> segment of derived/runtime-assets/<group>/<sha256>/<name>."""
     marker = uri.find(ASSET_PREFIX)
@@ -461,21 +607,7 @@ def pin_config_asset(
 
     _load_json_file(file)
     store = _store(uri)
-    document, generation, problem = _read_base(store)
-    env, deployment_dir = _deployment_env(_deployment_dir(deployment))
-    if document is not None and document.recipe is not None:
-        recipe = json.loads(json.dumps(document.recipe))
-        recipe_source = "store"
-    elif env.recipe_json:
-        recipe = json.loads(env.recipe_json)
-        recipe_source = f"env ({deployment_dir / 'egoexo-recipe.json'})"
-    else:
-        error("No recipe in the store and no --deployment recipe to start from.")
-        raise typer.Exit(1)
-    if document is not None and document.trusted_assets is not None:
-        assets = json.loads(json.dumps(document.trusted_assets))
-    else:
-        assets = json.loads(env.trusted_assets_json)
+    recipe, assets, recipe_source, generation, problem = _effective_documents(store, deployment)
     spec = recipe.get("preset", {}).get(stage)
     if not isinstance(spec, dict) or not isinstance(spec.get("method"), dict):
         error(f"Stage {stage!r} is not in the recipe preset.")

buildai_cli-0.3.152/cli/commands/egoexo_status.py

@@ -0,0 +1,215 @@
+"""Read-only EgoExo pipeline status: work items per capture and stage, plus Batch jobs.
+
+The processing API key is read from Secret Manager in process and never printed.
+Batch jobs are listed through the REST API with a server-side filter, because
+``gcloud batch jobs list`` filters on the client and takes minutes in this project.
+"""
+
+from __future__ import annotations
+
+import json
+import subprocess
+import time
+from collections import Counter, defaultdict
+from datetime import UTC, datetime, timedelta
+from pathlib import Path
+from typing import Any
+
+import httpx
+import typer
+
+from cli.config import resolve_api_url
+
+PROJECT = "data-470400"
+REGION = "us-central1"
+BUCKET = "buildai-ego-exo"
+ACTIVE_STATES = frozenset({"SCHEDULED", "QUEUED", "RUNNING"})
+# The worker reads its assignment under the prefix of the image it runs in.
+ASSIGNED_PREFIXES = ("EGOEXO_GPU", "EGOEXO_CPU")
+
+
+def _gcloud(*args: str) -> str:
+    """Run one gcloud command and return its stdout. Tests replace this function."""
+    return subprocess.run(["gcloud", *args], capture_output=True, text=True, check=True).stdout
+
+
+def _client() -> httpx.Client:
+    """HTTP client for the processing API and the Batch API. Tests replace this function."""
+    return httpx.Client(timeout=60.0)
+
+
+def _api_key(project: str) -> str:
+    """Read the processing API key from Secret Manager."""
+    return _gcloud(
+        "secrets",
+        "versions",
+        "access",
+        "latest",
+        "--secret=PROCESSING_API_KEY",
+        f"--project={project}",
+    ).strip()
+
+
+def _raw_capture_ids(bucket: str) -> list[str]:
+    """List capture UUIDs under raw/cn/<date>/."""
+    ids = []
+    for date in _gcloud("storage", "ls", f"gs://{bucket}/raw/cn/").split():
+        for capture in _gcloud("storage", "ls", date).split():
+            ids.append(capture.rstrip("/").rsplit("/", 1)[-1])
+    return sorted(ids)
+
+
+def _get_json(client: httpx.Client, url: str, token: str, params: dict | None = None) -> Any:
+    """GET JSON with a bearer token. Server errors are retried a few times."""
+    for attempt in range(5):
+        response = client.get(url, params=params, headers={"Authorization": f"Bearer {token}"})
+        if response.status_code < 500 or attempt == 4:
+            response.raise_for_status()
+            return response.json()
+        time.sleep(2 * (attempt + 1))
+    raise AssertionError("unreachable")
+
+
+def _capture_items(client: httpx.Client, api_url: str, key: str, capture_id: str) -> list[dict]:
+    """Page through one capture's work items."""
+    url = f"{api_url}/v1/processing/egoexo-captures/{capture_id}"
+    items: list[dict] = []
+    cursor = None
+    while True:
+        params = {"limit": 200} | ({"cursor": cursor} if cursor else {})
+        data = _get_json(client, url, key, params).get("data")
+        if not isinstance(data, dict):
+            return []
+        items.extend(data.get("items", []))
+        cursor = data.get("next_cursor")
+        if not data.get("truncated") or not cursor:
+            return items
+
+
+def _stage_of(item: dict) -> str:
+    """Derive the stage name from the request when present, else the work key."""
+    request = (item.get("metadata") or {}).get("request") or {}
+    return request.get("stage") or str(item.get("work_key", "?")).split(":", 1)[0]
+
+
+def _assigned_id(env: dict[str, str], suffix: str) -> str | None:
+    """Read the assigned work item or attempt id under the GPU or the CPU worker prefix."""
+    for prefix in ASSIGNED_PREFIXES:
+        value = env.get(f"{prefix}_ASSIGNED_{suffix}")
+        if value:
+            return value
+    return None
+
+
+def _batch_jobs(client: httpx.Client, project: str, region: str, since_hours: int) -> list[dict]:
+    """List recent assigned Batch jobs. Only job metadata and the assignment ids are kept."""
+    token = _gcloud("auth", "print-access-token").strip()
+    since = (datetime.now(UTC) - timedelta(hours=since_hours)).strftime("%Y-%m-%dT%H:%M:%SZ")
+    url = f"https://batch.googleapis.com/v1/projects/{project}/locations/{region}/jobs"
+    query = {
+        "filter": f'labels.lane="egoexo-assigned" AND createTime>"{since}"',
+        "pageSize": "500",
+    }
+    rows: list[dict] = []
+    page_token = None
+    for _ in range(200):
+        params = query | ({"pageToken": page_token} if page_token else {})
+        payload = _get_json(client, url, token, params)
+        for job in payload.get("jobs", []):
+            env = job["taskGroups"][0]["taskSpec"].get("environment", {}).get("variables", {})
+            status = job.get("status", {})
+            rows.append(
+                {
+                    "name": job["name"].rsplit("/", 1)[-1],
+                    "state": status.get("state"),
+                    "task_class": job.get("labels", {}).get("task-class"),
+                    "created": job.get("createTime"),
+                    "run_duration": status.get("runDuration"),
+                    "work_item_id": _assigned_id(env, "WORK_ITEM_ID"),
+                    "attempt_id": _assigned_id(env, "ATTEMPT_ID"),
+                    "machine_type": job["allocationPolicy"]["instances"][0]["policy"].get(
+                        "machineType"
+                    ),
+                    "events": [
+                        {"time": e.get("eventTime"), "description": e.get("description")}
+                        for e in status.get("statusEvents", [])
+                    ],
+                }
+            )
+        page_token = payload.get("nextPageToken")
+        if not page_token:
+            break
+    return sorted(rows, key=lambda row: row["created"] or "")
+
+
+def _capture_report(items: list[dict], *, include_items: bool) -> tuple[dict, str]:
+    """Summarize one capture's work items by stage and state. Returns (report, one line)."""
+    by_stage: dict[str, Counter] = defaultdict(Counter)
+    errors = []
+    for item in items:
+        by_stage[_stage_of(item)][item.get("state", "?")] += 1
+        if item.get("last_error"):
+            errors.append({"stage": _stage_of(item), "error": str(item["last_error"])[:200]})
+    report = {
+        "items": len(items),
+        "stages": {stage: dict(counts) for stage, counts in sorted(by_stage.items())},
+        "errors": errors[:20],
+    }
+    if include_items:
+        report["raw_items"] = items
+    summary = ", ".join(
+        f"{stage}:{'/'.join(f'{state}={n}' for state, n in sorted(counts.items()))}"
+        for stage, counts in sorted(by_stage.items())
+    )
+    return report, f"items={len(items)}  {summary or '(none)'}"
+
+
+def status(
+    captures: list[str] | None = typer.Option(
+        None,
+        "--capture",
+        help="Capture id. Repeat for several. Default: every capture under raw/cn/.",
+    ),
+    json_path: Path | None = typer.Option(
+        None, "--json", help="Write the full report to this path."
+    ),
+    items: bool = typer.Option(False, "--items", help="Include raw work items in the report."),
+    since_hours: int = typer.Option(24, "--since-hours", min=1, help="Batch job window."),
+    project: str = typer.Option(PROJECT, "--project", help="Google Cloud project."),
+) -> None:
+    """Show work items per capture and stage, and recent assigned Batch jobs. Reads only."""
+    key = _api_key(project)
+    api_url = resolve_api_url()
+    report: dict[str, Any] = {
+        "read_at": datetime.now(UTC).isoformat(),
+        "api_url": api_url,
+        "captures": {},
+        "batch_jobs": [],
+    }
+    with _client() as client:
+        for capture_id in captures or _raw_capture_ids(BUCKET):
+            try:
+                rows = _capture_items(client, api_url, key, capture_id)
+            except (httpx.HTTPError, ValueError) as exc:
+                report["captures"][capture_id] = {"error": str(exc)[:200]}
+                typer.echo(f"{capture_id}  ERROR {str(exc)[:120]}")
+                continue
+            capture, line = _capture_report(rows, include_items=items)
+            report["captures"][capture_id] = capture
+            typer.echo(f"{capture_id}  {line}")
+            for entry in capture["errors"][:5]:
+                typer.echo(f"    ! {entry['stage']}: {entry['error']}")
+        try:
+            report["batch_jobs"] = _batch_jobs(client, project, REGION, since_hours)
+        except (httpx.HTTPError, subprocess.CalledProcessError, KeyError, ValueError) as exc:
+            report["batch_jobs"] = [{"error": str(exc)[:200]}]
+    jobs = Counter((job.get("task_class"), job.get("state")) for job in report["batch_jobs"])
+    typer.echo(f"batch jobs ({since_hours}h): {dict(jobs) or 'none'}")
+    for job in report["batch_jobs"]:
+        if job.get("state") in ACTIVE_STATES and job.get("task_class") == "gpu_inference":
+            last = job["events"][-1]["description"] if job["events"] else ""
+            typer.echo(
+                f"  gpu {job['name']} {job['state']} item={job['work_item_id']} {last[:100]}"
+            )
+    if json_path is not None:
+        json_path.write_text(json.dumps(report, indent=2, default=str) + "\n")

{buildai_cli-0.3.151 → buildai_cli-0.3.152}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "buildai-cli"
-version = "0.3.151"
+version = "0.3.152"
 description = "Build AI CLI (Typer)"
 requires-python = ">=3.11"
 dependencies = [