buildai-cli 0.3.80__tar.gz → 0.3.82__tar.gz

{buildai_cli-0.3.80 → buildai_cli-0.3.82}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: buildai-cli
-Version: 0.3.80
+Version: 0.3.82
 Summary: Build AI CLI (Typer)
 Requires-Python: >=3.11
 Requires-Dist: httpx>=0.27.0

buildai_cli-0.3.82/cli/commands/grid.py

@@ -0,0 +1,440 @@
+"""Workspace commands for the GCS-native Build AI grid builder."""
+
+from __future__ import annotations
+
+from datetime import UTC, datetime
+from os import getenv
+from pathlib import Path
+from typing import Literal
+from uuid import uuid4
+
+import typer
+
+from cli.console import error, info, success
+
+app = typer.Typer(
+    help="Build manifest-native video grids in gs://buildai-grids.", no_args_is_help=True
+)
+
+ZoomMode = Literal["auto", "always", "never"]
+RunMode = Literal["local", "batch"]
+DEFAULT_GRID_BUILDER_BATCH_IMAGE = (
+    "us-central1-docker.pkg.dev/data-470400/buildai-services/grid-builder:latest"
+)
+
+
+@app.command("validate")
+def validate_manifest(
+    manifest: str = typer.Option(..., "--manifest", "-m", help="Local path or gs:// manifest URI."),
+) -> None:
+    """Validate a grid manifest without writing outputs."""
+
+    manifest_model, _state = _load_manifest(manifest)
+    success(
+        f"valid grid manifest: {manifest_model.grid_size}x{manifest_model.grid_size} "
+        f"({len(manifest_model.clips)} clips)"
+    )
+
+
+@app.command("plan")
+def plan_grid(
+    manifest: str = typer.Option(..., "--manifest", "-m", help="Local path or gs:// manifest URI."),
+    grid_key: str | None = typer.Option(
+        None, "--grid-key", help="Stable grid key for output prefix."
+    ),
+    zoom_out: ZoomMode = typer.Option("auto", "--zoom-out", help="auto, always, or never."),
+    write: bool = typer.Option(False, "--write", help="Write normalized manifest and plan to GCS."),
+) -> None:
+    """Create the deterministic render plan for a grid manifest."""
+
+    render_plan, state = _plan_from_manifest(
+        manifest, grid_key=grid_key, zoom_out=zoom_out, pin_sources=write
+    )
+    _print_plan(render_plan)
+    if write:
+        _write_plan(render_plan, state)
+        success(f"wrote plan: {render_plan.render_prefix}/plan.json")
+
+
+@app.command("build")
+def build_grid(
+    manifest: str = typer.Option(..., "--manifest", "-m", help="Local path or gs:// manifest URI."),
+    grid_key: str | None = typer.Option(
+        None, "--grid-key", help="Stable grid key for output prefix."
+    ),
+    zoom_out: ZoomMode = typer.Option("auto", "--zoom-out", help="auto, always, or never."),
+    mode: RunMode = typer.Option("batch", "--mode", help="local or batch."),
+    workers: int = typer.Option(
+        1,
+        "--workers",
+        "-w",
+        min=1,
+        help="Batch task count; local mode runs one resumable worker loop.",
+    ),
+    work_dir: Path = typer.Option(
+        Path(".grid-builder-work"), "--work-dir", help="Local scratch directory."
+    ),
+    encoder: str = typer.Option("hevc_nvenc", "--encoder", help="FFmpeg video encoder."),
+    image: str | None = typer.Option(
+        None,
+        "--image",
+        help="Batch worker image URI; defaults to the repo-owned grid-builder image.",
+    ),
+    project_id: str | None = typer.Option(
+        None, "--project-id", help="GCP project for Batch submission."
+    ),
+    region: str = typer.Option("us-central1", "--region", help="Batch region."),
+    service_account: str | None = typer.Option(
+        None, "--service-account", help="Batch worker service account."
+    ),
+    network: str | None = typer.Option(None, "--network", help="Batch VPC network."),
+    subnetwork: str | None = typer.Option(None, "--subnetwork", help="Batch VPC subnetwork."),
+    write: bool = typer.Option(False, "--write", help="Required for GCS mutations and rendering."),
+) -> None:
+    """Write a plan and execute or submit the grid render."""
+
+    if not write:
+        error("grid build mutates gs://buildai-grids; rerun with --write")
+        raise typer.Exit(1)
+
+    render_plan, state = _plan_from_manifest(
+        manifest,
+        grid_key=grid_key,
+        zoom_out=zoom_out,
+        pin_sources=True,
+        workers=workers,
+    )
+    _write_plan(render_plan, state)
+    if mode == "batch":
+        job_name = _submit_batch(
+            render_plan=render_plan,
+            image=image,
+            project_id=project_id,
+            region=region,
+            service_account=service_account,
+            network=network,
+            subnetwork=subnetwork,
+            workers=workers,
+            encoder=encoder,
+        )
+        success(f"submitted Batch job: {job_name}")
+    else:
+        summary = _execute_local(render_plan, state, work_dir=work_dir, encoder=encoder)
+        success(
+            f"local render pass complete: {len(summary.completed)} completed, "
+            f"{len(summary.skipped)} skipped"
+        )
+    info(render_plan.render_prefix)
+
+
+@app.command("resume")
+def resume_grid(
+    render_prefix: str = typer.Argument(..., help="Render prefix under gs://buildai-grids."),
+    mode: RunMode = typer.Option("batch", "--mode", help="local or batch."),
+    workers: int = typer.Option(
+        1,
+        "--workers",
+        "-w",
+        min=1,
+        help="Batch task count; local mode runs one resumable worker loop.",
+    ),
+    work_dir: Path = typer.Option(Path(".grid-builder-work"), "--work-dir"),
+    encoder: str = typer.Option("hevc_nvenc", "--encoder", help="FFmpeg video encoder."),
+    image: str | None = typer.Option(
+        None,
+        "--image",
+        help="Batch worker image URI; defaults to the repo-owned grid-builder image.",
+    ),
+    project_id: str | None = typer.Option(
+        None, "--project-id", help="GCP project for Batch submission."
+    ),
+    region: str = typer.Option("us-central1", "--region", help="Batch region."),
+    service_account: str | None = typer.Option(
+        None, "--service-account", help="Batch worker service account."
+    ),
+    network: str | None = typer.Option(None, "--network", help="Batch VPC network."),
+    subnetwork: str | None = typer.Option(None, "--subnetwork", help="Batch VPC subnetwork."),
+    write: bool = typer.Option(False, "--write", help="Required for GCS mutations and rendering."),
+) -> None:
+    """Resume a previously planned render, adopting completed receipts."""
+
+    if not write:
+        error("grid resume mutates gs://buildai-grids; rerun with --write")
+        raise typer.Exit(1)
+    state = _service("GcsState")()
+    render_plan = _load_plan_from_prefix(render_prefix, state)
+    if mode == "batch":
+        job_name = _submit_batch(
+            render_plan=render_plan,
+            image=image,
+            project_id=project_id,
+            region=region,
+            service_account=service_account,
+            network=network,
+            subnetwork=subnetwork,
+            workers=workers,
+            encoder=encoder,
+        )
+        success(f"submitted Batch job: {job_name}")
+    else:
+        summary = _execute_local(render_plan, state, work_dir=work_dir, encoder=encoder)
+        success(
+            f"local resume pass complete: {len(summary.completed)} completed, "
+            f"{len(summary.skipped)} skipped"
+        )
+
+
+@app.command("status")
+def grid_status(
+    render_prefix: str = typer.Argument(..., help="Render prefix under gs://buildai-grids."),
+) -> None:
+    """Show receipt status for a planned render."""
+
+    state = _service("GcsState")()
+    render_plan = _load_plan_from_prefix(render_prefix, state)
+    executor_cls = _service("GridTaskExecutor")
+    executor = executor_cls(
+        plan=render_plan,
+        state=state,
+        work_dir=Path(".grid-builder-status"),
+        owner="status",
+        handler=_noop_handler,
+    )
+    complete = sum(1 for task in render_plan.tasks if executor.receipt_is_valid(task))
+    pending = len(render_plan.tasks) - complete
+    info(f"render: {render_plan.render_id}")
+    info(f"grid: {render_plan.grid_key}")
+    info(f"tasks: {complete} complete, {pending} pending")
+    info(f"zoom_out: {'enabled' if render_plan.zoom_out_enabled else 'disabled'}")
+
+
+@app.command("verify")
+def verify_grid(
+    render_prefix: str = typer.Argument(..., help="Render prefix under gs://buildai-grids."),
+) -> None:
+    """Verify that every planned task has a valid receipt."""
+
+    state = _service("GcsState")()
+    render_plan = _load_plan_from_prefix(render_prefix, state)
+    executor_cls = _service("GridTaskExecutor")
+    executor = executor_cls(
+        plan=render_plan,
+        state=state,
+        work_dir=Path(".grid-builder-verify"),
+        owner="verify",
+        handler=_noop_handler,
+    )
+    missing = [task.task_id for task in render_plan.tasks if not executor.receipt_is_valid(task)]
+    if missing:
+        error(f"{len(missing)} tasks missing valid receipts")
+        for task_id in missing[:20]:
+            error(f"  {task_id}")
+        raise typer.Exit(1)
+    success("all grid builder task receipts are valid")
+
+
+def _load_manifest(manifest_uri: str, state=None):
+    """Load a local or GCS manifest while keeping service imports lazy."""
+
+    GridManifest = _service("GridManifest")
+    if manifest_uri.startswith("gs://"):
+        state = state or _service("GcsState")()
+        return GridManifest.model_validate(state.read_json(manifest_uri)), state
+    return GridManifest.model_validate_json(Path(manifest_uri).read_text(encoding="utf-8")), state
+
+
+def _plan_from_manifest(
+    manifest_uri: str,
+    *,
+    grid_key: str | None,
+    zoom_out: ZoomMode,
+    pin_sources: bool,
+    workers: int = 1,
+):
+    """Normalize a manifest and build a render plan."""
+
+    state = _service("GcsState")() if pin_sources or manifest_uri.startswith("gs://") else None
+    manifest_model, state = _load_manifest(manifest_uri, state=state)
+    normalize_manifest = _service("normalize_manifest")
+    build_render_plan = _service("build_render_plan")
+    RenderProfile = _service("RenderProfile")
+    ZoomOutMode = _service("ZoomOutMode")
+    resolver = state if pin_sources else None
+    normalized = normalize_manifest(manifest_model, resolver=resolver, grid_key=grid_key)
+    profile = RenderProfile(zoom_out_mode=ZoomOutMode(zoom_out), worker_count=workers)
+    return build_render_plan(normalized, profile), state
+
+
+def _write_plan(render_plan, state) -> None:
+    """Write immutable plan sidecars under the render prefix."""
+
+    if state is None:
+        raise RuntimeError("GCS state is required before writing a grid render plan")
+    sidecars = {
+        "manifest.normalized.json": render_plan.manifest,
+        "profile.json": render_plan.profile,
+        "plan.json": render_plan,
+        "metadata/status.json": {
+            "schema_key": "buildai.grid_builder.status",
+            "schema_version": "1.0",
+            "render_id": render_plan.render_id,
+            "grid_key": render_plan.grid_key,
+            "task_count": len(render_plan.tasks),
+            "zoom_out_enabled": render_plan.zoom_out_enabled,
+        },
+    }
+    for relative_path, payload in sidecars.items():
+        try:
+            state.upload_json(
+                f"{render_plan.render_prefix}/{relative_path}", payload, create_only=True
+            )
+        except FileExistsError:
+            pass
+
+
+def _execute_local(render_plan, state, *, work_dir: Path, encoder: str):
+    """Run the worker loop locally for a render plan."""
+
+    if state is None:
+        raise RuntimeError("GCS state is required before executing a grid render")
+    GridTaskExecutor = _service("GridTaskExecutor")
+    GridRenderHandler = _service("GridRenderHandler")
+    executor = GridTaskExecutor(
+        plan=render_plan,
+        state=state,
+        work_dir=work_dir / render_plan.render_id,
+        owner="local#0",
+        handler=GridRenderHandler(encoder=encoder),
+    )
+    return executor.execute_until_complete(poll_interval_sec=2.0)
+
+
+def _load_plan_from_prefix(render_prefix: str, state):
+    """Load a render plan from its GCS render prefix."""
+
+    RenderPlan = _service("RenderPlan")
+    return RenderPlan.model_validate(state.read_json(f"{render_prefix.rstrip('/')}/plan.json"))
+
+
+def _submit_batch(
+    *,
+    render_plan,
+    image: str | None,
+    project_id: str | None,
+    region: str,
+    service_account: str | None,
+    network: str | None,
+    subnetwork: str | None,
+    workers: int,
+    encoder: str,
+) -> str:
+    """Submit a Cloud Batch worker fleet for a grid render plan."""
+
+    resolved_image = _grid_builder_batch_image(image)
+    if not all([project_id, service_account, network, subnetwork]):
+        error("--mode=batch requires --project-id, --service-account, --network, and --subnetwork")
+        raise typer.Exit(1)
+    from infra.batch import BatchClient, BatchJobConfig, ProvisioningModel
+
+    job_id = _batch_job_id(render_plan.render_id)
+    client = BatchClient(
+        project_id=project_id,
+        region=region,
+        vpc_network=network,
+        vpc_subnetwork=subnetwork,
+        service_account=service_account,
+    )
+    config = BatchJobConfig(
+        image=resolved_image,
+        entrypoint="python",
+        commands=[
+            "-m",
+            "services.grid_builder",
+            "worker",
+            "--plan-uri",
+            f"{render_plan.render_prefix}/plan.json",
+            "--work-dir",
+            "/tmp/grid-builder",
+            "--encoder",
+            encoder,
+        ],
+        machine_type="g2-standard-4",
+        gpu_type="nvidia-l4",
+        gpu_count=1,
+        cpu_milli=4000,
+        memory_mib=16384,
+        boot_disk_gb=500,
+        service_account=service_account,
+    )
+    return client.create_job(
+        job_id=job_id,
+        task_count=workers,
+        config=config,
+        provisioning_model=ProvisioningModel.SPOT,
+        labels={"buildai-grid": "true", "render-id": render_plan.render_id},
+    )
+
+
+def _grid_builder_batch_image(image: str | None) -> str:
+    """Resolve the worker image while allowing operator overrides."""
+
+    return image or getenv("GRID_BUILDER_BATCH_IMAGE") or DEFAULT_GRID_BUILDER_BATCH_IMAGE
+
+
+def _print_plan(render_plan) -> None:
+    """Print the compact plan summary users need before launch."""
+
+    info(f"render_id: {render_plan.render_id}")
+    info(f"render_prefix: {render_plan.render_prefix}")
+    info(f"lod_sizes: {', '.join(str(size) for size in render_plan.lod_sizes)}")
+    info(f"zoom_out: {'enabled' if render_plan.zoom_out_enabled else 'disabled'}")
+    info(f"tasks: {len(render_plan.tasks)}")
+
+
+def _noop_handler(_task, _context) -> Path:
+    """Satisfy executor construction for receipt-only status commands."""
+
+    raise RuntimeError("status and verify commands do not execute render tasks")
+
+
+def _service(name: str):
+    """Import service symbols lazily so the standalone CLI remains lightweight."""
+
+    try:
+        import services.grid_builder as grid_builder
+    except ImportError as exc:
+        error("grid commands require a workspace install: run with `uv run buildai` from the repo")
+        raise typer.Exit(1) from exc
+
+    if name in {
+        "GridManifest",
+        "RenderPlan",
+        "RenderProfile",
+        "build_render_plan",
+        "normalize_manifest",
+    }:
+        return getattr(grid_builder, name)
+    if name == "GcsState":
+        from services.grid_builder.gcs_state import GcsState
+
+        return GcsState
+    if name == "GridTaskExecutor":
+        from services.grid_builder.executor import GridTaskExecutor
+
+        return GridTaskExecutor
+    if name == "GridRenderHandler":
+        from services.grid_builder.render_tasks import GridRenderHandler
+
+        return GridRenderHandler
+    if name == "ZoomOutMode":
+        from services.grid_builder.planner import ZoomOutMode
+
+        return ZoomOutMode
+    raise KeyError(name)
+
+
+def _batch_job_id(render_id: str) -> str:
+    """Return a unique Cloud Batch job id while preserving render traceability."""
+
+    attempt = datetime.now(UTC).strftime("%Y%m%d%H%M%S")
+    return f"grid-{render_id}-{attempt}-{uuid4().hex[:8]}"

{buildai_cli-0.3.80 → buildai_cli-0.3.82}/cli/main.py

@@ -38,6 +38,7 @@ def _bootstrap_repo_paths() -> None:
         return
 
     for package_root in (
+        repo_root,
         repo_root / "core",
         repo_root / "apps" / "buildai-api" / "shared",
     ):
@@ -319,10 +320,12 @@ def db_callback(
 if has_core():
     from cli.commands.db import app as db_app
     from cli.commands.gigcamera import app as gigcamera_app
+    from cli.commands.grid import app as grid_app
 
     db_app.callback()(db_callback)
     app.add_typer(db_app, name="db")
     app.add_typer(gigcamera_app, name="gigcamera")
+    app.add_typer(grid_app, name="grid")
 
 
 def main() -> None:

{buildai_cli-0.3.80 → buildai_cli-0.3.82}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "buildai-cli"
-version = "0.3.80"
+version = "0.3.82"
 description = "Build AI CLI (Typer)"
 requires-python = ">=3.11"
 dependencies = [