buildai-cli 0.3.95__tar.gz → 0.3.97__tar.gz

{buildai_cli-0.3.95 → buildai_cli-0.3.97}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: buildai-cli
-Version: 0.3.95
+Version: 0.3.97
 Summary: Build AI CLI (Typer)
 Requires-Python: >=3.11
 Requires-Dist: httpx>=0.27.0

{buildai_cli-0.3.95 → buildai_cli-0.3.97}/cli/commands/ego_frame_search.py

@@ -7,6 +7,7 @@ import json
 import os
 import subprocess
 from dataclasses import asdict, is_dataclass
+from pathlib import Path
 from typing import Any
 from urllib.parse import urlparse
 from uuid import UUID
@@ -86,6 +87,37 @@ def _settings_for_command(ctx: typer.Context, *, write: bool) -> object:
     return init_ops_context(ctx)
 
 
+def _clip_ids_from_options(
+    values: list[UUID],
+    file_path: Path | None,
+    *,
+    limit: int | None = None,
+) -> list[str]:
+    """Return unique clip UUID strings from repeatable CLI values and a newline file."""
+    ordered: list[UUID] = list(values)
+    if file_path is not None:
+        for line_number, raw_line in enumerate(file_path.read_text().splitlines(), start=1):
+            text = raw_line.strip()
+            if not text or text.startswith("#"):
+                continue
+            try:
+                ordered.append(UUID(text))
+            except ValueError as exc:
+                raise typer.BadParameter(
+                    f"invalid UUID on line {line_number} of {file_path}: {text}"
+                ) from exc
+    seen: set[UUID] = set()
+    deduped: list[str] = []
+    for clip_id in ordered:
+        if clip_id in seen:
+            continue
+        seen.add(clip_id)
+        deduped.append(str(clip_id))
+        if limit is not None and len(deduped) >= limit:
+            break
+    return deduped
+
+
 def _require_internal_admin_for_write(ctx: typer.Context, *, write: bool) -> None:
     """Require the explicit admin profile before direct DB writes."""
     if not write:
@@ -869,6 +901,47 @@ def membership_program(
     asyncio.run(run())
 
 
+@app.command("membership-bucket")
+def membership_bucket(
+    ctx: typer.Context,
+    corpus_key: str = typer.Option(..., "--corpus-key", help="Target corpus key."),
+    bucket_uri_prefix: str = typer.Option(..., "--bucket-uri-prefix", help="GCS URI prefix."),
+    run_id: str = typer.Option("manual", "--run-id", help="Membership materialization run id."),
+    min_source_video_size_bytes: int | None = typer.Option(
+        None,
+        "--min-source-video-size-bytes",
+        min=1,
+        help="Require source videos to be larger than this byte count.",
+    ),
+    limit: int | None = typer.Option(None, "--limit", min=1, help="Cap candidate clips."),
+    write: bool = typer.Option(False, "--write", help="Insert membership rows."),
+    format: Format = format_option(),
+) -> None:
+    """Dry-run or materialize corpus membership from video assets in a GCS bucket prefix."""
+    _require_internal_admin_for_write(ctx, write=write)
+    settings = _settings_for_command(ctx, write=write)
+
+    async def run() -> None:
+        from dal.embeddings import ego_frame_search as ego_search_dal
+
+        async with get_cli_context(settings, profile=(ctx.obj or {}).get("cli_profile")) as (
+            _db,
+            dal_ctx,
+        ):
+            summary = await ego_search_dal.materialize_bucket_membership(
+                dal_ctx,
+                corpus_key=corpus_key,
+                bucket_uri_prefix=bucket_uri_prefix,
+                membership_run_id=run_id,
+                min_source_video_size_bytes=min_source_video_size_bytes,
+                limit=limit,
+                write=write,
+            )
+        render(_plain(summary), format=format)
+
+    asyncio.run(run())
+
+
 @app.command("membership-index-space")
 def membership_index_space(
     ctx: typer.Context,
@@ -935,6 +1008,65 @@ def membership_gigcamera(
     asyncio.run(run())
 
 
+@app.command("sampled-frames-from-vectors")
+def sampled_frames_from_vectors(
+    ctx: typer.Context,
+    frame_uri_prefix: str = typer.Option(
+        ...,
+        "--frame-uri-prefix",
+        help="Frame-jpg GCS URI prefix to materialize as sampled frames.",
+    ),
+    clip_id: list[UUID] = typer.Option(
+        [],
+        "--clip-id",
+        help="Core clip id to include. Repeatable.",
+    ),
+    clip_id_file: Path | None = typer.Option(
+        None,
+        "--clip-id-file",
+        exists=True,
+        file_okay=True,
+        dir_okay=False,
+        readable=True,
+        help="Newline-delimited core clip ids to include.",
+    ),
+    space_id: int | None = typer.Option(None, "--space-id", help="Target embedding space id."),
+    space_key: str | None = typer.Option(None, "--space-key"),
+    selection_kind: str = typer.Option("near_90s", "--selection-kind"),
+    limit: int | None = typer.Option(None, "--limit", min=1, help="Cap candidate clips."),
+    write: bool = typer.Option(False, "--write", help="Insert missing sampled-frame rows."),
+    format: Format = format_option(),
+) -> None:
+    """Dry-run or materialize sampled-frame anchors from existing frame vectors."""
+    _require_internal_admin_for_write(ctx, write=write)
+    settings = _settings_for_command(ctx, write=write)
+
+    async def run() -> None:
+        from dal.embeddings import ego_frame_search as ego_search_dal
+
+        async with get_cli_context(settings, profile=(ctx.obj or {}).get("cli_profile")) as (
+            _db,
+            dal_ctx,
+        ):
+            target_space_id = await _resolve_space_id(
+                dal_ctx,
+                space_id=space_id,
+                space_key=space_key,
+            )
+            summary = await ego_search_dal.materialize_sampled_frames_from_existing_vectors(
+                dal_ctx,
+                target_space_id=target_space_id,
+                frame_uri_prefix=frame_uri_prefix,
+                clip_ids=_clip_ids_from_options(clip_id, clip_id_file, limit=None),
+                selection_kind=selection_kind,
+                limit=limit,
+                write=write,
+            )
+        render(_plain(summary), format=format)
+
+    asyncio.run(run())
+
+
 @app.command("membership-dataset")
 def membership_dataset(
     ctx: typer.Context,
@@ -1165,6 +1297,11 @@ def queue_missing_label_text_embeddings(
     ),
     model_config_hash: str | None = typer.Option(None, "--model-config-hash"),
     limit: int | None = typer.Option(None, "--limit", min=1, help="Cap selected labels."),
+    required_asset_uri_prefix: list[str] = typer.Option(
+        [],
+        "--required-asset-uri-prefix",
+        help="Require selected frame assets to start with this GCS URI prefix. Repeatable.",
+    ),
     idempotency_key: str = typer.Option(..., "--idempotency-key", help="Manifest idempotency key."),
     max_parallel_requests: int = typer.Option(4, "--max-parallel-requests", min=1),
     write: bool = typer.Option(False, "--write", help="Create the processing manifest."),
@@ -1204,6 +1341,8 @@ def queue_missing_label_text_embeddings(
                 "only_missing_text_embeddings": True,
                 "max_parallel_requests_per_worker": max_parallel_requests,
             }
+            if required_asset_uri_prefix:
+                selection_spec["required_asset_uri_prefixes"] = required_asset_uri_prefix
             sink_overrides = {
                 "embedding": {
                     "space_id": resolved_text_space_id,
@@ -1373,11 +1512,35 @@ def build_scann(
 @app.command("queue-missing-embeddings")
 def queue_missing_embeddings(
     ctx: typer.Context,
-    corpus_key: list[str] = typer.Option(..., "--corpus-key", help="Corpus key. Repeatable."),
+    corpus_key: list[str] = typer.Option([], "--corpus-key", help="Corpus key. Repeatable."),
+    clip_id: list[UUID] = typer.Option(
+        [],
+        "--clip-id",
+        help="Core clip id to include. Repeatable; bypasses corpus membership when set.",
+    ),
+    clip_id_file: Path | None = typer.Option(
+        None,
+        "--clip-id-file",
+        exists=True,
+        dir_okay=False,
+        readable=True,
+        help="Newline-delimited core clip ids to include.",
+    ),
     space_id: int | None = typer.Option(None, "--space-id", help="Target embedding space id."),
     space_key: str | None = typer.Option(_INDEX_QWEN_EGO_SPACE_KEY, "--space-key"),
     selection_kind: str = typer.Option("near_90s", "--selection-kind"),
     limit: int | None = typer.Option(None, "--limit", min=1, help="Cap selected frame rows."),
+    required_asset_uri_prefix: list[str] = typer.Option(
+        [],
+        "--required-asset-uri-prefix",
+        help="Require selected frame assets to start with this GCS URI prefix. Repeatable.",
+    ),
+    min_source_video_size_bytes: int | None = typer.Option(
+        None,
+        "--min-source-video-size-bytes",
+        min=1,
+        help="Require selected clips to have a source video larger than this byte count.",
+    ),
     idempotency_key: str = typer.Option(..., "--idempotency-key", help="Manifest idempotency key."),
     cost_cap_usd: float | None = typer.Option(None, "--cost-cap-usd", min=0.0),
     write: bool = typer.Option(False, "--write", help="Create the processing manifest."),
@@ -1387,6 +1550,93 @@ def queue_missing_embeddings(
     _require_internal_admin_for_write(ctx, write=write)
     settings = _settings_for_command(ctx, write=write)
 
+    async def run() -> None:
+        from dal.processing import media_jobs
+
+        async with get_cli_context(settings, profile=(ctx.obj or {}).get("cli_profile")) as (
+            _db,
+            dal_ctx,
+        ):
+            target_space_id = await _resolve_space_id(
+                dal_ctx,
+                space_id=space_id,
+                space_key=space_key,
+            )
+            clip_ids = _clip_ids_from_options(clip_id, clip_id_file, limit=limit)
+            if clip_ids:
+                selection_spec = {
+                    "kind": "by_clip_ids",
+                    "clip_ids": clip_ids,
+                    "selection_kind": selection_kind,
+                    "limit": None,
+                    "only_missing_vectors": True,
+                }
+            else:
+                if not corpus_key:
+                    raise typer.BadParameter("provide --corpus-key or --clip-id/--clip-id-file")
+                selection_spec = {
+                    "kind": "by_corpus",
+                    "corpus_keys": corpus_key,
+                    "selection_kind": selection_kind,
+                    "limit": limit,
+                    "only_missing_vectors": True,
+                }
+            if required_asset_uri_prefix:
+                selection_spec["required_asset_uri_prefixes"] = required_asset_uri_prefix
+            if min_source_video_size_bytes is not None:
+                selection_spec["min_source_video_size_bytes"] = min_source_video_size_bytes
+            sink_overrides = {"embedding": {"space_id": target_space_id}}
+            resource_overrides = {"cost_cap_usd": cost_cap_usd} if cost_cap_usd is not None else {}
+            if write:
+                manifest = await media_jobs.queue_processor_job(
+                    dal_ctx,
+                    processor_ref=media_jobs.EMBED_FRAMES_PROCESSOR_REF,
+                    selection_spec=selection_spec,
+                    sink_overrides=sink_overrides,
+                    resource_overrides=resource_overrides,
+                    submitted_by_principal=_SUBMITTED_BY,
+                    idempotency_key=idempotency_key,
+                )
+            else:
+                manifest = {
+                    "dry_run": True,
+                    "processor_ref": media_jobs.EMBED_FRAMES_PROCESSOR_REF,
+                    "selection_spec": selection_spec,
+                    "sink_overrides": sink_overrides,
+                    "resource_overrides": resource_overrides,
+                }
+        render(manifest, format=format)
+
+    asyncio.run(run())
+
+
+@app.command("queue-bucket-missing-embeddings")
+def queue_bucket_missing_embeddings(
+    ctx: typer.Context,
+    required_asset_uri_prefix: list[str] = typer.Option(
+        ...,
+        "--required-asset-uri-prefix",
+        help="Require selected frame assets to start with this GCS URI prefix. Repeatable.",
+    ),
+    space_id: int | None = typer.Option(None, "--space-id", help="Target embedding space id."),
+    space_key: str | None = typer.Option(_INDEX_QWEN_EGO_SPACE_KEY, "--space-key"),
+    selection_kind: str = typer.Option("near_90s", "--selection-kind"),
+    min_source_video_size_bytes: int | None = typer.Option(
+        None,
+        "--min-source-video-size-bytes",
+        min=1,
+        help="Require selected clips to have a source video larger than this byte count.",
+    ),
+    limit: int | None = typer.Option(None, "--limit", min=1, help="Cap selected frame rows."),
+    idempotency_key: str = typer.Option(..., "--idempotency-key", help="Manifest idempotency key."),
+    cost_cap_usd: float | None = typer.Option(None, "--cost-cap-usd", min=0.0),
+    write: bool = typer.Option(False, "--write", help="Create the processing manifest."),
+    format: Format = format_option(),
+) -> None:
+    """Queue missing-only frame embeddings directly from a bucket-scoped frame set."""
+    _require_internal_admin_for_write(ctx, write=write)
+    settings = _settings_for_command(ctx, write=write)
+
     async def run() -> None:
         from dal.processing import media_jobs
 
@@ -1400,12 +1650,14 @@ def queue_missing_embeddings(
                 space_key=space_key,
             )
             selection_spec = {
-                "kind": "by_corpus",
-                "corpus_keys": corpus_key,
+                "kind": "by_bucket",
                 "selection_kind": selection_kind,
                 "limit": limit,
                 "only_missing_vectors": True,
+                "required_asset_uri_prefixes": required_asset_uri_prefix,
             }
+            if min_source_video_size_bytes is not None:
+                selection_spec["min_source_video_size_bytes"] = min_source_video_size_bytes
             sink_overrides = {"embedding": {"space_id": target_space_id}}
             resource_overrides = {"cost_cap_usd": cost_cap_usd} if cost_cap_usd is not None else {}
             if write:
@@ -1434,9 +1686,55 @@ def queue_missing_embeddings(
 @app.command("queue-missing-sampled-frames")
 def queue_missing_sampled_frames(
     ctx: typer.Context,
-    corpus_key: list[str] = typer.Option(..., "--corpus-key", help="Corpus key. Repeatable."),
+    corpus_key: list[str] = typer.Option([], "--corpus-key", help="Corpus key. Repeatable."),
+    clip_id: list[UUID] = typer.Option(
+        [],
+        "--clip-id",
+        help="Core clip id to include. Repeatable; bypasses corpus membership when set.",
+    ),
+    clip_id_file: Path | None = typer.Option(
+        None,
+        "--clip-id-file",
+        exists=True,
+        dir_okay=False,
+        readable=True,
+        help="Newline-delimited core clip ids to include.",
+    ),
     selection_kind: str = typer.Option("near_90s", "--selection-kind"),
     limit: int | None = typer.Option(None, "--limit", min=1, help="Cap selected clips."),
+    required_asset_uri_prefix: list[str] = typer.Option(
+        [],
+        "--required-asset-uri-prefix",
+        help="Require selected video assets to start with this GCS URI prefix. Repeatable.",
+    ),
+    min_source_video_size_bytes: int | None = typer.Option(
+        None,
+        "--min-source-video-size-bytes",
+        min=1,
+        help="Require selected source videos to be larger than this byte count.",
+    ),
+    num_frames: int | None = typer.Option(
+        1,
+        "--num-frames",
+        min=1,
+        help="Override extractor frame count for this manifest.",
+    ),
+    single_frame_index: int | None = typer.Option(
+        5,
+        "--single-frame-index",
+        min=0,
+        help="Frame index to use when extracting one midpoint frame.",
+    ),
+    use_remote_gcs_seek: bool = typer.Option(
+        True,
+        "--remote-gcs-seek/--full-download",
+        help="Use authenticated GCS HTTP range reads for ffmpeg input.",
+    ),
+    ffmpeg_scale_flags: str | None = typer.Option(
+        "fast_bilinear",
+        "--ffmpeg-scale-flags",
+        help="ffmpeg scale filter flags for p104-compatible frame pixels.",
+    ),
     idempotency_key: str = typer.Option(..., "--idempotency-key", help="Manifest idempotency key."),
     write: bool = typer.Option(False, "--write", help="Create the processing manifest."),
     format: Format = format_option(),
@@ -1445,6 +1743,113 @@ def queue_missing_sampled_frames(
     _require_internal_admin_for_write(ctx, write=write)
     settings = _settings_for_command(ctx, write=write)
 
+    async def run() -> None:
+        from dal.processing import media_jobs
+
+        async with get_cli_context(settings, profile=(ctx.obj or {}).get("cli_profile")) as (
+            _db,
+            dal_ctx,
+        ):
+            clip_ids = _clip_ids_from_options(clip_id, clip_id_file, limit=limit)
+            if clip_ids:
+                selection_spec = {
+                    "kind": "by_ids",
+                    "ids": clip_ids,
+                    "selection_kind": selection_kind,
+                    "limit": None,
+                    "only_missing_sampled_frames": True,
+                    "require_zero_frames": False,
+                }
+            else:
+                if not corpus_key:
+                    raise typer.BadParameter("provide --corpus-key or --clip-id/--clip-id-file")
+                selection_spec = {
+                    "kind": "by_corpus",
+                    "corpus_keys": corpus_key,
+                    "selection_kind": selection_kind,
+                    "limit": limit,
+                    "only_missing_sampled_frames": True,
+                    "require_zero_frames": False,
+                }
+            if required_asset_uri_prefix:
+                selection_spec["required_asset_uri_prefixes"] = required_asset_uri_prefix
+            if min_source_video_size_bytes is not None:
+                selection_spec["min_source_video_size_bytes"] = min_source_video_size_bytes
+            if num_frames is not None:
+                selection_spec["num_frames"] = num_frames
+            if single_frame_index is not None:
+                selection_spec["single_frame_index"] = single_frame_index
+            if use_remote_gcs_seek:
+                selection_spec["use_remote_gcs_seek"] = True
+            if ffmpeg_scale_flags:
+                selection_spec["ffmpeg_scale_flags"] = ffmpeg_scale_flags
+            if write:
+                manifest = await media_jobs.queue_processor_job(
+                    dal_ctx,
+                    processor_ref=media_jobs.EXTRACT_FRAMES_PROCESSOR_REF,
+                    selection_spec=selection_spec,
+                    sink_overrides=None,
+                    resource_overrides=None,
+                    submitted_by_principal=_SUBMITTED_BY,
+                    idempotency_key=idempotency_key,
+                )
+            else:
+                manifest = {
+                    "dry_run": True,
+                    "processor_ref": media_jobs.EXTRACT_FRAMES_PROCESSOR_REF,
+                    "selection_spec": selection_spec,
+                }
+        render(manifest, format=format)
+
+    asyncio.run(run())
+
+
+@app.command("queue-bucket-missing-sampled-frames")
+def queue_bucket_missing_sampled_frames(
+    ctx: typer.Context,
+    required_asset_uri_prefix: list[str] = typer.Option(
+        ...,
+        "--required-asset-uri-prefix",
+        help="Require selected video assets to start with this GCS URI prefix. Repeatable.",
+    ),
+    selection_kind: str = typer.Option("near_90s", "--selection-kind"),
+    min_source_video_size_bytes: int | None = typer.Option(
+        None,
+        "--min-source-video-size-bytes",
+        min=1,
+        help="Require selected source videos to be larger than this byte count.",
+    ),
+    num_frames: int | None = typer.Option(
+        1,
+        "--num-frames",
+        min=1,
+        help="Override extractor frame count for this manifest.",
+    ),
+    single_frame_index: int | None = typer.Option(
+        5,
+        "--single-frame-index",
+        min=0,
+        help="Frame index to use when extracting one midpoint frame.",
+    ),
+    use_remote_gcs_seek: bool = typer.Option(
+        True,
+        "--remote-gcs-seek/--full-download",
+        help="Use authenticated GCS HTTP range reads for ffmpeg input.",
+    ),
+    ffmpeg_scale_flags: str | None = typer.Option(
+        "fast_bilinear",
+        "--ffmpeg-scale-flags",
+        help="ffmpeg scale filter flags for p104-compatible frame pixels.",
+    ),
+    limit: int | None = typer.Option(None, "--limit", min=1, help="Cap selected clips."),
+    idempotency_key: str = typer.Option(..., "--idempotency-key", help="Manifest idempotency key."),
+    write: bool = typer.Option(False, "--write", help="Create the processing manifest."),
+    format: Format = format_option(),
+) -> None:
+    """Queue missing sampled-frame extraction directly from a bucket-scoped video set."""
+    _require_internal_admin_for_write(ctx, write=write)
+    settings = _settings_for_command(ctx, write=write)
+
     async def run() -> None:
         from dal.processing import media_jobs
 
@@ -1453,13 +1858,23 @@ def queue_missing_sampled_frames(
             dal_ctx,
         ):
             selection_spec = {
-                "kind": "by_corpus",
-                "corpus_keys": corpus_key,
+                "kind": "by_bucket",
                 "selection_kind": selection_kind,
                 "limit": limit,
                 "only_missing_sampled_frames": True,
                 "require_zero_frames": False,
+                "required_asset_uri_prefixes": required_asset_uri_prefix,
             }
+            if min_source_video_size_bytes is not None:
+                selection_spec["min_source_video_size_bytes"] = min_source_video_size_bytes
+            if num_frames is not None:
+                selection_spec["num_frames"] = num_frames
+            if single_frame_index is not None:
+                selection_spec["single_frame_index"] = single_frame_index
+            if use_remote_gcs_seek:
+                selection_spec["use_remote_gcs_seek"] = True
+            if ffmpeg_scale_flags:
+                selection_spec["ffmpeg_scale_flags"] = ffmpeg_scale_flags
             if write:
                 manifest = await media_jobs.queue_processor_job(
                     dal_ctx,
@@ -1498,6 +1913,11 @@ def queue_missing_demian_labels(
     source_id: str = typer.Option("demian", "--source-id", help="Non-secret label source alias."),
     endpoint_url: str | None = typer.Option(None, "--endpoint-url"),
     limit: int | None = typer.Option(None, "--limit", min=1, help="Cap selected frames."),
+    required_asset_uri_prefix: list[str] = typer.Option(
+        [],
+        "--required-asset-uri-prefix",
+        help="Require selected frame assets to start with this GCS URI prefix. Repeatable.",
+    ),
     idempotency_key: str = typer.Option(..., "--idempotency-key", help="Manifest idempotency key."),
     cost_cap_usd: float | None = typer.Option(None, "--cost-cap-usd", min=0.0),
     estimated_cost_per_label_usd: float = typer.Option(
@@ -1549,6 +1969,8 @@ def queue_missing_demian_labels(
                 "endpoint_timeout_sec": endpoint_timeout_sec,
                 "allow_external_bearer": allow_external_bearer,
             }
+            if required_asset_uri_prefix:
+                selection_spec["required_asset_uri_prefixes"] = required_asset_uri_prefix
             sink_overrides = {
                 "observation": {
                     "table": "observations.frame_observations",

{buildai_cli-0.3.95 → buildai_cli-0.3.97}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "buildai-cli"
-version = "0.3.95"
+version = "0.3.97"
 description = "Build AI CLI (Typer)"
 requires-python = ">=3.11"
 dependencies = [