openrunner-sdk 2.41.7__tar.gz → 2.41.8__tar.gz

{openrunner_sdk-2.41.7 → openrunner_sdk-2.41.8}/.gitignore

@@ -153,6 +153,8 @@ activemq-data/
 # Environments
 .env
 .envrc
+.forge.env
+.forge-venv/
 
 # Docker secrets (real values — only examples tracked)
 secrets/

{openrunner_sdk-2.41.7 → openrunner_sdk-2.41.8}/PKG-INFO

@@ -1,10 +1,10 @@
-Metadata-Version: 2.4
+Metadata-Version: 2.5
 Name: openrunner-sdk
-Version: 2.41.7
+Version: 2.41.8
 Summary: OpenRunner SDK - W&B-compatible ML experiment tracking client
-Project-URL: Homepage, https://github.com/jqueguiner/openrunner
-Project-URL: Repository, https://github.com/jqueguiner/openrunner
-Project-URL: Issues, https://github.com/jqueguiner/openrunner/issues
+Project-URL: Homepage, https://github.com/gladiaio/openrunner
+Project-URL: Repository, https://github.com/gladiaio/openrunner
+Project-URL: Issues, https://github.com/gladiaio/openrunner/issues
 Author-email: JL Queguiner <jl@gladia.io>
 License: MIT
 Keywords: experiment-tracking,invention-disclosure,machine-learning,ml,mlops,patent,wandb
@@ -656,7 +656,7 @@ W&B env vars (`WANDB_API_KEY`, `WANDB_BASE_URL`) are also supported as fallback
 
 ## Self-Hosting
 
-OpenRunner is designed to be self-hosted. See the [main repo](https://github.com/jqueguiner/openrunner) for server setup with Docker Compose.
+OpenRunner is designed to be self-hosted. See the [main repo](https://github.com/gladiaio/openrunner) for server setup with Docker Compose.
 
 ## License
 

{openrunner_sdk-2.41.7 → openrunner_sdk-2.41.8}/README.md

@@ -541,7 +541,7 @@ W&B env vars (`WANDB_API_KEY`, `WANDB_BASE_URL`) are also supported as fallback
 
 ## Self-Hosting
 
-OpenRunner is designed to be self-hosted. See the [main repo](https://github.com/jqueguiner/openrunner) for server setup with Docker Compose.
+OpenRunner is designed to be self-hosted. See the [main repo](https://github.com/gladiaio/openrunner) for server setup with Docker Compose.
 
 ## License
 

{openrunner_sdk-2.41.7 → openrunner_sdk-2.41.8}/openrunner/__init__.py

@@ -93,6 +93,7 @@ import openrunner.datasets as datasets
 from openrunner.datasets import (
     DatasetInfo,
     DatasetsClient,
+    DeriveIncomplete,
     create_dataset,
     dataset_info,
     index_dataset,
@@ -101,6 +102,7 @@ from openrunner.datasets import (
     link_project,
     load_dataset,
     add_derivation,
+    derive_by_ids,
     derive_by_query,
     derived_rows,
     keyword_expand,
@@ -852,11 +854,21 @@ def alert(
         return None
 
 
-def log_artifact(artifact: Artifact) -> dict[str, Any] | None:
+def log_artifact(
+    artifact: "Artifact | str | Path",
+    *,
+    name: str | None = None,
+    type: str | None = None,
+    aliases: list[str] | None = None,
+) -> dict[str, Any] | None:
     """Upload an artifact to the active run. Never raises.
 
     Args:
-        artifact: The Artifact object with files added via add_file/add_dir.
+        artifact: An Artifact with files added via add_file/add_dir, OR a path
+            to a file or directory (W&B-style), built into an Artifact for you.
+        name: Artifact name when ``artifact`` is a path.
+        type: Advisory type when ``artifact`` is a path.
+        aliases: Aliases to set on the created version.
 
     Returns:
         Server response dict with version info, or None on failure.
@@ -865,7 +877,9 @@ def log_artifact(artifact: Artifact) -> dict[str, Any] | None:
         if _active_run is None:
             logger.warning("openrunner.log_artifact(): no active run")
             return None
-        return _active_run.log_artifact(artifact)
+        return _active_run.log_artifact(
+            artifact, name=name, type=type, aliases=aliases
+        )
     except Exception as e:
         logger.warning("openrunner.log_artifact() failed: %s", e)
         return None

{openrunner_sdk-2.41.7 → openrunner_sdk-2.41.8}/openrunner/artifact.py

@@ -29,6 +29,55 @@ def _sha256_file(path: Path, chunk_size: int = 8192) -> str:
     return h.hexdigest()
 
 
+def coerce_artifact(
+    source: "Artifact | str | Path",
+    *,
+    name: str | None = None,
+    type: str | None = None,
+    description: str | None = None,
+    metadata: dict[str, Any] | None = None,
+) -> "Artifact":
+    """Return an Artifact, building one from a path if that's what was given.
+
+    ``log_artifact`` historically took a fully built ``Artifact``. Users coming
+    from W&B -- which this repo actively courts, see
+    ``examples/migrate-from-wandb/`` -- write the path form instead::
+
+        run.log_artifact("out/final", name="my-model", type="model")
+
+    and used to get ``TypeError: unexpected keyword argument 'name'``. Accept
+    both. A directory becomes ``add_dir``, a file ``add_file``; ``name``
+    defaults to the path's own name so the common case needs no kwargs.
+    """
+    if isinstance(source, Artifact):
+        # Already built: kwargs only fill gaps, they never silently override
+        # what the caller put on the object.
+        if name and not source.name:
+            source.name = name
+        if type and source.type is None:
+            source.type = type
+        return source
+
+    path = Path(source)
+    if not path.exists():
+        raise FileNotFoundError(f"artifact path does not exist: {path}")
+    art = Artifact(
+        name or path.name or path.stem,
+        type,
+        description,
+        metadata,
+    )
+    if path.is_dir():
+        # Existing add_dir semantics: files are prefixed with the directory
+        # name. Kept as-is so a coerced artifact lays out identically to a
+        # hand-built one -- changing it would silently move every existing
+        # caller's files.
+        art.add_dir(str(path))
+    else:
+        art.add_file(str(path))
+    return art
+
+
 class Artifact:
     """An untyped, versioned bucket of files for experiment tracking.
 

{openrunner_sdk-2.41.7 → openrunner_sdk-2.41.8}/openrunner/cli.py

@@ -145,6 +145,45 @@ def _save_settings(settings: dict[str, Any]) -> None:
     )
 
 
+def _resolve_api_key(settings: dict[str, Any] | None = None) -> str:
+    """API key with the same precedence the SDK uses: env, then the
+    session-synced ~/.openrunner/session_config.json, then settings.json.
+
+    Historically the CLI read settings.json FIRST and most subcommands never
+    looked at session_config at all — so a stale key in settings.json made
+    `openrunner catalog ...` fail with "API key has expired" while
+    `openrunner datasets ...` (SDK-backed) worked with the fresh session key.
+    """
+    try:
+        from openrunner.settings import _global_creds
+        session = _global_creds()
+    except Exception:
+        session = {}
+    return (
+        os.environ.get("OPENRUNNER_API_KEY")
+        or os.environ.get("OPENRUN_API_KEY")
+        or session.get("api_key")
+        or (settings or _load_settings()).get("api_key")
+        or ""
+    )
+
+
+def _resolve_base_url(settings: dict[str, Any] | None = None) -> str:
+    """Base URL with the same precedence chain as ``_resolve_api_key``."""
+    try:
+        from openrunner.settings import _global_creds
+        session = _global_creds()
+    except Exception:
+        session = {}
+    return (
+        os.environ.get("OPENRUNNER_BASE_URL")
+        or os.environ.get("OPENRUN_BASE_URL")
+        or session.get("base_url")
+        or (settings or _load_settings()).get("base_url")
+        or ""
+    )
+
+
 # ---------------------------------------------------------------------------
 # CLI group
 # ---------------------------------------------------------------------------
@@ -420,7 +459,7 @@ def sync(
         --sync-tensorboard                Import TensorBoard event files as metrics
     """
     settings = _load_settings()
-    api_key = settings.get("api_key")
+    api_key = _resolve_api_key(settings)
     base_url = settings.get("base_url", "http://localhost:8000")
 
     if not api_key:
@@ -545,7 +584,7 @@ def ls(project: str | None) -> None:
     in the specified project.
     """
     settings = _load_settings()
-    api_key = settings.get("api_key")
+    api_key = _resolve_api_key(settings)
     base_url = settings.get("base_url", "http://localhost:8000")
 
     if not api_key:
@@ -607,7 +646,7 @@ def _get_client() -> "APIClient":
     import os
 
     settings = _load_settings()
-    api_key = os.environ.get("OPENRUNNER_API_KEY") or settings.get("api_key")
+    api_key = _resolve_api_key(settings)
     base_url = os.environ.get("OPENRUNNER_BASE_URL") or settings.get(
         "base_url", "http://localhost:8000"
     )
@@ -1534,7 +1573,7 @@ def launch(
     from openrunner.launch import launch as launch_fn
 
     settings = _load_settings()
-    api_key = settings.get("api_key")
+    api_key = _resolve_api_key(settings)
     if not api_key:
         click.echo("Error: Not logged in. Run 'openrunner login' first.", err=True)
         raise SystemExit(1)
@@ -1600,7 +1639,7 @@ def launch(
 def jobs(project: str, state: str | None) -> None:
     """List jobs for a project."""
     settings = _load_settings()
-    api_key = settings.get("api_key")
+    api_key = _resolve_api_key(settings)
     base_url = settings.get("base_url", "http://localhost:8000")
 
     if not api_key:
@@ -1656,7 +1695,7 @@ def job_logs(job_id: str) -> None:
     JOB_ID is the UUID of the job.
     """
     settings = _load_settings()
-    api_key = settings.get("api_key")
+    api_key = _resolve_api_key(settings)
     base_url = settings.get("base_url", "http://localhost:8000")
 
     if not api_key:
@@ -2244,7 +2283,7 @@ def docker(
     import subprocess
 
     settings = _load_settings()
-    api_key = os.environ.get("OPENRUNNER_API_KEY") or settings.get("api_key")
+    api_key = _resolve_api_key(settings)
     base_url = os.environ.get("OPENRUNNER_BASE_URL") or settings.get(
         "base_url", "http://localhost:8000"
     )
@@ -3579,8 +3618,8 @@ def catalog_tables(org: str):
     """List catalog tables and their row counts."""
     settings = _load_settings()
     client = APIClient(
-        base_url=settings.get("base_url", os.environ.get("OPENRUNNER_BASE_URL", "")),
-        api_key=settings.get("api_key", os.environ.get("OPENRUNNER_API_KEY", "")),
+        base_url=_resolve_base_url(settings),
+        api_key=_resolve_api_key(settings),
     )
     resp = client._request("get", f"/organizations/{org}/catalog/status")
     if resp.status_code != 200:
@@ -3618,8 +3657,8 @@ def catalog_init(org: str):
     """Initialize catalog tables on the configured storage connection."""
     settings = _load_settings()
     client = APIClient(
-        base_url=settings.get("base_url", os.environ.get("OPENRUNNER_BASE_URL", "")),
-        api_key=settings.get("api_key", os.environ.get("OPENRUNNER_API_KEY", "")),
+        base_url=_resolve_base_url(settings),
+        api_key=_resolve_api_key(settings),
     )
     resp = client._request("post", f"/organizations/{org}/catalog/init")
     if resp.status_code != 200:
@@ -3640,8 +3679,8 @@ def catalog_query(org: str, table: str, where: str, limit: int, offset: int, fmt
     """Query rows from a catalog table."""
     settings = _load_settings()
     client = APIClient(
-        base_url=settings.get("base_url", os.environ.get("OPENRUNNER_BASE_URL", "")),
-        api_key=settings.get("api_key", os.environ.get("OPENRUNNER_API_KEY", "")),
+        base_url=_resolve_base_url(settings),
+        api_key=_resolve_api_key(settings),
     )
     params = {"limit": limit, "offset": offset}
     if where:
@@ -3687,8 +3726,8 @@ def catalog_export(org: str, table: str, fmt: str, where: str, limit: int, outpu
     """Export catalog table data to file or stdout."""
     settings = _load_settings()
     client = APIClient(
-        base_url=settings.get("base_url", os.environ.get("OPENRUNNER_BASE_URL", "")),
-        api_key=settings.get("api_key", os.environ.get("OPENRUNNER_API_KEY", "")),
+        base_url=_resolve_base_url(settings),
+        api_key=_resolve_api_key(settings),
     )
     params: dict[str, Any] = {"format": fmt}
     if where:
@@ -3730,8 +3769,8 @@ def catalog_upload(org: str, table: str, path: str, validate: bool):
 
     settings = _load_settings()
     client = APIClient(
-        base_url=settings.get("base_url", os.environ.get("OPENRUNNER_BASE_URL", "")),
-        api_key=settings.get("api_key", os.environ.get("OPENRUNNER_API_KEY", "")),
+        base_url=_resolve_base_url(settings),
+        api_key=_resolve_api_key(settings),
     )
 
     # Collect files
@@ -3792,6 +3831,56 @@ def catalog_upload(org: str, table: str, path: str, validate: bool):
     click.echo(f"\nDone. {total_rows} total rows uploaded to '{table}'.")
 
 
+@catalog.command("merge")
+@click.option("--org", required=True, help="Organization ID or slug")
+@click.option("--table", required=True, help="Table name (e.g. transcriptions)")
+@click.option("--key", default="request_id", show_default=True,
+              help="Match column for the upsert")
+@click.option("--insert", is_flag=True, default=False,
+              help="Also insert rows whose key has no match (default: skip them)")
+@click.option("--batch", default=500, show_default=True, help="Rows per request")
+@click.argument("path", type=click.Path(exists=True))
+def catalog_merge(org: str, table: str, key: str, insert: bool, batch: int, path: str):
+    """Upsert JSONL rows into a catalog table by key (enrichment merge).
+
+    Unlike `catalog upload` (append-only), matched rows get ONLY the uploaded
+    columns updated; new columns are created automatically (nullable, typed
+    from the first non-null value). Nested values must be pre-serialized to
+    JSON strings.
+
+    Example:
+      openrunner catalog merge --org my-org --table transcriptions \\
+          --key request_id enriched.jsonl
+    """
+    settings = _load_settings()
+    client = APIClient(
+        base_url=_resolve_base_url(settings),
+        api_key=_resolve_api_key(settings),
+    )
+    lines = [ln for ln in Path(path).read_text(encoding="utf-8").splitlines() if ln.strip()]
+    total = 0
+    new_cols: set = set()
+    for start in range(0, len(lines), max(1, batch)):
+        chunk = lines[start:start + max(1, batch)]
+        resp = client._request(
+            "POST",
+            f"/organizations/{org}/catalog/tables/{table}/merge?key={key}"
+            + ("&insert=true" if insert else ""),
+            content=("\n".join(chunk)).encode("utf-8"),
+            headers={"Content-Type": "application/x-ndjson"},
+        )
+        if resp.status_code not in (200, 201):
+            click.echo(f"Merge failed at row {start + 1}: {resp.text}", err=True)
+            sys.exit(1)
+        result = resp.json()
+        total += result.get("rows_merged", 0)
+        new_cols.update(result.get("new_columns", []))
+        click.echo(f"  merged {total}/{len(lines)}")
+    if new_cols:
+        click.echo(f"New columns created: {', '.join(sorted(new_cols))}")
+    click.echo(f"Done. {total} rows merged into '{table}'.")
+
+
 # ---------------------------------------------------------------------------
 # patent-scan command
 # ---------------------------------------------------------------------------
@@ -3837,7 +3926,7 @@ def patent_scan(
     from openrunner.patent_scan import scan_commit, _run_git
 
     settings = _load_settings()
-    api_key = settings.get("api_key")
+    api_key = _resolve_api_key(settings)
     base_url = settings.get("base_url", "https://api.openrun.gladia.io")
     project = project or settings.get("project")
 
@@ -3954,7 +4043,7 @@ def patent_lab(project: str | None, min_score: int, status: str | None, limit: i
         openrunner patent-lab --min-score 70 --status detected
     """
     settings = _load_settings()
-    api_key = settings.get("api_key")
+    api_key = _resolve_api_key(settings)
     base_url = settings.get("base_url", "https://api.openrun.gladia.io")
     project = project or settings.get("project")
 
@@ -4372,7 +4461,7 @@ def gpu_credentials_add(org_id: str, provider: str, provider_key: str | None,
     from openrunner.gpu.api_integration import GpuAPIClient
 
     settings = _load_settings()
-    api_key = settings.get("api_key")
+    api_key = _resolve_api_key(settings)
     base_url = settings.get("base_url", "http://localhost:8000")
     if not api_key:
         click.echo("Error: Not logged in. Run 'openrunner login' first.", err=True)
@@ -4393,7 +4482,7 @@ def gpu_credentials_ls() -> None:
     from openrunner.gpu.api_integration import GpuAPIClient
 
     settings = _load_settings()
-    api_key = settings.get("api_key")
+    api_key = _resolve_api_key(settings)
     base_url = settings.get("base_url", "http://localhost:8000")
     if not api_key:
         click.echo("Error: Not logged in.", err=True)
@@ -4430,9 +4519,10 @@ def gpu_run(command, provider, gpu, gpu_count, image, disk_gb, env, ssh_keys, pr
 
     Forge is serverless (no pod / no SSH): COMMAND is the App script and it runs
     through the forge client, streaming the fleet's logs. The GPU is declared in
-    the script, so --gpu is not used.
+    the script, so --gpu is not used. -e KEY=VALUE reaches the App's
+    local_entrypoint as an environment variable.
 
-        openrunner gpu run --provider forge get_started.py
+        openrunner gpu run --provider forge -e EPOCHS=3 get_started.py
 
     Example:
 
@@ -4447,7 +4537,7 @@ def gpu_run(command, provider, gpu, gpu_count, image, disk_gb, env, ssh_keys, pr
         raise SystemExit(1)
 
     settings = _load_settings()
-    api_key = settings.get("api_key")
+    api_key = _resolve_api_key(settings)
     base_url = settings.get("base_url")
     if not api_key:
         click.echo("Error: Not logged in. Run 'openrunner login' first.", err=True)
@@ -4527,7 +4617,7 @@ def gpu_run(command, provider, gpu, gpu_count, image, disk_gb, env, ssh_keys, pr
                 from openrunner.gpu.runner import run_engine_serverside_sync
                 result = run_engine_serverside_sync(
                     provider=provider, command=command, project=project,
-                    emit=lambda m: click.echo(m),
+                    env=env_dict, emit=lambda m: click.echo(m),
                 )
         except Exception as exc:  # noqa: BLE001
             click.echo(f"gpu run failed: {exc}", err=True)
@@ -4617,7 +4707,7 @@ def _registry_http():
     import httpx
 
     settings = _load_settings()
-    api_key = settings.get("api_key")
+    api_key = _resolve_api_key(settings)
     base_url = settings.get("base_url", "http://localhost:8000")
     if not api_key:
         click.echo("Error: Not logged in. Run 'openrunner login' first.", err=True)
@@ -4815,7 +4905,7 @@ def registry_logout() -> None:
 def _drift_client_project(project: str | None):
     """Return (APIClient, project_id) resolving project name/slug -> UUID."""
     settings = _load_settings()
-    api_key = settings.get("api_key")
+    api_key = _resolve_api_key(settings)
     base_url = settings.get("base_url", "http://localhost:8000")
     if not api_key:
         click.echo("Error: Not logged in. Run 'openrunner login' first.", err=True)
@@ -5152,7 +5242,7 @@ def deploy_call(deployment_id, path, project, method, data, headers) -> None:
       openrunner deploy call <id> predict -p ORG/NAME -d '{"text":"hi"}'
     """
     settings = _load_settings()
-    api_key = settings.get("api_key")
+    api_key = _resolve_api_key(settings)
     base_url = settings.get("base_url", "http://localhost:8000").rstrip("/")
     if not api_key:
         click.echo("Error: Not logged in.", err=True); raise SystemExit(1)
@@ -5196,7 +5286,7 @@ def deploy_call(deployment_id, path, project, method, data, headers) -> None:
 def _plain_client():
     """Build an APIClient from settings, or exit if not logged in."""
     settings = _load_settings()
-    api_key = settings.get("api_key")
+    api_key = _resolve_api_key(settings)
     base_url = settings.get("base_url", "http://localhost:8000")
     if not api_key:
         click.echo("Error: Not logged in. Run 'openrunner login' first.", err=True)
@@ -5420,7 +5510,7 @@ def runner_start(org, name, labels, gpu, gpu_count, poll_interval, once, token,
         if not org or not name:
             click.echo("Error: pass --org and --name to register, or --token to run with a pre-minted token.", err=True)
             raise SystemExit(1)
-        api_key = settings.get("api_key")
+        api_key = _resolve_api_key(settings)
         if not api_key:
             click.echo("Error: Not logged in. Run 'openrunner login' first (or pass --token).", err=True)
             raise SystemExit(1)
@@ -5677,7 +5767,7 @@ def runner_launch(org, name, provider, gpu, gpu_count, labels, image, web, poll_
     from openrunner.gpu.api_integration import GpuAPIClient
 
     settings = _load_settings()
-    api_key = settings.get("api_key")
+    api_key = _resolve_api_key(settings)
     base_url = settings.get("base_url", "http://localhost:8000").rstrip("/")
     if not api_key:
         click.echo("Error: Not logged in. Run 'openrunner login' first.", err=True)

{openrunner_sdk-2.41.7 → openrunner_sdk-2.41.8}/openrunner/datasets.py

@@ -31,6 +31,20 @@ import httpx
 _UUID_RE = re.compile(r"^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$", re.I)
 
 
+class DeriveIncomplete(RuntimeError):
+    """A chunked ``derive_by_ids`` failed part-way through.
+
+    The dataset was created and holds ``ids[:resume_from]``. It is a valid
+    dataset, not a broken one — it is just short, and nothing about it says so.
+    Carries what is needed to append the remainder.
+    """
+
+    def __init__(self, msg: str, *, dataset_id: str, resume_from: int):
+        super().__init__(msg)
+        self.dataset_id = dataset_id
+        self.resume_from = resume_from
+
+
 def _check(r: "httpx.Response") -> "httpx.Response":
     """Raise with the API's actual error detail (FastAPI 422 lists the bad field)."""
     if r.is_success:
@@ -741,18 +755,39 @@ class DatasetsClient:
         _check(r)
         return r.json()
 
-    def reconcile_shards(self, repo_id: str, *, timeout: float = 1800.0) -> dict:
+    def reconcile_shards(self, repo_id: str, *, timeout: float = 1800.0,
+                         offset: int = 0, limit: int = 0) -> dict:
         """Rebuild the tar clip index (index.jsonl) by re-scanning shards on S3.
 
         Recovers byte offsets if index.jsonl was lost or an ``upload_sharded`` run
         was interrupted — the shard tars are the source of truth. Re-marks the
         dataset ``sharded`` so ``/clip`` resolves audio by Range GET.
+
+        ``offset``/``limit`` reconcile a slice of the (sorted) shards, MERGING into
+        the existing index — for datasets too large to scan in one request without
+        hitting the proxy timeout. ``limit=0`` scans everything at once.
         """
         ds = self._resolve(repo_id)
-        r = self._http.post(f"/datasets/{ds.id}/shards/reconcile", timeout=timeout)
+        r = self._http.post(f"/datasets/{ds.id}/shards/reconcile",
+                            params={"offset": offset, "limit": limit}, timeout=timeout)
         _check(r)
         return r.json()
 
+    def reconcile_shards_batched(self, repo_id: str, *, batch: int = 500,
+                                 timeout: float = 1800.0, progress=None) -> dict:
+        """Reconcile a large sharded dataset in ``batch``-shard slices so no single
+        request exceeds the proxy timeout. Loops until every shard is indexed."""
+        off, last = 0, None
+        while True:
+            last = self.reconcile_shards(repo_id, offset=off, limit=batch, timeout=timeout)
+            if progress:
+                progress(last)
+            nxt = last.get("next_offset")
+            if not nxt:
+                break
+            off = nxt
+        return last
+
     def set_storage_policy(self, repo_id: str, tier: str) -> dict:
         """1-click storage tier for a managed dataset's files.
 
@@ -908,6 +943,8 @@ class DatasetsClient:
     ) -> DatasetInfo:
         """Create (or append to) a dataset **derived from a query** over ``repo_id``.
 
+        For an explicit membership list rather than a query, see ``derive_by_ids``.
+
         ``mode``: "word" (``q`` = term / CSV; pairs with LLM ``keyword_expand``),
         "fts" (BM25), "exact" (phrase), "regex" (``q`` = a Python-``re`` pattern
         matched over the text column, bounded), "semantic" (vector — needs
@@ -936,6 +973,101 @@ class DatasetsClient:
         _check(r)
         return DatasetInfo._from_json(r.json())
 
+    def derive_by_ids(
+        self,
+        repo_id: str,
+        ids: "list[str]",
+        *,
+        name: str | None = None,
+        columns: "dict[str, list] | None" = None,
+        lang: str | None = None,
+        validate_ids: bool = True,
+        chunk_size: int = 50_000,
+        target_repo_id: str | None = None,
+        project_id: str | None = None,
+    ) -> DatasetInfo:
+        """Create (or append to) a dataset from an **explicit list of primary keys**.
+
+        For selections no query can express: rows picked by a boolean column the
+        search modes cannot reach, by an external model's output, or by a human
+        review pass. The alternative is reverse-engineering a query that happens
+        to return the same rows.
+
+        ``ids`` are the parent's primary keys (``debug_id`` for transcript
+        catalogs). Unknown ids are dropped server-side unless
+        ``validate_ids=False`` — pass that only when the ids were just read out
+        of the parent. Long lists are POSTed in ``chunk_size`` batches, the first
+        creating the dataset and the rest appending to it, so the caller does not
+        have to hold a 20 MB request body. Returns the dataset after the last
+        batch; ``num_examples`` reflects the deduplicated total.
+
+        ``columns`` attaches per-id metadata as member columns, one list per
+        column positionally aligned to ``ids``::
+
+            ds.derive_by_ids(parent, ids, name='org/medical',
+                             columns={'n_health': counts, 'source': sigs})
+
+        Types are preserved, so the ranking signal stays queryable: one dataset
+        plus ``n_health >= 20`` replaces the three nested datasets you would
+        otherwise have to build to express three thresholds over the same rows.
+        Chunking splits the columns alongside the ids, so a batch never carries
+        another batch's values.
+        """
+        if not ids:
+            raise ValueError("derive_by_ids needs a non-empty ids list")
+        if not (name or target_repo_id):
+            raise ValueError("derive_by_ids needs either name= (create) or target_repo_id= (append)")
+        if chunk_size < 1:
+            raise ValueError("chunk_size must be >= 1")
+        for _n, _v in (columns or {}).items():
+            if len(_v) != len(ids):
+                raise ValueError(
+                    f"columns[{_n!r}] has {len(_v)} values but ids has {len(ids)}; "
+                    "they are positional")
+        ds = self._resolve(repo_id)
+        nat = self._native_client()
+        info: DatasetInfo | None = None
+        target_id = str(self._resolve(target_repo_id).id) if target_repo_id else None
+        for i in range(0, len(ids), chunk_size):
+            batch = [str(x) for x in ids[i:i + chunk_size]]
+            body: dict[str, Any] = {
+                "mode": "ids", "ids": batch, "limit": len(batch),
+                "validate_ids": validate_ids,
+            }
+            if columns:
+                body["columns"] = {n: list(v[i:i + chunk_size])
+                                   for n, v in columns.items()}
+            if lang:
+                body["lang"] = lang
+            if target_id:                     # append into the existing target...
+                body["target_dataset_id"] = target_id
+            else:                             # ...or create on the first batch
+                body["name"] = name
+                if project_id:
+                    body["project_id"] = str(project_id)
+            try:
+                if nat is not None:
+                    info = DatasetInfo._from_json(nat.derive_dataset(str(ds.id), body))
+                else:
+                    r = self._http.post(f"/datasets/{ds.id}/derive", json=body, timeout=600.0)
+                    _check(r)
+                    info = DatasetInfo._from_json(r.json())
+            except Exception as e:
+                # A mid-loop failure leaves a dataset that exists and looks
+                # complete. Retrying the whole call re-sends batch 0 with name=
+                # and trips the duplicate-name check, so it can never resume —
+                # hand back what the caller needs to continue instead.
+                if target_id is None:
+                    raise
+                raise DeriveIncomplete(
+                    f"derive_by_ids failed on the batch starting at index {i} of "
+                    f"{len(ids)}: {e}. Dataset {target_id} holds ids[:{i}]; resume "
+                    f"with target_repo_id={target_id!r} and ids[{i}:].",
+                    dataset_id=target_id, resume_from=i,
+                ) from e
+            target_id = target_id or str(info.id)   # subsequent batches append
+        return info
+
     def derived_rows(self, repo_id: str, *, limit: int = 50, offset: int = 0) -> dict:
         """Page a query-derived dataset's rows: member keys + inherited parent
         fields (lineage-namespaced) + extra columns. Native core when available."""
@@ -1382,6 +1514,11 @@ def derive_by_query(repo_id: str, q: str, **kwargs: Any) -> "DatasetInfo":
     return _c().derive_by_query(repo_id, q, **kwargs)
 
 
+def derive_by_ids(repo_id: str, ids: "list[str]", **kwargs: Any) -> "DatasetInfo":
+    """Create/append a dataset from an explicit list of parent primary keys."""
+    return _c().derive_by_ids(repo_id, ids, **kwargs)
+
+
 def derived_rows(repo_id: str, **kwargs: Any) -> dict:
     """Page a query-derived dataset's rows (member keys + inherited parent fields)."""
     return _c().derived_rows(repo_id, **kwargs)