dh-cli 0.15.0__tar.gz → 0.16.0__tar.gz

{dh_cli-0.15.0 → dh_cli-0.16.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: dh-cli
-Version: 0.15.0
+Version: 0.16.0
 Summary: Dayhoff Labs developer CLI
 Author-email: Dayhoff Labs <dev@dayhofflabs.com>
 License: # PolyForm Noncommercial License 1.0.0

{dh_cli-0.15.0 → dh_cli-0.16.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "dh-cli"
-version = "0.15.0"
+version = "0.16.0"
 description = "Dayhoff Labs developer CLI"
 requires-python = ">=3.11"
 readme = "README.md"

{dh_cli-0.15.0 → dh_cli-0.16.0}/src/dh_cli/batch/__init__.py

@@ -15,6 +15,7 @@ from .commands.cancel import cancel
 from .commands.clean import clean
 from .commands.cost import cost
 from .commands.disco import disco
+from .commands.embed_esmc import embed_esmc
 from .commands.embed_t5 import embed_t5
 from .commands.equilibrator import equilibrator
 from .commands.esmfold2 import esmfold2
@@ -55,6 +56,7 @@ def batch_cli():
     \b
     Embedding Pipelines:
       embed-t5   Generate T5 protein embeddings
+      embed-esmc Generate ESMC protein embeddings
 
     \b
     Structure Prediction:
@@ -106,6 +108,7 @@ batch_cli.add_command(cost)
 
 # Register pipeline commands
 batch_cli.add_command(embed_t5, name="embed-t5")
+batch_cli.add_command(embed_esmc, name="embed-esmc")
 batch_cli.add_command(equilibrator, name="equilibrator")
 batch_cli.add_command(esmfold2)
 batch_cli.add_command(boltz)

{dh_cli-0.15.0 → dh_cli-0.16.0}/src/dh_cli/batch/commands/__init__.py

@@ -3,6 +3,7 @@
 from .boltz import boltz
 from .cancel import cancel
 from .cost import cost
+from .embed_esmc import embed_esmc
 from .embed_t5 import embed_t5
 from .list_jobs import list_jobs
 from .local import local
@@ -15,6 +16,7 @@ __all__ = [
     "boltz",
     "cancel",
     "cost",
+    "embed_esmc",
     "embed_t5",
     "list_jobs",
     "local",

dh_cli-0.16.0/src/dh_cli/batch/commands/embed_esmc.py

@@ -0,0 +1,244 @@
+"""ESMC embedding pipeline command."""
+
+from pathlib import Path
+
+import click
+
+from ..aws_batch import BatchClient, BatchError, resolve_dependency
+from ..job_id import generate_job_id, get_aws_username
+from ..manifest import (
+    BATCH_JOBS_BASE,
+    BatchConfig,
+    InputConfig,
+    JobManifest,
+    JobStatus,
+    OutputConfig,
+    create_job_directory,
+    save_local_stub,
+    save_manifest,
+    save_manifest_s3,
+)
+from ..s3_transport import s3_job_prefix, upload_directory
+from .embed_t5 import _count_sequences, _is_primordial_path, _split_fasta
+
+# A10G: ~79 seq/s vs T4 ~6 seq/s at 600M bf16 (T4 emulates bf16); ~$2.9 vs ~$13.7 per 1M seqs on spot.
+DEFAULT_QUEUE = "a10g-1x-spot"
+DEFAULT_WORKERS = 50
+DEFAULT_SEQS_PER_CHUNK = 5000
+DEFAULT_JOB_DEFINITION = "dayhoff-embed-esmc"
+DEFAULT_IMAGE_URI = "074735440724.dkr.ecr.us-east-1.amazonaws.com/dayhoff:esmc-latest"
+
+
+@click.command()
+@click.argument("input_fasta", type=click.Path(exists=True))
+@click.option(
+    "--model",
+    type=click.Choice(["300m", "600m"]),
+    default="600m",
+    show_default=True,
+    help="ESMC model size",
+)
+@click.option(
+    "--layer",
+    type=int,
+    default=-1,
+    show_default=True,
+    help="Hidden-state layer to mean-pool (0 = embeddings, -1 = final output)",
+)
+@click.option(
+    "--keep-alignment-tokens",
+    default="-",
+    show_default=True,
+    help='Which of "-.|" to keep as ESMC tokens; the rest are stripped ("" strips all)',
+)
+@click.option(
+    "--workers",
+    default=DEFAULT_WORKERS,
+    type=int,
+    help=f"Number of parallel workers [default: {DEFAULT_WORKERS}]",
+)
+@click.option("--queue", default=DEFAULT_QUEUE, help=f"Batch queue [default: {DEFAULT_QUEUE}]")
+@click.option(
+    "--seqs-per-chunk",
+    default=DEFAULT_SEQS_PER_CHUNK,
+    type=int,
+    help=f"Sequences per chunk [default: {DEFAULT_SEQS_PER_CHUNK}]",
+)
+@click.option("--dry-run", is_flag=True, help="Show plan without submitting")
+@click.option("--base-path", default=BATCH_JOBS_BASE, help="Base path for job data")
+@click.option("--after", "after", multiple=True, help="Job ID(s) to wait for before starting")
+def embed_esmc(
+    input_fasta,
+    model,
+    layer,
+    keep_alignment_tokens,
+    workers,
+    queue,
+    seqs_per_chunk,
+    dry_run,
+    base_path,
+    after,
+):
+    """Generate ESMC mean-pooled protein embeddings for a FASTA file.
+
+    Splits the input FASTA into chunks and embeds them in parallel as an
+    AWS Batch array job. Each worker writes an `ids`/`vectors` H5 file.
+
+    \b
+    Examples:
+      dh batch embed-esmc /primordial/proteins.fasta
+      dh batch embed-esmc proteins.fasta --model 300m --layer 20
+      dh batch embed-esmc msa.fasta --keep-alignment-tokens "-."
+
+    \b
+    After job completes:
+      dh batch status <job-id>
+      dh batch finalize <job-id> --output out.h5
+
+    `dh batch retry` does not carry the ESMC settings and refuses these jobs;
+    resubmit the failed chunks' sequences with `dh batch embed-esmc` instead.
+    """
+    input_path = Path(input_fasta).resolve()
+
+    click.echo(f"Counting sequences in {input_path}...")
+    num_sequences = _count_sequences(input_path)
+    click.echo(f"Found {num_sequences:,} sequences")
+
+    if num_sequences == 0:
+        click.echo(click.style("Error: No sequences found in input file", fg="red"), err=True)
+        raise SystemExit(1)
+
+    use_s3 = not _is_primordial_path(input_path)
+
+    num_chunks = min((num_sequences + seqs_per_chunk - 1) // seqs_per_chunk, workers)
+    actual_seqs_per_chunk = (num_sequences + num_chunks - 1) // num_chunks
+
+    job_id = generate_job_id("esmc")
+
+    click.echo()
+    click.echo(f"Job ID:           {job_id}")
+    click.echo(f"Input:            {input_path}")
+    click.echo(f"Sequences:        {num_sequences:,}")
+    click.echo(f"Model:            ESMC-{model}")
+    click.echo(f"Layer:            {layer}")
+    click.echo(f"Keep tokens:      {keep_alignment_tokens!r}")
+    click.echo(f"Chunks:           {num_chunks}")
+    click.echo(f"Seqs per chunk:   ~{actual_seqs_per_chunk:,}")
+    click.echo(f"Queue:            {queue}")
+    click.echo(f"Job definition:   {DEFAULT_JOB_DEFINITION}")
+    click.echo(f"Storage:          {'S3 (remote)' if use_s3 else 'Primordial'}")
+
+    if dry_run:
+        click.echo()
+        click.echo(click.style("Dry run - job not submitted", fg="yellow"))
+        return
+
+    if not click.confirm("\nSubmit job?", default=True):
+        click.echo("Cancelled.")
+        raise SystemExit(0)
+    click.echo()
+
+    s3_prefix = s3_job_prefix(job_id) if use_s3 else None
+    job_dir_path = Path(base_path) / job_id
+
+    if use_s3:
+        import tempfile
+
+        with tempfile.TemporaryDirectory() as tmpdir:
+            tmp_input_dir = Path(tmpdir) / "input"
+            tmp_input_dir.mkdir()
+            click.echo("Splitting FASTA into chunks...")
+            actual_chunks = _split_fasta(input_path, tmp_input_dir, actual_seqs_per_chunk)
+            click.echo(f"Created {actual_chunks} chunks")
+            click.echo("Uploading chunks to S3...")
+            upload_directory(tmp_input_dir, f"{s3_prefix}input/", glob="*.fasta")
+    else:
+        job_dir_path = create_job_directory(job_id, base_path)
+        click.echo(f"Created job directory: {job_dir_path}")
+        click.echo("Splitting FASTA into chunks...")
+        actual_chunks = _split_fasta(input_path, job_dir_path / "input", actual_seqs_per_chunk)
+        click.echo(f"Created {actual_chunks} chunks")
+
+    manifest = JobManifest(
+        job_id=job_id,
+        user=job_id.split("-")[0],
+        pipeline="embed-esmc",
+        status=JobStatus.PENDING,
+        image_uri=DEFAULT_IMAGE_URI,
+        storage_mode="s3" if use_s3 else "primordial",
+        s3_prefix=s3_prefix,
+        input=InputConfig(
+            source=str(input_path),
+            num_sequences=num_sequences,
+            num_chunks=actual_chunks,
+            sequences_per_chunk=actual_seqs_per_chunk,
+        ),
+        batch=BatchConfig(
+            queue=queue,
+            job_definition=DEFAULT_JOB_DEFINITION,
+            array_size=actual_chunks,
+        ),
+        output=OutputConfig(destination=None, finalized=False),
+        depends_on=list(after) if after else None,
+    )
+
+    def _save():
+        if use_s3:
+            save_manifest_s3(manifest)
+        else:
+            save_manifest(manifest, base_path)
+
+    _save()
+    if use_s3:
+        save_local_stub(job_id, s3_prefix)
+
+    try:
+        resolved = [resolve_dependency(jid, base_path) for jid in after]
+        depends_on = [{"jobId": aws_id} for aws_id in resolved if aws_id is not None] or None
+
+        environment = {
+            "JOB_DIR": str(job_dir_path),
+            "JOB_ID": job_id,
+            "ESMC_MODEL": model,
+            "ESMC_LAYER": str(layer),
+            "ESMC_KEEP_ALIGNMENT_TOKENS": keep_alignment_tokens,
+        }
+        if use_s3:
+            environment["STORAGE_MODE"] = "s3"
+            environment["S3_JOB_PREFIX"] = s3_prefix
+
+        batch_job_id = BatchClient().submit_job(
+            job_name=job_id,
+            job_definition=DEFAULT_JOB_DEFINITION,
+            job_queue=queue,
+            array_size=actual_chunks,
+            environment=environment,
+            timeout_seconds=6 * 3600,
+            retry_attempts=3,
+            depends_on=depends_on,
+            share_identifier=get_aws_username(),
+        )
+    except BatchError as e:
+        manifest.status = JobStatus.FAILED
+        manifest.error_message = str(e)
+        _save()
+        click.echo(click.style(f"✗ Failed to submit job: {e}", fg="red"), err=True)
+        raise SystemExit(1)
+
+    manifest.status = JobStatus.SUBMITTED
+    manifest.batch.job_id = batch_job_id
+    _save()
+
+    click.echo()
+    click.echo(click.style("✓ Job submitted successfully!", fg="green"))
+    click.echo()
+    click.echo(f"AWS Batch Job ID: {batch_job_id}")
+    click.echo()
+    click.echo("Next steps:")
+    click.echo(f"  Check status:  dh batch status {job_id}")
+    click.echo(f"  View logs:     dh batch logs {job_id}")
+    click.echo(f"  Cancel:        dh batch cancel {job_id}")
+    click.echo()
+    click.echo("After completion:")
+    finalize_output = "./embeddings.h5" if use_s3 else "/primordial/embeddings.h5"
+    click.echo(f"  Finalize:      dh batch finalize {job_id} --output {finalize_output}")

{dh_cli-0.15.0 → dh_cli-0.16.0}/src/dh_cli/batch/commands/finalize.py

@@ -85,7 +85,7 @@ def finalize(job_id, output, force, keep_intermediates, full_output, skip_dedup,
     output_path = Path(output).resolve()
 
     # For embed jobs: if output is an existing directory, append a filename
-    if manifest.pipeline in ("embed-t5", "embed") and output_path.is_dir():
+    if manifest.pipeline in ("embed-t5", "embed", "embed-esmc") and output_path.is_dir():
         output_path = output_path / f"{job_id}.h5"
         click.echo(f"Output is a directory, writing to: {output_path}")
 
@@ -127,7 +127,7 @@ def finalize(job_id, output, force, keep_intermediates, full_output, skip_dedup,
 
     # Finalize based on pipeline type
     click.echo()
-    if manifest.pipeline in ("embed-t5", "embed"):
+    if manifest.pipeline in ("embed-t5", "embed", "embed-esmc"):
         _finalize_embeddings(output_dir, output_path, skip_dedup=skip_dedup)
     elif manifest.pipeline == "boltz":
         # Check if S3 essential mode was used — output is already filtered

{dh_cli-0.15.0 → dh_cli-0.16.0}/src/dh_cli/batch/commands/retry.py

@@ -58,6 +58,19 @@ def retry(job_id, indices, queue, reslice, dry_run, base_path):
         click.echo(f"Job not found: {job_id}", err=True)
         raise SystemExit(1)
 
+    if manifest.pipeline == "embed-esmc":
+        click.echo(
+            click.style(
+                "Refusing to retry an embed-esmc job: retry does not carry the ESMC config "
+                "(ESMC_MODEL/ESMC_LAYER/ESMC_KEEP_ALIGNMENT_TOKENS), so retried chunks would "
+                "silently use the worker defaults. Resubmit the job, or run `dh batch embed-esmc` "
+                "on the failed chunks' sequences with the original options.",
+                fg="red",
+            ),
+            err=True,
+        )
+        raise SystemExit(1)
+
     # Get failed indices
     if indices:
         # User specified indices

{dh_cli-0.15.0 → dh_cli-0.16.0}/src/dh_cli/bedrock/cost_report.py

@@ -104,6 +104,7 @@ def classify_arn(arn: str) -> Principal:
 
 
 _BASE_MODELS = (
+    "opus-5-5",
     "opus-5",
     "opus-4-8",
     "opus-4-7",

{dh_cli-0.15.0 → dh_cli-0.16.0}/src/dh_cli/bedrock/pricing.yaml

@@ -1,6 +1,11 @@
 inputTokenCount_inclusive: false
 
 models:
+  opus-5-5:
+    input_per_1m: 4.00
+    output_per_1m: 20.00
+    cache_write_per_1m: 5.00
+    cache_read_per_1m: 0.20
   opus-5:
     input_per_1m: 5.00
     output_per_1m: 25.00

{dh_cli-0.15.0 → dh_cli-0.16.0}/tests/bedrock/test_resolve_base_model.py

@@ -43,6 +43,15 @@ import pytest
         ("anthropic.claude-fable-5-1", "fable-5-1"),
         ("us.anthropic.claude-fable-5-1", "fable-5-1"),
         ("global.anthropic.claude-fable-5-1", "fable-5-1"),
+        # Same trap, second occurrence: `opus-5` is a substring of
+        # `claude-opus-5-5`. Here the newer model is *cheaper* ($4/$20 vs
+        # $5/$25), so the silent failure over-reports 5.5 spend by ~25%.
+        ("anthropic.claude-opus-5", "opus-5"),
+        ("us.anthropic.claude-opus-5", "opus-5"),
+        ("global.anthropic.claude-opus-5", "opus-5"),
+        ("anthropic.claude-opus-5-5", "opus-5-5"),
+        ("us.anthropic.claude-opus-5-5", "opus-5-5"),
+        ("global.anthropic.claude-opus-5-5", "opus-5-5"),
     ],
 )
 def test_resolve_base_model_known(model_id: str, expected_base: str):
@@ -63,6 +72,8 @@ def test_resolve_prefers_the_most_specific_base():
         cost_report._BASE_MODELS = tuple(reversed(original))
         assert cost_report.resolve_base_model("global.anthropic.claude-fable-5-1") == "fable-5-1"
         assert cost_report.resolve_base_model("global.anthropic.claude-fable-5") == "fable-5"
+        assert cost_report.resolve_base_model("global.anthropic.claude-opus-5-5") == "opus-5-5"
+        assert cost_report.resolve_base_model("global.anthropic.claude-opus-5") == "opus-5"
     finally:
         cost_report._BASE_MODELS = original
 
@@ -91,6 +102,21 @@ def test_fable_51_cache_reads_are_priced_below_fable_5():
         assert pricing["fable-5-1"][rate] == pricing["fable-5"][rate]
 
 
+def test_opus_55_is_priced_below_opus_5():
+    """5.5 undercuts 5 on every dimension; a substring collapse hides it."""
+    from dh_cli.bedrock.cost_report import default_pricing_path, load_pricing
+
+    pricing = load_pricing(default_pricing_path())
+    assert pricing["opus-5-5"] == {
+        "input_per_1m": 4.00,
+        "output_per_1m": 20.00,
+        "cache_write_per_1m": 5.00,
+        "cache_read_per_1m": 0.20,
+    }
+    for rate in ("input_per_1m", "output_per_1m", "cache_write_per_1m", "cache_read_per_1m"):
+        assert pricing["opus-5-5"][rate] < pricing["opus-5"][rate]
+
+
 def test_resolve_base_model_unknown_raises():
     from dh_cli.bedrock.cost_report import UnknownModel, resolve_base_model
 

dh_cli-0.16.0/tests/test_embed_esmc_cli.py

@@ -0,0 +1,195 @@
+"""Tests for the `dh batch embed-esmc` CLI command, plus its finalize/retry hooks.
+
+All AWS + S3 interactions are mocked; no real Batch or S3 calls.
+"""
+
+import importlib
+import re
+import tempfile
+from pathlib import Path
+from unittest.mock import MagicMock, patch
+
+import h5py
+import numpy as np
+import pytest
+from click.testing import CliRunner
+
+from dh_cli.batch.manifest import BatchConfig, JobManifest, JobStatus, OutputConfig
+
+# `commands/__init__.py` re-exports Click commands under the module names, so
+# `from dh_cli.batch.commands import retry` would return the command, not the module.
+esmc_cmd = importlib.import_module("dh_cli.batch.commands.embed_esmc")
+fin = importlib.import_module("dh_cli.batch.commands.finalize")
+retry_cmd = importlib.import_module("dh_cli.batch.commands.retry")
+
+JOB_ID_RE = re.compile(r"^[a-z0-9]+-esmc-\d{8}-[0-9a-f]{8}$")
+
+
+@pytest.fixture
+def runner():
+    return CliRunner()
+
+
+@pytest.fixture
+def fasta():
+    with tempfile.TemporaryDirectory() as tmpdir:
+        path = Path(tmpdir) / "in.fasta"
+        path.write_text(">a\nMKTAYIAKQR\n>b\nMKT-AYIAK\n>c\nMSTNPKPQRK\n")
+        yield path
+
+
+def _invoke_submit(runner, input_path, args=(), primordial=False):
+    client = MagicMock()
+    client.submit_job.return_value = "aws-job-123"
+    captured = {}
+    with (
+        patch.object(esmc_cmd, "BatchClient", return_value=client),
+        patch.object(esmc_cmd, "resolve_dependency", return_value="dep-aws-id"),
+        patch.object(esmc_cmd, "upload_directory") as up,
+        patch.object(esmc_cmd, "save_manifest_s3", side_effect=lambda m: captured.update(manifest=m)),
+        patch.object(esmc_cmd, "save_manifest", side_effect=lambda m, b: captured.update(manifest=m)),
+        patch.object(esmc_cmd, "save_local_stub"),
+        patch.object(esmc_cmd, "get_aws_username", return_value="tester"),
+        patch.object(esmc_cmd, "_is_primordial_path", return_value=primordial),
+    ):
+        result = runner.invoke(
+            esmc_cmd.embed_esmc,
+            [str(input_path), "--base-path", str(Path(input_path).parent / "_jobs"), *args],
+            input="y\n",
+        )
+    return result, client, up, captured.get("manifest")
+
+
+def test_dry_run_prints_plan_no_submit(runner, fasta):
+    client = MagicMock()
+    with patch.object(esmc_cmd, "BatchClient", return_value=client):
+        result = runner.invoke(
+            esmc_cmd.embed_esmc,
+            [str(fasta), "--dry-run", "--model", "300m", "--layer", "12", "--keep-alignment-tokens", ".|"],
+        )
+    assert result.exit_code == 0, result.output
+    client.submit_job.assert_not_called()
+    out = result.output
+    assert "300m" in out
+    assert "12" in out
+    assert ".|" in out
+    assert "dayhoff-embed-esmc" in out
+    assert "Dry run" in out
+
+
+def test_model_6b_rejected(runner, fasta):
+    result = runner.invoke(esmc_cmd.embed_esmc, [str(fasta), "--dry-run", "--model", "6b"])
+    assert result.exit_code != 0
+    assert "Invalid value" in result.output
+
+
+def test_submit_default_env_s3_mode(runner, fasta):
+    result, client, up, _ = _invoke_submit(runner, fasta)
+    assert result.exit_code == 0, result.output
+    kwargs = client.submit_job.call_args.kwargs
+    assert kwargs["job_definition"] == "dayhoff-embed-esmc"
+    env = kwargs["environment"]
+    assert env["ESMC_MODEL"] == "600m"
+    assert env["ESMC_LAYER"] == "-1"
+    assert env["ESMC_KEEP_ALIGNMENT_TOKENS"] == "-"
+    assert env["STORAGE_MODE"] == "s3"
+    assert env["S3_JOB_PREFIX"]
+    assert env["JOB_ID"] == kwargs["job_name"]
+    up.assert_called()
+
+
+def test_submit_overrides_env_primordial_mode(runner, fasta):
+    result, client, up, _ = _invoke_submit(
+        runner,
+        fasta,
+        ["--model", "300m", "--layer", "20", "--keep-alignment-tokens", ""],
+        primordial=True,
+    )
+    assert result.exit_code == 0, result.output
+    env = client.submit_job.call_args.kwargs["environment"]
+    assert env["ESMC_MODEL"] == "300m"
+    assert env["ESMC_LAYER"] == "20"
+    assert env["ESMC_KEEP_ALIGNMENT_TOKENS"] == ""
+    assert "STORAGE_MODE" not in env
+    assert "S3_JOB_PREFIX" not in env
+    up.assert_not_called()
+
+
+def test_manifest_pipeline_and_job_id(runner, fasta):
+    result, client, _, manifest = _invoke_submit(runner, fasta)
+    assert result.exit_code == 0, result.output
+    assert manifest.pipeline == "embed-esmc"
+    assert manifest.batch.job_definition == "dayhoff-embed-esmc"
+    assert manifest.image_uri.endswith("dayhoff:esmc-latest")
+    assert JOB_ID_RE.match(client.submit_job.call_args.kwargs["job_name"])
+
+
+def test_default_queue_is_a10g_spot(runner, fasta):
+    # Set from the slice-4 smoke runs: A10G ~79 seq/s vs T4 ~6 seq/s at 600M bf16.
+    result, client, _, _ = _invoke_submit(runner, fasta)
+    assert result.exit_code == 0, result.output
+    assert client.submit_job.call_args.kwargs["job_queue"] == "a10g-1x-spot"
+
+
+def test_registered_in_batch_cli():
+    from dh_cli.batch import batch_cli
+
+    assert "embed-esmc" in batch_cli.commands
+    result = CliRunner().invoke(batch_cli, ["--help"])
+    assert "embed-esmc" in result.output
+
+
+def _manifest(pipeline="embed-esmc"):
+    return JobManifest(
+        job_id="tester-esmc-20260924-deadbeef",
+        user="tester",
+        pipeline=pipeline,
+        status=JobStatus.SUCCEEDED,
+        batch=BatchConfig(queue="t4-1x-spot", job_definition="dayhoff-embed-esmc", array_size=2),
+        output=OutputConfig(destination=None, finalized=False),
+    )
+
+
+def _write_h5(path: Path, ids: list[str], dim: int = 4):
+    with h5py.File(path, "w") as f:
+        f.create_dataset("ids", data=np.array(ids, dtype="S"))
+        f.create_dataset("vectors", data=np.random.rand(len(ids), dim).astype(np.float32))
+
+
+def test_finalize_merges_embed_h5(runner, tmp_path):
+    job_dir = tmp_path / "tester-esmc-20260924-deadbeef"
+    out_dir = job_dir / "output"
+    out_dir.mkdir(parents=True)
+    _write_h5(out_dir / "embed_000.h5", ["a", "b"])
+    _write_h5(out_dir / "embed_001.h5", ["c"])
+    for i in range(2):
+        (out_dir / f"embed_{i:03d}.done").touch()
+    dest = tmp_path / "merged.h5"
+    with (
+        patch.object(fin, "load_manifest", return_value=_manifest()),
+        patch.object(fin, "save_manifest"),
+        patch.object(fin, "get_job_dir", return_value=job_dir),
+        patch.object(fin, "_check_completion", return_value=[]),
+    ):
+        result = runner.invoke(
+            fin.finalize,
+            [job_dir.name, "--output", str(dest), "--keep-intermediates", "--base-path", str(tmp_path)],
+        )
+    assert result.exit_code == 0, result.output
+    with h5py.File(dest) as f:
+        assert sorted(x.decode() for x in f["ids"][:]) == ["a", "b", "c"]
+        assert f["vectors"].shape == (3, 4)
+
+
+def test_retry_refuses_embed_esmc(runner):
+    client = MagicMock()
+    with (
+        patch.object(retry_cmd, "load_manifest", return_value=_manifest()),
+        patch.object(retry_cmd, "_find_incomplete_chunks", return_value=[1]),
+        patch.object(retry_cmd, "BatchClient", return_value=client),
+    ):
+        result = runner.invoke(retry_cmd.retry, ["tester-esmc-20260924-deadbeef", "--indices", "1"])
+    assert result.exit_code != 0
+    assert "embed-esmc" in result.output
+    client.submit_job.assert_not_called()
+    client.submit_array_job_with_indices.assert_not_called()