dh-cli 0.10.0__tar.gz → 0.10.2__tar.gz

{dh_cli-0.10.0 → dh_cli-0.10.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dh-cli
-Version: 0.10.0
+Version: 0.10.2
 Summary: Dayhoff Labs developer CLI
 Author-email: Dayhoff Labs <dev@dayhofflabs.com>
 License: # PolyForm Noncommercial License 1.0.0

{dh_cli-0.10.0 → dh_cli-0.10.2}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "dh-cli"
-version = "0.10.0"
+version = "0.10.2"
 description = "Dayhoff Labs developer CLI"
 requires-python = ">=3.11"
 readme = "README.md"

{dh_cli-0.10.0 → dh_cli-0.10.2}/src/dh_cli/batch/__init__.py

@@ -15,6 +15,7 @@ from .commands.cancel import cancel
 from .commands.clean import clean
 from .commands.embed_t5 import embed_t5
 from .commands.equilibrator import equilibrator
+from .commands.esmfold2 import esmfold2
 from .commands.finalize import finalize
 from .commands.list_jobs import list_jobs
 from .commands.local import local
@@ -53,6 +54,7 @@ def batch_cli():
     \b
     Structure Prediction:
       boltz      Predict protein structures with Boltz
+      esmfold2   Predict protein structures with ESMFold2
 
     \b
     Sequence Design:
@@ -101,6 +103,7 @@ batch_cli.add_command(wait_for, name="wait-for")
 # Register pipeline commands
 batch_cli.add_command(embed_t5, name="embed-t5")
 batch_cli.add_command(equilibrator, name="equilibrator")
+batch_cli.add_command(esmfold2)
 batch_cli.add_command(boltz)
 batch_cli.add_command(protmpnn)
 batch_cli.add_command(orca)

dh_cli-0.10.2/src/dh_cli/batch/commands/esmfold2.py

@@ -0,0 +1,242 @@
+"""ESMFold2 protein structure prediction pipeline command."""
+
+import math
+import secrets
+import shutil
+from datetime import datetime
+from pathlib import Path
+
+import click
+
+from dh_batch.esmfold2_config import (
+    declares_at_least_one_sequence,
+    discover_esmfold2_configs,
+)
+
+from ..aws_batch import BatchClient, BatchError, resolve_dependency
+from ..job_id import get_aws_username
+from ..manifest import (
+    BATCH_JOBS_BASE,
+    BatchConfig,
+    InputConfig,
+    JobManifest,
+    JobStatus,
+    OutputConfig,
+    create_job_directory,
+    save_local_stub,
+    save_manifest,
+    save_manifest_s3,
+)
+from ..s3_transport import s3_job_prefix, upload_directory
+
+DEFAULT_QUEUE = "l40s-1x-spot"
+DEFAULT_JOB_DEFINITION = "dayhoff-esmfold2"
+DEFAULT_IMAGE_URI = "074735440724.dkr.ecr.us-east-1.amazonaws.com/dayhoff:esmfold2-latest"
+MAX_WORKERS = 50
+FILES_PER_WORKER = 1
+RETRY_ATTEMPTS = 5
+TIMEOUT_SECONDS = 2 * 3600
+
+
+def _generate_job_id() -> str:
+    """Job ID as ``{username}-esmfold2-{YYYYMMDD}-{random8}``.
+
+    Built here (not via ``generate_job_id``) so the full ``esmfold2`` pipeline
+    name survives — the generic helper truncates to 10 chars.
+    """
+    username = get_aws_username()
+    date_str = datetime.now().strftime("%Y%m%d")
+    return f"{username}-esmfold2-{date_str}-{secrets.token_hex(4)}"
+
+
+def _is_primordial_path(path: Path) -> bool:
+    """Check if a path is on Primordial (EFS)."""
+    return str(path).startswith("/primordial/")
+
+
+@click.command()
+@click.argument("input_dir", type=click.Path(exists=True))
+@click.option(
+    "--workers",
+    default=None,
+    type=int,
+    help="Number of parallel workers [default: ~1 per file]",
+)
+@click.option("--queue", default=DEFAULT_QUEUE, help=f"Batch queue [default: {DEFAULT_QUEUE}]")
+@click.option("--after", "after", multiple=True, help="Job ID(s) to wait for before starting")
+@click.option("--dry-run", is_flag=True, help="Show plan without submitting")
+@click.option("--base-path", default=BATCH_JOBS_BASE, help="Base path for job data")
+def esmfold2(input_dir, workers, queue, after, dry_run, base_path):
+    """Predict protein structures with ESMFold2.
+
+    Processes a directory of per-input YAML/JSON configs, each declaring the
+    entities to fold, and runs them in parallel using AWS Batch array jobs.
+
+    \b
+    Examples:
+      # Submit to AWS Batch (auto-sized workers)
+      dh batch esmfold2 input/
+
+      # Force a specific worker count
+      dh batch esmfold2 input/ --workers 20
+
+      # Preview what would run
+      dh batch esmfold2 input/ --dry-run
+    """
+    input_path = Path(input_dir).resolve()
+
+    click.echo(f"Scanning {input_path} for configs...")
+    try:
+        configs = discover_esmfold2_configs(input_path)
+    except ValueError as e:
+        raise click.ClickException(str(e))
+
+    for cfg in configs:
+        try:
+            declares_at_least_one_sequence(cfg)
+        except ValueError as e:
+            raise click.ClickException(f"Invalid config {cfg.name}: {e}")
+
+    num_files = len(configs)
+    click.echo(f"Found {num_files} config(s) to process")
+
+    if workers is None:
+        workers = max(1, min(math.ceil(num_files / FILES_PER_WORKER), MAX_WORKERS))
+    array_size = max(1, min(workers, num_files))
+
+    use_s3 = not _is_primordial_path(input_path)
+
+    job_id = _generate_job_id()
+
+    click.echo()
+    click.echo(f"Job ID:           {job_id}")
+    click.echo(f"Input:            {input_path}")
+    click.echo(f"Configs:          {num_files}")
+    click.echo(f"Workers:          {array_size}")
+    click.echo(f"Queue:            {queue}")
+    click.echo(f"Job definition:   {DEFAULT_JOB_DEFINITION}")
+    click.echo(f"Image:            {DEFAULT_IMAGE_URI}")
+    click.echo(f"Storage:          {'S3 (remote)' if use_s3 else 'Primordial'}")
+
+    if dry_run:
+        click.echo()
+        click.echo(click.style("Dry run - job not submitted", fg="yellow"))
+        return
+
+    if not click.confirm("\nSubmit job?", default=True):
+        click.echo("Cancelled.")
+        raise SystemExit(0)
+    click.echo()
+
+    client = BatchClient()
+
+    response = client.batch.describe_job_definitions(jobDefinitionName=DEFAULT_JOB_DEFINITION, status="ACTIVE")
+    if not response.get("jobDefinitions"):
+        raise click.ClickException(
+            "esmfold2 job definition not yet deployed; apply Terraform in environments/dev first"
+        )
+
+    s3_prefix = s3_job_prefix(job_id) if use_s3 else None
+    job_dir_path = Path(base_path) / job_id
+
+    if use_s3:
+        click.echo("Uploading configs to S3...")
+        upload_directory(input_path, f"{s3_prefix}input/", glob="*.yaml")
+        upload_directory(input_path, f"{s3_prefix}input/", glob="*.json")
+        click.echo(f"Uploaded {num_files} config(s) to S3")
+    else:
+        job_dir_path = create_job_directory(job_id, base_path)
+        click.echo(f"Created job directory: {job_dir_path}")
+        dest = job_dir_path / "input"
+        dest.mkdir(parents=True, exist_ok=True)
+        for cfg in configs:
+            shutil.copy2(cfg, dest / cfg.name)
+        click.echo(f"Copied {num_files} config(s)")
+
+    manifest = JobManifest(
+        job_id=job_id,
+        user=job_id.split("-")[0],
+        pipeline="esmfold2",
+        status=JobStatus.PENDING,
+        image_uri=DEFAULT_IMAGE_URI,
+        storage_mode="s3" if use_s3 else "primordial",
+        s3_prefix=s3_prefix,
+        input=InputConfig(
+            source=str(input_path),
+            num_sequences=num_files,
+            num_chunks=array_size,
+        ),
+        batch=BatchConfig(
+            queue=queue,
+            job_definition=DEFAULT_JOB_DEFINITION,
+            array_size=array_size,
+        ),
+        output=OutputConfig(
+            destination=None,
+            finalized=False,
+        ),
+        depends_on=list(after) if after else None,
+    )
+
+    if use_s3:
+        save_manifest_s3(manifest)
+        save_local_stub(job_id, s3_prefix)
+    else:
+        save_manifest(manifest, base_path)
+
+    try:
+        resolved = [resolve_dependency(jid, base_path) for jid in after]
+        depends_on = [{"jobId": aws_id} for aws_id in resolved if aws_id is not None] or None
+
+        environment = {
+            "JOB_DIR": str(job_dir_path),
+            "JOB_ID": job_id,
+            "BATCH_ARRAY_SIZE": str(array_size),
+        }
+        if use_s3:
+            environment["STORAGE_MODE"] = "s3"
+            environment["S3_JOB_PREFIX"] = s3_prefix
+
+        batch_job_id = client.submit_job(
+            job_name=job_id,
+            job_definition=DEFAULT_JOB_DEFINITION,
+            job_queue=queue,
+            array_size=array_size,
+            environment=environment,
+            timeout_seconds=TIMEOUT_SECONDS,
+            retry_attempts=RETRY_ATTEMPTS,
+            depends_on=depends_on,
+            share_identifier=get_aws_username(),
+        )
+
+        manifest.status = JobStatus.SUBMITTED
+        manifest.batch.job_id = batch_job_id
+        if use_s3:
+            save_manifest_s3(manifest)
+        else:
+            save_manifest(manifest, base_path)
+
+        click.echo()
+        click.echo(click.style("Job submitted successfully!", fg="green"))
+        click.echo()
+        click.echo(f"AWS Batch Job ID: {batch_job_id}")
+        if depends_on:
+            click.echo(f"Waiting on:       {', '.join(after)}")
+        click.echo()
+        click.echo("Next steps:")
+        click.echo(f"  Check status:  dh batch status {job_id}")
+        click.echo(f"  View logs:     dh batch logs {job_id}")
+        click.echo(f"  Cancel:        dh batch cancel {job_id}")
+        click.echo()
+        click.echo("After completion:")
+        click.echo(f"  Finalize:      dh batch finalize {job_id} --output ./results/")
+
+    except BatchError as e:
+        manifest.status = JobStatus.FAILED
+        manifest.error_message = str(e)
+        if use_s3:
+            save_manifest_s3(manifest)
+        else:
+            save_manifest(manifest, base_path)
+        click.echo(click.style(f"Failed to submit job: {e}", fg="red"), err=True)
+        raise SystemExit(1)

{dh_cli-0.10.0 → dh_cli-0.10.2}/src/dh_cli/batch/commands/finalize.py

@@ -146,6 +146,8 @@ def finalize(job_id, output, force, keep_intermediates, full_output, skip_dedup,
         _finalize_protmpnn(output_dir, output_path)
     elif manifest.pipeline == "rfd3":
         _finalize_rfd3(output_dir, output_path)
+    elif manifest.pipeline == "esmfold2":
+        _finalize_esmfold2(output_dir, output_path)
     elif manifest.pipeline == "equilibrator":
         _finalize_equilibrator(output_dir, output_path, job_id=job_id)
     else:
@@ -585,6 +587,40 @@ def _finalize_rfd3(output_dir: Path, output_path: Path):
     click.echo(click.style(f"Results: {output_path}", fg="green"))
 
 
+def _finalize_esmfold2(output_dir: Path, output_path: Path):
+    """Collate completed ESMFold2 per-config prediction dirs into the final output.
+
+    The worker promotes each config's structure + confidence files into
+    `output/<stem>/` and drops a `output/esmfold2_<stem>.done` marker only
+    when the prediction succeeded. Finalize copies each completed config's
+    directory into `output_path/<stem>/`, skipping configs without a `.done`
+    marker and the loose worker markers.
+    """
+    completed = []
+    for marker in sorted(output_dir.glob("esmfold2_*.done")):
+        stem = marker.name[len("esmfold2_") : -len(".done")]
+        if (output_dir / stem).is_dir():
+            completed.append(stem)
+
+    if not completed:
+        click.echo("No completed ESMFold2 configs found in output directory.", err=True)
+        raise SystemExit(1)
+
+    output_path.mkdir(parents=True, exist_ok=True)
+
+    copied_count = 0
+    for stem in completed:
+        dest = output_path / stem
+        if dest.exists():
+            click.echo(f"  Skipping {stem} (already exists)")
+            continue
+        shutil.copytree(output_dir / stem, dest)
+        copied_count += 1
+
+    click.echo(f"Collated {copied_count} ESMFold2 config(s)")
+    click.echo(click.style(f"Results: {output_path}", fg="green"))
+
+
 def _finalize_equilibrator(output_dir: Path, output_path: Path, job_id: str | None = None) -> None:
     """Combine per-chunk JSONL worker outputs into one sorted JSONL file.
 

{dh_cli-0.10.0 → dh_cli-0.10.2}/src/dh_cli/bedrock/cost_report.py

@@ -103,7 +103,7 @@ def classify_arn(arn: str) -> Principal:
     return Principal("unknown", arn)
 
 
-_BASE_MODELS = ("opus-4-7", "opus-4-6", "sonnet-4-6", "sonnet-4-5", "haiku-4-5")
+_BASE_MODELS = ("opus-4-8", "opus-4-7", "opus-4-6", "sonnet-4-6", "sonnet-4-5", "haiku-4-5", "fable-5")
 
 
 def resolve_base_model(model_id: str) -> str:

dh_cli-0.10.2/src/dh_cli/bedrock/pricing.yaml

@@ -0,0 +1,38 @@
+inputTokenCount_inclusive: false
+
+models:
+  opus-4-8:
+    input_per_1m: 5.00
+    output_per_1m: 25.00
+    cache_write_per_1m: 6.25
+    cache_read_per_1m: 0.50
+  opus-4-7:
+    input_per_1m: 5.00
+    output_per_1m: 25.00
+    cache_write_per_1m: 6.25
+    cache_read_per_1m: 0.50
+  opus-4-6:
+    input_per_1m: 5.00
+    output_per_1m: 25.00
+    cache_write_per_1m: 6.25
+    cache_read_per_1m: 0.50
+  fable-5:
+    input_per_1m: 10.00
+    output_per_1m: 50.00
+    cache_write_per_1m: 12.50
+    cache_read_per_1m: 1.00
+  sonnet-4-6:
+    input_per_1m: 3.00
+    output_per_1m: 15.00
+    cache_write_per_1m: 3.75
+    cache_read_per_1m: 0.30
+  sonnet-4-5:
+    input_per_1m: 3.00
+    output_per_1m: 15.00
+    cache_write_per_1m: 3.75
+    cache_read_per_1m: 0.30
+  haiku-4-5:
+    input_per_1m: 0.80
+    output_per_1m: 4.00
+    cache_write_per_1m: 1.00
+    cache_read_per_1m: 0.08

{dh_cli-0.10.0 → dh_cli-0.10.2}/src/dh_cli/engines_studios/engine_commands.py

@@ -28,7 +28,7 @@ def engine_cli():
     "--type",
     "engine_type",
     required=True,
-    type=click.Choice(["cpu", "cpumax", "t4", "a10g", "a100", "4_t4", "8_t4", "4_a10g", "8_a10g"]),
+    type=click.Choice(["cpu", "cpumax", "t4", "a10g", "l40s", "a100", "4_t4", "8_t4", "4_a10g", "8_a10g"]),
 )
 @click.option("--size", "boot_disk_size", type=int, help="Boot disk size in GB")
 @click.option(

{dh_cli-0.10.0 → dh_cli-0.10.2}/tests/bedrock/test_cost_calc.py

@@ -25,11 +25,11 @@ def test_cost_fixture_A_cache_write(load_fixture, pricing):
     from dh_cli.bedrock.cost_report import cost_of
 
     record = load_fixture("A_cache_write.json")
-    # 100  * 15.00 / 1e6 = 0.0015
-    # 2000 * 18.75 / 1e6 = 0.0375
-    # 500  * 75.00 / 1e6 = 0.0375
-    # total             = 0.0765
-    expected = 0.0015 + 0.0375 + 0.0375
+    # 100  * 5.00  / 1e6 = 0.0005
+    # 2000 * 6.25  / 1e6 = 0.0125
+    # 500  * 25.00 / 1e6 = 0.0125
+    # total             = 0.0255
+    expected = 0.0005 + 0.0125 + 0.0125
     assert cost_of(record, pricing) == _approx(expected)
 
 
@@ -38,10 +38,10 @@ def test_cost_fixture_B_cache_read(load_fixture, pricing):
     from dh_cli.bedrock.cost_report import cost_of
 
     record = load_fixture("B_cache_read.json")
-    # 1    * 15.00 / 1e6 = 0.000015
-    # 2000 * 1.50  / 1e6 = 0.003
-    # 500  * 75.00 / 1e6 = 0.0375
-    expected = 0.000015 + 0.003 + 0.0375
+    # 1    * 5.00  / 1e6 = 0.000005
+    # 2000 * 0.50  / 1e6 = 0.001
+    # 500  * 25.00 / 1e6 = 0.0125
+    expected = 0.000005 + 0.001 + 0.0125
     assert cost_of(record, pricing) == _approx(expected)
 
 
@@ -50,9 +50,9 @@ def test_cost_fixture_C_plain(load_fixture, pricing):
     from dh_cli.bedrock.cost_report import cost_of
 
     record = load_fixture("C_plain.json")
-    # 100 * 15.00 / 1e6 = 0.0015
-    # 500 * 75.00 / 1e6 = 0.0375
-    expected = 0.0015 + 0.0375
+    # 100 * 5.00  / 1e6 = 0.0005
+    # 500 * 25.00 / 1e6 = 0.0125
+    expected = 0.0005 + 0.0125
     assert cost_of(record, pricing) == _approx(expected)
 
 
@@ -103,11 +103,59 @@ def test_cost_exclusive_semantics_sanity(pricing):
         "output": {"outputTokenCount": 0},
     }
     # Exclusive (pricing as-loaded, flag = false):
-    #   100 * 15 / 1e6 + 40 * 1.50 / 1e6
+    #   100 * 5 / 1e6 + 40 * 0.50 / 1e6
     exclusive = cost_of(record, pricing)
     inclusive_pricing = {**pricing, "inputTokenCount_inclusive": True}
-    #   (100 - 40) * 15 / 1e6 + 40 * 1.50 / 1e6
+    #   (100 - 40) * 5 / 1e6 + 40 * 0.50 / 1e6
     inclusive = cost_of(record, inclusive_pricing)
     assert exclusive > inclusive
-    assert math.isclose(inclusive, (60 * 15 + 40 * 1.50) / 1_000_000, rel_tol=1e-9)
-    assert math.isclose(exclusive, (100 * 15 + 40 * 1.50) / 1_000_000, rel_tol=1e-9)
+    assert math.isclose(inclusive, (60 * 5 + 40 * 0.50) / 1_000_000, rel_tol=1e-9)
+    assert math.isclose(exclusive, (100 * 5 + 40 * 0.50) / 1_000_000, rel_tol=1e-9)
+
+
+def test_cost_opus_4_8_prices_nonzero(pricing):
+    """Opus 4.8 (global inference profile) must price at the Bedrock
+    Opus tier: input 5 / output 25 / cache_write 6.25 / cache_read 0.50."""
+    from dh_cli.bedrock.cost_report import cost_of
+
+    record = {
+        "modelId": "global.anthropic.claude-opus-4-8",
+        "input": {
+            "inputTokenCount": 100,
+            "cacheReadInputTokenCount": 2000,
+            "cacheWriteInputTokenCount": 1000,
+        },
+        "output": {"outputTokenCount": 500},
+    }
+    # 100  * 5.00  / 1e6 = 0.0005
+    # 500  * 25.00 / 1e6 = 0.0125
+    # 2000 * 0.50  / 1e6 = 0.001
+    # 1000 * 6.25  / 1e6 = 0.00625
+    expected = 0.0005 + 0.0125 + 0.001 + 0.00625
+    result = cost_of(record, pricing)
+    assert result > 0
+    assert result == _approx(expected)
+
+
+def test_cost_fable_5_prices_nonzero(pricing):
+    """Fable 5 (global inference profile) must price at the Fable tier:
+    input 10 / output 50 / cache_write 12.50 / cache_read 1.00."""
+    from dh_cli.bedrock.cost_report import cost_of
+
+    record = {
+        "modelId": "global.anthropic.claude-fable-5",
+        "input": {
+            "inputTokenCount": 100,
+            "cacheReadInputTokenCount": 2000,
+            "cacheWriteInputTokenCount": 1000,
+        },
+        "output": {"outputTokenCount": 500},
+    }
+    # 100  * 10.00 / 1e6 = 0.001
+    # 500  * 50.00 / 1e6 = 0.025
+    # 2000 * 1.00  / 1e6 = 0.002
+    # 1000 * 12.50 / 1e6 = 0.0125
+    expected = 0.001 + 0.025 + 0.002 + 0.0125
+    result = cost_of(record, pricing)
+    assert result > 0
+    assert result == _approx(expected)

{dh_cli-0.10.0 → dh_cli-0.10.2}/tests/bedrock/test_resolve_base_model.py

@@ -20,6 +20,9 @@ import pytest
         ("anthropic.claude-opus-4-7", "opus-4-7"),
         ("us.anthropic.claude-opus-4-7", "opus-4-7"),
         ("global.anthropic.claude-opus-4-7", "opus-4-7"),
+        ("anthropic.claude-opus-4-8", "opus-4-8"),
+        ("us.anthropic.claude-opus-4-8", "opus-4-8"),
+        ("global.anthropic.claude-opus-4-8", "opus-4-8"),
         ("anthropic.claude-opus-4-6-v1", "opus-4-6"),
         ("us.anthropic.claude-opus-4-6-v1", "opus-4-6"),
         ("global.anthropic.claude-opus-4-6-v1", "opus-4-6"),
@@ -29,6 +32,9 @@ import pytest
         ("anthropic.claude-haiku-4-5-20251001-v1:0", "haiku-4-5"),
         ("us.anthropic.claude-haiku-4-5-20251001-v1:0", "haiku-4-5"),
         ("global.anthropic.claude-haiku-4-5-20251001-v1:0", "haiku-4-5"),
+        ("anthropic.claude-fable-5", "fable-5"),
+        ("us.anthropic.claude-fable-5", "fable-5"),
+        ("global.anthropic.claude-fable-5", "fable-5"),
     ],
 )
 def test_resolve_base_model_known(model_id: str, expected_base: str):