dayhoff-tools 1.14.9__tar.gz → 1.14.11__tar.gz

{dayhoff_tools-1.14.9 → dayhoff_tools-1.14.11}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dayhoff-tools
-Version: 1.14.9
+Version: 1.14.11
 Summary: Common tools for all the repos at Dayhoff Labs
 Author: Daniel Martin-Alarcon
 Author-email: dma@dayhofflabs.com

{dayhoff_tools-1.14.9 → dayhoff_tools-1.14.11}/dayhoff_tools/batch/workers/base.py

@@ -30,12 +30,18 @@ def get_array_index() -> int:
 
     For array jobs, reads AWS_BATCH_JOB_ARRAY_INDEX.
     For retry jobs, maps from BATCH_RETRY_INDICES.
+    For resliced retry jobs, uses the raw array index (chunks are renumbered).
     For single jobs (array_size=1), defaults to 0.
 
     Returns:
         The array index this worker should process
     """
-    # Check for retry mode first
+    # For resliced retries, use raw array index (chunks are renumbered 0..N-1)
+    if os.environ.get("RESLICE_PREFIX"):
+        array_idx = os.environ.get("AWS_BATCH_JOB_ARRAY_INDEX", "0")
+        return int(array_idx)
+
+    # Check for retry mode (non-resliced)
     retry_indices = os.environ.get("BATCH_RETRY_INDICES")
     if retry_indices:
         # In retry mode, we have a list of indices and use array index to pick
@@ -69,6 +75,20 @@ def get_job_dir() -> Path:
     return Path(job_dir)
 
 
+def get_reslice_prefix() -> str | None:
+    """Get the reslice prefix from environment if set.
+
+    When RESLICE_PREFIX is set (e.g., 'r1'), files are named like:
+    - Input: chunk_r1_000.fasta
+    - Output: embed_r1_000.h5
+    - Done marker: embed_r1_000.done
+
+    Returns:
+        The reslice prefix or None if not in reslice mode
+    """
+    return os.environ.get("RESLICE_PREFIX")
+
+
 def get_input_file(index: int, job_dir: Path, prefix: str = "chunk") -> Path:
     """Get the input file path for a given index.
 
@@ -80,6 +100,9 @@ def get_input_file(index: int, job_dir: Path, prefix: str = "chunk") -> Path:
     Returns:
         Path to input file
     """
+    reslice = get_reslice_prefix()
+    if reslice:
+        return job_dir / "input" / f"{prefix}_{reslice}_{index:03d}.fasta"
     return job_dir / "input" / f"{prefix}_{index:03d}.fasta"
 
 
@@ -97,6 +120,9 @@ def get_output_file(
     Returns:
         Path to output file
     """
+    reslice = get_reslice_prefix()
+    if reslice:
+        return job_dir / "output" / f"{prefix}_{reslice}_{index:03d}{suffix}"
     return job_dir / "output" / f"{prefix}_{index:03d}{suffix}"
 
 
@@ -111,6 +137,9 @@ def get_done_marker(index: int, job_dir: Path, prefix: str = "embed") -> Path:
     Returns:
         Path to done marker file
     """
+    reslice = get_reslice_prefix()
+    if reslice:
+        return job_dir / "output" / f"{prefix}_{reslice}_{index:03d}.done"
     return job_dir / "output" / f"{prefix}_{index:03d}.done"
 
 

{dayhoff_tools-1.14.9 → dayhoff_tools-1.14.11}/dayhoff_tools/batch/workers/boltz.py

@@ -341,10 +341,31 @@ class BoltzProcessor:
         return expected_output_dir
 
 
+def _get_done_marker_for_file(job_dir: Path, file_stem: str) -> Path:
+    """Get the done marker path for a specific input file."""
+    return job_dir / "output" / f"boltz_{file_stem}.done"
+
+
+def _check_file_complete(job_dir: Path, file_stem: str) -> bool:
+    """Check if a specific file has been processed."""
+    return _get_done_marker_for_file(job_dir, file_stem).exists()
+
+
+def _mark_file_complete(job_dir: Path, file_stem: str):
+    """Mark a specific file as complete."""
+    done_marker = _get_done_marker_for_file(job_dir, file_stem)
+    done_marker.parent.mkdir(parents=True, exist_ok=True)
+    done_marker.touch()
+    logger.info(f"File {file_stem} marked complete: {done_marker}")
+
+
 def main():
-    """Boltz worker main entrypoint for AWS Batch array jobs."""
+    """Boltz worker main entrypoint for AWS Batch array jobs.
+
+    Each worker processes multiple files based on array index and total workers.
+    With N files and M workers, worker i processes files where file_index % M == i.
+    """
     from .base import (
-        check_already_complete,
         configure_worker_logging,
         get_array_index,
         get_job_dir,
@@ -358,34 +379,36 @@ def main():
         # Get configuration from environment
         index = get_array_index()
         job_dir = get_job_dir()
+        array_size = int(os.environ.get("BATCH_ARRAY_SIZE", "1"))
+        num_files = int(os.environ.get("BATCH_NUM_FILES", "0"))
 
         logger.info(f"Worker configuration:")
         logger.info(f"  Array index: {index}")
+        logger.info(f"  Array size: {array_size}")
+        logger.info(f"  Total files: {num_files}")
         logger.info(f"  Job directory: {job_dir}")
 
-        # Check idempotency
-        if check_already_complete(index, job_dir, prefix="boltz"):
-            logger.info("Exiting - complex already processed")
-            return
-
-        # Find input file by index
+        # Find all input files
         input_dir = job_dir / "input"
         input_files = sorted(input_dir.glob("*.yaml"))
+        total_files = len(input_files)
 
-        if index >= len(input_files):
-            logger.error(
-                f"Index {index} out of range. Found {len(input_files)} input files."
-            )
-            raise RuntimeError(f"Index {index} out of range")
+        if total_files == 0:
+            logger.error("No input files found")
+            raise RuntimeError("No input files found")
 
-        input_file = input_files[index]
-        logger.info(f"  Input file: {input_file}")
+        # Calculate which files this worker should process
+        # Worker i processes files where file_index % array_size == index
+        my_files = [f for i, f in enumerate(input_files) if i % array_size == index]
 
-        # Determine output directory
-        output_base = input_file.stem
-        output_dir = job_dir / "output" / output_base
+        logger.info(f"  Files assigned to this worker: {len(my_files)}")
 
-        # Get MSA directories
+        if not my_files:
+            logger.info("No files assigned to this worker, exiting successfully")
+            mark_complete(index, job_dir, prefix="boltz")
+            return
+
+        # Get MSA directories (shared across all files)
         job_msa_dir = job_dir / "msas"
         global_msa_dir = Path(os.environ.get("MSA_DIR", "/primordial/.cache/msas"))
 
@@ -408,7 +431,7 @@ def main():
         if boltz_options:
             logger.info(f"  Boltz options: {boltz_options}")
 
-        # Create processor and run
+        # Create processor (reused for all files)
         processor = BoltzProcessor(
             num_workers=None,  # Auto-detect
             boltz_options=boltz_options,
@@ -416,16 +439,60 @@ def main():
             cache_dir=cache_dir,
         )
 
-        # Ensure output directory exists
-        output_dir.parent.mkdir(parents=True, exist_ok=True)
+        # Process each assigned file
+        completed = 0
+        failed = 0
+
+        for file_idx, input_file in enumerate(my_files):
+            file_stem = input_file.stem
+
+            # Check if this file is already complete (idempotency)
+            if _check_file_complete(job_dir, file_stem):
+                logger.info(
+                    f"[{file_idx + 1}/{len(my_files)}] {file_stem}: "
+                    "already complete, skipping"
+                )
+                completed += 1
+                continue
+
+            logger.info(
+                f"[{file_idx + 1}/{len(my_files)}] Processing {file_stem}..."
+            )
+
+            try:
+                # Determine output directory
+                output_dir = job_dir / "output" / file_stem
+                output_dir.parent.mkdir(parents=True, exist_ok=True)
+
+                result_dir = processor.run(str(input_file), str(output_dir))
+
+                # Mark this file as complete
+                _mark_file_complete(job_dir, file_stem)
+
+                logger.info(
+                    f"[{file_idx + 1}/{len(my_files)}] {file_stem}: "
+                    f"completed successfully -> {result_dir}"
+                )
+                completed += 1
+
+            except Exception as e:
+                logger.error(
+                    f"[{file_idx + 1}/{len(my_files)}] {file_stem}: "
+                    f"failed with error: {e}"
+                )
+                failed += 1
+                # Continue processing other files even if one fails
 
-        result_dir = processor.run(str(input_file), str(output_dir))
+        # Summary
+        logger.info(f"Worker {index} finished: {completed} completed, {failed} failed")
 
-        # Mark as complete
+        # Mark worker as complete
         mark_complete(index, job_dir, prefix="boltz")
 
-        logger.info(f"Complex {input_file.stem} completed successfully")
-        logger.info(f"Output: {result_dir}")
+        if failed > 0:
+            logger.warning(f"{failed} file(s) failed to process")
+            # Don't exit with error - some files succeeded and are marked complete
+            # The failed files can be retried later
 
     except Exception as e:
         logger.exception(f"Worker failed with error: {e}")

{dayhoff_tools-1.14.9 → dayhoff_tools-1.14.11}/dayhoff_tools/cli/batch/commands/boltz.py

@@ -232,6 +232,8 @@ def _submit_batch_job(
             "JOB_ID": job_id,
             "BOLTZ_CACHE": "/primordial/.cache/boltz",
             "MSA_DIR": "/primordial/.cache/msas",
+            "BATCH_ARRAY_SIZE": str(array_size),
+            "BATCH_NUM_FILES": str(num_files),
         }
 
         batch_job_id = client.submit_job(

{dayhoff_tools-1.14.9 → dayhoff_tools-1.14.11}/dayhoff_tools/cli/batch/commands/finalize.py

@@ -118,7 +118,12 @@ def finalize(job_id, output, force, keep_intermediates, full_output, base_path):
 
 
 def _check_completion(job_id: str, base_path: str) -> list[int]:
-    """Check which chunks are incomplete (no .done marker)."""
+    """Check which chunks are incomplete (no .done marker).
+
+    Handles both original chunks (chunk_000.fasta) and resliced chunks
+    (chunk_r1_000.fasta). For original chunks that were resliced in a retry,
+    checks if all resliced chunks completed.
+    """
     job_dir = get_job_dir(job_id, base_path)
     input_dir = job_dir / "input"
     output_dir = job_dir / "output"
@@ -126,13 +131,53 @@ def _check_completion(job_id: str, base_path: str) -> list[int]:
     if not input_dir.exists():
         return []
 
+    # Load manifest to check for resliced retries
+    try:
+        manifest = load_manifest(job_id, base_path)
+        resliced_indices: set[int] = set()
+        reslice_info: dict[str, int] = {}  # prefix -> expected count
+
+        for retry in manifest.retries:
+            if retry.reslice_prefix and retry.reslice_count:
+                resliced_indices.update(retry.indices)
+                reslice_info[retry.reslice_prefix] = retry.reslice_count
+    except FileNotFoundError:
+        resliced_indices = set()
+        reslice_info = {}
+
     incomplete = []
-    for chunk_path in sorted(input_dir.glob("chunk_*.fasta")):
+
+    # Check original chunks (chunk_000.fasta pattern)
+    for chunk_path in sorted(input_dir.glob("chunk_[0-9][0-9][0-9].fasta")):
         idx_str = chunk_path.stem.split("_")[1]
         idx = int(idx_str)
+
+        # Check for original done marker
         done_marker = output_dir / f"embed_{idx:03d}.done"
-        if not done_marker.exists():
-            incomplete.append(idx)
+        if done_marker.exists():
+            continue
+
+        # Check if this chunk was resliced
+        if idx in resliced_indices:
+            # Find which retry covered this index and check if complete
+            is_covered = False
+            for retry in manifest.retries:
+                if (
+                    retry.reslice_prefix
+                    and retry.reslice_count
+                    and idx in retry.indices
+                ):
+                    # Check if all resliced chunks for this retry completed
+                    done_count = len(
+                        list(output_dir.glob(f"embed_{retry.reslice_prefix}_*.done"))
+                    )
+                    if done_count >= retry.reslice_count:
+                        is_covered = True
+                        break
+            if is_covered:
+                continue
+
+        incomplete.append(idx)
 
     return incomplete
 

dayhoff_tools-1.14.11/dayhoff_tools/cli/batch/commands/retry.py

@@ -0,0 +1,288 @@
+"""Retry command for re-running failed chunks."""
+
+from datetime import datetime
+from pathlib import Path
+
+import click
+
+from ..aws_batch import BatchClient, BatchError
+from ..job_id import generate_job_id
+from ..manifest import (
+    BATCH_JOBS_BASE,
+    JobStatus,
+    RetryInfo,
+    get_job_dir,
+    load_manifest,
+    save_manifest,
+)
+
+
+@click.command()
+@click.argument("job_id")
+@click.option("--indices", help="Specific indices to retry (comma-separated)")
+@click.option(
+    "--queue",
+    help="Override job queue (e.g., 't4-1x' for on-demand instead of spot)",
+)
+@click.option(
+    "--reslice",
+    type=int,
+    help="Reslice failed chunks into N thinner chunks (reduces interruption risk)",
+)
+@click.option(
+    "--dry-run", is_flag=True, help="Show what would be retried without submitting"
+)
+@click.option("--base-path", default=BATCH_JOBS_BASE, help="Base path for job data")
+def retry(job_id, indices, queue, reslice, dry_run, base_path):
+    """Retry failed chunks of a batch job.
+
+    Identifies failed array indices and submits a new job to retry only
+    those specific indices. Outputs go to the same job directory, so
+    finalization works normally after retries complete.
+
+    The --reslice option concatenates failed chunks and re-splits them into
+    thinner slices, reducing the time per worker and thus the risk of spot
+    interruptions. Resliced outputs are named with a prefix (e.g., embed_r1_000.h5)
+    and are automatically included in finalization.
+
+    \b
+    Examples:
+      dh batch retry dma-embed-20260109-a3f2              # Retry all failed
+      dh batch retry dma-embed-20260109-a3f2 --indices 5,12,27  # Retry specific indices
+      dh batch retry dma-embed-20260109-a3f2 --queue t4-1x     # Use on-demand (no spot interruptions)
+      dh batch retry dma-embed-20260109-a3f2 --reslice 40      # Reslice into 40 thinner chunks
+      dh batch retry dma-embed-20260109-a3f2 --dry-run   # Show what would be retried
+    """
+    # Load manifest
+    try:
+        manifest = load_manifest(job_id, base_path)
+    except FileNotFoundError:
+        click.echo(f"Job not found: {job_id}", err=True)
+        raise SystemExit(1)
+
+    # Get failed indices
+    if indices:
+        # User specified indices
+        retry_indices = [int(i.strip()) for i in indices.split(",")]
+    else:
+        # Auto-detect from .done markers
+        retry_indices = _find_incomplete_chunks(job_id, base_path)
+
+    if not retry_indices:
+        click.echo("No failed or incomplete chunks found. Nothing to retry.")
+        return
+
+    click.echo(f"Found {len(retry_indices)} chunks to retry: {retry_indices}")
+
+    # Check if we have the required info
+    if not manifest.batch:
+        click.echo("Job has no batch configuration.", err=True)
+        raise SystemExit(1)
+
+    # Generate retry job ID and reslice prefix
+    retry_num = len(manifest.retries) + 1
+    retry_id = f"{job_id}-r{retry_num}"
+    reslice_prefix = f"r{retry_num}" if reslice else None
+
+    job_dir = get_job_dir(job_id, base_path)
+
+    if reslice:
+        # Count sequences in failed chunks to estimate split
+        total_seqs = _count_sequences_in_chunks(job_dir, retry_indices)
+        seqs_per_chunk = max(1, total_seqs // reslice)
+        click.echo(f"Total sequences in failed chunks: {total_seqs:,}")
+        click.echo(f"Reslicing into {reslice} chunks (~{seqs_per_chunk:,} seqs each)")
+
+    if dry_run:
+        click.echo()
+        click.echo(click.style("Dry run - job not submitted", fg="yellow"))
+        return
+
+    click.echo()
+    click.echo(f"Retry job ID: {retry_id}")
+
+    # Handle reslicing if requested
+    if reslice:
+        click.echo(f"Reslice prefix: {reslice_prefix}")
+        actual_chunks = _reslice_failed_chunks(
+            job_dir, retry_indices, reslice_prefix, reslice
+        )
+        click.echo(f"Created {actual_chunks} resliced chunks")
+        array_size = actual_chunks
+    else:
+        array_size = len(retry_indices)
+
+    # Submit retry job
+    try:
+        client = BatchClient()
+
+        environment = {
+            "JOB_DIR": str(job_dir),
+            "JOB_ID": job_id,
+        }
+
+        # Use provided queue or fall back to original
+        job_queue = queue or manifest.batch.queue
+        if queue and queue != manifest.batch.queue:
+            click.echo(f"Using queue: {job_queue} (original: {manifest.batch.queue})")
+
+        if reslice:
+            # Resliced retry: use RESLICE_PREFIX, sequential indices 0..N-1
+            environment["RESLICE_PREFIX"] = reslice_prefix
+            batch_job_id = client.submit_job(
+                job_name=retry_id,
+                job_definition=manifest.batch.job_definition or "dayhoff-embed-t5",
+                job_queue=job_queue,
+                array_size=array_size,
+                environment=environment,
+                timeout_seconds=6 * 3600,
+                retry_attempts=5,
+            )
+        else:
+            # Standard retry: use BATCH_RETRY_INDICES mapping
+            environment["BATCH_RETRY_INDICES"] = ",".join(str(i) for i in retry_indices)
+            batch_job_id = client.submit_array_job_with_indices(
+                job_name=retry_id,
+                job_definition=manifest.batch.job_definition or "dayhoff-embed-t5",
+                job_queue=job_queue,
+                indices=retry_indices,
+                environment=environment,
+                timeout_seconds=6 * 3600,
+                retry_attempts=5,
+            )
+
+        # Update manifest with retry info
+        retry_info = RetryInfo(
+            retry_id=retry_id,
+            indices=retry_indices,
+            batch_job_id=batch_job_id,
+            reslice_prefix=reslice_prefix,
+            reslice_count=array_size if reslice else None,
+            created=datetime.utcnow(),
+        )
+        manifest.retries.append(retry_info)
+        manifest.status = JobStatus.RUNNING
+        save_manifest(manifest, base_path)
+
+        click.echo()
+        click.echo(click.style("✓ Retry job submitted successfully!", fg="green"))
+        click.echo()
+        click.echo(f"AWS Batch Job ID: {batch_job_id}")
+        click.echo()
+        click.echo("Next steps:")
+        click.echo(f"  Check status:  dh batch status {job_id}")
+        click.echo(f"  View logs:     dh batch logs {job_id}")
+
+    except BatchError as e:
+        click.echo(
+            click.style(f"✗ Failed to submit retry job: {e}", fg="red"), err=True
+        )
+        raise SystemExit(1)
+
+
+def _find_incomplete_chunks(job_id: str, base_path: str) -> list[int]:
+    """Find chunks that don't have .done markers."""
+    job_dir = get_job_dir(job_id, base_path)
+    input_dir = job_dir / "input"
+    output_dir = job_dir / "output"
+
+    if not input_dir.exists():
+        return []
+
+    # Find all original input chunks (not resliced ones)
+    input_chunks = sorted(input_dir.glob("chunk_[0-9][0-9][0-9].fasta"))
+    incomplete = []
+
+    for chunk_path in input_chunks:
+        # Extract index from filename (chunk_000.fasta -> 0)
+        idx_str = chunk_path.stem.split("_")[1]
+        idx = int(idx_str)
+
+        # Check for .done marker
+        done_marker = output_dir / f"embed_{idx:03d}.done"
+        if not done_marker.exists():
+            incomplete.append(idx)
+
+    return incomplete
+
+
+def _count_sequences_in_chunks(job_dir: Path, indices: list[int]) -> int:
+    """Count total sequences in the specified chunk files."""
+    input_dir = job_dir / "input"
+    total = 0
+
+    for idx in indices:
+        chunk_path = input_dir / f"chunk_{idx:03d}.fasta"
+        if chunk_path.exists():
+            with open(chunk_path) as f:
+                for line in f:
+                    if line.startswith(">"):
+                        total += 1
+
+    return total
+
+
+def _reslice_failed_chunks(
+    job_dir: Path, indices: list[int], reslice_prefix: str, num_chunks: int
+) -> int:
+    """Concatenate failed chunks and re-split into thinner slices.
+
+    Creates new chunk files named chunk_{prefix}_000.fasta, etc.
+
+    Args:
+        job_dir: Job directory path
+        indices: List of failed chunk indices
+        reslice_prefix: Prefix for new chunk files (e.g., 'r1')
+        num_chunks: Target number of new chunks
+
+    Returns:
+        Actual number of chunks created
+    """
+    from dayhoff_tools.fasta import split_fasta
+    import tempfile
+
+    input_dir = job_dir / "input"
+
+    # Concatenate all failed chunks into a temp file
+    with tempfile.NamedTemporaryFile(
+        mode="w", suffix=".fasta", delete=False
+    ) as tmp_file:
+        tmp_path = tmp_file.name
+        total_seqs = 0
+
+        for idx in indices:
+            chunk_path = input_dir / f"chunk_{idx:03d}.fasta"
+            if chunk_path.exists():
+                with open(chunk_path) as f:
+                    for line in f:
+                        tmp_file.write(line)
+                        if line.startswith(">"):
+                            total_seqs += 1
+
+    try:
+        # Calculate sequences per chunk
+        seqs_per_chunk = max(1, (total_seqs + num_chunks - 1) // num_chunks)
+
+        # Split into new chunks with reslice prefix
+        # split_fasta creates files like: chunk_r1_1.fasta, chunk_r1_2.fasta, etc.
+        actual_chunks = split_fasta(
+            fasta_file=tmp_path,
+            target_folder=str(input_dir),
+            base_name=f"chunk_{reslice_prefix}",
+            sequences_per_file=seqs_per_chunk,
+            max_files=num_chunks,
+            show_progress=True,
+        )
+
+        # Rename to zero-padded indices (chunk_r1_000.fasta, etc.)
+        for i in range(1, actual_chunks + 1):
+            old_name = input_dir / f"chunk_{reslice_prefix}_{i}.fasta"
+            new_name = input_dir / f"chunk_{reslice_prefix}_{i-1:03d}.fasta"
+            if old_name.exists():
+                old_name.rename(new_name)
+
+        return actual_chunks
+
+    finally:
+        # Clean up temp file
+        Path(tmp_path).unlink(missing_ok=True)

{dayhoff_tools-1.14.9 → dayhoff_tools-1.14.11}/dayhoff_tools/cli/batch/manifest.py

@@ -62,6 +62,12 @@ class RetryInfo(BaseModel):
     retry_id: str = Field(..., description="Retry job ID")
     indices: list[int] = Field(..., description="Array indices being retried")
     batch_job_id: str | None = Field(None, description="AWS Batch job ID for retry")
+    reslice_prefix: str | None = Field(
+        None, description="Reslice prefix if chunks were resliced (e.g., 'r1')"
+    )
+    reslice_count: int | None = Field(
+        None, description="Number of resliced chunks created"
+    )
     created: datetime = Field(default_factory=datetime.utcnow)
 
 

{dayhoff_tools-1.14.9 → dayhoff_tools-1.14.11}/pyproject.toml

@@ -5,7 +5,7 @@ build-backend = "poetry.core.masonry.api"
 
 [project]
 name = "dayhoff-tools"
-version = "1.14.9"
+version = "1.14.11"
 description = "Common tools for all the repos at Dayhoff Labs"
 authors = [
     {name = "Daniel Martin-Alarcon", email = "dma@dayhofflabs.com"}

dayhoff_tools-1.14.9/dayhoff_tools/cli/batch/commands/retry.py

@@ -1,146 +0,0 @@
-"""Retry command for re-running failed chunks."""
-
-from datetime import datetime
-
-import click
-
-from ..aws_batch import BatchClient, BatchError
-from ..job_id import generate_job_id
-from ..manifest import (
-    BATCH_JOBS_BASE,
-    JobStatus,
-    RetryInfo,
-    get_job_dir,
-    load_manifest,
-    save_manifest,
-)
-
-
-@click.command()
-@click.argument("job_id")
-@click.option("--indices", help="Specific indices to retry (comma-separated)")
-@click.option(
-    "--dry-run", is_flag=True, help="Show what would be retried without submitting"
-)
-@click.option("--base-path", default=BATCH_JOBS_BASE, help="Base path for job data")
-def retry(job_id, indices, dry_run, base_path):
-    """Retry failed chunks of a batch job.
-
-    Identifies failed array indices and submits a new job to retry only
-    those specific indices.
-
-    \b
-    Examples:
-      dh batch retry dma-embed-20260109-a3f2              # Retry all failed
-      dh batch retry dma-embed-20260109-a3f2 --indices 5,12,27  # Retry specific indices
-      dh batch retry dma-embed-20260109-a3f2 --dry-run   # Show what would be retried
-    """
-    # Load manifest
-    try:
-        manifest = load_manifest(job_id, base_path)
-    except FileNotFoundError:
-        click.echo(f"Job not found: {job_id}", err=True)
-        raise SystemExit(1)
-
-    # Get failed indices
-    if indices:
-        # User specified indices
-        retry_indices = [int(i.strip()) for i in indices.split(",")]
-    else:
-        # Auto-detect from .done markers
-        retry_indices = _find_incomplete_chunks(job_id, base_path)
-
-    if not retry_indices:
-        click.echo("No failed or incomplete chunks found. Nothing to retry.")
-        return
-
-    click.echo(f"Found {len(retry_indices)} chunks to retry: {retry_indices}")
-
-    if dry_run:
-        click.echo()
-        click.echo(click.style("Dry run - job not submitted", fg="yellow"))
-        return
-
-    # Check if we have the required info
-    if not manifest.batch:
-        click.echo("Job has no batch configuration.", err=True)
-        raise SystemExit(1)
-
-    # Generate retry job ID
-    retry_id = f"{job_id}-r{len(manifest.retries) + 1}"
-
-    click.echo()
-    click.echo(f"Retry job ID: {retry_id}")
-
-    # Submit retry job
-    try:
-        client = BatchClient()
-        job_dir = get_job_dir(job_id, base_path)
-
-        environment = {
-            "JOB_DIR": str(job_dir),
-            "JOB_ID": job_id,
-            "BATCH_RETRY_INDICES": ",".join(str(i) for i in retry_indices),
-        }
-
-        batch_job_id = client.submit_array_job_with_indices(
-            job_name=retry_id,
-            job_definition=manifest.batch.job_definition or "dayhoff-embed-t5",
-            job_queue=manifest.batch.queue,
-            indices=retry_indices,
-            environment=environment,
-            timeout_seconds=6 * 3600,
-            retry_attempts=3,
-        )
-
-        # Update manifest with retry info
-        retry_info = RetryInfo(
-            retry_id=retry_id,
-            indices=retry_indices,
-            batch_job_id=batch_job_id,
-            created=datetime.utcnow(),
-        )
-        manifest.retries.append(retry_info)
-        manifest.status = JobStatus.RUNNING
-        save_manifest(manifest, base_path)
-
-        click.echo()
-        click.echo(click.style("✓ Retry job submitted successfully!", fg="green"))
-        click.echo()
-        click.echo(f"AWS Batch Job ID: {batch_job_id}")
-        click.echo()
-        click.echo("Next steps:")
-        click.echo(f"  Check status:  dh batch status {job_id}")
-        click.echo(f"  View logs:     dh batch logs {job_id}")
-
-    except BatchError as e:
-        click.echo(
-            click.style(f"✗ Failed to submit retry job: {e}", fg="red"), err=True
-        )
-        raise SystemExit(1)
-
-
-def _find_incomplete_chunks(job_id: str, base_path: str) -> list[int]:
-    """Find chunks that don't have .done markers."""
-    job_dir = get_job_dir(job_id, base_path)
-    input_dir = job_dir / "input"
-    output_dir = job_dir / "output"
-
-    if not input_dir.exists():
-        return []
-
-    # Find all input chunks
-    input_chunks = sorted(input_dir.glob("chunk_*.fasta"))
-    incomplete = []
-
-    for chunk_path in input_chunks:
-        # Extract index from filename (chunk_000.fasta -> 0)
-        idx_str = chunk_path.stem.split("_")[1]
-        idx = int(idx_str)
-
-        # Check for .done marker
-        done_marker = output_dir / f"embed_{idx:03d}.done"
-        if not done_marker.exists():
-            incomplete.append(idx)
-
-    return incomplete