dayhoff-tools 1.14.10__tar.gz → 1.14.12__tar.gz

{dayhoff_tools-1.14.10 → dayhoff_tools-1.14.12}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dayhoff-tools
-Version: 1.14.10
+Version: 1.14.12
 Summary: Common tools for all the repos at Dayhoff Labs
 Author: Daniel Martin-Alarcon
 Author-email: dma@dayhofflabs.com

{dayhoff_tools-1.14.10 → dayhoff_tools-1.14.12}/dayhoff_tools/batch/workers/base.py

@@ -30,12 +30,18 @@ def get_array_index() -> int:
 
     For array jobs, reads AWS_BATCH_JOB_ARRAY_INDEX.
     For retry jobs, maps from BATCH_RETRY_INDICES.
+    For resliced retry jobs, uses the raw array index (chunks are renumbered).
     For single jobs (array_size=1), defaults to 0.
 
     Returns:
         The array index this worker should process
     """
-    # Check for retry mode first
+    # For resliced retries, use raw array index (chunks are renumbered 0..N-1)
+    if os.environ.get("RESLICE_PREFIX"):
+        array_idx = os.environ.get("AWS_BATCH_JOB_ARRAY_INDEX", "0")
+        return int(array_idx)
+
+    # Check for retry mode (non-resliced)
     retry_indices = os.environ.get("BATCH_RETRY_INDICES")
     if retry_indices:
         # In retry mode, we have a list of indices and use array index to pick
@@ -69,6 +75,20 @@ def get_job_dir() -> Path:
     return Path(job_dir)
 
 
+def get_reslice_prefix() -> str | None:
+    """Get the reslice prefix from environment if set.
+
+    When RESLICE_PREFIX is set (e.g., 'r1'), files are named like:
+    - Input: chunk_r1_000.fasta
+    - Output: embed_r1_000.h5
+    - Done marker: embed_r1_000.done
+
+    Returns:
+        The reslice prefix or None if not in reslice mode
+    """
+    return os.environ.get("RESLICE_PREFIX")
+
+
 def get_input_file(index: int, job_dir: Path, prefix: str = "chunk") -> Path:
     """Get the input file path for a given index.
 
@@ -80,6 +100,9 @@ def get_input_file(index: int, job_dir: Path, prefix: str = "chunk") -> Path:
     Returns:
         Path to input file
     """
+    reslice = get_reslice_prefix()
+    if reslice:
+        return job_dir / "input" / f"{prefix}_{reslice}_{index:03d}.fasta"
     return job_dir / "input" / f"{prefix}_{index:03d}.fasta"
 
 
@@ -97,6 +120,9 @@ def get_output_file(
     Returns:
         Path to output file
     """
+    reslice = get_reslice_prefix()
+    if reslice:
+        return job_dir / "output" / f"{prefix}_{reslice}_{index:03d}{suffix}"
     return job_dir / "output" / f"{prefix}_{index:03d}{suffix}"
 
 
@@ -111,6 +137,9 @@ def get_done_marker(index: int, job_dir: Path, prefix: str = "embed") -> Path:
     Returns:
         Path to done marker file
     """
+    reslice = get_reslice_prefix()
+    if reslice:
+        return job_dir / "output" / f"{prefix}_{reslice}_{index:03d}.done"
     return job_dir / "output" / f"{prefix}_{index:03d}.done"
 
 

{dayhoff_tools-1.14.10 → dayhoff_tools-1.14.12}/dayhoff_tools/cli/batch/commands/finalize.py

@@ -118,7 +118,12 @@ def finalize(job_id, output, force, keep_intermediates, full_output, base_path):
 
 
 def _check_completion(job_id: str, base_path: str) -> list[int]:
-    """Check which chunks are incomplete (no .done marker)."""
+    """Check which chunks are incomplete (no .done marker).
+
+    Handles both original chunks (chunk_000.fasta) and resliced chunks
+    (chunk_r1_000.fasta). For original chunks that were resliced in a retry,
+    checks if all resliced chunks completed.
+    """
     job_dir = get_job_dir(job_id, base_path)
     input_dir = job_dir / "input"
     output_dir = job_dir / "output"
@@ -126,13 +131,53 @@ def _check_completion(job_id: str, base_path: str) -> list[int]:
     if not input_dir.exists():
         return []
 
+    # Load manifest to check for resliced retries
+    try:
+        manifest = load_manifest(job_id, base_path)
+        resliced_indices: set[int] = set()
+        reslice_info: dict[str, int] = {}  # prefix -> expected count
+
+        for retry in manifest.retries:
+            if retry.reslice_prefix and retry.reslice_count:
+                resliced_indices.update(retry.indices)
+                reslice_info[retry.reslice_prefix] = retry.reslice_count
+    except FileNotFoundError:
+        resliced_indices = set()
+        reslice_info = {}
+
     incomplete = []
-    for chunk_path in sorted(input_dir.glob("chunk_*.fasta")):
+
+    # Check original chunks (chunk_000.fasta pattern)
+    for chunk_path in sorted(input_dir.glob("chunk_[0-9][0-9][0-9].fasta")):
         idx_str = chunk_path.stem.split("_")[1]
         idx = int(idx_str)
+
+        # Check for original done marker
         done_marker = output_dir / f"embed_{idx:03d}.done"
-        if not done_marker.exists():
-            incomplete.append(idx)
+        if done_marker.exists():
+            continue
+
+        # Check if this chunk was resliced
+        if idx in resliced_indices:
+            # Find which retry covered this index and check if complete
+            is_covered = False
+            for retry in manifest.retries:
+                if (
+                    retry.reslice_prefix
+                    and retry.reslice_count
+                    and idx in retry.indices
+                ):
+                    # Check if all resliced chunks for this retry completed
+                    done_count = len(
+                        list(output_dir.glob(f"embed_{retry.reslice_prefix}_*.done"))
+                    )
+                    if done_count >= retry.reslice_count:
+                        is_covered = True
+                        break
+            if is_covered:
+                continue
+
+        incomplete.append(idx)
 
     return incomplete
 

dayhoff_tools-1.14.12/dayhoff_tools/cli/batch/commands/retry.py

@@ -0,0 +1,288 @@
+"""Retry command for re-running failed chunks."""
+
+from datetime import datetime
+from pathlib import Path
+
+import click
+
+from ..aws_batch import BatchClient, BatchError
+from ..job_id import generate_job_id
+from ..manifest import (
+    BATCH_JOBS_BASE,
+    JobStatus,
+    RetryInfo,
+    get_job_dir,
+    load_manifest,
+    save_manifest,
+)
+
+
+@click.command()
+@click.argument("job_id")
+@click.option("--indices", help="Specific indices to retry (comma-separated)")
+@click.option(
+    "--queue",
+    help="Override job queue (e.g., 't4-1x' for on-demand instead of spot)",
+)
+@click.option(
+    "--reslice",
+    type=int,
+    help="Reslice failed chunks into N thinner chunks (reduces interruption risk)",
+)
+@click.option(
+    "--dry-run", is_flag=True, help="Show what would be retried without submitting"
+)
+@click.option("--base-path", default=BATCH_JOBS_BASE, help="Base path for job data")
+def retry(job_id, indices, queue, reslice, dry_run, base_path):
+    """Retry failed chunks of a batch job.
+
+    Identifies failed array indices and submits a new job to retry only
+    those specific indices. Outputs go to the same job directory, so
+    finalization works normally after retries complete.
+
+    The --reslice option concatenates failed chunks and re-splits them into
+    thinner slices, reducing the time per worker and thus the risk of spot
+    interruptions. Resliced outputs are named with a prefix (e.g., embed_r1_000.h5)
+    and are automatically included in finalization.
+
+    \b
+    Examples:
+      dh batch retry dma-embed-20260109-a3f2              # Retry all failed
+      dh batch retry dma-embed-20260109-a3f2 --indices 5,12,27  # Retry specific indices
+      dh batch retry dma-embed-20260109-a3f2 --queue t4-1x     # Use on-demand (no spot interruptions)
+      dh batch retry dma-embed-20260109-a3f2 --reslice 40      # Reslice into 40 thinner chunks
+      dh batch retry dma-embed-20260109-a3f2 --dry-run   # Show what would be retried
+    """
+    # Load manifest
+    try:
+        manifest = load_manifest(job_id, base_path)
+    except FileNotFoundError:
+        click.echo(f"Job not found: {job_id}", err=True)
+        raise SystemExit(1)
+
+    # Get failed indices
+    if indices:
+        # User specified indices
+        retry_indices = [int(i.strip()) for i in indices.split(",")]
+    else:
+        # Auto-detect from .done markers
+        retry_indices = _find_incomplete_chunks(job_id, base_path)
+
+    if not retry_indices:
+        click.echo("No failed or incomplete chunks found. Nothing to retry.")
+        return
+
+    click.echo(f"Found {len(retry_indices)} chunks to retry: {retry_indices}")
+
+    # Check if we have the required info
+    if not manifest.batch:
+        click.echo("Job has no batch configuration.", err=True)
+        raise SystemExit(1)
+
+    # Generate retry job ID and reslice prefix
+    retry_num = len(manifest.retries) + 1
+    retry_id = f"{job_id}-r{retry_num}"
+    reslice_prefix = f"r{retry_num}" if reslice else None
+
+    job_dir = get_job_dir(job_id, base_path)
+
+    if reslice:
+        # Count sequences in failed chunks to estimate split
+        total_seqs = _count_sequences_in_chunks(job_dir, retry_indices)
+        seqs_per_chunk = max(1, total_seqs // reslice)
+        click.echo(f"Total sequences in failed chunks: {total_seqs:,}")
+        click.echo(f"Reslicing into {reslice} chunks (~{seqs_per_chunk:,} seqs each)")
+
+    if dry_run:
+        click.echo()
+        click.echo(click.style("Dry run - job not submitted", fg="yellow"))
+        return
+
+    click.echo()
+    click.echo(f"Retry job ID: {retry_id}")
+
+    # Handle reslicing if requested
+    if reslice:
+        click.echo(f"Reslice prefix: {reslice_prefix}")
+        actual_chunks = _reslice_failed_chunks(
+            job_dir, retry_indices, reslice_prefix, reslice
+        )
+        click.echo(f"Created {actual_chunks} resliced chunks")
+        array_size = actual_chunks
+    else:
+        array_size = len(retry_indices)
+
+    # Submit retry job
+    try:
+        client = BatchClient()
+
+        environment = {
+            "JOB_DIR": str(job_dir),
+            "JOB_ID": job_id,
+        }
+
+        # Use provided queue or fall back to original
+        job_queue = queue or manifest.batch.queue
+        if queue and queue != manifest.batch.queue:
+            click.echo(f"Using queue: {job_queue} (original: {manifest.batch.queue})")
+
+        if reslice:
+            # Resliced retry: use RESLICE_PREFIX, sequential indices 0..N-1
+            environment["RESLICE_PREFIX"] = reslice_prefix
+            batch_job_id = client.submit_job(
+                job_name=retry_id,
+                job_definition=manifest.batch.job_definition or "dayhoff-embed-t5",
+                job_queue=job_queue,
+                array_size=array_size,
+                environment=environment,
+                timeout_seconds=6 * 3600,
+                retry_attempts=5,
+            )
+        else:
+            # Standard retry: use BATCH_RETRY_INDICES mapping
+            environment["BATCH_RETRY_INDICES"] = ",".join(str(i) for i in retry_indices)
+            batch_job_id = client.submit_array_job_with_indices(
+                job_name=retry_id,
+                job_definition=manifest.batch.job_definition or "dayhoff-embed-t5",
+                job_queue=job_queue,
+                indices=retry_indices,
+                environment=environment,
+                timeout_seconds=6 * 3600,
+                retry_attempts=5,
+            )
+
+        # Update manifest with retry info
+        retry_info = RetryInfo(
+            retry_id=retry_id,
+            indices=retry_indices,
+            batch_job_id=batch_job_id,
+            reslice_prefix=reslice_prefix,
+            reslice_count=array_size if reslice else None,
+            created=datetime.utcnow(),
+        )
+        manifest.retries.append(retry_info)
+        manifest.status = JobStatus.RUNNING
+        save_manifest(manifest, base_path)
+
+        click.echo()
+        click.echo(click.style("✓ Retry job submitted successfully!", fg="green"))
+        click.echo()
+        click.echo(f"AWS Batch Job ID: {batch_job_id}")
+        click.echo()
+        click.echo("Next steps:")
+        click.echo(f"  Check status:  dh batch status {job_id}")
+        click.echo(f"  View logs:     dh batch logs {job_id}")
+
+    except BatchError as e:
+        click.echo(
+            click.style(f"✗ Failed to submit retry job: {e}", fg="red"), err=True
+        )
+        raise SystemExit(1)
+
+
+def _find_incomplete_chunks(job_id: str, base_path: str) -> list[int]:
+    """Find chunks that don't have .done markers."""
+    job_dir = get_job_dir(job_id, base_path)
+    input_dir = job_dir / "input"
+    output_dir = job_dir / "output"
+
+    if not input_dir.exists():
+        return []
+
+    # Find all original input chunks (not resliced ones)
+    input_chunks = sorted(input_dir.glob("chunk_[0-9][0-9][0-9].fasta"))
+    incomplete = []
+
+    for chunk_path in input_chunks:
+        # Extract index from filename (chunk_000.fasta -> 0)
+        idx_str = chunk_path.stem.split("_")[1]
+        idx = int(idx_str)
+
+        # Check for .done marker
+        done_marker = output_dir / f"embed_{idx:03d}.done"
+        if not done_marker.exists():
+            incomplete.append(idx)
+
+    return incomplete
+
+
+def _count_sequences_in_chunks(job_dir: Path, indices: list[int]) -> int:
+    """Count total sequences in the specified chunk files."""
+    input_dir = job_dir / "input"
+    total = 0
+
+    for idx in indices:
+        chunk_path = input_dir / f"chunk_{idx:03d}.fasta"
+        if chunk_path.exists():
+            with open(chunk_path) as f:
+                for line in f:
+                    if line.startswith(">"):
+                        total += 1
+
+    return total
+
+
+def _reslice_failed_chunks(
+    job_dir: Path, indices: list[int], reslice_prefix: str, num_chunks: int
+) -> int:
+    """Concatenate failed chunks and re-split into thinner slices.
+
+    Creates new chunk files named chunk_{prefix}_000.fasta, etc.
+
+    Args:
+        job_dir: Job directory path
+        indices: List of failed chunk indices
+        reslice_prefix: Prefix for new chunk files (e.g., 'r1')
+        num_chunks: Target number of new chunks
+
+    Returns:
+        Actual number of chunks created
+    """
+    from dayhoff_tools.fasta import split_fasta
+    import tempfile
+
+    input_dir = job_dir / "input"
+
+    # Concatenate all failed chunks into a temp file
+    with tempfile.NamedTemporaryFile(
+        mode="w", suffix=".fasta", delete=False
+    ) as tmp_file:
+        tmp_path = tmp_file.name
+        total_seqs = 0
+
+        for idx in indices:
+            chunk_path = input_dir / f"chunk_{idx:03d}.fasta"
+            if chunk_path.exists():
+                with open(chunk_path) as f:
+                    for line in f:
+                        tmp_file.write(line)
+                        if line.startswith(">"):
+                            total_seqs += 1
+
+    try:
+        # Calculate sequences per chunk
+        seqs_per_chunk = max(1, (total_seqs + num_chunks - 1) // num_chunks)
+
+        # Split into new chunks with reslice prefix
+        # split_fasta creates files like: chunk_r1_1.fasta, chunk_r1_2.fasta, etc.
+        actual_chunks = split_fasta(
+            fasta_file=tmp_path,
+            target_folder=str(input_dir),
+            base_name=f"chunk_{reslice_prefix}",
+            sequences_per_file=seqs_per_chunk,
+            max_files=num_chunks,
+            show_progress=True,
+        )
+
+        # Rename to zero-padded indices (chunk_r1_000.fasta, etc.)
+        for i in range(1, actual_chunks + 1):
+            old_name = input_dir / f"chunk_{reslice_prefix}_{i}.fasta"
+            new_name = input_dir / f"chunk_{reslice_prefix}_{i-1:03d}.fasta"
+            if old_name.exists():
+                old_name.rename(new_name)
+
+        return actual_chunks
+
+    finally:
+        # Clean up temp file
+        Path(tmp_path).unlink(missing_ok=True)

{dayhoff_tools-1.14.10 → dayhoff_tools-1.14.12}/dayhoff_tools/cli/batch/commands/status.py

@@ -174,15 +174,41 @@ def _show_job_list(user, status_filter, pipeline, base_path):
     click.echo("Use 'dh batch status <job-id>' for details.")
 
 
+def _parse_retry_job_id(job_id: str) -> tuple[str, str | None]:
+    """Parse a job ID to extract parent job ID and retry suffix.
+
+    Args:
+        job_id: Job ID like 'dma-embed-20260120-63ec' or 'dma-embed-20260120-63ec-r1'
+
+    Returns:
+        Tuple of (parent_job_id, retry_id or None)
+    """
+    import re
+
+    # Check for retry suffix like -r1, -r2, etc.
+    match = re.match(r"^(.+)(-r\d+)$", job_id)
+    if match:
+        return match.group(1), job_id
+    return job_id, None
+
+
 def _show_job_details(job_id: str, base_path: str):
     """Show detailed status for a specific job."""
+    # Check if this is a retry job ID
+    parent_job_id, retry_id = _parse_retry_job_id(job_id)
+
     try:
-        manifest = load_manifest(job_id, base_path)
+        manifest = load_manifest(parent_job_id, base_path)
     except FileNotFoundError:
         click.echo(f"Job not found: {job_id}", err=True)
-        click.echo(f"Looking in: {base_path}/{job_id}/manifest.json", err=True)
+        click.echo(f"Looking in: {base_path}/{parent_job_id}/manifest.json", err=True)
         raise SystemExit(1)
 
+    # If showing a retry job, show retry-specific details
+    if retry_id:
+        _show_retry_details(manifest, retry_id)
+        return
+
     click.echo()
     click.echo(f"Job ID:    {manifest.job_id}")
     click.echo(f"Status:    {format_status(manifest.status)}")
@@ -235,7 +261,31 @@ def _show_job_details(job_id: str, base_path: str):
         click.echo()
         click.echo("Retries:")
         for retry in manifest.retries:
-            click.echo(f"  - {retry.retry_id}: indices {retry.indices}")
+            reslice_info = ""
+            if retry.reslice_prefix:
+                reslice_info = f" (resliced to {retry.reslice_count} chunks)"
+            click.echo(f"  - {retry.retry_id}: {len(retry.indices)} indices{reslice_info}")
+            click.echo(f"    Indices: {retry.indices}")
+            if retry.batch_job_id:
+                # Show brief status for retry job
+                try:
+                    client = BatchClient()
+                    array_status = client.get_array_job_status(retry.batch_job_id)
+                    if array_status.is_complete:
+                        pct = array_status.success_rate * 100
+                        color = "green" if pct == 100 else "yellow" if pct > 90 else "red"
+                        click.echo(
+                            f"    Status: Complete - {click.style(f'{pct:.0f}%', fg=color)} "
+                            f"({array_status.succeeded}/{array_status.total} succeeded)"
+                        )
+                    else:
+                        click.echo(
+                            f"    Status: Running - {array_status.succeeded}/{array_status.total} done, "
+                            f"{array_status.running} running"
+                        )
+                except BatchError:
+                    click.echo(f"    Status: (could not fetch)")
+            click.echo(f"    Details: dh batch status {retry.retry_id}")
 
     # Suggest next steps
     click.echo()
@@ -285,3 +335,48 @@ def _show_array_status(batch_job_id: str):
 
     except BatchError as e:
         click.echo(f"    (Could not fetch live status: {e})")
+
+
+def _show_retry_details(manifest, retry_id: str):
+    """Show detailed status for a retry job."""
+    # Find the retry info
+    retry_info = None
+    for retry in manifest.retries:
+        if retry.retry_id == retry_id:
+            retry_info = retry
+            break
+
+    if not retry_info:
+        click.echo(f"Retry job not found: {retry_id}", err=True)
+        click.echo(f"Known retries: {[r.retry_id for r in manifest.retries]}", err=True)
+        raise SystemExit(1)
+
+    click.echo()
+    click.echo(f"Retry Job:   {retry_id}")
+    click.echo(f"Parent Job:  {manifest.job_id}")
+    click.echo(f"Pipeline:    {manifest.pipeline}")
+    click.echo(f"User:        {manifest.user}")
+    click.echo(
+        f"Created:     {retry_info.created.isoformat()} ({format_time_ago(retry_info.created)})"
+    )
+
+    click.echo()
+    click.echo("Retry Config:")
+    click.echo(f"  Indices:   {retry_info.indices}")
+    if retry_info.reslice_prefix:
+        click.echo(f"  Reslice:   {retry_info.reslice_prefix} ({retry_info.reslice_count} chunks)")
+    else:
+        click.echo(f"  Reslice:   No (retrying original chunks)")
+
+    if retry_info.batch_job_id:
+        click.echo()
+        click.echo("Batch:")
+        click.echo(f"  AWS Job ID: {retry_info.batch_job_id}")
+
+        # Get live status from AWS Batch
+        _show_array_status(retry_info.batch_job_id)
+
+    click.echo()
+    click.echo("Next steps:")
+    click.echo(f"  View logs:      dh batch logs {manifest.job_id}")
+    click.echo(f"  Parent status:  dh batch status {manifest.job_id}")

{dayhoff_tools-1.14.10 → dayhoff_tools-1.14.12}/dayhoff_tools/cli/batch/manifest.py

@@ -62,6 +62,12 @@ class RetryInfo(BaseModel):
     retry_id: str = Field(..., description="Retry job ID")
     indices: list[int] = Field(..., description="Array indices being retried")
     batch_job_id: str | None = Field(None, description="AWS Batch job ID for retry")
+    reslice_prefix: str | None = Field(
+        None, description="Reslice prefix if chunks were resliced (e.g., 'r1')"
+    )
+    reslice_count: int | None = Field(
+        None, description="Number of resliced chunks created"
+    )
     created: datetime = Field(default_factory=datetime.utcnow)
 
 

{dayhoff_tools-1.14.10 → dayhoff_tools-1.14.12}/pyproject.toml

@@ -5,7 +5,7 @@ build-backend = "poetry.core.masonry.api"
 
 [project]
 name = "dayhoff-tools"
-version = "1.14.10"
+version = "1.14.12"
 description = "Common tools for all the repos at Dayhoff Labs"
 authors = [
     {name = "Daniel Martin-Alarcon", email = "dma@dayhofflabs.com"}

dayhoff_tools-1.14.10/dayhoff_tools/cli/batch/commands/retry.py

@@ -1,146 +0,0 @@
-"""Retry command for re-running failed chunks."""
-
-from datetime import datetime
-
-import click
-
-from ..aws_batch import BatchClient, BatchError
-from ..job_id import generate_job_id
-from ..manifest import (
-    BATCH_JOBS_BASE,
-    JobStatus,
-    RetryInfo,
-    get_job_dir,
-    load_manifest,
-    save_manifest,
-)
-
-
-@click.command()
-@click.argument("job_id")
-@click.option("--indices", help="Specific indices to retry (comma-separated)")
-@click.option(
-    "--dry-run", is_flag=True, help="Show what would be retried without submitting"
-)
-@click.option("--base-path", default=BATCH_JOBS_BASE, help="Base path for job data")
-def retry(job_id, indices, dry_run, base_path):
-    """Retry failed chunks of a batch job.
-
-    Identifies failed array indices and submits a new job to retry only
-    those specific indices.
-
-    \b
-    Examples:
-      dh batch retry dma-embed-20260109-a3f2              # Retry all failed
-      dh batch retry dma-embed-20260109-a3f2 --indices 5,12,27  # Retry specific indices
-      dh batch retry dma-embed-20260109-a3f2 --dry-run   # Show what would be retried
-    """
-    # Load manifest
-    try:
-        manifest = load_manifest(job_id, base_path)
-    except FileNotFoundError:
-        click.echo(f"Job not found: {job_id}", err=True)
-        raise SystemExit(1)
-
-    # Get failed indices
-    if indices:
-        # User specified indices
-        retry_indices = [int(i.strip()) for i in indices.split(",")]
-    else:
-        # Auto-detect from .done markers
-        retry_indices = _find_incomplete_chunks(job_id, base_path)
-
-    if not retry_indices:
-        click.echo("No failed or incomplete chunks found. Nothing to retry.")
-        return
-
-    click.echo(f"Found {len(retry_indices)} chunks to retry: {retry_indices}")
-
-    if dry_run:
-        click.echo()
-        click.echo(click.style("Dry run - job not submitted", fg="yellow"))
-        return
-
-    # Check if we have the required info
-    if not manifest.batch:
-        click.echo("Job has no batch configuration.", err=True)
-        raise SystemExit(1)
-
-    # Generate retry job ID
-    retry_id = f"{job_id}-r{len(manifest.retries) + 1}"
-
-    click.echo()
-    click.echo(f"Retry job ID: {retry_id}")
-
-    # Submit retry job
-    try:
-        client = BatchClient()
-        job_dir = get_job_dir(job_id, base_path)
-
-        environment = {
-            "JOB_DIR": str(job_dir),
-            "JOB_ID": job_id,
-            "BATCH_RETRY_INDICES": ",".join(str(i) for i in retry_indices),
-        }
-
-        batch_job_id = client.submit_array_job_with_indices(
-            job_name=retry_id,
-            job_definition=manifest.batch.job_definition or "dayhoff-embed-t5",
-            job_queue=manifest.batch.queue,
-            indices=retry_indices,
-            environment=environment,
-            timeout_seconds=6 * 3600,
-            retry_attempts=3,
-        )
-
-        # Update manifest with retry info
-        retry_info = RetryInfo(
-            retry_id=retry_id,
-            indices=retry_indices,
-            batch_job_id=batch_job_id,
-            created=datetime.utcnow(),
-        )
-        manifest.retries.append(retry_info)
-        manifest.status = JobStatus.RUNNING
-        save_manifest(manifest, base_path)
-
-        click.echo()
-        click.echo(click.style("✓ Retry job submitted successfully!", fg="green"))
-        click.echo()
-        click.echo(f"AWS Batch Job ID: {batch_job_id}")
-        click.echo()
-        click.echo("Next steps:")
-        click.echo(f"  Check status:  dh batch status {job_id}")
-        click.echo(f"  View logs:     dh batch logs {job_id}")
-
-    except BatchError as e:
-        click.echo(
-            click.style(f"✗ Failed to submit retry job: {e}", fg="red"), err=True
-        )
-        raise SystemExit(1)
-
-
-def _find_incomplete_chunks(job_id: str, base_path: str) -> list[int]:
-    """Find chunks that don't have .done markers."""
-    job_dir = get_job_dir(job_id, base_path)
-    input_dir = job_dir / "input"
-    output_dir = job_dir / "output"
-
-    if not input_dir.exists():
-        return []
-
-    # Find all input chunks
-    input_chunks = sorted(input_dir.glob("chunk_*.fasta"))
-    incomplete = []
-
-    for chunk_path in input_chunks:
-        # Extract index from filename (chunk_000.fasta -> 0)
-        idx_str = chunk_path.stem.split("_")[1]
-        idx = int(idx_str)
-
-        # Check for .done marker
-        done_marker = output_dir / f"embed_{idx:03d}.done"
-        if not done_marker.exists():
-            incomplete.append(idx)
-
-    return incomplete