dayhoff-tools 1.14.13__py3-none-any.whl → 1.14.14__py3-none-any.whl

dayhoff_tools/cli/batch/commands/finalize.py

@@ -34,8 +34,13 @@ from ..manifest import (
     is_flag=True,
     help="For Boltz: copy entire output directory (default: only essential files)",
 )
+@click.option(
+    "--skip-dedup",
+    is_flag=True,
+    help="Skip deduplication step (use if input has no duplicates)",
+)
 @click.option("--base-path", default=BATCH_JOBS_BASE, help="Base path for job data")
-def finalize(job_id, output, force, keep_intermediates, full_output, base_path):
+def finalize(job_id, output, force, keep_intermediates, full_output, skip_dedup, base_path):
     """Combine results and clean up job intermediates.
 
     For embedding jobs, combines H5 files into a single output file.
@@ -46,6 +51,9 @@ def finalize(job_id, output, force, keep_intermediates, full_output, base_path):
       # Embedding job - combine H5 files
       dh batch finalize dma-embed-20260109-a3f2 --output /primordial/embeddings.h5
 
+      # Skip deduplication (faster if input has no duplicates)
+      dh batch finalize dma-embed-20260109-a3f2 --output /primordial/embeddings.h5 --skip-dedup
+
       # Boltz job - extract essential files only (default)
       dh batch finalize dma-boltz-20260113-190a --output /primordial/structures/
 
@@ -92,7 +100,7 @@ def finalize(job_id, output, force, keep_intermediates, full_output, base_path):
     # Finalize based on pipeline type
     click.echo()
     if manifest.pipeline in ("embed-t5", "embed"):
-        _finalize_embeddings(output_dir, output_path)
+        _finalize_embeddings(output_dir, output_path, skip_dedup=skip_dedup)
     elif manifest.pipeline == "boltz":
         _finalize_boltz(output_dir, output_path, full_output=full_output)
     else:
@@ -182,7 +190,7 @@ def _check_completion(job_id: str, base_path: str) -> list[int]:
     return incomplete
 
 
-def _finalize_embeddings(output_dir: Path, output_path: Path):
+def _finalize_embeddings(output_dir: Path, output_path: Path, skip_dedup: bool = False):
     """Combine H5 embedding files into a single output."""
     h5_files = sorted(output_dir.glob("embed_*.h5"))
 
@@ -191,6 +199,8 @@ def _finalize_embeddings(output_dir: Path, output_path: Path):
         raise SystemExit(1)
 
     click.echo(f"Found {len(h5_files)} H5 files to combine")
+    if skip_dedup:
+        click.echo("Skipping deduplication (--skip-dedup)")
 
     # Check if output already exists
     if output_path.exists():
@@ -213,10 +223,9 @@ def _finalize_embeddings(output_dir: Path, output_path: Path):
             click.echo("Single chunk - copying directly...")
             shutil.copy2(h5_files[0], output_path)
         else:
-            # Multiple files - combine, deduplicate, and optimize
+            # Multiple files - combine and optionally deduplicate
             with tempfile.TemporaryDirectory() as tmpdir:
                 combined_path = Path(tmpdir) / "combined.h5"
-                deduped_path = Path(tmpdir) / "deduped.h5"
 
                 # Combine H5 files
                 click.echo("Combining H5 files...")
@@ -226,13 +235,17 @@ def _finalize_embeddings(output_dir: Path, output_path: Path):
                     output_file=str(combined_path),
                 )
 
-                # Deduplicate
-                click.echo("Deduplicating...")
-                deduplicate_h5_file(str(combined_path), str(deduped_path))
-
-                # Optimize chunks
-                click.echo("Optimizing chunks...")
-                optimize_protein_embedding_chunks(str(deduped_path), str(output_path))
+                if skip_dedup:
+                    # Skip dedup - optimize directly from combined
+                    click.echo("Optimizing chunks...")
+                    optimize_protein_embedding_chunks(str(combined_path), str(output_path))
+                else:
+                    # Full pipeline: combine -> dedup -> optimize
+                    deduped_path = Path(tmpdir) / "deduped.h5"
+                    click.echo("Deduplicating...")
+                    deduplicate_h5_file(str(combined_path), str(deduped_path))
+                    click.echo("Optimizing chunks...")
+                    optimize_protein_embedding_chunks(str(deduped_path), str(output_path))
 
         click.echo(click.style("✓ H5 files combined successfully", fg="green"))
 

{dayhoff_tools-1.14.13.dist-info → dayhoff_tools-1.14.14.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dayhoff-tools
-Version: 1.14.13
+Version: 1.14.14
 Summary: Common tools for all the repos at Dayhoff Labs
 Author: Daniel Martin-Alarcon
 Author-email: dma@dayhofflabs.com

{dayhoff_tools-1.14.13.dist-info → dayhoff_tools-1.14.14.dist-info}/RECORD

@@ -14,7 +14,7 @@ dayhoff_tools/cli/batch/commands/boltz.py,sha256=N0LksmtOpkvnEsR0SAUHxtksKPAsQjR
 dayhoff_tools/cli/batch/commands/cancel.py,sha256=kjvmCcFaMShyHfQjvR4WlII4njg4Fm4uffpWcY1qRWg,5299
 dayhoff_tools/cli/batch/commands/clean.py,sha256=nWOKbVM2nDuLMpyC038Q9aylOQxk2bq4N0JF65qJg-s,4570
 dayhoff_tools/cli/batch/commands/embed_t5.py,sha256=QXFydAw0wndevdzXF1cxikxMmvn1BuQ5p9lwutQFajU,11453
-dayhoff_tools/cli/batch/commands/finalize.py,sha256=xr3GFcMbvtU6UYiJI3UXhQqeaACSFzKIZOxz4GK-Dmo,12785
+dayhoff_tools/cli/batch/commands/finalize.py,sha256=OQUF9RiO8S55SCeQcFqExLKjYd-leL0Z_FOV0xMg7Dw,13497
 dayhoff_tools/cli/batch/commands/list_jobs.py,sha256=COfxZddDVUAHeTayNAB3ruYNhgrE3osgFxY2qzf33cg,4284
 dayhoff_tools/cli/batch/commands/local.py,sha256=dZeKhNakaM1jS-EoByAwg1nWspRRoOmYzcwzjEKBaIA,3226
 dayhoff_tools/cli/batch/commands/logs.py,sha256=ctgJksdzFmqBdD18ePPsZe2BpuJYtHz2xAaMPnUplmQ,5293
@@ -71,7 +71,7 @@ dayhoff_tools/intake/uniprot.py,sha256=BZYJQF63OtPcBBnQ7_P9gulxzJtqyorgyuDiPeOJq
 dayhoff_tools/logs.py,sha256=DKdeP0k0kliRcilwvX0mUB2eipO5BdWUeHwh-VnsICs,838
 dayhoff_tools/sqlite.py,sha256=jV55ikF8VpTfeQqqlHSbY8OgfyfHj8zgHNpZjBLos_E,18672
 dayhoff_tools/warehouse.py,sha256=UETBtZD3r7WgvURqfGbyHlT7cxoiVq8isjzMuerKw8I,24475
-dayhoff_tools-1.14.13.dist-info/METADATA,sha256=703JkMV45GgnJlsqHJVmlkwBGZntVuYUp3zNmhvT7Us,3185
-dayhoff_tools-1.14.13.dist-info/WHEEL,sha256=3ny-bZhpXrU6vSQ1UPG34FoxZBp3lVcvK0LkgUz6VLk,88
-dayhoff_tools-1.14.13.dist-info/entry_points.txt,sha256=iAf4jteNqW3cJm6CO6czLxjW3vxYKsyGLZ8WGmxamSc,49
-dayhoff_tools-1.14.13.dist-info/RECORD,,
+dayhoff_tools-1.14.14.dist-info/METADATA,sha256=QdEyPEUN_WWKDpvsKdWnN38FDpX6bqc4rsF4jGFbBDY,3185
+dayhoff_tools-1.14.14.dist-info/WHEEL,sha256=3ny-bZhpXrU6vSQ1UPG34FoxZBp3lVcvK0LkgUz6VLk,88
+dayhoff_tools-1.14.14.dist-info/entry_points.txt,sha256=iAf4jteNqW3cJm6CO6czLxjW3vxYKsyGLZ8WGmxamSc,49
+dayhoff_tools-1.14.14.dist-info/RECORD,,