dayhoff-tools 1.14.13__tar.gz → 1.14.15__tar.gz

{dayhoff_tools-1.14.13 → dayhoff_tools-1.14.15}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dayhoff-tools
-Version: 1.14.13
+Version: 1.14.15
 Summary: Common tools for all the repos at Dayhoff Labs
 Author: Daniel Martin-Alarcon
 Author-email: dma@dayhofflabs.com
@@ -25,7 +25,7 @@ Requires-Dist: fair-esm (>=2.0.0) ; extra == "full"
 Requires-Dist: firebase-admin (>=6.5.0)
 Requires-Dist: h5py (>=3.11.0) ; extra == "full"
 Requires-Dist: h5py (>=3.13.0) ; extra == "embedders"
-Requires-Dist: numpy (>=1.26.4) ; extra == "embedders"
+Requires-Dist: numpy (>=1.26.4,<2.0) ; extra == "embedders"
 Requires-Dist: pandas (>=2.2.0,<2.2.3) ; extra == "embedders"
 Requires-Dist: pandas (>=2.2.0,<2.2.3) ; extra == "full"
 Requires-Dist: pydantic (>=2.0.0) ; extra == "batch"
@@ -38,7 +38,6 @@ Requires-Dist: sentencepiece (>=0.2.0) ; extra == "embedders"
 Requires-Dist: sentencepiece (>=0.2.0) ; extra == "full"
 Requires-Dist: sqlalchemy (>=2.0.40,<3.0.0) ; extra == "full"
 Requires-Dist: toml (>=0.10)
-Requires-Dist: torch (>=2.4.0) ; extra == "embedders"
 Requires-Dist: tqdm (>=4.67.1) ; extra == "embedders"
 Requires-Dist: tqdm (>=4.67.1) ; extra == "full"
 Requires-Dist: transformers (==4.36.2) ; extra == "full"

{dayhoff_tools-1.14.13 → dayhoff_tools-1.14.15}/dayhoff_tools/cli/batch/commands/finalize.py

@@ -34,8 +34,13 @@ from ..manifest import (
     is_flag=True,
     help="For Boltz: copy entire output directory (default: only essential files)",
 )
+@click.option(
+    "--skip-dedup",
+    is_flag=True,
+    help="Skip deduplication step (use if input has no duplicates)",
+)
 @click.option("--base-path", default=BATCH_JOBS_BASE, help="Base path for job data")
-def finalize(job_id, output, force, keep_intermediates, full_output, base_path):
+def finalize(job_id, output, force, keep_intermediates, full_output, skip_dedup, base_path):
     """Combine results and clean up job intermediates.
 
     For embedding jobs, combines H5 files into a single output file.
@@ -46,6 +51,9 @@ def finalize(job_id, output, force, keep_intermediates, full_output, base_path):
       # Embedding job - combine H5 files
       dh batch finalize dma-embed-20260109-a3f2 --output /primordial/embeddings.h5
 
+      # Skip deduplication (faster if input has no duplicates)
+      dh batch finalize dma-embed-20260109-a3f2 --output /primordial/embeddings.h5 --skip-dedup
+
       # Boltz job - extract essential files only (default)
       dh batch finalize dma-boltz-20260113-190a --output /primordial/structures/
 
@@ -92,7 +100,7 @@ def finalize(job_id, output, force, keep_intermediates, full_output, base_path):
     # Finalize based on pipeline type
     click.echo()
     if manifest.pipeline in ("embed-t5", "embed"):
-        _finalize_embeddings(output_dir, output_path)
+        _finalize_embeddings(output_dir, output_path, skip_dedup=skip_dedup)
     elif manifest.pipeline == "boltz":
         _finalize_boltz(output_dir, output_path, full_output=full_output)
     else:
@@ -182,7 +190,7 @@ def _check_completion(job_id: str, base_path: str) -> list[int]:
     return incomplete
 
 
-def _finalize_embeddings(output_dir: Path, output_path: Path):
+def _finalize_embeddings(output_dir: Path, output_path: Path, skip_dedup: bool = False):
     """Combine H5 embedding files into a single output."""
     h5_files = sorted(output_dir.glob("embed_*.h5"))
 
@@ -191,6 +199,8 @@ def _finalize_embeddings(output_dir: Path, output_path: Path):
         raise SystemExit(1)
 
     click.echo(f"Found {len(h5_files)} H5 files to combine")
+    if skip_dedup:
+        click.echo("Skipping deduplication (--skip-dedup)")
 
     # Check if output already exists
     if output_path.exists():
@@ -213,10 +223,9 @@ def _finalize_embeddings(output_dir: Path, output_path: Path):
             click.echo("Single chunk - copying directly...")
             shutil.copy2(h5_files[0], output_path)
         else:
-            # Multiple files - combine, deduplicate, and optimize
+            # Multiple files - combine and optionally deduplicate
             with tempfile.TemporaryDirectory() as tmpdir:
                 combined_path = Path(tmpdir) / "combined.h5"
-                deduped_path = Path(tmpdir) / "deduped.h5"
 
                 # Combine H5 files
                 click.echo("Combining H5 files...")
@@ -226,13 +235,17 @@ def _finalize_embeddings(output_dir: Path, output_path: Path):
                     output_file=str(combined_path),
                 )
 
-                # Deduplicate
-                click.echo("Deduplicating...")
-                deduplicate_h5_file(str(combined_path), str(deduped_path))
-
-                # Optimize chunks
-                click.echo("Optimizing chunks...")
-                optimize_protein_embedding_chunks(str(deduped_path), str(output_path))
+                if skip_dedup:
+                    # Skip dedup - optimize directly from combined
+                    click.echo("Optimizing chunks...")
+                    optimize_protein_embedding_chunks(str(combined_path), str(output_path))
+                else:
+                    # Full pipeline: combine -> dedup -> optimize
+                    deduped_path = Path(tmpdir) / "deduped.h5"
+                    click.echo("Deduplicating...")
+                    deduplicate_h5_file(str(combined_path), str(deduped_path))
+                    click.echo("Optimizing chunks...")
+                    optimize_protein_embedding_chunks(str(deduped_path), str(output_path))
 
         click.echo(click.style("✓ H5 files combined successfully", fg="green"))
 

{dayhoff_tools-1.14.13 → dayhoff_tools-1.14.15}/dayhoff_tools/cli/utility_commands.py

@@ -110,24 +110,11 @@ def build_and_upload_wheel(bump_part: str = "patch"):
     publish_cmd = ["uv", "publish", "--token", token]
     print("Using UV_PUBLISH_TOKEN for authentication.")
 
-    # Find the primary manifest (prefer AWS, then Mac, then Workstation)
-    pyproject_path = None
-    for candidate in [
-        "pyproject.aws.toml",
-        "pyproject.mac.toml",
-        "pyproject.workstation.toml",
-    ]:
-        if Path(candidate).exists():
-            pyproject_path = candidate
-            break
-
-    if not pyproject_path:
-        print(
-            "Error: No platform-specific manifest found (pyproject.aws.toml, pyproject.mac.toml, or pyproject.workstation.toml)"
-        )
+    # Use standard pyproject.toml
+    pyproject_path = "pyproject.toml"
+    if not Path(pyproject_path).exists():
+        print("Error: pyproject.toml not found in current directory.")
         return
-
-    print(f"Using manifest: {pyproject_path}")
     current_version = None  # Initialize in case the first try block fails
 
     try:
@@ -190,90 +177,33 @@ def build_and_upload_wheel(bump_part: str = "patch"):
             f.write(new_content)
         print(f"Updated {pyproject_path} with version {new_version}")
 
-        # Mirror version in all other platform manifests (best-effort)
-        other_manifests = []
-        for candidate in [
-            "pyproject.aws.toml",
-            "pyproject.mac.toml",
-            "pyproject.workstation.toml",
-        ]:
-            if Path(candidate).exists() and candidate != pyproject_path:
-                other_manifests.append(Path(candidate))
-
-        for manifest_path in other_manifests:
-            try:
-                content = manifest_path.read_text()
-                pattern = re.compile(
-                    f'^version\s*=\s*"{re.escape(current_version)}"', re.MULTILINE
-                )
-                new_content, replacements = pattern.subn(
-                    f'version = "{new_version}"', content
-                )
-                if replacements > 0:
-                    manifest_path.write_text(new_content)
-                    print(f"Updated {manifest_path} with version {new_version}")
-            except Exception as e:
-                print(f"Warning: Could not update {manifest_path}: {e}")
         # --- End Version Bumping Logic ---
 
         # Build wheel and sdist
-        # UV expects pyproject.toml, so temporarily copy the platform manifest
-        backup_created = False
-        temp_pyproject_created = False
-        if pyproject_path != "pyproject.toml":
-            if Path("pyproject.toml").exists():
-                Path("pyproject.toml").rename("pyproject.toml.build.bak")
-                backup_created = True
-            Path(pyproject_path).read_text()
-            with open("pyproject.toml", "w") as f:
-                f.write(Path(pyproject_path).read_text())
-            temp_pyproject_created = True
+        build_cmd = ["uv", "build"]
+        print(f"Running command: {BLUE}{' '.join(build_cmd)}{RESET}")
+        subprocess.run(build_cmd, check=True)
 
-        try:
-            build_cmd = ["uv", "build"]
-            # Print command in blue
-            print(f"Running command: {BLUE}{' '.join(build_cmd)}{RESET}")
-            subprocess.run(build_cmd, check=True)
-
-            # Upload using uv publish with explicit arguments
-            # Print masked command in blue
-            print(f"Running command: {BLUE}{' '.join(publish_cmd_safe_print)}{RESET}")
-            subprocess.run(
-                publish_cmd,  # Use the actual command with token
-                check=True,
-            )
+        # Upload to PyPI
+        print(f"Running command: {BLUE}{' '.join(publish_cmd_safe_print)}{RESET}")
+        subprocess.run(publish_cmd, check=True)
 
-            print(f"Successfully built and uploaded version {new_version} to PyPI")
+        print(f"Successfully built and uploaded version {new_version} to PyPI")
 
-            # Re-install DHT in current venv when building from DHT itself
-            # (Keep temp pyproject.toml until after this step)
-            try:
-                proj_name = None
-                try:
-                    proj_toml = toml.load(pyproject_path)
-                    proj_name = (
-                        proj_toml.get("project", {}).get("name")
-                        if isinstance(proj_toml, dict)
-                        else None
-                    )
-                except Exception:
-                    pass
-                if proj_name == "dayhoff-tools":
-                    print("Re-installing dayhoff-tools into the active environment…")
-                    reinstall_cmd = ["uv", "pip", "install", "-e", ".[full]"]
-                    print(f"Running command: {BLUE}{' '.join(reinstall_cmd)}{RESET}")
-                    subprocess.run(reinstall_cmd, check=True)
-                    print("dayhoff-tools reinstalled in the current environment.")
-            except subprocess.CalledProcessError as e:
-                print(f"Warning: Failed to reinstall dayhoff-tools locally: {e}")
-
-        finally:
-            # Restore original state (always clean up, even if errors occurred)
-            if temp_pyproject_created:
-                if Path("pyproject.toml").exists():
-                    Path("pyproject.toml").unlink()
-                if backup_created and Path("pyproject.toml.build.bak").exists():
-                    Path("pyproject.toml.build.bak").rename("pyproject.toml")
+        # Re-install DHT in Pixi environment when building from DHT itself
+        try:
+            proj_toml = toml.load(pyproject_path)
+            proj_name = proj_toml.get("project", {}).get("name")
+            if proj_name == "dayhoff-tools":
+                print("Re-installing dayhoff-tools into the Pixi environment...")
+                reinstall_cmd = ["pixi", "install"]
+                print(f"Running command: {BLUE}{' '.join(reinstall_cmd)}{RESET}")
+                subprocess.run(reinstall_cmd, check=True)
+                print("dayhoff-tools reinstalled in the Pixi environment.")
+        except subprocess.CalledProcessError as e:
+            print(f"Warning: Failed to reinstall dayhoff-tools locally: {e}")
+        except Exception:
+            pass  # Not dayhoff-tools or couldn't read toml
 
     except FileNotFoundError:
         print(f"Error: {pyproject_path} not found.")
@@ -304,42 +234,18 @@ def build_and_upload_wheel(bump_part: str = "patch"):
                         f"Warning: Could not find version {new_version} to revert in {pyproject_path}."
                     )
 
-                # Also revert other platform manifests
-                for candidate in [
-                    "pyproject.aws.toml",
-                    "pyproject.mac.toml",
-                    "pyproject.workstation.toml",
-                ]:
-                    if Path(candidate).exists() and candidate != pyproject_path:
-                        try:
-                            content_revert = Path(candidate).read_text()
-                            reverted, num = pattern_revert.subn(
-                                f'version = "{current_version}"', content_revert
-                            )
-                            if num > 0:
-                                Path(candidate).write_text(reverted)
-                                print(f"Successfully reverted version in {candidate}.")
-                        except Exception as e2:
-                            print(
-                                f"Warning: Failed to revert version change in {candidate}: {e2}"
-                            )
             except Exception as revert_e:
-                print(
-                    f"Warning: Failed to revert version change in {pyproject_path}: {revert_e}"
-                )
+                print(f"Warning: Failed to revert version change: {revert_e}")
     except Exception as e:
         print(f"An unexpected error occurred: {e}")
-        # Also attempt rollback here if version was bumped
+        # Attempt rollback if version was bumped
         if current_version and "new_version" in locals() and new_version:
             try:
-                print(
-                    f"Attempting to revert version in {pyproject_path} back to {current_version} due to unexpected error..."
-                )
-                # (Same revert logic as above)
+                print(f"Attempting to revert version back to {current_version}...")
                 with open(pyproject_path, "r") as f:
                     content_revert = f.read()
                 pattern_revert = re.compile(
-                    f'^version\s*=\s*"{re.escape(new_version)}"', re.MULTILINE
+                    f'^version\\s*=\\s*"{re.escape(new_version)}"', re.MULTILINE
                 )
                 reverted_content, num_revert = pattern_revert.subn(
                     f'version = "{current_version}"', content_revert
@@ -347,34 +253,11 @@ def build_and_upload_wheel(bump_part: str = "patch"):
                 if num_revert > 0:
                     with open(pyproject_path, "w") as f:
                         f.write(reverted_content)
-                    print(f"Successfully reverted version in {pyproject_path}.")
+                    print("Successfully reverted version in pyproject.toml.")
                 else:
-                    print(
-                        f"Warning: Could not find version {new_version} to revert in {pyproject_path}."
-                    )
-                # Also revert other platform manifests
-                for candidate in [
-                    "pyproject.aws.toml",
-                    "pyproject.mac.toml",
-                    "pyproject.workstation.toml",
-                ]:
-                    if Path(candidate).exists() and candidate != pyproject_path:
-                        try:
-                            content_revert = Path(candidate).read_text()
-                            reverted, num = pattern_revert.subn(
-                                f'version = "{current_version}"', content_revert
-                            )
-                            if num > 0:
-                                Path(candidate).write_text(reverted)
-                                print(f"Successfully reverted version in {candidate}.")
-                        except Exception as e2:
-                            print(
-                                f"Warning: Failed to revert version change in {candidate}: {e2}"
-                            )
+                    print(f"Warning: Could not find version {new_version} to revert.")
             except Exception as revert_e:
-                print(
-                    f"Warning: Failed to revert version change in {pyproject_path}: {revert_e}"
-                )
+                print(f"Warning: Failed to revert version change: {revert_e}")
 
 
 # --- Dependency Management Commands ---

{dayhoff_tools-1.14.13 → dayhoff_tools-1.14.15}/pyproject.toml

@@ -1,96 +1,73 @@
-# This block is needed because we package DHT for PyPi.
+# =============================================================================
+# DAYHOFF-TOOLS: Package Configuration
+# =============================================================================
+# This file defines the package for PyPI distribution and editable installs.
+# Development environment is managed by pixi.toml, NOT this file.
+# =============================================================================
+
 [build-system]
 requires = ["poetry-core>=2.0"]
 build-backend = "poetry.core.masonry.api"
 
 [project]
 name = "dayhoff-tools"
-version = "1.14.13"
+version = "1.14.15"
 description = "Common tools for all the repos at Dayhoff Labs"
 authors = [
     {name = "Daniel Martin-Alarcon", email = "dma@dayhofflabs.com"}
 ]
 readme = "README.md"
+requires-python = ">=3.10,<4.0"
 
+# Core dependencies - minimal set for basic CLI functionality
 dependencies = [
+    "boto3>=1.36.8",
     "firebase-admin>=6.5.0",
     "pyyaml>=6.0",
+    "questionary>=2.0.1",
     "requests>=2.31.0",
-    "typer>=0.9.0",
     "toml>=0.10",
-    "questionary>=2.0.1",
+    "typer>=0.9.0",
     "tzdata>=2025.2",
-    "boto3>=1.36.8",
 ]
-requires-python = ">=3.10,<4.0"
 
 [project.optional-dependencies]
+# Full set of optional dependencies for all features
 full = [
     "biopython>=1.84",
     "docker>=7.1.0",
     "fair-esm>=2.0.0",
     "h5py>=3.11.0",
-    "pandas (>=2.2.0,<2.2.3)",
+    "pandas>=2.2.0,<2.2.3",
     "rdkit-pypi>=2022.9.5",
-    "sqlalchemy (>=2.0.40,<3.0.0)",
-    "transformers==4.36.2",
     "sentencepiece>=0.2.0",
+    "sqlalchemy>=2.0.40,<3.0.0",
     "tqdm>=4.67.1",
+    "transformers==4.36.2",
 ]
+
+# Embedding models (requires torch - user must install separately for their platform)
 embedders = [
     "biopython>=1.85",
     "fair-esm>=2.0.0",
     "h5py>=3.13.0",
-    "numpy>=1.26.4",
-    "pandas (>=2.2.0,<2.2.3)",
+    "numpy>=1.26.4,<2.0",
+    "pandas>=2.2.0,<2.2.3",
     "sentencepiece>=0.2.0",
-    "torch>=2.4.0",
     "tqdm>=4.67.1",
     "transformers>=4.36.2",
 ]
-# Boltz structure prediction dependencies
+
+# Boltz structure prediction
 boltz = [
     "ruamel.yaml>=0.17.0",
 ]
+
 # Batch job system (CLI and worker utilities)
 batch = [
     "click>=8.0.0",
     "pydantic>=2.0.0",
 ]
 
-[[tool.uv.index]]
-name = "torch-cuda"
-url = "https://download.pytorch.org/whl/cu121"
-explicit = true
-
-[tool.uv.sources]
-torch = { index = "torch-cuda" }
-torchvision = { index = "torch-cuda" }
-
-[dependency-groups]
-dev = [
-    "black>=25.1.0,<26",
-    "boto3>=1.36.8",
-    "colorlog>=6.8.2,<7",
-    "docker>=7.1.0",
-    "dvc>=3.48.2,<4",
-    "dvc-gs>=3.0.1,<4",
-    "flake8>=7.0.0,<8",
-    "google-cloud-storage>=2.14.0",
-    "ipykernel>=6.29.5,<7",
-    "ipywidgets>=8.1.3,<9",
-    "isort>=5.13.2,<6",
-    "nbformat>=5.10.4",
-    "numpy<2.0.0",
-    "pandas (>=2.2.0,<2.2.3)",
-    "pylance>=0.10.2,<1",
-    "pylint>=3.1.0,<4",
-    "pytest>=8.0.2,<9",
-    "pytest-cov>=4.1.0,<5",
-    "pytest-mock>=3.12.0,<4",
-    "torch==2.4.0+cu121",
-    "torchvision==0.19.0+cu121",
-]
-
 [project.scripts]
 dh = "dayhoff_tools.cli.main:app"