dayhoff-tools 1.8.0__tar.gz → 1.8.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {dayhoff_tools-1.8.0 → dayhoff_tools-1.8.2}/PKG-INFO +1 -1
- {dayhoff_tools-1.8.0 → dayhoff_tools-1.8.2}/dayhoff_tools/cli/engine_commands.py +135 -8
- {dayhoff_tools-1.8.0 → dayhoff_tools-1.8.2}/pyproject.toml +1 -1
- {dayhoff_tools-1.8.0 → dayhoff_tools-1.8.2}/README.md +0 -0
- {dayhoff_tools-1.8.0 → dayhoff_tools-1.8.2}/dayhoff_tools/__init__.py +0 -0
- {dayhoff_tools-1.8.0 → dayhoff_tools-1.8.2}/dayhoff_tools/chemistry/standardizer.py +0 -0
- {dayhoff_tools-1.8.0 → dayhoff_tools-1.8.2}/dayhoff_tools/chemistry/utils.py +0 -0
- {dayhoff_tools-1.8.0 → dayhoff_tools-1.8.2}/dayhoff_tools/cli/__init__.py +0 -0
- {dayhoff_tools-1.8.0 → dayhoff_tools-1.8.2}/dayhoff_tools/cli/cloud_commands.py +0 -0
- {dayhoff_tools-1.8.0 → dayhoff_tools-1.8.2}/dayhoff_tools/cli/main.py +0 -0
- {dayhoff_tools-1.8.0 → dayhoff_tools-1.8.2}/dayhoff_tools/cli/swarm_commands.py +0 -0
- {dayhoff_tools-1.8.0 → dayhoff_tools-1.8.2}/dayhoff_tools/cli/utility_commands.py +0 -0
- {dayhoff_tools-1.8.0 → dayhoff_tools-1.8.2}/dayhoff_tools/deployment/base.py +0 -0
- {dayhoff_tools-1.8.0 → dayhoff_tools-1.8.2}/dayhoff_tools/deployment/deploy_aws.py +0 -0
- {dayhoff_tools-1.8.0 → dayhoff_tools-1.8.2}/dayhoff_tools/deployment/deploy_gcp.py +0 -0
- {dayhoff_tools-1.8.0 → dayhoff_tools-1.8.2}/dayhoff_tools/deployment/deploy_utils.py +0 -0
- {dayhoff_tools-1.8.0 → dayhoff_tools-1.8.2}/dayhoff_tools/deployment/job_runner.py +0 -0
- {dayhoff_tools-1.8.0 → dayhoff_tools-1.8.2}/dayhoff_tools/deployment/processors.py +0 -0
- {dayhoff_tools-1.8.0 → dayhoff_tools-1.8.2}/dayhoff_tools/deployment/swarm.py +0 -0
- {dayhoff_tools-1.8.0 → dayhoff_tools-1.8.2}/dayhoff_tools/embedders.py +0 -0
- {dayhoff_tools-1.8.0 → dayhoff_tools-1.8.2}/dayhoff_tools/fasta.py +0 -0
- {dayhoff_tools-1.8.0 → dayhoff_tools-1.8.2}/dayhoff_tools/file_ops.py +0 -0
- {dayhoff_tools-1.8.0 → dayhoff_tools-1.8.2}/dayhoff_tools/h5.py +0 -0
- {dayhoff_tools-1.8.0 → dayhoff_tools-1.8.2}/dayhoff_tools/intake/gcp.py +0 -0
- {dayhoff_tools-1.8.0 → dayhoff_tools-1.8.2}/dayhoff_tools/intake/gtdb.py +0 -0
- {dayhoff_tools-1.8.0 → dayhoff_tools-1.8.2}/dayhoff_tools/intake/kegg.py +0 -0
- {dayhoff_tools-1.8.0 → dayhoff_tools-1.8.2}/dayhoff_tools/intake/mmseqs.py +0 -0
- {dayhoff_tools-1.8.0 → dayhoff_tools-1.8.2}/dayhoff_tools/intake/structure.py +0 -0
- {dayhoff_tools-1.8.0 → dayhoff_tools-1.8.2}/dayhoff_tools/intake/uniprot.py +0 -0
- {dayhoff_tools-1.8.0 → dayhoff_tools-1.8.2}/dayhoff_tools/logs.py +0 -0
- {dayhoff_tools-1.8.0 → dayhoff_tools-1.8.2}/dayhoff_tools/sqlite.py +0 -0
- {dayhoff_tools-1.8.0 → dayhoff_tools-1.8.2}/dayhoff_tools/warehouse.py +0 -0
@@ -9,7 +9,7 @@ import sys
|
|
9
9
|
import time
|
10
10
|
from datetime import datetime, timedelta, timezone
|
11
11
|
from pathlib import Path
|
12
|
-
from typing import Dict, List, Optional, Tuple
|
12
|
+
from typing import Any, Dict, List, Optional, Tuple
|
13
13
|
|
14
14
|
import boto3
|
15
15
|
import requests
|
@@ -647,7 +647,7 @@ def launch_engine(
|
|
647
647
|
) as progress:
|
648
648
|
progress.add_task("Creating engine...", total=None)
|
649
649
|
|
650
|
-
request_data = {
|
650
|
+
request_data: Dict[str, Any] = {
|
651
651
|
"name": name,
|
652
652
|
"user": username,
|
653
653
|
"engine_type": engine_type,
|
@@ -797,18 +797,50 @@ def engine_status(
|
|
797
797
|
|
798
798
|
engine_details = response.json()
|
799
799
|
engine = engine_details.get("engine", engine) # Use detailed info if available
|
800
|
-
idle_detector = engine_details.get("idle_detector", {})
|
800
|
+
idle_detector = engine_details.get("idle_detector", {}) or {}
|
801
801
|
attached_studios = engine_details.get("attached_studios", [])
|
802
802
|
|
803
803
|
# Calculate costs
|
804
804
|
launch_time = parse_launch_time(engine["launch_time"])
|
805
805
|
uptime = datetime.now(timezone.utc) - launch_time
|
806
806
|
hourly_cost = HOURLY_COSTS.get(engine["engine_type"], 0)
|
807
|
-
total_cost
|
807
|
+
# total_cost intentionally not shown in status view
|
808
808
|
|
809
809
|
stages_map = _fetch_init_stages([engine["instance_id"]])
|
810
810
|
stage_val = stages_map.get(engine["instance_id"], "-")
|
811
811
|
|
812
|
+
# Try to fetch actual boot time via SSM (best-effort)
|
813
|
+
boot_time_str: Optional[str] = None
|
814
|
+
try:
|
815
|
+
if engine["state"].lower() == "running":
|
816
|
+
ssm = boto3.client("ssm", region_name="us-east-1")
|
817
|
+
resp = ssm.send_command(
|
818
|
+
InstanceIds=[engine["instance_id"]],
|
819
|
+
DocumentName="AWS-RunShellScript",
|
820
|
+
Parameters={
|
821
|
+
"commands": ["uptime -s || who -b | awk '{print $3\" \"$4}'"]
|
822
|
+
},
|
823
|
+
)
|
824
|
+
cid = resp["Command"]["CommandId"]
|
825
|
+
time.sleep(1)
|
826
|
+
inv = ssm.get_command_invocation(
|
827
|
+
CommandId=cid, InstanceId=engine["instance_id"]
|
828
|
+
)
|
829
|
+
if inv.get("Status") == "Success":
|
830
|
+
boot_time_str = (
|
831
|
+
(inv.get("StandardOutputContent") or "").strip().splitlines()[0]
|
832
|
+
if inv.get("StandardOutputContent")
|
833
|
+
else None
|
834
|
+
)
|
835
|
+
except Exception:
|
836
|
+
boot_time_str = None
|
837
|
+
|
838
|
+
started_line = (
|
839
|
+
f"[bold]Started:[/bold] {boot_time_str} ({format_duration(uptime)} ago)"
|
840
|
+
if boot_time_str
|
841
|
+
else f"[bold]Started:[/bold] {launch_time.strftime('%Y-%m-%d %H:%M:%S')} ({format_duration(uptime)} ago)"
|
842
|
+
)
|
843
|
+
|
812
844
|
status_lines = [
|
813
845
|
f"[bold]Name:[/bold] {engine['name']}",
|
814
846
|
f"[bold]Instance:[/bold] {engine['instance_id']}",
|
@@ -816,10 +848,15 @@ def engine_status(
|
|
816
848
|
f"[bold]Status:[/bold] {format_status(engine['state'], engine.get('ready'))}",
|
817
849
|
f"[bold]User:[/bold] {engine['user']}",
|
818
850
|
f"[bold]IP:[/bold] {engine.get('public_ip', 'N/A')}",
|
819
|
-
|
820
|
-
f"[bold]
|
851
|
+
started_line,
|
852
|
+
f"[bold]$/hour:[/bold] ${hourly_cost:.2f}",
|
821
853
|
]
|
822
854
|
|
855
|
+
# Disk usage (like list --detailed)
|
856
|
+
if engine["state"].lower() == "running":
|
857
|
+
disk_usage = get_disk_usage_via_ssm(engine["instance_id"]) or "-"
|
858
|
+
status_lines.append(f"[bold]Disk:[/bold] {disk_usage}")
|
859
|
+
|
823
860
|
# Health report (only if bootstrap finished)
|
824
861
|
if stage_val == "finished":
|
825
862
|
try:
|
@@ -854,7 +891,80 @@ def engine_status(
|
|
854
891
|
except Exception:
|
855
892
|
pass
|
856
893
|
|
857
|
-
#
|
894
|
+
# Try to enrich/fallback idle-detector details from on-engine summary file via SSM
|
895
|
+
def _fetch_idle_summary_via_ssm(instance_id: str) -> Optional[Dict]:
|
896
|
+
try:
|
897
|
+
ssm = boto3.client("ssm", region_name="us-east-1")
|
898
|
+
res = ssm.send_command(
|
899
|
+
InstanceIds=[instance_id],
|
900
|
+
DocumentName="AWS-RunShellScript",
|
901
|
+
Parameters={
|
902
|
+
"commands": [
|
903
|
+
"cat /var/run/idle-detector/last_state.json 2>/dev/null || true",
|
904
|
+
],
|
905
|
+
"executionTimeout": ["5"],
|
906
|
+
},
|
907
|
+
)
|
908
|
+
cid = res["Command"]["CommandId"]
|
909
|
+
time.sleep(1)
|
910
|
+
inv = ssm.get_command_invocation(CommandId=cid, InstanceId=instance_id)
|
911
|
+
if inv["Status"] != "Success":
|
912
|
+
return None
|
913
|
+
content = inv["StandardOutputContent"].strip()
|
914
|
+
if not content:
|
915
|
+
return None
|
916
|
+
data = json.loads(content)
|
917
|
+
# Convert last_state schema to idle_detector schema used by CLI output
|
918
|
+
idle_info: Dict[str, Any] = {"available": True}
|
919
|
+
idle_info["status"] = "active" if not data.get("idle", True) else "idle"
|
920
|
+
# thresholds if present
|
921
|
+
if isinstance(data.get("timeout_sec"), (int, float)):
|
922
|
+
idle_info["idle_threshold"] = int(data["timeout_sec"]) # seconds
|
923
|
+
# keep raw reasons for sensor display
|
924
|
+
if isinstance(data.get("reasons"), list):
|
925
|
+
idle_info["_reasons_raw"] = data["reasons"]
|
926
|
+
# derive details from sensors
|
927
|
+
for r in data.get("reasons", []):
|
928
|
+
if not r.get("active"):
|
929
|
+
continue
|
930
|
+
sensor = (r.get("sensor") or "").lower()
|
931
|
+
forensic = r.get("forensic") or {}
|
932
|
+
if sensor == "ideconnectionsensor":
|
933
|
+
cnt = forensic.get("matches")
|
934
|
+
if isinstance(cnt, int):
|
935
|
+
idle_info["ide_connections"] = {"connection_count": cnt}
|
936
|
+
else:
|
937
|
+
idle_info["ide_connections"] = {"connection_count": 1}
|
938
|
+
elif sensor == "coffeelocksensor":
|
939
|
+
rem = forensic.get("remaining_sec")
|
940
|
+
if isinstance(rem, (int, float)) and rem > 0:
|
941
|
+
idle_info["coffee_lock"] = format_duration(
|
942
|
+
timedelta(seconds=int(rem))
|
943
|
+
)
|
944
|
+
elif sensor == "activeloginsensor":
|
945
|
+
# Provide a single summarized SSH session if available
|
946
|
+
sess = {
|
947
|
+
"tty": r.get("forensic", {}).get("tty", "pts/?"),
|
948
|
+
"pid": r.get("forensic", {}).get("pid", "?"),
|
949
|
+
"idle_time": r.get("forensic", {}).get("idle_sec", 0),
|
950
|
+
"from_ip": r.get("forensic", {}).get("remote_addr", "unknown"),
|
951
|
+
}
|
952
|
+
idle_info.setdefault("ssh_sessions", []).append(sess)
|
953
|
+
return idle_info
|
954
|
+
except Exception:
|
955
|
+
return None
|
956
|
+
|
957
|
+
# Always try to enrich from on-engine summary (fast, best-effort)
|
958
|
+
overlay = _fetch_idle_summary_via_ssm(engine["instance_id"])
|
959
|
+
if overlay:
|
960
|
+
# If API didn't indicate availability, replace entirely; otherwise fill gaps
|
961
|
+
if not idle_detector.get("available"):
|
962
|
+
idle_detector = overlay
|
963
|
+
else:
|
964
|
+
for k, v in overlay.items():
|
965
|
+
idle_detector.setdefault(k, v)
|
966
|
+
|
967
|
+
# Idle detector status (API and/or on-engine fallback)
|
858
968
|
if idle_detector.get("available"):
|
859
969
|
status_lines.append("")
|
860
970
|
status_lines.append("[bold]Idle Detector:[/bold]")
|
@@ -882,8 +992,13 @@ def engine_status(
|
|
882
992
|
if ssh_sessions:
|
883
993
|
status_lines.append(f" • [blue]SSH Sessions ({len(ssh_sessions)}):[/blue]")
|
884
994
|
for session in ssh_sessions:
|
995
|
+
idle_time = session.get("idle_time")
|
996
|
+
if isinstance(idle_time, (int, float)):
|
997
|
+
idle_disp = f"{int(idle_time)//60}m"
|
998
|
+
else:
|
999
|
+
idle_disp = str(idle_time) if idle_time else "0m"
|
885
1000
|
status_lines.append(
|
886
|
-
f" - {session
|
1001
|
+
f" - {session.get('tty', 'pts/?')} (pid {session.get('pid', '?')}, idle {idle_disp}) from {session.get('from_ip', 'unknown')}"
|
887
1002
|
)
|
888
1003
|
|
889
1004
|
# IDE connections
|
@@ -893,6 +1008,18 @@ def engine_status(
|
|
893
1008
|
f" • [magenta]🖥 IDE connected ({ide_conn['connection_count']} connections)[/magenta]"
|
894
1009
|
)
|
895
1010
|
|
1011
|
+
# Sensors contributing to ACTIVE
|
1012
|
+
reasons_raw = idle_detector.get("_reasons_raw")
|
1013
|
+
if isinstance(reasons_raw, list):
|
1014
|
+
active_sensors = [r for r in reasons_raw if r.get("active")]
|
1015
|
+
if active_sensors:
|
1016
|
+
status_lines.append("")
|
1017
|
+
status_lines.append("[bold]Active Sensors:[/bold]")
|
1018
|
+
for r in active_sensors:
|
1019
|
+
sensor = r.get("sensor", "Sensor")
|
1020
|
+
reason = r.get("reason") or "active"
|
1021
|
+
status_lines.append(f" • {sensor}: {reason}")
|
1022
|
+
|
896
1023
|
# Audit one-liner (best-effort SSM fetch)
|
897
1024
|
try:
|
898
1025
|
last_audit = _fetch_last_audit_via_ssm(engine["instance_id"])
|
@@ -5,7 +5,7 @@ build-backend = "poetry.core.masonry.api"
|
|
5
5
|
|
6
6
|
[project]
|
7
7
|
name = "dayhoff-tools"
|
8
|
-
version = "1.8.
|
8
|
+
version = "1.8.2"
|
9
9
|
description = "Common tools for all the repos at Dayhoff Labs"
|
10
10
|
authors = [
|
11
11
|
{name = "Daniel Martin-Alarcon", email = "dma@dayhofflabs.com"}
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|