dayhoff-tools 1.9.3__tar.gz → 1.9.5__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {dayhoff_tools-1.9.3 → dayhoff_tools-1.9.5}/PKG-INFO +1 -1
- {dayhoff_tools-1.9.3 → dayhoff_tools-1.9.5}/dayhoff_tools/cli/engine_commands.py +118 -34
- {dayhoff_tools-1.9.3 → dayhoff_tools-1.9.5}/pyproject.toml +1 -1
- {dayhoff_tools-1.9.3 → dayhoff_tools-1.9.5}/README.md +0 -0
- {dayhoff_tools-1.9.3 → dayhoff_tools-1.9.5}/dayhoff_tools/__init__.py +0 -0
- {dayhoff_tools-1.9.3 → dayhoff_tools-1.9.5}/dayhoff_tools/chemistry/standardizer.py +0 -0
- {dayhoff_tools-1.9.3 → dayhoff_tools-1.9.5}/dayhoff_tools/chemistry/utils.py +0 -0
- {dayhoff_tools-1.9.3 → dayhoff_tools-1.9.5}/dayhoff_tools/cli/__init__.py +0 -0
- {dayhoff_tools-1.9.3 → dayhoff_tools-1.9.5}/dayhoff_tools/cli/cloud_commands.py +0 -0
- {dayhoff_tools-1.9.3 → dayhoff_tools-1.9.5}/dayhoff_tools/cli/main.py +0 -0
- {dayhoff_tools-1.9.3 → dayhoff_tools-1.9.5}/dayhoff_tools/cli/swarm_commands.py +0 -0
- {dayhoff_tools-1.9.3 → dayhoff_tools-1.9.5}/dayhoff_tools/cli/utility_commands.py +0 -0
- {dayhoff_tools-1.9.3 → dayhoff_tools-1.9.5}/dayhoff_tools/deployment/base.py +0 -0
- {dayhoff_tools-1.9.3 → dayhoff_tools-1.9.5}/dayhoff_tools/deployment/deploy_aws.py +0 -0
- {dayhoff_tools-1.9.3 → dayhoff_tools-1.9.5}/dayhoff_tools/deployment/deploy_gcp.py +0 -0
- {dayhoff_tools-1.9.3 → dayhoff_tools-1.9.5}/dayhoff_tools/deployment/deploy_utils.py +0 -0
- {dayhoff_tools-1.9.3 → dayhoff_tools-1.9.5}/dayhoff_tools/deployment/job_runner.py +0 -0
- {dayhoff_tools-1.9.3 → dayhoff_tools-1.9.5}/dayhoff_tools/deployment/processors.py +0 -0
- {dayhoff_tools-1.9.3 → dayhoff_tools-1.9.5}/dayhoff_tools/deployment/swarm.py +0 -0
- {dayhoff_tools-1.9.3 → dayhoff_tools-1.9.5}/dayhoff_tools/embedders.py +0 -0
- {dayhoff_tools-1.9.3 → dayhoff_tools-1.9.5}/dayhoff_tools/fasta.py +0 -0
- {dayhoff_tools-1.9.3 → dayhoff_tools-1.9.5}/dayhoff_tools/file_ops.py +0 -0
- {dayhoff_tools-1.9.3 → dayhoff_tools-1.9.5}/dayhoff_tools/h5.py +0 -0
- {dayhoff_tools-1.9.3 → dayhoff_tools-1.9.5}/dayhoff_tools/intake/gcp.py +0 -0
- {dayhoff_tools-1.9.3 → dayhoff_tools-1.9.5}/dayhoff_tools/intake/gtdb.py +0 -0
- {dayhoff_tools-1.9.3 → dayhoff_tools-1.9.5}/dayhoff_tools/intake/kegg.py +0 -0
- {dayhoff_tools-1.9.3 → dayhoff_tools-1.9.5}/dayhoff_tools/intake/mmseqs.py +0 -0
- {dayhoff_tools-1.9.3 → dayhoff_tools-1.9.5}/dayhoff_tools/intake/structure.py +0 -0
- {dayhoff_tools-1.9.3 → dayhoff_tools-1.9.5}/dayhoff_tools/intake/uniprot.py +0 -0
- {dayhoff_tools-1.9.3 → dayhoff_tools-1.9.5}/dayhoff_tools/logs.py +0 -0
- {dayhoff_tools-1.9.3 → dayhoff_tools-1.9.5}/dayhoff_tools/sqlite.py +0 -0
- {dayhoff_tools-1.9.3 → dayhoff_tools-1.9.5}/dayhoff_tools/warehouse.py +0 -0
@@ -778,15 +778,24 @@ def engine_status(
|
|
778
778
|
return "[green]Active[/green]"
|
779
779
|
if running_state in ("stopped", "stopping"):
|
780
780
|
return "[dim]N/A[/dim]"
|
781
|
-
|
782
|
-
|
783
|
-
if
|
784
|
-
|
785
|
-
|
786
|
-
|
787
|
-
|
788
|
-
)
|
789
|
-
|
781
|
+
|
782
|
+
# If we don't have idle info at all, show N/A
|
783
|
+
if not idle_info.get("available"):
|
784
|
+
return "[dim]N/A[/dim]"
|
785
|
+
|
786
|
+
# If idle, show time/threshold if available, otherwise show uncertainty
|
787
|
+
if idle_info.get("status") == "idle":
|
788
|
+
idle_seconds_v = idle_info.get("idle_seconds")
|
789
|
+
thresh_v = idle_info.get("idle_threshold")
|
790
|
+
if isinstance(idle_seconds_v, (int, float)) and isinstance(thresh_v, (int, float)):
|
791
|
+
return f"[yellow]Idle {int(idle_seconds_v)//60}m/{int(thresh_v)//60}m[/yellow]"
|
792
|
+
elif isinstance(thresh_v, (int, float)):
|
793
|
+
return f"[yellow]Idle ?/{int(thresh_v)//60}m[/yellow]"
|
794
|
+
else:
|
795
|
+
return "[yellow]Idle ?/?[/yellow]"
|
796
|
+
|
797
|
+
# Default to N/A if we can't determine status
|
798
|
+
return "[dim]N/A[/dim]"
|
790
799
|
|
791
800
|
active_disp = _compute_active_disp(idle_detector)
|
792
801
|
|
@@ -894,9 +903,10 @@ def engine_status(
|
|
894
903
|
status_lines.append(
|
895
904
|
f" • GPU Drivers: {'OK' if health.get('drivers_ok') else 'MISSING'}"
|
896
905
|
)
|
897
|
-
|
898
|
-
|
906
|
+
idle_stat = health.get("idle_detector_service") or health.get(
|
907
|
+
"idle_detector_timer", "unknown"
|
899
908
|
)
|
909
|
+
status_lines.append(f" • Idle Detector: {idle_stat}")
|
900
910
|
except Exception:
|
901
911
|
pass
|
902
912
|
|
@@ -915,8 +925,12 @@ def engine_status(
|
|
915
925
|
},
|
916
926
|
)
|
917
927
|
cid = res["Command"]["CommandId"]
|
918
|
-
|
919
|
-
|
928
|
+
# Wait up to 2 seconds for SSM command to complete (was 1 second)
|
929
|
+
for _ in range(4): # 4 * 0.5 = 2 seconds
|
930
|
+
time.sleep(0.5)
|
931
|
+
inv = ssm.get_command_invocation(CommandId=cid, InstanceId=instance_id)
|
932
|
+
if inv["Status"] in ["Success", "Failed"]:
|
933
|
+
break
|
920
934
|
if inv["Status"] != "Success":
|
921
935
|
return None
|
922
936
|
content = inv["StandardOutputContent"].strip()
|
@@ -1036,23 +1050,10 @@ def engine_status(
|
|
1036
1050
|
status_lines.append(_sensor_line(" IDE ", "IDEConnectionSensor", "🖥"))
|
1037
1051
|
status_lines.append(_sensor_line("Docker", "DockerWorkloadSensor", "🐳"))
|
1038
1052
|
|
1039
|
-
#
|
1040
|
-
|
1041
|
-
|
1042
|
-
|
1043
|
-
thresh_secs = int(idle_detector.get("idle_threshold", 0))
|
1044
|
-
if thresh_secs > 0:
|
1045
|
-
active_disp = (
|
1046
|
-
f"[yellow]Idle {idle_secs//60}m/{thresh_secs//60}m[/yellow]"
|
1047
|
-
)
|
1048
|
-
# Rewrite top header line (index 0) to include updated display
|
1049
|
-
all_header = top_lines[0]
|
1050
|
-
# Replace the portion after two spaces (name and running state fixed)
|
1051
|
-
top_lines[0] = (
|
1052
|
-
f"[blue]{engine['name']}[/blue] {run_disp} {active_disp}\n"
|
1053
|
-
)
|
1054
|
-
except Exception:
|
1055
|
-
pass
|
1053
|
+
# Recompute display with latest idle detector data
|
1054
|
+
active_disp = _compute_active_disp(idle_detector)
|
1055
|
+
# Rewrite top header line (index 0) to include updated display
|
1056
|
+
top_lines[0] = f"[blue]{engine['name']}[/blue] {run_disp} {active_disp}\n"
|
1056
1057
|
|
1057
1058
|
# Combine top summary and details
|
1058
1059
|
all_lines = top_lines + status_lines
|
@@ -2630,6 +2631,9 @@ def idle_timeout_cmd(
|
|
2630
2631
|
set: Optional[str] = typer.Option(
|
2631
2632
|
None, "--set", "-s", help="New timeout (e.g., 2h30m, 45m)"
|
2632
2633
|
),
|
2634
|
+
status: bool = typer.Option(
|
2635
|
+
False, "--status", help="Show detailed idle status (faster than full engine status)"
|
2636
|
+
),
|
2633
2637
|
):
|
2634
2638
|
"""Show or set the engine idle-detector timeout."""
|
2635
2639
|
check_aws_sso()
|
@@ -2644,9 +2648,90 @@ def idle_timeout_cmd(
|
|
2644
2648
|
engine = resolve_engine(name_or_id, engines)
|
2645
2649
|
|
2646
2650
|
ssm = boto3.client("ssm", region_name="us-east-1")
|
2651
|
+
|
2652
|
+
# Handle --status flag for quick idle status check
|
2653
|
+
if status:
|
2654
|
+
console.print(f"[bold]Idle Status for {engine['name']}:[/bold]")
|
2655
|
+
|
2656
|
+
# Fetch state file with longer timeout for reliability
|
2657
|
+
try:
|
2658
|
+
resp = ssm.send_command(
|
2659
|
+
InstanceIds=[engine["instance_id"]],
|
2660
|
+
DocumentName="AWS-RunShellScript",
|
2661
|
+
Parameters={
|
2662
|
+
"commands": [
|
2663
|
+
"cat /var/run/idle-detector/last_state.json 2>/dev/null || echo '{}'"
|
2664
|
+
],
|
2665
|
+
"executionTimeout": ["10"],
|
2666
|
+
},
|
2667
|
+
)
|
2668
|
+
cid = resp["Command"]["CommandId"]
|
2669
|
+
|
2670
|
+
# Wait up to 3 seconds for result
|
2671
|
+
for _ in range(6): # 6 * 0.5 = 3 seconds
|
2672
|
+
time.sleep(0.5)
|
2673
|
+
inv = ssm.get_command_invocation(
|
2674
|
+
CommandId=cid, InstanceId=engine["instance_id"]
|
2675
|
+
)
|
2676
|
+
if inv["Status"] in ["Success", "Failed"]:
|
2677
|
+
break
|
2678
|
+
|
2679
|
+
if inv["Status"] == "Success":
|
2680
|
+
import json
|
2681
|
+
content = inv["StandardOutputContent"].strip()
|
2682
|
+
if content and content != "{}":
|
2683
|
+
data = json.loads(content)
|
2684
|
+
|
2685
|
+
# Extract key values
|
2686
|
+
is_idle = data.get("idle", False)
|
2687
|
+
timeout_sec = data.get("timeout_sec", "?")
|
2688
|
+
idle_seconds = data.get("idle_seconds", 0) if is_idle else 0
|
2689
|
+
|
2690
|
+
# Display status
|
2691
|
+
if is_idle:
|
2692
|
+
if isinstance(timeout_sec, int) and isinstance(idle_seconds, int):
|
2693
|
+
console.print(f" Status: [yellow]Idle {idle_seconds//60}m/{timeout_sec//60}m[/yellow]")
|
2694
|
+
remaining = max(0, timeout_sec - idle_seconds)
|
2695
|
+
console.print(f" Time until shutdown: [red]{remaining//60}m {remaining%60}s[/red]")
|
2696
|
+
else:
|
2697
|
+
console.print(f" Status: [yellow]Idle (timing unavailable)[/yellow]")
|
2698
|
+
else:
|
2699
|
+
console.print(f" Status: [green]Active[/green]")
|
2700
|
+
if isinstance(timeout_sec, int):
|
2701
|
+
console.print(f" Timeout: {timeout_sec//60}m ({timeout_sec}s)")
|
2702
|
+
|
2703
|
+
# Show activity sensors
|
2704
|
+
console.print("\n[bold]Activity Sensors:[/bold]")
|
2705
|
+
reasons = data.get("reasons", [])
|
2706
|
+
for r in reasons:
|
2707
|
+
sensor = r.get("sensor", "Unknown")
|
2708
|
+
active = r.get("active", False)
|
2709
|
+
reason = r.get("reason", "")
|
2710
|
+
|
2711
|
+
# Map sensor names to icons and labels
|
2712
|
+
sensor_map = {
|
2713
|
+
"CoffeeLockSensor": ("☕", "Coffee"),
|
2714
|
+
"ActiveLoginSensor": ("🐚", "SSH"),
|
2715
|
+
"IDEConnectionSensor": ("🖥", "IDE"),
|
2716
|
+
"DockerWorkloadSensor": ("🐳", "Docker"),
|
2717
|
+
}
|
2718
|
+
|
2719
|
+
icon, label = sensor_map.get(sensor, ("?", sensor))
|
2720
|
+
status_str = "[green]YES[/green]" if active else "[dim]no[/dim]"
|
2721
|
+
console.print(f" {icon} {label:8} {status_str} {reason if active else ''}")
|
2722
|
+
|
2723
|
+
else:
|
2724
|
+
console.print("[yellow]⚠️ Could not read idle detector state[/yellow]")
|
2725
|
+
else:
|
2726
|
+
console.print(f"[red]❌ Failed to get status: {inv.get('StandardErrorContent', 'Unknown error')}[/red]")
|
2727
|
+
|
2728
|
+
except Exception as e:
|
2729
|
+
console.print(f"[red]❌ Error fetching idle status: {e}[/red]")
|
2730
|
+
|
2731
|
+
return
|
2647
2732
|
|
2648
2733
|
if set is None:
|
2649
|
-
# Show current
|
2734
|
+
# Show current timeout setting
|
2650
2735
|
resp = ssm.send_command(
|
2651
2736
|
InstanceIds=[engine["instance_id"]],
|
2652
2737
|
DocumentName="AWS-RunShellScript",
|
@@ -2687,7 +2772,7 @@ def idle_timeout_cmd(
|
|
2687
2772
|
cmd = (
|
2688
2773
|
"sudo sed -i '/^IDLE_TIMEOUT_SECONDS=/d' /etc/engine.env && "
|
2689
2774
|
f"echo 'IDLE_TIMEOUT_SECONDS={seconds}' | sudo tee -a /etc/engine.env >/dev/null && "
|
2690
|
-
"sudo systemctl restart engine-idle-detector.
|
2775
|
+
"sudo systemctl restart engine-idle-detector.service"
|
2691
2776
|
)
|
2692
2777
|
|
2693
2778
|
resp = ssm.send_command(
|
@@ -2826,8 +2911,7 @@ def repair_engine(
|
|
2826
2911
|
"echo 'finished' | sudo tee /opt/dayhoff/state/engine-init.stage > /dev/null",
|
2827
2912
|
# Ensure SSM agent is running
|
2828
2913
|
"sudo systemctl restart amazon-ssm-agent 2>/dev/null || true",
|
2829
|
-
# Restart idle detector
|
2830
|
-
"sudo systemctl restart engine-idle-detector.timer 2>/dev/null || true",
|
2914
|
+
# Restart idle detector (service only)
|
2831
2915
|
"sudo systemctl restart engine-idle-detector.service 2>/dev/null || true",
|
2832
2916
|
# Report status
|
2833
2917
|
"echo '=== Repair Complete ===' && echo 'Sentinel: ' && ls -la /opt/dayhoff/first_boot_complete.sentinel",
|
@@ -5,7 +5,7 @@ build-backend = "poetry.core.masonry.api"
|
|
5
5
|
|
6
6
|
[project]
|
7
7
|
name = "dayhoff-tools"
|
8
|
-
version = "1.9.
|
8
|
+
version = "1.9.5"
|
9
9
|
description = "Common tools for all the repos at Dayhoff Labs"
|
10
10
|
authors = [
|
11
11
|
{name = "Daniel Martin-Alarcon", email = "dma@dayhofflabs.com"}
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|