dayhoff-tools 1.9.0__tar.gz → 1.9.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {dayhoff_tools-1.9.0 → dayhoff_tools-1.9.2}/PKG-INFO +1 -1
- {dayhoff_tools-1.9.0 → dayhoff_tools-1.9.2}/dayhoff_tools/cli/engine_commands.py +60 -164
- {dayhoff_tools-1.9.0 → dayhoff_tools-1.9.2}/pyproject.toml +1 -1
- {dayhoff_tools-1.9.0 → dayhoff_tools-1.9.2}/README.md +0 -0
- {dayhoff_tools-1.9.0 → dayhoff_tools-1.9.2}/dayhoff_tools/__init__.py +0 -0
- {dayhoff_tools-1.9.0 → dayhoff_tools-1.9.2}/dayhoff_tools/chemistry/standardizer.py +0 -0
- {dayhoff_tools-1.9.0 → dayhoff_tools-1.9.2}/dayhoff_tools/chemistry/utils.py +0 -0
- {dayhoff_tools-1.9.0 → dayhoff_tools-1.9.2}/dayhoff_tools/cli/__init__.py +0 -0
- {dayhoff_tools-1.9.0 → dayhoff_tools-1.9.2}/dayhoff_tools/cli/cloud_commands.py +0 -0
- {dayhoff_tools-1.9.0 → dayhoff_tools-1.9.2}/dayhoff_tools/cli/main.py +0 -0
- {dayhoff_tools-1.9.0 → dayhoff_tools-1.9.2}/dayhoff_tools/cli/swarm_commands.py +0 -0
- {dayhoff_tools-1.9.0 → dayhoff_tools-1.9.2}/dayhoff_tools/cli/utility_commands.py +0 -0
- {dayhoff_tools-1.9.0 → dayhoff_tools-1.9.2}/dayhoff_tools/deployment/base.py +0 -0
- {dayhoff_tools-1.9.0 → dayhoff_tools-1.9.2}/dayhoff_tools/deployment/deploy_aws.py +0 -0
- {dayhoff_tools-1.9.0 → dayhoff_tools-1.9.2}/dayhoff_tools/deployment/deploy_gcp.py +0 -0
- {dayhoff_tools-1.9.0 → dayhoff_tools-1.9.2}/dayhoff_tools/deployment/deploy_utils.py +0 -0
- {dayhoff_tools-1.9.0 → dayhoff_tools-1.9.2}/dayhoff_tools/deployment/job_runner.py +0 -0
- {dayhoff_tools-1.9.0 → dayhoff_tools-1.9.2}/dayhoff_tools/deployment/processors.py +0 -0
- {dayhoff_tools-1.9.0 → dayhoff_tools-1.9.2}/dayhoff_tools/deployment/swarm.py +0 -0
- {dayhoff_tools-1.9.0 → dayhoff_tools-1.9.2}/dayhoff_tools/embedders.py +0 -0
- {dayhoff_tools-1.9.0 → dayhoff_tools-1.9.2}/dayhoff_tools/fasta.py +0 -0
- {dayhoff_tools-1.9.0 → dayhoff_tools-1.9.2}/dayhoff_tools/file_ops.py +0 -0
- {dayhoff_tools-1.9.0 → dayhoff_tools-1.9.2}/dayhoff_tools/h5.py +0 -0
- {dayhoff_tools-1.9.0 → dayhoff_tools-1.9.2}/dayhoff_tools/intake/gcp.py +0 -0
- {dayhoff_tools-1.9.0 → dayhoff_tools-1.9.2}/dayhoff_tools/intake/gtdb.py +0 -0
- {dayhoff_tools-1.9.0 → dayhoff_tools-1.9.2}/dayhoff_tools/intake/kegg.py +0 -0
- {dayhoff_tools-1.9.0 → dayhoff_tools-1.9.2}/dayhoff_tools/intake/mmseqs.py +0 -0
- {dayhoff_tools-1.9.0 → dayhoff_tools-1.9.2}/dayhoff_tools/intake/structure.py +0 -0
- {dayhoff_tools-1.9.0 → dayhoff_tools-1.9.2}/dayhoff_tools/intake/uniprot.py +0 -0
- {dayhoff_tools-1.9.0 → dayhoff_tools-1.9.2}/dayhoff_tools/logs.py +0 -0
- {dayhoff_tools-1.9.0 → dayhoff_tools-1.9.2}/dayhoff_tools/sqlite.py +0 -0
- {dayhoff_tools-1.9.0 → dayhoff_tools-1.9.2}/dayhoff_tools/warehouse.py +0 -0
@@ -319,88 +319,6 @@ def format_status(state: str, ready: Optional[bool]) -> str:
|
|
319
319
|
return state
|
320
320
|
|
321
321
|
|
322
|
-
# --------------------------------------------------------------------------------
|
323
|
-
# Audit helpers (Phase 1 observability)
|
324
|
-
# --------------------------------------------------------------------------------
|
325
|
-
|
326
|
-
|
327
|
-
def _get_engine_audit_bucket() -> Optional[str]:
|
328
|
-
"""Return the engine audit bucket name from SSM Parameter Store, if configured."""
|
329
|
-
try:
|
330
|
-
ssm = boto3.client("ssm", region_name="us-east-1")
|
331
|
-
resp = ssm.get_parameter(Name="/dev/studio-manager/engine-audit-bucket")
|
332
|
-
return resp["Parameter"]["Value"]
|
333
|
-
except ClientError:
|
334
|
-
return None
|
335
|
-
|
336
|
-
|
337
|
-
def _fetch_last_audit_via_ssm(instance_id: str) -> Optional[Dict]:
|
338
|
-
"""Fetch last shutdown attempt audit from the engine via SSM (fast best-effort)."""
|
339
|
-
try:
|
340
|
-
ssm = boto3.client("ssm", region_name="us-east-1")
|
341
|
-
resp = ssm.send_command(
|
342
|
-
InstanceIds=[instance_id],
|
343
|
-
DocumentName="AWS-RunShellScript",
|
344
|
-
Parameters={
|
345
|
-
"commands": [
|
346
|
-
"cat /var/log/idle-detector/last_shutdown_attempt.json 2>/dev/null || true",
|
347
|
-
],
|
348
|
-
"executionTimeout": ["3"],
|
349
|
-
},
|
350
|
-
)
|
351
|
-
cid = resp["Command"]["CommandId"]
|
352
|
-
time.sleep(1)
|
353
|
-
inv = ssm.get_command_invocation(CommandId=cid, InstanceId=instance_id)
|
354
|
-
if inv["Status"] != "Success":
|
355
|
-
return None
|
356
|
-
content = inv["StandardOutputContent"].strip()
|
357
|
-
if not content:
|
358
|
-
return None
|
359
|
-
return json.loads(content)
|
360
|
-
except Exception:
|
361
|
-
return None
|
362
|
-
|
363
|
-
|
364
|
-
def _fetch_last_audit_via_s3(instance_id: str) -> Optional[Dict]:
|
365
|
-
"""Fetch the newest audit object from S3 if available."""
|
366
|
-
bucket = _get_engine_audit_bucket()
|
367
|
-
if not bucket:
|
368
|
-
return None
|
369
|
-
try:
|
370
|
-
s3 = boto3.client("s3", region_name="us-east-1")
|
371
|
-
paginator = s3.get_paginator("list_objects_v2")
|
372
|
-
newest = None
|
373
|
-
for page in paginator.paginate(
|
374
|
-
Bucket=bucket, Prefix=f"{instance_id}/", MaxKeys=1000
|
375
|
-
):
|
376
|
-
for obj in page.get("Contents", []):
|
377
|
-
lm = obj.get("LastModified")
|
378
|
-
if newest is None or (lm and lm > newest["LastModified"]):
|
379
|
-
newest = obj
|
380
|
-
if not newest:
|
381
|
-
return None
|
382
|
-
obj = s3.get_object(Bucket=bucket, Key=newest["Key"])
|
383
|
-
data = obj["Body"].read().decode("utf-8")
|
384
|
-
return json.loads(data)
|
385
|
-
except Exception:
|
386
|
-
return None
|
387
|
-
|
388
|
-
|
389
|
-
def _summarize_audit(audit: Dict) -> str:
|
390
|
-
"""Return a compact one-line summary for status output."""
|
391
|
-
ts = audit.get("ts", "?")
|
392
|
-
shutdown = audit.get("shutdown", {})
|
393
|
-
result = shutdown.get("result", "?")
|
394
|
-
detach = audit.get("detach", {})
|
395
|
-
num_detached = sum(
|
396
|
-
1 for r in detach.get("results", []) if r.get("status") == "success"
|
397
|
-
)
|
398
|
-
idle = audit.get("idle", {})
|
399
|
-
elapsed = int(idle.get("elapsed_sec", 0))
|
400
|
-
threshold = int(idle.get("threshold_sec", 0))
|
401
|
-
return f"Last shutdown attempt: {ts} result={result} detach={num_detached} idle={elapsed//60}/{threshold//60}m"
|
402
|
-
|
403
|
-
|
404
322
|
def resolve_engine(name_or_id: str, engines: List[Dict]) -> Dict:
|
405
323
|
"""Resolve engine by name or ID with interactive selection."""
|
406
324
|
# Exact ID match
|
@@ -1006,23 +924,55 @@ def engine_status(
|
|
1006
924
|
if not content:
|
1007
925
|
return None
|
1008
926
|
data = json.loads(content)
|
1009
|
-
# Convert last_state schema to idle_detector schema used by CLI output
|
927
|
+
# Convert last_state schema (new or old) to idle_detector schema used by CLI output
|
1010
928
|
idle_info: Dict[str, Any] = {"available": True}
|
1011
|
-
|
1012
|
-
#
|
929
|
+
|
930
|
+
# Active/idle
|
931
|
+
idle_flag = bool(data.get("idle", False))
|
932
|
+
idle_info["status"] = "idle" if idle_flag else "active"
|
933
|
+
|
934
|
+
# Threshold and elapsed
|
1013
935
|
if isinstance(data.get("timeout_sec"), (int, float)):
|
1014
936
|
idle_info["idle_threshold"] = int(data["timeout_sec"]) # seconds
|
1015
|
-
|
937
|
+
if isinstance(data.get("idle_seconds"), (int, float)):
|
938
|
+
idle_info["idle_seconds"] = int(data["idle_seconds"])
|
939
|
+
|
940
|
+
# Keep raw reasons for sensor display when available (new schema)
|
1016
941
|
if isinstance(data.get("reasons"), list):
|
1017
942
|
idle_info["_reasons_raw"] = data["reasons"]
|
1018
|
-
|
1019
|
-
|
943
|
+
else:
|
944
|
+
# Fallback: synthesize reasons from the old forensics layout
|
945
|
+
f_all = data.get("forensics", {}) or {}
|
946
|
+
synthesized = []
|
947
|
+
|
948
|
+
def _mk(sensor_name: str, key: str):
|
949
|
+
entry = f_all.get(key, {}) or {}
|
950
|
+
synthesized.append(
|
951
|
+
{
|
952
|
+
"sensor": sensor_name,
|
953
|
+
"active": bool(entry.get("active", False)),
|
954
|
+
"reason": entry.get("reason", ""),
|
955
|
+
"forensic": entry.get("forensic", {}),
|
956
|
+
}
|
957
|
+
)
|
958
|
+
|
959
|
+
_mk("CoffeeLockSensor", "coffee")
|
960
|
+
_mk("ActiveLoginSensor", "ssh")
|
961
|
+
_mk("IDEConnectionSensor", "ide")
|
962
|
+
_mk("DockerWorkloadSensor", "docker")
|
963
|
+
idle_info["_reasons_raw"] = synthesized
|
964
|
+
|
965
|
+
# Derive details from sensors
|
966
|
+
for r in idle_info.get("_reasons_raw", []):
|
1020
967
|
if not r.get("active"):
|
1021
968
|
continue
|
1022
969
|
sensor = (r.get("sensor") or "").lower()
|
1023
970
|
forensic = r.get("forensic") or {}
|
1024
971
|
if sensor == "ideconnectionsensor":
|
1025
|
-
|
972
|
+
# Prefer unique_pid_count written by new detector
|
973
|
+
cnt = forensic.get("unique_pid_count")
|
974
|
+
if not isinstance(cnt, int):
|
975
|
+
cnt = forensic.get("matches")
|
1026
976
|
if isinstance(cnt, int):
|
1027
977
|
idle_info["ide_connections"] = {"connection_count": cnt}
|
1028
978
|
else:
|
@@ -1034,12 +984,11 @@ def engine_status(
|
|
1034
984
|
timedelta(seconds=int(rem))
|
1035
985
|
)
|
1036
986
|
elif sensor == "activeloginsensor":
|
1037
|
-
# Provide a single summarized SSH session if available
|
1038
987
|
sess = {
|
1039
|
-
"tty":
|
1040
|
-
"pid":
|
1041
|
-
"idle_time":
|
1042
|
-
"from_ip":
|
988
|
+
"tty": forensic.get("tty", "pts/?"),
|
989
|
+
"pid": forensic.get("pid", "?"),
|
990
|
+
"idle_time": forensic.get("idle_sec", 0),
|
991
|
+
"from_ip": forensic.get("remote_addr", "unknown"),
|
1043
992
|
}
|
1044
993
|
idle_info.setdefault("ssh_sessions", []).append(sess)
|
1045
994
|
return idle_info
|
@@ -1081,15 +1030,23 @@ def engine_status(
|
|
1081
1030
|
status_lines.append(_sensor_line(" IDE ", "IDEConnectionSensor", "🖥"))
|
1082
1031
|
status_lines.append(_sensor_line("Docker", "DockerWorkloadSensor", "🐳"))
|
1083
1032
|
|
1084
|
-
|
1085
|
-
|
1086
|
-
|
1087
|
-
|
1088
|
-
|
1089
|
-
|
1090
|
-
|
1091
|
-
|
1092
|
-
|
1033
|
+
# If we have elapsed idle seconds and threshold, reflect that in the header
|
1034
|
+
try:
|
1035
|
+
if idle_detector.get("status") == "idle":
|
1036
|
+
idle_secs = int(idle_detector.get("idle_seconds", 0))
|
1037
|
+
thresh_secs = int(idle_detector.get("idle_threshold", 0))
|
1038
|
+
if thresh_secs > 0:
|
1039
|
+
active_disp = (
|
1040
|
+
f"[yellow]Idle {idle_secs//60}m/{thresh_secs//60}m[/yellow]"
|
1041
|
+
)
|
1042
|
+
# Rewrite top header line (index 0) to include updated display
|
1043
|
+
all_header = top_lines[0]
|
1044
|
+
# Replace the portion after two spaces (name and running state fixed)
|
1045
|
+
top_lines[0] = (
|
1046
|
+
f"[blue]{engine['name']}[/blue] {run_disp} {active_disp}\n"
|
1047
|
+
)
|
1048
|
+
except Exception:
|
1049
|
+
pass
|
1093
1050
|
|
1094
1051
|
# Combine top summary and details
|
1095
1052
|
all_lines = top_lines + status_lines
|
@@ -1128,67 +1085,6 @@ def engine_status(
|
|
1128
1085
|
console.print(f"[red]❌ Error fetching log: {e}[/red]")
|
1129
1086
|
|
1130
1087
|
|
1131
|
-
@engine_app.command("why")
|
1132
|
-
def engine_why(
|
1133
|
-
name_or_id: str = typer.Argument(help="Engine name or instance ID"),
|
1134
|
-
raw: bool = typer.Option(False, "--raw", help="Print raw audit JSON"),
|
1135
|
-
):
|
1136
|
-
"""Explain the last idle-detector shutdown attempt for an engine.
|
1137
|
-
|
1138
|
-
Tries SSM (on-instance file) first, then falls back to S3 audit bucket.
|
1139
|
-
"""
|
1140
|
-
check_aws_sso()
|
1141
|
-
|
1142
|
-
# Resolve engine
|
1143
|
-
response = make_api_request("GET", "/engines")
|
1144
|
-
if response.status_code != 200:
|
1145
|
-
console.print("[red]❌ Failed to fetch engines[/red]")
|
1146
|
-
raise typer.Exit(1)
|
1147
|
-
|
1148
|
-
engines = response.json().get("engines", [])
|
1149
|
-
engine = resolve_engine(name_or_id, engines)
|
1150
|
-
|
1151
|
-
audit = _fetch_last_audit_via_ssm(
|
1152
|
-
engine["instance_id"]
|
1153
|
-
) or _fetch_last_audit_via_s3(engine["instance_id"])
|
1154
|
-
if not audit:
|
1155
|
-
console.print("No audit found (engine may not have attempted shutdown yet).")
|
1156
|
-
raise typer.Exit(0)
|
1157
|
-
|
1158
|
-
if raw:
|
1159
|
-
console.print_json(data=audit)
|
1160
|
-
return
|
1161
|
-
|
1162
|
-
# Pretty summary
|
1163
|
-
status = _summarize_audit(audit)
|
1164
|
-
panel_lines = [
|
1165
|
-
f"[bold]{status}[/bold]",
|
1166
|
-
"",
|
1167
|
-
"[bold]Sensors:[/bold]",
|
1168
|
-
]
|
1169
|
-
for r in audit.get("idle", {}).get("reason_snapshot", []):
|
1170
|
-
active = "✓" if r.get("active") else "-"
|
1171
|
-
reason = r.get("reason") or ""
|
1172
|
-
sensor = r.get("sensor")
|
1173
|
-
panel_lines.append(f" {active} {sensor}: {reason}")
|
1174
|
-
panel_lines.append("")
|
1175
|
-
panel_lines.append("[bold]Detach results:[/bold]")
|
1176
|
-
for res in audit.get("detach", {}).get("results", []):
|
1177
|
-
panel_lines.append(
|
1178
|
-
f" - {res.get('studio_id')}: {res.get('status')} {res.get('error') or ''}"
|
1179
|
-
)
|
1180
|
-
s3_info = audit.get("s3", {})
|
1181
|
-
if s3_info.get("uploaded"):
|
1182
|
-
panel_lines.append("")
|
1183
|
-
panel_lines.append(
|
1184
|
-
f"[dim]S3: s3://{s3_info.get('bucket')}/{s3_info.get('key')}[/dim]"
|
1185
|
-
)
|
1186
|
-
|
1187
|
-
console.print(
|
1188
|
-
Panel("\n".join(panel_lines), title="Idle Shutdown Audit", border_style="blue")
|
1189
|
-
)
|
1190
|
-
|
1191
|
-
|
1192
1088
|
@engine_app.command("stop")
|
1193
1089
|
def stop_engine(
|
1194
1090
|
name_or_id: str = typer.Argument(help="Engine name or instance ID"),
|
@@ -5,7 +5,7 @@ build-backend = "poetry.core.masonry.api"
|
|
5
5
|
|
6
6
|
[project]
|
7
7
|
name = "dayhoff-tools"
|
8
|
-
version = "1.9.
|
8
|
+
version = "1.9.2"
|
9
9
|
description = "Common tools for all the repos at Dayhoff Labs"
|
10
10
|
authors = [
|
11
11
|
{name = "Daniel Martin-Alarcon", email = "dma@dayhofflabs.com"}
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|