machineconfig 2.2__py3-none-any.whl → 2.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of machineconfig might be problematic. Click here for more details.
- machineconfig/cluster/sessions_managers/enhanced_command_runner.py +0 -2
- machineconfig/cluster/sessions_managers/layout_types.py +29 -0
- machineconfig/cluster/sessions_managers/wt_local.py +68 -62
- machineconfig/cluster/sessions_managers/wt_local_manager.py +51 -22
- machineconfig/cluster/sessions_managers/wt_remote.py +30 -108
- machineconfig/cluster/sessions_managers/wt_remote_manager.py +14 -11
- machineconfig/cluster/sessions_managers/wt_utils/layout_generator.py +33 -37
- machineconfig/cluster/sessions_managers/wt_utils/process_monitor.py +22 -17
- machineconfig/cluster/sessions_managers/wt_utils/session_manager.py +59 -10
- machineconfig/cluster/sessions_managers/wt_utils/status_reporter.py +16 -14
- machineconfig/cluster/sessions_managers/zellij_local.py +75 -57
- machineconfig/cluster/sessions_managers/zellij_local_manager.py +51 -23
- machineconfig/cluster/sessions_managers/zellij_remote.py +47 -27
- machineconfig/cluster/sessions_managers/zellij_remote_manager.py +13 -12
- machineconfig/cluster/sessions_managers/zellij_utils/example_usage.py +14 -10
- machineconfig/cluster/sessions_managers/zellij_utils/layout_generator.py +31 -15
- machineconfig/cluster/sessions_managers/zellij_utils/process_monitor.py +47 -21
- machineconfig/cluster/sessions_managers/zellij_utils/session_manager.py +1 -1
- machineconfig/cluster/sessions_managers/zellij_utils/status_reporter.py +8 -7
- machineconfig/cluster/templates/utils.py +1 -1
- machineconfig/profile/create.py +4 -0
- machineconfig/scripts/python/devops.py +1 -0
- machineconfig/scripts/python/devops_devapps_install.py +1 -0
- machineconfig/scripts/python/fire_agents.py +6 -6
- machineconfig/scripts/python/fire_jobs.py +16 -64
- machineconfig/scripts/python/fire_jobs_args_helper.py +84 -0
- machineconfig/scripts/python/fire_jobs_layout_helper.py +66 -0
- machineconfig/scripts/python/helpers/helpers4.py +0 -1
- machineconfig/scripts/python/wifi_conn.py +0 -1
- machineconfig/utils/code.py +0 -1
- machineconfig/utils/installer_utils/installer_abc.py +0 -1
- machineconfig/utils/options.py +7 -7
- machineconfig/utils/path.py +12 -12
- machineconfig/utils/path_reduced.py +6 -1
- machineconfig/utils/ssh.py +11 -1
- machineconfig/utils/upgrade_packages.py +12 -12
- {machineconfig-2.2.dist-info → machineconfig-2.3.dist-info}/METADATA +1 -1
- {machineconfig-2.2.dist-info → machineconfig-2.3.dist-info}/RECORD +51 -59
- machineconfig-2.3.dist-info/entry_points.txt +2 -0
- machineconfig/cluster/sessions_managers/archive/create_zellij_template.py +0 -60
- machineconfig/cluster/sessions_managers/archive/session_managers.py +0 -183
- machineconfig/cluster/sessions_managers/demo_rich_zellij.py +0 -0
- machineconfig/jobs/__pycache__/__init__.cpython-313.pyc +0 -0
- machineconfig/jobs/python_generic_installers/__pycache__/__init__.cpython-313.pyc +0 -0
- machineconfig/jobs/python_linux_installers/__pycache__/__init__.cpython-313.pyc +0 -0
- machineconfig/scripts/__pycache__/__init__.cpython-313.pyc +0 -0
- machineconfig/scripts/python/__pycache__/__init__.cpython-313.pyc +0 -0
- machineconfig/scripts/python/__pycache__/devops.cpython-313.pyc +0 -0
- machineconfig/scripts/python/__pycache__/devops_devapps_install.cpython-313.pyc +0 -0
- machineconfig/scripts/python/__pycache__/devops_update_repos.cpython-313.pyc +0 -0
- machineconfig/scripts/python/__pycache__/fire_agents.cpython-313.pyc +0 -0
- /machineconfig/cluster/{cloud_manager.py → remote/cloud_manager.py} +0 -0
- /machineconfig/cluster/{data_transfer.py → remote/data_transfer.py} +0 -0
- /machineconfig/cluster/{distribute.py → remote/distribute.py} +0 -0
- /machineconfig/cluster/{file_manager.py → remote/file_manager.py} +0 -0
- /machineconfig/cluster/{job_params.py → remote/job_params.py} +0 -0
- /machineconfig/cluster/{loader_runner.py → remote/loader_runner.py} +0 -0
- /machineconfig/cluster/{remote_machine.py → remote/remote_machine.py} +0 -0
- /machineconfig/cluster/{script_execution.py → remote/script_execution.py} +0 -0
- /machineconfig/cluster/{script_notify_upon_completion.py → remote/script_notify_upon_completion.py} +0 -0
- /machineconfig/{cluster/sessions_managers/archive/__init__.py → scripts/python/fire_jobs_streamlit_helper.py} +0 -0
- {machineconfig-2.2.dist-info → machineconfig-2.3.dist-info}/WHEEL +0 -0
- {machineconfig-2.2.dist-info → machineconfig-2.3.dist-info}/top_level.txt +0 -0
|
@@ -2,10 +2,11 @@ from datetime import datetime
|
|
|
2
2
|
import json
|
|
3
3
|
import uuid
|
|
4
4
|
from pathlib import Path
|
|
5
|
-
from typing import Optional
|
|
5
|
+
from typing import Optional, Dict
|
|
6
6
|
from machineconfig.utils.utils5 import Scheduler
|
|
7
7
|
from machineconfig.cluster.sessions_managers.zellij_local import run_command_in_zellij_tab
|
|
8
8
|
from machineconfig.cluster.sessions_managers.zellij_remote import ZellijRemoteLayoutGenerator
|
|
9
|
+
from machineconfig.cluster.sessions_managers.layout_types import LayoutConfig
|
|
9
10
|
from machineconfig.logger import get_logger
|
|
10
11
|
|
|
11
12
|
|
|
@@ -14,13 +15,13 @@ logger = get_logger("cluster.sessions_managers.zellij_remote_manager")
|
|
|
14
15
|
|
|
15
16
|
|
|
16
17
|
class ZellijSessionManager:
|
|
17
|
-
def __init__(self,
|
|
18
|
+
def __init__(self, machine_layouts: Dict[str, LayoutConfig], session_name_prefix: str = "JobMgr"):
|
|
18
19
|
self.session_name_prefix = session_name_prefix
|
|
19
|
-
self.
|
|
20
|
+
self.machine_layouts = machine_layouts # Store the original config
|
|
20
21
|
self.managers: list[ZellijRemoteLayoutGenerator] = []
|
|
21
|
-
for machine,
|
|
22
|
+
for machine, layout_config in machine_layouts.items():
|
|
22
23
|
an_m = ZellijRemoteLayoutGenerator(remote_name=machine, session_name_prefix=self.session_name_prefix)
|
|
23
|
-
an_m.create_zellij_layout(
|
|
24
|
+
an_m.create_zellij_layout(layout_config=layout_config)
|
|
24
25
|
self.managers.append(an_m)
|
|
25
26
|
|
|
26
27
|
def ssh_to_all_machines(self) -> str:
|
|
@@ -93,13 +94,13 @@ class ZellijSessionManager:
|
|
|
93
94
|
session_dir = TMP_SERIALIAZATION_DIR / session_id
|
|
94
95
|
session_dir.mkdir(parents=True, exist_ok=True)
|
|
95
96
|
|
|
96
|
-
# Save the
|
|
97
|
-
config_file = session_dir / "
|
|
98
|
-
text = json.dumps(self.
|
|
97
|
+
# Save the machine_layouts configuration
|
|
98
|
+
config_file = session_dir / "machine_layouts.json"
|
|
99
|
+
text = json.dumps(self.machine_layouts, indent=2, ensure_ascii=False)
|
|
99
100
|
config_file.write_text(text, encoding="utf-8")
|
|
100
101
|
|
|
101
102
|
# Save session metadata
|
|
102
|
-
metadata = {"session_name_prefix": self.session_name_prefix, "created_at": str(datetime.now()), "num_managers": len(self.managers), "machines": list(self.
|
|
103
|
+
metadata = {"session_name_prefix": self.session_name_prefix, "created_at": str(datetime.now()), "num_managers": len(self.managers), "machines": list(self.machine_layouts.keys())}
|
|
103
104
|
metadata_file = session_dir / "metadata.json"
|
|
104
105
|
text = json.dumps(metadata, indent=2, ensure_ascii=False)
|
|
105
106
|
metadata_file.write_text(text, encoding="utf-8")
|
|
@@ -120,11 +121,11 @@ class ZellijSessionManager:
|
|
|
120
121
|
|
|
121
122
|
if not session_dir.exists():
|
|
122
123
|
raise FileNotFoundError(f"Session directory not found: {session_dir}")
|
|
123
|
-
config_file = session_dir / "
|
|
124
|
+
config_file = session_dir / "machine_layouts.json"
|
|
124
125
|
if not config_file.exists():
|
|
125
126
|
raise FileNotFoundError(f"Configuration file not found: {config_file}")
|
|
126
127
|
with open(config_file, "r", encoding="utf-8") as f:
|
|
127
|
-
|
|
128
|
+
machine_layouts = json.load(f)
|
|
128
129
|
|
|
129
130
|
# Load metadata
|
|
130
131
|
metadata_file = session_dir / "metadata.json"
|
|
@@ -134,7 +135,7 @@ class ZellijSessionManager:
|
|
|
134
135
|
metadata = json.load(f)
|
|
135
136
|
session_name_prefix = metadata.get("session_name_prefix", "JobMgr")
|
|
136
137
|
# Create new instance (this will create new managers)
|
|
137
|
-
instance = cls(
|
|
138
|
+
instance = cls(machine_layouts=machine_layouts, session_name_prefix=session_name_prefix)
|
|
138
139
|
# Load saved managers to restore their states
|
|
139
140
|
managers_dir = session_dir / "managers"
|
|
140
141
|
if managers_dir.exists():
|
|
@@ -4,17 +4,21 @@ Example usage of the modularized Zellij remote layout generator.
|
|
|
4
4
|
"""
|
|
5
5
|
|
|
6
6
|
from machineconfig.cluster.sessions_managers.zellij_remote import ZellijRemoteLayoutGenerator
|
|
7
|
+
from machineconfig.cluster.sessions_managers.layout_types import LayoutConfig
|
|
7
8
|
|
|
8
9
|
|
|
9
10
|
def example_usage():
|
|
10
11
|
"""Demonstrate the refactored modular usage."""
|
|
11
12
|
|
|
12
|
-
# Sample
|
|
13
|
-
|
|
14
|
-
"
|
|
15
|
-
"
|
|
16
|
-
|
|
17
|
-
|
|
13
|
+
# Sample layout configuration using new schema
|
|
14
|
+
sample_layout: LayoutConfig = {
|
|
15
|
+
"layoutName": "ExampleRemoteSession",
|
|
16
|
+
"layoutTabs": [
|
|
17
|
+
{"tabName": "🤖Bot1", "startDir": "~/code/bytesense/bithence", "command": "~/scripts/fire -mO go1.py bot1 --kw create_new_bot True"},
|
|
18
|
+
{"tabName": "🤖Bot2", "startDir": "~/code/bytesense/bithence", "command": "~/scripts/fire -mO go2.py bot2 --kw create_new_bot True"},
|
|
19
|
+
{"tabName": "📊Monitor", "startDir": "~", "command": "htop"},
|
|
20
|
+
{"tabName": "📝Logs", "startDir": "/var/log", "command": "tail -f /var/log/app.log"},
|
|
21
|
+
],
|
|
18
22
|
}
|
|
19
23
|
|
|
20
24
|
# Replace 'myserver' with an actual SSH config alias
|
|
@@ -26,11 +30,11 @@ def example_usage():
|
|
|
26
30
|
generator = ZellijRemoteLayoutGenerator(remote_name=remote_name, session_name_prefix=session_name)
|
|
27
31
|
|
|
28
32
|
# Create layout file
|
|
29
|
-
layout_path = generator.create_zellij_layout(
|
|
33
|
+
layout_path = generator.create_zellij_layout(sample_layout)
|
|
30
34
|
print(f"✅ Remote layout created successfully: {layout_path}")
|
|
31
35
|
|
|
32
36
|
# Preview the layout content
|
|
33
|
-
preview = generator.get_layout_preview(
|
|
37
|
+
preview = generator.get_layout_preview(sample_layout)
|
|
34
38
|
print(f"📄 Layout preview:\n{preview}")
|
|
35
39
|
|
|
36
40
|
# Check status using the modular components
|
|
@@ -44,11 +48,11 @@ def example_usage():
|
|
|
44
48
|
print(f"Remote executor: {generator.remote_executor.remote_name}")
|
|
45
49
|
|
|
46
50
|
# Use layout generator directly
|
|
47
|
-
layout_content = generator.layout_generator.generate_layout_content(
|
|
51
|
+
layout_content = generator.layout_generator.generate_layout_content(sample_layout)
|
|
48
52
|
print(f"Layout content length: {len(layout_content)} characters")
|
|
49
53
|
|
|
50
54
|
# Use process monitor directly
|
|
51
|
-
status = generator.process_monitor.check_all_commands_status(
|
|
55
|
+
status = generator.process_monitor.check_all_commands_status(sample_layout)
|
|
52
56
|
print(f"Command status check completed for {len(status)} commands")
|
|
53
57
|
|
|
54
58
|
print("\n✅ All modular components working correctly!")
|
|
@@ -6,11 +6,12 @@ Zellij layout generation utilities for creating KDL layout files.
|
|
|
6
6
|
import shlex
|
|
7
7
|
import random
|
|
8
8
|
import string
|
|
9
|
-
from typing import
|
|
9
|
+
from typing import List, Tuple
|
|
10
10
|
from pathlib import Path
|
|
11
11
|
import logging
|
|
12
12
|
|
|
13
13
|
from rich.console import Console
|
|
14
|
+
from machineconfig.cluster.sessions_managers.layout_types import LayoutConfig
|
|
14
15
|
|
|
15
16
|
logger = logging.getLogger(__name__)
|
|
16
17
|
console = Console()
|
|
@@ -77,36 +78,51 @@ class LayoutGenerator:
|
|
|
77
78
|
return tab_section
|
|
78
79
|
|
|
79
80
|
@staticmethod
|
|
80
|
-
def validate_tab_config(
|
|
81
|
-
"""Validate
|
|
82
|
-
if not
|
|
83
|
-
raise ValueError("
|
|
84
|
-
|
|
81
|
+
def validate_tab_config(layout_config: LayoutConfig) -> None:
|
|
82
|
+
"""Validate layout configuration format and content."""
|
|
83
|
+
if not layout_config:
|
|
84
|
+
raise ValueError("Layout configuration cannot be empty")
|
|
85
|
+
|
|
86
|
+
if not layout_config.get("layoutName", "").strip():
|
|
87
|
+
raise ValueError("Layout name cannot be empty")
|
|
88
|
+
|
|
89
|
+
layout_tabs = layout_config.get("layoutTabs", [])
|
|
90
|
+
if not layout_tabs:
|
|
91
|
+
raise ValueError("Layout must have at least one tab")
|
|
92
|
+
|
|
93
|
+
for tab in layout_tabs:
|
|
94
|
+
tab_name = tab.get("tabName", "")
|
|
95
|
+
command = tab.get("command", "")
|
|
96
|
+
start_dir = tab.get("startDir", "")
|
|
97
|
+
|
|
85
98
|
if not tab_name.strip():
|
|
86
99
|
raise ValueError(f"Invalid tab name: {tab_name}")
|
|
87
100
|
if not command.strip():
|
|
88
101
|
raise ValueError(f"Invalid command for tab '{tab_name}': {command}")
|
|
89
|
-
if not
|
|
90
|
-
raise ValueError(f"Invalid
|
|
102
|
+
if not start_dir.strip():
|
|
103
|
+
raise ValueError(f"Invalid startDir for tab '{tab_name}': {start_dir}")
|
|
91
104
|
|
|
92
|
-
def generate_layout_content(self,
|
|
105
|
+
def generate_layout_content(self, layout_config: LayoutConfig) -> str:
|
|
93
106
|
"""Generate complete KDL layout content."""
|
|
94
|
-
self.validate_tab_config(
|
|
107
|
+
self.validate_tab_config(layout_config)
|
|
95
108
|
|
|
96
109
|
layout_content = self.LAYOUT_TEMPLATE
|
|
97
|
-
for
|
|
98
|
-
|
|
110
|
+
for tab in layout_config["layoutTabs"]:
|
|
111
|
+
tab_name = tab["tabName"]
|
|
112
|
+
start_dir = tab["startDir"]
|
|
113
|
+
command = tab["command"]
|
|
114
|
+
layout_content += "\n" + self.create_tab_section(tab_name, start_dir, command)
|
|
99
115
|
layout_content += "\n}\n"
|
|
100
116
|
|
|
101
117
|
return layout_content
|
|
102
118
|
|
|
103
|
-
def create_layout_file(self,
|
|
119
|
+
def create_layout_file(self, layout_config: LayoutConfig, output_dir: Path, session_name: str) -> str:
|
|
104
120
|
"""Create a layout file and return its absolute path."""
|
|
105
|
-
self.validate_tab_config(
|
|
121
|
+
self.validate_tab_config(layout_config)
|
|
106
122
|
|
|
107
123
|
# Generate unique suffix for this layout
|
|
108
124
|
random_suffix = self.generate_random_suffix()
|
|
109
|
-
layout_content = self.generate_layout_content(
|
|
125
|
+
layout_content = self.generate_layout_content(layout_config)
|
|
110
126
|
|
|
111
127
|
try:
|
|
112
128
|
# Create output directory if it doesn't exist
|
|
@@ -6,8 +6,9 @@ Process monitoring and status checking utilities for remote commands.
|
|
|
6
6
|
import json
|
|
7
7
|
import shlex
|
|
8
8
|
import logging
|
|
9
|
-
from typing import Dict,
|
|
10
|
-
from .remote_executor import RemoteExecutor
|
|
9
|
+
from typing import Dict, Any
|
|
10
|
+
from machineconfig.cluster.sessions_managers.zellij_utils.remote_executor import RemoteExecutor
|
|
11
|
+
from machineconfig.cluster.sessions_managers.layout_types import LayoutConfig
|
|
11
12
|
|
|
12
13
|
logger = logging.getLogger(__name__)
|
|
13
14
|
|
|
@@ -18,20 +19,37 @@ class ProcessMonitor:
|
|
|
18
19
|
def __init__(self, remote_executor: RemoteExecutor):
|
|
19
20
|
self.remote_executor = remote_executor
|
|
20
21
|
|
|
21
|
-
def check_command_status(self, tab_name: str,
|
|
22
|
+
def check_command_status(self, tab_name: str, layout_config: LayoutConfig, use_verification: bool = True) -> Dict[str, Any]:
|
|
22
23
|
"""Check command status with optional process verification."""
|
|
23
|
-
|
|
24
|
-
|
|
24
|
+
# Find the tab with the given name
|
|
25
|
+
tab_config = None
|
|
26
|
+
for tab in layout_config["layoutTabs"]:
|
|
27
|
+
if tab["tabName"] == tab_name:
|
|
28
|
+
tab_config = tab
|
|
29
|
+
break
|
|
30
|
+
|
|
31
|
+
if tab_config is None:
|
|
32
|
+
return {"status": "unknown", "error": f"Tab '{tab_name}' not found in layout config", "running": False, "pid": None, "command": None, "remote": self.remote_executor.remote_name}
|
|
25
33
|
|
|
26
34
|
# Use the verified method by default for more accurate results
|
|
27
35
|
if use_verification:
|
|
28
|
-
return self.get_verified_process_status(tab_name,
|
|
36
|
+
return self.get_verified_process_status(tab_name, layout_config)
|
|
29
37
|
|
|
30
|
-
return self._basic_process_check(tab_name,
|
|
38
|
+
return self._basic_process_check(tab_name, layout_config)
|
|
31
39
|
|
|
32
|
-
def _basic_process_check(self, tab_name: str,
|
|
40
|
+
def _basic_process_check(self, tab_name: str, layout_config: LayoutConfig) -> Dict[str, Any]:
|
|
33
41
|
"""Basic process checking without verification."""
|
|
34
|
-
|
|
42
|
+
# Find the tab with the given name
|
|
43
|
+
tab_config = None
|
|
44
|
+
for tab in layout_config["layoutTabs"]:
|
|
45
|
+
if tab["tabName"] == tab_name:
|
|
46
|
+
tab_config = tab
|
|
47
|
+
break
|
|
48
|
+
|
|
49
|
+
if tab_config is None:
|
|
50
|
+
return {"status": "unknown", "error": f"Tab '{tab_name}' not found in layout config", "running": False, "command": None, "remote": self.remote_executor.remote_name}
|
|
51
|
+
|
|
52
|
+
command = tab_config["command"]
|
|
35
53
|
|
|
36
54
|
try:
|
|
37
55
|
check_script = self._create_process_check_script(command)
|
|
@@ -106,12 +124,19 @@ if __name__ == "__main__":
|
|
|
106
124
|
print(json.dumps(processes))
|
|
107
125
|
"""
|
|
108
126
|
|
|
109
|
-
def force_fresh_process_check(self, tab_name: str,
|
|
127
|
+
def force_fresh_process_check(self, tab_name: str, layout_config: LayoutConfig) -> Dict[str, Any]:
|
|
110
128
|
"""Force a fresh process check with additional validation."""
|
|
111
|
-
|
|
112
|
-
|
|
129
|
+
# Find the tab with the given name
|
|
130
|
+
tab_config = None
|
|
131
|
+
for tab in layout_config["layoutTabs"]:
|
|
132
|
+
if tab["tabName"] == tab_name:
|
|
133
|
+
tab_config = tab
|
|
134
|
+
break
|
|
135
|
+
|
|
136
|
+
if tab_config is None:
|
|
137
|
+
return {"status": "unknown", "error": f"Tab '{tab_name}' not found in layout config", "running": False, "command": None, "remote": self.remote_executor.remote_name}
|
|
113
138
|
|
|
114
|
-
|
|
139
|
+
command = tab_config["command"]
|
|
115
140
|
|
|
116
141
|
try:
|
|
117
142
|
# Get timestamp for freshness validation
|
|
@@ -230,9 +255,9 @@ if __name__ == "__main__":
|
|
|
230
255
|
except Exception:
|
|
231
256
|
return False
|
|
232
257
|
|
|
233
|
-
def get_verified_process_status(self, tab_name: str,
|
|
258
|
+
def get_verified_process_status(self, tab_name: str, layout_config: LayoutConfig) -> Dict[str, Any]:
|
|
234
259
|
"""Get process status with additional verification that processes are actually alive."""
|
|
235
|
-
status = self.force_fresh_process_check(tab_name,
|
|
260
|
+
status = self.force_fresh_process_check(tab_name, layout_config)
|
|
236
261
|
|
|
237
262
|
if status.get("running") and status.get("processes"):
|
|
238
263
|
verified_processes = []
|
|
@@ -252,13 +277,14 @@ if __name__ == "__main__":
|
|
|
252
277
|
|
|
253
278
|
return status
|
|
254
279
|
|
|
255
|
-
def check_all_commands_status(self,
|
|
256
|
-
"""Check status of all commands in the
|
|
257
|
-
if not
|
|
258
|
-
logger.warning("No
|
|
280
|
+
def check_all_commands_status(self, layout_config: LayoutConfig) -> Dict[str, Dict[str, Any]]:
|
|
281
|
+
"""Check status of all commands in the layout configuration."""
|
|
282
|
+
if not layout_config or not layout_config.get("layoutTabs"):
|
|
283
|
+
logger.warning("No layout configuration provided.")
|
|
259
284
|
return {}
|
|
260
285
|
|
|
261
286
|
status_report = {}
|
|
262
|
-
for
|
|
263
|
-
|
|
287
|
+
for tab in layout_config["layoutTabs"]:
|
|
288
|
+
tab_name = tab["tabName"]
|
|
289
|
+
status_report[tab_name] = self.check_command_status(tab_name, layout_config)
|
|
264
290
|
return status_report
|
|
@@ -9,7 +9,7 @@ from pathlib import Path
|
|
|
9
9
|
|
|
10
10
|
from rich.console import Console
|
|
11
11
|
|
|
12
|
-
from .remote_executor import RemoteExecutor
|
|
12
|
+
from machineconfig.cluster.sessions_managers.zellij_utils.remote_executor import RemoteExecutor
|
|
13
13
|
|
|
14
14
|
logger = logging.getLogger(__name__)
|
|
15
15
|
console = Console()
|
|
@@ -4,9 +4,10 @@ Status reporting utilities for Zellij remote layouts.
|
|
|
4
4
|
"""
|
|
5
5
|
|
|
6
6
|
import logging
|
|
7
|
-
from typing import Dict, Any
|
|
8
|
-
from .process_monitor import ProcessMonitor
|
|
9
|
-
from .session_manager import SessionManager
|
|
7
|
+
from typing import Dict, Any
|
|
8
|
+
from machineconfig.cluster.sessions_managers.zellij_utils.process_monitor import ProcessMonitor
|
|
9
|
+
from machineconfig.cluster.sessions_managers.zellij_utils.session_manager import SessionManager
|
|
10
|
+
from machineconfig.cluster.sessions_managers.layout_types import LayoutConfig
|
|
10
11
|
|
|
11
12
|
logger = logging.getLogger(__name__)
|
|
12
13
|
|
|
@@ -18,10 +19,10 @@ class StatusReporter:
|
|
|
18
19
|
self.process_monitor = process_monitor
|
|
19
20
|
self.session_manager = session_manager
|
|
20
21
|
|
|
21
|
-
def get_comprehensive_status(self,
|
|
22
|
+
def get_comprehensive_status(self, layout_config: LayoutConfig) -> Dict[str, Any]:
|
|
22
23
|
"""Get comprehensive status including Zellij session and all commands."""
|
|
23
24
|
zellij_status = self.session_manager.check_zellij_session_status()
|
|
24
|
-
commands_status = self.process_monitor.check_all_commands_status(
|
|
25
|
+
commands_status = self.process_monitor.check_all_commands_status(layout_config)
|
|
25
26
|
|
|
26
27
|
running_count = sum(1 for status in commands_status.values() if status.get("running", False))
|
|
27
28
|
total_count = len(commands_status)
|
|
@@ -38,9 +39,9 @@ class StatusReporter:
|
|
|
38
39
|
},
|
|
39
40
|
}
|
|
40
41
|
|
|
41
|
-
def print_status_report(self,
|
|
42
|
+
def print_status_report(self, layout_config: LayoutConfig) -> None:
|
|
42
43
|
"""Print a formatted status report to console."""
|
|
43
|
-
status = self.get_comprehensive_status(
|
|
44
|
+
status = self.get_comprehensive_status(layout_config)
|
|
44
45
|
remote_name = self.session_manager.remote_executor.remote_name
|
|
45
46
|
session_name = self.session_manager.session_name
|
|
46
47
|
|
machineconfig/profile/create.py
CHANGED
|
@@ -24,9 +24,12 @@ ERROR_LIST: list[Any] = [] # append to this after every exception captured.
|
|
|
24
24
|
|
|
25
25
|
SYSTEM = system.lower()
|
|
26
26
|
|
|
27
|
+
|
|
27
28
|
def get_other_systems(current_system: str) -> list[str]:
|
|
28
29
|
all_systems = ["linux", "windows", "darwin"]
|
|
29
30
|
return [s for s in all_systems if s != current_system.lower()]
|
|
31
|
+
|
|
32
|
+
|
|
30
33
|
OTHER_SYSTEMS = get_other_systems(SYSTEM)
|
|
31
34
|
|
|
32
35
|
|
|
@@ -59,6 +62,7 @@ def main_symlinks(choice: Optional[str] = None):
|
|
|
59
62
|
choice_selected = "all" # i.e. program_keys = program_keys
|
|
60
63
|
# overwrite = display_options(msg="Overwrite existing source file?", options=["yes", "no"], default="yes") == "yes"
|
|
61
64
|
from rich.prompt import Confirm
|
|
65
|
+
|
|
62
66
|
overwrite = Confirm.ask("Overwrite existing source file?", default=True)
|
|
63
67
|
else:
|
|
64
68
|
choice_selected = choice
|
|
@@ -74,6 +74,7 @@ def display_task_success(success: str) -> None:
|
|
|
74
74
|
|
|
75
75
|
def main(which: Optional[str] = None):
|
|
76
76
|
from machineconfig.utils.path_reduced import PathExtended
|
|
77
|
+
|
|
77
78
|
PathExtended(PROGRAM_PATH).delete(sure=True, verbose=False)
|
|
78
79
|
console.print(Panel("🚀 Initializing DevOps operation...", width=BOX_WIDTH, border_style="blue"))
|
|
79
80
|
|
|
@@ -39,6 +39,7 @@ def main(which: Optional[WHICH_CAT | str] = None):
|
|
|
39
39
|
# interactive installation
|
|
40
40
|
installers = [Installer.from_dict(d=vd, name=name) for __kat, vds in get_all_dicts(system=system()).items() for name, vd in vds.items()]
|
|
41
41
|
options = [x.get_description() for x in tqdm(installers, desc="✅ Checking installed programs")] + list(get_args(WHICH_CAT))
|
|
42
|
+
# print("s"*1000)
|
|
42
43
|
program_names = choose_multiple_options(msg="", options=options, header="🚀 CHOOSE DEV APP", default="AllEssentials")
|
|
43
44
|
|
|
44
45
|
total_program = ""
|
|
@@ -15,6 +15,7 @@ from math import ceil
|
|
|
15
15
|
from typing import Literal, TypeAlias, get_args, Iterable
|
|
16
16
|
|
|
17
17
|
from machineconfig.cluster.sessions_managers.zellij_local_manager import ZellijLocalManager
|
|
18
|
+
from machineconfig.cluster.sessions_managers.layout_types import TabConfig, LayoutConfig
|
|
18
19
|
from machineconfig.utils.utils2 import randstr
|
|
19
20
|
import random
|
|
20
21
|
# import time
|
|
@@ -23,7 +24,6 @@ AGENTS: TypeAlias = Literal[
|
|
|
23
24
|
"cursor-agent", "gemini", "crush", "q", "onlyPrepPromptFiles"
|
|
24
25
|
# warp terminal
|
|
25
26
|
]
|
|
26
|
-
TabConfig = dict[str, tuple[str, str]] # tab name -> (cwd, command)
|
|
27
27
|
DEFAULT_AGENT_CAP = 6
|
|
28
28
|
|
|
29
29
|
|
|
@@ -93,7 +93,7 @@ def _confirm(message: str, default_no: bool = True) -> bool:
|
|
|
93
93
|
return False
|
|
94
94
|
|
|
95
95
|
|
|
96
|
-
def launch_agents(repo_root: Path, prompts: list[str], agent: AGENTS, *, max_agents: int = DEFAULT_AGENT_CAP) -> TabConfig:
|
|
96
|
+
def launch_agents(repo_root: Path, prompts: list[str], agent: AGENTS, *, max_agents: int = DEFAULT_AGENT_CAP) -> list[TabConfig]:
|
|
97
97
|
"""Create tab configuration for a set of agent prompts.
|
|
98
98
|
|
|
99
99
|
If number of prompts exceeds max_agents, ask user for confirmation.
|
|
@@ -106,9 +106,9 @@ def launch_agents(repo_root: Path, prompts: list[str], agent: AGENTS, *, max_age
|
|
|
106
106
|
proceed = _confirm(message=(f"You are about to launch {len(prompts)} agents which exceeds the cap ({max_agents}). Proceed?"))
|
|
107
107
|
if not proceed:
|
|
108
108
|
print("Aborting per user choice.")
|
|
109
|
-
return
|
|
109
|
+
return []
|
|
110
110
|
|
|
111
|
-
tab_config: TabConfig =
|
|
111
|
+
tab_config: list[TabConfig] = []
|
|
112
112
|
tmp_dir = repo_root / ".ai" / f"tmp_prompts/{randstr()}"
|
|
113
113
|
tmp_dir.mkdir(parents=True, exist_ok=True)
|
|
114
114
|
|
|
@@ -173,7 +173,7 @@ echo "---------END OF AGENT OUTPUT---------"
|
|
|
173
173
|
"""
|
|
174
174
|
cmd_path.write_text(cmd_prefix + cmd + cmd_postfix, encoding="utf-8")
|
|
175
175
|
fire_cmd = f"bash {shlex.quote(str(cmd_path))}"
|
|
176
|
-
tab_config
|
|
176
|
+
tab_config.append(TabConfig(tabName=f"Agent{idx}", startDir=str(repo_root), command=fire_cmd))
|
|
177
177
|
|
|
178
178
|
print(f"Launching a template with #{len(tab_config)} agents")
|
|
179
179
|
return tab_config
|
|
@@ -228,7 +228,7 @@ def main(): # noqa: C901 - (complexity acceptable for CLI glue)
|
|
|
228
228
|
from machineconfig.utils.utils2 import randstr
|
|
229
229
|
|
|
230
230
|
random_name = randstr(length=3)
|
|
231
|
-
manager = ZellijLocalManager(
|
|
231
|
+
manager = ZellijLocalManager(session_layouts=[LayoutConfig(layoutName="Agents", layoutTabs=tab_config)], session_name_prefix=random_name)
|
|
232
232
|
manager.start_all_sessions()
|
|
233
233
|
manager.run_monitoring_routine()
|
|
234
234
|
|
|
@@ -18,52 +18,23 @@ from machineconfig.utils.source_of_truth import PROGRAM_PATH
|
|
|
18
18
|
from machineconfig.utils.path_reduced import PathExtended as PathExtended
|
|
19
19
|
from machineconfig.utils.io_save import save_toml
|
|
20
20
|
from machineconfig.utils.utils2 import randstr, read_toml
|
|
21
|
+
from machineconfig.scripts.python.fire_jobs_args_helper import get_args, FireJobArgs, extract_kwargs
|
|
21
22
|
import platform
|
|
22
23
|
from typing import Optional
|
|
23
|
-
import argparse
|
|
24
24
|
import os
|
|
25
25
|
|
|
26
26
|
|
|
27
|
-
|
|
27
|
+
def main(args: FireJobArgs) -> None:
|
|
28
|
+
if args.layout:
|
|
29
|
+
from machineconfig.scripts.python.fire_jobs_layout_helper import handle_layout_args
|
|
28
30
|
|
|
31
|
+
return handle_layout_args(args)
|
|
29
32
|
|
|
30
|
-
def main() -> None:
|
|
31
|
-
parser = argparse.ArgumentParser()
|
|
32
|
-
parser.add_argument("path", nargs="?", type=str, help="The directory containing the jobs", default=".")
|
|
33
|
-
parser.add_argument("function", nargs="?", type=str, help="Fuction to run", default=None)
|
|
34
|
-
parser.add_argument("--ve", "-v", type=str, help="virtual enviroment name", default="")
|
|
35
|
-
parser.add_argument("--cmd", "-B", action="store_true", help="Create a cmd fire command to launch the the job asynchronously.")
|
|
36
|
-
parser.add_argument("--interactive", "-i", action="store_true", help="Whether to run the job interactively using IPython")
|
|
37
|
-
parser.add_argument("--debug", "-d", action="store_true", help="debug")
|
|
38
|
-
parser.add_argument("--choose_function", "-c", action="store_true", help="debug")
|
|
39
|
-
parser.add_argument("--loop", "-l", action="store_true", help="infinite recusion (runs again after completion/interruption)")
|
|
40
|
-
parser.add_argument("--jupyter", "-j", action="store_true", help="open in a jupyter notebook")
|
|
41
|
-
parser.add_argument("--submit_to_cloud", "-C", action="store_true", help="submit to cloud compute")
|
|
42
|
-
parser.add_argument("--remote", "-r", action="store_true", help="launch on a remote machine")
|
|
43
|
-
parser.add_argument("--module", "-m", action="store_true", help="launch the main file")
|
|
44
|
-
parser.add_argument("--streamlit", "-S", action="store_true", help="run as streamlit app")
|
|
45
|
-
parser.add_argument("--environment", "-E", type=str, help="Choose ip, localhost, hostname or arbitrary url", default="")
|
|
46
|
-
parser.add_argument("--holdDirectory", "-D", action="store_true", help="hold current directory and avoid cd'ing to the script directory")
|
|
47
|
-
parser.add_argument("--PathExport", "-P", action="store_true", help="augment the PYTHONPATH with repo root.")
|
|
48
|
-
parser.add_argument("--git_pull", "-g", action="store_true", help="Start by pulling the git repo")
|
|
49
|
-
parser.add_argument("--optimized", "-O", action="store_true", help="Run the optimized version of the function")
|
|
50
|
-
parser.add_argument("--Nprocess", "-p", type=int, help="Number of processes to use", default=1)
|
|
51
|
-
parser.add_argument("--zellij_tab", "-z", type=str, dest="zellij_tab", help="open in a new zellij tab")
|
|
52
|
-
parser.add_argument("--watch", "-w", action="store_true", help="watch the file for changes")
|
|
53
|
-
parser.add_argument("--kw", nargs="*", default=None, help="keyword arguments to pass to the function in the form of k1 v1 k2 v2 ... (meaning k1=v1, k2=v2, etc)")
|
|
54
|
-
try:
|
|
55
|
-
args = parser.parse_args()
|
|
56
|
-
except Exception as ex:
|
|
57
|
-
print(f"❌ Failed to parse arguments: {ex}")
|
|
58
|
-
parser.print_help()
|
|
59
|
-
raise ex
|
|
60
33
|
path_obj = sanitize_path(args.path)
|
|
61
|
-
# print(f"Passed path sanitied to {path_obj}")
|
|
62
34
|
if not path_obj.exists():
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
if path_obj.is_dir():
|
|
35
|
+
suffixes = {".py", ".sh", ".ps1"}
|
|
36
|
+
choice_file = match_file_name(sub_string=args.path, search_root=PathExtended.cwd(), suffixes=suffixes)
|
|
37
|
+
elif path_obj.is_dir():
|
|
67
38
|
print(f"🔍 Searching recursively for Python, PowerShell and Shell scripts in directory `{path_obj}`")
|
|
68
39
|
files = search_for_files_of_interest(path_obj)
|
|
69
40
|
print(f"🔍 Got #{len(files)} results.")
|
|
@@ -73,24 +44,13 @@ def main() -> None:
|
|
|
73
44
|
choice_file = path_obj
|
|
74
45
|
repo_root = get_repo_root(str(choice_file))
|
|
75
46
|
print(f"💾 Selected file: {choice_file}.\nRepo root: {repo_root}")
|
|
76
|
-
|
|
77
47
|
ve_root_from_file, ipy_profile = get_ve_path_and_ipython_profile(choice_file)
|
|
78
48
|
if ipy_profile is None:
|
|
79
49
|
ipy_profile = "default"
|
|
80
50
|
activate_ve_line = get_ve_activate_line(ve_root=args.ve or ve_root_from_file or "$HOME/code/machineconfig/.venv")
|
|
81
51
|
|
|
82
|
-
# Convert args.kw to dictionary
|
|
83
52
|
if choice_file.suffix == ".py":
|
|
84
|
-
|
|
85
|
-
assert len(args.kw) % 2 == 0, f"args.kw must be a list of even length. Got {len(args.kw)}"
|
|
86
|
-
kwargs = dict(zip(args.kw[::2], args.kw[1::2]))
|
|
87
|
-
for key, value in kwargs.items():
|
|
88
|
-
if value in str2obj:
|
|
89
|
-
kwargs[key] = str2obj[value]
|
|
90
|
-
if args.function is None: # if user passed arguments and forgot to pass function, then assume they want to run the main function.
|
|
91
|
-
args.choose_function = True
|
|
92
|
-
else:
|
|
93
|
-
kwargs = {}
|
|
53
|
+
kwargs = extract_kwargs(args)
|
|
94
54
|
else:
|
|
95
55
|
kwargs = {}
|
|
96
56
|
|
|
@@ -296,22 +256,13 @@ python -m machineconfig.cluster.templates.cli_click --file {choice_file} """
|
|
|
296
256
|
command += f"--function {choice_function} "
|
|
297
257
|
|
|
298
258
|
if args.Nprocess > 1:
|
|
299
|
-
# lines = [f""" zellij action new-tab --name nProcess{randstr(2)}"""]
|
|
300
|
-
# command = command.replace(". activate_ve", ". $HOME/scripts/activate_ve")
|
|
301
|
-
# for an_arg in range(args.Nprocess):
|
|
302
|
-
# sub_command = f"{command} --idx={an_arg} --idx_max={args.Nprocess}"
|
|
303
|
-
# if args.optimized:
|
|
304
|
-
# sub_command = sub_command.replace("python ", "python -OO ")
|
|
305
|
-
# sub_command_path = PathExtended.tmpfile(suffix=".sh").write_text(sub_command, encoding="utf-8")
|
|
306
|
-
# lines.append(f"""zellij action new-pane -- bash {sub_command_path} """)
|
|
307
|
-
# lines.append("sleep 5") # python tends to freeze if you launch instances within 1 microsecond of each other
|
|
308
|
-
# command = "\n".join(lines)
|
|
309
|
-
tab_config = {}
|
|
310
|
-
for an_arg in range(args.Nprocess):
|
|
311
|
-
tab_config[f"tab{an_arg}"] = (str(PathExtended.cwd()), f"uv run -m fire {choice_file} {choice_function} --idx={an_arg} --idx_max={args.Nprocess}")
|
|
312
259
|
from machineconfig.cluster.sessions_managers.zellij_local import run_zellij_layout
|
|
260
|
+
from machineconfig.cluster.sessions_managers.layout_types import LayoutConfig
|
|
313
261
|
|
|
314
|
-
|
|
262
|
+
layout: LayoutConfig = {"layoutName": "fireNprocess", "layoutTabs": []}
|
|
263
|
+
for an_arg in range(args.Nprocess):
|
|
264
|
+
layout["layoutTabs"].append({"tabName": f"tab{an_arg}", "startDir": str(PathExtended.cwd()), "command": f"uv run -m fire {choice_file} {choice_function} --idx={an_arg} --idx_max={args.Nprocess}"})
|
|
265
|
+
run_zellij_layout(layout_config=layout)
|
|
315
266
|
return None
|
|
316
267
|
|
|
317
268
|
if args.optimized:
|
|
@@ -375,4 +326,5 @@ python -m machineconfig.cluster.templates.cli_click --file {choice_file} """
|
|
|
375
326
|
|
|
376
327
|
if __name__ == "__main__":
|
|
377
328
|
# options, func_args = parse_pyfile(file_path="C:/Users/aalsaf01/code/machineconfig/myresources/crocodile/core.py")
|
|
378
|
-
|
|
329
|
+
args = get_args()
|
|
330
|
+
main(args)
|