paramtui 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- manager/__init__.py +10 -0
- manager/commands/__init__.py +12 -0
- manager/commands/conda.py +61 -0
- manager/commands/files.py +186 -0
- manager/commands/help.py +165 -0
- manager/commands/job_templates.py +132 -0
- manager/commands/logs.py +42 -0
- manager/commands/modules.py +81 -0
- manager/commands/resources.py +65 -0
- manager/commands/settings.py +50 -0
- manager/commands/slurm.py +62 -0
- manager/commands/tunnel.py +23 -0
- manager/connection.py +136 -0
- manager/system.py +82 -0
- manager/templates.py +113 -0
- manager/ui/__init__.py +16 -0
- manager/ui/conda_manager.py +58 -0
- manager/ui/file_manager.py +112 -0
- manager/ui/help_menu.py +36 -0
- manager/ui/interactive.py +52 -0
- manager/ui/job_dashboard.py +48 -0
- manager/ui/job_templates.py +119 -0
- manager/ui/logs_menu.py +37 -0
- manager/ui/menus.py +128 -0
- manager/ui/modules_menu.py +52 -0
- manager/ui/quota.py +33 -0
- manager/ui/resources.py +39 -0
- manager/ui/settings_menu.py +37 -0
- manager/ui/styles.py +32 -0
- manager/ui/tunnel_menu.py +32 -0
- manager/ui.py +40 -0
- paramtui-0.1.0.dist-info/METADATA +45 -0
- paramtui-0.1.0.dist-info/RECORD +36 -0
- paramtui-0.1.0.dist-info/WHEEL +5 -0
- paramtui-0.1.0.dist-info/entry_points.txt +2 -0
- paramtui-0.1.0.dist-info/top_level.txt +1 -0
manager/__init__.py
ADDED
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
"""ParamTUI - SSH Manager & HPC Console.
|
|
2
|
+
|
|
3
|
+
A modular terminal user interface for managing HPC clusters.
|
|
4
|
+
"""
|
|
5
|
+
|
|
6
|
+
from manager.connection import SSHConnection
|
|
7
|
+
from manager.ui.menus import main_menu, connection_menu
|
|
8
|
+
|
|
9
|
+
__all__ = ['SSHConnection', 'main_menu', 'connection_menu']
|
|
10
|
+
__version__ = '1.0.0'
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
"""Commands subpackage - Backend operations for SSH management."""
|
|
2
|
+
|
|
3
|
+
from manager.commands.slurm import *
|
|
4
|
+
from manager.commands.conda import *
|
|
5
|
+
from manager.commands.files import *
|
|
6
|
+
from manager.commands.modules import *
|
|
7
|
+
from manager.commands.resources import *
|
|
8
|
+
from manager.commands.job_templates import *
|
|
9
|
+
from manager.commands.logs import *
|
|
10
|
+
from manager.commands.settings import *
|
|
11
|
+
from manager.commands.help import *
|
|
12
|
+
from manager.commands.tunnel import *
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
"""Conda environment management commands."""
|
|
2
|
+
|
|
3
|
+
from rich.console import Console
|
|
4
|
+
|
|
5
|
+
console = Console()
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
def conda_list_envs(ssh_conn):
|
|
9
|
+
"""List all conda environments."""
|
|
10
|
+
output = ssh_conn.execute_command(" conda env list")
|
|
11
|
+
if output:
|
|
12
|
+
console.print("[bold green]Conda Environments:[/bold green]")
|
|
13
|
+
console.print(output)
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def conda_activate_env(ssh_conn, env_name):
|
|
17
|
+
"""Activate a conda environment."""
|
|
18
|
+
console.print(f"[bold yellow]Activating environment '{env_name}'...[/bold yellow]")
|
|
19
|
+
output = ssh_conn.execute_command(f"source conda activate {env_name}")
|
|
20
|
+
if output and ("error" in output.lower() or "not found" in output.lower()):
|
|
21
|
+
console.print(f"[bold red]Failed to activate environment:[/bold red] {output}")
|
|
22
|
+
else:
|
|
23
|
+
console.print("[bold green]Environment activated![/bold green]")
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def conda_create_env(ssh_conn, env_name, python_version=None):
|
|
27
|
+
"""Create a new conda environment."""
|
|
28
|
+
if python_version:
|
|
29
|
+
cmd = f"conda create -n {env_name} python={python_version} -y"
|
|
30
|
+
else:
|
|
31
|
+
cmd = f"conda create -n {env_name} -y"
|
|
32
|
+
|
|
33
|
+
console.print(f"[bold yellow]Creating environment '{env_name}'...[/bold yellow]")
|
|
34
|
+
output = ssh_conn.execute_command(cmd)
|
|
35
|
+
if output:
|
|
36
|
+
console.print("[bold green]Environment created successfully![/bold green]")
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def conda_remove_env(ssh_conn, env_name):
|
|
40
|
+
"""Remove a conda environment."""
|
|
41
|
+
console.print(f"[bold yellow]Removing environment '{env_name}'...[/bold yellow]")
|
|
42
|
+
output = ssh_conn.execute_command(f"conda env remove -n {env_name} -y")
|
|
43
|
+
console.print("[bold green]Environment removed![/bold green]")
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def conda_install_package(ssh_conn, env_name, package):
|
|
47
|
+
"""Install a package in a conda environment."""
|
|
48
|
+
console.print(f"[bold yellow]Installing {package} in '{env_name}'...[/bold yellow]")
|
|
49
|
+
output = ssh_conn.execute_command(f"conda install -n {env_name} {package} -y")
|
|
50
|
+
if output:
|
|
51
|
+
console.print("[bold green]Package installed![/bold green]")
|
|
52
|
+
|
|
53
|
+
def conda_list_package(ssh_conn, env_name):
|
|
54
|
+
"""Install a package in a conda environment."""
|
|
55
|
+
console.print(f"[bold yellow]Packages installed in '{env_name}'...[/bold yellow]")
|
|
56
|
+
if env_name == 'base':
|
|
57
|
+
output = ssh_conn.execute_command(f"source ~/.bashrc && conda list -n base")
|
|
58
|
+
else:
|
|
59
|
+
output = ssh_conn.execute_command(f"source ~/.bashrc && conda list -n {env_name}")
|
|
60
|
+
if output:
|
|
61
|
+
console.print({output})
|
|
@@ -0,0 +1,186 @@
|
|
|
1
|
+
"""File management commands."""
|
|
2
|
+
|
|
3
|
+
import subprocess
|
|
4
|
+
from rich.console import Console
|
|
5
|
+
from rich.table import Table
|
|
6
|
+
from rich.panel import Panel
|
|
7
|
+
from rich.syntax import Syntax
|
|
8
|
+
from rich import box
|
|
9
|
+
|
|
10
|
+
console = Console()
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def file_browse_directory(ssh_conn, path="~"):
|
|
14
|
+
"""Browse a directory and show files with details."""
|
|
15
|
+
cmd = f"ls -la {path}"
|
|
16
|
+
output = ssh_conn.execute_command(cmd)
|
|
17
|
+
if output:
|
|
18
|
+
table = Table(title=f"📁 Directory: {path}", box=box.ROUNDED)
|
|
19
|
+
table.add_column("Permissions", style="cyan")
|
|
20
|
+
table.add_column("Owner", style="green")
|
|
21
|
+
table.add_column("Group", style="green")
|
|
22
|
+
table.add_column("Size", style="yellow", justify="right")
|
|
23
|
+
table.add_column("Modified", style="magenta")
|
|
24
|
+
table.add_column("Name", style="bold white")
|
|
25
|
+
|
|
26
|
+
lines = output.strip().split('\n')[1:]
|
|
27
|
+
for line in lines:
|
|
28
|
+
parts = line.split(None, 8)
|
|
29
|
+
if len(parts) >= 9:
|
|
30
|
+
perms = parts[0]
|
|
31
|
+
owner = parts[2]
|
|
32
|
+
group = parts[3]
|
|
33
|
+
size = parts[4]
|
|
34
|
+
date_str = f"{parts[5]} {parts[6]} {parts[7]}"
|
|
35
|
+
name = parts[8]
|
|
36
|
+
icon = "📁" if perms.startswith('d') else "📄"
|
|
37
|
+
table.add_row(perms, owner, group, size, date_str, f"{icon} {name}")
|
|
38
|
+
|
|
39
|
+
console.print(table)
|
|
40
|
+
return True
|
|
41
|
+
return False
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def file_get_home_path(ssh_conn):
|
|
45
|
+
"""Get user's home directory."""
|
|
46
|
+
return ssh_conn.execute_command("echo $HOME").strip()
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def file_get_scratch_path(ssh_conn):
|
|
50
|
+
"""Get user's scratch directory."""
|
|
51
|
+
output = ssh_conn.execute_command("echo ~/scratch")
|
|
52
|
+
return output.strip()
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def file_upload(ssh_conn, local_path, remote_path):
|
|
56
|
+
"""Upload a file to the remote server."""
|
|
57
|
+
try:
|
|
58
|
+
cmd = f"scp -P {ssh_conn.port} -o ControlPath={ssh_conn.control_path} {local_path} {ssh_conn.user}@{ssh_conn.host}:{remote_path}"
|
|
59
|
+
console.print(f"[bold yellow]Uploading {local_path} to {remote_path}...[/bold yellow]")
|
|
60
|
+
result = subprocess.run(cmd, shell=True, capture_output=True, text=True)
|
|
61
|
+
if result.returncode == 0:
|
|
62
|
+
console.print("[bold green]✓ Upload successful![/bold green]")
|
|
63
|
+
return True
|
|
64
|
+
else:
|
|
65
|
+
console.print(f"[bold red]✗ Upload failed: {result.stderr}[/bold red]")
|
|
66
|
+
return False
|
|
67
|
+
except Exception as e:
|
|
68
|
+
console.print(f"[bold red]✗ Upload error: {str(e)}[/bold red]")
|
|
69
|
+
return False
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def file_download(ssh_conn, remote_path, local_path):
|
|
73
|
+
"""Download a file from the remote server."""
|
|
74
|
+
try:
|
|
75
|
+
cmd = f"scp -P {ssh_conn.port} -o ControlPath={ssh_conn.control_path} {ssh_conn.user}@{ssh_conn.host}:{remote_path} {local_path}"
|
|
76
|
+
console.print(f"[bold yellow]Downloading {remote_path} to {local_path}...[/bold yellow]")
|
|
77
|
+
result = subprocess.run(cmd, shell=True, capture_output=True, text=True)
|
|
78
|
+
if result.returncode == 0:
|
|
79
|
+
console.print("[bold green]✓ Download successful![/bold green]")
|
|
80
|
+
return True
|
|
81
|
+
else:
|
|
82
|
+
console.print(f"[bold red]✗ Download failed: {result.stderr}[/bold red]")
|
|
83
|
+
return False
|
|
84
|
+
except Exception as e:
|
|
85
|
+
console.print(f"[bold red]✗ Download error: {str(e)}[/bold red]")
|
|
86
|
+
return False
|
|
87
|
+
|
|
88
|
+
def file_edit(ssh_conn, path, filename):
|
|
89
|
+
"""Edit file."""
|
|
90
|
+
console.print(f"[bold yellow]Starting nano editor...[/bold yellow]")
|
|
91
|
+
|
|
92
|
+
cmd = f"nano {path}/{filename}"
|
|
93
|
+
try:
|
|
94
|
+
subprocess.run(
|
|
95
|
+
f"ssh -S {ssh_conn.control_path} -t -p {ssh_conn.port} {ssh_conn.user}@{ssh_conn.host} '{cmd}'",
|
|
96
|
+
shell=True
|
|
97
|
+
)
|
|
98
|
+
except Exception as e:
|
|
99
|
+
console.print(f"[bold red]Session error: {str(e)}[/bold red]")
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
def file_create_directory(ssh_conn, path):
|
|
103
|
+
"""Create a new directory."""
|
|
104
|
+
output = ssh_conn.execute_command(f"mkdir -p {path} && echo 'SUCCESS'")
|
|
105
|
+
if output and 'SUCCESS' in output:
|
|
106
|
+
console.print(f"[bold green]✓ Directory created: {path}[/bold green]")
|
|
107
|
+
return True
|
|
108
|
+
console.print(f"[bold red]✗ Failed to create directory[/bold red]")
|
|
109
|
+
return False
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
def file_delete(ssh_conn, path, is_directory=False):
|
|
113
|
+
"""Delete a file or directory."""
|
|
114
|
+
cmd = f"rm -rf {path}" if is_directory else f"rm {path}"
|
|
115
|
+
output = ssh_conn.execute_command(f"{cmd} && echo 'SUCCESS'")
|
|
116
|
+
if output and 'SUCCESS' in output:
|
|
117
|
+
console.print(f"[bold green]✓ Deleted: {path}[/bold green]")
|
|
118
|
+
return True
|
|
119
|
+
console.print(f"[bold red]✗ Failed to delete[/bold red]")
|
|
120
|
+
return False
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
def file_rename(ssh_conn, old_path, new_path):
|
|
124
|
+
"""Rename/move a file or directory."""
|
|
125
|
+
output = ssh_conn.execute_command(f"mv {old_path} {new_path} && echo 'SUCCESS'")
|
|
126
|
+
if output and 'SUCCESS' in output:
|
|
127
|
+
console.print(f"[bold green]✓ Renamed: {old_path} → {new_path}[/bold green]")
|
|
128
|
+
return True
|
|
129
|
+
console.print(f"[bold red]✗ Failed to rename[/bold red]")
|
|
130
|
+
return False
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
def file_view_content(ssh_conn, path, lines=50):
|
|
134
|
+
"""View file content (for text files)."""
|
|
135
|
+
output = ssh_conn.execute_command(f"head -n {lines} {path}")
|
|
136
|
+
if output:
|
|
137
|
+
ext = path.split('.')[-1] if '.' in path else 'txt'
|
|
138
|
+
syntax = Syntax(output, ext, theme="monokai", line_numbers=True)
|
|
139
|
+
console.print(Panel(syntax, title=f"📄 {path}", border_style="green"))
|
|
140
|
+
return True
|
|
141
|
+
return False
|
|
142
|
+
|
|
143
|
+
def file_search(ssh_conn, path, pattern, depth):
|
|
144
|
+
"""Search for files by name pattern."""
|
|
145
|
+
console.print(f"[bold cyan]Searching for '{pattern}' in {path}...[/bold cyan]\n")
|
|
146
|
+
cmd1 = f'find {path} -maxdepth 1 -name "*{pattern}*" | head -50'
|
|
147
|
+
output = ssh_conn.execute_command(cmd1)
|
|
148
|
+
|
|
149
|
+
if output and output.strip() and not depth:
|
|
150
|
+
console.print("[bold green]Files found in depth:[/bold green]")
|
|
151
|
+
console.print(output)
|
|
152
|
+
return True
|
|
153
|
+
else:
|
|
154
|
+
cmd2 = f'find {path} -name "*{pattern}*" | head -50'
|
|
155
|
+
output = ssh_conn.execute_command(cmd2)
|
|
156
|
+
|
|
157
|
+
if output and output.strip():
|
|
158
|
+
console.print("[bold green]Files found (recursive search):[/bold green]")
|
|
159
|
+
console.print(output)
|
|
160
|
+
return True
|
|
161
|
+
else:
|
|
162
|
+
console.print("[yellow]No files found matching the pattern.[/yellow]")
|
|
163
|
+
return False
|
|
164
|
+
|
|
165
|
+
def file_disk_quota(ssh_conn, path="~"):
|
|
166
|
+
"""Get disk usage for a directory."""
|
|
167
|
+
output = ssh_conn.execute_command(f"du -h {path} | sort -hr | head -20")
|
|
168
|
+
if output:
|
|
169
|
+
table = Table(title=f"💾 Disk Usage: {path}", box=box.ROUNDED)
|
|
170
|
+
table.add_column("Size", style="yellow", justify="right")
|
|
171
|
+
table.add_column("Path", style="cyan")
|
|
172
|
+
|
|
173
|
+
for line in output.strip().split('\n'):
|
|
174
|
+
if line:
|
|
175
|
+
parts = line.split('\t', 1)
|
|
176
|
+
if len(parts) == 2:
|
|
177
|
+
table.add_row(parts[0], parts[1])
|
|
178
|
+
|
|
179
|
+
console.print(table)
|
|
180
|
+
|
|
181
|
+
quota_output = ssh_conn.execute_command("lfs quota -h ~/")
|
|
182
|
+
if quota_output:
|
|
183
|
+
console.print("\n[bold cyan]Quota Information:[/bold cyan]")
|
|
184
|
+
console.print(quota_output)
|
|
185
|
+
return True
|
|
186
|
+
return False
|
manager/commands/help.py
ADDED
|
@@ -0,0 +1,165 @@
|
|
|
1
|
+
"""Help and documentation commands."""
|
|
2
|
+
|
|
3
|
+
from rich.console import Console
|
|
4
|
+
from rich.panel import Panel
|
|
5
|
+
|
|
6
|
+
console = Console()
|
|
7
|
+
|
|
8
|
+
def help_slurm_cheatsheet():
|
|
9
|
+
"""Display SLURM cheat sheet."""
|
|
10
|
+
cheatsheet = """
|
|
11
|
+
[bold cyan]📋 SLURM Cheat Sheet[/bold cyan]
|
|
12
|
+
|
|
13
|
+
[bold green]Quick sbatch script example:[/bold green]
|
|
14
|
+
#!/bin/bash
|
|
15
|
+
#SBATCH --job-name=training
|
|
16
|
+
#SBATCH --output=out.%j.log
|
|
17
|
+
#SBATCH --error=err.%j.log
|
|
18
|
+
#SBATCH --nodes=1
|
|
19
|
+
#SBATCH --ntasks=1
|
|
20
|
+
#SBATCH --cpus-per-task=8
|
|
21
|
+
#SBATCH --gres=gpu:1
|
|
22
|
+
#SBATCH --mem=32G
|
|
23
|
+
#SBATCH --time=02:00:00
|
|
24
|
+
#SBATCH --partition=gpu
|
|
25
|
+
module purge
|
|
26
|
+
module load anaconda/3
|
|
27
|
+
source activate myenv
|
|
28
|
+
srun python train.py --epochs 10
|
|
29
|
+
|
|
30
|
+
[bold green]Common sbatch flags (CLI):[/bold green]
|
|
31
|
+
sbatch --job-name=name --output=out.%j.log --partition=gpu --gres=gpu:1 script.sh
|
|
32
|
+
sbatch --array=1-10%3 array_script.sh Run job arrays (limit concurrent with %)
|
|
33
|
+
sbatch --dependency=afterok:<jobid> script.sh Chain jobs on success
|
|
34
|
+
sbatch --wrap="python -u quick_task.py" Submit single command
|
|
35
|
+
|
|
36
|
+
[bold green]srun examples:[/bold green]
|
|
37
|
+
srun --pty --nodes=1 --ntasks=1 --cpus-per-task=4 bash Interactive shell in allocation
|
|
38
|
+
srun -n 16 --mpi=pmi2 ./mpi_program Run MPI across 16 tasks
|
|
39
|
+
srun --gres=gpu:2 python train.py Launch task within job
|
|
40
|
+
|
|
41
|
+
[bold green]salloc (interactive allocation):[/bold green]
|
|
42
|
+
salloc --nodes=1 --gres=gpu:1 --time=01:00:00
|
|
43
|
+
# then inside allocation:
|
|
44
|
+
srun --pty bash
|
|
45
|
+
# or run Jupyter:
|
|
46
|
+
srun --mem=16G --cpus-per-task=4 jupyter lab --no-browser --port=8888
|
|
47
|
+
|
|
48
|
+
[bold green]Monitoring & info:[/bold green]
|
|
49
|
+
squeue -u $USER Show current jobs
|
|
50
|
+
squeue -j <jobid> Show specific job
|
|
51
|
+
squeue -o "%.18i %.9P %.8j %.8u %.2t %.10M %.6D %R" Custom output format
|
|
52
|
+
sacct -j <jobid> --format=JobID,JobName,State,ExitCode,Elapsed Accounting
|
|
53
|
+
seff <jobid> Efficiency (CPU/GPU/memory usage)
|
|
54
|
+
sinfo -s Summary of partitions
|
|
55
|
+
sinfo -N -l Detailed node listing
|
|
56
|
+
scontrol show job <jobid> Full job details
|
|
57
|
+
scontrol show node <nodename> Node diagnostics
|
|
58
|
+
|
|
59
|
+
[bold green]Job control:[/bold green]
|
|
60
|
+
scancel <jobid> Cancel job
|
|
61
|
+
scancel -u $USER Cancel all your jobs
|
|
62
|
+
scontrol update JobId=<jobid> TimeLimit=HH:MM:SS Modify job (if permitted)
|
|
63
|
+
|
|
64
|
+
[bold green]Resource & environment tips:[/bold green]
|
|
65
|
+
--gres=gpu:V100:1 or --gres=gpu:1 Request specific GPU type if supported
|
|
66
|
+
--mem=32G Memory per node; use --mem-per-cpu for per-core
|
|
67
|
+
--cpus-per-task=8 Useful for multithreaded tasks
|
|
68
|
+
export OMP_NUM_THREADS=8 Match threads to cpus-per-task
|
|
69
|
+
echo $SLURM_JOB_ID Job ID accessible in scripts
|
|
70
|
+
echo $SLURM_NTASKS $SLURM_NNODES Common SLURM env vars
|
|
71
|
+
|
|
72
|
+
[bold green]Job arrays & parametrization:[/bold green]
|
|
73
|
+
#SBATCH --array=0-9
|
|
74
|
+
TASK_ID=${SLURM_ARRAY_TASK_ID}
|
|
75
|
+
INPUT=file_${TASK_ID}.txt
|
|
76
|
+
# run array with per-task inputs or offsets
|
|
77
|
+
|
|
78
|
+
[bold green]Checkpointing & restarts (pattern):[/bold green]
|
|
79
|
+
# write periodic checkpoints to $SCRATCH or $TMPDIR
|
|
80
|
+
# save model state with step number or epoch
|
|
81
|
+
# on restart, read latest checkpoint and submit continuation via --dependency
|
|
82
|
+
|
|
83
|
+
[bold green]Interactive GPU debugging:[/bold green]
|
|
84
|
+
srun --pty --gres=gpu:1 --mem=16G --cpus-per-task=4 --time=00:30:00 bash
|
|
85
|
+
# run nvidia-smi to inspect GPU allocation and processes
|
|
86
|
+
|
|
87
|
+
[bold green]Data transfer & staging tips:[/bold green]
|
|
88
|
+
rsync -avP local_dir user@host:/scratch/$USER/project/ Efficient sync of datasets
|
|
89
|
+
scp file user@host:/scratch/$USER/ Simple copy for small files
|
|
90
|
+
Use $SCRATCH or $TMPDIR on compute nodes for I/O-heavy operations, then rsync back
|
|
91
|
+
|
|
92
|
+
[bold green]Troubleshooting commands:[/bold green]
|
|
93
|
+
scontrol show job <jobid> Inspect job reasons for PENDING
|
|
94
|
+
scontrol show partition <part> See partition limits and nodes
|
|
95
|
+
journalctl -u slurmctld (admin-only) cluster controller logs
|
|
96
|
+
module avail | grep -i <name> Find available modules
|
|
97
|
+
|
|
98
|
+
[bold green]Useful patterns:[/bold green]
|
|
99
|
+
# Submit and track:
|
|
100
|
+
jid=$(sbatch --parsable train.sh)
|
|
101
|
+
squeue -j $jid
|
|
102
|
+
# Submit dependent job:
|
|
103
|
+
sbatch --dependency=afterok:$jid evaluate.sh
|
|
104
|
+
|
|
105
|
+
[bold green]Shortcuts & sensible defaults:[/bold green]
|
|
106
|
+
--time=01:00:00 small debug
|
|
107
|
+
--time=24:00:00 typical training epoch
|
|
108
|
+
--partition=debug short-queue for quick tests (if available)
|
|
109
|
+
|
|
110
|
+
[bold yellow]Remember:[/bold yellow] Check your cluster's documentation for partition names, available GPUs, and site-specific sbatch directives.
|
|
111
|
+
"""
|
|
112
|
+
console.print(Panel(cheatsheet, title="📚 SLURM Quick Reference", border_style="cyan"))
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
def help_common_errors():
|
|
116
|
+
"""Display common errors and fixes."""
|
|
117
|
+
errors = """
|
|
118
|
+
[bold cyan]🔧 Common Errors & Fixes[/bold cyan]
|
|
119
|
+
|
|
120
|
+
[bold red]Job stuck in PENDING with (Priority):[/bold red]
|
|
121
|
+
→ Wait for higher priority jobs to complete
|
|
122
|
+
→ Check partition limits: scontrol show partition
|
|
123
|
+
|
|
124
|
+
[bold red]Job stuck in PENDING with (Resources):[/bold red]
|
|
125
|
+
→ Requested resources unavailable
|
|
126
|
+
→ Try: reduce nodes/memory/time or different partition
|
|
127
|
+
|
|
128
|
+
[bold red]CUDA out of memory:[/bold red]
|
|
129
|
+
→ Reduce batch size
|
|
130
|
+
→ Use gradient accumulation
|
|
131
|
+
→ Request more GPU memory
|
|
132
|
+
|
|
133
|
+
[bold red]Module not found:[/bold red]
|
|
134
|
+
→ Run: module avail | grep -i <name>
|
|
135
|
+
→ Check for typos in module name
|
|
136
|
+
|
|
137
|
+
[bold red]Permission denied:[/bold red]
|
|
138
|
+
→ Check file permissions: ls -la
|
|
139
|
+
→ Use chmod to fix: chmod +x script.sh
|
|
140
|
+
|
|
141
|
+
[bold red]Connection timeout:[/bold red]
|
|
142
|
+
→ Check network connectivity
|
|
143
|
+
→ Verify SSH port and hostname
|
|
144
|
+
→ Try reconnecting
|
|
145
|
+
"""
|
|
146
|
+
console.print(Panel(errors, title="🔧 Troubleshooting Guide", border_style="yellow"))
|
|
147
|
+
|
|
148
|
+
|
|
149
|
+
def help_about():
|
|
150
|
+
"""Display about information."""
|
|
151
|
+
about_text = """
|
|
152
|
+
[bold cyan]PARAM SSH Manager & HPC Console[/bold cyan]
|
|
153
|
+
|
|
154
|
+
A modern terminal user interface for managing HPC clusters.
|
|
155
|
+
|
|
156
|
+
[bold green]Features:[/bold green]
|
|
157
|
+
• SSH connection management with ControlMaster
|
|
158
|
+
• SLURM job submission and monitoring
|
|
159
|
+
• File management (upload/download/browse)
|
|
160
|
+
• Conda environment management
|
|
161
|
+
|
|
162
|
+
[bold yellow]Version:[/bold yellow] 1.0.0
|
|
163
|
+
[bold yellow]Author:[/bold yellow] [link=https://github.com/ayush1512]Ayush Saxena[/link]
|
|
164
|
+
"""
|
|
165
|
+
console.print(Panel(about_text, title="📖 About", border_style="cyan"))
|
|
@@ -0,0 +1,132 @@
|
|
|
1
|
+
"""Job template management commands."""
|
|
2
|
+
|
|
3
|
+
import subprocess
|
|
4
|
+
from rich.console import Console
|
|
5
|
+
from rich.table import Table
|
|
6
|
+
from rich.panel import Panel
|
|
7
|
+
from rich import box
|
|
8
|
+
|
|
9
|
+
from manager.templates import JOB_TEMPLATES
|
|
10
|
+
|
|
11
|
+
console = Console()
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def template_list():
|
|
15
|
+
"""List available job templates."""
|
|
16
|
+
table = Table(title="📝 Available Job Templates", box=box.ROUNDED)
|
|
17
|
+
table.add_column("Template", style="cyan")
|
|
18
|
+
table.add_column("Description", style="green")
|
|
19
|
+
|
|
20
|
+
templates_info = {
|
|
21
|
+
"python": "Basic Python script execution",
|
|
22
|
+
"mpi": "MPI parallel job",
|
|
23
|
+
"cuda": "CUDA GPU job",
|
|
24
|
+
"pytorch": "PyTorch deep learning job",
|
|
25
|
+
"tensorflow": "TensorFlow deep learning job",
|
|
26
|
+
"jupyter": "Jupyter Notebook/Lab session",
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
for name, desc in templates_info.items():
|
|
30
|
+
table.add_row(name, desc)
|
|
31
|
+
|
|
32
|
+
console.print(table)
|
|
33
|
+
return list(templates_info.keys())
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def template_generate(template_name, **kwargs):
|
|
37
|
+
"""Generate a job script from template."""
|
|
38
|
+
if template_name not in JOB_TEMPLATES:
|
|
39
|
+
console.print(f"[bold red]Template '{template_name}' not found![/bold red]")
|
|
40
|
+
return None
|
|
41
|
+
|
|
42
|
+
try:
|
|
43
|
+
script = JOB_TEMPLATES[template_name].format(**kwargs)
|
|
44
|
+
return script
|
|
45
|
+
except KeyError as e:
|
|
46
|
+
console.print(f"[bold red]Missing parameter: {e}[/bold red]")
|
|
47
|
+
return None
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def template_save(ssh_conn, script_content, filename):
|
|
51
|
+
"""Save a job script to remote server."""
|
|
52
|
+
cmd = f"cat > {filename} << 'SCRIPT_EOF'\n{script_content}\nSCRIPT_EOF"
|
|
53
|
+
ssh_conn.execute_command(cmd)
|
|
54
|
+
console.print(f"[bold green]✓ Script saved to: {filename}[/bold green]")
|
|
55
|
+
return filename
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def template_submit(ssh_conn, script_path):
|
|
59
|
+
"""Submit a job script."""
|
|
60
|
+
output = ssh_conn.execute_command(f"sbatch {script_path}")
|
|
61
|
+
if output:
|
|
62
|
+
console.print(f"[bold green]✓ Job submitted![/bold green]")
|
|
63
|
+
console.print(output)
|
|
64
|
+
return True
|
|
65
|
+
return False
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def interactive_start_jupyter(ssh_conn, jupyter_type="notebook", conda_env="base", num_gpus=0, port=8888):
|
|
69
|
+
"""Start Jupyter Notebook/Lab via SLURM."""
|
|
70
|
+
template = JOB_TEMPLATES["jupyter"]
|
|
71
|
+
script = template.format(
|
|
72
|
+
job_name="jupyter",
|
|
73
|
+
conda_env=conda_env,
|
|
74
|
+
num_gpus=num_gpus,
|
|
75
|
+
port=port,
|
|
76
|
+
jupyter_type=jupyter_type,
|
|
77
|
+
user=ssh_conn.user,
|
|
78
|
+
host=ssh_conn.host,
|
|
79
|
+
ssh_port=ssh_conn.port
|
|
80
|
+
)
|
|
81
|
+
|
|
82
|
+
script_path = f"~/.jupyter_job_{port}.sh"
|
|
83
|
+
ssh_conn.execute_command(f"cat > {script_path} << 'EOF'\n{script}\nEOF")
|
|
84
|
+
|
|
85
|
+
output = ssh_conn.execute_command(f"sbatch {script_path}")
|
|
86
|
+
if output:
|
|
87
|
+
console.print(f"[bold green]✓ Jupyter {jupyter_type} job submitted![/bold green]")
|
|
88
|
+
console.print(output)
|
|
89
|
+
console.print(f"\n[bold cyan]Once the job starts, create an SSH tunnel:[/bold cyan]")
|
|
90
|
+
console.print(f"[yellow]ssh -L {port}:<node>:{port} {ssh_conn.user}@{ssh_conn.host} -p {ssh_conn.port}[/yellow]")
|
|
91
|
+
console.print(f"\n[bold cyan]Then open in browser:[/bold cyan] http://localhost:{port}")
|
|
92
|
+
return True
|
|
93
|
+
return False
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
def interactive_gpu_session(ssh_conn, num_gpus=1, time="02:00:00", mem="16G"):
|
|
97
|
+
"""Start an interactive GPU session."""
|
|
98
|
+
console.print(f"[bold yellow]Starting interactive GPU session ({num_gpus} GPU(s))...[/bold yellow]")
|
|
99
|
+
console.print("[dim]This will open an interactive shell. Type 'exit' to end the session.[/dim]\n")
|
|
100
|
+
|
|
101
|
+
cmd = f"srun --gres=gpu:{num_gpus} --time={time} --mem={mem} --pty bash"
|
|
102
|
+
try:
|
|
103
|
+
subprocess.run(
|
|
104
|
+
f"ssh -S {ssh_conn.control_path} -t -p {ssh_conn.port} {ssh_conn.user}@{ssh_conn.host} '{cmd}'",
|
|
105
|
+
shell=True
|
|
106
|
+
)
|
|
107
|
+
except Exception as e:
|
|
108
|
+
console.print(f"[bold red]Session error: {str(e)}[/bold red]")
|
|
109
|
+
|
|
110
|
+
def interactive_cpu_session(ssh_conn, num_cpus=1, time="02:00:00", core="40"):
|
|
111
|
+
"""Start an interactive GPU session."""
|
|
112
|
+
console.print(f"[bold yellow]Starting interactive CPU session ({num_cpus} CPU(s))...[/bold yellow]")
|
|
113
|
+
console.print("[dim]This will open an interactive shell. Type 'exit' to end the session.[/dim]\n")
|
|
114
|
+
|
|
115
|
+
cmd = f"srun --partition=cpu -N {num_cpus} --time={time} -c {core} --pty bash"
|
|
116
|
+
try:
|
|
117
|
+
subprocess.run(
|
|
118
|
+
f"ssh -S {ssh_conn.control_path} -t -p {ssh_conn.port} {ssh_conn.user}@{ssh_conn.host} '{cmd}'",
|
|
119
|
+
shell=True
|
|
120
|
+
)
|
|
121
|
+
except Exception as e:
|
|
122
|
+
console.print(f"[bold red]Session error: {str(e)}[/bold red]")
|
|
123
|
+
|
|
124
|
+
def interactive_list_notebooks(ssh_conn):
|
|
125
|
+
"""List running Jupyter jobs."""
|
|
126
|
+
output = ssh_conn.execute_command("squeue -u $USER -n jupyter_* -o '%.18i %.15j %.8T %.10M %.R' 2>/dev/null")
|
|
127
|
+
if output and len(output.strip().split('\n')) > 1:
|
|
128
|
+
console.print("[bold green]📓 Running Jupyter Sessions:[/bold green]")
|
|
129
|
+
console.print(output)
|
|
130
|
+
return True
|
|
131
|
+
console.print("[yellow]No active Jupyter sessions found.[/yellow]")
|
|
132
|
+
return False
|
manager/commands/logs.py
ADDED
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
"""Log management commands."""
|
|
2
|
+
|
|
3
|
+
from datetime import datetime, timedelta
|
|
4
|
+
from rich.console import Console
|
|
5
|
+
from rich.panel import Panel
|
|
6
|
+
|
|
7
|
+
from manager.commands.files import file_download
|
|
8
|
+
|
|
9
|
+
console = Console()
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def logs_get_session_history(ssh_conn, lines=50):
|
|
13
|
+
"""Get SSH session history."""
|
|
14
|
+
output = ssh_conn.execute_command(f"last -n {lines} $USER || last -n {lines}")
|
|
15
|
+
if output:
|
|
16
|
+
console.print(Panel(output, title="📋 Session History", border_style="cyan"))
|
|
17
|
+
return True
|
|
18
|
+
return False
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def logs_get_job_submission_history(ssh_conn, days=7):
|
|
22
|
+
"""Get job submission history."""
|
|
23
|
+
start_date = (datetime.now() - timedelta(days=days)).strftime("%Y-%m-%d")
|
|
24
|
+
output = ssh_conn.execute_command(f"sacct -u $USER -S {start_date} -o JobID,JobName,Submit,Start,End,State | head -50")
|
|
25
|
+
if output:
|
|
26
|
+
console.print(Panel(output, title=f"📝 Job Submission History (Last {days} days)", border_style="green"))
|
|
27
|
+
return True
|
|
28
|
+
return False
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def logs_get_error_logs(ssh_conn, pattern="*.err"):
|
|
32
|
+
"""Find and list recent error logs."""
|
|
33
|
+
output = ssh_conn.execute_command(f"find ~ -name '{pattern}' -mtime -7 -type f | head -20")
|
|
34
|
+
if output:
|
|
35
|
+
console.print("[bold cyan]📋 Recent Error Logs:[/bold cyan]")
|
|
36
|
+
for log in output.strip().split('\n'):
|
|
37
|
+
if log:
|
|
38
|
+
console.print(f" 📄 {log}")
|
|
39
|
+
return output.strip().split('\n')
|
|
40
|
+
console.print("[yellow]No recent error logs found.[/yellow]")
|
|
41
|
+
return []
|
|
42
|
+
|