machineconfig 7.53__py3-none-any.whl → 7.69__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of machineconfig might be problematic. Click here for more details.
- machineconfig/cluster/sessions_managers/utils/maker.py +21 -9
- machineconfig/jobs/installer/custom/boxes.py +2 -2
- machineconfig/jobs/installer/custom/hx.py +15 -12
- machineconfig/jobs/installer/custom_dev/cloudflare_warp_cli.py +23 -0
- machineconfig/jobs/installer/custom_dev/dubdb_adbc.py +1 -1
- machineconfig/jobs/installer/custom_dev/nerfont_windows_helper.py +1 -1
- machineconfig/jobs/installer/custom_dev/sysabc.py +39 -34
- machineconfig/jobs/installer/custom_dev/wezterm.py +0 -4
- machineconfig/jobs/installer/installer_data.json +103 -35
- machineconfig/jobs/installer/package_groups.py +28 -13
- machineconfig/scripts/__init__.py +0 -4
- machineconfig/scripts/linux/wrap_mcfg +1 -1
- machineconfig/scripts/python/ai/solutions/copilot/instructions/python/dev.instructions.md +3 -0
- machineconfig/scripts/python/croshell.py +22 -17
- machineconfig/scripts/python/devops.py +3 -4
- machineconfig/scripts/python/devops_navigator.py +0 -4
- machineconfig/scripts/python/env_manager/path_manager_tui.py +1 -1
- machineconfig/scripts/python/fire_jobs.py +17 -15
- machineconfig/scripts/python/ftpx.py +13 -11
- machineconfig/scripts/python/helpers/ast_search.py +74 -0
- machineconfig/scripts/python/helpers/repo_rag.py +325 -0
- machineconfig/scripts/python/helpers/symantic_search.py +25 -0
- machineconfig/scripts/python/helpers_cloud/cloud_copy.py +28 -21
- machineconfig/scripts/python/helpers_cloud/cloud_helpers.py +1 -1
- machineconfig/scripts/python/helpers_cloud/cloud_sync.py +8 -7
- machineconfig/scripts/python/helpers_croshell/crosh.py +2 -2
- machineconfig/scripts/python/helpers_devops/cli_config_dotfile.py +22 -13
- machineconfig/scripts/python/helpers_devops/cli_self.py +7 -6
- machineconfig/scripts/python/helpers_devops/cli_share_file.py +2 -2
- machineconfig/scripts/python/helpers_devops/cli_share_server.py +1 -1
- machineconfig/scripts/python/helpers_devops/cli_terminal.py +1 -1
- machineconfig/scripts/python/helpers_devops/cli_utils.py +2 -73
- machineconfig/scripts/python/helpers_devops/devops_backup_retrieve.py +4 -4
- machineconfig/scripts/python/helpers_fire_command/file_wrangler.py +2 -3
- machineconfig/scripts/python/helpers_fire_command/fire_jobs_route_helper.py +3 -4
- machineconfig/scripts/python/helpers_navigator/command_tree.py +50 -18
- machineconfig/scripts/python/helpers_repos/cloud_repo_sync.py +13 -5
- machineconfig/scripts/python/helpers_repos/count_lines_frontend.py +1 -1
- machineconfig/scripts/python/helpers_repos/entrypoint.py +2 -1
- machineconfig/scripts/python/helpers_repos/record.py +2 -1
- machineconfig/scripts/python/helpers_sessions/sessions_multiprocess.py +5 -5
- machineconfig/scripts/python/helpers_utils/download.py +152 -0
- machineconfig/scripts/python/helpers_utils/path.py +4 -2
- machineconfig/scripts/python/interactive.py +11 -14
- machineconfig/scripts/python/{machineconfig.py → mcfg_entry.py} +4 -0
- machineconfig/scripts/python/msearch.py +21 -2
- machineconfig/scripts/python/nw/devops_add_ssh_key.py +21 -5
- machineconfig/scripts/python/nw/ssh_debug_linux.py +7 -7
- machineconfig/scripts/python/nw/ssh_debug_windows.py +4 -4
- machineconfig/scripts/python/nw/wsl_windows_transfer.py +3 -2
- machineconfig/scripts/python/sessions.py +35 -20
- machineconfig/scripts/python/terminal.py +2 -2
- machineconfig/scripts/python/utils.py +12 -10
- machineconfig/scripts/windows/mounts/mount_ssh.ps1 +1 -1
- machineconfig/settings/lf/windows/lfcd.ps1 +1 -1
- machineconfig/settings/shells/pwsh/init.ps1 +1 -0
- machineconfig/settings/shells/wezterm/wezterm.lua +2 -0
- machineconfig/settings/shells/zsh/init.sh +0 -7
- machineconfig/settings/yazi/shell/yazi_cd.ps1 +29 -5
- machineconfig/setup_linux/web_shortcuts/interactive.sh +12 -11
- machineconfig/setup_windows/uv.ps1 +8 -1
- machineconfig/setup_windows/web_shortcuts/interactive.ps1 +12 -11
- machineconfig/setup_windows/web_shortcuts/quick_init.ps1 +4 -2
- machineconfig/utils/accessories.py +7 -4
- machineconfig/utils/code.py +6 -4
- machineconfig/utils/files/headers.py +2 -2
- machineconfig/utils/installer_utils/install_from_url.py +180 -0
- machineconfig/utils/installer_utils/installer_class.py +56 -46
- machineconfig/utils/installer_utils/{installer.py → installer_cli.py} +71 -65
- machineconfig/utils/{installer.py → installer_utils/installer_runner.py} +1 -25
- machineconfig/utils/meta.py +28 -15
- machineconfig/utils/options.py +4 -4
- machineconfig/utils/path_extended.py +40 -19
- machineconfig/utils/path_helper.py +33 -31
- machineconfig/utils/schemas/layouts/layout_types.py +1 -1
- machineconfig/utils/ssh.py +330 -99
- machineconfig/utils/ve.py +11 -4
- machineconfig-7.69.dist-info/METADATA +124 -0
- {machineconfig-7.53.dist-info → machineconfig-7.69.dist-info}/RECORD +85 -83
- {machineconfig-7.53.dist-info → machineconfig-7.69.dist-info}/entry_points.txt +2 -2
- machineconfig/jobs/installer/linux_scripts/pgsql.sh +0 -41
- machineconfig/scripts/python/explore.py +0 -49
- machineconfig/scripts/python/nw/add_ssh_key.py +0 -148
- machineconfig/settings/lf/linux/exe/fzf_nano.sh +0 -16
- machineconfig-7.53.dist-info/METADATA +0 -94
- /machineconfig/jobs/installer/linux_scripts/{warp-cli.sh → cloudflare_warp_cli.sh} +0 -0
- /machineconfig/scripts/{Restore-ThunderbirdProfile.ps1 → windows/mounts/Restore-ThunderbirdProfile.ps1} +0 -0
- /machineconfig/utils/installer_utils/{installer_abc.py → installer_locator_utils.py} +0 -0
- {machineconfig-7.53.dist-info → machineconfig-7.69.dist-info}/WHEEL +0 -0
- {machineconfig-7.53.dist-info → machineconfig-7.69.dist-info}/top_level.txt +0 -0
machineconfig/utils/ssh.py
CHANGED
|
@@ -7,17 +7,28 @@ import rich.console
|
|
|
7
7
|
from machineconfig.utils.terminal import Response
|
|
8
8
|
from machineconfig.utils.accessories import pprint, randstr
|
|
9
9
|
from machineconfig.utils.meta import lambda_to_python_script
|
|
10
|
+
|
|
10
11
|
UV_RUN_CMD = "$HOME/.local/bin/uv run" if platform.system() != "Windows" else """& "$env:USERPROFILE/.local/bin/uv" run"""
|
|
11
|
-
MACHINECONFIG_VERSION = "machineconfig>=7.
|
|
12
|
+
MACHINECONFIG_VERSION = "machineconfig>=7.69"
|
|
12
13
|
DEFAULT_PICKLE_SUBDIR = "tmp_results/tmp_scripts/ssh"
|
|
13
14
|
|
|
15
|
+
|
|
14
16
|
class SSH:
|
|
15
17
|
@staticmethod
|
|
16
18
|
def from_config_file(host: str) -> "SSH":
|
|
17
19
|
"""Create SSH instance from SSH config file entry."""
|
|
18
20
|
return SSH(host=host, username=None, hostname=None, ssh_key_path=None, password=None, port=22, enable_compression=False)
|
|
21
|
+
|
|
19
22
|
def __init__(
|
|
20
|
-
self,
|
|
23
|
+
self,
|
|
24
|
+
host: Optional[str],
|
|
25
|
+
username: Optional[str],
|
|
26
|
+
hostname: Optional[str],
|
|
27
|
+
ssh_key_path: Optional[str],
|
|
28
|
+
password: Optional[str],
|
|
29
|
+
port: int,
|
|
30
|
+
enable_compression: bool,
|
|
31
|
+
):
|
|
21
32
|
self.password = password
|
|
22
33
|
self.enable_compression = enable_compression
|
|
23
34
|
|
|
@@ -52,7 +63,9 @@ class SSH:
|
|
|
52
63
|
else:
|
|
53
64
|
ssh_key_path = wildcard_identity_file
|
|
54
65
|
except (FileNotFoundError, KeyError):
|
|
55
|
-
assert "@" in host or ":" in host,
|
|
66
|
+
assert "@" in host or ":" in host, (
|
|
67
|
+
f"Host must be in the form of `username@hostname:port` or `username@hostname` or `hostname:port`, but it is: {host}"
|
|
68
|
+
)
|
|
56
69
|
if "@" in host:
|
|
57
70
|
self.username, self.hostname = host.split("@")
|
|
58
71
|
else:
|
|
@@ -72,7 +85,10 @@ class SSH:
|
|
|
72
85
|
self.ssh = paramiko.SSHClient()
|
|
73
86
|
self.ssh.load_system_host_keys()
|
|
74
87
|
self.ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
|
75
|
-
pprint(
|
|
88
|
+
pprint(
|
|
89
|
+
dict(host=self.host, hostname=self.hostname, username=self.username, password="***", port=self.port, key_filename=self.ssh_key_path),
|
|
90
|
+
title="SSHing To",
|
|
91
|
+
)
|
|
76
92
|
sock = paramiko.ProxyCommand(self.proxycommand) if self.proxycommand is not None else None
|
|
77
93
|
try:
|
|
78
94
|
if password is None:
|
|
@@ -81,11 +97,31 @@ class SSH:
|
|
|
81
97
|
else:
|
|
82
98
|
allow_agent = False
|
|
83
99
|
look_for_keys = False
|
|
84
|
-
self.ssh.connect(
|
|
100
|
+
self.ssh.connect(
|
|
101
|
+
hostname=self.hostname,
|
|
102
|
+
username=self.username,
|
|
103
|
+
password=self.password,
|
|
104
|
+
port=self.port,
|
|
105
|
+
key_filename=self.ssh_key_path,
|
|
106
|
+
compress=self.enable_compression,
|
|
107
|
+
sock=sock,
|
|
108
|
+
allow_agent=allow_agent,
|
|
109
|
+
look_for_keys=look_for_keys,
|
|
110
|
+
) # type: ignore
|
|
85
111
|
except Exception as _err:
|
|
86
112
|
rich.console.Console().print_exception()
|
|
87
113
|
self.password = getpass.getpass(f"Enter password for {self.username}@{self.hostname}: ")
|
|
88
|
-
self.ssh.connect(
|
|
114
|
+
self.ssh.connect(
|
|
115
|
+
hostname=self.hostname,
|
|
116
|
+
username=self.username,
|
|
117
|
+
password=self.password,
|
|
118
|
+
port=self.port,
|
|
119
|
+
key_filename=self.ssh_key_path,
|
|
120
|
+
compress=self.enable_compression,
|
|
121
|
+
sock=sock,
|
|
122
|
+
allow_agent=False,
|
|
123
|
+
look_for_keys=False,
|
|
124
|
+
) # type: ignore
|
|
89
125
|
try:
|
|
90
126
|
self.sftp: Optional[paramiko.SFTPClient] = self.ssh.open_sftp()
|
|
91
127
|
except Exception as err:
|
|
@@ -100,7 +136,9 @@ class SSH:
|
|
|
100
136
|
self.task: Optional[Any] = None
|
|
101
137
|
|
|
102
138
|
def __enter__(self) -> "RichProgressWrapper":
|
|
103
|
-
self.progress = Progress(
|
|
139
|
+
self.progress = Progress(
|
|
140
|
+
SpinnerColumn(), TextColumn("[bold blue]{task.description}"), BarColumn(), FileSizeColumn(), TransferSpeedColumn()
|
|
141
|
+
)
|
|
104
142
|
self.progress.start()
|
|
105
143
|
self.task = self.progress.add_task("Transferring...", total=0)
|
|
106
144
|
return self
|
|
@@ -112,35 +150,49 @@ class SSH:
|
|
|
112
150
|
def view_bar(self, transferred: int, total: int) -> None:
|
|
113
151
|
if self.progress and self.task is not None:
|
|
114
152
|
self.progress.update(self.task, completed=transferred, total=total)
|
|
153
|
+
|
|
115
154
|
self.tqdm_wrap = RichProgressWrapper
|
|
116
155
|
from machineconfig.scripts.python.helpers_utils.path import get_machine_specs
|
|
156
|
+
|
|
117
157
|
self.local_specs: MachineSpecs = get_machine_specs()
|
|
118
|
-
resp = self.run_shell(
|
|
158
|
+
resp = self.run_shell(
|
|
159
|
+
command="""~/.local/bin/utils get-machine-specs """,
|
|
160
|
+
verbose_output=False,
|
|
161
|
+
description="Getting remote machine specs",
|
|
162
|
+
strict_stderr=False,
|
|
163
|
+
strict_return_code=False,
|
|
164
|
+
)
|
|
119
165
|
json_str = resp.op
|
|
120
166
|
import ast
|
|
167
|
+
|
|
121
168
|
self.remote_specs: MachineSpecs = cast(MachineSpecs, ast.literal_eval(json_str))
|
|
122
169
|
self.terminal_responses: list[Response] = []
|
|
123
|
-
|
|
170
|
+
|
|
124
171
|
from rich import inspect
|
|
125
|
-
|
|
172
|
+
|
|
126
173
|
local_info = dict(distro=self.local_specs.get("distro"), system=self.local_specs.get("system"), home_dir=self.local_specs.get("home_dir"))
|
|
127
174
|
remote_info = dict(distro=self.remote_specs.get("distro"), system=self.remote_specs.get("system"), home_dir=self.remote_specs.get("home_dir"))
|
|
128
|
-
|
|
175
|
+
|
|
129
176
|
console = rich.console.Console()
|
|
130
|
-
|
|
177
|
+
|
|
131
178
|
from io import StringIO
|
|
179
|
+
|
|
132
180
|
local_buffer = StringIO()
|
|
133
181
|
remote_buffer = StringIO()
|
|
134
|
-
|
|
182
|
+
|
|
135
183
|
local_console = rich.console.Console(file=local_buffer, width=40)
|
|
136
184
|
remote_console = rich.console.Console(file=remote_buffer, width=40)
|
|
137
|
-
|
|
138
|
-
inspect(
|
|
139
|
-
|
|
140
|
-
|
|
185
|
+
|
|
186
|
+
inspect(
|
|
187
|
+
type("LocalInfo", (object,), local_info)(), value=False, title="SSHing From", docs=False, dunder=False, sort=False, console=local_console
|
|
188
|
+
)
|
|
189
|
+
inspect(
|
|
190
|
+
type("RemoteInfo", (object,), remote_info)(), value=False, title="SSHing To", docs=False, dunder=False, sort=False, console=remote_console
|
|
191
|
+
)
|
|
192
|
+
|
|
141
193
|
local_lines = local_buffer.getvalue().split("\n")
|
|
142
194
|
remote_lines = remote_buffer.getvalue().split("\n")
|
|
143
|
-
|
|
195
|
+
|
|
144
196
|
max_lines = max(len(local_lines), len(remote_lines))
|
|
145
197
|
for i in range(max_lines):
|
|
146
198
|
left = local_lines[i] if i < len(local_lines) else ""
|
|
@@ -149,37 +201,59 @@ class SSH:
|
|
|
149
201
|
|
|
150
202
|
def __enter__(self) -> "SSH":
|
|
151
203
|
return self
|
|
204
|
+
|
|
152
205
|
def __exit__(self, exc_type: Any, exc_val: Any, exc_tb: Any) -> None:
|
|
153
206
|
self.close()
|
|
207
|
+
|
|
154
208
|
def close(self) -> None:
|
|
155
209
|
if self.sftp is not None:
|
|
156
210
|
self.sftp.close()
|
|
157
211
|
self.sftp = None
|
|
158
212
|
self.ssh.close()
|
|
213
|
+
|
|
159
214
|
def restart_computer(self) -> Response:
|
|
160
|
-
return self.run_shell(
|
|
215
|
+
return self.run_shell(
|
|
216
|
+
command="Restart-Computer -Force" if self.remote_specs["system"] == "Windows" else "sudo reboot",
|
|
217
|
+
verbose_output=True,
|
|
218
|
+
description="",
|
|
219
|
+
strict_stderr=False,
|
|
220
|
+
strict_return_code=False,
|
|
221
|
+
)
|
|
222
|
+
|
|
161
223
|
def send_ssh_key(self) -> Response:
|
|
162
224
|
self.copy_from_here(source_path="~/.ssh/id_rsa.pub", target_rel2home=None, compress_with_zip=False, recursive=False, overwrite_existing=False)
|
|
163
225
|
if self.remote_specs["system"] != "Windows":
|
|
164
226
|
raise RuntimeError("send_ssh_key is only supported for Windows remote machines")
|
|
165
227
|
code_url = "https://raw.githubusercontent.com/thisismygitrepo/machineconfig/refs/heads/main/src/machineconfig/setup_windows/openssh-server_add-sshkey.ps1"
|
|
166
228
|
import urllib.request
|
|
229
|
+
|
|
167
230
|
with urllib.request.urlopen(code_url) as response:
|
|
168
231
|
code = response.read().decode("utf-8")
|
|
169
232
|
return self.run_shell(command=code, verbose_output=True, description="", strict_stderr=False, strict_return_code=False)
|
|
170
233
|
|
|
171
234
|
def get_remote_repr(self, add_machine: bool = False) -> str:
|
|
172
|
-
return f"{self.username}@{self.hostname}:{self.port}" + (
|
|
235
|
+
return f"{self.username}@{self.hostname}:{self.port}" + (
|
|
236
|
+
f" [{self.remote_specs['system']}][{self.remote_specs['distro']}]" if add_machine else ""
|
|
237
|
+
)
|
|
238
|
+
|
|
173
239
|
def get_local_repr(self, add_machine: bool = False) -> str:
|
|
174
240
|
import getpass
|
|
241
|
+
|
|
175
242
|
return f"{getpass.getuser()}@{platform.node()}" + (f" [{platform.system()}][{self.local_specs['distro']}]" if add_machine else "")
|
|
243
|
+
|
|
176
244
|
def get_ssh_conn_str(self, command: str) -> str:
|
|
177
|
-
return
|
|
245
|
+
return (
|
|
246
|
+
"ssh "
|
|
247
|
+
+ (f" -i {self.ssh_key_path}" if self.ssh_key_path else "")
|
|
248
|
+
+ self.get_remote_repr(add_machine=False).replace(":", " -p ")
|
|
249
|
+
+ (f" -t {command} " if command != "" else " ")
|
|
250
|
+
)
|
|
251
|
+
|
|
178
252
|
def __repr__(self) -> str:
|
|
179
253
|
return f"local {self.get_local_repr(add_machine=True)} >>> SSH TO >>> remote {self.get_remote_repr(add_machine=True)}"
|
|
180
254
|
|
|
181
255
|
def run_locally(self, command: str) -> Response:
|
|
182
|
-
print(f"""💻 [LOCAL EXECUTION] Running command on node: {self.local_specs[
|
|
256
|
+
print(f"""💻 [LOCAL EXECUTION] Running command on node: {self.local_specs["system"]} Command: {command}""")
|
|
183
257
|
res = Response(cmd=command)
|
|
184
258
|
res.output.returncode = os.system(command)
|
|
185
259
|
return res
|
|
@@ -190,11 +264,13 @@ class SSH:
|
|
|
190
264
|
if verbose_output:
|
|
191
265
|
res.print()
|
|
192
266
|
else:
|
|
193
|
-
res.capture().print_if_unsuccessful(
|
|
267
|
+
res.capture().print_if_unsuccessful(
|
|
268
|
+
desc=description, strict_err=strict_stderr, strict_returncode=strict_return_code, assert_success=False
|
|
269
|
+
)
|
|
194
270
|
# self.terminal_responses.append(res)
|
|
195
271
|
return res
|
|
196
272
|
|
|
197
|
-
def _run_py_prep(self, python_code: str, uv_with: Optional[list[str]], uv_project_dir: Optional[str]
|
|
273
|
+
def _run_py_prep(self, python_code: str, uv_with: Optional[list[str]], uv_project_dir: Optional[str]) -> str:
|
|
198
274
|
py_path = Path.home().joinpath(f"{DEFAULT_PICKLE_SUBDIR}/runpy_{randstr()}.py")
|
|
199
275
|
py_path.parent.mkdir(parents=True, exist_ok=True)
|
|
200
276
|
py_path.write_text(python_code, encoding="utf-8")
|
|
@@ -210,10 +286,24 @@ class SSH:
|
|
|
210
286
|
uv_cmd = f"""{UV_RUN_CMD} {with_clause} python {py_path.relative_to(Path.home())}"""
|
|
211
287
|
return uv_cmd
|
|
212
288
|
|
|
213
|
-
def run_py(
|
|
214
|
-
|
|
289
|
+
def run_py(
|
|
290
|
+
self,
|
|
291
|
+
python_code: str,
|
|
292
|
+
uv_with: Optional[list[str]],
|
|
293
|
+
uv_project_dir: Optional[str],
|
|
294
|
+
description: str,
|
|
295
|
+
verbose_output: bool,
|
|
296
|
+
strict_stderr: bool,
|
|
297
|
+
strict_return_code: bool,
|
|
298
|
+
) -> Response:
|
|
215
299
|
uv_cmd = self._run_py_prep(python_code=python_code, uv_with=uv_with, uv_project_dir=uv_project_dir)
|
|
216
|
-
return self.run_shell(
|
|
300
|
+
return self.run_shell(
|
|
301
|
+
command=uv_cmd,
|
|
302
|
+
verbose_output=verbose_output,
|
|
303
|
+
description=description or f"run_py on {self.get_remote_repr(add_machine=False)}",
|
|
304
|
+
strict_stderr=strict_stderr,
|
|
305
|
+
strict_return_code=strict_return_code,
|
|
306
|
+
)
|
|
217
307
|
|
|
218
308
|
def run_lambda_function(self, func: Callable[..., Any], import_module: bool, uv_with: Optional[list[str]], uv_project_dir: Optional[str]):
|
|
219
309
|
command = lambda_to_python_script(lmb=func, in_global=True, import_module=import_module)
|
|
@@ -224,13 +314,20 @@ class SSH:
|
|
|
224
314
|
uv_cmd = self._run_py_prep(python_code=command, uv_with=uv_with, uv_project_dir=uv_project_dir)
|
|
225
315
|
if self.remote_specs["system"] == "Linux":
|
|
226
316
|
uv_cmd_modified = f'bash -l -c "{uv_cmd}"'
|
|
227
|
-
else:
|
|
317
|
+
else:
|
|
318
|
+
uv_cmd_modified = uv_cmd
|
|
228
319
|
# This works even withou the modified uv cmd:
|
|
229
320
|
# from machineconfig.utils.code import run_shell_script
|
|
230
321
|
# assert self.host is not None, "SSH host must be specified to run remote commands"
|
|
231
322
|
# process = run_shell_script(f"ssh {self.host} -n '. ~/.profile; . ~/.bashrc; {uv_cmd}'")
|
|
232
323
|
# return process
|
|
233
|
-
return self.run_shell(
|
|
324
|
+
return self.run_shell(
|
|
325
|
+
command=uv_cmd_modified,
|
|
326
|
+
verbose_output=True,
|
|
327
|
+
description=f"run_py_func {func.__name__} on {self.get_remote_repr(add_machine=False)}",
|
|
328
|
+
strict_stderr=True,
|
|
329
|
+
strict_return_code=True,
|
|
330
|
+
)
|
|
234
331
|
|
|
235
332
|
def _simple_sftp_get(self, remote_path: str, local_path: Path) -> None:
|
|
236
333
|
"""Simple SFTP get without any recursion or path expansion - for internal use only."""
|
|
@@ -241,9 +338,11 @@ class SSH:
|
|
|
241
338
|
|
|
242
339
|
def create_dir(self, path_rel2home: str, overwrite_existing: bool) -> None:
|
|
243
340
|
"""Helper to create a directory on remote machine and return its path."""
|
|
341
|
+
|
|
244
342
|
def create_target_dir(target_rel2home: str, overwrite: bool):
|
|
245
343
|
from pathlib import Path
|
|
246
344
|
import shutil
|
|
345
|
+
|
|
247
346
|
directory_path = Path(target_rel2home).expanduser()
|
|
248
347
|
if not directory_path.is_absolute():
|
|
249
348
|
directory_path = Path.home().joinpath(directory_path)
|
|
@@ -254,7 +353,10 @@ class SSH:
|
|
|
254
353
|
directory_path.unlink()
|
|
255
354
|
directory_path.parent.mkdir(parents=True, exist_ok=True)
|
|
256
355
|
directory_path.mkdir(parents=True, exist_ok=True)
|
|
257
|
-
|
|
356
|
+
|
|
357
|
+
command = lambda_to_python_script(
|
|
358
|
+
lmb=lambda: create_target_dir(target_rel2home=path_rel2home, overwrite=overwrite_existing), in_global=True, import_module=False
|
|
359
|
+
)
|
|
258
360
|
tmp_py_file = Path.home().joinpath(f"{DEFAULT_PICKLE_SUBDIR}/create_target_dir_{randstr()}.py")
|
|
259
361
|
tmp_py_file.parent.mkdir(parents=True, exist_ok=True)
|
|
260
362
|
tmp_py_file.write_text(command, encoding="utf-8")
|
|
@@ -262,54 +364,79 @@ class SSH:
|
|
|
262
364
|
assert self.sftp is not None
|
|
263
365
|
tmp_remote_path = ".tmp_pyfile.py"
|
|
264
366
|
self.sftp.put(localpath=str(tmp_py_file), remotepath=str(Path(self.remote_specs["home_dir"]).joinpath(tmp_remote_path)))
|
|
265
|
-
self.run_shell(
|
|
367
|
+
self.run_shell(
|
|
368
|
+
command=f"""{UV_RUN_CMD} python {tmp_remote_path}""",
|
|
369
|
+
verbose_output=False,
|
|
370
|
+
description=f"Creating target dir {path_rel2home}",
|
|
371
|
+
strict_stderr=True,
|
|
372
|
+
strict_return_code=True,
|
|
373
|
+
)
|
|
266
374
|
|
|
267
|
-
def copy_from_here(
|
|
268
|
-
|
|
375
|
+
def copy_from_here(
|
|
376
|
+
self, source_path: str, target_rel2home: Optional[str], compress_with_zip: bool, recursive: bool, overwrite_existing: bool
|
|
377
|
+
) -> None:
|
|
378
|
+
if self.sftp is None:
|
|
379
|
+
raise RuntimeError(f"SFTP connection not available for {self.hostname}. Cannot transfer files.")
|
|
269
380
|
source_obj = Path(source_path).expanduser().absolute()
|
|
270
|
-
if not source_obj.exists():
|
|
381
|
+
if not source_obj.exists():
|
|
382
|
+
raise RuntimeError(f"SSH Error: source `{source_obj}` does not exist!")
|
|
271
383
|
if target_rel2home is None:
|
|
272
|
-
try:
|
|
384
|
+
try:
|
|
385
|
+
target_rel2home = str(source_obj.relative_to(Path.home()))
|
|
273
386
|
except ValueError:
|
|
274
387
|
raise RuntimeError(f"If target is not specified, source must be relative to home directory, but got: {source_obj}")
|
|
275
388
|
if not compress_with_zip and source_obj.is_dir():
|
|
276
389
|
if not recursive:
|
|
277
|
-
raise RuntimeError(
|
|
390
|
+
raise RuntimeError(
|
|
391
|
+
f"SSH Error: source `{source_obj}` is a directory! Set `recursive=True` for recursive sending or `compress_with_zip=True` to zip it first."
|
|
392
|
+
)
|
|
278
393
|
file_paths_to_upload: list[Path] = [file_path for file_path in source_obj.rglob("*") if file_path.is_file()]
|
|
279
394
|
self.create_dir(path_rel2home=target_rel2home, overwrite_existing=overwrite_existing)
|
|
280
395
|
for idx, file_path in enumerate(file_paths_to_upload):
|
|
281
396
|
print(f" {idx + 1:03d}. {file_path}")
|
|
282
397
|
for file_path in file_paths_to_upload:
|
|
283
398
|
remote_file_target = Path(target_rel2home).joinpath(file_path.relative_to(source_obj))
|
|
284
|
-
self.copy_from_here(
|
|
399
|
+
self.copy_from_here(
|
|
400
|
+
source_path=str(file_path),
|
|
401
|
+
target_rel2home=str(remote_file_target),
|
|
402
|
+
compress_with_zip=False,
|
|
403
|
+
recursive=False,
|
|
404
|
+
overwrite_existing=overwrite_existing,
|
|
405
|
+
)
|
|
285
406
|
return None
|
|
286
407
|
if compress_with_zip:
|
|
287
408
|
print("🗜️ ZIPPING ...")
|
|
288
409
|
import shutil
|
|
410
|
+
|
|
289
411
|
zip_path = Path(str(source_obj) + "_archive")
|
|
290
412
|
if source_obj.is_dir():
|
|
291
413
|
shutil.make_archive(str(zip_path), "zip", source_obj)
|
|
292
414
|
else:
|
|
293
415
|
shutil.make_archive(str(zip_path), "zip", source_obj.parent, source_obj.name)
|
|
294
416
|
source_obj = Path(str(zip_path) + ".zip")
|
|
295
|
-
if not target_rel2home.endswith(".zip"):
|
|
417
|
+
if not target_rel2home.endswith(".zip"):
|
|
418
|
+
target_rel2home = target_rel2home + ".zip"
|
|
296
419
|
self.create_dir(path_rel2home=str(Path(target_rel2home).parent), overwrite_existing=overwrite_existing)
|
|
297
420
|
print(f"""📤 [SFTP UPLOAD] Sending file: {repr(source_obj)} ==> Remote Path: {target_rel2home}""")
|
|
298
421
|
try:
|
|
299
422
|
with self.tqdm_wrap(ascii=True, unit="b", unit_scale=True) as pbar:
|
|
300
423
|
if self.sftp is None: # type: ignore[unreachable]
|
|
301
424
|
raise RuntimeError(f"SFTP connection lost for {self.hostname}")
|
|
302
|
-
self.sftp.put(
|
|
425
|
+
self.sftp.put(
|
|
426
|
+
localpath=str(source_obj), remotepath=str(Path(self.remote_specs["home_dir"]).joinpath(target_rel2home)), callback=pbar.view_bar
|
|
427
|
+
)
|
|
303
428
|
except Exception:
|
|
304
429
|
if compress_with_zip and source_obj.exists() and str(source_obj).endswith("_archive.zip"):
|
|
305
430
|
source_obj.unlink()
|
|
306
431
|
raise
|
|
307
|
-
|
|
432
|
+
|
|
308
433
|
if compress_with_zip:
|
|
434
|
+
|
|
309
435
|
def unzip_archive(zip_file_path: str, overwrite_flag: bool) -> None:
|
|
310
436
|
from pathlib import Path
|
|
311
437
|
import shutil
|
|
312
438
|
import zipfile
|
|
439
|
+
|
|
313
440
|
archive_path = Path(zip_file_path).expanduser()
|
|
314
441
|
extraction_directory = archive_path.parent / archive_path.stem
|
|
315
442
|
if overwrite_flag and extraction_directory.exists():
|
|
@@ -317,38 +444,65 @@ class SSH:
|
|
|
317
444
|
with zipfile.ZipFile(archive_path, "r") as archive_handle:
|
|
318
445
|
archive_handle.extractall(extraction_directory)
|
|
319
446
|
archive_path.unlink()
|
|
320
|
-
|
|
447
|
+
|
|
448
|
+
command = lambda_to_python_script(
|
|
449
|
+
lmb=lambda: unzip_archive(
|
|
450
|
+
zip_file_path=str(Path(self.remote_specs["home_dir"]).joinpath(target_rel2home)), overwrite_flag=overwrite_existing
|
|
451
|
+
),
|
|
452
|
+
in_global=True,
|
|
453
|
+
import_module=False,
|
|
454
|
+
)
|
|
321
455
|
tmp_py_file = Path.home().joinpath(f"{DEFAULT_PICKLE_SUBDIR}/create_target_dir_{randstr()}.py")
|
|
322
456
|
tmp_py_file.parent.mkdir(parents=True, exist_ok=True)
|
|
323
457
|
tmp_py_file.write_text(command, encoding="utf-8")
|
|
324
458
|
remote_tmp_py = tmp_py_file.relative_to(Path.home()).as_posix()
|
|
325
459
|
self.copy_from_here(source_path=str(tmp_py_file), target_rel2home=None, compress_with_zip=False, recursive=False, overwrite_existing=True)
|
|
326
|
-
self.run_shell(
|
|
460
|
+
self.run_shell(
|
|
461
|
+
command=f"""{UV_RUN_CMD} python {remote_tmp_py}""",
|
|
462
|
+
verbose_output=False,
|
|
463
|
+
description=f"UNZIPPING {target_rel2home}",
|
|
464
|
+
strict_stderr=True,
|
|
465
|
+
strict_return_code=True,
|
|
466
|
+
)
|
|
327
467
|
source_obj.unlink()
|
|
328
468
|
tmp_py_file.unlink(missing_ok=True)
|
|
329
469
|
return None
|
|
330
470
|
|
|
331
471
|
def _check_remote_is_dir(self, source_path: Union[str, Path]) -> bool:
|
|
332
472
|
"""Helper to check if a remote path is a directory."""
|
|
473
|
+
|
|
333
474
|
def check_is_dir(path_to_check: str, json_output_path: str) -> bool:
|
|
334
475
|
from pathlib import Path
|
|
335
476
|
import json
|
|
477
|
+
|
|
336
478
|
is_directory = Path(path_to_check).expanduser().absolute().is_dir()
|
|
337
479
|
json_result_path = Path(json_output_path)
|
|
338
480
|
json_result_path.parent.mkdir(parents=True, exist_ok=True)
|
|
339
481
|
json_result_path.write_text(json.dumps(is_directory, indent=2), encoding="utf-8")
|
|
340
482
|
print(json_result_path.as_posix())
|
|
341
483
|
return is_directory
|
|
484
|
+
|
|
342
485
|
remote_json_output = Path.home().joinpath(f"{DEFAULT_PICKLE_SUBDIR}/return_{randstr()}.json").as_posix()
|
|
343
|
-
command = lambda_to_python_script(
|
|
344
|
-
|
|
486
|
+
command = lambda_to_python_script(
|
|
487
|
+
lmb=lambda: check_is_dir(path_to_check=str(source_path), json_output_path=remote_json_output), in_global=True, import_module=False
|
|
488
|
+
)
|
|
489
|
+
response = self.run_py(
|
|
490
|
+
python_code=command,
|
|
491
|
+
uv_with=[MACHINECONFIG_VERSION],
|
|
492
|
+
uv_project_dir=None,
|
|
493
|
+
description=f"Check if source `{source_path}` is a dir",
|
|
494
|
+
verbose_output=False,
|
|
495
|
+
strict_stderr=False,
|
|
496
|
+
strict_return_code=False,
|
|
497
|
+
)
|
|
345
498
|
remote_json_path = response.op.strip()
|
|
346
499
|
if not remote_json_path:
|
|
347
500
|
raise RuntimeError(f"Failed to check if {source_path} is directory - no response from remote")
|
|
348
|
-
|
|
501
|
+
|
|
349
502
|
local_json = Path.home().joinpath(f"{DEFAULT_PICKLE_SUBDIR}/local_{randstr()}.json")
|
|
350
503
|
self._simple_sftp_get(remote_path=remote_json_path, local_path=local_json)
|
|
351
504
|
import json
|
|
505
|
+
|
|
352
506
|
try:
|
|
353
507
|
result = json.loads(local_json.read_text(encoding="utf-8"))
|
|
354
508
|
except (json.JSONDecodeError, FileNotFoundError) as err:
|
|
@@ -361,28 +515,39 @@ class SSH:
|
|
|
361
515
|
|
|
362
516
|
def _expand_remote_path(self, source_path: Union[str, Path]) -> str:
|
|
363
517
|
"""Helper to expand a path on the remote machine."""
|
|
518
|
+
|
|
364
519
|
def expand_source(path_to_expand: str, json_output_path: str) -> str:
|
|
365
520
|
from pathlib import Path
|
|
366
521
|
import json
|
|
522
|
+
|
|
367
523
|
expanded_path_posix = Path(path_to_expand).expanduser().absolute().as_posix()
|
|
368
524
|
json_result_path = Path(json_output_path)
|
|
369
525
|
json_result_path.parent.mkdir(parents=True, exist_ok=True)
|
|
370
526
|
json_result_path.write_text(json.dumps(expanded_path_posix, indent=2), encoding="utf-8")
|
|
371
527
|
print(json_result_path.as_posix())
|
|
372
528
|
return expanded_path_posix
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
529
|
+
|
|
376
530
|
remote_json_output = Path.home().joinpath(f"{DEFAULT_PICKLE_SUBDIR}/return_{randstr()}.json").as_posix()
|
|
377
|
-
command = lambda_to_python_script(
|
|
378
|
-
|
|
531
|
+
command = lambda_to_python_script(
|
|
532
|
+
lmb=lambda: expand_source(path_to_expand=str(source_path), json_output_path=remote_json_output), in_global=True, import_module=False
|
|
533
|
+
)
|
|
534
|
+
response = self.run_py(
|
|
535
|
+
python_code=command,
|
|
536
|
+
uv_with=[MACHINECONFIG_VERSION],
|
|
537
|
+
uv_project_dir=None,
|
|
538
|
+
description="Resolving source path by expanding user",
|
|
539
|
+
verbose_output=False,
|
|
540
|
+
strict_stderr=False,
|
|
541
|
+
strict_return_code=False,
|
|
542
|
+
)
|
|
379
543
|
remote_json_path = response.op.strip()
|
|
380
544
|
if not remote_json_path:
|
|
381
545
|
raise RuntimeError(f"Could not resolve source path {source_path} - no response from remote")
|
|
382
|
-
|
|
546
|
+
|
|
383
547
|
local_json = Path.home().joinpath(f"{DEFAULT_PICKLE_SUBDIR}/local_{randstr()}.json")
|
|
384
548
|
self._simple_sftp_get(remote_path=remote_json_path, local_path=local_json)
|
|
385
549
|
import json
|
|
550
|
+
|
|
386
551
|
try:
|
|
387
552
|
result = json.loads(local_json.read_text(encoding="utf-8"))
|
|
388
553
|
except (json.JSONDecodeError, FileNotFoundError) as err:
|
|
@@ -393,45 +558,66 @@ class SSH:
|
|
|
393
558
|
assert isinstance(result, str), f"Could not resolve source path {source_path}"
|
|
394
559
|
return result
|
|
395
560
|
|
|
396
|
-
def copy_to_here(
|
|
561
|
+
def copy_to_here(
|
|
562
|
+
self,
|
|
563
|
+
source: Union[str, Path],
|
|
564
|
+
target: Optional[Union[str, Path]],
|
|
565
|
+
compress_with_zip: bool = False,
|
|
566
|
+
recursive: bool = False,
|
|
567
|
+
internal_call: bool = False,
|
|
568
|
+
) -> None:
|
|
397
569
|
if self.sftp is None:
|
|
398
570
|
raise RuntimeError(f"SFTP connection not available for {self.hostname}. Cannot transfer files.")
|
|
399
|
-
|
|
571
|
+
|
|
400
572
|
if not internal_call:
|
|
401
573
|
print(f"{'⬇️' * 5} SFTP DOWNLOADING FROM `{source}` TO `{target}`")
|
|
402
|
-
|
|
574
|
+
|
|
403
575
|
source_obj = Path(source)
|
|
404
576
|
expanded_source = self._expand_remote_path(source_path=source_obj)
|
|
405
|
-
|
|
577
|
+
|
|
406
578
|
if not compress_with_zip:
|
|
407
579
|
is_dir = self._check_remote_is_dir(source_path=expanded_source)
|
|
408
|
-
|
|
580
|
+
|
|
409
581
|
if is_dir:
|
|
410
582
|
if not recursive:
|
|
411
|
-
raise RuntimeError(
|
|
412
|
-
|
|
583
|
+
raise RuntimeError(
|
|
584
|
+
f"SSH Error: source `{source_obj}` is a directory! Set recursive=True for recursive transfer or compress_with_zip=True to zip it."
|
|
585
|
+
)
|
|
586
|
+
|
|
413
587
|
def search_files(directory_path: str, json_output_path: str) -> list[str]:
|
|
414
588
|
from pathlib import Path
|
|
415
589
|
import json
|
|
416
|
-
|
|
590
|
+
|
|
591
|
+
file_paths_list = [
|
|
592
|
+
file_path.as_posix() for file_path in Path(directory_path).expanduser().absolute().rglob("*") if file_path.is_file()
|
|
593
|
+
]
|
|
417
594
|
json_result_path = Path(json_output_path)
|
|
418
595
|
json_result_path.parent.mkdir(parents=True, exist_ok=True)
|
|
419
596
|
json_result_path.write_text(json.dumps(file_paths_list, indent=2), encoding="utf-8")
|
|
420
597
|
print(json_result_path.as_posix())
|
|
421
598
|
return file_paths_list
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
599
|
+
|
|
425
600
|
remote_json_output = Path.home().joinpath(f"{DEFAULT_PICKLE_SUBDIR}/return_{randstr()}.json").as_posix()
|
|
426
|
-
command = lambda_to_python_script(
|
|
427
|
-
|
|
601
|
+
command = lambda_to_python_script(
|
|
602
|
+
lmb=lambda: search_files(directory_path=expanded_source, json_output_path=remote_json_output), in_global=True, import_module=False
|
|
603
|
+
)
|
|
604
|
+
response = self.run_py(
|
|
605
|
+
python_code=command,
|
|
606
|
+
uv_with=[MACHINECONFIG_VERSION],
|
|
607
|
+
uv_project_dir=None,
|
|
608
|
+
description="Searching for files in source",
|
|
609
|
+
verbose_output=False,
|
|
610
|
+
strict_stderr=False,
|
|
611
|
+
strict_return_code=False,
|
|
612
|
+
)
|
|
428
613
|
remote_json_path = response.op.strip()
|
|
429
614
|
if not remote_json_path:
|
|
430
615
|
raise RuntimeError(f"Could not resolve source path {source} - no response from remote")
|
|
431
|
-
|
|
616
|
+
|
|
432
617
|
local_json = Path.home().joinpath(f"{DEFAULT_PICKLE_SUBDIR}/local_{randstr()}.json")
|
|
433
618
|
self._simple_sftp_get(remote_path=remote_json_path, local_path=local_json)
|
|
434
619
|
import json
|
|
620
|
+
|
|
435
621
|
try:
|
|
436
622
|
source_list_str = json.loads(local_json.read_text(encoding="utf-8"))
|
|
437
623
|
except (json.JSONDecodeError, FileNotFoundError) as err:
|
|
@@ -441,11 +627,13 @@ class SSH:
|
|
|
441
627
|
local_json.unlink()
|
|
442
628
|
assert isinstance(source_list_str, list), f"Could not resolve source path {source}"
|
|
443
629
|
file_paths_to_download = [Path(file_path_str) for file_path_str in source_list_str]
|
|
444
|
-
|
|
630
|
+
|
|
445
631
|
if target is None:
|
|
632
|
+
|
|
446
633
|
def collapse_to_home_dir(absolute_path: str, json_output_path: str) -> str:
|
|
447
634
|
from pathlib import Path
|
|
448
635
|
import json
|
|
636
|
+
|
|
449
637
|
source_absolute_path = Path(absolute_path).expanduser().absolute()
|
|
450
638
|
try:
|
|
451
639
|
relative_to_home = source_absolute_path.relative_to(Path.home())
|
|
@@ -457,19 +645,30 @@ class SSH:
|
|
|
457
645
|
return collapsed_path_posix
|
|
458
646
|
except ValueError:
|
|
459
647
|
raise RuntimeError(f"Source path must be relative to home directory: {source_absolute_path}")
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
648
|
+
|
|
463
649
|
remote_json_output = Path.home().joinpath(f"{DEFAULT_PICKLE_SUBDIR}/return_{randstr()}.json").as_posix()
|
|
464
|
-
command = lambda_to_python_script(
|
|
465
|
-
|
|
650
|
+
command = lambda_to_python_script(
|
|
651
|
+
lmb=lambda: collapse_to_home_dir(absolute_path=expanded_source, json_output_path=remote_json_output),
|
|
652
|
+
in_global=True,
|
|
653
|
+
import_module=False,
|
|
654
|
+
)
|
|
655
|
+
response = self.run_py(
|
|
656
|
+
python_code=command,
|
|
657
|
+
uv_with=[MACHINECONFIG_VERSION],
|
|
658
|
+
uv_project_dir=None,
|
|
659
|
+
description="Finding default target via relative source path",
|
|
660
|
+
verbose_output=False,
|
|
661
|
+
strict_stderr=False,
|
|
662
|
+
strict_return_code=False,
|
|
663
|
+
)
|
|
466
664
|
remote_json_path_dir = response.op.strip()
|
|
467
665
|
if not remote_json_path_dir:
|
|
468
666
|
raise RuntimeError("Could not resolve target path - no response from remote")
|
|
469
|
-
|
|
667
|
+
|
|
470
668
|
local_json_dir = Path.home().joinpath(f"{DEFAULT_PICKLE_SUBDIR}/local_{randstr()}.json")
|
|
471
669
|
self._simple_sftp_get(remote_path=remote_json_path_dir, local_path=local_json_dir)
|
|
472
670
|
import json
|
|
671
|
+
|
|
473
672
|
try:
|
|
474
673
|
target_dir_str = json.loads(local_json_dir.read_text(encoding="utf-8"))
|
|
475
674
|
except (json.JSONDecodeError, FileNotFoundError) as err:
|
|
@@ -479,24 +678,26 @@ class SSH:
|
|
|
479
678
|
local_json_dir.unlink()
|
|
480
679
|
assert isinstance(target_dir_str, str), "Could not resolve target path"
|
|
481
680
|
target = Path(target_dir_str)
|
|
482
|
-
|
|
681
|
+
|
|
483
682
|
target_dir = Path(target).expanduser().absolute()
|
|
484
|
-
|
|
683
|
+
|
|
485
684
|
for idx, file_path in enumerate(file_paths_to_download):
|
|
486
685
|
print(f" {idx + 1:03d}. {file_path}")
|
|
487
|
-
|
|
686
|
+
|
|
488
687
|
for file_path in file_paths_to_download:
|
|
489
688
|
local_file_target = target_dir.joinpath(Path(file_path).relative_to(expanded_source))
|
|
490
689
|
self.copy_to_here(source=file_path, target=local_file_target, compress_with_zip=False, recursive=False, internal_call=True)
|
|
491
|
-
|
|
690
|
+
|
|
492
691
|
return None
|
|
493
|
-
|
|
692
|
+
|
|
494
693
|
if compress_with_zip:
|
|
495
694
|
print("🗜️ ZIPPING ...")
|
|
695
|
+
|
|
496
696
|
def zip_source(path_to_zip: str, json_output_path: str) -> str:
|
|
497
697
|
from pathlib import Path
|
|
498
698
|
import shutil
|
|
499
699
|
import json
|
|
700
|
+
|
|
500
701
|
source_to_compress = Path(path_to_zip).expanduser().absolute()
|
|
501
702
|
archive_base_path = source_to_compress.parent / (source_to_compress.name + "_archive")
|
|
502
703
|
if source_to_compress.is_dir():
|
|
@@ -509,19 +710,28 @@ class SSH:
|
|
|
509
710
|
json_result_path.write_text(json.dumps(zip_file_path, indent=2), encoding="utf-8")
|
|
510
711
|
print(json_result_path.as_posix())
|
|
511
712
|
return zip_file_path
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
|
|
713
|
+
|
|
515
714
|
remote_json_output = Path.home().joinpath(f"{DEFAULT_PICKLE_SUBDIR}/return_{randstr()}.json").as_posix()
|
|
516
|
-
command = lambda_to_python_script(
|
|
517
|
-
|
|
715
|
+
command = lambda_to_python_script(
|
|
716
|
+
lmb=lambda: zip_source(path_to_zip=expanded_source, json_output_path=remote_json_output), in_global=True, import_module=False
|
|
717
|
+
)
|
|
718
|
+
response = self.run_py(
|
|
719
|
+
python_code=command,
|
|
720
|
+
uv_with=[MACHINECONFIG_VERSION],
|
|
721
|
+
uv_project_dir=None,
|
|
722
|
+
description=f"Zipping source file {source}",
|
|
723
|
+
verbose_output=False,
|
|
724
|
+
strict_stderr=False,
|
|
725
|
+
strict_return_code=False,
|
|
726
|
+
)
|
|
518
727
|
remote_json_path = response.op.strip()
|
|
519
728
|
if not remote_json_path:
|
|
520
729
|
raise RuntimeError(f"Could not zip {source} - no response from remote")
|
|
521
|
-
|
|
730
|
+
|
|
522
731
|
local_json = Path.home().joinpath(f"{DEFAULT_PICKLE_SUBDIR}/local_{randstr()}.json")
|
|
523
732
|
self._simple_sftp_get(remote_path=remote_json_path, local_path=local_json)
|
|
524
733
|
import json
|
|
734
|
+
|
|
525
735
|
try:
|
|
526
736
|
zipped_path = json.loads(local_json.read_text(encoding="utf-8"))
|
|
527
737
|
except (json.JSONDecodeError, FileNotFoundError) as err:
|
|
@@ -532,11 +742,13 @@ class SSH:
|
|
|
532
742
|
assert isinstance(zipped_path, str), f"Could not zip {source}"
|
|
533
743
|
source_obj = Path(zipped_path)
|
|
534
744
|
expanded_source = zipped_path
|
|
535
|
-
|
|
745
|
+
|
|
536
746
|
if target is None:
|
|
747
|
+
|
|
537
748
|
def collapse_to_home(absolute_path: str, json_output_path: str) -> str:
|
|
538
749
|
from pathlib import Path
|
|
539
750
|
import json
|
|
751
|
+
|
|
540
752
|
source_absolute_path = Path(absolute_path).expanduser().absolute()
|
|
541
753
|
try:
|
|
542
754
|
relative_to_home = source_absolute_path.relative_to(Path.home())
|
|
@@ -548,19 +760,28 @@ class SSH:
|
|
|
548
760
|
return collapsed_path_posix
|
|
549
761
|
except ValueError:
|
|
550
762
|
raise RuntimeError(f"Source path must be relative to home directory: {source_absolute_path}")
|
|
551
|
-
|
|
552
|
-
|
|
553
|
-
|
|
763
|
+
|
|
554
764
|
remote_json_output = Path.home().joinpath(f"{DEFAULT_PICKLE_SUBDIR}/return_{randstr()}.json").as_posix()
|
|
555
|
-
command = lambda_to_python_script(
|
|
556
|
-
|
|
765
|
+
command = lambda_to_python_script(
|
|
766
|
+
lmb=lambda: collapse_to_home(absolute_path=expanded_source, json_output_path=remote_json_output), in_global=True, import_module=False
|
|
767
|
+
)
|
|
768
|
+
response = self.run_py(
|
|
769
|
+
python_code=command,
|
|
770
|
+
uv_with=[MACHINECONFIG_VERSION],
|
|
771
|
+
uv_project_dir=None,
|
|
772
|
+
description="Finding default target via relative source path",
|
|
773
|
+
verbose_output=False,
|
|
774
|
+
strict_stderr=False,
|
|
775
|
+
strict_return_code=False,
|
|
776
|
+
)
|
|
557
777
|
remote_json_path = response.op.strip()
|
|
558
778
|
if not remote_json_path:
|
|
559
779
|
raise RuntimeError("Could not resolve target path - no response from remote")
|
|
560
|
-
|
|
780
|
+
|
|
561
781
|
local_json = Path.home().joinpath(f"{DEFAULT_PICKLE_SUBDIR}/local_{randstr()}.json")
|
|
562
782
|
self._simple_sftp_get(remote_path=remote_json_path, local_path=local_json)
|
|
563
783
|
import json
|
|
784
|
+
|
|
564
785
|
try:
|
|
565
786
|
target_str = json.loads(local_json.read_text(encoding="utf-8"))
|
|
566
787
|
except (json.JSONDecodeError, FileNotFoundError) as err:
|
|
@@ -571,13 +792,13 @@ class SSH:
|
|
|
571
792
|
assert isinstance(target_str, str), "Could not resolve target path"
|
|
572
793
|
target = Path(target_str)
|
|
573
794
|
assert str(target).startswith("~"), f"If target is not specified, source must be relative to home.\n{target=}"
|
|
574
|
-
|
|
795
|
+
|
|
575
796
|
target_obj = Path(target).expanduser().absolute()
|
|
576
797
|
target_obj.parent.mkdir(parents=True, exist_ok=True)
|
|
577
|
-
|
|
798
|
+
|
|
578
799
|
if compress_with_zip and target_obj.suffix != ".zip":
|
|
579
800
|
target_obj = target_obj.with_suffix(target_obj.suffix + ".zip")
|
|
580
|
-
|
|
801
|
+
|
|
581
802
|
print(f"""📥 [DOWNLOAD] Receiving: {expanded_source} ==> Local Path: {target_obj}""")
|
|
582
803
|
try:
|
|
583
804
|
with self.tqdm_wrap(ascii=True, unit="b", unit_scale=True) as pbar:
|
|
@@ -588,31 +809,41 @@ class SSH:
|
|
|
588
809
|
if target_obj.exists():
|
|
589
810
|
target_obj.unlink()
|
|
590
811
|
raise
|
|
591
|
-
|
|
812
|
+
|
|
592
813
|
if compress_with_zip:
|
|
593
814
|
import zipfile
|
|
815
|
+
|
|
594
816
|
extract_to = target_obj.parent / target_obj.stem
|
|
595
817
|
with zipfile.ZipFile(target_obj, "r") as zip_ref:
|
|
596
818
|
zip_ref.extractall(extract_to)
|
|
597
819
|
target_obj.unlink()
|
|
598
820
|
target_obj = extract_to
|
|
599
|
-
|
|
821
|
+
|
|
600
822
|
def delete_temp_zip(path_to_delete: str) -> None:
|
|
601
823
|
from pathlib import Path
|
|
602
824
|
import shutil
|
|
825
|
+
|
|
603
826
|
file_or_dir_path = Path(path_to_delete)
|
|
604
827
|
if file_or_dir_path.exists():
|
|
605
828
|
if file_or_dir_path.is_dir():
|
|
606
829
|
shutil.rmtree(file_or_dir_path)
|
|
607
830
|
else:
|
|
608
831
|
file_or_dir_path.unlink()
|
|
609
|
-
|
|
610
|
-
|
|
832
|
+
|
|
611
833
|
command = lambda_to_python_script(lmb=lambda: delete_temp_zip(path_to_delete=expanded_source), in_global=True, import_module=False)
|
|
612
|
-
self.run_py(
|
|
613
|
-
|
|
834
|
+
self.run_py(
|
|
835
|
+
python_code=command,
|
|
836
|
+
uv_with=[MACHINECONFIG_VERSION],
|
|
837
|
+
uv_project_dir=None,
|
|
838
|
+
description="Cleaning temp zip files @ remote.",
|
|
839
|
+
verbose_output=False,
|
|
840
|
+
strict_stderr=True,
|
|
841
|
+
strict_return_code=True,
|
|
842
|
+
)
|
|
843
|
+
|
|
614
844
|
print("\n")
|
|
615
845
|
return None
|
|
616
846
|
|
|
847
|
+
|
|
617
848
|
if __name__ == "__main__":
|
|
618
849
|
ssh = SSH(host="p51s", username=None, hostname=None, ssh_key_path=None, password=None, port=22, enable_compression=False)
|