machineconfig 3.99__py3-none-any.whl → 7.66__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of machineconfig might be problematic. Click here for more details.
- machineconfig/__init__.py +0 -28
- machineconfig/cluster/remote/distribute.py +0 -1
- machineconfig/cluster/remote/file_manager.py +0 -2
- machineconfig/cluster/remote/script_execution.py +1 -2
- machineconfig/cluster/sessions_managers/{enhanced_command_runner.py → helpers/enhanced_command_runner.py} +4 -6
- machineconfig/cluster/sessions_managers/helpers/load_balancer_helper.py +145 -0
- machineconfig/cluster/sessions_managers/utils/load_balancer.py +53 -0
- machineconfig/cluster/sessions_managers/utils/maker.py +69 -0
- machineconfig/cluster/sessions_managers/wt_local.py +128 -330
- machineconfig/cluster/sessions_managers/wt_local_manager.py +53 -187
- machineconfig/cluster/sessions_managers/wt_remote.py +51 -43
- machineconfig/cluster/sessions_managers/wt_remote_manager.py +49 -197
- machineconfig/cluster/sessions_managers/wt_utils/layout_generator.py +6 -19
- machineconfig/cluster/sessions_managers/wt_utils/manager_persistence.py +52 -0
- machineconfig/cluster/sessions_managers/wt_utils/monitoring_helpers.py +50 -0
- machineconfig/cluster/sessions_managers/wt_utils/status_reporter.py +4 -2
- machineconfig/cluster/sessions_managers/wt_utils/status_reporting.py +76 -0
- machineconfig/cluster/sessions_managers/wt_utils/wt_helpers.py +199 -0
- machineconfig/cluster/sessions_managers/zellij_local.py +81 -375
- machineconfig/cluster/sessions_managers/zellij_local_manager.py +22 -172
- machineconfig/cluster/sessions_managers/zellij_remote.py +40 -41
- machineconfig/cluster/sessions_managers/zellij_remote_manager.py +13 -10
- machineconfig/cluster/sessions_managers/zellij_utils/example_usage.py +4 -8
- machineconfig/cluster/sessions_managers/zellij_utils/layout_generator.py +5 -20
- machineconfig/cluster/sessions_managers/zellij_utils/process_monitor.py +3 -9
- machineconfig/cluster/sessions_managers/zellij_utils/status_reporter.py +3 -1
- machineconfig/cluster/sessions_managers/zellij_utils/zellij_local_helper.py +298 -0
- machineconfig/cluster/sessions_managers/zellij_utils/zellij_local_helper_restart.py +77 -0
- machineconfig/cluster/sessions_managers/zellij_utils/zellij_local_helper_with_panes.py +228 -0
- machineconfig/cluster/sessions_managers/zellij_utils/zellij_local_manager_helper.py +165 -0
- machineconfig/jobs/{python → installer}/check_installations.py +2 -16
- machineconfig/jobs/installer/custom/boxes.py +61 -0
- machineconfig/jobs/installer/custom/gh.py +69 -53
- machineconfig/jobs/installer/custom/hx.py +77 -20
- machineconfig/jobs/installer/custom_dev/alacritty.py +45 -30
- machineconfig/jobs/installer/custom_dev/brave.py +43 -35
- machineconfig/jobs/installer/custom_dev/bypass_paywall.py +31 -20
- machineconfig/jobs/installer/custom_dev/cloudflare_warp_cli.py +23 -0
- machineconfig/jobs/installer/custom_dev/code.py +33 -21
- machineconfig/jobs/installer/custom_dev/dubdb_adbc.py +30 -0
- machineconfig/jobs/installer/custom_dev/espanso.py +64 -41
- machineconfig/jobs/installer/custom_dev/goes.py +41 -36
- machineconfig/jobs/installer/custom_dev/lvim.py +49 -33
- machineconfig/jobs/installer/custom_dev/nerdfont.py +71 -47
- machineconfig/jobs/installer/custom_dev/nerfont_windows_helper.py +32 -26
- machineconfig/jobs/installer/custom_dev/redis.py +51 -33
- machineconfig/jobs/installer/custom_dev/sysabc.py +119 -0
- machineconfig/jobs/installer/custom_dev/wezterm.py +55 -39
- machineconfig/jobs/installer/custom_dev/winget.py +1 -0
- machineconfig/jobs/installer/installer_data.json +3406 -0
- machineconfig/jobs/installer/linux_scripts/brave.sh +4 -14
- machineconfig/jobs/installer/linux_scripts/{warp-cli.sh → cloudflare_warp_cli.sh} +5 -17
- machineconfig/jobs/installer/linux_scripts/docker.sh +5 -17
- machineconfig/jobs/installer/linux_scripts/docker_start.sh +6 -14
- machineconfig/jobs/installer/linux_scripts/edge.sh +3 -11
- machineconfig/jobs/{linux/msc → installer/linux_scripts}/lid.sh +2 -8
- machineconfig/jobs/installer/linux_scripts/nerdfont.sh +5 -17
- machineconfig/jobs/{linux/msc → installer/linux_scripts}/network.sh +2 -8
- machineconfig/jobs/installer/linux_scripts/ngrok.sh +6 -0
- machineconfig/jobs/installer/linux_scripts/q.sh +9 -0
- machineconfig/jobs/installer/linux_scripts/redis.sh +6 -17
- machineconfig/jobs/installer/linux_scripts/vscode.sh +5 -17
- machineconfig/jobs/installer/linux_scripts/wezterm.sh +4 -12
- machineconfig/jobs/installer/package_groups.py +255 -0
- machineconfig/logger.py +0 -1
- machineconfig/profile/backup.toml +49 -0
- machineconfig/profile/bash_shell_profiles.md +11 -0
- machineconfig/profile/create_helper.py +74 -0
- machineconfig/profile/create_links.py +288 -0
- machineconfig/profile/create_links_export.py +100 -0
- machineconfig/profile/create_shell_profile.py +136 -0
- machineconfig/profile/mapper.toml +258 -0
- machineconfig/scripts/Restore-ThunderbirdProfile.ps1 +92 -0
- machineconfig/scripts/__init__.py +0 -4
- machineconfig/scripts/linux/{share_cloud.sh → other/share_cloud.sh} +14 -25
- machineconfig/scripts/linux/wrap_mcfg +47 -0
- machineconfig/scripts/nu/wrap_mcfg.nu +69 -0
- machineconfig/scripts/python/agents.py +198 -0
- machineconfig/scripts/python/ai/command_runner/command_runner.sh +9 -0
- machineconfig/scripts/python/ai/command_runner/prompt.txt +9 -0
- machineconfig/scripts/python/ai/generate_files.py +307 -42
- machineconfig/scripts/python/ai/initai.py +3 -28
- machineconfig/scripts/python/ai/scripts/lint_and_type_check.ps1 +17 -18
- machineconfig/scripts/python/ai/scripts/lint_and_type_check.sh +17 -18
- machineconfig/scripts/python/ai/solutions/_shared.py +9 -1
- machineconfig/scripts/python/ai/solutions/copilot/instructions/python/dev.instructions.md +1 -1
- machineconfig/scripts/python/ai/solutions/copilot/prompts/pyright_fix.md +16 -0
- machineconfig/scripts/python/ai/solutions/generic.py +27 -4
- machineconfig/scripts/python/ai/vscode_tasks.py +37 -0
- machineconfig/scripts/python/cloud.py +29 -0
- machineconfig/scripts/python/croshell.py +117 -181
- machineconfig/scripts/python/define.py +31 -0
- machineconfig/scripts/python/devops.py +44 -124
- machineconfig/scripts/python/devops_navigator.py +10 -0
- machineconfig/scripts/python/env_manager/__init__.py +1 -0
- machineconfig/scripts/python/env_manager/path_manager_backend.py +47 -0
- machineconfig/scripts/python/env_manager/path_manager_tui.py +228 -0
- machineconfig/scripts/python/explore.py +49 -0
- machineconfig/scripts/python/fire_jobs.py +106 -244
- machineconfig/scripts/python/ftpx.py +125 -68
- machineconfig/scripts/python/helpers_agents/agentic_frameworks/fire_crush.json +14 -0
- machineconfig/scripts/python/helpers_agents/agentic_frameworks/fire_crush.py +37 -0
- machineconfig/scripts/python/helpers_agents/agentic_frameworks/fire_cursor_agents.py +22 -0
- machineconfig/scripts/python/helpers_agents/agentic_frameworks/fire_gemini.py +42 -0
- machineconfig/scripts/python/helpers_agents/agentic_frameworks/fire_qwen.py +30 -0
- machineconfig/scripts/python/helpers_agents/fire_agents_help_launch.py +110 -0
- machineconfig/scripts/python/helpers_agents/fire_agents_helper_types.py +34 -0
- machineconfig/scripts/python/helpers_agents/fire_agents_load_balancer.py +22 -0
- machineconfig/scripts/python/helpers_agents/templates/prompt.txt +6 -0
- machineconfig/scripts/python/helpers_agents/templates/template.ps1 +14 -0
- machineconfig/scripts/python/helpers_agents/templates/template.sh +24 -0
- machineconfig/scripts/python/{cloud_copy.py → helpers_cloud/cloud_copy.py} +30 -23
- machineconfig/scripts/python/{cloud_mount.py → helpers_cloud/cloud_mount.py} +11 -19
- machineconfig/scripts/python/{cloud_sync.py → helpers_cloud/cloud_sync.py} +12 -18
- machineconfig/scripts/python/{helpers → helpers_cloud}/helpers2.py +3 -3
- machineconfig/scripts/python/helpers_croshell/crosh.py +39 -0
- machineconfig/scripts/python/{start_slidev.py → helpers_croshell/start_slidev.py} +17 -7
- machineconfig/scripts/python/helpers_devops/cli_config.py +95 -0
- machineconfig/scripts/python/helpers_devops/cli_config_dotfile.py +89 -0
- machineconfig/scripts/python/helpers_devops/cli_data.py +25 -0
- machineconfig/scripts/python/helpers_devops/cli_nw.py +134 -0
- machineconfig/scripts/python/helpers_devops/cli_repos.py +182 -0
- machineconfig/scripts/python/helpers_devops/cli_self.py +134 -0
- machineconfig/scripts/python/helpers_devops/cli_share_file.py +137 -0
- machineconfig/scripts/python/helpers_devops/cli_share_server.py +141 -0
- machineconfig/scripts/python/helpers_devops/cli_terminal.py +156 -0
- machineconfig/scripts/python/helpers_devops/cli_utils.py +96 -0
- machineconfig/scripts/python/{devops_backup_retrieve.py → helpers_devops/devops_backup_retrieve.py} +7 -10
- machineconfig/scripts/python/helpers_devops/devops_status.py +511 -0
- machineconfig/scripts/python/helpers_devops/devops_update_repos.py +269 -0
- machineconfig/scripts/python/helpers_devops/themes/choose_pwsh_theme.ps1 +81 -0
- machineconfig/scripts/python/helpers_devops/themes/choose_starship_theme.bash +3 -0
- machineconfig/scripts/python/{choose_wezterm_theme.py → helpers_devops/themes/choose_wezterm_theme.py} +2 -2
- machineconfig/scripts/python/helpers_fire_command/__init__.py +0 -0
- machineconfig/scripts/python/{helpers/helpers4.py → helpers_fire_command/file_wrangler.py} +57 -87
- machineconfig/scripts/python/helpers_fire_command/fire_jobs_args_helper.py +145 -0
- machineconfig/scripts/python/helpers_fire_command/fire_jobs_route_helper.py +110 -0
- machineconfig/scripts/python/helpers_fire_command/fire_jobs_streamlit_helper.py +0 -0
- machineconfig/scripts/python/helpers_msearch/__init__.py +5 -0
- machineconfig/scripts/{linux → python/helpers_msearch/scripts_linux}/fzfag +1 -1
- machineconfig/scripts/{linux → python/helpers_msearch/scripts_linux}/fzfg +1 -1
- machineconfig/scripts/{linux → python/helpers_msearch/scripts_linux}/fzfrga +1 -1
- machineconfig/scripts/python/helpers_navigator/__init__.py +20 -0
- machineconfig/scripts/python/helpers_navigator/command_builder.py +111 -0
- machineconfig/scripts/python/helpers_navigator/command_detail.py +44 -0
- machineconfig/scripts/python/helpers_navigator/command_tree.py +588 -0
- machineconfig/scripts/python/helpers_navigator/data_models.py +28 -0
- machineconfig/scripts/python/helpers_navigator/main_app.py +272 -0
- machineconfig/scripts/python/helpers_navigator/search_bar.py +15 -0
- machineconfig/scripts/python/helpers_repos/action.py +209 -0
- machineconfig/scripts/python/helpers_repos/action_helper.py +150 -0
- machineconfig/scripts/python/{repos_helper_clone.py → helpers_repos/clone.py} +6 -7
- machineconfig/scripts/python/helpers_repos/cloud_repo_sync.py +218 -0
- machineconfig/scripts/python/helpers_repos/count_lines.py +348 -0
- machineconfig/scripts/python/helpers_repos/count_lines_frontend.py +17 -0
- machineconfig/scripts/python/helpers_repos/entrypoint.py +77 -0
- machineconfig/scripts/python/helpers_repos/grource.py +340 -0
- machineconfig/scripts/python/{repos_helper_record.py → helpers_repos/record.py} +7 -4
- machineconfig/scripts/python/helpers_repos/sync.py +66 -0
- machineconfig/scripts/python/{repos_helper_update.py → helpers_repos/update.py} +3 -3
- machineconfig/scripts/python/helpers_sessions/__init__.py +0 -0
- machineconfig/scripts/python/helpers_sessions/sessions_multiprocess.py +58 -0
- machineconfig/scripts/python/helpers_utils/download.py +152 -0
- machineconfig/scripts/python/helpers_utils/path.py +108 -0
- machineconfig/scripts/python/interactive.py +79 -160
- machineconfig/scripts/python/machineconfig.py +63 -0
- machineconfig/scripts/python/msearch.py +21 -0
- machineconfig/scripts/python/nw/__init__.py +0 -0
- machineconfig/scripts/python/{devops_add_identity.py → nw/devops_add_identity.py} +1 -3
- machineconfig/scripts/python/{devops_add_ssh_key.py → nw/devops_add_ssh_key.py} +74 -44
- machineconfig/scripts/{linux → python/nw}/mount_nfs +1 -1
- machineconfig/scripts/python/{mount_nfs.py → nw/mount_nfs.py} +19 -16
- machineconfig/scripts/{linux → python/nw}/mount_nw_drive +1 -2
- machineconfig/scripts/python/{mount_ssh.py → nw/mount_ssh.py} +7 -8
- machineconfig/scripts/python/{onetimeshare.py → nw/onetimeshare.py} +0 -1
- machineconfig/scripts/python/nw/ssh_debug_linux.py +391 -0
- machineconfig/scripts/python/nw/ssh_debug_windows.py +338 -0
- machineconfig/scripts/python/{wifi_conn.py → nw/wifi_conn.py} +1 -53
- machineconfig/scripts/python/{wsl_windows_transfer.py → nw/wsl_windows_transfer.py} +6 -5
- machineconfig/scripts/python/sessions.py +167 -0
- machineconfig/scripts/python/terminal.py +127 -0
- machineconfig/scripts/python/utils.py +66 -0
- machineconfig/scripts/windows/{mount_nfs.ps1 → mounts/mount_nfs.ps1} +1 -3
- machineconfig/scripts/windows/{mount_ssh.ps1 → mounts/mount_ssh.ps1} +1 -1
- machineconfig/scripts/windows/{share_smb.ps1 → mounts/share_smb.ps1} +0 -6
- machineconfig/scripts/windows/wrap_mcfg.ps1 +60 -0
- machineconfig/settings/broot/br.sh +0 -4
- machineconfig/settings/broot/conf.toml +1 -1
- machineconfig/settings/helix/config.toml +16 -0
- machineconfig/settings/helix/languages.toml +13 -4
- machineconfig/settings/helix/yazi-picker.sh +12 -0
- machineconfig/settings/lf/linux/exe/lfcd.sh +1 -0
- machineconfig/settings/lf/linux/exe/previewer.sh +9 -3
- machineconfig/settings/lf/linux/lfrc +10 -12
- machineconfig/settings/lf/windows/fzf_edit.ps1 +2 -2
- machineconfig/settings/lf/windows/lfrc +18 -38
- machineconfig/settings/lf/windows/mkfile.ps1 +1 -1
- machineconfig/settings/linters/.ruff.toml +1 -1
- machineconfig/settings/lvim/windows/archive/config_additional.lua +0 -6
- machineconfig/settings/marimo/marimo.toml +80 -0
- machineconfig/settings/marimo/snippets/globalize.py +34 -0
- machineconfig/settings/pistol/pistol.conf +1 -1
- machineconfig/settings/shells/bash/init.sh +55 -31
- machineconfig/settings/shells/nushell/config.nu +1 -34
- machineconfig/settings/shells/nushell/init.nu +127 -0
- machineconfig/settings/shells/pwsh/init.ps1 +60 -43
- machineconfig/settings/shells/starship/starship.toml +16 -0
- machineconfig/settings/shells/wezterm/wezterm.lua +2 -0
- machineconfig/settings/shells/wt/settings.json +32 -17
- machineconfig/settings/shells/zsh/init.sh +89 -0
- machineconfig/settings/svim/linux/init.toml +0 -4
- machineconfig/settings/svim/windows/init.toml +0 -3
- machineconfig/settings/yazi/init.lua +57 -0
- machineconfig/settings/yazi/keymap_linux.toml +79 -0
- machineconfig/settings/yazi/keymap_windows.toml +78 -0
- machineconfig/settings/yazi/shell/yazi_cd.ps1 +33 -0
- machineconfig/settings/yazi/shell/yazi_cd.sh +8 -0
- machineconfig/settings/yazi/yazi.toml +13 -0
- machineconfig/setup_linux/__init__.py +10 -0
- machineconfig/setup_linux/apps_desktop.sh +89 -0
- machineconfig/setup_linux/apps_gui.sh +64 -0
- machineconfig/setup_linux/{nix → others}/cli_installation.sh +9 -29
- machineconfig/setup_linux/ssh/openssh_all.sh +25 -0
- machineconfig/setup_linux/ssh/openssh_wsl.sh +38 -0
- machineconfig/setup_linux/uv.sh +15 -0
- machineconfig/setup_linux/web_shortcuts/interactive.sh +26 -6
- machineconfig/setup_mac/__init__.py +16 -0
- machineconfig/setup_mac/apps_gui.sh +248 -0
- machineconfig/setup_mac/ssh/openssh_setup.sh +114 -0
- machineconfig/setup_mac/uv.sh +36 -0
- machineconfig/setup_windows/__init__.py +8 -0
- machineconfig/setup_windows/others/power_options.ps1 +7 -0
- machineconfig/setup_windows/ssh/add-sshkey.ps1 +29 -0
- machineconfig/setup_windows/ssh/add_identity.ps1 +11 -0
- machineconfig/setup_windows/ssh/openssh-server.ps1 +37 -0
- machineconfig/setup_windows/uv.ps1 +10 -0
- machineconfig/setup_windows/web_shortcuts/interactive.ps1 +27 -9
- machineconfig/setup_windows/web_shortcuts/quick_init.ps1 +16 -0
- machineconfig/setup_windows/wt_and_pwsh/set_wt_settings.py +37 -23
- machineconfig/utils/accessories.py +7 -5
- machineconfig/utils/cloud/onedrive/README.md +139 -0
- machineconfig/utils/code.py +140 -93
- machineconfig/utils/files/art/fat_croco.txt +10 -0
- machineconfig/utils/files/art/halfwit_croco.txt +9 -0
- machineconfig/utils/files/art/happy_croco.txt +22 -0
- machineconfig/utils/files/art/water_croco.txt +11 -0
- machineconfig/utils/files/ascii_art.py +118 -0
- machineconfig/utils/files/dbms.py +257 -0
- machineconfig/utils/files/headers.py +68 -0
- machineconfig/utils/files/ouch/__init__.py +0 -0
- machineconfig/utils/files/ouch/decompress.py +45 -0
- machineconfig/utils/files/read.py +95 -0
- machineconfig/utils/installer_utils/github_release_bulk.py +2 -12
- machineconfig/utils/installer_utils/installer_class.py +68 -126
- machineconfig/utils/installer_utils/installer_cli.py +181 -0
- machineconfig/utils/installer_utils/{installer_abc.py → installer_locator_utils.py} +38 -85
- machineconfig/utils/{installer.py → installer_utils/installer_runner.py} +69 -69
- machineconfig/utils/io.py +77 -23
- machineconfig/utils/links.py +309 -100
- machineconfig/utils/meta.py +255 -0
- machineconfig/utils/notifications.py +1 -1
- machineconfig/utils/options.py +10 -25
- machineconfig/utils/path_extended.py +94 -104
- machineconfig/utils/path_helper.py +75 -22
- machineconfig/utils/procs.py +50 -74
- machineconfig/utils/scheduler.py +94 -97
- machineconfig/utils/scheduling.py +0 -3
- machineconfig/utils/schemas/fire_agents/fire_agents_input.py +5 -17
- machineconfig/utils/schemas/installer/installer_types.py +0 -1
- machineconfig/utils/schemas/layouts/layout_types.py +2 -1
- machineconfig/utils/source_of_truth.py +3 -6
- machineconfig/utils/ssh.py +742 -254
- machineconfig/utils/ssh_utils/utils.py +0 -0
- machineconfig/utils/terminal.py +3 -140
- machineconfig/utils/tst.py +20 -0
- machineconfig/utils/upgrade_packages.py +109 -28
- machineconfig/utils/ve.py +13 -5
- machineconfig-7.66.dist-info/METADATA +124 -0
- machineconfig-7.66.dist-info/RECORD +451 -0
- machineconfig-7.66.dist-info/entry_points.txt +15 -0
- machineconfig/cluster/templates/utils.py +0 -51
- machineconfig/jobs/installer/linux_scripts/pgsql.sh +0 -49
- machineconfig/jobs/installer/linux_scripts/timescaledb.sh +0 -85
- machineconfig/jobs/installer/packages_custom_dev.json +0 -226
- machineconfig/jobs/installer/packages_custom_essential.json +0 -39
- machineconfig/jobs/installer/packages_github_dev.json +0 -1110
- machineconfig/jobs/installer/packages_github_essential.json +0 -804
- machineconfig/jobs/linux/msc/cli_agents.sh +0 -37
- machineconfig/jobs/python/create_bootable_media.py +0 -16
- machineconfig/jobs/python/python_cargo_build_share.py +0 -59
- machineconfig/jobs/python/python_ve_symlink.py +0 -29
- machineconfig/jobs/python/tasks.py +0 -3
- machineconfig/jobs/python/vscode/api.py +0 -49
- machineconfig/jobs/python/vscode/sync_code.py +0 -58
- machineconfig/jobs/windows/archive/archive_pygraphviz.ps1 +0 -14
- machineconfig/jobs/windows/start_terminal.ps1 +0 -6
- machineconfig/jobs/windows/startup_file.cmd +0 -2
- machineconfig/profile/create.py +0 -170
- machineconfig/profile/shell.py +0 -176
- machineconfig/scripts/cloud/init.sh +0 -119
- machineconfig/scripts/linux/choose_wezterm_theme +0 -3
- machineconfig/scripts/linux/cloud_copy +0 -2
- machineconfig/scripts/linux/cloud_mount +0 -2
- machineconfig/scripts/linux/cloud_repo_sync +0 -2
- machineconfig/scripts/linux/cloud_sync +0 -2
- machineconfig/scripts/linux/croshell +0 -3
- machineconfig/scripts/linux/devops +0 -2
- machineconfig/scripts/linux/fire +0 -2
- machineconfig/scripts/linux/fire_agents +0 -2
- machineconfig/scripts/linux/ftpx +0 -2
- machineconfig/scripts/linux/fzf2g +0 -21
- machineconfig/scripts/linux/fzffg +0 -25
- machineconfig/scripts/linux/gh_models +0 -2
- machineconfig/scripts/linux/initai +0 -2
- machineconfig/scripts/linux/kill_process +0 -2
- machineconfig/scripts/linux/programs +0 -21
- machineconfig/scripts/linux/repos +0 -2
- machineconfig/scripts/linux/scheduler +0 -2
- machineconfig/scripts/linux/share_smb +0 -1
- machineconfig/scripts/linux/start_slidev +0 -2
- machineconfig/scripts/linux/start_terminals +0 -3
- machineconfig/scripts/linux/warp-cli.sh +0 -122
- machineconfig/scripts/linux/wifi_conn +0 -2
- machineconfig/scripts/linux/z_ls +0 -104
- machineconfig/scripts/python/ai/solutions/copilot/prompts/allLintersAndTypeCheckers.prompt.md +0 -5
- machineconfig/scripts/python/cloud_repo_sync.py +0 -186
- machineconfig/scripts/python/devops_devapps_install.py +0 -159
- machineconfig/scripts/python/devops_update_repos.py +0 -180
- machineconfig/scripts/python/dotfile.py +0 -52
- machineconfig/scripts/python/fire_agents.py +0 -175
- machineconfig/scripts/python/fire_agents_help_launch.py +0 -143
- machineconfig/scripts/python/fire_agents_load_balancer.py +0 -50
- machineconfig/scripts/python/fire_jobs_args_helper.py +0 -75
- machineconfig/scripts/python/fire_jobs_layout_helper.py +0 -74
- machineconfig/scripts/python/get_zellij_cmd.py +0 -15
- machineconfig/scripts/python/gh_models.py +0 -104
- machineconfig/scripts/python/helpers/repo_sync_helpers.py +0 -114
- machineconfig/scripts/python/repos.py +0 -80
- machineconfig/scripts/python/repos_helper_action.py +0 -335
- machineconfig/scripts/python/share_terminal.py +0 -104
- machineconfig/scripts/python/snapshot.py +0 -25
- machineconfig/scripts/python/start_terminals.py +0 -121
- machineconfig/scripts/python/t4.py +0 -17
- machineconfig/scripts/windows/choose_wezterm_theme.ps1 +0 -1
- machineconfig/scripts/windows/cloud_copy.ps1 +0 -1
- machineconfig/scripts/windows/cloud_mount.ps1 +0 -1
- machineconfig/scripts/windows/cloud_repo_sync.ps1 +0 -1
- machineconfig/scripts/windows/cloud_sync.ps1 +0 -1
- machineconfig/scripts/windows/croshell.ps1 +0 -1
- machineconfig/scripts/windows/devops.ps1 +0 -1
- machineconfig/scripts/windows/dotfile.ps1 +0 -1
- machineconfig/scripts/windows/fire.ps1 +0 -1
- machineconfig/scripts/windows/ftpx.ps1 +0 -1
- machineconfig/scripts/windows/gpt.ps1 +0 -1
- machineconfig/scripts/windows/grep.ps1 +0 -2
- machineconfig/scripts/windows/initai.ps1 +0 -1
- machineconfig/scripts/windows/kill_process.ps1 +0 -1
- machineconfig/scripts/windows/nano.ps1 +0 -3
- machineconfig/scripts/windows/pomodoro.ps1 +0 -1
- machineconfig/scripts/windows/reload_path.ps1 +0 -3
- machineconfig/scripts/windows/repos.ps1 +0 -1
- machineconfig/scripts/windows/scheduler.ps1 +0 -1
- machineconfig/scripts/windows/snapshot.ps1 +0 -1
- machineconfig/scripts/windows/start_slidev.ps1 +0 -1
- machineconfig/scripts/windows/start_terminals.ps1 +0 -1
- machineconfig/scripts/windows/wifi_conn.ps1 +0 -2
- machineconfig/scripts/windows/wsl_rdp_windows_port_forwarding.ps1 +0 -46
- machineconfig/scripts/windows/wsl_ssh_windows_port_forwarding.ps1 +0 -76
- machineconfig/settings/lf/linux/exe/fzf_nano.sh +0 -16
- machineconfig/setup_linux/others/openssh-server_add_pub_key.sh +0 -57
- machineconfig/setup_linux/web_shortcuts/ascii_art.sh +0 -93
- machineconfig/setup_linux/web_shortcuts/croshell.sh +0 -11
- machineconfig/setup_linux/web_shortcuts/ssh.sh +0 -52
- machineconfig/setup_windows/web_shortcuts/all.ps1 +0 -18
- machineconfig/setup_windows/web_shortcuts/ascii_art.ps1 +0 -36
- machineconfig/setup_windows/web_shortcuts/croshell.ps1 +0 -16
- machineconfig/setup_windows/web_shortcuts/ssh.ps1 +0 -11
- machineconfig/utils/ai/generate_file_checklist.py +0 -68
- machineconfig-3.99.dist-info/METADATA +0 -167
- machineconfig-3.99.dist-info/RECORD +0 -409
- machineconfig-3.99.dist-info/entry_points.txt +0 -18
- machineconfig/cluster/{templates → remote}/run_cloud.py +0 -0
- machineconfig/cluster/{templates → remote}/run_cluster.py +0 -0
- machineconfig/cluster/{templates → remote}/run_remote.py +0 -0
- machineconfig/scripts/linux/{share_nfs → other/share_nfs} +0 -0
- machineconfig/scripts/linux/{start_docker → other/start_docker} +0 -0
- machineconfig/scripts/linux/{switch_ip → other/switch_ip} +0 -0
- machineconfig/{jobs/python → scripts/python/helpers_agents}/__init__.py +0 -0
- machineconfig/scripts/python/{helpers → helpers_agents/agentic_frameworks}/__init__.py +0 -0
- machineconfig/scripts/python/{fire_agents_help_search.py → helpers_agents/fire_agents_help_search.py} +0 -0
- machineconfig/{jobs/windows/msc/cli_agents.bat → scripts/python/helpers_cloud/__init__.py} +0 -0
- machineconfig/scripts/python/{helpers → helpers_cloud}/cloud_helpers.py +1 -1
- /machineconfig/scripts/python/{helpers → helpers_cloud}/helpers5.py +0 -0
- /machineconfig/{jobs/windows/msc/cli_agents.ps1 → scripts/python/helpers_croshell/__init__.py} +0 -0
- /machineconfig/scripts/python/{pomodoro.py → helpers_croshell/pomodoro.py} +0 -0
- /machineconfig/scripts/python/{scheduler.py → helpers_croshell/scheduler.py} +0 -0
- /machineconfig/scripts/python/{viewer.py → helpers_croshell/viewer.py} +0 -0
- /machineconfig/scripts/python/{viewer_template.py → helpers_croshell/viewer_template.py} +0 -0
- /machineconfig/scripts/python/{fire_jobs_streamlit_helper.py → helpers_devops/__init__.py} +0 -0
- /machineconfig/scripts/{windows/share_nfs.ps1 → python/helpers_devops/themes/__init__.py} +0 -0
- /machineconfig/{settings/yazi/keymap.toml → scripts/python/helpers_devops/themes/choose_starship_theme.ps1} +0 -0
- /machineconfig/scripts/python/{cloud_manager.py → helpers_fire_command/cloud_manager.py} +0 -0
- /machineconfig/scripts/{linux → python/helpers_msearch/scripts_linux}/skrg +0 -0
- /machineconfig/scripts/{windows → python/helpers_msearch/scripts_windows}/fzfb.ps1 +0 -0
- /machineconfig/scripts/{windows → python/helpers_msearch/scripts_windows}/fzfg.ps1 +0 -0
- /machineconfig/scripts/{windows → python/helpers_msearch/scripts_windows}/fzfrga.bat +0 -0
- /machineconfig/scripts/{linux → python/nw}/mount_drive +0 -0
- /machineconfig/scripts/python/{mount_nw_drive.py → nw/mount_nw_drive.py} +0 -0
- /machineconfig/scripts/{linux → python/nw}/mount_smb +0 -0
- /machineconfig/scripts/windows/{mount_nw.ps1 → mounts/mount_nw.ps1} +0 -0
- /machineconfig/scripts/windows/{mount_smb.ps1 → mounts/mount_smb.ps1} +0 -0
- /machineconfig/scripts/windows/{share_cloud.cmd → mounts/share_cloud.cmd} +0 -0
- /machineconfig/scripts/windows/{unlock_bitlocker.ps1 → mounts/unlock_bitlocker.ps1} +0 -0
- /machineconfig/setup_linux/{web_shortcuts → others}/android.sh +0 -0
- /machineconfig/{jobs/windows/archive → setup_windows/ssh}/openssh-server_add_key.ps1 +0 -0
- /machineconfig/{jobs/windows/archive → setup_windows/ssh}/openssh-server_copy-ssh-id.ps1 +0 -0
- {machineconfig-3.99.dist-info → machineconfig-7.66.dist-info}/WHEEL +0 -0
- {machineconfig-3.99.dist-info → machineconfig-7.66.dist-info}/top_level.txt +0 -0
machineconfig/utils/ssh.py
CHANGED
|
@@ -1,54 +1,42 @@
|
|
|
1
|
-
from typing import Optional, Any, Union,
|
|
1
|
+
from typing import Callable, Optional, Any, Union, cast
|
|
2
2
|
import os
|
|
3
|
-
from
|
|
3
|
+
from pathlib import Path
|
|
4
|
+
import platform
|
|
5
|
+
from machineconfig.scripts.python.helpers_utils.path import MachineSpecs
|
|
4
6
|
import rich.console
|
|
5
|
-
from machineconfig.utils.terminal import
|
|
6
|
-
from machineconfig.utils.
|
|
7
|
-
from machineconfig.utils.
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
def scout(source: PLike, z: bool = False, r: bool = False) -> Scout:
|
|
21
|
-
source_full = PathExtended(source).expanduser().absolute()
|
|
22
|
-
source_rel2home = source_full.collapseuser()
|
|
23
|
-
exists = source_full.exists()
|
|
24
|
-
is_dir = source_full.is_dir() if exists else False
|
|
25
|
-
if z and exists:
|
|
26
|
-
try:
|
|
27
|
-
source_full = source_full.zip()
|
|
28
|
-
except Exception as ex:
|
|
29
|
-
raise Exception(f"Could not zip {source_full} due to {ex}") from ex # type: ignore # pylint: disable=W0719
|
|
30
|
-
source_rel2home = source_full.zip()
|
|
31
|
-
if r and exists and is_dir:
|
|
32
|
-
files = [item.collapseuser() for item in source_full.search(folders=False, r=True)]
|
|
33
|
-
else:
|
|
34
|
-
files = None
|
|
35
|
-
return Scout(source_full=source_full, source_rel2home=source_rel2home, exists=exists, is_dir=is_dir, files=files)
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
class SSH: # inferior alternative: https://github.com/fabric/fabric
|
|
7
|
+
from machineconfig.utils.terminal import Response
|
|
8
|
+
from machineconfig.utils.accessories import pprint, randstr
|
|
9
|
+
from machineconfig.utils.meta import lambda_to_python_script
|
|
10
|
+
|
|
11
|
+
UV_RUN_CMD = "$HOME/.local/bin/uv run" if platform.system() != "Windows" else """& "$env:USERPROFILE/.local/bin/uv" run"""
|
|
12
|
+
MACHINECONFIG_VERSION = "machineconfig>=7.66"
|
|
13
|
+
DEFAULT_PICKLE_SUBDIR = "tmp_results/tmp_scripts/ssh"
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
class SSH:
|
|
17
|
+
@staticmethod
|
|
18
|
+
def from_config_file(host: str) -> "SSH":
|
|
19
|
+
"""Create SSH instance from SSH config file entry."""
|
|
20
|
+
return SSH(host=host, username=None, hostname=None, ssh_key_path=None, password=None, port=22, enable_compression=False)
|
|
21
|
+
|
|
39
22
|
def __init__(
|
|
40
|
-
self,
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
23
|
+
self,
|
|
24
|
+
host: Optional[str],
|
|
25
|
+
username: Optional[str],
|
|
26
|
+
hostname: Optional[str],
|
|
27
|
+
ssh_key_path: Optional[str],
|
|
28
|
+
password: Optional[str],
|
|
29
|
+
port: int,
|
|
30
|
+
enable_compression: bool,
|
|
31
|
+
):
|
|
32
|
+
self.password = password
|
|
33
|
+
self.enable_compression = enable_compression
|
|
45
34
|
|
|
46
35
|
self.host: Optional[str] = None
|
|
47
36
|
self.hostname: str
|
|
48
37
|
self.username: str
|
|
49
38
|
self.port: int = port
|
|
50
39
|
self.proxycommand: Optional[str] = None
|
|
51
|
-
import platform
|
|
52
40
|
import paramiko # type: ignore
|
|
53
41
|
import getpass
|
|
54
42
|
|
|
@@ -56,26 +44,28 @@ class SSH: # inferior alternative: https://github.com/fabric/fabric
|
|
|
56
44
|
try:
|
|
57
45
|
import paramiko.config as pconfig
|
|
58
46
|
|
|
59
|
-
config = pconfig.SSHConfig.from_path(str(
|
|
47
|
+
config = pconfig.SSHConfig.from_path(str(Path.home().joinpath(".ssh/config")))
|
|
60
48
|
config_dict = config.lookup(host)
|
|
61
49
|
self.hostname = config_dict["hostname"]
|
|
62
50
|
self.username = config_dict["user"]
|
|
63
51
|
self.host = host
|
|
64
52
|
self.port = int(config_dict.get("port", port))
|
|
65
|
-
|
|
66
|
-
if isinstance(
|
|
67
|
-
|
|
53
|
+
identity_file_value = config_dict.get("identityfile", ssh_key_path)
|
|
54
|
+
if isinstance(identity_file_value, list):
|
|
55
|
+
ssh_key_path = identity_file_value[0]
|
|
68
56
|
else:
|
|
69
|
-
|
|
57
|
+
ssh_key_path = identity_file_value
|
|
70
58
|
self.proxycommand = config_dict.get("proxycommand", None)
|
|
71
|
-
if
|
|
72
|
-
|
|
73
|
-
if isinstance(
|
|
74
|
-
|
|
59
|
+
if ssh_key_path is not None:
|
|
60
|
+
wildcard_identity_file = config.lookup("*").get("identityfile", ssh_key_path)
|
|
61
|
+
if isinstance(wildcard_identity_file, list):
|
|
62
|
+
ssh_key_path = wildcard_identity_file[0]
|
|
75
63
|
else:
|
|
76
|
-
|
|
64
|
+
ssh_key_path = wildcard_identity_file
|
|
77
65
|
except (FileNotFoundError, KeyError):
|
|
78
|
-
assert "@" in host or ":" in host,
|
|
66
|
+
assert "@" in host or ":" in host, (
|
|
67
|
+
f"Host must be in the form of `username@hostname:port` or `username@hostname` or `hostname:port`, but it is: {host}"
|
|
68
|
+
)
|
|
79
69
|
if "@" in host:
|
|
80
70
|
self.username, self.hostname = host.split("@")
|
|
81
71
|
else:
|
|
@@ -91,31 +81,52 @@ class SSH: # inferior alternative: https://github.com/fabric/fabric
|
|
|
91
81
|
print(f"Provided values: host={host}, username={username}, hostname={hostname}")
|
|
92
82
|
raise ValueError("Either host or username and hostname must be provided.")
|
|
93
83
|
|
|
94
|
-
self.
|
|
84
|
+
self.ssh_key_path = str(Path(ssh_key_path).expanduser().absolute()) if ssh_key_path is not None else None
|
|
95
85
|
self.ssh = paramiko.SSHClient()
|
|
96
86
|
self.ssh.load_system_host_keys()
|
|
97
87
|
self.ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
|
98
|
-
pprint(
|
|
88
|
+
pprint(
|
|
89
|
+
dict(host=self.host, hostname=self.hostname, username=self.username, password="***", port=self.port, key_filename=self.ssh_key_path),
|
|
90
|
+
title="SSHing To",
|
|
91
|
+
)
|
|
99
92
|
sock = paramiko.ProxyCommand(self.proxycommand) if self.proxycommand is not None else None
|
|
100
93
|
try:
|
|
101
|
-
if
|
|
94
|
+
if password is None:
|
|
102
95
|
allow_agent = True
|
|
103
96
|
look_for_keys = True
|
|
104
97
|
else:
|
|
105
98
|
allow_agent = False
|
|
106
99
|
look_for_keys = False
|
|
107
|
-
self.ssh.connect(
|
|
100
|
+
self.ssh.connect(
|
|
101
|
+
hostname=self.hostname,
|
|
102
|
+
username=self.username,
|
|
103
|
+
password=self.password,
|
|
104
|
+
port=self.port,
|
|
105
|
+
key_filename=self.ssh_key_path,
|
|
106
|
+
compress=self.enable_compression,
|
|
107
|
+
sock=sock,
|
|
108
|
+
allow_agent=allow_agent,
|
|
109
|
+
look_for_keys=look_for_keys,
|
|
110
|
+
) # type: ignore
|
|
108
111
|
except Exception as _err:
|
|
109
112
|
rich.console.Console().print_exception()
|
|
110
|
-
self.
|
|
111
|
-
self.ssh.connect(
|
|
113
|
+
self.password = getpass.getpass(f"Enter password for {self.username}@{self.hostname}: ")
|
|
114
|
+
self.ssh.connect(
|
|
115
|
+
hostname=self.hostname,
|
|
116
|
+
username=self.username,
|
|
117
|
+
password=self.password,
|
|
118
|
+
port=self.port,
|
|
119
|
+
key_filename=self.ssh_key_path,
|
|
120
|
+
compress=self.enable_compression,
|
|
121
|
+
sock=sock,
|
|
122
|
+
allow_agent=False,
|
|
123
|
+
look_for_keys=False,
|
|
124
|
+
) # type: ignore
|
|
112
125
|
try:
|
|
113
126
|
self.sftp: Optional[paramiko.SFTPClient] = self.ssh.open_sftp()
|
|
114
127
|
except Exception as err:
|
|
115
128
|
self.sftp = None
|
|
116
|
-
print(f"""⚠️ WARNING: Failed to open SFTP connection to {hostname}.
|
|
117
|
-
Error Details: {err}\nData transfer may be affected!""")
|
|
118
|
-
|
|
129
|
+
print(f"""⚠️ WARNING: Failed to open SFTP connection to {self.hostname}. Error Details: {err}\nData transfer may be affected!""")
|
|
119
130
|
from rich.progress import Progress, SpinnerColumn, TextColumn, BarColumn, FileSizeColumn, TransferSpeedColumn
|
|
120
131
|
|
|
121
132
|
class RichProgressWrapper:
|
|
@@ -125,7 +136,9 @@ class SSH: # inferior alternative: https://github.com/fabric/fabric
|
|
|
125
136
|
self.task: Optional[Any] = None
|
|
126
137
|
|
|
127
138
|
def __enter__(self) -> "RichProgressWrapper":
|
|
128
|
-
self.progress = Progress(
|
|
139
|
+
self.progress = Progress(
|
|
140
|
+
SpinnerColumn(), TextColumn("[bold blue]{task.description}"), BarColumn(), FileSizeColumn(), TransferSpeedColumn()
|
|
141
|
+
)
|
|
129
142
|
self.progress.start()
|
|
130
143
|
self.task = self.progress.add_task("Transferring...", total=0)
|
|
131
144
|
return self
|
|
@@ -139,223 +152,698 @@ class SSH: # inferior alternative: https://github.com/fabric/fabric
|
|
|
139
152
|
self.progress.update(self.task, completed=transferred, total=total)
|
|
140
153
|
|
|
141
154
|
self.tqdm_wrap = RichProgressWrapper
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
self.
|
|
155
|
+
from machineconfig.scripts.python.helpers_utils.path import get_machine_specs
|
|
156
|
+
|
|
157
|
+
self.local_specs: MachineSpecs = get_machine_specs()
|
|
158
|
+
resp = self.run_shell(
|
|
159
|
+
command="""~/.local/bin/utils get-machine-specs """,
|
|
160
|
+
verbose_output=False,
|
|
161
|
+
description="Getting remote machine specs",
|
|
162
|
+
strict_stderr=False,
|
|
163
|
+
strict_return_code=False,
|
|
164
|
+
)
|
|
165
|
+
json_str = resp.op
|
|
166
|
+
import ast
|
|
167
|
+
|
|
168
|
+
self.remote_specs: MachineSpecs = cast(MachineSpecs, ast.literal_eval(json_str))
|
|
145
169
|
self.terminal_responses: list[Response] = []
|
|
146
|
-
self.platform = platform
|
|
147
170
|
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
171
|
+
from rich import inspect
|
|
172
|
+
|
|
173
|
+
local_info = dict(distro=self.local_specs.get("distro"), system=self.local_specs.get("system"), home_dir=self.local_specs.get("home_dir"))
|
|
174
|
+
remote_info = dict(distro=self.remote_specs.get("distro"), system=self.remote_specs.get("system"), home_dir=self.remote_specs.get("home_dir"))
|
|
175
|
+
|
|
176
|
+
console = rich.console.Console()
|
|
177
|
+
|
|
178
|
+
from io import StringIO
|
|
179
|
+
|
|
180
|
+
local_buffer = StringIO()
|
|
181
|
+
remote_buffer = StringIO()
|
|
182
|
+
|
|
183
|
+
local_console = rich.console.Console(file=local_buffer, width=40)
|
|
184
|
+
remote_console = rich.console.Console(file=remote_buffer, width=40)
|
|
185
|
+
|
|
186
|
+
inspect(
|
|
187
|
+
type("LocalInfo", (object,), local_info)(), value=False, title="SSHing From", docs=False, dunder=False, sort=False, console=local_console
|
|
188
|
+
)
|
|
189
|
+
inspect(
|
|
190
|
+
type("RemoteInfo", (object,), remote_info)(), value=False, title="SSHing To", docs=False, dunder=False, sort=False, console=remote_console
|
|
191
|
+
)
|
|
192
|
+
|
|
193
|
+
local_lines = local_buffer.getvalue().split("\n")
|
|
194
|
+
remote_lines = remote_buffer.getvalue().split("\n")
|
|
195
|
+
|
|
196
|
+
max_lines = max(len(local_lines), len(remote_lines))
|
|
197
|
+
for i in range(max_lines):
|
|
198
|
+
left = local_lines[i] if i < len(local_lines) else ""
|
|
199
|
+
right = remote_lines[i] if i < len(remote_lines) else ""
|
|
200
|
+
console.print(f"{left:<42} {right}")
|
|
201
|
+
|
|
202
|
+
def __enter__(self) -> "SSH":
|
|
203
|
+
return self
|
|
204
|
+
|
|
205
|
+
def __exit__(self, exc_type: Any, exc_val: Any, exc_tb: Any) -> None:
|
|
206
|
+
self.close()
|
|
207
|
+
|
|
208
|
+
def close(self) -> None:
|
|
209
|
+
if self.sftp is not None:
|
|
210
|
+
self.sftp.close()
|
|
211
|
+
self.sftp = None
|
|
212
|
+
self.ssh.close()
|
|
213
|
+
|
|
214
|
+
def restart_computer(self) -> Response:
|
|
215
|
+
return self.run_shell(
|
|
216
|
+
command="Restart-Computer -Force" if self.remote_specs["system"] == "Windows" else "sudo reboot",
|
|
217
|
+
verbose_output=True,
|
|
218
|
+
description="",
|
|
219
|
+
strict_stderr=False,
|
|
220
|
+
strict_return_code=False,
|
|
221
|
+
)
|
|
222
|
+
|
|
223
|
+
def send_ssh_key(self) -> Response:
|
|
224
|
+
self.copy_from_here(source_path="~/.ssh/id_rsa.pub", target_rel2home=None, compress_with_zip=False, recursive=False, overwrite_existing=False)
|
|
225
|
+
if self.remote_specs["system"] != "Windows":
|
|
226
|
+
raise RuntimeError("send_ssh_key is only supported for Windows remote machines")
|
|
178
227
|
code_url = "https://raw.githubusercontent.com/thisismygitrepo/machineconfig/refs/heads/main/src/machineconfig/setup_windows/openssh-server_add-sshkey.ps1"
|
|
179
|
-
|
|
180
|
-
self.run(code)
|
|
228
|
+
import urllib.request
|
|
181
229
|
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
return self.
|
|
230
|
+
with urllib.request.urlopen(code_url) as response:
|
|
231
|
+
code = response.read().decode("utf-8")
|
|
232
|
+
return self.run_shell(command=code, verbose_output=True, description="", strict_stderr=False, strict_return_code=False)
|
|
185
233
|
|
|
186
234
|
def get_remote_repr(self, add_machine: bool = False) -> str:
|
|
187
|
-
return f"{self.username}@{self.hostname}:{self.port}" + (
|
|
235
|
+
return f"{self.username}@{self.hostname}:{self.port}" + (
|
|
236
|
+
f" [{self.remote_specs['system']}][{self.remote_specs['distro']}]" if add_machine else ""
|
|
237
|
+
)
|
|
188
238
|
|
|
189
239
|
def get_local_repr(self, add_machine: bool = False) -> str:
|
|
190
240
|
import getpass
|
|
191
241
|
|
|
192
|
-
return f"{getpass.getuser()}@{
|
|
242
|
+
return f"{getpass.getuser()}@{platform.node()}" + (f" [{platform.system()}][{self.local_specs['distro']}]" if add_machine else "")
|
|
193
243
|
|
|
194
|
-
def
|
|
244
|
+
def get_ssh_conn_str(self, command: str) -> str:
|
|
245
|
+
return (
|
|
246
|
+
"ssh "
|
|
247
|
+
+ (f" -i {self.ssh_key_path}" if self.ssh_key_path else "")
|
|
248
|
+
+ self.get_remote_repr(add_machine=False).replace(":", " -p ")
|
|
249
|
+
+ (f" -t {command} " if command != "" else " ")
|
|
250
|
+
)
|
|
251
|
+
|
|
252
|
+
def __repr__(self) -> str:
|
|
195
253
|
return f"local {self.get_local_repr(add_machine=True)} >>> SSH TO >>> remote {self.get_remote_repr(add_machine=True)}"
|
|
196
254
|
|
|
197
|
-
def run_locally(self, command: str):
|
|
198
|
-
print(f"""💻 [LOCAL EXECUTION] Running command on node: {self.
|
|
255
|
+
def run_locally(self, command: str) -> Response:
|
|
256
|
+
print(f"""💻 [LOCAL EXECUTION] Running command on node: {self.local_specs["system"]} Command: {command}""")
|
|
199
257
|
res = Response(cmd=command)
|
|
200
258
|
res.output.returncode = os.system(command)
|
|
201
259
|
return res
|
|
202
260
|
|
|
203
|
-
def
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
raw = self.ssh.exec_command(cmd)
|
|
208
|
-
res = Response(stdin=raw[0], stdout=raw[1], stderr=raw[2], cmd=cmd, desc=desc) # type: ignore
|
|
209
|
-
if not verbose:
|
|
210
|
-
res.capture().print_if_unsuccessful(desc=desc, strict_err=strict_err, strict_returncode=strict_returncode, assert_success=False)
|
|
211
|
-
else:
|
|
261
|
+
def run_shell(self, command: str, verbose_output: bool, description: str, strict_stderr: bool, strict_return_code: bool) -> Response:
|
|
262
|
+
raw = self.ssh.exec_command(command)
|
|
263
|
+
res = Response(stdin=raw[0], stdout=raw[1], stderr=raw[2], cmd=command, desc=description) # type: ignore
|
|
264
|
+
if verbose_output:
|
|
212
265
|
res.print()
|
|
213
|
-
|
|
266
|
+
else:
|
|
267
|
+
res.capture().print_if_unsuccessful(
|
|
268
|
+
desc=description, strict_err=strict_stderr, strict_returncode=strict_return_code, assert_success=False
|
|
269
|
+
)
|
|
270
|
+
# self.terminal_responses.append(res)
|
|
214
271
|
return res
|
|
215
272
|
|
|
216
|
-
def
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
273
|
+
def _run_py_prep(self, python_code: str, uv_with: Optional[list[str]], uv_project_dir: Optional[str]) -> str:
|
|
274
|
+
py_path = Path.home().joinpath(f"{DEFAULT_PICKLE_SUBDIR}/runpy_{randstr()}.py")
|
|
275
|
+
py_path.parent.mkdir(parents=True, exist_ok=True)
|
|
276
|
+
py_path.write_text(python_code, encoding="utf-8")
|
|
277
|
+
self.copy_from_here(source_path=str(py_path), target_rel2home=None, compress_with_zip=False, recursive=False, overwrite_existing=False)
|
|
278
|
+
if uv_with is not None and len(uv_with) > 0:
|
|
279
|
+
with_clause = " --with " + '"' + ",".join(uv_with) + '"'
|
|
280
|
+
else:
|
|
281
|
+
with_clause = ""
|
|
282
|
+
if uv_project_dir is not None:
|
|
283
|
+
with_clause += f" --project {uv_project_dir}"
|
|
284
|
+
else:
|
|
285
|
+
with_clause += ""
|
|
286
|
+
uv_cmd = f"""{UV_RUN_CMD} {with_clause} python {py_path.relative_to(Path.home())}"""
|
|
287
|
+
return uv_cmd
|
|
288
|
+
|
|
289
|
+
def run_py(
|
|
290
|
+
self,
|
|
291
|
+
python_code: str,
|
|
292
|
+
uv_with: Optional[list[str]],
|
|
293
|
+
uv_project_dir: Optional[str],
|
|
294
|
+
description: str,
|
|
295
|
+
verbose_output: bool,
|
|
296
|
+
strict_stderr: bool,
|
|
297
|
+
strict_return_code: bool,
|
|
298
|
+
) -> Response:
|
|
299
|
+
uv_cmd = self._run_py_prep(python_code=python_code, uv_with=uv_with, uv_project_dir=uv_project_dir)
|
|
300
|
+
return self.run_shell(
|
|
301
|
+
command=uv_cmd,
|
|
302
|
+
verbose_output=verbose_output,
|
|
303
|
+
description=description or f"run_py on {self.get_remote_repr(add_machine=False)}",
|
|
304
|
+
strict_stderr=strict_stderr,
|
|
305
|
+
strict_return_code=strict_return_code,
|
|
306
|
+
)
|
|
307
|
+
|
|
308
|
+
def run_lambda_function(self, func: Callable[..., Any], import_module: bool, uv_with: Optional[list[str]], uv_project_dir: Optional[str]):
|
|
309
|
+
command = lambda_to_python_script(lmb=func, in_global=True, import_module=import_module)
|
|
310
|
+
# turns ou that the code below for some reason runs but zellij doesn't start, looks like things are assigned to different user.
|
|
311
|
+
# return self.run_py(python_code=command, uv_with=uv_with, uv_project_dir=uv_project_dir,
|
|
312
|
+
# description=f"run_py_func {func.__name__} on {self.get_remote_repr(add_machine=False)}",
|
|
313
|
+
# verbose_output=True, strict_stderr=True, strict_return_code=True)
|
|
314
|
+
uv_cmd = self._run_py_prep(python_code=command, uv_with=uv_with, uv_project_dir=uv_project_dir)
|
|
315
|
+
if self.remote_specs["system"] == "Linux":
|
|
316
|
+
uv_cmd_modified = f'bash -l -c "{uv_cmd}"'
|
|
317
|
+
else:
|
|
318
|
+
uv_cmd_modified = uv_cmd
|
|
319
|
+
# This works even withou the modified uv cmd:
|
|
320
|
+
# from machineconfig.utils.code import run_shell_script
|
|
321
|
+
# assert self.host is not None, "SSH host must be specified to run remote commands"
|
|
322
|
+
# process = run_shell_script(f"ssh {self.host} -n '. ~/.profile; . ~/.bashrc; {uv_cmd}'")
|
|
323
|
+
# return process
|
|
324
|
+
return self.run_shell(
|
|
325
|
+
command=uv_cmd_modified,
|
|
326
|
+
verbose_output=True,
|
|
327
|
+
description=f"run_py_func {func.__name__} on {self.get_remote_repr(add_machine=False)}",
|
|
328
|
+
strict_stderr=True,
|
|
329
|
+
strict_return_code=True,
|
|
330
|
+
)
|
|
331
|
+
|
|
332
|
+
def _simple_sftp_get(self, remote_path: str, local_path: Path) -> None:
|
|
333
|
+
"""Simple SFTP get without any recursion or path expansion - for internal use only."""
|
|
334
|
+
if self.sftp is None:
|
|
335
|
+
raise RuntimeError(f"SFTP connection not available for {self.hostname}")
|
|
336
|
+
local_path.parent.mkdir(parents=True, exist_ok=True)
|
|
337
|
+
self.sftp.get(remotepath=remote_path, localpath=str(local_path))
|
|
338
|
+
|
|
339
|
+
def create_dir(self, path_rel2home: str, overwrite_existing: bool) -> None:
|
|
340
|
+
"""Helper to create a directory on remote machine and return its path."""
|
|
341
|
+
|
|
342
|
+
def create_target_dir(target_rel2home: str, overwrite: bool):
|
|
343
|
+
from pathlib import Path
|
|
344
|
+
import shutil
|
|
345
|
+
|
|
346
|
+
directory_path = Path(target_rel2home).expanduser()
|
|
347
|
+
if not directory_path.is_absolute():
|
|
348
|
+
directory_path = Path.home().joinpath(directory_path)
|
|
349
|
+
if overwrite and directory_path.exists():
|
|
350
|
+
if directory_path.is_dir():
|
|
351
|
+
shutil.rmtree(directory_path)
|
|
352
|
+
else:
|
|
353
|
+
directory_path.unlink()
|
|
354
|
+
directory_path.parent.mkdir(parents=True, exist_ok=True)
|
|
355
|
+
directory_path.mkdir(parents=True, exist_ok=True)
|
|
230
356
|
|
|
231
|
-
|
|
357
|
+
command = lambda_to_python_script(
|
|
358
|
+
lmb=lambda: create_target_dir(target_rel2home=path_rel2home, overwrite=overwrite_existing), in_global=True, import_module=False
|
|
359
|
+
)
|
|
360
|
+
tmp_py_file = Path.home().joinpath(f"{DEFAULT_PICKLE_SUBDIR}/create_target_dir_{randstr()}.py")
|
|
361
|
+
tmp_py_file.parent.mkdir(parents=True, exist_ok=True)
|
|
362
|
+
tmp_py_file.write_text(command, encoding="utf-8")
|
|
363
|
+
# self.copy_from_here(source_path=str(tmp_py_file), target_rel2home=".tmp_file.py", compress_with_zip=False, recursive=False, overwrite_existing=True)
|
|
364
|
+
assert self.sftp is not None
|
|
365
|
+
tmp_remote_path = ".tmp_pyfile.py"
|
|
366
|
+
self.sftp.put(localpath=str(tmp_py_file), remotepath=str(Path(self.remote_specs["home_dir"]).joinpath(tmp_remote_path)))
|
|
367
|
+
self.run_shell(
|
|
368
|
+
command=f"""{UV_RUN_CMD} python {tmp_remote_path}""",
|
|
369
|
+
verbose_output=False,
|
|
370
|
+
description=f"Creating target dir {path_rel2home}",
|
|
371
|
+
strict_stderr=True,
|
|
372
|
+
strict_return_code=True,
|
|
373
|
+
)
|
|
232
374
|
|
|
233
|
-
def copy_from_here(
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
375
|
+
def copy_from_here(
|
|
376
|
+
self, source_path: str, target_rel2home: Optional[str], compress_with_zip: bool, recursive: bool, overwrite_existing: bool
|
|
377
|
+
) -> None:
|
|
378
|
+
if self.sftp is None:
|
|
379
|
+
raise RuntimeError(f"SFTP connection not available for {self.hostname}. Cannot transfer files.")
|
|
380
|
+
source_obj = Path(source_path).expanduser().absolute()
|
|
237
381
|
if not source_obj.exists():
|
|
238
|
-
raise RuntimeError(f"
|
|
239
|
-
if
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
target
|
|
244
|
-
if not
|
|
245
|
-
if
|
|
246
|
-
raise RuntimeError(
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
382
|
+
raise RuntimeError(f"SSH Error: source `{source_obj}` does not exist!")
|
|
383
|
+
if target_rel2home is None:
|
|
384
|
+
try:
|
|
385
|
+
target_rel2home = str(source_obj.relative_to(Path.home()))
|
|
386
|
+
except ValueError:
|
|
387
|
+
raise RuntimeError(f"If target is not specified, source must be relative to home directory, but got: {source_obj}")
|
|
388
|
+
if not compress_with_zip and source_obj.is_dir():
|
|
389
|
+
if not recursive:
|
|
390
|
+
raise RuntimeError(
|
|
391
|
+
f"SSH Error: source `{source_obj}` is a directory! Set `recursive=True` for recursive sending or `compress_with_zip=True` to zip it first."
|
|
392
|
+
)
|
|
393
|
+
file_paths_to_upload: list[Path] = [file_path for file_path in source_obj.rglob("*") if file_path.is_file()]
|
|
394
|
+
self.create_dir(path_rel2home=target_rel2home, overwrite_existing=overwrite_existing)
|
|
395
|
+
for idx, file_path in enumerate(file_paths_to_upload):
|
|
396
|
+
print(f" {idx + 1:03d}. {file_path}")
|
|
397
|
+
for file_path in file_paths_to_upload:
|
|
398
|
+
remote_file_target = Path(target_rel2home).joinpath(file_path.relative_to(source_obj))
|
|
399
|
+
self.copy_from_here(
|
|
400
|
+
source_path=str(file_path),
|
|
401
|
+
target_rel2home=str(remote_file_target),
|
|
402
|
+
compress_with_zip=False,
|
|
403
|
+
recursive=False,
|
|
404
|
+
overwrite_existing=overwrite_existing,
|
|
405
|
+
)
|
|
406
|
+
return None
|
|
407
|
+
if compress_with_zip:
|
|
263
408
|
print("🗜️ ZIPPING ...")
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
409
|
+
import shutil
|
|
410
|
+
|
|
411
|
+
zip_path = Path(str(source_obj) + "_archive")
|
|
412
|
+
if source_obj.is_dir():
|
|
413
|
+
shutil.make_archive(str(zip_path), "zip", source_obj)
|
|
414
|
+
else:
|
|
415
|
+
shutil.make_archive(str(zip_path), "zip", source_obj.parent, source_obj.name)
|
|
416
|
+
source_obj = Path(str(zip_path) + ".zip")
|
|
417
|
+
if not target_rel2home.endswith(".zip"):
|
|
418
|
+
target_rel2home = target_rel2home + ".zip"
|
|
419
|
+
self.create_dir(path_rel2home=str(Path(target_rel2home).parent), overwrite_existing=overwrite_existing)
|
|
420
|
+
print(f"""📤 [SFTP UPLOAD] Sending file: {repr(source_obj)} ==> Remote Path: {target_rel2home}""")
|
|
421
|
+
try:
|
|
422
|
+
with self.tqdm_wrap(ascii=True, unit="b", unit_scale=True) as pbar:
|
|
423
|
+
if self.sftp is None: # type: ignore[unreachable]
|
|
424
|
+
raise RuntimeError(f"SFTP connection lost for {self.hostname}")
|
|
425
|
+
self.sftp.put(
|
|
426
|
+
localpath=str(source_obj), remotepath=str(Path(self.remote_specs["home_dir"]).joinpath(target_rel2home)), callback=pbar.view_bar
|
|
427
|
+
)
|
|
428
|
+
except Exception:
|
|
429
|
+
if compress_with_zip and source_obj.exists() and str(source_obj).endswith("_archive.zip"):
|
|
430
|
+
source_obj.unlink()
|
|
431
|
+
raise
|
|
432
|
+
|
|
433
|
+
if compress_with_zip:
|
|
434
|
+
|
|
435
|
+
def unzip_archive(zip_file_path: str, overwrite_flag: bool) -> None:
|
|
436
|
+
from pathlib import Path
|
|
437
|
+
import shutil
|
|
438
|
+
import zipfile
|
|
439
|
+
|
|
440
|
+
archive_path = Path(zip_file_path).expanduser()
|
|
441
|
+
extraction_directory = archive_path.parent / archive_path.stem
|
|
442
|
+
if overwrite_flag and extraction_directory.exists():
|
|
443
|
+
shutil.rmtree(extraction_directory)
|
|
444
|
+
with zipfile.ZipFile(archive_path, "r") as archive_handle:
|
|
445
|
+
archive_handle.extractall(extraction_directory)
|
|
446
|
+
archive_path.unlink()
|
|
447
|
+
|
|
448
|
+
command = lambda_to_python_script(
|
|
449
|
+
lmb=lambda: unzip_archive(
|
|
450
|
+
zip_file_path=str(Path(self.remote_specs["home_dir"]).joinpath(target_rel2home)), overwrite_flag=overwrite_existing
|
|
451
|
+
),
|
|
452
|
+
in_global=True,
|
|
453
|
+
import_module=False,
|
|
454
|
+
)
|
|
455
|
+
tmp_py_file = Path.home().joinpath(f"{DEFAULT_PICKLE_SUBDIR}/create_target_dir_{randstr()}.py")
|
|
456
|
+
tmp_py_file.parent.mkdir(parents=True, exist_ok=True)
|
|
457
|
+
tmp_py_file.write_text(command, encoding="utf-8")
|
|
458
|
+
remote_tmp_py = tmp_py_file.relative_to(Path.home()).as_posix()
|
|
459
|
+
self.copy_from_here(source_path=str(tmp_py_file), target_rel2home=None, compress_with_zip=False, recursive=False, overwrite_existing=True)
|
|
460
|
+
self.run_shell(
|
|
461
|
+
command=f"""{UV_RUN_CMD} python {remote_tmp_py}""",
|
|
462
|
+
verbose_output=False,
|
|
463
|
+
description=f"UNZIPPING {target_rel2home}",
|
|
464
|
+
strict_stderr=True,
|
|
465
|
+
strict_return_code=True,
|
|
466
|
+
)
|
|
467
|
+
source_obj.unlink()
|
|
468
|
+
tmp_py_file.unlink(missing_ok=True)
|
|
469
|
+
return None
|
|
470
|
+
|
|
471
|
+
def _check_remote_is_dir(self, source_path: Union[str, Path]) -> bool:
|
|
472
|
+
"""Helper to check if a remote path is a directory."""
|
|
473
|
+
|
|
474
|
+
def check_is_dir(path_to_check: str, json_output_path: str) -> bool:
|
|
475
|
+
from pathlib import Path
|
|
476
|
+
import json
|
|
477
|
+
|
|
478
|
+
is_directory = Path(path_to_check).expanduser().absolute().is_dir()
|
|
479
|
+
json_result_path = Path(json_output_path)
|
|
480
|
+
json_result_path.parent.mkdir(parents=True, exist_ok=True)
|
|
481
|
+
json_result_path.write_text(json.dumps(is_directory, indent=2), encoding="utf-8")
|
|
482
|
+
print(json_result_path.as_posix())
|
|
483
|
+
return is_directory
|
|
484
|
+
|
|
485
|
+
remote_json_output = Path.home().joinpath(f"{DEFAULT_PICKLE_SUBDIR}/return_{randstr()}.json").as_posix()
|
|
486
|
+
command = lambda_to_python_script(
|
|
487
|
+
lmb=lambda: check_is_dir(path_to_check=str(source_path), json_output_path=remote_json_output), in_global=True, import_module=False
|
|
488
|
+
)
|
|
489
|
+
response = self.run_py(
|
|
490
|
+
python_code=command,
|
|
491
|
+
uv_with=[MACHINECONFIG_VERSION],
|
|
492
|
+
uv_project_dir=None,
|
|
493
|
+
description=f"Check if source `{source_path}` is a dir",
|
|
494
|
+
verbose_output=False,
|
|
495
|
+
strict_stderr=False,
|
|
496
|
+
strict_return_code=False,
|
|
272
497
|
)
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
498
|
+
remote_json_path = response.op.strip()
|
|
499
|
+
if not remote_json_path:
|
|
500
|
+
raise RuntimeError(f"Failed to check if {source_path} is directory - no response from remote")
|
|
501
|
+
|
|
502
|
+
local_json = Path.home().joinpath(f"{DEFAULT_PICKLE_SUBDIR}/local_{randstr()}.json")
|
|
503
|
+
self._simple_sftp_get(remote_path=remote_json_path, local_path=local_json)
|
|
504
|
+
import json
|
|
505
|
+
|
|
506
|
+
try:
|
|
507
|
+
result = json.loads(local_json.read_text(encoding="utf-8"))
|
|
508
|
+
except (json.JSONDecodeError, FileNotFoundError) as err:
|
|
509
|
+
raise RuntimeError(f"Failed to check if {source_path} is directory - invalid JSON response: {err}") from err
|
|
510
|
+
finally:
|
|
511
|
+
if local_json.exists():
|
|
512
|
+
local_json.unlink()
|
|
513
|
+
assert isinstance(result, bool), f"Failed to check if {source_path} is directory"
|
|
514
|
+
return result
|
|
515
|
+
|
|
516
|
+
def _expand_remote_path(self, source_path: Union[str, Path]) -> str:
|
|
517
|
+
"""Helper to expand a path on the remote machine."""
|
|
518
|
+
|
|
519
|
+
def expand_source(path_to_expand: str, json_output_path: str) -> str:
|
|
520
|
+
from pathlib import Path
|
|
521
|
+
import json
|
|
522
|
+
|
|
523
|
+
expanded_path_posix = Path(path_to_expand).expanduser().absolute().as_posix()
|
|
524
|
+
json_result_path = Path(json_output_path)
|
|
525
|
+
json_result_path.parent.mkdir(parents=True, exist_ok=True)
|
|
526
|
+
json_result_path.write_text(json.dumps(expanded_path_posix, indent=2), encoding="utf-8")
|
|
527
|
+
print(json_result_path.as_posix())
|
|
528
|
+
return expanded_path_posix
|
|
529
|
+
|
|
530
|
+
remote_json_output = Path.home().joinpath(f"{DEFAULT_PICKLE_SUBDIR}/return_{randstr()}.json").as_posix()
|
|
531
|
+
command = lambda_to_python_script(
|
|
532
|
+
lmb=lambda: expand_source(path_to_expand=str(source_path), json_output_path=remote_json_output), in_global=True, import_module=False
|
|
533
|
+
)
|
|
534
|
+
response = self.run_py(
|
|
535
|
+
python_code=command,
|
|
536
|
+
uv_with=[MACHINECONFIG_VERSION],
|
|
537
|
+
uv_project_dir=None,
|
|
538
|
+
description="Resolving source path by expanding user",
|
|
539
|
+
verbose_output=False,
|
|
540
|
+
strict_stderr=False,
|
|
541
|
+
strict_return_code=False,
|
|
542
|
+
)
|
|
543
|
+
remote_json_path = response.op.strip()
|
|
544
|
+
if not remote_json_path:
|
|
545
|
+
raise RuntimeError(f"Could not resolve source path {source_path} - no response from remote")
|
|
546
|
+
|
|
547
|
+
local_json = Path.home().joinpath(f"{DEFAULT_PICKLE_SUBDIR}/local_{randstr()}.json")
|
|
548
|
+
self._simple_sftp_get(remote_path=remote_json_path, local_path=local_json)
|
|
549
|
+
import json
|
|
550
|
+
|
|
551
|
+
try:
|
|
552
|
+
result = json.loads(local_json.read_text(encoding="utf-8"))
|
|
553
|
+
except (json.JSONDecodeError, FileNotFoundError) as err:
|
|
554
|
+
raise RuntimeError(f"Could not resolve source path {source_path} - invalid JSON response: {err}") from err
|
|
555
|
+
finally:
|
|
556
|
+
if local_json.exists():
|
|
557
|
+
local_json.unlink()
|
|
558
|
+
assert isinstance(result, str), f"Could not resolve source path {source_path}"
|
|
559
|
+
return result
|
|
560
|
+
|
|
561
|
+
def copy_to_here(
|
|
562
|
+
self,
|
|
563
|
+
source: Union[str, Path],
|
|
564
|
+
target: Optional[Union[str, Path]],
|
|
565
|
+
compress_with_zip: bool = False,
|
|
566
|
+
recursive: bool = False,
|
|
567
|
+
internal_call: bool = False,
|
|
568
|
+
) -> None:
|
|
569
|
+
if self.sftp is None:
|
|
570
|
+
raise RuntimeError(f"SFTP connection not available for {self.hostname}. Cannot transfer files.")
|
|
571
|
+
|
|
572
|
+
if not internal_call:
|
|
285
573
|
print(f"{'⬇️' * 5} SFTP DOWNLOADING FROM `{source}` TO `{target}`")
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
574
|
+
|
|
575
|
+
source_obj = Path(source)
|
|
576
|
+
expanded_source = self._expand_remote_path(source_path=source_obj)
|
|
577
|
+
|
|
578
|
+
if not compress_with_zip:
|
|
579
|
+
is_dir = self._check_remote_is_dir(source_path=expanded_source)
|
|
580
|
+
|
|
581
|
+
if is_dir:
|
|
582
|
+
if not recursive:
|
|
583
|
+
raise RuntimeError(
|
|
584
|
+
f"SSH Error: source `{source_obj}` is a directory! Set recursive=True for recursive transfer or compress_with_zip=True to zip it."
|
|
585
|
+
)
|
|
586
|
+
|
|
587
|
+
def search_files(directory_path: str, json_output_path: str) -> list[str]:
|
|
588
|
+
from pathlib import Path
|
|
589
|
+
import json
|
|
590
|
+
|
|
591
|
+
file_paths_list = [
|
|
592
|
+
file_path.as_posix() for file_path in Path(directory_path).expanduser().absolute().rglob("*") if file_path.is_file()
|
|
593
|
+
]
|
|
594
|
+
json_result_path = Path(json_output_path)
|
|
595
|
+
json_result_path.parent.mkdir(parents=True, exist_ok=True)
|
|
596
|
+
json_result_path.write_text(json.dumps(file_paths_list, indent=2), encoding="utf-8")
|
|
597
|
+
print(json_result_path.as_posix())
|
|
598
|
+
return file_paths_list
|
|
599
|
+
|
|
600
|
+
remote_json_output = Path.home().joinpath(f"{DEFAULT_PICKLE_SUBDIR}/return_{randstr()}.json").as_posix()
|
|
601
|
+
command = lambda_to_python_script(
|
|
602
|
+
lmb=lambda: search_files(directory_path=expanded_source, json_output_path=remote_json_output), in_global=True, import_module=False
|
|
603
|
+
)
|
|
604
|
+
response = self.run_py(
|
|
605
|
+
python_code=command,
|
|
606
|
+
uv_with=[MACHINECONFIG_VERSION],
|
|
607
|
+
uv_project_dir=None,
|
|
608
|
+
description="Searching for files in source",
|
|
609
|
+
verbose_output=False,
|
|
610
|
+
strict_stderr=False,
|
|
611
|
+
strict_return_code=False,
|
|
612
|
+
)
|
|
613
|
+
remote_json_path = response.op.strip()
|
|
614
|
+
if not remote_json_path:
|
|
615
|
+
raise RuntimeError(f"Could not resolve source path {source} - no response from remote")
|
|
616
|
+
|
|
617
|
+
local_json = Path.home().joinpath(f"{DEFAULT_PICKLE_SUBDIR}/local_{randstr()}.json")
|
|
618
|
+
self._simple_sftp_get(remote_path=remote_json_path, local_path=local_json)
|
|
619
|
+
import json
|
|
620
|
+
|
|
621
|
+
try:
|
|
622
|
+
source_list_str = json.loads(local_json.read_text(encoding="utf-8"))
|
|
623
|
+
except (json.JSONDecodeError, FileNotFoundError) as err:
|
|
624
|
+
raise RuntimeError(f"Could not resolve source path {source} - invalid JSON response: {err}") from err
|
|
625
|
+
finally:
|
|
626
|
+
if local_json.exists():
|
|
627
|
+
local_json.unlink()
|
|
628
|
+
assert isinstance(source_list_str, list), f"Could not resolve source path {source}"
|
|
629
|
+
file_paths_to_download = [Path(file_path_str) for file_path_str in source_list_str]
|
|
630
|
+
|
|
631
|
+
if target is None:
|
|
632
|
+
|
|
633
|
+
def collapse_to_home_dir(absolute_path: str, json_output_path: str) -> str:
|
|
634
|
+
from pathlib import Path
|
|
635
|
+
import json
|
|
636
|
+
|
|
637
|
+
source_absolute_path = Path(absolute_path).expanduser().absolute()
|
|
638
|
+
try:
|
|
639
|
+
relative_to_home = source_absolute_path.relative_to(Path.home())
|
|
640
|
+
collapsed_path_posix = (Path("~") / relative_to_home).as_posix()
|
|
641
|
+
json_result_path = Path(json_output_path)
|
|
642
|
+
json_result_path.parent.mkdir(parents=True, exist_ok=True)
|
|
643
|
+
json_result_path.write_text(json.dumps(collapsed_path_posix, indent=2), encoding="utf-8")
|
|
644
|
+
print(json_result_path.as_posix())
|
|
645
|
+
return collapsed_path_posix
|
|
646
|
+
except ValueError:
|
|
647
|
+
raise RuntimeError(f"Source path must be relative to home directory: {source_absolute_path}")
|
|
648
|
+
|
|
649
|
+
remote_json_output = Path.home().joinpath(f"{DEFAULT_PICKLE_SUBDIR}/return_{randstr()}.json").as_posix()
|
|
650
|
+
command = lambda_to_python_script(
|
|
651
|
+
lmb=lambda: collapse_to_home_dir(absolute_path=expanded_source, json_output_path=remote_json_output),
|
|
652
|
+
in_global=True,
|
|
653
|
+
import_module=False,
|
|
654
|
+
)
|
|
655
|
+
response = self.run_py(
|
|
656
|
+
python_code=command,
|
|
657
|
+
uv_with=[MACHINECONFIG_VERSION],
|
|
658
|
+
uv_project_dir=None,
|
|
659
|
+
description="Finding default target via relative source path",
|
|
660
|
+
verbose_output=False,
|
|
661
|
+
strict_stderr=False,
|
|
662
|
+
strict_return_code=False,
|
|
663
|
+
)
|
|
664
|
+
remote_json_path_dir = response.op.strip()
|
|
665
|
+
if not remote_json_path_dir:
|
|
666
|
+
raise RuntimeError("Could not resolve target path - no response from remote")
|
|
667
|
+
|
|
668
|
+
local_json_dir = Path.home().joinpath(f"{DEFAULT_PICKLE_SUBDIR}/local_{randstr()}.json")
|
|
669
|
+
self._simple_sftp_get(remote_path=remote_json_path_dir, local_path=local_json_dir)
|
|
670
|
+
import json
|
|
671
|
+
|
|
672
|
+
try:
|
|
673
|
+
target_dir_str = json.loads(local_json_dir.read_text(encoding="utf-8"))
|
|
674
|
+
except (json.JSONDecodeError, FileNotFoundError) as err:
|
|
675
|
+
raise RuntimeError(f"Could not resolve target path - invalid JSON response: {err}") from err
|
|
676
|
+
finally:
|
|
677
|
+
if local_json_dir.exists():
|
|
678
|
+
local_json_dir.unlink()
|
|
679
|
+
assert isinstance(target_dir_str, str), "Could not resolve target path"
|
|
680
|
+
target = Path(target_dir_str)
|
|
681
|
+
|
|
682
|
+
target_dir = Path(target).expanduser().absolute()
|
|
683
|
+
|
|
684
|
+
for idx, file_path in enumerate(file_paths_to_download):
|
|
685
|
+
print(f" {idx + 1:03d}. {file_path}")
|
|
686
|
+
|
|
687
|
+
for file_path in file_paths_to_download:
|
|
688
|
+
local_file_target = target_dir.joinpath(Path(file_path).relative_to(expanded_source))
|
|
689
|
+
self.copy_to_here(source=file_path, target=local_file_target, compress_with_zip=False, recursive=False, internal_call=True)
|
|
690
|
+
|
|
691
|
+
return None
|
|
692
|
+
|
|
693
|
+
if compress_with_zip:
|
|
694
|
+
print("🗜️ ZIPPING ...")
|
|
695
|
+
|
|
696
|
+
def zip_source(path_to_zip: str, json_output_path: str) -> str:
|
|
697
|
+
from pathlib import Path
|
|
698
|
+
import shutil
|
|
699
|
+
import json
|
|
700
|
+
|
|
701
|
+
source_to_compress = Path(path_to_zip).expanduser().absolute()
|
|
702
|
+
archive_base_path = source_to_compress.parent / (source_to_compress.name + "_archive")
|
|
703
|
+
if source_to_compress.is_dir():
|
|
704
|
+
shutil.make_archive(str(archive_base_path), "zip", source_to_compress)
|
|
705
|
+
else:
|
|
706
|
+
shutil.make_archive(str(archive_base_path), "zip", source_to_compress.parent, source_to_compress.name)
|
|
707
|
+
zip_file_path = str(archive_base_path) + ".zip"
|
|
708
|
+
json_result_path = Path(json_output_path)
|
|
709
|
+
json_result_path.parent.mkdir(parents=True, exist_ok=True)
|
|
710
|
+
json_result_path.write_text(json.dumps(zip_file_path, indent=2), encoding="utf-8")
|
|
711
|
+
print(json_result_path.as_posix())
|
|
712
|
+
return zip_file_path
|
|
713
|
+
|
|
714
|
+
remote_json_output = Path.home().joinpath(f"{DEFAULT_PICKLE_SUBDIR}/return_{randstr()}.json").as_posix()
|
|
715
|
+
command = lambda_to_python_script(
|
|
716
|
+
lmb=lambda: zip_source(path_to_zip=expanded_source, json_output_path=remote_json_output), in_global=True, import_module=False
|
|
717
|
+
)
|
|
718
|
+
response = self.run_py(
|
|
719
|
+
python_code=command,
|
|
720
|
+
uv_with=[MACHINECONFIG_VERSION],
|
|
721
|
+
uv_project_dir=None,
|
|
722
|
+
description=f"Zipping source file {source}",
|
|
723
|
+
verbose_output=False,
|
|
724
|
+
strict_stderr=False,
|
|
725
|
+
strict_return_code=False,
|
|
726
|
+
)
|
|
727
|
+
remote_json_path = response.op.strip()
|
|
728
|
+
if not remote_json_path:
|
|
729
|
+
raise RuntimeError(f"Could not zip {source} - no response from remote")
|
|
730
|
+
|
|
731
|
+
local_json = Path.home().joinpath(f"{DEFAULT_PICKLE_SUBDIR}/local_{randstr()}.json")
|
|
732
|
+
self._simple_sftp_get(remote_path=remote_json_path, local_path=local_json)
|
|
733
|
+
import json
|
|
734
|
+
|
|
735
|
+
try:
|
|
736
|
+
zipped_path = json.loads(local_json.read_text(encoding="utf-8"))
|
|
737
|
+
except (json.JSONDecodeError, FileNotFoundError) as err:
|
|
738
|
+
raise RuntimeError(f"Could not zip {source} - invalid JSON response: {err}") from err
|
|
739
|
+
finally:
|
|
740
|
+
if local_json.exists():
|
|
741
|
+
local_json.unlink()
|
|
742
|
+
assert isinstance(zipped_path, str), f"Could not zip {source}"
|
|
743
|
+
source_obj = Path(zipped_path)
|
|
744
|
+
expanded_source = zipped_path
|
|
745
|
+
|
|
300
746
|
if target is None:
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
747
|
+
|
|
748
|
+
def collapse_to_home(absolute_path: str, json_output_path: str) -> str:
|
|
749
|
+
from pathlib import Path
|
|
750
|
+
import json
|
|
751
|
+
|
|
752
|
+
source_absolute_path = Path(absolute_path).expanduser().absolute()
|
|
753
|
+
try:
|
|
754
|
+
relative_to_home = source_absolute_path.relative_to(Path.home())
|
|
755
|
+
collapsed_path_posix = (Path("~") / relative_to_home).as_posix()
|
|
756
|
+
json_result_path = Path(json_output_path)
|
|
757
|
+
json_result_path.parent.mkdir(parents=True, exist_ok=True)
|
|
758
|
+
json_result_path.write_text(json.dumps(collapsed_path_posix, indent=2), encoding="utf-8")
|
|
759
|
+
print(json_result_path.as_posix())
|
|
760
|
+
return collapsed_path_posix
|
|
761
|
+
except ValueError:
|
|
762
|
+
raise RuntimeError(f"Source path must be relative to home directory: {source_absolute_path}")
|
|
763
|
+
|
|
764
|
+
remote_json_output = Path.home().joinpath(f"{DEFAULT_PICKLE_SUBDIR}/return_{randstr()}.json").as_posix()
|
|
765
|
+
command = lambda_to_python_script(
|
|
766
|
+
lmb=lambda: collapse_to_home(absolute_path=expanded_source, json_output_path=remote_json_output), in_global=True, import_module=False
|
|
767
|
+
)
|
|
768
|
+
response = self.run_py(
|
|
769
|
+
python_code=command,
|
|
770
|
+
uv_with=[MACHINECONFIG_VERSION],
|
|
771
|
+
uv_project_dir=None,
|
|
772
|
+
description="Finding default target via relative source path",
|
|
773
|
+
verbose_output=False,
|
|
774
|
+
strict_stderr=False,
|
|
775
|
+
strict_return_code=False,
|
|
776
|
+
)
|
|
777
|
+
remote_json_path = response.op.strip()
|
|
778
|
+
if not remote_json_path:
|
|
779
|
+
raise RuntimeError("Could not resolve target path - no response from remote")
|
|
780
|
+
|
|
781
|
+
local_json = Path.home().joinpath(f"{DEFAULT_PICKLE_SUBDIR}/local_{randstr()}.json")
|
|
782
|
+
self._simple_sftp_get(remote_path=remote_json_path, local_path=local_json)
|
|
783
|
+
import json
|
|
784
|
+
|
|
785
|
+
try:
|
|
786
|
+
target_str = json.loads(local_json.read_text(encoding="utf-8"))
|
|
787
|
+
except (json.JSONDecodeError, FileNotFoundError) as err:
|
|
788
|
+
raise RuntimeError(f"Could not resolve target path - invalid JSON response: {err}") from err
|
|
789
|
+
finally:
|
|
790
|
+
if local_json.exists():
|
|
791
|
+
local_json.unlink()
|
|
792
|
+
assert isinstance(target_str, str), "Could not resolve target path"
|
|
793
|
+
target = Path(target_str)
|
|
794
|
+
assert str(target).startswith("~"), f"If target is not specified, source must be relative to home.\n{target=}"
|
|
795
|
+
|
|
796
|
+
target_obj = Path(target).expanduser().absolute()
|
|
308
797
|
target_obj.parent.mkdir(parents=True, exist_ok=True)
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
798
|
+
|
|
799
|
+
if compress_with_zip and target_obj.suffix != ".zip":
|
|
800
|
+
target_obj = target_obj.with_suffix(target_obj.suffix + ".zip")
|
|
801
|
+
|
|
802
|
+
print(f"""📥 [DOWNLOAD] Receiving: {expanded_source} ==> Local Path: {target_obj}""")
|
|
803
|
+
try:
|
|
804
|
+
with self.tqdm_wrap(ascii=True, unit="b", unit_scale=True) as pbar:
|
|
805
|
+
if self.sftp is None: # type: ignore[unreachable]
|
|
806
|
+
raise RuntimeError(f"SFTP connection lost for {self.hostname}")
|
|
807
|
+
self.sftp.get(remotepath=expanded_source, localpath=str(target_obj), callback=pbar.view_bar) # type: ignore
|
|
808
|
+
except Exception:
|
|
809
|
+
if target_obj.exists():
|
|
810
|
+
target_obj.unlink()
|
|
811
|
+
raise
|
|
812
|
+
|
|
813
|
+
if compress_with_zip:
|
|
814
|
+
import zipfile
|
|
815
|
+
|
|
816
|
+
extract_to = target_obj.parent / target_obj.stem
|
|
817
|
+
with zipfile.ZipFile(target_obj, "r") as zip_ref:
|
|
818
|
+
zip_ref.extractall(extract_to)
|
|
819
|
+
target_obj.unlink()
|
|
820
|
+
target_obj = extract_to
|
|
821
|
+
|
|
822
|
+
def delete_temp_zip(path_to_delete: str) -> None:
|
|
823
|
+
from pathlib import Path
|
|
824
|
+
import shutil
|
|
825
|
+
|
|
826
|
+
file_or_dir_path = Path(path_to_delete)
|
|
827
|
+
if file_or_dir_path.exists():
|
|
828
|
+
if file_or_dir_path.is_dir():
|
|
829
|
+
shutil.rmtree(file_or_dir_path)
|
|
830
|
+
else:
|
|
831
|
+
file_or_dir_path.unlink()
|
|
832
|
+
|
|
833
|
+
command = lambda_to_python_script(lmb=lambda: delete_temp_zip(path_to_delete=expanded_source), in_global=True, import_module=False)
|
|
834
|
+
self.run_py(
|
|
835
|
+
python_code=command,
|
|
836
|
+
uv_with=[MACHINECONFIG_VERSION],
|
|
837
|
+
uv_project_dir=None,
|
|
838
|
+
description="Cleaning temp zip files @ remote.",
|
|
839
|
+
verbose_output=False,
|
|
840
|
+
strict_stderr=True,
|
|
841
|
+
strict_return_code=True,
|
|
842
|
+
)
|
|
843
|
+
|
|
351
844
|
print("\n")
|
|
352
|
-
return
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
# print("\nSummary of operations performed:")
|
|
358
|
-
# print(df.to_pandas().to_markdown())
|
|
359
|
-
# if ((df.select('returncode').to_series().to_list()[2:] == [None] * (len(df) - 2)) and (df.select('err').to_series().to_list()[2:] == [''] * (len(df) - 2))): print("\nAll operations completed successfully.\n")
|
|
360
|
-
# else: print("\nSome operations failed. \n")
|
|
361
|
-
# return df
|
|
845
|
+
return None
|
|
846
|
+
|
|
847
|
+
|
|
848
|
+
if __name__ == "__main__":
|
|
849
|
+
ssh = SSH(host="p51s", username=None, hostname=None, ssh_key_path=None, password=None, port=22, enable_compression=False)
|