ezhpcy 0.4.2__tar.gz

ezhpcy-0.4.2/PKG-INFO

@@ -0,0 +1,168 @@
+Metadata-Version: 2.4
+Name: ezhpcy
+Version: 0.4.2
+Summary: A small tool to establish an SSH tunnel to an HPC compute node.
+License-Expression: MIT
+Requires-Dist: pydantic-extra-types>=2.11.1
+Requires-Dist: pydantic-settings>=2.14.2
+Requires-Dist: pydantic>=2.13.5
+Requires-Dist: jinja2>=3.1.6
+Requires-Dist: asyncssh>=2.24.1
+Requires-Dist: rich>=15.0.0
+Requires-Dist: cyclopts>=5.2.0
+Requires-Dist: keyring>=25.7.0
+Requires-Dist: secretstorage>=3.5.0 ; sys_platform == 'linux'
+Requires-Dist: py-machineid>=1.0.0
+Requires-Dist: platformdirs>=4.9.4
+Requires-Dist: filelock>=4.2.0
+Requires-Python: >=3.15
+Description-Content-Type: text/markdown
+
+# EzHPCy
+
+EzHPCy is a small(ish) tool that makes it easy to establish an SSH connection directly
+to a compute node on an HPC system such as the one(s) found at
+Technical University of Denmark ([DTU]).
+
+This allows the user to connect to the compute node using the remote server capabilities
+of their favourite editor.
+
+Key benefits:
+
+- If you use **notebooks**, **debuggers**, and other such editor features, they now run on the compute node instead of login node.
+- Any **terminals** you open in the connected editor are on the compute node.
+- Any LLM extensions now run on the compute not, not login node. This reduces congestion on the login nodes significantly.
+- Works with many editors and other tools (any compliant OpenSSH clients)
+
+It works for networks that have the following properties:
+
+- Compute nodes may or may not have OpenSSH server running
+- User can expose (large, non-root) port on compute nodes
+- Login node can be reached via SSH
+- Login node can reach compute nodes over network
+- Login node and compute node share a filesystem
+
+Both batch jobs and interactive jobs are supported, though you should probably only
+use interactive nodes to avoid invoking the ire of the HPC administrators and
+wasting scarce resources on the shared facility.
+
+Supported schedulers are:
+
+- [IBM Spectrum LSF]: Batch and interactive
+- [PBS Pro/OpenPBS]: Only interactive tested
+
+**Please use EzHPCy responsibly!**
+If you are not familiar with HPC systems, please familiarise yourself with their
+operation and the guidelines of your local HPC system to avoid accidentally
+abusing the resources that have kindly and generously been made available to you.
+
+## Coarse Overview
+
+The operating principle of EzHPCy can be seen in the diagram below. In essence:
+
+- Start an unprivileged, secure OpenSSH server on compute node
+- Forward traffic to this through login node via SSH (`direct-tcpip`)
+
+```mermaid
+sequenceDiagram
+    participant S as ssh ALIAS
+    participant T as ezhpcy tunnel PROFILE
+    participant L as HPC Login Node
+    participant W as HPC Compute Node
+
+    T->>L: Connect
+    T->>L: Submit job
+    L->>W: Start sshd
+    T->>T: Expose sshd as ALIAS
+
+    S->>T: Connect to ALIAS
+    T->>W: Forward via login node
+    S->>W: SSH session
+
+    T->>L: Ctrl+C: cancel job
+```
+
+## Security Considerations
+
+Significant care has been given to the security of EzHPCy.
+The networking setup is relatively simple with a small attack surface.
+Industry standard implementations are used (OpenSSH, Python's `cryptography` library)
+and only a standard, secure protocol is used (SSH).
+
+Importantly, all traffic still goes through the login node and is authenticated using
+the authentication scheme set up by the HPC administrators.
+Nothing is circumvented or bypassed (see slight caveat below).
+
+Because EzHPCy controls both the server and can instruct the client on which
+authentication scheme to use for connections, it is able to harden the server
+significantly:
+
+- Generates a per-host key-pair (`ed25519`) and instructs clients to only trust this key
+- Generates a per-profile key-pair (`ed25519`) and instructs server to only trust this key
+
+This ensures that nobody else on the network can disguise themselves as a server to the
+client, and similarly ensures that no one else on the HPC network can log in as a client.
+
+Locally EzHPCy forwards all traffic on a chosen port to the remote server,
+at which point authentication is done. This means that a local process on the client
+system (such as a malicious user on the same system) can send traffic to the EzHPCy
+managed OpenSSH server, though this would be immediately dropped as the malicious
+user on the local client system would have no means with which to authenticate itself.
+
+**Slight caveat:** EzHPCy allows you to securely store your password in the
+keyring of your local operating system, if you so choose.
+This does not weaken the security of the authentication scheme, but means
+that if you leave your local PC unlocked and unattended,
+someone could use this to access the HPC.
+This is no different to the case where you had a regular connection to the HPC
+open and left your PC unattended.
+As such, this should not be a security concern to any reasonable security practitioner.
+_This is disabled by default and somewhat a somewhat hidden feature to ensure
+that users have some basic grasp of the terminal._
+
+## Implementation
+
+EzHPCy is unapologetically modern and requires Python 3.15 to run.
+
+## License
+
+EzHPCy is released under the [MIT License](LICENSE).
+
+## History
+
+Previous, unreleased versions of EzHPCy used a proxy interface with an IPC protocol (see [Janelia] acknowledgement below).
+
+A significant refactoring was done to switch to [`asyncssh`](https://github.com/ronf/asyncssh) from [`paramiko`](https://www.paramiko.org/) motivated by security, maintainance, and the desire to switch to an `async` implementation.
+
+With this in mind, the CLI interface was first refactored to use the excellent [`cyclopts`](https://cyclopts.readthedocs.io/en/stable/) library by [Brian Pugh](https://github.com/BrianPugh) instead of [`typer`](https://typer.tiangolo.com/) in order to reduce the number of hacks required to achieve the interface I wanted and simultaneously getting native `async` functionality.
+
+## Acknowledgements
+
+I had wondered for a while whether an easier instrumentation of the [DTU] HPC system
+might be possible, in a way that would enable me to connect directly to a compute node
+using my editor or other SSH-capable tools.
+
+I stumbled upon [Janelia]'s configuration [found here](https://github.com/JaneliaSciComp/ssh_cluster_tunnel_for_vscode)
+for their HPC system, which used OpenSSH's
+`ProxyCommand` feature to tunnel through their HPC setup.
+The first decent versions of EzHPCy (v0.1, v0.2) also used `ProxyCommand` and
+a local TCP server in combination with a simple IPC protocol,
+which worked quite well, but was overly complex and had a larger attack surface.
+
+## LLM Disclosure
+
+**This README has been written entirely by a human; please read it!**
+
+The very first versions of EzHPCy were made without the use of agentic programming.
+
+Later refactoring and expansion of EzHPCy has made extensive use of LLMs, though
+under somewhat careful review.
+_I have found that even with the latest models,
+they have to be kept on a relatively short leash,
+lest they take over the project entirely and end up producing something
+overly complex and entirely opaque._
+
+[DTU]: https://www.dtu.dk/english
+[Janelia]: https://www.hhmi.org/research/janelia
+[PBS Pro/OpenPBS]: https://www.openpbs.org/
+[IBM Spectrum LSF]: https://www.ibm.com/docs/en/spectrum-lsf

ezhpcy-0.4.2/README.md

@@ -0,0 +1,148 @@
+# EzHPCy
+
+EzHPCy is a small(ish) tool that makes it easy to establish an SSH connection directly
+to a compute node on an HPC system such as the one(s) found at
+Technical University of Denmark ([DTU]).
+
+This allows the user to connect to the compute node using the remote server capabilities
+of their favourite editor.
+
+Key benefits:
+
+- If you use **notebooks**, **debuggers**, and other such editor features, they now run on the compute node instead of login node.
+- Any **terminals** you open in the connected editor are on the compute node.
+- Any LLM extensions now run on the compute not, not login node. This reduces congestion on the login nodes significantly.
+- Works with many editors and other tools (any compliant OpenSSH clients)
+
+It works for networks that have the following properties:
+
+- Compute nodes may or may not have OpenSSH server running
+- User can expose (large, non-root) port on compute nodes
+- Login node can be reached via SSH
+- Login node can reach compute nodes over network
+- Login node and compute node share a filesystem
+
+Both batch jobs and interactive jobs are supported, though you should probably only
+use interactive nodes to avoid invoking the ire of the HPC administrators and
+wasting scarce resources on the shared facility.
+
+Supported schedulers are:
+
+- [IBM Spectrum LSF]: Batch and interactive
+- [PBS Pro/OpenPBS]: Only interactive tested
+
+**Please use EzHPCy responsibly!**
+If you are not familiar with HPC systems, please familiarise yourself with their
+operation and the guidelines of your local HPC system to avoid accidentally
+abusing the resources that have kindly and generously been made available to you.
+
+## Coarse Overview
+
+The operating principle of EzHPCy can be seen in the diagram below. In essence:
+
+- Start an unprivileged, secure OpenSSH server on compute node
+- Forward traffic to this through login node via SSH (`direct-tcpip`)
+
+```mermaid
+sequenceDiagram
+    participant S as ssh ALIAS
+    participant T as ezhpcy tunnel PROFILE
+    participant L as HPC Login Node
+    participant W as HPC Compute Node
+
+    T->>L: Connect
+    T->>L: Submit job
+    L->>W: Start sshd
+    T->>T: Expose sshd as ALIAS
+
+    S->>T: Connect to ALIAS
+    T->>W: Forward via login node
+    S->>W: SSH session
+
+    T->>L: Ctrl+C: cancel job
+```
+
+## Security Considerations
+
+Significant care has been given to the security of EzHPCy.
+The networking setup is relatively simple with a small attack surface.
+Industry standard implementations are used (OpenSSH, Python's `cryptography` library)
+and only a standard, secure protocol is used (SSH).
+
+Importantly, all traffic still goes through the login node and is authenticated using
+the authentication scheme set up by the HPC administrators.
+Nothing is circumvented or bypassed (see slight caveat below).
+
+Because EzHPCy controls both the server and can instruct the client on which
+authentication scheme to use for connections, it is able to harden the server
+significantly:
+
+- Generates a per-host key-pair (`ed25519`) and instructs clients to only trust this key
+- Generates a per-profile key-pair (`ed25519`) and instructs server to only trust this key
+
+This ensures that nobody else on the network can disguise themselves as a server to the
+client, and similarly ensures that no one else on the HPC network can log in as a client.
+
+Locally EzHPCy forwards all traffic on a chosen port to the remote server,
+at which point authentication is done. This means that a local process on the client
+system (such as a malicious user on the same system) can send traffic to the EzHPCy
+managed OpenSSH server, though this would be immediately dropped as the malicious
+user on the local client system would have no means with which to authenticate itself.
+
+**Slight caveat:** EzHPCy allows you to securely store your password in the
+keyring of your local operating system, if you so choose.
+This does not weaken the security of the authentication scheme, but means
+that if you leave your local PC unlocked and unattended,
+someone could use this to access the HPC.
+This is no different to the case where you had a regular connection to the HPC
+open and left your PC unattended.
+As such, this should not be a security concern to any reasonable security practitioner.
+_This is disabled by default and somewhat a somewhat hidden feature to ensure
+that users have some basic grasp of the terminal._
+
+## Implementation
+
+EzHPCy is unapologetically modern and requires Python 3.15 to run.
+
+## License
+
+EzHPCy is released under the [MIT License](LICENSE).
+
+## History
+
+Previous, unreleased versions of EzHPCy used a proxy interface with an IPC protocol (see [Janelia] acknowledgement below).
+
+A significant refactoring was done to switch to [`asyncssh`](https://github.com/ronf/asyncssh) from [`paramiko`](https://www.paramiko.org/) motivated by security, maintainance, and the desire to switch to an `async` implementation.
+
+With this in mind, the CLI interface was first refactored to use the excellent [`cyclopts`](https://cyclopts.readthedocs.io/en/stable/) library by [Brian Pugh](https://github.com/BrianPugh) instead of [`typer`](https://typer.tiangolo.com/) in order to reduce the number of hacks required to achieve the interface I wanted and simultaneously getting native `async` functionality.
+
+## Acknowledgements
+
+I had wondered for a while whether an easier instrumentation of the [DTU] HPC system
+might be possible, in a way that would enable me to connect directly to a compute node
+using my editor or other SSH-capable tools.
+
+I stumbled upon [Janelia]'s configuration [found here](https://github.com/JaneliaSciComp/ssh_cluster_tunnel_for_vscode)
+for their HPC system, which used OpenSSH's
+`ProxyCommand` feature to tunnel through their HPC setup.
+The first decent versions of EzHPCy (v0.1, v0.2) also used `ProxyCommand` and
+a local TCP server in combination with a simple IPC protocol,
+which worked quite well, but was overly complex and had a larger attack surface.
+
+## LLM Disclosure
+
+**This README has been written entirely by a human; please read it!**
+
+The very first versions of EzHPCy were made without the use of agentic programming.
+
+Later refactoring and expansion of EzHPCy has made extensive use of LLMs, though
+under somewhat careful review.
+_I have found that even with the latest models,
+they have to be kept on a relatively short leash,
+lest they take over the project entirely and end up producing something
+overly complex and entirely opaque._
+
+[DTU]: https://www.dtu.dk/english
+[Janelia]: https://www.hhmi.org/research/janelia
+[PBS Pro/OpenPBS]: https://www.openpbs.org/
+[IBM Spectrum LSF]: https://www.ibm.com/docs/en/spectrum-lsf

ezhpcy-0.4.2/pyproject.toml

@@ -0,0 +1,99 @@
+[project]
+name = "ezhpcy"
+version = "0.4.2"
+license = "MIT"
+readme = "README.md"
+description = "A small tool to establish an SSH tunnel to an HPC compute node."
+requires-python = ">=3.15"
+dependencies = [
+    "pydantic-extra-types>=2.11.1",
+    "pydantic-settings>=2.14.2",
+    "pydantic>=2.13.5",
+    "jinja2>=3.1.6",
+    "asyncssh>=2.24.1",
+    "rich>=15.0.0",
+    "cyclopts>=5.2.0",
+    "keyring>=25.7.0",
+    "secretstorage>=3.5.0 ; sys_platform == 'linux'",
+    "py-machineid>=1.0.0",
+    "platformdirs>=4.9.4",
+    "filelock>=4.2.0",
+]
+
+[project.scripts]
+ezhpcy = "ezhpcy.cli:main"
+
+[dependency-groups]
+format = ["ruff==0.17.1"]
+pre-commit = [
+    "prek==0.5.6",
+    { include-group = "typecheck" },
+]
+test = [
+    "pytest>=9.1.1",
+    "pytest-asyncio>=1.4.0",
+    "pytest-cov>=7.1.0",
+]
+typecheck = ["pyrefly==1.4.0"]
+
+[[dependency-groups.dev]]
+include-group = "format"
+
+[[dependency-groups.dev]]
+include-group = "pre-commit"
+
+[[dependency-groups.dev]]
+include-group = "test"
+
+[[dependency-groups.dev]]
+include-group = "typecheck"
+
+[tool.pytest.ini_options]
+asyncio_mode = "auto"
+asyncio_default_fixture_loop_scope = "function"
+markers = ["real_machine_id: read this machine's ID instead of the fixed one tests get"]
+
+[tool.coverage.run]
+source = ["ezhpcy"]
+branch = true
+
+[tool.coverage.report]
+show_missing = true
+skip_covered = true
+
+[tool.ruff.lint]
+preview = true
+extend-select = [
+    "A",
+    "ASYNC",
+    "B",
+    "DTZ",
+    "EXE",
+    "FURB",
+    "I",
+    "INP",
+    "LOG",
+    "PTH",
+    "T10",
+    "TID",
+    "UP",
+    "YTT",
+]
+ignore = ["RUF106"]
+
+[tool.ruff.lint.isort]
+combine-as-imports = true
+
+[tool.ruff.lint.per-file-ignores]
+"tests/**" = ["RUF069"]
+"scripts/**" = ["INP"]
+
+[tool.pyrefly]
+project-includes = [
+    "src/**/*.py*",
+    "src/**/*.ipynb",
+]
+
+[build-system]
+requires = ["uv_build>=0.13.1,<0.14"]
+build-backend = "uv_build"

ezhpcy-0.4.2/pyproject.toml.orig

@@ -0,0 +1,123 @@
+[project]
+name = "ezhpcy"
+version = "0.4.2"
+
+license = "MIT"
+readme = "README.md"
+description = "A small tool to establish an SSH tunnel to an HPC compute node."
+
+requires-python = ">=3.15"
+dependencies = [
+    # Pydantic
+    "pydantic-extra-types>=2.11.1",
+    "pydantic-settings>=2.14.2",
+    "pydantic>=2.13.5",
+    # Templates
+    "jinja2>=3.1.6",
+    # SSH
+    "asyncssh>=2.24.1",
+    # CLI
+    "rich>=15.0.0",
+    "cyclopts>=5.2.0",
+    # Keyring, secretstorage is for keyring on Linux
+    "keyring>=25.7.0",
+    "secretstorage>=3.5.0 ; sys_platform == 'linux'",
+    # Utils
+    "py-machineid>=1.0.0",
+    "platformdirs>=4.9.4",
+    "filelock>=4.2.0",
+]
+
+[project.scripts]
+ezhpcy = "ezhpcy.cli:main"
+
+[dependency-groups]
+dev = [
+    { include-group = "format" },
+    { include-group = "pre-commit" },
+    { include-group = "test" },
+    { include-group = "typecheck" },
+]
+
+format = [
+    "ruff==0.17.1",
+]
+pre-commit = [
+    "prek==0.5.6", # keep in sync with .github/workflows/pre-commit.yaml
+    { include-group = "typecheck" }, # For pyrefly that uses pyrefly from deps
+]
+test = [
+    "pytest>=9.1.1",
+    "pytest-asyncio>=1.4.0",
+    "pytest-cov>=7.1.0",
+]
+typecheck = [
+    "pyrefly==1.4.0",
+]
+
+# Tests
+[tool.pytest.ini_options]
+# `async def` tests and fixtures run on asyncio without a marker
+asyncio_mode = "auto"
+asyncio_default_fixture_loop_scope = "function"
+markers = [
+    "real_machine_id: read this machine's ID instead of the fixed one tests get",
+]
+
+# Test coverage
+[tool.coverage.run]
+source = ["ezhpcy"]
+branch = true
+
+[tool.coverage.report]
+show_missing = true
+skip_covered = true
+
+# Ruff - Linting
+[tool.ruff.lint]
+preview = true
+extend-select = [
+    "A",     # flake8-builtins
+    "ASYNC", # flake8-async
+    "B",     # flake8-bugbear
+    "DTZ",   # flake8-datetimetz
+    "EXE",   # flake8-executable
+    "FURB",  # refurb
+    "I",     # isort
+    "INP",   # flake8-no-pep420
+    "LOG",   # flake8-logging
+    "PTH",   # flake8-use-pathlib
+    "T10",   # flake8-debugger
+    "TID",   # flake8-tidy-imports
+    "UP",    # pyupgrade
+    "YTT",   # flake8-2020
+]
+ignore = [
+    "RUF106", # rule-codecs-in-suppression-comments
+]
+isort.combine-as-imports = true
+
+[tool.ruff.lint.per-file-ignores]
+# Tests
+"tests/**" = [
+    "RUF069", # float-equality-comparison, in tests we sometimes want eq. check
+]
+
+# Standalone scripts
+"scripts/**" = [
+    "INP", # no-pep420, scripts are standalone and not part of a package
+]
+
+# Pyrefly - Type checking
+[tool.pyrefly]
+project-includes = [
+    "src/**/*.py*",
+    "src/**/*.ipynb",
+]
+
+# Build system
+[build-system]
+# Keep the lower bound in sync with the setup-uv `version` in .github/workflows/
+# (pre-commit.yaml, tests.yml, publish.yml x2)
+requires = ["uv_build>=0.13.1,<0.14"]
+build-backend = "uv_build"