deployr 1.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
deployr-1.1.0/LICENSE ADDED
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Harikrishna T P
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
deployr-1.1.0/PKG-INFO ADDED
@@ -0,0 +1,136 @@
1
+ Metadata-Version: 2.4
2
+ Name: deployr
3
+ Version: 1.1.0
4
+ Summary: Interactive and scriptable CLI for managing Wikimedia Toolforge tools, webservices, and Kubernetes jobs.
5
+ Author-email: Harikrishna T P <tpharikrishna5@gmail.com>
6
+ License-Expression: MIT
7
+ Project-URL: Homepage, https://github.com/angrezichatterbox/toolforge_toolkit
8
+ Project-URL: Repository, https://github.com/angrezichatterbox/toolforge_toolkit
9
+ Project-URL: Issues, https://github.com/angrezichatterbox/toolforge_toolkit/issues
10
+ Keywords: wikimedia,toolforge,cli,deployment,kubernetes,ssh
11
+ Classifier: Development Status :: 4 - Beta
12
+ Classifier: Environment :: Console
13
+ Classifier: Intended Audience :: Developers
14
+ Classifier: Operating System :: POSIX :: Linux
15
+ Classifier: Programming Language :: Python :: 3
16
+ Classifier: Programming Language :: Python :: 3.9
17
+ Classifier: Programming Language :: Python :: 3.10
18
+ Classifier: Programming Language :: Python :: 3.11
19
+ Classifier: Programming Language :: Python :: 3.12
20
+ Classifier: Topic :: Software Development :: Build Tools
21
+ Classifier: Topic :: System :: Systems Administration
22
+ Requires-Python: >=3.9
23
+ Description-Content-Type: text/markdown
24
+ License-File: LICENSE
25
+ Requires-Dist: click>=8.1.0
26
+ Dynamic: license-file
27
+
28
+ # Deployr
29
+
30
+ > **Deployr** is an interactive and scriptable CLI for managing [Wikimedia Toolforge](https://wikitech.wikimedia.org/wiki/Portal:Toolforge) tools.
31
+
32
+ [![PyPI version](https://badge.fury.io/py/deployr.svg)](https://badge.fury.io/py/deployr)
33
+ [![Python 3.9+](https://img.shields.io/badge/python-3.9+-blue.svg)](https://www.python.org/)
34
+ [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](LICENSE)
35
+
36
+ ---
37
+
38
+ ## Features
39
+
40
+ - 🔐 **SSH session management** — connect to the bastion or drop into a tool shell
41
+ - 🌐 **Webservice control** — start, stop, restart your Toolforge webservice
42
+ - 🚀 **Guided Flask deployment** — bundle, upload, venv-build, and restart in one command
43
+ - ⚙️ **Kubernetes jobs** — run, list, delete, and tail logs of Toolforge jobs
44
+ - 📁 **File uploads** — securely transfer files via the two-step SCP handshake
45
+ - 🔌 **SSH DB tunnels** — forward Wikimedia database ports to your local machine
46
+
47
+ ---
48
+
49
+ ## Installation
50
+
51
+ ```bash
52
+ pip install deployr
53
+ ```
54
+
55
+ **Prerequisites:**
56
+ - A [Wikitech](https://wikitech.wikimedia.org) developer account
57
+ - Your SSH public key registered in [Wikitech Preferences](https://wikitech.wikimedia.org/wiki/Special:Preferences)
58
+ - A registered Toolforge tool account
59
+
60
+ ---
61
+
62
+ ## Usage
63
+
64
+ ### Interactive Console (default)
65
+ ```bash
66
+ deployr
67
+ ```
68
+
69
+ ### Scriptable Subcommands
70
+
71
+ ```bash
72
+ # SSH
73
+ deployr ssh # Connect to bastion
74
+ deployr shell # Switch to tool shell (become)
75
+
76
+ # Webservice
77
+ deployr webservice status
78
+ deployr webservice start --type python3.11
79
+ deployr webservice stop
80
+ deployr webservice restart
81
+ deployr webservice deploy app.py --python python3.11
82
+
83
+ # Kubernetes Jobs
84
+ deployr jobs list
85
+ deployr jobs run my-job "python3 script.py" --image python3.11
86
+ deployr jobs run daily-job "python3 report.py" --image python3.11 --schedule "0 0 * * *"
87
+ deployr jobs delete my-job
88
+ deployr jobs logs my-job # stdout log
89
+ deployr jobs logs my-job --err # stderr log
90
+
91
+ # File Transfer
92
+ deployr upload ./my-project .
93
+
94
+ # Database Tunnel
95
+ deployr tunnel --local-port 3306 --remote-host tools.db.svc.wikimedia.cloud
96
+
97
+ # Configuration
98
+ deployr configure
99
+ ```
100
+
101
+ ---
102
+
103
+ ## First-Time Setup
104
+
105
+ On first run, Deployr will prompt you for:
106
+
107
+ | Field | Example |
108
+ |---|---|
109
+ | Wikimedia Username | `your-username` |
110
+ | Default Tool Name | `my-tool` *(without `tools.` prefix)* |
111
+ | Path to SSH Key | `~/.ssh/id_ed25519` |
112
+ | Bastion Host | `login.toolforge.org` |
113
+
114
+ Config is saved to `~/.toolforge_config.json`.
115
+
116
+ ---
117
+
118
+ ## Deploying a Flask App
119
+
120
+ ```bash
121
+ deployr webservice deploy ./app.py --app-var app --python python3.11
122
+ ```
123
+
124
+ This will:
125
+ 1. Bundle your project (excluding `.git`, `.venv`, `__pycache__`)
126
+ 2. Generate an `app.py` wrapper if your entry point is non-standard
127
+ 3. SCP the bundle to remote staging
128
+ 4. Launch a Kubernetes job to build the virtual environment
129
+ 5. Install `requirements.txt` inside the Toolforge Python container
130
+ 6. Restart the webservice
131
+
132
+ ---
133
+
134
+ ## License
135
+
136
+ MIT © Harikrishna T P
@@ -0,0 +1,109 @@
1
+ # Deployr
2
+
3
+ > **Deployr** is an interactive and scriptable CLI for managing [Wikimedia Toolforge](https://wikitech.wikimedia.org/wiki/Portal:Toolforge) tools.
4
+
5
+ [![PyPI version](https://badge.fury.io/py/deployr.svg)](https://badge.fury.io/py/deployr)
6
+ [![Python 3.9+](https://img.shields.io/badge/python-3.9+-blue.svg)](https://www.python.org/)
7
+ [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](LICENSE)
8
+
9
+ ---
10
+
11
+ ## Features
12
+
13
+ - 🔐 **SSH session management** — connect to the bastion or drop into a tool shell
14
+ - 🌐 **Webservice control** — start, stop, restart your Toolforge webservice
15
+ - 🚀 **Guided Flask deployment** — bundle, upload, venv-build, and restart in one command
16
+ - ⚙️ **Kubernetes jobs** — run, list, delete, and tail logs of Toolforge jobs
17
+ - 📁 **File uploads** — securely transfer files via the two-step SCP handshake
18
+ - 🔌 **SSH DB tunnels** — forward Wikimedia database ports to your local machine
19
+
20
+ ---
21
+
22
+ ## Installation
23
+
24
+ ```bash
25
+ pip install deployr
26
+ ```
27
+
28
+ **Prerequisites:**
29
+ - A [Wikitech](https://wikitech.wikimedia.org) developer account
30
+ - Your SSH public key registered in [Wikitech Preferences](https://wikitech.wikimedia.org/wiki/Special:Preferences)
31
+ - A registered Toolforge tool account
32
+
33
+ ---
34
+
35
+ ## Usage
36
+
37
+ ### Interactive Console (default)
38
+ ```bash
39
+ deployr
40
+ ```
41
+
42
+ ### Scriptable Subcommands
43
+
44
+ ```bash
45
+ # SSH
46
+ deployr ssh # Connect to bastion
47
+ deployr shell # Switch to tool shell (become)
48
+
49
+ # Webservice
50
+ deployr webservice status
51
+ deployr webservice start --type python3.11
52
+ deployr webservice stop
53
+ deployr webservice restart
54
+ deployr webservice deploy app.py --python python3.11
55
+
56
+ # Kubernetes Jobs
57
+ deployr jobs list
58
+ deployr jobs run my-job "python3 script.py" --image python3.11
59
+ deployr jobs run daily-job "python3 report.py" --image python3.11 --schedule "0 0 * * *"
60
+ deployr jobs delete my-job
61
+ deployr jobs logs my-job # stdout log
62
+ deployr jobs logs my-job --err # stderr log
63
+
64
+ # File Transfer
65
+ deployr upload ./my-project .
66
+
67
+ # Database Tunnel
68
+ deployr tunnel --local-port 3306 --remote-host tools.db.svc.wikimedia.cloud
69
+
70
+ # Configuration
71
+ deployr configure
72
+ ```
73
+
74
+ ---
75
+
76
+ ## First-Time Setup
77
+
78
+ On first run, Deployr will prompt you for:
79
+
80
+ | Field | Example |
81
+ |---|---|
82
+ | Wikimedia Username | `your-username` |
83
+ | Default Tool Name | `my-tool` *(without `tools.` prefix)* |
84
+ | Path to SSH Key | `~/.ssh/id_ed25519` |
85
+ | Bastion Host | `login.toolforge.org` |
86
+
87
+ Config is saved to `~/.toolforge_config.json`.
88
+
89
+ ---
90
+
91
+ ## Deploying a Flask App
92
+
93
+ ```bash
94
+ deployr webservice deploy ./app.py --app-var app --python python3.11
95
+ ```
96
+
97
+ This will:
98
+ 1. Bundle your project (excluding `.git`, `.venv`, `__pycache__`)
99
+ 2. Generate an `app.py` wrapper if your entry point is non-standard
100
+ 3. SCP the bundle to remote staging
101
+ 4. Launch a Kubernetes job to build the virtual environment
102
+ 5. Install `requirements.txt` inside the Toolforge Python container
103
+ 6. Restart the webservice
104
+
105
+ ---
106
+
107
+ ## License
108
+
109
+ MIT © Harikrishna T P
@@ -0,0 +1,2 @@
1
+ """Deployr — Toolforge Deployment Suite."""
2
+ __version__ = "1.1.0"
@@ -0,0 +1,831 @@
1
+ #!/usr/bin/env python3
2
+ """
3
+ Wikimedia Toolforge Manager CLI
4
+ An interactive and scriptable command-line utility to manage Toolforge tools,
5
+ run jobs, control webservices, SSH into the bastion host, and transfer files.
6
+ """
7
+
8
+ import os
9
+ import sys
10
+ import json
11
+ import subprocess
12
+ import shutil
13
+ import tempfile
14
+ import uuid
15
+ from pathlib import Path
16
+ import click
17
+
18
+ # Try importing readline for better input editing support
19
+ try:
20
+ import readline
21
+ except ImportError:
22
+ pass
23
+
24
+ CONFIG_FILE_NAME = ".toolforge_config.json"
25
+
26
+ def get_config_path():
27
+ """Returns the path to the configuration file in the user's home directory."""
28
+ return Path.home() / CONFIG_FILE_NAME
29
+
30
+ def load_config():
31
+ """Loads configuration from the home directory, or returns defaults."""
32
+ config_path = get_config_path()
33
+ if config_path.is_file():
34
+ try:
35
+ with open(config_path, "r", encoding="utf-8") as f:
36
+ return json.load(f)
37
+ except Exception as e:
38
+ click.secho(f"Error loading config from {config_path}: {e}", fg="red")
39
+ return {
40
+ "username": "",
41
+ "tool_name": "",
42
+ "ssh_key": "",
43
+ "bastion_host": "login.toolforge.org"
44
+ }
45
+
46
+ def save_config(config):
47
+ """Saves the configuration to the user's home directory."""
48
+ config_path = get_config_path()
49
+ try:
50
+ with open(config_path, "w", encoding="utf-8") as f:
51
+ json.dump(config, f, indent=4)
52
+ click.secho(f"Configuration saved to {config_path}", fg="green")
53
+ return True
54
+ except Exception as e:
55
+ click.secho(f"Error saving config: {e}", fg="red")
56
+ return False
57
+
58
+ def print_header():
59
+ """Prints a beautiful CLI header."""
60
+ click.echo("\n" + "=" * 65)
61
+ click.secho(" ____ __ ", fg="cyan", bold=True)
62
+ click.secho(" / __ \\___ ____ / /___ __ ______ ", fg="cyan", bold=True)
63
+ click.secho(" / / / / _ \\/ __ \\/ / __ \\/ / / / ___/ ", fg="cyan", bold=True)
64
+ click.secho(" / /_/ / __/ /_/ / / /_/ / /_/ / / ", fg="cyan", bold=True)
65
+ click.secho("/_____/\\___/ .___/_/\\____/\\__, /_/ ", fg="cyan", bold=True)
66
+ click.secho(" /_/ /____/ ", fg="cyan", bold=True)
67
+ click.secho(" Deployr - Toolforge Deployment Suite v1.1.0 ", fg="blue", bold=True)
68
+ click.echo("=" * 65 + "\n")
69
+
70
+ def configure_settings(config):
71
+ """Configures or updates settings."""
72
+ click.secho("\n--- Configure Toolforge CLI Settings ---", fg="cyan", bold=True)
73
+ config["username"] = click.prompt("Wikimedia Username", default=config.get("username", ""))
74
+ config["tool_name"] = click.prompt("Default Tool Name (without 'tools.' prefix)", default=config.get("tool_name", ""))
75
+
76
+ ssh_key_default = config.get("ssh_key", "")
77
+ config["ssh_key"] = click.prompt("Path to Private SSH Key (blank for default SSH agent)", default=ssh_key_default, show_default=False)
78
+
79
+ config["bastion_host"] = click.prompt("Bastion Host", default=config.get("bastion_host", "login.toolforge.org"))
80
+
81
+ save_config(config)
82
+ return config
83
+
84
+ def check_config(config):
85
+ """Ensures username and tool name are set; configures them if missing."""
86
+ if not config.get("username"):
87
+ click.secho("No Wikimedia username found. Let's configure it first!", fg="yellow")
88
+ config = configure_settings(config)
89
+ return config
90
+
91
+ def get_ssh_cmd_base(config, tty=False):
92
+ """Constructs the base SSH command list."""
93
+ cmd = ["ssh"]
94
+ if tty:
95
+ cmd.append("-t")
96
+ if config.get("ssh_key"):
97
+ cmd.extend(["-i", os.path.expanduser(config["ssh_key"])])
98
+
99
+ username = config["username"]
100
+ host = config["bastion_host"]
101
+ cmd.append(f"{username}@{host}")
102
+ return cmd
103
+
104
+ def run_ssh_session(config, command=None, as_tool=False):
105
+ """
106
+ Runs an interactive or non-interactive SSH session.
107
+ If command is provided, executes that command.
108
+ If as_tool is True, runs 'become <tool>' first.
109
+ """
110
+ cmd = get_ssh_cmd_base(config, tty=True)
111
+
112
+ # Target command building
113
+ remote_cmd = ""
114
+ if as_tool:
115
+ tool_name = config.get("tool_name")
116
+ if not tool_name:
117
+ tool_name = click.prompt("Enter tool name")
118
+ if not tool_name:
119
+ click.secho("Tool name required to run as tool.", fg="red")
120
+ return False
121
+
122
+ if command:
123
+ # Run specific command inside the tool shell
124
+ remote_cmd = f"sudo -i -u tools.{tool_name} {command}"
125
+ else:
126
+ # Drop into tool bash shell
127
+ remote_cmd = f"become {tool_name}"
128
+ else:
129
+ if command:
130
+ remote_cmd = command
131
+
132
+ if remote_cmd:
133
+ cmd.append(remote_cmd)
134
+
135
+ try:
136
+ click.secho("Connecting to Toolforge...", fg="blue")
137
+ if remote_cmd:
138
+ click.secho(f"Executing: {remote_cmd}\n", fg="yellow")
139
+ subprocess.run(cmd, check=True)
140
+ return True
141
+ except subprocess.CalledProcessError as e:
142
+ click.secho(f"\nSSH session exited with error: {e}", fg="red")
143
+ return False
144
+ except KeyboardInterrupt:
145
+ click.secho(f"\nSSH session interrupted.", fg="yellow")
146
+ return True
147
+
148
+ def run_tool_command_capture(config, command):
149
+ """
150
+ Runs a command on the bastion as the tool, captures and returns stdout/stderr.
151
+ This is used for non-interactive commands where we want to parse or display results in the UI.
152
+ """
153
+ tool_name = config.get("tool_name")
154
+ if not tool_name:
155
+ click.secho("Default tool name is not set!", fg="red")
156
+ return None, "No tool name configured"
157
+
158
+ # We construct a base SSH command without allocating a tty to safely capture output
159
+ ssh_base = get_ssh_cmd_base(config, tty=False)
160
+
161
+ # Execute command inside the tool context using sudo
162
+ remote_cmd = f"sudo -i -u tools.{tool_name} {command}"
163
+ ssh_base.append(remote_cmd)
164
+
165
+ try:
166
+ result = subprocess.run(ssh_base, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, check=True)
167
+ return result.stdout, None
168
+ except subprocess.CalledProcessError as e:
169
+ return None, e.stderr or e.stdout or str(e)
170
+
171
+ def upload_files(config, local_path=None, dest_path=None):
172
+ """Uploads files/directories to the tool's directory using scp or rsync."""
173
+ tool_name = config.get("tool_name")
174
+ if not tool_name:
175
+ click.secho("Default tool name is not set!", fg="red")
176
+ return
177
+
178
+ if local_path is None:
179
+ click.secho("\n--- Copy Files to Toolforge ---", fg="cyan", bold=True)
180
+ local_path = click.prompt("Enter local file/directory path to upload", type=click.Path(exists=True))
181
+
182
+ local_path_expanded = os.path.expanduser(local_path)
183
+
184
+ if dest_path is None:
185
+ dest_path = click.prompt("Enter destination path relative to tool home (e.g. '.' or 'public_html')", default=".")
186
+
187
+ click.secho("\nMethod: We will copy to your personal directory first, then move it to the tool's folder.", fg="yellow")
188
+ personal_dest = f"~/tf_transfer_{tool_name}"
189
+
190
+ scp_cmd = ["scp", "-r"]
191
+ if config.get("ssh_key"):
192
+ scp_cmd.extend(["-i", os.path.expanduser(config["ssh_key"])])
193
+
194
+ scp_cmd.extend([local_path_expanded, f"{config['username']}@{config['bastion_host']}:{personal_dest}"])
195
+
196
+ try:
197
+ click.secho("Uploading to temporary personal storage on bastion...", fg="blue")
198
+ subprocess.run(scp_cmd, check=True)
199
+
200
+ # Now, run a command on bastion to move/copy it to the tool's folder using sudo
201
+ click.secho(f"Moving files to tool's home folder (/data/project/{tool_name}/{dest_path})...", fg="blue")
202
+
203
+ # Clean destination path if it's '.'
204
+ remote_dest = f"/data/project/{tool_name}/{dest_path if dest_path != '.' else ''}"
205
+
206
+ # We run ssh to copy files from user home to tools folder
207
+ move_cmd = (
208
+ f"sudo -u tools.{tool_name} mkdir -p {remote_dest} && "
209
+ f"sudo -u tools.{tool_name} cp -r {personal_dest}/* {remote_dest}/ 2>/dev/null || "
210
+ f"sudo -u tools.{tool_name} cp -r {personal_dest} {remote_dest}/ && "
211
+ f"rm -rf {personal_dest}"
212
+ )
213
+
214
+ ssh_cmd = get_ssh_cmd_base(config, tty=False)
215
+ ssh_cmd.append(move_cmd)
216
+ subprocess.run(ssh_cmd, check=True)
217
+
218
+ click.secho("Successfully uploaded files to toolforge!", fg="green")
219
+ except Exception as e:
220
+ click.secho(f"File upload failed: {e}", fg="red")
221
+
222
+ def deploy_flask_app(config, local_flask_path=None, app_var_name=None, local_req_path=None, upload_mode=None, python_version=None):
223
+ """Deploys a local Flask application to Toolforge automatically."""
224
+ tool_name = config.get("tool_name")
225
+ if not tool_name:
226
+ click.secho("Default tool name is not set!", fg="red")
227
+ return
228
+
229
+ if local_flask_path is None:
230
+ click.secho("\n--- Deploy Python/Flask Web App ---", fg="cyan", bold=True)
231
+ click.echo("This helper will package your Flask application, generate the required")
232
+ click.echo("WSGI entrypoint ('wsgi.py'), set up a virtual environment on Toolforge,")
233
+ click.echo("install dependencies, and start/restart the Python web service.")
234
+
235
+ local_flask_path = click.prompt("Enter local Flask app entry file (e.g. app.py, main.py)", type=click.Path(exists=True))
236
+
237
+ local_flask_expanded = os.path.expanduser(local_flask_path)
238
+ flask_dir = os.path.dirname(os.path.abspath(local_flask_expanded))
239
+
240
+ if app_var_name is None:
241
+ app_var_name = click.prompt("Flask application variable name (inside that file)", default="app")
242
+
243
+ if local_req_path is None:
244
+ req_default = os.path.join(flask_dir, "requirements.txt")
245
+ if os.path.exists(req_default):
246
+ if click.confirm(f"Found requirements.txt at {req_default}. Use it?", default=True):
247
+ local_req_path = req_default
248
+ else:
249
+ local_req_path = click.prompt("Enter local requirements.txt path (optional, press Enter if none)", default="", show_default=False)
250
+ else:
251
+ local_req_path = click.prompt("Enter local requirements.txt path (optional, press Enter if none)", default="", show_default=False)
252
+
253
+ if local_req_path:
254
+ local_req_expanded = os.path.expanduser(local_req_path)
255
+ if not os.path.exists(local_req_expanded):
256
+ click.secho(f"Local requirements file does not exist: {local_req_path}", fg="red")
257
+ return
258
+ else:
259
+ local_req_expanded = ""
260
+
261
+ if upload_mode is None:
262
+ click.echo(f"\nDeployment options:")
263
+ click.echo("1. Upload ONLY the Flask entry file")
264
+ click.echo("2. Upload ENTIRE directory containing the file (Recommended for multi-file apps)")
265
+ upload_choice = click.prompt("Choose option", default="2", type=click.Choice(["1", "2"]))
266
+ upload_mode = "dir" if upload_choice == "2" else "file"
267
+ elif upload_mode not in ("dir", "file"):
268
+ click.secho(f"Invalid upload mode: {upload_mode}. Must be 'dir' or 'file'.", fg="red")
269
+ return
270
+
271
+ if python_version is None:
272
+ python_version = click.prompt("Target Python runtime version (e.g., python3.11, python3.9)", default="python3.11")
273
+
274
+ click.secho("\nPreparing deployment package...", fg="blue")
275
+
276
+ # Create local temporary directory
277
+ temp_dir = tempfile.mkdtemp()
278
+ flask_file_name = os.path.basename(local_flask_expanded)
279
+ module_name = os.path.splitext(flask_file_name)[0]
280
+
281
+ try:
282
+ if upload_mode == "dir":
283
+ src_dir = os.path.dirname(os.path.abspath(local_flask_expanded))
284
+
285
+ def ignore_patterns(path, names):
286
+ ignored = []
287
+ for name in names:
288
+ if name in ('__pycache__', '.git', '.venv', 'venv', 'node_modules') or name.endswith('.pyc'):
289
+ ignored.append(name)
290
+ return ignored
291
+
292
+ for item in os.listdir(src_dir):
293
+ s = os.path.join(src_dir, item)
294
+ d = os.path.join(temp_dir, item)
295
+ if os.path.isdir(s):
296
+ if item not in ('__pycache__', '.git', '.venv', 'venv', 'node_modules'):
297
+ shutil.copytree(s, d, ignore=ignore_patterns)
298
+ else:
299
+ if not item.endswith('.pyc'):
300
+ shutil.copy2(s, d)
301
+ else:
302
+ shutil.copy2(local_flask_expanded, os.path.join(temp_dir, flask_file_name))
303
+
304
+ # Copy requirements.txt to temp_dir if provided explicitly or found during dir copy
305
+ if local_req_expanded and not os.path.exists(os.path.join(temp_dir, "requirements.txt")):
306
+ shutil.copy2(local_req_expanded, os.path.join(temp_dir, "requirements.txt"))
307
+
308
+ # If the flask app file is not app.py or the app variable is not app,
309
+ # we generate a wrapper app.py so that uWSGI can find it natively.
310
+ if flask_file_name != "app.py" or app_var_name != "app":
311
+ app_wrapper_content = f"""import sys
312
+ import os
313
+
314
+ # Add directory to sys.path
315
+ sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
316
+
317
+ try:
318
+ from {module_name} import {app_var_name} as app
319
+ except Exception as e:
320
+ print("Error importing Flask application: " + str(e))
321
+ raise e
322
+ """
323
+ with open(os.path.join(temp_dir, "app.py"), "w", encoding="utf-8") as f:
324
+ f.write(app_wrapper_content)
325
+
326
+ # Generate random unique ID for remote temporary directory
327
+ deploy_id = str(uuid.uuid4())[:8]
328
+ remote_temp = f"/tmp/tf_deploy_{tool_name}_{deploy_id}"
329
+
330
+ # Generate the deploy.sh script inside the temporary directory
331
+ deploy_sh_content = f"""#!/bin/bash
332
+ set -e
333
+
334
+ echo "Preparing www/python/src directory..."
335
+ mkdir -p /data/project/{tool_name}/www/python/src
336
+
337
+ # Empty www/python/src to prevent old leftover file conflicts
338
+ echo "Clearing old code in www/python/src..."
339
+ rm -rf /data/project/{tool_name}/www/python/src/* 2>/dev/null || true
340
+
341
+ echo "Copying new code to www/python/src..."
342
+ cp -r {remote_temp}/* /data/project/{tool_name}/www/python/src/
343
+ chmod -R 755 /data/project/{tool_name}/www/python/src
344
+
345
+ # Remove deploy.sh from www/python/src to keep it clean
346
+ rm -f /data/project/{tool_name}/www/python/src/deploy.sh
347
+
348
+ if [ -f "/data/project/{tool_name}/www/python/src/requirements.txt" ]; then
349
+ echo "requirements.txt detected. Checking Python Virtual Environment..."
350
+ echo "Creating virtual environment and installing requirements via a Toolforge Kubernetes Job..."
351
+ echo "This ensures we match the exact {python_version} environment of your web service."
352
+
353
+ # Delete old venv first to ensure a clean slate
354
+ rm -rf /data/project/{tool_name}/www/python/venv
355
+
356
+ # Delete any existing setup-venv job to prevent name collision
357
+ toolforge jobs delete setup-venv 2>/dev/null || true
358
+
359
+ # Run the setup via a one-off Kubernetes Job with the matching image
360
+ toolforge jobs run setup-venv \\
361
+ --command "/bin/bash -c 'python3 -m venv /data/project/{tool_name}/www/python/venv && /data/project/{tool_name}/www/python/venv/bin/pip install --upgrade pip && /data/project/{tool_name}/www/python/venv/bin/pip install -r /data/project/{tool_name}/www/python/src/requirements.txt'" \\
362
+ --image {python_version} \\
363
+ --wait
364
+
365
+ # Clean up the job definition after run
366
+ toolforge jobs delete setup-venv
367
+ fi
368
+
369
+ echo "Deploy finished. Web service starting..."
370
+ toolforge webservice {python_version} restart || toolforge webservice {python_version} start
371
+ """
372
+ with open(os.path.join(temp_dir, "deploy.sh"), "w", encoding="utf-8") as f:
373
+ f.write(deploy_sh_content)
374
+
375
+ # SCP local temp dir to remote /tmp
376
+ scp_cmd = ["scp", "-r"]
377
+ if config.get("ssh_key"):
378
+ scp_cmd.extend(["-i", os.path.expanduser(config["ssh_key"])])
379
+
380
+ scp_cmd.extend([temp_dir, f"{config['username']}@{config['bastion_host']}:{remote_temp}"])
381
+
382
+ click.secho("Uploading deployment bundle to remote staging...", fg="blue")
383
+ subprocess.run(scp_cmd, check=True)
384
+
385
+ # Give permission so tools user can read files inside /tmp folder
386
+ click.secho("Adjusting permissions...", fg="blue")
387
+ chmod_cmd = f"chmod -R 777 {remote_temp}"
388
+ ssh_chmod = get_ssh_cmd_base(config, tty=False)
389
+ ssh_chmod.append(chmod_cmd)
390
+ subprocess.run(ssh_chmod, check=True)
391
+
392
+ # Clean up old source directories owned by the SSH user first to prevent permission-denied issues
393
+ click.secho("Clearing old application directories using SSH user to resolve folder permissions...", fg="blue")
394
+ clear_cmd = f"rm -rf /data/project/{tool_name}/public_html /data/project/{tool_name}/www/python/src 2>/dev/null || true"
395
+ ssh_clear = get_ssh_cmd_base(config, tty=False)
396
+ ssh_clear.append(clear_cmd)
397
+ subprocess.run(ssh_clear, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
398
+
399
+ click.secho("Running installation and restarting webservice as tool user...", fg="blue")
400
+ # Run it inside tool's bash
401
+ run_ssh_session(config, f"bash {remote_temp}/deploy.sh", as_tool=True)
402
+
403
+ # Clean up remote temp
404
+ click.secho("Cleaning up remote staging folder...", fg="blue")
405
+ cleanup_cmd = f"rm -rf {remote_temp}"
406
+ ssh_cleanup = get_ssh_cmd_base(config, tty=False)
407
+ ssh_cleanup.append(cleanup_cmd)
408
+ subprocess.run(ssh_cleanup, check=True)
409
+
410
+ click.secho(f"\n★ Flask application deployed successfully! ★", fg="green", bold=True)
411
+ click.secho(f"Deployment URL: https://{tool_name}.toolforge.org/", fg="cyan", bold=True)
412
+
413
+ except Exception as e:
414
+ click.secho(f"\nDeployment failed: {e}", fg="red")
415
+ finally:
416
+ # Clean up local temp folder
417
+ if os.path.exists(temp_dir):
418
+ shutil.rmtree(temp_dir)
419
+
420
+ def _draw_box(title, lines, width=54):
421
+ """Draws a Unicode box with a title and content lines."""
422
+ click.secho(f"╔{'═' * width}╗", fg="cyan")
423
+ click.secho(f"║ {click.style(title, bold=True, fg='cyan'):<{width + 9}}║", fg="cyan")
424
+ click.secho(f"╠{'═' * width}╣", fg="cyan")
425
+ for line in lines:
426
+ click.secho(f"║ {line:<{width - 2}}║", fg="cyan")
427
+ click.secho(f"╚{'═' * width}╝", fg="cyan")
428
+
429
+ def manage_webservice(config):
430
+ """Sub-menu to manage Toolforge web services."""
431
+ tool_name = config.get("tool_name")
432
+ if not tool_name:
433
+ click.secho("Default tool name is not set!", fg="red")
434
+ return
435
+
436
+ while True:
437
+ click.clear()
438
+ _draw_box(
439
+ f"Web Service Management [{tool_name}]",
440
+ [
441
+ click.style(" 1", fg="green", bold=True) + " View Web Service Status",
442
+ click.style(" 2", fg="green", bold=True) + " Start Web Service",
443
+ click.style(" 3", fg="red", bold=True) + " Stop Web Service",
444
+ click.style(" 4", fg="yellow", bold=True) + " Restart Web Service",
445
+ click.style(" 5", fg="blue", bold=True) + " Deploy Python/Flask Web App",
446
+ "",
447
+ click.style(" 6", bold=True) + " ← Back to Main Menu",
448
+ ]
449
+ )
450
+
451
+ choice = click.prompt("\nChoose an option", default="1", type=click.Choice(["1", "2", "3", "4", "5", "6"]))
452
+
453
+ if choice == "1":
454
+ out, err = run_tool_command_capture(config, "toolforge webservice status")
455
+ if out:
456
+ click.secho("\n● Webservice Status", fg="green", bold=True)
457
+ click.echo(out)
458
+ else:
459
+ click.secho(f"\n✗ Error fetching status: {err}", fg="red")
460
+ click.pause(info="\nPress any key to continue...")
461
+ elif choice == "2":
462
+ ws_type = click.prompt("Framework/type (e.g. python3.11, node20, php8.2, buildservice)", default="buildservice")
463
+ click.secho(f"\n▶ Starting '{ws_type}'...", fg="blue")
464
+ run_ssh_session(config, f"toolforge webservice {ws_type} start", as_tool=True)
465
+ click.pause(info="\nPress any key to continue...")
466
+ elif choice == "3":
467
+ if click.confirm(click.style("\n⚠ Are you sure you want to STOP the webservice?", fg="red", bold=True), default=False):
468
+ click.secho("■ Stopping webservice...", fg="red")
469
+ run_ssh_session(config, "toolforge webservice stop", as_tool=True)
470
+ click.pause(info="\nPress any key to continue...")
471
+ else:
472
+ click.secho("Cancelled.", fg="yellow")
473
+ elif choice == "4":
474
+ if click.confirm(click.style("\n↺ Restart the webservice?", fg="yellow", bold=True), default=True):
475
+ click.secho("↺ Restarting webservice...", fg="yellow")
476
+ run_ssh_session(config, "toolforge webservice restart", as_tool=True)
477
+ click.pause(info="\nPress any key to continue...")
478
+ elif choice == "5":
479
+ deploy_flask_app(config)
480
+ elif choice == "6":
481
+ break
482
+
483
+ def manage_jobs(config):
484
+ """Sub-menu to manage Kubernetes Jobs."""
485
+ tool_name = config.get("tool_name")
486
+ if not tool_name:
487
+ click.secho("Default tool name is not set!", fg="red")
488
+ return
489
+
490
+ while True:
491
+ click.clear()
492
+ _draw_box(
493
+ f"Kubernetes Jobs [{tool_name}]",
494
+ [
495
+ click.style(" 1", fg="green", bold=True) + " List Jobs",
496
+ click.style(" 2", fg="blue", bold=True) + " Run a New Job (One-off or Scheduled)",
497
+ click.style(" 3", fg="red", bold=True) + " Delete a Job",
498
+ click.style(" 4", fg="cyan", bold=True) + " View Job Logs",
499
+ "",
500
+ click.style(" 5", bold=True) + " ← Back to Main Menu",
501
+ ]
502
+ )
503
+
504
+ choice = click.prompt("\nChoose an option", default="1", type=click.Choice(["1", "2", "3", "4", "5"]))
505
+
506
+ if choice == "1":
507
+ out, err = run_tool_command_capture(config, "toolforge jobs list")
508
+ if out:
509
+ click.secho("\n● Kubernetes Jobs", fg="green", bold=True)
510
+ click.echo_via_pager(out)
511
+ else:
512
+ click.secho(f"\n✗ Error fetching jobs list: {err}", fg="red")
513
+ click.pause(info="\nPress any key to continue...")
514
+ elif choice == "2":
515
+ job_name = click.prompt("\nJob name")
516
+ if not job_name:
517
+ click.secho("Job name is required.", fg="red")
518
+ continue
519
+ command = click.prompt("Command to run (e.g. python3 my_script.py)")
520
+ if not command:
521
+ click.secho("Command is required.", fg="red")
522
+ continue
523
+ image = click.prompt("Container image", default="python3.11")
524
+ schedule = click.prompt("Cron schedule (optional, blank = one-off)", default="", show_default=False)
525
+
526
+ run_cmd = f"toolforge jobs run {job_name} --command \"{command}\" --image {image}"
527
+ if schedule:
528
+ run_cmd += f" --schedule \"{schedule}\""
529
+
530
+ click.secho(f"\n▶ Submitting job '{job_name}'...", fg="blue")
531
+ run_ssh_session(config, run_cmd, as_tool=True)
532
+ click.pause(info="\nPress any key to continue...")
533
+
534
+ elif choice == "3":
535
+ job_name = click.prompt("\nJob name to delete")
536
+ if not job_name:
537
+ continue
538
+ if click.confirm(click.style(f"\n⚠ Permanently delete job '{job_name}'?", fg="red", bold=True), default=False):
539
+ click.secho(f"✗ Deleting job '{job_name}'...", fg="red")
540
+ run_ssh_session(config, f"toolforge jobs delete {job_name}", as_tool=True)
541
+ click.pause(info="\nPress any key to continue...")
542
+ else:
543
+ click.secho("Cancelled.", fg="yellow")
544
+
545
+ elif choice == "4":
546
+ job_name = click.prompt("\nJob name to check logs")
547
+ if not job_name:
548
+ continue
549
+ log_choice = click.prompt(
550
+ "Log type",
551
+ default="out",
552
+ type=click.Choice(["out", "err"]),
553
+ show_choices=True
554
+ )
555
+ log_file = f"/data/project/{tool_name}/{job_name}.{log_choice}"
556
+ click.secho(f"\n📄 Fetching {log_file} ...", fg="blue")
557
+ # capture and page the output
558
+ out, err = run_tool_command_capture(
559
+ config,
560
+ f"tail -n 200 {log_file} 2>/dev/null || echo 'No logs found at {log_file}'"
561
+ )
562
+ if out:
563
+ click.echo_via_pager(out)
564
+ else:
565
+ click.secho(f"✗ Could not fetch logs: {err}", fg="red")
566
+ click.pause(info="\nPress any key to continue...")
567
+
568
+ elif choice == "5":
569
+ break
570
+
571
+ def setup_ssh_tunnel(config, local_port=None, remote_host=None, remote_port=None):
572
+ """Establishes an SSH tunnel/port forwarding for database access or similar."""
573
+ click.secho("\n--- Establish Port Forwarding / SSH Tunnel ---", fg="cyan", bold=True)
574
+ click.echo("Toolforge databases are only accessible from within the Toolforge cluster.")
575
+ click.echo("This feature allows you to forward a database or service port to your local machine.")
576
+
577
+ if local_port is None:
578
+ local_port = click.prompt("Enter Local Port (e.g., 3306 for MySQL, 8080 for web)", default="3306")
579
+
580
+ if remote_host is None:
581
+ click.echo("\nDatabase Server names depend on the project (e.g. 'enwiki.web.db.svc.wikimedia.cloud' or 'tools.db.svc.wikimedia.cloud')")
582
+ remote_host = click.prompt("Enter Remote Host/IP on Toolforge", default="tools.db.svc.wikimedia.cloud")
583
+
584
+ if remote_port is None:
585
+ remote_port = click.prompt("Enter Remote Port", default="3306")
586
+
587
+ # We construct the SSH tunnel command
588
+ ssh_tunnel_cmd = ["ssh", "-N", "-L", f"{local_port}:{remote_host}:{remote_port}"]
589
+
590
+ if config.get("ssh_key"):
591
+ ssh_tunnel_cmd.extend(["-i", os.path.expanduser(config["ssh_key"])])
592
+
593
+ ssh_tunnel_cmd.append(f"{config['username']}@{config['bastion_host']}")
594
+
595
+ click.secho(f"\nStarting tunnel: Local port {local_port} -> {remote_host}:{remote_port}", fg="green")
596
+ click.secho("Press Ctrl+C to terminate the tunnel.", fg="yellow")
597
+ click.secho(f"Command running: {' '.join(ssh_tunnel_cmd)}", fg="blue")
598
+
599
+ try:
600
+ subprocess.run(ssh_tunnel_cmd, check=True)
601
+ except KeyboardInterrupt:
602
+ click.secho(f"\nSSH Tunnel terminated.", fg="green")
603
+ except Exception as e:
604
+ click.secho(f"\nError establishing SSH Tunnel: {e}", fg="red")
605
+
606
+ def interactive_console(config):
607
+ """Runs the main interactive loop of the toolforge manager CLI."""
608
+ while True:
609
+ click.clear()
610
+ print_header()
611
+
612
+ # ── Status bar ──────────────────────────────────────────────────────
613
+ tool = config.get('tool_name') or click.style('not set', fg='red')
614
+ user = click.style(config['username'], fg='cyan', bold=True)
615
+ host = click.style(config['bastion_host'], fg='yellow')
616
+ click.echo(f" Tool: {click.style(tool, fg='magenta', bold=True)} User: {user} Host: {host}")
617
+ click.echo()
618
+
619
+ # ── Main menu box ───────────────────────────────────────────────────
620
+ # NOTE: click.style() injects invisible ANSI bytes that break f-string
621
+ # width formatting. We build each row manually: badge + visible text +
622
+ # explicit spaces to reach column W, then the right border.
623
+ W = 54 # visible inner width
624
+
625
+ def _row(badge_styled, badge_visible_len, label, total_inner=W):
626
+ # total_inner = 2 (left pad) + badge_visible_len + 1 (space) + label + padding + 0 (right border handled outside)
627
+ used = 2 + badge_visible_len + 1 + len(label)
628
+ pad = total_inner - used
629
+ return f"║ {badge_styled} {label}{' ' * pad}║"
630
+
631
+ click.secho(f"╔{'═' * W}╗", fg="cyan")
632
+ click.secho(f"║{' ── SSH & Shell':^{W}}║", fg="cyan")
633
+ click.secho(f"╠{'═' * W}╣", fg="cyan")
634
+ click.secho(_row(click.style(' 1 ', fg='black', bg='green'), 3, "SSH to Bastion (Interactive)"), fg="cyan")
635
+ click.secho(_row(click.style(' 2 ', fg='black', bg='green'), 3, "Switch to Tool Shell (become)"), fg="cyan")
636
+ click.secho(f"╠{'═' * W}╣", fg="cyan")
637
+ click.secho(f"║{' ── Manage':^{W}}║", fg="cyan")
638
+ click.secho(f"╠{'═' * W}╣", fg="cyan")
639
+ click.secho(_row(click.style(' 3 ', fg='black', bg='cyan'), 3, "Web Service Control"), fg="cyan")
640
+ click.secho(_row(click.style(' 4 ', fg='black', bg='cyan'), 3, "Kubernetes Jobs"), fg="cyan")
641
+ click.secho(_row(click.style(' 5 ', fg='black', bg='cyan'), 3, "Upload Files / Deploy Code"), fg="cyan")
642
+ click.secho(_row(click.style(' 6 ', fg='black', bg='cyan'), 3, "SSH Database Tunnel"), fg="cyan")
643
+ click.secho(f"╠{'═' * W}╣", fg="cyan")
644
+ click.secho(_row(click.style(' 7 ', fg='black', bg='white'), 3, "Settings"), fg="cyan")
645
+ click.secho(_row(click.style(' 8 ', fg='black', bg='red'), 3, "Exit"), fg="cyan")
646
+ click.secho(f"╚{'═' * W}╝", fg="cyan")
647
+
648
+ choice = click.prompt("\n Choose", default="1", type=click.Choice(["1","2","3","4","5","6","7","8"]))
649
+
650
+ if choice == "1":
651
+ run_ssh_session(config)
652
+ elif choice == "2":
653
+ run_ssh_session(config, as_tool=True)
654
+ elif choice == "3":
655
+ manage_webservice(config)
656
+ elif choice == "4":
657
+ manage_jobs(config)
658
+ elif choice == "5":
659
+ click.clear()
660
+ click.secho("\n Upload / Deploy", fg="cyan", bold=True)
661
+ click.echo(" 1. Copy arbitrary files/directories (scp)")
662
+ click.echo(" 2. Deploy a Python/Flask Web App (guided)")
663
+ deploy_choice = click.prompt(" Choose", default="2", type=click.Choice(["1", "2"]))
664
+ if deploy_choice == "1":
665
+ upload_files(config)
666
+ else:
667
+ deploy_flask_app(config)
668
+ elif choice == "6":
669
+ setup_ssh_tunnel(config)
670
+ elif choice == "7":
671
+ config = configure_settings(config)
672
+ elif choice == "8":
673
+ click.secho("\nThank you for using Deployr. Goodbye!\n", fg="green")
674
+ break
675
+
676
+ # ── CLICK COMMAND LINE INTERFACE DEFINITION ──────────────────────────────────
677
+
678
+ @click.group(invoke_without_command=True)
679
+ @click.pass_context
680
+ def cli(ctx):
681
+ """Deployr - Interactive and scriptable management suite for Wikimedia Toolforge."""
682
+ if ctx.invoked_subcommand is None:
683
+ config = load_config()
684
+ print_header()
685
+ config = check_config(config)
686
+ interactive_console(config)
687
+
688
+ @cli.command("ssh")
689
+ def cli_ssh():
690
+ """Connect directly to the Toolforge bastion server."""
691
+ config = load_config()
692
+ config = check_config(config)
693
+ run_ssh_session(config)
694
+
695
+ @cli.command("shell")
696
+ def cli_shell():
697
+ """Switch context directly to your tool's bash shell (become)."""
698
+ config = load_config()
699
+ config = check_config(config)
700
+ run_ssh_session(config, as_tool=True)
701
+
702
+ @cli.group("webservice")
703
+ def cli_webservice():
704
+ """Manage and deploy web services on Toolforge."""
705
+ pass
706
+
707
+ @cli_webservice.command("status")
708
+ def cli_ws_status():
709
+ """Get the current running status of the webservice."""
710
+ config = load_config()
711
+ config = check_config(config)
712
+ out, err = run_tool_command_capture(config, "toolforge webservice status")
713
+ if out:
714
+ click.secho(f"\n{out}", fg="green")
715
+ else:
716
+ click.secho(f"\nError: {err}", fg="red")
717
+
718
+ @cli_webservice.command("start")
719
+ @click.option("--type", "-t", default="buildservice", help="Webservice framework type (e.g. python3.11, php8.2, buildservice).")
720
+ def cli_ws_start(type):
721
+ """Start the webservice on Toolforge."""
722
+ config = load_config()
723
+ config = check_config(config)
724
+ run_ssh_session(config, f"toolforge webservice {type} start", as_tool=True)
725
+
726
+ @cli_webservice.command("stop")
727
+ def cli_ws_stop():
728
+ """Stop the running webservice on Toolforge."""
729
+ config = load_config()
730
+ config = check_config(config)
731
+ run_ssh_session(config, "toolforge webservice stop", as_tool=True)
732
+
733
+ @cli_webservice.command("restart")
734
+ def cli_ws_restart():
735
+ """Restart the webservice on Toolforge."""
736
+ config = load_config()
737
+ config = check_config(config)
738
+ run_ssh_session(config, "toolforge webservice restart", as_tool=True)
739
+
740
+ @cli_webservice.command("deploy")
741
+ @click.argument("entry_file", type=click.Path(exists=True))
742
+ @click.option("--app-var", default="app", help="Flask app variable name (e.g. app, myapp)")
743
+ @click.option("--requirements", type=click.Path(exists=True), help="Path to local requirements.txt file")
744
+ @click.option("--mode", type=click.Choice(["dir", "file"]), default="dir", help="Upload a single file or the whole directory")
745
+ @click.option("--python", default="python3.11", help="Target Toolforge Python version (e.g. python3.11, python3.9)")
746
+ def cli_ws_deploy(entry_file, app_var, requirements, mode, python):
747
+ """Deploy a Flask application to Toolforge automatically."""
748
+ config = load_config()
749
+ config = check_config(config)
750
+ deploy_flask_app(config, entry_file, app_var, requirements, mode, python)
751
+
752
+ @cli.group("jobs")
753
+ def cli_jobs():
754
+ """Manage Kubernetes jobs running under the tool account."""
755
+ pass
756
+
757
+ @cli_jobs.command("list")
758
+ def cli_jobs_list():
759
+ """List all scheduled and continuous jobs."""
760
+ config = load_config()
761
+ config = check_config(config)
762
+ out, err = run_tool_command_capture(config, "toolforge jobs list")
763
+ if out:
764
+ click.secho(f"\n{out}", fg="green")
765
+ else:
766
+ click.secho(f"\nError: {err}", fg="red")
767
+
768
+ @cli_jobs.command("run")
769
+ @click.argument("name")
770
+ @click.argument("command")
771
+ @click.option("--image", default="python3.11", help="Image tag to run under")
772
+ @click.option("--schedule", help="Cron schedule expression (optional)")
773
+ def cli_jobs_run(name, command, image, schedule):
774
+ """Run a new Kubernetes job on the cluster."""
775
+ config = load_config()
776
+ config = check_config(config)
777
+ run_cmd = f"toolforge jobs run {name} --command \"{command}\" --image {image}"
778
+ if schedule:
779
+ run_cmd += f" --schedule \"{schedule}\""
780
+ run_ssh_session(config, run_cmd, as_tool=True)
781
+
782
+ @cli_jobs.command("delete")
783
+ @click.argument("name")
784
+ def cli_jobs_delete(name):
785
+ """Delete a running or scheduled Kubernetes job."""
786
+ config = load_config()
787
+ config = check_config(config)
788
+ run_ssh_session(config, f"toolforge jobs delete {name}", as_tool=True)
789
+
790
+ @cli_jobs.command("logs")
791
+ @click.argument("name")
792
+ @click.option("--err", is_flag=True, help="Fetch standard error (.err) log instead of standard out")
793
+ def cli_jobs_logs(name, err):
794
+ """Tail the logs of a Kubernetes job (shows last 50 lines)."""
795
+ config = load_config()
796
+ config = check_config(config)
797
+ suffix = "err" if err else "out"
798
+ log_file = f"/data/project/{config['tool_name']}/{name}.{suffix}"
799
+ run_ssh_session(config, f"tail -n 50 {log_file} 2>/dev/null || echo 'No logs found at {log_file}'", as_tool=True)
800
+
801
+ @cli.command("upload")
802
+ @click.argument("local_path", type=click.Path(exists=True))
803
+ @click.argument("dest_path", default=".")
804
+ def cli_upload(local_path, dest_path):
805
+ """Upload a file or directory to Toolforge."""
806
+ config = load_config()
807
+ config = check_config(config)
808
+ upload_files(config, local_path, dest_path)
809
+
810
+ @cli.command("tunnel")
811
+ @click.option("--local-port", "-l", default="3306", help="Local port to bind to")
812
+ @click.option("--remote-host", "-h", default="tools.db.svc.wikimedia.cloud", help="Remote Wikimedia service host name")
813
+ @click.option("--remote-port", "-r", default="3306", help="Remote port of the service")
814
+ def cli_tunnel(local_port, remote_host, remote_port):
815
+ """Open an SSH tunnel to query Wikimedia database servers locally."""
816
+ config = load_config()
817
+ config = check_config(config)
818
+ setup_ssh_tunnel(config, local_port, remote_host, remote_port)
819
+
820
+ @cli.command("configure")
821
+ def cli_configure():
822
+ """Configure your Wikimedia developer credentials."""
823
+ config = load_config()
824
+ configure_settings(config)
825
+
826
+ if __name__ == "__main__":
827
+ try:
828
+ cli()
829
+ except KeyboardInterrupt:
830
+ click.secho(f"\n\nCLI terminated by user. Goodbye!\n", fg="yellow")
831
+ sys.exit(0)
@@ -0,0 +1,136 @@
1
+ Metadata-Version: 2.4
2
+ Name: deployr
3
+ Version: 1.1.0
4
+ Summary: Interactive and scriptable CLI for managing Wikimedia Toolforge tools, webservices, and Kubernetes jobs.
5
+ Author-email: Harikrishna T P <tpharikrishna5@gmail.com>
6
+ License-Expression: MIT
7
+ Project-URL: Homepage, https://github.com/angrezichatterbox/toolforge_toolkit
8
+ Project-URL: Repository, https://github.com/angrezichatterbox/toolforge_toolkit
9
+ Project-URL: Issues, https://github.com/angrezichatterbox/toolforge_toolkit/issues
10
+ Keywords: wikimedia,toolforge,cli,deployment,kubernetes,ssh
11
+ Classifier: Development Status :: 4 - Beta
12
+ Classifier: Environment :: Console
13
+ Classifier: Intended Audience :: Developers
14
+ Classifier: Operating System :: POSIX :: Linux
15
+ Classifier: Programming Language :: Python :: 3
16
+ Classifier: Programming Language :: Python :: 3.9
17
+ Classifier: Programming Language :: Python :: 3.10
18
+ Classifier: Programming Language :: Python :: 3.11
19
+ Classifier: Programming Language :: Python :: 3.12
20
+ Classifier: Topic :: Software Development :: Build Tools
21
+ Classifier: Topic :: System :: Systems Administration
22
+ Requires-Python: >=3.9
23
+ Description-Content-Type: text/markdown
24
+ License-File: LICENSE
25
+ Requires-Dist: click>=8.1.0
26
+ Dynamic: license-file
27
+
28
+ # Deployr
29
+
30
+ > **Deployr** is an interactive and scriptable CLI for managing [Wikimedia Toolforge](https://wikitech.wikimedia.org/wiki/Portal:Toolforge) tools.
31
+
32
+ [![PyPI version](https://badge.fury.io/py/deployr.svg)](https://badge.fury.io/py/deployr)
33
+ [![Python 3.9+](https://img.shields.io/badge/python-3.9+-blue.svg)](https://www.python.org/)
34
+ [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](LICENSE)
35
+
36
+ ---
37
+
38
+ ## Features
39
+
40
+ - 🔐 **SSH session management** — connect to the bastion or drop into a tool shell
41
+ - 🌐 **Webservice control** — start, stop, restart your Toolforge webservice
42
+ - 🚀 **Guided Flask deployment** — bundle, upload, venv-build, and restart in one command
43
+ - ⚙️ **Kubernetes jobs** — run, list, delete, and tail logs of Toolforge jobs
44
+ - 📁 **File uploads** — securely transfer files via the two-step SCP handshake
45
+ - 🔌 **SSH DB tunnels** — forward Wikimedia database ports to your local machine
46
+
47
+ ---
48
+
49
+ ## Installation
50
+
51
+ ```bash
52
+ pip install deployr
53
+ ```
54
+
55
+ **Prerequisites:**
56
+ - A [Wikitech](https://wikitech.wikimedia.org) developer account
57
+ - Your SSH public key registered in [Wikitech Preferences](https://wikitech.wikimedia.org/wiki/Special:Preferences)
58
+ - A registered Toolforge tool account
59
+
60
+ ---
61
+
62
+ ## Usage
63
+
64
+ ### Interactive Console (default)
65
+ ```bash
66
+ deployr
67
+ ```
68
+
69
+ ### Scriptable Subcommands
70
+
71
+ ```bash
72
+ # SSH
73
+ deployr ssh # Connect to bastion
74
+ deployr shell # Switch to tool shell (become)
75
+
76
+ # Webservice
77
+ deployr webservice status
78
+ deployr webservice start --type python3.11
79
+ deployr webservice stop
80
+ deployr webservice restart
81
+ deployr webservice deploy app.py --python python3.11
82
+
83
+ # Kubernetes Jobs
84
+ deployr jobs list
85
+ deployr jobs run my-job "python3 script.py" --image python3.11
86
+ deployr jobs run daily-job "python3 report.py" --image python3.11 --schedule "0 0 * * *"
87
+ deployr jobs delete my-job
88
+ deployr jobs logs my-job # stdout log
89
+ deployr jobs logs my-job --err # stderr log
90
+
91
+ # File Transfer
92
+ deployr upload ./my-project .
93
+
94
+ # Database Tunnel
95
+ deployr tunnel --local-port 3306 --remote-host tools.db.svc.wikimedia.cloud
96
+
97
+ # Configuration
98
+ deployr configure
99
+ ```
100
+
101
+ ---
102
+
103
+ ## First-Time Setup
104
+
105
+ On first run, Deployr will prompt you for:
106
+
107
+ | Field | Example |
108
+ |---|---|
109
+ | Wikimedia Username | `your-username` |
110
+ | Default Tool Name | `my-tool` *(without `tools.` prefix)* |
111
+ | Path to SSH Key | `~/.ssh/id_ed25519` |
112
+ | Bastion Host | `login.toolforge.org` |
113
+
114
+ Config is saved to `~/.toolforge_config.json`.
115
+
116
+ ---
117
+
118
+ ## Deploying a Flask App
119
+
120
+ ```bash
121
+ deployr webservice deploy ./app.py --app-var app --python python3.11
122
+ ```
123
+
124
+ This will:
125
+ 1. Bundle your project (excluding `.git`, `.venv`, `__pycache__`)
126
+ 2. Generate an `app.py` wrapper if your entry point is non-standard
127
+ 3. SCP the bundle to remote staging
128
+ 4. Launch a Kubernetes job to build the virtual environment
129
+ 5. Install `requirements.txt` inside the Toolforge Python container
130
+ 6. Restart the webservice
131
+
132
+ ---
133
+
134
+ ## License
135
+
136
+ MIT © Harikrishna T P
@@ -0,0 +1,11 @@
1
+ LICENSE
2
+ README.md
3
+ pyproject.toml
4
+ deployr/__init__.py
5
+ deployr/cli.py
6
+ deployr.egg-info/PKG-INFO
7
+ deployr.egg-info/SOURCES.txt
8
+ deployr.egg-info/dependency_links.txt
9
+ deployr.egg-info/entry_points.txt
10
+ deployr.egg-info/requires.txt
11
+ deployr.egg-info/top_level.txt
@@ -0,0 +1,2 @@
1
+ [console_scripts]
2
+ deployr = deployr.cli:cli
@@ -0,0 +1 @@
1
+ click>=8.1.0
@@ -0,0 +1 @@
1
+ deployr
@@ -0,0 +1,42 @@
1
+ [build-system]
2
+ requires = ["setuptools>=61", "wheel"]
3
+ build-backend = "setuptools.build_meta"
4
+
5
+ [project]
6
+ name = "deployr"
7
+ version = "1.1.0"
8
+ description = "Interactive and scriptable CLI for managing Wikimedia Toolforge tools, webservices, and Kubernetes jobs."
9
+ readme = "README.md"
10
+ license = "MIT"
11
+ license-files = ["LICENSE"]
12
+ authors = [{ name = "Harikrishna T P", email = "tpharikrishna5@gmail.com" }]
13
+ keywords = ["wikimedia", "toolforge", "cli", "deployment", "kubernetes", "ssh"]
14
+ classifiers = [
15
+ "Development Status :: 4 - Beta",
16
+ "Environment :: Console",
17
+ "Intended Audience :: Developers",
18
+ "Operating System :: POSIX :: Linux",
19
+ "Programming Language :: Python :: 3",
20
+ "Programming Language :: Python :: 3.9",
21
+ "Programming Language :: Python :: 3.10",
22
+ "Programming Language :: Python :: 3.11",
23
+ "Programming Language :: Python :: 3.12",
24
+ "Topic :: Software Development :: Build Tools",
25
+ "Topic :: System :: Systems Administration",
26
+ ]
27
+ requires-python = ">=3.9"
28
+ dependencies = [
29
+ "click>=8.1.0",
30
+ ]
31
+
32
+ [project.urls]
33
+ Homepage = "https://github.com/angrezichatterbox/toolforge_toolkit"
34
+ Repository = "https://github.com/angrezichatterbox/toolforge_toolkit"
35
+ Issues = "https://github.com/angrezichatterbox/toolforge_toolkit/issues"
36
+
37
+ [project.scripts]
38
+ deployr = "deployr.cli:cli"
39
+
40
+ [tool.setuptools.packages.find]
41
+ where = ["."]
42
+ include = ["deployr*"]
@@ -0,0 +1,4 @@
1
+ [egg_info]
2
+ tag_build =
3
+ tag_date = 0
4
+