dockhand-cli 0.3.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dockhand/__init__.py +335 -0
- dockhand/build.py +35 -0
- dockhand/client/__init__.py +38 -0
- dockhand/client/base.py +37 -0
- dockhand/client/local.py +45 -0
- dockhand/client/ssh.py +53 -0
- dockhand/config.py +355 -0
- dockhand/constants.py +2 -0
- dockhand/download.py +53 -0
- dockhand/error.py +10 -0
- dockhand/history.py +171 -0
- dockhand/manage.py +459 -0
- dockhand/queue.py +174 -0
- dockhand/resubmit.py +49 -0
- dockhand/submit.py +163 -0
- dockhand/sync.py +50 -0
- dockhand/tagging.py +48 -0
- dockhand/transport.py +212 -0
- dockhand/tunnel.py +62 -0
- dockhand/volumes.py +149 -0
- dockhand_cli-0.3.1.dist-info/METADATA +478 -0
- dockhand_cli-0.3.1.dist-info/RECORD +25 -0
- dockhand_cli-0.3.1.dist-info/WHEEL +4 -0
- dockhand_cli-0.3.1.dist-info/entry_points.txt +2 -0
- dockhand_cli-0.3.1.dist-info/licenses/LICENSE +7 -0
dockhand/submit.py
ADDED
|
@@ -0,0 +1,163 @@
|
|
|
1
|
+
"""Docker container submission and queuing."""
|
|
2
|
+
from typing import List
|
|
3
|
+
|
|
4
|
+
import typer
|
|
5
|
+
from git import InvalidGitRepositoryError, Repo
|
|
6
|
+
from rich.progress import Progress, SpinnerColumn, TextColumn
|
|
7
|
+
|
|
8
|
+
from dockhand.client import get_client
|
|
9
|
+
from dockhand.config import DockerConfig, cli_config
|
|
10
|
+
from dockhand.history import add_to_history
|
|
11
|
+
from dockhand.sync import execute_sync
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def _build_docker_run_cmd(
|
|
15
|
+
config: DockerConfig,
|
|
16
|
+
commands: List[str],
|
|
17
|
+
imagename: str,
|
|
18
|
+
gpus: str | None,
|
|
19
|
+
effective_ports: list[str] | None,
|
|
20
|
+
mount_code: bool = True,
|
|
21
|
+
run_flags: list[str] | None = None,
|
|
22
|
+
) -> str:
|
|
23
|
+
"""Build the docker run command string.
|
|
24
|
+
|
|
25
|
+
When ``mount_code`` is True (mount delivery) the project directory is bind-mounted
|
|
26
|
+
over ``containerworkdir`` and ``preserve_paths`` anonymous volumes are layered on
|
|
27
|
+
top. When False (bake delivery) the code is already baked into ``imagename``, so no
|
|
28
|
+
code mount or preserve volumes are emitted — only data volumes.
|
|
29
|
+
"""
|
|
30
|
+
code_mounts = []
|
|
31
|
+
if mount_code:
|
|
32
|
+
if config.containerworkdir == "/":
|
|
33
|
+
typer.echo(
|
|
34
|
+
"Warning: containerworkdir is '/' — mounting code at the container root "
|
|
35
|
+
"will shadow the entire filesystem.",
|
|
36
|
+
err=True,
|
|
37
|
+
)
|
|
38
|
+
# Implicit code mount: remote project path → containerworkdir
|
|
39
|
+
code_mounts.append(f"-v {cli_config.remote_path}:{config.containerworkdir}:rw")
|
|
40
|
+
|
|
41
|
+
# Anonymous volumes layered on top of the code mount for paths that must keep the
|
|
42
|
+
# image's build-time contents (e.g. a virtualenv or node_modules baked in at build
|
|
43
|
+
# time) instead of being shadowed by the bind-mounted host project directory.
|
|
44
|
+
workdir = config.containerworkdir.rstrip("/")
|
|
45
|
+
code_mounts += [f"-v {workdir}/{path.strip('/')}" for path in config.preserve_paths]
|
|
46
|
+
|
|
47
|
+
data_volumes = []
|
|
48
|
+
if config.volumes is not None:
|
|
49
|
+
data_volumes = [f"-v {v['hostpath']}:{v['containerpath']}:{v['permissions']}" for v in config.volumes]
|
|
50
|
+
|
|
51
|
+
gpu_flags = [f"--gpus {gpus}"] if gpus is not None else []
|
|
52
|
+
port_flags = [f"-p {mapping}" for mapping in effective_ports] if effective_ports is not None else []
|
|
53
|
+
|
|
54
|
+
# Run flags come from the transport (e.g. ["--rm"] for the queue, or
|
|
55
|
+
# ["-d", "--name", ...] for a detached direct run).
|
|
56
|
+
run_flags = ["--rm"] if run_flags is None else run_flags
|
|
57
|
+
|
|
58
|
+
return " ".join(
|
|
59
|
+
[
|
|
60
|
+
"docker",
|
|
61
|
+
"run",
|
|
62
|
+
*run_flags,
|
|
63
|
+
*code_mounts,
|
|
64
|
+
*data_volumes,
|
|
65
|
+
*gpu_flags,
|
|
66
|
+
*port_flags,
|
|
67
|
+
imagename,
|
|
68
|
+
*commands,
|
|
69
|
+
]
|
|
70
|
+
)
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def _get_branch() -> str | None:
|
|
74
|
+
try:
|
|
75
|
+
with Repo(cli_config.project_root) as repo:
|
|
76
|
+
return repo.active_branch.name
|
|
77
|
+
except (InvalidGitRepositoryError, Exception):
|
|
78
|
+
return None
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def execute_submit(
|
|
82
|
+
config: DockerConfig,
|
|
83
|
+
commands: List[str],
|
|
84
|
+
sync: bool,
|
|
85
|
+
imagename: str | None = None,
|
|
86
|
+
gpus: str | None = None,
|
|
87
|
+
ports: list[str] | None = None,
|
|
88
|
+
urgent: bool = False,
|
|
89
|
+
slots: int | None = None,
|
|
90
|
+
image_ref: str | None = None,
|
|
91
|
+
) -> int:
|
|
92
|
+
"""Optionally sync code, then start a container run. Returns the local job ID.
|
|
93
|
+
|
|
94
|
+
Runs through the queue (task spooler) when ``queue.enabled``, otherwise starts a
|
|
95
|
+
detached container directly. When ``image_ref`` is given (e.g. resubmitting a baked
|
|
96
|
+
job), that exact pre-built image is run verbatim — no re-resolution and no rebuild.
|
|
97
|
+
"""
|
|
98
|
+
from dockhand.build import execute_build
|
|
99
|
+
from dockhand.history import reserve_local_id
|
|
100
|
+
from dockhand.tagging import resolve_image_ref
|
|
101
|
+
from dockhand.transport import get_transport
|
|
102
|
+
|
|
103
|
+
if sync:
|
|
104
|
+
execute_sync(confirm_changes=True)
|
|
105
|
+
|
|
106
|
+
imagename = imagename or config.imagename
|
|
107
|
+
gpus = gpus if gpus is not None else config.gpus
|
|
108
|
+
effective_ports = ports if ports is not None else config.ports
|
|
109
|
+
effective_slots = slots if slots is not None else cli_config.queue.slots
|
|
110
|
+
|
|
111
|
+
transport = get_transport()
|
|
112
|
+
local_id = reserve_local_id()
|
|
113
|
+
|
|
114
|
+
if image_ref is not None:
|
|
115
|
+
# Rerun an exact pre-built image (baked); no re-resolution, no rebuild.
|
|
116
|
+
run_image = image_ref
|
|
117
|
+
mount_code = False
|
|
118
|
+
else:
|
|
119
|
+
delivery = config.resolve_code_delivery(cli_config.queue.enabled)
|
|
120
|
+
if delivery == "bake":
|
|
121
|
+
# Bake the code into an image and run that; no code mount at run time. Queued
|
|
122
|
+
# jobs get an immutable per-submit tag so they stay pinned to their code.
|
|
123
|
+
run_image = resolve_image_ref(imagename, unique=cli_config.queue.enabled)
|
|
124
|
+
execute_build(config, sync=False, dockerfile=config.dockerfile, imagename=run_image)
|
|
125
|
+
mount_code = False
|
|
126
|
+
else:
|
|
127
|
+
run_image = imagename
|
|
128
|
+
mount_code = True
|
|
129
|
+
|
|
130
|
+
docker_cmd = _build_docker_run_cmd(
|
|
131
|
+
config,
|
|
132
|
+
commands,
|
|
133
|
+
run_image,
|
|
134
|
+
gpus,
|
|
135
|
+
effective_ports,
|
|
136
|
+
mount_code=mount_code,
|
|
137
|
+
run_flags=transport.run_flags(local_id),
|
|
138
|
+
)
|
|
139
|
+
|
|
140
|
+
with get_client() as client:
|
|
141
|
+
with Progress(SpinnerColumn(), TextColumn("[progress.description]{task.description}")) as progress:
|
|
142
|
+
task = progress.add_task(description="Submitting job", total=None)
|
|
143
|
+
handle = transport.submit(
|
|
144
|
+
client, docker_cmd, local_id=local_id, slots=effective_slots, urgent=urgent
|
|
145
|
+
)
|
|
146
|
+
progress.update(task, completed=True)
|
|
147
|
+
|
|
148
|
+
host = cli_config.ssh.hostname if cli_config.ssh else "localhost"
|
|
149
|
+
add_to_history(
|
|
150
|
+
config,
|
|
151
|
+
commands=commands,
|
|
152
|
+
local_id=local_id,
|
|
153
|
+
handle=handle,
|
|
154
|
+
image_ref=run_image,
|
|
155
|
+
branch=_get_branch(),
|
|
156
|
+
ports=effective_ports,
|
|
157
|
+
host=host,
|
|
158
|
+
)
|
|
159
|
+
|
|
160
|
+
verb = "Queued" if cli_config.queue.enabled else "Started"
|
|
161
|
+
label = "urgent job" if (urgent and cli_config.queue.enabled) else "job"
|
|
162
|
+
typer.echo(f"{verb} {label} #{local_id}")
|
|
163
|
+
return local_id
|
dockhand/sync.py
ADDED
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
import subprocess
|
|
2
|
+
|
|
3
|
+
import typer
|
|
4
|
+
from git import Repo
|
|
5
|
+
from rich.progress import Progress, SpinnerColumn, TextColumn
|
|
6
|
+
from rich.prompt import Confirm
|
|
7
|
+
|
|
8
|
+
from dockhand.config import cli_config
|
|
9
|
+
from dockhand.error import error_and_exit
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def execute_sync(confirm_changes: bool = True):
|
|
13
|
+
if confirm_changes:
|
|
14
|
+
check_and_confirm_changes()
|
|
15
|
+
|
|
16
|
+
ssh = cli_config.ssh
|
|
17
|
+
source = "./"
|
|
18
|
+
destination = f"{ssh.user}@{ssh.hostname}:{cli_config.remote_path}"
|
|
19
|
+
command = [
|
|
20
|
+
"rsync",
|
|
21
|
+
"-avz",
|
|
22
|
+
"-e",
|
|
23
|
+
f"ssh -i {ssh.identityfile}",
|
|
24
|
+
"--exclude-from=.gitignore",
|
|
25
|
+
"--delete",
|
|
26
|
+
source,
|
|
27
|
+
destination,
|
|
28
|
+
]
|
|
29
|
+
|
|
30
|
+
with Progress(SpinnerColumn(), TextColumn("[progress.description]{task.description}")) as progress:
|
|
31
|
+
task = progress.add_task(description="Syncing", total=None)
|
|
32
|
+
progress.start()
|
|
33
|
+
try:
|
|
34
|
+
subprocess.run(command, check=True, capture_output=True)
|
|
35
|
+
except subprocess.CalledProcessError as e:
|
|
36
|
+
error_and_exit(f"Sync failed:\n{e.stderr.decode()}")
|
|
37
|
+
progress.update(task, completed=True)
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def check_and_confirm_changes():
|
|
41
|
+
with Repo(cli_config.project_root) as repo:
|
|
42
|
+
if repo.is_dirty() or len(repo.untracked_files) != 0:
|
|
43
|
+
prompt = (
|
|
44
|
+
"You have uncommitted changes.\n"
|
|
45
|
+
+ "This may cause problems with switching branches.\n"
|
|
46
|
+
+ "Do you want to continue synchronizing?"
|
|
47
|
+
)
|
|
48
|
+
confirmed = Confirm.ask(prompt, show_default=True, default=True)
|
|
49
|
+
if not confirmed:
|
|
50
|
+
raise typer.Exit()
|
dockhand/tagging.py
ADDED
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
"""Image tag resolution for baked code delivery."""
|
|
2
|
+
import hashlib
|
|
3
|
+
import time
|
|
4
|
+
|
|
5
|
+
from git import Repo
|
|
6
|
+
|
|
7
|
+
from dockhand.config import cli_config
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
def resolve_image_ref(imagename: str, unique: bool) -> str:
|
|
11
|
+
"""Return the image ref to build and run for baked code delivery.
|
|
12
|
+
|
|
13
|
+
``unique=True`` (queued jobs) produces an immutable, content-addressed tag so a
|
|
14
|
+
job waiting in the queue is pinned to the exact code it was submitted with, and
|
|
15
|
+
a later submit cannot retroactively change it. ``unique=False`` (immediate,
|
|
16
|
+
unqueued run) reuses a single tag since there is no drift window to guard against.
|
|
17
|
+
"""
|
|
18
|
+
if not unique:
|
|
19
|
+
return imagename # implicit :latest, rebuilt on each submit
|
|
20
|
+
return f"{imagename}:{_content_tag()}"
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def _content_tag() -> str:
|
|
24
|
+
"""Derive a tag from the project's git state.
|
|
25
|
+
|
|
26
|
+
A clean worktree maps to its commit sha (fully reproducible). A dirty worktree
|
|
27
|
+
folds tracked changes and untracked filenames into the tag so distinct working
|
|
28
|
+
states get distinct images while identical states dedupe. Note that untracked
|
|
29
|
+
file *contents* are not hashed — commit before queued submits for a guaranteed
|
|
30
|
+
immutable snapshot. Falls back to a time-based tag outside a git repo.
|
|
31
|
+
"""
|
|
32
|
+
try:
|
|
33
|
+
repo = Repo(cli_config.project_root)
|
|
34
|
+
sha = repo.head.commit.hexsha[:12]
|
|
35
|
+
dirty = repo.is_dirty(untracked_files=True)
|
|
36
|
+
diff = repo.git.diff() if dirty else ""
|
|
37
|
+
untracked = sorted(repo.untracked_files) if dirty else []
|
|
38
|
+
except Exception:
|
|
39
|
+
return f"ts-{int(time.time())}"
|
|
40
|
+
|
|
41
|
+
if not dirty:
|
|
42
|
+
return sha
|
|
43
|
+
|
|
44
|
+
h = hashlib.sha256()
|
|
45
|
+
h.update(diff.encode())
|
|
46
|
+
for path in untracked:
|
|
47
|
+
h.update(path.encode())
|
|
48
|
+
return f"{sha}-dirty-{h.hexdigest()[:8]}"
|
dockhand/transport.py
ADDED
|
@@ -0,0 +1,212 @@
|
|
|
1
|
+
"""Execution transports: task-spooler queue vs. direct ``docker run``.
|
|
2
|
+
|
|
3
|
+
When ``queue.enabled`` is true, jobs are submitted to task spooler (``tsp``) and run
|
|
4
|
+
in submission order. When it is false there is no queue, so jobs are started
|
|
5
|
+
immediately as detached containers (``docker run -d``) and managed directly with
|
|
6
|
+
``docker logs``/``stop``/``ps``/``rm``.
|
|
7
|
+
|
|
8
|
+
Both backends expose the same interface so ``submit`` and the management commands
|
|
9
|
+
(``logs``/``stop``/``jobs``/``remove``) don't care which is active. A job's transport
|
|
10
|
+
is recorded in history, so per-job commands dispatch to the backend that created it.
|
|
11
|
+
"""
|
|
12
|
+
from abc import ABC, abstractmethod
|
|
13
|
+
|
|
14
|
+
import typer
|
|
15
|
+
|
|
16
|
+
from dockhand.client.base import Client
|
|
17
|
+
from dockhand.config import cli_config
|
|
18
|
+
from dockhand.error import error_and_exit
|
|
19
|
+
from dockhand.queue import ts_get_job, ts_kill, ts_list, ts_make_urgent, ts_remove, ts_submit
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def _container_name(local_id: int) -> str:
|
|
23
|
+
return f"dockhand-{local_id}"
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def get_transport() -> "Transport":
|
|
27
|
+
"""Select the transport for a new submission based on ``queue.enabled``."""
|
|
28
|
+
return TaskSpoolerTransport() if cli_config.queue.enabled else DockerTransport()
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def transport_for_entry(entry: dict) -> "Transport":
|
|
32
|
+
"""Select the transport that created an existing history entry.
|
|
33
|
+
|
|
34
|
+
Old entries predate the ``transport`` field and were always task spooler.
|
|
35
|
+
"""
|
|
36
|
+
name = entry.get("transport", TaskSpoolerTransport.name)
|
|
37
|
+
return _TRANSPORTS.get(name, TaskSpoolerTransport())
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def entry_handle(entry: dict):
|
|
41
|
+
"""The backend handle for an entry (tsp job id, or container name)."""
|
|
42
|
+
return entry.get("handle", entry.get("ts_job_id"))
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
class Transport(ABC):
|
|
46
|
+
name: str
|
|
47
|
+
|
|
48
|
+
@abstractmethod
|
|
49
|
+
def run_flags(self, local_id: int) -> list[str]:
|
|
50
|
+
"""Flags injected right after ``docker run`` (e.g. ``--rm`` or ``-d --name``)."""
|
|
51
|
+
|
|
52
|
+
@abstractmethod
|
|
53
|
+
def submit(self, client: Client, docker_cmd: str, *, local_id: int, slots: int, urgent: bool) -> dict:
|
|
54
|
+
"""Start the job. Returns a handle dict merged into the history entry."""
|
|
55
|
+
|
|
56
|
+
@abstractmethod
|
|
57
|
+
def list_jobs(self, client: Client) -> list[dict]:
|
|
58
|
+
"""Live jobs as dicts: ``{handle, state, command}`` (states normalized)."""
|
|
59
|
+
|
|
60
|
+
@abstractmethod
|
|
61
|
+
def container_name(self, entry: dict) -> str | None:
|
|
62
|
+
"""The docker container name backing this job, for ``docker inspect`` (e.g. exact
|
|
63
|
+
start time). None if the job predates deterministic container naming."""
|
|
64
|
+
|
|
65
|
+
@abstractmethod
|
|
66
|
+
def logs(self, client: Client, entry: dict, *, n: int | None, follow: bool) -> int:
|
|
67
|
+
"""Stream a job's logs. Returns the command's exit code."""
|
|
68
|
+
|
|
69
|
+
@abstractmethod
|
|
70
|
+
def stop(self, client: Client, entry: dict) -> bool:
|
|
71
|
+
"""Stop a running job."""
|
|
72
|
+
|
|
73
|
+
@abstractmethod
|
|
74
|
+
def remove(self, client: Client, entry: dict) -> bool:
|
|
75
|
+
"""Remove a job that hasn't started (queued) / clean up its container."""
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
class TaskSpoolerTransport(Transport):
|
|
79
|
+
name = "task_spooler"
|
|
80
|
+
|
|
81
|
+
def run_flags(self, local_id: int) -> list[str]:
|
|
82
|
+
# Named (as well as --rm) so a running job's container can still be inspected for
|
|
83
|
+
# its exact start time — ts itself doesn't expose one.
|
|
84
|
+
return ["--rm", "--name", _container_name(local_id)]
|
|
85
|
+
|
|
86
|
+
def submit(self, client, docker_cmd, *, local_id, slots, urgent):
|
|
87
|
+
job_id = ts_submit(client, docker_cmd, cwd=cli_config.remote_path, slots=slots)
|
|
88
|
+
if urgent:
|
|
89
|
+
ts_make_urgent(client, job_id, cwd=cli_config.remote_path)
|
|
90
|
+
return {"transport": self.name, "handle": job_id, "ts_job_id": job_id}
|
|
91
|
+
|
|
92
|
+
def list_jobs(self, client):
|
|
93
|
+
jobs = ts_list(client, cwd=cli_config.remote_path)
|
|
94
|
+
return [
|
|
95
|
+
{
|
|
96
|
+
"handle": j["id"],
|
|
97
|
+
"state": j["state"],
|
|
98
|
+
"command": j["command"],
|
|
99
|
+
"duration_seconds": j.get("duration_seconds"),
|
|
100
|
+
}
|
|
101
|
+
for j in jobs
|
|
102
|
+
]
|
|
103
|
+
|
|
104
|
+
def container_name(self, entry):
|
|
105
|
+
local_id = entry.get("local_id")
|
|
106
|
+
# Only jobs submitted after this transport started naming containers have one.
|
|
107
|
+
return _container_name(local_id) if local_id is not None else None
|
|
108
|
+
|
|
109
|
+
def logs(self, client, entry, *, n, follow):
|
|
110
|
+
job_id = entry_handle(entry)
|
|
111
|
+
if follow:
|
|
112
|
+
cmd = f"tail -f $(tsp -o {job_id})"
|
|
113
|
+
elif n is not None:
|
|
114
|
+
cmd = f"tail -n {n} $(tsp -o {job_id})"
|
|
115
|
+
else:
|
|
116
|
+
cmd = f"cat $(tsp -o {job_id})"
|
|
117
|
+
returncode, _ = client.run(cmd, cwd=cli_config.remote_path)
|
|
118
|
+
return returncode
|
|
119
|
+
|
|
120
|
+
def stop(self, client, entry):
|
|
121
|
+
job_id = entry_handle(entry)
|
|
122
|
+
job = ts_get_job(client, job_id, cwd=cli_config.remote_path)
|
|
123
|
+
if job is None:
|
|
124
|
+
error_and_exit(f"Job (tsp {job_id}) not found in task spooler.")
|
|
125
|
+
if job["state"] != "running":
|
|
126
|
+
error_and_exit(f"Job is not running (state: {job['state']}). Use 'remove' for queued jobs.")
|
|
127
|
+
return ts_kill(client, job_id, cwd=cli_config.remote_path)
|
|
128
|
+
|
|
129
|
+
def remove(self, client, entry):
|
|
130
|
+
job_id = entry_handle(entry)
|
|
131
|
+
job = ts_get_job(client, job_id, cwd=cli_config.remote_path)
|
|
132
|
+
if job and job["state"] != "queued":
|
|
133
|
+
error_and_exit(f"Job is not queued (state: {job['state']}). Use 'stop' to terminate running jobs.")
|
|
134
|
+
return ts_remove(client, job_id, cwd=cli_config.remote_path)
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
# docker container State → the queue-style state labels the UI already knows.
|
|
138
|
+
_DOCKER_STATE_MAP = {
|
|
139
|
+
"running": "running",
|
|
140
|
+
"created": "queued",
|
|
141
|
+
"restarting": "running",
|
|
142
|
+
"paused": "running",
|
|
143
|
+
"exited": "finished",
|
|
144
|
+
"dead": "failed",
|
|
145
|
+
"removing": "finished",
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
|
|
149
|
+
class DockerTransport(Transport):
|
|
150
|
+
name = "docker"
|
|
151
|
+
|
|
152
|
+
def run_flags(self, local_id: int) -> list[str]:
|
|
153
|
+
# Detached, named, and NOT --rm so logs survive after the container exits.
|
|
154
|
+
return ["-d", "--name", _container_name(local_id)]
|
|
155
|
+
|
|
156
|
+
def submit(self, client, docker_cmd, *, local_id, slots, urgent):
|
|
157
|
+
if urgent:
|
|
158
|
+
typer.echo("Warning: --urgent has no effect without a queue.", err=True)
|
|
159
|
+
returncode, _ = client.run(docker_cmd, cwd=cli_config.remote_path, capture=True)
|
|
160
|
+
if returncode != 0:
|
|
161
|
+
error_and_exit(f"docker run failed (exit {returncode}).")
|
|
162
|
+
return {"transport": self.name, "handle": _container_name(local_id)}
|
|
163
|
+
|
|
164
|
+
def list_jobs(self, client):
|
|
165
|
+
fmt = "{{.Names}}\t{{.State}}\t{{.Command}}"
|
|
166
|
+
returncode, stdout = client.run(
|
|
167
|
+
f"docker ps -a --filter name=dockhand- --format '{fmt}'", cwd=cli_config.remote_path, capture=True
|
|
168
|
+
)
|
|
169
|
+
if returncode != 0:
|
|
170
|
+
return []
|
|
171
|
+
jobs = []
|
|
172
|
+
for line in stdout.strip().split("\n"):
|
|
173
|
+
if not line.strip():
|
|
174
|
+
continue
|
|
175
|
+
parts = line.split("\t")
|
|
176
|
+
if len(parts) < 2:
|
|
177
|
+
continue
|
|
178
|
+
name, state = parts[0], parts[1]
|
|
179
|
+
command = parts[2].strip('"') if len(parts) > 2 else ""
|
|
180
|
+
jobs.append({"handle": name, "state": _DOCKER_STATE_MAP.get(state, state), "command": command})
|
|
181
|
+
return jobs
|
|
182
|
+
|
|
183
|
+
def container_name(self, entry):
|
|
184
|
+
return entry_handle(entry)
|
|
185
|
+
|
|
186
|
+
def logs(self, client, entry, *, n, follow):
|
|
187
|
+
name = entry_handle(entry)
|
|
188
|
+
flags = []
|
|
189
|
+
if follow:
|
|
190
|
+
flags.append("-f")
|
|
191
|
+
if n is not None:
|
|
192
|
+
flags.append(f"--tail {n}")
|
|
193
|
+
cmd = f"docker logs {' '.join(flags)} {name}".replace(" ", " ")
|
|
194
|
+
returncode, _ = client.run(cmd, cwd=cli_config.remote_path)
|
|
195
|
+
return returncode
|
|
196
|
+
|
|
197
|
+
def stop(self, client, entry):
|
|
198
|
+
name = entry_handle(entry)
|
|
199
|
+
returncode, _ = client.run(f"docker stop {name}", cwd=cli_config.remote_path, capture=True)
|
|
200
|
+
return returncode == 0
|
|
201
|
+
|
|
202
|
+
def remove(self, client, entry):
|
|
203
|
+
# No queue to dequeue from — remove the container record (force-remove if running).
|
|
204
|
+
name = entry_handle(entry)
|
|
205
|
+
returncode, _ = client.run(f"docker rm -f {name}", cwd=cli_config.remote_path, capture=True)
|
|
206
|
+
return returncode == 0
|
|
207
|
+
|
|
208
|
+
|
|
209
|
+
_TRANSPORTS = {
|
|
210
|
+
TaskSpoolerTransport.name: TaskSpoolerTransport(),
|
|
211
|
+
DockerTransport.name: DockerTransport(),
|
|
212
|
+
}
|
dockhand/tunnel.py
ADDED
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
"""SSH local port forwarding to docker container ports."""
|
|
2
|
+
import time
|
|
3
|
+
from contextlib import ExitStack
|
|
4
|
+
|
|
5
|
+
import fabric
|
|
6
|
+
import typer
|
|
7
|
+
|
|
8
|
+
from dockhand.config import cli_config
|
|
9
|
+
from dockhand.error import error_and_exit
|
|
10
|
+
from dockhand.history import load_history
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def execute_tunnel(*, container_id: str | None, ports: list[str] | None):
|
|
14
|
+
"""Forward ports to localhost via SSH tunnel.
|
|
15
|
+
|
|
16
|
+
If ports are provided, use those. Otherwise read from the last (or specified) history entry.
|
|
17
|
+
"""
|
|
18
|
+
# Resolve port list: explicit override takes priority over history
|
|
19
|
+
if ports:
|
|
20
|
+
effective_ports = ports
|
|
21
|
+
else:
|
|
22
|
+
history = load_history()
|
|
23
|
+
if not history:
|
|
24
|
+
error_and_exit("No container history found. Run a container first.")
|
|
25
|
+
|
|
26
|
+
if container_id is not None:
|
|
27
|
+
entry = next((e for e in history if e["container_id"] == container_id), None)
|
|
28
|
+
if entry is None:
|
|
29
|
+
error_and_exit(f"Container ID '{container_id}' not found in history.")
|
|
30
|
+
else:
|
|
31
|
+
entry = history[-1]
|
|
32
|
+
|
|
33
|
+
effective_ports = entry["config"].get("ports") or []
|
|
34
|
+
if not effective_ports:
|
|
35
|
+
error_and_exit("No port mappings in history. Use -p to specify ports explicitly.")
|
|
36
|
+
|
|
37
|
+
# Parse "hostPort:containerPort" → forward local hostPort to remote hostPort
|
|
38
|
+
host_ports = []
|
|
39
|
+
for mapping in effective_ports:
|
|
40
|
+
parts = mapping.split(":")
|
|
41
|
+
if len(parts) != 2:
|
|
42
|
+
error_and_exit(f"Unexpected port mapping format: '{mapping}'. Expected 'hostPort:containerPort'.")
|
|
43
|
+
host_ports.append(int(parts[0]))
|
|
44
|
+
|
|
45
|
+
ssh = cli_config.ssh
|
|
46
|
+
conn = fabric.Connection(
|
|
47
|
+
host=ssh.hostname,
|
|
48
|
+
user=ssh.user,
|
|
49
|
+
connect_kwargs={"key_filename": ssh.identityfile},
|
|
50
|
+
)
|
|
51
|
+
|
|
52
|
+
typer.echo(f"Forwarding ports: {', '.join(str(p) for p in host_ports)}")
|
|
53
|
+
typer.echo("Press Ctrl+C to stop the tunnel.")
|
|
54
|
+
|
|
55
|
+
with ExitStack() as stack:
|
|
56
|
+
for port in host_ports:
|
|
57
|
+
stack.enter_context(conn.forward_local(port, remote_host="localhost", remote_port=port))
|
|
58
|
+
try:
|
|
59
|
+
while True:
|
|
60
|
+
time.sleep(1)
|
|
61
|
+
except KeyboardInterrupt:
|
|
62
|
+
typer.echo("\nTunnel closed.")
|
dockhand/volumes.py
ADDED
|
@@ -0,0 +1,149 @@
|
|
|
1
|
+
"""Docker volume inspection and path resolution."""
|
|
2
|
+
|
|
3
|
+
from rich.console import Console
|
|
4
|
+
from rich.tree import Tree
|
|
5
|
+
|
|
6
|
+
from dockhand.client import get_client
|
|
7
|
+
from dockhand.config import DockerConfig, cli_config
|
|
8
|
+
|
|
9
|
+
# Default scan depth when --depth is not specified.
|
|
10
|
+
# Keeps the find command fast on large volumes; use --depth N to go deeper.
|
|
11
|
+
DEFAULT_SCAN_DEPTH = 5
|
|
12
|
+
|
|
13
|
+
# Directories pruned from every find scan.
|
|
14
|
+
_PRUNE_DIRS = {".git", "__pycache__", ".venv", "venv", "node_modules", ".mypy_cache", ".ruff_cache", ".pytest_cache"}
|
|
15
|
+
_PRUNE_EXPR = " -o ".join(f"-name {d}" for d in sorted(_PRUNE_DIRS))
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def _workdir_relative(containerpath: str, workdir: str) -> str:
|
|
19
|
+
"""Convert an absolute container path to a workdir-relative path."""
|
|
20
|
+
workdir = workdir.rstrip("/")
|
|
21
|
+
containerpath = containerpath.rstrip("/")
|
|
22
|
+
if containerpath.startswith(workdir + "/"):
|
|
23
|
+
return containerpath[len(workdir) + 1 :]
|
|
24
|
+
if containerpath == workdir:
|
|
25
|
+
return "."
|
|
26
|
+
return containerpath
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def _resolve_to_host(relative_path: str, config: DockerConfig) -> tuple[str, str] | None:
|
|
30
|
+
"""Map a workdir-relative path to the host path where it lives.
|
|
31
|
+
|
|
32
|
+
Checks data volumes first (most specific match wins), then falls back to
|
|
33
|
+
the code mount (project root → containerworkdir).
|
|
34
|
+
"""
|
|
35
|
+
workdir = config.containerworkdir.rstrip("/")
|
|
36
|
+
for volume in config.volumes or []:
|
|
37
|
+
containerpath = volume["containerpath"].rstrip("/")
|
|
38
|
+
hostpath = volume["hostpath"].rstrip("/")
|
|
39
|
+
vol_relative = _workdir_relative(containerpath, workdir)
|
|
40
|
+
if vol_relative == ".":
|
|
41
|
+
# Volume is mounted at workdir itself — matches any relative path.
|
|
42
|
+
return hostpath + "/" + relative_path, hostpath
|
|
43
|
+
if relative_path.startswith(vol_relative + "/") or relative_path == vol_relative:
|
|
44
|
+
suffix = relative_path[len(vol_relative) :]
|
|
45
|
+
return hostpath + suffix, hostpath
|
|
46
|
+
# Fall back to the code mount (project root → containerworkdir).
|
|
47
|
+
if cli_config.remote_path:
|
|
48
|
+
return cli_config.remote_path.rstrip("/") + "/" + relative_path, cli_config.remote_path
|
|
49
|
+
return None
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def _build_tree(paths: list[str], strip_prefix: str) -> dict:
|
|
53
|
+
"""Convert a flat list of absolute paths into a nested dict.
|
|
54
|
+
|
|
55
|
+
Each key is a path component; files map to an empty dict,
|
|
56
|
+
directories to a non-empty dict of their children.
|
|
57
|
+
"""
|
|
58
|
+
root: dict = {}
|
|
59
|
+
prefix = strip_prefix.rstrip("/")
|
|
60
|
+
for path in paths:
|
|
61
|
+
path = path.strip()
|
|
62
|
+
if not path:
|
|
63
|
+
continue
|
|
64
|
+
if path.startswith(prefix + "/"):
|
|
65
|
+
relative = path[len(prefix) + 1 :]
|
|
66
|
+
elif path == prefix:
|
|
67
|
+
continue
|
|
68
|
+
else:
|
|
69
|
+
relative = path
|
|
70
|
+
node = root
|
|
71
|
+
for part in relative.split("/"):
|
|
72
|
+
if part:
|
|
73
|
+
node = node.setdefault(part, {})
|
|
74
|
+
return root
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
def _dict_to_rich_tree(d: dict, tree: Tree, remaining_depth: int | None = None) -> None:
|
|
78
|
+
"""Recursively populate a rich Tree from a nested dict.
|
|
79
|
+
|
|
80
|
+
Directories (non-empty children) are listed before files, sorted alphabetically.
|
|
81
|
+
remaining_depth: how many more directory levels to expand fully; None = unlimited.
|
|
82
|
+
Directories at the limit are shown with a '...' child to signal truncated content.
|
|
83
|
+
"""
|
|
84
|
+
dirs = {k: v for k, v in d.items() if v}
|
|
85
|
+
files = {k: v for k, v in d.items() if not v}
|
|
86
|
+
for name in sorted(dirs):
|
|
87
|
+
branch = tree.add(f"[bold blue]{name}/[/bold blue]")
|
|
88
|
+
if remaining_depth is None:
|
|
89
|
+
_dict_to_rich_tree(dirs[name], branch, None)
|
|
90
|
+
elif remaining_depth > 1:
|
|
91
|
+
_dict_to_rich_tree(dirs[name], branch, remaining_depth - 1)
|
|
92
|
+
else:
|
|
93
|
+
branch.add("[dim]...[/dim]")
|
|
94
|
+
for name in sorted(files):
|
|
95
|
+
tree.add(name)
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def execute_volumes(
|
|
99
|
+
config: DockerConfig,
|
|
100
|
+
depth: int | None = None,
|
|
101
|
+
imagename: str | None = None,
|
|
102
|
+
volumes: list | None = None,
|
|
103
|
+
) -> None:
|
|
104
|
+
"""List volume files as they appear inside the container, rooted at containerworkdir."""
|
|
105
|
+
volumes = volumes if volumes is not None else (config.volumes or [])
|
|
106
|
+
workdir = (config.containerworkdir or "/").rstrip("/") or "/"
|
|
107
|
+
|
|
108
|
+
# Build the full list of mounts: code mount first, then data volumes.
|
|
109
|
+
# The code mount maps the local project root to containerworkdir, exactly as submit does.
|
|
110
|
+
mounts: list[dict] = []
|
|
111
|
+
if cli_config.remote_path:
|
|
112
|
+
mounts.append({"hostpath": cli_config.remote_path, "containerpath": workdir})
|
|
113
|
+
mounts.extend(volumes)
|
|
114
|
+
|
|
115
|
+
# Scan depth for find: depth+1 ensures folder nodes at the display limit are non-empty.
|
|
116
|
+
# Without --depth, use DEFAULT_SCAN_DEPTH to keep find fast on large volumes.
|
|
117
|
+
scan_depth = depth + 1 if depth is not None else DEFAULT_SCAN_DEPTH
|
|
118
|
+
|
|
119
|
+
container_paths: list[str] = []
|
|
120
|
+
|
|
121
|
+
with get_client() as client:
|
|
122
|
+
for mount in mounts:
|
|
123
|
+
hostpath = mount["hostpath"].rstrip("/")
|
|
124
|
+
containerpath = mount["containerpath"].rstrip("/")
|
|
125
|
+
|
|
126
|
+
# cd into hostpath so find outputs relative paths (./a/b/file) — avoids
|
|
127
|
+
# any prefix-stripping issues caused by tilde expansion or symlinks.
|
|
128
|
+
# Replace ~ with $HOME so it expands inside bash -l -c "...".
|
|
129
|
+
hostpath_cmd = hostpath.replace("~", "$HOME")
|
|
130
|
+
find = f"find . -maxdepth {scan_depth} \\( {_PRUNE_EXPR} \\) -prune -o -type f -print 2>/dev/null"
|
|
131
|
+
cmd = f"cd {hostpath_cmd} && {find}"
|
|
132
|
+
exit_code, stdout = client.run(cmd, cwd=None, capture=True)
|
|
133
|
+
|
|
134
|
+
if exit_code != 0 or not stdout.strip():
|
|
135
|
+
continue
|
|
136
|
+
|
|
137
|
+
for line in stdout.splitlines():
|
|
138
|
+
# find . outputs ./relative/path — strip the leading ./
|
|
139
|
+
rel = line.strip().removeprefix("./")
|
|
140
|
+
if rel:
|
|
141
|
+
container_paths.append(containerpath + "/" + rel)
|
|
142
|
+
|
|
143
|
+
tree = Tree(f"[bold]{workdir}[/bold]")
|
|
144
|
+
if not container_paths:
|
|
145
|
+
tree.add("[dim](empty or inaccessible)[/dim]")
|
|
146
|
+
else:
|
|
147
|
+
_dict_to_rich_tree(_build_tree(container_paths, workdir), tree, remaining_depth=depth)
|
|
148
|
+
|
|
149
|
+
Console().print(tree)
|