dockhand-cli 0.3.1__py3-none-any.whl

dockhand/config.py

@@ -0,0 +1,355 @@
+import dataclasses
+import json
+from hashlib import sha256
+from pathlib import Path
+from typing import List
+
+import typer
+
+from dockhand.constants import CONFIG_FILENAME, HISTORY_FILENAME
+from dockhand.error import error_and_exit
+
+
+@dataclasses.dataclass
+class DockerVolumesConfig:
+    hostpath: str
+    containerpath: str
+    permissions: str
+
+
+@dataclasses.dataclass
+class DockerResubmitConfig:
+    container_id: str | None
+    commands: List[str] | None
+    imagename: str | None
+    gpus: str | None
+
+
+CODE_DELIVERY_MODES = ("mount", "bake")
+
+
+@dataclasses.dataclass
+class DockerConfig:
+    dockerfile: str
+    imagename: str
+    volumes: list[DockerVolumesConfig]
+    ports: list[str]
+    gpus: str
+    containerworkdir: str
+    preserve_paths: list[str] = dataclasses.field(default_factory=list)
+    # None means "derive from queue.enabled" (bake when queued, mount otherwise).
+    code_delivery: str | None = None
+
+    @classmethod
+    def load(cls, config: dict):
+        if "docker" not in config:
+            return None
+
+        docker = config["docker"]
+        if "dockerfile" not in docker:
+            error_and_exit('"dockerfile" not found in docker config.')
+
+        if "volumes" not in docker:
+            error_and_exit('"volumes" not found in docker config.')
+
+        if "imagename" not in docker:
+            error_and_exit('"imagename" not found in docker config.')
+
+        if "gpus" not in docker:
+            docker["gpus"] = None
+
+        if "ports" not in docker:
+            docker["ports"] = None
+
+        if "containerworkdir" not in docker:
+            docker["containerworkdir"] = "/"
+
+        if "preserve_paths" not in docker:
+            docker["preserve_paths"] = []
+
+        code_delivery = docker.get("code_delivery")
+        if code_delivery is not None and code_delivery not in CODE_DELIVERY_MODES:
+            error_and_exit(
+                f"Invalid value for code_delivery in docker config. Expected one of {CODE_DELIVERY_MODES} "
+                f"but got {code_delivery!r}."
+            )
+
+        # Only pass fields that DockerConfig expects
+        return cls(
+            dockerfile=docker["dockerfile"],
+            imagename=docker["imagename"],
+            volumes=docker["volumes"],
+            ports=docker["ports"],
+            gpus=docker["gpus"],
+            containerworkdir=docker["containerworkdir"],
+            preserve_paths=docker["preserve_paths"],
+            code_delivery=code_delivery,
+        )
+
+    def resolve_code_delivery(self, queue_enabled: bool) -> str:
+        """Resolve the effective delivery mode, applying the queue-derived default."""
+        if self.code_delivery is not None:
+            return self.code_delivery
+        return "bake" if queue_enabled else "mount"
+
+    @classmethod
+    def validate(cls, config: dict) -> dict:
+        if not isinstance(config, dict):
+            error_and_exit(f"Invalid type for ssh option in config. Expected dictionary but got {type(config)}.")
+
+        output = {}
+
+        dockerfile = config.get("dockerfile")
+        if dockerfile is not None:
+            if not isinstance(dockerfile, str):
+                error_and_exit(f"Invalid type for dockerfile. Expected string, got {type(dockerfile).__name__}.")
+            output["dockerfile"] = dockerfile
+
+        imagename = config.get("imagename")
+        if imagename is not None:
+            if not isinstance(imagename, str):
+                error_and_exit(
+                    f"Invalid type for compose_file option in docker config. Expected string but got {type(imagename)}."
+                )
+            output["imagename"] = imagename
+
+        volumes = config.get("volumes")
+        if volumes is not None:
+            if not isinstance(volumes, str):
+                error_and_exit(
+                    f"Invalid type for compose_file option in docker config. Expected string but got {type(volumes)}."
+                )
+            output["volumes"] = volumes
+
+        ports = config.get("ports")
+        if ports is not None:
+            if not isinstance(ports, str):
+                error_and_exit(
+                    f"Invalid type for compose_file option in docker config. Expected string but got {type(ports)}."
+                )
+            output["ports"] = ports
+
+        gpus = config.get("gpus")
+        if gpus is not None:
+            if not isinstance(gpus, str):
+                error_and_exit(
+                    f"Invalid type for gpus option in docker config. Expected string but got {type(gpus).__name__}."
+                )
+            output["gpus"] = gpus
+
+        preserve_paths = config.get("preserve_paths")
+        if preserve_paths is not None:
+            if not isinstance(preserve_paths, list) or not all(isinstance(p, str) for p in preserve_paths):
+                error_and_exit(
+                    "Invalid value for preserve_paths option in docker config. Expected a list of strings but got "
+                    f"{preserve_paths!r}."
+                )
+            output["preserve_paths"] = preserve_paths
+
+        return output
+
+
+@dataclasses.dataclass
+class QueueConfig:
+    enabled: bool = False
+    tool: str = "task_spooler"
+    slots: int = 1
+
+    @classmethod
+    def load(cls, config: dict):
+        queue = config.get("queue", {})
+        if not isinstance(queue, dict):
+            error_and_exit(f"Invalid type for queue option in config. Expected dictionary but got {type(queue)}.")
+        enabled = queue.get("enabled", False)
+        tool = queue.get("tool", "task_spooler")
+        slots = cls._load_slots(queue, config.get("docker") or {})
+        return cls(enabled=enabled, tool=tool, slots=slots)
+
+    @staticmethod
+    def _load_slots(queue: dict, docker: dict) -> int:
+        slots = queue.get("slots")
+        if slots is None and isinstance(docker, dict) and "slots" in docker:
+            # Back-compat: slots used to live under the docker block.
+            typer.echo(
+                "Warning: 'docker.slots' is deprecated — move it to 'queue.slots'.",
+                err=True,
+            )
+            slots = docker["slots"]
+        if slots is None:
+            return 1
+        if not isinstance(slots, int) or isinstance(slots, bool) or slots < 1:
+            error_and_exit(
+                f"Invalid value for slots option in queue config. Expected a positive integer but got {slots!r}."
+            )
+        return slots
+
+
+@dataclasses.dataclass
+class SSHConfig:
+    hostname: str
+    user: str
+    identityfile: str
+
+    @classmethod
+    def load(cls, config: dict):
+        if "ssh" not in config:
+            return None
+
+        ssh = config["ssh"]
+        ssh = SSHConfig.validate(ssh)
+
+        if "hostname" not in ssh:
+            error_and_exit('"hostname" not found in SSH config.')
+
+        if "user" not in ssh:
+            error_and_exit('"user" not found in SSH config.')
+
+        if "identityfile" not in ssh:
+            error_and_exit('"identityfile" not found in SSH config')
+
+        return cls(**ssh)
+
+    @classmethod
+    def validate(cls, config: dict) -> dict:
+        if not isinstance(config, dict):
+            error_and_exit(f"Invalid type for ssh option in config. Expected dictionary but got {type(config)}.")
+
+        output = {}
+
+        hostname = config.get("hostname")
+        if hostname is not None:
+            if not isinstance(hostname, str):
+                error_and_exit(f"Invalid type for host option in ssh config. Expected string but got {type(hostname)}.")
+            output["hostname"] = hostname
+
+        user = config.get("user")
+        if user is not None:
+            if not isinstance(user, str):
+                error_and_exit(f"Invalid type for user option in ssh config. Expected string but got {type(user)}.")
+            output["user"] = user
+
+        identityfile = config.get("identityfile")
+        if identityfile is not None:
+            if not isinstance(identityfile, str):
+                error_and_exit(
+                    f"Invalid type for identityfile option in ssh config. Expected string but got {type(identityfile)}."
+                )
+            output["identityfile"] = str(Path(identityfile).expanduser())
+
+        return output
+
+
+@dataclasses.dataclass
+class CLIConfig:
+    history_path: Path
+    project_root: Path
+    remote_path: str
+    sync: bool
+    profiles: dict | None
+    ssh: SSHConfig | None
+    docker: DockerConfig | None
+    queue: QueueConfig = dataclasses.field(default_factory=QueueConfig)
+
+    @classmethod
+    def load(cls):
+        project_root = cls.get_project_root()
+
+        git_path = project_root / ".git"
+        if not git_path.exists():
+            error_and_exit(f"Could not find git repository at '{git_path}'.")
+
+        path = project_root / CONFIG_FILENAME
+
+        try:
+            config = json.loads(path.read_text())
+        except json.JSONDecodeError as e:
+            error_and_exit(f"Error while parsing config file at '{path}':\n{e}")
+
+        if not isinstance(config, dict):
+            error_and_exit(f"Invalid type for config. Expected dictionary but got {type(config)}.")
+
+        profiles = config.get("profiles")
+        if profiles is not None and not isinstance(profiles, dict):
+            error_and_exit(f"Invalid type for profiles option in config. Expected dictionary but got {type(profiles)}.")
+
+        history_path = cls.load_history_path(config, project_root)
+        remote_path = cls.load_remote_path(config, project_root)
+        sync = config.get("sync", True)
+        ssh = SSHConfig.load(config)
+        docker = DockerConfig.load(config)
+        queue = QueueConfig.load(config)
+
+        return cls(
+            history_path=history_path,
+            profiles=profiles,
+            project_root=project_root,
+            remote_path=remote_path,
+            sync=sync,
+            ssh=ssh,
+            docker=docker,
+            queue=queue,
+        )
+
+    @classmethod
+    def get_project_root(cls) -> Path:
+        """Assume that config file exist in the project root and use that to get the project root."""
+        root = Path("/")
+        current_path = Path.cwd()
+        while current_path != root:
+            if (current_path / CONFIG_FILENAME).exists():
+                return current_path
+            current_path = current_path.parent
+
+        if (root / CONFIG_FILENAME).exists():
+            return root
+
+        error_and_exit(
+            f"Could not find project root. Make sure that '{CONFIG_FILENAME}' exists in the root of the project."
+        )
+
+    @classmethod
+    def load_history_path(cls, config: dict, project_root: Path) -> Path:
+        if "history_path" in config:
+            history_path = config["history_path"]
+            if not isinstance(history_path, str):
+                error_and_exit(
+                    f"Invalid type for history_path option in config. Expected string but got {type(history_path)}."
+                )
+            return Path(history_path)
+        return project_root / HISTORY_FILENAME
+
+    @classmethod
+    def load_remote_path(cls, config: dict, project_root: Path) -> str:
+        if "remote_path" in config:
+            return config["remote_path"]
+
+        name = project_root.name
+        hash = sha256(str(project_root).encode()).hexdigest()[:8]
+        return f"~/{name}-{hash}"
+
+    def check_ssh(self, msg: str = "SSH configuration is required for this command."):
+        if self.ssh is None:
+            error_and_exit(msg)
+
+    def check_docker(self, msg: str = "Docker configuration is required for this command"):
+        if self.docker is None:
+            error_and_exit(msg)
+
+    def load_profile(self, name: str):
+        if name not in self.profiles:
+            error_and_exit(f"Profile '{name}' not found in config.")
+
+        profile = self.profiles[name]
+
+        if "history_path" in profile:
+            self.history_path = CLIConfig.load_history_path(profile, self.project_root)
+
+        if "remote_path" in profile:
+            self.remote_path = profile["remote_path"]
+
+        if "ssh" in profile:
+            ssh = SSHConfig.validate(profile["ssh"])
+            self.ssh = dataclasses.replace(self.ssh, **ssh)
+
+
+cli_config = CLIConfig.load()

dockhand/constants.py

@@ -0,0 +1,2 @@
+CONFIG_FILENAME = ".dockhand.json"
+HISTORY_FILENAME = ".dockhand_history.json"

dockhand/download.py

@@ -0,0 +1,53 @@
+"""Download files from docker volumes."""
+import subprocess
+from pathlib import Path
+
+from rich.progress import Progress, SpinnerColumn, TextColumn
+
+from dockhand.config import DockerConfig, cli_config
+from dockhand.error import error_and_exit
+from dockhand.volumes import _resolve_to_host
+
+
+def execute_download(config: DockerConfig, path: str, local_path: str | None = None):
+    """Download a file from a docker volume by its workdir-relative path."""
+    result = _resolve_to_host(path, config)
+    if result is None:
+        error_and_exit(f"Path '{path}' does not match any configured docker volume.")
+
+    host_path, _ = result
+    ssh = cli_config.ssh
+
+    # If no local_path provided, mirror the remote path structure locally.
+    # For directory downloads (trailing slash), use the directory itself as the destination
+    # so rsync places contents at reports/figures/ rather than reports/.
+    # For file downloads, use the parent directory so rsync places the file correctly.
+    if local_path is None:
+        if path.endswith("/"):
+            dest = cli_config.project_root / Path(path.rstrip("/"))
+            dest.mkdir(parents=True, exist_ok=True)
+            local_path = str(dest)
+        else:
+            parent_dir = cli_config.project_root / Path(path).parent
+            parent_dir.mkdir(parents=True, exist_ok=True)
+            local_path = str(parent_dir)
+
+    with Progress(
+        SpinnerColumn(),
+        TextColumn("[progress.description]{task.description}"),
+    ) as progress:
+        task = progress.add_task(description="Downloading", total=None)
+        progress.start()
+        try:
+            command = [
+                "rsync",
+                "-avz",
+                "-e",
+                f"ssh -i {ssh.identityfile}",
+                f"{ssh.user}@{ssh.hostname}:{host_path}",
+                local_path,
+            ]
+            subprocess.run(command, check=True, capture_output=True)
+        except subprocess.CalledProcessError as e:
+            error_and_exit(f"Download failed:\n{e.stderr.decode()}")
+        progress.update(task, completed=True)

dockhand/error.py

@@ -0,0 +1,10 @@
+import sys
+
+from rich import print as rich_print
+from rich.panel import Panel
+
+
+def error_and_exit(message: str, code: int = 1):
+    panel = Panel(message, border_style="red", title="Error", title_align="left", highlight=True)
+    rich_print(panel)
+    sys.exit(code)

dockhand/history.py

@@ -0,0 +1,171 @@
+"""Docker container run history management and tracking."""
+
+import json
+import time
+from datetime import datetime
+from pathlib import Path
+from typing import List
+
+import typer
+from rich.console import Console
+from rich.table import Table
+from rich.text import Text
+
+from dockhand.config import DockerConfig
+from dockhand.constants import HISTORY_FILENAME
+
+DOCKER_HISTORY_FILE = Path(".dockhand_history.json")
+
+
+def load_history() -> list[dict]:
+    """Load container run history from disk."""
+    # path = DOCKER_HISTORY_FILE
+    path = Path(HISTORY_FILENAME)
+    if not path.exists():
+        return []
+    return json.loads(path.read_text())
+
+
+def save_history(history: list[dict]):
+    """Save container run history to disk."""
+    # path = DOCKER_HISTORY_FILE
+    path = Path(HISTORY_FILENAME)
+    path.write_text(json.dumps(history))
+
+
+def _next_local_id(history: list[dict]) -> int:
+    if not history:
+        return 1
+    return max((e.get("local_id", 0) for e in history), default=0) + 1
+
+
+def reserve_local_id() -> int:
+    """Peek the next local job ID without persisting it.
+
+    Submitting needs the ID before the job runs (e.g. to name the container), so it
+    is reserved here and passed to :func:`add_to_history` once the job has started.
+    """
+    return _next_local_id(load_history())
+
+
+def add_to_history(
+    config: DockerConfig,
+    commands: List[str],
+    *,
+    local_id: int,
+    handle: dict,
+    image_ref: str | None = None,
+    branch: str | None = None,
+    ports: list[str] | None = None,
+    host: str | None = None,
+) -> int:
+    """Add a started job to the history file. Returns the local job ID.
+
+    ``handle`` carries the transport-specific job handle (e.g. ``transport`` name and
+    a ``ts_job_id`` or container ``handle``) and is merged into the entry. ``image_ref``
+    is the exact image that ran (a resolved baked tag, or the base image name for mount
+    delivery) so the job can be reproduced verbatim on resubmit.
+    """
+    history = load_history()
+    _d = {
+        "gpus": config.gpus,
+        "volumes": config.volumes,
+        "imagename": config.imagename,
+        "commands": commands,
+        "ports": ports,
+    }
+    if image_ref is not None:
+        _d["image_ref"] = image_ref
+    if branch is not None:
+        _d["branch"] = branch
+    entry = {
+        "local_id": local_id,
+        "timestamp": time.time(),
+        "config": _d,
+        **handle,
+    }
+    if host is not None:
+        entry["host"] = host
+    history.append(entry)
+    save_history(history)
+    return local_id
+
+
+def get_history_entry(local_id: int) -> dict | None:
+    """Look up a history entry by local job ID."""
+    history = load_history()
+    for entry in reversed(history):
+        if entry.get("local_id") == local_id:
+            return entry
+    return None
+
+
+def mark_stopped(local_id: int):
+    """Flag a history entry as user-stopped, so ``jobs`` can show it distinctly."""
+    history = load_history()
+    for entry in history:
+        if entry.get("local_id") == local_id:
+            entry["stopped"] = True
+    save_history(history)
+
+
+def mark_job_time(local_id: int, *, started_at: float | None = None, ended_at: float | None = None):
+    """Record start/end timestamps for a job, without overwriting ones already set.
+
+    Neither the queue nor docker exposes exact start/end timestamps cheaply, so these
+    are stamped lazily the first time a command happens to observe the job in that
+    state (see ``manage.py``).
+    """
+    history = load_history()
+    changed = False
+    for entry in history:
+        if entry.get("local_id") != local_id:
+            continue
+        if started_at is not None and "started_at" not in entry:
+            entry["started_at"] = started_at
+            changed = True
+        if ended_at is not None and "ended_at" not in entry:
+            entry["ended_at"] = ended_at
+            changed = True
+    if changed:
+        save_history(history)
+
+
+def execute_history(config: DockerConfig):
+    """Show history of past Docker runs."""
+    history_file = Path(HISTORY_FILENAME)
+    if not history_file.exists():
+        typer.echo(f"No history found in '{history_file}'. You might not have submitted any jobs yet.")
+        return
+
+    history = load_history()
+
+    table = Table(title="Docker Run History", show_lines=True)
+    table.add_column("Job ID", justify="right", style="bold")
+    table.add_column("Host")
+    table.add_column("Timestamp")
+    table.add_column("Branch")
+    table.add_column("GPU(s)")
+    table.add_column("Volume(s)")
+    table.add_column("Imagename")
+    table.add_column("Commands")
+
+    for entry in history:
+        local_id = str(entry.get("local_id", "-"))
+        host = entry.get("host") or "-"
+        timestamp = datetime.fromtimestamp(entry["timestamp"]).strftime("%Y-%m-%d %H:%M:%S")
+        _config = entry["config"]
+        branch = _config.get("branch") or "-"
+        gpus = _config["gpus"] if _config["gpus"] else "-"
+        volumes = (
+            "\n".join(
+                [f"{v['hostpath']}:{v['containerpath']}:{v['permissions']}" for v in (_config.get("volumes") or [])]
+            )
+            or "-"
+        )
+        imagename = _config["imagename"]
+        cmds = _config.get("commands") or _config.get("arguments") or []
+        commands_str = " ".join(cmds)
+        table.add_row(local_id, host, timestamp, branch, gpus, volumes, imagename, Text(commands_str))
+
+    Console().print(table)