poly-hammer-worker 0.1.0.dev5__tar.gz → 0.1.1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -6,6 +6,7 @@ node_modules/
6
6
  scratches
7
7
  backend/reports
8
8
  .terraform
9
+ .terraform-*
9
10
  terraform.tfvars
10
11
  .vite/
11
12
  secret.env
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: poly-hammer-worker
3
- Version: 0.1.0.dev5
3
+ Version: 0.1.1
4
4
  Summary: Poly Hammer self-hosted GPU worker agent
5
5
  License-Expression: MIT
6
6
  Requires-Python: >=3.13
@@ -11,7 +11,7 @@ Requires-Dist: rich>=13.0
11
11
  Requires-Dist: typer>=0.12
12
12
  Description-Content-Type: text/markdown
13
13
 
14
- # ph-worker — Poly Hammer Self-Hosted GPU Worker
14
+ # Poly Hammer Self-Hosted GPU Worker
15
15
 
16
16
  A CLI agent that connects your GPU workstation to the Poly Hammer Portal, letting you run AI inference jobs on your own hardware instead of consuming cloud credits.
17
17
 
@@ -19,22 +19,16 @@ A CLI agent that connects your GPU workstation to the Poly Hammer Portal, lettin
19
19
 
20
20
  - **NVIDIA GPU** with CUDA drivers installed
21
21
  - **Docker** with [NVIDIA Container Toolkit](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/latest/install-guide.html) (`--gpus` support)
22
- - **Python 3.11+**
22
+ - **Python 3.13+**
23
23
  - An active Poly Hammer Portal account with a registered worker key
24
24
 
25
- ## Installation
26
-
27
- ```bash
28
- cd workers/self-hosted/client
29
- uv sync
30
- ```
31
25
 
32
26
  ## Quick Start
33
27
 
34
28
  ### 1. Check GPU compatibility
35
29
 
36
30
  ```bash
37
- ph-worker check-gpu
31
+ uvx poly-hammer-worker check-gpu
38
32
  ```
39
33
 
40
34
  This will display your GPU model, VRAM, CUDA version, CPU cores, and RAM.
@@ -46,14 +40,14 @@ Navigate to the **Workers** page in the Poly Hammer Portal and click **Add Worke
46
40
  ### 3. Start the worker
47
41
 
48
42
  ```bash
49
- ph-worker start --worker-key ph_worker_XXXXXXXX
43
+ uvx poly-hammer-worker start --worker-key ph_worker_XXXXXXXX
50
44
  ```
51
45
 
52
46
  Or use environment variables:
53
47
 
54
48
  ```bash
55
49
  export PH_WORKER_KEY=ph_worker_XXXXXXXX
56
- export PH_PORTAL_URL=https://portal.polyhammer.com # default
50
+ export PH_PORTAL_URL=https://api.portal.polyhammer.com # default
57
51
  ph-worker start
58
52
  ```
59
53
 
@@ -86,18 +80,7 @@ Portal API ←→ ph-worker agent ←→ Docker (GPU container)
86
80
  | Option | Env Var | Default | Description |
87
81
  |--------|---------|---------|-------------|
88
82
  | `--worker-key` | `PH_WORKER_KEY` | (required) | Worker API key from the portal |
89
- | `--portal-url` | `PH_PORTAL_URL` | `https://portal.polyhammer.com` | Portal API base URL |
90
-
91
- ## Docker Image Tags
92
-
93
- Worker images are published to GHCR with CUDA-versioned tags:
94
-
95
- | Image | Tag | Description |
96
- |-------|-----|-------------|
97
- | `ghcr.io/poly-hammer/hy-motion-worker` | `cuda13.1` | HY-Motion models (latest stable) |
98
- | `ghcr.io/poly-hammer/mdm-worker` | `cuda13.1` | MDM models (latest stable) |
99
- | | `cuda13.1-<sha>` | Pinned to a specific commit |
100
- | | `latest` | Latest build (any CUDA version) |
83
+ | `--portal-url` | `PH_PORTAL_URL` | `https://api.portal.polyhammer.com` | Portal API base URL |
101
84
 
102
85
  ## Troubleshooting
103
86
 
@@ -116,10 +99,3 @@ Worker images are published to GHCR with CUDA-versioned tags:
116
99
  - Heartbeats are sent every 60 seconds; the portal marks workers offline after 120 seconds
117
100
  - Check network connectivity to the portal URL
118
101
  - Verify your worker key is correct
119
-
120
- ## Development
121
-
122
- ```bash
123
- cd workers/self-hosted/client
124
- uv sync --group dev
125
- ```
@@ -1,4 +1,4 @@
1
- # ph-worker — Poly Hammer Self-Hosted GPU Worker
1
+ # Poly Hammer Self-Hosted GPU Worker
2
2
 
3
3
  A CLI agent that connects your GPU workstation to the Poly Hammer Portal, letting you run AI inference jobs on your own hardware instead of consuming cloud credits.
4
4
 
@@ -6,22 +6,16 @@ A CLI agent that connects your GPU workstation to the Poly Hammer Portal, lettin
6
6
 
7
7
  - **NVIDIA GPU** with CUDA drivers installed
8
8
  - **Docker** with [NVIDIA Container Toolkit](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/latest/install-guide.html) (`--gpus` support)
9
- - **Python 3.11+**
9
+ - **Python 3.13+**
10
10
  - An active Poly Hammer Portal account with a registered worker key
11
11
 
12
- ## Installation
13
-
14
- ```bash
15
- cd workers/self-hosted/client
16
- uv sync
17
- ```
18
12
 
19
13
  ## Quick Start
20
14
 
21
15
  ### 1. Check GPU compatibility
22
16
 
23
17
  ```bash
24
- ph-worker check-gpu
18
+ uvx poly-hammer-worker check-gpu
25
19
  ```
26
20
 
27
21
  This will display your GPU model, VRAM, CUDA version, CPU cores, and RAM.
@@ -33,14 +27,14 @@ Navigate to the **Workers** page in the Poly Hammer Portal and click **Add Worke
33
27
  ### 3. Start the worker
34
28
 
35
29
  ```bash
36
- ph-worker start --worker-key ph_worker_XXXXXXXX
30
+ uvx poly-hammer-worker start --worker-key ph_worker_XXXXXXXX
37
31
  ```
38
32
 
39
33
  Or use environment variables:
40
34
 
41
35
  ```bash
42
36
  export PH_WORKER_KEY=ph_worker_XXXXXXXX
43
- export PH_PORTAL_URL=https://portal.polyhammer.com # default
37
+ export PH_PORTAL_URL=https://api.portal.polyhammer.com # default
44
38
  ph-worker start
45
39
  ```
46
40
 
@@ -73,18 +67,7 @@ Portal API ←→ ph-worker agent ←→ Docker (GPU container)
73
67
  | Option | Env Var | Default | Description |
74
68
  |--------|---------|---------|-------------|
75
69
  | `--worker-key` | `PH_WORKER_KEY` | (required) | Worker API key from the portal |
76
- | `--portal-url` | `PH_PORTAL_URL` | `https://portal.polyhammer.com` | Portal API base URL |
77
-
78
- ## Docker Image Tags
79
-
80
- Worker images are published to GHCR with CUDA-versioned tags:
81
-
82
- | Image | Tag | Description |
83
- |-------|-----|-------------|
84
- | `ghcr.io/poly-hammer/hy-motion-worker` | `cuda13.1` | HY-Motion models (latest stable) |
85
- | `ghcr.io/poly-hammer/mdm-worker` | `cuda13.1` | MDM models (latest stable) |
86
- | | `cuda13.1-<sha>` | Pinned to a specific commit |
87
- | | `latest` | Latest build (any CUDA version) |
70
+ | `--portal-url` | `PH_PORTAL_URL` | `https://api.portal.polyhammer.com` | Portal API base URL |
88
71
 
89
72
  ## Troubleshooting
90
73
 
@@ -103,10 +86,3 @@ Worker images are published to GHCR with CUDA-versioned tags:
103
86
  - Heartbeats are sent every 60 seconds; the portal marks workers offline after 120 seconds
104
87
  - Check network connectivity to the portal URL
105
88
  - Verify your worker key is correct
106
-
107
- ## Development
108
-
109
- ```bash
110
- cd workers/self-hosted/client
111
- uv sync --group dev
112
- ```
@@ -12,11 +12,12 @@ import httpx
12
12
  logger = logging.getLogger(__name__)
13
13
 
14
14
  # Endpoints (relative to portal base URL)
15
- _POLL_PATH = "/api/v1/worker-jobs/poll"
16
- _COMPLETE_PATH = "/api/v1/worker-jobs/{job_id}/complete"
17
- _FAIL_PATH = "/api/v1/worker-jobs/{job_id}/fail"
18
- _PROGRESS_PATH = "/api/v1/worker-jobs/{job_id}/progress"
19
- _HEARTBEAT_PATH = "/api/v1/worker-jobs/heartbeat"
15
+ _POLL_PATH = "/v1/worker-jobs/poll"
16
+ _COMPLETE_PATH = "/v1/worker-jobs/{job_id}/complete"
17
+ _FAIL_PATH = "/v1/worker-jobs/{job_id}/fail"
18
+ _PROGRESS_PATH = "/v1/worker-jobs/{job_id}/progress"
19
+ _HEARTBEAT_PATH = "/v1/worker-jobs/heartbeat"
20
+ _DISCONNECT_PATH = "/v1/worker-jobs/disconnect"
20
21
 
21
22
 
22
23
  class PortalClient:
@@ -93,15 +94,21 @@ class PortalClient:
93
94
 
94
95
  def heartbeat(
95
96
  self,
96
- supported_models: list[str] | None = None,
97
97
  hardware_info: dict | None = None,
98
+ available_images: list[dict] | None = None,
99
+ disk_free_gb: float | None = None,
100
+ assigned_service_id: str | None = None,
98
101
  ) -> None:
99
102
  """Send a heartbeat to keep the worker marked ONLINE."""
100
103
  body: dict[str, Any] = {}
101
- if supported_models is not None:
102
- body["supported_models"] = supported_models
103
104
  if hardware_info is not None:
104
105
  body["hardware_info"] = hardware_info
106
+ if available_images is not None:
107
+ body["available_images"] = available_images
108
+ if disk_free_gb is not None:
109
+ body["disk_free_gb"] = disk_free_gb
110
+ if assigned_service_id is not None:
111
+ body["assigned_service_id"] = assigned_service_id
105
112
 
106
113
  try:
107
114
  resp = self._client.post(_HEARTBEAT_PATH, json=body)
@@ -109,5 +116,13 @@ class PortalClient:
109
116
  except httpx.HTTPError:
110
117
  logger.warning("Heartbeat failed (will retry)")
111
118
 
119
+ def disconnect(self) -> None:
120
+ """Notify the portal that this worker is going offline."""
121
+ try:
122
+ resp = self._client.post(_DISCONNECT_PATH)
123
+ resp.raise_for_status()
124
+ except httpx.HTTPError:
125
+ logger.warning("Disconnect notification failed (non-fatal)")
126
+
112
127
  def close(self) -> None:
113
128
  self._client.close()
@@ -0,0 +1,22 @@
1
+ """ASCII art banner for the ph-worker CLI."""
2
+
3
+ from rich.console import Console
4
+ from rich.text import Text
5
+
6
+ _BANNER_ART = r"""
7
+ ____ _ _ __ __ _
8
+ | _ \| | | | \ \ / /__ _ __| | _____ _ __
9
+ | |_) | |_| | \ \ /\ / / _ \| '__| |/ / _ \ '__|
10
+ | __/| _ | \ V V / (_) | | | < __/ |
11
+ |_| |_| |_| \_/\_/ \___/|_| |_|\_\___|_|
12
+
13
+ Poly Hammer · Self-Hosted Inference Agent
14
+ """
15
+
16
+ GOLD = "color(178)" # 256-color gold / yellow-gold
17
+
18
+
19
+ def print_banner(console: Console) -> None:
20
+ """Print the ph-worker banner in gold."""
21
+ text = Text(_BANNER_ART, style=GOLD)
22
+ console.print(text)
@@ -70,23 +70,55 @@ def start(
70
70
  help="Optional container registry URL to pull images from if not found locally.",
71
71
  ),
72
72
  ] = None,
73
+ service: Annotated[
74
+ str | None,
75
+ typer.Option(
76
+ "--service",
77
+ "-s",
78
+ envvar="PH_SERVICE",
79
+ help="Service ID to assign this worker to (e.g. 'text_to_motion').",
80
+ ),
81
+ ] = None,
82
+ disk_check_interval: Annotated[
83
+ int,
84
+ typer.Option(
85
+ "--disk-check-interval",
86
+ envvar="PH_DISK_CHECK_INTERVAL",
87
+ help="Seconds between disk space checks (default 300).",
88
+ ),
89
+ ] = 300,
90
+ detach: Annotated[
91
+ bool,
92
+ typer.Option(
93
+ "--detach",
94
+ help="Register, display settings, perform one successful poll, then exit.",
95
+ ),
96
+ ] = False,
73
97
  ) -> None:
74
98
  """Start the worker agent loop.
75
99
 
76
100
  The worker will:
77
101
  1. Detect local GPU/CUDA capabilities
78
- 2. Connect to the portal and send heartbeats
79
- 3. Long-poll for inference jobs
80
- 4. Use locally-built Docker images (or pull from --registry if set)
81
- 5. Run inference and upload results
102
+ 2. Discover portal-compatible Docker images via OCI labels
103
+ 3. Connect to the portal and send heartbeats (with images + disk info)
104
+ 4. Long-poll for inference jobs
105
+ 5. Use locally-built Docker images (or pull from --registry if set)
106
+ 6. Run inference and upload results
82
107
  """
108
+ from poly_hammer_worker.logging_config import configure_logging
83
109
  from poly_hammer_worker.worker_loop import run_worker_loop
84
110
 
111
+ log_file = configure_logging()
112
+ console.print(f" Logs: {log_file}\n")
113
+
85
114
  run_worker_loop(
86
115
  worker_key=worker_key,
87
116
  portal_url=portal_url,
88
117
  auto_cleanup=auto_cleanup,
89
118
  registry_url=registry,
119
+ detach=detach,
120
+ service=service,
121
+ disk_check_interval=disk_check_interval,
90
122
  console=console,
91
123
  )
92
124
 
@@ -98,7 +130,10 @@ def check_gpu() -> None:
98
130
 
99
131
  info = detect_hardware()
100
132
 
101
- console.print("\n[bold]GPU / Hardware Info[/bold]\n")
133
+ from poly_hammer_worker.banner import print_banner
134
+
135
+ print_banner(console)
136
+ console.print("[bold]GPU / Hardware Info[/bold]\n")
102
137
  console.print(f" GPU Model: {info.get('gpu_model', 'N/A')}")
103
138
  console.print(f" VRAM: {info.get('vram_gb', 'N/A')} GB")
104
139
  console.print(f" CUDA Version: {info.get('cuda_version', 'N/A')}")
@@ -9,9 +9,12 @@ Handles:
9
9
 
10
10
  import json
11
11
  import logging
12
+ import os
13
+ import platform
12
14
  import shutil
13
15
  import time
14
16
  from collections.abc import Callable
17
+ from pathlib import Path
15
18
 
16
19
  import docker
17
20
  import docker.errors
@@ -34,15 +37,6 @@ class DockerRunError(Exception):
34
37
  super().__init__(f"Container exited with code {exit_code}: {stderr}")
35
38
 
36
39
 
37
- def resolve_image_tag(docker_image: str, cuda_version: str) -> str:
38
- """Build the fully-qualified image reference.
39
-
40
- The backend stores bare image names (e.g. ``ph-hy-motion``).
41
- This appends the CUDA version tag matching the local GPU.
42
- """
43
- return f"{docker_image}:cuda{cuda_version}"
44
-
45
-
46
40
  class InsufficientDiskError(Exception):
47
41
  """Raised when there isn't enough disk space to pull an image."""
48
42
 
@@ -80,6 +74,48 @@ def get_docker_root() -> str:
80
74
  return "/var/lib/docker"
81
75
 
82
76
 
77
+ def _resolve_docker_host_storage_path() -> str:
78
+ """Resolve the host filesystem path where Docker actually stores data.
79
+
80
+ On Linux the Docker root (e.g. ``/var/lib/docker``) is a real local
81
+ path. On Windows and macOS, Docker Desktop runs inside a VM and the
82
+ reported root is a Linux path that does not exist on the host. This
83
+ function returns the host-side directory whose partition should be
84
+ checked for free space.
85
+ """
86
+ system = platform.system()
87
+
88
+ if system == "Linux":
89
+ return get_docker_root()
90
+
91
+ if system == "Windows":
92
+ # Docker Desktop (WSL2 backend) stores its ext4.vhdx here
93
+ local_app_data = os.environ.get("LOCALAPPDATA", "")
94
+ if local_app_data:
95
+ docker_wsl = os.path.join(local_app_data, "Docker", "wsl")
96
+ if os.path.isdir(docker_wsl):
97
+ return docker_wsl
98
+ # Older Docker Desktop installations
99
+ program_data = os.environ.get("ProgramData", r"C:\ProgramData")
100
+ docker_pd = os.path.join(program_data, "DockerDesktop")
101
+ if os.path.isdir(docker_pd):
102
+ return docker_pd
103
+ # Last resort: Docker Desktop defaults to C: drive
104
+ return os.environ.get("SystemDrive", "C:\\")
105
+
106
+ if system == "Darwin":
107
+ # Docker Desktop for Mac stores its VM disk image here
108
+ docker_data = (
109
+ Path.home() / "Library" / "Containers" / "com.docker.docker" / "Data"
110
+ )
111
+ if docker_data.is_dir():
112
+ return str(docker_data)
113
+ return str(Path.home())
114
+
115
+ # Unknown OS — try the Docker root directly
116
+ return get_docker_root()
117
+
118
+
83
119
  def get_free_disk_gb(path: str | None = None) -> float:
84
120
  """Return free disk space in GB on the partition containing *path*.
85
121
 
@@ -87,8 +123,12 @@ def get_free_disk_gb(path: str | None = None) -> float:
87
123
  where images will actually be stored.
88
124
  """
89
125
  if path is None:
90
- path = get_docker_root()
91
- usage = shutil.disk_usage(path)
126
+ path = _resolve_docker_host_storage_path()
127
+ try:
128
+ usage = shutil.disk_usage(path)
129
+ except (FileNotFoundError, OSError):
130
+ logger.debug("Cannot stat %s, falling back to SystemDrive", path)
131
+ usage = shutil.disk_usage(os.environ.get("SystemDrive", "/"))
92
132
  return round(usage.free / (1024**3), 2)
93
133
 
94
134
 
@@ -251,6 +291,7 @@ def run_inference_container(
251
291
  job_payload: dict,
252
292
  on_progress: Callable[[float], None] | None = None,
253
293
  on_status: Callable[[str], None] | None = None,
294
+ on_log: Callable[[str], None] | None = None,
254
295
  ) -> dict:
255
296
  """Run an inference container and return the result dict.
256
297
 
@@ -266,6 +307,8 @@ def run_inference_container(
266
307
  Called with progress float (0.0–1.0) as the container reports progress.
267
308
  on_status:
268
309
  Called with human-readable status strings (pulling, running, etc.).
310
+ on_log:
311
+ Called with each line of container output (both stdout and stderr).
269
312
 
270
313
  Returns
271
314
  -------
@@ -275,14 +318,12 @@ def run_inference_container(
275
318
  ------
276
319
  DockerRunError — If the container exits non-zero or emits an error message.
277
320
  """
278
- stdin_bytes = json.dumps(job_payload).encode()
279
-
280
321
  if on_status:
281
322
  on_status(f"Creating container from {image_ref}")
282
323
 
283
324
  container = client.containers.create(
284
325
  image_ref,
285
- stdin_open=True,
326
+ environment={"JOB_PAYLOAD": json.dumps(job_payload)},
286
327
  detach=True,
287
328
  device_requests=[
288
329
  docker.types.DeviceRequest(count=-1, capabilities=[["gpu"]]),
@@ -295,26 +336,26 @@ def run_inference_container(
295
336
  try:
296
337
  container.start()
297
338
 
298
- # Attach and pipe job JSON into stdin
299
- sock = container.attach_socket(params={"stdin": 1, "stream": 1})
300
- sock._sock.sendall(stdin_bytes)
301
- sock._sock.shutdown(1) # EOF on stdin
302
- sock.close()
303
-
304
339
  if on_status:
305
340
  on_status("Inference running …")
306
341
 
307
- # Stream stdout for JSON-line progress messages
342
+ # Stream both stdout and stderr for JSON-line progress + container logs
343
+ stderr_lines: list[str] = []
308
344
  for raw_line in container.logs(
309
- stream=True, follow=True, stdout=True, stderr=False
345
+ stream=True, follow=True, stdout=True, stderr=True
310
346
  ):
311
347
  line = raw_line.decode("utf-8", errors="replace").strip()
312
348
  if not line:
313
349
  continue
350
+
351
+ if on_log:
352
+ on_log(line)
353
+
314
354
  try:
315
355
  msg = json.loads(line)
316
356
  except json.JSONDecodeError:
317
- logger.debug("Non-JSON output: %s", line[:200])
357
+ # Non-JSON output is stderr or unstructured stdout
358
+ stderr_lines.append(line)
318
359
  continue
319
360
 
320
361
  msg_type = msg.get("type")
@@ -335,10 +376,8 @@ def run_inference_container(
335
376
  raise DockerRunError(exit_code, error_msg)
336
377
 
337
378
  if exit_code != 0:
338
- stderr_logs = container.logs(stdout=False, stderr=True).decode(
339
- "utf-8", errors="replace"
340
- )[-2000:]
341
- raise DockerRunError(exit_code, stderr_logs)
379
+ stderr_tail = "\n".join(stderr_lines)[-2000:]
380
+ raise DockerRunError(exit_code, stderr_tail)
342
381
 
343
382
  if result_msg is None:
344
383
  raise DockerRunError(
@@ -12,7 +12,7 @@ import subprocess
12
12
 
13
13
  import psutil
14
14
 
15
- from poly_hammer_worker.docker_runner import get_docker_root
15
+ from poly_hammer_worker.docker_runner import get_free_disk_gb
16
16
 
17
17
  logger = logging.getLogger(__name__)
18
18
 
@@ -106,9 +106,7 @@ def _get_cuda_version_from_smi(nvidia_smi: str) -> str | None:
106
106
  def _get_docker_disk_free_gb() -> float | None:
107
107
  """Return free GB on the partition where Docker stores images."""
108
108
  try:
109
- docker_root = get_docker_root()
110
- usage = shutil.disk_usage(docker_root)
111
- return round(usage.free / (1024**3), 1)
109
+ return get_free_disk_gb()
112
110
  except Exception:
113
111
  logger.debug("Could not check Docker disk free space", exc_info=True)
114
112
  return None
@@ -0,0 +1,188 @@
1
+ """
2
+ Docker image discovery for Poly Hammer Portal compatible images.
3
+
4
+ Scans local Docker images for ``ai.poly-hammer.*`` OCI labels and returns
5
+ structured metadata for each compatible image. Each image may contain
6
+ multiple models with independent hardware requirements, parsed from
7
+ per-model labels (``ai.poly-hammer.model.<id>.*``).
8
+
9
+ Required labels (image is skipped if any are missing):
10
+ - ai.poly-hammer.portal-compatible = "true"
11
+ - ai.poly-hammer.service-id
12
+ - ai.poly-hammer.model-ids (comma-separated model IDs)
13
+
14
+ Optional labels:
15
+ - ai.poly-hammer.min-cuda-version Minimum host CUDA version required.
16
+ When set **and** *host_cuda_version* is passed to
17
+ :func:`discover_local_images`, images whose minimum exceeds the
18
+ host capability are skipped with a warning.
19
+
20
+ Per-model labels (optional, looked up for each model ID):
21
+ - ai.poly-hammer.model.<id>.name
22
+ - ai.poly-hammer.model.<id>.min-vram-gb
23
+ - ai.poly-hammer.model.<id>.min-ram-gb
24
+ """
25
+
26
+ import logging
27
+ from typing import Any
28
+
29
+ import docker
30
+
31
+ logger = logging.getLogger(__name__)
32
+
33
+ _PREFIX = "ai.poly-hammer."
34
+
35
+ _REQUIRED_LABELS = frozenset(
36
+ {
37
+ "portal-compatible",
38
+ "service-id",
39
+ "model-ids",
40
+ }
41
+ )
42
+
43
+
44
+ def discover_local_images(
45
+ client: docker.DockerClient,
46
+ host_cuda_version: str | None = None,
47
+ ) -> list[dict[str, Any]]:
48
+ """Scan local Docker images for portal-compatible OCI labels.
49
+
50
+ Returns a list of dicts matching the ``WorkerImageInfo`` schema — one
51
+ entry per Docker image. Each entry contains a ``models`` list with
52
+ per-model metadata (name, hardware requirements).
53
+
54
+ If *host_cuda_version* is provided (e.g. ``"13.1"``), images whose
55
+ ``ai.poly-hammer.min-cuda-version`` label exceeds it are skipped.
56
+ """
57
+ results: list[dict[str, Any]] = []
58
+
59
+ try:
60
+ images = client.images.list()
61
+ except docker.errors.DockerException:
62
+ logger.exception("Failed to list Docker images")
63
+ return results
64
+
65
+ for image in images:
66
+ labels = image.labels or {}
67
+
68
+ # Quick check: skip images without the portal-compatible flag
69
+ if labels.get(f"{_PREFIX}portal-compatible") != "true":
70
+ continue
71
+
72
+ # Extract all ai.poly-hammer.* labels (strip prefix)
73
+ ph_labels: dict[str, str] = {}
74
+ for key, value in labels.items():
75
+ if key.startswith(_PREFIX):
76
+ ph_labels[key[len(_PREFIX) :]] = value
77
+
78
+ # Validate required labels
79
+ missing = _REQUIRED_LABELS - ph_labels.keys()
80
+ if missing:
81
+ tag = _best_tag(image)
82
+ logger.warning(
83
+ "Image %s has portal-compatible flag but missing required labels: %s",
84
+ tag,
85
+ ", ".join(sorted(missing)),
86
+ )
87
+ continue
88
+
89
+ image_ref = _best_tag(image)
90
+ if not image_ref:
91
+ logger.debug("Skipping untagged image %s", image.short_id)
92
+ continue
93
+
94
+ service_id = ph_labels["service-id"]
95
+ min_cuda_version: str | None = ph_labels.get("min-cuda-version")
96
+
97
+ # Host compatibility check: skip images that need a newer CUDA
98
+ if min_cuda_version and host_cuda_version:
99
+ if _parse_version(min_cuda_version) > _parse_version(host_cuda_version):
100
+ logger.warning(
101
+ "Image %s requires CUDA >= %s but host has %s — skipping",
102
+ image_ref,
103
+ min_cuda_version,
104
+ host_cuda_version,
105
+ )
106
+ continue
107
+
108
+ # Image size in GB (Docker reports bytes)
109
+ image_size_gb = round(image.attrs.get("Size", 0) / (1024**3), 2)
110
+
111
+ # Parse comma-separated model IDs
112
+ model_ids = [m.strip() for m in ph_labels["model-ids"].split(",") if m.strip()]
113
+
114
+ # Build per-model metadata from ai.poly-hammer.model.<id>.* labels
115
+ models: list[dict[str, Any]] = []
116
+ for model_id in model_ids:
117
+ model_prefix = f"model.{model_id}."
118
+ model_name = ph_labels.get(f"{model_prefix}name")
119
+ min_vram_gb = _parse_int(ph_labels.get(f"{model_prefix}min-vram-gb", "0"))
120
+ min_ram_gb = _parse_int(ph_labels.get(f"{model_prefix}min-ram-gb", "0"))
121
+ models.append(
122
+ {
123
+ "model_id": model_id,
124
+ "model_name": model_name,
125
+ "min_vram_gb": min_vram_gb,
126
+ "min_ram_gb": min_ram_gb,
127
+ }
128
+ )
129
+
130
+ results.append(
131
+ {
132
+ "image_ref": image_ref,
133
+ "service_id": service_id,
134
+ "min_cuda_version": min_cuda_version,
135
+ "image_size_gb": image_size_gb,
136
+ "models": models,
137
+ }
138
+ )
139
+
140
+ total_models = sum(len(img["models"]) for img in results)
141
+ logger.info(
142
+ "Discovered %d portal-compatible image(s) with %d model(s)",
143
+ len(results),
144
+ total_models,
145
+ )
146
+ return results
147
+
148
+
149
+ def _best_tag(image: Any) -> str | None:
150
+ """Return the most useful tag for an image, or None if untagged."""
151
+ tags = image.tags
152
+ if tags:
153
+ return tags[0]
154
+ return None
155
+
156
+
157
+ def _parse_int(value: str) -> int:
158
+ """Parse an integer from a label value, defaulting to 0."""
159
+ try:
160
+ return int(value)
161
+ except (ValueError, TypeError):
162
+ return 0
163
+
164
+
165
+ def _parse_version(version: str) -> tuple[int, ...]:
166
+ """Parse a dotted version string into a tuple of ints for comparison."""
167
+ try:
168
+ return tuple(int(p) for p in version.split("."))
169
+ except (ValueError, TypeError):
170
+ return (0,)
171
+
172
+
173
+ def resolve_image_for_model(
174
+ available_images: list[dict[str, Any]], model_id: str
175
+ ) -> str | None:
176
+ """Find the local Docker image reference that supports a given model ID.
177
+
178
+ Searches the list returned by :func:`discover_local_images` for the
179
+ first image whose ``models`` list contains the requested *model_id*.
180
+
181
+ Returns the ``image_ref`` (e.g. ``"hy-motion:1.0"``) or ``None`` if
182
+ no matching image is found.
183
+ """
184
+ for img in available_images:
185
+ for model in img.get("models", []):
186
+ if model.get("model_id") == model_id:
187
+ return img.get("image_ref")
188
+ return None
@@ -0,0 +1,73 @@
1
+ """
2
+ Logging configuration for the ph-worker agent.
3
+
4
+ Configures both console and rotating file handlers with timestamps.
5
+ Log files are stored in a platform-appropriate directory and automatically
6
+ cleaned up after 30 days.
7
+ """
8
+
9
+ import logging
10
+ import os
11
+ import sys
12
+ from logging.handlers import TimedRotatingFileHandler
13
+ from pathlib import Path
14
+
15
+ _LOG_FORMAT = "[%(levelname)s] %(asctime)s %(name)s %(message)s"
16
+ _LOG_DATE_FORMAT = "%H:%M:%S %d-%m-%Y"
17
+ _RETENTION_DAYS = 30
18
+
19
+
20
+ def _log_dir() -> Path:
21
+ """Return the log directory, creating it if needed.
22
+
23
+ Uses ``PH_WORKER_LOG_DIR`` env var if set, otherwise falls back to
24
+ ``~/.poly-hammer-worker/logs/``.
25
+ """
26
+ env = os.environ.get("PH_WORKER_LOG_DIR")
27
+ if env:
28
+ path = Path(env)
29
+ else:
30
+ path = Path.home() / ".poly-hammer-worker" / "logs"
31
+ path.mkdir(parents=True, exist_ok=True)
32
+ return path
33
+
34
+
35
+ def configure_logging(*, verbose: bool = False) -> Path:
36
+ """Set up root logger with console + rotating file handlers.
37
+
38
+ Returns the log file path so the CLI can display it.
39
+ """
40
+ level = logging.DEBUG if verbose else logging.INFO
41
+ root = logging.getLogger()
42
+ root.setLevel(level)
43
+
44
+ # Remove any pre-existing handlers (e.g. from basicConfig)
45
+ root.handlers.clear()
46
+
47
+ formatter = logging.Formatter(_LOG_FORMAT, datefmt=_LOG_DATE_FORMAT)
48
+
49
+ # ---- Console handler (stderr) ----
50
+ console_handler = logging.StreamHandler(sys.stderr)
51
+ console_handler.setLevel(level)
52
+ console_handler.setFormatter(formatter)
53
+ root.addHandler(console_handler)
54
+
55
+ # ---- Rotating file handler (daily rotation, 30-day retention) ----
56
+ log_file = _log_dir() / "ph-worker.log"
57
+ file_handler = TimedRotatingFileHandler(
58
+ filename=str(log_file),
59
+ when="midnight",
60
+ interval=1,
61
+ backupCount=_RETENTION_DAYS,
62
+ encoding="utf-8",
63
+ )
64
+ file_handler.setLevel(logging.DEBUG) # always capture debug in file
65
+ file_handler.setFormatter(formatter)
66
+ root.addHandler(file_handler)
67
+
68
+ # ---- Quiet noisy third-party loggers unless verbose ----
69
+ if not verbose:
70
+ for name in ("httpx", "httpcore", "urllib3", "docker", "hpack"):
71
+ logging.getLogger(name).setLevel(logging.WARNING)
72
+
73
+ return log_file
@@ -8,6 +8,7 @@ Responsibilities:
8
8
  """
9
9
 
10
10
  import logging
11
+ import queue
11
12
  import signal
12
13
  import threading
13
14
  import time
@@ -21,15 +22,15 @@ from poly_hammer_worker.docker_runner import (
21
22
  InsufficientDiskError,
22
23
  ensure_image,
23
24
  get_free_disk_gb,
24
- resolve_image_tag,
25
25
  run_inference_container,
26
26
  )
27
27
  from poly_hammer_worker.hardware import detect_hardware
28
+ from poly_hammer_worker.image_discovery import resolve_image_for_model
28
29
 
29
30
  logger = logging.getLogger(__name__)
30
31
 
31
32
  # Intervals (seconds)
32
- HEARTBEAT_INTERVAL = 60 # well within the 120s backend timeout
33
+ HEARTBEAT_INTERVAL = 30 # well within the 60s backend timeout
33
34
  POLL_BACKOFF_BASE = 5
34
35
  POLL_BACKOFF_MAX = 30
35
36
 
@@ -40,9 +41,19 @@ def run_worker_loop(
40
41
  console: Console,
41
42
  auto_cleanup: bool = False,
42
43
  registry_url: str | None = None,
44
+ detach: bool = False,
45
+ service: str | None = None,
46
+ disk_check_interval: int = 300,
43
47
  ) -> None:
44
- """Blocking top-level loop. Runs until SIGINT / SIGTERM."""
45
- console.print("[bold]ph-worker[/bold] starting …\n")
48
+ """Blocking top-level loop. Runs until SIGINT / SIGTERM.
49
+
50
+ If *detach* is True the loop registers, displays settings, waits for
51
+ one successful poll round, then exits cleanly.
52
+ """
53
+ from poly_hammer_worker.banner import print_banner
54
+ from poly_hammer_worker.image_discovery import discover_local_images
55
+
56
+ print_banner(console)
46
57
 
47
58
  # ---- Hardware detection ---------------------------------------------------
48
59
  hw = detect_hardware()
@@ -67,16 +78,49 @@ def run_worker_loop(
67
78
  console.print(f" Registry: {registry_url}")
68
79
  else:
69
80
  console.print(" Registry: [dim]local-only (no remote pull)[/dim]")
81
+ if service:
82
+ console.print(f" Service: {service}")
70
83
  console.print()
71
84
 
72
85
  # ---- Clients --------------------------------------------------------------
73
86
  portal = PortalClient(portal_url, worker_key)
74
87
  docker_client = docker.from_env()
75
88
 
89
+ # ---- Image discovery (initial) -------------------------------------------
90
+ available_images = discover_local_images(
91
+ docker_client, host_cuda_version=cuda_version
92
+ )
93
+ if available_images:
94
+ total_models = sum(len(img["models"]) for img in available_images)
95
+ console.print(
96
+ f"[green]Discovered {len(available_images)} image(s) with {total_models} model(s)[/green]"
97
+ )
98
+ for img in available_images:
99
+ console.print(f" {img['image_ref']} service={img['service_id']}")
100
+ for model in img["models"]:
101
+ console.print(f" - {model['model_id']}")
102
+ else:
103
+ console.print(
104
+ "[yellow]No portal-compatible Docker images found locally[/yellow]"
105
+ )
106
+
107
+ if service:
108
+ matching = [i for i in available_images if i["service_id"] == service]
109
+ if not matching:
110
+ console.print(
111
+ f"[yellow bold]Warning: no local images match service '{service}'[/yellow bold]"
112
+ )
113
+
114
+ console.print()
115
+
76
116
  # ---- Shutdown flag --------------------------------------------------------
77
117
  shutdown = threading.Event()
78
118
 
79
119
  def _handle_signal(sig: int, _frame: object) -> None:
120
+ if shutdown.is_set():
121
+ # Second signal — force immediate exit
122
+ console.print("\n[red bold]Forced exit[/red bold]")
123
+ raise SystemExit(1)
80
124
  console.print(f"\n[yellow]Received signal {sig}, shutting down …[/yellow]")
81
125
  shutdown.set()
82
126
 
@@ -84,9 +128,33 @@ def run_worker_loop(
84
128
  signal.signal(signal.SIGTERM, _handle_signal)
85
129
 
86
130
  # ---- Heartbeat thread -----------------------------------------------------
131
+ _last_disk_check: float = 0.0
132
+
87
133
  def _heartbeat_loop() -> None:
134
+ nonlocal _last_disk_check, available_images
88
135
  while not shutdown.is_set():
89
- portal.heartbeat(hardware_info=hw)
136
+ # Re-discover images on each heartbeat (cheap — Docker API list)
137
+ available_images = discover_local_images(
138
+ docker_client, host_cuda_version=cuda_version
139
+ )
140
+
141
+ # Throttled disk space check
142
+ now = time.monotonic()
143
+ disk_gb: float | None = None
144
+ if now - _last_disk_check >= disk_check_interval:
145
+ disk_gb = get_free_disk_gb()
146
+ _last_disk_check = now
147
+
148
+ try:
149
+ portal.heartbeat(
150
+ hardware_info=hw,
151
+ available_images=available_images,
152
+ disk_free_gb=disk_gb,
153
+ assigned_service_id=service,
154
+ )
155
+ logger.info("Heartbeat sent")
156
+ except Exception:
157
+ logger.warning("Heartbeat failed", exc_info=True)
90
158
  shutdown.wait(HEARTBEAT_INTERVAL)
91
159
 
92
160
  hb_thread = threading.Thread(target=_heartbeat_loop, daemon=True, name="heartbeat")
@@ -98,23 +166,51 @@ def run_worker_loop(
98
166
  console.print("[bold]Polling for jobs …[/bold]\n")
99
167
 
100
168
  while not shutdown.is_set():
101
- try:
102
- job = portal.poll_for_job()
103
- except Exception:
104
- logger.exception("Poll error")
169
+ # Run poll in a daemon thread so Ctrl+C is never blocked by I/O
170
+ poll_q: queue.Queue[tuple[str, object]] = queue.Queue()
171
+
172
+ def _do_poll() -> None:
173
+ try:
174
+ poll_q.put(("ok", portal.poll_for_job()))
175
+ except Exception as exc:
176
+ poll_q.put(("error", exc))
177
+
178
+ poll_thread = threading.Thread(target=_do_poll, daemon=True)
179
+ poll_thread.start()
180
+
181
+ # Main thread sleeps in short intervals so signals are handled fast
182
+ while poll_thread.is_alive() and not shutdown.is_set():
183
+ poll_thread.join(timeout=0.5)
184
+
185
+ if shutdown.is_set():
186
+ break
187
+
188
+ tag, payload = poll_q.get_nowait()
189
+
190
+ if tag == "error":
191
+ logger.exception("Poll error", exc_info=payload)
105
192
  console.print(f"[red]Poll error — retrying in {backoff}s[/red]")
106
193
  shutdown.wait(backoff)
107
194
  backoff = min(backoff * 2, POLL_BACKOFF_MAX)
108
195
  continue
109
196
 
197
+ job = payload
198
+
110
199
  if job is None:
111
- # No job available — wait briefly then re-poll
200
+ # Successful poll with no job available
201
+ if detach:
202
+ console.print("[green]Poll successful — detaching.[/green]")
203
+ break
112
204
  shutdown.wait(POLL_BACKOFF_BASE)
113
205
  continue
114
206
 
115
207
  # Reset backoff on successful poll
116
208
  backoff = POLL_BACKOFF_BASE
117
209
 
210
+ if detach:
211
+ console.print("[green]Poll successful — detaching.[/green]")
212
+ break
213
+
118
214
  job_id = str(job["job_id"])
119
215
  model_id = job.get("model_id", "unknown")
120
216
  console.print(
@@ -125,7 +221,7 @@ def run_worker_loop(
125
221
  portal=portal,
126
222
  docker_client=docker_client,
127
223
  job=job,
128
- cuda_version=cuda_version,
224
+ available_images=available_images,
129
225
  auto_cleanup=auto_cleanup,
130
226
  registry_url=registry_url,
131
227
  console=console,
@@ -133,6 +229,7 @@ def run_worker_loop(
133
229
 
134
230
  # ---- Cleanup --------------------------------------------------------------
135
231
  console.print("\n[bold]Shutting down …[/bold]")
232
+ portal.disconnect()
136
233
  portal.close()
137
234
  console.print("[green]Done.[/green]")
138
235
 
@@ -141,21 +238,26 @@ def _execute_job(
141
238
  portal: PortalClient,
142
239
  docker_client: docker.DockerClient,
143
240
  job: dict,
144
- cuda_version: str,
241
+ available_images: list[dict],
145
242
  console: Console,
146
243
  auto_cleanup: bool = False,
147
244
  registry_url: str | None = None,
148
245
  ) -> None:
149
246
  """Ensure image (local-first, optional registry pull), run container, report outcome."""
150
247
  job_id = str(job["job_id"])
151
- docker_image = job.get("docker_image", "")
248
+ model_id = job.get("model_id", "")
152
249
 
153
- if not docker_image:
154
- portal.fail_job(job_id, "Job missing docker_image field", "NO_IMAGE")
155
- console.print("[red] Job missing docker_image — skipped[/red]")
250
+ image_ref = resolve_image_for_model(available_images, model_id)
251
+ if not image_ref:
252
+ portal.fail_job(
253
+ job_id,
254
+ f"No local Docker image found for model '{model_id}'. "
255
+ "Build the appropriate image and restart the worker.",
256
+ "NO_IMAGE",
257
+ )
258
+ console.print(f"[red] No local image for model '{model_id}' — skipped[/red]")
156
259
  return
157
260
 
158
- image_ref = resolve_image_tag(docker_image, cuda_version)
159
261
  start_time = time.monotonic()
160
262
 
161
263
  try:
@@ -169,22 +271,42 @@ def _execute_job(
169
271
  )
170
272
 
171
273
  # ---- Run container ----------------------------------------------------
274
+ container_logger = logging.getLogger(f"{__name__}.container")
275
+
276
+ def _on_log(line: str) -> None:
277
+ container_logger.info(line)
278
+ console.print(f" [dim]│ {line}[/dim]", highlight=False)
279
+
172
280
  result = run_inference_container(
173
281
  docker_client,
174
282
  image_ref,
175
283
  job_payload=job,
176
284
  on_progress=lambda p: _on_progress(portal, job_id, p, console),
177
285
  on_status=lambda s: console.print(f" [dim]{s}[/dim]"),
286
+ on_log=_on_log,
178
287
  )
179
288
 
180
289
  elapsed = time.monotonic() - start_time
181
290
 
182
291
  # ---- Report completion ------------------------------------------------
292
+ # The container uploads to S3 via the presigned URL but doesn't
293
+ # echo the key back in its result JSON. Fall back to the s3_key
294
+ # from the job payload so the backend can store it.
295
+ result_url = result.get("result_url") or job.get("s3_key", "")
296
+
297
+ # Build result_metadata: use explicit field if present, otherwise
298
+ # collect useful top-level fields the container emitted.
299
+ result_metadata = result.get("result_metadata") or {
300
+ k: v
301
+ for k, v in result.items()
302
+ if k not in ("type", "status", "result_url", "result_metadata")
303
+ }
304
+
183
305
  portal.complete_job(
184
306
  job_id,
185
307
  elapsed_seconds=elapsed,
186
- result_url=result.get("result_url"),
187
- result_metadata=result.get("result_metadata"),
308
+ result_url=result_url,
309
+ result_metadata=result_metadata,
188
310
  )
189
311
  console.print(f" [green]Completed in {elapsed:.1f}s[/green]")
190
312
 
@@ -1,6 +1,6 @@
1
1
  [project]
2
2
  name = "poly-hammer-worker"
3
- version = "0.1.0.dev5"
3
+ version = "0.1.1"
4
4
  description = "Poly Hammer self-hosted GPU worker agent"
5
5
  readme = "README.md"
6
6
  requires-python = ">=3.13"
@@ -14,7 +14,8 @@ dependencies = [
14
14
  ]
15
15
 
16
16
  [project.scripts]
17
- ph-worker = "ph_worker.cli:app"
17
+ ph-worker = "poly_hammer_worker.cli:app"
18
+ poly-hammer-worker = "poly_hammer_worker.cli:app"
18
19
 
19
20
  [dependency-groups]
20
21
  dev = ["ruff", "mypy", "pytest"]
@@ -296,7 +296,7 @@ wheels = [
296
296
 
297
297
  [[package]]
298
298
  name = "poly-hammer-worker"
299
- version = "0.1.0"
299
+ version = "0.1.1"
300
300
  source = { editable = "." }
301
301
  dependencies = [
302
302
  { name = "docker" },