poly-hammer-worker 0.1.0.dev5__tar.gz → 0.1.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {poly_hammer_worker-0.1.0.dev5 → poly_hammer_worker-0.1.1}/.gitignore +1 -0
- {poly_hammer_worker-0.1.0.dev5 → poly_hammer_worker-0.1.1}/PKG-INFO +7 -31
- {poly_hammer_worker-0.1.0.dev5 → poly_hammer_worker-0.1.1}/README.md +6 -30
- {poly_hammer_worker-0.1.0.dev5 → poly_hammer_worker-0.1.1}/poly_hammer_worker/api_client.py +23 -8
- poly_hammer_worker-0.1.1/poly_hammer_worker/banner.py +22 -0
- {poly_hammer_worker-0.1.0.dev5 → poly_hammer_worker-0.1.1}/poly_hammer_worker/cli.py +40 -5
- {poly_hammer_worker-0.1.0.dev5 → poly_hammer_worker-0.1.1}/poly_hammer_worker/docker_runner.py +66 -27
- {poly_hammer_worker-0.1.0.dev5 → poly_hammer_worker-0.1.1}/poly_hammer_worker/hardware.py +2 -4
- poly_hammer_worker-0.1.1/poly_hammer_worker/image_discovery.py +188 -0
- poly_hammer_worker-0.1.1/poly_hammer_worker/logging_config.py +73 -0
- {poly_hammer_worker-0.1.0.dev5 → poly_hammer_worker-0.1.1}/poly_hammer_worker/worker_loop.py +141 -19
- {poly_hammer_worker-0.1.0.dev5 → poly_hammer_worker-0.1.1}/pyproject.toml +3 -2
- {poly_hammer_worker-0.1.0.dev5 → poly_hammer_worker-0.1.1}/uv.lock +1 -1
- {poly_hammer_worker-0.1.0.dev5 → poly_hammer_worker-0.1.1}/poly_hammer_worker/__init__.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: poly-hammer-worker
|
|
3
|
-
Version: 0.1.
|
|
3
|
+
Version: 0.1.1
|
|
4
4
|
Summary: Poly Hammer self-hosted GPU worker agent
|
|
5
5
|
License-Expression: MIT
|
|
6
6
|
Requires-Python: >=3.13
|
|
@@ -11,7 +11,7 @@ Requires-Dist: rich>=13.0
|
|
|
11
11
|
Requires-Dist: typer>=0.12
|
|
12
12
|
Description-Content-Type: text/markdown
|
|
13
13
|
|
|
14
|
-
#
|
|
14
|
+
# Poly Hammer Self-Hosted GPU Worker
|
|
15
15
|
|
|
16
16
|
A CLI agent that connects your GPU workstation to the Poly Hammer Portal, letting you run AI inference jobs on your own hardware instead of consuming cloud credits.
|
|
17
17
|
|
|
@@ -19,22 +19,16 @@ A CLI agent that connects your GPU workstation to the Poly Hammer Portal, lettin
|
|
|
19
19
|
|
|
20
20
|
- **NVIDIA GPU** with CUDA drivers installed
|
|
21
21
|
- **Docker** with [NVIDIA Container Toolkit](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/latest/install-guide.html) (`--gpus` support)
|
|
22
|
-
- **Python 3.
|
|
22
|
+
- **Python 3.13+**
|
|
23
23
|
- An active Poly Hammer Portal account with a registered worker key
|
|
24
24
|
|
|
25
|
-
## Installation
|
|
26
|
-
|
|
27
|
-
```bash
|
|
28
|
-
cd workers/self-hosted/client
|
|
29
|
-
uv sync
|
|
30
|
-
```
|
|
31
25
|
|
|
32
26
|
## Quick Start
|
|
33
27
|
|
|
34
28
|
### 1. Check GPU compatibility
|
|
35
29
|
|
|
36
30
|
```bash
|
|
37
|
-
|
|
31
|
+
uvx poly-hammer-worker check-gpu
|
|
38
32
|
```
|
|
39
33
|
|
|
40
34
|
This will display your GPU model, VRAM, CUDA version, CPU cores, and RAM.
|
|
@@ -46,14 +40,14 @@ Navigate to the **Workers** page in the Poly Hammer Portal and click **Add Worke
|
|
|
46
40
|
### 3. Start the worker
|
|
47
41
|
|
|
48
42
|
```bash
|
|
49
|
-
|
|
43
|
+
uvx poly-hammer-worker start --worker-key ph_worker_XXXXXXXX
|
|
50
44
|
```
|
|
51
45
|
|
|
52
46
|
Or use environment variables:
|
|
53
47
|
|
|
54
48
|
```bash
|
|
55
49
|
export PH_WORKER_KEY=ph_worker_XXXXXXXX
|
|
56
|
-
export PH_PORTAL_URL=https://portal.polyhammer.com # default
|
|
50
|
+
export PH_PORTAL_URL=https://api.portal.polyhammer.com # default
|
|
57
51
|
ph-worker start
|
|
58
52
|
```
|
|
59
53
|
|
|
@@ -86,18 +80,7 @@ Portal API ←→ ph-worker agent ←→ Docker (GPU container)
|
|
|
86
80
|
| Option | Env Var | Default | Description |
|
|
87
81
|
|--------|---------|---------|-------------|
|
|
88
82
|
| `--worker-key` | `PH_WORKER_KEY` | (required) | Worker API key from the portal |
|
|
89
|
-
| `--portal-url` | `PH_PORTAL_URL` | `https://portal.polyhammer.com` | Portal API base URL |
|
|
90
|
-
|
|
91
|
-
## Docker Image Tags
|
|
92
|
-
|
|
93
|
-
Worker images are published to GHCR with CUDA-versioned tags:
|
|
94
|
-
|
|
95
|
-
| Image | Tag | Description |
|
|
96
|
-
|-------|-----|-------------|
|
|
97
|
-
| `ghcr.io/poly-hammer/hy-motion-worker` | `cuda13.1` | HY-Motion models (latest stable) |
|
|
98
|
-
| `ghcr.io/poly-hammer/mdm-worker` | `cuda13.1` | MDM models (latest stable) |
|
|
99
|
-
| | `cuda13.1-<sha>` | Pinned to a specific commit |
|
|
100
|
-
| | `latest` | Latest build (any CUDA version) |
|
|
83
|
+
| `--portal-url` | `PH_PORTAL_URL` | `https://api.portal.polyhammer.com` | Portal API base URL |
|
|
101
84
|
|
|
102
85
|
## Troubleshooting
|
|
103
86
|
|
|
@@ -116,10 +99,3 @@ Worker images are published to GHCR with CUDA-versioned tags:
|
|
|
116
99
|
- Heartbeats are sent every 60 seconds; the portal marks workers offline after 120 seconds
|
|
117
100
|
- Check network connectivity to the portal URL
|
|
118
101
|
- Verify your worker key is correct
|
|
119
|
-
|
|
120
|
-
## Development
|
|
121
|
-
|
|
122
|
-
```bash
|
|
123
|
-
cd workers/self-hosted/client
|
|
124
|
-
uv sync --group dev
|
|
125
|
-
```
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
#
|
|
1
|
+
# Poly Hammer Self-Hosted GPU Worker
|
|
2
2
|
|
|
3
3
|
A CLI agent that connects your GPU workstation to the Poly Hammer Portal, letting you run AI inference jobs on your own hardware instead of consuming cloud credits.
|
|
4
4
|
|
|
@@ -6,22 +6,16 @@ A CLI agent that connects your GPU workstation to the Poly Hammer Portal, lettin
|
|
|
6
6
|
|
|
7
7
|
- **NVIDIA GPU** with CUDA drivers installed
|
|
8
8
|
- **Docker** with [NVIDIA Container Toolkit](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/latest/install-guide.html) (`--gpus` support)
|
|
9
|
-
- **Python 3.
|
|
9
|
+
- **Python 3.13+**
|
|
10
10
|
- An active Poly Hammer Portal account with a registered worker key
|
|
11
11
|
|
|
12
|
-
## Installation
|
|
13
|
-
|
|
14
|
-
```bash
|
|
15
|
-
cd workers/self-hosted/client
|
|
16
|
-
uv sync
|
|
17
|
-
```
|
|
18
12
|
|
|
19
13
|
## Quick Start
|
|
20
14
|
|
|
21
15
|
### 1. Check GPU compatibility
|
|
22
16
|
|
|
23
17
|
```bash
|
|
24
|
-
|
|
18
|
+
uvx poly-hammer-worker check-gpu
|
|
25
19
|
```
|
|
26
20
|
|
|
27
21
|
This will display your GPU model, VRAM, CUDA version, CPU cores, and RAM.
|
|
@@ -33,14 +27,14 @@ Navigate to the **Workers** page in the Poly Hammer Portal and click **Add Worke
|
|
|
33
27
|
### 3. Start the worker
|
|
34
28
|
|
|
35
29
|
```bash
|
|
36
|
-
|
|
30
|
+
uvx poly-hammer-worker start --worker-key ph_worker_XXXXXXXX
|
|
37
31
|
```
|
|
38
32
|
|
|
39
33
|
Or use environment variables:
|
|
40
34
|
|
|
41
35
|
```bash
|
|
42
36
|
export PH_WORKER_KEY=ph_worker_XXXXXXXX
|
|
43
|
-
export PH_PORTAL_URL=https://portal.polyhammer.com # default
|
|
37
|
+
export PH_PORTAL_URL=https://api.portal.polyhammer.com # default
|
|
44
38
|
ph-worker start
|
|
45
39
|
```
|
|
46
40
|
|
|
@@ -73,18 +67,7 @@ Portal API ←→ ph-worker agent ←→ Docker (GPU container)
|
|
|
73
67
|
| Option | Env Var | Default | Description |
|
|
74
68
|
|--------|---------|---------|-------------|
|
|
75
69
|
| `--worker-key` | `PH_WORKER_KEY` | (required) | Worker API key from the portal |
|
|
76
|
-
| `--portal-url` | `PH_PORTAL_URL` | `https://portal.polyhammer.com` | Portal API base URL |
|
|
77
|
-
|
|
78
|
-
## Docker Image Tags
|
|
79
|
-
|
|
80
|
-
Worker images are published to GHCR with CUDA-versioned tags:
|
|
81
|
-
|
|
82
|
-
| Image | Tag | Description |
|
|
83
|
-
|-------|-----|-------------|
|
|
84
|
-
| `ghcr.io/poly-hammer/hy-motion-worker` | `cuda13.1` | HY-Motion models (latest stable) |
|
|
85
|
-
| `ghcr.io/poly-hammer/mdm-worker` | `cuda13.1` | MDM models (latest stable) |
|
|
86
|
-
| | `cuda13.1-<sha>` | Pinned to a specific commit |
|
|
87
|
-
| | `latest` | Latest build (any CUDA version) |
|
|
70
|
+
| `--portal-url` | `PH_PORTAL_URL` | `https://api.portal.polyhammer.com` | Portal API base URL |
|
|
88
71
|
|
|
89
72
|
## Troubleshooting
|
|
90
73
|
|
|
@@ -103,10 +86,3 @@ Worker images are published to GHCR with CUDA-versioned tags:
|
|
|
103
86
|
- Heartbeats are sent every 60 seconds; the portal marks workers offline after 120 seconds
|
|
104
87
|
- Check network connectivity to the portal URL
|
|
105
88
|
- Verify your worker key is correct
|
|
106
|
-
|
|
107
|
-
## Development
|
|
108
|
-
|
|
109
|
-
```bash
|
|
110
|
-
cd workers/self-hosted/client
|
|
111
|
-
uv sync --group dev
|
|
112
|
-
```
|
|
@@ -12,11 +12,12 @@ import httpx
|
|
|
12
12
|
logger = logging.getLogger(__name__)
|
|
13
13
|
|
|
14
14
|
# Endpoints (relative to portal base URL)
|
|
15
|
-
_POLL_PATH = "/
|
|
16
|
-
_COMPLETE_PATH = "/
|
|
17
|
-
_FAIL_PATH = "/
|
|
18
|
-
_PROGRESS_PATH = "/
|
|
19
|
-
_HEARTBEAT_PATH = "/
|
|
15
|
+
_POLL_PATH = "/v1/worker-jobs/poll"
|
|
16
|
+
_COMPLETE_PATH = "/v1/worker-jobs/{job_id}/complete"
|
|
17
|
+
_FAIL_PATH = "/v1/worker-jobs/{job_id}/fail"
|
|
18
|
+
_PROGRESS_PATH = "/v1/worker-jobs/{job_id}/progress"
|
|
19
|
+
_HEARTBEAT_PATH = "/v1/worker-jobs/heartbeat"
|
|
20
|
+
_DISCONNECT_PATH = "/v1/worker-jobs/disconnect"
|
|
20
21
|
|
|
21
22
|
|
|
22
23
|
class PortalClient:
|
|
@@ -93,15 +94,21 @@ class PortalClient:
|
|
|
93
94
|
|
|
94
95
|
def heartbeat(
|
|
95
96
|
self,
|
|
96
|
-
supported_models: list[str] | None = None,
|
|
97
97
|
hardware_info: dict | None = None,
|
|
98
|
+
available_images: list[dict] | None = None,
|
|
99
|
+
disk_free_gb: float | None = None,
|
|
100
|
+
assigned_service_id: str | None = None,
|
|
98
101
|
) -> None:
|
|
99
102
|
"""Send a heartbeat to keep the worker marked ONLINE."""
|
|
100
103
|
body: dict[str, Any] = {}
|
|
101
|
-
if supported_models is not None:
|
|
102
|
-
body["supported_models"] = supported_models
|
|
103
104
|
if hardware_info is not None:
|
|
104
105
|
body["hardware_info"] = hardware_info
|
|
106
|
+
if available_images is not None:
|
|
107
|
+
body["available_images"] = available_images
|
|
108
|
+
if disk_free_gb is not None:
|
|
109
|
+
body["disk_free_gb"] = disk_free_gb
|
|
110
|
+
if assigned_service_id is not None:
|
|
111
|
+
body["assigned_service_id"] = assigned_service_id
|
|
105
112
|
|
|
106
113
|
try:
|
|
107
114
|
resp = self._client.post(_HEARTBEAT_PATH, json=body)
|
|
@@ -109,5 +116,13 @@ class PortalClient:
|
|
|
109
116
|
except httpx.HTTPError:
|
|
110
117
|
logger.warning("Heartbeat failed (will retry)")
|
|
111
118
|
|
|
119
|
+
def disconnect(self) -> None:
|
|
120
|
+
"""Notify the portal that this worker is going offline."""
|
|
121
|
+
try:
|
|
122
|
+
resp = self._client.post(_DISCONNECT_PATH)
|
|
123
|
+
resp.raise_for_status()
|
|
124
|
+
except httpx.HTTPError:
|
|
125
|
+
logger.warning("Disconnect notification failed (non-fatal)")
|
|
126
|
+
|
|
112
127
|
def close(self) -> None:
|
|
113
128
|
self._client.close()
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
"""ASCII art banner for the ph-worker CLI."""
|
|
2
|
+
|
|
3
|
+
from rich.console import Console
|
|
4
|
+
from rich.text import Text
|
|
5
|
+
|
|
6
|
+
_BANNER_ART = r"""
|
|
7
|
+
____ _ _ __ __ _
|
|
8
|
+
| _ \| | | | \ \ / /__ _ __| | _____ _ __
|
|
9
|
+
| |_) | |_| | \ \ /\ / / _ \| '__| |/ / _ \ '__|
|
|
10
|
+
| __/| _ | \ V V / (_) | | | < __/ |
|
|
11
|
+
|_| |_| |_| \_/\_/ \___/|_| |_|\_\___|_|
|
|
12
|
+
|
|
13
|
+
Poly Hammer · Self-Hosted Inference Agent
|
|
14
|
+
"""
|
|
15
|
+
|
|
16
|
+
GOLD = "color(178)" # 256-color gold / yellow-gold
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def print_banner(console: Console) -> None:
|
|
20
|
+
"""Print the ph-worker banner in gold."""
|
|
21
|
+
text = Text(_BANNER_ART, style=GOLD)
|
|
22
|
+
console.print(text)
|
|
@@ -70,23 +70,55 @@ def start(
|
|
|
70
70
|
help="Optional container registry URL to pull images from if not found locally.",
|
|
71
71
|
),
|
|
72
72
|
] = None,
|
|
73
|
+
service: Annotated[
|
|
74
|
+
str | None,
|
|
75
|
+
typer.Option(
|
|
76
|
+
"--service",
|
|
77
|
+
"-s",
|
|
78
|
+
envvar="PH_SERVICE",
|
|
79
|
+
help="Service ID to assign this worker to (e.g. 'text_to_motion').",
|
|
80
|
+
),
|
|
81
|
+
] = None,
|
|
82
|
+
disk_check_interval: Annotated[
|
|
83
|
+
int,
|
|
84
|
+
typer.Option(
|
|
85
|
+
"--disk-check-interval",
|
|
86
|
+
envvar="PH_DISK_CHECK_INTERVAL",
|
|
87
|
+
help="Seconds between disk space checks (default 300).",
|
|
88
|
+
),
|
|
89
|
+
] = 300,
|
|
90
|
+
detach: Annotated[
|
|
91
|
+
bool,
|
|
92
|
+
typer.Option(
|
|
93
|
+
"--detach",
|
|
94
|
+
help="Register, display settings, perform one successful poll, then exit.",
|
|
95
|
+
),
|
|
96
|
+
] = False,
|
|
73
97
|
) -> None:
|
|
74
98
|
"""Start the worker agent loop.
|
|
75
99
|
|
|
76
100
|
The worker will:
|
|
77
101
|
1. Detect local GPU/CUDA capabilities
|
|
78
|
-
2.
|
|
79
|
-
3.
|
|
80
|
-
4.
|
|
81
|
-
5.
|
|
102
|
+
2. Discover portal-compatible Docker images via OCI labels
|
|
103
|
+
3. Connect to the portal and send heartbeats (with images + disk info)
|
|
104
|
+
4. Long-poll for inference jobs
|
|
105
|
+
5. Use locally-built Docker images (or pull from --registry if set)
|
|
106
|
+
6. Run inference and upload results
|
|
82
107
|
"""
|
|
108
|
+
from poly_hammer_worker.logging_config import configure_logging
|
|
83
109
|
from poly_hammer_worker.worker_loop import run_worker_loop
|
|
84
110
|
|
|
111
|
+
log_file = configure_logging()
|
|
112
|
+
console.print(f" Logs: {log_file}\n")
|
|
113
|
+
|
|
85
114
|
run_worker_loop(
|
|
86
115
|
worker_key=worker_key,
|
|
87
116
|
portal_url=portal_url,
|
|
88
117
|
auto_cleanup=auto_cleanup,
|
|
89
118
|
registry_url=registry,
|
|
119
|
+
detach=detach,
|
|
120
|
+
service=service,
|
|
121
|
+
disk_check_interval=disk_check_interval,
|
|
90
122
|
console=console,
|
|
91
123
|
)
|
|
92
124
|
|
|
@@ -98,7 +130,10 @@ def check_gpu() -> None:
|
|
|
98
130
|
|
|
99
131
|
info = detect_hardware()
|
|
100
132
|
|
|
101
|
-
|
|
133
|
+
from poly_hammer_worker.banner import print_banner
|
|
134
|
+
|
|
135
|
+
print_banner(console)
|
|
136
|
+
console.print("[bold]GPU / Hardware Info[/bold]\n")
|
|
102
137
|
console.print(f" GPU Model: {info.get('gpu_model', 'N/A')}")
|
|
103
138
|
console.print(f" VRAM: {info.get('vram_gb', 'N/A')} GB")
|
|
104
139
|
console.print(f" CUDA Version: {info.get('cuda_version', 'N/A')}")
|
{poly_hammer_worker-0.1.0.dev5 → poly_hammer_worker-0.1.1}/poly_hammer_worker/docker_runner.py
RENAMED
|
@@ -9,9 +9,12 @@ Handles:
|
|
|
9
9
|
|
|
10
10
|
import json
|
|
11
11
|
import logging
|
|
12
|
+
import os
|
|
13
|
+
import platform
|
|
12
14
|
import shutil
|
|
13
15
|
import time
|
|
14
16
|
from collections.abc import Callable
|
|
17
|
+
from pathlib import Path
|
|
15
18
|
|
|
16
19
|
import docker
|
|
17
20
|
import docker.errors
|
|
@@ -34,15 +37,6 @@ class DockerRunError(Exception):
|
|
|
34
37
|
super().__init__(f"Container exited with code {exit_code}: {stderr}")
|
|
35
38
|
|
|
36
39
|
|
|
37
|
-
def resolve_image_tag(docker_image: str, cuda_version: str) -> str:
|
|
38
|
-
"""Build the fully-qualified image reference.
|
|
39
|
-
|
|
40
|
-
The backend stores bare image names (e.g. ``ph-hy-motion``).
|
|
41
|
-
This appends the CUDA version tag matching the local GPU.
|
|
42
|
-
"""
|
|
43
|
-
return f"{docker_image}:cuda{cuda_version}"
|
|
44
|
-
|
|
45
|
-
|
|
46
40
|
class InsufficientDiskError(Exception):
|
|
47
41
|
"""Raised when there isn't enough disk space to pull an image."""
|
|
48
42
|
|
|
@@ -80,6 +74,48 @@ def get_docker_root() -> str:
|
|
|
80
74
|
return "/var/lib/docker"
|
|
81
75
|
|
|
82
76
|
|
|
77
|
+
def _resolve_docker_host_storage_path() -> str:
|
|
78
|
+
"""Resolve the host filesystem path where Docker actually stores data.
|
|
79
|
+
|
|
80
|
+
On Linux the Docker root (e.g. ``/var/lib/docker``) is a real local
|
|
81
|
+
path. On Windows and macOS, Docker Desktop runs inside a VM and the
|
|
82
|
+
reported root is a Linux path that does not exist on the host. This
|
|
83
|
+
function returns the host-side directory whose partition should be
|
|
84
|
+
checked for free space.
|
|
85
|
+
"""
|
|
86
|
+
system = platform.system()
|
|
87
|
+
|
|
88
|
+
if system == "Linux":
|
|
89
|
+
return get_docker_root()
|
|
90
|
+
|
|
91
|
+
if system == "Windows":
|
|
92
|
+
# Docker Desktop (WSL2 backend) stores its ext4.vhdx here
|
|
93
|
+
local_app_data = os.environ.get("LOCALAPPDATA", "")
|
|
94
|
+
if local_app_data:
|
|
95
|
+
docker_wsl = os.path.join(local_app_data, "Docker", "wsl")
|
|
96
|
+
if os.path.isdir(docker_wsl):
|
|
97
|
+
return docker_wsl
|
|
98
|
+
# Older Docker Desktop installations
|
|
99
|
+
program_data = os.environ.get("ProgramData", r"C:\ProgramData")
|
|
100
|
+
docker_pd = os.path.join(program_data, "DockerDesktop")
|
|
101
|
+
if os.path.isdir(docker_pd):
|
|
102
|
+
return docker_pd
|
|
103
|
+
# Last resort: Docker Desktop defaults to C: drive
|
|
104
|
+
return os.environ.get("SystemDrive", "C:\\")
|
|
105
|
+
|
|
106
|
+
if system == "Darwin":
|
|
107
|
+
# Docker Desktop for Mac stores its VM disk image here
|
|
108
|
+
docker_data = (
|
|
109
|
+
Path.home() / "Library" / "Containers" / "com.docker.docker" / "Data"
|
|
110
|
+
)
|
|
111
|
+
if docker_data.is_dir():
|
|
112
|
+
return str(docker_data)
|
|
113
|
+
return str(Path.home())
|
|
114
|
+
|
|
115
|
+
# Unknown OS — try the Docker root directly
|
|
116
|
+
return get_docker_root()
|
|
117
|
+
|
|
118
|
+
|
|
83
119
|
def get_free_disk_gb(path: str | None = None) -> float:
|
|
84
120
|
"""Return free disk space in GB on the partition containing *path*.
|
|
85
121
|
|
|
@@ -87,8 +123,12 @@ def get_free_disk_gb(path: str | None = None) -> float:
|
|
|
87
123
|
where images will actually be stored.
|
|
88
124
|
"""
|
|
89
125
|
if path is None:
|
|
90
|
-
path =
|
|
91
|
-
|
|
126
|
+
path = _resolve_docker_host_storage_path()
|
|
127
|
+
try:
|
|
128
|
+
usage = shutil.disk_usage(path)
|
|
129
|
+
except (FileNotFoundError, OSError):
|
|
130
|
+
logger.debug("Cannot stat %s, falling back to SystemDrive", path)
|
|
131
|
+
usage = shutil.disk_usage(os.environ.get("SystemDrive", "/"))
|
|
92
132
|
return round(usage.free / (1024**3), 2)
|
|
93
133
|
|
|
94
134
|
|
|
@@ -251,6 +291,7 @@ def run_inference_container(
|
|
|
251
291
|
job_payload: dict,
|
|
252
292
|
on_progress: Callable[[float], None] | None = None,
|
|
253
293
|
on_status: Callable[[str], None] | None = None,
|
|
294
|
+
on_log: Callable[[str], None] | None = None,
|
|
254
295
|
) -> dict:
|
|
255
296
|
"""Run an inference container and return the result dict.
|
|
256
297
|
|
|
@@ -266,6 +307,8 @@ def run_inference_container(
|
|
|
266
307
|
Called with progress float (0.0–1.0) as the container reports progress.
|
|
267
308
|
on_status:
|
|
268
309
|
Called with human-readable status strings (pulling, running, etc.).
|
|
310
|
+
on_log:
|
|
311
|
+
Called with each line of container output (both stdout and stderr).
|
|
269
312
|
|
|
270
313
|
Returns
|
|
271
314
|
-------
|
|
@@ -275,14 +318,12 @@ def run_inference_container(
|
|
|
275
318
|
------
|
|
276
319
|
DockerRunError — If the container exits non-zero or emits an error message.
|
|
277
320
|
"""
|
|
278
|
-
stdin_bytes = json.dumps(job_payload).encode()
|
|
279
|
-
|
|
280
321
|
if on_status:
|
|
281
322
|
on_status(f"Creating container from {image_ref}")
|
|
282
323
|
|
|
283
324
|
container = client.containers.create(
|
|
284
325
|
image_ref,
|
|
285
|
-
|
|
326
|
+
environment={"JOB_PAYLOAD": json.dumps(job_payload)},
|
|
286
327
|
detach=True,
|
|
287
328
|
device_requests=[
|
|
288
329
|
docker.types.DeviceRequest(count=-1, capabilities=[["gpu"]]),
|
|
@@ -295,26 +336,26 @@ def run_inference_container(
|
|
|
295
336
|
try:
|
|
296
337
|
container.start()
|
|
297
338
|
|
|
298
|
-
# Attach and pipe job JSON into stdin
|
|
299
|
-
sock = container.attach_socket(params={"stdin": 1, "stream": 1})
|
|
300
|
-
sock._sock.sendall(stdin_bytes)
|
|
301
|
-
sock._sock.shutdown(1) # EOF on stdin
|
|
302
|
-
sock.close()
|
|
303
|
-
|
|
304
339
|
if on_status:
|
|
305
340
|
on_status("Inference running …")
|
|
306
341
|
|
|
307
|
-
# Stream stdout for JSON-line progress
|
|
342
|
+
# Stream both stdout and stderr for JSON-line progress + container logs
|
|
343
|
+
stderr_lines: list[str] = []
|
|
308
344
|
for raw_line in container.logs(
|
|
309
|
-
stream=True, follow=True, stdout=True, stderr=
|
|
345
|
+
stream=True, follow=True, stdout=True, stderr=True
|
|
310
346
|
):
|
|
311
347
|
line = raw_line.decode("utf-8", errors="replace").strip()
|
|
312
348
|
if not line:
|
|
313
349
|
continue
|
|
350
|
+
|
|
351
|
+
if on_log:
|
|
352
|
+
on_log(line)
|
|
353
|
+
|
|
314
354
|
try:
|
|
315
355
|
msg = json.loads(line)
|
|
316
356
|
except json.JSONDecodeError:
|
|
317
|
-
|
|
357
|
+
# Non-JSON output is stderr or unstructured stdout
|
|
358
|
+
stderr_lines.append(line)
|
|
318
359
|
continue
|
|
319
360
|
|
|
320
361
|
msg_type = msg.get("type")
|
|
@@ -335,10 +376,8 @@ def run_inference_container(
|
|
|
335
376
|
raise DockerRunError(exit_code, error_msg)
|
|
336
377
|
|
|
337
378
|
if exit_code != 0:
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
)[-2000:]
|
|
341
|
-
raise DockerRunError(exit_code, stderr_logs)
|
|
379
|
+
stderr_tail = "\n".join(stderr_lines)[-2000:]
|
|
380
|
+
raise DockerRunError(exit_code, stderr_tail)
|
|
342
381
|
|
|
343
382
|
if result_msg is None:
|
|
344
383
|
raise DockerRunError(
|
|
@@ -12,7 +12,7 @@ import subprocess
|
|
|
12
12
|
|
|
13
13
|
import psutil
|
|
14
14
|
|
|
15
|
-
from poly_hammer_worker.docker_runner import
|
|
15
|
+
from poly_hammer_worker.docker_runner import get_free_disk_gb
|
|
16
16
|
|
|
17
17
|
logger = logging.getLogger(__name__)
|
|
18
18
|
|
|
@@ -106,9 +106,7 @@ def _get_cuda_version_from_smi(nvidia_smi: str) -> str | None:
|
|
|
106
106
|
def _get_docker_disk_free_gb() -> float | None:
|
|
107
107
|
"""Return free GB on the partition where Docker stores images."""
|
|
108
108
|
try:
|
|
109
|
-
|
|
110
|
-
usage = shutil.disk_usage(docker_root)
|
|
111
|
-
return round(usage.free / (1024**3), 1)
|
|
109
|
+
return get_free_disk_gb()
|
|
112
110
|
except Exception:
|
|
113
111
|
logger.debug("Could not check Docker disk free space", exc_info=True)
|
|
114
112
|
return None
|
|
@@ -0,0 +1,188 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Docker image discovery for Poly Hammer Portal compatible images.
|
|
3
|
+
|
|
4
|
+
Scans local Docker images for ``ai.poly-hammer.*`` OCI labels and returns
|
|
5
|
+
structured metadata for each compatible image. Each image may contain
|
|
6
|
+
multiple models with independent hardware requirements, parsed from
|
|
7
|
+
per-model labels (``ai.poly-hammer.model.<id>.*``).
|
|
8
|
+
|
|
9
|
+
Required labels (image is skipped if any are missing):
|
|
10
|
+
- ai.poly-hammer.portal-compatible = "true"
|
|
11
|
+
- ai.poly-hammer.service-id
|
|
12
|
+
- ai.poly-hammer.model-ids (comma-separated model IDs)
|
|
13
|
+
|
|
14
|
+
Optional labels:
|
|
15
|
+
- ai.poly-hammer.min-cuda-version Minimum host CUDA version required.
|
|
16
|
+
When set **and** *host_cuda_version* is passed to
|
|
17
|
+
:func:`discover_local_images`, images whose minimum exceeds the
|
|
18
|
+
host capability are skipped with a warning.
|
|
19
|
+
|
|
20
|
+
Per-model labels (optional, looked up for each model ID):
|
|
21
|
+
- ai.poly-hammer.model.<id>.name
|
|
22
|
+
- ai.poly-hammer.model.<id>.min-vram-gb
|
|
23
|
+
- ai.poly-hammer.model.<id>.min-ram-gb
|
|
24
|
+
"""
|
|
25
|
+
|
|
26
|
+
import logging
|
|
27
|
+
from typing import Any
|
|
28
|
+
|
|
29
|
+
import docker
|
|
30
|
+
|
|
31
|
+
logger = logging.getLogger(__name__)
|
|
32
|
+
|
|
33
|
+
_PREFIX = "ai.poly-hammer."
|
|
34
|
+
|
|
35
|
+
_REQUIRED_LABELS = frozenset(
|
|
36
|
+
{
|
|
37
|
+
"portal-compatible",
|
|
38
|
+
"service-id",
|
|
39
|
+
"model-ids",
|
|
40
|
+
}
|
|
41
|
+
)
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def discover_local_images(
|
|
45
|
+
client: docker.DockerClient,
|
|
46
|
+
host_cuda_version: str | None = None,
|
|
47
|
+
) -> list[dict[str, Any]]:
|
|
48
|
+
"""Scan local Docker images for portal-compatible OCI labels.
|
|
49
|
+
|
|
50
|
+
Returns a list of dicts matching the ``WorkerImageInfo`` schema — one
|
|
51
|
+
entry per Docker image. Each entry contains a ``models`` list with
|
|
52
|
+
per-model metadata (name, hardware requirements).
|
|
53
|
+
|
|
54
|
+
If *host_cuda_version* is provided (e.g. ``"13.1"``), images whose
|
|
55
|
+
``ai.poly-hammer.min-cuda-version`` label exceeds it are skipped.
|
|
56
|
+
"""
|
|
57
|
+
results: list[dict[str, Any]] = []
|
|
58
|
+
|
|
59
|
+
try:
|
|
60
|
+
images = client.images.list()
|
|
61
|
+
except docker.errors.DockerException:
|
|
62
|
+
logger.exception("Failed to list Docker images")
|
|
63
|
+
return results
|
|
64
|
+
|
|
65
|
+
for image in images:
|
|
66
|
+
labels = image.labels or {}
|
|
67
|
+
|
|
68
|
+
# Quick check: skip images without the portal-compatible flag
|
|
69
|
+
if labels.get(f"{_PREFIX}portal-compatible") != "true":
|
|
70
|
+
continue
|
|
71
|
+
|
|
72
|
+
# Extract all ai.poly-hammer.* labels (strip prefix)
|
|
73
|
+
ph_labels: dict[str, str] = {}
|
|
74
|
+
for key, value in labels.items():
|
|
75
|
+
if key.startswith(_PREFIX):
|
|
76
|
+
ph_labels[key[len(_PREFIX) :]] = value
|
|
77
|
+
|
|
78
|
+
# Validate required labels
|
|
79
|
+
missing = _REQUIRED_LABELS - ph_labels.keys()
|
|
80
|
+
if missing:
|
|
81
|
+
tag = _best_tag(image)
|
|
82
|
+
logger.warning(
|
|
83
|
+
"Image %s has portal-compatible flag but missing required labels: %s",
|
|
84
|
+
tag,
|
|
85
|
+
", ".join(sorted(missing)),
|
|
86
|
+
)
|
|
87
|
+
continue
|
|
88
|
+
|
|
89
|
+
image_ref = _best_tag(image)
|
|
90
|
+
if not image_ref:
|
|
91
|
+
logger.debug("Skipping untagged image %s", image.short_id)
|
|
92
|
+
continue
|
|
93
|
+
|
|
94
|
+
service_id = ph_labels["service-id"]
|
|
95
|
+
min_cuda_version: str | None = ph_labels.get("min-cuda-version")
|
|
96
|
+
|
|
97
|
+
# Host compatibility check: skip images that need a newer CUDA
|
|
98
|
+
if min_cuda_version and host_cuda_version:
|
|
99
|
+
if _parse_version(min_cuda_version) > _parse_version(host_cuda_version):
|
|
100
|
+
logger.warning(
|
|
101
|
+
"Image %s requires CUDA >= %s but host has %s — skipping",
|
|
102
|
+
image_ref,
|
|
103
|
+
min_cuda_version,
|
|
104
|
+
host_cuda_version,
|
|
105
|
+
)
|
|
106
|
+
continue
|
|
107
|
+
|
|
108
|
+
# Image size in GB (Docker reports bytes)
|
|
109
|
+
image_size_gb = round(image.attrs.get("Size", 0) / (1024**3), 2)
|
|
110
|
+
|
|
111
|
+
# Parse comma-separated model IDs
|
|
112
|
+
model_ids = [m.strip() for m in ph_labels["model-ids"].split(",") if m.strip()]
|
|
113
|
+
|
|
114
|
+
# Build per-model metadata from ai.poly-hammer.model.<id>.* labels
|
|
115
|
+
models: list[dict[str, Any]] = []
|
|
116
|
+
for model_id in model_ids:
|
|
117
|
+
model_prefix = f"model.{model_id}."
|
|
118
|
+
model_name = ph_labels.get(f"{model_prefix}name")
|
|
119
|
+
min_vram_gb = _parse_int(ph_labels.get(f"{model_prefix}min-vram-gb", "0"))
|
|
120
|
+
min_ram_gb = _parse_int(ph_labels.get(f"{model_prefix}min-ram-gb", "0"))
|
|
121
|
+
models.append(
|
|
122
|
+
{
|
|
123
|
+
"model_id": model_id,
|
|
124
|
+
"model_name": model_name,
|
|
125
|
+
"min_vram_gb": min_vram_gb,
|
|
126
|
+
"min_ram_gb": min_ram_gb,
|
|
127
|
+
}
|
|
128
|
+
)
|
|
129
|
+
|
|
130
|
+
results.append(
|
|
131
|
+
{
|
|
132
|
+
"image_ref": image_ref,
|
|
133
|
+
"service_id": service_id,
|
|
134
|
+
"min_cuda_version": min_cuda_version,
|
|
135
|
+
"image_size_gb": image_size_gb,
|
|
136
|
+
"models": models,
|
|
137
|
+
}
|
|
138
|
+
)
|
|
139
|
+
|
|
140
|
+
total_models = sum(len(img["models"]) for img in results)
|
|
141
|
+
logger.info(
|
|
142
|
+
"Discovered %d portal-compatible image(s) with %d model(s)",
|
|
143
|
+
len(results),
|
|
144
|
+
total_models,
|
|
145
|
+
)
|
|
146
|
+
return results
|
|
147
|
+
|
|
148
|
+
|
|
149
|
+
def _best_tag(image: Any) -> str | None:
|
|
150
|
+
"""Return the most useful tag for an image, or None if untagged."""
|
|
151
|
+
tags = image.tags
|
|
152
|
+
if tags:
|
|
153
|
+
return tags[0]
|
|
154
|
+
return None
|
|
155
|
+
|
|
156
|
+
|
|
157
|
+
def _parse_int(value: str) -> int:
|
|
158
|
+
"""Parse an integer from a label value, defaulting to 0."""
|
|
159
|
+
try:
|
|
160
|
+
return int(value)
|
|
161
|
+
except (ValueError, TypeError):
|
|
162
|
+
return 0
|
|
163
|
+
|
|
164
|
+
|
|
165
|
+
def _parse_version(version: str) -> tuple[int, ...]:
|
|
166
|
+
"""Parse a dotted version string into a tuple of ints for comparison."""
|
|
167
|
+
try:
|
|
168
|
+
return tuple(int(p) for p in version.split("."))
|
|
169
|
+
except (ValueError, TypeError):
|
|
170
|
+
return (0,)
|
|
171
|
+
|
|
172
|
+
|
|
173
|
+
def resolve_image_for_model(
|
|
174
|
+
available_images: list[dict[str, Any]], model_id: str
|
|
175
|
+
) -> str | None:
|
|
176
|
+
"""Find the local Docker image reference that supports a given model ID.
|
|
177
|
+
|
|
178
|
+
Searches the list returned by :func:`discover_local_images` for the
|
|
179
|
+
first image whose ``models`` list contains the requested *model_id*.
|
|
180
|
+
|
|
181
|
+
Returns the ``image_ref`` (e.g. ``"hy-motion:1.0"``) or ``None`` if
|
|
182
|
+
no matching image is found.
|
|
183
|
+
"""
|
|
184
|
+
for img in available_images:
|
|
185
|
+
for model in img.get("models", []):
|
|
186
|
+
if model.get("model_id") == model_id:
|
|
187
|
+
return img.get("image_ref")
|
|
188
|
+
return None
|
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Logging configuration for the ph-worker agent.
|
|
3
|
+
|
|
4
|
+
Configures both console and rotating file handlers with timestamps.
|
|
5
|
+
Log files are stored in a platform-appropriate directory and automatically
|
|
6
|
+
cleaned up after 30 days.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
import logging
|
|
10
|
+
import os
|
|
11
|
+
import sys
|
|
12
|
+
from logging.handlers import TimedRotatingFileHandler
|
|
13
|
+
from pathlib import Path
|
|
14
|
+
|
|
15
|
+
_LOG_FORMAT = "[%(levelname)s] %(asctime)s %(name)s %(message)s"
|
|
16
|
+
_LOG_DATE_FORMAT = "%H:%M:%S %d-%m-%Y"
|
|
17
|
+
_RETENTION_DAYS = 30
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def _log_dir() -> Path:
|
|
21
|
+
"""Return the log directory, creating it if needed.
|
|
22
|
+
|
|
23
|
+
Uses ``PH_WORKER_LOG_DIR`` env var if set, otherwise falls back to
|
|
24
|
+
``~/.poly-hammer-worker/logs/``.
|
|
25
|
+
"""
|
|
26
|
+
env = os.environ.get("PH_WORKER_LOG_DIR")
|
|
27
|
+
if env:
|
|
28
|
+
path = Path(env)
|
|
29
|
+
else:
|
|
30
|
+
path = Path.home() / ".poly-hammer-worker" / "logs"
|
|
31
|
+
path.mkdir(parents=True, exist_ok=True)
|
|
32
|
+
return path
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def configure_logging(*, verbose: bool = False) -> Path:
|
|
36
|
+
"""Set up root logger with console + rotating file handlers.
|
|
37
|
+
|
|
38
|
+
Returns the log file path so the CLI can display it.
|
|
39
|
+
"""
|
|
40
|
+
level = logging.DEBUG if verbose else logging.INFO
|
|
41
|
+
root = logging.getLogger()
|
|
42
|
+
root.setLevel(level)
|
|
43
|
+
|
|
44
|
+
# Remove any pre-existing handlers (e.g. from basicConfig)
|
|
45
|
+
root.handlers.clear()
|
|
46
|
+
|
|
47
|
+
formatter = logging.Formatter(_LOG_FORMAT, datefmt=_LOG_DATE_FORMAT)
|
|
48
|
+
|
|
49
|
+
# ---- Console handler (stderr) ----
|
|
50
|
+
console_handler = logging.StreamHandler(sys.stderr)
|
|
51
|
+
console_handler.setLevel(level)
|
|
52
|
+
console_handler.setFormatter(formatter)
|
|
53
|
+
root.addHandler(console_handler)
|
|
54
|
+
|
|
55
|
+
# ---- Rotating file handler (daily rotation, 30-day retention) ----
|
|
56
|
+
log_file = _log_dir() / "ph-worker.log"
|
|
57
|
+
file_handler = TimedRotatingFileHandler(
|
|
58
|
+
filename=str(log_file),
|
|
59
|
+
when="midnight",
|
|
60
|
+
interval=1,
|
|
61
|
+
backupCount=_RETENTION_DAYS,
|
|
62
|
+
encoding="utf-8",
|
|
63
|
+
)
|
|
64
|
+
file_handler.setLevel(logging.DEBUG) # always capture debug in file
|
|
65
|
+
file_handler.setFormatter(formatter)
|
|
66
|
+
root.addHandler(file_handler)
|
|
67
|
+
|
|
68
|
+
# ---- Quiet noisy third-party loggers unless verbose ----
|
|
69
|
+
if not verbose:
|
|
70
|
+
for name in ("httpx", "httpcore", "urllib3", "docker", "hpack"):
|
|
71
|
+
logging.getLogger(name).setLevel(logging.WARNING)
|
|
72
|
+
|
|
73
|
+
return log_file
|
{poly_hammer_worker-0.1.0.dev5 → poly_hammer_worker-0.1.1}/poly_hammer_worker/worker_loop.py
RENAMED
|
@@ -8,6 +8,7 @@ Responsibilities:
|
|
|
8
8
|
"""
|
|
9
9
|
|
|
10
10
|
import logging
|
|
11
|
+
import queue
|
|
11
12
|
import signal
|
|
12
13
|
import threading
|
|
13
14
|
import time
|
|
@@ -21,15 +22,15 @@ from poly_hammer_worker.docker_runner import (
|
|
|
21
22
|
InsufficientDiskError,
|
|
22
23
|
ensure_image,
|
|
23
24
|
get_free_disk_gb,
|
|
24
|
-
resolve_image_tag,
|
|
25
25
|
run_inference_container,
|
|
26
26
|
)
|
|
27
27
|
from poly_hammer_worker.hardware import detect_hardware
|
|
28
|
+
from poly_hammer_worker.image_discovery import resolve_image_for_model
|
|
28
29
|
|
|
29
30
|
logger = logging.getLogger(__name__)
|
|
30
31
|
|
|
31
32
|
# Intervals (seconds)
|
|
32
|
-
HEARTBEAT_INTERVAL =
|
|
33
|
+
HEARTBEAT_INTERVAL = 30 # well within the 60s backend timeout
|
|
33
34
|
POLL_BACKOFF_BASE = 5
|
|
34
35
|
POLL_BACKOFF_MAX = 30
|
|
35
36
|
|
|
@@ -40,9 +41,19 @@ def run_worker_loop(
|
|
|
40
41
|
console: Console,
|
|
41
42
|
auto_cleanup: bool = False,
|
|
42
43
|
registry_url: str | None = None,
|
|
44
|
+
detach: bool = False,
|
|
45
|
+
service: str | None = None,
|
|
46
|
+
disk_check_interval: int = 300,
|
|
43
47
|
) -> None:
|
|
44
|
-
"""Blocking top-level loop. Runs until SIGINT / SIGTERM.
|
|
45
|
-
|
|
48
|
+
"""Blocking top-level loop. Runs until SIGINT / SIGTERM.
|
|
49
|
+
|
|
50
|
+
If *detach* is True the loop registers, displays settings, waits for
|
|
51
|
+
one successful poll round, then exits cleanly.
|
|
52
|
+
"""
|
|
53
|
+
from poly_hammer_worker.banner import print_banner
|
|
54
|
+
from poly_hammer_worker.image_discovery import discover_local_images
|
|
55
|
+
|
|
56
|
+
print_banner(console)
|
|
46
57
|
|
|
47
58
|
# ---- Hardware detection ---------------------------------------------------
|
|
48
59
|
hw = detect_hardware()
|
|
@@ -67,16 +78,49 @@ def run_worker_loop(
|
|
|
67
78
|
console.print(f" Registry: {registry_url}")
|
|
68
79
|
else:
|
|
69
80
|
console.print(" Registry: [dim]local-only (no remote pull)[/dim]")
|
|
81
|
+
if service:
|
|
82
|
+
console.print(f" Service: {service}")
|
|
70
83
|
console.print()
|
|
71
84
|
|
|
72
85
|
# ---- Clients --------------------------------------------------------------
|
|
73
86
|
portal = PortalClient(portal_url, worker_key)
|
|
74
87
|
docker_client = docker.from_env()
|
|
75
88
|
|
|
89
|
+
# ---- Image discovery (initial) -------------------------------------------
|
|
90
|
+
available_images = discover_local_images(
|
|
91
|
+
docker_client, host_cuda_version=cuda_version
|
|
92
|
+
)
|
|
93
|
+
if available_images:
|
|
94
|
+
total_models = sum(len(img["models"]) for img in available_images)
|
|
95
|
+
console.print(
|
|
96
|
+
f"[green]Discovered {len(available_images)} image(s) with {total_models} model(s)[/green]"
|
|
97
|
+
)
|
|
98
|
+
for img in available_images:
|
|
99
|
+
console.print(f" {img['image_ref']} service={img['service_id']}")
|
|
100
|
+
for model in img["models"]:
|
|
101
|
+
console.print(f" - {model['model_id']}")
|
|
102
|
+
else:
|
|
103
|
+
console.print(
|
|
104
|
+
"[yellow]No portal-compatible Docker images found locally[/yellow]"
|
|
105
|
+
)
|
|
106
|
+
|
|
107
|
+
if service:
|
|
108
|
+
matching = [i for i in available_images if i["service_id"] == service]
|
|
109
|
+
if not matching:
|
|
110
|
+
console.print(
|
|
111
|
+
f"[yellow bold]Warning: no local images match service '{service}'[/yellow bold]"
|
|
112
|
+
)
|
|
113
|
+
|
|
114
|
+
console.print()
|
|
115
|
+
|
|
76
116
|
# ---- Shutdown flag --------------------------------------------------------
|
|
77
117
|
shutdown = threading.Event()
|
|
78
118
|
|
|
79
119
|
def _handle_signal(sig: int, _frame: object) -> None:
|
|
120
|
+
if shutdown.is_set():
|
|
121
|
+
# Second signal — force immediate exit
|
|
122
|
+
console.print("\n[red bold]Forced exit[/red bold]")
|
|
123
|
+
raise SystemExit(1)
|
|
80
124
|
console.print(f"\n[yellow]Received signal {sig}, shutting down …[/yellow]")
|
|
81
125
|
shutdown.set()
|
|
82
126
|
|
|
@@ -84,9 +128,33 @@ def run_worker_loop(
|
|
|
84
128
|
signal.signal(signal.SIGTERM, _handle_signal)
|
|
85
129
|
|
|
86
130
|
# ---- Heartbeat thread -----------------------------------------------------
|
|
131
|
+
_last_disk_check: float = 0.0
|
|
132
|
+
|
|
87
133
|
def _heartbeat_loop() -> None:
|
|
134
|
+
nonlocal _last_disk_check, available_images
|
|
88
135
|
while not shutdown.is_set():
|
|
89
|
-
|
|
136
|
+
# Re-discover images on each heartbeat (cheap — Docker API list)
|
|
137
|
+
available_images = discover_local_images(
|
|
138
|
+
docker_client, host_cuda_version=cuda_version
|
|
139
|
+
)
|
|
140
|
+
|
|
141
|
+
# Throttled disk space check
|
|
142
|
+
now = time.monotonic()
|
|
143
|
+
disk_gb: float | None = None
|
|
144
|
+
if now - _last_disk_check >= disk_check_interval:
|
|
145
|
+
disk_gb = get_free_disk_gb()
|
|
146
|
+
_last_disk_check = now
|
|
147
|
+
|
|
148
|
+
try:
|
|
149
|
+
portal.heartbeat(
|
|
150
|
+
hardware_info=hw,
|
|
151
|
+
available_images=available_images,
|
|
152
|
+
disk_free_gb=disk_gb,
|
|
153
|
+
assigned_service_id=service,
|
|
154
|
+
)
|
|
155
|
+
logger.info("Heartbeat sent")
|
|
156
|
+
except Exception:
|
|
157
|
+
logger.warning("Heartbeat failed", exc_info=True)
|
|
90
158
|
shutdown.wait(HEARTBEAT_INTERVAL)
|
|
91
159
|
|
|
92
160
|
hb_thread = threading.Thread(target=_heartbeat_loop, daemon=True, name="heartbeat")
|
|
@@ -98,23 +166,51 @@ def run_worker_loop(
|
|
|
98
166
|
console.print("[bold]Polling for jobs …[/bold]\n")
|
|
99
167
|
|
|
100
168
|
while not shutdown.is_set():
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
169
|
+
# Run poll in a daemon thread so Ctrl+C is never blocked by I/O
|
|
170
|
+
poll_q: queue.Queue[tuple[str, object]] = queue.Queue()
|
|
171
|
+
|
|
172
|
+
def _do_poll() -> None:
|
|
173
|
+
try:
|
|
174
|
+
poll_q.put(("ok", portal.poll_for_job()))
|
|
175
|
+
except Exception as exc:
|
|
176
|
+
poll_q.put(("error", exc))
|
|
177
|
+
|
|
178
|
+
poll_thread = threading.Thread(target=_do_poll, daemon=True)
|
|
179
|
+
poll_thread.start()
|
|
180
|
+
|
|
181
|
+
# Main thread sleeps in short intervals so signals are handled fast
|
|
182
|
+
while poll_thread.is_alive() and not shutdown.is_set():
|
|
183
|
+
poll_thread.join(timeout=0.5)
|
|
184
|
+
|
|
185
|
+
if shutdown.is_set():
|
|
186
|
+
break
|
|
187
|
+
|
|
188
|
+
tag, payload = poll_q.get_nowait()
|
|
189
|
+
|
|
190
|
+
if tag == "error":
|
|
191
|
+
logger.exception("Poll error", exc_info=payload)
|
|
105
192
|
console.print(f"[red]Poll error — retrying in {backoff}s[/red]")
|
|
106
193
|
shutdown.wait(backoff)
|
|
107
194
|
backoff = min(backoff * 2, POLL_BACKOFF_MAX)
|
|
108
195
|
continue
|
|
109
196
|
|
|
197
|
+
job = payload
|
|
198
|
+
|
|
110
199
|
if job is None:
|
|
111
|
-
#
|
|
200
|
+
# Successful poll with no job available
|
|
201
|
+
if detach:
|
|
202
|
+
console.print("[green]Poll successful — detaching.[/green]")
|
|
203
|
+
break
|
|
112
204
|
shutdown.wait(POLL_BACKOFF_BASE)
|
|
113
205
|
continue
|
|
114
206
|
|
|
115
207
|
# Reset backoff on successful poll
|
|
116
208
|
backoff = POLL_BACKOFF_BASE
|
|
117
209
|
|
|
210
|
+
if detach:
|
|
211
|
+
console.print("[green]Poll successful — detaching.[/green]")
|
|
212
|
+
break
|
|
213
|
+
|
|
118
214
|
job_id = str(job["job_id"])
|
|
119
215
|
model_id = job.get("model_id", "unknown")
|
|
120
216
|
console.print(
|
|
@@ -125,7 +221,7 @@ def run_worker_loop(
|
|
|
125
221
|
portal=portal,
|
|
126
222
|
docker_client=docker_client,
|
|
127
223
|
job=job,
|
|
128
|
-
|
|
224
|
+
available_images=available_images,
|
|
129
225
|
auto_cleanup=auto_cleanup,
|
|
130
226
|
registry_url=registry_url,
|
|
131
227
|
console=console,
|
|
@@ -133,6 +229,7 @@ def run_worker_loop(
|
|
|
133
229
|
|
|
134
230
|
# ---- Cleanup --------------------------------------------------------------
|
|
135
231
|
console.print("\n[bold]Shutting down …[/bold]")
|
|
232
|
+
portal.disconnect()
|
|
136
233
|
portal.close()
|
|
137
234
|
console.print("[green]Done.[/green]")
|
|
138
235
|
|
|
@@ -141,21 +238,26 @@ def _execute_job(
|
|
|
141
238
|
portal: PortalClient,
|
|
142
239
|
docker_client: docker.DockerClient,
|
|
143
240
|
job: dict,
|
|
144
|
-
|
|
241
|
+
available_images: list[dict],
|
|
145
242
|
console: Console,
|
|
146
243
|
auto_cleanup: bool = False,
|
|
147
244
|
registry_url: str | None = None,
|
|
148
245
|
) -> None:
|
|
149
246
|
"""Ensure image (local-first, optional registry pull), run container, report outcome."""
|
|
150
247
|
job_id = str(job["job_id"])
|
|
151
|
-
|
|
248
|
+
model_id = job.get("model_id", "")
|
|
152
249
|
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
250
|
+
image_ref = resolve_image_for_model(available_images, model_id)
|
|
251
|
+
if not image_ref:
|
|
252
|
+
portal.fail_job(
|
|
253
|
+
job_id,
|
|
254
|
+
f"No local Docker image found for model '{model_id}'. "
|
|
255
|
+
"Build the appropriate image and restart the worker.",
|
|
256
|
+
"NO_IMAGE",
|
|
257
|
+
)
|
|
258
|
+
console.print(f"[red] No local image for model '{model_id}' — skipped[/red]")
|
|
156
259
|
return
|
|
157
260
|
|
|
158
|
-
image_ref = resolve_image_tag(docker_image, cuda_version)
|
|
159
261
|
start_time = time.monotonic()
|
|
160
262
|
|
|
161
263
|
try:
|
|
@@ -169,22 +271,42 @@ def _execute_job(
|
|
|
169
271
|
)
|
|
170
272
|
|
|
171
273
|
# ---- Run container ----------------------------------------------------
|
|
274
|
+
container_logger = logging.getLogger(f"{__name__}.container")
|
|
275
|
+
|
|
276
|
+
def _on_log(line: str) -> None:
|
|
277
|
+
container_logger.info(line)
|
|
278
|
+
console.print(f" [dim]│ {line}[/dim]", highlight=False)
|
|
279
|
+
|
|
172
280
|
result = run_inference_container(
|
|
173
281
|
docker_client,
|
|
174
282
|
image_ref,
|
|
175
283
|
job_payload=job,
|
|
176
284
|
on_progress=lambda p: _on_progress(portal, job_id, p, console),
|
|
177
285
|
on_status=lambda s: console.print(f" [dim]{s}[/dim]"),
|
|
286
|
+
on_log=_on_log,
|
|
178
287
|
)
|
|
179
288
|
|
|
180
289
|
elapsed = time.monotonic() - start_time
|
|
181
290
|
|
|
182
291
|
# ---- Report completion ------------------------------------------------
|
|
292
|
+
# The container uploads to S3 via the presigned URL but doesn't
|
|
293
|
+
# echo the key back in its result JSON. Fall back to the s3_key
|
|
294
|
+
# from the job payload so the backend can store it.
|
|
295
|
+
result_url = result.get("result_url") or job.get("s3_key", "")
|
|
296
|
+
|
|
297
|
+
# Build result_metadata: use explicit field if present, otherwise
|
|
298
|
+
# collect useful top-level fields the container emitted.
|
|
299
|
+
result_metadata = result.get("result_metadata") or {
|
|
300
|
+
k: v
|
|
301
|
+
for k, v in result.items()
|
|
302
|
+
if k not in ("type", "status", "result_url", "result_metadata")
|
|
303
|
+
}
|
|
304
|
+
|
|
183
305
|
portal.complete_job(
|
|
184
306
|
job_id,
|
|
185
307
|
elapsed_seconds=elapsed,
|
|
186
|
-
result_url=
|
|
187
|
-
result_metadata=
|
|
308
|
+
result_url=result_url,
|
|
309
|
+
result_metadata=result_metadata,
|
|
188
310
|
)
|
|
189
311
|
console.print(f" [green]Completed in {elapsed:.1f}s[/green]")
|
|
190
312
|
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "poly-hammer-worker"
|
|
3
|
-
version = "0.1.
|
|
3
|
+
version = "0.1.1"
|
|
4
4
|
description = "Poly Hammer self-hosted GPU worker agent"
|
|
5
5
|
readme = "README.md"
|
|
6
6
|
requires-python = ">=3.13"
|
|
@@ -14,7 +14,8 @@ dependencies = [
|
|
|
14
14
|
]
|
|
15
15
|
|
|
16
16
|
[project.scripts]
|
|
17
|
-
ph-worker = "
|
|
17
|
+
ph-worker = "poly_hammer_worker.cli:app"
|
|
18
|
+
poly-hammer-worker = "poly_hammer_worker.cli:app"
|
|
18
19
|
|
|
19
20
|
[dependency-groups]
|
|
20
21
|
dev = ["ruff", "mypy", "pytest"]
|
|
File without changes
|