hexaqueue-worker 0.1.0__tar.gz

hexaqueue_worker-0.1.0/PKG-INFO

@@ -0,0 +1,50 @@
+Metadata-Version: 2.4
+Name: hexaqueue-worker
+Version: 0.1.0
+Summary: Hexaqueue Worker - Lightweight Compute Node Execution Daemon
+Author: Richard West
+Author-email: Richard West <dopplereffect.us@gmail.com>
+License-Expression: Apache-2.0
+Requires-Dist: hexaqueue-core
+Requires-Dist: hexastack-auth>=0.3.0
+Requires-Dist: hexastack-core>=0.3.0
+Requires-Dist: hexastack-grpc>=0.3.0
+Requires-Dist: hexastack-logging>=0.3.0
+Requires-Python: >=3.13
+Description-Content-Type: text/markdown
+
+# 🐝 `hexaqueue-worker`
+
+> Lightweight compute node execution daemon with GPU isolation and real-time streaming.
+
+[![PyPI: hexaqueue-worker](https://img.shields.io/pypi/v/hexaqueue-worker.svg)](https://pypi.org/project/hexaqueue-worker/)
+[![Python 3.13+](https://img.shields.io/badge/python-3.13+-blue.svg)](https://www.python.org/downloads/)
+[![Coverage](https://codecov.io/github/TheTrueSCU/hexaqueue/graph/badge.svg?component=hexaqueue_worker)](https://codecov.io/github/TheTrueSCU/hexaqueue)
+[![License: Apache 2.0](https://img.shields.io/badge/license-Apache%202.0-blue.svg)](../../LICENSE)
+
+---
+
+## 🏗️ Architecture & Dependencies
+
+```mermaid
+graph TD
+    subgraph S1["hexaqueue-worker"]
+        RUNNER["Execution Runtimes (cgroups v2, Docker/Podman, Apptainer)"]
+        GPU["GPU Masking (NVML & CUDA_VISIBLE_DEVICES)"]
+        STREAM["Log & Telemetry gRPC Streaming"]
+    end
+
+    subgraph S2["Internal & Hexastack Dependencies"]
+        HQ_CORE["hexaqueue-core"]
+        HS_AUTH["hexastack-auth (SPIFFE mTLS)"]
+        HS_CORE["hexastack-core"]
+        HS_GRPC["hexastack-grpc (Bidirectional Streams)"]
+        HS_LOG["hexastack-logging (structlog)"]
+    end
+
+    S1 --> HQ_CORE
+    S1 --> HS_AUTH
+    S1 --> HS_CORE
+    S1 --> HS_GRPC
+    S1 --> HS_LOG
+```

hexaqueue_worker-0.1.0/README.md

@@ -0,0 +1,35 @@
+# 🐝 `hexaqueue-worker`
+
+> Lightweight compute node execution daemon with GPU isolation and real-time streaming.
+
+[![PyPI: hexaqueue-worker](https://img.shields.io/pypi/v/hexaqueue-worker.svg)](https://pypi.org/project/hexaqueue-worker/)
+[![Python 3.13+](https://img.shields.io/badge/python-3.13+-blue.svg)](https://www.python.org/downloads/)
+[![Coverage](https://codecov.io/github/TheTrueSCU/hexaqueue/graph/badge.svg?component=hexaqueue_worker)](https://codecov.io/github/TheTrueSCU/hexaqueue)
+[![License: Apache 2.0](https://img.shields.io/badge/license-Apache%202.0-blue.svg)](../../LICENSE)
+
+---
+
+## 🏗️ Architecture & Dependencies
+
+```mermaid
+graph TD
+    subgraph S1["hexaqueue-worker"]
+        RUNNER["Execution Runtimes (cgroups v2, Docker/Podman, Apptainer)"]
+        GPU["GPU Masking (NVML & CUDA_VISIBLE_DEVICES)"]
+        STREAM["Log & Telemetry gRPC Streaming"]
+    end
+
+    subgraph S2["Internal & Hexastack Dependencies"]
+        HQ_CORE["hexaqueue-core"]
+        HS_AUTH["hexastack-auth (SPIFFE mTLS)"]
+        HS_CORE["hexastack-core"]
+        HS_GRPC["hexastack-grpc (Bidirectional Streams)"]
+        HS_LOG["hexastack-logging (structlog)"]
+    end
+
+    S1 --> HQ_CORE
+    S1 --> HS_AUTH
+    S1 --> HS_CORE
+    S1 --> HS_GRPC
+    S1 --> HS_LOG
+```

hexaqueue_worker-0.1.0/pyproject.toml

@@ -0,0 +1,25 @@
+[project]
+name = "hexaqueue-worker"
+version = "0.1.0"
+description = "Hexaqueue Worker - Lightweight Compute Node Execution Daemon"
+readme = "README.md"
+license = "Apache-2.0"
+requires-python = ">=3.13"
+dependencies = [
+    "hexaqueue-core",
+    "hexastack-auth>=0.3.0",
+    "hexastack-core>=0.3.0",
+    "hexastack-grpc>=0.3.0",
+    "hexastack-logging>=0.3.0",
+]
+
+[[project.authors]]
+name = "Richard West"
+email = "dopplereffect.us@gmail.com"
+
+[build-system]
+requires = ["uv_build>=0.12.3,<0.13.0"]
+build-backend = "uv_build"
+
+[tool.uv.sources.hexaqueue-core]
+workspace = true

hexaqueue_worker-0.1.0/pyproject.toml.orig

@@ -0,0 +1,24 @@
+[project]
+name = "hexaqueue-worker"
+version = "0.1.0"
+description = "Hexaqueue Worker - Lightweight Compute Node Execution Daemon"
+readme = "README.md"
+license = "Apache-2.0"
+authors = [
+    { name = "Richard West", email = "dopplereffect.us@gmail.com" }
+]
+requires-python = ">=3.13"
+dependencies = [
+    "hexaqueue-core",
+    "hexastack-auth>=0.3.0",
+    "hexastack-core>=0.3.0",
+    "hexastack-grpc>=0.3.0",
+    "hexastack-logging>=0.3.0",
+]
+
+[build-system]
+requires = ["uv_build>=0.12.3,<0.13.0"]
+build-backend = "uv_build"
+
+[tool.uv.sources]
+hexaqueue-core = { workspace = true }

hexaqueue_worker-0.1.0/src/hexaqueue_worker/__init__.py

@@ -0,0 +1,14 @@
+"""Hexaqueue Worker - Lightweight Compute Node Execution Daemon."""
+
+from hexaqueue_worker.adapters.local import LocalSubprocessWorker
+from hexaqueue_worker.domain.models import WorkerConfig, WorkerMetrics
+from hexaqueue_worker.infra.bootstrap import WorkerBootstrapper
+from hexaqueue_worker.ports.worker import WorkerDaemonPort
+
+__all__ = [
+    "LocalSubprocessWorker",
+    "WorkerBootstrapper",
+    "WorkerConfig",
+    "WorkerDaemonPort",
+    "WorkerMetrics",
+]

hexaqueue_worker-0.1.0/src/hexaqueue_worker/adapters/__init__.py

@@ -0,0 +1,7 @@
+"""Adapters package for hexaqueue-worker."""
+
+from hexaqueue_worker.adapters.local import LocalSubprocessWorker
+
+__all__ = [
+    "LocalSubprocessWorker",
+]

hexaqueue_worker-0.1.0/src/hexaqueue_worker/adapters/local.py

@@ -0,0 +1,179 @@
+"""Local subprocess compute node worker daemon adapter.
+
+Notes/Architectural Intent:
+    Polls ready jobs from JobQueuePort, allocates isolated scratch storage via StorageVolumePort,
+    runs commands via ExecutionRuntimePort, and updates terminal outcomes via SchedulerControllerPort.
+    Ensures zero cross-job pollution by cleaning up ephemeral scratch allocations in a finally block.
+"""
+
+import asyncio
+import contextlib
+from typing import Any
+
+from hexaqueue_core.adapters.runtime.local import LocalSubprocessExecutionRuntimeAdapter
+from hexaqueue_core.adapters.storage.local import LocalDiskStorageVolumeAdapter
+from hexaqueue_core.domain.job import JobSpec
+from hexaqueue_core.ports.logging import LogStreamPort
+from hexaqueue_core.ports.queue import JobQueuePort
+from hexaqueue_core.ports.runtime import ExecutionRuntimePort, ProcessExecutionResult
+from hexaqueue_core.ports.storage import StorageVolumePort, VolumeAllocation
+from hexaqueue_worker.domain.models import WorkerConfig, WorkerMetrics
+from hexaqueue_worker.ports.worker import WorkerDaemonPort
+
+
+class LocalSubprocessWorker(WorkerDaemonPort):
+    """Local subprocess execution worker daemon.
+
+    Args:
+        queue: JobQueuePort to dequeue ready jobs from.
+        controller: Optional SchedulerControllerPort to update job terminal outcomes.
+        runtime: Optional ExecutionRuntimePort (defaults to LocalSubprocessExecutionRuntimeAdapter).
+        storage: Optional StorageVolumePort (defaults to LocalStorageVolumeAdapter).
+        log_port: Optional LogStreamPort for streaming stdout/stderr.
+        config: Optional WorkerConfig for concurrency and polling parameters.
+    """
+
+    def __init__(
+        self,
+        queue: JobQueuePort,
+        controller: Any | None = None,
+        runtime: ExecutionRuntimePort | None = None,
+        storage: StorageVolumePort | None = None,
+        log_port: LogStreamPort | None = None,
+        config: WorkerConfig | None = None,
+    ) -> None:
+        self._queue = queue
+        self._controller = controller
+        self._log_port = log_port
+        self._runtime = runtime or LocalSubprocessExecutionRuntimeAdapter(
+            log_port=self._log_port
+        )
+        self._storage = storage or LocalDiskStorageVolumeAdapter()
+        self._config = config or WorkerConfig()
+
+        self._semaphore = asyncio.Semaphore(self._config.concurrency)
+        self._active_jobs: set[str] = set()
+        self._running = False
+        self._worker_tasks: set[asyncio.Task[None]] = set()
+        self._loop_task: asyncio.Task[None] | None = None
+
+        self._total_executed = 0
+        self._total_completed = 0
+        self._total_failed = 0
+        self._lock = asyncio.Lock()
+
+    async def start(self) -> None:
+        """Start worker polling loop in background task."""
+        async with self._lock:
+            if self._running:
+                return
+            self._running = True
+            self._loop_task = asyncio.create_task(self._poll_loop())
+
+    async def stop(self) -> None:
+        """Gracefully stop worker loop and await in-flight tasks."""
+        async with self._lock:
+            if not self._running:
+                return
+            self._running = False
+
+        if self._loop_task:
+            self._loop_task.cancel()
+            with contextlib.suppress(asyncio.CancelledError):
+                await self._loop_task
+            self._loop_task = None
+
+        if self._worker_tasks:
+            await asyncio.gather(*self._worker_tasks, return_exceptions=True)
+            self._worker_tasks.clear()
+
+    async def _poll_loop(self) -> None:
+        """Main queue polling loop."""
+        while self._running:
+            try:
+                # Wait for concurrency slot
+                await self._semaphore.acquire()
+                job = await self._queue.dequeue(
+                    timeout_seconds=self._config.poll_interval_seconds
+                )
+                if job is None:
+                    self._semaphore.release()
+                    await asyncio.sleep(self._config.poll_interval_seconds)
+                    continue
+
+                task = asyncio.create_task(self._process_job_slot(job))
+                self._worker_tasks.add(task)
+                task.add_done_callback(self._worker_tasks.discard)
+            except asyncio.CancelledError:
+                break
+            except Exception:
+                await asyncio.sleep(self._config.poll_interval_seconds)
+
+    async def _process_job_slot(self, job: JobSpec) -> None:
+        """Process job with acquired semaphore slot."""
+        try:
+            await self.execute_job(job)
+        finally:
+            self._semaphore.release()
+
+    async def execute_job(self, job: JobSpec) -> ProcessExecutionResult:
+        """Execute a single job within isolated scratch storage and report status."""
+        async with self._lock:
+            self._active_jobs.add(job.id)
+            self._total_executed += 1
+
+        scratch_vol: VolumeAllocation | None = None
+        try:
+            # 1. Allocate isolated scratch workspace
+            scratch_vol = await self._storage.allocate_scratch(
+                job_id=job.id,
+                size_mb=job.resources.scratch_mb,
+                base_dir=self._config.scratch_base_dir,
+            )
+
+            # 2. Execute process via runtime
+            result = await self._runtime.execute(
+                job=job,
+                scratch_volume=scratch_vol,
+                environment=job.env,
+            )
+
+            # 3. Notify controller if configured
+            if self._controller:
+                await self._controller.update_job_outcome(
+                    job_id=job.id,
+                    outcome=result.outcome,
+                    reason=result.error_message,
+                )
+
+            async with self._lock:
+                if result.exit_code == 0:
+                    self._total_completed += 1
+                else:
+                    self._total_failed += 1
+
+            return result
+        finally:
+            # 4. Clean up scratch storage
+            if scratch_vol and scratch_vol.is_ephemeral:
+                await self._storage.cleanup_scratch(scratch_vol.volume_id)
+
+            async with self._lock:
+                self._active_jobs.discard(job.id)
+
+    async def get_metrics(self) -> WorkerMetrics:
+        """Retrieve real-time operational worker metrics."""
+        async with self._lock:
+            return WorkerMetrics(
+                worker_id=self._config.worker_id,
+                active_jobs=len(self._active_jobs),
+                total_executed=self._total_executed,
+                total_completed=self._total_completed,
+                total_failed=self._total_failed,
+                is_running=self._running,
+            )
+
+
+__all__ = [
+    "LocalSubprocessWorker",
+]

hexaqueue_worker-0.1.0/src/hexaqueue_worker/domain/__init__.py

@@ -0,0 +1,8 @@
+"""Domain package for hexaqueue-worker."""
+
+from hexaqueue_worker.domain.models import WorkerConfig, WorkerMetrics
+
+__all__ = [
+    "WorkerConfig",
+    "WorkerMetrics",
+]

hexaqueue_worker-0.1.0/src/hexaqueue_worker/domain/models.py

@@ -0,0 +1,70 @@
+"""Domain models for worker configuration and metrics.
+
+Notes/Architectural Intent:
+    Defines configuration structures and status metrics for worker daemon operations,
+    specifying concurrency thresholds, polling intervals, and scratch volume base paths.
+"""
+
+from pydantic import BaseModel, ConfigDict, Field
+
+
+class WorkerConfig(BaseModel):
+    """Configuration options for a compute worker daemon instance.
+
+    Args:
+        worker_id: Unique worker identifier.
+        concurrency: Maximum number of concurrent jobs to execute.
+        poll_interval_seconds: Polling interval when waiting for queued jobs.
+        scratch_base_dir: Optional base directory for isolating scratch workspaces.
+        grace_period_seconds: Timeout to wait after SIGTERM before SIGKILL.
+    """
+
+    model_config = ConfigDict(frozen=True, extra="forbid")
+
+    concurrency: int = Field(
+        default=4, ge=1, description="Maximum concurrent job execution limit"
+    )
+    grace_period_seconds: int = Field(
+        default=15, ge=1, description="Grace period before SIGKILL"
+    )
+    poll_interval_seconds: float = Field(
+        default=0.1, gt=0.0, description="Queue polling interval in seconds"
+    )
+    scratch_base_dir: str | None = Field(
+        default=None, description="Base directory for scratch storage allocations"
+    )
+    worker_id: str = Field(
+        default="local-worker", description="Unique worker instance identifier"
+    )
+
+
+class WorkerMetrics(BaseModel):
+    """Real-time operational metrics of a compute worker.
+
+    Args:
+        worker_id: Unique worker identifier.
+        active_jobs: Count of jobs currently executing.
+        total_executed: Total count of jobs executed since startup.
+        total_failed: Total count of failed jobs.
+        total_completed: Total count of successfully completed jobs.
+        is_running: Whether the worker execution loop is active.
+    """
+
+    model_config = ConfigDict(frozen=True, extra="forbid")
+
+    active_jobs: int = Field(ge=0, description="Number of currently executing jobs")
+    is_running: bool = Field(description="Whether the worker daemon is running")
+    total_completed: int = Field(
+        default=0, ge=0, description="Total completed jobs count"
+    )
+    total_executed: int = Field(
+        default=0, ge=0, description="Total executed jobs count"
+    )
+    total_failed: int = Field(default=0, ge=0, description="Total failed jobs count")
+    worker_id: str = Field(description="Worker instance identifier")
+
+
+__all__ = [
+    "WorkerConfig",
+    "WorkerMetrics",
+]

hexaqueue_worker-0.1.0/src/hexaqueue_worker/infra/__init__.py

@@ -0,0 +1,7 @@
+"""Infra package for hexaqueue-worker."""
+
+from hexaqueue_worker.infra.bootstrap import WorkerBootstrapper
+
+__all__ = [
+    "WorkerBootstrapper",
+]

hexaqueue_worker-0.1.0/src/hexaqueue_worker/infra/bootstrap.py

@@ -0,0 +1,23 @@
+"""Bootstrap lifecycle handler for hexaqueue-worker."""
+
+from hexaqueue_core.infra.bootstrap import HexaqueueBootstrapContext
+from hexaqueue_core.infra.registries import HexaqueueConfigRegistry
+from hexaqueue_core.ports.bootstrap import BootstrapperPort
+
+
+class WorkerBootstrapper(BootstrapperPort):
+    """Bootstrapper for hexaqueue-worker services and registries."""
+
+    name: str = "worker"
+    order: int = 40
+
+    def register_config(self, registry: HexaqueueConfigRegistry) -> None:
+        """Register worker configuration models."""
+
+    def configure(self, context: HexaqueueBootstrapContext) -> None:
+        """Configure worker services in application DI container."""
+
+
+__all__ = [
+    "WorkerBootstrapper",
+]

hexaqueue_worker-0.1.0/src/hexaqueue_worker/ports/__init__.py

@@ -0,0 +1,7 @@
+"""Ports package for hexaqueue-worker."""
+
+from hexaqueue_worker.ports.worker import WorkerDaemonPort
+
+__all__ = [
+    "WorkerDaemonPort",
+]

hexaqueue_worker-0.1.0/src/hexaqueue_worker/ports/worker.py

@@ -0,0 +1,48 @@
+"""Worker daemon port interface.
+
+Notes/Architectural Intent:
+    Defines the contract for worker daemons managing task polling, isolated workspace provisioning,
+    command execution via ExecutionRuntimePort, and outcome notification to SchedulerControllerPort.
+"""
+
+from abc import ABC, abstractmethod
+
+from hexaqueue_core.domain.job import JobSpec
+from hexaqueue_core.ports.runtime import ProcessExecutionResult
+from hexaqueue_worker.domain.models import WorkerMetrics
+
+
+class WorkerDaemonPort(ABC):
+    """Abstract port interface for worker compute daemons."""
+
+    @abstractmethod
+    async def start(self) -> None:
+        """Start worker loop pulling and executing jobs."""
+
+    @abstractmethod
+    async def stop(self) -> None:
+        """Gracefully stop worker loop and await in-flight tasks."""
+
+    @abstractmethod
+    async def execute_job(self, job: JobSpec) -> ProcessExecutionResult:
+        """Execute a single job within isolated scratch storage and report status.
+
+        Args:
+            job: JobSpec to execute.
+
+        Returns:
+            ProcessExecutionResult summarizing execution outcome.
+        """
+
+    @abstractmethod
+    async def get_metrics(self) -> WorkerMetrics:
+        """Retrieve real-time operational worker metrics.
+
+        Returns:
+            WorkerMetrics instance.
+        """
+
+
+__all__ = [
+    "WorkerDaemonPort",
+]