ls-algorithm-plugin-sdk 0.2.5__py3-none-any.whl

algorithm_plugin_sdk/context.py

@@ -0,0 +1,307 @@
+from __future__ import annotations
+
+import copy
+import logging
+import threading
+from dataclasses import dataclass, field
+from datetime import datetime, timezone
+from typing import Any, Callable, Literal, Mapping
+
+from .errors import ExecutionCancelled
+from .models import DatasetResult
+
+ProgressStatus = Literal[
+    "pending",
+    "running",
+    "succeeded",
+    "failed",
+    "skipped",
+    "cancelled",
+]
+
+
+def utc_now() -> str:
+    return datetime.now(timezone.utc).isoformat(timespec="milliseconds").replace("+00:00", "Z")
+
+
+@dataclass(slots=True)
+class EpisodeProgress:
+    episode: str
+    status: ProgressStatus = "pending"
+    stage: str = "pending"
+    completed: float | None = None
+    total: float | None = None
+    percent: float | None = None
+    message: str = ""
+    metrics: dict[str, Any] = field(default_factory=dict)
+    updated_at: str = field(default_factory=utc_now)
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "episode": self.episode,
+            "status": self.status,
+            "stage": self.stage,
+            "completed": self.completed,
+            "total": self.total,
+            "percent": self.percent,
+            "message": self.message,
+            "metrics": copy.deepcopy(self.metrics),
+            "updatedAt": self.updated_at,
+        }
+
+
+@dataclass(slots=True)
+class DatasetProgress:
+    dataset: str
+    status: ProgressStatus = "pending"
+    stage: str = "pending"
+    completed: float | None = None
+    total: float | None = None
+    percent: float | None = None
+    message: str = ""
+    metrics: dict[str, Any] = field(default_factory=dict)
+    episodes: dict[str, EpisodeProgress] = field(default_factory=dict)
+    updated_at: str = field(default_factory=utc_now)
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "dataset": self.dataset,
+            "status": self.status,
+            "stage": self.stage,
+            "completed": self.completed,
+            "total": self.total,
+            "percent": self.percent,
+            "message": self.message,
+            "metrics": copy.deepcopy(self.metrics),
+            "episodes": {
+                episode: progress.to_dict()
+                for episode, progress in self.episodes.items()
+            },
+            "updatedAt": self.updated_at,
+        }
+
+
+@dataclass(frozen=True, slots=True)
+class ProgressSnapshot:
+    execution_id: str
+    sequence: int
+    timestamp: str
+    datasets: dict[str, DatasetProgress]
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "executionId": self.execution_id,
+            "sequence": self.sequence,
+            "timestamp": self.timestamp,
+            "datasets": {
+                dataset: progress.to_dict()
+                for dataset, progress in self.datasets.items()
+            },
+        }
+
+
+ProgressCallback = Callable[[ProgressSnapshot], None]
+
+
+class ExecutionContext:
+    """Thread-safe execution controls and full-dataset progress aggregation."""
+
+    def __init__(
+        self,
+        execution_id: str,
+        datasets: list[str],
+        *,
+        scratch_dir: str | None = None,
+        deadline: datetime | None = None,
+        logger: logging.Logger | None = None,
+        progress_callback: ProgressCallback | None = None,
+    ) -> None:
+        self.execution_id = execution_id
+        self.scratch_dir = scratch_dir
+        self.deadline = deadline
+        self.logger = logger or logging.getLogger(
+            f"algorithm_plugin_sdk.execution.{execution_id}"
+        )
+        self._progress = {
+            dataset: DatasetProgress(dataset=dataset)
+            for dataset in datasets
+        }
+        self._progress_callback = progress_callback
+        self._sequence = 0
+        self._lock = threading.RLock()
+        self._report_lock = threading.RLock()
+        self._cancel_event = threading.Event()
+        self._cancel_reason: str | None = None
+
+    def report_progress(
+        self,
+        *,
+        dataset: str,
+        episode: str | None = None,
+        stage: str,
+        completed: int | float | None = None,
+        total: int | float | None = None,
+        message: str = "",
+        metrics: Mapping[str, Any] | None = None,
+        status: ProgressStatus = "running",
+    ) -> None:
+        if dataset not in self._progress:
+            raise ValueError(f"progress dataset is not in the request: {dataset}")
+        if not isinstance(stage, str) or not stage.strip():
+            raise ValueError("progress stage must be a non-empty string")
+        if episode is not None and (not isinstance(episode, str) or not episode.strip()):
+            raise ValueError("progress episode must be a non-empty string or None")
+        completed_value = self._number(completed, "completed")
+        total_value = self._number(total, "total")
+        if total_value is not None and total_value < 0:
+            raise ValueError("progress total cannot be negative")
+        if completed_value is not None and completed_value < 0:
+            raise ValueError("progress completed cannot be negative")
+
+        with self._report_lock:
+            with self._lock:
+                dataset_progress = self._progress[dataset]
+                if episode is None:
+                    progress = dataset_progress
+                else:
+                    progress = dataset_progress.episodes.setdefault(
+                        episode,
+                        EpisodeProgress(episode=episode),
+                    )
+                    if dataset_progress.status == "pending":
+                        dataset_progress.status = "running"
+                    dataset_progress.updated_at = utc_now()
+                self._update_progress(
+                    progress,
+                    status=status,
+                    stage=stage,
+                    completed=completed_value,
+                    total=total_value,
+                    message=message,
+                    metrics=metrics,
+                )
+                snapshot = self._next_snapshot_locked()
+            self._emit(snapshot)
+
+    def mark_dataset_result(self, result: DatasetResult) -> None:
+        with self._report_lock:
+            with self._lock:
+                progress = self._progress[result.dataset]
+                progress.status = result.status
+                progress.stage = "completed"
+                if progress.total is not None and result.status == "succeeded":
+                    progress.completed = progress.total
+                    progress.percent = 100.0
+                progress.message = result.error or result.merge_message or ""
+                progress.metrics = {
+                    **progress.metrics,
+                    **result.metrics,
+                    "mergeStatus": result.merge_status,
+                }
+                progress.updated_at = utc_now()
+                for episode in progress.episodes.values():
+                    if episode.status in {"pending", "running"}:
+                        episode.status = result.status
+                        episode.stage = "completed"
+                        if episode.total is not None and result.status == "succeeded":
+                            episode.completed = episode.total
+                            episode.percent = 100.0
+                        episode.updated_at = utc_now()
+                snapshot = self._next_snapshot_locked()
+            self._emit(snapshot)
+
+    def mark_unfinished(self, status: ProgressStatus, message: str) -> None:
+        with self._report_lock:
+            with self._lock:
+                changed = False
+                for progress in self._progress.values():
+                    if progress.status in {"pending", "running"}:
+                        progress.status = status
+                        progress.stage = "completed"
+                        progress.message = message
+                        progress.updated_at = utc_now()
+                        for episode in progress.episodes.values():
+                            if episode.status in {"pending", "running"}:
+                                episode.status = status
+                                episode.stage = "completed"
+                                episode.message = message
+                                episode.updated_at = utc_now()
+                        changed = True
+                snapshot = self._next_snapshot_locked() if changed else None
+            if snapshot is not None:
+                self._emit(snapshot)
+
+    def snapshot(self) -> ProgressSnapshot:
+        with self._lock:
+            return self._snapshot_locked()
+
+    def cancel(self, reason: str = "cancelled") -> None:
+        with self._lock:
+            self._cancel_reason = reason
+            self._cancel_event.set()
+
+    def is_cancelled(self) -> bool:
+        if self._cancel_event.is_set():
+            return True
+        if self.deadline is not None and datetime.now(timezone.utc) >= self.deadline:
+            self.cancel("deadline exceeded")
+            return True
+        return False
+
+    def raise_if_cancelled(self) -> None:
+        if self.is_cancelled():
+            raise ExecutionCancelled(self._cancel_reason or "execution cancelled")
+
+    @staticmethod
+    def _number(value: int | float | None, name: str) -> float | None:
+        if value is None:
+            return None
+        if isinstance(value, bool) or not isinstance(value, (int, float)):
+            raise ValueError(f"progress {name} must be numeric or None")
+        return float(value)
+
+    @staticmethod
+    def _percent(completed: float | None, total: float | None) -> float | None:
+        if completed is None or total is None or total <= 0:
+            return None
+        return round(max(0.0, min(100.0, completed / total * 100.0)), 4)
+
+    def _update_progress(
+        self,
+        progress: DatasetProgress | EpisodeProgress,
+        *,
+        status: ProgressStatus,
+        stage: str,
+        completed: float | None,
+        total: float | None,
+        message: str,
+        metrics: Mapping[str, Any] | None,
+    ) -> None:
+        progress.status = status
+        progress.stage = stage
+        progress.completed = completed
+        progress.total = total
+        progress.percent = self._percent(completed, total)
+        progress.message = message
+        progress.metrics = dict(metrics or {})
+        progress.updated_at = utc_now()
+
+    def _next_snapshot_locked(self) -> ProgressSnapshot:
+        self._sequence += 1
+        return self._snapshot_locked()
+
+    def _snapshot_locked(self) -> ProgressSnapshot:
+        return ProgressSnapshot(
+            execution_id=self.execution_id,
+            sequence=self._sequence,
+            timestamp=utc_now(),
+            datasets=copy.deepcopy(self._progress),
+        )
+
+    def _emit(self, snapshot: ProgressSnapshot) -> None:
+        if self._progress_callback is None:
+            return
+        try:
+            self._progress_callback(snapshot)
+        except Exception:
+            self.logger.exception("progress callback failed")

algorithm_plugin_sdk/deployment.py

@@ -0,0 +1,317 @@
+from __future__ import annotations
+
+import configparser
+import json
+import os
+from dataclasses import dataclass
+from importlib import metadata
+from pathlib import Path
+from typing import Any
+from urllib.parse import urlparse
+
+from .loader import ALGORITHM_ENTRY_POINT_GROUP, load_algorithm
+from .release import ReleaseManifest
+
+CONFIG_SCHEMA = "algorithm-plugin-sdk.deployment/v1"
+DEFAULT_CONFIG_NAME = "algorithm-plugin.json"
+
+
+def repository_root(start: str | Path | None = None) -> Path:
+    current = Path(start or Path.cwd()).expanduser().resolve()
+    if current.is_file():
+        current = current.parent
+    for candidate in (current, *current.parents):
+        if any(
+            (candidate / marker).exists()
+            for marker in ("release-manifest.json", "pyproject.toml", "setup.cfg", ".git")
+        ):
+            return candidate
+    raise RuntimeError(f"cannot find a repository from {current}")
+
+
+def _setup_cfg_entry_points(root: Path) -> list[tuple[str, str]]:
+    path = root / "setup.cfg"
+    if not path.is_file():
+        return []
+    parser = configparser.ConfigParser(interpolation=None)
+    parser.read(path, encoding="utf-8")
+    if not parser.has_section("options.entry_points"):
+        return []
+    raw = parser.get("options.entry_points", ALGORITHM_ENTRY_POINT_GROUP, fallback="")
+    entries: list[tuple[str, str]] = []
+    for line in raw.splitlines():
+        name, separator, reference = line.partition("=")
+        if separator and name.strip() and reference.strip():
+            entries.append((name.strip(), reference.strip()))
+    return entries
+
+
+def discover_algorithm_reference(
+    root: str | Path,
+    requested: str | None = None,
+) -> tuple[str, str | None]:
+    """Find the repository's sole Algorithm entry point.
+
+    The class reference from setup.cfg is preferred so config generation also works
+    from an editable source checkout. Installed entry points are the fallback.
+    """
+    repository = Path(root).resolve()
+    configured = _setup_cfg_entry_points(repository)
+    if requested:
+        for name, reference in configured:
+            if requested in {name, reference}:
+                return reference, name
+        return requested, None
+    if len(configured) == 1:
+        name, reference = configured[0]
+        return reference, name
+    if len(configured) > 1:
+        names = ", ".join(name for name, _ in configured)
+        raise RuntimeError(f"multiple algorithms found; select one explicitly: {names}")
+
+    installed = list(metadata.entry_points(group=ALGORITHM_ENTRY_POINT_GROUP))
+    if len(installed) == 1:
+        return installed[0].name, installed[0].name
+    if not installed:
+        raise RuntimeError(
+            "no Algorithm entry point found; declare algorithm_plugin_sdk.algorithms"
+        )
+    names = ", ".join(sorted(entry.name for entry in installed))
+    raise RuntimeError(f"multiple installed algorithms found; select one: {names}")
+
+
+def _absolute(value: str | Path | None, root: Path) -> str | None:
+    if value is None or not str(value).strip():
+        return None
+    path = Path(value).expanduser()
+    if not path.is_absolute():
+        path = root / path
+    return str(path.resolve())
+
+
+@dataclass(slots=True)
+class DeploymentConfig:
+    path: Path
+    payload: dict[str, Any]
+
+    @classmethod
+    def generate(
+        cls,
+        *,
+        output: str | Path,
+        root: str | Path,
+        algorithm: str | None,
+        host: str,
+        port: int,
+        webui: bool,
+        max_concurrency: int,
+        scratch_dir: str | None,
+        token: str | None,
+        registration: dict[str, str] | None,
+        gpu_ids: list[int] | None = None,
+        environment: dict[str, str] | None = None,
+    ) -> "DeploymentConfig":
+        repository = repository_root(root)
+        reference, entry_point = discover_algorithm_reference(repository, algorithm)
+        instance = load_algorithm(reference)
+        release = ReleaseManifest.discover(instance, start=repository, required=True)
+        assert release is not None
+        if port < 1 or port > 65535:
+            raise ValueError("service port must be in [1, 65535]")
+        if max_concurrency < 1:
+            raise ValueError("max concurrency must be positive")
+        configured_gpu_ids = list(gpu_ids or [])
+        if any(
+            isinstance(gpu_id, bool) or not isinstance(gpu_id, int) or gpu_id < 0
+            for gpu_id in configured_gpu_ids
+        ) or len(configured_gpu_ids) != len(set(configured_gpu_ids)):
+            raise ValueError("GPU IDs must be unique non-negative integers")
+
+        output_path = Path(output).expanduser()
+        if not output_path.is_absolute():
+            output_path = repository / output_path
+        output_path = output_path.resolve()
+        state_dir = output_path.parent / ".algorithm-plugin"
+        payload: dict[str, Any] = {
+            "schemaVersion": CONFIG_SCHEMA,
+            "algorithm": {
+                "reference": reference,
+                "entryPoint": entry_point,
+            },
+            "repository": str(repository),
+            "releaseManifest": str(repository / "release-manifest.json"),
+            "service": {
+                "host": host,
+                "port": port,
+                "webui": webui,
+                "maxConcurrency": max_concurrency,
+                "scratchDir": _absolute(scratch_dir, repository),
+                "gpuIds": configured_gpu_ids,
+                "token": token or None,
+            },
+            "registration": registration,
+            "environment": dict(sorted((environment or {}).items())),
+            "runtime": {
+                "pidFile": str(state_dir / "service.pid.json"),
+                "logFile": str(state_dir / "service.log"),
+            },
+        }
+        config = cls(output_path, payload)
+        config.refresh()
+        return config
+
+    @classmethod
+    def load(
+        cls,
+        path: str | Path,
+        *,
+        refresh: bool = True,
+    ) -> "DeploymentConfig":
+        config_path = Path(path).expanduser().resolve()
+        try:
+            payload = json.loads(config_path.read_text(encoding="utf-8"))
+        except FileNotFoundError as exc:
+            raise RuntimeError(f"deployment config is missing: {config_path}") from exc
+        except json.JSONDecodeError as exc:
+            raise RuntimeError(f"deployment config is invalid JSON: {exc}") from exc
+        if not isinstance(payload, dict) or payload.get("schemaVersion") != CONFIG_SCHEMA:
+            raise RuntimeError(f"deployment config schema must be {CONFIG_SCHEMA}")
+        config = cls(config_path, payload)
+        if refresh:
+            config.refresh()
+        return config
+
+    def refresh(self) -> None:
+        algorithm = self.payload.get("algorithm")
+        if not isinstance(algorithm, dict) or not algorithm.get("reference"):
+            raise RuntimeError("deployment config has no algorithm reference")
+        repository = Path(str(self.payload.get("repository") or "")).expanduser()
+        if not repository.is_absolute() or not repository.is_dir():
+            raise RuntimeError(f"configured repository is unavailable: {repository}")
+        instance = load_algorithm(str(algorithm["reference"]))
+        release_path = Path(str(self.payload.get("releaseManifest") or ""))
+        release = ReleaseManifest.load(
+            release_path,
+            expected_algorithm_type=instance.metadata().name.replace("-", "_"),
+        )
+        if release.version != instance.metadata().version:
+            raise RuntimeError(
+                "Algorithm metadata version does not match release-manifest.json: "
+                f"{instance.metadata().version} != {release.version}"
+            )
+        algorithm.update(
+            {
+                "type": release.algorithm_type,
+                "name": release.name,
+                "version": release.version,
+                "implementationDigest": release.implementation_digest,
+                "buildRevision": release.build_revision,
+            }
+        )
+        self._validate_service()
+        self._validate_registration()
+        environment = self.payload.get("environment", {})
+        if not isinstance(environment, dict) or not all(
+            isinstance(key, str) and isinstance(value, str)
+            for key, value in environment.items()
+        ):
+            raise RuntimeError("deployment environment must be a string map")
+
+    def _validate_service(self) -> None:
+        service = self.payload.get("service")
+        if not isinstance(service, dict):
+            raise RuntimeError("deployment config has no service settings")
+        port = int(service.get("port", 0))
+        concurrency = int(service.get("maxConcurrency", 0))
+        if not 1 <= port <= 65535:
+            raise RuntimeError("service port must be in [1, 65535]")
+        if concurrency < 1:
+            raise RuntimeError("service maxConcurrency must be positive")
+        gpu_ids = service.get("gpuIds")
+        if not isinstance(gpu_ids, list) or any(
+            isinstance(gpu_id, bool) or not isinstance(gpu_id, int) or gpu_id < 0
+            for gpu_id in gpu_ids
+        ) or len(gpu_ids) != len(set(gpu_ids)):
+            raise RuntimeError("service gpuIds must be unique non-negative integers")
+
+    def _validate_registration(self) -> None:
+        registration = self.payload.get("registration")
+        if registration is None:
+            return
+        required = (
+            "computeUrl", "apiKey", "publicUrl", "inputRoot", "workspaceRoot",
+            "cluster", "namespace", "nodeName",
+        )
+        if not isinstance(registration, dict):
+            raise RuntimeError("registration must be an object or null")
+        missing = [name for name in required if not str(registration.get(name) or "").strip()]
+        if missing:
+            raise RuntimeError("registration is missing: " + ", ".join(missing))
+        for name in ("computeUrl", "publicUrl"):
+            parsed = urlparse(str(registration[name]).strip())
+            if parsed.scheme not in {"http", "https"} or not parsed.hostname:
+                raise RuntimeError(f"registration {name} must be an http or https URL")
+        for name in ("inputRoot", "workspaceRoot"):
+            path = Path(str(registration[name])).expanduser()
+            if not path.is_absolute() or not path.is_dir():
+                raise RuntimeError(
+                    f"registration {name} must be an existing absolute directory"
+                )
+        workspace = Path(str(registration["workspaceRoot"])).expanduser()
+        if not os.access(workspace, os.W_OK):
+            raise RuntimeError("registration workspaceRoot must be writable")
+
+    def save(self) -> None:
+        self.path.parent.mkdir(parents=True, exist_ok=True)
+        temporary = self.path.with_name(f".{self.path.name}.tmp")
+        temporary.write_text(
+            json.dumps(self.payload, ensure_ascii=False, indent=2, sort_keys=True) + "\n",
+            encoding="utf-8",
+        )
+        os.chmod(temporary, 0o600)
+        temporary.replace(self.path)
+
+    @property
+    def algorithm_reference(self) -> str:
+        return str(self.payload["algorithm"]["reference"])
+
+    @property
+    def repository(self) -> Path:
+        return Path(self.payload["repository"])
+
+    @property
+    def pid_file(self) -> Path:
+        return Path(self.payload["runtime"]["pidFile"])
+
+    @property
+    def log_file(self) -> Path:
+        return Path(self.payload["runtime"]["logFile"])
+
+    def process_environment(self) -> dict[str, str]:
+        values = os.environ.copy()
+        values.update(self.payload.get("environment", {}))
+        registration = self.payload.get("registration")
+        if registration is None:
+            values["LDP_PLUGIN_REGISTRATION_DISABLED"] = "1"
+        else:
+            values.pop("LDP_PLUGIN_REGISTRATION_DISABLED", None)
+            values.update(
+                {
+                    "LDP_COMPUTE_URL": str(registration["computeUrl"]),
+                    "LDP_INTERNAL_API_KEY": str(registration["apiKey"]),
+                    "LDP_PLUGIN_PUBLIC_URL": str(registration["publicUrl"]),
+                    "LDP_JOB_INPUT_ROOT": str(registration["inputRoot"]),
+                    "LDP_JOB_WORKSPACE_ROOT": str(registration["workspaceRoot"]),
+                    "LDP_CLUSTER": str(registration["cluster"]),
+                    "POD_NAMESPACE": str(registration["namespace"]),
+                    "NODE_NAME": str(registration["nodeName"]),
+                }
+            )
+            if registration.get("instanceKey"):
+                values["LDP_PLUGIN_INSTANCE_KEY"] = str(registration["instanceKey"])
+        token = self.payload["service"].get("token")
+        if token:
+            values["LDP_PLUGIN_TOKEN"] = str(token)
+        else:
+            values.pop("LDP_PLUGIN_TOKEN", None)
+        return values

algorithm_plugin_sdk/errors.py

@@ -0,0 +1,27 @@
+class AlgorithmPluginError(RuntimeError):
+    """Base error raised by the SDK."""
+
+
+class InvalidRequest(AlgorithmPluginError, ValueError):
+    """The execution request is structurally invalid."""
+
+
+class InvalidAlgorithmResult(AlgorithmPluginError, ValueError):
+    """The algorithm returned a result that violates the SDK contract."""
+
+
+class ExecutionCancelled(AlgorithmPluginError):
+    """The execution was cancelled by its caller or deadline."""
+
+
+class AlgorithmLoadError(AlgorithmPluginError, ImportError):
+    """An algorithm class could not be imported or instantiated."""
+
+class ExecutionNotFound(KeyError):
+    pass
+
+class ExecutionNotFinished(RuntimeError):
+    pass
+
+class IdempotencyConflict(RuntimeError):
+    pass

algorithm_plugin_sdk/examples/__init__.py

@@ -0,0 +1 @@
+"""Runnable example algorithms packaged with the SDK."""