ls-algorithm-plugin-sdk 0.2.5__py3-none-any.whl

algorithm_plugin_sdk/examples/example_algorithm.py

@@ -0,0 +1,119 @@
+from __future__ import annotations
+
+import json
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from pathlib import Path
+
+from algorithm_plugin_sdk import (
+    Algorithm,
+    AlgorithmInput,
+    AlgorithmMetadata,
+    AlgorithmRequest,
+    AlgorithmResult,
+    DatasetResult,
+    ExecutionContext
+)
+
+
+class ExampleAlgorithm(Algorithm):
+    """A packaged example that demonstrates algorithm-owned scheduling."""
+
+    @classmethod
+    def metadata(cls) -> AlgorithmMetadata:
+        return AlgorithmMetadata(
+            name="example-dataset-algorithm",
+            version="0.1.1",
+            description="Demonstrates algorithm-owned scheduling and merging",
+            supports_merge=True,
+        )
+
+    def execute(
+        self,
+        request: AlgorithmRequest,
+        context: ExecutionContext,
+    ) -> AlgorithmResult:
+        results: list[DatasetResult] = []
+        worker_count = max(1, len(request.gpu_ids))
+        with ThreadPoolExecutor(max_workers=worker_count) as executor:
+            futures = {
+                executor.submit(
+                    self._process_dataset,
+                    algorithm_input,
+                    request,
+                    context,
+                    request.gpu_ids[index % len(request.gpu_ids)]
+                    if request.gpu_ids
+                    else None,
+                ): algorithm_input.dataset
+                for index, algorithm_input in enumerate(request.inputs)
+            }
+            for future in as_completed(futures):
+                results.append(future.result())
+        return AlgorithmResult(datasets=results)
+
+    def _process_dataset(
+        self,
+        algorithm_input: AlgorithmInput,
+        request: AlgorithmRequest,
+        context: ExecutionContext,
+        gpu_id: int | None,
+    ) -> DatasetResult:
+        dataset = algorithm_input.dataset
+        context.raise_if_cancelled()
+        context.report_progress(
+            dataset=dataset,
+            stage="inference",
+            completed=0,
+            total=1,
+            metrics={"gpuId": gpu_id},
+        )
+        output_dir = Path(algorithm_input.output)
+        output_dir.mkdir(parents=True, exist_ok=True)
+        artifact_path = output_dir / "prediction.json"
+        artifact_path.write_text(
+            json.dumps(
+                {
+                    "dataset": dataset,
+                    "inputs": algorithm_input.to_dict(),
+                    "gpuId": gpu_id,
+                    "parameters": request.parameters,
+                },
+                indent=2,
+            ),
+            encoding="utf-8",
+        )
+
+        merge_status = "not_requested"
+        merge_message = None
+        if request.merge:
+            try:
+                self._merge(dataset, artifact_path)
+                merge_status = "succeeded"
+            except Exception as exc:
+                merge_status = "skipped"
+                merge_message = f"{type(exc).__name__}: {exc}"
+
+        context.report_progress(
+            dataset=dataset,
+            stage="completed",
+            completed=1,
+            total=1,
+            metrics={"gpuId": gpu_id, "mergeStatus": merge_status},
+        )
+        return DatasetResult(
+            dataset=dataset,
+            status="succeeded",
+            artifacts={"prediction": str(artifact_path)},
+            merge_status=merge_status,
+            merge_message=merge_message,
+        )
+
+    @staticmethod
+    def _merge(dataset: str, artifact_path: Path) -> None:
+        dataset_path = Path(dataset)
+        if not dataset_path.is_dir():
+            raise FileNotFoundError(dataset)
+        (dataset_path / "example_prediction.json").write_text(
+            artifact_path.read_text(encoding="utf-8"),
+            encoding="utf-8",
+        )

algorithm_plugin_sdk/examples/simulated_algorithm.py

@@ -0,0 +1,92 @@
+from __future__ import annotations
+
+import random
+import time
+from concurrent.futures import ThreadPoolExecutor, as_completed
+
+from algorithm_plugin_sdk import (
+    Algorithm,
+    AlgorithmMetadata,
+    AlgorithmRequest,
+    AlgorithmResult,
+    DatasetResult,
+    ExecutionContext
+)
+
+
+class SimulatedAlgorithm(Algorithm):
+    """A filesystem-free example for observing multi-dataset progress."""
+
+    @classmethod
+    def metadata(cls) -> AlgorithmMetadata:
+        return AlgorithmMetadata(
+            name="simulated-dataset-algorithm",
+            version="0.1.3",
+            description="Simulates concurrent dataset work without filesystem access",
+        )
+
+    def execute(
+        self,
+        request: AlgorithmRequest,
+        context: ExecutionContext,
+    ) -> AlgorithmResult:
+        datasets = [item.dataset for item in request.inputs]
+        worker_count = max(1, len(datasets))
+        with ThreadPoolExecutor(max_workers=worker_count) as executor:
+            futures = [
+                executor.submit(self._process_dataset, dataset, context)
+                for dataset in datasets
+            ]
+            results = [future.result() for future in as_completed(futures)]
+        return AlgorithmResult(datasets=results)
+
+    @staticmethod
+    def _process_dataset(
+        dataset: str,
+        context: ExecutionContext,
+    ) -> DatasetResult:
+        wait_seconds = random.uniform(2, 5)
+        run_seconds = random.uniform(3, 10)
+        context.report_progress(
+            dataset=dataset,
+            stage="waiting",
+            completed=0,
+            total=run_seconds,
+            message=f"waiting {wait_seconds:.1f}s before execution",
+            metrics={"waitSeconds": round(wait_seconds, 2), "runSeconds": round(run_seconds, 2)},
+        )
+        SimulatedAlgorithm._wait(wait_seconds, context)
+
+        started = time.monotonic()
+        while True:
+            context.raise_if_cancelled()
+            elapsed = min(time.monotonic() - started, run_seconds)
+            context.report_progress(
+                dataset=dataset,
+                stage="running",
+                completed=elapsed,
+                total=run_seconds,
+                message=f"simulated work for {run_seconds:.1f}s",
+            )
+            if elapsed >= run_seconds:
+                break
+            time.sleep(0.1)
+
+        return DatasetResult(
+            dataset=dataset,
+            status="succeeded",
+            metrics={
+                "waitSeconds": round(wait_seconds, 2),
+                "runSeconds": round(run_seconds, 2),
+            },
+        )
+
+    @staticmethod
+    def _wait(seconds: float, context: ExecutionContext) -> None:
+        deadline = time.monotonic() + seconds
+        while True:
+            context.raise_if_cancelled()
+            remaining = deadline - time.monotonic()
+            if remaining <= 0:
+                return
+            time.sleep(min(remaining, 0.1))

algorithm_plugin_sdk/loader.py

@@ -0,0 +1,71 @@
+from __future__ import annotations
+
+import importlib
+from importlib import metadata
+
+from .algorithm import Algorithm
+from .errors import AlgorithmLoadError
+
+ALGORITHM_ENTRY_POINT_GROUP = "algorithm_plugin_sdk.algorithms"
+
+
+def load_algorithm(reference: str) -> Algorithm:
+    """Load an algorithm by entry-point name or ``package.module:Class``."""
+    if ":" not in reference:
+        return _load_registered_algorithm(reference)
+    module_name, separator, class_name = reference.partition(":")
+    if not separator or not module_name or not class_name:
+        raise AlgorithmLoadError(
+            "algorithm reference must use package.module:AlgorithmClass"
+        )
+    try:
+        module = importlib.import_module(module_name)
+    except Exception as exc:
+        raise AlgorithmLoadError(f"cannot import algorithm module {module_name}: {exc}") from exc
+    try:
+        algorithm_class = getattr(module, class_name)
+    except AttributeError as exc:
+        raise AlgorithmLoadError(
+            f"algorithm class does not exist: {reference}"
+        ) from exc
+    return _instantiate_algorithm(algorithm_class, reference)
+
+
+def _load_registered_algorithm(name: str) -> Algorithm:
+    if not name:
+        raise AlgorithmLoadError("algorithm name cannot be empty")
+    candidates = list(
+        metadata.entry_points(group=ALGORITHM_ENTRY_POINT_GROUP, name=name)
+    )
+    if not candidates:
+        available = sorted(
+            entry.name
+            for entry in metadata.entry_points(group=ALGORITHM_ENTRY_POINT_GROUP)
+        )
+        suffix = f"; available: {', '.join(available)}" if available else ""
+        raise AlgorithmLoadError(f"registered algorithm does not exist: {name}{suffix}")
+    if len(candidates) != 1:
+        raise AlgorithmLoadError(f"registered algorithm name is ambiguous: {name}")
+    try:
+        algorithm_class = candidates[0].load()
+    except Exception as exc:
+        raise AlgorithmLoadError(
+            f"cannot load registered algorithm {name}: {exc}"
+        ) from exc
+    return _instantiate_algorithm(algorithm_class, name)
+
+
+def _instantiate_algorithm(
+    algorithm_class: object,
+    reference: str,
+) -> Algorithm:
+    if not isinstance(algorithm_class, type) or not issubclass(
+        algorithm_class, Algorithm
+    ):
+        raise AlgorithmLoadError(
+            f"algorithm class must inherit Algorithm: {reference}"
+        )
+    try:
+        return algorithm_class()
+    except Exception as exc:
+        raise AlgorithmLoadError(f"cannot instantiate algorithm {reference}: {exc}") from exc

algorithm_plugin_sdk/models.py

@@ -0,0 +1,322 @@
+from __future__ import annotations
+
+import json
+from collections.abc import Iterator
+from dataclasses import dataclass, field
+from typing import Any, Literal, Mapping
+
+from .errors import InvalidAlgorithmResult, InvalidRequest
+
+DatasetStatus = Literal["succeeded", "failed", "skipped"]
+MergeStatus = Literal["not_requested", "succeeded", "failed", "skipped"]
+ResultStatus = Literal["succeeded", "partial", "failed"]
+
+
+def _json_dict(value: Mapping[str, Any], name: str, error_type: type[ValueError]) -> dict[str, Any]:
+    if not isinstance(value, Mapping):
+        raise error_type(f"{name} must be a mapping")
+    result = dict(value)
+    if any(not isinstance(key, str) for key in result):
+        raise error_type(f"{name} keys must be strings")
+    try:
+        json.dumps(result)
+    except (TypeError, ValueError) as exc:
+        raise error_type(f"{name} must be JSON serializable") from exc
+    return result
+
+
+def _validate_workspace(workspace: dict[str, str]):
+    if not isinstance(workspace, Mapping):
+        raise InvalidRequest("workspace must be a mapping or None")
+    unknown = set(workspace) - {
+            "inputRoot",
+            "outputRoot",
+            "scratchRoot",
+        }
+    if unknown:
+        raise InvalidRequest(
+                f"unknown workspace fields: {sorted(unknown)}"
+            )
+    workspace = {
+            name: workspace.get(name)
+            for name in ("inputRoot", "outputRoot", "scratchRoot")
+        }
+    for name, value in workspace.items():
+        if not isinstance(value, str) or not value.strip():
+            raise InvalidRequest(
+                    f"workspace.{name} must be a non-empty string"
+                )
+    return workspace
+
+
+@dataclass(frozen=True, slots=True)
+class AlgorithmMetadata:
+    name: str
+    version: str
+    description: str = ""
+    supports_merge: bool = False
+    parameters: dict[str, Any] = field(default_factory=dict)
+    input_paths: dict[str, Any] = field(
+        default_factory=lambda: {
+            "dataset": {
+                "type": "directory",
+                "required": True,
+                "description": "Input dataset directory",
+            }
+        }
+    )
+
+    def __post_init__(self) -> None:
+        if not isinstance(self.name, str) or not self.name.strip():
+            raise ValueError("metadata.name must be a non-empty string")
+        if not isinstance(self.version, str) or not self.version.strip():
+            raise ValueError("metadata.version must be a non-empty string")
+        object.__setattr__(
+            self,
+            "parameters",
+            _json_dict(self.parameters, "metadata.parameters", ValueError),
+        )
+        object.__setattr__(
+            self,
+            "input_paths",
+            _json_dict(self.input_paths, "metadata.input_paths", ValueError),
+        )
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "name": self.name,
+            "version": self.version,
+            "description": self.description,
+            "supportsMerge": self.supports_merge,
+            "parameters": self.parameters,
+            "inputPaths": self.input_paths,
+        }
+
+
+@dataclass(frozen=True, slots=True)
+class AlgorithmInput(Mapping[str, str]):
+    paths: dict[str, str]
+    output: str
+
+    def __post_init__(self) -> None:
+        if not isinstance(self.paths, Mapping):
+            raise InvalidRequest("input must be a mapping")
+        paths = dict(self.paths)
+        if any(not isinstance(name, str) or not name.strip() for name in paths):
+            raise InvalidRequest("input path names must be non-empty strings")
+        if any(not isinstance(path, str) or not path.strip() for path in paths.values()):
+            raise InvalidRequest("input paths must be non-empty strings")
+        if "dataset" not in paths:
+            raise InvalidRequest("input must contain a dataset path")
+        if not isinstance(self.output, str) or not self.output.strip():
+            raise InvalidRequest("input output must be a non-empty string")
+        object.__setattr__(self, "paths", paths)
+
+    @property
+    def dataset(self) -> str:
+        return self.paths["dataset"]
+
+    def __getitem__(self, name: str) -> str:
+        return self.paths[name]
+
+    def __iter__(self) -> Iterator[str]:
+        return iter(self.paths)
+
+    def __len__(self) -> int:
+        return len(self.paths)
+
+    def to_dict(self) -> dict[str, str]:
+        return dict(self.paths)
+
+
+@dataclass(frozen=True, slots=True)
+class AlgorithmRequest:
+    inputs: list[AlgorithmInput]
+    merge: bool = False
+    gpu_ids: list[int] = field(default_factory=list)
+    parameters: dict[str, Any] = field(default_factory=dict)
+    workspace: dict[str, str] | None = None
+
+    def __post_init__(self) -> None:
+        inputs = list(self.inputs)
+        gpu_ids = list(self.gpu_ids)
+        parameters = _json_dict(self.parameters, "parameters", InvalidRequest)
+        if not inputs:
+            raise InvalidRequest("inputs cannot be empty")
+        if any(not isinstance(item, AlgorithmInput) for item in inputs):
+            raise InvalidRequest("inputs must contain AlgorithmInput objects")
+        datasets = [item.dataset for item in inputs]
+        outputs = [item.output for item in inputs]
+        if len(datasets) != len(set(datasets)):
+            raise InvalidRequest("inputs cannot contain duplicate dataset paths")
+        if len(outputs) != len(set(outputs)):
+            raise InvalidRequest("inputs cannot contain duplicate output paths")
+        if not isinstance(self.merge, bool):
+            raise InvalidRequest("merge must be a boolean")
+        if any(isinstance(gpu_id, bool) or not isinstance(gpu_id, int) for gpu_id in gpu_ids):
+            raise InvalidRequest("gpu_ids must contain integers")
+        if any(gpu_id < 0 for gpu_id in gpu_ids):
+            raise InvalidRequest("gpu_ids cannot contain negative values")
+        if len(gpu_ids) != len(set(gpu_ids)):
+            raise InvalidRequest("gpu_ids cannot contain duplicate values")
+        workspace = None
+        if self.workspace is not None:
+            workspace = _validate_workspace(self.workspace)
+        object.__setattr__(self, "inputs", inputs)
+        object.__setattr__(self, "gpu_ids", gpu_ids)
+        object.__setattr__(self, "parameters", parameters)
+        object.__setattr__(self, "workspace", workspace)
+
+    @classmethod
+    def from_dict(cls, value: Mapping[str, Any]) -> "AlgorithmRequest":
+        if not isinstance(value, Mapping):
+            raise InvalidRequest("request body must be a mapping")
+        known = {"input", "workspace", "merge", "parameters"}
+        unknown = set(value) - known
+        if unknown:
+            raise InvalidRequest(f"unknown request fields: {sorted(unknown)}")
+        raw_inputs = value.get("input", [])
+        if not isinstance(raw_inputs, list) or not raw_inputs:
+            raise InvalidRequest("input must be a non-empty list")
+        parsed_inputs = []
+        for index, item in enumerate(raw_inputs):
+            if not isinstance(item, Mapping):
+                raise InvalidRequest(f"input[{index}] must be a mapping")
+            item_unknown = set(item) - {"input_dataset", "output"}
+            if item_unknown:
+                raise InvalidRequest(
+                    f"unknown input[{index}] fields: {sorted(item_unknown)}"
+                )
+            parsed_inputs.append(
+                AlgorithmInput(
+                    {"dataset": item.get("input_dataset")},
+                    output=item.get("output"),
+                )
+            )
+        return cls(
+            inputs=parsed_inputs,
+            merge=value.get("merge", False),
+            parameters=value.get("parameters", {}),
+            workspace=(
+                dict(value["workspace"])
+                if isinstance(value.get("workspace"), Mapping)
+                else value.get("workspace")
+            ),
+        )
+
+    def to_dict(self) -> dict[str, Any]:
+        payload = {
+            "input": [
+                {"input_dataset": item.dataset, "output": item.output}
+                for item in self.inputs
+            ],
+            "merge": self.merge,
+            "parameters": dict(self.parameters),
+        }
+        if self.workspace is not None:
+            payload["workspace"] = dict(self.workspace)
+        return payload
+
+
+@dataclass(slots=True)
+class DatasetResult:
+    dataset: str
+    status: DatasetStatus
+    artifacts: dict[str, str] = field(default_factory=dict)
+    metrics: dict[str, Any] = field(default_factory=dict)
+    error: str | None = None
+    merge_status: MergeStatus = "not_requested"
+    merge_message: str | None = None
+
+    def validate(self) -> None:
+        if not isinstance(self.dataset, str) or not self.dataset.strip():
+            raise InvalidAlgorithmResult("result.dataset must be a non-empty string")
+        if self.status not in {"succeeded", "failed", "skipped"}:
+            raise InvalidAlgorithmResult(f"invalid dataset status: {self.status}")
+        if self.merge_status not in {"not_requested", "succeeded", "failed", "skipped"}:
+            raise InvalidAlgorithmResult(f"invalid merge status: {self.merge_status}")
+        if not isinstance(self.artifacts, dict):
+            raise InvalidAlgorithmResult("result.artifacts must be a dict")
+        for name, path in self.artifacts.items():
+            if not isinstance(name, str) or not name.strip():
+                raise InvalidAlgorithmResult("artifact names must be non-empty strings")
+            if not isinstance(path, str) or not path.strip():
+                raise InvalidAlgorithmResult("artifact paths must be non-empty strings")
+        self.metrics = _json_dict(
+            self.metrics,
+            "result.metrics",
+            InvalidAlgorithmResult,
+        )
+        if self.error is not None and not isinstance(self.error, str):
+            raise InvalidAlgorithmResult("result.error must be a string or None")
+        if self.merge_message is not None and not isinstance(self.merge_message, str):
+            raise InvalidAlgorithmResult("result.merge_message must be a string or None")
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "dataset": self.dataset,
+            "status": self.status,
+            "artifacts": dict(self.artifacts),
+            "metrics": dict(self.metrics),
+            "error": self.error,
+            "mergeStatus": self.merge_status,
+            "mergeMessage": self.merge_message,
+        }
+
+
+@dataclass(slots=True)
+class AlgorithmResult:
+    datasets: list[DatasetResult]
+    metrics: dict[str, Any] = field(default_factory=dict)
+    warnings: list[str] = field(default_factory=list)
+
+    @property
+    def status(self) -> ResultStatus:
+        successful = sum(
+            result.status == "succeeded" and result.merge_status != "failed"
+            for result in self.datasets
+        )
+        if successful == len(self.datasets):
+            return "succeeded"
+        if successful == 0:
+            return "failed"
+        return "partial"
+
+    @classmethod
+    def from_datasets(cls, datasets: list[DatasetResult]) -> "AlgorithmResult":
+        return cls(datasets=list(datasets))
+
+    def validate(self) -> None:
+        if not isinstance(self.datasets, list):
+            raise InvalidAlgorithmResult("result.datasets must be a list")
+        for result in self.datasets:
+            if not isinstance(result, DatasetResult):
+                raise InvalidAlgorithmResult(
+                    "result.datasets must contain DatasetResult objects"
+                )
+            result.validate()
+        self.metrics = _json_dict(
+            self.metrics,
+            "result.metrics",
+            InvalidAlgorithmResult,
+        )
+        if not isinstance(self.warnings, list) or any(
+            not isinstance(warning, str) for warning in self.warnings
+        ):
+            raise InvalidAlgorithmResult("result.warnings must be a list of strings")
+
+    def to_dict(self) -> dict[str, Any]:
+        successful = sum(
+            result.status == "succeeded" and result.merge_status != "failed"
+            for result in self.datasets
+        )
+        return {
+            "status": self.status,
+            "datasets": [result.to_dict() for result in self.datasets],
+            "metrics": {
+                "totalDatasets": len(self.datasets),
+                "successfulDatasets": successful,
+                **self.metrics,
+            },
+            "warnings": list(self.warnings),
+        }