ocean-runner 0.2.18__tar.gz → 0.2.21__tar.gz

{ocean_runner-0.2.18 → ocean_runner-0.2.21}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ocean-runner
-Version: 0.2.18
+Version: 0.2.21
 Summary: A fluent API for OceanProtocol algorithms
 Project-URL: Homepage, https://github.com/AgrospAI/ocean-runner
 Project-URL: Issues, https://github.com/AgrospAI/ocean-runner/issues
@@ -17,8 +17,12 @@ Classifier: License :: OSI Approved :: MIT License
 Classifier: Operating System :: OS Independent
 Classifier: Programming Language :: Python :: 3
 Requires-Python: >=3.10
+Requires-Dist: aiofiles>=25.1.0
 Requires-Dist: oceanprotocol-job-details>=0.2.8
+Requires-Dist: pydantic-settings>=2.12.0
+Requires-Dist: pydantic>=2.12.5
 Requires-Dist: pytest>=8.4.2
+Requires-Dist: types-aiofiles>=25.1.0.20251011
 Description-Content-Type: text/markdown
 
 # ocean-runner

{ocean_runner-0.2.18 → ocean_runner-0.2.21}/ocean_runner/__init__.py

@@ -1,4 +1,4 @@
 from ocean_runner.config import Config, Environment
 from ocean_runner.runner import Algorithm
 
-__all__ = [Config, Algorithm, Environment]
+__all__ = [Config, Algorithm, Environment]  # type: ignore

ocean_runner-0.2.21/ocean_runner/config.py

@@ -0,0 +1,71 @@
+from enum import StrEnum, auto
+from logging import Logger
+from pathlib import Path
+from typing import Generic, Sequence, TypeVar
+
+from pydantic import BaseModel, ConfigDict, Field
+from pydantic_settings import BaseSettings
+
+InputT = TypeVar("InputT")
+
+DEFAULT = "DEFAULT"
+
+
+class Keys(StrEnum):
+    SECRET = auto()
+    BASE_DIR = auto()
+    TRANSFORMATION_DID = auto()
+    DIDS = auto()
+
+
+class Environment(BaseSettings):
+    """Environment configuration loaded from environment variables"""
+
+    base_dir: str | Path | None = Field(
+        default_factory=lambda: Path("/data"),
+        validation_alias=Keys.BASE_DIR.value,
+        description="Base data directory, defaults to '/data'",
+    )
+
+    dids: str | list[Path] | None = Field(
+        default=None,
+        validation_alias=Keys.DIDS.value,
+        description='Datasets DID\'s, format: ["XXXX"]',
+    )
+
+    transformation_did: str = Field(
+        default=DEFAULT,
+        validation_alias=Keys.TRANSFORMATION_DID.value,
+        description="Transformation (algorithm) DID",
+    )
+
+    secret: str = Field(
+        default=DEFAULT,
+        validation_alias=Keys.SECRET.value,
+        description="Super secret secret",
+    )
+
+
+class Config(BaseModel, Generic[InputT]):
+    """Algorithm overall configuration"""
+
+    model_config = ConfigDict(arbitrary_types_allowed=True)
+
+    custom_input: InputT | None = Field(
+        default=None,
+        description="Algorithm's custom input types, must be a dataclass_json",
+    )
+
+    logger: Logger | None = Field(
+        default=None,
+        description="Logger to use in the algorithm",
+    )
+
+    source_paths: Sequence[Path] = Field(
+        default_factory=lambda: [Path("/algorithm/src")],
+        description="Paths that should be included so the code executes correctly",
+    )
+
+    environment: Environment = Field(
+        default_factory=Environment, description="Environment configuration"
+    )

ocean_runner-0.2.21/ocean_runner/runner.py

@@ -0,0 +1,194 @@
+from __future__ import annotations
+
+import asyncio
+import inspect
+from dataclasses import InitVar, asdict, dataclass, field
+from logging import Logger
+from pathlib import Path
+from typing import Awaitable, Callable, Generic, TypeAlias, TypeVar
+
+from oceanprotocol_job_details import JobDetails  # type: ignore
+
+from ocean_runner.config import Config
+
+InputT = TypeVar("InputT")
+ResultT = TypeVar("ResultT")
+T = TypeVar("T")
+
+
+Algo: TypeAlias = "Algorithm[InputT, ResultT]"
+ValidateFuncT: TypeAlias = Callable[[Algo], None | Awaitable[None] | None]
+RunFuncT: TypeAlias = Callable[[Algo], ResultT | Awaitable[ResultT]]
+SaveFuncT: TypeAlias = Callable[[Algo, ResultT, Path], Awaitable[None] | None]
+ErrorFuncT: TypeAlias = Callable[[Algo, Exception], Awaitable[None] | None]
+
+
+def default_error_callback(algorithm: Algorithm, error: Exception) -> None:
+    algorithm.logger.exception("Error during algorithm execution")
+    raise error
+
+
+def default_validation(algorithm: Algorithm) -> None:
+    algorithm.logger.info("Validating input using default validation")
+    assert algorithm.job_details.ddos, "DDOs missing"
+    assert algorithm.job_details.files, "Files missing"
+
+
+async def default_save(algorithm: Algorithm, result: ResultT, base: Path) -> None:
+    import aiofiles
+
+    algorithm.logger.info("Saving results using default save")
+    async with aiofiles.open(base / "result.txt", "w+") as f:
+        await f.write(str(result))
+
+
+async def execute(
+    function: Callable[..., T | Awaitable[T]],
+    *args,
+    **kwargs,
+) -> T:
+    result = function(*args, **kwargs)
+
+    if inspect.isawaitable(result):
+        return await result
+
+    return result
+
+
+@dataclass(slots=True)
+class Functions(Generic[InputT, ResultT]):
+    validate: ValidateFuncT = field(default=default_validation, init=False)
+    run: RunFuncT | None = field(default=None, init=False)
+    save: SaveFuncT = field(default=default_save, init=False)
+    error: ErrorFuncT = field(default=default_error_callback, init=False)
+
+
+@dataclass
+class Algorithm(Generic[InputT, ResultT]):
+    """
+    A configurable algorithm runner that behaves like a FastAPI app:
+      - You register `validate`, `run`, and `save_results` via decorators.
+      - You execute the full pipeline by calling `app()`.
+    """
+
+    config: InitVar[Config[InputT] | None] = field(default=None)
+
+    logger: Logger = field(init=False, repr=False)
+
+    _job_details: JobDetails[InputT] = field(init=False)
+    _result: ResultT | None = field(default=None, init=False)
+    _functions: Functions[InputT, ResultT] = field(
+        default_factory=Functions, init=False, repr=False
+    )
+
+    def __post_init__(self, config: Config[InputT] | None) -> None:
+        configuration = config or Config()
+
+        # Configure logger
+        if configuration.logger:
+            self.logger = configuration.logger
+        else:
+            import logging
+
+            logging.basicConfig(
+                level=logging.DEBUG,
+                format="%(asctime)s | %(levelname)-8s | %(name)s | %(message)s",
+                datefmt="%Y-%m-%d %H:%M:%S",
+            )
+            self.logger = logging.getLogger(__name__)
+
+        # Normalize base_dir
+        if isinstance(configuration.environment.base_dir, str):
+            configuration.environment.base_dir = Path(
+                configuration.environment.base_dir
+            )
+
+        # Extend sys.path for custom imports
+        if configuration.source_paths:
+            import sys
+
+            sys.path.extend(
+                [str(path.absolute()) for path in configuration.source_paths]
+            )
+            self.logger.debug(
+                f"Added [{len(configuration.source_paths)}] entries to PATH"
+            )
+
+        self.configuration = configuration
+
+    class Error(RuntimeError): ...
+
+    @property
+    def job_details(self) -> JobDetails:
+        if not self._job_details:
+            raise Algorithm.Error("JobDetails not initialized or missing")
+        return self._job_details
+
+    @property
+    def result(self) -> ResultT:
+        if self._result is None:
+            raise Algorithm.Error("Result missing, run the algorithm first")
+        return self._result
+
+    # ---------------------------
+    # Decorators (FastAPI-style)
+    # ---------------------------
+
+    def validate(self, fn: ValidateFuncT) -> ValidateFuncT:
+        self._functions.validate = fn
+        return fn
+
+    def run(self, fn: RunFuncT) -> RunFuncT:
+        self._functions.run = fn
+        return fn
+
+    def save_results(self, fn: SaveFuncT) -> SaveFuncT:
+        self._functions.save = fn
+        return fn
+
+    def on_error(self, fn: ErrorFuncT) -> ErrorFuncT:
+        self._functions.error = fn
+        return fn
+
+    # ---------------------------
+    # Execution Pipeline
+    # ---------------------------
+
+    async def execute(self) -> ResultT | None:
+        # Load job details
+        self._job_details = JobDetails.load(
+            _type=self.configuration.custom_input,
+            base_dir=self.configuration.environment.base_dir,
+            dids=self.configuration.environment.dids,
+            transformation_did=self.configuration.environment.transformation_did,
+            secret=self.configuration.environment.secret,
+        )
+
+        self.logger.info("Loaded JobDetails")
+        self.logger.debug(asdict(self.job_details))
+
+        try:
+            await execute(self._functions.validate, self)
+
+            if self._functions.run:
+                self.logger.info("Running algorithm...")
+                self._result = await execute(self._functions.run, self)
+            else:
+                self.logger.error("No run() function defined. Skipping execution.")
+                self._result = None
+
+            await execute(
+                self._functions.save,
+                algorithm=self,
+                result=self._result,
+                base=self.job_details.paths.outputs,
+            )
+
+        except Exception as e:
+            await execute(self._functions.error, self, e)
+
+        return self._result
+
+    def __call__(self) -> ResultT | None:
+        """Executes the algorithm pipeline: validate → run → save_results."""
+        return asyncio.run(self.execute())

{ocean_runner-0.2.18 → ocean_runner-0.2.21}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "ocean-runner"
-version = "0.2.18"
+version = "0.2.21"
 description = "A fluent API for OceanProtocol algorithms"
 authors = [
     { name = "AgrospAI", email = "agrospai@udl.cat" },
@@ -14,7 +14,14 @@ classifiers = [
     "Operating System :: OS Independent",
     "License :: OSI Approved :: MIT License",
 ]
-dependencies = ["oceanprotocol-job-details>=0.2.8", "pytest>=8.4.2"]
+dependencies = [
+    "aiofiles>=25.1.0",
+    "oceanprotocol-job-details>=0.2.8",
+    "pydantic>=2.12.5",
+    "pydantic-settings>=2.12.0",
+    "pytest>=8.4.2",
+    "types-aiofiles>=25.1.0.20251011",
+]
 
 [project.urls]
 Homepage = "https://github.com/AgrospAI/ocean-runner"
@@ -29,8 +36,16 @@ pythonpath = "ocean_runner"
 requires = ["hatchling"]
 build-backend = "hatchling.build"
 
+[dependency-groups]
+dev = [
+    "mypy>=1.19.1",
+]
+
 [tool.hatch.build.targets.sdist]
 include = ["ocean_runner"]
 
 [tool.hatch.build.targets.wheel]
 include = ["ocean_runner"]
+
+[tool.mypy]
+plugins = ['pydantic.mypy']

ocean_runner-0.2.18/ocean_runner/config.py

@@ -1,55 +0,0 @@
-import os
-from dataclasses import asdict, dataclass, field
-from logging import Logger
-from pathlib import Path
-from typing import Iterable, TypeVar
-
-T = TypeVar("T")
-
-DEFAULT = "DEFAULT"
-
-
-@dataclass
-class Environment:
-    """Environment variables mock"""
-
-    base_dir: str | None = field(
-        default_factory=lambda: os.environ.get("BASE_DIR", None),
-    )
-    """Base data directory, defaults to '/data'"""
-
-    dids: str = field(
-        default_factory=lambda: os.environ.get("DIDS", None),
-    )
-    """Datasets DID's, format: '["XXXX"]'"""
-
-    transformation_did: str = field(
-        default_factory=lambda: os.environ.get("TRANSFORMATION_DID", DEFAULT),
-    )
-    """Transformation (algorithm) DID"""
-
-    secret: str = field(
-        default_factory=lambda: os.environ.get("SECRET", DEFAULT),
-    )
-    """Super secret secret"""
-
-    dict = asdict
-
-
-@dataclass
-class Config:
-    """Algorithm overall configuration"""
-
-    custom_input: T | None = None
-    """Algorithm's custom input types, must be a dataclass_json"""
-
-    logger: Logger | None = None
-    """Logger to use in the algorithm"""
-
-    source_paths: Iterable[Path] = field(
-        default_factory=lambda: [Path("/algorithm/src")]
-    )
-    """Paths that should be included so the code executes correctly"""
-
-    environment: Environment = field(default_factory=lambda: Environment())
-    """Mock of environment data"""

ocean_runner-0.2.18/ocean_runner/runner.py

@@ -1,190 +0,0 @@
-from __future__ import annotations
-
-from dataclasses import InitVar, asdict, dataclass, field
-from logging import Logger
-from pathlib import Path
-from typing import Callable, Generic, TypeVar
-
-from oceanprotocol_job_details import JobDetails
-
-from ocean_runner.config import Config
-
-JobDetailsT = TypeVar("JobDetailsT")
-ResultT = TypeVar("ResultT")
-
-
-def default_error_callback(algorithm: Algorithm, e: Exception) -> None:
-    algorithm.logger.exception("Error during algorithm execution")
-    raise e
-
-
-def default_validation(algorithm: Algorithm) -> None:
-    algorithm.logger.info("Validating input using default validation")
-    assert algorithm.job_details.ddos, "DDOs missing"
-    assert algorithm.job_details.files, "Files missing"
-
-
-def default_save(*, result: ResultT, base: Path, algorithm: Algorithm) -> None:
-    algorithm.logger.info("Saving results using default save")
-    with open(base / "result.txt", "w+") as f:
-        f.write(str(result))
-
-
-@dataclass
-class Algorithm(Generic[JobDetailsT, ResultT]):
-    """
-    A configurable algorithm runner that behaves like a FastAPI app:
-      - You register `validate`, `run`, and `save_results` via decorators.
-      - You execute the full pipeline by calling `app()`.
-    """
-
-    config: InitVar[Config | None] = None
-    logger: Logger = field(init=False)
-    _job_details: JobDetails[JobDetailsT] = field(init=False)
-    _result: ResultT | None = field(default=None, init=False)
-
-    # Decorator-registered callbacks
-    _validate_fn: Callable[[Algorithm], None] | None = field(
-        default=None,
-        init=False,
-        repr=False,
-    )
-
-    _run_fn: Callable[[Algorithm], ResultT] | None = field(
-        default=None,
-        init=False,
-        repr=False,
-    )
-
-    _save_fn: Callable[[ResultT, Path, Algorithm], None] | None = field(
-        default=None,
-        init=False,
-        repr=False,
-    )
-
-    _error_callback: Callable[[Algorithm, Exception], None] | None = field(
-        default=None,
-        init=False,
-        repr=False,
-    )
-
-    def __post_init__(self, config: Config | None) -> None:
-        config: Config = config or Config()
-
-        # Configure logger
-        if config.logger:
-            self.logger = config.logger
-        else:
-            import logging
-
-            logging.basicConfig(
-                level=logging.DEBUG,
-                format="%(asctime)s | %(levelname)-8s | %(name)s | %(message)s",
-                datefmt="%Y-%m-%d %H:%M:%S",
-            )
-            self.logger = logging.getLogger("ocean_runner")
-
-        # Normalize base_dir
-        if isinstance(config.environment.base_dir, str):
-            config.environment.base_dir = Path(config.environment.base_dir)
-
-        # Extend sys.path for custom imports
-        if config.source_paths:
-            import sys
-
-            sys.path.extend([str(path.absolute()) for path in config.source_paths])
-            self.logger.debug(f"Added [{len(config.source_paths)}] entries to PATH")
-
-        self.config = config
-
-    class Error(RuntimeError): ...
-
-    @property
-    def job_details(self) -> JobDetails:
-        if not self._job_details:
-            raise Algorithm.Error("JobDetails not initialized or missing")
-        return self._job_details
-
-    @property
-    def result(self) -> ResultT:
-        if self._result is None:
-            raise Algorithm.Error("Result missing, run the algorithm first")
-        return self._result
-
-    # ---------------------------
-    # Decorators (FastAPI-style)
-    # ---------------------------
-
-    def validate(self, fn: Callable[[], None]) -> Callable[[], None]:
-        self._validate_fn = fn
-        return fn
-
-    def run(self, fn: Callable[[], ResultT]) -> Callable[[], ResultT]:
-        self._run_fn = fn
-        return fn
-
-    def save_results(self, fn: Callable[[ResultT, Path], None]) -> Callable:
-        self._save_fn = fn
-        return fn
-
-    def on_error(self, fn: Callable[[Exception], None]) -> Callable:
-        self._error_callback = fn
-        return fn
-
-    # ---------------------------
-    # Execution Pipeline
-    # ---------------------------
-
-    def __call__(self) -> ResultT | None:
-        """Executes the algorithm pipeline: validate → run → save_results."""
-        # Load job details
-        self._job_details = JobDetails.load(
-            _type=self.config.custom_input,
-            base_dir=self.config.environment.base_dir,
-            dids=self.config.environment.dids,
-            transformation_did=self.config.environment.transformation_did,
-            secret=self.config.environment.secret,
-        )
-
-        self.logger.info("Loaded JobDetails")
-        self.logger.debug(asdict(self.job_details))
-
-        try:
-            # Validation step
-            if self._validate_fn:
-                self.logger.info("Running custom validation...")
-                self._validate_fn()
-            else:
-                self.logger.info("Running default validation...")
-                default_validation(self)
-
-            # Run step
-            if self._run_fn:
-                self.logger.info("Running algorithm...")
-                self._result = self._run_fn()
-            else:
-                self.logger.warning("No run() function defined. Skipping execution.")
-                self._result = None
-
-            # Save step
-            if self._save_fn:
-                self.logger.info("Saving results...")
-                self._save_fn(
-                    self._result,
-                    self.job_details.paths.outputs,
-                )
-            else:
-                self.logger.info("No save_results() defined. Using default.")
-                default_save(
-                    result=self._result,
-                    base=self.job_details.paths.outputs,
-                    algorithm=self,
-                )
-
-        except Exception as e:
-            if self._error_callback:
-                self._error_callback(e)
-            else:
-                default_error_callback(self, e)
-
-        return self._result