oceanprotocol-job-details 0.3.12__tar.gz → 0.3.14__tar.gz

{oceanprotocol_job_details-0.3.12 → oceanprotocol_job_details-0.3.14}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: oceanprotocol-job-details
-Version: 0.3.12
+Version: 0.3.14
 Summary: A Python package to get details from OceanProtocol jobs
 Project-URL: Homepage, https://github.com/AgrospAI/oceanprotocol-job-details
 Project-URL: Issues, https://github.com/AgrospAI/oceanprotocol-job-details/issues
@@ -72,8 +72,9 @@ class InputParameters(BaseModel):
 job_details = load_job_details({"base_dir": "...", "transformation_did": "..."}, InputParameters)
 
 # Usage
-job_details.input_parameters.foo
-job_details.input_parameters.foo.bar
+parameters = await job_details.input_parameters()
+parameters.foo
+parameters.foo.bar
 ```
 
 The values to fill the custom `InputParameters` will be parsed from the `algoCustomData.json` located next to the input data directories.
@@ -96,7 +97,7 @@ _, file_path = next(job_details.inputs())
 
 ```bash
 data        # Root /data directory
-├── ddos    # Contains the loaded dataset's DDO
+├── ddos    # Contains the loaded dataset's DDO (metadata)
 │   ├── 17feb...e42 # DDO file
 │   └── ... # One DDO per loaded dataset
 ├── inputs  # Datasets dir

{oceanprotocol_job_details-0.3.12 → oceanprotocol_job_details-0.3.14}/README.md

@@ -46,8 +46,9 @@ class InputParameters(BaseModel):
 job_details = load_job_details({"base_dir": "...", "transformation_did": "..."}, InputParameters)
 
 # Usage
-job_details.input_parameters.foo
-job_details.input_parameters.foo.bar
+parameters = await job_details.input_parameters()
+parameters.foo
+parameters.foo.bar
 ```
 
 The values to fill the custom `InputParameters` will be parsed from the `algoCustomData.json` located next to the input data directories.
@@ -70,7 +71,7 @@ _, file_path = next(job_details.inputs())
 
 ```bash
 data        # Root /data directory
-├── ddos    # Contains the loaded dataset's DDO
+├── ddos    # Contains the loaded dataset's DDO (metadata)
 │   ├── 17feb...e42 # DDO file
 │   └── ... # One DDO per loaded dataset
 ├── inputs  # Datasets dir

{oceanprotocol_job_details-0.3.12 → oceanprotocol_job_details-0.3.14}/oceanprotocol_job_details/di.py

@@ -8,7 +8,7 @@ from oceanprotocol_job_details.loaders.impl.job_details import JobDetailsLoader
 from oceanprotocol_job_details.domain import Paths
 
 
-InputParametersT = TypeVar("InputParametersT", BaseModel, None)
+InputParametersT = TypeVar("InputParametersT", bound=BaseModel)
 
 
 class Container(containers.DeclarativeContainer, Generic[InputParametersT]):
@@ -26,7 +26,7 @@ class Container(containers.DeclarativeContainer, Generic[InputParametersT]):
 
     files = providers.Factory(lambda loader: loader.load(), loader=file_loader)
     ddo_loader = providers.Factory(DDOLoader, files=files)
-    ddos = providers.Factory(lambda loader: loader.load(), loader=ddo_loader)
+    metadata = providers.Factory(lambda loader: loader.load(), loader=ddo_loader)
 
     job_details_loader: providers.Factory[JobDetailsLoader[InputParametersT]] = (
         providers.Factory(
@@ -34,6 +34,6 @@ class Container(containers.DeclarativeContainer, Generic[InputParametersT]):
             files=files,
             secret=config.secret,
             paths=paths,
-            ddos=ddos,
+            metadata=metadata,
         )
     )

oceanprotocol_job_details-0.3.14/oceanprotocol_job_details/domain/__init__.py

@@ -0,0 +1,10 @@
+from oceanprotocol_job_details.domain.ddo import DDO
+from oceanprotocol_job_details.domain.derived import (
+    DID,
+    DDOMetadata,
+    DIDPaths,
+    Files,
+    Paths,
+)
+
+__all__ = ["DDO", "DID", "DDOMetadata", "Paths", "DIDPaths", "Files"]

oceanprotocol_job_details-0.3.12/oceanprotocol_job_details/domain.py → oceanprotocol_job_details-0.3.14/oceanprotocol_job_details/domain/ddo.py

@@ -1,7 +1,5 @@
 # mypy: disable-error-code=explicit-any
-from dataclasses import InitVar, dataclass, field
-from pathlib import Path
-from typing import Generator, List, Optional, Sequence, TypeAlias
+from typing import Optional
 
 from pydantic import BaseModel, ConfigDict, Field, JsonValue
 
@@ -124,64 +122,3 @@ class DDO(BaseModel):
     purgatory: Purgatory
 
     model_config = ConfigDict(populate_by_name=True)
-
-
-@dataclass(frozen=True)
-class DIDPaths:
-    did: str
-    ddo: Path = field(repr=False)
-
-    files: InitVar[Generator[Path, None, None]]
-
-    _input: List[Path] = field(init=False, repr=False)
-
-    def __post_init__(self, files: Generator[Path, None, None]) -> None:
-        assert self.ddo.exists(), f"DDO {self.ddo} does not exist"
-
-        object.__setattr__(self, "_input", list(files))
-
-    @property
-    def input_files(self) -> List[Path]:
-        return self._input
-
-    def __len__(self) -> int:
-        return len(self._input)
-
-
-Files: TypeAlias = Sequence[DIDPaths]
-
-
-@dataclass(frozen=True)
-class Paths:
-    """Configuration class for the Ocean Protocol Job Details"""
-
-    base_dir: InitVar[Path | None] = None
-
-    _base: Path = field(init=False, repr=False)
-
-    def __post_init__(self, base_dir: Path | None) -> None:
-        object.__setattr__(self, "_base", base_dir if base_dir else Path("/data"))
-
-    @property
-    def data(self) -> Path:
-        return self._base
-
-    @property
-    def inputs(self) -> Path:
-        return self.data / "inputs"
-
-    @property
-    def ddos(self) -> Path:
-        return self.data / "ddos"
-
-    @property
-    def outputs(self) -> Path:
-        return self.data / "outputs"
-
-    @property
-    def logs(self) -> Path:
-        return self.data / "logs"
-
-    @property
-    def algorithm_custom_parameters(self) -> Path:
-        return self.inputs / "algoCustomData.json"

oceanprotocol_job_details-0.3.14/oceanprotocol_job_details/domain/derived.py

@@ -0,0 +1,71 @@
+from dataclasses import InitVar, dataclass, field
+from pathlib import Path
+from typing import Dict, Generator, List, Sequence, TypeAlias
+
+from oceanprotocol_job_details.domain.ddo import DDO
+
+DID: TypeAlias = str
+
+
+@dataclass(frozen=True)
+class DIDPaths:
+    did: DID
+    ddo: Path = field(repr=False)
+
+    files: InitVar[Generator[Path, None, None]]
+
+    _input: List[Path] = field(init=False, repr=False)
+
+    def __post_init__(self, files: Generator[Path, None, None]) -> None:
+        assert self.ddo.exists(), f"DDO {self.ddo} does not exist"
+
+        object.__setattr__(self, "_input", list(files))
+
+    @property
+    def input_files(self) -> List[Path]:
+        return self._input
+
+    def __len__(self) -> int:
+        return len(self._input)
+
+
+Files: TypeAlias = Sequence[DIDPaths]
+
+
+@dataclass(frozen=True)
+class Paths:
+    """Configuration class for the Ocean Protocol Job Details"""
+
+    base_dir: InitVar[Path | None] = None
+
+    _base: Path = field(init=False, repr=False)
+
+    def __post_init__(self, base_dir: Path | None) -> None:
+        object.__setattr__(self, "_base", base_dir if base_dir else Path("/data"))
+
+    @property
+    def data(self) -> Path:
+        return self._base
+
+    @property
+    def inputs(self) -> Path:
+        return self.data / "inputs"
+
+    @property
+    def ddos(self) -> Path:
+        return self.data / "ddos"
+
+    @property
+    def outputs(self) -> Path:
+        return self.data / "outputs"
+
+    @property
+    def logs(self) -> Path:
+        return self.data / "logs"
+
+    @property
+    def algorithm_custom_parameters(self) -> Path:
+        return self.inputs / "algoCustomData.json"
+
+
+DDOMetadata: TypeAlias = Dict[DID, DDO]

oceanprotocol_job_details-0.3.14/oceanprotocol_job_details/executors.py

@@ -0,0 +1,40 @@
+# mypy: disable-error-code=explicit-any
+import asyncio
+import inspect
+from functools import partial
+from typing import Any, Callable, Coroutine, TypeGuard, TypeVar, cast
+
+T = TypeVar("T")
+
+
+def is_coro_function(
+    obj: Any,
+) -> TypeGuard[Callable[..., Coroutine[Any, Any, T]]]:
+    return inspect.iscoroutinefunction(obj)
+
+
+def is_coro(obj: Any) -> TypeGuard[Coroutine[Any, Any, T]]:
+    return inspect.iscoroutine(obj)
+
+
+async def run_in_executor(
+    obj: Callable[..., T]
+    | Callable[..., Coroutine[Any, Any, T]]
+    | Coroutine[Any, Any, T],
+    *args: Any,
+    **kwargs: Any,
+) -> T:
+    if is_coro_function(obj):
+        return await obj(*args, **kwargs)
+
+    if is_coro(obj):
+        return await obj
+
+    if callable(obj):
+        loop = asyncio.get_running_loop()
+
+        # just to comply with mypy
+        func = partial(obj, *args, **kwargs)
+        return await loop.run_in_executor(None, cast(Callable[[], T], func))
+
+    return cast(T, obj)

{oceanprotocol_job_details-0.3.12 → oceanprotocol_job_details-0.3.14}/oceanprotocol_job_details/helpers.py

@@ -6,7 +6,7 @@ from oceanprotocol_job_details.di import Container
 from oceanprotocol_job_details.ocean import JobDetails
 from oceanprotocol_job_details.settings import JobSettings
 
-InputParametersT = TypeVar("InputParametersT", BaseModel, None)
+InputParametersT = TypeVar("InputParametersT", bound=BaseModel)
 
 
 def create_container(config: Dict[str, Any]) -> Container[InputParametersT]:  # type: ignore[explicit-any]

oceanprotocol_job_details-0.3.14/oceanprotocol_job_details/loaders/impl/ddo.py

@@ -0,0 +1,26 @@
+from dataclasses import InitVar, dataclass, field
+from pathlib import Path
+from typing import List, Tuple, final
+
+import orjson
+
+from oceanprotocol_job_details.domain import DDO, DID, DDOMetadata, Files
+
+
+@final
+@dataclass(frozen=True)
+class DDOLoader:
+    files: InitVar[Files]
+    """The files to load the DDOs from"""
+
+    _files: List[Tuple[DID, Path]] = field(init=False)
+
+    def __post_init__(self, files: Files) -> None:
+        assert files is not None and len(files) != 0, "Missing files"
+        object.__setattr__(self, "_files", [(f.did, f.ddo) for f in files])
+
+    def load(self) -> DDOMetadata:
+        return {
+            did: DDO.model_validate(orjson.loads(path.read_bytes()))
+            for did, path in self._files
+        }

{oceanprotocol_job_details-0.3.12 → oceanprotocol_job_details-0.3.14}/oceanprotocol_job_details/loaders/impl/files.py

@@ -1,6 +1,6 @@
 from __future__ import annotations
 
-from dataclasses import InitVar, dataclass, field
+from dataclasses import dataclass, field
 from logging import Logger
 from pathlib import Path
 from typing import Literal, final

oceanprotocol_job_details-0.3.14/oceanprotocol_job_details/loaders/impl/job_details.py

@@ -0,0 +1,28 @@
+from dataclasses import dataclass
+from typing import Generic, Type, TypeVar, final
+
+from pydantic import BaseModel, Secret
+
+from oceanprotocol_job_details.domain import DDOMetadata, Files, Paths
+from oceanprotocol_job_details.ocean import JobDetails
+
+InputParameterT = TypeVar("InputParameterT", bound=BaseModel)
+
+
+@final
+@dataclass(frozen=True)
+class JobDetailsLoader(Generic[InputParameterT]):
+    input_type: Type[InputParameterT] | None
+    files: Files
+    secret: Secret[str] | None
+    paths: Paths
+    metadata: DDOMetadata
+
+    def load(self) -> JobDetails[InputParameterT]:
+        return JobDetails[InputParameterT](
+            files=self.files,
+            secret=self.secret,
+            metadata=self.metadata,
+            paths=self.paths,
+            input_type=self.input_type,
+        )

{oceanprotocol_job_details-0.3.12 → oceanprotocol_job_details-0.3.14}/oceanprotocol_job_details/ocean.py

@@ -1,29 +1,27 @@
-from __future__ import annotations
-
-import asyncio
-from functools import cached_property
 from pathlib import Path
 from typing import Generator, Generic, Tuple, Type, TypeVar, final
 
 import aiofiles
+import orjson
 from pydantic import BaseModel, ConfigDict, Secret
 
-from oceanprotocol_job_details.domain import DDO, Files, Paths
-from oceanprotocol_job_details.executors import run_in_executor
+from oceanprotocol_job_details.domain import DDOMetadata, Files, Paths
 
-InputParametersT = TypeVar("InputParametersT", BaseModel, None)
+InputParametersT = TypeVar("InputParametersT", bound=BaseModel)
 
 
 @final
 class JobDetails(BaseModel, Generic[InputParametersT]):  # type: ignore[explicit-any]
     files: Files
-    ddos: list[DDO]
+    metadata: DDOMetadata
     paths: Paths
     input_type: Type[InputParametersT] | None
-    secret: Secret[str] | None = None
+    secret: Secret[str] | None
 
     model_config = ConfigDict(arbitrary_types_allowed=True, frozen=True)
 
+    _input_parameters: InputParametersT | None = None
+
     def inputs(self) -> Generator[Tuple[int, Path], None, None]:
         yield from (
             (idx, file)
@@ -31,9 +29,14 @@ class JobDetails(BaseModel, Generic[InputParametersT]):  # type: ignore[explicit
             for file in files.input_files
         )
 
-    @cached_property
-    def input_parameters(self) -> InputParametersT | None:
-        return run_in_executor(self.ainput_parameters())
+    async def input_parameters(self) -> InputParametersT | None:
+        current = self._input_parameters
+
+        if current is None:
+            current = await self.ainput_parameters()
+            object.__setattr__(self, "_input_parameters", current)
+
+        return current
 
     async def ainput_parameters(self) -> InputParametersT | None:
         if self.input_type is None:
@@ -45,4 +48,4 @@ class JobDetails(BaseModel, Generic[InputParametersT]):  # type: ignore[explicit
 
         raw = raw.strip()
         assert raw is not None, f"Empty file {path}"
-        return self.input_type.model_validate_json(raw)  # type: ignore
+        return self.input_type.model_validate(orjson.loads(raw))

{oceanprotocol_job_details-0.3.12 → oceanprotocol_job_details-0.3.14}/oceanprotocol_job_details/settings.py

@@ -4,7 +4,7 @@ from pathlib import Path
 from typing import Self
 
 import orjson
-from pydantic import Field, field_validator, model_validator
+from pydantic import Field, Secret, field_validator, model_validator
 from pydantic_settings import BaseSettings, SettingsConfigDict
 
 
@@ -12,7 +12,7 @@ class JobSettings(BaseSettings):  # type: ignore[explicit-any]
     base_dir: Path = Field(alias="BASE_DIR")
     dids: list[str] = Field(default_factory=list, alias="DIDS")
     transformation_did: str = Field(alias="TRANSFORMATION_DID")
-    secret: str | None = Field(default=None, alias="SECRET")
+    secret: Secret[str] | None = Field(default=None, alias="SECRET")
     logger: Logger = Field(default_factory=lambda: getLogger(__name__))
 
     model_config = SettingsConfigDict(
@@ -35,7 +35,7 @@ class JobSettings(BaseSettings):  # type: ignore[explicit-any]
 
     @model_validator(mode="after")
     def validate_dids(self) -> Self:
-        if not self.dids:
+        if len(self.dids) == 0:
             self.dids.extend(
                 [f.name for f in (self.base_dir / "ddos").glob("*") if f.is_file()]
             )

{oceanprotocol_job_details-0.3.12 → oceanprotocol_job_details-0.3.14}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "oceanprotocol-job-details"
-version = "0.3.12"
+version = "0.3.14"
 description = "A Python package to get details from OceanProtocol jobs"
 authors = [
     { name = "Agrospai", email = "agrospai@udl.cat" },
@@ -27,6 +27,8 @@ Homepage = "https://github.com/AgrospAI/oceanprotocol-job-details"
 Issues = "https://github.com/AgrospAI/oceanprotocol-job-details/issues"
 
 [tool.pytest.ini_options]
+asyncio_mode = "auto"
+asyncio_default_fixture_loop_scope = "function"
 log_level = "INFO"
 pythonpath = "oceanprotocol_job_details"
 log_cli = true
@@ -51,4 +53,9 @@ warn_return_any = true
 disallow_any_explicit = true
 
 [dependency-groups]
-dev = ["mypy>=1.15.0", "pytest>=8.3.4", "types-aiofiles>=25.1.0.20251011"]
+dev = [
+    "mypy>=1.15.0",
+    "pytest>=8.3.4",
+    "pytest-asyncio>=1.3.0",
+    "types-aiofiles>=25.1.0.20251011",
+]

oceanprotocol_job_details-0.3.12/oceanprotocol_job_details/executors.py

@@ -1,24 +0,0 @@
-import asyncio
-import inspect
-from typing import Any, Callable, Coroutine, TypeVar
-
-T = TypeVar("T")
-
-
-def run_in_executor(obj: Callable[..., Any] | Coroutine[Any, Any, T]) -> T:
-    if callable(obj) and not inspect.iscoroutinefunction(obj):
-        return obj()
-
-    if inspect.iscoroutinefunction(obj):
-        obj = obj()
-
-    if not inspect.iscoroutine(obj):
-        return obj
-
-    try:
-        loop = asyncio.get_running_loop()
-    except RuntimeError:
-        return asyncio.run(obj)
-
-    future = asyncio.run_coroutine_threadsafe(obj, loop)
-    return future.result()

oceanprotocol_job_details-0.3.12/oceanprotocol_job_details/loaders/impl/ddo.py

@@ -1,24 +0,0 @@
-from __future__ import annotations
-
-from dataclasses import InitVar, dataclass, field
-from pathlib import Path
-from typing import final
-
-from oceanprotocol_job_details.domain import DDO, Files
-
-
-@final
-@dataclass(frozen=True)
-class DDOLoader:
-    files: InitVar[Files]
-    """The files to load the DDOs from"""
-
-    _ddo_paths: list[Path] = field(init=False)
-
-    def __post_init__(self, files: Files) -> None:
-        assert files is not None and len(files) != 0, "Missing files"
-
-        object.__setattr__(self, "_ddo_paths", [f.ddo for f in files])
-
-    def load(self) -> list[DDO]:
-        return [DDO.model_validate_json(p.read_text()) for p in self._ddo_paths]

oceanprotocol_job_details-0.3.12/oceanprotocol_job_details/loaders/impl/job_details.py

@@ -1,29 +0,0 @@
-from dataclasses import dataclass, field
-from types import NoneType
-from typing import Generic, Type, TypeVar, final
-
-from pydantic import BaseModel
-
-from oceanprotocol_job_details.domain import DDO, Files, Paths
-from oceanprotocol_job_details.ocean import JobDetails
-
-T = TypeVar("T", BaseModel, None)
-
-
-@final
-@dataclass(frozen=True)
-class JobDetailsLoader(Generic[T]):
-    input_type: Type[T] = field(repr=False)
-    files: Files
-    secret: str
-    paths: Paths
-    ddos: list[DDO]
-
-    def load(self) -> JobDetails[T]:
-        return JobDetails[T](
-            files=self.files,
-            secret=self.secret,
-            ddos=self.ddos,
-            paths=self.paths,
-            input_type=self.input_type,
-        )