oceanprotocol-job-details 0.3.13__py3-none-any.whl → 0.3.14__py3-none-any.whl

oceanprotocol_job_details/di.py

@@ -8,7 +8,7 @@ from oceanprotocol_job_details.loaders.impl.job_details import JobDetailsLoader
 from oceanprotocol_job_details.domain import Paths
 
 
-InputParametersT = TypeVar("InputParametersT", BaseModel, None)
+InputParametersT = TypeVar("InputParametersT", bound=BaseModel)
 
 
 class Container(containers.DeclarativeContainer, Generic[InputParametersT]):
@@ -26,7 +26,7 @@ class Container(containers.DeclarativeContainer, Generic[InputParametersT]):
 
     files = providers.Factory(lambda loader: loader.load(), loader=file_loader)
     ddo_loader = providers.Factory(DDOLoader, files=files)
-    ddos = providers.Factory(lambda loader: loader.load(), loader=ddo_loader)
+    metadata = providers.Factory(lambda loader: loader.load(), loader=ddo_loader)
 
     job_details_loader: providers.Factory[JobDetailsLoader[InputParametersT]] = (
         providers.Factory(
@@ -34,6 +34,6 @@ class Container(containers.DeclarativeContainer, Generic[InputParametersT]):
             files=files,
             secret=config.secret,
             paths=paths,
-            ddos=ddos,
+            metadata=metadata,
         )
     )

oceanprotocol_job_details/domain/__init__.py

@@ -0,0 +1,10 @@
+from oceanprotocol_job_details.domain.ddo import DDO
+from oceanprotocol_job_details.domain.derived import (
+    DID,
+    DDOMetadata,
+    DIDPaths,
+    Files,
+    Paths,
+)
+
+__all__ = ["DDO", "DID", "DDOMetadata", "Paths", "DIDPaths", "Files"]

oceanprotocol_job_details/{domain.py → domain/ddo.py}

@@ -1,7 +1,5 @@
 # mypy: disable-error-code=explicit-any
-from dataclasses import InitVar, dataclass, field
-from pathlib import Path
-from typing import Generator, List, Optional, Sequence, TypeAlias
+from typing import Optional
 
 from pydantic import BaseModel, ConfigDict, Field, JsonValue
 
@@ -124,64 +122,3 @@ class DDO(BaseModel):
     purgatory: Purgatory
 
     model_config = ConfigDict(populate_by_name=True)
-
-
-@dataclass(frozen=True)
-class DIDPaths:
-    did: str
-    ddo: Path = field(repr=False)
-
-    files: InitVar[Generator[Path, None, None]]
-
-    _input: List[Path] = field(init=False, repr=False)
-
-    def __post_init__(self, files: Generator[Path, None, None]) -> None:
-        assert self.ddo.exists(), f"DDO {self.ddo} does not exist"
-
-        object.__setattr__(self, "_input", list(files))
-
-    @property
-    def input_files(self) -> List[Path]:
-        return self._input
-
-    def __len__(self) -> int:
-        return len(self._input)
-
-
-Files: TypeAlias = Sequence[DIDPaths]
-
-
-@dataclass(frozen=True)
-class Paths:
-    """Configuration class for the Ocean Protocol Job Details"""
-
-    base_dir: InitVar[Path | None] = None
-
-    _base: Path = field(init=False, repr=False)
-
-    def __post_init__(self, base_dir: Path | None) -> None:
-        object.__setattr__(self, "_base", base_dir if base_dir else Path("/data"))
-
-    @property
-    def data(self) -> Path:
-        return self._base
-
-    @property
-    def inputs(self) -> Path:
-        return self.data / "inputs"
-
-    @property
-    def ddos(self) -> Path:
-        return self.data / "ddos"
-
-    @property
-    def outputs(self) -> Path:
-        return self.data / "outputs"
-
-    @property
-    def logs(self) -> Path:
-        return self.data / "logs"
-
-    @property
-    def algorithm_custom_parameters(self) -> Path:
-        return self.inputs / "algoCustomData.json"

oceanprotocol_job_details/domain/derived.py

@@ -0,0 +1,71 @@
+from dataclasses import InitVar, dataclass, field
+from pathlib import Path
+from typing import Dict, Generator, List, Sequence, TypeAlias
+
+from oceanprotocol_job_details.domain.ddo import DDO
+
+DID: TypeAlias = str
+
+
+@dataclass(frozen=True)
+class DIDPaths:
+    did: DID
+    ddo: Path = field(repr=False)
+
+    files: InitVar[Generator[Path, None, None]]
+
+    _input: List[Path] = field(init=False, repr=False)
+
+    def __post_init__(self, files: Generator[Path, None, None]) -> None:
+        assert self.ddo.exists(), f"DDO {self.ddo} does not exist"
+
+        object.__setattr__(self, "_input", list(files))
+
+    @property
+    def input_files(self) -> List[Path]:
+        return self._input
+
+    def __len__(self) -> int:
+        return len(self._input)
+
+
+Files: TypeAlias = Sequence[DIDPaths]
+
+
+@dataclass(frozen=True)
+class Paths:
+    """Configuration class for the Ocean Protocol Job Details"""
+
+    base_dir: InitVar[Path | None] = None
+
+    _base: Path = field(init=False, repr=False)
+
+    def __post_init__(self, base_dir: Path | None) -> None:
+        object.__setattr__(self, "_base", base_dir if base_dir else Path("/data"))
+
+    @property
+    def data(self) -> Path:
+        return self._base
+
+    @property
+    def inputs(self) -> Path:
+        return self.data / "inputs"
+
+    @property
+    def ddos(self) -> Path:
+        return self.data / "ddos"
+
+    @property
+    def outputs(self) -> Path:
+        return self.data / "outputs"
+
+    @property
+    def logs(self) -> Path:
+        return self.data / "logs"
+
+    @property
+    def algorithm_custom_parameters(self) -> Path:
+        return self.inputs / "algoCustomData.json"
+
+
+DDOMetadata: TypeAlias = Dict[DID, DDO]

oceanprotocol_job_details/executors.py

@@ -1,25 +1,40 @@
+# mypy: disable-error-code=explicit-any
 import asyncio
 import inspect
-from typing import Any, Callable, Coroutine, TypeVar
+from functools import partial
+from typing import Any, Callable, Coroutine, TypeGuard, TypeVar, cast
 
 T = TypeVar("T")
 
 
+def is_coro_function(
+    obj: Any,
+) -> TypeGuard[Callable[..., Coroutine[Any, Any, T]]]:
+    return inspect.iscoroutinefunction(obj)
+
+
+def is_coro(obj: Any) -> TypeGuard[Coroutine[Any, Any, T]]:
+    return inspect.iscoroutine(obj)
+
+
 async def run_in_executor(
     obj: Callable[..., T]
     | Callable[..., Coroutine[Any, Any, T]]
     | Coroutine[Any, Any, T],
-    *args,
-    **kwargs,
+    *args: Any,
+    **kwargs: Any,
 ) -> T:
-    if inspect.iscoroutinefunction(obj):
+    if is_coro_function(obj):
         return await obj(*args, **kwargs)
 
-    if inspect.iscoroutine(obj):
+    if is_coro(obj):
         return await obj
 
     if callable(obj):
         loop = asyncio.get_running_loop()
-        return await loop.run_in_executor(None, obj, *args, **kwargs)
 
-    return obj
+        # just to comply with mypy
+        func = partial(obj, *args, **kwargs)
+        return await loop.run_in_executor(None, cast(Callable[[], T], func))
+
+    return cast(T, obj)

oceanprotocol_job_details/helpers.py

@@ -6,7 +6,7 @@ from oceanprotocol_job_details.di import Container
 from oceanprotocol_job_details.ocean import JobDetails
 from oceanprotocol_job_details.settings import JobSettings
 
-InputParametersT = TypeVar("InputParametersT", BaseModel, None)
+InputParametersT = TypeVar("InputParametersT", bound=BaseModel)
 
 
 def create_container(config: Dict[str, Any]) -> Container[InputParametersT]:  # type: ignore[explicit-any]

oceanprotocol_job_details/loaders/impl/ddo.py

@@ -1,10 +1,10 @@
-from __future__ import annotations
-
 from dataclasses import InitVar, dataclass, field
 from pathlib import Path
-from typing import final
+from typing import List, Tuple, final
+
+import orjson
 
-from oceanprotocol_job_details.domain import DDO, Files
+from oceanprotocol_job_details.domain import DDO, DID, DDOMetadata, Files
 
 
 @final
@@ -13,12 +13,14 @@ class DDOLoader:
     files: InitVar[Files]
     """The files to load the DDOs from"""
 
-    _ddo_paths: list[Path] = field(init=False)
+    _files: List[Tuple[DID, Path]] = field(init=False)
 
     def __post_init__(self, files: Files) -> None:
         assert files is not None and len(files) != 0, "Missing files"
+        object.__setattr__(self, "_files", [(f.did, f.ddo) for f in files])
 
-        object.__setattr__(self, "_ddo_paths", [f.ddo for f in files])
-
-    def load(self) -> list[DDO]:
-        return [DDO.model_validate_json(p.read_text()) for p in self._ddo_paths]
+    def load(self) -> DDOMetadata:
+        return {
+            did: DDO.model_validate(orjson.loads(path.read_bytes()))
+            for did, path in self._files
+        }

oceanprotocol_job_details/loaders/impl/files.py

@@ -1,6 +1,6 @@
 from __future__ import annotations
 
-from dataclasses import InitVar, dataclass, field
+from dataclasses import dataclass, field
 from logging import Logger
 from pathlib import Path
 from typing import Literal, final

oceanprotocol_job_details/loaders/impl/job_details.py

@@ -1,29 +1,28 @@
-from dataclasses import dataclass, field
-from types import NoneType
+from dataclasses import dataclass
 from typing import Generic, Type, TypeVar, final
 
-from pydantic import BaseModel
+from pydantic import BaseModel, Secret
 
-from oceanprotocol_job_details.domain import DDO, Files, Paths
+from oceanprotocol_job_details.domain import DDOMetadata, Files, Paths
 from oceanprotocol_job_details.ocean import JobDetails
 
-T = TypeVar("T", BaseModel, None)
+InputParameterT = TypeVar("InputParameterT", bound=BaseModel)
 
 
 @final
 @dataclass(frozen=True)
-class JobDetailsLoader(Generic[T]):
-    input_type: Type[T] = field(repr=False)
+class JobDetailsLoader(Generic[InputParameterT]):
+    input_type: Type[InputParameterT] | None
     files: Files
-    secret: str
+    secret: Secret[str] | None
     paths: Paths
-    ddos: list[DDO]
+    metadata: DDOMetadata
 
-    def load(self) -> JobDetails[T]:
-        return JobDetails[T](
+    def load(self) -> JobDetails[InputParameterT]:
+        return JobDetails[InputParameterT](
             files=self.files,
             secret=self.secret,
-            ddos=self.ddos,
+            metadata=self.metadata,
             paths=self.paths,
             input_type=self.input_type,
         )

oceanprotocol_job_details/ocean.py

@@ -1,26 +1,22 @@
-from __future__ import annotations
-
-import asyncio
-from functools import cached_property
 from pathlib import Path
 from typing import Generator, Generic, Tuple, Type, TypeVar, final
 
 import aiofiles
+import orjson
 from pydantic import BaseModel, ConfigDict, Secret
 
-from oceanprotocol_job_details.domain import DDO, Files, Paths
-from oceanprotocol_job_details.executors import run_in_executor
+from oceanprotocol_job_details.domain import DDOMetadata, Files, Paths
 
-InputParametersT = TypeVar("InputParametersT", BaseModel, None)
+InputParametersT = TypeVar("InputParametersT", bound=BaseModel)
 
 
 @final
 class JobDetails(BaseModel, Generic[InputParametersT]):  # type: ignore[explicit-any]
     files: Files
-    ddos: list[DDO]
+    metadata: DDOMetadata
     paths: Paths
     input_type: Type[InputParametersT] | None
-    secret: Secret[str] | None = None
+    secret: Secret[str] | None
 
     model_config = ConfigDict(arbitrary_types_allowed=True, frozen=True)
 
@@ -33,13 +29,16 @@ class JobDetails(BaseModel, Generic[InputParametersT]):  # type: ignore[explicit
             for file in files.input_files
         )
 
-    async def input_parameters(self) -> InputParametersT:
-        if self._input_parameters is None:
-            input_parameters = await self.ainput_parameters()
-            object.__setattr__(self, "_input_parameters", input_parameters)
-        return self._input_parameters
+    async def input_parameters(self) -> InputParametersT | None:
+        current = self._input_parameters
+
+        if current is None:
+            current = await self.ainput_parameters()
+            object.__setattr__(self, "_input_parameters", current)
+
+        return current
 
-    async def ainput_parameters(self) -> InputParametersT:
+    async def ainput_parameters(self) -> InputParametersT | None:
         if self.input_type is None:
             return None
 
@@ -49,4 +48,4 @@ class JobDetails(BaseModel, Generic[InputParametersT]):  # type: ignore[explicit
 
         raw = raw.strip()
         assert raw is not None, f"Empty file {path}"
-        return self.input_type.model_validate_json(raw)  # type: ignore
+        return self.input_type.model_validate(orjson.loads(raw))

oceanprotocol_job_details/settings.py

@@ -4,7 +4,7 @@ from pathlib import Path
 from typing import Self
 
 import orjson
-from pydantic import Field, field_validator, model_validator
+from pydantic import Field, Secret, field_validator, model_validator
 from pydantic_settings import BaseSettings, SettingsConfigDict
 
 
@@ -12,7 +12,7 @@ class JobSettings(BaseSettings):  # type: ignore[explicit-any]
     base_dir: Path = Field(alias="BASE_DIR")
     dids: list[str] = Field(default_factory=list, alias="DIDS")
     transformation_did: str = Field(alias="TRANSFORMATION_DID")
-    secret: str | None = Field(default=None, alias="SECRET")
+    secret: Secret[str] | None = Field(default=None, alias="SECRET")
     logger: Logger = Field(default_factory=lambda: getLogger(__name__))
 
     model_config = SettingsConfigDict(
@@ -35,7 +35,7 @@ class JobSettings(BaseSettings):  # type: ignore[explicit-any]
 
     @model_validator(mode="after")
     def validate_dids(self) -> Self:
-        if not self.dids:
+        if len(self.dids) == 0:
             self.dids.extend(
                 [f.name for f in (self.base_dir / "ddos").glob("*") if f.is_file()]
             )

{oceanprotocol_job_details-0.3.13.dist-info → oceanprotocol_job_details-0.3.14.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: oceanprotocol-job-details
-Version: 0.3.13
+Version: 0.3.14
 Summary: A Python package to get details from OceanProtocol jobs
 Project-URL: Homepage, https://github.com/AgrospAI/oceanprotocol-job-details
 Project-URL: Issues, https://github.com/AgrospAI/oceanprotocol-job-details/issues
@@ -97,7 +97,7 @@ _, file_path = next(job_details.inputs())
 
 ```bash
 data        # Root /data directory
-├── ddos    # Contains the loaded dataset's DDO
+├── ddos    # Contains the loaded dataset's DDO (metadata)
 │   ├── 17feb...e42 # DDO file
 │   └── ... # One DDO per loaded dataset
 ├── inputs  # Datasets dir

oceanprotocol_job_details-0.3.14.dist-info/RECORD

@@ -0,0 +1,20 @@
+oceanprotocol_job_details/__init__.py,sha256=sq_C8Ey4_4oHiX9cO1wne7ZsWqs86WWgHfKQ5PMwwak,218
+oceanprotocol_job_details/di.py,sha256=QQ7Gs1zbDdGRJ6MrJsW3VmaFLL90edcQd-x2DNR_I5A,1328
+oceanprotocol_job_details/executors.py,sha256=CTkJkFmNYg50Z7U6JO6Jc85biKE7yhWBHByfPMOT0wo,962
+oceanprotocol_job_details/helpers.py,sha256=nRmIu53eX3kzOdivcTCjr06xofjo-Z88_hMt7tcJzYg,1064
+oceanprotocol_job_details/ocean.py,sha256=8J2fChXmVevJGm8EWTi0QrnqGM8iOoOqJPqyM4PTJlQ,1572
+oceanprotocol_job_details/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+oceanprotocol_job_details/settings.py,sha256=MFrZXc1tPXyUsFCE9v2xqEamU3kBHhjzH2qLi0ov384,1367
+oceanprotocol_job_details/domain/__init__.py,sha256=GA0Fl0GB4_ioMNfdLAPHpGjo9oImy_1DqNWX9yMJqRo,243
+oceanprotocol_job_details/domain/ddo.py,sha256=sLNuxue6PYGs1fXR7FCuzS1Ny0gybJL1kwZF_aptkzE,2307
+oceanprotocol_job_details/domain/derived.py,sha256=GBq_HDWJnVbyj0KejYfOEb6FL3kVrn6may4cKQYtsws,1677
+oceanprotocol_job_details/loaders/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+oceanprotocol_job_details/loaders/loader.py,sha256=36X2s_0lN89kCUpItxEXfIzuBBNJySebP2B_tdWK2E0,186
+oceanprotocol_job_details/loaders/impl/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+oceanprotocol_job_details/loaders/impl/ddo.py,sha256=jKesMAOTtbHeXtOM_aWyOiX4GDfT3IRdmiub1FkuMCs,748
+oceanprotocol_job_details/loaders/impl/files.py,sha256=01wZAhR41lh3lAiJrMMjNguWI_JlY28oCix2zN9CNXo,1136
+oceanprotocol_job_details/loaders/impl/job_details.py,sha256=06Zy8SdIpEmrqbmA8de-InMPrrh3PygzcgOVBPr5Ks8,806
+oceanprotocol_job_details-0.3.14.dist-info/METADATA,sha256=sNjiZklJFJoGCakjHdeei9e69lKhBFuBipBvYIMPCYs,4520
+oceanprotocol_job_details-0.3.14.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
+oceanprotocol_job_details-0.3.14.dist-info/licenses/LICENSE,sha256=ni3ix7P_GxK1W3VGC4fJ3o6QoCngCEpSuTJwO4nkpbw,1055
+oceanprotocol_job_details-0.3.14.dist-info/RECORD,,

oceanprotocol_job_details-0.3.13.dist-info/RECORD

@@ -1,18 +0,0 @@
-oceanprotocol_job_details/__init__.py,sha256=sq_C8Ey4_4oHiX9cO1wne7ZsWqs86WWgHfKQ5PMwwak,218
-oceanprotocol_job_details/di.py,sha256=H4_n9NrNuRB2s9uarSkwDYzjD1i7sV8dySRfHt21QWM,1316
-oceanprotocol_job_details/domain.py,sha256=ifw-hKFAJj0Gl_wJuH-51LQx4KPsCyEVDsT10xg9uBw,3842
-oceanprotocol_job_details/executors.py,sha256=Csc1MaUf0-Ve4Wp80qIanL7reRTT2qA3W3QrSE2jQO4,556
-oceanprotocol_job_details/helpers.py,sha256=ABm3oIRwPd-4XeCOIszCbfL2wkUJqVJJ2bqy3hR4jyw,1064
-oceanprotocol_job_details/ocean.py,sha256=kq7tAHJ36s7piJ_WpXVaTmSAMe87tuwHDeMQB-xS0T4,1702
-oceanprotocol_job_details/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-oceanprotocol_job_details/settings.py,sha256=o_1Hn2vl5hMk7bAkdS7GjE4nKOAyHm7dScO2_o2sPuY,1345
-oceanprotocol_job_details/loaders/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-oceanprotocol_job_details/loaders/loader.py,sha256=36X2s_0lN89kCUpItxEXfIzuBBNJySebP2B_tdWK2E0,186
-oceanprotocol_job_details/loaders/impl/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-oceanprotocol_job_details/loaders/impl/ddo.py,sha256=XthrQFhmP85XSVzVjBlLePtTowGR3BAsmVp3jngiQ08,668
-oceanprotocol_job_details/loaders/impl/files.py,sha256=Y2vFBT2T9w9zrdpmf550-LQJxwtNPUGa0UU6bBzk9AU,1145
-oceanprotocol_job_details/loaders/impl/job_details.py,sha256=QwlUaG9KozkI1wX66oDTPg4TjGkvSsi8O-TctF6eWvo,724
-oceanprotocol_job_details-0.3.13.dist-info/METADATA,sha256=9jFrlbS9oWNQXrHj_jY3ia16bA3HvdBeMt6nMdUguHk,4509
-oceanprotocol_job_details-0.3.13.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
-oceanprotocol_job_details-0.3.13.dist-info/licenses/LICENSE,sha256=ni3ix7P_GxK1W3VGC4fJ3o6QoCngCEpSuTJwO4nkpbw,1055
-oceanprotocol_job_details-0.3.13.dist-info/RECORD,,