oceanprotocol-job-details 0.1.5__tar.gz → 0.2.1__tar.gz

{oceanprotocol_job_details-0.1.5 → oceanprotocol_job_details-0.2.1}/.gitignore

@@ -3,4 +3,5 @@ dist
 
 *egg-info
 __pycache__
-.pytest_cache
+.pytest_cache
+.vscode

{oceanprotocol_job_details-0.1.5 → oceanprotocol_job_details-0.2.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: oceanprotocol-job-details
-Version: 0.1.5
+Version: 0.2.1
 Summary: A Python package to get details from OceanProtocol jobs
 Project-URL: Homepage, https://github.com/AgrospAI/oceanprotocol-job-details
 Project-URL: Issues, https://github.com/AgrospAI/oceanprotocol-job-details/issues
@@ -18,6 +18,7 @@ Classifier: Operating System :: OS Independent
 Classifier: Programming Language :: Python :: 3
 Requires-Python: >=3.10
 Requires-Dist: dataclasses-json>=0.6.7
+Requires-Dist: dependency-injector>=4.48.2
 Requires-Dist: orjson>=3.11.3
 Description-Content-Type: text/markdown
 
@@ -36,10 +37,10 @@ pip install oceanprotocol-job-details
 As a simple library, we only need to import the main object and use it once:
 
 ```Python
-from oceanprotocol_job_details.job_details import OceanProtocolJobDetails
+from oceanprotocol_job_details import JobDetails
 
 # Having no algorithm input parameters
-job_details = OceanProtocolJobDetails().load()  # type: ignore
+job_details = JobDetails.load()
 
 ```
 
@@ -48,15 +49,16 @@ If our algorithm has custom input parameters and we want to load them into our a
 ```Python
 
 from dataclasses import dataclass
-from oceanprotocol_job_details.job_details import OceanProtocolJobDetails
-from oceanprotocol_job_details.ocean import JobDetails
+from oceanprotocol_job_details import JobDetails
+
 
 @dataclass
-class Input:
+class InputParameters:
     name: str
     age: int
 
-job_details: JobDetails[Input] = OceanProtocolJobDetails(Input).load()
+
+job_details: JobDetails[InputParameters] = JobDetails.load(InputParameters)
 
 # Usage (is type hinted)
 job_details.input_parameters.name
@@ -64,20 +66,11 @@ job_details.input_parameters.age
 
 ```
 
-Assumes the following directory structure:
-```
-<ROOT_FOLDER>
-└───data
-    ├───ddos
-    ├───transformation
-    ├───inputs
-    └───logs
-```
+Assumes the directory structure of OceanProtocol algorithms.
 
 ### Core functionalities
 
-Given the Ocean Protocol job details structure as in [https://github.com/GX4FM-Base-X/pontus-x-ontology](Pontus-X Ontology), parses the passed algorithm parameters into an object to use in your algorithms.
+Given the Ocean Protocol job details structure, parses the passed algorithm parameters into an object to use in your algorithms.
 
-1. Parsing JSON
-1. Validation
-1. Metadata and service extraction
+1. Input parameter JSON parsing and validation
+1. Metadata and service extraction from the directory structure.

oceanprotocol_job_details-0.2.1/README.md

@@ -0,0 +1,52 @@
+A Python package to get details from OceanProtocol jobs
+
+---
+
+## Installation
+
+```
+pip install oceanprotocol-job-details
+```
+
+## Usage 
+
+As a simple library, we only need to import the main object and use it once:
+
+```Python
+from oceanprotocol_job_details import JobDetails
+
+# Having no algorithm input parameters
+job_details = JobDetails.load()
+
+```
+
+If our algorithm has custom input parameters and we want to load them into our algorithm, we can do it as follows:
+
+```Python
+
+from dataclasses import dataclass
+from oceanprotocol_job_details import JobDetails
+
+
+@dataclass
+class InputParameters:
+    name: str
+    age: int
+
+
+job_details: JobDetails[InputParameters] = JobDetails.load(InputParameters)
+
+# Usage (is type hinted)
+job_details.input_parameters.name
+job_details.input_parameters.age
+
+```
+
+Assumes the directory structure of OceanProtocol algorithms.
+
+### Core functionalities
+
+Given the Ocean Protocol job details structure, parses the passed algorithm parameters into an object to use in your algorithms.
+
+1. Input parameter JSON parsing and validation
+1. Metadata and service extraction from the directory structure.

oceanprotocol_job_details-0.2.1/oceanprotocol_job_details/__init__.py

@@ -0,0 +1,4 @@
+from ocean import JobDetails
+
+
+__all__ = [JobDetails]

oceanprotocol_job_details-0.2.1/oceanprotocol_job_details/di.py

@@ -0,0 +1,44 @@
+from dependency_injector import containers, providers
+
+from oceanprotocol_job_details.loaders.impl.ddo import DDOLoader
+from oceanprotocol_job_details.loaders.impl.files import FilesLoader
+from oceanprotocol_job_details.loaders.impl.job_details import JobDetailsLoader
+from oceanprotocol_job_details.paths import Paths
+
+
+class Container(containers.DeclarativeContainer):
+
+    config = providers.Configuration()
+
+    paths = providers.Singleton(Paths)
+
+    file_loader = providers.Factory(
+        FilesLoader,
+        dids=config.dids,
+        transformation_did=config.transformation_did,
+        paths=paths,
+    )
+
+    files = providers.Factory(
+        lambda loader: loader.load(),
+        loader=file_loader,
+    )
+
+    # DDOLoader depends on Files loaded from FilesLoader
+    ddo_loader = providers.Factory(
+        DDOLoader,
+        files=files,
+    )
+
+    ddos = providers.Factory(
+        lambda loader: loader.load(),
+        loader=ddo_loader,
+    )
+
+    job_details_loader = providers.Factory(
+        JobDetailsLoader,
+        files=files,
+        secret=config.secret,
+        paths=paths,
+        ddos=ddos,
+    )

{oceanprotocol_job_details-0.1.5 → oceanprotocol_job_details-0.2.1}/oceanprotocol_job_details/loaders/impl/ddo.py

@@ -1,24 +1,30 @@
+from __future__ import annotations
+
 from dataclasses import InitVar, dataclass, field
 from pathlib import Path
-from typing import final
+from typing import TYPE_CHECKING, final
 
-from oceanprotocol_job_details.ocean import DDO
+if TYPE_CHECKING:
+    from oceanprotocol_job_details.ocean import DDO, Files
 
 
 @final
 @dataclass(frozen=True)
 class DDOLoader:
-    ddo_paths: InitVar[list[Path]]
+
+    files: InitVar[list[Files]]
     """The files to load the DDOs from"""
 
     _ddo_paths: list[Path] = field(init=False)
 
-    def __post_init__(self, ddo_paths: list[Path]) -> None:
-        assert ddo_paths, "Missing DDO paths"
+    def __post_init__(self, files: list[Files]) -> None:
+        assert files, "Missing files"
 
-        object.__setattr__(self, "_ddo_paths", ddo_paths)
+        object.__setattr__(self, "_ddo_paths", [f.ddo for f in files])
 
     def load(self) -> list[DDO]:
+        from oceanprotocol_job_details.ocean import DDO
+
         ddos = []
         for path in self._ddo_paths:
             with open(path, "r") as f:

{oceanprotocol_job_details-0.1.5 → oceanprotocol_job_details-0.2.1}/oceanprotocol_job_details/loaders/impl/files.py

@@ -1,53 +1,28 @@
+from __future__ import annotations
+
 import json
 from dataclasses import InitVar, dataclass, field
-from pathlib import Path
-from typing import Iterator, Sequence, final
-
-from oceanprotocol_job_details.config import config
-
-
-@dataclass(frozen=True)
-class DIDPaths:
-    did: str
-    ddo: Path
-    input_files: Sequence[Path]
-
-    def __post_init__(self) -> None:
-        assert self.ddo.exists(), f"DDO {self.ddo} does not exist"
-        for input_file in self.input_files:
-            assert input_file.exists(), f"File {input_file} does not exist"
+from typing import TYPE_CHECKING, Sequence, final
 
-    def __len__(self) -> int:
-        return len(self.input_files)
+from oceanprotocol_job_details.paths import Paths
 
-
-@dataclass(frozen=True)
-class Files:
-    _files: Sequence[DIDPaths]
-
-    @property
-    def files(self) -> Sequence[DIDPaths]:
-        return self._files
-
-    def __getitem__(self, index: int) -> DIDPaths:
-        return self.files[index]
-
-    def __iter__(self) -> Iterator[DIDPaths]:
-        return iter(self.files)
-
-    def __len__(self) -> int:
-        return len(self.files)
+if TYPE_CHECKING:
+    from oceanprotocol_job_details.ocean import DIDPaths, Files
 
 
 @final
 @dataclass(frozen=True)
 class FilesLoader:
+
     dids: InitVar[str | None]
     """Input DIDs"""
 
     transformation_did: InitVar[str | None]
     """DID for the transformation algorithm"""
 
+    paths: Paths
+    """Path configurations of the project"""
+
     _dids: Sequence[str] = field(init=False)
     _transformation_did: str = field(init=False)
 
@@ -63,13 +38,15 @@ class FilesLoader:
         object.__setattr__(self, "_transformation_did", transformation_did)
 
     def load(self) -> Files:
+        from oceanprotocol_job_details.ocean import DIDPaths, Files
+
         files: list[DIDPaths] = []
         for did in self._dids:
-            base = Path(config.path_inputs) / did
+            base = self.paths.inputs / did
             files.append(
                 DIDPaths(
                     did=did,
-                    ddo=Path(config.path_ddos) / did,
+                    ddo=self.paths.ddos / did,
                     input_files=list(base.iterdir()),
                 )
             )

oceanprotocol_job_details-0.2.1/oceanprotocol_job_details/loaders/impl/job_details.py

@@ -0,0 +1,35 @@
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import TYPE_CHECKING, Generic, Type, TypeVar, final
+
+from oceanprotocol_job_details.paths import Paths
+
+if TYPE_CHECKING:
+    from oceanprotocol_job_details.ocean import DDO, Files, JobDetails
+
+
+T = TypeVar("T")
+
+
+@final
+@dataclass(frozen=True)
+class JobDetailsLoader(Generic[T]):
+
+    _type: Type[T] = field(repr=False)
+
+    files: Files
+    secret: str
+    paths: Paths
+    ddos: list[DDO]
+
+    def load(self) -> JobDetails[T]:
+        from oceanprotocol_job_details.ocean import JobDetails
+
+        return JobDetails(
+            files=self.files,
+            secret=self.secret,
+            ddos=self.ddos,
+            paths=self.paths,
+            _type=self._type,
+        )

{oceanprotocol_job_details-0.1.5 → oceanprotocol_job_details-0.2.1}/oceanprotocol_job_details/ocean.py

@@ -1,17 +1,28 @@
+from __future__ import annotations
+
+import logging
+import os
 from dataclasses import dataclass, field
 from functools import cached_property
-from typing import Any, Generic, Optional, Type, TypeVar, final
+from pathlib import Path
+from typing import Any, Generic, Iterator, Optional, Sequence, Type, TypeVar, final
 
 import orjson
-
 from dataclasses_json import config as dc_config
 from dataclasses_json import dataclass_json
 
-from oceanprotocol_job_details.config import config
-from oceanprotocol_job_details.loaders.impl.files import Files
+from oceanprotocol_job_details.di import Container
+from oceanprotocol_job_details.paths import Paths
 
 T = TypeVar("T")
 
+logging.basicConfig(
+    level=logging.INFO,
+    format="%(asctime)s [%(threadName)s] [%(levelname)s]  %(message)s",
+    handlers=[logging.StreamHandler()],
+)
+logger = logging.getLogger(__name__)
+
 
 @dataclass_json
 @dataclass
@@ -29,7 +40,7 @@ class Credentials:
 
 @dataclass_json
 @dataclass
-class Container:
+class DockerContainer:
     image: str
     tag: str
     entrypoint: str
@@ -38,7 +49,7 @@ class Container:
 @dataclass_json
 @dataclass
 class Algorithm:  # type: ignore
-    container: Container
+    container: DockerContainer
     language: str
     version: str
     consumerParameters: Any  # type: ignore
@@ -157,6 +168,39 @@ class DDO:
     purgatory: Purgatory
 
 
+@dataclass(frozen=True)
+class DIDPaths:
+    did: str
+    ddo: Path
+    input_files: Sequence[Path]
+
+    def __post_init__(self) -> None:
+        assert self.ddo.exists(), f"DDO {self.ddo} does not exist"
+        for input_file in self.input_files:
+            assert input_file.exists(), f"File {input_file} does not exist"
+
+    def __len__(self) -> int:
+        return len(self.input_files)
+
+
+@dataclass(frozen=True)
+class Files:
+    _files: Sequence[DIDPaths]
+
+    @property
+    def files(self) -> Sequence[DIDPaths]:
+        return self._files
+
+    def __getitem__(self, index: int) -> DIDPaths:
+        return self.files[index]
+
+    def __iter__(self) -> Iterator[DIDPaths]:
+        return iter(self.files)
+
+    def __len__(self) -> int:
+        return len(self.files)
+
+
 def _normalize_json(value):
     if isinstance(value, str):
         try:
@@ -171,6 +215,12 @@ def _normalize_json(value):
     return value
 
 
+@final
+@dataclass_json
+@dataclass
+class _EmptyJobDetails: ...
+
+
 @final
 @dataclass_json
 @dataclass(frozen=True)
@@ -181,6 +231,9 @@ class JobDetails(Generic[T]):
     ddos: list[DDO]
     """list of paths to the DDOs"""
 
+    paths: Paths
+    """Configuration paths"""
+
     # Store the type explicitly to avoid issues
     _type: Type[T] = field(repr=False)
 
@@ -195,11 +248,11 @@ class JobDetails(Generic[T]):
     def input_parameters(self) -> T:
         """Read the input parameters and return them in an instance of the dataclass T"""
 
-        with open(config.path_algorithm_custom_parameters, "r") as f:
+        with open(self.paths.algorithm_custom_parameters, "r") as f:
             raw = f.read().strip()
             if not raw:
                 raise ValueError(
-                    f"Custom parameters file {config.path_algorithm_custom_parameters} is empty"
+                    f"Custom parameters file {self.paths.algorithm_custom_parameters} is empty"
                 )
             try:
                 parsed = _normalize_json(orjson.loads(raw))
@@ -209,3 +262,39 @@ class JobDetails(Generic[T]):
                     f"Failed to parse input paramers into {self._type.__name__}: {e}\n"
                     f"Raw content: {raw}"
                 ) from e
+
+    @classmethod
+    def load(
+        cls,
+        _type: Type[T] | None = None,
+        dids: str | None = None,
+        transformation_did: str | None = None,
+        secret: str | None = None,
+    ) -> JobDetails[T]:
+        """Load a JobDetails instance that holds the runtime details.
+
+        Loading it will check the following:
+        1. That the needed environment variables are set.
+        1. That the ocean protocol contains the needed data based on the passed environment variables.
+
+        Those needed environment variables are:
+        - DIDS: The DIDs of the inputs
+        - TRANSFORMATION_DID: The DID of the transformation algorithm
+        - SECRET (optional): A really secret secret
+
+        """
+
+        if _type is None:
+            _type = _EmptyJobDetails
+
+        container = Container()
+        container.config.from_dict(
+            {
+                "dids": dids or os.environ.get("DIDS"),
+                "transformation_did": transformation_did
+                or os.environ.get("TRANSFORMATION_DID"),
+                "secret": secret or os.environ.get("SECRET"),
+            }
+        )
+
+        return container.job_details_loader(_type=_type).load()

oceanprotocol_job_details-0.2.1/oceanprotocol_job_details/paths.py

@@ -0,0 +1,28 @@
+from dataclasses import dataclass
+from logging import getLogger
+from pathlib import Path
+
+logger = getLogger(__name__)
+
+
+@dataclass
+class Paths:
+    """Configuration class for the Ocean Protocol Job Details"""
+
+    data: Path = Path("/data")
+    """The path to the data directory"""
+
+    inputs: Path = data / "inputs"
+    """The path to the inputs directory"""
+
+    ddos: Path = data / "ddos"
+    """The path to the DDOs directory"""
+
+    outputs: Path = data / "outputs"
+    """The path to the outputs directory"""
+
+    logs: Path = data / "logs"
+    """The path to the logs directory"""
+
+    algorithm_custom_parameters: Path = inputs / "algoCustomData.json"
+    """The path to the algorithm's custom parameters file"""

{oceanprotocol_job_details-0.1.5 → oceanprotocol_job_details-0.2.1}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "oceanprotocol-job-details"
-version = "0.1.5"
+version = "0.2.1"
 description = "A Python package to get details from OceanProtocol jobs"
 authors = [
     { name = "Christian López García", email = "christian.lopez@udl.cat" },
@@ -13,7 +13,11 @@ classifiers = [
     "Operating System :: OS Independent",
     "License :: OSI Approved :: MIT License",
 ]
-dependencies = ["dataclasses-json>=0.6.7", "orjson>=3.11.3"]
+dependencies = [
+    "dataclasses-json>=0.6.7",
+    "dependency-injector>=4.48.2",
+    "orjson>=3.11.3",
+]
 
 [project.urls]
 Homepage = "https://github.com/AgrospAI/oceanprotocol-job-details"

oceanprotocol_job_details-0.1.5/README.md

@@ -1,60 +0,0 @@
-A Python package to get details from OceanProtocol jobs
-
----
-
-## Installation
-
-```
-pip install oceanprotocol-job-details
-```
-
-## Usage 
-
-As a simple library, we only need to import the main object and use it once:
-
-```Python
-from oceanprotocol_job_details.job_details import OceanProtocolJobDetails
-
-# Having no algorithm input parameters
-job_details = OceanProtocolJobDetails().load()  # type: ignore
-
-```
-
-If our algorithm has custom input parameters and we want to load them into our algorithm, we can do it as follows:
-
-```Python
-
-from dataclasses import dataclass
-from oceanprotocol_job_details.job_details import OceanProtocolJobDetails
-from oceanprotocol_job_details.ocean import JobDetails
-
-@dataclass
-class Input:
-    name: str
-    age: int
-
-job_details: JobDetails[Input] = OceanProtocolJobDetails(Input).load()
-
-# Usage (is type hinted)
-job_details.input_parameters.name
-job_details.input_parameters.age
-
-```
-
-Assumes the following directory structure:
-```
-<ROOT_FOLDER>
-└───data
-    ├───ddos
-    ├───transformation
-    ├───inputs
-    └───logs
-```
-
-### Core functionalities
-
-Given the Ocean Protocol job details structure as in [https://github.com/GX4FM-Base-X/pontus-x-ontology](Pontus-X Ontology), parses the passed algorithm parameters into an object to use in your algorithms.
-
-1. Parsing JSON
-1. Validation
-1. Metadata and service extraction

oceanprotocol_job_details-0.1.5/oceanprotocol_job_details/config.py

@@ -1,54 +0,0 @@
-from dataclasses import dataclass, fields
-from logging import getLogger
-from pathlib import Path
-
-logger = getLogger(__name__)
-
-
-@dataclass
-class Config:
-    """Configuration class for the Ocean Protocol Job Details"""
-
-    path_data: Path = Path("/data")
-    """The path to the data directory"""
-
-    path_inputs: Path = path_data / "inputs"
-    """The path to the inputs directory"""
-
-    path_ddos: Path = path_data / "ddos"
-    """The path to the DDOs directory"""
-
-    path_outputs: Path = path_data / "outputs"
-    """The path to the outputs directory"""
-
-    path_logs: Path = path_data / "logs"
-    """The path to the logs directory"""
-
-    path_algorithm_custom_parameters: Path = path_inputs / "algoCustomData.json"
-    """The path to the algorithm's custom parameters file"""
-
-
-config = Config()
-
-
-def update_config_from(base: Path) -> None:
-    """Updates the configuration to use the new base path, ensures that the base path exists.
-
-    Args:
-        base (Path): The new base path to use.
-    """
-
-    logger.info(f"Updating config to use base path: {base}")
-
-    base.mkdir(parents=True, exist_ok=True)
-
-    for field in fields(config):
-        current_value = getattr(config, field.name)
-        if not isinstance(current_value, Path):
-            raise ValueError(f"Field {field.name} is n|ot a Path")
-
-        rel_path = Path(current_value).relative_to("/data")
-        object.__setattr__(config, field.name, base / rel_path)
-
-
-__all__ = ["config"]

oceanprotocol_job_details-0.1.5/oceanprotocol_job_details/job_details.py

@@ -1,47 +0,0 @@
-import logging
-from dataclasses import dataclass
-from typing import Generic, Type, TypeVar
-
-from dataclasses_json import dataclass_json
-
-from oceanprotocol_job_details.loaders.impl.job_details import JobDetailsLoader
-from oceanprotocol_job_details.loaders.loader import Loader
-from oceanprotocol_job_details.ocean import JobDetails
-
-logging.basicConfig(
-    level=logging.INFO,
-    format="%(asctime)s [%(threadName)s] [%(levelname)s]  %(message)s",
-    handlers=[logging.StreamHandler()],
-)
-logger = logging.getLogger(__name__)
-
-
-@dataclass_json
-@dataclass
-class _EmptyJobDetails: ...
-
-
-T = TypeVar("T")
-
-
-class OceanProtocolJobDetails(Generic[T]):
-    """The JobDetails class is a dataclass that holds the details of the current job.
-
-    Loading it will check the following:
-    1. That the needed environment variables are set
-    1. That the ocean protocol contains the needed data based on the passed environment variables
-
-    Those needed environment variables are:
-    - DIDS: The DIDs of the inputs
-    - TRANSFORMATION_DID: The DID of the transformation algorithm
-    - SECRET (optional): A really secret secret
-
-    """
-
-    def __init__(self, _type: Type[T] | None = None) -> None:
-        if _type is None:
-            _type = _EmptyJobDetails  # type: ignore[assignment]
-        self.job_details_loader: Loader[JobDetails[T]] = JobDetailsLoader(_type)  # type: ignore[arg-type]
-
-    def load(self) -> JobDetails[T]:
-        return self.job_details_loader.load()

oceanprotocol_job_details-0.1.5/oceanprotocol_job_details/loaders/impl/job_details.py

@@ -1,26 +0,0 @@
-import os
-from dataclasses import dataclass, field
-from typing import Generic, Type, TypeVar, final
-
-from oceanprotocol_job_details.loaders.impl.ddo import DDOLoader
-from oceanprotocol_job_details.loaders.impl.files import FilesLoader
-from oceanprotocol_job_details.ocean import JobDetails
-
-T = TypeVar("T")
-
-
-@final
-@dataclass(frozen=True)
-class JobDetailsLoader(Generic[T]):
-    _type: Type[T] = field(repr=False)
-    
-
-    def load(self) -> JobDetails[T]:
-        dids = os.environ.get("DIDS")
-        transformation_did = os.environ.get("TRANSFORMATION_DID")
-        secret = os.environ.get("SECRET")
-
-        files = FilesLoader(dids, transformation_did).load()
-        ddos = DDOLoader([f.ddo for f in files]).load()
-
-        return JobDetails(files=files, secret=secret, ddos=ddos, _type=self._type)

oceanprotocol_job_details-0.1.5/oceanprotocol_job_details/utils.py

@@ -1,33 +0,0 @@
-from logging import getLogger
-from typing import Mapping, Optional, TypeVar
-
-T = TypeVar("T")
-logger = getLogger(__name__)
-
-
-def get(
-    map: Mapping[str, T],
-    key: str,
-    default: Optional[T] = None,
-) -> T | None:
-    """Get the value of a key from a dictionary, if not found return the default value if given, otherwise raise a KeyError
-
-    :param map: original map to get the item from
-    :type map: Mapping[str, T]
-    :param key: key to get the value from
-    :type key: str
-    :param default: default value if missing, defaults to None
-    :type default: Optional[T], optional
-    :raises KeyError: if the value is missing and no default is provided
-    :return: value of the key
-    :rtype: T
-    """
-
-    if key in map.keys():
-        return map.get(key)
-
-    if default is None:
-        raise KeyError(f"Key {key} not found")
-
-    logger.info(f"Key {key} not found, returning default value {default}")
-    return default