oceanprotocol-job-details 0.0.5__tar.gz → 0.0.6__tar.gz

{oceanprotocol_job_details-0.0.5 → oceanprotocol_job_details-0.0.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: oceanprotocol-job-details
-Version: 0.0.5
+Version: 0.0.6
 Summary: A Python package to get details from OceanProtocol jobs
 License: Copyright 2025 Agrospai
          

{oceanprotocol_job_details-0.0.5 → oceanprotocol_job_details-0.0.6}/oceanprotocol_job_details/dataclasses/constants.py

@@ -25,11 +25,14 @@ class _Paths:
     """Common paths used in the Ocean Protocol directories"""
 
     DATA: Path = Path("data")
+
     INPUTS: Path = DATA / "inputs"
     DDOS: Path = DATA / "ddos"
     OUTPUTS: Path = DATA / "outputs"
     LOGS: Path = DATA / "logs"
 
+    ALGORITHM_CUSTOM_PARAMETERS: Path = INPUTS / "algoCustomData.json"
+
 
 DidKeys = _DidKeys()
 ServiceType = _ServiceType()

oceanprotocol_job_details-0.0.6/oceanprotocol_job_details/dataclasses/job_details.py

@@ -0,0 +1,96 @@
+import json
+import logging
+import os
+from dataclasses import InitVar, dataclass
+from pathlib import Path
+from typing import Any, Mapping, Optional, Sequence
+
+from oceanprotocol_job_details.dataclasses.constants import Paths
+
+_MetadataType = Mapping[str, Any]
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass(frozen=True)
+class Parameters:
+    """Custom data for the algorithm, such as the algorithm's parameters"""
+
+    parameters: _MetadataType
+    """The parameters used by the algorithm"""
+
+
+@dataclass(frozen=True)
+class Algorithm:
+    """Details of the algorithm used to process the data"""
+
+    did: str
+    """The DID of the algorithm used to process the data"""
+
+    ddo: Path
+    """The DDO path of the algorithm used to process the data"""
+
+
+@dataclass
+class JobDetails:
+    """Details of the current job, such as the used inputs and algorithm"""
+
+    root: Path
+    """The root folder of the Ocean Protocol directories"""
+
+    dids: Sequence[Path]
+    """Identifiers for the inputs"""
+
+    files: Mapping[str, Sequence[Path]]
+    """Paths to the input files"""
+
+    secret: Optional[str]
+    """The secret used to process the data"""
+
+    algorithm: Optional[Algorithm]
+    """Details of the used algorithm"""
+
+    # Cache parameters, should not be included as _fields_ of the class
+    _parameters: InitVar[Optional[_MetadataType]] = None
+
+    def __post_init__(self, _):
+        os.makedirs(self.root / Paths.LOGS, exist_ok=True)
+
+        logging.getLogger().addHandler(
+            logging.FileHandler(
+                self.root / Paths.LOGS / "job_details.log",
+                mode="w",
+            )
+        )
+
+    @property
+    def parameters(self, parameters: Optional[Path] = None) -> _MetadataType:
+        """Parameters for algorithm job, read from default path"""
+
+        if parameters is None:
+            parameters = self.root / Paths.ALGORITHM_CUSTOM_PARAMETERS
+
+        if self._parameters is None:
+            if not parameters.exists():
+                logging.warning(
+                    f"Parameters file {parameters} not found, supplying empty"
+                )
+                self._parameters = {}
+            else:
+                # Load the parameters from filesystem
+                with open(parameters, "r") as f:
+                    try:
+                        self._parameters = json.load(f)
+                    except json.JSONDecodeError as e:
+                        self._parameters = {}
+                        logger.warning(
+                            f"Error loading parameters file {parameters}: {e}"
+                        )
+
+        return self._parameters
+
+
+del _MetadataType
+
+
+__all__ = ["Algorithm", "Parameters", "JobDetails"]

{oceanprotocol_job_details-0.0.5 → oceanprotocol_job_details-0.0.6}/oceanprotocol_job_details/job_details.py

@@ -1,7 +1,19 @@
-from typing import Literal, Optional
+import logging
+import os
+from typing import Any, Literal, Mapping, Optional
+
 from oceanprotocol_job_details.dataclasses.job_details import JobDetails
+from oceanprotocol_job_details.loaders.impl.map import Keys, Map
 from oceanprotocol_job_details.loaders.loader import Loader
-from oceanprotocol_job_details.loaders.impl.environment import EnvironmentLoader
+
+# Logging setup for the module
+logging.basicConfig(
+    level=logging.INFO,
+    format="%(asctime)s [%(threadName)-12.12s] [%(levelname)-5.5s]  %(message)s",
+    handlers=[
+        logging.StreamHandler(),
+    ],
+)
 
 _Implementations = Literal["env"]
 
@@ -11,13 +23,15 @@ class OceanProtocolJobDetails(Loader[JobDetails]):
 
     def __init__(
         self,
-        implementation: Optional[_Implementations] = "env",
+        implementation: Optional[_Implementations] = "map",
+        mapper: Mapping[str, Any] = os.environ,
+        keys: Keys = Keys(),
         *args,
         **kwargs,
     ):
-        if implementation == "env":
+        if implementation == "map":
             # As there are not more implementations, we can use the EnvironmentLoader directly
-            self._loader = lambda: EnvironmentLoader(*args, **kwargs)
+            self._loader = lambda: Map(mapper=mapper, keys=keys, *args, **kwargs)
         else:
             raise NotImplementedError(f"Implementation {implementation} not supported")
 

oceanprotocol_job_details-0.0.5/oceanprotocol_job_details/loaders/impl/environment.py → oceanprotocol_job_details-0.0.6/oceanprotocol_job_details/loaders/impl/map.py

@@ -1,23 +1,20 @@
 """Loads the current Job Details from the environment variables, could be abstracted to a more general 'mapper loader' but won't, since right now it fits our needs"""
 
-import os
-from collections.abc import Mapping, Sequence
 from dataclasses import dataclass
-from json import load, loads
+from json import JSONDecodeError, load, loads
+from logging import getLogger
 from pathlib import Path
-from typing import Optional, final
+from typing import Mapping, Optional, Sequence, final
 
-from oceanprotocol_job_details.dataclasses.constants import (
-    DidKeys,
-    Paths,
-    ServiceType,
-)
+from oceanprotocol_job_details.dataclasses.constants import DidKeys, Paths, ServiceType
 from oceanprotocol_job_details.dataclasses.job_details import Algorithm, JobDetails
 from oceanprotocol_job_details.loaders.loader import Loader
 
+logger = getLogger(__name__)
+
 
 @dataclass(frozen=True)
-class _Keys:
+class Keys:
     """Environment keys passed to the algorithm"""
 
     ROOT: str = "ROOT_FOLDER"
@@ -26,17 +23,15 @@ class _Keys:
     DIDS: str = "DIDS"
 
 
-Keys = _Keys()
-del _Keys
-
-
 @final
-class EnvironmentLoader(Loader[JobDetails]):
+class Map(Loader[JobDetails]):
     """Loads the current Job Details from the environment variables"""
 
-    def __init__(self, mapper: Mapping[str, str] = os.environ):
-        super().__init__()
-        self.mapper = mapper
+    def __init__(self, mapper: Mapping[str, str], keys: Keys, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+
+        self._mapper = mapper
+        self._keys = keys
 
     def load(self, *args, **kwargs) -> JobDetails:
         root, dids = self._root(), self._dids()
@@ -44,29 +39,34 @@ class EnvironmentLoader(Loader[JobDetails]):
         return JobDetails(
             root=root,
             dids=dids,
-            metadata=self._metadata(),
             files=self._files(root, dids),
-            algorithm=self._algorithm(root=root),
+            algorithm=self._algorithm(root),
             secret=self._secret(),
         )
 
     def _root(self) -> Path:
-        root = Path(self.mapper.get(Keys.ROOT, Path.home()))
+        root = Path(self._mapper.get(self._keys.ROOT, Path.home()))
 
         if not root.exists():
             raise FileNotFoundError(f"Root folder {root} does not exist")
 
         return root
 
-    def _dids(self) -> Sequence[str]:
-        return loads(self.mapper.get(Keys.DIDS)) if Keys.DIDS in self.mapper else []
+    def _dids(self) -> Sequence[Path]:
+        return (
+            loads(self._mapper.get(self._keys.DIDS))
+            if self._keys.DIDS in self._mapper
+            else []
+        )
 
     def _files(
         self,
         root: Path,
         dids: Optional[Sequence[Path]],
     ) -> Mapping[str, Sequence[Path]]:
+
         files: Mapping[str, Sequence[Path]] = {}
+
         for did in dids:
             # Retrieve DDO from disk
             file_path = root / Paths.DDOS / did
@@ -74,27 +74,30 @@ class EnvironmentLoader(Loader[JobDetails]):
                 raise FileNotFoundError(f"DDO file {file_path} does not exist")
 
             with open(file_path, "r") as f:
-                ddo = load(f)
+                try:
+                    ddo = load(f)
+                except JSONDecodeError as e:
+                    logger.warning(f"Error loading DDO file {file_path}: {e}")
+                    continue
+
                 for service in ddo[DidKeys.SERVICE]:
-                    if service[DidKeys.SERVICE_TYPE] == ServiceType.METADATA:
-                        base_path = root / Paths.INPUTS / did
-                        files[did] = [
-                            base_path / str(idx)
-                            for idx in range(
-                                len(
-                                    service[DidKeys.ATTRIBUTES][DidKeys.MAIN][
-                                        DidKeys.FILES
-                                    ]
-                                )
+                    if service[DidKeys.SERVICE_TYPE] != ServiceType.METADATA:
+                        continue
+
+                    did_path = root / Paths.INPUTS / did
+                    files[did] = [
+                        did_path / str(idx)
+                        for idx in range(
+                            len(
+                                service[DidKeys.ATTRIBUTES][DidKeys.MAIN][DidKeys.FILES]
                             )
-                        ]
-        return files
+                        )
+                    ]
 
-    def _metadata(self) -> Mapping[str, str]:
-        return {}
+        return files
 
-    def _algorithm(self, root: Path) -> Algorithm:
-        did = self.mapper.get(Keys.ALGORITHM, None)
+    def _algorithm(self, root: Path) -> Optional[Algorithm]:
+        did = self._mapper.get(self._keys.ALGORITHM, None)
 
         if not did:
             return None
@@ -103,10 +106,7 @@ class EnvironmentLoader(Loader[JobDetails]):
         if not ddo.exists():
             raise FileNotFoundError(f"DDO file {ddo} does not exist")
 
-        return Algorithm(
-            did=did,
-            ddo=ddo,
-        )
+        return Algorithm(did, ddo)
 
-    def _secret(self) -> str:
-        return self.mapper.get(Keys.SECRET, "")
+    def _secret(self) -> Optional[str]:
+        return self._mapper.get(self._keys.SECRET, None)

{oceanprotocol_job_details-0.0.5 → oceanprotocol_job_details-0.0.6}/pyproject.toml

@@ -2,7 +2,7 @@
 license = { file = "LICENSE" }
 
 name = "oceanprotocol-job-details"
-version = "0.0.5"
+version = "0.0.6"
 authors = [
     { name = "Christian López García", email = "christian.lopez@udl.cat" },
 ]

oceanprotocol_job_details-0.0.5/oceanprotocol_job_details/dataclasses/job_details.py

@@ -1,35 +0,0 @@
-from dataclasses import dataclass
-from pathlib import Path
-from typing import Any, Mapping, Optional, Sequence
-
-
-@dataclass(frozen=True)
-class Algorithm:
-    did: str
-    """The DID of the algorithm used to process the data"""
-
-    ddo: Path
-    """The DDO path of the algorithm used to process the data"""
-
-
-@dataclass(frozen=True)
-class JobDetails:
-    """Details of the current job, such as the used inputs and algorithm"""
-
-    root: Path
-    """The root folder of the Ocean Protocol directories"""
-
-    dids: Optional[Sequence[Path]]
-    """Identifiers for the inputs"""
-
-    metadata: Mapping[str, Any]
-    """TODO: To define"""
-
-    files: Mapping[str, Sequence[Path]]
-    """Paths to the input files"""
-
-    secret: Optional[str]
-    """The secret used to process the data"""
-
-    algorithm: Optional[Algorithm]
-    """Details of the used algorithm"""