oceanprotocol-job-details 0.0.9__py3-none-any.whl → 0.0.11__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- oceanprotocol_job_details/dataclasses/constants.py +2 -6
- oceanprotocol_job_details/dataclasses/job_details.py +8 -29
- oceanprotocol_job_details/dataclasses/ocean.py +67 -0
- oceanprotocol_job_details/job_details.py +30 -21
- oceanprotocol_job_details/loaders/impl/map.py +64 -80
- oceanprotocol_job_details/loaders/impl/utils.py +57 -0
- oceanprotocol_job_details/loaders/loader.py +2 -9
- {oceanprotocol_job_details-0.0.9.dist-info → oceanprotocol_job_details-0.0.11.dist-info}/METADATA +28 -16
- oceanprotocol_job_details-0.0.11.dist-info/RECORD +15 -0
- {oceanprotocol_job_details-0.0.9.dist-info → oceanprotocol_job_details-0.0.11.dist-info}/WHEEL +1 -1
- oceanprotocol_job_details-0.0.9.dist-info/RECORD +0 -13
- {oceanprotocol_job_details-0.0.9.dist-info → oceanprotocol_job_details-0.0.11.dist-info/licenses}/LICENSE +0 -0
|
@@ -2,9 +2,7 @@ from dataclasses import dataclass
|
|
|
2
2
|
from pathlib import Path
|
|
3
3
|
|
|
4
4
|
|
|
5
|
-
@dataclass(
|
|
6
|
-
frozen=True,
|
|
7
|
-
)
|
|
5
|
+
@dataclass(frozen=True)
|
|
8
6
|
class _DidKeys:
|
|
9
7
|
"""Common keys inside the DIDs"""
|
|
10
8
|
|
|
@@ -15,9 +13,7 @@ class _DidKeys:
|
|
|
15
13
|
FILES: str = "files"
|
|
16
14
|
|
|
17
15
|
|
|
18
|
-
@dataclass(
|
|
19
|
-
frozen=True,
|
|
20
|
-
)
|
|
16
|
+
@dataclass(frozen=True)
|
|
21
17
|
class _ServiceType:
|
|
22
18
|
"""Service types inside the DIDs"""
|
|
23
19
|
|
|
@@ -1,10 +1,11 @@
|
|
|
1
|
-
import json
|
|
2
1
|
import logging
|
|
3
2
|
import os
|
|
4
3
|
from dataclasses import InitVar, dataclass
|
|
5
4
|
from pathlib import Path
|
|
6
5
|
from typing import Any, Mapping, Optional, Sequence
|
|
7
6
|
|
|
7
|
+
from orjson import JSONDecodeError, loads
|
|
8
|
+
|
|
8
9
|
from oceanprotocol_job_details.dataclasses.constants import Paths
|
|
9
10
|
|
|
10
11
|
_MetadataType = Mapping[str, Any]
|
|
@@ -12,9 +13,7 @@ _MetadataType = Mapping[str, Any]
|
|
|
12
13
|
logger = logging.getLogger(__name__)
|
|
13
14
|
|
|
14
15
|
|
|
15
|
-
@dataclass(
|
|
16
|
-
frozen=True,
|
|
17
|
-
)
|
|
16
|
+
@dataclass(frozen=True)
|
|
18
17
|
class Parameters:
|
|
19
18
|
"""Custom data for the algorithm, such as the algorithm's parameters"""
|
|
20
19
|
|
|
@@ -22,9 +21,7 @@ class Parameters:
|
|
|
22
21
|
"""The parameters used by the algorithm"""
|
|
23
22
|
|
|
24
23
|
|
|
25
|
-
@dataclass(
|
|
26
|
-
frozen=True,
|
|
27
|
-
)
|
|
24
|
+
@dataclass(frozen=True)
|
|
28
25
|
class Algorithm:
|
|
29
26
|
"""Details of the algorithm used to process the data"""
|
|
30
27
|
|
|
@@ -54,24 +51,8 @@ class JobDetails:
|
|
|
54
51
|
# Cache parameters, should not be included as _fields_ of the class
|
|
55
52
|
_parameters: InitVar[Optional[_MetadataType]] = None
|
|
56
53
|
|
|
57
|
-
def __post_init__(
|
|
58
|
-
self,
|
|
59
|
-
_,
|
|
60
|
-
):
|
|
61
|
-
os.makedirs(Paths.LOGS, exist_ok=True)
|
|
62
|
-
|
|
63
|
-
logging.getLogger().addHandler(
|
|
64
|
-
logging.FileHandler(
|
|
65
|
-
Paths.LOGS / "job_details.log",
|
|
66
|
-
mode="w",
|
|
67
|
-
)
|
|
68
|
-
)
|
|
69
|
-
|
|
70
54
|
@property
|
|
71
|
-
def parameters(
|
|
72
|
-
self,
|
|
73
|
-
parameters: Optional[Path] = None,
|
|
74
|
-
) -> _MetadataType:
|
|
55
|
+
def parameters(self, parameters: Optional[Path] = None) -> _MetadataType:
|
|
75
56
|
"""Parameters for algorithm job, read from default path"""
|
|
76
57
|
|
|
77
58
|
if parameters is None:
|
|
@@ -79,16 +60,14 @@ class JobDetails:
|
|
|
79
60
|
|
|
80
61
|
if self._parameters is None:
|
|
81
62
|
if not parameters.exists():
|
|
82
|
-
logging.warning(
|
|
83
|
-
f"Parameters file {parameters} not found, supplying empty"
|
|
84
|
-
)
|
|
63
|
+
logging.warning(f"Missing parameters file: {parameters} not found")
|
|
85
64
|
self._parameters = {}
|
|
86
65
|
else:
|
|
87
66
|
# Load the parameters from filesystem
|
|
88
67
|
with open(parameters, "r") as f:
|
|
89
68
|
try:
|
|
90
|
-
self._parameters =
|
|
91
|
-
except
|
|
69
|
+
self._parameters = loads(f.read())
|
|
70
|
+
except JSONDecodeError as e:
|
|
92
71
|
self._parameters = {}
|
|
93
72
|
logger.warning(
|
|
94
73
|
f"Error loading parameters file {parameters}: {e}"
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
from dataclasses import Field
|
|
2
|
+
from datetime import datetime
|
|
3
|
+
from typing import Annotated, Any, List, Optional
|
|
4
|
+
|
|
5
|
+
from pydantic import BaseModel, HttpUrl
|
|
6
|
+
|
|
7
|
+
"""Base classes for the Ocean Protocol algorithm structure"""
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
class Credential:
|
|
11
|
+
type: Annotated[str, Field(frozen=True)]
|
|
12
|
+
values: Annotated[List[str], Field(frozen=True)]
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
class Credentials:
|
|
16
|
+
allow: Optional[Annotated[List[Credential], Field(frozen=True)]] = []
|
|
17
|
+
deny: Optional[Annotated[List[Credential], Field(frozen=True)]] = []
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
class Metadata(BaseModel):
|
|
21
|
+
"""Base class for the Metadata structure"""
|
|
22
|
+
|
|
23
|
+
description: Annotated[str, Field(frozen=True)]
|
|
24
|
+
name: Annotated[str, Field(frozen=True)]
|
|
25
|
+
type: Annotated[str, Field(frozen=True)]
|
|
26
|
+
author: Annotated[str, Field(frozen=True)]
|
|
27
|
+
license: Annotated[str, Field(frozen=True)]
|
|
28
|
+
|
|
29
|
+
algorithm: Any
|
|
30
|
+
tags: Optional[Annotated[List[str], Field(frozen=True)]] = None
|
|
31
|
+
created: Optional[Annotated[datetime, Field(frozen=True)]] = None
|
|
32
|
+
updated: Optional[Annotated[datetime, Field(frozen=True)]] = None
|
|
33
|
+
copyrightHolder: Optional[Annotated[str, Field(frozen=True)]] = None
|
|
34
|
+
links: Optional[Annotated[List[HttpUrl], Field(frozen=True)]] = None
|
|
35
|
+
contentLanguage: Optional[Annotated[str, Field(frozen=True)]] = None
|
|
36
|
+
categories: Optional[Annotated[List[str], Field(frozen=True)]] = None
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
class Service(BaseModel):
|
|
40
|
+
"""Base class for the Service structure"""
|
|
41
|
+
|
|
42
|
+
id: Annotated[str, Field(frozen=True)]
|
|
43
|
+
type: Annotated[str, Field(frozen=True)]
|
|
44
|
+
timeout: Annotated[int, Field(frozen=True)]
|
|
45
|
+
files: Annotated[str, Field(frozen=True)]
|
|
46
|
+
datatokenAddress: Annotated[str, Field(frozen=True)]
|
|
47
|
+
serviceEndpoint: Annotated[HttpUrl, Field(frozen=True)]
|
|
48
|
+
|
|
49
|
+
compute: Any
|
|
50
|
+
consumerParameters: Any
|
|
51
|
+
additionalInformation: Any
|
|
52
|
+
name: Optional[Annotated[str, Field(frozen=True)]] = None
|
|
53
|
+
description: Optional[Annotated[str, Field(frozen=True)]] = None
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
class DDO(BaseModel):
|
|
57
|
+
"""DDO structure in Ocean Protocol"""
|
|
58
|
+
|
|
59
|
+
id: Annotated[str, Field(frozen=True)]
|
|
60
|
+
context: Annotated[List[str], Field(frozen=True)]
|
|
61
|
+
version: Annotated[str, Field(frozen=True)]
|
|
62
|
+
chainId: Annotated[int, Field(frozen=True)]
|
|
63
|
+
nftAddress: Annotated[str, Field(frozen=True)]
|
|
64
|
+
metadata: Annotated[Metadata, Field(frozen=True)]
|
|
65
|
+
services: Annotated[List[Service], Field(frozen=True)]
|
|
66
|
+
|
|
67
|
+
credentials: Annotated[Optional[str], Field(frozen=True)] = None
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
from ctypes import ArgumentError
|
|
1
2
|
import logging
|
|
2
3
|
import os
|
|
3
4
|
from typing import Any, Literal, Mapping, Optional
|
|
@@ -10,17 +11,13 @@ from oceanprotocol_job_details.loaders.loader import Loader
|
|
|
10
11
|
logging.basicConfig(
|
|
11
12
|
level=logging.INFO,
|
|
12
13
|
format="%(asctime)s [%(threadName)s] [%(levelname)s] %(message)s",
|
|
13
|
-
handlers=[
|
|
14
|
-
logging.StreamHandler(),
|
|
15
|
-
],
|
|
14
|
+
handlers=[logging.StreamHandler()],
|
|
16
15
|
)
|
|
17
16
|
|
|
18
|
-
_Implementations = Literal["
|
|
17
|
+
_Implementations = Literal["map"]
|
|
19
18
|
|
|
20
19
|
|
|
21
|
-
class OceanProtocolJobDetails(
|
|
22
|
-
Loader[JobDetails],
|
|
23
|
-
):
|
|
20
|
+
class OceanProtocolJobDetails(Loader[JobDetails]):
|
|
24
21
|
"""Decorator that loads the JobDetails from the given implementation"""
|
|
25
22
|
|
|
26
23
|
def __init__(
|
|
@@ -31,21 +28,33 @@ class OceanProtocolJobDetails(
|
|
|
31
28
|
*args,
|
|
32
29
|
**kwargs,
|
|
33
30
|
):
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
)
|
|
42
|
-
else:
|
|
43
|
-
raise NotImplementedError(f"Implementation {implementation} not supported")
|
|
44
|
-
|
|
45
|
-
def load(
|
|
46
|
-
self,
|
|
47
|
-
) -> JobDetails:
|
|
31
|
+
match implementation.lower():
|
|
32
|
+
case "map":
|
|
33
|
+
self._loader = lambda: Map(mapper=mapper, keys=keys, *args, **kwargs)
|
|
34
|
+
case _:
|
|
35
|
+
raise ArgumentError(f"Implementation {implementation} not valid")
|
|
36
|
+
|
|
37
|
+
def load(self) -> JobDetails:
|
|
48
38
|
return self._loader().load()
|
|
49
39
|
|
|
50
40
|
|
|
51
41
|
del _Implementations
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def _main():
|
|
45
|
+
"""Main function to test functionalities"""
|
|
46
|
+
|
|
47
|
+
# Re-define logging configuration
|
|
48
|
+
logging.basicConfig(
|
|
49
|
+
level=logging.DEBUG,
|
|
50
|
+
format="%(asctime)s [%(threadName)s] [%(levelname)s] %(message)s",
|
|
51
|
+
handlers=[logging.StreamHandler()],
|
|
52
|
+
force=True,
|
|
53
|
+
)
|
|
54
|
+
|
|
55
|
+
job_details = OceanProtocolJobDetails().load()
|
|
56
|
+
logging.info(f"Loaded job details: {job_details}")
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
if __name__ == "__main__":
|
|
60
|
+
_main()
|
|
@@ -1,21 +1,21 @@
|
|
|
1
1
|
"""Loads the current Job Details from the environment variables, could be abstracted to a more general 'mapper loader' but won't, since right now it fits our needs"""
|
|
2
2
|
|
|
3
|
-
from dataclasses import dataclass
|
|
4
|
-
from json import JSONDecodeError, load, loads
|
|
3
|
+
from dataclasses import dataclass
|
|
5
4
|
from logging import getLogger
|
|
6
5
|
from pathlib import Path
|
|
7
6
|
from typing import Mapping, Optional, Sequence, final
|
|
8
7
|
|
|
8
|
+
from orjson import JSONDecodeError, loads
|
|
9
|
+
|
|
9
10
|
from oceanprotocol_job_details.dataclasses.constants import DidKeys, Paths, ServiceType
|
|
10
11
|
from oceanprotocol_job_details.dataclasses.job_details import Algorithm, JobDetails
|
|
12
|
+
from oceanprotocol_job_details.loaders.impl.utils import do, execute_predicate
|
|
11
13
|
from oceanprotocol_job_details.loaders.loader import Loader
|
|
12
14
|
|
|
13
15
|
logger = getLogger(__name__)
|
|
14
16
|
|
|
15
17
|
|
|
16
|
-
@dataclass(
|
|
17
|
-
frozen=True,
|
|
18
|
-
)
|
|
18
|
+
@dataclass(frozen=True)
|
|
19
19
|
class Keys:
|
|
20
20
|
"""Environment keys passed to the algorithm"""
|
|
21
21
|
|
|
@@ -25,44 +25,42 @@ class Keys:
|
|
|
25
25
|
DIDS: str = "DIDS"
|
|
26
26
|
|
|
27
27
|
|
|
28
|
+
def _update_paths_from_root(root: Path):
|
|
29
|
+
"""Update the default from a root folder
|
|
30
|
+
|
|
31
|
+
:param root: root folder to update the paths
|
|
32
|
+
:type root: Path
|
|
33
|
+
"""
|
|
34
|
+
|
|
35
|
+
Paths.DATA = root / "data"
|
|
36
|
+
Paths.INPUTS = Paths.DATA / "inputs"
|
|
37
|
+
Paths.DDOS = Paths.DATA / "ddos"
|
|
38
|
+
Paths.OUTPUTS = Paths.DATA / "outputs"
|
|
39
|
+
Paths.LOGS = Paths.DATA / "logs"
|
|
40
|
+
Paths.ALGORITHM_CUSTOM_PARAMETERS = Paths.INPUTS / "algoCustomData.json"
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def _files_from_service(service):
|
|
44
|
+
return service[DidKeys.ATTRIBUTES][DidKeys.MAIN][DidKeys.FILES]
|
|
45
|
+
|
|
46
|
+
|
|
28
47
|
@final
|
|
29
|
-
class Map(
|
|
30
|
-
Loader[JobDetails],
|
|
31
|
-
):
|
|
48
|
+
class Map(Loader[JobDetails]):
|
|
32
49
|
"""Loads the current Job Details from the environment variables"""
|
|
33
50
|
|
|
34
|
-
def __init__(
|
|
35
|
-
self,
|
|
36
|
-
mapper: Mapping[str, str],
|
|
37
|
-
keys: Keys,
|
|
38
|
-
*args,
|
|
39
|
-
**kwargs,
|
|
40
|
-
) -> None:
|
|
41
|
-
super().__init__(*args, **kwargs)
|
|
42
|
-
|
|
51
|
+
def __init__(self, mapper: Mapping[str, str], keys: Keys, *args, **kwargs) -> None:
|
|
43
52
|
self._mapper = mapper
|
|
44
53
|
self._keys = keys
|
|
45
54
|
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
# Update the rest of paths
|
|
51
|
-
Paths.DATA = Path(root) / "data"
|
|
52
|
-
Paths.INPUTS = Paths.DATA / "inputs"
|
|
53
|
-
Paths.DDOS = Paths.DATA / "ddos"
|
|
54
|
-
Paths.OUTPUTS = Paths.DATA / "outputs"
|
|
55
|
-
Paths.LOGS = Paths.DATA / "logs"
|
|
56
|
-
Paths.ALGORITHM_CUSTOM_PARAMETERS = Paths.INPUTS / "algoCustomData.json"
|
|
57
|
-
|
|
55
|
+
execute_predicate(
|
|
56
|
+
lambda: _update_paths_from_root(Path(self._mapper[Keys.ROOT_FOLDER])),
|
|
57
|
+
lambda: Keys.ROOT_FOLDER in self._mapper,
|
|
58
|
+
)
|
|
58
59
|
|
|
59
|
-
def load(
|
|
60
|
-
self
|
|
61
|
-
*args,
|
|
62
|
-
**kwargs,
|
|
63
|
-
) -> JobDetails:
|
|
64
|
-
dids = self._dids()
|
|
60
|
+
def load(self, *args, **kwargs) -> JobDetails:
|
|
61
|
+
return self._from_dids(self._dids())
|
|
65
62
|
|
|
63
|
+
def _from_dids(self, dids: Sequence[Path]) -> JobDetails:
|
|
66
64
|
return JobDetails(
|
|
67
65
|
dids=dids,
|
|
68
66
|
files=self._files(dids),
|
|
@@ -70,65 +68,51 @@ class Map(
|
|
|
70
68
|
secret=self._secret(),
|
|
71
69
|
)
|
|
72
70
|
|
|
73
|
-
def _dids(
|
|
74
|
-
self,
|
|
75
|
-
) -> Sequence[Path]:
|
|
76
|
-
return (
|
|
77
|
-
loads(self._mapper.get(self._keys.DIDS))
|
|
78
|
-
if self._keys.DIDS in self._mapper
|
|
79
|
-
else []
|
|
80
|
-
)
|
|
71
|
+
def _dids(self) -> Sequence[Path]:
|
|
72
|
+
return loads(self._mapper.get(self._keys.DIDS, []))
|
|
81
73
|
|
|
82
|
-
def _files(
|
|
83
|
-
|
|
84
|
-
dids: Optional[Sequence[Path]],
|
|
85
|
-
) -> Mapping[str, Sequence[Path]]:
|
|
86
|
-
files: Mapping[str, Sequence[Path]] = {}
|
|
74
|
+
def _files(self, dids: Optional[Sequence[Path]]) -> Mapping[str, Sequence[Path]]:
|
|
75
|
+
"""Iterate through the given DIDs and retrieve their respective filepaths
|
|
87
76
|
|
|
77
|
+
:param dids: dids to read the files from
|
|
78
|
+
:type dids: Optional[Sequence[Path]]
|
|
79
|
+
:raises FileNotFoundError: if the DDO file does not exist
|
|
80
|
+
:return: _description_
|
|
81
|
+
:rtype: Mapping[str, Sequence[Path]]
|
|
82
|
+
"""
|
|
83
|
+
|
|
84
|
+
files: Mapping[str, Sequence[Path]] = {}
|
|
88
85
|
for did in dids:
|
|
89
|
-
#
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
with open(
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
except JSONDecodeError as e:
|
|
98
|
-
logger.warning(f"Error loading DDO file {file_path}: {e}")
|
|
86
|
+
# For each given DID, check if the DDO file exists and read its metadata
|
|
87
|
+
|
|
88
|
+
ddo_path = Paths.DDOS / did
|
|
89
|
+
do(lambda: ddo_path.exists(), exc=FileNotFoundError("Missing DDO file"))
|
|
90
|
+
|
|
91
|
+
with open(ddo_path, "r") as f:
|
|
92
|
+
ddo = do(lambda: loads(f.read()), JSONDecodeError)
|
|
93
|
+
if not ddo:
|
|
99
94
|
continue
|
|
100
95
|
|
|
101
|
-
for service in ddo[DidKeys.SERVICE]:
|
|
96
|
+
for service in do(lambda: ddo[DidKeys.SERVICE], KeyError, default=[]):
|
|
102
97
|
if service[DidKeys.SERVICE_TYPE] != ServiceType.METADATA:
|
|
103
|
-
continue
|
|
104
|
-
|
|
105
|
-
did_path = Paths.INPUTS / did
|
|
106
|
-
files[did] = [
|
|
107
|
-
did_path / str(idx)
|
|
108
|
-
for idx in range(
|
|
109
|
-
len(
|
|
110
|
-
service[DidKeys.ATTRIBUTES][DidKeys.MAIN][DidKeys.FILES]
|
|
111
|
-
)
|
|
112
|
-
)
|
|
113
|
-
]
|
|
98
|
+
continue # Only read the metadata of the services
|
|
114
99
|
|
|
100
|
+
files_n = do(lambda: len(_files_from_service(service)), KeyError)
|
|
101
|
+
ddo_path = Paths.INPUTS / did
|
|
102
|
+
files[did] = [ddo_path / str(idx) for idx in range(files_n)]
|
|
115
103
|
return files
|
|
116
104
|
|
|
117
|
-
def _algorithm(
|
|
118
|
-
self,
|
|
119
|
-
) -> Optional[Algorithm]:
|
|
105
|
+
def _algorithm(self) -> Optional[Algorithm]:
|
|
120
106
|
did = self._mapper.get(self._keys.ALGORITHM, None)
|
|
121
|
-
|
|
122
107
|
if not did:
|
|
123
108
|
return None
|
|
124
109
|
|
|
125
110
|
ddo = Paths.DDOS / did
|
|
126
|
-
if not ddo.exists():
|
|
127
|
-
raise FileNotFoundError(f"DDO file {ddo} does not exist")
|
|
128
111
|
|
|
129
|
-
return Algorithm(
|
|
112
|
+
return Algorithm(
|
|
113
|
+
did,
|
|
114
|
+
do(lambda: ddo.exists() and ddo, exc=FileNotFoundError("Missing DDO file")),
|
|
115
|
+
)
|
|
130
116
|
|
|
131
|
-
def _secret(
|
|
132
|
-
self,
|
|
133
|
-
) -> Optional[str]:
|
|
117
|
+
def _secret(self) -> Optional[str]:
|
|
134
118
|
return self._mapper.get(self._keys.SECRET, None)
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
from logging import WARNING, getLogger
|
|
2
|
+
from typing import Callable, TypeVar
|
|
3
|
+
|
|
4
|
+
logger = getLogger(__name__)
|
|
5
|
+
R = TypeVar("R")
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
def do(
|
|
9
|
+
function: Callable[[], R],
|
|
10
|
+
exception: Exception = Exception,
|
|
11
|
+
*,
|
|
12
|
+
log_level=WARNING,
|
|
13
|
+
default: R = None,
|
|
14
|
+
exc=False,
|
|
15
|
+
) -> R:
|
|
16
|
+
"""Executes a function and logs the exception if it fails
|
|
17
|
+
|
|
18
|
+
:param function: function to call
|
|
19
|
+
:type function: Callable
|
|
20
|
+
:param exception: exception to catch
|
|
21
|
+
:type exception: Exception
|
|
22
|
+
:param log_level: logging level to use
|
|
23
|
+
:type log_level: int
|
|
24
|
+
:param default: default value to return if the function fails
|
|
25
|
+
:type default: R
|
|
26
|
+
:param exc: if the exception should be raised
|
|
27
|
+
:type exc: bool
|
|
28
|
+
:return: result of the function and if it was successful
|
|
29
|
+
:rtype: R
|
|
30
|
+
"""
|
|
31
|
+
|
|
32
|
+
try:
|
|
33
|
+
return function()
|
|
34
|
+
except exception as e:
|
|
35
|
+
logger.log(log_level, e)
|
|
36
|
+
if exc:
|
|
37
|
+
if isinstance(exc, Exception):
|
|
38
|
+
raise exc from e
|
|
39
|
+
raise e
|
|
40
|
+
return default
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def execute_predicate(
|
|
44
|
+
function: Callable[[], R],
|
|
45
|
+
predicate: Callable[[], bool],
|
|
46
|
+
) -> R | bool:
|
|
47
|
+
"""Executes a function if the predicate is true"
|
|
48
|
+
|
|
49
|
+
:param function: function to call
|
|
50
|
+
:type function: Callable
|
|
51
|
+
:param predicate: predicate to check
|
|
52
|
+
:type predicate: Callable
|
|
53
|
+
:return: result of the function and if it was successful
|
|
54
|
+
:rtype: R | bool
|
|
55
|
+
"""
|
|
56
|
+
|
|
57
|
+
return predicate() and function()
|
|
@@ -5,16 +5,9 @@ from typing import Generic, TypeVar
|
|
|
5
5
|
T = TypeVar("T")
|
|
6
6
|
|
|
7
7
|
|
|
8
|
-
class Loader(
|
|
9
|
-
ABC,
|
|
10
|
-
Generic[T],
|
|
11
|
-
):
|
|
8
|
+
class Loader(ABC, Generic[T]):
|
|
12
9
|
@abstractmethod
|
|
13
|
-
def load(
|
|
14
|
-
self,
|
|
15
|
-
*args,
|
|
16
|
-
**kwargs,
|
|
17
|
-
) -> T:
|
|
10
|
+
def load(self, *args, **kwargs) -> T:
|
|
18
11
|
"""Load an instance of the given type"""
|
|
19
12
|
pass
|
|
20
13
|
|
{oceanprotocol_job_details-0.0.9.dist-info → oceanprotocol_job_details-0.0.11.dist-info}/METADATA
RENAMED
|
@@ -1,23 +1,25 @@
|
|
|
1
|
-
Metadata-Version: 2.
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
2
|
Name: oceanprotocol-job-details
|
|
3
|
-
Version: 0.0.
|
|
3
|
+
Version: 0.0.11
|
|
4
4
|
Summary: A Python package to get details from OceanProtocol jobs
|
|
5
|
-
License: Copyright 2025 Agrospai
|
|
6
|
-
|
|
7
|
-
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
|
|
8
|
-
|
|
9
|
-
The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
|
|
10
|
-
|
|
11
|
-
THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
|
12
|
-
Author: Christian López García
|
|
13
|
-
Author-email: christian.lopez@udl.cat
|
|
14
|
-
Requires-Python: >=3.9
|
|
15
|
-
Classifier: Programming Language :: Python :: 3
|
|
16
|
-
Classifier: Operating System :: OS Independent
|
|
17
|
-
Classifier: License :: OSI Approved :: MIT License
|
|
18
|
-
Requires-Dist: pytest (>=8.3.4,<9.0.0)
|
|
19
5
|
Project-URL: Homepage, https://github.com/AgrospAI/oceanprotocol-job-details
|
|
20
6
|
Project-URL: Issues, https://github.com/AgrospAI/oceanprotocol-job-details/issues
|
|
7
|
+
Author-email: Christian López García <christian.lopez@udl.cat>
|
|
8
|
+
License: Copyright 2025 Agrospai
|
|
9
|
+
|
|
10
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
|
|
13
|
+
|
|
14
|
+
THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
|
15
|
+
License-File: LICENSE
|
|
16
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
17
|
+
Classifier: Operating System :: OS Independent
|
|
18
|
+
Classifier: Programming Language :: Python :: 3
|
|
19
|
+
Requires-Python: >=3.10
|
|
20
|
+
Requires-Dist: orjson>=3.10.15
|
|
21
|
+
Requires-Dist: pydantic>=2.10.6
|
|
22
|
+
Requires-Dist: pytest<9,>=8.3.4
|
|
21
23
|
Description-Content-Type: text/markdown
|
|
22
24
|
|
|
23
25
|
A Python package to get details from OceanProtocol jobs
|
|
@@ -50,6 +52,15 @@ Assumes the following directory structure:
|
|
|
50
52
|
└───logs
|
|
51
53
|
```
|
|
52
54
|
|
|
55
|
+
### Core functionalities
|
|
56
|
+
|
|
57
|
+
Given the Ocean Protocol job details structure as in [https://github.com/GX4FM-Base-X/pontus-x-ontology](Pontus-X Ontology), parses the passed algorithm parameters into an object to use in your algorithms.
|
|
58
|
+
|
|
59
|
+
1. Parsing JSON
|
|
60
|
+
1. Validation
|
|
61
|
+
1. Metadata and service extraction
|
|
62
|
+
|
|
63
|
+
|
|
53
64
|
### Advanced Usage (not recommended)
|
|
54
65
|
|
|
55
66
|
If instead of the environment variables, we want to use another kind of mapping, can pass it as a parameter and it will work as long as it has the same key values (Can be implemented in a more generic way, but there is no need right now).
|
|
@@ -69,3 +80,4 @@ custom_mapper = {
|
|
|
69
80
|
job_details = OceanProtocolJobDetails(mapper=custom_mapper).load()
|
|
70
81
|
```
|
|
71
82
|
|
|
83
|
+
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
oceanprotocol_job_details/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
2
|
+
oceanprotocol_job_details/job_details.py,sha256=dlQABSV8TQf66vaeFImyw7c4Nms417pRKiDR9d4qDhY,1668
|
|
3
|
+
oceanprotocol_job_details/dataclasses/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
4
|
+
oceanprotocol_job_details/dataclasses/constants.py,sha256=EHUhbaFpixtK5Kq2V92gbWg-jxYR-tO6ie4Ntb_FPGQ,836
|
|
5
|
+
oceanprotocol_job_details/dataclasses/job_details.py,sha256=jho4PQLyqh1iB6e5wlM9ATSZreqAo7J9hOeFdETSHgo,2311
|
|
6
|
+
oceanprotocol_job_details/dataclasses/ocean.py,sha256=u4Mc5cHPnJz9oc1a43kkWfwEqRgsFsb-Y3UP2eBpoN4,2480
|
|
7
|
+
oceanprotocol_job_details/loaders/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
8
|
+
oceanprotocol_job_details/loaders/loader.py,sha256=JwR6OSkzIQQkeeyAU-ad_F89W9WNvoRwvHQY7Q3zIXI,256
|
|
9
|
+
oceanprotocol_job_details/loaders/impl/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
10
|
+
oceanprotocol_job_details/loaders/impl/map.py,sha256=5DF5HilWhmF_lq4j3VdAbyCbGRPGQPkn_-l7K-TOVmM,4090
|
|
11
|
+
oceanprotocol_job_details/loaders/impl/utils.py,sha256=rxoG8PrXRSKKDYPZh-wkWOXzMfdWXrkZs0f04OfDCIs,1446
|
|
12
|
+
oceanprotocol_job_details-0.0.11.dist-info/METADATA,sha256=nuTtra3JG0LP5wcG5PkdC5fdYGirZbQhDE6TRAdajlw,3382
|
|
13
|
+
oceanprotocol_job_details-0.0.11.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
|
|
14
|
+
oceanprotocol_job_details-0.0.11.dist-info/licenses/LICENSE,sha256=ni3ix7P_GxK1W3VGC4fJ3o6QoCngCEpSuTJwO4nkpbw,1055
|
|
15
|
+
oceanprotocol_job_details-0.0.11.dist-info/RECORD,,
|
|
@@ -1,13 +0,0 @@
|
|
|
1
|
-
oceanprotocol_job_details/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
2
|
-
oceanprotocol_job_details/dataclasses/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
3
|
-
oceanprotocol_job_details/dataclasses/constants.py,sha256=4bUtp5JA4Sp22CW7L3oLrGG2SmYpAEOCXd59k5rKFyg,850
|
|
4
|
-
oceanprotocol_job_details/dataclasses/job_details.py,sha256=iH52jd6XCplP2Z40qDO0lbFCTm7suVNPyLVgYkeOtQ0,2640
|
|
5
|
-
oceanprotocol_job_details/job_details.py,sha256=E5z_4ZlvSJSi1IJslSTi3X36UI_FHeaZkOyfCxxeJXw,1366
|
|
6
|
-
oceanprotocol_job_details/loaders/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
7
|
-
oceanprotocol_job_details/loaders/impl/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
8
|
-
oceanprotocol_job_details/loaders/impl/map.py,sha256=c3GNrLk14Hc3vu8OJFwcUb_23ZVzbkG3iaJ4rVi4sZs,3861
|
|
9
|
-
oceanprotocol_job_details/loaders/loader.py,sha256=ZedS7sA3K3uAIvc-yLRjgHIAen6_XyHO6sXhqm55kYo,298
|
|
10
|
-
oceanprotocol_job_details-0.0.9.dist-info/LICENSE,sha256=ni3ix7P_GxK1W3VGC4fJ3o6QoCngCEpSuTJwO4nkpbw,1055
|
|
11
|
-
oceanprotocol_job_details-0.0.9.dist-info/METADATA,sha256=Zd2nclEtjLf62kw2GTNNqI9Z0FDsQSfESePhoRaIQ0E,3013
|
|
12
|
-
oceanprotocol_job_details-0.0.9.dist-info/WHEEL,sha256=IYZQI976HJqqOpQU6PHkJ8fb3tMNBFjg-Cn-pwAbaFM,88
|
|
13
|
-
oceanprotocol_job_details-0.0.9.dist-info/RECORD,,
|
|
File without changes
|