oceanprotocol-job-details 0.2.5__py3-none-any.whl → 0.3.3__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,4 +1,4 @@
1
+ from .helpers import create_container, load_job_details
1
2
  from .ocean import JobDetails
2
3
 
3
-
4
- __all__ = [JobDetails]
4
+ __all__ = [JobDetails, load_job_details, create_container] # type: ignore
@@ -1,47 +1,39 @@
1
+ from typing import Generic, TypeVar
1
2
  from dependency_injector import containers, providers
3
+ from pydantic import BaseModel
2
4
 
3
5
  from oceanprotocol_job_details.loaders.impl.ddo import DDOLoader
4
6
  from oceanprotocol_job_details.loaders.impl.files import FilesLoader
5
7
  from oceanprotocol_job_details.loaders.impl.job_details import JobDetailsLoader
6
- from oceanprotocol_job_details.paths import Paths
8
+ from oceanprotocol_job_details.domain import Paths
7
9
 
8
10
 
9
- class Container(containers.DeclarativeContainer):
11
+ InputParametersT = TypeVar("InputParametersT", bound=BaseModel)
10
12
 
13
+
14
+ class Container(containers.DeclarativeContainer, Generic[InputParametersT]):
11
15
  config = providers.Configuration()
12
16
 
13
- paths = providers.Singleton(
14
- Paths,
15
- base_dir=config.base_dir,
16
- )
17
+ paths = providers.Singleton(Paths, base_dir=config.base_dir)
17
18
 
18
- file_loader = providers.Factory(
19
+ file_loader = providers.Singleton(
19
20
  FilesLoader,
20
21
  dids=config.dids,
21
22
  transformation_did=config.transformation_did,
22
23
  paths=paths,
24
+ logger=config.logger,
23
25
  )
24
26
 
25
- files = providers.Factory(
26
- lambda loader: loader.load(),
27
- loader=file_loader,
28
- )
29
-
30
- # DDOLoader depends on Files loaded from FilesLoader
31
- ddo_loader = providers.Factory(
32
- DDOLoader,
33
- files=files,
34
- )
35
-
36
- ddos = providers.Factory(
37
- lambda loader: loader.load(),
38
- loader=ddo_loader,
39
- )
40
-
41
- job_details_loader = providers.Factory(
42
- JobDetailsLoader,
43
- files=files,
44
- secret=config.secret,
45
- paths=paths,
46
- ddos=ddos,
27
+ files = providers.Factory(lambda loader: loader.load(), loader=file_loader)
28
+ ddo_loader = providers.Factory(DDOLoader, files=files)
29
+ ddos = providers.Factory(lambda loader: loader.load(), loader=ddo_loader)
30
+
31
+ job_details_loader: providers.Factory[JobDetailsLoader[InputParametersT]] = (
32
+ providers.Factory(
33
+ JobDetailsLoader,
34
+ files=files,
35
+ secret=config.secret,
36
+ paths=paths,
37
+ ddos=ddos,
38
+ )
47
39
  )
@@ -0,0 +1,187 @@
1
+ # mypy: disable-error-code=explicit-any
2
+ from dataclasses import InitVar, dataclass, field
3
+ from pathlib import Path
4
+ from typing import Generator, List, Optional, Sequence, TypeAlias, TypeVar
5
+
6
+ from pydantic import BaseModel, ConfigDict, Field, JsonValue
7
+
8
+
9
+ class Credential(BaseModel):
10
+ type: str
11
+ values: list[str]
12
+
13
+
14
+ class Credentials(BaseModel):
15
+ allow: list[Credential]
16
+ deny: list[Credential]
17
+
18
+
19
+ class DockerContainer(BaseModel):
20
+ image: str
21
+ tag: str
22
+ entrypoint: str
23
+
24
+
25
+ class Algorithm(BaseModel):
26
+ container: DockerContainer
27
+ language: str
28
+ version: str
29
+ consumerParameters: JsonValue
30
+
31
+
32
+ class Metadata(BaseModel):
33
+ description: str
34
+ name: str
35
+ type: str
36
+ author: str
37
+ license: str
38
+ algorithm: Optional[Algorithm] = None
39
+ tags: Optional[list[str]] = None
40
+ created: Optional[str] = None
41
+ updated: Optional[str] = None
42
+ copyrightHolder: Optional[str] = None
43
+ links: Optional[list[str]] = None
44
+ contentLanguage: Optional[str] = None
45
+ categories: Optional[list[str]] = None
46
+
47
+
48
+ class ConsumerParameters(BaseModel):
49
+ name: str
50
+ type: str
51
+ label: str
52
+ required: bool
53
+ description: str
54
+ default: str
55
+ option: Optional[list[str]] = None
56
+
57
+
58
+ class Service(BaseModel):
59
+ id: str
60
+ type: str
61
+ timeout: int
62
+ files: str
63
+ datatokenAddress: str
64
+ serviceEndpoint: str
65
+ additionalInformation: Optional[str] = None
66
+ name: Optional[str] = None
67
+ description: Optional[str] = None
68
+
69
+
70
+ class Event(BaseModel):
71
+ tx: str
72
+ block: int
73
+ from_: str = Field(alias="from")
74
+ contract: str
75
+ datetime: str
76
+
77
+ model_config = ConfigDict(populate_by_name=True)
78
+
79
+
80
+ class NFT(BaseModel):
81
+ address: str
82
+ name: str
83
+ symbol: str
84
+ state: int
85
+ tokenURI: str
86
+ owner: str
87
+ created: str
88
+
89
+
90
+ class DataToken(BaseModel):
91
+ address: str
92
+ name: str
93
+ symbol: str
94
+ serviceId: str
95
+
96
+
97
+ class Price(BaseModel):
98
+ value: int
99
+
100
+
101
+ class Stats(BaseModel):
102
+ allocated: int
103
+ orders: int
104
+ price: Price
105
+
106
+
107
+ class Purgatory(BaseModel):
108
+ state: bool
109
+
110
+
111
+ class DDO(BaseModel):
112
+ id: str
113
+ context: list[str] = Field(alias="@context")
114
+ nftAddress: str
115
+ chainId: int
116
+ version: str
117
+ metadata: Metadata
118
+ services: list[Service]
119
+ credentials: Credentials
120
+ event: Event
121
+ nft: NFT
122
+ datatokens: list[DataToken]
123
+ stats: Stats
124
+ purgatory: Purgatory
125
+
126
+ model_config = ConfigDict(populate_by_name=True)
127
+
128
+
129
+ @dataclass(frozen=True)
130
+ class DIDPaths:
131
+ did: str
132
+ ddo: Path = field(repr=False)
133
+
134
+ files: InitVar[Generator[Path, None, None]]
135
+
136
+ _input: List[Path] = field(init=False, repr=False)
137
+
138
+ def __post_init__(self, files: Generator[Path, None, None]) -> None:
139
+ assert self.ddo.exists(), f"DDO {self.ddo} does not exist"
140
+
141
+ object.__setattr__(self, "_input", list(files))
142
+
143
+ @property
144
+ def input_files(self) -> List[Path]:
145
+ return self._input
146
+
147
+ def __len__(self) -> int:
148
+ return len(self._input)
149
+
150
+
151
+ Files: TypeAlias = Sequence[DIDPaths]
152
+
153
+
154
+ @dataclass(frozen=True)
155
+ class Paths:
156
+ """Configuration class for the Ocean Protocol Job Details"""
157
+
158
+ base_dir: InitVar[Path | None] = None
159
+
160
+ _base: Path = field(init=False, repr=False)
161
+
162
+ def __post_init__(self, base_dir: Path | None) -> None:
163
+ object.__setattr__(self, "_base", base_dir if base_dir else Path("/data"))
164
+
165
+ @property
166
+ def data(self) -> Path:
167
+ return self._base
168
+
169
+ @property
170
+ def inputs(self) -> Path:
171
+ return self.data / "inputs"
172
+
173
+ @property
174
+ def ddos(self) -> Path:
175
+ return self.data / "ddos"
176
+
177
+ @property
178
+ def outputs(self) -> Path:
179
+ return self.data / "outputs"
180
+
181
+ @property
182
+ def logs(self) -> Path:
183
+ return self.data / "logs"
184
+
185
+ @property
186
+ def algorithm_custom_parameters(self) -> Path:
187
+ return self.inputs / "algoCustomData.json"
@@ -0,0 +1,29 @@
1
+ from typing import Any, Dict, Type, TypeVar
2
+
3
+ from pydantic import BaseModel, JsonValue
4
+
5
+ from oceanprotocol_job_details.di import Container
6
+ from oceanprotocol_job_details.ocean import JobDetails
7
+ from oceanprotocol_job_details.settings import JobSettings
8
+
9
+ InputParametersT = TypeVar("InputParametersT", bound=BaseModel)
10
+
11
+
12
+ def create_container(config: Dict[str, Any]) -> Container[InputParametersT]: # type: ignore[explicit-any]
13
+ """Return a fully configured Container from a config dict."""
14
+ container = Container[InputParametersT]()
15
+ settings = JobSettings(**config)
16
+ container.config.from_pydantic(settings)
17
+ return container
18
+
19
+
20
+ def load_job_details(
21
+ config: Dict[str, JsonValue],
22
+ input_type: Type[InputParametersT],
23
+ ) -> JobDetails[InputParametersT]:
24
+ """
25
+ Load JobDetails for a given input_type using the config.
26
+ Returns a fully initialized JobDetails instance.
27
+ """
28
+ container: Container[InputParametersT] = create_container(config)
29
+ return container.job_details_loader(input_type=input_type).load()
@@ -2,31 +2,23 @@ from __future__ import annotations
2
2
 
3
3
  from dataclasses import InitVar, dataclass, field
4
4
  from pathlib import Path
5
- from typing import TYPE_CHECKING, final
5
+ from typing import final
6
6
 
7
- if TYPE_CHECKING:
8
- from oceanprotocol_job_details.ocean import DDO, Files
7
+ from oceanprotocol_job_details.domain import DDO, Files
9
8
 
10
9
 
11
10
  @final
12
11
  @dataclass(frozen=True)
13
12
  class DDOLoader:
14
-
15
- files: InitVar[list[Files]]
13
+ files: InitVar[Files]
16
14
  """The files to load the DDOs from"""
17
15
 
18
16
  _ddo_paths: list[Path] = field(init=False)
19
17
 
20
- def __post_init__(self, files: list[Files]) -> None:
21
- assert files, "Missing files"
18
+ def __post_init__(self, files: Files) -> None:
19
+ assert files is not None and len(files) != 0, "Missing files"
22
20
 
23
21
  object.__setattr__(self, "_ddo_paths", [f.ddo for f in files])
24
22
 
25
23
  def load(self) -> list[DDO]:
26
- from oceanprotocol_job_details.ocean import DDO
27
-
28
- ddos = []
29
- for path in self._ddo_paths:
30
- with open(path, "r") as f:
31
- ddos.append(DDO.from_json(f.read())) # type: ignore
32
- return ddos
24
+ return [DDO.model_validate_json(p.read_text()) for p in self._ddo_paths]
@@ -1,54 +1,44 @@
1
1
  from __future__ import annotations
2
2
 
3
- import json
4
3
  from dataclasses import InitVar, dataclass, field
5
- from typing import TYPE_CHECKING, Sequence, final
4
+ from logging import Logger
5
+ from pathlib import Path
6
+ from typing import Literal, final
6
7
 
7
- from oceanprotocol_job_details.paths import Paths
8
-
9
- if TYPE_CHECKING:
10
- from oceanprotocol_job_details.ocean import DIDPaths, Files
8
+ from oceanprotocol_job_details.domain import DIDPaths, Files, Paths
11
9
 
12
10
 
13
11
  @final
14
12
  @dataclass(frozen=True)
15
13
  class FilesLoader:
14
+ paths: Paths
15
+ """Path configurations of the project"""
16
16
 
17
- dids: InitVar[str | None]
17
+ logger: Logger = field(repr=False)
18
+ """Logger to use"""
19
+
20
+ dids: list[str]
18
21
  """Input DIDs"""
19
22
 
20
- transformation_did: InitVar[str | None]
23
+ transformation_did: str
21
24
  """DID for the transformation algorithm"""
22
25
 
23
- paths: Paths
24
- """Path configurations of the project"""
25
-
26
- _dids: Sequence[str] = field(init=False)
27
- _transformation_did: str = field(init=False)
26
+ def __post_init__(self) -> None:
27
+ assert self.dids, "Missing input DIDs"
28
28
 
29
- def __post_init__(
30
- self,
31
- dids: str | None,
32
- transformation_did: str | None,
33
- ) -> None:
34
- assert dids, "Missing DIDs"
35
- assert transformation_did, "Missing transformation DID"
36
-
37
- object.__setattr__(self, "_dids", json.loads(dids))
38
- object.__setattr__(self, "_transformation_did", transformation_did)
29
+ def calculate_path(self, did: str, path_type: Literal["input", "ddo"]) -> Path:
30
+ match path_type:
31
+ case "ddo":
32
+ return self.paths.ddos / did
33
+ case "input":
34
+ return self.paths.inputs / did
39
35
 
40
36
  def load(self) -> Files:
41
- from oceanprotocol_job_details.ocean import DIDPaths, Files
42
-
43
- files: list[DIDPaths] = []
44
- for did in self._dids:
45
- base = self.paths.inputs / did
46
- files.append(
47
- DIDPaths(
48
- did=did,
49
- ddo=self.paths.ddos / did,
50
- input_files=list(base.iterdir()),
51
- )
37
+ return [
38
+ DIDPaths(
39
+ did=did,
40
+ ddo=self.calculate_path(did, "ddo"),
41
+ files=self.calculate_path(did, "input").iterdir(),
52
42
  )
53
-
54
- return Files(files)
43
+ for did in self.dids
44
+ ]
@@ -1,35 +1,28 @@
1
- from __future__ import annotations
2
-
3
1
  from dataclasses import dataclass, field
4
- from typing import TYPE_CHECKING, Generic, Type, TypeVar, final
5
-
6
- from oceanprotocol_job_details.paths import Paths
2
+ from typing import Generic, Type, TypeVar, final
7
3
 
8
- if TYPE_CHECKING:
9
- from oceanprotocol_job_details.ocean import DDO, Files, JobDetails
4
+ from pydantic import BaseModel
10
5
 
6
+ from oceanprotocol_job_details.domain import DDO, Files, Paths
7
+ from oceanprotocol_job_details.ocean import JobDetails
11
8
 
12
- T = TypeVar("T")
9
+ T = TypeVar("T", bound=BaseModel)
13
10
 
14
11
 
15
12
  @final
16
13
  @dataclass(frozen=True)
17
14
  class JobDetailsLoader(Generic[T]):
18
-
19
- _type: Type[T] = field(repr=False)
20
-
15
+ input_type: Type[T] = field(repr=False)
21
16
  files: Files
22
17
  secret: str
23
18
  paths: Paths
24
19
  ddos: list[DDO]
25
20
 
26
21
  def load(self) -> JobDetails[T]:
27
- from oceanprotocol_job_details.ocean import JobDetails
28
-
29
- return JobDetails(
22
+ return JobDetails[T](
30
23
  files=self.files,
31
24
  secret=self.secret,
32
25
  ddos=self.ddos,
33
26
  paths=self.paths,
34
- _type=self._type,
27
+ input_type=self.input_type,
35
28
  )
@@ -4,10 +4,6 @@ T = TypeVar("T", covariant=True)
4
4
 
5
5
 
6
6
  class Loader(Protocol[T]):
7
-
8
7
  def load(self) -> T:
9
8
  """Load an instance of the given type"""
10
9
  ...
11
-
12
-
13
- del T
@@ -1,319 +1,51 @@
1
1
  from __future__ import annotations
2
2
 
3
- import logging
4
- import os
5
- from dataclasses import dataclass, field
3
+ import asyncio
6
4
  from functools import cached_property
7
5
  from pathlib import Path
8
- from typing import (
9
- Any,
10
- Generator,
11
- Generic,
12
- Iterator,
13
- Optional,
14
- Sequence,
15
- Type,
16
- TypeVar,
17
- final,
18
- )
6
+ from typing import Generator, Generic, Tuple, Type, TypeVar, final
19
7
 
20
- import orjson
21
- from dataclasses_json import config as dc_config
22
- from dataclasses_json import dataclass_json
8
+ import aiofiles
9
+ from pydantic import BaseModel, ConfigDict, Secret, model_validator
23
10
 
24
- from oceanprotocol_job_details.di import Container
25
- from oceanprotocol_job_details.paths import Paths
11
+ from oceanprotocol_job_details.domain import DDO, Files, Paths
26
12
 
27
- T = TypeVar("T")
28
-
29
- logging.basicConfig(
30
- level=logging.INFO,
31
- format="%(asctime)s [%(threadName)s] [%(levelname)s] %(message)s",
32
- handlers=[logging.StreamHandler()],
33
- )
34
- logger = logging.getLogger(__name__)
35
-
36
-
37
- @dataclass_json
38
- @dataclass
39
- class Credential:
40
- type: str
41
- values: list[str]
42
-
43
-
44
- @dataclass_json
45
- @dataclass
46
- class Credentials:
47
- allow: list[Credential]
48
- deny: list[Credential]
49
-
50
-
51
- @dataclass_json
52
- @dataclass
53
- class DockerContainer:
54
- image: str
55
- tag: str
56
- entrypoint: str
57
-
58
-
59
- @dataclass_json
60
- @dataclass
61
- class Algorithm: # type: ignore
62
- container: DockerContainer
63
- language: str
64
- version: str
65
- consumerParameters: Any # type: ignore
66
-
67
-
68
- @dataclass_json
69
- @dataclass
70
- class Metadata:
71
- description: str
72
- name: str
73
- type: str
74
- author: str
75
- license: str
76
- algorithm: Optional[Algorithm] = None
77
- tags: Optional[list[str]] = None
78
- created: Optional[str] = None
79
- updated: Optional[str] = None
80
- copyrightHolder: Optional[str] = None
81
- links: Optional[list[str]] = None
82
- contentLanguage: Optional[str] = None
83
- categories: Optional[list[str]] = None
84
-
85
-
86
- @dataclass_json
87
- @dataclass
88
- class ConsumerParameters:
89
- name: str
90
- type: str
91
- label: str
92
- required: bool
93
- description: str
94
- default: str
95
- option: Optional[list[str]] = None
96
-
97
-
98
- @dataclass_json
99
- @dataclass
100
- class Service:
101
- id: str
102
- type: str
103
- timeout: int
104
- files: str
105
- datatokenAddress: str
106
- serviceEndpoint: str
107
- additionalInformation: Optional[str] = None
108
- name: Optional[str] = None
109
- description: Optional[str] = None
110
-
111
-
112
- @dataclass_json
113
- @dataclass
114
- class Event:
115
- tx: str
116
- block: int
117
- from_: str = field(metadata=dc_config(field_name="from"))
118
- contract: str
119
- datetime: str
120
-
121
-
122
- @dataclass_json
123
- @dataclass
124
- class NFT:
125
- address: str
126
- name: str
127
- symbol: str
128
- state: int
129
- tokenURI: str
130
- owner: str
131
- created: str
132
-
133
-
134
- @dataclass_json
135
- @dataclass
136
- class DataToken:
137
- address: str
138
- name: str
139
- symbol: str
140
- serviceId: str
141
-
142
-
143
- @dataclass_json
144
- @dataclass
145
- class Price:
146
- value: int
147
-
148
-
149
- @dataclass_json
150
- @dataclass
151
- class Stats:
152
- allocated: int
153
- orders: int
154
- price: Price
155
-
156
-
157
- @dataclass_json
158
- @dataclass
159
- class Purgatory:
160
- state: bool
161
-
162
-
163
- @dataclass_json
164
- @dataclass
165
- class DDO:
166
- id: str
167
- context: list[str] = field(metadata=dc_config(field_name="@context"))
168
- nftAddress: str
169
- chainId: int
170
- version: str
171
- metadata: Metadata
172
- services: list[Service]
173
- credentials: Credentials
174
- event: Event
175
- nft: NFT
176
- datatokens: list[DataToken]
177
- stats: Stats
178
- purgatory: Purgatory
179
-
180
-
181
- @dataclass(frozen=True)
182
- class DIDPaths:
183
- did: str
184
- ddo: Path
185
- input_files: Sequence[Path]
186
-
187
- def __post_init__(self) -> None:
188
- assert self.ddo.exists(), f"DDO {self.ddo} does not exist"
189
- for input_file in self.input_files:
190
- assert input_file.exists(), f"File {input_file} does not exist"
191
-
192
- def __len__(self) -> int:
193
- return len(self.input_files)
194
-
195
-
196
- @dataclass(frozen=True)
197
- class Files:
198
- _files: Sequence[DIDPaths]
199
-
200
- @property
201
- def files(self) -> Sequence[DIDPaths]:
202
- return self._files
203
-
204
- def __getitem__(self, index: int) -> DIDPaths:
205
- return self.files[index]
206
-
207
- def __iter__(self) -> Iterator[DIDPaths]:
208
- return iter(self.files)
209
-
210
- def __len__(self) -> int:
211
- return len(self.files)
212
-
213
-
214
- def _normalize_json(value):
215
- if isinstance(value, str):
216
- try:
217
- decoded = orjson.loads(value)
218
- return _normalize_json(decoded) # recurse if nested again
219
- except orjson.JSONDecodeError:
220
- return value
221
- elif isinstance(value, dict):
222
- return {k: _normalize_json(v) for k, v in value.items()}
223
- elif isinstance(value, list):
224
- return [_normalize_json(v) for v in value]
225
- return value
226
-
227
-
228
- @final
229
- @dataclass_json
230
- @dataclass
231
- class _EmptyJobDetails: ...
13
+ InputParemetersT = TypeVar("InputParemetersT", bound=BaseModel)
232
14
 
233
15
 
234
16
  @final
235
- @dataclass_json
236
- @dataclass(frozen=True)
237
- class JobDetails(Generic[T]):
17
+ class JobDetails(BaseModel, Generic[InputParemetersT]): # type: ignore[explicit-any]
238
18
  files: Files
239
- """The input filepaths"""
240
-
241
19
  ddos: list[DDO]
242
- """list of paths to the DDOs"""
243
-
244
20
  paths: Paths
245
- """Configuration paths"""
21
+ input_type: Type[InputParemetersT]
22
+ secret: Secret[str] | None = None
246
23
 
247
- # Store the type explicitly to avoid issues
248
- _type: Type[T] = field(repr=False)
24
+ model_config = ConfigDict(arbitrary_types_allowed=True, frozen=True)
249
25
 
250
- secret: str | None = None
251
- """Shh it's a secret"""
252
-
253
- def __post_init__(self) -> None:
254
- if not hasattr(self._type, "__dataclass_fields__"):
255
- raise TypeError(f"{self._type} is not a dataclass type")
26
+ @model_validator(mode="after")
27
+ def validate_type(self) -> JobDetails[InputParemetersT]:
28
+ assert issubclass(self.input_type, BaseModel), (
29
+ f"{self.input_type} must be subtype of pydantic.BaseModel"
30
+ )
31
+ return self
256
32
 
257
- def next_path(self) -> Generator[tuple[int, Path], None, None]:
258
- for idx, did_files in enumerate(self.files):
259
- for file in did_files.input_files:
260
- yield (idx, file)
33
+ def inputs(self) -> Generator[Tuple[int, Path], None, None]:
34
+ yield from (
35
+ (idx, file)
36
+ for idx, files in enumerate(self.files)
37
+ for file in files.input_files
38
+ )
261
39
 
262
40
  @cached_property
263
- def input_parameters(self) -> T:
264
- """Read the input parameters and return them in an instance of the dataclass T"""
41
+ def input_parameters(self) -> InputParemetersT:
42
+ return asyncio.run(self.ainput_parameters())
265
43
 
266
- with open(self.paths.algorithm_custom_parameters, "r") as f:
267
- raw = f.read().strip()
268
- if not raw:
269
- raise ValueError(
270
- f"Custom parameters file {self.paths.algorithm_custom_parameters} is empty"
271
- )
272
- try:
273
- parsed = _normalize_json(orjson.loads(raw))
274
- return dataclass_json(self._type).from_dict(parsed) # type: ignore
275
- except Exception as e:
276
- raise ValueError(
277
- f"Failed to parse input paramers into {self._type.__name__}: {e}\n"
278
- f"Raw content: {raw}"
279
- ) from e
280
-
281
- @classmethod
282
- def load(
283
- cls,
284
- _type: Type[T] | None = None,
285
- *,
286
- base_dir: str | None = None,
287
- dids: str | None = None,
288
- transformation_did: str | None = None,
289
- secret: str | None = None,
290
- ) -> JobDetails[T]:
291
- """Load a JobDetails instance that holds the runtime details.
292
-
293
- Loading it will check the following:
294
- 1. That the needed environment variables are set.
295
- 1. That the ocean protocol contains the needed data based on the passed environment variables.
296
-
297
- Those needed environment variables are:
298
- - BASE_DIR: Base directory to read the data from, parent of the ddos, inputs, outputs and logs directories.
299
- - DIDS: The DIDs of the inputs
300
- - TRANSFORMATION_DID: The DID of the transformation algorithm
301
- - SECRET (optional): A really secret secret
302
-
303
- """
304
-
305
- if _type is None:
306
- _type = _EmptyJobDetails
307
-
308
- container = Container()
309
- container.config.from_dict(
310
- {
311
- "base_dir": base_dir or os.environ.get("BASE_DIR", None),
312
- "dids": dids or os.environ.get("DIDS"),
313
- "transformation_did": transformation_did
314
- or os.environ.get("TRANSFORMATION_DID"),
315
- "secret": secret or os.environ.get("SECRET"),
316
- }
317
- )
44
+ async def ainput_parameters(self) -> InputParemetersT:
45
+ path = self.paths.algorithm_custom_parameters
46
+ async with aiofiles.open(path) as f:
47
+ raw = await f.read()
318
48
 
319
- return container.job_details_loader(_type=_type).load()
49
+ raw = raw.strip()
50
+ assert raw is not None, f"Empty file {path}"
51
+ return self.input_type.model_validate_json(raw)
@@ -0,0 +1,37 @@
1
+ # mypy: disable-error-code=call-overload
2
+ from logging import Logger, getLogger
3
+ from pathlib import Path
4
+ from typing import Self
5
+
6
+ import orjson
7
+ from pydantic import Field, field_validator, model_validator
8
+ from pydantic_settings import BaseSettings, SettingsConfigDict
9
+
10
+
11
+ class JobSettings(BaseSettings): # type: ignore[explicit-any]
12
+ base_dir: Path = Field(alias="BASE_DIR")
13
+ dids: list[str] = Field(default_factory=list, alias="DIDS")
14
+ transformation_did: str = Field(alias="TRANSFORMATION_DID")
15
+ secret: str | None = Field(default=None, alias="SECRET")
16
+ logger: Logger = Field(default_factory=lambda: getLogger(__name__))
17
+
18
+ model_config = SettingsConfigDict(
19
+ extra="forbid",
20
+ populate_by_name=True,
21
+ arbitrary_types_allowed=True,
22
+ )
23
+
24
+ @field_validator("dids", mode="before")
25
+ @classmethod
26
+ def split_dids(cls, v: list[str] | str) -> list[str]:
27
+ if isinstance(v, str):
28
+ data = orjson.loads(v)
29
+ assert isinstance(data, list)
30
+ return data
31
+ return v
32
+
33
+ @model_validator(mode="after")
34
+ def validate_dids(self) -> Self:
35
+ if not self.dids:
36
+ self.dids.extend([f.name for f in (self.base_dir / "ddos").glob("*")])
37
+ return self
@@ -0,0 +1,114 @@
1
+ Metadata-Version: 2.4
2
+ Name: oceanprotocol-job-details
3
+ Version: 0.3.3
4
+ Summary: A Python package to get details from OceanProtocol jobs
5
+ Project-URL: Homepage, https://github.com/AgrospAI/oceanprotocol-job-details
6
+ Project-URL: Issues, https://github.com/AgrospAI/oceanprotocol-job-details/issues
7
+ Author-email: Agrospai <agrospai@udl.cat>, Christian López García <christian.lopez@udl.cat>
8
+ License: Copyright 2025 Agrospai
9
+
10
+ Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
13
+
14
+ THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
15
+ License-File: LICENSE
16
+ Classifier: License :: OSI Approved :: MIT License
17
+ Classifier: Operating System :: OS Independent
18
+ Classifier: Programming Language :: Python :: 3
19
+ Requires-Python: >=3.10
20
+ Requires-Dist: aiofiles>=25.1.0
21
+ Requires-Dist: dependency-injector>=4.48.2
22
+ Requires-Dist: orjson>=3.11.3
23
+ Requires-Dist: pydantic-settings>=2.12.0
24
+ Requires-Dist: pydantic>=2.12.5
25
+ Description-Content-Type: text/markdown
26
+
27
+ A Python package to get details from OceanProtocol jobs
28
+
29
+ ---
30
+
31
+ ## Installation
32
+
33
+ ```bash
34
+ pip install oceanprotocol-job-details
35
+ ```
36
+
37
+ ```bash
38
+ uv add oceanprotocol-job-details
39
+ ```
40
+
41
+ ## Usage
42
+
43
+ As a simple library, we only need to import `load_job_details` and run it. It will:
44
+
45
+ 1. Fetch the needed parameters to populate the `JobDetails` instance from the environment variables or use the passed values to the function.
46
+ 1. Look for the files corresponding to the passed DIDs in the filesystem according to the [Ocean Protocol Structure](#oceanprotocol-structure) and load them into the `JobDetails` instance.
47
+
48
+ ### Minimal Example
49
+
50
+ ```python
51
+ from oceanprotocol_job_details import load_job_details
52
+
53
+ class InputParameters(BaseModel): ...
54
+
55
+ job_details = load_job_details({}, InputParameters)
56
+ ```
57
+
58
+ ### Custom Input Parameters
59
+
60
+ If our algorithm has custom input parameters and we want to load them into our algorithm, we can do it as follows:
61
+
62
+ ```python
63
+ from pydantic import BaseModel
64
+ from oceanprotocol_job_details import load_job_details
65
+
66
+
67
+ class Foo(BaseModel):
68
+ bar: str
69
+
70
+
71
+ class InputParameters(BaseModel):
72
+ # Allows for nested types
73
+ foo: Foo
74
+
75
+
76
+ job_details = load_job_details({}, InputParameters)
77
+
78
+ # Usage
79
+ job_details.input_parameters.foo
80
+ job_details.input_parameters.foo.bar
81
+ ```
82
+
83
+ The values to fill the custom `InputParameters` will be parsed from the `algoCustomData.json` located next to the input data directories.
84
+
85
+ ### Iterating Input Files the clean way
86
+
87
+ ```python
88
+ from oceanprotocol_job_details import load_job_details
89
+
90
+
91
+ job_details = load_job_details
92
+
93
+ for idx, file_path in job_details.inputs():
94
+ ...
95
+
96
+ _, file_path = next(job_details.inputs())
97
+ ```
98
+
99
+ ## OceanProtocol Structure
100
+
101
+ ```bash
102
+ data # Root /data directory
103
+ ├── ddos # Contains the loaded dataset's DDO
104
+ │ ├── 17feb...e42 # DDO file
105
+ │ └── ... # One DDO per loaded dataset
106
+ ├── inputs # Datasets dir
107
+ │ ├── 17feb...e42 # Dir holding the data of its name DID, contains files named 0..X
108
+ │ │ └── 0 # Data file
109
+ │ └── algoCustomData.json # Custom algorithm input data
110
+ ├── logs # Algorithm output logs dir
111
+ └── outputs # Algorithm output files dir
112
+ ```
113
+
114
+ > **_Note:_** Even though it's possible that the algorithm is passed multiple datasets, right now the implementation only allows to use **one dataset** per algorithm execution, so **normally** the executing job will only have **one ddo**, **one dir** inside inputs, and **one data file** named `0`.
@@ -0,0 +1,16 @@
1
+ oceanprotocol_job_details/__init__.py,sha256=nJMrZsEC5F1n9WF-v5QV095Yyc8UkhFw0AzD9o7X0IE,162
2
+ oceanprotocol_job_details/di.py,sha256=lsogbmjvmPfkd0mjLvn9vYLIZebwJm5RNraWt7WE5LA,1316
3
+ oceanprotocol_job_details/domain.py,sha256=2_USbeA_7VIEYS8DVb2MW6dCZasjiqIxQaGUnNUKspY,3851
4
+ oceanprotocol_job_details/helpers.py,sha256=ubH_KjAROqYvn0mkbA0-89vpdKIhVNGZ0h9pQLfPNow,1045
5
+ oceanprotocol_job_details/ocean.py,sha256=q8rgT5ycA2ifey3XNhUW0bcJwfMp7hpKU-6EVDeKV1o,1620
6
+ oceanprotocol_job_details/settings.py,sha256=zgIYPzaXjsgcmuhT7L2ipSP-2eNaodugHZr0rn2Z420,1248
7
+ oceanprotocol_job_details/loaders/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
8
+ oceanprotocol_job_details/loaders/loader.py,sha256=36X2s_0lN89kCUpItxEXfIzuBBNJySebP2B_tdWK2E0,186
9
+ oceanprotocol_job_details/loaders/impl/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
10
+ oceanprotocol_job_details/loaders/impl/ddo.py,sha256=XthrQFhmP85XSVzVjBlLePtTowGR3BAsmVp3jngiQ08,668
11
+ oceanprotocol_job_details/loaders/impl/files.py,sha256=Y2vFBT2T9w9zrdpmf550-LQJxwtNPUGa0UU6bBzk9AU,1145
12
+ oceanprotocol_job_details/loaders/impl/job_details.py,sha256=7mEdeTo-cmsWuqWPdN7btjLjo6p5Oa1_acjSLWL5tb8,697
13
+ oceanprotocol_job_details-0.3.3.dist-info/METADATA,sha256=pDXdsInFTf36wYjsqWSd_ygeCDggozjueHlhF6OyvlQ,4416
14
+ oceanprotocol_job_details-0.3.3.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
15
+ oceanprotocol_job_details-0.3.3.dist-info/licenses/LICENSE,sha256=ni3ix7P_GxK1W3VGC4fJ3o6QoCngCEpSuTJwO4nkpbw,1055
16
+ oceanprotocol_job_details-0.3.3.dist-info/RECORD,,
@@ -1,4 +1,4 @@
1
1
  Wheel-Version: 1.0
2
- Generator: hatchling 1.27.0
2
+ Generator: hatchling 1.28.0
3
3
  Root-Is-Purelib: true
4
4
  Tag: py3-none-any
@@ -1,38 +0,0 @@
1
- from dataclasses import InitVar, dataclass, field
2
- from pathlib import Path
3
-
4
-
5
- @dataclass
6
- class Paths:
7
- """Configuration class for the Ocean Protocol Job Details"""
8
-
9
- base_dir: InitVar[Path | None]
10
-
11
- _base: Path = field(init=False)
12
-
13
- def __post_init__(self, base_dir: Path | None) -> None:
14
- self._base = base_dir or Path("/data")
15
-
16
- @property
17
- def data(self) -> Path:
18
- return self._base
19
-
20
- @property
21
- def inputs(self) -> Path:
22
- return self.data / "inputs"
23
-
24
- @property
25
- def ddos(self) -> Path:
26
- return self.data / "ddos"
27
-
28
- @property
29
- def outputs(self) -> Path:
30
- return self.data / "outputs"
31
-
32
- @property
33
- def logs(self) -> Path:
34
- return self.data / "logs"
35
-
36
- @property
37
- def algorithm_custom_parameters(self) -> Path:
38
- return self.inputs / "algoCustomData.json"
@@ -1,76 +0,0 @@
1
- Metadata-Version: 2.4
2
- Name: oceanprotocol-job-details
3
- Version: 0.2.5
4
- Summary: A Python package to get details from OceanProtocol jobs
5
- Project-URL: Homepage, https://github.com/AgrospAI/oceanprotocol-job-details
6
- Project-URL: Issues, https://github.com/AgrospAI/oceanprotocol-job-details/issues
7
- Author-email: Christian López García <christian.lopez@udl.cat>
8
- License: Copyright 2025 Agrospai
9
-
10
- Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
11
-
12
- The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
13
-
14
- THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
15
- License-File: LICENSE
16
- Classifier: License :: OSI Approved :: MIT License
17
- Classifier: Operating System :: OS Independent
18
- Classifier: Programming Language :: Python :: 3
19
- Requires-Python: >=3.10
20
- Requires-Dist: dataclasses-json>=0.6.7
21
- Requires-Dist: dependency-injector>=4.48.2
22
- Requires-Dist: orjson>=3.11.3
23
- Description-Content-Type: text/markdown
24
-
25
- A Python package to get details from OceanProtocol jobs
26
-
27
- ---
28
-
29
- ## Installation
30
-
31
- ```
32
- pip install oceanprotocol-job-details
33
- ```
34
-
35
- ## Usage
36
-
37
- As a simple library, we only need to import the main object and use it once:
38
-
39
- ```Python
40
- from oceanprotocol_job_details import JobDetails
41
-
42
- # Having no algorithm input parameters
43
- job_details = JobDetails.load()
44
-
45
- ```
46
-
47
- If our algorithm has custom input parameters and we want to load them into our algorithm, we can do it as follows:
48
-
49
- ```Python
50
-
51
- from dataclasses import dataclass
52
- from oceanprotocol_job_details import JobDetails
53
-
54
-
55
- @dataclass
56
- class InputParameters:
57
- name: str
58
- age: int
59
-
60
-
61
- job_details: JobDetails[InputParameters] = JobDetails.load(InputParameters)
62
-
63
- # Usage (is type hinted)
64
- job_details.input_parameters.name
65
- job_details.input_parameters.age
66
-
67
- ```
68
-
69
- Assumes the directory structure of OceanProtocol algorithms.
70
-
71
- ### Core functionalities
72
-
73
- Given the Ocean Protocol job details structure, parses the passed algorithm parameters into an object to use in your algorithms.
74
-
75
- 1. Input parameter JSON parsing and validation
76
- 1. Metadata and service extraction from the directory structure.
@@ -1,14 +0,0 @@
1
- oceanprotocol_job_details/__init__.py,sha256=C67wv7fy5ZT5FtbGD-oQeSoLU-x6e2ts-820koFo034,55
2
- oceanprotocol_job_details/di.py,sha256=j2BvpaGpiQwt9MSiViYPFup4MnzTHm44Zq12_TQ07kY,1180
3
- oceanprotocol_job_details/ocean.py,sha256=uABP0JZdCpd3NrTmfXWGuG8XOEVx680-jjRDV0hHync,7373
4
- oceanprotocol_job_details/paths.py,sha256=N90IF8OQCBELULGPaBv9yALApa-lC2dsVKXMWLdqa14,851
5
- oceanprotocol_job_details/loaders/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
6
- oceanprotocol_job_details/loaders/loader.py,sha256=HIzsVKCuGP7ghfM7ppN3ANVybvsA64wr3h8I68mqS6A,195
7
- oceanprotocol_job_details/loaders/impl/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
8
- oceanprotocol_job_details/loaders/impl/ddo.py,sha256=_xl0PozuvIm0n6jU--Znk6qfw1A6OLBq3SERbDGIF74,844
9
- oceanprotocol_job_details/loaders/impl/files.py,sha256=oFkA_0Ma5NBgWvVEk_rhyDIDrAam_zjesh0-bxZaIU8,1443
10
- oceanprotocol_job_details/loaders/impl/job_details.py,sha256=wf0xNAG4tESq57vqkdtMQ8BdiyS91j5f7FL8Gfwbjh4,770
11
- oceanprotocol_job_details-0.2.5.dist-info/METADATA,sha256=o5sx14PqA0kau6UYfYy_NMGhatGf0cWcaKhS7HGDbWw,2948
12
- oceanprotocol_job_details-0.2.5.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
13
- oceanprotocol_job_details-0.2.5.dist-info/licenses/LICENSE,sha256=ni3ix7P_GxK1W3VGC4fJ3o6QoCngCEpSuTJwO4nkpbw,1055
14
- oceanprotocol_job_details-0.2.5.dist-info/RECORD,,