oceanprotocol-job-details 0.2.5__py3-none-any.whl → 0.3.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- oceanprotocol_job_details/__init__.py +2 -2
- oceanprotocol_job_details/di.py +21 -29
- oceanprotocol_job_details/domain.py +187 -0
- oceanprotocol_job_details/helpers.py +29 -0
- oceanprotocol_job_details/loaders/impl/ddo.py +6 -14
- oceanprotocol_job_details/loaders/impl/files.py +26 -36
- oceanprotocol_job_details/loaders/impl/job_details.py +8 -15
- oceanprotocol_job_details/loaders/loader.py +0 -4
- oceanprotocol_job_details/ocean.py +31 -299
- oceanprotocol_job_details/settings.py +37 -0
- oceanprotocol_job_details-0.3.3.dist-info/METADATA +114 -0
- oceanprotocol_job_details-0.3.3.dist-info/RECORD +16 -0
- {oceanprotocol_job_details-0.2.5.dist-info → oceanprotocol_job_details-0.3.3.dist-info}/WHEEL +1 -1
- oceanprotocol_job_details/paths.py +0 -38
- oceanprotocol_job_details-0.2.5.dist-info/METADATA +0 -76
- oceanprotocol_job_details-0.2.5.dist-info/RECORD +0 -14
- {oceanprotocol_job_details-0.2.5.dist-info → oceanprotocol_job_details-0.3.3.dist-info}/licenses/LICENSE +0 -0
oceanprotocol_job_details/di.py
CHANGED
|
@@ -1,47 +1,39 @@
|
|
|
1
|
+
from typing import Generic, TypeVar
|
|
1
2
|
from dependency_injector import containers, providers
|
|
3
|
+
from pydantic import BaseModel
|
|
2
4
|
|
|
3
5
|
from oceanprotocol_job_details.loaders.impl.ddo import DDOLoader
|
|
4
6
|
from oceanprotocol_job_details.loaders.impl.files import FilesLoader
|
|
5
7
|
from oceanprotocol_job_details.loaders.impl.job_details import JobDetailsLoader
|
|
6
|
-
from oceanprotocol_job_details.
|
|
8
|
+
from oceanprotocol_job_details.domain import Paths
|
|
7
9
|
|
|
8
10
|
|
|
9
|
-
|
|
11
|
+
InputParametersT = TypeVar("InputParametersT", bound=BaseModel)
|
|
10
12
|
|
|
13
|
+
|
|
14
|
+
class Container(containers.DeclarativeContainer, Generic[InputParametersT]):
|
|
11
15
|
config = providers.Configuration()
|
|
12
16
|
|
|
13
|
-
paths = providers.Singleton(
|
|
14
|
-
Paths,
|
|
15
|
-
base_dir=config.base_dir,
|
|
16
|
-
)
|
|
17
|
+
paths = providers.Singleton(Paths, base_dir=config.base_dir)
|
|
17
18
|
|
|
18
|
-
file_loader = providers.
|
|
19
|
+
file_loader = providers.Singleton(
|
|
19
20
|
FilesLoader,
|
|
20
21
|
dids=config.dids,
|
|
21
22
|
transformation_did=config.transformation_did,
|
|
22
23
|
paths=paths,
|
|
24
|
+
logger=config.logger,
|
|
23
25
|
)
|
|
24
26
|
|
|
25
|
-
files = providers.Factory(
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
lambda loader: loader.load(),
|
|
38
|
-
loader=ddo_loader,
|
|
39
|
-
)
|
|
40
|
-
|
|
41
|
-
job_details_loader = providers.Factory(
|
|
42
|
-
JobDetailsLoader,
|
|
43
|
-
files=files,
|
|
44
|
-
secret=config.secret,
|
|
45
|
-
paths=paths,
|
|
46
|
-
ddos=ddos,
|
|
27
|
+
files = providers.Factory(lambda loader: loader.load(), loader=file_loader)
|
|
28
|
+
ddo_loader = providers.Factory(DDOLoader, files=files)
|
|
29
|
+
ddos = providers.Factory(lambda loader: loader.load(), loader=ddo_loader)
|
|
30
|
+
|
|
31
|
+
job_details_loader: providers.Factory[JobDetailsLoader[InputParametersT]] = (
|
|
32
|
+
providers.Factory(
|
|
33
|
+
JobDetailsLoader,
|
|
34
|
+
files=files,
|
|
35
|
+
secret=config.secret,
|
|
36
|
+
paths=paths,
|
|
37
|
+
ddos=ddos,
|
|
38
|
+
)
|
|
47
39
|
)
|
|
@@ -0,0 +1,187 @@
|
|
|
1
|
+
# mypy: disable-error-code=explicit-any
|
|
2
|
+
from dataclasses import InitVar, dataclass, field
|
|
3
|
+
from pathlib import Path
|
|
4
|
+
from typing import Generator, List, Optional, Sequence, TypeAlias, TypeVar
|
|
5
|
+
|
|
6
|
+
from pydantic import BaseModel, ConfigDict, Field, JsonValue
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
class Credential(BaseModel):
|
|
10
|
+
type: str
|
|
11
|
+
values: list[str]
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
class Credentials(BaseModel):
|
|
15
|
+
allow: list[Credential]
|
|
16
|
+
deny: list[Credential]
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
class DockerContainer(BaseModel):
|
|
20
|
+
image: str
|
|
21
|
+
tag: str
|
|
22
|
+
entrypoint: str
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
class Algorithm(BaseModel):
|
|
26
|
+
container: DockerContainer
|
|
27
|
+
language: str
|
|
28
|
+
version: str
|
|
29
|
+
consumerParameters: JsonValue
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
class Metadata(BaseModel):
|
|
33
|
+
description: str
|
|
34
|
+
name: str
|
|
35
|
+
type: str
|
|
36
|
+
author: str
|
|
37
|
+
license: str
|
|
38
|
+
algorithm: Optional[Algorithm] = None
|
|
39
|
+
tags: Optional[list[str]] = None
|
|
40
|
+
created: Optional[str] = None
|
|
41
|
+
updated: Optional[str] = None
|
|
42
|
+
copyrightHolder: Optional[str] = None
|
|
43
|
+
links: Optional[list[str]] = None
|
|
44
|
+
contentLanguage: Optional[str] = None
|
|
45
|
+
categories: Optional[list[str]] = None
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
class ConsumerParameters(BaseModel):
|
|
49
|
+
name: str
|
|
50
|
+
type: str
|
|
51
|
+
label: str
|
|
52
|
+
required: bool
|
|
53
|
+
description: str
|
|
54
|
+
default: str
|
|
55
|
+
option: Optional[list[str]] = None
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
class Service(BaseModel):
|
|
59
|
+
id: str
|
|
60
|
+
type: str
|
|
61
|
+
timeout: int
|
|
62
|
+
files: str
|
|
63
|
+
datatokenAddress: str
|
|
64
|
+
serviceEndpoint: str
|
|
65
|
+
additionalInformation: Optional[str] = None
|
|
66
|
+
name: Optional[str] = None
|
|
67
|
+
description: Optional[str] = None
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
class Event(BaseModel):
|
|
71
|
+
tx: str
|
|
72
|
+
block: int
|
|
73
|
+
from_: str = Field(alias="from")
|
|
74
|
+
contract: str
|
|
75
|
+
datetime: str
|
|
76
|
+
|
|
77
|
+
model_config = ConfigDict(populate_by_name=True)
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
class NFT(BaseModel):
|
|
81
|
+
address: str
|
|
82
|
+
name: str
|
|
83
|
+
symbol: str
|
|
84
|
+
state: int
|
|
85
|
+
tokenURI: str
|
|
86
|
+
owner: str
|
|
87
|
+
created: str
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
class DataToken(BaseModel):
|
|
91
|
+
address: str
|
|
92
|
+
name: str
|
|
93
|
+
symbol: str
|
|
94
|
+
serviceId: str
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
class Price(BaseModel):
|
|
98
|
+
value: int
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
class Stats(BaseModel):
|
|
102
|
+
allocated: int
|
|
103
|
+
orders: int
|
|
104
|
+
price: Price
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
class Purgatory(BaseModel):
|
|
108
|
+
state: bool
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
class DDO(BaseModel):
|
|
112
|
+
id: str
|
|
113
|
+
context: list[str] = Field(alias="@context")
|
|
114
|
+
nftAddress: str
|
|
115
|
+
chainId: int
|
|
116
|
+
version: str
|
|
117
|
+
metadata: Metadata
|
|
118
|
+
services: list[Service]
|
|
119
|
+
credentials: Credentials
|
|
120
|
+
event: Event
|
|
121
|
+
nft: NFT
|
|
122
|
+
datatokens: list[DataToken]
|
|
123
|
+
stats: Stats
|
|
124
|
+
purgatory: Purgatory
|
|
125
|
+
|
|
126
|
+
model_config = ConfigDict(populate_by_name=True)
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
@dataclass(frozen=True)
|
|
130
|
+
class DIDPaths:
|
|
131
|
+
did: str
|
|
132
|
+
ddo: Path = field(repr=False)
|
|
133
|
+
|
|
134
|
+
files: InitVar[Generator[Path, None, None]]
|
|
135
|
+
|
|
136
|
+
_input: List[Path] = field(init=False, repr=False)
|
|
137
|
+
|
|
138
|
+
def __post_init__(self, files: Generator[Path, None, None]) -> None:
|
|
139
|
+
assert self.ddo.exists(), f"DDO {self.ddo} does not exist"
|
|
140
|
+
|
|
141
|
+
object.__setattr__(self, "_input", list(files))
|
|
142
|
+
|
|
143
|
+
@property
|
|
144
|
+
def input_files(self) -> List[Path]:
|
|
145
|
+
return self._input
|
|
146
|
+
|
|
147
|
+
def __len__(self) -> int:
|
|
148
|
+
return len(self._input)
|
|
149
|
+
|
|
150
|
+
|
|
151
|
+
Files: TypeAlias = Sequence[DIDPaths]
|
|
152
|
+
|
|
153
|
+
|
|
154
|
+
@dataclass(frozen=True)
|
|
155
|
+
class Paths:
|
|
156
|
+
"""Configuration class for the Ocean Protocol Job Details"""
|
|
157
|
+
|
|
158
|
+
base_dir: InitVar[Path | None] = None
|
|
159
|
+
|
|
160
|
+
_base: Path = field(init=False, repr=False)
|
|
161
|
+
|
|
162
|
+
def __post_init__(self, base_dir: Path | None) -> None:
|
|
163
|
+
object.__setattr__(self, "_base", base_dir if base_dir else Path("/data"))
|
|
164
|
+
|
|
165
|
+
@property
|
|
166
|
+
def data(self) -> Path:
|
|
167
|
+
return self._base
|
|
168
|
+
|
|
169
|
+
@property
|
|
170
|
+
def inputs(self) -> Path:
|
|
171
|
+
return self.data / "inputs"
|
|
172
|
+
|
|
173
|
+
@property
|
|
174
|
+
def ddos(self) -> Path:
|
|
175
|
+
return self.data / "ddos"
|
|
176
|
+
|
|
177
|
+
@property
|
|
178
|
+
def outputs(self) -> Path:
|
|
179
|
+
return self.data / "outputs"
|
|
180
|
+
|
|
181
|
+
@property
|
|
182
|
+
def logs(self) -> Path:
|
|
183
|
+
return self.data / "logs"
|
|
184
|
+
|
|
185
|
+
@property
|
|
186
|
+
def algorithm_custom_parameters(self) -> Path:
|
|
187
|
+
return self.inputs / "algoCustomData.json"
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
from typing import Any, Dict, Type, TypeVar
|
|
2
|
+
|
|
3
|
+
from pydantic import BaseModel, JsonValue
|
|
4
|
+
|
|
5
|
+
from oceanprotocol_job_details.di import Container
|
|
6
|
+
from oceanprotocol_job_details.ocean import JobDetails
|
|
7
|
+
from oceanprotocol_job_details.settings import JobSettings
|
|
8
|
+
|
|
9
|
+
InputParametersT = TypeVar("InputParametersT", bound=BaseModel)
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def create_container(config: Dict[str, Any]) -> Container[InputParametersT]: # type: ignore[explicit-any]
|
|
13
|
+
"""Return a fully configured Container from a config dict."""
|
|
14
|
+
container = Container[InputParametersT]()
|
|
15
|
+
settings = JobSettings(**config)
|
|
16
|
+
container.config.from_pydantic(settings)
|
|
17
|
+
return container
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def load_job_details(
|
|
21
|
+
config: Dict[str, JsonValue],
|
|
22
|
+
input_type: Type[InputParametersT],
|
|
23
|
+
) -> JobDetails[InputParametersT]:
|
|
24
|
+
"""
|
|
25
|
+
Load JobDetails for a given input_type using the config.
|
|
26
|
+
Returns a fully initialized JobDetails instance.
|
|
27
|
+
"""
|
|
28
|
+
container: Container[InputParametersT] = create_container(config)
|
|
29
|
+
return container.job_details_loader(input_type=input_type).load()
|
|
@@ -2,31 +2,23 @@ from __future__ import annotations
|
|
|
2
2
|
|
|
3
3
|
from dataclasses import InitVar, dataclass, field
|
|
4
4
|
from pathlib import Path
|
|
5
|
-
from typing import
|
|
5
|
+
from typing import final
|
|
6
6
|
|
|
7
|
-
|
|
8
|
-
from oceanprotocol_job_details.ocean import DDO, Files
|
|
7
|
+
from oceanprotocol_job_details.domain import DDO, Files
|
|
9
8
|
|
|
10
9
|
|
|
11
10
|
@final
|
|
12
11
|
@dataclass(frozen=True)
|
|
13
12
|
class DDOLoader:
|
|
14
|
-
|
|
15
|
-
files: InitVar[list[Files]]
|
|
13
|
+
files: InitVar[Files]
|
|
16
14
|
"""The files to load the DDOs from"""
|
|
17
15
|
|
|
18
16
|
_ddo_paths: list[Path] = field(init=False)
|
|
19
17
|
|
|
20
|
-
def __post_init__(self, files:
|
|
21
|
-
assert files, "Missing files"
|
|
18
|
+
def __post_init__(self, files: Files) -> None:
|
|
19
|
+
assert files is not None and len(files) != 0, "Missing files"
|
|
22
20
|
|
|
23
21
|
object.__setattr__(self, "_ddo_paths", [f.ddo for f in files])
|
|
24
22
|
|
|
25
23
|
def load(self) -> list[DDO]:
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
ddos = []
|
|
29
|
-
for path in self._ddo_paths:
|
|
30
|
-
with open(path, "r") as f:
|
|
31
|
-
ddos.append(DDO.from_json(f.read())) # type: ignore
|
|
32
|
-
return ddos
|
|
24
|
+
return [DDO.model_validate_json(p.read_text()) for p in self._ddo_paths]
|
|
@@ -1,54 +1,44 @@
|
|
|
1
1
|
from __future__ import annotations
|
|
2
2
|
|
|
3
|
-
import json
|
|
4
3
|
from dataclasses import InitVar, dataclass, field
|
|
5
|
-
from
|
|
4
|
+
from logging import Logger
|
|
5
|
+
from pathlib import Path
|
|
6
|
+
from typing import Literal, final
|
|
6
7
|
|
|
7
|
-
from oceanprotocol_job_details.
|
|
8
|
-
|
|
9
|
-
if TYPE_CHECKING:
|
|
10
|
-
from oceanprotocol_job_details.ocean import DIDPaths, Files
|
|
8
|
+
from oceanprotocol_job_details.domain import DIDPaths, Files, Paths
|
|
11
9
|
|
|
12
10
|
|
|
13
11
|
@final
|
|
14
12
|
@dataclass(frozen=True)
|
|
15
13
|
class FilesLoader:
|
|
14
|
+
paths: Paths
|
|
15
|
+
"""Path configurations of the project"""
|
|
16
16
|
|
|
17
|
-
|
|
17
|
+
logger: Logger = field(repr=False)
|
|
18
|
+
"""Logger to use"""
|
|
19
|
+
|
|
20
|
+
dids: list[str]
|
|
18
21
|
"""Input DIDs"""
|
|
19
22
|
|
|
20
|
-
transformation_did:
|
|
23
|
+
transformation_did: str
|
|
21
24
|
"""DID for the transformation algorithm"""
|
|
22
25
|
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
_dids: Sequence[str] = field(init=False)
|
|
27
|
-
_transformation_did: str = field(init=False)
|
|
26
|
+
def __post_init__(self) -> None:
|
|
27
|
+
assert self.dids, "Missing input DIDs"
|
|
28
28
|
|
|
29
|
-
def
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
assert transformation_did, "Missing transformation DID"
|
|
36
|
-
|
|
37
|
-
object.__setattr__(self, "_dids", json.loads(dids))
|
|
38
|
-
object.__setattr__(self, "_transformation_did", transformation_did)
|
|
29
|
+
def calculate_path(self, did: str, path_type: Literal["input", "ddo"]) -> Path:
|
|
30
|
+
match path_type:
|
|
31
|
+
case "ddo":
|
|
32
|
+
return self.paths.ddos / did
|
|
33
|
+
case "input":
|
|
34
|
+
return self.paths.inputs / did
|
|
39
35
|
|
|
40
36
|
def load(self) -> Files:
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
files.append(
|
|
47
|
-
DIDPaths(
|
|
48
|
-
did=did,
|
|
49
|
-
ddo=self.paths.ddos / did,
|
|
50
|
-
input_files=list(base.iterdir()),
|
|
51
|
-
)
|
|
37
|
+
return [
|
|
38
|
+
DIDPaths(
|
|
39
|
+
did=did,
|
|
40
|
+
ddo=self.calculate_path(did, "ddo"),
|
|
41
|
+
files=self.calculate_path(did, "input").iterdir(),
|
|
52
42
|
)
|
|
53
|
-
|
|
54
|
-
|
|
43
|
+
for did in self.dids
|
|
44
|
+
]
|
|
@@ -1,35 +1,28 @@
|
|
|
1
|
-
from __future__ import annotations
|
|
2
|
-
|
|
3
1
|
from dataclasses import dataclass, field
|
|
4
|
-
from typing import
|
|
5
|
-
|
|
6
|
-
from oceanprotocol_job_details.paths import Paths
|
|
2
|
+
from typing import Generic, Type, TypeVar, final
|
|
7
3
|
|
|
8
|
-
|
|
9
|
-
from oceanprotocol_job_details.ocean import DDO, Files, JobDetails
|
|
4
|
+
from pydantic import BaseModel
|
|
10
5
|
|
|
6
|
+
from oceanprotocol_job_details.domain import DDO, Files, Paths
|
|
7
|
+
from oceanprotocol_job_details.ocean import JobDetails
|
|
11
8
|
|
|
12
|
-
T = TypeVar("T")
|
|
9
|
+
T = TypeVar("T", bound=BaseModel)
|
|
13
10
|
|
|
14
11
|
|
|
15
12
|
@final
|
|
16
13
|
@dataclass(frozen=True)
|
|
17
14
|
class JobDetailsLoader(Generic[T]):
|
|
18
|
-
|
|
19
|
-
_type: Type[T] = field(repr=False)
|
|
20
|
-
|
|
15
|
+
input_type: Type[T] = field(repr=False)
|
|
21
16
|
files: Files
|
|
22
17
|
secret: str
|
|
23
18
|
paths: Paths
|
|
24
19
|
ddos: list[DDO]
|
|
25
20
|
|
|
26
21
|
def load(self) -> JobDetails[T]:
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
return JobDetails(
|
|
22
|
+
return JobDetails[T](
|
|
30
23
|
files=self.files,
|
|
31
24
|
secret=self.secret,
|
|
32
25
|
ddos=self.ddos,
|
|
33
26
|
paths=self.paths,
|
|
34
|
-
|
|
27
|
+
input_type=self.input_type,
|
|
35
28
|
)
|
|
@@ -1,319 +1,51 @@
|
|
|
1
1
|
from __future__ import annotations
|
|
2
2
|
|
|
3
|
-
import
|
|
4
|
-
import os
|
|
5
|
-
from dataclasses import dataclass, field
|
|
3
|
+
import asyncio
|
|
6
4
|
from functools import cached_property
|
|
7
5
|
from pathlib import Path
|
|
8
|
-
from typing import
|
|
9
|
-
Any,
|
|
10
|
-
Generator,
|
|
11
|
-
Generic,
|
|
12
|
-
Iterator,
|
|
13
|
-
Optional,
|
|
14
|
-
Sequence,
|
|
15
|
-
Type,
|
|
16
|
-
TypeVar,
|
|
17
|
-
final,
|
|
18
|
-
)
|
|
6
|
+
from typing import Generator, Generic, Tuple, Type, TypeVar, final
|
|
19
7
|
|
|
20
|
-
import
|
|
21
|
-
from
|
|
22
|
-
from dataclasses_json import dataclass_json
|
|
8
|
+
import aiofiles
|
|
9
|
+
from pydantic import BaseModel, ConfigDict, Secret, model_validator
|
|
23
10
|
|
|
24
|
-
from oceanprotocol_job_details.
|
|
25
|
-
from oceanprotocol_job_details.paths import Paths
|
|
11
|
+
from oceanprotocol_job_details.domain import DDO, Files, Paths
|
|
26
12
|
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
logging.basicConfig(
|
|
30
|
-
level=logging.INFO,
|
|
31
|
-
format="%(asctime)s [%(threadName)s] [%(levelname)s] %(message)s",
|
|
32
|
-
handlers=[logging.StreamHandler()],
|
|
33
|
-
)
|
|
34
|
-
logger = logging.getLogger(__name__)
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
@dataclass_json
|
|
38
|
-
@dataclass
|
|
39
|
-
class Credential:
|
|
40
|
-
type: str
|
|
41
|
-
values: list[str]
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
@dataclass_json
|
|
45
|
-
@dataclass
|
|
46
|
-
class Credentials:
|
|
47
|
-
allow: list[Credential]
|
|
48
|
-
deny: list[Credential]
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
@dataclass_json
|
|
52
|
-
@dataclass
|
|
53
|
-
class DockerContainer:
|
|
54
|
-
image: str
|
|
55
|
-
tag: str
|
|
56
|
-
entrypoint: str
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
@dataclass_json
|
|
60
|
-
@dataclass
|
|
61
|
-
class Algorithm: # type: ignore
|
|
62
|
-
container: DockerContainer
|
|
63
|
-
language: str
|
|
64
|
-
version: str
|
|
65
|
-
consumerParameters: Any # type: ignore
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
@dataclass_json
|
|
69
|
-
@dataclass
|
|
70
|
-
class Metadata:
|
|
71
|
-
description: str
|
|
72
|
-
name: str
|
|
73
|
-
type: str
|
|
74
|
-
author: str
|
|
75
|
-
license: str
|
|
76
|
-
algorithm: Optional[Algorithm] = None
|
|
77
|
-
tags: Optional[list[str]] = None
|
|
78
|
-
created: Optional[str] = None
|
|
79
|
-
updated: Optional[str] = None
|
|
80
|
-
copyrightHolder: Optional[str] = None
|
|
81
|
-
links: Optional[list[str]] = None
|
|
82
|
-
contentLanguage: Optional[str] = None
|
|
83
|
-
categories: Optional[list[str]] = None
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
@dataclass_json
|
|
87
|
-
@dataclass
|
|
88
|
-
class ConsumerParameters:
|
|
89
|
-
name: str
|
|
90
|
-
type: str
|
|
91
|
-
label: str
|
|
92
|
-
required: bool
|
|
93
|
-
description: str
|
|
94
|
-
default: str
|
|
95
|
-
option: Optional[list[str]] = None
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
@dataclass_json
|
|
99
|
-
@dataclass
|
|
100
|
-
class Service:
|
|
101
|
-
id: str
|
|
102
|
-
type: str
|
|
103
|
-
timeout: int
|
|
104
|
-
files: str
|
|
105
|
-
datatokenAddress: str
|
|
106
|
-
serviceEndpoint: str
|
|
107
|
-
additionalInformation: Optional[str] = None
|
|
108
|
-
name: Optional[str] = None
|
|
109
|
-
description: Optional[str] = None
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
@dataclass_json
|
|
113
|
-
@dataclass
|
|
114
|
-
class Event:
|
|
115
|
-
tx: str
|
|
116
|
-
block: int
|
|
117
|
-
from_: str = field(metadata=dc_config(field_name="from"))
|
|
118
|
-
contract: str
|
|
119
|
-
datetime: str
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
@dataclass_json
|
|
123
|
-
@dataclass
|
|
124
|
-
class NFT:
|
|
125
|
-
address: str
|
|
126
|
-
name: str
|
|
127
|
-
symbol: str
|
|
128
|
-
state: int
|
|
129
|
-
tokenURI: str
|
|
130
|
-
owner: str
|
|
131
|
-
created: str
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
@dataclass_json
|
|
135
|
-
@dataclass
|
|
136
|
-
class DataToken:
|
|
137
|
-
address: str
|
|
138
|
-
name: str
|
|
139
|
-
symbol: str
|
|
140
|
-
serviceId: str
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
@dataclass_json
|
|
144
|
-
@dataclass
|
|
145
|
-
class Price:
|
|
146
|
-
value: int
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
@dataclass_json
|
|
150
|
-
@dataclass
|
|
151
|
-
class Stats:
|
|
152
|
-
allocated: int
|
|
153
|
-
orders: int
|
|
154
|
-
price: Price
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
@dataclass_json
|
|
158
|
-
@dataclass
|
|
159
|
-
class Purgatory:
|
|
160
|
-
state: bool
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
@dataclass_json
|
|
164
|
-
@dataclass
|
|
165
|
-
class DDO:
|
|
166
|
-
id: str
|
|
167
|
-
context: list[str] = field(metadata=dc_config(field_name="@context"))
|
|
168
|
-
nftAddress: str
|
|
169
|
-
chainId: int
|
|
170
|
-
version: str
|
|
171
|
-
metadata: Metadata
|
|
172
|
-
services: list[Service]
|
|
173
|
-
credentials: Credentials
|
|
174
|
-
event: Event
|
|
175
|
-
nft: NFT
|
|
176
|
-
datatokens: list[DataToken]
|
|
177
|
-
stats: Stats
|
|
178
|
-
purgatory: Purgatory
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
@dataclass(frozen=True)
|
|
182
|
-
class DIDPaths:
|
|
183
|
-
did: str
|
|
184
|
-
ddo: Path
|
|
185
|
-
input_files: Sequence[Path]
|
|
186
|
-
|
|
187
|
-
def __post_init__(self) -> None:
|
|
188
|
-
assert self.ddo.exists(), f"DDO {self.ddo} does not exist"
|
|
189
|
-
for input_file in self.input_files:
|
|
190
|
-
assert input_file.exists(), f"File {input_file} does not exist"
|
|
191
|
-
|
|
192
|
-
def __len__(self) -> int:
|
|
193
|
-
return len(self.input_files)
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
@dataclass(frozen=True)
|
|
197
|
-
class Files:
|
|
198
|
-
_files: Sequence[DIDPaths]
|
|
199
|
-
|
|
200
|
-
@property
|
|
201
|
-
def files(self) -> Sequence[DIDPaths]:
|
|
202
|
-
return self._files
|
|
203
|
-
|
|
204
|
-
def __getitem__(self, index: int) -> DIDPaths:
|
|
205
|
-
return self.files[index]
|
|
206
|
-
|
|
207
|
-
def __iter__(self) -> Iterator[DIDPaths]:
|
|
208
|
-
return iter(self.files)
|
|
209
|
-
|
|
210
|
-
def __len__(self) -> int:
|
|
211
|
-
return len(self.files)
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
def _normalize_json(value):
|
|
215
|
-
if isinstance(value, str):
|
|
216
|
-
try:
|
|
217
|
-
decoded = orjson.loads(value)
|
|
218
|
-
return _normalize_json(decoded) # recurse if nested again
|
|
219
|
-
except orjson.JSONDecodeError:
|
|
220
|
-
return value
|
|
221
|
-
elif isinstance(value, dict):
|
|
222
|
-
return {k: _normalize_json(v) for k, v in value.items()}
|
|
223
|
-
elif isinstance(value, list):
|
|
224
|
-
return [_normalize_json(v) for v in value]
|
|
225
|
-
return value
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
@final
|
|
229
|
-
@dataclass_json
|
|
230
|
-
@dataclass
|
|
231
|
-
class _EmptyJobDetails: ...
|
|
13
|
+
InputParemetersT = TypeVar("InputParemetersT", bound=BaseModel)
|
|
232
14
|
|
|
233
15
|
|
|
234
16
|
@final
|
|
235
|
-
|
|
236
|
-
@dataclass(frozen=True)
|
|
237
|
-
class JobDetails(Generic[T]):
|
|
17
|
+
class JobDetails(BaseModel, Generic[InputParemetersT]): # type: ignore[explicit-any]
|
|
238
18
|
files: Files
|
|
239
|
-
"""The input filepaths"""
|
|
240
|
-
|
|
241
19
|
ddos: list[DDO]
|
|
242
|
-
"""list of paths to the DDOs"""
|
|
243
|
-
|
|
244
20
|
paths: Paths
|
|
245
|
-
|
|
21
|
+
input_type: Type[InputParemetersT]
|
|
22
|
+
secret: Secret[str] | None = None
|
|
246
23
|
|
|
247
|
-
|
|
248
|
-
_type: Type[T] = field(repr=False)
|
|
24
|
+
model_config = ConfigDict(arbitrary_types_allowed=True, frozen=True)
|
|
249
25
|
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
26
|
+
@model_validator(mode="after")
|
|
27
|
+
def validate_type(self) -> JobDetails[InputParemetersT]:
|
|
28
|
+
assert issubclass(self.input_type, BaseModel), (
|
|
29
|
+
f"{self.input_type} must be subtype of pydantic.BaseModel"
|
|
30
|
+
)
|
|
31
|
+
return self
|
|
256
32
|
|
|
257
|
-
def
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
33
|
+
def inputs(self) -> Generator[Tuple[int, Path], None, None]:
|
|
34
|
+
yield from (
|
|
35
|
+
(idx, file)
|
|
36
|
+
for idx, files in enumerate(self.files)
|
|
37
|
+
for file in files.input_files
|
|
38
|
+
)
|
|
261
39
|
|
|
262
40
|
@cached_property
|
|
263
|
-
def input_parameters(self) ->
|
|
264
|
-
|
|
41
|
+
def input_parameters(self) -> InputParemetersT:
|
|
42
|
+
return asyncio.run(self.ainput_parameters())
|
|
265
43
|
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
f"Custom parameters file {self.paths.algorithm_custom_parameters} is empty"
|
|
271
|
-
)
|
|
272
|
-
try:
|
|
273
|
-
parsed = _normalize_json(orjson.loads(raw))
|
|
274
|
-
return dataclass_json(self._type).from_dict(parsed) # type: ignore
|
|
275
|
-
except Exception as e:
|
|
276
|
-
raise ValueError(
|
|
277
|
-
f"Failed to parse input paramers into {self._type.__name__}: {e}\n"
|
|
278
|
-
f"Raw content: {raw}"
|
|
279
|
-
) from e
|
|
280
|
-
|
|
281
|
-
@classmethod
|
|
282
|
-
def load(
|
|
283
|
-
cls,
|
|
284
|
-
_type: Type[T] | None = None,
|
|
285
|
-
*,
|
|
286
|
-
base_dir: str | None = None,
|
|
287
|
-
dids: str | None = None,
|
|
288
|
-
transformation_did: str | None = None,
|
|
289
|
-
secret: str | None = None,
|
|
290
|
-
) -> JobDetails[T]:
|
|
291
|
-
"""Load a JobDetails instance that holds the runtime details.
|
|
292
|
-
|
|
293
|
-
Loading it will check the following:
|
|
294
|
-
1. That the needed environment variables are set.
|
|
295
|
-
1. That the ocean protocol contains the needed data based on the passed environment variables.
|
|
296
|
-
|
|
297
|
-
Those needed environment variables are:
|
|
298
|
-
- BASE_DIR: Base directory to read the data from, parent of the ddos, inputs, outputs and logs directories.
|
|
299
|
-
- DIDS: The DIDs of the inputs
|
|
300
|
-
- TRANSFORMATION_DID: The DID of the transformation algorithm
|
|
301
|
-
- SECRET (optional): A really secret secret
|
|
302
|
-
|
|
303
|
-
"""
|
|
304
|
-
|
|
305
|
-
if _type is None:
|
|
306
|
-
_type = _EmptyJobDetails
|
|
307
|
-
|
|
308
|
-
container = Container()
|
|
309
|
-
container.config.from_dict(
|
|
310
|
-
{
|
|
311
|
-
"base_dir": base_dir or os.environ.get("BASE_DIR", None),
|
|
312
|
-
"dids": dids or os.environ.get("DIDS"),
|
|
313
|
-
"transformation_did": transformation_did
|
|
314
|
-
or os.environ.get("TRANSFORMATION_DID"),
|
|
315
|
-
"secret": secret or os.environ.get("SECRET"),
|
|
316
|
-
}
|
|
317
|
-
)
|
|
44
|
+
async def ainput_parameters(self) -> InputParemetersT:
|
|
45
|
+
path = self.paths.algorithm_custom_parameters
|
|
46
|
+
async with aiofiles.open(path) as f:
|
|
47
|
+
raw = await f.read()
|
|
318
48
|
|
|
319
|
-
|
|
49
|
+
raw = raw.strip()
|
|
50
|
+
assert raw is not None, f"Empty file {path}"
|
|
51
|
+
return self.input_type.model_validate_json(raw)
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
# mypy: disable-error-code=call-overload
|
|
2
|
+
from logging import Logger, getLogger
|
|
3
|
+
from pathlib import Path
|
|
4
|
+
from typing import Self
|
|
5
|
+
|
|
6
|
+
import orjson
|
|
7
|
+
from pydantic import Field, field_validator, model_validator
|
|
8
|
+
from pydantic_settings import BaseSettings, SettingsConfigDict
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
class JobSettings(BaseSettings): # type: ignore[explicit-any]
|
|
12
|
+
base_dir: Path = Field(alias="BASE_DIR")
|
|
13
|
+
dids: list[str] = Field(default_factory=list, alias="DIDS")
|
|
14
|
+
transformation_did: str = Field(alias="TRANSFORMATION_DID")
|
|
15
|
+
secret: str | None = Field(default=None, alias="SECRET")
|
|
16
|
+
logger: Logger = Field(default_factory=lambda: getLogger(__name__))
|
|
17
|
+
|
|
18
|
+
model_config = SettingsConfigDict(
|
|
19
|
+
extra="forbid",
|
|
20
|
+
populate_by_name=True,
|
|
21
|
+
arbitrary_types_allowed=True,
|
|
22
|
+
)
|
|
23
|
+
|
|
24
|
+
@field_validator("dids", mode="before")
|
|
25
|
+
@classmethod
|
|
26
|
+
def split_dids(cls, v: list[str] | str) -> list[str]:
|
|
27
|
+
if isinstance(v, str):
|
|
28
|
+
data = orjson.loads(v)
|
|
29
|
+
assert isinstance(data, list)
|
|
30
|
+
return data
|
|
31
|
+
return v
|
|
32
|
+
|
|
33
|
+
@model_validator(mode="after")
|
|
34
|
+
def validate_dids(self) -> Self:
|
|
35
|
+
if not self.dids:
|
|
36
|
+
self.dids.extend([f.name for f in (self.base_dir / "ddos").glob("*")])
|
|
37
|
+
return self
|
|
@@ -0,0 +1,114 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: oceanprotocol-job-details
|
|
3
|
+
Version: 0.3.3
|
|
4
|
+
Summary: A Python package to get details from OceanProtocol jobs
|
|
5
|
+
Project-URL: Homepage, https://github.com/AgrospAI/oceanprotocol-job-details
|
|
6
|
+
Project-URL: Issues, https://github.com/AgrospAI/oceanprotocol-job-details/issues
|
|
7
|
+
Author-email: Agrospai <agrospai@udl.cat>, Christian López García <christian.lopez@udl.cat>
|
|
8
|
+
License: Copyright 2025 Agrospai
|
|
9
|
+
|
|
10
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
|
|
13
|
+
|
|
14
|
+
THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
|
15
|
+
License-File: LICENSE
|
|
16
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
17
|
+
Classifier: Operating System :: OS Independent
|
|
18
|
+
Classifier: Programming Language :: Python :: 3
|
|
19
|
+
Requires-Python: >=3.10
|
|
20
|
+
Requires-Dist: aiofiles>=25.1.0
|
|
21
|
+
Requires-Dist: dependency-injector>=4.48.2
|
|
22
|
+
Requires-Dist: orjson>=3.11.3
|
|
23
|
+
Requires-Dist: pydantic-settings>=2.12.0
|
|
24
|
+
Requires-Dist: pydantic>=2.12.5
|
|
25
|
+
Description-Content-Type: text/markdown
|
|
26
|
+
|
|
27
|
+
A Python package to get details from OceanProtocol jobs
|
|
28
|
+
|
|
29
|
+
---
|
|
30
|
+
|
|
31
|
+
## Installation
|
|
32
|
+
|
|
33
|
+
```bash
|
|
34
|
+
pip install oceanprotocol-job-details
|
|
35
|
+
```
|
|
36
|
+
|
|
37
|
+
```bash
|
|
38
|
+
uv add oceanprotocol-job-details
|
|
39
|
+
```
|
|
40
|
+
|
|
41
|
+
## Usage
|
|
42
|
+
|
|
43
|
+
As a simple library, we only need to import `load_job_details` and run it. It will:
|
|
44
|
+
|
|
45
|
+
1. Fetch the needed parameters to populate the `JobDetails` instance from the environment variables or use the passed values to the function.
|
|
46
|
+
1. Look for the files corresponding to the passed DIDs in the filesystem according to the [Ocean Protocol Structure](#oceanprotocol-structure) and load them into the `JobDetails` instance.
|
|
47
|
+
|
|
48
|
+
### Minimal Example
|
|
49
|
+
|
|
50
|
+
```python
|
|
51
|
+
from oceanprotocol_job_details import load_job_details
|
|
52
|
+
|
|
53
|
+
class InputParameters(BaseModel): ...
|
|
54
|
+
|
|
55
|
+
job_details = load_job_details({}, InputParameters)
|
|
56
|
+
```
|
|
57
|
+
|
|
58
|
+
### Custom Input Parameters
|
|
59
|
+
|
|
60
|
+
If our algorithm has custom input parameters and we want to load them into our algorithm, we can do it as follows:
|
|
61
|
+
|
|
62
|
+
```python
|
|
63
|
+
from pydantic import BaseModel
|
|
64
|
+
from oceanprotocol_job_details import load_job_details
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
class Foo(BaseModel):
|
|
68
|
+
bar: str
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
class InputParameters(BaseModel):
|
|
72
|
+
# Allows for nested types
|
|
73
|
+
foo: Foo
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
job_details = load_job_details({}, InputParameters)
|
|
77
|
+
|
|
78
|
+
# Usage
|
|
79
|
+
job_details.input_parameters.foo
|
|
80
|
+
job_details.input_parameters.foo.bar
|
|
81
|
+
```
|
|
82
|
+
|
|
83
|
+
The values to fill the custom `InputParameters` will be parsed from the `algoCustomData.json` located next to the input data directories.
|
|
84
|
+
|
|
85
|
+
### Iterating Input Files the clean way
|
|
86
|
+
|
|
87
|
+
```python
|
|
88
|
+
from oceanprotocol_job_details import load_job_details
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
job_details = load_job_details
|
|
92
|
+
|
|
93
|
+
for idx, file_path in job_details.inputs():
|
|
94
|
+
...
|
|
95
|
+
|
|
96
|
+
_, file_path = next(job_details.inputs())
|
|
97
|
+
```
|
|
98
|
+
|
|
99
|
+
## OceanProtocol Structure
|
|
100
|
+
|
|
101
|
+
```bash
|
|
102
|
+
data # Root /data directory
|
|
103
|
+
├── ddos # Contains the loaded dataset's DDO
|
|
104
|
+
│ ├── 17feb...e42 # DDO file
|
|
105
|
+
│ └── ... # One DDO per loaded dataset
|
|
106
|
+
├── inputs # Datasets dir
|
|
107
|
+
│ ├── 17feb...e42 # Dir holding the data of its name DID, contains files named 0..X
|
|
108
|
+
│ │ └── 0 # Data file
|
|
109
|
+
│ └── algoCustomData.json # Custom algorithm input data
|
|
110
|
+
├── logs # Algorithm output logs dir
|
|
111
|
+
└── outputs # Algorithm output files dir
|
|
112
|
+
```
|
|
113
|
+
|
|
114
|
+
> **_Note:_** Even though it's possible that the algorithm is passed multiple datasets, right now the implementation only allows to use **one dataset** per algorithm execution, so **normally** the executing job will only have **one ddo**, **one dir** inside inputs, and **one data file** named `0`.
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
oceanprotocol_job_details/__init__.py,sha256=nJMrZsEC5F1n9WF-v5QV095Yyc8UkhFw0AzD9o7X0IE,162
|
|
2
|
+
oceanprotocol_job_details/di.py,sha256=lsogbmjvmPfkd0mjLvn9vYLIZebwJm5RNraWt7WE5LA,1316
|
|
3
|
+
oceanprotocol_job_details/domain.py,sha256=2_USbeA_7VIEYS8DVb2MW6dCZasjiqIxQaGUnNUKspY,3851
|
|
4
|
+
oceanprotocol_job_details/helpers.py,sha256=ubH_KjAROqYvn0mkbA0-89vpdKIhVNGZ0h9pQLfPNow,1045
|
|
5
|
+
oceanprotocol_job_details/ocean.py,sha256=q8rgT5ycA2ifey3XNhUW0bcJwfMp7hpKU-6EVDeKV1o,1620
|
|
6
|
+
oceanprotocol_job_details/settings.py,sha256=zgIYPzaXjsgcmuhT7L2ipSP-2eNaodugHZr0rn2Z420,1248
|
|
7
|
+
oceanprotocol_job_details/loaders/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
8
|
+
oceanprotocol_job_details/loaders/loader.py,sha256=36X2s_0lN89kCUpItxEXfIzuBBNJySebP2B_tdWK2E0,186
|
|
9
|
+
oceanprotocol_job_details/loaders/impl/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
10
|
+
oceanprotocol_job_details/loaders/impl/ddo.py,sha256=XthrQFhmP85XSVzVjBlLePtTowGR3BAsmVp3jngiQ08,668
|
|
11
|
+
oceanprotocol_job_details/loaders/impl/files.py,sha256=Y2vFBT2T9w9zrdpmf550-LQJxwtNPUGa0UU6bBzk9AU,1145
|
|
12
|
+
oceanprotocol_job_details/loaders/impl/job_details.py,sha256=7mEdeTo-cmsWuqWPdN7btjLjo6p5Oa1_acjSLWL5tb8,697
|
|
13
|
+
oceanprotocol_job_details-0.3.3.dist-info/METADATA,sha256=pDXdsInFTf36wYjsqWSd_ygeCDggozjueHlhF6OyvlQ,4416
|
|
14
|
+
oceanprotocol_job_details-0.3.3.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
|
|
15
|
+
oceanprotocol_job_details-0.3.3.dist-info/licenses/LICENSE,sha256=ni3ix7P_GxK1W3VGC4fJ3o6QoCngCEpSuTJwO4nkpbw,1055
|
|
16
|
+
oceanprotocol_job_details-0.3.3.dist-info/RECORD,,
|
|
@@ -1,38 +0,0 @@
|
|
|
1
|
-
from dataclasses import InitVar, dataclass, field
|
|
2
|
-
from pathlib import Path
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
@dataclass
|
|
6
|
-
class Paths:
|
|
7
|
-
"""Configuration class for the Ocean Protocol Job Details"""
|
|
8
|
-
|
|
9
|
-
base_dir: InitVar[Path | None]
|
|
10
|
-
|
|
11
|
-
_base: Path = field(init=False)
|
|
12
|
-
|
|
13
|
-
def __post_init__(self, base_dir: Path | None) -> None:
|
|
14
|
-
self._base = base_dir or Path("/data")
|
|
15
|
-
|
|
16
|
-
@property
|
|
17
|
-
def data(self) -> Path:
|
|
18
|
-
return self._base
|
|
19
|
-
|
|
20
|
-
@property
|
|
21
|
-
def inputs(self) -> Path:
|
|
22
|
-
return self.data / "inputs"
|
|
23
|
-
|
|
24
|
-
@property
|
|
25
|
-
def ddos(self) -> Path:
|
|
26
|
-
return self.data / "ddos"
|
|
27
|
-
|
|
28
|
-
@property
|
|
29
|
-
def outputs(self) -> Path:
|
|
30
|
-
return self.data / "outputs"
|
|
31
|
-
|
|
32
|
-
@property
|
|
33
|
-
def logs(self) -> Path:
|
|
34
|
-
return self.data / "logs"
|
|
35
|
-
|
|
36
|
-
@property
|
|
37
|
-
def algorithm_custom_parameters(self) -> Path:
|
|
38
|
-
return self.inputs / "algoCustomData.json"
|
|
@@ -1,76 +0,0 @@
|
|
|
1
|
-
Metadata-Version: 2.4
|
|
2
|
-
Name: oceanprotocol-job-details
|
|
3
|
-
Version: 0.2.5
|
|
4
|
-
Summary: A Python package to get details from OceanProtocol jobs
|
|
5
|
-
Project-URL: Homepage, https://github.com/AgrospAI/oceanprotocol-job-details
|
|
6
|
-
Project-URL: Issues, https://github.com/AgrospAI/oceanprotocol-job-details/issues
|
|
7
|
-
Author-email: Christian López García <christian.lopez@udl.cat>
|
|
8
|
-
License: Copyright 2025 Agrospai
|
|
9
|
-
|
|
10
|
-
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
|
|
11
|
-
|
|
12
|
-
The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
|
|
13
|
-
|
|
14
|
-
THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
|
15
|
-
License-File: LICENSE
|
|
16
|
-
Classifier: License :: OSI Approved :: MIT License
|
|
17
|
-
Classifier: Operating System :: OS Independent
|
|
18
|
-
Classifier: Programming Language :: Python :: 3
|
|
19
|
-
Requires-Python: >=3.10
|
|
20
|
-
Requires-Dist: dataclasses-json>=0.6.7
|
|
21
|
-
Requires-Dist: dependency-injector>=4.48.2
|
|
22
|
-
Requires-Dist: orjson>=3.11.3
|
|
23
|
-
Description-Content-Type: text/markdown
|
|
24
|
-
|
|
25
|
-
A Python package to get details from OceanProtocol jobs
|
|
26
|
-
|
|
27
|
-
---
|
|
28
|
-
|
|
29
|
-
## Installation
|
|
30
|
-
|
|
31
|
-
```
|
|
32
|
-
pip install oceanprotocol-job-details
|
|
33
|
-
```
|
|
34
|
-
|
|
35
|
-
## Usage
|
|
36
|
-
|
|
37
|
-
As a simple library, we only need to import the main object and use it once:
|
|
38
|
-
|
|
39
|
-
```Python
|
|
40
|
-
from oceanprotocol_job_details import JobDetails
|
|
41
|
-
|
|
42
|
-
# Having no algorithm input parameters
|
|
43
|
-
job_details = JobDetails.load()
|
|
44
|
-
|
|
45
|
-
```
|
|
46
|
-
|
|
47
|
-
If our algorithm has custom input parameters and we want to load them into our algorithm, we can do it as follows:
|
|
48
|
-
|
|
49
|
-
```Python
|
|
50
|
-
|
|
51
|
-
from dataclasses import dataclass
|
|
52
|
-
from oceanprotocol_job_details import JobDetails
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
@dataclass
|
|
56
|
-
class InputParameters:
|
|
57
|
-
name: str
|
|
58
|
-
age: int
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
job_details: JobDetails[InputParameters] = JobDetails.load(InputParameters)
|
|
62
|
-
|
|
63
|
-
# Usage (is type hinted)
|
|
64
|
-
job_details.input_parameters.name
|
|
65
|
-
job_details.input_parameters.age
|
|
66
|
-
|
|
67
|
-
```
|
|
68
|
-
|
|
69
|
-
Assumes the directory structure of OceanProtocol algorithms.
|
|
70
|
-
|
|
71
|
-
### Core functionalities
|
|
72
|
-
|
|
73
|
-
Given the Ocean Protocol job details structure, parses the passed algorithm parameters into an object to use in your algorithms.
|
|
74
|
-
|
|
75
|
-
1. Input parameter JSON parsing and validation
|
|
76
|
-
1. Metadata and service extraction from the directory structure.
|
|
@@ -1,14 +0,0 @@
|
|
|
1
|
-
oceanprotocol_job_details/__init__.py,sha256=C67wv7fy5ZT5FtbGD-oQeSoLU-x6e2ts-820koFo034,55
|
|
2
|
-
oceanprotocol_job_details/di.py,sha256=j2BvpaGpiQwt9MSiViYPFup4MnzTHm44Zq12_TQ07kY,1180
|
|
3
|
-
oceanprotocol_job_details/ocean.py,sha256=uABP0JZdCpd3NrTmfXWGuG8XOEVx680-jjRDV0hHync,7373
|
|
4
|
-
oceanprotocol_job_details/paths.py,sha256=N90IF8OQCBELULGPaBv9yALApa-lC2dsVKXMWLdqa14,851
|
|
5
|
-
oceanprotocol_job_details/loaders/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
6
|
-
oceanprotocol_job_details/loaders/loader.py,sha256=HIzsVKCuGP7ghfM7ppN3ANVybvsA64wr3h8I68mqS6A,195
|
|
7
|
-
oceanprotocol_job_details/loaders/impl/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
8
|
-
oceanprotocol_job_details/loaders/impl/ddo.py,sha256=_xl0PozuvIm0n6jU--Znk6qfw1A6OLBq3SERbDGIF74,844
|
|
9
|
-
oceanprotocol_job_details/loaders/impl/files.py,sha256=oFkA_0Ma5NBgWvVEk_rhyDIDrAam_zjesh0-bxZaIU8,1443
|
|
10
|
-
oceanprotocol_job_details/loaders/impl/job_details.py,sha256=wf0xNAG4tESq57vqkdtMQ8BdiyS91j5f7FL8Gfwbjh4,770
|
|
11
|
-
oceanprotocol_job_details-0.2.5.dist-info/METADATA,sha256=o5sx14PqA0kau6UYfYy_NMGhatGf0cWcaKhS7HGDbWw,2948
|
|
12
|
-
oceanprotocol_job_details-0.2.5.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
|
|
13
|
-
oceanprotocol_job_details-0.2.5.dist-info/licenses/LICENSE,sha256=ni3ix7P_GxK1W3VGC4fJ3o6QoCngCEpSuTJwO4nkpbw,1055
|
|
14
|
-
oceanprotocol_job_details-0.2.5.dist-info/RECORD,,
|
|
File without changes
|