runnable 0.26.0__py3-none-any.whl → 0.27.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- extensions/catalog/minio.py +7 -4
- extensions/run_log_store/any_path.py +104 -0
- extensions/run_log_store/chunked_fs.py +13 -9
- extensions/run_log_store/file_system.py +6 -60
- extensions/run_log_store/generic_chunked.py +17 -11
- extensions/run_log_store/minio.py +111 -0
- {runnable-0.26.0.dist-info → runnable-0.27.0.dist-info}/METADATA +1 -1
- {runnable-0.26.0.dist-info → runnable-0.27.0.dist-info}/RECORD +11 -9
- {runnable-0.26.0.dist-info → runnable-0.27.0.dist-info}/entry_points.txt +1 -0
- {runnable-0.26.0.dist-info → runnable-0.27.0.dist-info}/WHEEL +0 -0
- {runnable-0.26.0.dist-info → runnable-0.27.0.dist-info}/licenses/LICENSE +0 -0
extensions/catalog/minio.py
CHANGED
@@ -4,6 +4,7 @@ from pathlib import Path
|
|
4
4
|
from typing import Any
|
5
5
|
|
6
6
|
from cloudpathlib import CloudPath, S3Client, S3Path
|
7
|
+
from pydantic import Field, SecretStr
|
7
8
|
|
8
9
|
from extensions.catalog.any_path import AnyPathCatalog
|
9
10
|
from runnable import defaults
|
@@ -25,9 +26,9 @@ def get_minio_client(
|
|
25
26
|
class MinioCatalog(AnyPathCatalog):
|
26
27
|
service_name: str = "minio"
|
27
28
|
|
28
|
-
endpoint_url: str = "http://localhost:9002"
|
29
|
-
aws_access_key_id:
|
30
|
-
aws_secret_access_key:
|
29
|
+
endpoint_url: str = Field(default="http://localhost:9002")
|
30
|
+
aws_access_key_id: SecretStr = SecretStr(secret_value="minioadmin")
|
31
|
+
aws_secret_access_key: SecretStr = SecretStr(secret_value="minioadmin")
|
31
32
|
bucket: str = "runnable"
|
32
33
|
|
33
34
|
def get_summary(self) -> dict[str, Any]:
|
@@ -44,7 +45,9 @@ class MinioCatalog(AnyPathCatalog):
|
|
44
45
|
return S3Path(
|
45
46
|
f"s3://{self.bucket}/{run_id}/{self.compute_data_folder}".strip("."),
|
46
47
|
client=get_minio_client(
|
47
|
-
self.endpoint_url,
|
48
|
+
self.endpoint_url,
|
49
|
+
self.aws_access_key_id.get_secret_value(),
|
50
|
+
self.aws_secret_access_key.get_secret_value(),
|
48
51
|
),
|
49
52
|
)
|
50
53
|
|
@@ -0,0 +1,104 @@
|
|
1
|
+
import logging
|
2
|
+
from abc import abstractmethod
|
3
|
+
from typing import Any, Dict
|
4
|
+
|
5
|
+
from runnable import defaults, exceptions
|
6
|
+
from runnable.datastore import BaseRunLogStore, RunLog
|
7
|
+
|
8
|
+
logger = logging.getLogger(defaults.LOGGER_NAME)
|
9
|
+
|
10
|
+
|
11
|
+
class AnyPathRunLogStore(BaseRunLogStore):
|
12
|
+
"""
|
13
|
+
In this type of Run Log store, we use a file system to store the JSON run log.
|
14
|
+
|
15
|
+
Every single run is stored as a different file which makes it compatible across other store types.
|
16
|
+
|
17
|
+
When to use:
|
18
|
+
When locally testing a pipeline and have the need to compare across runs.
|
19
|
+
Its fully featured and perfectly fine if your local environment is where you would do everything.
|
20
|
+
|
21
|
+
Do not use:
|
22
|
+
If you need parallelization on local, this run log would not support it.
|
23
|
+
|
24
|
+
Example config:
|
25
|
+
|
26
|
+
run_log:
|
27
|
+
type: file-system
|
28
|
+
config:
|
29
|
+
log_folder: The folder to out the logs. Defaults to .run_log_store
|
30
|
+
|
31
|
+
"""
|
32
|
+
|
33
|
+
service_name: str = "file-system"
|
34
|
+
log_folder: str = defaults.LOG_LOCATION_FOLDER
|
35
|
+
|
36
|
+
@property
|
37
|
+
def log_folder_name(self):
|
38
|
+
return self.log_folder
|
39
|
+
|
40
|
+
def get_summary(self) -> Dict[str, Any]:
|
41
|
+
summary = {"Type": self.service_name, "Location": self.log_folder}
|
42
|
+
|
43
|
+
return summary
|
44
|
+
|
45
|
+
@abstractmethod
|
46
|
+
def write_to_path(self, run_log: RunLog): ...
|
47
|
+
|
48
|
+
@abstractmethod
|
49
|
+
def read_from_path(self, run_id: str) -> RunLog: ...
|
50
|
+
|
51
|
+
def create_run_log(
|
52
|
+
self,
|
53
|
+
run_id: str,
|
54
|
+
dag_hash: str = "",
|
55
|
+
use_cached: bool = False,
|
56
|
+
tag: str = "",
|
57
|
+
original_run_id: str = "",
|
58
|
+
status: str = defaults.CREATED,
|
59
|
+
) -> RunLog:
|
60
|
+
"""
|
61
|
+
# Creates a Run log
|
62
|
+
# Adds it to the db
|
63
|
+
"""
|
64
|
+
|
65
|
+
try:
|
66
|
+
self.get_run_log_by_id(run_id=run_id, full=False)
|
67
|
+
raise exceptions.RunLogExistsError(run_id=run_id)
|
68
|
+
except exceptions.RunLogNotFoundError:
|
69
|
+
pass
|
70
|
+
|
71
|
+
logger.info(f"{self.service_name} Creating a Run Log for : {run_id}")
|
72
|
+
run_log = RunLog(
|
73
|
+
run_id=run_id,
|
74
|
+
dag_hash=dag_hash,
|
75
|
+
tag=tag,
|
76
|
+
status=status,
|
77
|
+
)
|
78
|
+
self.write_to_path(run_log)
|
79
|
+
return run_log
|
80
|
+
|
81
|
+
def get_run_log_by_id(
|
82
|
+
self,
|
83
|
+
run_id: str,
|
84
|
+
full: bool = False,
|
85
|
+
) -> RunLog:
|
86
|
+
"""
|
87
|
+
# Returns the run_log defined by id
|
88
|
+
# Raises Exception if not found
|
89
|
+
"""
|
90
|
+
try:
|
91
|
+
logger.info(f"{self.service_name} Getting a Run Log for : {run_id}")
|
92
|
+
run_log = self.read_from_path(run_id)
|
93
|
+
return run_log
|
94
|
+
except FileNotFoundError as e:
|
95
|
+
raise exceptions.RunLogNotFoundError(run_id) from e
|
96
|
+
|
97
|
+
def put_run_log(self, run_log: RunLog):
|
98
|
+
"""
|
99
|
+
# Puts the run_log into the database
|
100
|
+
"""
|
101
|
+
logger.info(
|
102
|
+
f"{self.service_name} Putting the run log in the DB: {run_log.run_id}"
|
103
|
+
)
|
104
|
+
self.write_to_path(run_log)
|
@@ -2,14 +2,16 @@ import json
|
|
2
2
|
import logging
|
3
3
|
from pathlib import Path
|
4
4
|
from string import Template
|
5
|
-
from typing import Any, Dict, Optional,
|
5
|
+
from typing import Any, Dict, Optional, Union
|
6
|
+
|
7
|
+
from cloudpathlib import CloudPath
|
6
8
|
|
7
9
|
from extensions.run_log_store.generic_chunked import ChunkedRunLogStore
|
8
10
|
from runnable import defaults, utils
|
9
11
|
|
10
12
|
logger = logging.getLogger(defaults.LOGGER_NAME)
|
11
13
|
|
12
|
-
|
14
|
+
MixT = Union[CloudPath, Path]
|
13
15
|
|
14
16
|
|
15
17
|
class ChunkedFileSystemRunLogStore(ChunkedRunLogStore):
|
@@ -28,7 +30,7 @@ class ChunkedFileSystemRunLogStore(ChunkedRunLogStore):
|
|
28
30
|
|
29
31
|
def get_matches(
|
30
32
|
self, run_id: str, name: str, multiple_allowed: bool = False
|
31
|
-
) -> Optional[Union[
|
33
|
+
) -> Optional[Union[list[Path], list[CloudPath], MixT]]:
|
32
34
|
"""
|
33
35
|
Get contents of files matching the pattern name*
|
34
36
|
|
@@ -78,7 +80,7 @@ class ChunkedFileSystemRunLogStore(ChunkedRunLogStore):
|
|
78
80
|
|
79
81
|
return str(name) + ".json"
|
80
82
|
|
81
|
-
def _store(self, run_id: str, contents: dict, name:
|
83
|
+
def _store(self, run_id: str, contents: dict, name: MixT, insert=False):
|
82
84
|
"""
|
83
85
|
Store the contents against the name in the folder.
|
84
86
|
|
@@ -87,15 +89,16 @@ class ChunkedFileSystemRunLogStore(ChunkedRunLogStore):
|
|
87
89
|
contents (dict): The dict to store
|
88
90
|
name (str): The name to store as
|
89
91
|
"""
|
92
|
+
log_folder_with_run_id = self.log_folder_with_run_id(run_id=run_id)
|
90
93
|
if insert:
|
91
|
-
name =
|
94
|
+
name = log_folder_with_run_id / name
|
92
95
|
|
93
|
-
utils.safe_make_dir(
|
96
|
+
utils.safe_make_dir(log_folder_with_run_id)
|
94
97
|
|
95
|
-
with open(self.safe_suffix_json(name), "w") as fw:
|
98
|
+
with open(log_folder_with_run_id / self.safe_suffix_json(name.name), "w") as fw:
|
96
99
|
json.dump(contents, fw, ensure_ascii=True, indent=4)
|
97
100
|
|
98
|
-
def _retrieve(self,
|
101
|
+
def _retrieve(self, run_id: str, name: MixT) -> dict:
|
99
102
|
"""
|
100
103
|
Does the job of retrieving from the folder.
|
101
104
|
|
@@ -106,8 +109,9 @@ class ChunkedFileSystemRunLogStore(ChunkedRunLogStore):
|
|
106
109
|
dict: The contents
|
107
110
|
"""
|
108
111
|
contents: dict = {}
|
112
|
+
log_folder_with_run_id = self.log_folder_with_run_id(run_id=run_id)
|
109
113
|
|
110
|
-
with open(self.safe_suffix_json(name), "r") as fr:
|
114
|
+
with open(log_folder_with_run_id / self.safe_suffix_json(name.name), "r") as fr:
|
111
115
|
contents = json.load(fr)
|
112
116
|
|
113
117
|
return contents
|
@@ -3,13 +3,14 @@ import logging
|
|
3
3
|
from pathlib import Path
|
4
4
|
from typing import Any, Dict
|
5
5
|
|
6
|
-
from
|
7
|
-
from runnable
|
6
|
+
from extensions.run_log_store.any_path import AnyPathRunLogStore
|
7
|
+
from runnable import defaults, utils
|
8
|
+
from runnable.datastore import RunLog
|
8
9
|
|
9
10
|
logger = logging.getLogger(defaults.LOGGER_NAME)
|
10
11
|
|
11
12
|
|
12
|
-
class FileSystemRunLogstore(
|
13
|
+
class FileSystemRunLogstore(AnyPathRunLogStore):
|
13
14
|
"""
|
14
15
|
In this type of Run Log store, we use a file system to store the JSON run log.
|
15
16
|
|
@@ -43,7 +44,7 @@ class FileSystemRunLogstore(BaseRunLogStore):
|
|
43
44
|
|
44
45
|
return summary
|
45
46
|
|
46
|
-
def
|
47
|
+
def write_to_path(self, run_log: RunLog):
|
47
48
|
"""
|
48
49
|
Write the run log to the folder
|
49
50
|
|
@@ -60,7 +61,7 @@ class FileSystemRunLogstore(BaseRunLogStore):
|
|
60
61
|
with json_file_path.open("w") as fw:
|
61
62
|
json.dump(run_log.model_dump(), fw, ensure_ascii=True, indent=4) # pylint: disable=no-member
|
62
63
|
|
63
|
-
def
|
64
|
+
def read_from_path(self, run_id: str) -> RunLog:
|
64
65
|
"""
|
65
66
|
Look into the run log folder for the run log for the run id.
|
66
67
|
|
@@ -88,58 +89,3 @@ class FileSystemRunLogstore(BaseRunLogStore):
|
|
88
89
|
json_str = json.load(fr)
|
89
90
|
run_log = RunLog(**json_str) # pylint: disable=no-member
|
90
91
|
return run_log
|
91
|
-
|
92
|
-
def create_run_log(
|
93
|
-
self,
|
94
|
-
run_id: str,
|
95
|
-
dag_hash: str = "",
|
96
|
-
use_cached: bool = False,
|
97
|
-
tag: str = "",
|
98
|
-
original_run_id: str = "",
|
99
|
-
status: str = defaults.CREATED,
|
100
|
-
) -> RunLog:
|
101
|
-
"""
|
102
|
-
# Creates a Run log
|
103
|
-
# Adds it to the db
|
104
|
-
"""
|
105
|
-
|
106
|
-
try:
|
107
|
-
self.get_run_log_by_id(run_id=run_id, full=False)
|
108
|
-
raise exceptions.RunLogExistsError(run_id=run_id)
|
109
|
-
except exceptions.RunLogNotFoundError:
|
110
|
-
pass
|
111
|
-
|
112
|
-
logger.info(f"{self.service_name} Creating a Run Log for : {run_id}")
|
113
|
-
run_log = RunLog(
|
114
|
-
run_id=run_id,
|
115
|
-
dag_hash=dag_hash,
|
116
|
-
tag=tag,
|
117
|
-
status=status,
|
118
|
-
)
|
119
|
-
self.write_to_folder(run_log)
|
120
|
-
return run_log
|
121
|
-
|
122
|
-
def get_run_log_by_id(
|
123
|
-
self,
|
124
|
-
run_id: str,
|
125
|
-
full: bool = False,
|
126
|
-
) -> RunLog:
|
127
|
-
"""
|
128
|
-
# Returns the run_log defined by id
|
129
|
-
# Raises Exception if not found
|
130
|
-
"""
|
131
|
-
try:
|
132
|
-
logger.info(f"{self.service_name} Getting a Run Log for : {run_id}")
|
133
|
-
run_log = self.get_from_folder(run_id)
|
134
|
-
return run_log
|
135
|
-
except FileNotFoundError as e:
|
136
|
-
raise exceptions.RunLogNotFoundError(run_id) from e
|
137
|
-
|
138
|
-
def put_run_log(self, run_log: RunLog):
|
139
|
-
"""
|
140
|
-
# Puts the run_log into the database
|
141
|
-
"""
|
142
|
-
logger.info(
|
143
|
-
f"{self.service_name} Putting the run log in the DB: {run_log.run_id}"
|
144
|
-
)
|
145
|
-
self.write_to_folder(run_log)
|
@@ -4,7 +4,9 @@ from abc import abstractmethod
|
|
4
4
|
from enum import Enum
|
5
5
|
from pathlib import Path
|
6
6
|
from string import Template
|
7
|
-
from typing import Any, Dict, Optional,
|
7
|
+
from typing import Any, Dict, Optional, Union
|
8
|
+
|
9
|
+
from cloudpathlib import CloudPath
|
8
10
|
|
9
11
|
from runnable import defaults, exceptions
|
10
12
|
from runnable.datastore import (
|
@@ -21,7 +23,7 @@ from runnable.datastore import (
|
|
21
23
|
logger = logging.getLogger(defaults.LOGGER_NAME)
|
22
24
|
|
23
25
|
|
24
|
-
|
26
|
+
MixT = Union[CloudPath, Path] # Holds str, path
|
25
27
|
|
26
28
|
|
27
29
|
class EntityNotFoundError(Exception):
|
@@ -87,7 +89,7 @@ class ChunkedRunLogStore(BaseRunLogStore):
|
|
87
89
|
@abstractmethod
|
88
90
|
def get_matches(
|
89
91
|
self, run_id: str, name: str, multiple_allowed: bool = False
|
90
|
-
) -> Optional[Union[
|
92
|
+
) -> Optional[Union[list[Path], list[CloudPath], MixT]]:
|
91
93
|
"""
|
92
94
|
Get contents of persistence layer matching the pattern name*
|
93
95
|
|
@@ -98,7 +100,7 @@ class ChunkedRunLogStore(BaseRunLogStore):
|
|
98
100
|
...
|
99
101
|
|
100
102
|
@abstractmethod
|
101
|
-
def _store(self, run_id: str, contents: dict, name:
|
103
|
+
def _store(self, run_id: str, contents: dict, name: MixT, insert: bool = False):
|
102
104
|
"""
|
103
105
|
Store the contents against the name in the persistence layer.
|
104
106
|
|
@@ -110,7 +112,7 @@ class ChunkedRunLogStore(BaseRunLogStore):
|
|
110
112
|
...
|
111
113
|
|
112
114
|
@abstractmethod
|
113
|
-
def _retrieve(self, name:
|
115
|
+
def _retrieve(self, run_id: str, name: MixT) -> dict:
|
114
116
|
"""
|
115
117
|
Does the job of retrieving from the persistent layer.
|
116
118
|
|
@@ -140,7 +142,7 @@ class ChunkedRunLogStore(BaseRunLogStore):
|
|
140
142
|
insert = False
|
141
143
|
|
142
144
|
if match:
|
143
|
-
existing_contents = self._retrieve(name=match) # type: ignore
|
145
|
+
existing_contents = self._retrieve(run_id=run_id, name=match) # type: ignore
|
144
146
|
contents = dict(existing_contents, **contents)
|
145
147
|
name_to_give = match # type: ignore
|
146
148
|
else:
|
@@ -149,7 +151,9 @@ class ChunkedRunLogStore(BaseRunLogStore):
|
|
149
151
|
)
|
150
152
|
insert = True
|
151
153
|
|
152
|
-
self._store(
|
154
|
+
self._store(
|
155
|
+
run_id=run_id, contents=contents, name=Path(name_to_give), insert=insert
|
156
|
+
)
|
153
157
|
|
154
158
|
def retrieve(
|
155
159
|
self, run_id: str, log_type: LogTypes, name: str = "", multiple_allowed=False
|
@@ -190,13 +194,13 @@ class ChunkedRunLogStore(BaseRunLogStore):
|
|
190
194
|
|
191
195
|
if matches:
|
192
196
|
if not multiple_allowed:
|
193
|
-
contents = self._retrieve(name=matches) # type: ignore
|
197
|
+
contents = self._retrieve(run_id=run_id, name=matches) # type: ignore
|
194
198
|
model = self.ModelTypes[log_type.name].value
|
195
199
|
return model(**contents)
|
196
200
|
|
197
201
|
models = []
|
198
202
|
for match in matches: # type: ignore
|
199
|
-
contents = self._retrieve(name=match)
|
203
|
+
contents = self._retrieve(run_id=run_id, name=match)
|
200
204
|
model = self.ModelTypes[log_type.name].value
|
201
205
|
models.append(model(**contents))
|
202
206
|
return models
|
@@ -225,7 +229,9 @@ class ChunkedRunLogStore(BaseRunLogStore):
|
|
225
229
|
# No branch logs are found
|
226
230
|
return {}
|
227
231
|
# Forcing get_matches to always return a list is a better design
|
228
|
-
|
232
|
+
|
233
|
+
assert isinstance(matches, list)
|
234
|
+
epoch_created = [str(match).split("-")[-1] for match in matches]
|
229
235
|
|
230
236
|
# sort matches by epoch created
|
231
237
|
epoch_created, matches = zip(*sorted(zip(epoch_created, matches))) # type: ignore
|
@@ -234,7 +240,7 @@ class ChunkedRunLogStore(BaseRunLogStore):
|
|
234
240
|
|
235
241
|
for match in matches:
|
236
242
|
model = self.ModelTypes[log_type.name].value
|
237
|
-
log_model = model(**self._retrieve(match))
|
243
|
+
log_model = model(**self._retrieve(run_id=run_id, name=match))
|
238
244
|
logs[log_model.internal_name] = log_model # type: ignore
|
239
245
|
|
240
246
|
return logs
|
@@ -0,0 +1,111 @@
|
|
1
|
+
import json
|
2
|
+
import logging
|
3
|
+
from functools import lru_cache
|
4
|
+
from typing import Any, Dict
|
5
|
+
|
6
|
+
from cloudpathlib import S3Client, S3Path
|
7
|
+
from pydantic import Field, SecretStr
|
8
|
+
|
9
|
+
from extensions.run_log_store.any_path import AnyPathRunLogStore
|
10
|
+
from runnable import defaults
|
11
|
+
from runnable.datastore import RunLog
|
12
|
+
|
13
|
+
logger = logging.getLogger(defaults.LOGGER_NAME)
|
14
|
+
|
15
|
+
|
16
|
+
@lru_cache
|
17
|
+
def get_minio_client(
|
18
|
+
endpoint_url: str, aws_access_key_id: str, aws_secret_access_key: str
|
19
|
+
) -> S3Client:
|
20
|
+
return S3Client(
|
21
|
+
endpoint_url=endpoint_url,
|
22
|
+
aws_access_key_id=aws_access_key_id,
|
23
|
+
aws_secret_access_key=aws_secret_access_key,
|
24
|
+
)
|
25
|
+
|
26
|
+
|
27
|
+
class MinioRunLogStore(AnyPathRunLogStore):
|
28
|
+
"""
|
29
|
+
In this type of Run Log store, we use a file system to store the JSON run log.
|
30
|
+
|
31
|
+
Every single run is stored as a different file which makes it compatible across other store types.
|
32
|
+
|
33
|
+
When to use:
|
34
|
+
When locally testing a pipeline and have the need to compare across runs.
|
35
|
+
Its fully featured and perfectly fine if your local environment is where you would do everything.
|
36
|
+
|
37
|
+
Do not use:
|
38
|
+
If you need parallelization on local, this run log would not support it.
|
39
|
+
|
40
|
+
Example config:
|
41
|
+
|
42
|
+
run_log:
|
43
|
+
type: file-system
|
44
|
+
config:
|
45
|
+
log_folder: The folder to out the logs. Defaults to .run_log_store
|
46
|
+
|
47
|
+
"""
|
48
|
+
|
49
|
+
service_name: str = "file-system"
|
50
|
+
|
51
|
+
endpoint_url: str = Field(default="http://localhost:9002")
|
52
|
+
aws_access_key_id: SecretStr = SecretStr(secret_value="minioadmin")
|
53
|
+
aws_secret_access_key: SecretStr = SecretStr(secret_value="minioadmin")
|
54
|
+
bucket: str = Field(default="runnable/run-logs")
|
55
|
+
|
56
|
+
def get_summary(self) -> Dict[str, Any]:
|
57
|
+
summary = {"Type": self.service_name, "Location": self.log_folder}
|
58
|
+
|
59
|
+
return summary
|
60
|
+
|
61
|
+
def get_run_log_bucket(self) -> S3Path:
|
62
|
+
run_id = self._context.run_id
|
63
|
+
|
64
|
+
return S3Path(
|
65
|
+
f"s3://{self.bucket}/{run_id}/",
|
66
|
+
client=get_minio_client(
|
67
|
+
self.endpoint_url,
|
68
|
+
self.aws_access_key_id.get_secret_value(),
|
69
|
+
self.aws_secret_access_key.get_secret_value(),
|
70
|
+
),
|
71
|
+
)
|
72
|
+
|
73
|
+
def write_to_path(self, run_log: RunLog):
|
74
|
+
"""
|
75
|
+
Write the run log to the folder
|
76
|
+
|
77
|
+
Args:
|
78
|
+
run_log (RunLog): The run log to be added to the database
|
79
|
+
"""
|
80
|
+
run_log_bucket = self.get_run_log_bucket()
|
81
|
+
run_log_bucket.mkdir(parents=True, exist_ok=True)
|
82
|
+
|
83
|
+
run_log_object = run_log_bucket / f"{run_log.run_id}.json"
|
84
|
+
run_log_object.write_text(
|
85
|
+
json.dumps(run_log.model_dump_json(), ensure_ascii=True, indent=4)
|
86
|
+
)
|
87
|
+
|
88
|
+
def read_from_path(self, run_id: str) -> RunLog:
|
89
|
+
"""
|
90
|
+
Look into the run log folder for the run log for the run id.
|
91
|
+
|
92
|
+
If the run log does not exist, raise an exception. If it does, decode it
|
93
|
+
as a RunLog and return it
|
94
|
+
|
95
|
+
Args:
|
96
|
+
run_id (str): The requested run id to retrieve the run log store
|
97
|
+
|
98
|
+
Raises:
|
99
|
+
FileNotFoundError: If the Run Log has not been found.
|
100
|
+
|
101
|
+
Returns:
|
102
|
+
RunLog: The decoded Run log
|
103
|
+
"""
|
104
|
+
run_log_bucket = self.get_run_log_bucket()
|
105
|
+
|
106
|
+
run_log_object = run_log_bucket / f"{run_id}.json"
|
107
|
+
|
108
|
+
run_log_text = json.loads(run_log_object.read_text())
|
109
|
+
run_log = RunLog(**json.loads(run_log_text))
|
110
|
+
|
111
|
+
return run_log
|
@@ -3,7 +3,7 @@ extensions/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
3
3
|
extensions/catalog/README.md,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
4
4
|
extensions/catalog/any_path.py,sha256=aNjphoPIyllUfY2uNDFWD1ErM3Px6izSGr0-oGowN8k,7263
|
5
5
|
extensions/catalog/file_system.py,sha256=T_qFPFfrmykoAMc1rjNi_DBb437me8WPRcFglwAK744,1767
|
6
|
-
extensions/catalog/minio.py,sha256=
|
6
|
+
extensions/catalog/minio.py,sha256=R3GvfCxN1GTcs4bQIAWh79_GHDTVd14gnpKlzwFeKUI,2363
|
7
7
|
extensions/catalog/pyproject.toml,sha256=lLNxY6v04c8I5QK_zKw_E6sJTArSJRA_V-79ktaA3Hk,279
|
8
8
|
extensions/catalog/s3.py,sha256=Sw5t8_kVRprn3uGGJCiHn7M9zw1CLaCOFj6YErtfG0o,287
|
9
9
|
extensions/job_executor/README.md,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
@@ -26,9 +26,11 @@ extensions/pipeline_executor/pyproject.toml,sha256=ykTX7srR10PBYb8LsIwEj8vIPPIEZ
|
|
26
26
|
extensions/pipeline_executor/retry.py,sha256=KGenhWrLLmOQgzMvqloXHDRJyoNs91t05rRW8aLW6FA,6969
|
27
27
|
extensions/run_log_store/README.md,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
28
28
|
extensions/run_log_store/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
29
|
-
extensions/run_log_store/
|
30
|
-
extensions/run_log_store/
|
31
|
-
extensions/run_log_store/
|
29
|
+
extensions/run_log_store/any_path.py,sha256=2Ff9-jd1mWazFXn4HvsF0MF-oQAdMNYrl698Kjpip4Q,2938
|
30
|
+
extensions/run_log_store/chunked_fs.py,sha256=_2KmTaxK-p1e_YJqnQwyOqkLoCr80y-Wg1X2XdCC_9k,3546
|
31
|
+
extensions/run_log_store/file_system.py,sha256=hhrbhSnuzv8yzBr6DAu45NT8-sawPP86WA2-LY70vjw,2781
|
32
|
+
extensions/run_log_store/generic_chunked.py,sha256=D08ADWK_rw4ed46_yeEankO-g9rwOjVYi4V-JaOOT6M,20445
|
33
|
+
extensions/run_log_store/minio.py,sha256=pTHUzCHnehJ5JZoIrlZGjb4Cl-6yinp3MgqLZPvIxq4,3403
|
32
34
|
extensions/run_log_store/pyproject.toml,sha256=YnmXsFvFG9uv_c0spLYBsNI_1sbktqxtHsOuClyvZ3g,288
|
33
35
|
extensions/run_log_store/db/implementation_FF.py,sha256=euTnh0xzNF0e_DyfHQ4W-kG1AwTr8u7OuO3_cZkR5bM,5237
|
34
36
|
extensions/run_log_store/db/integration_FF.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
@@ -53,8 +55,8 @@ runnable/sdk.py,sha256=T1nqDpLN9fULvvU9L-oY0EHqYdKUI9qk7oekLynm02Y,33568
|
|
53
55
|
runnable/secrets.py,sha256=PXcEJw-4WPzeWRLfsatcPPyr1zkqgHzdRWRcS9vvpvM,2354
|
54
56
|
runnable/tasks.py,sha256=X6xijut7ffwpfYDcXoN6y0AcRVd7fWHs676DJ00Kma4,29134
|
55
57
|
runnable/utils.py,sha256=hBr7oGwGL2VgfITlQCTz-a1iwvvf7Mfl-HY8UdENZac,19929
|
56
|
-
runnable-0.
|
57
|
-
runnable-0.
|
58
|
-
runnable-0.
|
59
|
-
runnable-0.
|
60
|
-
runnable-0.
|
58
|
+
runnable-0.27.0.dist-info/METADATA,sha256=Bmncq3RAxN2eZGtlWQf5X8O0CkWRqmiCQiY-rH90odM,10047
|
59
|
+
runnable-0.27.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
|
60
|
+
runnable-0.27.0.dist-info/entry_points.txt,sha256=VZ-VOpcr16mj9jwxxMycwGNJlG_cziICRIe-LQrDwig,1760
|
61
|
+
runnable-0.27.0.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
|
62
|
+
runnable-0.27.0.dist-info/RECORD,,
|
@@ -36,6 +36,7 @@ retry = extensions.pipeline_executor.retry:RetryExecutor
|
|
36
36
|
buffered = runnable.datastore:BufferRunLogstore
|
37
37
|
chunked-fs = extensions.run_log_store.chunked_fs:ChunkedFileSystemRunLogStore
|
38
38
|
file-system = extensions.run_log_store.file_system:FileSystemRunLogstore
|
39
|
+
minio = extensions.run_log_store.minio:MinioRunLogStore
|
39
40
|
|
40
41
|
[secrets]
|
41
42
|
do-nothing = runnable.secrets:DoNothingSecretManager
|
File without changes
|
File without changes
|