runnable 0.17.1__py3-none-any.whl → 0.19.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- extensions/README.md +0 -0
- extensions/__init__.py +0 -0
- extensions/catalog/README.md +0 -0
- extensions/catalog/file_system.py +253 -0
- extensions/catalog/pyproject.toml +14 -0
- extensions/job_executor/README.md +0 -0
- extensions/job_executor/__init__.py +160 -0
- extensions/job_executor/k8s.py +484 -0
- extensions/job_executor/k8s_job_spec.yaml +37 -0
- extensions/job_executor/local.py +61 -0
- extensions/job_executor/local_container.py +192 -0
- extensions/job_executor/pyproject.toml +16 -0
- extensions/nodes/README.md +0 -0
- extensions/nodes/nodes.py +954 -0
- extensions/nodes/pyproject.toml +15 -0
- extensions/pipeline_executor/README.md +0 -0
- extensions/pipeline_executor/__init__.py +644 -0
- extensions/pipeline_executor/argo.py +1307 -0
- extensions/pipeline_executor/argo_specification.yaml +51 -0
- extensions/pipeline_executor/local.py +62 -0
- extensions/pipeline_executor/local_container.py +362 -0
- extensions/pipeline_executor/mocked.py +161 -0
- extensions/pipeline_executor/pyproject.toml +16 -0
- extensions/pipeline_executor/retry.py +180 -0
- extensions/run_log_store/README.md +0 -0
- extensions/run_log_store/__init__.py +0 -0
- extensions/run_log_store/chunked_fs.py +113 -0
- extensions/run_log_store/db/implementation_FF.py +163 -0
- extensions/run_log_store/db/integration_FF.py +0 -0
- extensions/run_log_store/file_system.py +145 -0
- extensions/run_log_store/generic_chunked.py +599 -0
- extensions/run_log_store/pyproject.toml +15 -0
- extensions/secrets/README.md +0 -0
- extensions/secrets/dotenv.py +62 -0
- extensions/secrets/pyproject.toml +15 -0
- runnable/__init__.py +1 -0
- runnable/catalog.py +1 -2
- runnable/entrypoints.py +1 -5
- runnable/executor.py +1 -1
- runnable/parameters.py +0 -9
- runnable/utils.py +5 -25
- {runnable-0.17.1.dist-info → runnable-0.19.0.dist-info}/METADATA +1 -7
- runnable-0.19.0.dist-info/RECORD +58 -0
- {runnable-0.17.1.dist-info → runnable-0.19.0.dist-info}/entry_points.txt +1 -0
- runnable-0.17.1.dist-info/RECORD +0 -23
- {runnable-0.17.1.dist-info → runnable-0.19.0.dist-info}/WHEEL +0 -0
- {runnable-0.17.1.dist-info → runnable-0.19.0.dist-info}/licenses/LICENSE +0 -0
@@ -0,0 +1,192 @@
|
|
1
|
+
import logging
|
2
|
+
from pathlib import Path
|
3
|
+
from typing import Dict, List, Optional
|
4
|
+
|
5
|
+
from pydantic import Field
|
6
|
+
|
7
|
+
from extensions.job_executor import GenericJobExecutor
|
8
|
+
from runnable import console, defaults, utils
|
9
|
+
from runnable.datastore import DataCatalog
|
10
|
+
from runnable.tasks import BaseTaskType
|
11
|
+
|
12
|
+
logger = logging.getLogger(defaults.LOGGER_NAME)
|
13
|
+
|
14
|
+
|
15
|
+
class LocalContainerJobExecutor(GenericJobExecutor):
|
16
|
+
"""
|
17
|
+
The LocalJobExecutor is a job executor that runs the job locally.
|
18
|
+
"""
|
19
|
+
|
20
|
+
service_name: str = "local-container"
|
21
|
+
docker_image: str
|
22
|
+
mock: bool = False
|
23
|
+
auto_remove_container: bool = True
|
24
|
+
environment: Dict[str, str] = Field(default_factory=dict)
|
25
|
+
|
26
|
+
_is_local: bool = False
|
27
|
+
|
28
|
+
_container_log_location = "/tmp/run_logs/"
|
29
|
+
_container_catalog_location = "/tmp/catalog/"
|
30
|
+
_container_secrets_location = "/tmp/dotenv"
|
31
|
+
_volumes: Dict[str, Dict[str, str]] = {}
|
32
|
+
|
33
|
+
def submit_job(self, job: BaseTaskType, catalog_settings=Optional[List[str]]):
|
34
|
+
"""
|
35
|
+
This method gets invoked by the CLI.
|
36
|
+
"""
|
37
|
+
self._set_up_run_log()
|
38
|
+
self._mount_volumes()
|
39
|
+
|
40
|
+
# Call the container job
|
41
|
+
job_log = self._context.run_log_store.create_job_log()
|
42
|
+
self._context.run_log_store.add_job_log(
|
43
|
+
run_id=self._context.run_id, job_log=job_log
|
44
|
+
)
|
45
|
+
self.spin_container()
|
46
|
+
|
47
|
+
def execute_job(self, job: BaseTaskType, catalog_settings=Optional[List[str]]):
|
48
|
+
"""
|
49
|
+
Focusses on execution of the job.
|
50
|
+
"""
|
51
|
+
self._use_volumes()
|
52
|
+
logger.info("Trying to execute job")
|
53
|
+
|
54
|
+
job_log = self._context.run_log_store.get_job_log(run_id=self._context.run_id)
|
55
|
+
|
56
|
+
attempt_log = job.execute_command(
|
57
|
+
attempt_number=self.step_attempt_number,
|
58
|
+
mock=self.mock,
|
59
|
+
)
|
60
|
+
|
61
|
+
job_log.status = attempt_log.status
|
62
|
+
job_log.attempts.append(attempt_log)
|
63
|
+
|
64
|
+
data_catalogs_put: Optional[List[DataCatalog]] = self._sync_catalog(
|
65
|
+
catalog_settings=catalog_settings
|
66
|
+
)
|
67
|
+
logger.debug(f"data_catalogs_put: {data_catalogs_put}")
|
68
|
+
|
69
|
+
job_log.add_data_catalogs(data_catalogs_put or [])
|
70
|
+
|
71
|
+
console.print("Summary of job")
|
72
|
+
console.print(job_log.get_summary())
|
73
|
+
|
74
|
+
self._context.run_log_store.add_job_log(
|
75
|
+
run_id=self._context.run_id, job_log=job_log
|
76
|
+
)
|
77
|
+
|
78
|
+
def spin_container(self):
|
79
|
+
"""
|
80
|
+
This method spins up the container
|
81
|
+
"""
|
82
|
+
import docker # pylint: disable=C0415
|
83
|
+
|
84
|
+
try:
|
85
|
+
client = docker.from_env()
|
86
|
+
api_client = docker.APIClient()
|
87
|
+
except Exception as ex:
|
88
|
+
logger.exception("Could not get access to docker")
|
89
|
+
raise Exception(
|
90
|
+
"Could not get the docker socket file, do you have docker installed?"
|
91
|
+
) from ex
|
92
|
+
|
93
|
+
try:
|
94
|
+
command = utils.get_job_execution_command()
|
95
|
+
logger.info(f"Running the command {command}")
|
96
|
+
print(command)
|
97
|
+
|
98
|
+
docker_image = self.docker_image
|
99
|
+
environment = self.environment
|
100
|
+
|
101
|
+
container = client.containers.create(
|
102
|
+
image=docker_image,
|
103
|
+
command=command,
|
104
|
+
auto_remove=False,
|
105
|
+
volumes=self._volumes,
|
106
|
+
network_mode="host",
|
107
|
+
environment=environment,
|
108
|
+
)
|
109
|
+
|
110
|
+
# print(container.__dict__)
|
111
|
+
|
112
|
+
container.start()
|
113
|
+
stream = api_client.logs(
|
114
|
+
container=container.id, timestamps=True, stream=True, follow=True
|
115
|
+
)
|
116
|
+
while True:
|
117
|
+
try:
|
118
|
+
output = next(stream).decode("utf-8")
|
119
|
+
output = output.strip("\r\n")
|
120
|
+
logger.info(output)
|
121
|
+
print(output)
|
122
|
+
except StopIteration:
|
123
|
+
logger.info("Docker Run completed")
|
124
|
+
break
|
125
|
+
|
126
|
+
exit_status = api_client.inspect_container(container.id)["State"][
|
127
|
+
"ExitCode"
|
128
|
+
]
|
129
|
+
|
130
|
+
if self.auto_remove_container:
|
131
|
+
container.remove(force=True)
|
132
|
+
|
133
|
+
if exit_status != 0:
|
134
|
+
msg = f"Docker command failed with exit code {exit_status}"
|
135
|
+
raise Exception(msg)
|
136
|
+
|
137
|
+
except Exception as _e:
|
138
|
+
logger.exception("Problems with spinning/running the container")
|
139
|
+
raise _e
|
140
|
+
|
141
|
+
def _mount_volumes(self):
|
142
|
+
"""
|
143
|
+
Mount the volumes for the container
|
144
|
+
"""
|
145
|
+
match self._context.run_log_store.service_name:
|
146
|
+
case "file-system":
|
147
|
+
write_to = self._context.run_log_store.log_folder
|
148
|
+
self._volumes[str(Path(write_to).resolve())] = {
|
149
|
+
"bind": f"{self._container_log_location}",
|
150
|
+
"mode": "rw",
|
151
|
+
}
|
152
|
+
case "chunked-fs":
|
153
|
+
write_to = self._context.run_log_store.log_folder
|
154
|
+
self._volumes[str(Path(write_to).resolve())] = {
|
155
|
+
"bind": f"{self._container_log_location}",
|
156
|
+
"mode": "rw",
|
157
|
+
}
|
158
|
+
|
159
|
+
match self._context.catalog_handler.service_name:
|
160
|
+
case "file-system":
|
161
|
+
catalog_location = self._context.catalog_handler.catalog_location
|
162
|
+
self._volumes[str(Path(catalog_location).resolve())] = {
|
163
|
+
"bind": f"{self._container_catalog_location}",
|
164
|
+
"mode": "rw",
|
165
|
+
}
|
166
|
+
|
167
|
+
match self._context.secrets_handler.service_name:
|
168
|
+
case "dotenv":
|
169
|
+
secrets_location = self._context.secrets_handler.location
|
170
|
+
self._volumes[str(Path(secrets_location).resolve())] = {
|
171
|
+
"bind": f"{self._container_secrets_location}",
|
172
|
+
"mode": "ro",
|
173
|
+
}
|
174
|
+
|
175
|
+
def _use_volumes(self):
|
176
|
+
match self._context.run_log_store.service_name:
|
177
|
+
case "file-system":
|
178
|
+
self._context.run_log_store.log_folder = self._container_log_location
|
179
|
+
case "chunked-fs":
|
180
|
+
self._context.run_log_store.log_folder = self._container_log_location
|
181
|
+
|
182
|
+
match self._context.catalog_handler.service_name:
|
183
|
+
case "file-system":
|
184
|
+
self._context.catalog_handler.catalog_location = (
|
185
|
+
self._container_catalog_location
|
186
|
+
)
|
187
|
+
|
188
|
+
match self._context.secrets_handler.service_name:
|
189
|
+
case "dotenv":
|
190
|
+
self._context.secrets_handler.location = (
|
191
|
+
self._container_secrets_location
|
192
|
+
)
|
@@ -0,0 +1,16 @@
|
|
1
|
+
[project]
|
2
|
+
name = "job_executor"
|
3
|
+
version = "0.0.0"
|
4
|
+
description = "Add your description here"
|
5
|
+
readme = "README.md"
|
6
|
+
requires-python = ">=3.10"
|
7
|
+
dependencies = []
|
8
|
+
|
9
|
+
|
10
|
+
[build-system]
|
11
|
+
requires = ["hatchling"]
|
12
|
+
build-backend = "hatchling.build"
|
13
|
+
|
14
|
+
|
15
|
+
[tool.hatch.build.targets.wheel]
|
16
|
+
packages = ["."]
|
File without changes
|