runnable 0.17.1__py3-none-any.whl → 0.19.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (47) hide show
  1. extensions/README.md +0 -0
  2. extensions/__init__.py +0 -0
  3. extensions/catalog/README.md +0 -0
  4. extensions/catalog/file_system.py +253 -0
  5. extensions/catalog/pyproject.toml +14 -0
  6. extensions/job_executor/README.md +0 -0
  7. extensions/job_executor/__init__.py +160 -0
  8. extensions/job_executor/k8s.py +484 -0
  9. extensions/job_executor/k8s_job_spec.yaml +37 -0
  10. extensions/job_executor/local.py +61 -0
  11. extensions/job_executor/local_container.py +192 -0
  12. extensions/job_executor/pyproject.toml +16 -0
  13. extensions/nodes/README.md +0 -0
  14. extensions/nodes/nodes.py +954 -0
  15. extensions/nodes/pyproject.toml +15 -0
  16. extensions/pipeline_executor/README.md +0 -0
  17. extensions/pipeline_executor/__init__.py +644 -0
  18. extensions/pipeline_executor/argo.py +1307 -0
  19. extensions/pipeline_executor/argo_specification.yaml +51 -0
  20. extensions/pipeline_executor/local.py +62 -0
  21. extensions/pipeline_executor/local_container.py +362 -0
  22. extensions/pipeline_executor/mocked.py +161 -0
  23. extensions/pipeline_executor/pyproject.toml +16 -0
  24. extensions/pipeline_executor/retry.py +180 -0
  25. extensions/run_log_store/README.md +0 -0
  26. extensions/run_log_store/__init__.py +0 -0
  27. extensions/run_log_store/chunked_fs.py +113 -0
  28. extensions/run_log_store/db/implementation_FF.py +163 -0
  29. extensions/run_log_store/db/integration_FF.py +0 -0
  30. extensions/run_log_store/file_system.py +145 -0
  31. extensions/run_log_store/generic_chunked.py +599 -0
  32. extensions/run_log_store/pyproject.toml +15 -0
  33. extensions/secrets/README.md +0 -0
  34. extensions/secrets/dotenv.py +62 -0
  35. extensions/secrets/pyproject.toml +15 -0
  36. runnable/__init__.py +1 -0
  37. runnable/catalog.py +1 -2
  38. runnable/entrypoints.py +1 -5
  39. runnable/executor.py +1 -1
  40. runnable/parameters.py +0 -9
  41. runnable/utils.py +5 -25
  42. {runnable-0.17.1.dist-info → runnable-0.19.0.dist-info}/METADATA +1 -7
  43. runnable-0.19.0.dist-info/RECORD +58 -0
  44. {runnable-0.17.1.dist-info → runnable-0.19.0.dist-info}/entry_points.txt +1 -0
  45. runnable-0.17.1.dist-info/RECORD +0 -23
  46. {runnable-0.17.1.dist-info → runnable-0.19.0.dist-info}/WHEEL +0 -0
  47. {runnable-0.17.1.dist-info → runnable-0.19.0.dist-info}/licenses/LICENSE +0 -0
@@ -0,0 +1,192 @@
1
+ import logging
2
+ from pathlib import Path
3
+ from typing import Dict, List, Optional
4
+
5
+ from pydantic import Field
6
+
7
+ from extensions.job_executor import GenericJobExecutor
8
+ from runnable import console, defaults, utils
9
+ from runnable.datastore import DataCatalog
10
+ from runnable.tasks import BaseTaskType
11
+
12
+ logger = logging.getLogger(defaults.LOGGER_NAME)
13
+
14
+
15
+ class LocalContainerJobExecutor(GenericJobExecutor):
16
+ """
17
+ The LocalJobExecutor is a job executor that runs the job locally.
18
+ """
19
+
20
+ service_name: str = "local-container"
21
+ docker_image: str
22
+ mock: bool = False
23
+ auto_remove_container: bool = True
24
+ environment: Dict[str, str] = Field(default_factory=dict)
25
+
26
+ _is_local: bool = False
27
+
28
+ _container_log_location = "/tmp/run_logs/"
29
+ _container_catalog_location = "/tmp/catalog/"
30
+ _container_secrets_location = "/tmp/dotenv"
31
+ _volumes: Dict[str, Dict[str, str]] = {}
32
+
33
+ def submit_job(self, job: BaseTaskType, catalog_settings=Optional[List[str]]):
34
+ """
35
+ This method gets invoked by the CLI.
36
+ """
37
+ self._set_up_run_log()
38
+ self._mount_volumes()
39
+
40
+ # Call the container job
41
+ job_log = self._context.run_log_store.create_job_log()
42
+ self._context.run_log_store.add_job_log(
43
+ run_id=self._context.run_id, job_log=job_log
44
+ )
45
+ self.spin_container()
46
+
47
+ def execute_job(self, job: BaseTaskType, catalog_settings=Optional[List[str]]):
48
+ """
49
+ Focusses on execution of the job.
50
+ """
51
+ self._use_volumes()
52
+ logger.info("Trying to execute job")
53
+
54
+ job_log = self._context.run_log_store.get_job_log(run_id=self._context.run_id)
55
+
56
+ attempt_log = job.execute_command(
57
+ attempt_number=self.step_attempt_number,
58
+ mock=self.mock,
59
+ )
60
+
61
+ job_log.status = attempt_log.status
62
+ job_log.attempts.append(attempt_log)
63
+
64
+ data_catalogs_put: Optional[List[DataCatalog]] = self._sync_catalog(
65
+ catalog_settings=catalog_settings
66
+ )
67
+ logger.debug(f"data_catalogs_put: {data_catalogs_put}")
68
+
69
+ job_log.add_data_catalogs(data_catalogs_put or [])
70
+
71
+ console.print("Summary of job")
72
+ console.print(job_log.get_summary())
73
+
74
+ self._context.run_log_store.add_job_log(
75
+ run_id=self._context.run_id, job_log=job_log
76
+ )
77
+
78
+ def spin_container(self):
79
+ """
80
+ This method spins up the container
81
+ """
82
+ import docker # pylint: disable=C0415
83
+
84
+ try:
85
+ client = docker.from_env()
86
+ api_client = docker.APIClient()
87
+ except Exception as ex:
88
+ logger.exception("Could not get access to docker")
89
+ raise Exception(
90
+ "Could not get the docker socket file, do you have docker installed?"
91
+ ) from ex
92
+
93
+ try:
94
+ command = utils.get_job_execution_command()
95
+ logger.info(f"Running the command {command}")
96
+ print(command)
97
+
98
+ docker_image = self.docker_image
99
+ environment = self.environment
100
+
101
+ container = client.containers.create(
102
+ image=docker_image,
103
+ command=command,
104
+ auto_remove=False,
105
+ volumes=self._volumes,
106
+ network_mode="host",
107
+ environment=environment,
108
+ )
109
+
110
+ # print(container.__dict__)
111
+
112
+ container.start()
113
+ stream = api_client.logs(
114
+ container=container.id, timestamps=True, stream=True, follow=True
115
+ )
116
+ while True:
117
+ try:
118
+ output = next(stream).decode("utf-8")
119
+ output = output.strip("\r\n")
120
+ logger.info(output)
121
+ print(output)
122
+ except StopIteration:
123
+ logger.info("Docker Run completed")
124
+ break
125
+
126
+ exit_status = api_client.inspect_container(container.id)["State"][
127
+ "ExitCode"
128
+ ]
129
+
130
+ if self.auto_remove_container:
131
+ container.remove(force=True)
132
+
133
+ if exit_status != 0:
134
+ msg = f"Docker command failed with exit code {exit_status}"
135
+ raise Exception(msg)
136
+
137
+ except Exception as _e:
138
+ logger.exception("Problems with spinning/running the container")
139
+ raise _e
140
+
141
+ def _mount_volumes(self):
142
+ """
143
+ Mount the volumes for the container
144
+ """
145
+ match self._context.run_log_store.service_name:
146
+ case "file-system":
147
+ write_to = self._context.run_log_store.log_folder
148
+ self._volumes[str(Path(write_to).resolve())] = {
149
+ "bind": f"{self._container_log_location}",
150
+ "mode": "rw",
151
+ }
152
+ case "chunked-fs":
153
+ write_to = self._context.run_log_store.log_folder
154
+ self._volumes[str(Path(write_to).resolve())] = {
155
+ "bind": f"{self._container_log_location}",
156
+ "mode": "rw",
157
+ }
158
+
159
+ match self._context.catalog_handler.service_name:
160
+ case "file-system":
161
+ catalog_location = self._context.catalog_handler.catalog_location
162
+ self._volumes[str(Path(catalog_location).resolve())] = {
163
+ "bind": f"{self._container_catalog_location}",
164
+ "mode": "rw",
165
+ }
166
+
167
+ match self._context.secrets_handler.service_name:
168
+ case "dotenv":
169
+ secrets_location = self._context.secrets_handler.location
170
+ self._volumes[str(Path(secrets_location).resolve())] = {
171
+ "bind": f"{self._container_secrets_location}",
172
+ "mode": "ro",
173
+ }
174
+
175
+ def _use_volumes(self):
176
+ match self._context.run_log_store.service_name:
177
+ case "file-system":
178
+ self._context.run_log_store.log_folder = self._container_log_location
179
+ case "chunked-fs":
180
+ self._context.run_log_store.log_folder = self._container_log_location
181
+
182
+ match self._context.catalog_handler.service_name:
183
+ case "file-system":
184
+ self._context.catalog_handler.catalog_location = (
185
+ self._container_catalog_location
186
+ )
187
+
188
+ match self._context.secrets_handler.service_name:
189
+ case "dotenv":
190
+ self._context.secrets_handler.location = (
191
+ self._container_secrets_location
192
+ )
@@ -0,0 +1,16 @@
1
+ [project]
2
+ name = "job_executor"
3
+ version = "0.0.0"
4
+ description = "Add your description here"
5
+ readme = "README.md"
6
+ requires-python = ">=3.10"
7
+ dependencies = []
8
+
9
+
10
+ [build-system]
11
+ requires = ["hatchling"]
12
+ build-backend = "hatchling.build"
13
+
14
+
15
+ [tool.hatch.build.targets.wheel]
16
+ packages = ["."]
File without changes