odctl 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (57) hide show
  1. odctl/__init__.py +0 -0
  2. odctl/config.py +65 -0
  3. odctl/docker.py +295 -0
  4. odctl/main.py +520 -0
  5. odctl/planner.py +133 -0
  6. odctl/registry.py +66 -0
  7. odctl/resources/clickhouse/config.xml +52 -0
  8. odctl/resources/clickhouse/init.sql +2 -0
  9. odctl/resources/clickhouse/keeper_config.xml +26 -0
  10. odctl/resources/clickhouse/macros-11.xml +1 -0
  11. odctl/resources/clickhouse/macros-12.xml +1 -0
  12. odctl/resources/clickhouse/macros-21.xml +1 -0
  13. odctl/resources/clickhouse/macros-22.xml +1 -0
  14. odctl/resources/clickhouse/users.xml +56 -0
  15. odctl/resources/compose-analytics.yml +243 -0
  16. odctl/resources/compose-deps.yml +24 -0
  17. odctl/resources/compose-flink.yml +102 -0
  18. odctl/resources/compose-infra.yml +144 -0
  19. odctl/resources/compose-kafka.yml +206 -0
  20. odctl/resources/compose-metadata.yml +160 -0
  21. odctl/resources/compose-mlops.yml +74 -0
  22. odctl/resources/compose-obsv.yml +115 -0
  23. odctl/resources/compose-orch.yml +126 -0
  24. odctl/resources/compose-spark.yml +95 -0
  25. odctl/resources/compose-store.yml +103 -0
  26. odctl/resources/deps/Dockerfile +72 -0
  27. odctl/resources/deps/build.gradle +51 -0
  28. odctl/resources/deps/build_connectors.sh +71 -0
  29. odctl/resources/deps/fetch_flink_dependencies.sh +47 -0
  30. odctl/resources/deps/fetch_spark_dependencies.sh +40 -0
  31. odctl/resources/deps/fetch_standalone_plugins.sh +63 -0
  32. odctl/resources/docker/airflow/Dockerfile +23 -0
  33. odctl/resources/docker/ray-mlops/Dockerfile +62 -0
  34. odctl/resources/docker/spark/Dockerfile +16 -0
  35. odctl/resources/flink/flink1-conf.yml +58 -0
  36. odctl/resources/grafana/prometheus.yml +8 -0
  37. odctl/resources/marquez/marquez.yml +20 -0
  38. odctl/resources/mlops/serving/lightgbm_inference.py.example +42 -0
  39. odctl/resources/mlops/serving/vector_search.py.example +44 -0
  40. odctl/resources/postgres/01-init-databases.sh +47 -0
  41. odctl/resources/prometheus/alertmanager.yml +6 -0
  42. odctl/resources/prometheus/prometheus.yml +32 -0
  43. odctl/resources/registry.yml +289 -0
  44. odctl/resources/seaweed/s3.json +15 -0
  45. odctl/resources/spark/log4j2.properties +13 -0
  46. odctl/resources/spark/spark-defaults.conf +25 -0
  47. odctl/resources/trino/catalog/clickhouse.properties +4 -0
  48. odctl/resources/trino/catalog/iceberg.properties +9 -0
  49. odctl/resources/trino/catalog/postgres.properties +4 -0
  50. odctl/resources/trino/catalog/valkey.properties +6 -0
  51. odctl/ui.py +373 -0
  52. odctl/workspace.py +74 -0
  53. odctl-0.1.0.dist-info/METADATA +211 -0
  54. odctl-0.1.0.dist-info/RECORD +57 -0
  55. odctl-0.1.0.dist-info/WHEEL +4 -0
  56. odctl-0.1.0.dist-info/entry_points.txt +2 -0
  57. odctl-0.1.0.dist-info/licenses/LICENSE +201 -0
odctl/__init__.py ADDED
File without changes
odctl/config.py ADDED
@@ -0,0 +1,65 @@
1
+ import sys
2
+ from pathlib import Path
3
+
4
+
5
+ def get_internal_resources_dir() -> Path:
6
+ """
7
+ Determine the path to the bundled compose files, supporting PyInstaller.
8
+
9
+ Returns:
10
+ Path: The absolute path to the internal `resources` directory.
11
+ """
12
+ if getattr(sys, "frozen", False) and hasattr(sys, "_MEIPASS"):
13
+ # Running as a compiled PyInstaller binary
14
+ return Path(sys._MEIPASS) / "odctl" / "resources"
15
+ # Running as a normal Python script
16
+ return Path(__file__).parent.resolve() / "resources"
17
+
18
+
19
+ # Internal immutable resources bundled with the CLI
20
+ INTERNAL_RESOURCES_DIR = get_internal_resources_dir()
21
+
22
+
23
+ def get_workspace_dir() -> Path:
24
+ """
25
+ Get the path to the local mutable workspace.
26
+
27
+ Returns:
28
+ Path: The absolute path to the `.odctl` directory in the current working directory.
29
+ """
30
+ return Path.cwd() / ".odctl"
31
+
32
+
33
+ def get_active_dir() -> Path:
34
+ """
35
+ Determine the active configuration directory.
36
+
37
+ Returns:
38
+ Path: The local workspace directory if it exists; otherwise, falls back
39
+ to the internal resources directory.
40
+ """
41
+ workspace = get_workspace_dir()
42
+ return workspace if workspace.exists() else INTERNAL_RESOURCES_DIR
43
+
44
+
45
+ def get_registry_path() -> Path:
46
+ """
47
+ Get the path to the stack registry configuration file.
48
+
49
+ Returns:
50
+ Path: The path to `registry.yml` within the active directory.
51
+ """
52
+ return get_active_dir() / "registry.yml"
53
+
54
+
55
+ def get_compose_path(filename: str) -> Path:
56
+ """
57
+ Get the path to a specific docker-compose file.
58
+
59
+ Args:
60
+ filename (str): The name of the compose file (e.g., 'compose-infra.yml').
61
+
62
+ Returns:
63
+ Path: The path to the requested file within the active directory.
64
+ """
65
+ return get_active_dir() / filename
odctl/docker.py ADDED
@@ -0,0 +1,295 @@
1
+ import os
2
+ import sys
3
+ from typing import Any, Dict, List, Optional, Tuple
4
+
5
+ import yaml
6
+ from python_on_whales import DockerClient
7
+
8
+ from odctl.config import get_compose_path
9
+
10
+
11
+ def _create_client(
12
+ compose_files: List[str] | None = None, profiles: List[str] | None = None
13
+ ) -> DockerClient:
14
+ """
15
+ Create a Docker client that naturally honors the system's Docker context.
16
+
17
+ Args:
18
+ compose_files (List[str], optional): A list of paths to docker-compose files. Defaults to None.
19
+ profiles (List[str], optional): A list of compose profiles to activate. Defaults to None.
20
+
21
+ Returns:
22
+ DockerClient: An initialized python-on-whales Docker client.
23
+ """
24
+ kwargs: Dict[str, Any] = {}
25
+
26
+ if "DOCKER_HOST" in os.environ:
27
+ kwargs["host"] = os.environ["DOCKER_HOST"]
28
+
29
+ if compose_files:
30
+ kwargs["compose_files"] = compose_files
31
+
32
+ if profiles:
33
+ kwargs["compose_profiles"] = profiles
34
+
35
+ return DockerClient(**kwargs)
36
+
37
+
38
+ # Global client used for basic checks
39
+ client = _create_client()
40
+
41
+
42
+ def is_docker_running() -> bool:
43
+ """
44
+ Check if the Docker daemon is reachable.
45
+
46
+ Returns:
47
+ bool: True if the Docker engine responds to system info requests, False otherwise.
48
+ """
49
+ try:
50
+ # Changed from client.ping() to client.system.info()
51
+ client.system.info()
52
+ return True
53
+ except Exception:
54
+ return False
55
+
56
+
57
+ def get_stack_details(
58
+ compose_filename: str, profiles: List[str]
59
+ ) -> Tuple[List[str], List[str], List[str], List[str]]:
60
+ """
61
+ Resolve services, host port mappings, images, and volumes via direct YAML parsing.
62
+
63
+ Args:
64
+ compose_filename (str): The name of the compose file to parse.
65
+ profiles (List[str]): The active profiles to filter services by.
66
+
67
+ Returns:
68
+ Tuple[List[str], List[str], List[str], List[str]]: A tuple containing four sorted lists:
69
+ services, exposed host ports, container images, and named volumes.
70
+ Returns lists containing "File Error" if the file cannot be parsed.
71
+ """
72
+ path = get_compose_path(compose_filename)
73
+ if not path.exists():
74
+ return ["File Error"], ["File Error"], ["File Error"], ["File Error"]
75
+
76
+ try:
77
+ with open(path, "r") as f:
78
+ compose_dict = yaml.safe_load(f)
79
+
80
+ services = []
81
+ ports = []
82
+ images = []
83
+ volumes = []
84
+
85
+ # Parse the raw dictionary directly
86
+ for svc_name, svc_data in compose_dict.get("services", {}).items():
87
+ svc_profiles = svc_data.get("profiles", [])
88
+
89
+ # Check if this service belongs to the profiles we are querying
90
+ if any(p in profiles for p in svc_profiles):
91
+ services.append(svc_name)
92
+
93
+ if "image" in svc_data:
94
+ images.append(f"{svc_name} -> {svc_data['image']}")
95
+
96
+ if "ports" in svc_data:
97
+ for p in svc_data["ports"]:
98
+ if isinstance(p, str):
99
+ ports.append(f"{svc_name}:{p}")
100
+ elif isinstance(p, dict) and "published" in p:
101
+ target = p.get("target", p["published"])
102
+ ports.append(f"{svc_name}:{p['published']}:{target}")
103
+
104
+ if "volumes" in svc_data:
105
+ for v in svc_data["volumes"]:
106
+ if isinstance(v, str):
107
+ vol_name = v.split(":")[0]
108
+ # Only track named volumes, ignore local host bind mounts (./ or /)
109
+ if not vol_name.startswith((".", "/")):
110
+ volumes.append(vol_name)
111
+ elif isinstance(v, dict) and v.get("type") == "volume":
112
+ source = v.get("source")
113
+ if isinstance(source, str):
114
+ volumes.append(source)
115
+
116
+ return (
117
+ sorted(list(set(services))),
118
+ sorted(list(set(ports))),
119
+ sorted(list(set(images))),
120
+ sorted(list(set(volumes))),
121
+ )
122
+ except Exception as e:
123
+ err = f"Err: {type(e).__name__}"
124
+ return [err], [err], [err], [err]
125
+
126
+
127
+ def pull_stack_images(compose_filename: str, profiles: List[str]):
128
+ """
129
+ Pull required images for a specific stack.
130
+
131
+ Args:
132
+ compose_filename (str): The compose file defining the target stack.
133
+ profiles (List[str]): The profiles indicating which services to pull.
134
+ """
135
+ path = get_compose_path(compose_filename)
136
+ stack_client = _create_client(compose_files=[str(path)], profiles=profiles)
137
+ stack_client.compose.pull()
138
+
139
+
140
+ def launch_stack(compose_filename: str, profiles: List[str]):
141
+ """
142
+ Start the stack via docker compose up.
143
+
144
+ If the 'deps' profile is active, this blocks until the ephemeral init container
145
+ finishes. Otherwise, it detaches and waits for healthchecks.
146
+
147
+ Args:
148
+ compose_filename (str): The compose file defining the target stack.
149
+ profiles (List[str]): The profiles to launch.
150
+ """
151
+ path = get_compose_path(compose_filename)
152
+ stack_client = _create_client(compose_files=[str(path)], profiles=profiles)
153
+
154
+ # The 'deps' profile contains an ephemeral container that exits after copying files.
155
+ # Docker Compose's --wait flag fails if it sees a container exit (even with code 0).
156
+ if "deps" in profiles:
157
+ # detach=False blocks until the init container gracefully finishes its job
158
+ stack_client.compose.up(detach=False)
159
+ else:
160
+ # For all other long-running services, we detach and wait for healthchecks
161
+ stack_client.compose.up(detach=True, wait=True)
162
+
163
+
164
+ def stop_stack(
165
+ compose_filename: str, profiles: List[str], remove_volumes: bool = False
166
+ ):
167
+ """
168
+ Stop and remove containers and networks for a stack.
169
+
170
+ Args:
171
+ compose_filename (str): The compose file defining the target stack.
172
+ profiles (List[str]): The profiles to stop.
173
+ remove_volumes (bool, optional): If True, destroys named volumes associated
174
+ with the stack. Defaults to False.
175
+ """
176
+ path = get_compose_path(compose_filename)
177
+ stack_client = _create_client(compose_files=[str(path)], profiles=profiles)
178
+ # volumes=True will remove named volumes defined in the compose file
179
+ stack_client.compose.down(volumes=remove_volumes)
180
+
181
+
182
+ def get_managed_containers(execution_plan: Dict[str, List[str]]) -> List[Any]:
183
+ """
184
+ Fetch Docker containers matching the requested execution plan.
185
+
186
+ Args:
187
+ execution_plan (Dict[str, List[str]]): A mapping of compose files to target profiles.
188
+
189
+ Returns:
190
+ List[Any]: A list of python-on-whales Container objects managed by the ODCTL stack.
191
+ """
192
+ target_services = set()
193
+ for file, profs in execution_plan.items():
194
+ # Re-using your existing parser to know exactly which services to look for
195
+ services, _, _, _ = get_stack_details(file, profs)
196
+ target_services.update(services)
197
+
198
+ all_containers = client.container.list(all=True)
199
+ managed_containers = []
200
+
201
+ for c in all_containers:
202
+ # Safely access labels via the Container's Config block
203
+ labels = c.config.labels if c.config and c.config.labels else {}
204
+
205
+ project = labels.get("com.docker.compose.project", "")
206
+ service = labels.get("com.docker.compose.service", "")
207
+
208
+ # Strictly enforce that the container belongs to a ODCTL stack AND the requested profile
209
+ if project.startswith("odctl-") and service in target_services:
210
+ managed_containers.append(c)
211
+
212
+ return managed_containers
213
+
214
+
215
+ def _build_compose_client(execution_plan: Dict[str, List[str]]) -> DockerClient:
216
+ """
217
+ Build a unified DockerClient for multiple compose files and profiles.
218
+
219
+ Args:
220
+ execution_plan (Dict[str, List[str]]): A mapping of compose files to target profiles.
221
+
222
+ Returns:
223
+ DockerClient: A client configured to target all specified files and profiles simultaneously.
224
+ """
225
+ files = [str(get_compose_path(f)) for f in execution_plan.keys()]
226
+ # Flatten the lists of profiles and deduplicate them
227
+ profiles = list(set(p for profs in execution_plan.values() for p in profs))
228
+
229
+ # Initialize a single client capable of cross-file execution
230
+ return _create_client(compose_files=files, profiles=profiles)
231
+
232
+
233
+ def restart_managed_containers(execution_plan: Dict[str, List[str]]):
234
+ """
235
+ Restart containers across all requested profiles.
236
+
237
+ Args:
238
+ execution_plan (Dict[str, List[str]]): A mapping of compose files to target profiles.
239
+ """
240
+ compose_client = _build_compose_client(execution_plan)
241
+ compose_client.compose.restart()
242
+
243
+
244
+ def get_managed_logs(
245
+ execution_plan: Dict[str, List[str]],
246
+ follow: bool = False,
247
+ tail: str = "all",
248
+ timestamps: bool = False,
249
+ since: Optional[str] = None,
250
+ until: Optional[str] = None,
251
+ service: Optional[str] = None,
252
+ ):
253
+ """
254
+ Fetch or stream logs for the requested execution plan.
255
+
256
+ Args:
257
+ execution_plan (Dict[str, List[str]]): A mapping of compose files to target profiles.
258
+ follow (bool, optional): If True, streams the logs continuously. Defaults to False.
259
+ tail (str, optional): Number of lines to show from the end of the logs. Defaults to "all".
260
+ timestamps (bool, optional): If True, prints timestamps for each log line. Defaults to False.
261
+ since (str, optional): Show logs since a specific timestamp or relative time. Defaults to None.
262
+ until (str, optional): Show logs before a specific timestamp or relative time. Defaults to None.
263
+ service (str, optional): Restrict logs to a specific compose service name. Defaults to None.
264
+ """
265
+ compose_client = _build_compose_client(execution_plan)
266
+
267
+ kwargs: Dict[str, Any] = {}
268
+ if service:
269
+ # Compose expects a list of services to filter by
270
+ kwargs["services"] = [service]
271
+ if tail != "all":
272
+ kwargs["tail"] = tail
273
+ if timestamps:
274
+ kwargs["timestamps"] = True
275
+ if since:
276
+ kwargs["since"] = since
277
+ if until:
278
+ kwargs["until"] = until
279
+
280
+ try:
281
+ if follow:
282
+ # stream=True yields (source, bytes) tuples in real-time
283
+ for source, content in compose_client.compose.logs(
284
+ stream=True, follow=True, **kwargs
285
+ ):
286
+ # Write raw bytes to preserve Docker Compose's native formatting
287
+ sys.stdout.buffer.write(content)
288
+ sys.stdout.flush()
289
+ else:
290
+ # Fetches the log history as a single string block
291
+ logs_output = compose_client.compose.logs(**kwargs)
292
+ if logs_output:
293
+ print(logs_output)
294
+ except KeyboardInterrupt:
295
+ pass # Handle user pressing Ctrl+C gracefully