odctl 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- odctl/__init__.py +0 -0
- odctl/config.py +65 -0
- odctl/docker.py +295 -0
- odctl/main.py +520 -0
- odctl/planner.py +133 -0
- odctl/registry.py +66 -0
- odctl/resources/clickhouse/config.xml +52 -0
- odctl/resources/clickhouse/init.sql +2 -0
- odctl/resources/clickhouse/keeper_config.xml +26 -0
- odctl/resources/clickhouse/macros-11.xml +1 -0
- odctl/resources/clickhouse/macros-12.xml +1 -0
- odctl/resources/clickhouse/macros-21.xml +1 -0
- odctl/resources/clickhouse/macros-22.xml +1 -0
- odctl/resources/clickhouse/users.xml +56 -0
- odctl/resources/compose-analytics.yml +243 -0
- odctl/resources/compose-deps.yml +24 -0
- odctl/resources/compose-flink.yml +102 -0
- odctl/resources/compose-infra.yml +144 -0
- odctl/resources/compose-kafka.yml +206 -0
- odctl/resources/compose-metadata.yml +160 -0
- odctl/resources/compose-mlops.yml +74 -0
- odctl/resources/compose-obsv.yml +115 -0
- odctl/resources/compose-orch.yml +126 -0
- odctl/resources/compose-spark.yml +95 -0
- odctl/resources/compose-store.yml +103 -0
- odctl/resources/deps/Dockerfile +72 -0
- odctl/resources/deps/build.gradle +51 -0
- odctl/resources/deps/build_connectors.sh +71 -0
- odctl/resources/deps/fetch_flink_dependencies.sh +47 -0
- odctl/resources/deps/fetch_spark_dependencies.sh +40 -0
- odctl/resources/deps/fetch_standalone_plugins.sh +63 -0
- odctl/resources/docker/airflow/Dockerfile +23 -0
- odctl/resources/docker/ray-mlops/Dockerfile +62 -0
- odctl/resources/docker/spark/Dockerfile +16 -0
- odctl/resources/flink/flink1-conf.yml +58 -0
- odctl/resources/grafana/prometheus.yml +8 -0
- odctl/resources/marquez/marquez.yml +20 -0
- odctl/resources/mlops/serving/lightgbm_inference.py.example +42 -0
- odctl/resources/mlops/serving/vector_search.py.example +44 -0
- odctl/resources/postgres/01-init-databases.sh +47 -0
- odctl/resources/prometheus/alertmanager.yml +6 -0
- odctl/resources/prometheus/prometheus.yml +32 -0
- odctl/resources/registry.yml +289 -0
- odctl/resources/seaweed/s3.json +15 -0
- odctl/resources/spark/log4j2.properties +13 -0
- odctl/resources/spark/spark-defaults.conf +25 -0
- odctl/resources/trino/catalog/clickhouse.properties +4 -0
- odctl/resources/trino/catalog/iceberg.properties +9 -0
- odctl/resources/trino/catalog/postgres.properties +4 -0
- odctl/resources/trino/catalog/valkey.properties +6 -0
- odctl/ui.py +373 -0
- odctl/workspace.py +74 -0
- odctl-0.1.0.dist-info/METADATA +211 -0
- odctl-0.1.0.dist-info/RECORD +57 -0
- odctl-0.1.0.dist-info/WHEEL +4 -0
- odctl-0.1.0.dist-info/entry_points.txt +2 -0
- odctl-0.1.0.dist-info/licenses/LICENSE +201 -0
odctl/__init__.py
ADDED
|
File without changes
|
odctl/config.py
ADDED
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
import sys
|
|
2
|
+
from pathlib import Path
|
|
3
|
+
|
|
4
|
+
|
|
5
|
+
def get_internal_resources_dir() -> Path:
|
|
6
|
+
"""
|
|
7
|
+
Determine the path to the bundled compose files, supporting PyInstaller.
|
|
8
|
+
|
|
9
|
+
Returns:
|
|
10
|
+
Path: The absolute path to the internal `resources` directory.
|
|
11
|
+
"""
|
|
12
|
+
if getattr(sys, "frozen", False) and hasattr(sys, "_MEIPASS"):
|
|
13
|
+
# Running as a compiled PyInstaller binary
|
|
14
|
+
return Path(sys._MEIPASS) / "odctl" / "resources"
|
|
15
|
+
# Running as a normal Python script
|
|
16
|
+
return Path(__file__).parent.resolve() / "resources"
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
# Internal immutable resources bundled with the CLI
|
|
20
|
+
INTERNAL_RESOURCES_DIR = get_internal_resources_dir()
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def get_workspace_dir() -> Path:
|
|
24
|
+
"""
|
|
25
|
+
Get the path to the local mutable workspace.
|
|
26
|
+
|
|
27
|
+
Returns:
|
|
28
|
+
Path: The absolute path to the `.odctl` directory in the current working directory.
|
|
29
|
+
"""
|
|
30
|
+
return Path.cwd() / ".odctl"
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def get_active_dir() -> Path:
|
|
34
|
+
"""
|
|
35
|
+
Determine the active configuration directory.
|
|
36
|
+
|
|
37
|
+
Returns:
|
|
38
|
+
Path: The local workspace directory if it exists; otherwise, falls back
|
|
39
|
+
to the internal resources directory.
|
|
40
|
+
"""
|
|
41
|
+
workspace = get_workspace_dir()
|
|
42
|
+
return workspace if workspace.exists() else INTERNAL_RESOURCES_DIR
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def get_registry_path() -> Path:
|
|
46
|
+
"""
|
|
47
|
+
Get the path to the stack registry configuration file.
|
|
48
|
+
|
|
49
|
+
Returns:
|
|
50
|
+
Path: The path to `registry.yml` within the active directory.
|
|
51
|
+
"""
|
|
52
|
+
return get_active_dir() / "registry.yml"
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def get_compose_path(filename: str) -> Path:
|
|
56
|
+
"""
|
|
57
|
+
Get the path to a specific docker-compose file.
|
|
58
|
+
|
|
59
|
+
Args:
|
|
60
|
+
filename (str): The name of the compose file (e.g., 'compose-infra.yml').
|
|
61
|
+
|
|
62
|
+
Returns:
|
|
63
|
+
Path: The path to the requested file within the active directory.
|
|
64
|
+
"""
|
|
65
|
+
return get_active_dir() / filename
|
odctl/docker.py
ADDED
|
@@ -0,0 +1,295 @@
|
|
|
1
|
+
import os
|
|
2
|
+
import sys
|
|
3
|
+
from typing import Any, Dict, List, Optional, Tuple
|
|
4
|
+
|
|
5
|
+
import yaml
|
|
6
|
+
from python_on_whales import DockerClient
|
|
7
|
+
|
|
8
|
+
from odctl.config import get_compose_path
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def _create_client(
|
|
12
|
+
compose_files: List[str] | None = None, profiles: List[str] | None = None
|
|
13
|
+
) -> DockerClient:
|
|
14
|
+
"""
|
|
15
|
+
Create a Docker client that naturally honors the system's Docker context.
|
|
16
|
+
|
|
17
|
+
Args:
|
|
18
|
+
compose_files (List[str], optional): A list of paths to docker-compose files. Defaults to None.
|
|
19
|
+
profiles (List[str], optional): A list of compose profiles to activate. Defaults to None.
|
|
20
|
+
|
|
21
|
+
Returns:
|
|
22
|
+
DockerClient: An initialized python-on-whales Docker client.
|
|
23
|
+
"""
|
|
24
|
+
kwargs: Dict[str, Any] = {}
|
|
25
|
+
|
|
26
|
+
if "DOCKER_HOST" in os.environ:
|
|
27
|
+
kwargs["host"] = os.environ["DOCKER_HOST"]
|
|
28
|
+
|
|
29
|
+
if compose_files:
|
|
30
|
+
kwargs["compose_files"] = compose_files
|
|
31
|
+
|
|
32
|
+
if profiles:
|
|
33
|
+
kwargs["compose_profiles"] = profiles
|
|
34
|
+
|
|
35
|
+
return DockerClient(**kwargs)
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
# Global client used for basic checks
|
|
39
|
+
client = _create_client()
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def is_docker_running() -> bool:
|
|
43
|
+
"""
|
|
44
|
+
Check if the Docker daemon is reachable.
|
|
45
|
+
|
|
46
|
+
Returns:
|
|
47
|
+
bool: True if the Docker engine responds to system info requests, False otherwise.
|
|
48
|
+
"""
|
|
49
|
+
try:
|
|
50
|
+
# Changed from client.ping() to client.system.info()
|
|
51
|
+
client.system.info()
|
|
52
|
+
return True
|
|
53
|
+
except Exception:
|
|
54
|
+
return False
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def get_stack_details(
|
|
58
|
+
compose_filename: str, profiles: List[str]
|
|
59
|
+
) -> Tuple[List[str], List[str], List[str], List[str]]:
|
|
60
|
+
"""
|
|
61
|
+
Resolve services, host port mappings, images, and volumes via direct YAML parsing.
|
|
62
|
+
|
|
63
|
+
Args:
|
|
64
|
+
compose_filename (str): The name of the compose file to parse.
|
|
65
|
+
profiles (List[str]): The active profiles to filter services by.
|
|
66
|
+
|
|
67
|
+
Returns:
|
|
68
|
+
Tuple[List[str], List[str], List[str], List[str]]: A tuple containing four sorted lists:
|
|
69
|
+
services, exposed host ports, container images, and named volumes.
|
|
70
|
+
Returns lists containing "File Error" if the file cannot be parsed.
|
|
71
|
+
"""
|
|
72
|
+
path = get_compose_path(compose_filename)
|
|
73
|
+
if not path.exists():
|
|
74
|
+
return ["File Error"], ["File Error"], ["File Error"], ["File Error"]
|
|
75
|
+
|
|
76
|
+
try:
|
|
77
|
+
with open(path, "r") as f:
|
|
78
|
+
compose_dict = yaml.safe_load(f)
|
|
79
|
+
|
|
80
|
+
services = []
|
|
81
|
+
ports = []
|
|
82
|
+
images = []
|
|
83
|
+
volumes = []
|
|
84
|
+
|
|
85
|
+
# Parse the raw dictionary directly
|
|
86
|
+
for svc_name, svc_data in compose_dict.get("services", {}).items():
|
|
87
|
+
svc_profiles = svc_data.get("profiles", [])
|
|
88
|
+
|
|
89
|
+
# Check if this service belongs to the profiles we are querying
|
|
90
|
+
if any(p in profiles for p in svc_profiles):
|
|
91
|
+
services.append(svc_name)
|
|
92
|
+
|
|
93
|
+
if "image" in svc_data:
|
|
94
|
+
images.append(f"{svc_name} -> {svc_data['image']}")
|
|
95
|
+
|
|
96
|
+
if "ports" in svc_data:
|
|
97
|
+
for p in svc_data["ports"]:
|
|
98
|
+
if isinstance(p, str):
|
|
99
|
+
ports.append(f"{svc_name}:{p}")
|
|
100
|
+
elif isinstance(p, dict) and "published" in p:
|
|
101
|
+
target = p.get("target", p["published"])
|
|
102
|
+
ports.append(f"{svc_name}:{p['published']}:{target}")
|
|
103
|
+
|
|
104
|
+
if "volumes" in svc_data:
|
|
105
|
+
for v in svc_data["volumes"]:
|
|
106
|
+
if isinstance(v, str):
|
|
107
|
+
vol_name = v.split(":")[0]
|
|
108
|
+
# Only track named volumes, ignore local host bind mounts (./ or /)
|
|
109
|
+
if not vol_name.startswith((".", "/")):
|
|
110
|
+
volumes.append(vol_name)
|
|
111
|
+
elif isinstance(v, dict) and v.get("type") == "volume":
|
|
112
|
+
source = v.get("source")
|
|
113
|
+
if isinstance(source, str):
|
|
114
|
+
volumes.append(source)
|
|
115
|
+
|
|
116
|
+
return (
|
|
117
|
+
sorted(list(set(services))),
|
|
118
|
+
sorted(list(set(ports))),
|
|
119
|
+
sorted(list(set(images))),
|
|
120
|
+
sorted(list(set(volumes))),
|
|
121
|
+
)
|
|
122
|
+
except Exception as e:
|
|
123
|
+
err = f"Err: {type(e).__name__}"
|
|
124
|
+
return [err], [err], [err], [err]
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
def pull_stack_images(compose_filename: str, profiles: List[str]):
|
|
128
|
+
"""
|
|
129
|
+
Pull required images for a specific stack.
|
|
130
|
+
|
|
131
|
+
Args:
|
|
132
|
+
compose_filename (str): The compose file defining the target stack.
|
|
133
|
+
profiles (List[str]): The profiles indicating which services to pull.
|
|
134
|
+
"""
|
|
135
|
+
path = get_compose_path(compose_filename)
|
|
136
|
+
stack_client = _create_client(compose_files=[str(path)], profiles=profiles)
|
|
137
|
+
stack_client.compose.pull()
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
def launch_stack(compose_filename: str, profiles: List[str]):
|
|
141
|
+
"""
|
|
142
|
+
Start the stack via docker compose up.
|
|
143
|
+
|
|
144
|
+
If the 'deps' profile is active, this blocks until the ephemeral init container
|
|
145
|
+
finishes. Otherwise, it detaches and waits for healthchecks.
|
|
146
|
+
|
|
147
|
+
Args:
|
|
148
|
+
compose_filename (str): The compose file defining the target stack.
|
|
149
|
+
profiles (List[str]): The profiles to launch.
|
|
150
|
+
"""
|
|
151
|
+
path = get_compose_path(compose_filename)
|
|
152
|
+
stack_client = _create_client(compose_files=[str(path)], profiles=profiles)
|
|
153
|
+
|
|
154
|
+
# The 'deps' profile contains an ephemeral container that exits after copying files.
|
|
155
|
+
# Docker Compose's --wait flag fails if it sees a container exit (even with code 0).
|
|
156
|
+
if "deps" in profiles:
|
|
157
|
+
# detach=False blocks until the init container gracefully finishes its job
|
|
158
|
+
stack_client.compose.up(detach=False)
|
|
159
|
+
else:
|
|
160
|
+
# For all other long-running services, we detach and wait for healthchecks
|
|
161
|
+
stack_client.compose.up(detach=True, wait=True)
|
|
162
|
+
|
|
163
|
+
|
|
164
|
+
def stop_stack(
|
|
165
|
+
compose_filename: str, profiles: List[str], remove_volumes: bool = False
|
|
166
|
+
):
|
|
167
|
+
"""
|
|
168
|
+
Stop and remove containers and networks for a stack.
|
|
169
|
+
|
|
170
|
+
Args:
|
|
171
|
+
compose_filename (str): The compose file defining the target stack.
|
|
172
|
+
profiles (List[str]): The profiles to stop.
|
|
173
|
+
remove_volumes (bool, optional): If True, destroys named volumes associated
|
|
174
|
+
with the stack. Defaults to False.
|
|
175
|
+
"""
|
|
176
|
+
path = get_compose_path(compose_filename)
|
|
177
|
+
stack_client = _create_client(compose_files=[str(path)], profiles=profiles)
|
|
178
|
+
# volumes=True will remove named volumes defined in the compose file
|
|
179
|
+
stack_client.compose.down(volumes=remove_volumes)
|
|
180
|
+
|
|
181
|
+
|
|
182
|
+
def get_managed_containers(execution_plan: Dict[str, List[str]]) -> List[Any]:
|
|
183
|
+
"""
|
|
184
|
+
Fetch Docker containers matching the requested execution plan.
|
|
185
|
+
|
|
186
|
+
Args:
|
|
187
|
+
execution_plan (Dict[str, List[str]]): A mapping of compose files to target profiles.
|
|
188
|
+
|
|
189
|
+
Returns:
|
|
190
|
+
List[Any]: A list of python-on-whales Container objects managed by the ODCTL stack.
|
|
191
|
+
"""
|
|
192
|
+
target_services = set()
|
|
193
|
+
for file, profs in execution_plan.items():
|
|
194
|
+
# Re-using your existing parser to know exactly which services to look for
|
|
195
|
+
services, _, _, _ = get_stack_details(file, profs)
|
|
196
|
+
target_services.update(services)
|
|
197
|
+
|
|
198
|
+
all_containers = client.container.list(all=True)
|
|
199
|
+
managed_containers = []
|
|
200
|
+
|
|
201
|
+
for c in all_containers:
|
|
202
|
+
# Safely access labels via the Container's Config block
|
|
203
|
+
labels = c.config.labels if c.config and c.config.labels else {}
|
|
204
|
+
|
|
205
|
+
project = labels.get("com.docker.compose.project", "")
|
|
206
|
+
service = labels.get("com.docker.compose.service", "")
|
|
207
|
+
|
|
208
|
+
# Strictly enforce that the container belongs to a ODCTL stack AND the requested profile
|
|
209
|
+
if project.startswith("odctl-") and service in target_services:
|
|
210
|
+
managed_containers.append(c)
|
|
211
|
+
|
|
212
|
+
return managed_containers
|
|
213
|
+
|
|
214
|
+
|
|
215
|
+
def _build_compose_client(execution_plan: Dict[str, List[str]]) -> DockerClient:
|
|
216
|
+
"""
|
|
217
|
+
Build a unified DockerClient for multiple compose files and profiles.
|
|
218
|
+
|
|
219
|
+
Args:
|
|
220
|
+
execution_plan (Dict[str, List[str]]): A mapping of compose files to target profiles.
|
|
221
|
+
|
|
222
|
+
Returns:
|
|
223
|
+
DockerClient: A client configured to target all specified files and profiles simultaneously.
|
|
224
|
+
"""
|
|
225
|
+
files = [str(get_compose_path(f)) for f in execution_plan.keys()]
|
|
226
|
+
# Flatten the lists of profiles and deduplicate them
|
|
227
|
+
profiles = list(set(p for profs in execution_plan.values() for p in profs))
|
|
228
|
+
|
|
229
|
+
# Initialize a single client capable of cross-file execution
|
|
230
|
+
return _create_client(compose_files=files, profiles=profiles)
|
|
231
|
+
|
|
232
|
+
|
|
233
|
+
def restart_managed_containers(execution_plan: Dict[str, List[str]]):
|
|
234
|
+
"""
|
|
235
|
+
Restart containers across all requested profiles.
|
|
236
|
+
|
|
237
|
+
Args:
|
|
238
|
+
execution_plan (Dict[str, List[str]]): A mapping of compose files to target profiles.
|
|
239
|
+
"""
|
|
240
|
+
compose_client = _build_compose_client(execution_plan)
|
|
241
|
+
compose_client.compose.restart()
|
|
242
|
+
|
|
243
|
+
|
|
244
|
+
def get_managed_logs(
|
|
245
|
+
execution_plan: Dict[str, List[str]],
|
|
246
|
+
follow: bool = False,
|
|
247
|
+
tail: str = "all",
|
|
248
|
+
timestamps: bool = False,
|
|
249
|
+
since: Optional[str] = None,
|
|
250
|
+
until: Optional[str] = None,
|
|
251
|
+
service: Optional[str] = None,
|
|
252
|
+
):
|
|
253
|
+
"""
|
|
254
|
+
Fetch or stream logs for the requested execution plan.
|
|
255
|
+
|
|
256
|
+
Args:
|
|
257
|
+
execution_plan (Dict[str, List[str]]): A mapping of compose files to target profiles.
|
|
258
|
+
follow (bool, optional): If True, streams the logs continuously. Defaults to False.
|
|
259
|
+
tail (str, optional): Number of lines to show from the end of the logs. Defaults to "all".
|
|
260
|
+
timestamps (bool, optional): If True, prints timestamps for each log line. Defaults to False.
|
|
261
|
+
since (str, optional): Show logs since a specific timestamp or relative time. Defaults to None.
|
|
262
|
+
until (str, optional): Show logs before a specific timestamp or relative time. Defaults to None.
|
|
263
|
+
service (str, optional): Restrict logs to a specific compose service name. Defaults to None.
|
|
264
|
+
"""
|
|
265
|
+
compose_client = _build_compose_client(execution_plan)
|
|
266
|
+
|
|
267
|
+
kwargs: Dict[str, Any] = {}
|
|
268
|
+
if service:
|
|
269
|
+
# Compose expects a list of services to filter by
|
|
270
|
+
kwargs["services"] = [service]
|
|
271
|
+
if tail != "all":
|
|
272
|
+
kwargs["tail"] = tail
|
|
273
|
+
if timestamps:
|
|
274
|
+
kwargs["timestamps"] = True
|
|
275
|
+
if since:
|
|
276
|
+
kwargs["since"] = since
|
|
277
|
+
if until:
|
|
278
|
+
kwargs["until"] = until
|
|
279
|
+
|
|
280
|
+
try:
|
|
281
|
+
if follow:
|
|
282
|
+
# stream=True yields (source, bytes) tuples in real-time
|
|
283
|
+
for source, content in compose_client.compose.logs(
|
|
284
|
+
stream=True, follow=True, **kwargs
|
|
285
|
+
):
|
|
286
|
+
# Write raw bytes to preserve Docker Compose's native formatting
|
|
287
|
+
sys.stdout.buffer.write(content)
|
|
288
|
+
sys.stdout.flush()
|
|
289
|
+
else:
|
|
290
|
+
# Fetches the log history as a single string block
|
|
291
|
+
logs_output = compose_client.compose.logs(**kwargs)
|
|
292
|
+
if logs_output:
|
|
293
|
+
print(logs_output)
|
|
294
|
+
except KeyboardInterrupt:
|
|
295
|
+
pass # Handle user pressing Ctrl+C gracefully
|