odctl 0.2.2__tar.gz → 0.3.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {odctl-0.2.2 → odctl-0.3.1}/.github/renovate.json +10 -0
- odctl-0.3.1/.github/scripts/smoke.sh +196 -0
- {odctl-0.2.2 → odctl-0.3.1}/.github/workflows/e2e-test.yml +33 -27
- odctl-0.3.1/.github/workflows/pipeline.yml +69 -0
- odctl-0.3.1/.github/workflows/publish.yml +90 -0
- {odctl-0.2.2 → odctl-0.3.1}/PKG-INFO +3 -3
- {odctl-0.2.2 → odctl-0.3.1}/README.md +2 -2
- {odctl-0.2.2 → odctl-0.3.1}/pyproject.toml +1 -1
- {odctl-0.2.2 → odctl-0.3.1}/src/odctl/docker.py +6 -2
- {odctl-0.2.2 → odctl-0.3.1}/src/odctl/main.py +7 -7
- {odctl-0.2.2 → odctl-0.3.1}/src/odctl/resources/compose-flink.yml +38 -30
- {odctl-0.2.2 → odctl-0.3.1}/src/odctl/resources/compose-infra.yml +2 -2
- {odctl-0.2.2 → odctl-0.3.1}/src/odctl/resources/compose-kafka.yml +5 -5
- {odctl-0.2.2 → odctl-0.3.1}/src/odctl/resources/compose-metadata.yml +2 -2
- {odctl-0.2.2 → odctl-0.3.1}/src/odctl/resources/compose-store.yml +2 -2
- {odctl-0.2.2 → odctl-0.3.1}/src/odctl/resources/deps/Dockerfile +11 -6
- {odctl-0.2.2 → odctl-0.3.1}/src/odctl/resources/deps/build.gradle +4 -2
- {odctl-0.2.2 → odctl-0.3.1}/src/odctl/resources/deps/fetch_flink_dependencies.sh +18 -14
- {odctl-0.2.2 → odctl-0.3.1}/src/odctl/resources/deps/fetch_spark_dependencies.sh +6 -2
- {odctl-0.2.2 → odctl-0.3.1}/src/odctl/resources/deps/fetch_standalone_plugins.sh +6 -2
- odctl-0.3.1/src/odctl/resources/deps/versions.env +19 -0
- odctl-0.2.2/src/odctl/resources/flink/flink1-conf.yml → odctl-0.3.1/src/odctl/resources/flink/flink-conf.yml +15 -5
- {odctl-0.2.2 → odctl-0.3.1}/src/odctl/resources/prometheus/prometheus.yml +4 -4
- {odctl-0.2.2 → odctl-0.3.1}/src/odctl/resources/registry.yml +10 -10
- {odctl-0.2.2 → odctl-0.3.1}/tests/test_docker.py +3 -1
- odctl-0.2.2/.github/workflows/pipeline.yml +0 -34
- odctl-0.2.2/.github/workflows/publish.yml +0 -34
- {odctl-0.2.2 → odctl-0.3.1}/.github/ISSUE_TEMPLATE/bug_report.md +0 -0
- {odctl-0.2.2 → odctl-0.3.1}/.github/ISSUE_TEMPLATE/feature_request.md +0 -0
- {odctl-0.2.2 → odctl-0.3.1}/.github/workflows/build-deps.yml +0 -0
- {odctl-0.2.2 → odctl-0.3.1}/.github/workflows/build-platform-images.yml +0 -0
- {odctl-0.2.2 → odctl-0.3.1}/.github/workflows/ci.yml +0 -0
- {odctl-0.2.2 → odctl-0.3.1}/.gitignore +0 -0
- {odctl-0.2.2 → odctl-0.3.1}/.pre-commit-config.yaml +0 -0
- {odctl-0.2.2 → odctl-0.3.1}/.vscode/settings.json +0 -0
- {odctl-0.2.2 → odctl-0.3.1}/LICENSE +0 -0
- {odctl-0.2.2 → odctl-0.3.1}/image/diagram.png +0 -0
- {odctl-0.2.2 → odctl-0.3.1}/image/diagram1.drawio +0 -0
- {odctl-0.2.2 → odctl-0.3.1}/image/diagram2.drawio +0 -0
- {odctl-0.2.2 → odctl-0.3.1}/src/odctl/__init__.py +0 -0
- {odctl-0.2.2 → odctl-0.3.1}/src/odctl/config.py +0 -0
- {odctl-0.2.2 → odctl-0.3.1}/src/odctl/planner.py +0 -0
- {odctl-0.2.2 → odctl-0.3.1}/src/odctl/registry.py +0 -0
- {odctl-0.2.2 → odctl-0.3.1}/src/odctl/resources/clickhouse/config.xml +0 -0
- {odctl-0.2.2 → odctl-0.3.1}/src/odctl/resources/clickhouse/init.sql +0 -0
- {odctl-0.2.2 → odctl-0.3.1}/src/odctl/resources/clickhouse/keeper_config.xml +0 -0
- {odctl-0.2.2 → odctl-0.3.1}/src/odctl/resources/clickhouse/macros-11.xml +0 -0
- {odctl-0.2.2 → odctl-0.3.1}/src/odctl/resources/clickhouse/macros-12.xml +0 -0
- {odctl-0.2.2 → odctl-0.3.1}/src/odctl/resources/clickhouse/macros-21.xml +0 -0
- {odctl-0.2.2 → odctl-0.3.1}/src/odctl/resources/clickhouse/macros-22.xml +0 -0
- {odctl-0.2.2 → odctl-0.3.1}/src/odctl/resources/clickhouse/users.xml +0 -0
- {odctl-0.2.2 → odctl-0.3.1}/src/odctl/resources/compose-analytics.yml +0 -0
- {odctl-0.2.2 → odctl-0.3.1}/src/odctl/resources/compose-deps.yml +0 -0
- {odctl-0.2.2 → odctl-0.3.1}/src/odctl/resources/compose-mlops.yml +0 -0
- {odctl-0.2.2 → odctl-0.3.1}/src/odctl/resources/compose-obsv.yml +0 -0
- {odctl-0.2.2 → odctl-0.3.1}/src/odctl/resources/compose-orch.yml +0 -0
- {odctl-0.2.2 → odctl-0.3.1}/src/odctl/resources/compose-spark.yml +0 -0
- {odctl-0.2.2 → odctl-0.3.1}/src/odctl/resources/deps/build_connectors.sh +0 -0
- {odctl-0.2.2 → odctl-0.3.1}/src/odctl/resources/docker/airflow/Dockerfile +0 -0
- {odctl-0.2.2 → odctl-0.3.1}/src/odctl/resources/docker/ray-mlops/Dockerfile +0 -0
- {odctl-0.2.2 → odctl-0.3.1}/src/odctl/resources/docker/spark/Dockerfile +0 -0
- {odctl-0.2.2 → odctl-0.3.1}/src/odctl/resources/grafana/prometheus.yml +0 -0
- {odctl-0.2.2 → odctl-0.3.1}/src/odctl/resources/marquez/marquez.yml +0 -0
- {odctl-0.2.2 → odctl-0.3.1}/src/odctl/resources/mlops/serving/lightgbm_inference.py.example +0 -0
- {odctl-0.2.2 → odctl-0.3.1}/src/odctl/resources/mlops/serving/vector_search.py.example +0 -0
- {odctl-0.2.2 → odctl-0.3.1}/src/odctl/resources/postgres/01-init-databases.sh +0 -0
- {odctl-0.2.2 → odctl-0.3.1}/src/odctl/resources/prometheus/alertmanager.yml +0 -0
- {odctl-0.2.2 → odctl-0.3.1}/src/odctl/resources/seaweed/s3.json +0 -0
- {odctl-0.2.2 → odctl-0.3.1}/src/odctl/resources/spark/log4j2.properties +0 -0
- {odctl-0.2.2 → odctl-0.3.1}/src/odctl/resources/spark/spark-defaults.conf +0 -0
- {odctl-0.2.2 → odctl-0.3.1}/src/odctl/resources/trino/catalog/clickhouse.properties +0 -0
- {odctl-0.2.2 → odctl-0.3.1}/src/odctl/resources/trino/catalog/iceberg.properties +0 -0
- {odctl-0.2.2 → odctl-0.3.1}/src/odctl/resources/trino/catalog/kafka.properties +0 -0
- {odctl-0.2.2 → odctl-0.3.1}/src/odctl/resources/trino/catalog/postgres.properties +0 -0
- {odctl-0.2.2 → odctl-0.3.1}/src/odctl/resources/trino/catalog/valkey.properties +0 -0
- {odctl-0.2.2 → odctl-0.3.1}/src/odctl/ui.py +0 -0
- {odctl-0.2.2 → odctl-0.3.1}/src/odctl/workspace.py +0 -0
- {odctl-0.2.2 → odctl-0.3.1}/tests/conftest.py +0 -0
- {odctl-0.2.2 → odctl-0.3.1}/tests/test_cli.py +0 -0
- {odctl-0.2.2 → odctl-0.3.1}/tests/test_cli_extended.py +0 -0
- {odctl-0.2.2 → odctl-0.3.1}/tests/test_config.py +0 -0
- {odctl-0.2.2 → odctl-0.3.1}/tests/test_planner.py +0 -0
- {odctl-0.2.2 → odctl-0.3.1}/tests/test_registry.py +0 -0
- {odctl-0.2.2 → odctl-0.3.1}/tests/test_workspace.py +0 -0
- {odctl-0.2.2 → odctl-0.3.1}/uv.lock +0 -0
|
@@ -12,6 +12,16 @@
|
|
|
12
12
|
]
|
|
13
13
|
},
|
|
14
14
|
"packageRules": [
|
|
15
|
+
{
|
|
16
|
+
"description": "Flink stays on 2.1: Iceberg publishes iceberg-flink-runtime for 2.0 and 2.1 only, and fluss-flink has no 2.0 build. Raise this when Iceberg ships a runtime for a newer minor. See issue #9.",
|
|
17
|
+
"matchPackageNames": ["flink", "org.apache.flink:{/,}**"],
|
|
18
|
+
"allowedVersions": "<2.2"
|
|
19
|
+
},
|
|
20
|
+
{
|
|
21
|
+
"description": "Fluss stays on 0.8.x because 0.9.x requires Flink 2.2, which has no Iceberg runtime. Moves together with the Flink ceiling above.",
|
|
22
|
+
"matchPackageNames": ["apache/fluss", "org.apache.fluss:{/,}**"],
|
|
23
|
+
"allowedVersions": "<0.9"
|
|
24
|
+
},
|
|
15
25
|
{
|
|
16
26
|
"description": "Auto-merge non-breaking minor and patch updates if CI passes",
|
|
17
27
|
"matchUpdateTypes": ["minor", "patch"],
|
|
@@ -0,0 +1,196 @@
|
|
|
1
|
+
#!/usr/bin/env bash
|
|
2
|
+
# Functional smoke test for one odctl profile.
|
|
3
|
+
#
|
|
4
|
+
# Liveness is not enough: every bug fixed in 0.2.x and 0.3.0 left its containers
|
|
5
|
+
# running and healthy while the profile was unusable. Kafka consumer groups hung
|
|
6
|
+
# forever, Spark could not start a SparkContext, Flink could not find a connector
|
|
7
|
+
# factory, ClickHouse could not consume from the broker, the Trino ClickHouse
|
|
8
|
+
# catalog pointed at a hostname that did not exist, and the Flink Iceberg sink
|
|
9
|
+
# committed nothing while its checkpoints succeeded. Each assertion below exists
|
|
10
|
+
# because something shipped broken past a liveness check.
|
|
11
|
+
#
|
|
12
|
+
# Usage: smoke.sh <profile>
|
|
13
|
+
set -uo pipefail
|
|
14
|
+
|
|
15
|
+
PROFILE="${1:?usage: smoke.sh <profile>}"
|
|
16
|
+
|
|
17
|
+
fail() { echo "❌ $PROFILE: $*"; exit 1; }
|
|
18
|
+
pass() { echo "✅ $PROFILE: $*"; }
|
|
19
|
+
|
|
20
|
+
# Retry a command until it succeeds or the budget runs out.
|
|
21
|
+
retry() {
|
|
22
|
+
local tries="$1" delay="$2"; shift 2
|
|
23
|
+
for ((i = 1; i <= tries; i++)); do
|
|
24
|
+
if "$@" >/dev/null 2>&1; then return 0; fi
|
|
25
|
+
sleep "$delay"
|
|
26
|
+
done
|
|
27
|
+
return 1
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
http_ok() { curl -fsS -o /dev/null --max-time 10 "$1"; }
|
|
31
|
+
|
|
32
|
+
# Some endpoints answer 403 or 404 to an unauthenticated probe, which still
|
|
33
|
+
# proves the service is listening.
|
|
34
|
+
http_reachable() {
|
|
35
|
+
local code
|
|
36
|
+
code=$(curl -s -o /dev/null --max-time 10 -w '%{http_code}' "$1")
|
|
37
|
+
[ -n "$code" ] && [ "$code" != "000" ]
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
ch() { docker exec "$1" clickhouse-client --password password -q "$2" 2>&1; }
|
|
41
|
+
|
|
42
|
+
kafka_topic() {
|
|
43
|
+
docker exec kafka /opt/kafka/bin/kafka-topics.sh --bootstrap-server broker-1:19092 "$@"
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
smoke_kafka() {
|
|
47
|
+
local container="$1"
|
|
48
|
+
retry 30 5 docker exec "$container" /opt/kafka/bin/kafka-topics.sh \
|
|
49
|
+
--bootstrap-server broker-1:19092 --list || fail "broker never became reachable"
|
|
50
|
+
docker exec "$container" /opt/kafka/bin/kafka-topics.sh --bootstrap-server broker-1:19092 \
|
|
51
|
+
--create --topic smoke --partitions 1 --replication-factor 1 >/dev/null 2>&1
|
|
52
|
+
for i in 1 2 3 4 5; do echo "{\"n\":$i}"; done | docker exec -i "$container" \
|
|
53
|
+
/opt/kafka/bin/kafka-console-producer.sh --bootstrap-server broker-1:19092 --topic smoke >/dev/null 2>&1 \
|
|
54
|
+
|| fail "produce failed"
|
|
55
|
+
# Group mode on purpose. Assign mode bypasses the coordinator and would pass
|
|
56
|
+
# even when __consumer_offsets cannot be created on a single broker.
|
|
57
|
+
local got
|
|
58
|
+
got=$(docker exec "$container" /opt/kafka/bin/kafka-console-consumer.sh \
|
|
59
|
+
--bootstrap-server broker-1:19092 --topic smoke --from-beginning \
|
|
60
|
+
--max-messages 5 --timeout-ms 60000 2>/dev/null | grep -c '"n"')
|
|
61
|
+
[ "$got" -eq 5 ] || fail "group-mode consume returned $got of 5 messages"
|
|
62
|
+
pass "produced and consumed 5 messages through a consumer group"
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
smoke_flink() {
|
|
66
|
+
retry 40 5 http_ok "http://127.0.0.1:8082/config" || fail "JobManager REST never answered"
|
|
67
|
+
local out
|
|
68
|
+
out=$(docker exec -i flink-jobmanager /opt/flink/bin/sql-client.sh 2>&1 <<'SQL'
|
|
69
|
+
CREATE CATALOG ice WITH ('type'='iceberg','catalog-type'='rest','uri'='http://catalog:8181','warehouse'='s3://warehouse','s3.endpoint'='http://seaweed:8333','s3.path-style-access'='true','s3.access-key-id'='user','s3.secret-access-key'='password');
|
|
70
|
+
SHOW CATALOGS;
|
|
71
|
+
SQL
|
|
72
|
+
)
|
|
73
|
+
grep -q "ice" <<<"$out" || { echo "$out" | tail -20; fail "Iceberg catalog did not register (connector factory or Hadoop classes missing)"; }
|
|
74
|
+
# The image ships --add-opens in config.yaml; mounting a replacement wipes them
|
|
75
|
+
# and Kryo then fails on java.nio buffers at checkpoint time.
|
|
76
|
+
docker exec flink-jobmanager sh -c 'ps ax | grep -q "add-opens=java.base/java.nio"' \
|
|
77
|
+
|| fail "JVM module flags missing from the TaskManager command line"
|
|
78
|
+
pass "Iceberg catalog registered and JVM module flags present"
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
smoke_spark() {
|
|
82
|
+
local out
|
|
83
|
+
out=$(docker exec spark-master /opt/spark/bin/spark-sql -e "
|
|
84
|
+
CREATE NAMESPACE IF NOT EXISTS iceberg.smoke;
|
|
85
|
+
CREATE OR REPLACE TABLE iceberg.smoke.t (id BIGINT) USING iceberg;
|
|
86
|
+
INSERT INTO iceberg.smoke.t VALUES (1),(2),(3);
|
|
87
|
+
SELECT count(*) FROM iceberg.smoke.t;" 2>&1)
|
|
88
|
+
grep -qE "^3$" <<<"$out" || { echo "$out" | tail -20; fail "spark-sql could not round-trip an Iceberg table"; }
|
|
89
|
+
docker exec spark-master sh -c 'ls /tmp/spark-events | head -1' >/dev/null 2>&1 \
|
|
90
|
+
|| fail "no event log written, so the event log directory is not writable"
|
|
91
|
+
pass "spark-sql wrote and read an Iceberg table, event logging works"
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
smoke_ch_lite() {
|
|
95
|
+
retry 30 5 docker exec ch-11 clickhouse-client --password password -q "SELECT 1" \
|
|
96
|
+
|| fail "ch-11 never answered"
|
|
97
|
+
# Databases come from /docker-entrypoint-initdb.d on every fresh server.
|
|
98
|
+
ch ch-11 "SHOW DATABASES" | grep -q feature_store || fail "feature_store database missing"
|
|
99
|
+
ch ch-11 "DROP TABLE IF EXISTS default.smoke" >/dev/null
|
|
100
|
+
ch ch-11 "CREATE TABLE default.smoke (id UInt32) ENGINE=MergeTree ORDER BY id" >/dev/null
|
|
101
|
+
ch ch-11 "INSERT INTO default.smoke SELECT number FROM numbers(10)" >/dev/null
|
|
102
|
+
[ "$(ch ch-11 "SELECT count() FROM default.smoke")" = "10" ] || fail "MergeTree round trip failed"
|
|
103
|
+
pass "databases initialised and MergeTree round trip works"
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
smoke_ch_full() {
|
|
107
|
+
retry 30 5 docker exec ch-21 clickhouse-client --password password -q "SELECT 1" \
|
|
108
|
+
|| fail "ch-21 never answered"
|
|
109
|
+
# Exercises keeper and the dual-stack listeners: replicated DDL fails with
|
|
110
|
+
# "Connection refused" when keeper listens on IPv4 only.
|
|
111
|
+
ch ch-11 "DROP TABLE IF EXISTS default.repl ON CLUSTER '{cluster}' SYNC" >/dev/null 2>&1
|
|
112
|
+
ch ch-11 "CREATE TABLE default.repl ON CLUSTER '{cluster}' (id UInt32) ENGINE=ReplicatedMergeTree ORDER BY id" >/dev/null \
|
|
113
|
+
|| fail "replicated DDL failed, check keeper reachability"
|
|
114
|
+
ch ch-11 "INSERT INTO default.repl SELECT number FROM numbers(50)" >/dev/null
|
|
115
|
+
retry 12 5 bash -c '[ "$(docker exec ch-12 clickhouse-client --password password -q "SELECT count() FROM default.repl" 2>/dev/null)" = "50" ]' \
|
|
116
|
+
|| fail "rows never replicated to the sibling replica"
|
|
117
|
+
[ "$(ch ch-21 "SELECT count() FROM default.repl")" = "0" ] || fail "shard 2 unexpectedly holds shard 1 data"
|
|
118
|
+
pass "replicated table converged on shard 1 and stayed off shard 2"
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
# Catalog registration finishes after the health endpoint starts answering, so
|
|
122
|
+
# poll rather than asserting once. Catalogs load even when their backing service
|
|
123
|
+
# is absent, which is why this can be required in a trino-only profile group.
|
|
124
|
+
catalogs_loaded() {
|
|
125
|
+
local cat
|
|
126
|
+
cat=$(docker exec trino trino --execute "SHOW CATALOGS" 2>/dev/null | tr -d '"')
|
|
127
|
+
for expected in iceberg kafka clickhouse postgres; do
|
|
128
|
+
grep -q "^${expected}$" <<<"$cat" || return 1
|
|
129
|
+
done
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
smoke_trino() {
|
|
133
|
+
retry 40 5 http_ok "http://127.0.0.1:8080/v1/info" || fail "Trino never answered"
|
|
134
|
+
retry 24 5 catalogs_loaded || {
|
|
135
|
+
docker exec trino trino --execute "SHOW CATALOGS" 2>&1 | tail -10
|
|
136
|
+
fail "not every catalog loaded within two minutes"
|
|
137
|
+
}
|
|
138
|
+
docker exec trino trino --execute "SELECT 1" >/dev/null 2>&1 || fail "query execution failed"
|
|
139
|
+
pass "all catalogs loaded and a query ran"
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
smoke_infra() {
|
|
143
|
+
case "$PROFILE" in
|
|
144
|
+
postgres)
|
|
145
|
+
retry 30 5 docker exec postgres pg_isready -U user || fail "postgres never became ready"
|
|
146
|
+
docker exec postgres psql -U user -d odctl -c "SELECT 1" >/dev/null 2>&1 || fail "psql query failed"
|
|
147
|
+
pass "accepting connections and running queries" ;;
|
|
148
|
+
storage)
|
|
149
|
+
retry 30 5 http_reachable "http://127.0.0.1:8333" || fail "S3 API never answered"
|
|
150
|
+
pass "S3 API reachable" ;;
|
|
151
|
+
catalog)
|
|
152
|
+
retry 30 5 http_ok "http://127.0.0.1:8181/v1/config" || fail "REST catalog never answered"
|
|
153
|
+
curl -fsS -X POST -H 'Content-Type: application/json' \
|
|
154
|
+
-d '{"namespace":["smoke"]}' "http://127.0.0.1:8181/v1/namespaces" >/dev/null 2>&1
|
|
155
|
+
curl -fsS "http://127.0.0.1:8181/v1/namespaces" 2>/dev/null | grep -q smoke \
|
|
156
|
+
|| fail "namespace create or list failed"
|
|
157
|
+
pass "namespace created and listed through the REST API" ;;
|
|
158
|
+
valkey)
|
|
159
|
+
retry 30 5 docker exec valkey valkey-cli -u redis://user:password@localhost:6379 ping \
|
|
160
|
+
|| fail "valkey never answered PING"
|
|
161
|
+
pass "responding to PING" ;;
|
|
162
|
+
esac
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
# A profile with no functional assertion yet still has to expose its endpoint.
|
|
166
|
+
smoke_http_only() {
|
|
167
|
+
local url="$1"
|
|
168
|
+
retry 60 5 http_ok "$url" || fail "no HTTP response from $url"
|
|
169
|
+
pass "HTTP endpoint answering at $url"
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
case "$PROFILE" in
|
|
173
|
+
kafka-lite) smoke_kafka kafka ;;
|
|
174
|
+
kafka-full) smoke_kafka kafka-1 ;;
|
|
175
|
+
flink-lite|flink-full) smoke_flink ;;
|
|
176
|
+
spark-lite|spark-full) smoke_spark ;;
|
|
177
|
+
ch-lite) smoke_ch_lite ;;
|
|
178
|
+
ch-full) smoke_ch_full ;;
|
|
179
|
+
trino) smoke_trino ;;
|
|
180
|
+
postgres|storage|catalog|valkey) smoke_infra ;;
|
|
181
|
+
metabase) smoke_http_only "http://127.0.0.1:3000/api/health" ;;
|
|
182
|
+
airflow) smoke_http_only "http://127.0.0.1:8085/api/v2/monitor/health" ;;
|
|
183
|
+
mlflow) smoke_http_only "http://127.0.0.1:5000/health" ;;
|
|
184
|
+
ray-serve) smoke_http_only "http://127.0.0.1:8265" ;;
|
|
185
|
+
lineage) smoke_http_only "http://127.0.0.1:5002/api/v1/namespaces" ;;
|
|
186
|
+
telemetry) smoke_http_only "http://127.0.0.1:19090/-/ready" ;;
|
|
187
|
+
metadata) smoke_http_only "http://127.0.0.1:8585/api/v1/system/version" ;;
|
|
188
|
+
fluss)
|
|
189
|
+
retry 40 5 bash -c 'docker ps --format "{{.Names}}" | grep -q fluss-coordinator' \
|
|
190
|
+
|| fail "coordinator container never appeared"
|
|
191
|
+
pass "coordinator running" ;;
|
|
192
|
+
*)
|
|
193
|
+
echo "ℹ️ $PROFILE: no functional assertion defined, checking containers only"
|
|
194
|
+
[ "$(docker ps -q | wc -l)" -ge 1 ] || fail "no containers running"
|
|
195
|
+
pass "containers running" ;;
|
|
196
|
+
esac
|
|
@@ -3,8 +3,6 @@ name: End-to-End Integration Tests
|
|
|
3
3
|
on:
|
|
4
4
|
workflow_call:
|
|
5
5
|
|
|
6
|
-
|
|
7
|
-
|
|
8
6
|
jobs:
|
|
9
7
|
e2e:
|
|
10
8
|
runs-on: ubuntu-latest
|
|
@@ -12,18 +10,30 @@ jobs:
|
|
|
12
10
|
contents: read
|
|
13
11
|
packages: read
|
|
14
12
|
strategy:
|
|
13
|
+
fail-fast: false
|
|
15
14
|
matrix:
|
|
15
|
+
# Profiles are grouped only where they share a large image, so the pull
|
|
16
|
+
# is amortised: the lite and full variants of Kafka, Flink, Spark and
|
|
17
|
+
# ClickHouse. Everything else runs alone, which keeps the failure signal
|
|
18
|
+
# per profile and shortens the critical path.
|
|
16
19
|
profile_group:
|
|
17
|
-
- "metadata"
|
|
18
|
-
- "airflow"
|
|
19
20
|
- "kafka-lite kafka-full"
|
|
20
|
-
- "
|
|
21
|
+
- "flink-lite flink-full"
|
|
21
22
|
- "spark-lite spark-full"
|
|
22
23
|
- "ch-lite ch-full"
|
|
23
|
-
- "
|
|
24
|
-
- "
|
|
25
|
-
- "
|
|
26
|
-
- "
|
|
24
|
+
- "fluss"
|
|
25
|
+
- "trino"
|
|
26
|
+
- "metabase"
|
|
27
|
+
- "mlflow"
|
|
28
|
+
- "ray-serve"
|
|
29
|
+
- "lineage"
|
|
30
|
+
- "telemetry"
|
|
31
|
+
- "metadata"
|
|
32
|
+
- "airflow"
|
|
33
|
+
- "postgres"
|
|
34
|
+
- "storage"
|
|
35
|
+
- "catalog"
|
|
36
|
+
- "valkey"
|
|
27
37
|
steps:
|
|
28
38
|
- name: Checkout repository
|
|
29
39
|
uses: actions/checkout@v4
|
|
@@ -49,31 +59,27 @@ jobs:
|
|
|
49
59
|
|
|
50
60
|
- name: Test Profile Group
|
|
51
61
|
run: |
|
|
52
|
-
|
|
62
|
+
chmod +x .github/scripts/smoke.sh
|
|
53
63
|
for profile in ${{ matrix.profile_group }}; do
|
|
54
64
|
echo "::group::Testing Profile: $profile"
|
|
55
65
|
echo "🚀 Launching $profile..."
|
|
56
|
-
odctl up $profile
|
|
66
|
+
odctl up "$profile"
|
|
57
67
|
|
|
58
|
-
echo "⏳
|
|
59
|
-
|
|
60
|
-
for i in {1..15}; do
|
|
61
|
-
RUNNING=$(docker ps -q | wc -l)
|
|
62
|
-
if [ "$RUNNING" -ge 1 ]; then
|
|
63
|
-
echo "✅ Containers are running!"
|
|
64
|
-
docker ps
|
|
65
|
-
odctl ps --all
|
|
66
|
-
SUCCESS=1
|
|
67
|
-
break
|
|
68
|
-
fi
|
|
69
|
-
echo "Waiting... ($i/15)"
|
|
70
|
-
sleep 2
|
|
71
|
-
done
|
|
68
|
+
echo "⏳ Containers:"
|
|
69
|
+
docker ps
|
|
72
70
|
|
|
73
|
-
|
|
74
|
-
|
|
71
|
+
# Liveness alone has never been enough here: every bug fixed in 0.2.x
|
|
72
|
+
# and 0.3.0 left healthy containers behind a broken profile, so the
|
|
73
|
+
# smoke script asserts the profile can actually do its job.
|
|
74
|
+
echo "🔍 Smoke testing $profile..."
|
|
75
|
+
if ! .github/scripts/smoke.sh "$profile"; then
|
|
76
|
+
echo "::error::Smoke test failed for $profile"
|
|
75
77
|
docker ps -a
|
|
76
78
|
odctl ps --all
|
|
79
|
+
for c in $(docker ps -aq); do
|
|
80
|
+
echo "---- logs: $(docker inspect --format '{{.Name}}' "$c") ----"
|
|
81
|
+
docker logs --tail 40 "$c" 2>&1 || true
|
|
82
|
+
done
|
|
77
83
|
exit 1
|
|
78
84
|
fi
|
|
79
85
|
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
name: Main Pipeline
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
branches: ["main"]
|
|
6
|
+
tags: ["v*"]
|
|
7
|
+
pull_request:
|
|
8
|
+
branches: ["main"]
|
|
9
|
+
workflow_dispatch:
|
|
10
|
+
|
|
11
|
+
concurrency:
|
|
12
|
+
group: ${{ github.workflow }}-${{ github.ref }}
|
|
13
|
+
cancel-in-progress: true
|
|
14
|
+
|
|
15
|
+
permissions:
|
|
16
|
+
contents: read
|
|
17
|
+
packages: write
|
|
18
|
+
|
|
19
|
+
jobs:
|
|
20
|
+
# Which parts of the tree moved. Used to skip the multi-arch image builds on
|
|
21
|
+
# changes that cannot affect them, which is most changes.
|
|
22
|
+
changes:
|
|
23
|
+
runs-on: ubuntu-latest
|
|
24
|
+
permissions:
|
|
25
|
+
contents: read
|
|
26
|
+
pull-requests: read
|
|
27
|
+
outputs:
|
|
28
|
+
images: ${{ steps.filter.outputs.images }}
|
|
29
|
+
steps:
|
|
30
|
+
- uses: actions/checkout@v4
|
|
31
|
+
- uses: dorny/paths-filter@v3
|
|
32
|
+
id: filter
|
|
33
|
+
with:
|
|
34
|
+
filters: |
|
|
35
|
+
images:
|
|
36
|
+
- 'src/odctl/resources/deps/**'
|
|
37
|
+
- 'src/odctl/resources/docker/**'
|
|
38
|
+
- '.github/workflows/build-deps.yml'
|
|
39
|
+
- '.github/workflows/build-platform-images.yml'
|
|
40
|
+
|
|
41
|
+
ci:
|
|
42
|
+
uses: ./.github/workflows/ci.yml
|
|
43
|
+
|
|
44
|
+
# Images are pushed only on a tag, so on any other ref this is a
|
|
45
|
+
# build-and-discard. Run it when the build inputs changed, so a broken
|
|
46
|
+
# Dockerfile surfaces before release instead of at tag time, and skip it
|
|
47
|
+
# otherwise. That removes two multi-arch builds from the typical change.
|
|
48
|
+
build-platform:
|
|
49
|
+
needs: [ci, changes]
|
|
50
|
+
if: >-
|
|
51
|
+
startsWith(github.ref, 'refs/tags/v') ||
|
|
52
|
+
github.event_name == 'workflow_dispatch' ||
|
|
53
|
+
needs.changes.outputs.images == 'true'
|
|
54
|
+
uses: ./.github/workflows/build-platform-images.yml
|
|
55
|
+
|
|
56
|
+
build-deps:
|
|
57
|
+
needs: [ci, changes]
|
|
58
|
+
if: >-
|
|
59
|
+
startsWith(github.ref, 'refs/tags/v') ||
|
|
60
|
+
github.event_name == 'workflow_dispatch' ||
|
|
61
|
+
needs.changes.outputs.images == 'true'
|
|
62
|
+
uses: ./.github/workflows/build-deps.yml
|
|
63
|
+
|
|
64
|
+
# Tags and manual runs only: the e2e job pulls images at the CLI's own version,
|
|
65
|
+
# and those exist on ghcr only once a tag has published them.
|
|
66
|
+
e2e:
|
|
67
|
+
needs: [build-platform, build-deps]
|
|
68
|
+
if: startsWith(github.ref, 'refs/tags/v') || github.event_name == 'workflow_dispatch'
|
|
69
|
+
uses: ./.github/workflows/e2e-test.yml
|
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
name: Publish to PyPI
|
|
2
|
+
|
|
3
|
+
# Publishing stays tied to publishing a release, so the release notes remain the
|
|
4
|
+
# trigger. What is new is the gate below: a release event fires independently of
|
|
5
|
+
# CI, which is how v0.3.0 reached PyPI while the pipeline's e2e job was failing.
|
|
6
|
+
# This job now waits for the tag's Main Pipeline run and refuses to publish
|
|
7
|
+
# unless it succeeded.
|
|
8
|
+
on:
|
|
9
|
+
release:
|
|
10
|
+
types: [published]
|
|
11
|
+
workflow_dispatch:
|
|
12
|
+
inputs:
|
|
13
|
+
tag:
|
|
14
|
+
description: "Tag to publish, for example v0.3.1"
|
|
15
|
+
required: true
|
|
16
|
+
skip_pipeline_check:
|
|
17
|
+
description: "Publish even if the tag's pipeline is not green"
|
|
18
|
+
type: boolean
|
|
19
|
+
default: false
|
|
20
|
+
|
|
21
|
+
concurrency:
|
|
22
|
+
group: ${{ github.workflow }}-${{ github.event.release.tag_name || inputs.tag }}
|
|
23
|
+
cancel-in-progress: false
|
|
24
|
+
|
|
25
|
+
jobs:
|
|
26
|
+
pypi-publish:
|
|
27
|
+
name: Build and publish Python package
|
|
28
|
+
runs-on: ubuntu-latest
|
|
29
|
+
# These permissions are strictly required for PyPI OIDC Trusted Publishing;
|
|
30
|
+
# actions:read is what lets the gate query the pipeline's conclusion.
|
|
31
|
+
permissions:
|
|
32
|
+
contents: read
|
|
33
|
+
id-token: write
|
|
34
|
+
actions: read
|
|
35
|
+
|
|
36
|
+
steps:
|
|
37
|
+
- name: Checkout source code
|
|
38
|
+
uses: actions/checkout@v7
|
|
39
|
+
with:
|
|
40
|
+
ref: ${{ github.event.release.tag_name || inputs.tag }}
|
|
41
|
+
|
|
42
|
+
- name: Require a green Main Pipeline for this tag
|
|
43
|
+
if: ${{ !inputs.skip_pipeline_check }}
|
|
44
|
+
env:
|
|
45
|
+
GH_TOKEN: ${{ github.token }}
|
|
46
|
+
TAG: ${{ github.event.release.tag_name || inputs.tag }}
|
|
47
|
+
run: |
|
|
48
|
+
set -euo pipefail
|
|
49
|
+
# Wait rather than fail outright: a release is usually published moments
|
|
50
|
+
# after the tag is pushed, so the pipeline is still running e2e.
|
|
51
|
+
deadline=$(( SECONDS + 45 * 60 ))
|
|
52
|
+
while :; do
|
|
53
|
+
run=$(gh api "repos/${GITHUB_REPOSITORY}/actions/runs?event=push&per_page=100" \
|
|
54
|
+
--jq "[.workflow_runs[]
|
|
55
|
+
| select(.name == \"Main Pipeline\" and .head_branch == \"${TAG}\")]
|
|
56
|
+
| sort_by(.created_at) | last // empty")
|
|
57
|
+
if [ -z "$run" ]; then
|
|
58
|
+
echo "::error::No Main Pipeline run found for tag ${TAG}. Push the tag first."
|
|
59
|
+
exit 1
|
|
60
|
+
fi
|
|
61
|
+
status=$(jq -r '.status' <<<"$run")
|
|
62
|
+
concl=$(jq -r '.conclusion' <<<"$run")
|
|
63
|
+
url=$(jq -r '.html_url' <<<"$run")
|
|
64
|
+
if [ "$status" = "completed" ]; then
|
|
65
|
+
if [ "$concl" = "success" ]; then
|
|
66
|
+
echo "✅ Main Pipeline for ${TAG} succeeded: $url"
|
|
67
|
+
exit 0
|
|
68
|
+
fi
|
|
69
|
+
echo "::error::Main Pipeline for ${TAG} concluded '${concl}', refusing to publish: $url"
|
|
70
|
+
exit 1
|
|
71
|
+
fi
|
|
72
|
+
if [ "$SECONDS" -ge "$deadline" ]; then
|
|
73
|
+
echo "::error::Main Pipeline for ${TAG} still '${status}' after 45 minutes: $url"
|
|
74
|
+
exit 1
|
|
75
|
+
fi
|
|
76
|
+
echo "⏳ Main Pipeline for ${TAG} is ${status}; waiting: $url"
|
|
77
|
+
sleep 30
|
|
78
|
+
done
|
|
79
|
+
|
|
80
|
+
- name: Install uv
|
|
81
|
+
uses: astral-sh/setup-uv@v7
|
|
82
|
+
|
|
83
|
+
- name: Set up Python
|
|
84
|
+
run: uv python install 3.10
|
|
85
|
+
|
|
86
|
+
- name: Build Wheel and sdist
|
|
87
|
+
run: uv build
|
|
88
|
+
|
|
89
|
+
- name: Publish package distributions to PyPI
|
|
90
|
+
uses: pypa/gh-action-pypi-publish@release/v1
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: odctl
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.3.1
|
|
4
4
|
Summary: Orchestrator for the Open Data Stack
|
|
5
5
|
Project-URL: Homepage, https://github.com/jaehyeon-kim/odctl
|
|
6
6
|
Project-URL: Repository, https://github.com/jaehyeon-kim/odctl
|
|
@@ -123,7 +123,7 @@ odctl list
|
|
|
123
123
|
Bring up a robust data engineering environment.
|
|
124
124
|
|
|
125
125
|
```bash
|
|
126
|
-
odctl up kafka-lite
|
|
126
|
+
odctl up kafka-lite flink-lite spark-lite
|
|
127
127
|
```
|
|
128
128
|
|
|
129
129
|
_Note: You do not need to memorize dependencies. The CLI will automatically detect that these profiles require foundational infrastructure and will launch PostgreSQL, SeaweedFS (S3), and the Iceberg REST Catalog for you before starting the target compute engines._
|
|
@@ -169,7 +169,7 @@ $ odctl list -d
|
|
|
169
169
|
$ odctl explain kafka-lite
|
|
170
170
|
|
|
171
171
|
# Launch specific profiles
|
|
172
|
-
$ odctl up
|
|
172
|
+
$ odctl up flink-lite kafka-lite spark-lite
|
|
173
173
|
|
|
174
174
|
# Complete teardown and wipe all data
|
|
175
175
|
$ odctl down --all --volumes
|
|
@@ -87,7 +87,7 @@ odctl list
|
|
|
87
87
|
Bring up a robust data engineering environment.
|
|
88
88
|
|
|
89
89
|
```bash
|
|
90
|
-
odctl up kafka-lite
|
|
90
|
+
odctl up kafka-lite flink-lite spark-lite
|
|
91
91
|
```
|
|
92
92
|
|
|
93
93
|
_Note: You do not need to memorize dependencies. The CLI will automatically detect that these profiles require foundational infrastructure and will launch PostgreSQL, SeaweedFS (S3), and the Iceberg REST Catalog for you before starting the target compute engines._
|
|
@@ -133,7 +133,7 @@ $ odctl list -d
|
|
|
133
133
|
$ odctl explain kafka-lite
|
|
134
134
|
|
|
135
135
|
# Launch specific profiles
|
|
136
|
-
$ odctl up
|
|
136
|
+
$ odctl up flink-lite kafka-lite spark-lite
|
|
137
137
|
|
|
138
138
|
# Complete teardown and wipe all data
|
|
139
139
|
$ odctl down --all --volumes
|
|
@@ -175,8 +175,12 @@ def stop_stack(
|
|
|
175
175
|
"""
|
|
176
176
|
path = get_compose_path(compose_filename)
|
|
177
177
|
stack_client = _create_client(compose_files=[str(path)], profiles=profiles)
|
|
178
|
-
# volumes=True will remove named volumes defined in the compose file
|
|
179
|
-
|
|
178
|
+
# volumes=True will remove named volumes defined in the compose file.
|
|
179
|
+
# remove_orphans catches containers whose service no longer exists in the
|
|
180
|
+
# compose file, which teardown would otherwise leave running forever: an
|
|
181
|
+
# upgrade that drops or renames a service (ch-init-lite in 0.2.0, the
|
|
182
|
+
# flink1-* services in 0.3.0) leaves the old container holding its name.
|
|
183
|
+
stack_client.compose.down(volumes=remove_volumes, remove_orphans=True)
|
|
180
184
|
|
|
181
185
|
|
|
182
186
|
def get_managed_containers(execution_plan: Dict[str, List[str]]) -> List[Any]:
|
|
@@ -36,7 +36,7 @@ Provides commands to inspect, provision, and tear down curated Docker Compose st
|
|
|
36
36
|
[dim]# See exactly what the airflow profile provisions[/dim]\n
|
|
37
37
|
$ [bold cyan]odctl explain kafka-lite[/bold cyan]\n\n
|
|
38
38
|
[dim]# Launch specific profiles and their dependencies[/dim]\n
|
|
39
|
-
$ [bold cyan]odctl up
|
|
39
|
+
$ [bold cyan]odctl up flink-lite kafka-lite spark-lite[/bold cyan]\n\n
|
|
40
40
|
[dim]# Complete teardown and wipe all data[/dim]\n
|
|
41
41
|
$ [bold cyan]odctl down --all --volumes[/bold cyan]
|
|
42
42
|
""",
|
|
@@ -201,8 +201,8 @@ def init(
|
|
|
201
201
|
[bold underline]Examples:[/bold underline]\n
|
|
202
202
|
[dim]# Pre-fetch images for all profiles[/dim]\n
|
|
203
203
|
$ [bold cyan]odctl pull --all[/bold cyan]\n\n
|
|
204
|
-
[dim]# Pre-fetch images just for Flink
|
|
205
|
-
$ [bold cyan]odctl pull
|
|
204
|
+
[dim]# Pre-fetch images just for Flink and Spark[/dim]\n
|
|
205
|
+
$ [bold cyan]odctl pull flink-lite kafka-lite[/bold cyan]
|
|
206
206
|
""",
|
|
207
207
|
)
|
|
208
208
|
def pull(
|
|
@@ -243,8 +243,8 @@ def pull(
|
|
|
243
243
|
rich_help_panel="Cluster Lifecycle",
|
|
244
244
|
epilog="""
|
|
245
245
|
[bold underline]Examples:[/bold underline]\n
|
|
246
|
-
[dim]# Launch Clickhouse, Flink
|
|
247
|
-
$ [bold cyan]odctl up ch-lite
|
|
246
|
+
[dim]# Launch Clickhouse, Flink, and their dependencies[/dim]\n
|
|
247
|
+
$ [bold cyan]odctl up ch-lite flink-lite[/bold cyan]\n\n
|
|
248
248
|
[dim]# Preview what would be launched for Airflow[/dim]\n
|
|
249
249
|
$ [bold cyan]odctl up airflow --dry-run[/bold cyan]\n\n
|
|
250
250
|
[dim]# Force pull latest images before launching[/dim]\n
|
|
@@ -464,9 +464,9 @@ def info():
|
|
|
464
464
|
epilog="""
|
|
465
465
|
[bold underline]Examples:[/bold underline]\n
|
|
466
466
|
[dim]# Tail the last 50 lines of all Flink containers and follow live[/dim]\n
|
|
467
|
-
$ [bold cyan]odctl logs
|
|
467
|
+
$ [bold cyan]odctl logs flink-lite -n 50 -f[/bold cyan]\n\n
|
|
468
468
|
[dim]# View logs strictly for the JobManager service[/dim]\n
|
|
469
|
-
$ [bold cyan]odctl logs
|
|
469
|
+
$ [bold cyan]odctl logs flink-lite -s jobmanager[/bold cyan]\n\n
|
|
470
470
|
[dim]# Show logs with timestamps for the last 10 minutes[/dim]\n
|
|
471
471
|
$ [bold cyan]odctl logs kafka-lite --since 10m -t[/bold cyan]
|
|
472
472
|
""",
|