odctl 0.2.2__tar.gz → 0.3.1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (85) hide show
  1. {odctl-0.2.2 → odctl-0.3.1}/.github/renovate.json +10 -0
  2. odctl-0.3.1/.github/scripts/smoke.sh +196 -0
  3. {odctl-0.2.2 → odctl-0.3.1}/.github/workflows/e2e-test.yml +33 -27
  4. odctl-0.3.1/.github/workflows/pipeline.yml +69 -0
  5. odctl-0.3.1/.github/workflows/publish.yml +90 -0
  6. {odctl-0.2.2 → odctl-0.3.1}/PKG-INFO +3 -3
  7. {odctl-0.2.2 → odctl-0.3.1}/README.md +2 -2
  8. {odctl-0.2.2 → odctl-0.3.1}/pyproject.toml +1 -1
  9. {odctl-0.2.2 → odctl-0.3.1}/src/odctl/docker.py +6 -2
  10. {odctl-0.2.2 → odctl-0.3.1}/src/odctl/main.py +7 -7
  11. {odctl-0.2.2 → odctl-0.3.1}/src/odctl/resources/compose-flink.yml +38 -30
  12. {odctl-0.2.2 → odctl-0.3.1}/src/odctl/resources/compose-infra.yml +2 -2
  13. {odctl-0.2.2 → odctl-0.3.1}/src/odctl/resources/compose-kafka.yml +5 -5
  14. {odctl-0.2.2 → odctl-0.3.1}/src/odctl/resources/compose-metadata.yml +2 -2
  15. {odctl-0.2.2 → odctl-0.3.1}/src/odctl/resources/compose-store.yml +2 -2
  16. {odctl-0.2.2 → odctl-0.3.1}/src/odctl/resources/deps/Dockerfile +11 -6
  17. {odctl-0.2.2 → odctl-0.3.1}/src/odctl/resources/deps/build.gradle +4 -2
  18. {odctl-0.2.2 → odctl-0.3.1}/src/odctl/resources/deps/fetch_flink_dependencies.sh +18 -14
  19. {odctl-0.2.2 → odctl-0.3.1}/src/odctl/resources/deps/fetch_spark_dependencies.sh +6 -2
  20. {odctl-0.2.2 → odctl-0.3.1}/src/odctl/resources/deps/fetch_standalone_plugins.sh +6 -2
  21. odctl-0.3.1/src/odctl/resources/deps/versions.env +19 -0
  22. odctl-0.2.2/src/odctl/resources/flink/flink1-conf.yml → odctl-0.3.1/src/odctl/resources/flink/flink-conf.yml +15 -5
  23. {odctl-0.2.2 → odctl-0.3.1}/src/odctl/resources/prometheus/prometheus.yml +4 -4
  24. {odctl-0.2.2 → odctl-0.3.1}/src/odctl/resources/registry.yml +10 -10
  25. {odctl-0.2.2 → odctl-0.3.1}/tests/test_docker.py +3 -1
  26. odctl-0.2.2/.github/workflows/pipeline.yml +0 -34
  27. odctl-0.2.2/.github/workflows/publish.yml +0 -34
  28. {odctl-0.2.2 → odctl-0.3.1}/.github/ISSUE_TEMPLATE/bug_report.md +0 -0
  29. {odctl-0.2.2 → odctl-0.3.1}/.github/ISSUE_TEMPLATE/feature_request.md +0 -0
  30. {odctl-0.2.2 → odctl-0.3.1}/.github/workflows/build-deps.yml +0 -0
  31. {odctl-0.2.2 → odctl-0.3.1}/.github/workflows/build-platform-images.yml +0 -0
  32. {odctl-0.2.2 → odctl-0.3.1}/.github/workflows/ci.yml +0 -0
  33. {odctl-0.2.2 → odctl-0.3.1}/.gitignore +0 -0
  34. {odctl-0.2.2 → odctl-0.3.1}/.pre-commit-config.yaml +0 -0
  35. {odctl-0.2.2 → odctl-0.3.1}/.vscode/settings.json +0 -0
  36. {odctl-0.2.2 → odctl-0.3.1}/LICENSE +0 -0
  37. {odctl-0.2.2 → odctl-0.3.1}/image/diagram.png +0 -0
  38. {odctl-0.2.2 → odctl-0.3.1}/image/diagram1.drawio +0 -0
  39. {odctl-0.2.2 → odctl-0.3.1}/image/diagram2.drawio +0 -0
  40. {odctl-0.2.2 → odctl-0.3.1}/src/odctl/__init__.py +0 -0
  41. {odctl-0.2.2 → odctl-0.3.1}/src/odctl/config.py +0 -0
  42. {odctl-0.2.2 → odctl-0.3.1}/src/odctl/planner.py +0 -0
  43. {odctl-0.2.2 → odctl-0.3.1}/src/odctl/registry.py +0 -0
  44. {odctl-0.2.2 → odctl-0.3.1}/src/odctl/resources/clickhouse/config.xml +0 -0
  45. {odctl-0.2.2 → odctl-0.3.1}/src/odctl/resources/clickhouse/init.sql +0 -0
  46. {odctl-0.2.2 → odctl-0.3.1}/src/odctl/resources/clickhouse/keeper_config.xml +0 -0
  47. {odctl-0.2.2 → odctl-0.3.1}/src/odctl/resources/clickhouse/macros-11.xml +0 -0
  48. {odctl-0.2.2 → odctl-0.3.1}/src/odctl/resources/clickhouse/macros-12.xml +0 -0
  49. {odctl-0.2.2 → odctl-0.3.1}/src/odctl/resources/clickhouse/macros-21.xml +0 -0
  50. {odctl-0.2.2 → odctl-0.3.1}/src/odctl/resources/clickhouse/macros-22.xml +0 -0
  51. {odctl-0.2.2 → odctl-0.3.1}/src/odctl/resources/clickhouse/users.xml +0 -0
  52. {odctl-0.2.2 → odctl-0.3.1}/src/odctl/resources/compose-analytics.yml +0 -0
  53. {odctl-0.2.2 → odctl-0.3.1}/src/odctl/resources/compose-deps.yml +0 -0
  54. {odctl-0.2.2 → odctl-0.3.1}/src/odctl/resources/compose-mlops.yml +0 -0
  55. {odctl-0.2.2 → odctl-0.3.1}/src/odctl/resources/compose-obsv.yml +0 -0
  56. {odctl-0.2.2 → odctl-0.3.1}/src/odctl/resources/compose-orch.yml +0 -0
  57. {odctl-0.2.2 → odctl-0.3.1}/src/odctl/resources/compose-spark.yml +0 -0
  58. {odctl-0.2.2 → odctl-0.3.1}/src/odctl/resources/deps/build_connectors.sh +0 -0
  59. {odctl-0.2.2 → odctl-0.3.1}/src/odctl/resources/docker/airflow/Dockerfile +0 -0
  60. {odctl-0.2.2 → odctl-0.3.1}/src/odctl/resources/docker/ray-mlops/Dockerfile +0 -0
  61. {odctl-0.2.2 → odctl-0.3.1}/src/odctl/resources/docker/spark/Dockerfile +0 -0
  62. {odctl-0.2.2 → odctl-0.3.1}/src/odctl/resources/grafana/prometheus.yml +0 -0
  63. {odctl-0.2.2 → odctl-0.3.1}/src/odctl/resources/marquez/marquez.yml +0 -0
  64. {odctl-0.2.2 → odctl-0.3.1}/src/odctl/resources/mlops/serving/lightgbm_inference.py.example +0 -0
  65. {odctl-0.2.2 → odctl-0.3.1}/src/odctl/resources/mlops/serving/vector_search.py.example +0 -0
  66. {odctl-0.2.2 → odctl-0.3.1}/src/odctl/resources/postgres/01-init-databases.sh +0 -0
  67. {odctl-0.2.2 → odctl-0.3.1}/src/odctl/resources/prometheus/alertmanager.yml +0 -0
  68. {odctl-0.2.2 → odctl-0.3.1}/src/odctl/resources/seaweed/s3.json +0 -0
  69. {odctl-0.2.2 → odctl-0.3.1}/src/odctl/resources/spark/log4j2.properties +0 -0
  70. {odctl-0.2.2 → odctl-0.3.1}/src/odctl/resources/spark/spark-defaults.conf +0 -0
  71. {odctl-0.2.2 → odctl-0.3.1}/src/odctl/resources/trino/catalog/clickhouse.properties +0 -0
  72. {odctl-0.2.2 → odctl-0.3.1}/src/odctl/resources/trino/catalog/iceberg.properties +0 -0
  73. {odctl-0.2.2 → odctl-0.3.1}/src/odctl/resources/trino/catalog/kafka.properties +0 -0
  74. {odctl-0.2.2 → odctl-0.3.1}/src/odctl/resources/trino/catalog/postgres.properties +0 -0
  75. {odctl-0.2.2 → odctl-0.3.1}/src/odctl/resources/trino/catalog/valkey.properties +0 -0
  76. {odctl-0.2.2 → odctl-0.3.1}/src/odctl/ui.py +0 -0
  77. {odctl-0.2.2 → odctl-0.3.1}/src/odctl/workspace.py +0 -0
  78. {odctl-0.2.2 → odctl-0.3.1}/tests/conftest.py +0 -0
  79. {odctl-0.2.2 → odctl-0.3.1}/tests/test_cli.py +0 -0
  80. {odctl-0.2.2 → odctl-0.3.1}/tests/test_cli_extended.py +0 -0
  81. {odctl-0.2.2 → odctl-0.3.1}/tests/test_config.py +0 -0
  82. {odctl-0.2.2 → odctl-0.3.1}/tests/test_planner.py +0 -0
  83. {odctl-0.2.2 → odctl-0.3.1}/tests/test_registry.py +0 -0
  84. {odctl-0.2.2 → odctl-0.3.1}/tests/test_workspace.py +0 -0
  85. {odctl-0.2.2 → odctl-0.3.1}/uv.lock +0 -0
@@ -12,6 +12,16 @@
12
12
  ]
13
13
  },
14
14
  "packageRules": [
15
+ {
16
+ "description": "Flink stays on 2.1: Iceberg publishes iceberg-flink-runtime for 2.0 and 2.1 only, and fluss-flink has no 2.0 build. Raise this when Iceberg ships a runtime for a newer minor. See issue #9.",
17
+ "matchPackageNames": ["flink", "org.apache.flink:{/,}**"],
18
+ "allowedVersions": "<2.2"
19
+ },
20
+ {
21
+ "description": "Fluss stays on 0.8.x because 0.9.x requires Flink 2.2, which has no Iceberg runtime. Moves together with the Flink ceiling above.",
22
+ "matchPackageNames": ["apache/fluss", "org.apache.fluss:{/,}**"],
23
+ "allowedVersions": "<0.9"
24
+ },
15
25
  {
16
26
  "description": "Auto-merge non-breaking minor and patch updates if CI passes",
17
27
  "matchUpdateTypes": ["minor", "patch"],
@@ -0,0 +1,196 @@
1
+ #!/usr/bin/env bash
2
+ # Functional smoke test for one odctl profile.
3
+ #
4
+ # Liveness is not enough: every bug fixed in 0.2.x and 0.3.0 left its containers
5
+ # running and healthy while the profile was unusable. Kafka consumer groups hung
6
+ # forever, Spark could not start a SparkContext, Flink could not find a connector
7
+ # factory, ClickHouse could not consume from the broker, the Trino ClickHouse
8
+ # catalog pointed at a hostname that did not exist, and the Flink Iceberg sink
9
+ # committed nothing while its checkpoints succeeded. Each assertion below exists
10
+ # because something shipped broken past a liveness check.
11
+ #
12
+ # Usage: smoke.sh <profile>
13
+ set -uo pipefail
14
+
15
+ PROFILE="${1:?usage: smoke.sh <profile>}"
16
+
17
+ fail() { echo "❌ $PROFILE: $*"; exit 1; }
18
+ pass() { echo "✅ $PROFILE: $*"; }
19
+
20
+ # Retry a command until it succeeds or the budget runs out.
21
+ retry() {
22
+ local tries="$1" delay="$2"; shift 2
23
+ for ((i = 1; i <= tries; i++)); do
24
+ if "$@" >/dev/null 2>&1; then return 0; fi
25
+ sleep "$delay"
26
+ done
27
+ return 1
28
+ }
29
+
30
+ http_ok() { curl -fsS -o /dev/null --max-time 10 "$1"; }
31
+
32
+ # Some endpoints answer 403 or 404 to an unauthenticated probe, which still
33
+ # proves the service is listening.
34
+ http_reachable() {
35
+ local code
36
+ code=$(curl -s -o /dev/null --max-time 10 -w '%{http_code}' "$1")
37
+ [ -n "$code" ] && [ "$code" != "000" ]
38
+ }
39
+
40
+ ch() { docker exec "$1" clickhouse-client --password password -q "$2" 2>&1; }
41
+
42
+ kafka_topic() {
43
+ docker exec kafka /opt/kafka/bin/kafka-topics.sh --bootstrap-server broker-1:19092 "$@"
44
+ }
45
+
46
+ smoke_kafka() {
47
+ local container="$1"
48
+ retry 30 5 docker exec "$container" /opt/kafka/bin/kafka-topics.sh \
49
+ --bootstrap-server broker-1:19092 --list || fail "broker never became reachable"
50
+ docker exec "$container" /opt/kafka/bin/kafka-topics.sh --bootstrap-server broker-1:19092 \
51
+ --create --topic smoke --partitions 1 --replication-factor 1 >/dev/null 2>&1
52
+ for i in 1 2 3 4 5; do echo "{\"n\":$i}"; done | docker exec -i "$container" \
53
+ /opt/kafka/bin/kafka-console-producer.sh --bootstrap-server broker-1:19092 --topic smoke >/dev/null 2>&1 \
54
+ || fail "produce failed"
55
+ # Group mode on purpose. Assign mode bypasses the coordinator and would pass
56
+ # even when __consumer_offsets cannot be created on a single broker.
57
+ local got
58
+ got=$(docker exec "$container" /opt/kafka/bin/kafka-console-consumer.sh \
59
+ --bootstrap-server broker-1:19092 --topic smoke --from-beginning \
60
+ --max-messages 5 --timeout-ms 60000 2>/dev/null | grep -c '"n"')
61
+ [ "$got" -eq 5 ] || fail "group-mode consume returned $got of 5 messages"
62
+ pass "produced and consumed 5 messages through a consumer group"
63
+ }
64
+
65
+ smoke_flink() {
66
+ retry 40 5 http_ok "http://127.0.0.1:8082/config" || fail "JobManager REST never answered"
67
+ local out
68
+ out=$(docker exec -i flink-jobmanager /opt/flink/bin/sql-client.sh 2>&1 <<'SQL'
69
+ CREATE CATALOG ice WITH ('type'='iceberg','catalog-type'='rest','uri'='http://catalog:8181','warehouse'='s3://warehouse','s3.endpoint'='http://seaweed:8333','s3.path-style-access'='true','s3.access-key-id'='user','s3.secret-access-key'='password');
70
+ SHOW CATALOGS;
71
+ SQL
72
+ )
73
+ grep -q "ice" <<<"$out" || { echo "$out" | tail -20; fail "Iceberg catalog did not register (connector factory or Hadoop classes missing)"; }
74
+ # The image ships --add-opens in config.yaml; mounting a replacement wipes them
75
+ # and Kryo then fails on java.nio buffers at checkpoint time.
76
+ docker exec flink-jobmanager sh -c 'ps ax | grep -q "add-opens=java.base/java.nio"' \
77
+ || fail "JVM module flags missing from the TaskManager command line"
78
+ pass "Iceberg catalog registered and JVM module flags present"
79
+ }
80
+
81
+ smoke_spark() {
82
+ local out
83
+ out=$(docker exec spark-master /opt/spark/bin/spark-sql -e "
84
+ CREATE NAMESPACE IF NOT EXISTS iceberg.smoke;
85
+ CREATE OR REPLACE TABLE iceberg.smoke.t (id BIGINT) USING iceberg;
86
+ INSERT INTO iceberg.smoke.t VALUES (1),(2),(3);
87
+ SELECT count(*) FROM iceberg.smoke.t;" 2>&1)
88
+ grep -qE "^3$" <<<"$out" || { echo "$out" | tail -20; fail "spark-sql could not round-trip an Iceberg table"; }
89
+ docker exec spark-master sh -c 'ls /tmp/spark-events | head -1' >/dev/null 2>&1 \
90
+ || fail "no event log written, so the event log directory is not writable"
91
+ pass "spark-sql wrote and read an Iceberg table, event logging works"
92
+ }
93
+
94
+ smoke_ch_lite() {
95
+ retry 30 5 docker exec ch-11 clickhouse-client --password password -q "SELECT 1" \
96
+ || fail "ch-11 never answered"
97
+ # Databases come from /docker-entrypoint-initdb.d on every fresh server.
98
+ ch ch-11 "SHOW DATABASES" | grep -q feature_store || fail "feature_store database missing"
99
+ ch ch-11 "DROP TABLE IF EXISTS default.smoke" >/dev/null
100
+ ch ch-11 "CREATE TABLE default.smoke (id UInt32) ENGINE=MergeTree ORDER BY id" >/dev/null
101
+ ch ch-11 "INSERT INTO default.smoke SELECT number FROM numbers(10)" >/dev/null
102
+ [ "$(ch ch-11 "SELECT count() FROM default.smoke")" = "10" ] || fail "MergeTree round trip failed"
103
+ pass "databases initialised and MergeTree round trip works"
104
+ }
105
+
106
+ smoke_ch_full() {
107
+ retry 30 5 docker exec ch-21 clickhouse-client --password password -q "SELECT 1" \
108
+ || fail "ch-21 never answered"
109
+ # Exercises keeper and the dual-stack listeners: replicated DDL fails with
110
+ # "Connection refused" when keeper listens on IPv4 only.
111
+ ch ch-11 "DROP TABLE IF EXISTS default.repl ON CLUSTER '{cluster}' SYNC" >/dev/null 2>&1
112
+ ch ch-11 "CREATE TABLE default.repl ON CLUSTER '{cluster}' (id UInt32) ENGINE=ReplicatedMergeTree ORDER BY id" >/dev/null \
113
+ || fail "replicated DDL failed, check keeper reachability"
114
+ ch ch-11 "INSERT INTO default.repl SELECT number FROM numbers(50)" >/dev/null
115
+ retry 12 5 bash -c '[ "$(docker exec ch-12 clickhouse-client --password password -q "SELECT count() FROM default.repl" 2>/dev/null)" = "50" ]' \
116
+ || fail "rows never replicated to the sibling replica"
117
+ [ "$(ch ch-21 "SELECT count() FROM default.repl")" = "0" ] || fail "shard 2 unexpectedly holds shard 1 data"
118
+ pass "replicated table converged on shard 1 and stayed off shard 2"
119
+ }
120
+
121
+ # Catalog registration finishes after the health endpoint starts answering, so
122
+ # poll rather than asserting once. Catalogs load even when their backing service
123
+ # is absent, which is why this can be required in a trino-only profile group.
124
+ catalogs_loaded() {
125
+ local cat
126
+ cat=$(docker exec trino trino --execute "SHOW CATALOGS" 2>/dev/null | tr -d '"')
127
+ for expected in iceberg kafka clickhouse postgres; do
128
+ grep -q "^${expected}$" <<<"$cat" || return 1
129
+ done
130
+ }
131
+
132
+ smoke_trino() {
133
+ retry 40 5 http_ok "http://127.0.0.1:8080/v1/info" || fail "Trino never answered"
134
+ retry 24 5 catalogs_loaded || {
135
+ docker exec trino trino --execute "SHOW CATALOGS" 2>&1 | tail -10
136
+ fail "not every catalog loaded within two minutes"
137
+ }
138
+ docker exec trino trino --execute "SELECT 1" >/dev/null 2>&1 || fail "query execution failed"
139
+ pass "all catalogs loaded and a query ran"
140
+ }
141
+
142
+ smoke_infra() {
143
+ case "$PROFILE" in
144
+ postgres)
145
+ retry 30 5 docker exec postgres pg_isready -U user || fail "postgres never became ready"
146
+ docker exec postgres psql -U user -d odctl -c "SELECT 1" >/dev/null 2>&1 || fail "psql query failed"
147
+ pass "accepting connections and running queries" ;;
148
+ storage)
149
+ retry 30 5 http_reachable "http://127.0.0.1:8333" || fail "S3 API never answered"
150
+ pass "S3 API reachable" ;;
151
+ catalog)
152
+ retry 30 5 http_ok "http://127.0.0.1:8181/v1/config" || fail "REST catalog never answered"
153
+ curl -fsS -X POST -H 'Content-Type: application/json' \
154
+ -d '{"namespace":["smoke"]}' "http://127.0.0.1:8181/v1/namespaces" >/dev/null 2>&1
155
+ curl -fsS "http://127.0.0.1:8181/v1/namespaces" 2>/dev/null | grep -q smoke \
156
+ || fail "namespace create or list failed"
157
+ pass "namespace created and listed through the REST API" ;;
158
+ valkey)
159
+ retry 30 5 docker exec valkey valkey-cli -u redis://user:password@localhost:6379 ping \
160
+ || fail "valkey never answered PING"
161
+ pass "responding to PING" ;;
162
+ esac
163
+ }
164
+
165
+ # A profile with no functional assertion yet still has to expose its endpoint.
166
+ smoke_http_only() {
167
+ local url="$1"
168
+ retry 60 5 http_ok "$url" || fail "no HTTP response from $url"
169
+ pass "HTTP endpoint answering at $url"
170
+ }
171
+
172
+ case "$PROFILE" in
173
+ kafka-lite) smoke_kafka kafka ;;
174
+ kafka-full) smoke_kafka kafka-1 ;;
175
+ flink-lite|flink-full) smoke_flink ;;
176
+ spark-lite|spark-full) smoke_spark ;;
177
+ ch-lite) smoke_ch_lite ;;
178
+ ch-full) smoke_ch_full ;;
179
+ trino) smoke_trino ;;
180
+ postgres|storage|catalog|valkey) smoke_infra ;;
181
+ metabase) smoke_http_only "http://127.0.0.1:3000/api/health" ;;
182
+ airflow) smoke_http_only "http://127.0.0.1:8085/api/v2/monitor/health" ;;
183
+ mlflow) smoke_http_only "http://127.0.0.1:5000/health" ;;
184
+ ray-serve) smoke_http_only "http://127.0.0.1:8265" ;;
185
+ lineage) smoke_http_only "http://127.0.0.1:5002/api/v1/namespaces" ;;
186
+ telemetry) smoke_http_only "http://127.0.0.1:19090/-/ready" ;;
187
+ metadata) smoke_http_only "http://127.0.0.1:8585/api/v1/system/version" ;;
188
+ fluss)
189
+ retry 40 5 bash -c 'docker ps --format "{{.Names}}" | grep -q fluss-coordinator' \
190
+ || fail "coordinator container never appeared"
191
+ pass "coordinator running" ;;
192
+ *)
193
+ echo "ℹ️ $PROFILE: no functional assertion defined, checking containers only"
194
+ [ "$(docker ps -q | wc -l)" -ge 1 ] || fail "no containers running"
195
+ pass "containers running" ;;
196
+ esac
@@ -3,8 +3,6 @@ name: End-to-End Integration Tests
3
3
  on:
4
4
  workflow_call:
5
5
 
6
-
7
-
8
6
  jobs:
9
7
  e2e:
10
8
  runs-on: ubuntu-latest
@@ -12,18 +10,30 @@ jobs:
12
10
  contents: read
13
11
  packages: read
14
12
  strategy:
13
+ fail-fast: false
15
14
  matrix:
15
+ # Profiles are grouped only where they share a large image, so the pull
16
+ # is amortised: the lite and full variants of Kafka, Flink, Spark and
17
+ # ClickHouse. Everything else runs alone, which keeps the failure signal
18
+ # per profile and shortens the critical path.
16
19
  profile_group:
17
- - "metadata"
18
- - "airflow"
19
20
  - "kafka-lite kafka-full"
20
- - "flink1-lite flink1-full fluss"
21
+ - "flink-lite flink-full"
21
22
  - "spark-lite spark-full"
22
23
  - "ch-lite ch-full"
23
- - "trino metabase"
24
- - "mlflow ray-serve"
25
- - "lineage telemetry"
26
- - "postgres storage catalog valkey"
24
+ - "fluss"
25
+ - "trino"
26
+ - "metabase"
27
+ - "mlflow"
28
+ - "ray-serve"
29
+ - "lineage"
30
+ - "telemetry"
31
+ - "metadata"
32
+ - "airflow"
33
+ - "postgres"
34
+ - "storage"
35
+ - "catalog"
36
+ - "valkey"
27
37
  steps:
28
38
  - name: Checkout repository
29
39
  uses: actions/checkout@v4
@@ -49,31 +59,27 @@ jobs:
49
59
 
50
60
  - name: Test Profile Group
51
61
  run: |
52
- # Split the space-separated group into an array
62
+ chmod +x .github/scripts/smoke.sh
53
63
  for profile in ${{ matrix.profile_group }}; do
54
64
  echo "::group::Testing Profile: $profile"
55
65
  echo "🚀 Launching $profile..."
56
- odctl up $profile
66
+ odctl up "$profile"
57
67
 
58
- echo "⏳ Validating containers..."
59
- SUCCESS=0
60
- for i in {1..15}; do
61
- RUNNING=$(docker ps -q | wc -l)
62
- if [ "$RUNNING" -ge 1 ]; then
63
- echo "✅ Containers are running!"
64
- docker ps
65
- odctl ps --all
66
- SUCCESS=1
67
- break
68
- fi
69
- echo "Waiting... ($i/15)"
70
- sleep 2
71
- done
68
+ echo "⏳ Containers:"
69
+ docker ps
72
70
 
73
- if [ "$SUCCESS" -eq 0 ]; then
74
- echo "❌ Timeout reached: No containers are running for $profile!"
71
+ # Liveness alone has never been enough here: every bug fixed in 0.2.x
72
+ # and 0.3.0 left healthy containers behind a broken profile, so the
73
+ # smoke script asserts the profile can actually do its job.
74
+ echo "🔍 Smoke testing $profile..."
75
+ if ! .github/scripts/smoke.sh "$profile"; then
76
+ echo "::error::Smoke test failed for $profile"
75
77
  docker ps -a
76
78
  odctl ps --all
79
+ for c in $(docker ps -aq); do
80
+ echo "---- logs: $(docker inspect --format '{{.Name}}' "$c") ----"
81
+ docker logs --tail 40 "$c" 2>&1 || true
82
+ done
77
83
  exit 1
78
84
  fi
79
85
 
@@ -0,0 +1,69 @@
1
+ name: Main Pipeline
2
+
3
+ on:
4
+ push:
5
+ branches: ["main"]
6
+ tags: ["v*"]
7
+ pull_request:
8
+ branches: ["main"]
9
+ workflow_dispatch:
10
+
11
+ concurrency:
12
+ group: ${{ github.workflow }}-${{ github.ref }}
13
+ cancel-in-progress: true
14
+
15
+ permissions:
16
+ contents: read
17
+ packages: write
18
+
19
+ jobs:
20
+ # Which parts of the tree moved. Used to skip the multi-arch image builds on
21
+ # changes that cannot affect them, which is most changes.
22
+ changes:
23
+ runs-on: ubuntu-latest
24
+ permissions:
25
+ contents: read
26
+ pull-requests: read
27
+ outputs:
28
+ images: ${{ steps.filter.outputs.images }}
29
+ steps:
30
+ - uses: actions/checkout@v4
31
+ - uses: dorny/paths-filter@v3
32
+ id: filter
33
+ with:
34
+ filters: |
35
+ images:
36
+ - 'src/odctl/resources/deps/**'
37
+ - 'src/odctl/resources/docker/**'
38
+ - '.github/workflows/build-deps.yml'
39
+ - '.github/workflows/build-platform-images.yml'
40
+
41
+ ci:
42
+ uses: ./.github/workflows/ci.yml
43
+
44
+ # Images are pushed only on a tag, so on any other ref this is a
45
+ # build-and-discard. Run it when the build inputs changed, so a broken
46
+ # Dockerfile surfaces before release instead of at tag time, and skip it
47
+ # otherwise. That removes two multi-arch builds from the typical change.
48
+ build-platform:
49
+ needs: [ci, changes]
50
+ if: >-
51
+ startsWith(github.ref, 'refs/tags/v') ||
52
+ github.event_name == 'workflow_dispatch' ||
53
+ needs.changes.outputs.images == 'true'
54
+ uses: ./.github/workflows/build-platform-images.yml
55
+
56
+ build-deps:
57
+ needs: [ci, changes]
58
+ if: >-
59
+ startsWith(github.ref, 'refs/tags/v') ||
60
+ github.event_name == 'workflow_dispatch' ||
61
+ needs.changes.outputs.images == 'true'
62
+ uses: ./.github/workflows/build-deps.yml
63
+
64
+ # Tags and manual runs only: the e2e job pulls images at the CLI's own version,
65
+ # and those exist on ghcr only once a tag has published them.
66
+ e2e:
67
+ needs: [build-platform, build-deps]
68
+ if: startsWith(github.ref, 'refs/tags/v') || github.event_name == 'workflow_dispatch'
69
+ uses: ./.github/workflows/e2e-test.yml
@@ -0,0 +1,90 @@
1
+ name: Publish to PyPI
2
+
3
+ # Publishing stays tied to publishing a release, so the release notes remain the
4
+ # trigger. What is new is the gate below: a release event fires independently of
5
+ # CI, which is how v0.3.0 reached PyPI while the pipeline's e2e job was failing.
6
+ # This job now waits for the tag's Main Pipeline run and refuses to publish
7
+ # unless it succeeded.
8
+ on:
9
+ release:
10
+ types: [published]
11
+ workflow_dispatch:
12
+ inputs:
13
+ tag:
14
+ description: "Tag to publish, for example v0.3.1"
15
+ required: true
16
+ skip_pipeline_check:
17
+ description: "Publish even if the tag's pipeline is not green"
18
+ type: boolean
19
+ default: false
20
+
21
+ concurrency:
22
+ group: ${{ github.workflow }}-${{ github.event.release.tag_name || inputs.tag }}
23
+ cancel-in-progress: false
24
+
25
+ jobs:
26
+ pypi-publish:
27
+ name: Build and publish Python package
28
+ runs-on: ubuntu-latest
29
+ # These permissions are strictly required for PyPI OIDC Trusted Publishing;
30
+ # actions:read is what lets the gate query the pipeline's conclusion.
31
+ permissions:
32
+ contents: read
33
+ id-token: write
34
+ actions: read
35
+
36
+ steps:
37
+ - name: Checkout source code
38
+ uses: actions/checkout@v7
39
+ with:
40
+ ref: ${{ github.event.release.tag_name || inputs.tag }}
41
+
42
+ - name: Require a green Main Pipeline for this tag
43
+ if: ${{ !inputs.skip_pipeline_check }}
44
+ env:
45
+ GH_TOKEN: ${{ github.token }}
46
+ TAG: ${{ github.event.release.tag_name || inputs.tag }}
47
+ run: |
48
+ set -euo pipefail
49
+ # Wait rather than fail outright: a release is usually published moments
50
+ # after the tag is pushed, so the pipeline is still running e2e.
51
+ deadline=$(( SECONDS + 45 * 60 ))
52
+ while :; do
53
+ run=$(gh api "repos/${GITHUB_REPOSITORY}/actions/runs?event=push&per_page=100" \
54
+ --jq "[.workflow_runs[]
55
+ | select(.name == \"Main Pipeline\" and .head_branch == \"${TAG}\")]
56
+ | sort_by(.created_at) | last // empty")
57
+ if [ -z "$run" ]; then
58
+ echo "::error::No Main Pipeline run found for tag ${TAG}. Push the tag first."
59
+ exit 1
60
+ fi
61
+ status=$(jq -r '.status' <<<"$run")
62
+ concl=$(jq -r '.conclusion' <<<"$run")
63
+ url=$(jq -r '.html_url' <<<"$run")
64
+ if [ "$status" = "completed" ]; then
65
+ if [ "$concl" = "success" ]; then
66
+ echo "✅ Main Pipeline for ${TAG} succeeded: $url"
67
+ exit 0
68
+ fi
69
+ echo "::error::Main Pipeline for ${TAG} concluded '${concl}', refusing to publish: $url"
70
+ exit 1
71
+ fi
72
+ if [ "$SECONDS" -ge "$deadline" ]; then
73
+ echo "::error::Main Pipeline for ${TAG} still '${status}' after 45 minutes: $url"
74
+ exit 1
75
+ fi
76
+ echo "⏳ Main Pipeline for ${TAG} is ${status}; waiting: $url"
77
+ sleep 30
78
+ done
79
+
80
+ - name: Install uv
81
+ uses: astral-sh/setup-uv@v7
82
+
83
+ - name: Set up Python
84
+ run: uv python install 3.10
85
+
86
+ - name: Build Wheel and sdist
87
+ run: uv build
88
+
89
+ - name: Publish package distributions to PyPI
90
+ uses: pypa/gh-action-pypi-publish@release/v1
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: odctl
3
- Version: 0.2.2
3
+ Version: 0.3.1
4
4
  Summary: Orchestrator for the Open Data Stack
5
5
  Project-URL: Homepage, https://github.com/jaehyeon-kim/odctl
6
6
  Project-URL: Repository, https://github.com/jaehyeon-kim/odctl
@@ -123,7 +123,7 @@ odctl list
123
123
  Bring up a robust data engineering environment.
124
124
 
125
125
  ```bash
126
- odctl up kafka-lite flink1-lite spark-lite
126
+ odctl up kafka-lite flink-lite spark-lite
127
127
  ```
128
128
 
129
129
  _Note: You do not need to memorize dependencies. The CLI will automatically detect that these profiles require foundational infrastructure and will launch PostgreSQL, SeaweedFS (S3), and the Iceberg REST Catalog for you before starting the target compute engines._
@@ -169,7 +169,7 @@ $ odctl list -d
169
169
  $ odctl explain kafka-lite
170
170
 
171
171
  # Launch specific profiles
172
- $ odctl up flink1-lite kafka-lite spark-lite
172
+ $ odctl up flink-lite kafka-lite spark-lite
173
173
 
174
174
  # Complete teardown and wipe all data
175
175
  $ odctl down --all --volumes
@@ -87,7 +87,7 @@ odctl list
87
87
  Bring up a robust data engineering environment.
88
88
 
89
89
  ```bash
90
- odctl up kafka-lite flink1-lite spark-lite
90
+ odctl up kafka-lite flink-lite spark-lite
91
91
  ```
92
92
 
93
93
  _Note: You do not need to memorize dependencies. The CLI will automatically detect that these profiles require foundational infrastructure and will launch PostgreSQL, SeaweedFS (S3), and the Iceberg REST Catalog for you before starting the target compute engines._
@@ -133,7 +133,7 @@ $ odctl list -d
133
133
  $ odctl explain kafka-lite
134
134
 
135
135
  # Launch specific profiles
136
- $ odctl up flink1-lite kafka-lite spark-lite
136
+ $ odctl up flink-lite kafka-lite spark-lite
137
137
 
138
138
  # Complete teardown and wipe all data
139
139
  $ odctl down --all --volumes
@@ -1,6 +1,6 @@
1
1
  [project]
2
2
  name = "odctl"
3
- version = "0.2.2"
3
+ version = "0.3.1"
4
4
  description = "Orchestrator for the Open Data Stack"
5
5
  readme = "README.md"
6
6
  authors = [
@@ -175,8 +175,12 @@ def stop_stack(
175
175
  """
176
176
  path = get_compose_path(compose_filename)
177
177
  stack_client = _create_client(compose_files=[str(path)], profiles=profiles)
178
- # volumes=True will remove named volumes defined in the compose file
179
- stack_client.compose.down(volumes=remove_volumes)
178
+ # volumes=True will remove named volumes defined in the compose file.
179
+ # remove_orphans catches containers whose service no longer exists in the
180
+ # compose file, which teardown would otherwise leave running forever: an
181
+ # upgrade that drops or renames a service (ch-init-lite in 0.2.0, the
182
+ # flink1-* services in 0.3.0) leaves the old container holding its name.
183
+ stack_client.compose.down(volumes=remove_volumes, remove_orphans=True)
180
184
 
181
185
 
182
186
  def get_managed_containers(execution_plan: Dict[str, List[str]]) -> List[Any]:
@@ -36,7 +36,7 @@ Provides commands to inspect, provision, and tear down curated Docker Compose st
36
36
  [dim]# See exactly what the airflow profile provisions[/dim]\n
37
37
  $ [bold cyan]odctl explain kafka-lite[/bold cyan]\n\n
38
38
  [dim]# Launch specific profiles and their dependencies[/dim]\n
39
- $ [bold cyan]odctl up flink1-lite kafka-lite spark-lite[/bold cyan]\n\n
39
+ $ [bold cyan]odctl up flink-lite kafka-lite spark-lite[/bold cyan]\n\n
40
40
  [dim]# Complete teardown and wipe all data[/dim]\n
41
41
  $ [bold cyan]odctl down --all --volumes[/bold cyan]
42
42
  """,
@@ -201,8 +201,8 @@ def init(
201
201
  [bold underline]Examples:[/bold underline]\n
202
202
  [dim]# Pre-fetch images for all profiles[/dim]\n
203
203
  $ [bold cyan]odctl pull --all[/bold cyan]\n\n
204
- [dim]# Pre-fetch images just for Flink 1.x and Spark[/dim]\n
205
- $ [bold cyan]odctl pull flink1-lite kafka-lite[/bold cyan]
204
+ [dim]# Pre-fetch images just for Flink and Spark[/dim]\n
205
+ $ [bold cyan]odctl pull flink-lite kafka-lite[/bold cyan]
206
206
  """,
207
207
  )
208
208
  def pull(
@@ -243,8 +243,8 @@ def pull(
243
243
  rich_help_panel="Cluster Lifecycle",
244
244
  epilog="""
245
245
  [bold underline]Examples:[/bold underline]\n
246
- [dim]# Launch Clickhouse, Flink 1.x, and their dependencies[/dim]\n
247
- $ [bold cyan]odctl up ch-lite flink1-lite[/bold cyan]\n\n
246
+ [dim]# Launch Clickhouse, Flink, and their dependencies[/dim]\n
247
+ $ [bold cyan]odctl up ch-lite flink-lite[/bold cyan]\n\n
248
248
  [dim]# Preview what would be launched for Airflow[/dim]\n
249
249
  $ [bold cyan]odctl up airflow --dry-run[/bold cyan]\n\n
250
250
  [dim]# Force pull latest images before launching[/dim]\n
@@ -464,9 +464,9 @@ def info():
464
464
  epilog="""
465
465
  [bold underline]Examples:[/bold underline]\n
466
466
  [dim]# Tail the last 50 lines of all Flink containers and follow live[/dim]\n
467
- $ [bold cyan]odctl logs flink1-lite -n 50 -f[/bold cyan]\n\n
467
+ $ [bold cyan]odctl logs flink-lite -n 50 -f[/bold cyan]\n\n
468
468
  [dim]# View logs strictly for the JobManager service[/dim]\n
469
- $ [bold cyan]odctl logs flink1-lite -s jobmanager-1[/bold cyan]\n\n
469
+ $ [bold cyan]odctl logs flink-lite -s jobmanager[/bold cyan]\n\n
470
470
  [dim]# Show logs with timestamps for the last 10 minutes[/dim]\n
471
471
  $ [bold cyan]odctl logs kafka-lite --since 10m -t[/bold cyan]
472
472
  """,