odctl 0.3.0__tar.gz → 0.3.1__tar.gz

odctl-0.3.1/.github/scripts/smoke.sh

@@ -0,0 +1,196 @@
+#!/usr/bin/env bash
+# Functional smoke test for one odctl profile.
+#
+# Liveness is not enough: every bug fixed in 0.2.x and 0.3.0 left its containers
+# running and healthy while the profile was unusable. Kafka consumer groups hung
+# forever, Spark could not start a SparkContext, Flink could not find a connector
+# factory, ClickHouse could not consume from the broker, the Trino ClickHouse
+# catalog pointed at a hostname that did not exist, and the Flink Iceberg sink
+# committed nothing while its checkpoints succeeded. Each assertion below exists
+# because something shipped broken past a liveness check.
+#
+# Usage: smoke.sh <profile>
+set -uo pipefail
+
+PROFILE="${1:?usage: smoke.sh <profile>}"
+
+fail() { echo "❌ $PROFILE: $*"; exit 1; }
+pass() { echo "✅ $PROFILE: $*"; }
+
+# Retry a command until it succeeds or the budget runs out.
+retry() {
+  local tries="$1" delay="$2"; shift 2
+  for ((i = 1; i <= tries; i++)); do
+    if "$@" >/dev/null 2>&1; then return 0; fi
+    sleep "$delay"
+  done
+  return 1
+}
+
+http_ok() { curl -fsS -o /dev/null --max-time 10 "$1"; }
+
+# Some endpoints answer 403 or 404 to an unauthenticated probe, which still
+# proves the service is listening.
+http_reachable() {
+  local code
+  code=$(curl -s -o /dev/null --max-time 10 -w '%{http_code}' "$1")
+  [ -n "$code" ] && [ "$code" != "000" ]
+}
+
+ch() { docker exec "$1" clickhouse-client --password password -q "$2" 2>&1; }
+
+kafka_topic() {
+  docker exec kafka /opt/kafka/bin/kafka-topics.sh --bootstrap-server broker-1:19092 "$@"
+}
+
+smoke_kafka() {
+  local container="$1"
+  retry 30 5 docker exec "$container" /opt/kafka/bin/kafka-topics.sh \
+    --bootstrap-server broker-1:19092 --list || fail "broker never became reachable"
+  docker exec "$container" /opt/kafka/bin/kafka-topics.sh --bootstrap-server broker-1:19092 \
+    --create --topic smoke --partitions 1 --replication-factor 1 >/dev/null 2>&1
+  for i in 1 2 3 4 5; do echo "{\"n\":$i}"; done | docker exec -i "$container" \
+    /opt/kafka/bin/kafka-console-producer.sh --bootstrap-server broker-1:19092 --topic smoke >/dev/null 2>&1 \
+    || fail "produce failed"
+  # Group mode on purpose. Assign mode bypasses the coordinator and would pass
+  # even when __consumer_offsets cannot be created on a single broker.
+  local got
+  got=$(docker exec "$container" /opt/kafka/bin/kafka-console-consumer.sh \
+    --bootstrap-server broker-1:19092 --topic smoke --from-beginning \
+    --max-messages 5 --timeout-ms 60000 2>/dev/null | grep -c '"n"')
+  [ "$got" -eq 5 ] || fail "group-mode consume returned $got of 5 messages"
+  pass "produced and consumed 5 messages through a consumer group"
+}
+
+smoke_flink() {
+  retry 40 5 http_ok "http://127.0.0.1:8082/config" || fail "JobManager REST never answered"
+  local out
+  out=$(docker exec -i flink-jobmanager /opt/flink/bin/sql-client.sh 2>&1 <<'SQL'
+CREATE CATALOG ice WITH ('type'='iceberg','catalog-type'='rest','uri'='http://catalog:8181','warehouse'='s3://warehouse','s3.endpoint'='http://seaweed:8333','s3.path-style-access'='true','s3.access-key-id'='user','s3.secret-access-key'='password');
+SHOW CATALOGS;
+SQL
+)
+  grep -q "ice" <<<"$out" || { echo "$out" | tail -20; fail "Iceberg catalog did not register (connector factory or Hadoop classes missing)"; }
+  # The image ships --add-opens in config.yaml; mounting a replacement wipes them
+  # and Kryo then fails on java.nio buffers at checkpoint time.
+  docker exec flink-jobmanager sh -c 'ps ax | grep -q "add-opens=java.base/java.nio"' \
+    || fail "JVM module flags missing from the TaskManager command line"
+  pass "Iceberg catalog registered and JVM module flags present"
+}
+
+smoke_spark() {
+  local out
+  out=$(docker exec spark-master /opt/spark/bin/spark-sql -e "
+    CREATE NAMESPACE IF NOT EXISTS iceberg.smoke;
+    CREATE OR REPLACE TABLE iceberg.smoke.t (id BIGINT) USING iceberg;
+    INSERT INTO iceberg.smoke.t VALUES (1),(2),(3);
+    SELECT count(*) FROM iceberg.smoke.t;" 2>&1)
+  grep -qE "^3$" <<<"$out" || { echo "$out" | tail -20; fail "spark-sql could not round-trip an Iceberg table"; }
+  docker exec spark-master sh -c 'ls /tmp/spark-events | head -1' >/dev/null 2>&1 \
+    || fail "no event log written, so the event log directory is not writable"
+  pass "spark-sql wrote and read an Iceberg table, event logging works"
+}
+
+smoke_ch_lite() {
+  retry 30 5 docker exec ch-11 clickhouse-client --password password -q "SELECT 1" \
+    || fail "ch-11 never answered"
+  # Databases come from /docker-entrypoint-initdb.d on every fresh server.
+  ch ch-11 "SHOW DATABASES" | grep -q feature_store || fail "feature_store database missing"
+  ch ch-11 "DROP TABLE IF EXISTS default.smoke" >/dev/null
+  ch ch-11 "CREATE TABLE default.smoke (id UInt32) ENGINE=MergeTree ORDER BY id" >/dev/null
+  ch ch-11 "INSERT INTO default.smoke SELECT number FROM numbers(10)" >/dev/null
+  [ "$(ch ch-11 "SELECT count() FROM default.smoke")" = "10" ] || fail "MergeTree round trip failed"
+  pass "databases initialised and MergeTree round trip works"
+}
+
+smoke_ch_full() {
+  retry 30 5 docker exec ch-21 clickhouse-client --password password -q "SELECT 1" \
+    || fail "ch-21 never answered"
+  # Exercises keeper and the dual-stack listeners: replicated DDL fails with
+  # "Connection refused" when keeper listens on IPv4 only.
+  ch ch-11 "DROP TABLE IF EXISTS default.repl ON CLUSTER '{cluster}' SYNC" >/dev/null 2>&1
+  ch ch-11 "CREATE TABLE default.repl ON CLUSTER '{cluster}' (id UInt32) ENGINE=ReplicatedMergeTree ORDER BY id" >/dev/null \
+    || fail "replicated DDL failed, check keeper reachability"
+  ch ch-11 "INSERT INTO default.repl SELECT number FROM numbers(50)" >/dev/null
+  retry 12 5 bash -c '[ "$(docker exec ch-12 clickhouse-client --password password -q "SELECT count() FROM default.repl" 2>/dev/null)" = "50" ]' \
+    || fail "rows never replicated to the sibling replica"
+  [ "$(ch ch-21 "SELECT count() FROM default.repl")" = "0" ] || fail "shard 2 unexpectedly holds shard 1 data"
+  pass "replicated table converged on shard 1 and stayed off shard 2"
+}
+
+# Catalog registration finishes after the health endpoint starts answering, so
+# poll rather than asserting once. Catalogs load even when their backing service
+# is absent, which is why this can be required in a trino-only profile group.
+catalogs_loaded() {
+  local cat
+  cat=$(docker exec trino trino --execute "SHOW CATALOGS" 2>/dev/null | tr -d '"')
+  for expected in iceberg kafka clickhouse postgres; do
+    grep -q "^${expected}$" <<<"$cat" || return 1
+  done
+}
+
+smoke_trino() {
+  retry 40 5 http_ok "http://127.0.0.1:8080/v1/info" || fail "Trino never answered"
+  retry 24 5 catalogs_loaded || {
+    docker exec trino trino --execute "SHOW CATALOGS" 2>&1 | tail -10
+    fail "not every catalog loaded within two minutes"
+  }
+  docker exec trino trino --execute "SELECT 1" >/dev/null 2>&1 || fail "query execution failed"
+  pass "all catalogs loaded and a query ran"
+}
+
+smoke_infra() {
+  case "$PROFILE" in
+    postgres)
+      retry 30 5 docker exec postgres pg_isready -U user || fail "postgres never became ready"
+      docker exec postgres psql -U user -d odctl -c "SELECT 1" >/dev/null 2>&1 || fail "psql query failed"
+      pass "accepting connections and running queries" ;;
+    storage)
+      retry 30 5 http_reachable "http://127.0.0.1:8333" || fail "S3 API never answered"
+      pass "S3 API reachable" ;;
+    catalog)
+      retry 30 5 http_ok "http://127.0.0.1:8181/v1/config" || fail "REST catalog never answered"
+      curl -fsS -X POST -H 'Content-Type: application/json' \
+        -d '{"namespace":["smoke"]}' "http://127.0.0.1:8181/v1/namespaces" >/dev/null 2>&1
+      curl -fsS "http://127.0.0.1:8181/v1/namespaces" 2>/dev/null | grep -q smoke \
+        || fail "namespace create or list failed"
+      pass "namespace created and listed through the REST API" ;;
+    valkey)
+      retry 30 5 docker exec valkey valkey-cli -u redis://user:password@localhost:6379 ping \
+        || fail "valkey never answered PING"
+      pass "responding to PING" ;;
+  esac
+}
+
+# A profile with no functional assertion yet still has to expose its endpoint.
+smoke_http_only() {
+  local url="$1"
+  retry 60 5 http_ok "$url" || fail "no HTTP response from $url"
+  pass "HTTP endpoint answering at $url"
+}
+
+case "$PROFILE" in
+  kafka-lite)            smoke_kafka kafka ;;
+  kafka-full)            smoke_kafka kafka-1 ;;
+  flink-lite|flink-full) smoke_flink ;;
+  spark-lite|spark-full) smoke_spark ;;
+  ch-lite)               smoke_ch_lite ;;
+  ch-full)               smoke_ch_full ;;
+  trino)                 smoke_trino ;;
+  postgres|storage|catalog|valkey) smoke_infra ;;
+  metabase)   smoke_http_only "http://127.0.0.1:3000/api/health" ;;
+  airflow)    smoke_http_only "http://127.0.0.1:8085/api/v2/monitor/health" ;;
+  mlflow)     smoke_http_only "http://127.0.0.1:5000/health" ;;
+  ray-serve)  smoke_http_only "http://127.0.0.1:8265" ;;
+  lineage)    smoke_http_only "http://127.0.0.1:5002/api/v1/namespaces" ;;
+  telemetry)  smoke_http_only "http://127.0.0.1:19090/-/ready" ;;
+  metadata)   smoke_http_only "http://127.0.0.1:8585/api/v1/system/version" ;;
+  fluss)
+    retry 40 5 bash -c 'docker ps --format "{{.Names}}" | grep -q fluss-coordinator' \
+      || fail "coordinator container never appeared"
+    pass "coordinator running" ;;
+  *)
+    echo "ℹ️  $PROFILE: no functional assertion defined, checking containers only"
+    [ "$(docker ps -q | wc -l)" -ge 1 ] || fail "no containers running"
+    pass "containers running" ;;
+esac

{odctl-0.3.0 → odctl-0.3.1}/.github/workflows/e2e-test.yml

@@ -3,8 +3,6 @@ name: End-to-End Integration Tests
 on:
   workflow_call:
 
-
-
 jobs:
   e2e:
     runs-on: ubuntu-latest
@@ -12,18 +10,30 @@ jobs:
       contents: read
       packages: read
     strategy:
+      fail-fast: false
       matrix:
+        # Profiles are grouped only where they share a large image, so the pull
+        # is amortised: the lite and full variants of Kafka, Flink, Spark and
+        # ClickHouse. Everything else runs alone, which keeps the failure signal
+        # per profile and shortens the critical path.
         profile_group:
-          - "metadata"
-          - "airflow"
           - "kafka-lite kafka-full"
-          - "flink1-lite flink1-full fluss"
+          - "flink-lite flink-full"
           - "spark-lite spark-full"
           - "ch-lite ch-full"
-          - "trino metabase"
-          - "mlflow ray-serve"
-          - "lineage telemetry"
-          - "postgres storage catalog valkey"
+          - "fluss"
+          - "trino"
+          - "metabase"
+          - "mlflow"
+          - "ray-serve"
+          - "lineage"
+          - "telemetry"
+          - "metadata"
+          - "airflow"
+          - "postgres"
+          - "storage"
+          - "catalog"
+          - "valkey"
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4
@@ -49,31 +59,27 @@ jobs:
 
       - name: Test Profile Group
         run: |
-          # Split the space-separated group into an array
+          chmod +x .github/scripts/smoke.sh
           for profile in ${{ matrix.profile_group }}; do
             echo "::group::Testing Profile: $profile"
             echo "🚀 Launching $profile..."
-            odctl up $profile
+            odctl up "$profile"
 
-            echo "⏳ Validating containers..."
-            SUCCESS=0
-            for i in {1..15}; do
-              RUNNING=$(docker ps -q | wc -l)
-              if [ "$RUNNING" -ge 1 ]; then
-                echo "✅ Containers are running!"
-                docker ps
-                odctl ps --all
-                SUCCESS=1
-                break
-              fi
-              echo "Waiting... ($i/15)"
-              sleep 2
-            done
+            echo "⏳ Containers:"
+            docker ps
 
-            if [ "$SUCCESS" -eq 0 ]; then
-              echo "❌ Timeout reached: No containers are running for $profile!"
+            # Liveness alone has never been enough here: every bug fixed in 0.2.x
+            # and 0.3.0 left healthy containers behind a broken profile, so the
+            # smoke script asserts the profile can actually do its job.
+            echo "🔍 Smoke testing $profile..."
+            if ! .github/scripts/smoke.sh "$profile"; then
+              echo "::error::Smoke test failed for $profile"
               docker ps -a
               odctl ps --all
+              for c in $(docker ps -aq); do
+                echo "---- logs: $(docker inspect --format '{{.Name}}' "$c") ----"
+                docker logs --tail 40 "$c" 2>&1 || true
+              done
               exit 1
             fi
 

odctl-0.3.1/.github/workflows/pipeline.yml

@@ -0,0 +1,69 @@
+name: Main Pipeline
+
+on:
+  push:
+    branches: ["main"]
+    tags: ["v*"]
+  pull_request:
+    branches: ["main"]
+  workflow_dispatch:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+permissions:
+  contents: read
+  packages: write
+
+jobs:
+  # Which parts of the tree moved. Used to skip the multi-arch image builds on
+  # changes that cannot affect them, which is most changes.
+  changes:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      pull-requests: read
+    outputs:
+      images: ${{ steps.filter.outputs.images }}
+    steps:
+      - uses: actions/checkout@v4
+      - uses: dorny/paths-filter@v3
+        id: filter
+        with:
+          filters: |
+            images:
+              - 'src/odctl/resources/deps/**'
+              - 'src/odctl/resources/docker/**'
+              - '.github/workflows/build-deps.yml'
+              - '.github/workflows/build-platform-images.yml'
+
+  ci:
+    uses: ./.github/workflows/ci.yml
+
+  # Images are pushed only on a tag, so on any other ref this is a
+  # build-and-discard. Run it when the build inputs changed, so a broken
+  # Dockerfile surfaces before release instead of at tag time, and skip it
+  # otherwise. That removes two multi-arch builds from the typical change.
+  build-platform:
+    needs: [ci, changes]
+    if: >-
+      startsWith(github.ref, 'refs/tags/v') ||
+      github.event_name == 'workflow_dispatch' ||
+      needs.changes.outputs.images == 'true'
+    uses: ./.github/workflows/build-platform-images.yml
+
+  build-deps:
+    needs: [ci, changes]
+    if: >-
+      startsWith(github.ref, 'refs/tags/v') ||
+      github.event_name == 'workflow_dispatch' ||
+      needs.changes.outputs.images == 'true'
+    uses: ./.github/workflows/build-deps.yml
+
+  # Tags and manual runs only: the e2e job pulls images at the CLI's own version,
+  # and those exist on ghcr only once a tag has published them.
+  e2e:
+    needs: [build-platform, build-deps]
+    if: startsWith(github.ref, 'refs/tags/v') || github.event_name == 'workflow_dispatch'
+    uses: ./.github/workflows/e2e-test.yml

odctl-0.3.1/.github/workflows/publish.yml

@@ -0,0 +1,90 @@
+name: Publish to PyPI
+
+# Publishing stays tied to publishing a release, so the release notes remain the
+# trigger. What is new is the gate below: a release event fires independently of
+# CI, which is how v0.3.0 reached PyPI while the pipeline's e2e job was failing.
+# This job now waits for the tag's Main Pipeline run and refuses to publish
+# unless it succeeded.
+on:
+  release:
+    types: [published]
+  workflow_dispatch:
+    inputs:
+      tag:
+        description: "Tag to publish, for example v0.3.1"
+        required: true
+      skip_pipeline_check:
+        description: "Publish even if the tag's pipeline is not green"
+        type: boolean
+        default: false
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.release.tag_name || inputs.tag }}
+  cancel-in-progress: false
+
+jobs:
+  pypi-publish:
+    name: Build and publish Python package
+    runs-on: ubuntu-latest
+    # These permissions are strictly required for PyPI OIDC Trusted Publishing;
+    # actions:read is what lets the gate query the pipeline's conclusion.
+    permissions:
+      contents: read
+      id-token: write
+      actions: read
+
+    steps:
+      - name: Checkout source code
+        uses: actions/checkout@v7
+        with:
+          ref: ${{ github.event.release.tag_name || inputs.tag }}
+
+      - name: Require a green Main Pipeline for this tag
+        if: ${{ !inputs.skip_pipeline_check }}
+        env:
+          GH_TOKEN: ${{ github.token }}
+          TAG: ${{ github.event.release.tag_name || inputs.tag }}
+        run: |
+          set -euo pipefail
+          # Wait rather than fail outright: a release is usually published moments
+          # after the tag is pushed, so the pipeline is still running e2e.
+          deadline=$(( SECONDS + 45 * 60 ))
+          while :; do
+            run=$(gh api "repos/${GITHUB_REPOSITORY}/actions/runs?event=push&per_page=100" \
+                    --jq "[.workflow_runs[]
+                           | select(.name == \"Main Pipeline\" and .head_branch == \"${TAG}\")]
+                           | sort_by(.created_at) | last // empty")
+            if [ -z "$run" ]; then
+              echo "::error::No Main Pipeline run found for tag ${TAG}. Push the tag first."
+              exit 1
+            fi
+            status=$(jq -r '.status'     <<<"$run")
+            concl=$(jq  -r '.conclusion' <<<"$run")
+            url=$(jq    -r '.html_url'   <<<"$run")
+            if [ "$status" = "completed" ]; then
+              if [ "$concl" = "success" ]; then
+                echo "✅ Main Pipeline for ${TAG} succeeded: $url"
+                exit 0
+              fi
+              echo "::error::Main Pipeline for ${TAG} concluded '${concl}', refusing to publish: $url"
+              exit 1
+            fi
+            if [ "$SECONDS" -ge "$deadline" ]; then
+              echo "::error::Main Pipeline for ${TAG} still '${status}' after 45 minutes: $url"
+              exit 1
+            fi
+            echo "⏳ Main Pipeline for ${TAG} is ${status}; waiting: $url"
+            sleep 30
+          done
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v7
+
+      - name: Set up Python
+        run: uv python install 3.10
+
+      - name: Build Wheel and sdist
+        run: uv build
+
+      - name: Publish package distributions to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1

{odctl-0.3.0 → odctl-0.3.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: odctl
-Version: 0.3.0
+Version: 0.3.1
 Summary: Orchestrator for the Open Data Stack
 Project-URL: Homepage, https://github.com/jaehyeon-kim/odctl
 Project-URL: Repository, https://github.com/jaehyeon-kim/odctl

{odctl-0.3.0 → odctl-0.3.1}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "odctl"
-version = "0.3.0"
+version = "0.3.1"
 description = "Orchestrator for the Open Data Stack"
 readme = "README.md"
 authors = [

{odctl-0.3.0 → odctl-0.3.1}/src/odctl/docker.py

@@ -175,8 +175,12 @@ def stop_stack(
     """
     path = get_compose_path(compose_filename)
     stack_client = _create_client(compose_files=[str(path)], profiles=profiles)
-    # volumes=True will remove named volumes defined in the compose file
-    stack_client.compose.down(volumes=remove_volumes)
+    # volumes=True will remove named volumes defined in the compose file.
+    # remove_orphans catches containers whose service no longer exists in the
+    # compose file, which teardown would otherwise leave running forever: an
+    # upgrade that drops or renames a service (ch-init-lite in 0.2.0, the
+    # flink1-* services in 0.3.0) leaves the old container holding its name.
+    stack_client.compose.down(volumes=remove_volumes, remove_orphans=True)
 
 
 def get_managed_containers(execution_plan: Dict[str, List[str]]) -> List[Any]:

{odctl-0.3.0 → odctl-0.3.1}/src/odctl/resources/compose-infra.yml

@@ -53,7 +53,7 @@ services:
   # Object Storage (SeaweedFS - Single Node)
   # ==========================================
   seaweed:
-    image: chrislusf/seaweedfs:4.22
+    image: &seaweed-image chrislusf/seaweedfs:4.22
     container_name: seaweed
     ports:
       - "8333:8333" # S3 API
@@ -78,7 +78,7 @@ services:
       retries: 10
 
   seaweed-init:
-    image: chrislusf/seaweedfs:4.22
+    image: *seaweed-image
     container_name: seaweed-init
     depends_on:
       seaweed:

{odctl-0.3.0 → odctl-0.3.1}/src/odctl/resources/compose-kafka.yml

@@ -19,7 +19,7 @@ services:
   # [LITE] Single Broker Cluster (Container: kafka)
   # ==========================================
   kafka:
-    image: apache/kafka:4.3.0
+    image: &kafka-image apache/kafka:4.3.0
     container_name: kafka
     ports:
       - "9092:9092"
@@ -54,7 +54,7 @@ services:
   # [FULL] 3-Node Broker Cluster
   # ==========================================
   kafka-1:
-    image: apache/kafka:4.3.0
+    image: *kafka-image
     container_name: kafka-1
     ports:
       - "9092:9092"
@@ -74,7 +74,7 @@ services:
     mem_limit: 1500m
 
   kafka-2:
-    image: apache/kafka:4.3.0
+    image: *kafka-image
     container_name: kafka-2
     ports:
       - "9093:9093"
@@ -94,7 +94,7 @@ services:
     mem_limit: 1500m
 
   kafka-3:
-    image: apache/kafka:4.3.0
+    image: *kafka-image
     container_name: kafka-3
     ports:
       - "9094:9094"
@@ -117,7 +117,7 @@ services:
   # Open Source Kafka Connect
   # ==========================================
   connect:
-    image: apache/kafka:4.3.0
+    image: *kafka-image
     container_name: connect
     restart: unless-stopped
     ports:

{odctl-0.3.0 → odctl-0.3.1}/src/odctl/resources/compose-metadata.yml

@@ -33,7 +33,7 @@ services:
   # [PROFILE: metadata] OpenMetadata DB Migration
   # ==========================================
   openmetadata-migrate:
-    image: docker.getcollate.io/openmetadata/server:1.12.6
+    image: &om-server-image docker.getcollate.io/openmetadata/server:1.12.6
     container_name: openmetadata-migrate
     command: "./bootstrap/openmetadata-ops.sh migrate"
     profiles:
@@ -63,7 +63,7 @@ services:
   # [PROFILE: metadata] OpenMetadata Server
   # ==========================================
   openmetadata-server:
-    image: docker.getcollate.io/openmetadata/server:1.12.6
+    image: *om-server-image
     container_name: openmetadata-server
     restart: always
     profiles:

{odctl-0.3.0 → odctl-0.3.1}/src/odctl/resources/compose-store.yml

@@ -7,7 +7,7 @@ services:
   # [PROFILE: fluss] Streaming Storage Engine
   # ==========================================
   fluss-coordinator:
-    image: apache/fluss:0.8.0-incubating
+    image: &fluss-image apache/fluss:0.8.0-incubating
     container_name: fluss-coordinator
     command: coordinatorServer
     profiles: ["fluss"]
@@ -28,7 +28,7 @@ services:
         remote.data.dir: /tmp/fluss/remote-data
 
   fluss-tablet-1:
-    image: apache/fluss:0.8.0-incubating
+    image: *fluss-image
     container_name: fluss-tablet-1
     command: tabletServer
     profiles: ["fluss"]

{odctl-0.3.0 → odctl-0.3.1}/src/odctl/resources/deps/Dockerfile

@@ -15,10 +15,13 @@ RUN --mount=type=cache,target=/root/.gradle \
 # ==========================================
 FROM gradle:8.6-jdk17 AS gradle-resolver
 WORKDIR /app
-COPY build.gradle .
-# Let Gradle do the heavy transitive lifting for both Connect and Flink!
+COPY build.gradle versions.env ./
+# Let Gradle do the heavy transitive lifting for both Connect and Flink, using the
+# same pins the fetch scripts read so the two cannot drift. See issue #5.
 RUN --mount=type=cache,target=/home/gradle/.gradle \
-    gradle copyAllDependencies --no-daemon
+    . ./versions.env \
+    && gradle copyAllDependencies --no-daemon \
+        -PflinkMinor="${FLINK_MINOR}" -PhadoopVersion="${HADOOP_V}"
 
 # ==========================================
 # Stage 3: Direct Downloader (Uber JARs & ZIPs)
@@ -29,6 +32,7 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
     apt-get update && apt-get install -y curl tar unzip && rm -rf /var/lib/apt/lists/*
 WORKDIR /downloads
 # Copy the standalone fetch scripts
+COPY versions.env .
 COPY fetch_standalone_plugins.sh .
 COPY fetch_flink_dependencies.sh .
 COPY fetch_spark_dependencies.sh .

{odctl-0.3.0 → odctl-0.3.1}/src/odctl/resources/deps/build.gradle

@@ -11,8 +11,10 @@ repositories {
 
 ext {
     confluentVersion = '8.2.0'
-    flinkVersion = '2.1.3'
-    hadoopVersion = '3.3.6'
+    // Supplied by the Dockerfile from versions.env. The '+' resolves the latest
+    // patch of the pinned minor, which is what the fetch scripts do too.
+    flinkVersion = "${project.findProperty('flinkMinor') ?: '2.1'}.+"
+    hadoopVersion = project.findProperty('hadoopVersion') ?: '3.3.6'
 }
 
 configurations {

{odctl-0.3.0 → odctl-0.3.1}/src/odctl/resources/deps/fetch_flink_dependencies.sh

@@ -9,21 +9,20 @@ fetch_artifact() {
     if [ "$DRY_RUN" -eq 1 ]; then echo " 🔍 Checking: $2"; curl -sL -I -f "$2" > /dev/null || exit 1;
     else echo " ⬇️  Downloading: $2"; curl -sL -f -o "$1" "$2"; fi
 }
+# Versions come from versions.env so a bump is one edit rather than four kept
+# in step. See issue #5.
+# shellcheck source=versions.env
+. "$(dirname "$0")/versions.env"
+
 get_maven_version() {
     curl -sL "https://repo1.maven.org/maven2/${1}/maven-metadata.xml" | grep -Eo "<version>${2}</version>" | sort -V | tail -1 | sed -E 's#</?version>##g' || true
 }
 
 echo "▶️  Resolving Flink Versions..."
 
-# 🔒 PINNED: Flink 2.1, not the latest 2.x. Iceberg publishes
-# iceberg-flink-runtime for 2.0 and 2.1 only, and Fluss has no 2.0 build, so 2.1
-# is the one minor where every connector this stack needs exists. See issue #9.
-FLINK_MINOR="2.1"
 FLINK_V=$(get_maven_version "org/apache/flink/flink-core" "${FLINK_MINOR}\.[0-9]+")
 if [ -z "$FLINK_V" ]; then echo "❌ Error: Could not resolve Flink ${FLINK_MINOR} patch version!"; exit 1; fi
 
-# 🔒 HARDCODED: Lock Iceberg to 1.11.0 to match Spark and PyIceberg sidecar
-ICEBERG_V="1.11.0"
 
 echo "▶️  Fetching Flink Dependencies for ${FLINK_MINOR} (Iceberg ${ICEBERG_V})..."
 make_dir "flink/2.x"

{odctl-0.3.0 → odctl-0.3.1}/src/odctl/resources/deps/fetch_spark_dependencies.sh

@@ -9,6 +9,11 @@ fetch_artifact() {
     if [ "$DRY_RUN" -eq 1 ]; then echo " 🔍 Checking: $2"; curl -sL -I -f "$2" > /dev/null || exit 1;
     else echo " ⬇️  Downloading: $2"; curl -sL -f -o "$1" "$2"; fi
 }
+# Versions come from versions.env so a bump is one edit rather than several
+# kept in step. See issue #5.
+# shellcheck source=versions.env
+. "$(dirname "$0")/versions.env"
+
 get_maven_version() {
     curl -sL "https://repo1.maven.org/maven2/${1}/maven-metadata.xml" | grep -Eo "<version>${2}</version>" | sort -V | tail -1 | sed -E 's#</?version>##g' || true
 }
@@ -16,9 +21,8 @@ get_maven_version() {
 echo "▶️  Resolving Spark Versions..."
 SCALA_V="2.13"
 
-# 🔒 HARDCODED: Lock to Spark 4.1.x and Iceberg 1.11.0 to ensure stack stability
+# 🔒 HARDCODED: Lock to Spark 4.1.x; Iceberg comes from versions.env
 SPARK_COMPAT_MINOR="4.1"
-ICEBERG_V="1.11.0"
 
 # Keep OpenLineage dynamic as it relies on frequent patch updates
 OL_SPARK_V=$(get_maven_version "io/openlineage/openlineage-spark_${SCALA_V}" "[0-9]+\.[0-9]+\.[0-9]+")

{odctl-0.3.0 → odctl-0.3.1}/src/odctl/resources/deps/fetch_standalone_plugins.sh

@@ -9,14 +9,18 @@ fetch_artifact() {
     if [ "$DRY_RUN" -eq 1 ]; then echo " 🔍 Checking: $2"; curl -sL -I -f "$2" > /dev/null || exit 1;
     else echo " ⬇️  Downloading: $2"; curl -sL -f -o "$1" "$2"; fi
 }
+# Versions come from versions.env so a bump is one edit rather than several
+# kept in step. See issue #5.
+# shellcheck source=versions.env
+. "$(dirname "$0")/versions.env"
+
 get_maven_version() {
     curl -sL "https://repo1.maven.org/maven2/${1}/maven-metadata.xml" | grep -Eo "<version>${2}</version>" | sort -V | tail -1 | sed -E 's#</?version>##g' || true
 }
 
 echo "▶️  Resolving Shared & Connector Versions..."
 
-# 🔒 HARDCODED: Lock core dependencies to ensure absolute stack stability (May 2026)
-ICEBERG_V="1.11.0"
+# 🔒 HARDCODED: Lock core dependencies; Iceberg comes from versions.env
 DEB_V="3.5.1.Final"
 POSTGRES_V="42.7.3"  # Latest stable JDBC 42.x series
 

odctl-0.3.1/src/odctl/resources/deps/versions.env

@@ -0,0 +1,19 @@
+# Single source of truth for the versions the deps image builds against.
+#
+# Sourced by every fetch script and passed into the Gradle stage as build args,
+# so a bump is one edit here rather than four in step. See issue #5.
+#
+# The one value that lives outside this file is the Flink runtime image tag in
+# compose-flink.yml, which Docker Compose resolves before any of this runs. Keep
+# its minor equal to FLINK_MINOR below.
+
+# Flink minor series. Pinned to 2.1 because Iceberg publishes
+# iceberg-flink-runtime for 2.0 and 2.1 only, and fluss-flink has no 2.0 build.
+FLINK_MINOR=2.1
+
+# Iceberg client libraries, shared by the Spark, Flink, and standalone fetches so
+# every engine and the PyIceberg sidecar agree.
+ICEBERG_V=1.11.0
+
+# Hadoop client jars, used by the Gradle stage for the Flink classpath.
+HADOOP_V=3.3.6

{odctl-0.3.0 → odctl-0.3.1}/src/odctl/resources/registry.yml

@@ -235,7 +235,7 @@ stacks:
     usage:
       mlflow: |
         • MLflow Tracking UI & Artifact Registry Server:
-          * External (Localhost): http://localhost:5050
+          * External (Localhost): http://localhost:5000
           * Internal (Docker): http://mlflow:5000
       ray-serve: |
         • Ray Serve Inference API Proxy:

{odctl-0.3.0 → odctl-0.3.1}/tests/test_docker.py

@@ -65,7 +65,9 @@ def test_docker_actions(monkeypatch):
     mock_client.compose.up.assert_called_with(detach=False)
 
     docker.stop_stack("dummy.yml", ["prof1"], remove_volumes=True)
-    mock_client.compose.down.assert_called_with(volumes=True)
+    # remove_orphans is what clears containers whose service was dropped
+    # from the compose file.
+    mock_client.compose.down.assert_called_with(volumes=True, remove_orphans=True)
 
 
 def test_restart_managed_containers(monkeypatch):

odctl-0.3.0/.github/workflows/pipeline.yml

@@ -1,34 +0,0 @@
-name: Main Pipeline
-
-on:
-  push:
-    branches: ["main"]
-    tags: ["v*"]
-  pull_request:
-    branches: ["main"]
-  workflow_dispatch:
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.ref }}
-  cancel-in-progress: true
-
-permissions:
-  contents: read
-  packages: write
-
-jobs:
-  ci:
-    uses: ./.github/workflows/ci.yml
-
-  build-platform:
-    needs: ci
-    uses: ./.github/workflows/build-platform-images.yml
-
-  build-deps:
-    needs: ci
-    uses: ./.github/workflows/build-deps.yml
-
-  e2e:
-    needs: [build-platform, build-deps]
-    if: startsWith(github.ref, 'refs/tags/v') || github.event_name == 'workflow_dispatch'
-    uses: ./.github/workflows/e2e-test.yml

odctl-0.3.0/.github/workflows/publish.yml

@@ -1,34 +0,0 @@
-name: Publish to PyPI
-
-on:
-  release:
-    types: [published] # ONLY triggers when a drafted release is published
-
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.ref }}
-  cancel-in-progress: true
-jobs:
-  pypi-publish:
-    name: Build and publish Python package
-    runs-on: ubuntu-latest
-    # These permissions are strictly required for PyPI OIDC Trusted Publishing
-    permissions:
-      contents: read
-      id-token: write
-
-    steps:
-      - name: Checkout source code
-        uses: actions/checkout@v7
-
-      - name: Install uv
-        uses: astral-sh/setup-uv@v7
-
-      - name: Set up Python
-        run: uv python install 3.10
-
-      - name: Build Wheel and sdist
-        run: uv build
-
-      - name: Publish package distributions to PyPI
-        uses: pypa/gh-action-pypi-publish@release/v1