odctl 0.2.2__tar.gz → 0.3.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (81) hide show
  1. {odctl-0.2.2 → odctl-0.3.0}/.github/renovate.json +10 -0
  2. {odctl-0.2.2 → odctl-0.3.0}/PKG-INFO +3 -3
  3. {odctl-0.2.2 → odctl-0.3.0}/README.md +2 -2
  4. {odctl-0.2.2 → odctl-0.3.0}/pyproject.toml +1 -1
  5. {odctl-0.2.2 → odctl-0.3.0}/src/odctl/main.py +7 -7
  6. {odctl-0.2.2 → odctl-0.3.0}/src/odctl/resources/compose-flink.yml +38 -30
  7. {odctl-0.2.2 → odctl-0.3.0}/src/odctl/resources/compose-store.yml +2 -2
  8. {odctl-0.2.2 → odctl-0.3.0}/src/odctl/resources/deps/Dockerfile +4 -3
  9. {odctl-0.2.2 → odctl-0.3.0}/src/odctl/resources/deps/build.gradle +1 -1
  10. {odctl-0.2.2 → odctl-0.3.0}/src/odctl/resources/deps/fetch_flink_dependencies.sh +17 -12
  11. odctl-0.2.2/src/odctl/resources/flink/flink1-conf.yml → odctl-0.3.0/src/odctl/resources/flink/flink-conf.yml +15 -5
  12. {odctl-0.2.2 → odctl-0.3.0}/src/odctl/resources/prometheus/prometheus.yml +4 -4
  13. {odctl-0.2.2 → odctl-0.3.0}/src/odctl/resources/registry.yml +9 -9
  14. {odctl-0.2.2 → odctl-0.3.0}/.github/ISSUE_TEMPLATE/bug_report.md +0 -0
  15. {odctl-0.2.2 → odctl-0.3.0}/.github/ISSUE_TEMPLATE/feature_request.md +0 -0
  16. {odctl-0.2.2 → odctl-0.3.0}/.github/workflows/build-deps.yml +0 -0
  17. {odctl-0.2.2 → odctl-0.3.0}/.github/workflows/build-platform-images.yml +0 -0
  18. {odctl-0.2.2 → odctl-0.3.0}/.github/workflows/ci.yml +0 -0
  19. {odctl-0.2.2 → odctl-0.3.0}/.github/workflows/e2e-test.yml +0 -0
  20. {odctl-0.2.2 → odctl-0.3.0}/.github/workflows/pipeline.yml +0 -0
  21. {odctl-0.2.2 → odctl-0.3.0}/.github/workflows/publish.yml +0 -0
  22. {odctl-0.2.2 → odctl-0.3.0}/.gitignore +0 -0
  23. {odctl-0.2.2 → odctl-0.3.0}/.pre-commit-config.yaml +0 -0
  24. {odctl-0.2.2 → odctl-0.3.0}/.vscode/settings.json +0 -0
  25. {odctl-0.2.2 → odctl-0.3.0}/LICENSE +0 -0
  26. {odctl-0.2.2 → odctl-0.3.0}/image/diagram.png +0 -0
  27. {odctl-0.2.2 → odctl-0.3.0}/image/diagram1.drawio +0 -0
  28. {odctl-0.2.2 → odctl-0.3.0}/image/diagram2.drawio +0 -0
  29. {odctl-0.2.2 → odctl-0.3.0}/src/odctl/__init__.py +0 -0
  30. {odctl-0.2.2 → odctl-0.3.0}/src/odctl/config.py +0 -0
  31. {odctl-0.2.2 → odctl-0.3.0}/src/odctl/docker.py +0 -0
  32. {odctl-0.2.2 → odctl-0.3.0}/src/odctl/planner.py +0 -0
  33. {odctl-0.2.2 → odctl-0.3.0}/src/odctl/registry.py +0 -0
  34. {odctl-0.2.2 → odctl-0.3.0}/src/odctl/resources/clickhouse/config.xml +0 -0
  35. {odctl-0.2.2 → odctl-0.3.0}/src/odctl/resources/clickhouse/init.sql +0 -0
  36. {odctl-0.2.2 → odctl-0.3.0}/src/odctl/resources/clickhouse/keeper_config.xml +0 -0
  37. {odctl-0.2.2 → odctl-0.3.0}/src/odctl/resources/clickhouse/macros-11.xml +0 -0
  38. {odctl-0.2.2 → odctl-0.3.0}/src/odctl/resources/clickhouse/macros-12.xml +0 -0
  39. {odctl-0.2.2 → odctl-0.3.0}/src/odctl/resources/clickhouse/macros-21.xml +0 -0
  40. {odctl-0.2.2 → odctl-0.3.0}/src/odctl/resources/clickhouse/macros-22.xml +0 -0
  41. {odctl-0.2.2 → odctl-0.3.0}/src/odctl/resources/clickhouse/users.xml +0 -0
  42. {odctl-0.2.2 → odctl-0.3.0}/src/odctl/resources/compose-analytics.yml +0 -0
  43. {odctl-0.2.2 → odctl-0.3.0}/src/odctl/resources/compose-deps.yml +0 -0
  44. {odctl-0.2.2 → odctl-0.3.0}/src/odctl/resources/compose-infra.yml +0 -0
  45. {odctl-0.2.2 → odctl-0.3.0}/src/odctl/resources/compose-kafka.yml +0 -0
  46. {odctl-0.2.2 → odctl-0.3.0}/src/odctl/resources/compose-metadata.yml +0 -0
  47. {odctl-0.2.2 → odctl-0.3.0}/src/odctl/resources/compose-mlops.yml +0 -0
  48. {odctl-0.2.2 → odctl-0.3.0}/src/odctl/resources/compose-obsv.yml +0 -0
  49. {odctl-0.2.2 → odctl-0.3.0}/src/odctl/resources/compose-orch.yml +0 -0
  50. {odctl-0.2.2 → odctl-0.3.0}/src/odctl/resources/compose-spark.yml +0 -0
  51. {odctl-0.2.2 → odctl-0.3.0}/src/odctl/resources/deps/build_connectors.sh +0 -0
  52. {odctl-0.2.2 → odctl-0.3.0}/src/odctl/resources/deps/fetch_spark_dependencies.sh +0 -0
  53. {odctl-0.2.2 → odctl-0.3.0}/src/odctl/resources/deps/fetch_standalone_plugins.sh +0 -0
  54. {odctl-0.2.2 → odctl-0.3.0}/src/odctl/resources/docker/airflow/Dockerfile +0 -0
  55. {odctl-0.2.2 → odctl-0.3.0}/src/odctl/resources/docker/ray-mlops/Dockerfile +0 -0
  56. {odctl-0.2.2 → odctl-0.3.0}/src/odctl/resources/docker/spark/Dockerfile +0 -0
  57. {odctl-0.2.2 → odctl-0.3.0}/src/odctl/resources/grafana/prometheus.yml +0 -0
  58. {odctl-0.2.2 → odctl-0.3.0}/src/odctl/resources/marquez/marquez.yml +0 -0
  59. {odctl-0.2.2 → odctl-0.3.0}/src/odctl/resources/mlops/serving/lightgbm_inference.py.example +0 -0
  60. {odctl-0.2.2 → odctl-0.3.0}/src/odctl/resources/mlops/serving/vector_search.py.example +0 -0
  61. {odctl-0.2.2 → odctl-0.3.0}/src/odctl/resources/postgres/01-init-databases.sh +0 -0
  62. {odctl-0.2.2 → odctl-0.3.0}/src/odctl/resources/prometheus/alertmanager.yml +0 -0
  63. {odctl-0.2.2 → odctl-0.3.0}/src/odctl/resources/seaweed/s3.json +0 -0
  64. {odctl-0.2.2 → odctl-0.3.0}/src/odctl/resources/spark/log4j2.properties +0 -0
  65. {odctl-0.2.2 → odctl-0.3.0}/src/odctl/resources/spark/spark-defaults.conf +0 -0
  66. {odctl-0.2.2 → odctl-0.3.0}/src/odctl/resources/trino/catalog/clickhouse.properties +0 -0
  67. {odctl-0.2.2 → odctl-0.3.0}/src/odctl/resources/trino/catalog/iceberg.properties +0 -0
  68. {odctl-0.2.2 → odctl-0.3.0}/src/odctl/resources/trino/catalog/kafka.properties +0 -0
  69. {odctl-0.2.2 → odctl-0.3.0}/src/odctl/resources/trino/catalog/postgres.properties +0 -0
  70. {odctl-0.2.2 → odctl-0.3.0}/src/odctl/resources/trino/catalog/valkey.properties +0 -0
  71. {odctl-0.2.2 → odctl-0.3.0}/src/odctl/ui.py +0 -0
  72. {odctl-0.2.2 → odctl-0.3.0}/src/odctl/workspace.py +0 -0
  73. {odctl-0.2.2 → odctl-0.3.0}/tests/conftest.py +0 -0
  74. {odctl-0.2.2 → odctl-0.3.0}/tests/test_cli.py +0 -0
  75. {odctl-0.2.2 → odctl-0.3.0}/tests/test_cli_extended.py +0 -0
  76. {odctl-0.2.2 → odctl-0.3.0}/tests/test_config.py +0 -0
  77. {odctl-0.2.2 → odctl-0.3.0}/tests/test_docker.py +0 -0
  78. {odctl-0.2.2 → odctl-0.3.0}/tests/test_planner.py +0 -0
  79. {odctl-0.2.2 → odctl-0.3.0}/tests/test_registry.py +0 -0
  80. {odctl-0.2.2 → odctl-0.3.0}/tests/test_workspace.py +0 -0
  81. {odctl-0.2.2 → odctl-0.3.0}/uv.lock +0 -0
@@ -12,6 +12,16 @@
12
12
  ]
13
13
  },
14
14
  "packageRules": [
15
+ {
16
+ "description": "Flink stays on 2.1: Iceberg publishes iceberg-flink-runtime for 2.0 and 2.1 only, and fluss-flink has no 2.0 build. Raise this when Iceberg ships a runtime for a newer minor. See issue #9.",
17
+ "matchPackageNames": ["flink", "org.apache.flink:{/,}**"],
18
+ "allowedVersions": "<2.2"
19
+ },
20
+ {
21
+ "description": "Fluss stays on 0.8.x because 0.9.x requires Flink 2.2, which has no Iceberg runtime. Moves together with the Flink ceiling above.",
22
+ "matchPackageNames": ["apache/fluss", "org.apache.fluss:{/,}**"],
23
+ "allowedVersions": "<0.9"
24
+ },
15
25
  {
16
26
  "description": "Auto-merge non-breaking minor and patch updates if CI passes",
17
27
  "matchUpdateTypes": ["minor", "patch"],
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: odctl
3
- Version: 0.2.2
3
+ Version: 0.3.0
4
4
  Summary: Orchestrator for the Open Data Stack
5
5
  Project-URL: Homepage, https://github.com/jaehyeon-kim/odctl
6
6
  Project-URL: Repository, https://github.com/jaehyeon-kim/odctl
@@ -123,7 +123,7 @@ odctl list
123
123
  Bring up a robust data engineering environment.
124
124
 
125
125
  ```bash
126
- odctl up kafka-lite flink1-lite spark-lite
126
+ odctl up kafka-lite flink-lite spark-lite
127
127
  ```
128
128
 
129
129
  _Note: You do not need to memorize dependencies. The CLI will automatically detect that these profiles require foundational infrastructure and will launch PostgreSQL, SeaweedFS (S3), and the Iceberg REST Catalog for you before starting the target compute engines._
@@ -169,7 +169,7 @@ $ odctl list -d
169
169
  $ odctl explain kafka-lite
170
170
 
171
171
  # Launch specific profiles
172
- $ odctl up flink1-lite kafka-lite spark-lite
172
+ $ odctl up flink-lite kafka-lite spark-lite
173
173
 
174
174
  # Complete teardown and wipe all data
175
175
  $ odctl down --all --volumes
@@ -87,7 +87,7 @@ odctl list
87
87
  Bring up a robust data engineering environment.
88
88
 
89
89
  ```bash
90
- odctl up kafka-lite flink1-lite spark-lite
90
+ odctl up kafka-lite flink-lite spark-lite
91
91
  ```
92
92
 
93
93
  _Note: You do not need to memorize dependencies. The CLI will automatically detect that these profiles require foundational infrastructure and will launch PostgreSQL, SeaweedFS (S3), and the Iceberg REST Catalog for you before starting the target compute engines._
@@ -133,7 +133,7 @@ $ odctl list -d
133
133
  $ odctl explain kafka-lite
134
134
 
135
135
  # Launch specific profiles
136
- $ odctl up flink1-lite kafka-lite spark-lite
136
+ $ odctl up flink-lite kafka-lite spark-lite
137
137
 
138
138
  # Complete teardown and wipe all data
139
139
  $ odctl down --all --volumes
@@ -1,6 +1,6 @@
1
1
  [project]
2
2
  name = "odctl"
3
- version = "0.2.2"
3
+ version = "0.3.0"
4
4
  description = "Orchestrator for the Open Data Stack"
5
5
  readme = "README.md"
6
6
  authors = [
@@ -36,7 +36,7 @@ Provides commands to inspect, provision, and tear down curated Docker Compose st
36
36
  [dim]# See exactly what the airflow profile provisions[/dim]\n
37
37
  $ [bold cyan]odctl explain kafka-lite[/bold cyan]\n\n
38
38
  [dim]# Launch specific profiles and their dependencies[/dim]\n
39
- $ [bold cyan]odctl up flink1-lite kafka-lite spark-lite[/bold cyan]\n\n
39
+ $ [bold cyan]odctl up flink-lite kafka-lite spark-lite[/bold cyan]\n\n
40
40
  [dim]# Complete teardown and wipe all data[/dim]\n
41
41
  $ [bold cyan]odctl down --all --volumes[/bold cyan]
42
42
  """,
@@ -201,8 +201,8 @@ def init(
201
201
  [bold underline]Examples:[/bold underline]\n
202
202
  [dim]# Pre-fetch images for all profiles[/dim]\n
203
203
  $ [bold cyan]odctl pull --all[/bold cyan]\n\n
204
- [dim]# Pre-fetch images just for Flink 1.x and Spark[/dim]\n
205
- $ [bold cyan]odctl pull flink1-lite kafka-lite[/bold cyan]
204
+ [dim]# Pre-fetch images just for Flink and Spark[/dim]\n
205
+ $ [bold cyan]odctl pull flink-lite kafka-lite[/bold cyan]
206
206
  """,
207
207
  )
208
208
  def pull(
@@ -243,8 +243,8 @@ def pull(
243
243
  rich_help_panel="Cluster Lifecycle",
244
244
  epilog="""
245
245
  [bold underline]Examples:[/bold underline]\n
246
- [dim]# Launch Clickhouse, Flink 1.x, and their dependencies[/dim]\n
247
- $ [bold cyan]odctl up ch-lite flink1-lite[/bold cyan]\n\n
246
+ [dim]# Launch Clickhouse, Flink, and their dependencies[/dim]\n
247
+ $ [bold cyan]odctl up ch-lite flink-lite[/bold cyan]\n\n
248
248
  [dim]# Preview what would be launched for Airflow[/dim]\n
249
249
  $ [bold cyan]odctl up airflow --dry-run[/bold cyan]\n\n
250
250
  [dim]# Force pull latest images before launching[/dim]\n
@@ -464,9 +464,9 @@ def info():
464
464
  epilog="""
465
465
  [bold underline]Examples:[/bold underline]\n
466
466
  [dim]# Tail the last 50 lines of all Flink containers and follow live[/dim]\n
467
- $ [bold cyan]odctl logs flink1-lite -n 50 -f[/bold cyan]\n\n
467
+ $ [bold cyan]odctl logs flink-lite -n 50 -f[/bold cyan]\n\n
468
468
  [dim]# View logs strictly for the JobManager service[/dim]\n
469
- $ [bold cyan]odctl logs flink1-lite -s jobmanager-1[/bold cyan]\n\n
469
+ $ [bold cyan]odctl logs flink-lite -s jobmanager[/bold cyan]\n\n
470
470
  [dim]# Show logs with timestamps for the last 10 minutes[/dim]\n
471
471
  $ [bold cyan]odctl logs kafka-lite --since 10m -t[/bold cyan]
472
472
  """,
@@ -4,14 +4,22 @@ x-flink-common: &flink-common
4
4
  networks:
5
5
  - odctl
6
6
  volumes:
7
- - ./flink/flink1-conf.yml:/opt/flink/conf/config.yaml:ro
7
+ - ./flink/flink-conf.yml:/opt/flink/conf/config.yaml:ro
8
8
  - odctl-shared-deps:/mnt/shared-deps:ro
9
9
 
10
- x-flink1-base: &flink1-base
10
+ x-flink-base: &flink-base
11
11
  <<: *flink-common
12
- image: flink:1.20.5-java17
12
+ # Pinned to 2.1 because Iceberg publishes iceberg-flink-runtime for 2.0 and
13
+ # 2.1 only, and Fluss has no 2.0 build. See issue #9.
14
+ image: flink:2.1-java17
13
15
  environment:
14
16
  TZ: ${TZ:-UTC}
17
+ # SeaweedFS ignores the region, but the AWS SDK refuses to build a client
18
+ # without one, so an Iceberg catalog that omits client.region fails its
19
+ # commits with "Unable to load region" while checkpoints keep succeeding.
20
+ AWS_REGION: us-east-1
21
+ AWS_ACCESS_KEY_ID: user
22
+ AWS_SECRET_ACCESS_KEY: password
15
23
  entrypoint:
16
24
  - /bin/bash
17
25
  - -c
@@ -19,7 +27,7 @@ x-flink1-base: &flink1-base
19
27
  # Load dependencies into lib/, not usrlib/. Only lib/ reaches the system
20
28
  # classloader, which is where SQL connector factories are looked up.
21
29
  cp /mnt/shared-deps/shared/*.jar /opt/flink/lib/ 2>/dev/null || true
22
- cp /mnt/shared-deps/flink/1.x/*.jar /opt/flink/lib/ 2>/dev/null || true
30
+ cp /mnt/shared-deps/flink/2.x/*.jar /opt/flink/lib/ 2>/dev/null || true
23
31
  cp /mnt/shared-deps/connect/confluent-converters/*.jar /opt/flink/lib/ 2>/dev/null || true
24
32
 
25
33
  # S3 Plugin Linking
@@ -30,10 +38,10 @@ x-flink1-base: &flink1-base
30
38
  - --
31
39
 
32
40
  services:
33
- jobmanager-1:
34
- <<: *flink1-base
35
- container_name: flink-jobmanager-1
36
- profiles: ["flink1-lite", "flink1-full"]
41
+ jobmanager:
42
+ <<: *flink-base
43
+ container_name: flink-jobmanager
44
+ profiles: ["flink-lite", "flink-full"]
37
45
  command: jobmanager
38
46
  ports:
39
47
  - "8082:8081"
@@ -44,51 +52,51 @@ services:
44
52
  timeout: 5s
45
53
  retries: 5
46
54
 
47
- taskmanager-1-a:
48
- <<: *flink1-base
49
- container_name: flink-taskmanager-1-a
50
- profiles: ["flink1-lite", "flink1-full"]
55
+ taskmanager-a:
56
+ <<: *flink-base
57
+ container_name: flink-taskmanager-a
58
+ profiles: ["flink-lite", "flink-full"]
51
59
  command: taskmanager
52
60
  mem_limit: 3500m
53
61
  depends_on:
54
- jobmanager-1:
62
+ jobmanager:
55
63
  condition: service_healthy
56
64
 
57
- taskmanager-1-b:
58
- <<: *flink1-base
59
- container_name: flink-taskmanager-1-b
60
- profiles: ["flink1-full"]
65
+ taskmanager-b:
66
+ <<: *flink-base
67
+ container_name: flink-taskmanager-b
68
+ profiles: ["flink-full"]
61
69
  command: taskmanager
62
70
  mem_limit: 3500m
63
71
  depends_on:
64
- jobmanager-1:
72
+ jobmanager:
65
73
  condition: service_healthy
66
74
 
67
- taskmanager-1-c:
68
- <<: *flink1-base
69
- container_name: flink-taskmanager-1-c
70
- profiles: ["flink1-full"]
75
+ taskmanager-c:
76
+ <<: *flink-base
77
+ container_name: flink-taskmanager-c
78
+ profiles: ["flink-full"]
71
79
  command: taskmanager
72
80
  mem_limit: 3500m
73
81
  depends_on:
74
- jobmanager-1:
82
+ jobmanager:
75
83
  condition: service_healthy
76
84
 
77
- sql-gateway-1:
78
- <<: *flink1-base
79
- container_name: flink-sql-gateway-1
80
- profiles: ["flink1-lite", "flink1-full"]
85
+ sql-gateway:
86
+ <<: *flink-base
87
+ container_name: flink-sql-gateway
88
+ profiles: ["flink-lite", "flink-full"]
81
89
  command: >
82
90
  bin/sql-gateway.sh start-foreground
83
91
  -Dsql-gateway.endpoint.rest.address=0.0.0.0
84
92
  -Dsql-gateway.endpoint.rest.port=8083
85
- -Djobmanager.rpc.address=jobmanager-1
86
- -Drest.address=jobmanager-1
93
+ -Djobmanager.rpc.address=jobmanager
94
+ -Drest.address=jobmanager
87
95
  mem_limit: 1g
88
96
  ports:
89
97
  - "8084:8083"
90
98
  depends_on:
91
- jobmanager-1:
99
+ jobmanager:
92
100
  condition: service_healthy
93
101
 
94
102
  networks:
@@ -7,7 +7,7 @@ services:
7
7
  # [PROFILE: fluss] Streaming Storage Engine
8
8
  # ==========================================
9
9
  fluss-coordinator:
10
- image: apache/fluss:0.9.0-incubating
10
+ image: apache/fluss:0.8.0-incubating
11
11
  container_name: fluss-coordinator
12
12
  command: coordinatorServer
13
13
  profiles: ["fluss"]
@@ -28,7 +28,7 @@ services:
28
28
  remote.data.dir: /tmp/fluss/remote-data
29
29
 
30
30
  fluss-tablet-1:
31
- image: apache/fluss:0.9.0-incubating
31
+ image: apache/fluss:0.8.0-incubating
32
32
  container_name: fluss-tablet-1
33
33
  command: tabletServer
34
34
  profiles: ["fluss"]
@@ -54,10 +54,11 @@ COPY --from=source-builder /build/dist/ /workspace/connect/
54
54
  # Grab perfectly resolved Schema Converters
55
55
  COPY --from=gradle-resolver /app/build/confluent-converters/ /workspace/connect/confluent-converters/
56
56
 
57
- # Grab Gradle-resolved Flink Parquet & Hadoop JARs
58
- COPY --from=gradle-resolver /app/build/flink-jars/ /workspace/flink/1.x/
57
+ # Grab Gradle-resolved Flink Parquet & Hadoop JARs (Hadoop is what the Iceberg
58
+ # catalog needs for org.apache.hadoop.conf.Configuration)
59
+ COPY --from=gradle-resolver /app/build/flink-jars/ /workspace/flink/2.x/
59
60
 
60
- # Grab standalone Uber JARs & ZIPs (Docker safely merges flink/1.x/ folders)
61
+ # Grab standalone Uber JARs & ZIPs (Docker safely merges flink/2.x/ folders)
61
62
  COPY --from=downloader /downloads/connect /workspace/connect/
62
63
  COPY --from=downloader /downloads/flink /workspace/flink/
63
64
  COPY --from=downloader /downloads/spark /workspace/spark/
@@ -11,7 +11,7 @@ repositories {
11
11
 
12
12
  ext {
13
13
  confluentVersion = '8.2.0'
14
- flinkVersion = '1.20.4'
14
+ flinkVersion = '2.1.3'
15
15
  hadoopVersion = '3.3.6'
16
16
  }
17
17
 
@@ -15,39 +15,44 @@ get_maven_version() {
15
15
 
16
16
  echo "▶️ Resolving Flink Versions..."
17
17
 
18
- # Dynamically fetch the latest Flink 1.x patch
19
- FLINK_V=$(get_maven_version "org/apache/flink/flink-core" "1\.[0-9]+\.[0-9]+")
20
- if [ -z "$FLINK_V" ]; then echo "❌ Error: Could not resolve Flink core version!"; exit 1; fi
21
- FLINK_MINOR=$(echo "$FLINK_V" | grep -Eo '^[0-9]+\.[0-9]+' || true)
18
+ # 🔒 PINNED: Flink 2.1, not the latest 2.x. Iceberg publishes
19
+ # iceberg-flink-runtime for 2.0 and 2.1 only, and Fluss has no 2.0 build, so 2.1
20
+ # is the one minor where every connector this stack needs exists. See issue #9.
21
+ FLINK_MINOR="2.1"
22
+ FLINK_V=$(get_maven_version "org/apache/flink/flink-core" "${FLINK_MINOR}\.[0-9]+")
23
+ if [ -z "$FLINK_V" ]; then echo "❌ Error: Could not resolve Flink ${FLINK_MINOR} patch version!"; exit 1; fi
22
24
 
23
25
  # 🔒 HARDCODED: Lock Iceberg to 1.11.0 to match Spark and PyIceberg sidecar
24
26
  ICEBERG_V="1.11.0"
25
27
 
26
28
  echo "▶️ Fetching Flink Dependencies for ${FLINK_MINOR} (Iceberg ${ICEBERG_V})..."
27
- make_dir "flink/1.x"
29
+ make_dir "flink/2.x"
28
30
 
29
31
  # Iceberg Flink Runtime
30
- fetch_artifact "flink/1.x/iceberg-runtime.jar" "https://repo1.maven.org/maven2/org/apache/iceberg/iceberg-flink-runtime-${FLINK_MINOR}/${ICEBERG_V}/iceberg-flink-runtime-${FLINK_MINOR}-${ICEBERG_V}.jar"
32
+ fetch_artifact "flink/2.x/iceberg-runtime.jar" "https://repo1.maven.org/maven2/org/apache/iceberg/iceberg-flink-runtime-${FLINK_MINOR}/${ICEBERG_V}/iceberg-flink-runtime-${FLINK_MINOR}-${ICEBERG_V}.jar"
31
33
 
32
34
  # Kafka (Flink's independent connector cycle)
33
35
  K_V=$(get_maven_version "org/apache/flink/flink-sql-connector-kafka" "[0-9]+\.[0-9]+\.[0-9]+-${FLINK_MINOR}")
34
36
  if [ -z "$K_V" ]; then echo "❌ Error: Kafka connector for Flink ${FLINK_MINOR} is not published yet!"; exit 1; fi
35
- fetch_artifact "flink/1.x/kafka.jar" "https://repo1.maven.org/maven2/org/apache/flink/flink-sql-connector-kafka/${K_V}/flink-sql-connector-kafka-${K_V}.jar"
37
+ fetch_artifact "flink/2.x/kafka.jar" "https://repo1.maven.org/maven2/org/apache/flink/flink-sql-connector-kafka/${K_V}/flink-sql-connector-kafka-${K_V}.jar"
36
38
 
37
39
  # Avro with Confluent Schema Registry. This uber JAR is a superset of
38
40
  # flink-sql-avro: it registers the plain 'avro' format factory as well as
39
41
  # 'avro-confluent' and 'debezium-avro-confluent', so adding both would duplicate
40
42
  # the Avro classes and register the same factory twice.
41
- fetch_artifact "flink/1.x/avro-confluent-registry.jar" "https://repo1.maven.org/maven2/org/apache/flink/flink-sql-avro-confluent-registry/${FLINK_V}/flink-sql-avro-confluent-registry-${FLINK_V}.jar"
43
+ fetch_artifact "flink/2.x/avro-confluent-registry.jar" "https://repo1.maven.org/maven2/org/apache/flink/flink-sql-avro-confluent-registry/${FLINK_V}/flink-sql-avro-confluent-registry-${FLINK_V}.jar"
42
44
 
43
- # JDBC
44
- J_V=$(get_maven_version "org/apache/flink/flink-connector-jdbc" "[0-9]+\.[0-9]+\.[0-9]+-${FLINK_MINOR}")
45
+ # JDBC. Flink 2.x split this connector: the core jar carries the connector and
46
+ # each dialect ships separately, so Postgres needs both. The Postgres driver
47
+ # itself comes from fetch_standalone_plugins.sh into shared/.
48
+ J_V=$(get_maven_version "org/apache/flink/flink-connector-jdbc-core" "[0-9]+\.[0-9]+\.[0-9]+-${FLINK_MINOR}")
45
49
  if [ -z "$J_V" ]; then echo "❌ Error: JDBC connector for Flink ${FLINK_MINOR} is not published yet!"; exit 1; fi
46
- fetch_artifact "flink/1.x/jdbc.jar" "https://repo1.maven.org/maven2/org/apache/flink/flink-connector-jdbc/${J_V}/flink-connector-jdbc-${J_V}.jar"
50
+ fetch_artifact "flink/2.x/jdbc-core.jar" "https://repo1.maven.org/maven2/org/apache/flink/flink-connector-jdbc-core/${J_V}/flink-connector-jdbc-core-${J_V}.jar"
51
+ fetch_artifact "flink/2.x/jdbc-postgres.jar" "https://repo1.maven.org/maven2/org/apache/flink/flink-connector-jdbc-postgres/${J_V}/flink-connector-jdbc-postgres-${J_V}.jar"
47
52
 
48
53
  # Fluss
49
54
  F_V=$(get_maven_version "org/apache/fluss/fluss-flink-${FLINK_MINOR}" "[0-9]+\.[0-9]+\.[0-9]+(-incubating)?")
50
55
  if [ -z "$F_V" ]; then echo "❌ Error: Fluss connector for Flink ${FLINK_MINOR} is not published yet!"; exit 1; fi
51
- fetch_artifact "flink/1.x/fluss.jar" "https://repo1.maven.org/maven2/org/apache/fluss/fluss-flink-${FLINK_MINOR}/${F_V}/fluss-flink-${FLINK_MINOR}-${F_V}.jar"
56
+ fetch_artifact "flink/2.x/fluss.jar" "https://repo1.maven.org/maven2/org/apache/fluss/fluss-flink-${FLINK_MINOR}/${F_V}/fluss-flink-${FLINK_MINOR}-${F_V}.jar"
52
57
 
53
58
  echo "✅ Flink dependencies complete!"
@@ -1,7 +1,7 @@
1
1
  # ======================================================================
2
2
  # Core Process & Memory
3
3
  # ======================================================================
4
- jobmanager.rpc.address: jobmanager-1
4
+ jobmanager.rpc.address: jobmanager
5
5
  jobmanager.bind-address: 0.0.0.0
6
6
  jobmanager.memory.process.size: 1024m
7
7
  taskmanager.memory.process.size: 2560m
@@ -10,15 +10,15 @@ taskmanager.numberOfTaskSlots: 5
10
10
  # ======================================================================
11
11
  # State Backend (RocksDB)
12
12
  # ======================================================================
13
- state.backend: rocksdb
13
+ state.backend.type: rocksdb
14
14
  state.backend.rocksdb.localdir: /tmp/rocksdb
15
- state.backend.rocksdb.incremental: true
15
+ execution.checkpointing.incremental: true
16
16
  state.backend.rocksdb.predefined-options: FLASH_SSD_OPTIMIZED
17
17
  state.backend.rocksdb.thread.num: 4
18
18
  state.backend.rocksdb.memory.managed: true
19
19
 
20
- state.checkpoints.dir: s3://flink-checkpoints/1.x/
21
- state.savepoints.dir: s3://flink-savepoints/1.x/
20
+ execution.checkpointing.dir: s3://flink-checkpoints/2.x/
21
+ execution.checkpointing.savepoint-dir: s3://flink-savepoints/2.x/
22
22
 
23
23
  # ======================================================================
24
24
  # SeaweedFS (S3 Presto Plugin) Configuration
@@ -56,3 +56,13 @@ classloader.resolve-order: child-first
56
56
 
57
57
  # Enable streaming Iceberg queries with OPTIONS hints
58
58
  table.dynamic-table-options.enabled: true
59
+
60
+ # ======================================================================
61
+ # JVM module access
62
+ # Mounting this file replaces the image's config.yaml, which is where Flink
63
+ # ships its --add-opens and --add-exports defaults. Without them, Kryo
64
+ # serialization of java.nio buffers fails on Java 17 with
65
+ # InaccessibleObjectException and every checkpoint of a job that touches them
66
+ # dies. Copied verbatim from the flink:2.1-java17 image.
67
+ # ======================================================================
68
+ env.java.opts.all: --add-exports=java.rmi/sun.rmi.registry=ALL-UNNAMED --add-exports=jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED --add-exports=jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED --add-exports=jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED --add-exports=jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED --add-exports=jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED --add-exports=java.security.jgss/sun.security.krb5=ALL-UNNAMED --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.text=ALL-UNNAMED --add-opens=java.base/java.time=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.locks=ALL-UNNAMED
@@ -17,10 +17,10 @@ scrape_configs:
17
17
  metrics_path: "/metrics"
18
18
  static_configs:
19
19
  - targets:
20
- - "jobmanager-1:9249"
21
- - "taskmanager-1-a:9249"
22
- - "taskmanager-1-b:9249"
23
- - "taskmanager-1-c:9249"
20
+ - "jobmanager:9249"
21
+ - "taskmanager-a:9249"
22
+ - "taskmanager-b:9249"
23
+ - "taskmanager-c:9249"
24
24
 
25
25
  - job_name: "clickhouse"
26
26
  metrics_path: "/metrics"
@@ -106,27 +106,27 @@ stacks:
106
106
  flink:
107
107
  file: "compose-flink.yml"
108
108
  description: "Stateful, real-time stream processing engine. Lite uses 1 TaskManager; Full uses 3 TaskManagers."
109
- profiles: ["flink1-lite", "flink1-full"]
109
+ profiles: ["flink-lite", "flink-full"]
110
110
  capacities: ["stream_processing"]
111
111
  depends_on:
112
- flink1-lite: ["deps", "storage", "catalog"]
113
- flink1-full: ["deps", "storage", "catalog"]
112
+ flink-lite: ["deps", "storage", "catalog"]
113
+ flink-full: ["deps", "storage", "catalog"]
114
114
  role: "Executes continuous streaming pipelines. Lite uses 1 TaskManager; Full uses 3 TaskManagers."
115
115
  usage:
116
- flink1-lite: |
116
+ flink-lite: |
117
117
  • Flink JobManager UI (1 TaskManager):
118
118
  - External (Localhost): http://localhost:8082
119
- - Internal (Docker): http://jobmanager-1:8081
119
+ - Internal (Docker): http://jobmanager:8081
120
120
  • SQL Gateway REST:
121
121
  - External (Localhost): http://localhost:8084
122
- - Internal (Docker): http://flink-sql-gateway-1:8083
123
- flink1-full: |
122
+ - Internal (Docker): http://flink-sql-gateway:8083
123
+ flink-full: |
124
124
  • Flink JobManager UI (3 TaskManagers):
125
125
  - External (Localhost): http://localhost:8082
126
- - Internal (Docker): http://jobmanager-1:8081
126
+ - Internal (Docker): http://jobmanager:8081
127
127
  • SQL Gateway REST:
128
128
  - External (Localhost): http://localhost:8084
129
- - Internal (Docker): http://flink-sql-gateway-1:8083
129
+ - Internal (Docker): http://flink-sql-gateway:8083
130
130
 
131
131
  spark:
132
132
  file: "compose-spark.yml"
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes