odctl 0.2.1__tar.gz → 0.3.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {odctl-0.2.1 → odctl-0.3.0}/.github/renovate.json +10 -0
- {odctl-0.2.1 → odctl-0.3.0}/PKG-INFO +3 -3
- {odctl-0.2.1 → odctl-0.3.0}/README.md +2 -2
- {odctl-0.2.1 → odctl-0.3.0}/pyproject.toml +1 -1
- {odctl-0.2.1 → odctl-0.3.0}/src/odctl/main.py +7 -7
- {odctl-0.2.1 → odctl-0.3.0}/src/odctl/resources/compose-flink.yml +38 -30
- {odctl-0.2.1 → odctl-0.3.0}/src/odctl/resources/compose-infra.yml +7 -4
- {odctl-0.2.1 → odctl-0.3.0}/src/odctl/resources/compose-store.yml +2 -2
- {odctl-0.2.1 → odctl-0.3.0}/src/odctl/resources/deps/Dockerfile +4 -3
- {odctl-0.2.1 → odctl-0.3.0}/src/odctl/resources/deps/build.gradle +1 -1
- {odctl-0.2.1 → odctl-0.3.0}/src/odctl/resources/deps/fetch_flink_dependencies.sh +17 -12
- odctl-0.2.1/src/odctl/resources/flink/flink1-conf.yml → odctl-0.3.0/src/odctl/resources/flink/flink-conf.yml +15 -5
- {odctl-0.2.1 → odctl-0.3.0}/src/odctl/resources/prometheus/prometheus.yml +4 -4
- {odctl-0.2.1 → odctl-0.3.0}/src/odctl/resources/registry.yml +9 -9
- {odctl-0.2.1 → odctl-0.3.0}/.github/ISSUE_TEMPLATE/bug_report.md +0 -0
- {odctl-0.2.1 → odctl-0.3.0}/.github/ISSUE_TEMPLATE/feature_request.md +0 -0
- {odctl-0.2.1 → odctl-0.3.0}/.github/workflows/build-deps.yml +0 -0
- {odctl-0.2.1 → odctl-0.3.0}/.github/workflows/build-platform-images.yml +0 -0
- {odctl-0.2.1 → odctl-0.3.0}/.github/workflows/ci.yml +0 -0
- {odctl-0.2.1 → odctl-0.3.0}/.github/workflows/e2e-test.yml +0 -0
- {odctl-0.2.1 → odctl-0.3.0}/.github/workflows/pipeline.yml +0 -0
- {odctl-0.2.1 → odctl-0.3.0}/.github/workflows/publish.yml +0 -0
- {odctl-0.2.1 → odctl-0.3.0}/.gitignore +0 -0
- {odctl-0.2.1 → odctl-0.3.0}/.pre-commit-config.yaml +0 -0
- {odctl-0.2.1 → odctl-0.3.0}/.vscode/settings.json +0 -0
- {odctl-0.2.1 → odctl-0.3.0}/LICENSE +0 -0
- {odctl-0.2.1 → odctl-0.3.0}/image/diagram.png +0 -0
- {odctl-0.2.1 → odctl-0.3.0}/image/diagram1.drawio +0 -0
- {odctl-0.2.1 → odctl-0.3.0}/image/diagram2.drawio +0 -0
- {odctl-0.2.1 → odctl-0.3.0}/src/odctl/__init__.py +0 -0
- {odctl-0.2.1 → odctl-0.3.0}/src/odctl/config.py +0 -0
- {odctl-0.2.1 → odctl-0.3.0}/src/odctl/docker.py +0 -0
- {odctl-0.2.1 → odctl-0.3.0}/src/odctl/planner.py +0 -0
- {odctl-0.2.1 → odctl-0.3.0}/src/odctl/registry.py +0 -0
- {odctl-0.2.1 → odctl-0.3.0}/src/odctl/resources/clickhouse/config.xml +0 -0
- {odctl-0.2.1 → odctl-0.3.0}/src/odctl/resources/clickhouse/init.sql +0 -0
- {odctl-0.2.1 → odctl-0.3.0}/src/odctl/resources/clickhouse/keeper_config.xml +0 -0
- {odctl-0.2.1 → odctl-0.3.0}/src/odctl/resources/clickhouse/macros-11.xml +0 -0
- {odctl-0.2.1 → odctl-0.3.0}/src/odctl/resources/clickhouse/macros-12.xml +0 -0
- {odctl-0.2.1 → odctl-0.3.0}/src/odctl/resources/clickhouse/macros-21.xml +0 -0
- {odctl-0.2.1 → odctl-0.3.0}/src/odctl/resources/clickhouse/macros-22.xml +0 -0
- {odctl-0.2.1 → odctl-0.3.0}/src/odctl/resources/clickhouse/users.xml +0 -0
- {odctl-0.2.1 → odctl-0.3.0}/src/odctl/resources/compose-analytics.yml +0 -0
- {odctl-0.2.1 → odctl-0.3.0}/src/odctl/resources/compose-deps.yml +0 -0
- {odctl-0.2.1 → odctl-0.3.0}/src/odctl/resources/compose-kafka.yml +0 -0
- {odctl-0.2.1 → odctl-0.3.0}/src/odctl/resources/compose-metadata.yml +0 -0
- {odctl-0.2.1 → odctl-0.3.0}/src/odctl/resources/compose-mlops.yml +0 -0
- {odctl-0.2.1 → odctl-0.3.0}/src/odctl/resources/compose-obsv.yml +0 -0
- {odctl-0.2.1 → odctl-0.3.0}/src/odctl/resources/compose-orch.yml +0 -0
- {odctl-0.2.1 → odctl-0.3.0}/src/odctl/resources/compose-spark.yml +0 -0
- {odctl-0.2.1 → odctl-0.3.0}/src/odctl/resources/deps/build_connectors.sh +0 -0
- {odctl-0.2.1 → odctl-0.3.0}/src/odctl/resources/deps/fetch_spark_dependencies.sh +0 -0
- {odctl-0.2.1 → odctl-0.3.0}/src/odctl/resources/deps/fetch_standalone_plugins.sh +0 -0
- {odctl-0.2.1 → odctl-0.3.0}/src/odctl/resources/docker/airflow/Dockerfile +0 -0
- {odctl-0.2.1 → odctl-0.3.0}/src/odctl/resources/docker/ray-mlops/Dockerfile +0 -0
- {odctl-0.2.1 → odctl-0.3.0}/src/odctl/resources/docker/spark/Dockerfile +0 -0
- {odctl-0.2.1 → odctl-0.3.0}/src/odctl/resources/grafana/prometheus.yml +0 -0
- {odctl-0.2.1 → odctl-0.3.0}/src/odctl/resources/marquez/marquez.yml +0 -0
- {odctl-0.2.1 → odctl-0.3.0}/src/odctl/resources/mlops/serving/lightgbm_inference.py.example +0 -0
- {odctl-0.2.1 → odctl-0.3.0}/src/odctl/resources/mlops/serving/vector_search.py.example +0 -0
- {odctl-0.2.1 → odctl-0.3.0}/src/odctl/resources/postgres/01-init-databases.sh +0 -0
- {odctl-0.2.1 → odctl-0.3.0}/src/odctl/resources/prometheus/alertmanager.yml +0 -0
- {odctl-0.2.1 → odctl-0.3.0}/src/odctl/resources/seaweed/s3.json +0 -0
- {odctl-0.2.1 → odctl-0.3.0}/src/odctl/resources/spark/log4j2.properties +0 -0
- {odctl-0.2.1 → odctl-0.3.0}/src/odctl/resources/spark/spark-defaults.conf +0 -0
- {odctl-0.2.1 → odctl-0.3.0}/src/odctl/resources/trino/catalog/clickhouse.properties +0 -0
- {odctl-0.2.1 → odctl-0.3.0}/src/odctl/resources/trino/catalog/iceberg.properties +0 -0
- {odctl-0.2.1 → odctl-0.3.0}/src/odctl/resources/trino/catalog/kafka.properties +0 -0
- {odctl-0.2.1 → odctl-0.3.0}/src/odctl/resources/trino/catalog/postgres.properties +0 -0
- {odctl-0.2.1 → odctl-0.3.0}/src/odctl/resources/trino/catalog/valkey.properties +0 -0
- {odctl-0.2.1 → odctl-0.3.0}/src/odctl/ui.py +0 -0
- {odctl-0.2.1 → odctl-0.3.0}/src/odctl/workspace.py +0 -0
- {odctl-0.2.1 → odctl-0.3.0}/tests/conftest.py +0 -0
- {odctl-0.2.1 → odctl-0.3.0}/tests/test_cli.py +0 -0
- {odctl-0.2.1 → odctl-0.3.0}/tests/test_cli_extended.py +0 -0
- {odctl-0.2.1 → odctl-0.3.0}/tests/test_config.py +0 -0
- {odctl-0.2.1 → odctl-0.3.0}/tests/test_docker.py +0 -0
- {odctl-0.2.1 → odctl-0.3.0}/tests/test_planner.py +0 -0
- {odctl-0.2.1 → odctl-0.3.0}/tests/test_registry.py +0 -0
- {odctl-0.2.1 → odctl-0.3.0}/tests/test_workspace.py +0 -0
- {odctl-0.2.1 → odctl-0.3.0}/uv.lock +0 -0
|
@@ -12,6 +12,16 @@
|
|
|
12
12
|
]
|
|
13
13
|
},
|
|
14
14
|
"packageRules": [
|
|
15
|
+
{
|
|
16
|
+
"description": "Flink stays on 2.1: Iceberg publishes iceberg-flink-runtime for 2.0 and 2.1 only, and fluss-flink has no 2.0 build. Raise this when Iceberg ships a runtime for a newer minor. See issue #9.",
|
|
17
|
+
"matchPackageNames": ["flink", "org.apache.flink:{/,}**"],
|
|
18
|
+
"allowedVersions": "<2.2"
|
|
19
|
+
},
|
|
20
|
+
{
|
|
21
|
+
"description": "Fluss stays on 0.8.x because 0.9.x requires Flink 2.2, which has no Iceberg runtime. Moves together with the Flink ceiling above.",
|
|
22
|
+
"matchPackageNames": ["apache/fluss", "org.apache.fluss:{/,}**"],
|
|
23
|
+
"allowedVersions": "<0.9"
|
|
24
|
+
},
|
|
15
25
|
{
|
|
16
26
|
"description": "Auto-merge non-breaking minor and patch updates if CI passes",
|
|
17
27
|
"matchUpdateTypes": ["minor", "patch"],
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: odctl
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.3.0
|
|
4
4
|
Summary: Orchestrator for the Open Data Stack
|
|
5
5
|
Project-URL: Homepage, https://github.com/jaehyeon-kim/odctl
|
|
6
6
|
Project-URL: Repository, https://github.com/jaehyeon-kim/odctl
|
|
@@ -123,7 +123,7 @@ odctl list
|
|
|
123
123
|
Bring up a robust data engineering environment.
|
|
124
124
|
|
|
125
125
|
```bash
|
|
126
|
-
odctl up kafka-lite
|
|
126
|
+
odctl up kafka-lite flink-lite spark-lite
|
|
127
127
|
```
|
|
128
128
|
|
|
129
129
|
_Note: You do not need to memorize dependencies. The CLI will automatically detect that these profiles require foundational infrastructure and will launch PostgreSQL, SeaweedFS (S3), and the Iceberg REST Catalog for you before starting the target compute engines._
|
|
@@ -169,7 +169,7 @@ $ odctl list -d
|
|
|
169
169
|
$ odctl explain kafka-lite
|
|
170
170
|
|
|
171
171
|
# Launch specific profiles
|
|
172
|
-
$ odctl up
|
|
172
|
+
$ odctl up flink-lite kafka-lite spark-lite
|
|
173
173
|
|
|
174
174
|
# Complete teardown and wipe all data
|
|
175
175
|
$ odctl down --all --volumes
|
|
@@ -87,7 +87,7 @@ odctl list
|
|
|
87
87
|
Bring up a robust data engineering environment.
|
|
88
88
|
|
|
89
89
|
```bash
|
|
90
|
-
odctl up kafka-lite
|
|
90
|
+
odctl up kafka-lite flink-lite spark-lite
|
|
91
91
|
```
|
|
92
92
|
|
|
93
93
|
_Note: You do not need to memorize dependencies. The CLI will automatically detect that these profiles require foundational infrastructure and will launch PostgreSQL, SeaweedFS (S3), and the Iceberg REST Catalog for you before starting the target compute engines._
|
|
@@ -133,7 +133,7 @@ $ odctl list -d
|
|
|
133
133
|
$ odctl explain kafka-lite
|
|
134
134
|
|
|
135
135
|
# Launch specific profiles
|
|
136
|
-
$ odctl up
|
|
136
|
+
$ odctl up flink-lite kafka-lite spark-lite
|
|
137
137
|
|
|
138
138
|
# Complete teardown and wipe all data
|
|
139
139
|
$ odctl down --all --volumes
|
|
@@ -36,7 +36,7 @@ Provides commands to inspect, provision, and tear down curated Docker Compose st
|
|
|
36
36
|
[dim]# See exactly what the airflow profile provisions[/dim]\n
|
|
37
37
|
$ [bold cyan]odctl explain kafka-lite[/bold cyan]\n\n
|
|
38
38
|
[dim]# Launch specific profiles and their dependencies[/dim]\n
|
|
39
|
-
$ [bold cyan]odctl up
|
|
39
|
+
$ [bold cyan]odctl up flink-lite kafka-lite spark-lite[/bold cyan]\n\n
|
|
40
40
|
[dim]# Complete teardown and wipe all data[/dim]\n
|
|
41
41
|
$ [bold cyan]odctl down --all --volumes[/bold cyan]
|
|
42
42
|
""",
|
|
@@ -201,8 +201,8 @@ def init(
|
|
|
201
201
|
[bold underline]Examples:[/bold underline]\n
|
|
202
202
|
[dim]# Pre-fetch images for all profiles[/dim]\n
|
|
203
203
|
$ [bold cyan]odctl pull --all[/bold cyan]\n\n
|
|
204
|
-
[dim]# Pre-fetch images just for Flink
|
|
205
|
-
$ [bold cyan]odctl pull
|
|
204
|
+
[dim]# Pre-fetch images just for Flink and Spark[/dim]\n
|
|
205
|
+
$ [bold cyan]odctl pull flink-lite kafka-lite[/bold cyan]
|
|
206
206
|
""",
|
|
207
207
|
)
|
|
208
208
|
def pull(
|
|
@@ -243,8 +243,8 @@ def pull(
|
|
|
243
243
|
rich_help_panel="Cluster Lifecycle",
|
|
244
244
|
epilog="""
|
|
245
245
|
[bold underline]Examples:[/bold underline]\n
|
|
246
|
-
[dim]# Launch Clickhouse, Flink
|
|
247
|
-
$ [bold cyan]odctl up ch-lite
|
|
246
|
+
[dim]# Launch Clickhouse, Flink, and their dependencies[/dim]\n
|
|
247
|
+
$ [bold cyan]odctl up ch-lite flink-lite[/bold cyan]\n\n
|
|
248
248
|
[dim]# Preview what would be launched for Airflow[/dim]\n
|
|
249
249
|
$ [bold cyan]odctl up airflow --dry-run[/bold cyan]\n\n
|
|
250
250
|
[dim]# Force pull latest images before launching[/dim]\n
|
|
@@ -464,9 +464,9 @@ def info():
|
|
|
464
464
|
epilog="""
|
|
465
465
|
[bold underline]Examples:[/bold underline]\n
|
|
466
466
|
[dim]# Tail the last 50 lines of all Flink containers and follow live[/dim]\n
|
|
467
|
-
$ [bold cyan]odctl logs
|
|
467
|
+
$ [bold cyan]odctl logs flink-lite -n 50 -f[/bold cyan]\n\n
|
|
468
468
|
[dim]# View logs strictly for the JobManager service[/dim]\n
|
|
469
|
-
$ [bold cyan]odctl logs
|
|
469
|
+
$ [bold cyan]odctl logs flink-lite -s jobmanager[/bold cyan]\n\n
|
|
470
470
|
[dim]# Show logs with timestamps for the last 10 minutes[/dim]\n
|
|
471
471
|
$ [bold cyan]odctl logs kafka-lite --since 10m -t[/bold cyan]
|
|
472
472
|
""",
|
|
@@ -4,14 +4,22 @@ x-flink-common: &flink-common
|
|
|
4
4
|
networks:
|
|
5
5
|
- odctl
|
|
6
6
|
volumes:
|
|
7
|
-
- ./flink/
|
|
7
|
+
- ./flink/flink-conf.yml:/opt/flink/conf/config.yaml:ro
|
|
8
8
|
- odctl-shared-deps:/mnt/shared-deps:ro
|
|
9
9
|
|
|
10
|
-
x-
|
|
10
|
+
x-flink-base: &flink-base
|
|
11
11
|
<<: *flink-common
|
|
12
|
-
|
|
12
|
+
# Pinned to 2.1 because Iceberg publishes iceberg-flink-runtime for 2.0 and
|
|
13
|
+
# 2.1 only, and Fluss has no 2.0 build. See issue #9.
|
|
14
|
+
image: flink:2.1-java17
|
|
13
15
|
environment:
|
|
14
16
|
TZ: ${TZ:-UTC}
|
|
17
|
+
# SeaweedFS ignores the region, but the AWS SDK refuses to build a client
|
|
18
|
+
# without one, so an Iceberg catalog that omits client.region fails its
|
|
19
|
+
# commits with "Unable to load region" while checkpoints keep succeeding.
|
|
20
|
+
AWS_REGION: us-east-1
|
|
21
|
+
AWS_ACCESS_KEY_ID: user
|
|
22
|
+
AWS_SECRET_ACCESS_KEY: password
|
|
15
23
|
entrypoint:
|
|
16
24
|
- /bin/bash
|
|
17
25
|
- -c
|
|
@@ -19,7 +27,7 @@ x-flink1-base: &flink1-base
|
|
|
19
27
|
# Load dependencies into lib/, not usrlib/. Only lib/ reaches the system
|
|
20
28
|
# classloader, which is where SQL connector factories are looked up.
|
|
21
29
|
cp /mnt/shared-deps/shared/*.jar /opt/flink/lib/ 2>/dev/null || true
|
|
22
|
-
cp /mnt/shared-deps/flink/
|
|
30
|
+
cp /mnt/shared-deps/flink/2.x/*.jar /opt/flink/lib/ 2>/dev/null || true
|
|
23
31
|
cp /mnt/shared-deps/connect/confluent-converters/*.jar /opt/flink/lib/ 2>/dev/null || true
|
|
24
32
|
|
|
25
33
|
# S3 Plugin Linking
|
|
@@ -30,10 +38,10 @@ x-flink1-base: &flink1-base
|
|
|
30
38
|
- --
|
|
31
39
|
|
|
32
40
|
services:
|
|
33
|
-
jobmanager
|
|
34
|
-
<<: *
|
|
35
|
-
container_name: flink-jobmanager
|
|
36
|
-
profiles: ["
|
|
41
|
+
jobmanager:
|
|
42
|
+
<<: *flink-base
|
|
43
|
+
container_name: flink-jobmanager
|
|
44
|
+
profiles: ["flink-lite", "flink-full"]
|
|
37
45
|
command: jobmanager
|
|
38
46
|
ports:
|
|
39
47
|
- "8082:8081"
|
|
@@ -44,51 +52,51 @@ services:
|
|
|
44
52
|
timeout: 5s
|
|
45
53
|
retries: 5
|
|
46
54
|
|
|
47
|
-
taskmanager-
|
|
48
|
-
<<: *
|
|
49
|
-
container_name: flink-taskmanager-
|
|
50
|
-
profiles: ["
|
|
55
|
+
taskmanager-a:
|
|
56
|
+
<<: *flink-base
|
|
57
|
+
container_name: flink-taskmanager-a
|
|
58
|
+
profiles: ["flink-lite", "flink-full"]
|
|
51
59
|
command: taskmanager
|
|
52
60
|
mem_limit: 3500m
|
|
53
61
|
depends_on:
|
|
54
|
-
jobmanager
|
|
62
|
+
jobmanager:
|
|
55
63
|
condition: service_healthy
|
|
56
64
|
|
|
57
|
-
taskmanager-
|
|
58
|
-
<<: *
|
|
59
|
-
container_name: flink-taskmanager-
|
|
60
|
-
profiles: ["
|
|
65
|
+
taskmanager-b:
|
|
66
|
+
<<: *flink-base
|
|
67
|
+
container_name: flink-taskmanager-b
|
|
68
|
+
profiles: ["flink-full"]
|
|
61
69
|
command: taskmanager
|
|
62
70
|
mem_limit: 3500m
|
|
63
71
|
depends_on:
|
|
64
|
-
jobmanager
|
|
72
|
+
jobmanager:
|
|
65
73
|
condition: service_healthy
|
|
66
74
|
|
|
67
|
-
taskmanager-
|
|
68
|
-
<<: *
|
|
69
|
-
container_name: flink-taskmanager-
|
|
70
|
-
profiles: ["
|
|
75
|
+
taskmanager-c:
|
|
76
|
+
<<: *flink-base
|
|
77
|
+
container_name: flink-taskmanager-c
|
|
78
|
+
profiles: ["flink-full"]
|
|
71
79
|
command: taskmanager
|
|
72
80
|
mem_limit: 3500m
|
|
73
81
|
depends_on:
|
|
74
|
-
jobmanager
|
|
82
|
+
jobmanager:
|
|
75
83
|
condition: service_healthy
|
|
76
84
|
|
|
77
|
-
sql-gateway
|
|
78
|
-
<<: *
|
|
79
|
-
container_name: flink-sql-gateway
|
|
80
|
-
profiles: ["
|
|
85
|
+
sql-gateway:
|
|
86
|
+
<<: *flink-base
|
|
87
|
+
container_name: flink-sql-gateway
|
|
88
|
+
profiles: ["flink-lite", "flink-full"]
|
|
81
89
|
command: >
|
|
82
90
|
bin/sql-gateway.sh start-foreground
|
|
83
91
|
-Dsql-gateway.endpoint.rest.address=0.0.0.0
|
|
84
92
|
-Dsql-gateway.endpoint.rest.port=8083
|
|
85
|
-
-Djobmanager.rpc.address=jobmanager
|
|
86
|
-
-Drest.address=jobmanager
|
|
93
|
+
-Djobmanager.rpc.address=jobmanager
|
|
94
|
+
-Drest.address=jobmanager
|
|
87
95
|
mem_limit: 1g
|
|
88
96
|
ports:
|
|
89
97
|
- "8084:8083"
|
|
90
98
|
depends_on:
|
|
91
|
-
jobmanager
|
|
99
|
+
jobmanager:
|
|
92
100
|
condition: service_healthy
|
|
93
101
|
|
|
94
102
|
networks:
|
|
@@ -124,10 +124,13 @@ services:
|
|
|
124
124
|
- CATALOG_WAREHOUSE=s3://warehouse/
|
|
125
125
|
- CATALOG_IO__IMPL=org.apache.iceberg.aws.s3.S3FileIO
|
|
126
126
|
- CATALOG_S3_ENDPOINT=http://seaweed:8333
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
-
|
|
130
|
-
-
|
|
127
|
+
# The image maps a double underscore to a hyphen and a single one to a dot,
|
|
128
|
+
# so hyphenated Iceberg properties need __ or they resolve to names Iceberg
|
|
129
|
+
# ignores. With path-style access silently dropped, S3FileIO addressed
|
|
130
|
+
# buckets virtual-host style and SeaweedFS read the namespace as the bucket.
|
|
131
|
+
- CATALOG_S3_ACCESS__KEY__ID=user
|
|
132
|
+
- CATALOG_S3_SECRET__ACCESS__KEY=password
|
|
133
|
+
- CATALOG_S3_PATH__STYLE__ACCESS=true
|
|
131
134
|
# Explicit AWS SDK overrides (just in case)
|
|
132
135
|
- AWS_REGION=us-east-1
|
|
133
136
|
- AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID:-user}
|
|
@@ -7,7 +7,7 @@ services:
|
|
|
7
7
|
# [PROFILE: fluss] Streaming Storage Engine
|
|
8
8
|
# ==========================================
|
|
9
9
|
fluss-coordinator:
|
|
10
|
-
image: apache/fluss:0.
|
|
10
|
+
image: apache/fluss:0.8.0-incubating
|
|
11
11
|
container_name: fluss-coordinator
|
|
12
12
|
command: coordinatorServer
|
|
13
13
|
profiles: ["fluss"]
|
|
@@ -28,7 +28,7 @@ services:
|
|
|
28
28
|
remote.data.dir: /tmp/fluss/remote-data
|
|
29
29
|
|
|
30
30
|
fluss-tablet-1:
|
|
31
|
-
image: apache/fluss:0.
|
|
31
|
+
image: apache/fluss:0.8.0-incubating
|
|
32
32
|
container_name: fluss-tablet-1
|
|
33
33
|
command: tabletServer
|
|
34
34
|
profiles: ["fluss"]
|
|
@@ -54,10 +54,11 @@ COPY --from=source-builder /build/dist/ /workspace/connect/
|
|
|
54
54
|
# Grab perfectly resolved Schema Converters
|
|
55
55
|
COPY --from=gradle-resolver /app/build/confluent-converters/ /workspace/connect/confluent-converters/
|
|
56
56
|
|
|
57
|
-
# Grab Gradle-resolved Flink Parquet & Hadoop JARs
|
|
58
|
-
|
|
57
|
+
# Grab Gradle-resolved Flink Parquet & Hadoop JARs (Hadoop is what the Iceberg
|
|
58
|
+
# catalog needs for org.apache.hadoop.conf.Configuration)
|
|
59
|
+
COPY --from=gradle-resolver /app/build/flink-jars/ /workspace/flink/2.x/
|
|
59
60
|
|
|
60
|
-
# Grab standalone Uber JARs & ZIPs (Docker safely merges flink/
|
|
61
|
+
# Grab standalone Uber JARs & ZIPs (Docker safely merges flink/2.x/ folders)
|
|
61
62
|
COPY --from=downloader /downloads/connect /workspace/connect/
|
|
62
63
|
COPY --from=downloader /downloads/flink /workspace/flink/
|
|
63
64
|
COPY --from=downloader /downloads/spark /workspace/spark/
|
|
@@ -15,39 +15,44 @@ get_maven_version() {
|
|
|
15
15
|
|
|
16
16
|
echo "▶️ Resolving Flink Versions..."
|
|
17
17
|
|
|
18
|
-
#
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
FLINK_MINOR
|
|
18
|
+
# 🔒 PINNED: Flink 2.1, not the latest 2.x. Iceberg publishes
|
|
19
|
+
# iceberg-flink-runtime for 2.0 and 2.1 only, and Fluss has no 2.0 build, so 2.1
|
|
20
|
+
# is the one minor where every connector this stack needs exists. See issue #9.
|
|
21
|
+
FLINK_MINOR="2.1"
|
|
22
|
+
FLINK_V=$(get_maven_version "org/apache/flink/flink-core" "${FLINK_MINOR}\.[0-9]+")
|
|
23
|
+
if [ -z "$FLINK_V" ]; then echo "❌ Error: Could not resolve Flink ${FLINK_MINOR} patch version!"; exit 1; fi
|
|
22
24
|
|
|
23
25
|
# 🔒 HARDCODED: Lock Iceberg to 1.11.0 to match Spark and PyIceberg sidecar
|
|
24
26
|
ICEBERG_V="1.11.0"
|
|
25
27
|
|
|
26
28
|
echo "▶️ Fetching Flink Dependencies for ${FLINK_MINOR} (Iceberg ${ICEBERG_V})..."
|
|
27
|
-
make_dir "flink/
|
|
29
|
+
make_dir "flink/2.x"
|
|
28
30
|
|
|
29
31
|
# Iceberg Flink Runtime
|
|
30
|
-
fetch_artifact "flink/
|
|
32
|
+
fetch_artifact "flink/2.x/iceberg-runtime.jar" "https://repo1.maven.org/maven2/org/apache/iceberg/iceberg-flink-runtime-${FLINK_MINOR}/${ICEBERG_V}/iceberg-flink-runtime-${FLINK_MINOR}-${ICEBERG_V}.jar"
|
|
31
33
|
|
|
32
34
|
# Kafka (Flink's independent connector cycle)
|
|
33
35
|
K_V=$(get_maven_version "org/apache/flink/flink-sql-connector-kafka" "[0-9]+\.[0-9]+\.[0-9]+-${FLINK_MINOR}")
|
|
34
36
|
if [ -z "$K_V" ]; then echo "❌ Error: Kafka connector for Flink ${FLINK_MINOR} is not published yet!"; exit 1; fi
|
|
35
|
-
fetch_artifact "flink/
|
|
37
|
+
fetch_artifact "flink/2.x/kafka.jar" "https://repo1.maven.org/maven2/org/apache/flink/flink-sql-connector-kafka/${K_V}/flink-sql-connector-kafka-${K_V}.jar"
|
|
36
38
|
|
|
37
39
|
# Avro with Confluent Schema Registry. This uber JAR is a superset of
|
|
38
40
|
# flink-sql-avro: it registers the plain 'avro' format factory as well as
|
|
39
41
|
# 'avro-confluent' and 'debezium-avro-confluent', so adding both would duplicate
|
|
40
42
|
# the Avro classes and register the same factory twice.
|
|
41
|
-
fetch_artifact "flink/
|
|
43
|
+
fetch_artifact "flink/2.x/avro-confluent-registry.jar" "https://repo1.maven.org/maven2/org/apache/flink/flink-sql-avro-confluent-registry/${FLINK_V}/flink-sql-avro-confluent-registry-${FLINK_V}.jar"
|
|
42
44
|
|
|
43
|
-
# JDBC
|
|
44
|
-
|
|
45
|
+
# JDBC. Flink 2.x split this connector: the core jar carries the connector and
|
|
46
|
+
# each dialect ships separately, so Postgres needs both. The Postgres driver
|
|
47
|
+
# itself comes from fetch_standalone_plugins.sh into shared/.
|
|
48
|
+
J_V=$(get_maven_version "org/apache/flink/flink-connector-jdbc-core" "[0-9]+\.[0-9]+\.[0-9]+-${FLINK_MINOR}")
|
|
45
49
|
if [ -z "$J_V" ]; then echo "❌ Error: JDBC connector for Flink ${FLINK_MINOR} is not published yet!"; exit 1; fi
|
|
46
|
-
fetch_artifact "flink/
|
|
50
|
+
fetch_artifact "flink/2.x/jdbc-core.jar" "https://repo1.maven.org/maven2/org/apache/flink/flink-connector-jdbc-core/${J_V}/flink-connector-jdbc-core-${J_V}.jar"
|
|
51
|
+
fetch_artifact "flink/2.x/jdbc-postgres.jar" "https://repo1.maven.org/maven2/org/apache/flink/flink-connector-jdbc-postgres/${J_V}/flink-connector-jdbc-postgres-${J_V}.jar"
|
|
47
52
|
|
|
48
53
|
# Fluss
|
|
49
54
|
F_V=$(get_maven_version "org/apache/fluss/fluss-flink-${FLINK_MINOR}" "[0-9]+\.[0-9]+\.[0-9]+(-incubating)?")
|
|
50
55
|
if [ -z "$F_V" ]; then echo "❌ Error: Fluss connector for Flink ${FLINK_MINOR} is not published yet!"; exit 1; fi
|
|
51
|
-
fetch_artifact "flink/
|
|
56
|
+
fetch_artifact "flink/2.x/fluss.jar" "https://repo1.maven.org/maven2/org/apache/fluss/fluss-flink-${FLINK_MINOR}/${F_V}/fluss-flink-${FLINK_MINOR}-${F_V}.jar"
|
|
52
57
|
|
|
53
58
|
echo "✅ Flink dependencies complete!"
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
# ======================================================================
|
|
2
2
|
# Core Process & Memory
|
|
3
3
|
# ======================================================================
|
|
4
|
-
jobmanager.rpc.address: jobmanager
|
|
4
|
+
jobmanager.rpc.address: jobmanager
|
|
5
5
|
jobmanager.bind-address: 0.0.0.0
|
|
6
6
|
jobmanager.memory.process.size: 1024m
|
|
7
7
|
taskmanager.memory.process.size: 2560m
|
|
@@ -10,15 +10,15 @@ taskmanager.numberOfTaskSlots: 5
|
|
|
10
10
|
# ======================================================================
|
|
11
11
|
# State Backend (RocksDB)
|
|
12
12
|
# ======================================================================
|
|
13
|
-
state.backend: rocksdb
|
|
13
|
+
state.backend.type: rocksdb
|
|
14
14
|
state.backend.rocksdb.localdir: /tmp/rocksdb
|
|
15
|
-
|
|
15
|
+
execution.checkpointing.incremental: true
|
|
16
16
|
state.backend.rocksdb.predefined-options: FLASH_SSD_OPTIMIZED
|
|
17
17
|
state.backend.rocksdb.thread.num: 4
|
|
18
18
|
state.backend.rocksdb.memory.managed: true
|
|
19
19
|
|
|
20
|
-
|
|
21
|
-
|
|
20
|
+
execution.checkpointing.dir: s3://flink-checkpoints/2.x/
|
|
21
|
+
execution.checkpointing.savepoint-dir: s3://flink-savepoints/2.x/
|
|
22
22
|
|
|
23
23
|
# ======================================================================
|
|
24
24
|
# SeaweedFS (S3 Presto Plugin) Configuration
|
|
@@ -56,3 +56,13 @@ classloader.resolve-order: child-first
|
|
|
56
56
|
|
|
57
57
|
# Enable streaming Iceberg queries with OPTIONS hints
|
|
58
58
|
table.dynamic-table-options.enabled: true
|
|
59
|
+
|
|
60
|
+
# ======================================================================
|
|
61
|
+
# JVM module access
|
|
62
|
+
# Mounting this file replaces the image's config.yaml, which is where Flink
|
|
63
|
+
# ships its --add-opens and --add-exports defaults. Without them, Kryo
|
|
64
|
+
# serialization of java.nio buffers fails on Java 17 with
|
|
65
|
+
# InaccessibleObjectException and every checkpoint of a job that touches them
|
|
66
|
+
# dies. Copied verbatim from the flink:2.1-java17 image.
|
|
67
|
+
# ======================================================================
|
|
68
|
+
env.java.opts.all: --add-exports=java.rmi/sun.rmi.registry=ALL-UNNAMED --add-exports=jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED --add-exports=jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED --add-exports=jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED --add-exports=jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED --add-exports=jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED --add-exports=java.security.jgss/sun.security.krb5=ALL-UNNAMED --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.text=ALL-UNNAMED --add-opens=java.base/java.time=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.locks=ALL-UNNAMED
|
|
@@ -17,10 +17,10 @@ scrape_configs:
|
|
|
17
17
|
metrics_path: "/metrics"
|
|
18
18
|
static_configs:
|
|
19
19
|
- targets:
|
|
20
|
-
- "jobmanager
|
|
21
|
-
- "taskmanager-
|
|
22
|
-
- "taskmanager-
|
|
23
|
-
- "taskmanager-
|
|
20
|
+
- "jobmanager:9249"
|
|
21
|
+
- "taskmanager-a:9249"
|
|
22
|
+
- "taskmanager-b:9249"
|
|
23
|
+
- "taskmanager-c:9249"
|
|
24
24
|
|
|
25
25
|
- job_name: "clickhouse"
|
|
26
26
|
metrics_path: "/metrics"
|
|
@@ -106,27 +106,27 @@ stacks:
|
|
|
106
106
|
flink:
|
|
107
107
|
file: "compose-flink.yml"
|
|
108
108
|
description: "Stateful, real-time stream processing engine. Lite uses 1 TaskManager; Full uses 3 TaskManagers."
|
|
109
|
-
profiles: ["
|
|
109
|
+
profiles: ["flink-lite", "flink-full"]
|
|
110
110
|
capacities: ["stream_processing"]
|
|
111
111
|
depends_on:
|
|
112
|
-
|
|
113
|
-
|
|
112
|
+
flink-lite: ["deps", "storage", "catalog"]
|
|
113
|
+
flink-full: ["deps", "storage", "catalog"]
|
|
114
114
|
role: "Executes continuous streaming pipelines. Lite uses 1 TaskManager; Full uses 3 TaskManagers."
|
|
115
115
|
usage:
|
|
116
|
-
|
|
116
|
+
flink-lite: |
|
|
117
117
|
• Flink JobManager UI (1 TaskManager):
|
|
118
118
|
- External (Localhost): http://localhost:8082
|
|
119
|
-
- Internal (Docker): http://jobmanager
|
|
119
|
+
- Internal (Docker): http://jobmanager:8081
|
|
120
120
|
• SQL Gateway REST:
|
|
121
121
|
- External (Localhost): http://localhost:8084
|
|
122
|
-
- Internal (Docker): http://flink-sql-gateway
|
|
123
|
-
|
|
122
|
+
- Internal (Docker): http://flink-sql-gateway:8083
|
|
123
|
+
flink-full: |
|
|
124
124
|
• Flink JobManager UI (3 TaskManagers):
|
|
125
125
|
- External (Localhost): http://localhost:8082
|
|
126
|
-
- Internal (Docker): http://jobmanager
|
|
126
|
+
- Internal (Docker): http://jobmanager:8081
|
|
127
127
|
• SQL Gateway REST:
|
|
128
128
|
- External (Localhost): http://localhost:8084
|
|
129
|
-
- Internal (Docker): http://flink-sql-gateway
|
|
129
|
+
- Internal (Docker): http://flink-sql-gateway:8083
|
|
130
130
|
|
|
131
131
|
spark:
|
|
132
132
|
file: "compose-spark.yml"
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|