odctl 0.1.0__tar.gz → 0.2.0__tar.gz

{odctl-0.1.0 → odctl-0.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: odctl
-Version: 0.1.0
+Version: 0.2.0
 Summary: Orchestrator for the Open Data Stack
 Project-URL: Homepage, https://github.com/jaehyeon-kim/odctl
 Project-URL: Repository, https://github.com/jaehyeon-kim/odctl

{odctl-0.1.0 → odctl-0.2.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "odctl"
-version = "0.1.0"
+version = "0.2.0"
 description = "Orchestrator for the Open Data Stack"
 readme = "README.md"
 authors = [

{odctl-0.1.0 → odctl-0.2.0}/src/odctl/config.py

@@ -1,5 +1,15 @@
 import sys
 from pathlib import Path
+from typing import Optional
+
+# Workspace directory requested via the global `--workspace` flag (set in main.py).
+_workspace_override: Optional[Path] = None
+
+
+def set_workspace_override(path: Optional[Path]) -> None:
+    """Record the workspace directory requested via the global `--workspace` flag."""
+    global _workspace_override
+    _workspace_override = path.resolve() if path is not None else None
 
 
 def get_internal_resources_dir() -> Path:
@@ -25,8 +35,11 @@ def get_workspace_dir() -> Path:
     Get the path to the local mutable workspace.
 
     Returns:
-        Path: The absolute path to the `.odctl` directory in the current working directory.
+        Path: The directory supplied via `--workspace`, or the `.odctl` directory in
+        the current working directory when no override is set.
     """
+    if _workspace_override is not None:
+        return _workspace_override
     return Path.cwd() / ".odctl"
 
 

{odctl-0.1.0 → odctl-0.2.0}/src/odctl/main.py

@@ -1,9 +1,10 @@
+import logging
 from pathlib import Path
 from typing import List, Optional
 
 import typer
 
-from odctl import ui
+from odctl import config, ui
 from odctl.docker import (
     get_managed_containers,
     get_managed_logs,
@@ -51,14 +52,20 @@ def main(
         help="Enable debug-level logging across all commands.",
         rich_help_panel="Global Options",
     ),
-    workspace: Path = typer.Option(
-        "./.odctl",
+    workspace: Optional[Path] = typer.Option(
+        None,
         "--workspace",
         "-w",
-        help="Path to the ODCTL workspace directory.",
+        help="Path to the ODCTL workspace directory (default: ./.odctl).",
         rich_help_panel="Global Options",
     ),
 ):
+    logging.basicConfig(
+        level=logging.DEBUG if verbose else logging.WARNING,
+        format="%(levelname)s %(name)s: %(message)s",
+        force=True,
+    )
+    config.set_workspace_override(workspace)
     ctx.obj = {"verbose": verbose, "workspace": workspace}
 
 
@@ -183,7 +190,9 @@ def init(
             "⚠️  This will wipe out local modifications. Are you sure?", abort=True
         )
     init_workspace(force=force)
-    ui.print_success("Workspace initialized! You can now edit any file in ./.odctl/")
+    ui.print_success(
+        f"Workspace initialized! You can now edit any file in {get_workspace_dir()}/"
+    )
 
 
 @app.command(

{odctl-0.1.0 → odctl-0.2.0}/src/odctl/resources/clickhouse/config.xml

@@ -9,6 +9,13 @@
     <background_pool_size>16</background_pool_size>
     <background_schedule_pool_size>8</background_schedule_pool_size>
 
+    <!-- Docker networks hand out IPv6 addresses, and container names can resolve
+         to the AAAA record only, so listening on IPv4 alone leaves peers with
+         connection refused. :: is dual-stack and covers IPv4 too, so it comes
+         first; the 0.0.0.0 entry is a fallback for hosts without IPv6, where
+         the :: bind fails harmlessly thanks to listen_try. -->
+    <listen_try>1</listen_try>
+    <listen_host>::</listen_host>
     <listen_host>0.0.0.0</listen_host>
 
     <zookeeper>

{odctl-0.1.0 → odctl-0.2.0}/src/odctl/resources/clickhouse/keeper_config.xml

@@ -4,6 +4,13 @@
         <level>information</level>
         <console>1</console>
     </logger>
+    <!-- Docker networks hand out IPv6 addresses, and container names can resolve
+         to the AAAA record only, so listening on IPv4 alone leaves peers with
+         connection refused. :: is dual-stack and covers IPv4 too, so it comes
+         first; the 0.0.0.0 entry is a fallback for hosts without IPv6, where
+         the :: bind fails harmlessly thanks to listen_try. -->
+    <listen_try>1</listen_try>
+    <listen_host>::</listen_host>
     <listen_host>0.0.0.0</listen_host>
     <keeper_server>
         <tcp_port>9181</tcp_port>

{odctl-0.1.0 → odctl-0.2.0}/src/odctl/resources/compose-analytics.yml

@@ -1,7 +1,7 @@
 name: odctl-analytics
 
 x-clickhouse-base: &clickhouse-base
-  image: clickhouse/clickhouse-server:24.3-alpine
+  image: clickhouse/clickhouse-server:25.8-alpine
   networks:
     - odctl
   mem_limit: 4g
@@ -32,7 +32,7 @@ services:
   # [SHARED] Consensus Node
   # ==========================================
   ch-keeper:
-    image: clickhouse/clickhouse-keeper:24.3-alpine
+    image: clickhouse/clickhouse-keeper:25.8-alpine
     container_name: ch-keeper
     # Keeper must run in BOTH profiles
     profiles:
@@ -83,6 +83,7 @@ services:
       - ./clickhouse/config.xml:/etc/clickhouse-server/config.d/cluster-config.xml
       - ./clickhouse/users.xml:/etc/clickhouse-server/users.d/custom-users.xml
       - ./clickhouse/macros-11.xml:/etc/clickhouse-server/config.d/macros.xml
+      - ./clickhouse/init.sql:/docker-entrypoint-initdb.d/init.sql:ro
 
   ch-12:
     <<: *clickhouse-base
@@ -95,6 +96,7 @@ services:
       - ./clickhouse/config.xml:/etc/clickhouse-server/config.d/cluster-config.xml
       - ./clickhouse/users.xml:/etc/clickhouse-server/users.d/custom-users.xml
       - ./clickhouse/macros-12.xml:/etc/clickhouse-server/config.d/macros.xml
+      - ./clickhouse/init.sql:/docker-entrypoint-initdb.d/init.sql:ro
 
   # ==========================================
   # Shard 2 (Active ONLY in multi profile)
@@ -110,6 +112,7 @@ services:
       - ./clickhouse/config.xml:/etc/clickhouse-server/config.d/cluster-config.xml
       - ./clickhouse/users.xml:/etc/clickhouse-server/users.d/custom-users.xml
       - ./clickhouse/macros-21.xml:/etc/clickhouse-server/config.d/macros.xml
+      - ./clickhouse/init.sql:/docker-entrypoint-initdb.d/init.sql:ro
 
   ch-22:
     <<: *clickhouse-base
@@ -122,68 +125,7 @@ services:
       - ./clickhouse/config.xml:/etc/clickhouse-server/config.d/cluster-config.xml
       - ./clickhouse/users.xml:/etc/clickhouse-server/users.d/custom-users.xml
       - ./clickhouse/macros-22.xml:/etc/clickhouse-server/config.d/macros.xml
-
-  # ==========================================
-  # [INIT: ch-lite] Database Initialization
-  # ==========================================
-  ch-init-lite:
-    image: clickhouse/clickhouse-server:24.3-alpine
-    container_name: ch-init-lite
-    profiles: ["ch-lite"]
-    networks:
-      - odctl
-    depends_on:
-      ch-11:
-        condition: service_healthy
-      ch-12:
-        condition: service_healthy
-    entrypoint: >
-      /bin/sh -c "
-      echo '[ch-lite] Creating databases on Shard 1...' &&
-      clickhouse-client --host ch-11 --user default --password password --query 'CREATE DATABASE IF NOT EXISTS feature_store;' &&
-      clickhouse-client --host ch-11 --user default --password password --query 'CREATE DATABASE IF NOT EXISTS odctl;' &&
-      clickhouse-client --host ch-12 --user default --password password --query 'CREATE DATABASE IF NOT EXISTS feature_store;' &&
-      clickhouse-client --host ch-12 --user default --password password --query 'CREATE DATABASE IF NOT EXISTS odctl;' &&
-      echo '✅ [ch-lite] Databases created successfully!' &&
-      sleep infinity
-      "
-
-  # ==========================================
-  # [INIT: ch-full] Database Initialization
-  # ==========================================
-  ch-init-full:
-    image: clickhouse/clickhouse-server:24.3-alpine
-    container_name: ch-init-full
-    profiles: ["ch-full"]
-    networks:
-      - odctl
-    depends_on:
-      ch-11:
-        condition: service_healthy
-      ch-12:
-        condition: service_healthy
-      ch-21:
-        condition: service_healthy
-      ch-22:
-        condition: service_healthy
-    entrypoint: >
-      /bin/sh -c "
-      echo '[ch-full] Creating databases on Shard 1...' &&
-      clickhouse-client --host ch-11 --user default --password password --query 'CREATE DATABASE IF NOT EXISTS feature_store;' &&
-      clickhouse-client --host ch-11 --user default --password password --query 'CREATE DATABASE IF NOT EXISTS odctl;' &&
-      clickhouse-client --host ch-12 --user default --password password --query 'CREATE DATABASE IF NOT EXISTS feature_store;' &&
-      clickhouse-client --host ch-12 --user default --password password --query 'CREATE DATABASE IF NOT EXISTS odctl;' &&
-
-      echo '[ch-full] Creating databases on Shard 2...' &&
-      clickhouse-client --host ch-21 --user default --password password --query 'CREATE DATABASE IF NOT EXISTS feature_store;' &&
-      clickhouse-client --host ch-21 --user default --password password --query 'CREATE DATABASE IF NOT EXISTS odctl;' &&
-      clickhouse-client --host ch-22 --user default --password password --query 'CREATE DATABASE IF NOT EXISTS feature_store;' &&
-      clickhouse-client --host ch-22 --user default --password password --query 'CREATE DATABASE IF NOT EXISTS odctl;' &&
-
-      echo '✅ [ch-full] All databases created across the cluster!' &&
-      sleep infinity
-      "
-
+      - ./clickhouse/init.sql:/docker-entrypoint-initdb.d/init.sql:ro
 
   # ==========================================
   # [PROFILE: trino] Federated SQL Engine

{odctl-0.1.0 → odctl-0.2.0}/src/odctl/resources/compose-flink.yml

@@ -9,18 +9,18 @@ x-flink-common: &flink-common
 
 x-flink1-base: &flink1-base
   <<: *flink-common
-  image: flink:1.20.4-java17
+  image: flink:1.20.5-java17
   environment:
     TZ: ${TZ:-UTC}
   entrypoint:
     - /bin/bash
     - -c
     - |
-      # Load dependencies
-      mkdir -p /opt/flink/usrlib
-      cp /mnt/shared-deps/shared/*.jar /opt/flink/usrlib/ 2>/dev/null || true
-      cp /mnt/shared-deps/flink/1.x/*.jar /opt/flink/usrlib/ 2>/dev/null || true
-      cp /mnt/shared-deps/connect/confluent-converters/*.jar /opt/flink/usrlib/ 2>/dev/null || true
+      # Load dependencies into lib/, not usrlib/. Only lib/ reaches the system
+      # classloader, which is where SQL connector factories are looked up.
+      cp /mnt/shared-deps/shared/*.jar /opt/flink/lib/ 2>/dev/null || true
+      cp /mnt/shared-deps/flink/1.x/*.jar /opt/flink/lib/ 2>/dev/null || true
+      cp /mnt/shared-deps/connect/confluent-converters/*.jar /opt/flink/lib/ 2>/dev/null || true
 
       # S3 Plugin Linking
       echo "Linking S3 FileSystem Plugin..."

{odctl-0.1.0 → odctl-0.2.0}/src/odctl/resources/compose-kafka.yml

@@ -39,6 +39,14 @@ services:
       KAFKA_CONTROLLER_QUORUM_VOTERS: "1@kafka:19091"
       KAFKA_NUM_PARTITIONS: "3"
       KAFKA_DEFAULT_REPLICATION_FACTOR: "1"
+      # Internal topics default to a replication factor of 3, which a single
+      # broker can never satisfy. Without these, __consumer_offsets is never
+      # created and every consumer group waits on the coordinator forever.
+      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: "1"
+      KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: "1"
+      KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: "1"
+      KAFKA_SHARE_COORDINATOR_STATE_TOPIC_REPLICATION_FACTOR: "1"
+      KAFKA_SHARE_COORDINATOR_STATE_TOPIC_MIN_ISR: "1"
       <<: *kafka_broker_common_env_vars
     mem_limit: 1500m
 

{odctl-0.1.0 → odctl-0.2.0}/src/odctl/resources/compose-spark.yml

@@ -12,6 +12,12 @@ x-spark-base: &spark-base
   environment:
     TZ: ${TZ:-UTC}
     SPARK_DIST_CLASSPATH: "/mnt/shared-deps/shared/*:/mnt/shared-deps/spark/*"
+    # Picked up by boto3 and by any AWS SDK client that falls back to the
+    # default credential chain
+    AWS_REGION: us-east-1
+    AWS_ACCESS_KEY_ID: user
+    AWS_SECRET_ACCESS_KEY: password
+    SPARK_WORKER_MEMORY: "1500m"
 
 services:
   spark-master:
@@ -23,8 +29,6 @@ services:
       - "9080:8080"
       - "7077:7077"
     mem_limit: 1g
-    environment:
-      SPARK_WORKER_MEMORY: "1500m"
     healthcheck:
       test: ["CMD", "curl", "-f", "http://localhost:8080"]
       interval: 10s
@@ -39,8 +43,6 @@ services:
     ports:
       - "9081:8081"
     mem_limit: 2g
-    environment:
-      SPARK_WORKER_MEMORY: "1500m"
     depends_on:
       spark-master:
         condition: service_healthy
@@ -53,8 +55,6 @@ services:
     ports:
       - "9082:8081"
     mem_limit: 2g
-    environment:
-      SPARK_WORKER_MEMORY: "1500m"
     depends_on:
       spark-master:
         condition: service_healthy
@@ -67,8 +67,6 @@ services:
     ports:
       - "9083:8081"
     mem_limit: 2g
-    environment:
-      SPARK_WORKER_MEMORY: "1500m"
     depends_on:
       spark-master:
         condition: service_healthy

{odctl-0.1.0 → odctl-0.2.0}/src/odctl/resources/deps/fetch_flink_dependencies.sh

@@ -10,7 +10,7 @@ fetch_artifact() {
     else echo " ⬇️  Downloading: $2"; curl -sL -f -o "$1" "$2"; fi
 }
 get_maven_version() {
-    curl -sL "https://repo1.maven.org/maven2/${1}/maven-metadata.xml" | grep -Eo "<version>${2}</version>" | sort -V | tail -1 | sed 's/<\/\?version>//g' || true
+    curl -sL "https://repo1.maven.org/maven2/${1}/maven-metadata.xml" | grep -Eo "<version>${2}</version>" | sort -V | tail -1 | sed -E 's#</?version>##g' || true
 }
 
 echo "▶️  Resolving Flink Versions..."
@@ -34,6 +34,12 @@ K_V=$(get_maven_version "org/apache/flink/flink-sql-connector-kafka" "[0-9]+\.[0
 if [ -z "$K_V" ]; then echo "❌ Error: Kafka connector for Flink ${FLINK_MINOR} is not published yet!"; exit 1; fi
 fetch_artifact "flink/1.x/kafka.jar" "https://repo1.maven.org/maven2/org/apache/flink/flink-sql-connector-kafka/${K_V}/flink-sql-connector-kafka-${K_V}.jar"
 
+# Avro with Confluent Schema Registry. This uber JAR is a superset of
+# flink-sql-avro: it registers the plain 'avro' format factory as well as
+# 'avro-confluent' and 'debezium-avro-confluent', so adding both would duplicate
+# the Avro classes and register the same factory twice.
+fetch_artifact "flink/1.x/avro-confluent-registry.jar" "https://repo1.maven.org/maven2/org/apache/flink/flink-sql-avro-confluent-registry/${FLINK_V}/flink-sql-avro-confluent-registry-${FLINK_V}.jar"
+
 # JDBC
 J_V=$(get_maven_version "org/apache/flink/flink-connector-jdbc" "[0-9]+\.[0-9]+\.[0-9]+-${FLINK_MINOR}")
 if [ -z "$J_V" ]; then echo "❌ Error: JDBC connector for Flink ${FLINK_MINOR} is not published yet!"; exit 1; fi

{odctl-0.1.0 → odctl-0.2.0}/src/odctl/resources/deps/fetch_spark_dependencies.sh

@@ -10,7 +10,7 @@ fetch_artifact() {
     else echo " ⬇️  Downloading: $2"; curl -sL -f -o "$1" "$2"; fi
 }
 get_maven_version() {
-    curl -sL "https://repo1.maven.org/maven2/${1}/maven-metadata.xml" | grep -Eo "<version>${2}</version>" | sort -V | tail -1 | sed 's/<\/\?version>//g' || true
+    curl -sL "https://repo1.maven.org/maven2/${1}/maven-metadata.xml" | grep -Eo "<version>${2}</version>" | sort -V | tail -1 | sed -E 's#</?version>##g' || true
 }
 
 echo "▶️  Resolving Spark Versions..."

{odctl-0.1.0 → odctl-0.2.0}/src/odctl/resources/deps/fetch_standalone_plugins.sh

@@ -10,7 +10,7 @@ fetch_artifact() {
     else echo " ⬇️  Downloading: $2"; curl -sL -f -o "$1" "$2"; fi
 }
 get_maven_version() {
-    curl -sL "https://repo1.maven.org/maven2/${1}/maven-metadata.xml" | grep -Eo "<version>${2}</version>" | sort -V | tail -1 | sed 's/<\/\?version>//g' || true
+    curl -sL "https://repo1.maven.org/maven2/${1}/maven-metadata.xml" | grep -Eo "<version>${2}</version>" | sort -V | tail -1 | sed -E 's#</?version>##g' || true
 }
 
 echo "▶️  Resolving Shared & Connector Versions..."

{odctl-0.1.0 → odctl-0.2.0}/src/odctl/resources/docker/spark/Dockerfile

@@ -12,5 +12,9 @@ RUN --mount=type=cache,target=/root/.cache/pip \
     'mlflow~=3.12.0' \
     'boto3'
 
+# Pre-create the event log directory owned by the spark user. A named volume
+# mounted here inherits this ownership, so spark-submit can write its logs.
+RUN mkdir -p /tmp/spark-events && chown 185:185 /tmp/spark-events
+
 # Revert to the official Apache Spark non-root user ID
 USER 185

{odctl-0.1.0 → odctl-0.2.0}/src/odctl/resources/registry.yml

@@ -202,6 +202,8 @@ stacks:
         • Trino UI & JDBC connection (user: user, password: none):
           * External (Localhost): http://localhost:8080
           * Internal (Docker): http://trino:8080
+        • Catalogs: iceberg, kafka (JSON topics), kafka_avro (Avro via
+          the schema registry), clickhouse, postgres, valkey
       metabase: |
         • Metabase Dashboard Engine:
           * External (Localhost): http://localhost:3000

{odctl-0.1.0 → odctl-0.2.0}/src/odctl/resources/spark/spark-defaults.conf

@@ -17,6 +17,12 @@ spark.sql.catalog.iceberg.s3.endpoint  http://seaweed:8333
 spark.sql.catalog.iceberg.s3.path-style-access true
 spark.sql.defaultCatalog               iceberg
 
+# SeaweedFS ignores the region, but the AWS SDK refuses to build a client
+# without one, and the default credential chain finds nothing in a container.
+spark.sql.catalog.iceberg.s3.access-key-id      user
+spark.sql.catalog.iceberg.s3.secret-access-key  password
+spark.sql.catalog.iceberg.client.region         us-east-1
+
 # ========================================
 # History Server & Event Logging (Local Volume)
 # ========================================

{odctl-0.1.0 → odctl-0.2.0}/src/odctl/resources/trino/catalog/clickhouse.properties

@@ -1,4 +1,4 @@
 connector.name=clickhouse
-connection-url=jdbc:clickhouse://clickhouse-11:8123/feature_store
+connection-url=jdbc:clickhouse://ch-11:8123/feature_store
 connection-user=user
 connection-password=password

odctl-0.2.0/src/odctl/resources/trino/catalog/kafka.properties

@@ -0,0 +1,7 @@
+connector.name=kafka
+kafka.nodes=broker-1:19092
+kafka.default-schema=default
+kafka.table-names=web_events,logs
+# Expose _message, _key and the other underscore columns so raw JSON topics can
+# be read without a table description file
+kafka.hide-internal-columns=false

odctl-0.2.0/src/odctl/resources/trino/catalog/kafka_avro.properties

@@ -0,0 +1,10 @@
+connector.name=kafka
+kafka.nodes=broker-1:19092
+kafka.default-schema=default
+# Tables are discovered from the schema registry, so topics carrying Confluent
+# Avro appear without being listed and their fields arrive typed. This supplier
+# replaces file-based discovery rather than extending it, which is why plain JSON
+# topics live in the separate 'kafka' catalog.
+kafka.table-description-supplier=CONFLUENT
+kafka.confluent-schema-registry-url=http://karapace:8081
+kafka.hide-internal-columns=false

{odctl-0.1.0 → odctl-0.2.0}/src/odctl/workspace.py

@@ -1,6 +1,5 @@
 import shutil
 from importlib.metadata import PackageNotFoundError, version
-from pathlib import Path
 
 from odctl.config import INTERNAL_RESOURCES_DIR, get_workspace_dir
 
@@ -34,11 +33,11 @@ def init_workspace(force: bool = False):
     # Nuke the existing workspace if force is True
     if force and workspace.exists():
         shutil.rmtree(workspace)
-        print(f"🗑️  Removed existing workspace at {workspace.relative_to(Path.cwd())}/")
+        print(f"🗑️  Removed existing workspace at {workspace}/")
 
     if not workspace.exists():
         workspace.mkdir(parents=True)
-        print(f"📁 Created workspace at {workspace.relative_to(Path.cwd())}/")
+        print(f"📁 Created workspace at {workspace}/")
 
     # Iterate over everything in the internal resources
     for item in INTERNAL_RESOURCES_DIR.iterdir():

{odctl-0.1.0 → odctl-0.2.0}/tests/test_workspace.py

@@ -10,7 +10,6 @@ def test_init_workspace(tmp_path, monkeypatch):
     monkeypatch.setattr(
         "odctl.workspace.get_workspace_dir", lambda: tmp_path / ".odctl"
     )
-    monkeypatch.setattr("odctl.workspace.Path.cwd", lambda: tmp_path)
 
     # Mock INTERNAL_RESOURCES_DIR to point to a temporary internal dir
     internal_dir = tmp_path / "internal"

{odctl-0.1.0 → odctl-0.2.0}/uv.lock

@@ -387,7 +387,7 @@ wheels = [
 
 [[package]]
 name = "odctl"
-version = "0.0.10"
+version = "0.1.1"
 source = { editable = "." }
 dependencies = [
     { name = "pydantic" },