odctl 0.1.0__tar.gz

odctl-0.1.0/.github/ISSUE_TEMPLATE/bug_report.md

@@ -0,0 +1,46 @@
+---
+name: Bug Report
+about: Create a report to help us improve the Open Data Stack
+title: "[BUG] "
+labels: bug
+assignees: ""
+---
+
+**Describe the bug**
+
+A clear and concise description of what the bug is.
+
+**To Reproduce**
+
+Steps to reproduce the behavior:
+
+1. What profiles were you trying to launch? (e.g., `dml up airflow spark ckh1`)
+2. Run this CLI command: '...'
+3. See error
+
+**Expected behavior**
+
+A clear and concise description of what you expected to happen.
+
+**Traceback / Error Logs**
+
+If applicable, paste the full CLI error or the specific container logs here (you can fetch container logs using `dml logs <profile> -s <service>`):
+
+```text
+# Paste logs here
+
+
+```
+
+**Environment (please complete the following information):**
+
+* OS: [e.g. Ubuntu 24.04, macOS Sonoma, Windows 11 / WSL2]
+* Python Version: [e.g. 3.12]
+* `dml-cli` Version: [e.g. 0.0.3]
+* Docker Version: [e.g. Docker Desktop 4.28.0, Docker Engine 25.0]
+
+## **Additional context**
+
+* Did you customize any files in your local `./.dml/` workspace? (e.g., changing ports in `compose-orch.yml` or editing `registry.yml`)
+* How much RAM/CPU is allocated to your Docker daemon? (Many Open Data profiles are resource-heavy)
+* Add any other context about the problem here.

odctl-0.1.0/.github/ISSUE_TEMPLATE/feature_request.md

@@ -0,0 +1,44 @@
+---
+name: Feature Request
+about: Suggest an idea for the Open Data Stack
+title: "[FEATURE] "
+labels: enhancement
+assignees: ""
+---
+
+**Is your feature request related to a problem? Please describe.**
+
+A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
+
+**Describe the solution you'd like**
+
+A clear and concise description of what you want to happen. (e.g., "I want a new profile added to `compose-orch.yml` for Dagster", or "Add a dedicated connector script for MongoDB CDC.")
+
+**Describe alternatives you've considered**
+
+A clear and concise description of any alternative solutions or workarounds you've tried using the current stack.
+
+**Example Use Case (Optional but helpful)**
+
+Show a tiny snippet of how you would want the CLI or registry to look if this feature existed:
+
+```bash
+# Example of the CLI command you wish existed
+dml up dagster --pull
+
+```
+
+*or*
+
+```yaml
+# Example addition to registry.yml
+  dagster:
+    file: "compose-orch.yml"
+    profiles: ["dagster"]
+
+```
+
+## **Additional context**
+
+* Does this feature require any specific base infrastructure (like Postgres or SeaweedFS) to operate?
+* Add any other context, architectural diagrams, or links to the requested tool's documentation here.

odctl-0.1.0/.github/renovate.json

@@ -0,0 +1,70 @@
+{
+  "$schema": "https://docs.renovatebot.com/renovate-schema.json",
+  "extends": ["config:base", ":dependencyDashboard", "docker:enableMajor"],
+  "schedule": ["every weekend"],
+  "docker-compose": {
+    "fileMatch": ["(^|/)compose-[^/]+\\.ya?ml$"]
+  },
+  "dockerfile": {
+    "fileMatch": [
+      "^src/dml/resources/docker/.*/Dockerfile$",
+      "^src/dml/resources/deps/Dockerfile$"
+    ]
+  },
+  "packageRules": [
+    {
+      "description": "Auto-merge non-breaking minor and patch updates if CI passes",
+      "matchUpdateTypes": ["minor", "patch"],
+      "automerge": true,
+      "automergeType": "pr"
+    },
+    {
+      "description": "Group all Python dependencies from pyproject.toml into a single PR",
+      "matchDatasources": ["pypi"],
+      "groupName": "Python Packages"
+    },
+    {
+      "description": "Group all Apache Airflow Providers into a single PR",
+      "matchPackagePatterns": ["^apache-airflow-providers-"],
+      "groupName": "Airflow Providers"
+    },
+    {
+      "description": "Group Spark and PySpark updates together to prevent Py4J mismatches",
+      "matchPackageNames": ["apache/spark", "pyspark"],
+      "groupName": "Spark Ecosystem"
+    },
+    {
+      "description": "Group Observability stack updates together",
+      "matchPackageNames": [
+        "prom/prometheus",
+        "prom/alertmanager",
+        "grafana/grafana"
+      ],
+      "groupName": "Observability Stack"
+    },
+    {
+      "description": "Group ClickHouse Server and Keeper to maintain cluster consensus",
+      "matchPackageNames": [
+        "clickhouse/clickhouse-server",
+        "clickhouse/clickhouse-keeper"
+      ],
+      "groupName": "ClickHouse Ecosystem"
+    },
+    {
+      "description": "Group OpenMetadata Server and Ingestion to maintain API compatibility",
+      "matchPackageNames": [
+        "docker.getcollate.io/openmetadata/server",
+        "docker.getcollate.io/openmetadata/ingestion"
+      ],
+      "groupName": "OpenMetadata Ecosystem"
+    },
+    {
+      "description": "Group Marquez API and Web UI together",
+      "matchPackageNames": [
+        "marquezproject/marquez",
+        "marquezproject/marquez-web"
+      ],
+      "groupName": "Lineage Ecosystem"
+    }
+  ]
+}

odctl-0.1.0/.github/workflows/build-deps.yml

@@ -0,0 +1,113 @@
+name: Build Open Data Dependencies Image
+
+on:
+  workflow_call:
+
+env:
+  REGISTRY: ghcr.io
+  IMAGE_NAME: ${{ github.repository }}/deps
+
+jobs:
+  build:
+    name: Build (${{ matrix.platform.arch }})
+    runs-on: ${{ matrix.platform.os }}
+    permissions:
+      contents: read
+      packages: write
+    strategy:
+      fail-fast: false
+      matrix:
+        platform:
+          - os: ubuntu-latest
+            arch: linux/amd64
+            name: amd64
+          - os: ubuntu-24.04-arm
+            arch: linux/arm64
+            name: arm64
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Log in to the Container registry
+        uses: docker/login-action@v3
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Extract Docker metadata
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+
+      - name: Build and push by digest
+        id: build
+        uses: docker/build-push-action@v5
+        with:
+          context: src/odctl/resources/deps
+          platforms: ${{ matrix.platform.arch }}
+          labels: ${{ steps.meta.outputs.labels }}
+          outputs: type=image,name=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }},push-by-digest=true,name-canonical=true,push=${{ startsWith(github.ref, 'refs/tags/v') }}
+          cache-from: type=registry,ref=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:buildcache-${{ matrix.platform.name }}
+          cache-to: type=registry,ref=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:buildcache-${{ matrix.platform.name }},mode=max
+
+      - name: Export digest
+        if: startsWith(github.ref, 'refs/tags/v')
+        run: |
+          mkdir -p /tmp/digests
+          digest="${{ steps.build.outputs.digest }}"
+          touch "/tmp/digests/${digest#sha256:}"
+
+      - name: Upload digest
+        if: startsWith(github.ref, 'refs/tags/v')
+        uses: actions/upload-artifact@v4
+        with:
+          name: digests-deps-${{ matrix.platform.name }}
+          path: /tmp/digests/*
+          if-no-files-found: error
+          retention-days: 1
+
+  merge:
+    name: Merge & Tag
+    needs: build
+    if: startsWith(github.ref, 'refs/tags/v')
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+    steps:
+      - name: Download digests
+        uses: actions/download-artifact@v4
+        with:
+          pattern: digests-deps-*
+          path: /tmp/digests
+          merge-multiple: true
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Log in to the Container registry
+        uses: docker/login-action@v3
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Extract Docker metadata
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+          tags: |
+            type=semver,pattern={{version}}
+            type=raw,value=latest
+
+      - name: Create manifest list and push
+        working-directory: /tmp/digests
+        run: |
+          docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \
+            $(printf '${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}@sha256:%s ' *)

odctl-0.1.0/.github/workflows/build-platform-images.yml

@@ -0,0 +1,117 @@
+name: Build Platform Images
+
+on:
+  workflow_call:
+
+env:
+  REGISTRY: ghcr.io
+
+jobs:
+  build:
+    name: Build (${{ matrix.component }}, ${{ matrix.platform.arch }})
+    runs-on: ${{ matrix.platform.os }}
+    permissions:
+      contents: read
+      packages: write
+    strategy:
+      fail-fast: false
+      matrix:
+        component: [airflow, spark, ray-mlops]
+        platform:
+          - os: ubuntu-latest
+            arch: linux/amd64
+            name: amd64
+          - os: ubuntu-24.04-arm
+            arch: linux/arm64
+            name: arm64
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Log in to the Container registry
+        uses: docker/login-action@v3
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Extract Docker metadata
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: ${{ env.REGISTRY }}/${{ github.repository }}/${{ matrix.component }}
+
+      - name: Build and push by digest
+        id: build
+        uses: docker/build-push-action@v5
+        with:
+          context: src/odctl/resources/docker/${{ matrix.component }}
+          platforms: ${{ matrix.platform.arch }}
+          labels: ${{ steps.meta.outputs.labels }}
+          outputs: type=image,name=${{ env.REGISTRY }}/${{ github.repository }}/${{ matrix.component }},push-by-digest=true,name-canonical=true,push=${{ startsWith(github.ref, 'refs/tags/v') }}
+          cache-from: type=registry,ref=${{ env.REGISTRY }}/${{ github.repository }}/${{ matrix.component }}:buildcache-${{ matrix.platform.name }}
+          cache-to: type=registry,ref=${{ env.REGISTRY }}/${{ github.repository }}/${{ matrix.component }}:buildcache-${{ matrix.platform.name }},mode=max
+
+      - name: Export digest
+        if: startsWith(github.ref, 'refs/tags/v')
+        run: |
+          mkdir -p /tmp/digests
+          digest="${{ steps.build.outputs.digest }}"
+          touch "/tmp/digests/${digest#sha256:}"
+
+      - name: Upload digest
+        if: startsWith(github.ref, 'refs/tags/v')
+        uses: actions/upload-artifact@v4
+        with:
+          name: digests-${{ matrix.component }}-${{ matrix.platform.name }}
+          path: /tmp/digests/*
+          if-no-files-found: error
+          retention-days: 1
+
+  merge:
+    name: Merge & Tag (${{ matrix.component }})
+    needs: build
+    if: startsWith(github.ref, 'refs/tags/v')
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+    strategy:
+      fail-fast: false
+      matrix:
+        component: [airflow, spark, ray-mlops]
+    steps:
+      - name: Download digests
+        uses: actions/download-artifact@v4
+        with:
+          pattern: digests-${{ matrix.component }}-*
+          path: /tmp/digests
+          merge-multiple: true
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Log in to the Container registry
+        uses: docker/login-action@v3
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Extract Docker metadata
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: ${{ env.REGISTRY }}/${{ github.repository }}/${{ matrix.component }}
+          tags: |
+            type=semver,pattern={{version}}
+            type=raw,value=latest
+
+      - name: Create manifest list and push
+        working-directory: /tmp/digests
+        run: |
+          docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \
+            $(printf '${{ env.REGISTRY }}/${{ github.repository }}/${{ matrix.component }}@sha256:%s ' *)

odctl-0.1.0/.github/workflows/ci.yml

@@ -0,0 +1,39 @@
+name: CI
+
+on:
+  workflow_call:
+  workflow_dispatch:
+
+
+jobs:
+  test:
+    name: Python ${{ matrix.python-version }}
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"]
+    steps:
+      - name: Checkout source code
+        uses: actions/checkout@v7
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v7
+        with:
+          enable-cache: true
+
+      - name: Set up Python ${{ matrix.python-version }}
+        run: uv python install ${{ matrix.python-version }}
+
+      - name: Run pre-commit
+        uses: pre-commit/action@v3.0.1
+
+      - name: Run Tests
+        # Target your test folder and output coverage for src/odctl
+        run: uv run --with pytest-cov pytest tests/ --cov=src/odctl --cov-report=xml
+
+      # - name: Upload coverage reports to Codecov
+      #   uses: codecov/codecov-action@v4
+      #   with:
+      #     token: ${{ secrets.CODECOV_TOKEN }}
+      #     file: ./coverage.xml

odctl-0.1.0/.github/workflows/e2e-test.yml

@@ -0,0 +1,83 @@
+name: End-to-End Integration Tests
+
+on:
+  workflow_call:
+
+
+
+jobs:
+  e2e:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: read
+    strategy:
+      matrix:
+        profile_group:
+          - "metadata"
+          - "airflow"
+          - "kafka-lite kafka-full"
+          - "flink1-lite flink1-full fluss"
+          - "spark-lite spark-full"
+          - "ch-lite ch-full"
+          - "trino metabase"
+          - "mlflow ray-serve"
+          - "lineage telemetry"
+          - "postgres storage catalog valkey"
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v4
+
+      - name: Set up Python
+        run: uv python install 3.10
+
+      - name: Install odctl
+        run: uv tool install .
+
+      - name: Initialize odctl workspace
+        run: yes | odctl init --force
+
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Test Profile Group
+        run: |
+          # Split the space-separated group into an array
+          for profile in ${{ matrix.profile_group }}; do
+            echo "::group::Testing Profile: $profile"
+            echo "🚀 Launching $profile..."
+            odctl up $profile
+
+            echo "⏳ Validating containers..."
+            SUCCESS=0
+            for i in {1..15}; do
+              RUNNING=$(docker ps -q | wc -l)
+              if [ "$RUNNING" -ge 1 ]; then
+                echo "✅ Containers are running!"
+                docker ps
+                odctl ps --all
+                SUCCESS=1
+                break
+              fi
+              echo "Waiting... ($i/15)"
+              sleep 2
+            done
+
+            if [ "$SUCCESS" -eq 0 ]; then
+              echo "❌ Timeout reached: No containers are running for $profile!"
+              docker ps -a
+              odctl ps --all
+              exit 1
+            fi
+
+            echo "🧹 Tearing down $profile..."
+            yes | odctl down --all --volumes
+            echo "::endgroup::"
+          done

odctl-0.1.0/.github/workflows/pipeline.yml

@@ -0,0 +1,34 @@
+name: Main Pipeline
+
+on:
+  push:
+    branches: ["main"]
+    tags: ["v*"]
+  pull_request:
+    branches: ["main"]
+  workflow_dispatch:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+permissions:
+  contents: read
+  packages: write
+
+jobs:
+  ci:
+    uses: ./.github/workflows/ci.yml
+
+  build-platform:
+    needs: ci
+    uses: ./.github/workflows/build-platform-images.yml
+
+  build-deps:
+    needs: ci
+    uses: ./.github/workflows/build-deps.yml
+
+  e2e:
+    needs: [build-platform, build-deps]
+    if: startsWith(github.ref, 'refs/tags/v') || github.event_name == 'workflow_dispatch'
+    uses: ./.github/workflows/e2e-test.yml

odctl-0.1.0/.github/workflows/publish.yml

@@ -0,0 +1,34 @@
+name: Publish to PyPI
+
+on:
+  release:
+    types: [published] # ONLY triggers when a drafted release is published
+
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+jobs:
+  pypi-publish:
+    name: Build and publish Python package
+    runs-on: ubuntu-latest
+    # These permissions are strictly required for PyPI OIDC Trusted Publishing
+    permissions:
+      contents: read
+      id-token: write
+
+    steps:
+      - name: Checkout source code
+        uses: actions/checkout@v7
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v7
+
+      - name: Set up Python
+        run: uv python install 3.10
+
+      - name: Build Wheel and sdist
+        run: uv build
+
+      - name: Publish package distributions to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1