odctl 1.2.1__tar.gz → 1.3.0__tar.gz

{odctl-1.2.1 → odctl-1.3.0}/.github/renovate.json

@@ -77,6 +77,11 @@
     }
   ],
   "packageRules": [
+    {
+      "description": "Kafka stays below 4.4: Kafka 4.4.0 moved AppInfoParser to org.apache.kafka.common.utils.internals, and the Iceberg Kafka Connect sink still calls the old class, so its task fails on a 4.4 Connect worker. Raise this when an Iceberg release no longer uses it. See issue #161.",
+      "matchPackageNames": ["apache/kafka"],
+      "allowedVersions": "<4.4"
+    },
     {
       "description": "Flink stays on 2.2: it is the newest minor with Kafka and JDBC connector releases. Raise this when both connectors publish a build for the next minor, and move FLINK_MINOR in versions.env with it. See issue #9.",
       "matchPackageNames": ["flink", "org.apache.flink:{/,}**"],

{odctl-1.2.1 → odctl-1.3.0}/.github/scripts/select-profiles.py

@@ -85,7 +85,7 @@ ALSO_RUNS = {
 SNIPPET_PREFIX = "docs/snippets/"
 SNIPPET_GROUPS = {
     "getting-started": ["kafka-lite kafka-full"],
-    "how-it-works": ["storage", "catalog", "trino", "airflow"],
+    "how-it-works": ["storage", "catalog", "trino", "airflow", "metadata"],
     "kafka": ["kafka-lite kafka-full"],
     "kafka-connect": ["kafka-lite kafka-full"],
     "iceberg-ingestion": ["kafka-lite kafka-full", "flink-lite flink-full"],

{odctl-1.2.1 → odctl-1.3.0}/.github/scripts/smoke.sh

@@ -1094,7 +1094,7 @@ smoke_fluss() {
   pass "Flink wrote 3 rows to a Fluss table and read them back"
 }
 
-# OpenMetadata: the version endpoint answers through every failure the 2.0.1
+# OpenMetadata: the version endpoint answers through every failure an
 # upgrade could cause. The ingestion container runs its own Airflow 3, whose
 # health path moved to /api/v2/monitor/health, and a stale image or a failed
 # migration both leave a server that still reports a version. Drive the whole
@@ -1110,14 +1110,23 @@ smoke_metadata() {
   pass "schema migration completed"
   assert_metrics http://openmetadata-server:8586/prometheus
 
-  # The principal domain odctl sets is open-data.local, not the upstream
+  # The principal domain odctl sets is odctl.io, not the upstream
   # default. A wrong domain fails with a misleading invalid password error.
   local token
   token=$(om_step login 'echo "$TOKEN"' 2>/dev/null | tail -1)
-  [ -n "$token" ] || fail "admin@open-data.local could not log in"
+  [ -n "$token" ] || fail "admin@odctl.io could not log in"
   pass "admin logged in and received a JWT"
   local auth="Authorization: Bearer $token"
 
+  # user is the second admin. Its entry in AUTHORIZER_ADMIN_PRINCIPALS,
+  # user:password, sets its first password.
+  local code
+  code=$(curl -s -o /dev/null -w '%{http_code}' -H 'Content-Type: application/json' \
+    -d "{\"email\":\"user@odctl.io\",\"password\":\"$(printf password | base64)\"}" \
+    http://127.0.0.1:8585/api/v1/users/login)
+  [ "$code" = "200" ] || fail "user@odctl.io could not log in with the password password (HTTP $code)"
+  pass "user@odctl.io logged in with the password password"
+
   # Airflow 3 moved this path. The old /health returns 404 with a message saying so.
   retry 60 5 http_ok "http://127.0.0.1:8087/api/v2/monitor/health" \
     || fail "ingestion Airflow never answered on 8087"
@@ -1126,6 +1135,20 @@ smoke_metadata() {
     || fail "the server cannot reach the ingestion container, so no pipeline can deploy"
   pass "ingestion Airflow healthy and reachable from the server"
 
+  # AIRFLOW_ADMIN_USER and AIRFLOW_ADMIN_PASSWORD set this login. A new token
+  # answers 201, and a wrong login 401.
+  code=$(curl -s -o /dev/null -w '%{http_code}' -H 'Content-Type: application/json' \
+    -d '{"username":"user","password":"password"}' http://127.0.0.1:8087/auth/token)
+  [ "$code" = "201" ] || fail "the ingestion Airflow refused user / password (HTTP $code)"
+  pass "ingestion Airflow accepted user / password"
+
+  # openmetadata-init sets AutoPilot from METADATA_AUTOPILOT, false by default.
+  retry 24 5 om_autopilot_is "$token" False \
+    || fail "AutoPilot does not report active: false after the start"
+  [ "$(docker inspect -f '{{.State.ExitCode}}' openmetadata-init)" = "0" ] \
+    || fail "openmetadata-init did not exit with code 0"
+  pass "AutoPilot reports active: false and openmetadata-init exited 0"
+
   # A real ingestion, with the guide's steps. Postgres is already running, so it
   # is its own source.
   local svc
@@ -1142,23 +1165,11 @@ smoke_metadata() {
   # Airflow's dag_processor has to parse the new file before it can be
   # triggered, so the guide's trigger can come too early. Trigger again here.
   if ! grep -q 'has been triggered' <<<"$out"; then
-    local triggered=no
-    for _ in $(seq 1 24); do
-      if curl -fsS --max-time 90 -X POST -H "$auth" \
-          "http://127.0.0.1:8585/api/v1/services/ingestionPipelines/trigger/$pipe" 2>/dev/null \
-          | grep -q 'has been triggered'; then triggered=yes; break; fi
-      sleep 5
-    done
-    [ "$triggered" = "yes" ] || fail "the pipeline never triggered"
+    om_trigger "$token" "$pipe" || fail "the pipeline never triggered"
   fi
 
-  local state=""
-  for _ in $(seq 1 30); do
-    state=$(TOKEN="$token" om_step status 2>/dev/null \
-      | python3 -c 'import json,sys; s=json.load(sys.stdin).get("pipelineStatuses") or []; print(s[0]["pipelineState"] if s else "")' 2>/dev/null)
-    case "$state" in success|failed|partialSuccess) break ;; esac
-    sleep 10
-  done
+  local state
+  state=$(om_run_state "$token" om_step "")
   [ "$state" = "success" ] || fail "ingestion run finished in state ${state:-none}, not success"
   pass "ingestion run reached success"
 
@@ -1191,8 +1202,214 @@ smoke_metadata() {
   done
   pass "Context Center pages and memories both answer"
 
+  smoke_metadata_connector "$token"
+  smoke_metadata_autopilot "$token"
+
   TOKEN="$token" om_step remove >/dev/null 2>&1 || fail "the guide's remove step failed"
-  pass "the guide's remove step deleted the service and its pipeline"
+  code=$(curl -s -o /dev/null -w '%{http_code}' -H "$auth" \
+    http://127.0.0.1:8585/api/v1/services/databaseServices/name/demo_connector)
+  [ "$code" = "404" ] || fail "the guide's remove step left the demo_connector service (HTTP $code)"
+  retry 12 5 bash -c '! docker exec openmetadata-ingestion test -e /opt/airflow/connectors/demo_connector.py' \
+    || fail "the ingestion container kept the connector after the remove step deleted it from S3"
+  pass "the guide's remove step deleted both services, their pipelines and the uploaded connector"
+}
+
+# Trigger an ingestion pipeline until the server accepts it: Airflow has to parse
+# a new DAG file first. Usage: om_trigger <token> <pipeline id>
+om_trigger() {
+  local _
+  for _ in $(seq 1 24); do
+    if curl -fsS --max-time 90 -X POST -H "Authorization: Bearer $1" \
+        "http://127.0.0.1:8585/api/v1/services/ingestionPipelines/trigger/$2" 2>/dev/null \
+        | grep -q 'has been triggered'; then return 0; fi
+    sleep 5
+  done
+  return 1
+}
+
+# The latest run of the pipeline a guide's status step reads, as "<run id> <state>",
+# or nothing when there is no run yet. The step is the guide's own: om_step for
+# demo_pg, om_connector_step for the custom connector.
+# Usage: om_latest_run <token> <om_step|om_connector_step>
+om_latest_run() {
+  TOKEN="$1" "$2" status 2>/dev/null | python3 -c '
+import json, sys
+runs = json.load(sys.stdin).get("pipelineStatuses") or []
+if runs:
+    print(runs[0]["runId"], runs[0]["pipelineState"])' 2>/dev/null
+}
+
+# Wait for the first run whose id is not <previous run id> to finish, and print its
+# state. Usage: om_run_state <token> <om_step|om_connector_step> <previous run id>
+om_run_state() {
+  local _ run state
+  for _ in $(seq 1 30); do
+    read -r run state < <(om_latest_run "$1" "$2")
+    if [ -n "${run:-}" ] && [ "$run" != "$3" ]; then
+      case "$state" in success|failed|partialSuccess) echo "$state"; return ;; esac
+    fi
+    sleep 10
+  done
+  echo ""
+}
+
+# Whether the guide's autopilot step prints <value>. Usage: om_autopilot_is <token> <True|False>
+om_autopilot_is() {
+  [ "$(TOKEN="$1" om_step autopilot 2>/dev/null | tail -1)" = "$2" ]
+}
+
+# Add a service the way the UI does: create it, then trigger AutoPilotApplication with
+# the service's entityLink. Usage: om_add_service_like_the_ui <token> <service name>
+om_add_service_like_the_ui() {
+  local auth="Authorization: Bearer $1"
+  curl -fsS -X PUT -H "$auth" -H 'Content-Type: application/json' \
+    -d "{\"name\":\"$2\",\"serviceType\":\"CustomDatabase\",\"connection\":{\"config\":{\"type\":\"CustomDatabase\",\"sourcePythonClass\":\"demo_connector.DemoSource\"}}}" \
+    http://127.0.0.1:8585/api/v1/services/databaseServices >/dev/null || fail "could not create the $2 service"
+  curl -fsS -X POST -H "$auth" -H 'Content-Type: application/json' \
+    -d "{\"entityLink\":\"<#E::databaseService::$2>\"}" \
+    http://127.0.0.1:8585/api/v1/apps/trigger/AutoPilotApplication >/dev/null \
+    || fail "the server refused the AutoPilot trigger the UI sends"
+}
+
+# The number of agents a service has. Usage: om_agent_count <token> <service name>
+om_agent_count() {
+  curl -fsS -H "Authorization: Bearer $1" "http://127.0.0.1:8585/api/v1/services/ingestionPipelines?service=$2" \
+    | python3 -c 'import json,sys; print(json.load(sys.stdin)["paging"]["total"])'
+}
+
+om_delete_service() {
+  curl -s -o /dev/null -X DELETE -H "Authorization: Bearer $1" \
+    "http://127.0.0.1:8585/api/v1/services/databaseServices/name/$2?hardDelete=true&recursive=true"
+}
+
+# With AutoPilot off, a service added the way the UI adds one gets no agent. The
+# check waits 60 seconds, as long as the on-case below allows AutoPilot to create its agent.
+smoke_metadata_autopilot() {
+  local agents
+  om_add_service_like_the_ui "$1" demo_ui
+  sleep 60
+  agents=$(om_agent_count "$1" demo_ui)
+  om_delete_service "$1" demo_ui
+  [ "$agents" = "0" ] || fail "a service added the way the UI adds one got $agents agents with AutoPilot off"
+  pass "a service added the way the UI adds one, with its AutoPilot trigger, got no agent"
+}
+
+# The same two calls with AutoPilot on create at least one agent. It proves
+# METADATA_AUTOPILOT=true works and that the off-case above could have failed.
+smoke_metadata_autopilot_on() {
+  local agents=0 _
+  om_add_service_like_the_ui "$1" demo_ui_on
+  for _ in $(seq 1 12); do
+    agents=$(om_agent_count "$1" demo_ui_on)
+    [ "${agents:-0}" -ge 1 ] && break
+    sleep 5
+  done
+  om_delete_service "$1" demo_ui_on
+  [ "${agents:-0}" -ge 1 ] || fail "a service added the way the UI adds one got no agent with AutoPilot on"
+  pass "with METADATA_AUTOPILOT true, a service added the way the UI adds one got $agents agent(s)"
+}
+
+# The How It Works example turns AutoPilot on and recreates metadata. While it is on,
+# check the UI's calls create an agent, then put the key back and check it is off.
+how_it_works_autopilot() {
+  local out
+  out=$(run_snippet how-it-works/commands.sh metadata-autopilot 2>&1) || { echo "$out" | tail -20; fail "the metadata-autopilot example failed"; }
+  grep -qE -- 'AutoPilot active: true' <<<"$out" || { echo "$out" | tail -20; fail "the metadata-autopilot example printed no 'AutoPilot active: true'"; }
+  pass "How It Works' metadata-autopilot example printed 'AutoPilot active: true'"
+  smoke_metadata_autopilot_on "$(om_token)"
+  mv .odctl/.env.bak .odctl/.env || fail "the metadata-autopilot example left no .env.bak"
+  odctl recreate metadata >/dev/null 2>&1 || fail "could not recreate metadata after the metadata-autopilot example"
+  retry 24 5 om_autopilot_is "$(om_token)" False \
+    || fail "AutoPilot is not off again after the key was put back"
+  pass "AutoPilot is off again after the key was put back"
+}
+
+# Run a connector section until its output has <text>. Usage: om_connector_says <section> <text>
+om_connector_says() {
+  om_connector_step "$1" 2>/dev/null | grep -qF "$2"
+}
+
+# The guide's custom connector steps, run from the folder that holds the file, as
+# the guide tells the reader. Usage: om_connector_step <section> [extra commands]
+om_connector_step() {
+  local code
+  code=$(snippet openmetadata/connector/commands.sh "$1")
+  (cd "$CONNECTOR_DIR" && bash -o pipefail -c 'AUTH="Authorization: Bearer ${TOKEN:-}"
+'"$code"'
+'"${2:-}")
+}
+
+# A custom connector uploaded to s3://openmetadata/connectors is importable in the
+# ingestion container, and a service and an agent made through the API run it.
+# Then a changed connector is used by the next run with no restart, which is why
+# nothing here restarts the container: its start time must not move.
+smoke_metadata_connector() {
+  local token="$1" auth="Authorization: Bearer $1" started svc pipe out run state agents
+  CONNECTOR_DIR=$(mktemp -d)
+  cp "$SNIPPETS/openmetadata/connector/demo_connector.py" "$CONNECTOR_DIR/"
+  started=$(docker inspect -f '{{.State.StartedAt}}' openmetadata-ingestion)
+
+  om_connector_step upload >/dev/null 2>&1 || fail "could not upload the connector to s3://openmetadata/connectors"
+  retry 12 5 om_connector_says copied demo_connector.py \
+    || fail "the ingestion container never copied the connector from S3"
+  pass "the connector reached /opt/airflow/connectors in the ingestion container"
+
+  svc=$(TOKEN="$token" om_connector_step service 'echo "$SVC"' 2>/dev/null | tail -1)
+  [ -n "$svc" ] || fail "could not create the CustomDatabase service through the API"
+  out=$(TOKEN="$token" SVC="$svc" om_connector_step agent 'echo; echo "$PIPE"' 2>/dev/null)
+  pipe=$(tail -1 <<<"$out")
+  [ -n "$pipe" ] || fail "could not create the custom connector's agent"
+  grep -q 'has been created' <<<"$out" || { echo "$out"; fail "the custom connector's agent did not deploy a DAG"; }
+  grep -q 'has been triggered' <<<"$out" || om_trigger "$token" "$pipe" || fail "the custom connector's agent never triggered"
+  state=$(om_run_state "$token" om_connector_step "")
+  [ "$state" = "success" ] || fail "the custom connector's run finished in state ${state:-none}, not success"
+  out=$(TOKEN="$token" om_connector_step table 2>/dev/null)
+  grep -q '"fullyQualifiedName":"demo_connector.demo_db.public.orders"' <<<"$out" \
+    && grep -q '"name":"id"' <<<"$out" && grep -q '"name":"amount"' <<<"$out" \
+    || fail "the custom connector's table orders, with its columns id and amount, is not in the catalogue"
+  pass "the custom connector's agent ran and its table orders is in the catalogue"
+
+  # A service made through the API starts no AutoPilot agents. The UI does.
+  agents=$(curl -fsS -H "$auth" "http://127.0.0.1:8585/api/v1/services/ingestionPipelines?service=demo_connector" \
+    | python3 -c 'import json,sys; print(json.load(sys.stdin)["paging"]["total"])')
+  [ "$agents" = "1" ] || fail "the demo_connector service has $agents agents, not only its own"
+  pass "the demo_connector service has only its own agent"
+
+  # Change the connector, upload it again and trigger the same agent.
+  read -r run _ < <(om_latest_run "$token" om_connector_step)
+  om_connector_step change >/dev/null 2>&1 || fail "could not change the connector"
+  om_connector_step upload >/dev/null 2>&1 || fail "could not upload the changed connector"
+  retry 12 5 om_connector_says copied-change 1 \
+    || fail "the ingestion container never copied the changed connector"
+  TOKEN="$token" PIPE="$pipe" om_connector_step retrigger 2>/dev/null | grep -q 'has been triggered' \
+    || fail "the custom connector's agent did not trigger again"
+  state=$(om_run_state "$token" om_connector_step "$run")
+  [ "$state" = "success" ] || fail "the second run finished in state ${state:-none}, not success"
+  TOKEN="$token" om_connector_step changed-table 2>/dev/null \
+    | grep -q '"name":"orders_v2"' || fail "the changed connector's table orders_v2 is not in the catalogue"
+  [ "$(docker inspect -f '{{.State.StartedAt}}' openmetadata-ingestion)" = "$started" ] \
+    || fail "the ingestion container restarted while the connector changed"
+  pass "the changed connector was used by the next run, with no restart"
+  rm -rf "$CONNECTOR_DIR"
+}
+
+# The guide's runnable example needs demo_connector.py in the folder it runs in.
+smoke_metadata_example() {
+  local out
+  cp "$SNIPPETS/openmetadata/connector/demo_connector.py" .
+  out=$(bash "$SNIPPETS/openmetadata/example.sh" 2>&1) || { rm -f demo_connector.py; echo "$out" | tail -20; fail "the openmetadata example failed"; }
+  rm -f demo_connector.py
+  local want
+  for want in 'autopilot active: False' 'demo_pg ingestion: success' 'demo_connector ingestion: success' \
+      '^custom table: orders$' 'removed demo_pg and demo_connector'; do
+    grep -qE -- "$want" <<<"$out" || { echo "$out" | tail -20; fail "the openmetadata example printed no '$want'"; }
+  done
+  pass "the openmetadata guide's runnable example ran both services and printed its results"
+}
+
+# An admin token, from the guide's login step.
+om_token() {
+  om_step login 'echo "$TOKEN"' 2>/dev/null | tail -1
 }
 
 # The OpenMetadata guide's steps share shell variables: login sets TOKEN, the
@@ -1546,7 +1763,13 @@ case "$PROFILE" in
     run_example mlflow '"predictions": \[0, 0\]' "$(mlflow_runner) bash" ;;
   lineage)    smoke_lineage; run_example lineage 'COMPLETED' ;;
   telemetry)  smoke_telemetry; run_example telemetry '"42"' ;;
-  metadata)   smoke_metadata; run_example openmetadata 'removed demo_pg' ;;
+  metadata)
+    smoke_metadata
+    smoke_metadata_example
+    how_it_works_edit metadata-deps .odctl/.env metadata '^1\.4\.3$'
+    how_it_works_autopilot
+    retry 60 5 http_ok "http://127.0.0.1:8087/api/v2/monitor/health" \
+      || fail "ingestion Airflow did not answer after the package was put back" ;;
   fluss)      smoke_fluss; run_example fluss '\| +3 +\| +c +\|' ;;
   feast)
     smoke_feast

{odctl-1.2.1 → odctl-1.3.0}/.github/workflows/ci.yml

@@ -20,7 +20,7 @@ jobs:
       - name: Install uv
         # Pinned to a full version on purpose: setup-uv stopped publishing
         # moving major tags at v8, so astral-sh/setup-uv@v10 does not exist.
-        uses: astral-sh/setup-uv@v10.2.0
+        uses: astral-sh/setup-uv@v10.3.2
         with:
           enable-cache: true
           # The default key holds the runner's Python, not the matrix one, so

{odctl-1.2.1 → odctl-1.3.0}/.github/workflows/e2e-test.yml

@@ -67,7 +67,7 @@ jobs:
       - name: Install uv
         # Pinned to a full version on purpose: setup-uv stopped publishing
         # moving major tags at v8, so astral-sh/setup-uv@v10 does not exist.
-        uses: astral-sh/setup-uv@v10.2.0
+        uses: astral-sh/setup-uv@v10.3.2
 
       - name: Set up Python
         run: uv python install 3.10

{odctl-1.2.1 → odctl-1.3.0}/.github/workflows/pipeline.yml

@@ -48,7 +48,7 @@ jobs:
       # A tag runs every profile. A pull request runs only the groups its
       # changed files affect, because the full matrix pulls roughly 10 GB per
       # entry and paying that on every push is not worth the signal.
-      - uses: astral-sh/setup-uv@v10.2.0
+      - uses: astral-sh/setup-uv@v10.3.2
       - name: Choose profile groups
         id: profiles
         run: |
@@ -109,7 +109,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v7
-      - uses: astral-sh/setup-uv@v10.2.0
+      - uses: astral-sh/setup-uv@v10.3.2
         with:
           enable-cache: true
           cache-suffix: docs
@@ -132,7 +132,7 @@ jobs:
       - uses: actions/checkout@v7
         with:
           fetch-depth: 0 # mike needs the full history to update the gh-pages branch
-      - uses: astral-sh/setup-uv@v10.2.0
+      - uses: astral-sh/setup-uv@v10.3.2
       - name: Deploy the versioned docs
         run: |
           # Commit as the owner, so the gh-pages history adds no bot contributor.

{odctl-1.2.1 → odctl-1.3.0}/.github/workflows/publish.yml

@@ -125,7 +125,7 @@ jobs:
       - name: Install uv
         # Pinned to a full version on purpose: setup-uv stopped publishing
         # moving major tags at v8, so astral-sh/setup-uv@v10 does not exist.
-        uses: astral-sh/setup-uv@v10.2.0
+        uses: astral-sh/setup-uv@v10.3.2
 
       - name: Set up Python
         run: uv python install 3.10

odctl-1.3.0/CLAUDE.md

@@ -0,0 +1,20 @@
+# odctl
+
+A CLI that starts a local open data stack with Docker Compose profiles: Kafka, Flink, Spark, Iceberg, Trino, ClickHouse, MLflow, Feast, Airflow, Temporal, Marquez, OpenMetadata, Grafana and more.
+
+`CONTRIBUTING.md` holds the set-up, the code map, the test commands, the rules for code and docs, how to add a profile or a command, and the release steps. Read it before a change. This file holds only what applies to an agent.
+
+## Running the Tests
+
+- Run the unit suite, `pytest tests/`, in full once before each commit, never after each step.
+- Run the strict build at most once per commit.
+
+## Owner Decisions
+
+- Show any Renovate config change to the owner before making it.
+- Real projects find defects the profile e2e tests miss, so run the benchtop projects on a released odctl from time to time, when the owner asks. It is not a release step: each project starts a full compose stack, which takes too long to do at every release.
+
+## Git
+
+- Commit or push only when asked. Do not create branches unless asked.
+- Merge a bot PR with the owner as author: `gh pr merge <n> --squash --author-email <maintainer email>`. A plain squash merge adds the bot to the contributors list. If a bot commit lands on `main`, rewrite it with the maintainer as author and rename the default branch and back so GitHub recounts contributors. Do this only with the owner's approval: the rewrite changes pushed history and needs a force push, and each force push is approved on its own.

odctl-1.3.0/CONTRIBUTING.md

@@ -0,0 +1,99 @@
+# Contributing to odctl
+
+This file is for people who change odctl itself: the CLI, its compose files and images, its tests and its documentation. To use odctl, start with the [documentation](https://jaehyeon.me/odctl/). For a change larger than a fix, such as a new profile, open an issue first, so the design is agreed before the work starts. The [roadmap](https://jaehyeon.me/odctl/latest/about/roadmap/) shows what is planned, and [How It Works](https://jaehyeon.me/odctl/latest/how-it-works/) shows how the parts fit together.
+
+## Set Up
+
+odctl uses [uv](https://docs.astral.sh/uv/) and needs Docker. Clone the repository, install the dependencies and the project in development mode, and install the pre-commit hooks, which run the formatting and type checks on each commit:
+
+```bash
+git clone https://github.com/jaehyeon-kim/odctl.git
+cd odctl
+uv sync
+uvx pre-commit install
+```
+
+Exporting the diagrams also needs draw.io, Chrome and ffmpeg.
+
+## Code Map
+
+| Path | Contents |
+|---|---|
+| `src/odctl/main.py` | the Typer CLI (`list`, `explain`, `ps`, `info`, `init`, `pull`, `up`, `down`, `logs`, `restart`, `recreate`) |
+| `src/odctl/registry.py`, `resources/registry.yml` | profiles, the services in each, dependencies, ports and usage text |
+| `src/odctl/planner.py`, `docker.py`, `projects.py`, `logs.py`, `workspace.py`, `config.py`, `ui.py` | startup order, Docker Compose calls, `restart` and `recreate`, and `logs`, each with one Compose project per file as `up` runs them, the `.odctl` workspace, settings, terminal output |
+| `src/odctl/resources/compose-*.yml` | one compose file per area (kafka, flink, spark, store, analytics, mlops, orch, obsv, metadata, infra, deps) |
+| `src/odctl/resources/docker/` | odctl's own images (postgres, mlflow, airflow, spark), published to `ghcr.io/jaehyeon-kim/odctl/<name>:<version>` |
+| `src/odctl/resources/deps/` | the `deps` image: connector and engine jars built or downloaded into the `odctl-shared-deps` volume; versions in `versions.env` |
+| `scripts/gen_pages.py` | builds a page per technology and the CLI reference at docs build time, from `registry.yml`, the compose files and `--help`; run directly, it writes `skills/odctl/SKILL.md`, with every command, option and `.env` key, which a test keeps in step |
+| `scripts/local_links.py`, `scripts/llmstxt_sections.py`, `scripts/release_links.py` | MkDocs hooks: clickable local addresses, the `llms.txt` sections from the nav, and links to files on `main` pointed at the release tag `v<version>` |
+| `scripts/diagrams.py`, `docs/assets/diagrams/*.drawio` | exports each draw.io diagram to the PNG, or GIF for steps, that the docs use, and writes a still PNG of each GIF's last frame beside it; `_legend.drawio` holds the shared shapes; `tests/test_diagrams.py` checks the exports and the stills |
+| `docs/images/` | screenshots of the services' web UIs and the terminal |
+| `image/` | the README's diagram and recording, with the recording's still; they stay here because the PyPI page of every released version links them on `main` |
+| `.github/scripts/smoke.sh` | the e2e assertions per profile group, which run the guide snippets and each guide's Runnable Example; `select-profiles.py` chooses the groups |
+| `docs/snippets/` | the code each guide, Getting Started and How It Works show, one folder per page |
+| `skills/odctl/SKILL.md` | the odctl skill for AI assistants, generated by `scripts/gen_pages.py` |
+| `docs/` | MkDocs Material site, versioned with mike; `docs/about/` holds the public Roadmap and the Changelog |
+| `scripts/release_notes.py` | prints one version's Changelog section, the text of its GitHub release |
+
+## Tests
+
+```bash
+uv run pytest tests/ -q
+uvx pre-commit run --all-files                    # ruff, ruff format, mypy, whitespace, YAML, TOML
+DISABLE_MKDOCS_2_WARNING=true uv run mkdocs build --strict
+uv run mkdocs serve -a 127.0.0.1:8000             # docs at http://127.0.0.1:8000/odctl/
+uv run python scripts/gen_pages.py                # regenerate skills/odctl/SKILL.md
+uv run python scripts/diagrams.py [name ...]      # export diagrams; needs draw.io, Chrome and ffmpeg
+```
+
+Each layer checks one thing:
+
+- `tests/` checks the CLI, the registry, the compose files, the docs and the generated pages, with no container running.
+- The strict docs build fails on a broken link or anchor.
+- The e2e tests start each profile group and run `.github/scripts/smoke.sh` on it. A running container is not enough: the script checks that the profile does its job, such as a Kafka consumer reading a message or a Flink job committing to Iceberg. It also runs each guide's snippets and Runnable Example.
+
+Run the unit suite in full once before each commit. While fixing, run only the failed tests (`pytest --lf`) and the tests of the files you changed. Run the strict build when `docs/`, `mkdocs.yml` or an input of `scripts/gen_pages.py` changed: `registry.yml`, a compose file, or a command's `--help` text.
+
+A pull request runs pre-commit, the unit tests on each supported Python version, the strict docs build, and the image builds when an image input changed. e2e runs only on a tag and on a manual run, because odctl runs images tagged with its own version, and the images of an unreleased version do not exist yet. To check a profile on your machine, do what the e2e job does: install the CLI with `uv tool install .`, run `odctl init`, start the profile with `odctl up <profile>`, then run `.github/scripts/smoke.sh <profile>`. Run one `odctl up` at a time. When no image definition changed since the last release, you can give the last release's images the new version's tag for the test, and remove those tags afterwards.
+
+## Rules
+
+- **Images follow the CLI version.** odctl runs its own images at its own version (`TAG`), never `latest`: Docker does not pull a tag it already has, so a cached `latest` keeps running old code. Third-party images are pinned to a version and updated by Renovate.
+- **Bind to 127.0.0.1.** Every published port binds to `127.0.0.1`, not all interfaces.
+- **One profile per tool**, with `-lite` and `-full` variants where a tool has a small and a clustered form (`kafka-lite`, `kafka-full`).
+- **Fixed container names.** Every service has a `container_name`, so `odctl up` cannot scale a service; several copies are separate named services (Flink's `taskmanager-a` and `taskmanager-b`).
+- **A feature merges only with its docs.** Each command, option, profile and `.env` key needs a tested example on a guide, Getting Started or How It Works, a reference entry (the CLI reference, a profile page or How It Works), and a place in the skill; a command also needs a line in the README's Commands section. `test_every_feature_has_its_docs` checks this, and its short list of exceptions gives a reason for each.
+- **Renovate** opens its pull requests on Mondays, with image updates grouped by compose file. Agree a change to its configuration (`.github/renovate.json`) with the maintainer first.
+
+## Add a Profile
+
+1. Add the services to the compose file of their area in `src/odctl/resources/`, following the Rules: a pinned image, a `container_name`, and ports bound to `127.0.0.1`.
+2. Add the profile to `src/odctl/resources/registry.yml`: its compose file, description, profiles, dependencies, role and usage text. `scripts/gen_pages.py` builds its page from this entry and the compose file.
+3. Add its checks to `.github/scripts/smoke.sh`, and its group to `GROUPS` in `.github/scripts/select-profiles.py`.
+4. Add the technology to the README diagram, `image/diagram.drawio`, then export it. `test_readme_diagram_names_every_technology` fails until you do.
+5. Give it a tested example on a guide, Getting Started or How It Works, then run `scripts/gen_pages.py` to update the skill.
+
+## Add a Command or Option
+
+Add it to `src/odctl/main.py` with its `--help` text, which the CLI reference is built from. Give it a tested example on a guide, Getting Started or How It Works, and a command a line in the README's Commands section. Then run `scripts/gen_pages.py` to update the skill.
+
+## Write the Docs
+
+- **Generated pages.** The profile pages and the CLI reference are generated by `scripts/gen_pages.py` during the build, from `registry.yml`, the compose files and the commands. Change those, not the pages, and do not copy versions or ports into hand-written guides.
+- **Code blocks are copies.** Guide code lives in `docs/snippets/<guide>/`, one section per step (`# --8<-- [start:name]`), and Getting Started and How It Works follow the same rule. A block labelled `title="path:section"` or `title="path"` is a copy of that section or file, checked by `tests/test_docs.py`. Pages use copies, not `--8<--` includes, so the Markdown reads correctly on GitHub. Every guide ends with a Runnable Example, `example.sh` or `example.py` in its folder. `smoke.sh` runs the same sections and examples, and `select-profiles.py` maps each snippet folder to the e2e groups that run it.
+- **Guides.** Each guide says which pages it builds on, the output of every step, and where a value it quotes is set. A page has at most 10 `##` headings and 15 in all.
+- **Writing.** Plain English for readers whose first language is not English. Headings use title case, keeping product names such as `odctl` as they are, and do not start with "The". No em or en dashes. Each Markdown paragraph is one line, never hard-wrapped.
+- **Diagrams.** Copy shapes from `docs/assets/diagrams/_legend.drawio`, and export with `scripts/diagrams.py`.
+
+## Pull Requests
+
+Work on a branch from `main` and open a pull request against `main`. Before you push, run pre-commit, the unit tests, and the strict build when the docs changed. Describe each change to ports, profiles, `.env` keys, volumes or workspace files in the pull request, so that the release can add an Upgrading line to the Changelog (`docs/about/changelog.md`).
+
+## Releases
+
+The maintainer releases. These steps are written down so that each release is the same:
+
+1. Bump `version` in `pyproject.toml`, run `uv lock`, and add the release to `docs/about/changelog.md`, with an Upgrading line for any change to ports, profiles, `.env` keys, volumes or workspace files. When it reaches `main`, the docs deploy as that version, aliased `latest`.
+2. Once the `main` pipeline for that commit is green, push tag `vX.Y.Z` on it. A tag on a commit whose checks fail builds no images and cannot be released until it is moved. The tag pipeline builds and pushes the `X.Y.Z` images, then runs e2e for every profile group (about an hour).
+3. Publish the GitHub release with that version's Changelog section as its text, so the two never differ: `python scripts/release_notes.py X.Y.Z > /tmp/notes.md`, then `gh release create vX.Y.Z --title vX.Y.Z --notes-file /tmp/notes.md`. `publish.yml` waits for a green tag pipeline and checks that every `X.Y.Z` image is in ghcr, then uploads to PyPI.

{odctl-1.2.1 → odctl-1.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: odctl
-Version: 1.2.1
+Version: 1.3.0
 Summary: Run an open data stack on your machine, with its services already connected, from one command.
 Project-URL: Homepage, https://github.com/jaehyeon-kim/odctl
 Project-URL: Repository, https://github.com/jaehyeon-kim/odctl
@@ -106,30 +106,9 @@ Projects that use odctl:
 - [desflow](https://github.com/jaehyeon-kim/desflow) ([docs](https://jaehyeon.me/desflow/)): a SimPy library for simulations that stream to Kafka, PostgreSQL, Redis and Iceberg; its examples and integration tests run their infrastructure with odctl.
 - [agentic-analytics-system](https://github.com/jaehyeon-kim/agentic-analytics-system): conversational analytics over an Iceberg lakehouse, with Trino, Iceberg and object storage started by odctl.
 
-## Development
-
-Pull requests are welcome. To work on the CLI:
-
-1. Clone the repository and install [uv](https://docs.astral.sh/uv/).
-2. Install the dependencies and the project in development mode:
-   ```bash
-   uv sync
-   ```
-3. Install the pre-commit hooks, which run the formatting and type checks:
-   ```bash
-   uvx pre-commit install
-   ```
-4. Run the tests:
-   ```bash
-   uv run pytest tests/
-   ```
-5. Build the documentation site, then serve it while you edit. It is served at `http://127.0.0.1:8000/odctl/`:
-   ```bash
-   uv run mkdocs build --strict
-   uv run mkdocs serve
-   ```
-
-The profile pages and the CLI reference are generated by `scripts/gen_pages.py` during the build, so edit `registry.yml`, the compose files or the commands rather than those pages.
+## Contributing
+
+Pull requests are welcome. [CONTRIBUTING.md](https://github.com/jaehyeon-kim/odctl/blob/main/CONTRIBUTING.md) explains how to set up the repository, run the tests, add a profile or a command, and write the docs.
 
 ## License
 

{odctl-1.2.1 → odctl-1.3.0}/README.md

@@ -69,30 +69,9 @@ Projects that use odctl:
 - [desflow](https://github.com/jaehyeon-kim/desflow) ([docs](https://jaehyeon.me/desflow/)): a SimPy library for simulations that stream to Kafka, PostgreSQL, Redis and Iceberg; its examples and integration tests run their infrastructure with odctl.
 - [agentic-analytics-system](https://github.com/jaehyeon-kim/agentic-analytics-system): conversational analytics over an Iceberg lakehouse, with Trino, Iceberg and object storage started by odctl.
 
-## Development
-
-Pull requests are welcome. To work on the CLI:
-
-1. Clone the repository and install [uv](https://docs.astral.sh/uv/).
-2. Install the dependencies and the project in development mode:
-   ```bash
-   uv sync
-   ```
-3. Install the pre-commit hooks, which run the formatting and type checks:
-   ```bash
-   uvx pre-commit install
-   ```
-4. Run the tests:
-   ```bash
-   uv run pytest tests/
-   ```
-5. Build the documentation site, then serve it while you edit. It is served at `http://127.0.0.1:8000/odctl/`:
-   ```bash
-   uv run mkdocs build --strict
-   uv run mkdocs serve
-   ```
-
-The profile pages and the CLI reference are generated by `scripts/gen_pages.py` during the build, so edit `registry.yml`, the compose files or the commands rather than those pages.
+## Contributing
+
+Pull requests are welcome. [CONTRIBUTING.md](https://github.com/jaehyeon-kim/odctl/blob/main/CONTRIBUTING.md) explains how to set up the repository, run the tests, add a profile or a command, and write the docs.
 
 ## License
 

{odctl-1.2.1 → odctl-1.3.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "odctl"
-version = "1.2.1"
+version = "1.3.0"
 description = "Run an open data stack on your machine, with its services already connected, from one command."
 readme = "README.md"
 authors = [

{odctl-1.2.1 → odctl-1.3.0}/scripts/gen_pages.py

@@ -418,6 +418,8 @@ ENV_KEYS = {
     "TZ": "The time zone of the services that read it, `UTC` by default.",
     "_AIRFLOW_PIP_DEPS": "Extra Python packages installed in every new Airflow container.",
     "_MLOPS_PIP_DEPS": "Extra Python packages installed in every new `mlflow` and `mlflow-serve` container.",
+    "_METADATA_PIP_DEPS": "Extra Python packages installed in every new `openmetadata-ingestion` container.",
+    "METADATA_AUTOPILOT": "`true` turns on OpenMetadata's AutoPilot, which creates and runs default agents for a service added in the UI; it is `false` by default.",
     "MODEL_URI": "The model `mlflow-serve` serves, such as `models:/<name>@<alias>`; empty means it waits idle.",
 }