odctl 1.2.2__tar.gz → 1.4.0__tar.gz

{odctl-1.2.2 → odctl-1.4.0}/.github/scripts/select-profiles.py

@@ -45,6 +45,11 @@ GROUPS = [
     "lineage",
     "telemetry",
     "metadata",
+    # The part before the colon is the profile to start. metadata:deps and
+    # metadata:autopilot start the same profile as metadata. Each examples group
+    # recreates metadata twice, which is why each runs alone, in parallel with the others.
+    "metadata:deps",
+    "metadata:autopilot",
     "airflow",
     "temporal",
     "postgres",
@@ -70,13 +75,14 @@ REGISTRY = Path("src/odctl/resources/registry.yml")
 
 # smoke.sh starts some profiles from another group's case: the kafka case starts
 # catalog, the flink case starts kafka-lite and feast, the spark case starts
-# kafka-lite, and the fluss and feast cases start flink-lite. A change to that
-# compose file needs these groups as well as the ones the registry maps to it.
+# kafka-lite, the fluss and feast cases start flink-lite, and the airflow case
+# starts feast. A change to that compose file needs these groups as well as the
+# ones the registry maps to it.
 ALSO_RUNS = {
     "compose-infra.yml": ["kafka-lite kafka-full"],
     "compose-kafka.yml": ["flink-lite flink-full", "spark-lite spark-full"],
     "compose-flink.yml": ["fluss", "feast"],
-    "compose-mlops.yml": ["flink-lite flink-full"],
+    "compose-mlops.yml": ["flink-lite flink-full", "airflow"],
 }
 
 # smoke.sh runs the guide commands in docs/snippets/<guide>/, so a change there
@@ -85,7 +91,15 @@ ALSO_RUNS = {
 SNIPPET_PREFIX = "docs/snippets/"
 SNIPPET_GROUPS = {
     "getting-started": ["kafka-lite kafka-full"],
-    "how-it-works": ["storage", "catalog", "trino", "airflow"],
+    "how-it-works": [
+        "storage",
+        "catalog",
+        "trino",
+        "airflow",
+        "mlflow",
+        "metadata:deps",
+        "metadata:autopilot",
+    ],
     "kafka": ["kafka-lite kafka-full"],
     "kafka-connect": ["kafka-lite kafka-full"],
     "iceberg-ingestion": ["kafka-lite kafka-full", "flink-lite flink-full"],
@@ -100,10 +114,11 @@ SNIPPET_GROUPS = {
     "airflow-dags": ["airflow"],
     "temporal-workers": ["temporal"],
     "mlflow": ["mlflow"],
-    "feast-iceberg": ["feast", "flink-lite flink-full", "mlflow"],
+    "feast-iceberg": ["feast", "flink-lite flink-full", "mlflow", "airflow"],
+    "feast-vector-retrieval": ["feast"],
     "evidently-reports": ["evidently"],
     "lineage": ["lineage"],
-    "openmetadata": ["metadata"],
+    "openmetadata": ["metadata", "metadata:autopilot"],
     "telemetry": ["telemetry"],
 }
 
@@ -174,7 +189,8 @@ def select(changed: list, registry_path: Path = REGISTRY) -> list:
 
     selected = []
     for group in GROUPS:
-        files = {mapping.get(p) for p in group.split()}
+        # A group name may carry a suffix after a colon, as metadata:deps does.
+        files = {mapping.get(p.split(":")[0]) for p in group.split()}
         also = any(group in ALSO_RUNS.get(name, []) for name in touched_files)
         if files & touched_files or group in snippet_groups or also:
             selected.append(group)

{odctl-1.2.2 → odctl-1.4.0}/.github/scripts/smoke.sh

@@ -9,10 +9,10 @@
 # committed nothing while its checkpoints succeeded. Each assertion below exists
 # because something shipped broken past a liveness check.
 #
-# Usage: smoke.sh <profile>
+# Usage: smoke.sh <profile or group name, such as metadata:deps>
 set -uo pipefail
 
-PROFILE="${1:?usage: smoke.sh <profile>}"
+PROFILE="${1:?usage: smoke.sh <profile or group name, such as metadata:deps>}"
 MB_USER="smoke@example.com"
 MB_PASS="Smoke-test-1234"
 
@@ -880,6 +880,104 @@ if bad:
 
   # Airflow pushes OTLP to the telemetry collector rather than being scraped.
   assert_pushed_metrics 'airflow_.+' Airflow
+
+  smoke_airflow_materialize
+}
+
+# Feast guide's On a Schedule section: a DAG that builds a FeatureStore from
+# FEAST_REGISTRY and calls materialize_incremental. It needs the registry entry and
+# the Iceberg table the guide's earlier steps make, so this runs those from the
+# host as the guide does, then uploads the DAG, runs it and reads the value it
+# copied into Valkey. The feature view source has no catalog address, so the
+# registry entry works from the host here and from the Airflow container in the DAG.
+# Its own project, so that the guide's Runnable Example can still register demo.
+smoke_airflow_materialize() {
+  odctl up feast >/dev/null 2>&1 || fail "could not start feast for the materialise DAG"
+  retry 30 5 http_ok "http://127.0.0.1:6566/health" || fail "Feast never answered on :6566"
+
+  local work="${TMPDIR:-/tmp}/odctl-airflow-feast-smoke"
+  rm -rf "$work"; mkdir -p "$work"
+  uv venv "$work/.venv" >/dev/null 2>&1 || fail "could not create a venv for the feast client"
+  VIRTUAL_ENV="$work/.venv" uv pip install -q \
+    "feast[duckdb,iceberg,redis,postgres]==0.66.0" "pyarrow" >/dev/null 2>&1 \
+    || fail "could not install the feast client"
+
+  cp -r "$SNIPPETS/feast-iceberg/feature_repo" "$work/"
+  sed 's/^project: demo$/project: airflow_smoke/' "$work/feature_repo/feature_store.yaml" \
+    > "$work/feature_store.yaml" && mv "$work/feature_store.yaml" "$work/feature_repo/feature_store.yaml"
+  sed 's/project="demo"/project="airflow_smoke"/' "$SNIPPETS/feast-iceberg/feast_materialize.py" \
+    > "$work/feast_materialize.py"
+  snippet feast-iceberg/steps.py create-table > "$work/build.py"
+  snippet feast-iceberg/steps.py append > "$work/append.py"
+  snippet feast-iceberg/steps.py read > "$work/read.py"
+
+  local py="$work/.venv/bin/python" log="$work/step.log"
+  run_step() {
+    local what="$1"; shift
+    if ! (cd "$work" && "$@") >"$log" 2>&1; then
+      echo "---- $what ----"
+      tail -25 "$log"
+      fail "$what"
+    fi
+  }
+
+  # A table left by the Runnable Example would make the guide's create step fail.
+  local code
+  code=$(curl -sS --max-time 10 -o /dev/null -w '%{http_code}' -X DELETE "http://127.0.0.1:8181/v1/namespaces/demo/tables/driver_stats?purgeRequested=true")
+  case "$code" in 204|404) ;; *) fail "could not delete demo.driver_stats before the guide's create step (HTTP $code)";; esac
+  eval "$(snippet feast-iceberg/commands.sh catalog-env)"
+  run_step "could not create the Iceberg table through the catalog" "$py" build.py
+  # Delete the view, so that materialise starts again and the check reads this run's values.
+  (cd "$work" && "$work/.venv/bin/feast" -c feature_repo delete driver_stats) >/dev/null 2>&1 || true
+  run_step "feast apply or materialize failed without a catalog address in the source" \
+    env PATH="$work/.venv/bin:$PATH" bash -e -o pipefail -c "$(snippet feast-iceberg/commands.sh apply-materialize)"
+  pass "the guide's source, with no catalog address, applied and materialised from the host"
+
+  run_step "could not upload the materialise DAG to s3://airflow/dags" \
+    run_snippet feast-iceberg/commands.sh dag-upload
+  retry 30 10 bash -c \
+    "docker exec airflow airflow dags list 2>/dev/null | grep -q feast_materialize" \
+    || fail "the S3 DAG bundle never delivered the materialise DAG, or it was never parsed"
+
+  # The new row is newer than the host's materialise, so only the DAG can copy it.
+  run_step "could not append a row to the Iceberg table" "$py" append.py
+  run_step "the guide's unpause and trigger commands failed" \
+    run_snippet feast-iceberg/commands.sh dag-run
+
+  # Unpausing also starts a scheduled run, so wait until every run has finished.
+  local states=""
+  for _ in $(seq 1 30); do
+    states=$(docker exec airflow airflow dags list-runs feast_materialize -o plain 2>/dev/null \
+      | grep -E "^feast_materialize" | tr -s ' ' | cut -d' ' -f3)
+    grep -qE 'queued|running' <<<"$states" || [ -z "$states" ] || break
+    sleep 10
+  done
+  [ -n "$states" ] && ! grep -qvx success <<<"$states" \
+    || fail "the materialise DAG runs ended in states: ${states:-none}"
+  pass "the materialise DAG ran to success inside the Airflow container"
+
+  local got
+  got=$(cd "$work" && "$py" read.py 2>"$log") || { tail -25 "$log"; fail "the online read failed"; }
+  [ "$got" = "{'driver_id': [1001, 1002], 'conv_rate': [0.6000000238418579, 0.75]}" ] \
+    || fail "Valkey holds $got, not the 0.6 the DAG should have copied"
+  pass "the DAG copied the new value from Iceberg into Valkey"
+
+  # Leave no hourly run behind: a DAG that stays unpaused fails on its next run once
+  # the table below is gone, and the next smoke run would read that failed run.
+  docker exec airflow airflow dags pause feast_materialize >/dev/null 2>&1 \
+    || fail "could not pause the materialise DAG"
+  docker exec airflow python -c '
+import boto3
+boto3.client("s3", endpoint_url="http://seaweed:8333").delete_object(
+    Bucket="airflow", Key="dags/feast_materialize.py")' >/dev/null 2>&1 \
+    || fail "could not delete the materialise DAG from s3://airflow/dags"
+
+  # No feast teardown: it leaves the view's row in feature_view_version_history, and
+  # a second run of this check would then fail to apply the view again.
+  curl -fsS --max-time 10 -X DELETE \
+    "http://127.0.0.1:8181/v1/namespaces/demo/tables/driver_stats?purgeRequested=true" >/dev/null 2>&1 || true
+  curl -fsS --max-time 10 -X DELETE "http://127.0.0.1:8181/v1/namespaces/demo" >/dev/null 2>&1 || true
+  rm -rf "$work"
 }
 
 # deps is a one-shot copy into a shared volume, so there is no container to
@@ -1094,7 +1192,7 @@ smoke_fluss() {
   pass "Flink wrote 3 rows to a Fluss table and read them back"
 }
 
-# OpenMetadata: the version endpoint answers through every failure the 2.0.1
+# OpenMetadata: the version endpoint answers through every failure an
 # upgrade could cause. The ingestion container runs its own Airflow 3, whose
 # health path moved to /api/v2/monitor/health, and a stale image or a failed
 # migration both leave a server that still reports a version. Drive the whole
@@ -1110,14 +1208,23 @@ smoke_metadata() {
   pass "schema migration completed"
   assert_metrics http://openmetadata-server:8586/prometheus
 
-  # The principal domain odctl sets is open-data.local, not the upstream
+  # The principal domain odctl sets is odctl.io, not the upstream
   # default. A wrong domain fails with a misleading invalid password error.
   local token
   token=$(om_step login 'echo "$TOKEN"' 2>/dev/null | tail -1)
-  [ -n "$token" ] || fail "admin@open-data.local could not log in"
+  [ -n "$token" ] || fail "admin@odctl.io could not log in"
   pass "admin logged in and received a JWT"
   local auth="Authorization: Bearer $token"
 
+  # user is the second admin. Its entry in AUTHORIZER_ADMIN_PRINCIPALS,
+  # user:password, sets its first password.
+  local code
+  code=$(curl -s -o /dev/null -w '%{http_code}' -H 'Content-Type: application/json' \
+    -d "{\"email\":\"user@odctl.io\",\"password\":\"$(printf password | base64)\"}" \
+    http://127.0.0.1:8585/api/v1/users/login)
+  [ "$code" = "200" ] || fail "user@odctl.io could not log in with the password password (HTTP $code)"
+  pass "user@odctl.io logged in with the password password"
+
   # Airflow 3 moved this path. The old /health returns 404 with a message saying so.
   retry 60 5 http_ok "http://127.0.0.1:8087/api/v2/monitor/health" \
     || fail "ingestion Airflow never answered on 8087"
@@ -1126,6 +1233,20 @@ smoke_metadata() {
     || fail "the server cannot reach the ingestion container, so no pipeline can deploy"
   pass "ingestion Airflow healthy and reachable from the server"
 
+  # AIRFLOW_ADMIN_USER and AIRFLOW_ADMIN_PASSWORD set this login. A new token
+  # answers 201, and a wrong login 401.
+  code=$(curl -s -o /dev/null -w '%{http_code}' -H 'Content-Type: application/json' \
+    -d '{"username":"user","password":"password"}' http://127.0.0.1:8087/auth/token)
+  [ "$code" = "201" ] || fail "the ingestion Airflow refused user / password (HTTP $code)"
+  pass "ingestion Airflow accepted user / password"
+
+  # openmetadata-init sets AutoPilot from METADATA_AUTOPILOT, false by default.
+  retry 24 5 om_autopilot_is "$token" False \
+    || fail "AutoPilot does not report active: false after the start"
+  [ "$(docker inspect -f '{{.State.ExitCode}}' openmetadata-init)" = "0" ] \
+    || fail "openmetadata-init did not exit with code 0"
+  pass "AutoPilot reports active: false and openmetadata-init exited 0"
+
   # A real ingestion, with the guide's steps. Postgres is already running, so it
   # is its own source.
   local svc
@@ -1142,23 +1263,11 @@ smoke_metadata() {
   # Airflow's dag_processor has to parse the new file before it can be
   # triggered, so the guide's trigger can come too early. Trigger again here.
   if ! grep -q 'has been triggered' <<<"$out"; then
-    local triggered=no
-    for _ in $(seq 1 24); do
-      if curl -fsS --max-time 90 -X POST -H "$auth" \
-          "http://127.0.0.1:8585/api/v1/services/ingestionPipelines/trigger/$pipe" 2>/dev/null \
-          | grep -q 'has been triggered'; then triggered=yes; break; fi
-      sleep 5
-    done
-    [ "$triggered" = "yes" ] || fail "the pipeline never triggered"
+    om_trigger "$token" "$pipe" || fail "the pipeline never triggered"
   fi
 
-  local state=""
-  for _ in $(seq 1 30); do
-    state=$(TOKEN="$token" om_step status 2>/dev/null \
-      | python3 -c 'import json,sys; s=json.load(sys.stdin).get("pipelineStatuses") or []; print(s[0]["pipelineState"] if s else "")' 2>/dev/null)
-    case "$state" in success|failed|partialSuccess) break ;; esac
-    sleep 10
-  done
+  local state
+  state=$(om_run_state "$token" om_step "")
   [ "$state" = "success" ] || fail "ingestion run finished in state ${state:-none}, not success"
   pass "ingestion run reached success"
 
@@ -1191,8 +1300,214 @@ smoke_metadata() {
   done
   pass "Context Center pages and memories both answer"
 
+  smoke_metadata_connector "$token"
+  smoke_metadata_autopilot "$token"
+
   TOKEN="$token" om_step remove >/dev/null 2>&1 || fail "the guide's remove step failed"
-  pass "the guide's remove step deleted the service and its pipeline"
+  code=$(curl -s -o /dev/null -w '%{http_code}' -H "$auth" \
+    http://127.0.0.1:8585/api/v1/services/databaseServices/name/demo_connector)
+  [ "$code" = "404" ] || fail "the guide's remove step left the demo_connector service (HTTP $code)"
+  retry 12 5 bash -c '! docker exec openmetadata-ingestion test -e /opt/airflow/connectors/demo_connector.py' \
+    || fail "the ingestion container kept the connector after the remove step deleted it from S3"
+  pass "the guide's remove step deleted both services, their pipelines and the uploaded connector"
+}
+
+# Trigger an ingestion pipeline until the server accepts it: Airflow has to parse
+# a new DAG file first. Usage: om_trigger <token> <pipeline id>
+om_trigger() {
+  local _
+  for _ in $(seq 1 24); do
+    if curl -fsS --max-time 90 -X POST -H "Authorization: Bearer $1" \
+        "http://127.0.0.1:8585/api/v1/services/ingestionPipelines/trigger/$2" 2>/dev/null \
+        | grep -q 'has been triggered'; then return 0; fi
+    sleep 5
+  done
+  return 1
+}
+
+# The latest run of the pipeline a guide's status step reads, as "<run id> <state>",
+# or nothing when there is no run yet. The step is the guide's own: om_step for
+# demo_pg, om_connector_step for the custom connector.
+# Usage: om_latest_run <token> <om_step|om_connector_step>
+om_latest_run() {
+  TOKEN="$1" "$2" status 2>/dev/null | python3 -c '
+import json, sys
+runs = json.load(sys.stdin).get("pipelineStatuses") or []
+if runs:
+    print(runs[0]["runId"], runs[0]["pipelineState"])' 2>/dev/null
+}
+
+# Wait for the first run whose id is not <previous run id> to finish, and print its
+# state. Usage: om_run_state <token> <om_step|om_connector_step> <previous run id>
+om_run_state() {
+  local _ run state
+  for _ in $(seq 1 30); do
+    read -r run state < <(om_latest_run "$1" "$2")
+    if [ -n "${run:-}" ] && [ "$run" != "$3" ]; then
+      case "$state" in success|failed|partialSuccess) echo "$state"; return ;; esac
+    fi
+    sleep 10
+  done
+  echo ""
+}
+
+# Whether the guide's autopilot step prints <value>. Usage: om_autopilot_is <token> <True|False>
+om_autopilot_is() {
+  [ "$(TOKEN="$1" om_step autopilot 2>/dev/null | tail -1)" = "$2" ]
+}
+
+# Add a service the way the UI does: create it, then trigger AutoPilotApplication with
+# the service's entityLink. Usage: om_add_service_like_the_ui <token> <service name>
+om_add_service_like_the_ui() {
+  local auth="Authorization: Bearer $1"
+  curl -fsS -X PUT -H "$auth" -H 'Content-Type: application/json' \
+    -d "{\"name\":\"$2\",\"serviceType\":\"CustomDatabase\",\"connection\":{\"config\":{\"type\":\"CustomDatabase\",\"sourcePythonClass\":\"demo_connector.DemoSource\"}}}" \
+    http://127.0.0.1:8585/api/v1/services/databaseServices >/dev/null || fail "could not create the $2 service"
+  curl -fsS -X POST -H "$auth" -H 'Content-Type: application/json' \
+    -d "{\"entityLink\":\"<#E::databaseService::$2>\"}" \
+    http://127.0.0.1:8585/api/v1/apps/trigger/AutoPilotApplication >/dev/null \
+    || fail "the server refused the AutoPilot trigger the UI sends"
+}
+
+# The number of agents a service has. Usage: om_agent_count <token> <service name>
+om_agent_count() {
+  curl -fsS -H "Authorization: Bearer $1" "http://127.0.0.1:8585/api/v1/services/ingestionPipelines?service=$2" \
+    | python3 -c 'import json,sys; print(json.load(sys.stdin)["paging"]["total"])'
+}
+
+om_delete_service() {
+  curl -s -o /dev/null -X DELETE -H "Authorization: Bearer $1" \
+    "http://127.0.0.1:8585/api/v1/services/databaseServices/name/$2?hardDelete=true&recursive=true"
+}
+
+# With AutoPilot off, a service added the way the UI adds one gets no agent. The
+# check waits 60 seconds, as long as the on-case below allows AutoPilot to create its agent.
+smoke_metadata_autopilot() {
+  local agents
+  om_add_service_like_the_ui "$1" demo_ui
+  sleep 60
+  agents=$(om_agent_count "$1" demo_ui)
+  om_delete_service "$1" demo_ui
+  [ "$agents" = "0" ] || fail "a service added the way the UI adds one got $agents agents with AutoPilot off"
+  pass "a service added the way the UI adds one, with its AutoPilot trigger, got no agent"
+}
+
+# The same two calls with AutoPilot on create at least one agent. It proves
+# METADATA_AUTOPILOT=true works and that the off-case above could have failed.
+smoke_metadata_autopilot_on() {
+  local agents=0 _
+  om_add_service_like_the_ui "$1" demo_ui_on
+  for _ in $(seq 1 12); do
+    agents=$(om_agent_count "$1" demo_ui_on)
+    [ "${agents:-0}" -ge 1 ] && break
+    sleep 5
+  done
+  om_delete_service "$1" demo_ui_on
+  [ "${agents:-0}" -ge 1 ] || fail "a service added the way the UI adds one got no agent with AutoPilot on"
+  pass "with METADATA_AUTOPILOT true, a service added the way the UI adds one got $agents agent(s)"
+}
+
+# The How It Works example turns AutoPilot on and recreates metadata. While it is on,
+# check the UI's calls create an agent, then put the key back and check it is off.
+how_it_works_autopilot() {
+  local out
+  out=$(run_snippet how-it-works/commands.sh metadata-autopilot 2>&1) || { echo "$out" | tail -20; fail "the metadata-autopilot example failed"; }
+  grep -qE -- 'AutoPilot active: true' <<<"$out" || { echo "$out" | tail -20; fail "the metadata-autopilot example printed no 'AutoPilot active: true'"; }
+  pass "How It Works' metadata-autopilot example printed 'AutoPilot active: true'"
+  smoke_metadata_autopilot_on "$(om_token)"
+  mv .odctl/.env.bak .odctl/.env || fail "the metadata-autopilot example left no .env.bak"
+  odctl recreate metadata >/dev/null 2>&1 || fail "could not recreate metadata after the metadata-autopilot example"
+  retry 24 5 om_autopilot_is "$(om_token)" False \
+    || fail "AutoPilot is not off again after the key was put back"
+  pass "AutoPilot is off again after the key was put back"
+}
+
+# Run a connector section until its output has <text>. Usage: om_connector_says <section> <text>
+om_connector_says() {
+  om_connector_step "$1" 2>/dev/null | grep -qF "$2"
+}
+
+# The guide's custom connector steps, run from the folder that holds the file, as
+# the guide tells the reader. Usage: om_connector_step <section> [extra commands]
+om_connector_step() {
+  local code
+  code=$(snippet openmetadata/connector/commands.sh "$1")
+  (cd "$CONNECTOR_DIR" && bash -o pipefail -c 'AUTH="Authorization: Bearer ${TOKEN:-}"
+'"$code"'
+'"${2:-}")
+}
+
+# A custom connector uploaded to s3://openmetadata/connectors is importable in the
+# ingestion container, and a service and an agent made through the API run it.
+# Then a changed connector is used by the next run with no restart, which is why
+# nothing here restarts the container: its start time must not move.
+smoke_metadata_connector() {
+  local token="$1" auth="Authorization: Bearer $1" started svc pipe out run state agents
+  CONNECTOR_DIR=$(mktemp -d)
+  cp "$SNIPPETS/openmetadata/connector/demo_connector.py" "$CONNECTOR_DIR/"
+  started=$(docker inspect -f '{{.State.StartedAt}}' openmetadata-ingestion)
+
+  om_connector_step upload >/dev/null 2>&1 || fail "could not upload the connector to s3://openmetadata/connectors"
+  retry 12 5 om_connector_says copied demo_connector.py \
+    || fail "the ingestion container never copied the connector from S3"
+  pass "the connector reached /opt/airflow/connectors in the ingestion container"
+
+  svc=$(TOKEN="$token" om_connector_step service 'echo "$SVC"' 2>/dev/null | tail -1)
+  [ -n "$svc" ] || fail "could not create the CustomDatabase service through the API"
+  out=$(TOKEN="$token" SVC="$svc" om_connector_step agent 'echo; echo "$PIPE"' 2>/dev/null)
+  pipe=$(tail -1 <<<"$out")
+  [ -n "$pipe" ] || fail "could not create the custom connector's agent"
+  grep -q 'has been created' <<<"$out" || { echo "$out"; fail "the custom connector's agent did not deploy a DAG"; }
+  grep -q 'has been triggered' <<<"$out" || om_trigger "$token" "$pipe" || fail "the custom connector's agent never triggered"
+  state=$(om_run_state "$token" om_connector_step "")
+  [ "$state" = "success" ] || fail "the custom connector's run finished in state ${state:-none}, not success"
+  out=$(TOKEN="$token" om_connector_step table 2>/dev/null)
+  grep -q '"fullyQualifiedName":"demo_connector.demo_db.public.orders"' <<<"$out" \
+    && grep -q '"name":"id"' <<<"$out" && grep -q '"name":"amount"' <<<"$out" \
+    || fail "the custom connector's table orders, with its columns id and amount, is not in the catalogue"
+  pass "the custom connector's agent ran and its table orders is in the catalogue"
+
+  # A service made through the API starts no AutoPilot agents. The UI does.
+  agents=$(curl -fsS -H "$auth" "http://127.0.0.1:8585/api/v1/services/ingestionPipelines?service=demo_connector" \
+    | python3 -c 'import json,sys; print(json.load(sys.stdin)["paging"]["total"])')
+  [ "$agents" = "1" ] || fail "the demo_connector service has $agents agents, not only its own"
+  pass "the demo_connector service has only its own agent"
+
+  # Change the connector, upload it again and trigger the same agent.
+  read -r run _ < <(om_latest_run "$token" om_connector_step)
+  om_connector_step change >/dev/null 2>&1 || fail "could not change the connector"
+  om_connector_step upload >/dev/null 2>&1 || fail "could not upload the changed connector"
+  retry 12 5 om_connector_says copied-change 1 \
+    || fail "the ingestion container never copied the changed connector"
+  TOKEN="$token" PIPE="$pipe" om_connector_step retrigger 2>/dev/null | grep -q 'has been triggered' \
+    || fail "the custom connector's agent did not trigger again"
+  state=$(om_run_state "$token" om_connector_step "$run")
+  [ "$state" = "success" ] || fail "the second run finished in state ${state:-none}, not success"
+  TOKEN="$token" om_connector_step changed-table 2>/dev/null \
+    | grep -q '"name":"orders_v2"' || fail "the changed connector's table orders_v2 is not in the catalogue"
+  [ "$(docker inspect -f '{{.State.StartedAt}}' openmetadata-ingestion)" = "$started" ] \
+    || fail "the ingestion container restarted while the connector changed"
+  pass "the changed connector was used by the next run, with no restart"
+  rm -rf "$CONNECTOR_DIR"
+}
+
+# The guide's runnable example needs demo_connector.py in the folder it runs in.
+smoke_metadata_example() {
+  local out
+  cp "$SNIPPETS/openmetadata/connector/demo_connector.py" .
+  out=$(bash "$SNIPPETS/openmetadata/example.sh" 2>&1) || { rm -f demo_connector.py; echo "$out" | tail -20; fail "the openmetadata example failed"; }
+  rm -f demo_connector.py
+  local want
+  for want in 'autopilot active: False' 'demo_pg ingestion: success' 'demo_connector ingestion: success' \
+      '^custom table: orders$' 'removed demo_pg and demo_connector'; do
+    grep -qE -- "$want" <<<"$out" || { echo "$out" | tail -20; fail "the openmetadata example printed no '$want'"; }
+  done
+  pass "the openmetadata guide's runnable example ran both services and printed its results"
+}
+
+# An admin token, from the guide's login step.
+om_token() {
+  om_step login 'echo "$TOKEN"' 2>/dev/null | tail -1
 }
 
 # The OpenMetadata guide's steps share shell variables: login sets TOKEN, the
@@ -1210,7 +1525,9 @@ om_step() {
 # Feast keeps no feature values of its own, so liveness proves nothing here.
 # This builds a real Iceberg table through the catalog, registers a feature view
 # over it, takes a point-in-time join, materializes to Valkey and reads it back.
-# A failure in any of those is a broken profile even when both containers are up.
+# It also retrieves the nearest rows from a second repository on PostgreSQL with
+# pgvector. A failure in any of those is a broken profile even when both
+# containers are up.
 smoke_feast() {
   retry 60 5 http_ok "http://127.0.0.1:8890/api/v1/projects" || fail "no HTTP response from the feast UI on :8890"
   pass "feast UI answering, registry reachable"
@@ -1225,17 +1542,16 @@ smoke_feast() {
   # the published image carries feast[minimal], which has no iceberg or duckdb.
   uv venv "$work/.venv" >/dev/null 2>&1 || fail "could not create a venv for the feast client"
   VIRTUAL_ENV="$work/.venv" uv pip install -q \
-    "feast[duckdb,iceberg,redis,postgres]==0.66.0" "pyarrow" >/dev/null 2>&1 \
-    || fail "could not install the feast client"
+    "feast[duckdb,iceberg,redis,postgres]==0.66.0" "pyarrow" "mcp>=1.24,<2" >/dev/null 2>&1 \
+    || fail "could not install the feast and mcp clients"
 
-  # Repository from the guide, with warehouse="" in its source. Feast's REST client
-  # puts warehouse into the URL path as a prefix for Polaris and Nessie style
-  # catalogs, and Iceberg's REST fixture server serves the spec with no prefix,
-  # so a real warehouse gives HTTP 400 "Ambiguous URI empty segment".
+  # Repository from the guide, whose source passes warehouse="" so that PyIceberg
+  # takes the warehouse from PYICEBERG_CATALOG__ODCTL__WAREHOUSE.
   cp "$SNIPPETS/feast-iceberg/feature_repo/"* "$work/feature_repo/"
-  # Its own project, because the guide's Runnable Example registers the same
-  # feature view in demo afterwards. feast teardown leaves the view's row in
-  # feature_view_version_history, so a second apply in demo fails on its key.
+  # Its own project, so that the guide's Runnable Example can still register demo.
+  # No feast teardown: it deletes every project in the registry, and it leaves each
+  # view's row in feature_view_version_history, so the next apply of that view fails
+  # on its key. feast delete removes one view and its online values, and the view applies again.
   sed 's/^project: demo$/project: smoke/' "$work/feature_repo/feature_store.yaml" \
     > "$work/feature_store.yaml" && mv "$work/feature_store.yaml" "$work/feature_repo/feature_store.yaml"
 
@@ -1286,11 +1602,17 @@ PYONLINE
     fi
   }
 
+  # A table left by the Runnable Example or by a reader would make the guide's create step fail.
+  local code
+  code=$(curl -sS --max-time 10 -o /dev/null -w '%{http_code}' -X DELETE "http://127.0.0.1:8181/v1/namespaces/demo/tables/driver_stats?purgeRequested=true")
+  case "$code" in 204|404) ;; *) fail "could not delete demo.driver_stats before the guide's create step (HTTP $code)";; esac
   run_step "could not create the Iceberg table through the catalog" "$py" build.py
   pass "Iceberg table created through the REST catalog on SeaweedFS"
 
+  # Delete the view, so that materialise starts again and the check reads this run's values.
+  (cd "$work" && "$feast_bin" -c feature_repo delete driver_stats) >/dev/null 2>&1 || true
   run_step "feast apply or materialize failed" \
-    env PATH="$work/.venv/bin:$PATH" bash -o pipefail -c "$(snippet feast-iceberg/commands.sh apply-materialize)"
+    env PATH="$work/.venv/bin:$PATH" bash -e -o pipefail -c "$(snippet feast-iceberg/commands.sh apply-materialize)"
   pass "feast apply registered the feature view, and materialize wrote to valkey"
 
   run_step "the point-in-time join returned the wrong values" "$py" check.py
@@ -1299,7 +1621,25 @@ PYONLINE
   run_step "the online read from valkey returned the wrong values" "$py" online.py
   pass "get_online_features read the materialised values from valkey"
 
-  (cd "$work" && "$feast_bin" -c feature_repo teardown) >/dev/null 2>&1 || true
+  # Vector retrieval, from the vector page: a second repository, with
+  # PostgreSQL and pgvector as its online store. Its own project, because the
+  # Runnable Example registers vector_demo afterwards.
+  mkdir -p "$work/vector_repo"
+  cp "$SNIPPETS/feast-vector-retrieval/vector_repo/"* "$work/vector_repo/"
+  sed 's/^project: vector_demo$/project: vector_smoke/' "$work/vector_repo/feature_store.yaml" \
+    > "$work/feature_store.yaml" && mv "$work/feature_store.yaml" "$work/vector_repo/feature_store.yaml"
+  snippet feast-vector-retrieval/steps.py write > "$work/vector_write.py"
+  snippet feast-vector-retrieval/steps.py retrieve > "$work/vector_retrieve.py"
+  run_step "feast apply failed on the vector repository" \
+    env PATH="$work/.venv/bin:$PATH" bash -o pipefail -c "$(snippet feast-vector-retrieval/commands.sh apply)"
+  run_step "writing the documents to the PostgreSQL online store failed" "$py" vector_write.py
+  local nearest
+  nearest=$(cd "$work" && "$py" vector_retrieve.py 2>"$log") \
+    || { tail -25 "$log"; fail "retrieve_online_documents_v2 failed"; }
+  [ "$nearest" = "['Feast', 'Iceberg']" ] \
+    || fail "retrieve_online_documents_v2 returned $nearest, not the two nearest documents"
+  pass "retrieve_online_documents_v2 returned the nearest rows from pgvector"
+
   curl -fsS --max-time 10 -X DELETE \
     "http://127.0.0.1:8181/v1/namespaces/demo/tables/driver_stats?purgeRequested=true" >/dev/null 2>&1 || true
   curl -fsS --max-time 10 -X DELETE "http://127.0.0.1:8181/v1/namespaces/demo" >/dev/null 2>&1 || true
@@ -1307,6 +1647,31 @@ PYONLINE
   # case removes $work after that.
 }
 
+# The checks run after the guide's Runnable Example, so they reuse its push
+# definitions in feast-serve; the HTTP push then stores a newer value than the
+# example's Flink push. Read it back until the feature server's cached registry
+# knows the feature view, then read it through the MCP endpoint.
+smoke_feast_push_and_mcp() {
+  local work="${TMPDIR:-/tmp}/odctl-feast-smoke"
+  local out
+  out=$(cd "$work" && run_snippet feast-iceberg/commands.sh push-http 2>&1) \
+    || { echo "$out" | tail -20; fail "the HTTP push to /push failed"; }
+  [ "$out" = "200" ] || { echo "$out" | tail -20; fail "the HTTP push to /push answered '$out', not 200"; }
+  retry 24 5 snippet_says feast-iceberg/commands.sh push-read -F 0.699999988079071 \
+    || fail "the value pushed over HTTP was not read back from /get-online-features"
+  pass "a value pushed over HTTP was read back from the feature server"
+
+  cp "$SNIPPETS/feast-iceberg/mcp_client.py" "$work/"
+  out=$(cd "$work" && "$work/.venv/bin/python" mcp_client.py 2>&1) \
+    || { echo "$out" | tail -20; fail "the MCP client failed"; }
+  grep -qx 'get_online_features_get_online_features_post' <<<"$out" \
+    && grep -qx 'push_push_post' <<<"$out" \
+    || { echo "$out" | tail -20; fail "the MCP endpoint did not list Feast's tools"; }
+  grep -qxF '[0.699999988079071]' <<<"$out" \
+    || { echo "$out" | tail -20; fail "the MCP tool did not return the pushed value"; }
+  pass "an MCP client listed Feast's tools and read the pushed value"
+}
+
 # Temporal: a healthy server proves only that it answers. Workflows run in the
 # caller's worker, so this runs one with the temporalio SDK from the host: an
 # activity that fails once and succeeds on retry, and a workflow that waits for
@@ -1546,11 +1911,26 @@ case "$PROFILE" in
     run_example mlflow '"predictions": \[0, 0\]' "$(mlflow_runner) bash" ;;
   lineage)    smoke_lineage; run_example lineage 'COMPLETED' ;;
   telemetry)  smoke_telemetry; run_example telemetry '"42"' ;;
-  metadata)   smoke_metadata; run_example openmetadata 'removed demo_pg' ;;
+  metadata)
+    smoke_metadata
+    smoke_metadata_example ;;
+  # Two How It Works examples that recreate metadata, each in a group of its own
+  # so that they run in parallel with the checks above and with each other. Each
+  # recreates metadata twice, once for the example and once to restore it.
+  metadata:deps)
+    how_it_works_edit metadata-deps .odctl/.env metadata '^1\.4\.3$' ;;
+  metadata:autopilot)
+    how_it_works_autopilot
+    retry 60 5 http_ok "http://127.0.0.1:8087/api/v2/monitor/health" \
+      || fail "ingestion Airflow did not answer after the package was put back" ;;
   fluss)      smoke_fluss; run_example fluss '\| +3 +\| +c +\|' ;;
   feast)
     smoke_feast
     run_example feast-iceberg '0.8999999761581421' "env PATH=${TMPDIR:-/tmp}/odctl-feast-smoke/.venv/bin:$PATH bash"
+    smoke_feast_push_and_mcp
+    run_example feast-vector-retrieval "\['Feast', 'Iceberg'\]" "env PATH=${TMPDIR:-/tmp}/odctl-feast-smoke/.venv/bin:$PATH bash"
+    # The two examples write their repositories into the current folder, as a reader's would.
+    rm -rf feature_repo vector_repo push_definitions.py
     rm -rf "${TMPDIR:-/tmp}/odctl-feast-smoke" ;;
   evidently)  smoke_evidently; run_example evidently-reports 'report pushed to project' "$(evidently_runner)" ;;
   *)

{odctl-1.2.2 → odctl-1.4.0}/.github/workflows/ci.yml

@@ -20,7 +20,7 @@ jobs:
       - name: Install uv
         # Pinned to a full version on purpose: setup-uv stopped publishing
         # moving major tags at v8, so astral-sh/setup-uv@v10 does not exist.
-        uses: astral-sh/setup-uv@v10.3.1
+        uses: astral-sh/setup-uv@v10.3.2
         with:
           enable-cache: true
           # The default key holds the runner's Python, not the matrix one, so