odctl 1.3.0__tar.gz → 1.4.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (155) hide show
  1. {odctl-1.3.0 → odctl-1.4.0}/.github/scripts/select-profiles.py +23 -7
  2. {odctl-1.3.0 → odctl-1.4.0}/.github/scripts/smoke.sh +173 -16
  3. {odctl-1.3.0 → odctl-1.4.0}/.github/workflows/e2e-test.yml +8 -4
  4. {odctl-1.3.0 → odctl-1.4.0}/CONTRIBUTING.md +2 -2
  5. {odctl-1.3.0 → odctl-1.4.0}/PKG-INFO +2 -2
  6. {odctl-1.3.0 → odctl-1.4.0}/README.md +1 -1
  7. {odctl-1.3.0 → odctl-1.4.0}/mkdocs.yml +4 -0
  8. {odctl-1.3.0 → odctl-1.4.0}/pyproject.toml +1 -1
  9. {odctl-1.3.0 → odctl-1.4.0}/skills/odctl/SKILL.md +2 -0
  10. {odctl-1.3.0 → odctl-1.4.0}/src/odctl/resources/feast/feature_store.yaml +10 -5
  11. {odctl-1.3.0 → odctl-1.4.0}/src/odctl/resources/registry.yml +2 -0
  12. {odctl-1.3.0 → odctl-1.4.0}/tests/test_docs.py +35 -3
  13. {odctl-1.3.0 → odctl-1.4.0}/tests/test_feast_profile.py +11 -0
  14. odctl-1.4.0/tests/test_named_paths.py +62 -0
  15. odctl-1.4.0/tests/test_roadmap.py +55 -0
  16. {odctl-1.3.0 → odctl-1.4.0}/uv.lock +1 -1
  17. {odctl-1.3.0 → odctl-1.4.0}/.github/ISSUE_TEMPLATE/bug_report.md +0 -0
  18. {odctl-1.3.0 → odctl-1.4.0}/.github/ISSUE_TEMPLATE/feature_request.md +0 -0
  19. {odctl-1.3.0 → odctl-1.4.0}/.github/renovate.json +0 -0
  20. {odctl-1.3.0 → odctl-1.4.0}/.github/scripts/spark_avro_producer.py +0 -0
  21. {odctl-1.3.0 → odctl-1.4.0}/.github/workflows/build-deps.yml +0 -0
  22. {odctl-1.3.0 → odctl-1.4.0}/.github/workflows/build-platform-images.yml +0 -0
  23. {odctl-1.3.0 → odctl-1.4.0}/.github/workflows/ci.yml +0 -0
  24. {odctl-1.3.0 → odctl-1.4.0}/.github/workflows/pipeline.yml +0 -0
  25. {odctl-1.3.0 → odctl-1.4.0}/.github/workflows/publish.yml +0 -0
  26. {odctl-1.3.0 → odctl-1.4.0}/.gitignore +0 -0
  27. {odctl-1.3.0 → odctl-1.4.0}/.pre-commit-config.yaml +0 -0
  28. {odctl-1.3.0 → odctl-1.4.0}/.vscode/settings.json +0 -0
  29. {odctl-1.3.0 → odctl-1.4.0}/CLAUDE.md +0 -0
  30. {odctl-1.3.0 → odctl-1.4.0}/LICENSE +0 -0
  31. {odctl-1.3.0 → odctl-1.4.0}/image/demo.gif +0 -0
  32. {odctl-1.3.0 → odctl-1.4.0}/image/demo.png +0 -0
  33. {odctl-1.3.0 → odctl-1.4.0}/image/diagram.drawio +0 -0
  34. {odctl-1.3.0 → odctl-1.4.0}/image/diagram.png +0 -0
  35. {odctl-1.3.0 → odctl-1.4.0}/scripts/diagrams.py +0 -0
  36. {odctl-1.3.0 → odctl-1.4.0}/scripts/gen_pages.py +0 -0
  37. {odctl-1.3.0 → odctl-1.4.0}/scripts/llmstxt_sections.py +0 -0
  38. {odctl-1.3.0 → odctl-1.4.0}/scripts/local_links.py +0 -0
  39. {odctl-1.3.0 → odctl-1.4.0}/scripts/release_links.py +0 -0
  40. {odctl-1.3.0 → odctl-1.4.0}/scripts/release_notes.py +0 -0
  41. {odctl-1.3.0 → odctl-1.4.0}/src/odctl/__init__.py +0 -0
  42. {odctl-1.3.0 → odctl-1.4.0}/src/odctl/config.py +0 -0
  43. {odctl-1.3.0 → odctl-1.4.0}/src/odctl/docker.py +0 -0
  44. {odctl-1.3.0 → odctl-1.4.0}/src/odctl/logs.py +0 -0
  45. {odctl-1.3.0 → odctl-1.4.0}/src/odctl/main.py +0 -0
  46. {odctl-1.3.0 → odctl-1.4.0}/src/odctl/planner.py +0 -0
  47. {odctl-1.3.0 → odctl-1.4.0}/src/odctl/projects.py +0 -0
  48. {odctl-1.3.0 → odctl-1.4.0}/src/odctl/registry.py +0 -0
  49. {odctl-1.3.0 → odctl-1.4.0}/src/odctl/resources/airflow/odctl_log_config.py +0 -0
  50. {odctl-1.3.0 → odctl-1.4.0}/src/odctl/resources/clickhouse/config.xml +0 -0
  51. {odctl-1.3.0 → odctl-1.4.0}/src/odctl/resources/clickhouse/init.sql +0 -0
  52. {odctl-1.3.0 → odctl-1.4.0}/src/odctl/resources/clickhouse/keeper_config.xml +0 -0
  53. {odctl-1.3.0 → odctl-1.4.0}/src/odctl/resources/clickhouse/macros-11-lite.xml +0 -0
  54. {odctl-1.3.0 → odctl-1.4.0}/src/odctl/resources/clickhouse/macros-11.xml +0 -0
  55. {odctl-1.3.0 → odctl-1.4.0}/src/odctl/resources/clickhouse/macros-12-lite.xml +0 -0
  56. {odctl-1.3.0 → odctl-1.4.0}/src/odctl/resources/clickhouse/macros-12.xml +0 -0
  57. {odctl-1.3.0 → odctl-1.4.0}/src/odctl/resources/clickhouse/macros-21.xml +0 -0
  58. {odctl-1.3.0 → odctl-1.4.0}/src/odctl/resources/clickhouse/macros-22.xml +0 -0
  59. {odctl-1.3.0 → odctl-1.4.0}/src/odctl/resources/clickhouse/users.xml +0 -0
  60. {odctl-1.3.0 → odctl-1.4.0}/src/odctl/resources/compose-analytics.yml +0 -0
  61. {odctl-1.3.0 → odctl-1.4.0}/src/odctl/resources/compose-deps.yml +0 -0
  62. {odctl-1.3.0 → odctl-1.4.0}/src/odctl/resources/compose-flink.yml +0 -0
  63. {odctl-1.3.0 → odctl-1.4.0}/src/odctl/resources/compose-infra.yml +0 -0
  64. {odctl-1.3.0 → odctl-1.4.0}/src/odctl/resources/compose-kafka.yml +0 -0
  65. {odctl-1.3.0 → odctl-1.4.0}/src/odctl/resources/compose-metadata.yml +0 -0
  66. {odctl-1.3.0 → odctl-1.4.0}/src/odctl/resources/compose-mlops.yml +0 -0
  67. {odctl-1.3.0 → odctl-1.4.0}/src/odctl/resources/compose-obsv.yml +0 -0
  68. {odctl-1.3.0 → odctl-1.4.0}/src/odctl/resources/compose-orch.yml +0 -0
  69. {odctl-1.3.0 → odctl-1.4.0}/src/odctl/resources/compose-spark.yml +0 -0
  70. {odctl-1.3.0 → odctl-1.4.0}/src/odctl/resources/compose-store.yml +0 -0
  71. {odctl-1.3.0 → odctl-1.4.0}/src/odctl/resources/deps/Dockerfile +0 -0
  72. {odctl-1.3.0 → odctl-1.4.0}/src/odctl/resources/deps/build.gradle +0 -0
  73. {odctl-1.3.0 → odctl-1.4.0}/src/odctl/resources/deps/build_connectors.sh +0 -0
  74. {odctl-1.3.0 → odctl-1.4.0}/src/odctl/resources/deps/fetch_flink_dependencies.sh +0 -0
  75. {odctl-1.3.0 → odctl-1.4.0}/src/odctl/resources/deps/fetch_spark_dependencies.sh +0 -0
  76. {odctl-1.3.0 → odctl-1.4.0}/src/odctl/resources/deps/fetch_standalone_plugins.sh +0 -0
  77. {odctl-1.3.0 → odctl-1.4.0}/src/odctl/resources/deps/versions.env +0 -0
  78. {odctl-1.3.0 → odctl-1.4.0}/src/odctl/resources/docker/airflow/Dockerfile +0 -0
  79. {odctl-1.3.0 → odctl-1.4.0}/src/odctl/resources/docker/mlflow/Dockerfile +0 -0
  80. {odctl-1.3.0 → odctl-1.4.0}/src/odctl/resources/docker/postgres/Dockerfile +0 -0
  81. {odctl-1.3.0 → odctl-1.4.0}/src/odctl/resources/docker/spark/Dockerfile +0 -0
  82. {odctl-1.3.0 → odctl-1.4.0}/src/odctl/resources/evidently/config.yaml +0 -0
  83. {odctl-1.3.0 → odctl-1.4.0}/src/odctl/resources/flink/flink-conf.yml +0 -0
  84. {odctl-1.3.0 → odctl-1.4.0}/src/odctl/resources/grafana/dashboards/airflow.json +0 -0
  85. {odctl-1.3.0 → odctl-1.4.0}/src/odctl/resources/grafana/dashboards/clickhouse.json +0 -0
  86. {odctl-1.3.0 → odctl-1.4.0}/src/odctl/resources/grafana/dashboards/feast.json +0 -0
  87. {odctl-1.3.0 → odctl-1.4.0}/src/odctl/resources/grafana/dashboards/flink.json +0 -0
  88. {odctl-1.3.0 → odctl-1.4.0}/src/odctl/resources/grafana/dashboards/fluss.json +0 -0
  89. {odctl-1.3.0 → odctl-1.4.0}/src/odctl/resources/grafana/dashboards/iceberg-catalog.json +0 -0
  90. {odctl-1.3.0 → odctl-1.4.0}/src/odctl/resources/grafana/dashboards/kafka-connect.json +0 -0
  91. {odctl-1.3.0 → odctl-1.4.0}/src/odctl/resources/grafana/dashboards/kafka.json +0 -0
  92. {odctl-1.3.0 → odctl-1.4.0}/src/odctl/resources/grafana/dashboards/karapace.json +0 -0
  93. {odctl-1.3.0 → odctl-1.4.0}/src/odctl/resources/grafana/dashboards/marquez.json +0 -0
  94. {odctl-1.3.0 → odctl-1.4.0}/src/odctl/resources/grafana/dashboards/metabase.json +0 -0
  95. {odctl-1.3.0 → odctl-1.4.0}/src/odctl/resources/grafana/dashboards/mlflow.json +0 -0
  96. {odctl-1.3.0 → odctl-1.4.0}/src/odctl/resources/grafana/dashboards/openmetadata.json +0 -0
  97. {odctl-1.3.0 → odctl-1.4.0}/src/odctl/resources/grafana/dashboards/postgresql.json +0 -0
  98. {odctl-1.3.0 → odctl-1.4.0}/src/odctl/resources/grafana/dashboards/seaweedfs.json +0 -0
  99. {odctl-1.3.0 → odctl-1.4.0}/src/odctl/resources/grafana/dashboards/spark.json +0 -0
  100. {odctl-1.3.0 → odctl-1.4.0}/src/odctl/resources/grafana/dashboards/temporal.json +0 -0
  101. {odctl-1.3.0 → odctl-1.4.0}/src/odctl/resources/grafana/dashboards/trino.json +0 -0
  102. {odctl-1.3.0 → odctl-1.4.0}/src/odctl/resources/grafana/dashboards/valkey.json +0 -0
  103. {odctl-1.3.0 → odctl-1.4.0}/src/odctl/resources/grafana/odctl-dashboards.yaml +0 -0
  104. {odctl-1.3.0 → odctl-1.4.0}/src/odctl/resources/grafana/sources.yml +0 -0
  105. {odctl-1.3.0 → odctl-1.4.0}/src/odctl/resources/jmx/jmx-exporter.yml +0 -0
  106. {odctl-1.3.0 → odctl-1.4.0}/src/odctl/resources/marquez/marquez.yml +0 -0
  107. {odctl-1.3.0 → odctl-1.4.0}/src/odctl/resources/openmetadata/autopilot_init.py +0 -0
  108. {odctl-1.3.0 → odctl-1.4.0}/src/odctl/resources/openmetadata/connector_sync.py +0 -0
  109. {odctl-1.3.0 → odctl-1.4.0}/src/odctl/resources/otelcol/odctl-receivers.yaml +0 -0
  110. {odctl-1.3.0 → odctl-1.4.0}/src/odctl/resources/postgres/01-init-databases.sh +0 -0
  111. {odctl-1.3.0 → odctl-1.4.0}/src/odctl/resources/prometheus/prometheus.yml +0 -0
  112. {odctl-1.3.0 → odctl-1.4.0}/src/odctl/resources/seaweed/s3.json +0 -0
  113. {odctl-1.3.0 → odctl-1.4.0}/src/odctl/resources/spark/log4j2.properties +0 -0
  114. {odctl-1.3.0 → odctl-1.4.0}/src/odctl/resources/spark/metrics.properties +0 -0
  115. {odctl-1.3.0 → odctl-1.4.0}/src/odctl/resources/spark/spark-defaults.conf +0 -0
  116. {odctl-1.3.0 → odctl-1.4.0}/src/odctl/resources/trino/access-control.properties +0 -0
  117. {odctl-1.3.0 → odctl-1.4.0}/src/odctl/resources/trino/catalog/clickhouse.properties +0 -0
  118. {odctl-1.3.0 → odctl-1.4.0}/src/odctl/resources/trino/catalog/iceberg.properties +0 -0
  119. {odctl-1.3.0 → odctl-1.4.0}/src/odctl/resources/trino/catalog/kafka.properties +0 -0
  120. {odctl-1.3.0 → odctl-1.4.0}/src/odctl/resources/trino/catalog/postgres.properties +0 -0
  121. {odctl-1.3.0 → odctl-1.4.0}/src/odctl/resources/trino/catalog/valkey.properties +0 -0
  122. {odctl-1.3.0 → odctl-1.4.0}/src/odctl/resources/trino/group-provider.properties +0 -0
  123. {odctl-1.3.0 → odctl-1.4.0}/src/odctl/resources/trino/groups.txt +0 -0
  124. {odctl-1.3.0 → odctl-1.4.0}/src/odctl/resources/trino/rules.json +0 -0
  125. {odctl-1.3.0 → odctl-1.4.0}/src/odctl/ui.py +0 -0
  126. {odctl-1.3.0 → odctl-1.4.0}/src/odctl/workspace.py +0 -0
  127. {odctl-1.3.0 → odctl-1.4.0}/tests/conftest.py +0 -0
  128. {odctl-1.3.0 → odctl-1.4.0}/tests/docs_helpers.py +0 -0
  129. {odctl-1.3.0 → odctl-1.4.0}/tests/test_airflow_profile.py +0 -0
  130. {odctl-1.3.0 → odctl-1.4.0}/tests/test_autopilot_init.py +0 -0
  131. {odctl-1.3.0 → odctl-1.4.0}/tests/test_cli.py +0 -0
  132. {odctl-1.3.0 → odctl-1.4.0}/tests/test_cli_extended.py +0 -0
  133. {odctl-1.3.0 → odctl-1.4.0}/tests/test_compose_profiles.py +0 -0
  134. {odctl-1.3.0 → odctl-1.4.0}/tests/test_config.py +0 -0
  135. {odctl-1.3.0 → odctl-1.4.0}/tests/test_confirmed_pages.py +0 -0
  136. {odctl-1.3.0 → odctl-1.4.0}/tests/test_connector_sync.py +0 -0
  137. {odctl-1.3.0 → odctl-1.4.0}/tests/test_diagrams.py +0 -0
  138. {odctl-1.3.0 → odctl-1.4.0}/tests/test_docker.py +0 -0
  139. {odctl-1.3.0 → odctl-1.4.0}/tests/test_evidently_profile.py +0 -0
  140. {odctl-1.3.0 → odctl-1.4.0}/tests/test_feature_docs.py +0 -0
  141. {odctl-1.3.0 → odctl-1.4.0}/tests/test_grafana_dashboards.py +0 -0
  142. {odctl-1.3.0 → odctl-1.4.0}/tests/test_logs.py +0 -0
  143. {odctl-1.3.0 → odctl-1.4.0}/tests/test_metadata_profile.py +0 -0
  144. {odctl-1.3.0 → odctl-1.4.0}/tests/test_metrics_collector.py +0 -0
  145. {odctl-1.3.0 → odctl-1.4.0}/tests/test_metrics_jvm.py +0 -0
  146. {odctl-1.3.0 → odctl-1.4.0}/tests/test_metrics_python_services.py +0 -0
  147. {odctl-1.3.0 → odctl-1.4.0}/tests/test_planner.py +0 -0
  148. {odctl-1.3.0 → odctl-1.4.0}/tests/test_postgres_image.py +0 -0
  149. {odctl-1.3.0 → odctl-1.4.0}/tests/test_projects.py +0 -0
  150. {odctl-1.3.0 → odctl-1.4.0}/tests/test_registry.py +0 -0
  151. {odctl-1.3.0 → odctl-1.4.0}/tests/test_release_links.py +0 -0
  152. {odctl-1.3.0 → odctl-1.4.0}/tests/test_release_notes.py +0 -0
  153. {odctl-1.3.0 → odctl-1.4.0}/tests/test_telemetry_profile.py +0 -0
  154. {odctl-1.3.0 → odctl-1.4.0}/tests/test_temporal_profile.py +0 -0
  155. {odctl-1.3.0 → odctl-1.4.0}/tests/test_workspace.py +0 -0
@@ -45,6 +45,11 @@ GROUPS = [
45
45
  "lineage",
46
46
  "telemetry",
47
47
  "metadata",
48
+ # The part before the colon is the profile to start. metadata:deps and
49
+ # metadata:autopilot start the same profile as metadata. Each examples group
50
+ # recreates metadata twice, which is why each runs alone, in parallel with the others.
51
+ "metadata:deps",
52
+ "metadata:autopilot",
48
53
  "airflow",
49
54
  "temporal",
50
55
  "postgres",
@@ -70,13 +75,14 @@ REGISTRY = Path("src/odctl/resources/registry.yml")
70
75
 
71
76
  # smoke.sh starts some profiles from another group's case: the kafka case starts
72
77
  # catalog, the flink case starts kafka-lite and feast, the spark case starts
73
- # kafka-lite, and the fluss and feast cases start flink-lite. A change to that
74
- # compose file needs these groups as well as the ones the registry maps to it.
78
+ # kafka-lite, the fluss and feast cases start flink-lite, and the airflow case
79
+ # starts feast. A change to that compose file needs these groups as well as the
80
+ # ones the registry maps to it.
75
81
  ALSO_RUNS = {
76
82
  "compose-infra.yml": ["kafka-lite kafka-full"],
77
83
  "compose-kafka.yml": ["flink-lite flink-full", "spark-lite spark-full"],
78
84
  "compose-flink.yml": ["fluss", "feast"],
79
- "compose-mlops.yml": ["flink-lite flink-full"],
85
+ "compose-mlops.yml": ["flink-lite flink-full", "airflow"],
80
86
  }
81
87
 
82
88
  # smoke.sh runs the guide commands in docs/snippets/<guide>/, so a change there
@@ -85,7 +91,15 @@ ALSO_RUNS = {
85
91
  SNIPPET_PREFIX = "docs/snippets/"
86
92
  SNIPPET_GROUPS = {
87
93
  "getting-started": ["kafka-lite kafka-full"],
88
- "how-it-works": ["storage", "catalog", "trino", "airflow", "metadata"],
94
+ "how-it-works": [
95
+ "storage",
96
+ "catalog",
97
+ "trino",
98
+ "airflow",
99
+ "mlflow",
100
+ "metadata:deps",
101
+ "metadata:autopilot",
102
+ ],
89
103
  "kafka": ["kafka-lite kafka-full"],
90
104
  "kafka-connect": ["kafka-lite kafka-full"],
91
105
  "iceberg-ingestion": ["kafka-lite kafka-full", "flink-lite flink-full"],
@@ -100,10 +114,11 @@ SNIPPET_GROUPS = {
100
114
  "airflow-dags": ["airflow"],
101
115
  "temporal-workers": ["temporal"],
102
116
  "mlflow": ["mlflow"],
103
- "feast-iceberg": ["feast", "flink-lite flink-full", "mlflow"],
117
+ "feast-iceberg": ["feast", "flink-lite flink-full", "mlflow", "airflow"],
118
+ "feast-vector-retrieval": ["feast"],
104
119
  "evidently-reports": ["evidently"],
105
120
  "lineage": ["lineage"],
106
- "openmetadata": ["metadata"],
121
+ "openmetadata": ["metadata", "metadata:autopilot"],
107
122
  "telemetry": ["telemetry"],
108
123
  }
109
124
 
@@ -174,7 +189,8 @@ def select(changed: list, registry_path: Path = REGISTRY) -> list:
174
189
 
175
190
  selected = []
176
191
  for group in GROUPS:
177
- files = {mapping.get(p) for p in group.split()}
192
+ # A group name may carry a suffix after a colon, as metadata:deps does.
193
+ files = {mapping.get(p.split(":")[0]) for p in group.split()}
178
194
  also = any(group in ALSO_RUNS.get(name, []) for name in touched_files)
179
195
  if files & touched_files or group in snippet_groups or also:
180
196
  selected.append(group)
@@ -9,10 +9,10 @@
9
9
  # committed nothing while its checkpoints succeeded. Each assertion below exists
10
10
  # because something shipped broken past a liveness check.
11
11
  #
12
- # Usage: smoke.sh <profile>
12
+ # Usage: smoke.sh <profile or group name, such as metadata:deps>
13
13
  set -uo pipefail
14
14
 
15
- PROFILE="${1:?usage: smoke.sh <profile>}"
15
+ PROFILE="${1:?usage: smoke.sh <profile or group name, such as metadata:deps>}"
16
16
  MB_USER="smoke@example.com"
17
17
  MB_PASS="Smoke-test-1234"
18
18
 
@@ -880,6 +880,104 @@ if bad:
880
880
 
881
881
  # Airflow pushes OTLP to the telemetry collector rather than being scraped.
882
882
  assert_pushed_metrics 'airflow_.+' Airflow
883
+
884
+ smoke_airflow_materialize
885
+ }
886
+
887
+ # Feast guide's On a Schedule section: a DAG that builds a FeatureStore from
888
+ # FEAST_REGISTRY and calls materialize_incremental. It needs the registry entry and
889
+ # the Iceberg table the guide's earlier steps make, so this runs those from the
890
+ # host as the guide does, then uploads the DAG, runs it and reads the value it
891
+ # copied into Valkey. The feature view source has no catalog address, so the
892
+ # registry entry works from the host here and from the Airflow container in the DAG.
893
+ # Its own project, so that the guide's Runnable Example can still register demo.
894
+ smoke_airflow_materialize() {
895
+ odctl up feast >/dev/null 2>&1 || fail "could not start feast for the materialise DAG"
896
+ retry 30 5 http_ok "http://127.0.0.1:6566/health" || fail "Feast never answered on :6566"
897
+
898
+ local work="${TMPDIR:-/tmp}/odctl-airflow-feast-smoke"
899
+ rm -rf "$work"; mkdir -p "$work"
900
+ uv venv "$work/.venv" >/dev/null 2>&1 || fail "could not create a venv for the feast client"
901
+ VIRTUAL_ENV="$work/.venv" uv pip install -q \
902
+ "feast[duckdb,iceberg,redis,postgres]==0.66.0" "pyarrow" >/dev/null 2>&1 \
903
+ || fail "could not install the feast client"
904
+
905
+ cp -r "$SNIPPETS/feast-iceberg/feature_repo" "$work/"
906
+ sed 's/^project: demo$/project: airflow_smoke/' "$work/feature_repo/feature_store.yaml" \
907
+ > "$work/feature_store.yaml" && mv "$work/feature_store.yaml" "$work/feature_repo/feature_store.yaml"
908
+ sed 's/project="demo"/project="airflow_smoke"/' "$SNIPPETS/feast-iceberg/feast_materialize.py" \
909
+ > "$work/feast_materialize.py"
910
+ snippet feast-iceberg/steps.py create-table > "$work/build.py"
911
+ snippet feast-iceberg/steps.py append > "$work/append.py"
912
+ snippet feast-iceberg/steps.py read > "$work/read.py"
913
+
914
+ local py="$work/.venv/bin/python" log="$work/step.log"
915
+ run_step() {
916
+ local what="$1"; shift
917
+ if ! (cd "$work" && "$@") >"$log" 2>&1; then
918
+ echo "---- $what ----"
919
+ tail -25 "$log"
920
+ fail "$what"
921
+ fi
922
+ }
923
+
924
+ # A table left by the Runnable Example would make the guide's create step fail.
925
+ local code
926
+ code=$(curl -sS --max-time 10 -o /dev/null -w '%{http_code}' -X DELETE "http://127.0.0.1:8181/v1/namespaces/demo/tables/driver_stats?purgeRequested=true")
927
+ case "$code" in 204|404) ;; *) fail "could not delete demo.driver_stats before the guide's create step (HTTP $code)";; esac
928
+ eval "$(snippet feast-iceberg/commands.sh catalog-env)"
929
+ run_step "could not create the Iceberg table through the catalog" "$py" build.py
930
+ # Delete the view, so that materialise starts again and the check reads this run's values.
931
+ (cd "$work" && "$work/.venv/bin/feast" -c feature_repo delete driver_stats) >/dev/null 2>&1 || true
932
+ run_step "feast apply or materialize failed without a catalog address in the source" \
933
+ env PATH="$work/.venv/bin:$PATH" bash -e -o pipefail -c "$(snippet feast-iceberg/commands.sh apply-materialize)"
934
+ pass "the guide's source, with no catalog address, applied and materialised from the host"
935
+
936
+ run_step "could not upload the materialise DAG to s3://airflow/dags" \
937
+ run_snippet feast-iceberg/commands.sh dag-upload
938
+ retry 30 10 bash -c \
939
+ "docker exec airflow airflow dags list 2>/dev/null | grep -q feast_materialize" \
940
+ || fail "the S3 DAG bundle never delivered the materialise DAG, or it was never parsed"
941
+
942
+ # The new row is newer than the host's materialise, so only the DAG can copy it.
943
+ run_step "could not append a row to the Iceberg table" "$py" append.py
944
+ run_step "the guide's unpause and trigger commands failed" \
945
+ run_snippet feast-iceberg/commands.sh dag-run
946
+
947
+ # Unpausing also starts a scheduled run, so wait until every run has finished.
948
+ local states=""
949
+ for _ in $(seq 1 30); do
950
+ states=$(docker exec airflow airflow dags list-runs feast_materialize -o plain 2>/dev/null \
951
+ | grep -E "^feast_materialize" | tr -s ' ' | cut -d' ' -f3)
952
+ grep -qE 'queued|running' <<<"$states" || [ -z "$states" ] || break
953
+ sleep 10
954
+ done
955
+ [ -n "$states" ] && ! grep -qvx success <<<"$states" \
956
+ || fail "the materialise DAG runs ended in states: ${states:-none}"
957
+ pass "the materialise DAG ran to success inside the Airflow container"
958
+
959
+ local got
960
+ got=$(cd "$work" && "$py" read.py 2>"$log") || { tail -25 "$log"; fail "the online read failed"; }
961
+ [ "$got" = "{'driver_id': [1001, 1002], 'conv_rate': [0.6000000238418579, 0.75]}" ] \
962
+ || fail "Valkey holds $got, not the 0.6 the DAG should have copied"
963
+ pass "the DAG copied the new value from Iceberg into Valkey"
964
+
965
+ # Leave no hourly run behind: a DAG that stays unpaused fails on its next run once
966
+ # the table below is gone, and the next smoke run would read that failed run.
967
+ docker exec airflow airflow dags pause feast_materialize >/dev/null 2>&1 \
968
+ || fail "could not pause the materialise DAG"
969
+ docker exec airflow python -c '
970
+ import boto3
971
+ boto3.client("s3", endpoint_url="http://seaweed:8333").delete_object(
972
+ Bucket="airflow", Key="dags/feast_materialize.py")' >/dev/null 2>&1 \
973
+ || fail "could not delete the materialise DAG from s3://airflow/dags"
974
+
975
+ # No feast teardown: it leaves the view's row in feature_view_version_history, and
976
+ # a second run of this check would then fail to apply the view again.
977
+ curl -fsS --max-time 10 -X DELETE \
978
+ "http://127.0.0.1:8181/v1/namespaces/demo/tables/driver_stats?purgeRequested=true" >/dev/null 2>&1 || true
979
+ curl -fsS --max-time 10 -X DELETE "http://127.0.0.1:8181/v1/namespaces/demo" >/dev/null 2>&1 || true
980
+ rm -rf "$work"
883
981
  }
884
982
 
885
983
  # deps is a one-shot copy into a shared volume, so there is no container to
@@ -1427,7 +1525,9 @@ om_step() {
1427
1525
  # Feast keeps no feature values of its own, so liveness proves nothing here.
1428
1526
  # This builds a real Iceberg table through the catalog, registers a feature view
1429
1527
  # over it, takes a point-in-time join, materializes to Valkey and reads it back.
1430
- # A failure in any of those is a broken profile even when both containers are up.
1528
+ # It also retrieves the nearest rows from a second repository on PostgreSQL with
1529
+ # pgvector. A failure in any of those is a broken profile even when both
1530
+ # containers are up.
1431
1531
  smoke_feast() {
1432
1532
  retry 60 5 http_ok "http://127.0.0.1:8890/api/v1/projects" || fail "no HTTP response from the feast UI on :8890"
1433
1533
  pass "feast UI answering, registry reachable"
@@ -1442,17 +1542,16 @@ smoke_feast() {
1442
1542
  # the published image carries feast[minimal], which has no iceberg or duckdb.
1443
1543
  uv venv "$work/.venv" >/dev/null 2>&1 || fail "could not create a venv for the feast client"
1444
1544
  VIRTUAL_ENV="$work/.venv" uv pip install -q \
1445
- "feast[duckdb,iceberg,redis,postgres]==0.66.0" "pyarrow" >/dev/null 2>&1 \
1446
- || fail "could not install the feast client"
1545
+ "feast[duckdb,iceberg,redis,postgres]==0.66.0" "pyarrow" "mcp>=1.24,<2" >/dev/null 2>&1 \
1546
+ || fail "could not install the feast and mcp clients"
1447
1547
 
1448
- # Repository from the guide, with warehouse="" in its source. Feast's REST client
1449
- # puts warehouse into the URL path as a prefix for Polaris and Nessie style
1450
- # catalogs, and Iceberg's REST fixture server serves the spec with no prefix,
1451
- # so a real warehouse gives HTTP 400 "Ambiguous URI empty segment".
1548
+ # Repository from the guide, whose source passes warehouse="" so that PyIceberg
1549
+ # takes the warehouse from PYICEBERG_CATALOG__ODCTL__WAREHOUSE.
1452
1550
  cp "$SNIPPETS/feast-iceberg/feature_repo/"* "$work/feature_repo/"
1453
- # Its own project, because the guide's Runnable Example registers the same
1454
- # feature view in demo afterwards. feast teardown leaves the view's row in
1455
- # feature_view_version_history, so a second apply in demo fails on its key.
1551
+ # Its own project, so that the guide's Runnable Example can still register demo.
1552
+ # No feast teardown: it deletes every project in the registry, and it leaves each
1553
+ # view's row in feature_view_version_history, so the next apply of that view fails
1554
+ # on its key. feast delete removes one view and its online values, and the view applies again.
1456
1555
  sed 's/^project: demo$/project: smoke/' "$work/feature_repo/feature_store.yaml" \
1457
1556
  > "$work/feature_store.yaml" && mv "$work/feature_store.yaml" "$work/feature_repo/feature_store.yaml"
1458
1557
 
@@ -1503,11 +1602,17 @@ PYONLINE
1503
1602
  fi
1504
1603
  }
1505
1604
 
1605
+ # A table left by the Runnable Example or by a reader would make the guide's create step fail.
1606
+ local code
1607
+ code=$(curl -sS --max-time 10 -o /dev/null -w '%{http_code}' -X DELETE "http://127.0.0.1:8181/v1/namespaces/demo/tables/driver_stats?purgeRequested=true")
1608
+ case "$code" in 204|404) ;; *) fail "could not delete demo.driver_stats before the guide's create step (HTTP $code)";; esac
1506
1609
  run_step "could not create the Iceberg table through the catalog" "$py" build.py
1507
1610
  pass "Iceberg table created through the REST catalog on SeaweedFS"
1508
1611
 
1612
+ # Delete the view, so that materialise starts again and the check reads this run's values.
1613
+ (cd "$work" && "$feast_bin" -c feature_repo delete driver_stats) >/dev/null 2>&1 || true
1509
1614
  run_step "feast apply or materialize failed" \
1510
- env PATH="$work/.venv/bin:$PATH" bash -o pipefail -c "$(snippet feast-iceberg/commands.sh apply-materialize)"
1615
+ env PATH="$work/.venv/bin:$PATH" bash -e -o pipefail -c "$(snippet feast-iceberg/commands.sh apply-materialize)"
1511
1616
  pass "feast apply registered the feature view, and materialize wrote to valkey"
1512
1617
 
1513
1618
  run_step "the point-in-time join returned the wrong values" "$py" check.py
@@ -1516,7 +1621,25 @@ PYONLINE
1516
1621
  run_step "the online read from valkey returned the wrong values" "$py" online.py
1517
1622
  pass "get_online_features read the materialised values from valkey"
1518
1623
 
1519
- (cd "$work" && "$feast_bin" -c feature_repo teardown) >/dev/null 2>&1 || true
1624
+ # Vector retrieval, from the vector page: a second repository, with
1625
+ # PostgreSQL and pgvector as its online store. Its own project, because the
1626
+ # Runnable Example registers vector_demo afterwards.
1627
+ mkdir -p "$work/vector_repo"
1628
+ cp "$SNIPPETS/feast-vector-retrieval/vector_repo/"* "$work/vector_repo/"
1629
+ sed 's/^project: vector_demo$/project: vector_smoke/' "$work/vector_repo/feature_store.yaml" \
1630
+ > "$work/feature_store.yaml" && mv "$work/feature_store.yaml" "$work/vector_repo/feature_store.yaml"
1631
+ snippet feast-vector-retrieval/steps.py write > "$work/vector_write.py"
1632
+ snippet feast-vector-retrieval/steps.py retrieve > "$work/vector_retrieve.py"
1633
+ run_step "feast apply failed on the vector repository" \
1634
+ env PATH="$work/.venv/bin:$PATH" bash -o pipefail -c "$(snippet feast-vector-retrieval/commands.sh apply)"
1635
+ run_step "writing the documents to the PostgreSQL online store failed" "$py" vector_write.py
1636
+ local nearest
1637
+ nearest=$(cd "$work" && "$py" vector_retrieve.py 2>"$log") \
1638
+ || { tail -25 "$log"; fail "retrieve_online_documents_v2 failed"; }
1639
+ [ "$nearest" = "['Feast', 'Iceberg']" ] \
1640
+ || fail "retrieve_online_documents_v2 returned $nearest, not the two nearest documents"
1641
+ pass "retrieve_online_documents_v2 returned the nearest rows from pgvector"
1642
+
1520
1643
  curl -fsS --max-time 10 -X DELETE \
1521
1644
  "http://127.0.0.1:8181/v1/namespaces/demo/tables/driver_stats?purgeRequested=true" >/dev/null 2>&1 || true
1522
1645
  curl -fsS --max-time 10 -X DELETE "http://127.0.0.1:8181/v1/namespaces/demo" >/dev/null 2>&1 || true
@@ -1524,6 +1647,31 @@ PYONLINE
1524
1647
  # case removes $work after that.
1525
1648
  }
1526
1649
 
1650
+ # The checks run after the guide's Runnable Example, so they reuse its push
1651
+ # definitions in feast-serve; the HTTP push then stores a newer value than the
1652
+ # example's Flink push. Read it back until the feature server's cached registry
1653
+ # knows the feature view, then read it through the MCP endpoint.
1654
+ smoke_feast_push_and_mcp() {
1655
+ local work="${TMPDIR:-/tmp}/odctl-feast-smoke"
1656
+ local out
1657
+ out=$(cd "$work" && run_snippet feast-iceberg/commands.sh push-http 2>&1) \
1658
+ || { echo "$out" | tail -20; fail "the HTTP push to /push failed"; }
1659
+ [ "$out" = "200" ] || { echo "$out" | tail -20; fail "the HTTP push to /push answered '$out', not 200"; }
1660
+ retry 24 5 snippet_says feast-iceberg/commands.sh push-read -F 0.699999988079071 \
1661
+ || fail "the value pushed over HTTP was not read back from /get-online-features"
1662
+ pass "a value pushed over HTTP was read back from the feature server"
1663
+
1664
+ cp "$SNIPPETS/feast-iceberg/mcp_client.py" "$work/"
1665
+ out=$(cd "$work" && "$work/.venv/bin/python" mcp_client.py 2>&1) \
1666
+ || { echo "$out" | tail -20; fail "the MCP client failed"; }
1667
+ grep -qx 'get_online_features_get_online_features_post' <<<"$out" \
1668
+ && grep -qx 'push_push_post' <<<"$out" \
1669
+ || { echo "$out" | tail -20; fail "the MCP endpoint did not list Feast's tools"; }
1670
+ grep -qxF '[0.699999988079071]' <<<"$out" \
1671
+ || { echo "$out" | tail -20; fail "the MCP tool did not return the pushed value"; }
1672
+ pass "an MCP client listed Feast's tools and read the pushed value"
1673
+ }
1674
+
1527
1675
  # Temporal: a healthy server proves only that it answers. Workflows run in the
1528
1676
  # caller's worker, so this runs one with the temporalio SDK from the host: an
1529
1677
  # activity that fails once and succeeds on retry, and a workflow that waits for
@@ -1765,8 +1913,13 @@ case "$PROFILE" in
1765
1913
  telemetry) smoke_telemetry; run_example telemetry '"42"' ;;
1766
1914
  metadata)
1767
1915
  smoke_metadata
1768
- smoke_metadata_example
1769
- how_it_works_edit metadata-deps .odctl/.env metadata '^1\.4\.3$'
1916
+ smoke_metadata_example ;;
1917
+ # Two How It Works examples that recreate metadata, each in a group of its own
1918
+ # so that they run in parallel with the checks above and with each other. Each
1919
+ # recreates metadata twice, once for the example and once to restore it.
1920
+ metadata:deps)
1921
+ how_it_works_edit metadata-deps .odctl/.env metadata '^1\.4\.3$' ;;
1922
+ metadata:autopilot)
1770
1923
  how_it_works_autopilot
1771
1924
  retry 60 5 http_ok "http://127.0.0.1:8087/api/v2/monitor/health" \
1772
1925
  || fail "ingestion Airflow did not answer after the package was put back" ;;
@@ -1774,6 +1927,10 @@ case "$PROFILE" in
1774
1927
  feast)
1775
1928
  smoke_feast
1776
1929
  run_example feast-iceberg '0.8999999761581421' "env PATH=${TMPDIR:-/tmp}/odctl-feast-smoke/.venv/bin:$PATH bash"
1930
+ smoke_feast_push_and_mcp
1931
+ run_example feast-vector-retrieval "\['Feast', 'Iceberg'\]" "env PATH=${TMPDIR:-/tmp}/odctl-feast-smoke/.venv/bin:$PATH bash"
1932
+ # The two examples write their repositories into the current folder, as a reader's would.
1933
+ rm -rf feature_repo vector_repo push_definitions.py
1777
1934
  rm -rf "${TMPDIR:-/tmp}/odctl-feast-smoke" ;;
1778
1935
  evidently) smoke_evidently; run_example evidently-reports 'report pushed to project' "$(evidently_runner)" ;;
1779
1936
  *)
@@ -89,9 +89,13 @@ jobs:
89
89
  run: |
90
90
  chmod +x .github/scripts/smoke.sh
91
91
  for profile in ${{ matrix.profile_group }}; do
92
+ # A name such as metadata:deps is a group of its own for the same
93
+ # profile: odctl starts the part before the colon, and smoke.sh
94
+ # gets the whole name to choose its checks.
95
+ base="${profile%%:*}"
92
96
  echo "::group::Testing Profile: $profile"
93
- echo "🚀 Launching $profile..."
94
- odctl up "$profile"
97
+ echo "🚀 Launching $base..."
98
+ odctl up "$base"
95
99
 
96
100
  echo "⏳ Containers:"
97
101
  docker ps
@@ -116,8 +120,8 @@ jobs:
116
120
  # named teardown did not until 0.4.3, so compose rejected the
117
121
  # project with "depends on undefined service" and only the named
118
122
  # form could catch it. Naming the profile is also what a user runs.
119
- echo "🧹 Tearing down $profile by name..."
120
- yes | odctl down "$profile" --volumes
123
+ echo "🧹 Tearing down $base by name..."
124
+ yes | odctl down "$base" --volumes
121
125
 
122
126
  # Sweep afterwards so a failure above cannot leak state into the
123
127
  # next profile in the group. This is a no-op when the named
@@ -55,7 +55,7 @@ Each layer checks one thing:
55
55
 
56
56
  Run the unit suite in full once before each commit. While fixing, run only the failed tests (`pytest --lf`) and the tests of the files you changed. Run the strict build when `docs/`, `mkdocs.yml` or an input of `scripts/gen_pages.py` changed: `registry.yml`, a compose file, or a command's `--help` text.
57
57
 
58
- A pull request runs pre-commit, the unit tests on each supported Python version, the strict docs build, and the image builds when an image input changed. e2e runs only on a tag and on a manual run, because odctl runs images tagged with its own version, and the images of an unreleased version do not exist yet. To check a profile on your machine, do what the e2e job does: install the CLI with `uv tool install .`, run `odctl init`, start the profile with `odctl up <profile>`, then run `.github/scripts/smoke.sh <profile>`. Run one `odctl up` at a time. When no image definition changed since the last release, you can give the last release's images the new version's tag for the test, and remove those tags afterwards.
58
+ A pull request runs pre-commit, the unit tests on each supported Python version, the strict docs build, and the image builds when an image input changed. e2e runs only on a tag and on a manual run, because odctl runs images tagged with its own version, and the images of an unreleased version do not exist yet. To check a profile on your machine, do what the e2e job does: install the CLI with `uv tool install .`, run `odctl init`, start the profile with `odctl up <profile>`, then run `.github/scripts/smoke.sh <profile>`. For a group name with a colon, such as `metadata:deps`, the e2e job starts the profile before the colon (`odctl up metadata`) and passes the whole name to `smoke.sh` (`smoke.sh metadata:deps`). Run one `odctl up` at a time. When no image definition changed since the last release, you can give the last release's images the new version's tag for the test, and remove those tags afterwards.
59
59
 
60
60
  ## Rules
61
61
 
@@ -94,6 +94,6 @@ Work on a branch from `main` and open a pull request against `main`. Before you
94
94
 
95
95
  The maintainer releases. These steps are written down so that each release is the same:
96
96
 
97
- 1. Bump `version` in `pyproject.toml`, run `uv lock`, and add the release to `docs/about/changelog.md`, with an Upgrading line for any change to ports, profiles, `.env` keys, volumes or workspace files. When it reaches `main`, the docs deploy as that version, aliased `latest`.
97
+ 1. Bump `version` in `pyproject.toml`, run `uv lock`, and add the release to `docs/about/changelog.md`, with an Upgrading line for any change to ports, profiles, `.env` keys, volumes or workspace files. The same commit updates the public roadmap, `docs/about/roadmap.md`: the release moves from Next Releases to Released, the next release becomes the current entry (`key: now`), and every planned entry names all the issues planned for it; `tests/test_roadmap.py` fails otherwise. When it reaches `main`, the docs deploy as that version, aliased `latest`.
98
98
  2. Once the `main` pipeline for that commit is green, push tag `vX.Y.Z` on it. A tag on a commit whose checks fail builds no images and cannot be released until it is moved. The tag pipeline builds and pushes the `X.Y.Z` images, then runs e2e for every profile group (about an hour).
99
99
  3. Publish the GitHub release with that version's Changelog section as its text, so the two never differ: `python scripts/release_notes.py X.Y.Z > /tmp/notes.md`, then `gh release create vX.Y.Z --title vX.Y.Z --notes-file /tmp/notes.md`. `publish.yml` waits for a green tag pipeline and checks that every `X.Y.Z` image is in ghcr, then uploads to PyPI.
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.5
2
2
  Name: odctl
3
- Version: 1.3.0
3
+ Version: 1.4.0
4
4
  Summary: Run an open data stack on your machine, with its services already connected, from one command.
5
5
  Project-URL: Homepage, https://github.com/jaehyeon-kim/odctl
6
6
  Project-URL: Repository, https://github.com/jaehyeon-kim/odctl
@@ -94,7 +94,7 @@ Each command has its own section in the CLI reference, with its options and exam
94
94
 
95
95
  ## Roadmap
96
96
 
97
- odctl follows semantic versioning from 1.0, so a later 1.x release does not break a project that pins `odctl>=1.0,<2`. The next releases add an MCP endpoint and streaming features for Feast, custom OpenMetadata connectors, a written list of what 1.x keeps stable with JSON output, an MCP server for AI agents, a Langfuse profile, and snapshot and restore of a profile's data. The [roadmap page](https://jaehyeon.me/odctl/latest/about/roadmap/) shows each step in order.
97
+ odctl follows semantic versioning from 1.0, so a later 1.x release does not break a project that pins `odctl>=1.0,<2`. The next releases add a written list of what 1.x keeps stable with JSON output, an MCP server for AI agents, a Langfuse profile, and lineage and alert rules. The [roadmap page](https://jaehyeon.me/odctl/latest/about/roadmap/) shows each step in order.
98
98
 
99
99
  ## Related Reading
100
100
 
@@ -57,7 +57,7 @@ Each command has its own section in the CLI reference, with its options and exam
57
57
 
58
58
  ## Roadmap
59
59
 
60
- odctl follows semantic versioning from 1.0, so a later 1.x release does not break a project that pins `odctl>=1.0,<2`. The next releases add an MCP endpoint and streaming features for Feast, custom OpenMetadata connectors, a written list of what 1.x keeps stable with JSON output, an MCP server for AI agents, a Langfuse profile, and snapshot and restore of a profile's data. The [roadmap page](https://jaehyeon.me/odctl/latest/about/roadmap/) shows each step in order.
60
+ odctl follows semantic versioning from 1.0, so a later 1.x release does not break a project that pins `odctl>=1.0,<2`. The next releases add a written list of what 1.x keeps stable with JSON output, an MCP server for AI agents, a Langfuse profile, and lineage and alert rules. The [roadmap page](https://jaehyeon.me/odctl/latest/about/roadmap/) shows each step in order.
61
61
 
62
62
  ## Related Reading
63
63
 
@@ -67,6 +67,9 @@ exclude_docs: |
67
67
  # A link to a heading that does not exist is a warning, which --strict fails.
68
68
  validation:
69
69
  anchors: warn
70
+ nav:
71
+ omitted_files: warn
72
+ not_found: warn
70
73
 
71
74
  extra_css:
72
75
  - stylesheets/extra.css
@@ -109,6 +112,7 @@ nav:
109
112
  - Temporal Workers: guides/temporal-workers.md
110
113
  - ML and AI:
111
114
  - Feast with Iceberg: guides/feast-iceberg.md
115
+ - Feast Vector Retrieval: guides/feast-vector-retrieval.md
112
116
  - MLflow: guides/mlflow.md
113
117
  - Evidently Reports: guides/evidently-reports.md
114
118
  - Hybrid Search: guides/hybrid-search.md
@@ -1,6 +1,6 @@
1
1
  [project]
2
2
  name = "odctl"
3
- version = "1.3.0"
3
+ version = "1.4.0"
4
4
  description = "Run an open data stack on your machine, with its services already connected, from one command."
5
5
  readme = "README.md"
6
6
  authors = [
@@ -369,6 +369,8 @@ Feast: Experiment tracking, model registry and model serving (MLflow), the featu
369
369
  • Online feature server for get_online_features:
370
370
  * External (host): http://127.0.0.1:6566
371
371
  * Internal (Docker): http://feast-serve:6566
372
+ • MCP endpoint, for agents that list Feast's tools and read features:
373
+ http://127.0.0.1:6566/mcp
372
374
  • Liveness: http://127.0.0.1:6566/health
373
375
  ```
374
376
 
@@ -8,11 +8,8 @@
8
8
  # half runs in the caller's process and the published image has feast[minimal].
9
9
  #
10
10
  # On an IcebergSource against the catalog profile, pass warehouse="" and
11
- # catalog_name="odctl". Feast's REST client puts warehouse into the URL path as
12
- # a prefix for Polaris and Nessie style catalogs, and Iceberg's REST fixture server
13
- # serves the specification with no prefix, so a real warehouse returns
14
- # HTTP 400 "Ambiguous URI empty segment". PyIceberg still gets the real
15
- # warehouse, from the PYICEBERG_CATALOG__ODCTL__* variables in compose-mlops.yml.
11
+ # catalog_name="odctl", so that PyIceberg takes the catalog address and the
12
+ # warehouse from the PYICEBERG_CATALOG__ODCTL__* variables in compose-mlops.yml.
16
13
  project: odctl
17
14
  provider: local
18
15
 
@@ -33,4 +30,12 @@ online_store:
33
30
  connection_string: "valkey:6379,username=user,password=password"
34
31
  key_ttl_seconds: 604800
35
32
 
33
+ # MCP endpoint at /mcp on the feature server, so an agent can list Feast's tools
34
+ # and read features. http is Streamable HTTP, which the MCP specification of
35
+ # 2025-03-26 put in place of the older HTTP with SSE transport (sse).
36
+ feature_server:
37
+ type: mcp
38
+ mcp_enabled: true
39
+ mcp_transport: http
40
+
36
41
  entity_key_serialization_version: 3
@@ -269,6 +269,8 @@ stacks:
269
269
  • Online feature server for get_online_features:
270
270
  * External (host): http://127.0.0.1:6566
271
271
  * Internal (Docker): http://feast-serve:6566
272
+ • MCP endpoint, for agents that list Feast's tools and read features:
273
+ http://127.0.0.1:6566/mcp
272
274
  • Liveness: http://127.0.0.1:6566/health
273
275
  evidently: |
274
276
  • Evidently UI & API (no login, and writes are open):
@@ -285,8 +285,8 @@ def test_snippet_changes_select_the_groups_that_run_them():
285
285
  assert "spark-lite spark-full" in selector.select(
286
286
  ["src/odctl/resources/compose-kafka.yml"], registry
287
287
  )
288
- assert "flink-lite flink-full" in selector.select(
289
- ["src/odctl/resources/compose-mlops.yml"], registry
288
+ assert {"flink-lite flink-full", "airflow"} <= set(
289
+ selector.select(["src/odctl/resources/compose-mlops.yml"], registry)
290
290
  )
291
291
  assert "flink-lite flink-full" in selector.select(
292
292
  ["src/odctl/resources/compose-kafka.yml"], registry
@@ -305,7 +305,7 @@ def test_snippet_changes_select_the_groups_that_run_them():
305
305
  ]
306
306
  assert set(
307
307
  selector.select(["docs/snippets/feast-iceberg/steps.sql"], registry)
308
- ) == {"feast", "flink-lite flink-full", "mlflow"}
308
+ ) == {"feast", "flink-lite flink-full", "mlflow", "airflow"}
309
309
 
310
310
 
311
311
  def test_pages_keep_within_the_heading_limits():
@@ -328,3 +328,35 @@ def test_pages_keep_within_the_heading_limits():
328
328
  f"{page.relative_to(REPO)}: {len(second)} ##, {len(headings)} in all"
329
329
  )
330
330
  assert not over, over
331
+
332
+
333
+ def test_metadata_example_groups_are_selected_wherever_metadata_is():
334
+ selector = load(
335
+ "select_profiles", REPO / ".github" / "scripts" / "select-profiles.py"
336
+ )
337
+ registry = INTERNAL_RESOURCES_DIR / "registry.yml"
338
+ groups = ["metadata", "metadata:deps", "metadata:autopilot"]
339
+ # The two snippet folders the metadata cases read select exactly the metadata
340
+ # groups whose smoke.sh case runs them; how-it-works also selects mlflow.
341
+ for changed, runs_it in (
342
+ ("docs/snippets/openmetadata/commands.sh", {"metadata", "metadata:autopilot"}),
343
+ (
344
+ "docs/snippets/how-it-works/commands.sh",
345
+ {"metadata:deps", "metadata:autopilot", "mlflow"},
346
+ ),
347
+ ):
348
+ selected = set(selector.select([changed], registry))
349
+ assert runs_it <= selected, changed
350
+ assert not (set(groups) - runs_it) & selected, changed
351
+ assert (
352
+ selector.select(["src/odctl/resources/compose-metadata.yml"], registry)
353
+ == groups
354
+ )
355
+ for group in ("metadata:deps", "metadata:autopilot"):
356
+ assert group not in selector.select(
357
+ ["src/odctl/resources/compose-orch.yml"], registry
358
+ )
359
+ # Without a case, smoke.sh falls through to its default and passes silently.
360
+ smoke = (REPO / ".github" / "scripts" / "smoke.sh").read_text()
361
+ for group in groups:
362
+ assert re.search(rf"^ {group}\)", smoke, re.M), group
@@ -69,3 +69,14 @@ def test_feast_ui_waits_for_a_healthy_feast_serve():
69
69
  assert services["feast-ui"]["depends_on"] == {
70
70
  "feast-serve": {"condition": "service_healthy"}
71
71
  }
72
+
73
+
74
+ def test_the_bundled_repository_serves_mcp_and_defines_no_features():
75
+ repo = get_internal_resources_dir() / "feast"
76
+ config = yaml.safe_load((repo / "feature_store.yaml").read_text())
77
+ assert config["feature_server"] == {
78
+ "type": "mcp",
79
+ "mcp_enabled": True,
80
+ "mcp_transport": "http",
81
+ }
82
+ assert [p.name for p in repo.iterdir()] == ["feature_store.yaml"]
@@ -0,0 +1,62 @@
1
+ """The files for contributors and assistants name only files that exist.
2
+
3
+ `README.md`, `CLAUDE.md` and `CONTRIBUTING.md` name files in backticks, often as a bare
4
+ name inside a folder's row of the code map. A file moved or renamed without these docs
5
+ leaves the name stale, so every name that looks like a path must be the end of a
6
+ tracked path.
7
+ """
8
+
9
+ import re
10
+ import subprocess
11
+ from pathlib import Path
12
+
13
+ ROOT = Path(__file__).resolve().parents[1]
14
+ _DOCS = ["README.md", "CLAUDE.md", "CONTRIBUTING.md"]
15
+ # Names of files a reader or a command creates, not files in the repository.
16
+ _NOT_FILES = {"llms.txt"}
17
+ _SUFFIXES = (
18
+ ".py",
19
+ ".md",
20
+ ".yml",
21
+ ".yaml",
22
+ ".json",
23
+ ".txt",
24
+ ".toml",
25
+ ".sh",
26
+ ".drawio",
27
+ ".csv",
28
+ )
29
+
30
+
31
+ def _tracked_paths() -> set[str]:
32
+ listed = subprocess.run(
33
+ ["git", "-C", str(ROOT), "ls-files"], capture_output=True, text=True, check=True
34
+ )
35
+ paths = set()
36
+ for line in listed.stdout.splitlines():
37
+ parts = line.split("/")
38
+ for start in range(len(parts)):
39
+ for end in range(start + 1, len(parts) + 1):
40
+ paths.add("/".join(parts[start:end]))
41
+ return paths
42
+
43
+
44
+ def _named_paths(text: str) -> set[str]:
45
+ names = set()
46
+ for name in re.findall(r"`([^`\s]+)`", text):
47
+ if re.search(r"[<>*{}$~:=()]|^-|^\.|^http", name) or name in _NOT_FILES:
48
+ continue
49
+ if "/" in name or name.endswith(_SUFFIXES):
50
+ names.add(name.rstrip("/"))
51
+ return names
52
+
53
+
54
+ def test_named_paths_exist():
55
+ tracked = _tracked_paths()
56
+ missing = [
57
+ f"{doc}: {name}"
58
+ for doc in _DOCS
59
+ for name in sorted(_named_paths((ROOT / doc).read_text()))
60
+ if name not in tracked
61
+ ]
62
+ assert missing == []