odctl 1.3.0__tar.gz → 1.4.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {odctl-1.3.0 → odctl-1.4.0}/.github/scripts/select-profiles.py +23 -7
- {odctl-1.3.0 → odctl-1.4.0}/.github/scripts/smoke.sh +173 -16
- {odctl-1.3.0 → odctl-1.4.0}/.github/workflows/e2e-test.yml +8 -4
- {odctl-1.3.0 → odctl-1.4.0}/CONTRIBUTING.md +2 -2
- {odctl-1.3.0 → odctl-1.4.0}/PKG-INFO +2 -2
- {odctl-1.3.0 → odctl-1.4.0}/README.md +1 -1
- {odctl-1.3.0 → odctl-1.4.0}/mkdocs.yml +4 -0
- {odctl-1.3.0 → odctl-1.4.0}/pyproject.toml +1 -1
- {odctl-1.3.0 → odctl-1.4.0}/skills/odctl/SKILL.md +2 -0
- {odctl-1.3.0 → odctl-1.4.0}/src/odctl/resources/feast/feature_store.yaml +10 -5
- {odctl-1.3.0 → odctl-1.4.0}/src/odctl/resources/registry.yml +2 -0
- {odctl-1.3.0 → odctl-1.4.0}/tests/test_docs.py +35 -3
- {odctl-1.3.0 → odctl-1.4.0}/tests/test_feast_profile.py +11 -0
- odctl-1.4.0/tests/test_named_paths.py +62 -0
- odctl-1.4.0/tests/test_roadmap.py +55 -0
- {odctl-1.3.0 → odctl-1.4.0}/uv.lock +1 -1
- {odctl-1.3.0 → odctl-1.4.0}/.github/ISSUE_TEMPLATE/bug_report.md +0 -0
- {odctl-1.3.0 → odctl-1.4.0}/.github/ISSUE_TEMPLATE/feature_request.md +0 -0
- {odctl-1.3.0 → odctl-1.4.0}/.github/renovate.json +0 -0
- {odctl-1.3.0 → odctl-1.4.0}/.github/scripts/spark_avro_producer.py +0 -0
- {odctl-1.3.0 → odctl-1.4.0}/.github/workflows/build-deps.yml +0 -0
- {odctl-1.3.0 → odctl-1.4.0}/.github/workflows/build-platform-images.yml +0 -0
- {odctl-1.3.0 → odctl-1.4.0}/.github/workflows/ci.yml +0 -0
- {odctl-1.3.0 → odctl-1.4.0}/.github/workflows/pipeline.yml +0 -0
- {odctl-1.3.0 → odctl-1.4.0}/.github/workflows/publish.yml +0 -0
- {odctl-1.3.0 → odctl-1.4.0}/.gitignore +0 -0
- {odctl-1.3.0 → odctl-1.4.0}/.pre-commit-config.yaml +0 -0
- {odctl-1.3.0 → odctl-1.4.0}/.vscode/settings.json +0 -0
- {odctl-1.3.0 → odctl-1.4.0}/CLAUDE.md +0 -0
- {odctl-1.3.0 → odctl-1.4.0}/LICENSE +0 -0
- {odctl-1.3.0 → odctl-1.4.0}/image/demo.gif +0 -0
- {odctl-1.3.0 → odctl-1.4.0}/image/demo.png +0 -0
- {odctl-1.3.0 → odctl-1.4.0}/image/diagram.drawio +0 -0
- {odctl-1.3.0 → odctl-1.4.0}/image/diagram.png +0 -0
- {odctl-1.3.0 → odctl-1.4.0}/scripts/diagrams.py +0 -0
- {odctl-1.3.0 → odctl-1.4.0}/scripts/gen_pages.py +0 -0
- {odctl-1.3.0 → odctl-1.4.0}/scripts/llmstxt_sections.py +0 -0
- {odctl-1.3.0 → odctl-1.4.0}/scripts/local_links.py +0 -0
- {odctl-1.3.0 → odctl-1.4.0}/scripts/release_links.py +0 -0
- {odctl-1.3.0 → odctl-1.4.0}/scripts/release_notes.py +0 -0
- {odctl-1.3.0 → odctl-1.4.0}/src/odctl/__init__.py +0 -0
- {odctl-1.3.0 → odctl-1.4.0}/src/odctl/config.py +0 -0
- {odctl-1.3.0 → odctl-1.4.0}/src/odctl/docker.py +0 -0
- {odctl-1.3.0 → odctl-1.4.0}/src/odctl/logs.py +0 -0
- {odctl-1.3.0 → odctl-1.4.0}/src/odctl/main.py +0 -0
- {odctl-1.3.0 → odctl-1.4.0}/src/odctl/planner.py +0 -0
- {odctl-1.3.0 → odctl-1.4.0}/src/odctl/projects.py +0 -0
- {odctl-1.3.0 → odctl-1.4.0}/src/odctl/registry.py +0 -0
- {odctl-1.3.0 → odctl-1.4.0}/src/odctl/resources/airflow/odctl_log_config.py +0 -0
- {odctl-1.3.0 → odctl-1.4.0}/src/odctl/resources/clickhouse/config.xml +0 -0
- {odctl-1.3.0 → odctl-1.4.0}/src/odctl/resources/clickhouse/init.sql +0 -0
- {odctl-1.3.0 → odctl-1.4.0}/src/odctl/resources/clickhouse/keeper_config.xml +0 -0
- {odctl-1.3.0 → odctl-1.4.0}/src/odctl/resources/clickhouse/macros-11-lite.xml +0 -0
- {odctl-1.3.0 → odctl-1.4.0}/src/odctl/resources/clickhouse/macros-11.xml +0 -0
- {odctl-1.3.0 → odctl-1.4.0}/src/odctl/resources/clickhouse/macros-12-lite.xml +0 -0
- {odctl-1.3.0 → odctl-1.4.0}/src/odctl/resources/clickhouse/macros-12.xml +0 -0
- {odctl-1.3.0 → odctl-1.4.0}/src/odctl/resources/clickhouse/macros-21.xml +0 -0
- {odctl-1.3.0 → odctl-1.4.0}/src/odctl/resources/clickhouse/macros-22.xml +0 -0
- {odctl-1.3.0 → odctl-1.4.0}/src/odctl/resources/clickhouse/users.xml +0 -0
- {odctl-1.3.0 → odctl-1.4.0}/src/odctl/resources/compose-analytics.yml +0 -0
- {odctl-1.3.0 → odctl-1.4.0}/src/odctl/resources/compose-deps.yml +0 -0
- {odctl-1.3.0 → odctl-1.4.0}/src/odctl/resources/compose-flink.yml +0 -0
- {odctl-1.3.0 → odctl-1.4.0}/src/odctl/resources/compose-infra.yml +0 -0
- {odctl-1.3.0 → odctl-1.4.0}/src/odctl/resources/compose-kafka.yml +0 -0
- {odctl-1.3.0 → odctl-1.4.0}/src/odctl/resources/compose-metadata.yml +0 -0
- {odctl-1.3.0 → odctl-1.4.0}/src/odctl/resources/compose-mlops.yml +0 -0
- {odctl-1.3.0 → odctl-1.4.0}/src/odctl/resources/compose-obsv.yml +0 -0
- {odctl-1.3.0 → odctl-1.4.0}/src/odctl/resources/compose-orch.yml +0 -0
- {odctl-1.3.0 → odctl-1.4.0}/src/odctl/resources/compose-spark.yml +0 -0
- {odctl-1.3.0 → odctl-1.4.0}/src/odctl/resources/compose-store.yml +0 -0
- {odctl-1.3.0 → odctl-1.4.0}/src/odctl/resources/deps/Dockerfile +0 -0
- {odctl-1.3.0 → odctl-1.4.0}/src/odctl/resources/deps/build.gradle +0 -0
- {odctl-1.3.0 → odctl-1.4.0}/src/odctl/resources/deps/build_connectors.sh +0 -0
- {odctl-1.3.0 → odctl-1.4.0}/src/odctl/resources/deps/fetch_flink_dependencies.sh +0 -0
- {odctl-1.3.0 → odctl-1.4.0}/src/odctl/resources/deps/fetch_spark_dependencies.sh +0 -0
- {odctl-1.3.0 → odctl-1.4.0}/src/odctl/resources/deps/fetch_standalone_plugins.sh +0 -0
- {odctl-1.3.0 → odctl-1.4.0}/src/odctl/resources/deps/versions.env +0 -0
- {odctl-1.3.0 → odctl-1.4.0}/src/odctl/resources/docker/airflow/Dockerfile +0 -0
- {odctl-1.3.0 → odctl-1.4.0}/src/odctl/resources/docker/mlflow/Dockerfile +0 -0
- {odctl-1.3.0 → odctl-1.4.0}/src/odctl/resources/docker/postgres/Dockerfile +0 -0
- {odctl-1.3.0 → odctl-1.4.0}/src/odctl/resources/docker/spark/Dockerfile +0 -0
- {odctl-1.3.0 → odctl-1.4.0}/src/odctl/resources/evidently/config.yaml +0 -0
- {odctl-1.3.0 → odctl-1.4.0}/src/odctl/resources/flink/flink-conf.yml +0 -0
- {odctl-1.3.0 → odctl-1.4.0}/src/odctl/resources/grafana/dashboards/airflow.json +0 -0
- {odctl-1.3.0 → odctl-1.4.0}/src/odctl/resources/grafana/dashboards/clickhouse.json +0 -0
- {odctl-1.3.0 → odctl-1.4.0}/src/odctl/resources/grafana/dashboards/feast.json +0 -0
- {odctl-1.3.0 → odctl-1.4.0}/src/odctl/resources/grafana/dashboards/flink.json +0 -0
- {odctl-1.3.0 → odctl-1.4.0}/src/odctl/resources/grafana/dashboards/fluss.json +0 -0
- {odctl-1.3.0 → odctl-1.4.0}/src/odctl/resources/grafana/dashboards/iceberg-catalog.json +0 -0
- {odctl-1.3.0 → odctl-1.4.0}/src/odctl/resources/grafana/dashboards/kafka-connect.json +0 -0
- {odctl-1.3.0 → odctl-1.4.0}/src/odctl/resources/grafana/dashboards/kafka.json +0 -0
- {odctl-1.3.0 → odctl-1.4.0}/src/odctl/resources/grafana/dashboards/karapace.json +0 -0
- {odctl-1.3.0 → odctl-1.4.0}/src/odctl/resources/grafana/dashboards/marquez.json +0 -0
- {odctl-1.3.0 → odctl-1.4.0}/src/odctl/resources/grafana/dashboards/metabase.json +0 -0
- {odctl-1.3.0 → odctl-1.4.0}/src/odctl/resources/grafana/dashboards/mlflow.json +0 -0
- {odctl-1.3.0 → odctl-1.4.0}/src/odctl/resources/grafana/dashboards/openmetadata.json +0 -0
- {odctl-1.3.0 → odctl-1.4.0}/src/odctl/resources/grafana/dashboards/postgresql.json +0 -0
- {odctl-1.3.0 → odctl-1.4.0}/src/odctl/resources/grafana/dashboards/seaweedfs.json +0 -0
- {odctl-1.3.0 → odctl-1.4.0}/src/odctl/resources/grafana/dashboards/spark.json +0 -0
- {odctl-1.3.0 → odctl-1.4.0}/src/odctl/resources/grafana/dashboards/temporal.json +0 -0
- {odctl-1.3.0 → odctl-1.4.0}/src/odctl/resources/grafana/dashboards/trino.json +0 -0
- {odctl-1.3.0 → odctl-1.4.0}/src/odctl/resources/grafana/dashboards/valkey.json +0 -0
- {odctl-1.3.0 → odctl-1.4.0}/src/odctl/resources/grafana/odctl-dashboards.yaml +0 -0
- {odctl-1.3.0 → odctl-1.4.0}/src/odctl/resources/grafana/sources.yml +0 -0
- {odctl-1.3.0 → odctl-1.4.0}/src/odctl/resources/jmx/jmx-exporter.yml +0 -0
- {odctl-1.3.0 → odctl-1.4.0}/src/odctl/resources/marquez/marquez.yml +0 -0
- {odctl-1.3.0 → odctl-1.4.0}/src/odctl/resources/openmetadata/autopilot_init.py +0 -0
- {odctl-1.3.0 → odctl-1.4.0}/src/odctl/resources/openmetadata/connector_sync.py +0 -0
- {odctl-1.3.0 → odctl-1.4.0}/src/odctl/resources/otelcol/odctl-receivers.yaml +0 -0
- {odctl-1.3.0 → odctl-1.4.0}/src/odctl/resources/postgres/01-init-databases.sh +0 -0
- {odctl-1.3.0 → odctl-1.4.0}/src/odctl/resources/prometheus/prometheus.yml +0 -0
- {odctl-1.3.0 → odctl-1.4.0}/src/odctl/resources/seaweed/s3.json +0 -0
- {odctl-1.3.0 → odctl-1.4.0}/src/odctl/resources/spark/log4j2.properties +0 -0
- {odctl-1.3.0 → odctl-1.4.0}/src/odctl/resources/spark/metrics.properties +0 -0
- {odctl-1.3.0 → odctl-1.4.0}/src/odctl/resources/spark/spark-defaults.conf +0 -0
- {odctl-1.3.0 → odctl-1.4.0}/src/odctl/resources/trino/access-control.properties +0 -0
- {odctl-1.3.0 → odctl-1.4.0}/src/odctl/resources/trino/catalog/clickhouse.properties +0 -0
- {odctl-1.3.0 → odctl-1.4.0}/src/odctl/resources/trino/catalog/iceberg.properties +0 -0
- {odctl-1.3.0 → odctl-1.4.0}/src/odctl/resources/trino/catalog/kafka.properties +0 -0
- {odctl-1.3.0 → odctl-1.4.0}/src/odctl/resources/trino/catalog/postgres.properties +0 -0
- {odctl-1.3.0 → odctl-1.4.0}/src/odctl/resources/trino/catalog/valkey.properties +0 -0
- {odctl-1.3.0 → odctl-1.4.0}/src/odctl/resources/trino/group-provider.properties +0 -0
- {odctl-1.3.0 → odctl-1.4.0}/src/odctl/resources/trino/groups.txt +0 -0
- {odctl-1.3.0 → odctl-1.4.0}/src/odctl/resources/trino/rules.json +0 -0
- {odctl-1.3.0 → odctl-1.4.0}/src/odctl/ui.py +0 -0
- {odctl-1.3.0 → odctl-1.4.0}/src/odctl/workspace.py +0 -0
- {odctl-1.3.0 → odctl-1.4.0}/tests/conftest.py +0 -0
- {odctl-1.3.0 → odctl-1.4.0}/tests/docs_helpers.py +0 -0
- {odctl-1.3.0 → odctl-1.4.0}/tests/test_airflow_profile.py +0 -0
- {odctl-1.3.0 → odctl-1.4.0}/tests/test_autopilot_init.py +0 -0
- {odctl-1.3.0 → odctl-1.4.0}/tests/test_cli.py +0 -0
- {odctl-1.3.0 → odctl-1.4.0}/tests/test_cli_extended.py +0 -0
- {odctl-1.3.0 → odctl-1.4.0}/tests/test_compose_profiles.py +0 -0
- {odctl-1.3.0 → odctl-1.4.0}/tests/test_config.py +0 -0
- {odctl-1.3.0 → odctl-1.4.0}/tests/test_confirmed_pages.py +0 -0
- {odctl-1.3.0 → odctl-1.4.0}/tests/test_connector_sync.py +0 -0
- {odctl-1.3.0 → odctl-1.4.0}/tests/test_diagrams.py +0 -0
- {odctl-1.3.0 → odctl-1.4.0}/tests/test_docker.py +0 -0
- {odctl-1.3.0 → odctl-1.4.0}/tests/test_evidently_profile.py +0 -0
- {odctl-1.3.0 → odctl-1.4.0}/tests/test_feature_docs.py +0 -0
- {odctl-1.3.0 → odctl-1.4.0}/tests/test_grafana_dashboards.py +0 -0
- {odctl-1.3.0 → odctl-1.4.0}/tests/test_logs.py +0 -0
- {odctl-1.3.0 → odctl-1.4.0}/tests/test_metadata_profile.py +0 -0
- {odctl-1.3.0 → odctl-1.4.0}/tests/test_metrics_collector.py +0 -0
- {odctl-1.3.0 → odctl-1.4.0}/tests/test_metrics_jvm.py +0 -0
- {odctl-1.3.0 → odctl-1.4.0}/tests/test_metrics_python_services.py +0 -0
- {odctl-1.3.0 → odctl-1.4.0}/tests/test_planner.py +0 -0
- {odctl-1.3.0 → odctl-1.4.0}/tests/test_postgres_image.py +0 -0
- {odctl-1.3.0 → odctl-1.4.0}/tests/test_projects.py +0 -0
- {odctl-1.3.0 → odctl-1.4.0}/tests/test_registry.py +0 -0
- {odctl-1.3.0 → odctl-1.4.0}/tests/test_release_links.py +0 -0
- {odctl-1.3.0 → odctl-1.4.0}/tests/test_release_notes.py +0 -0
- {odctl-1.3.0 → odctl-1.4.0}/tests/test_telemetry_profile.py +0 -0
- {odctl-1.3.0 → odctl-1.4.0}/tests/test_temporal_profile.py +0 -0
- {odctl-1.3.0 → odctl-1.4.0}/tests/test_workspace.py +0 -0
|
@@ -45,6 +45,11 @@ GROUPS = [
|
|
|
45
45
|
"lineage",
|
|
46
46
|
"telemetry",
|
|
47
47
|
"metadata",
|
|
48
|
+
# The part before the colon is the profile to start. metadata:deps and
|
|
49
|
+
# metadata:autopilot start the same profile as metadata. Each examples group
|
|
50
|
+
# recreates metadata twice, which is why each runs alone, in parallel with the others.
|
|
51
|
+
"metadata:deps",
|
|
52
|
+
"metadata:autopilot",
|
|
48
53
|
"airflow",
|
|
49
54
|
"temporal",
|
|
50
55
|
"postgres",
|
|
@@ -70,13 +75,14 @@ REGISTRY = Path("src/odctl/resources/registry.yml")
|
|
|
70
75
|
|
|
71
76
|
# smoke.sh starts some profiles from another group's case: the kafka case starts
|
|
72
77
|
# catalog, the flink case starts kafka-lite and feast, the spark case starts
|
|
73
|
-
# kafka-lite,
|
|
74
|
-
# compose file needs these groups as well as the
|
|
78
|
+
# kafka-lite, the fluss and feast cases start flink-lite, and the airflow case
|
|
79
|
+
# starts feast. A change to that compose file needs these groups as well as the
|
|
80
|
+
# ones the registry maps to it.
|
|
75
81
|
ALSO_RUNS = {
|
|
76
82
|
"compose-infra.yml": ["kafka-lite kafka-full"],
|
|
77
83
|
"compose-kafka.yml": ["flink-lite flink-full", "spark-lite spark-full"],
|
|
78
84
|
"compose-flink.yml": ["fluss", "feast"],
|
|
79
|
-
"compose-mlops.yml": ["flink-lite flink-full"],
|
|
85
|
+
"compose-mlops.yml": ["flink-lite flink-full", "airflow"],
|
|
80
86
|
}
|
|
81
87
|
|
|
82
88
|
# smoke.sh runs the guide commands in docs/snippets/<guide>/, so a change there
|
|
@@ -85,7 +91,15 @@ ALSO_RUNS = {
|
|
|
85
91
|
SNIPPET_PREFIX = "docs/snippets/"
|
|
86
92
|
SNIPPET_GROUPS = {
|
|
87
93
|
"getting-started": ["kafka-lite kafka-full"],
|
|
88
|
-
"how-it-works": [
|
|
94
|
+
"how-it-works": [
|
|
95
|
+
"storage",
|
|
96
|
+
"catalog",
|
|
97
|
+
"trino",
|
|
98
|
+
"airflow",
|
|
99
|
+
"mlflow",
|
|
100
|
+
"metadata:deps",
|
|
101
|
+
"metadata:autopilot",
|
|
102
|
+
],
|
|
89
103
|
"kafka": ["kafka-lite kafka-full"],
|
|
90
104
|
"kafka-connect": ["kafka-lite kafka-full"],
|
|
91
105
|
"iceberg-ingestion": ["kafka-lite kafka-full", "flink-lite flink-full"],
|
|
@@ -100,10 +114,11 @@ SNIPPET_GROUPS = {
|
|
|
100
114
|
"airflow-dags": ["airflow"],
|
|
101
115
|
"temporal-workers": ["temporal"],
|
|
102
116
|
"mlflow": ["mlflow"],
|
|
103
|
-
"feast-iceberg": ["feast", "flink-lite flink-full", "mlflow"],
|
|
117
|
+
"feast-iceberg": ["feast", "flink-lite flink-full", "mlflow", "airflow"],
|
|
118
|
+
"feast-vector-retrieval": ["feast"],
|
|
104
119
|
"evidently-reports": ["evidently"],
|
|
105
120
|
"lineage": ["lineage"],
|
|
106
|
-
"openmetadata": ["metadata"],
|
|
121
|
+
"openmetadata": ["metadata", "metadata:autopilot"],
|
|
107
122
|
"telemetry": ["telemetry"],
|
|
108
123
|
}
|
|
109
124
|
|
|
@@ -174,7 +189,8 @@ def select(changed: list, registry_path: Path = REGISTRY) -> list:
|
|
|
174
189
|
|
|
175
190
|
selected = []
|
|
176
191
|
for group in GROUPS:
|
|
177
|
-
|
|
192
|
+
# A group name may carry a suffix after a colon, as metadata:deps does.
|
|
193
|
+
files = {mapping.get(p.split(":")[0]) for p in group.split()}
|
|
178
194
|
also = any(group in ALSO_RUNS.get(name, []) for name in touched_files)
|
|
179
195
|
if files & touched_files or group in snippet_groups or also:
|
|
180
196
|
selected.append(group)
|
|
@@ -9,10 +9,10 @@
|
|
|
9
9
|
# committed nothing while its checkpoints succeeded. Each assertion below exists
|
|
10
10
|
# because something shipped broken past a liveness check.
|
|
11
11
|
#
|
|
12
|
-
# Usage: smoke.sh <profile>
|
|
12
|
+
# Usage: smoke.sh <profile or group name, such as metadata:deps>
|
|
13
13
|
set -uo pipefail
|
|
14
14
|
|
|
15
|
-
PROFILE="${1:?usage: smoke.sh <profile>}"
|
|
15
|
+
PROFILE="${1:?usage: smoke.sh <profile or group name, such as metadata:deps>}"
|
|
16
16
|
MB_USER="smoke@example.com"
|
|
17
17
|
MB_PASS="Smoke-test-1234"
|
|
18
18
|
|
|
@@ -880,6 +880,104 @@ if bad:
|
|
|
880
880
|
|
|
881
881
|
# Airflow pushes OTLP to the telemetry collector rather than being scraped.
|
|
882
882
|
assert_pushed_metrics 'airflow_.+' Airflow
|
|
883
|
+
|
|
884
|
+
smoke_airflow_materialize
|
|
885
|
+
}
|
|
886
|
+
|
|
887
|
+
# Feast guide's On a Schedule section: a DAG that builds a FeatureStore from
|
|
888
|
+
# FEAST_REGISTRY and calls materialize_incremental. It needs the registry entry and
|
|
889
|
+
# the Iceberg table the guide's earlier steps make, so this runs those from the
|
|
890
|
+
# host as the guide does, then uploads the DAG, runs it and reads the value it
|
|
891
|
+
# copied into Valkey. The feature view source has no catalog address, so the
|
|
892
|
+
# registry entry works from the host here and from the Airflow container in the DAG.
|
|
893
|
+
# Its own project, so that the guide's Runnable Example can still register demo.
|
|
894
|
+
smoke_airflow_materialize() {
|
|
895
|
+
odctl up feast >/dev/null 2>&1 || fail "could not start feast for the materialise DAG"
|
|
896
|
+
retry 30 5 http_ok "http://127.0.0.1:6566/health" || fail "Feast never answered on :6566"
|
|
897
|
+
|
|
898
|
+
local work="${TMPDIR:-/tmp}/odctl-airflow-feast-smoke"
|
|
899
|
+
rm -rf "$work"; mkdir -p "$work"
|
|
900
|
+
uv venv "$work/.venv" >/dev/null 2>&1 || fail "could not create a venv for the feast client"
|
|
901
|
+
VIRTUAL_ENV="$work/.venv" uv pip install -q \
|
|
902
|
+
"feast[duckdb,iceberg,redis,postgres]==0.66.0" "pyarrow" >/dev/null 2>&1 \
|
|
903
|
+
|| fail "could not install the feast client"
|
|
904
|
+
|
|
905
|
+
cp -r "$SNIPPETS/feast-iceberg/feature_repo" "$work/"
|
|
906
|
+
sed 's/^project: demo$/project: airflow_smoke/' "$work/feature_repo/feature_store.yaml" \
|
|
907
|
+
> "$work/feature_store.yaml" && mv "$work/feature_store.yaml" "$work/feature_repo/feature_store.yaml"
|
|
908
|
+
sed 's/project="demo"/project="airflow_smoke"/' "$SNIPPETS/feast-iceberg/feast_materialize.py" \
|
|
909
|
+
> "$work/feast_materialize.py"
|
|
910
|
+
snippet feast-iceberg/steps.py create-table > "$work/build.py"
|
|
911
|
+
snippet feast-iceberg/steps.py append > "$work/append.py"
|
|
912
|
+
snippet feast-iceberg/steps.py read > "$work/read.py"
|
|
913
|
+
|
|
914
|
+
local py="$work/.venv/bin/python" log="$work/step.log"
|
|
915
|
+
run_step() {
|
|
916
|
+
local what="$1"; shift
|
|
917
|
+
if ! (cd "$work" && "$@") >"$log" 2>&1; then
|
|
918
|
+
echo "---- $what ----"
|
|
919
|
+
tail -25 "$log"
|
|
920
|
+
fail "$what"
|
|
921
|
+
fi
|
|
922
|
+
}
|
|
923
|
+
|
|
924
|
+
# A table left by the Runnable Example would make the guide's create step fail.
|
|
925
|
+
local code
|
|
926
|
+
code=$(curl -sS --max-time 10 -o /dev/null -w '%{http_code}' -X DELETE "http://127.0.0.1:8181/v1/namespaces/demo/tables/driver_stats?purgeRequested=true")
|
|
927
|
+
case "$code" in 204|404) ;; *) fail "could not delete demo.driver_stats before the guide's create step (HTTP $code)";; esac
|
|
928
|
+
eval "$(snippet feast-iceberg/commands.sh catalog-env)"
|
|
929
|
+
run_step "could not create the Iceberg table through the catalog" "$py" build.py
|
|
930
|
+
# Delete the view, so that materialise starts again and the check reads this run's values.
|
|
931
|
+
(cd "$work" && "$work/.venv/bin/feast" -c feature_repo delete driver_stats) >/dev/null 2>&1 || true
|
|
932
|
+
run_step "feast apply or materialize failed without a catalog address in the source" \
|
|
933
|
+
env PATH="$work/.venv/bin:$PATH" bash -e -o pipefail -c "$(snippet feast-iceberg/commands.sh apply-materialize)"
|
|
934
|
+
pass "the guide's source, with no catalog address, applied and materialised from the host"
|
|
935
|
+
|
|
936
|
+
run_step "could not upload the materialise DAG to s3://airflow/dags" \
|
|
937
|
+
run_snippet feast-iceberg/commands.sh dag-upload
|
|
938
|
+
retry 30 10 bash -c \
|
|
939
|
+
"docker exec airflow airflow dags list 2>/dev/null | grep -q feast_materialize" \
|
|
940
|
+
|| fail "the S3 DAG bundle never delivered the materialise DAG, or it was never parsed"
|
|
941
|
+
|
|
942
|
+
# The new row is newer than the host's materialise, so only the DAG can copy it.
|
|
943
|
+
run_step "could not append a row to the Iceberg table" "$py" append.py
|
|
944
|
+
run_step "the guide's unpause and trigger commands failed" \
|
|
945
|
+
run_snippet feast-iceberg/commands.sh dag-run
|
|
946
|
+
|
|
947
|
+
# Unpausing also starts a scheduled run, so wait until every run has finished.
|
|
948
|
+
local states=""
|
|
949
|
+
for _ in $(seq 1 30); do
|
|
950
|
+
states=$(docker exec airflow airflow dags list-runs feast_materialize -o plain 2>/dev/null \
|
|
951
|
+
| grep -E "^feast_materialize" | tr -s ' ' | cut -d' ' -f3)
|
|
952
|
+
grep -qE 'queued|running' <<<"$states" || [ -z "$states" ] || break
|
|
953
|
+
sleep 10
|
|
954
|
+
done
|
|
955
|
+
[ -n "$states" ] && ! grep -qvx success <<<"$states" \
|
|
956
|
+
|| fail "the materialise DAG runs ended in states: ${states:-none}"
|
|
957
|
+
pass "the materialise DAG ran to success inside the Airflow container"
|
|
958
|
+
|
|
959
|
+
local got
|
|
960
|
+
got=$(cd "$work" && "$py" read.py 2>"$log") || { tail -25 "$log"; fail "the online read failed"; }
|
|
961
|
+
[ "$got" = "{'driver_id': [1001, 1002], 'conv_rate': [0.6000000238418579, 0.75]}" ] \
|
|
962
|
+
|| fail "Valkey holds $got, not the 0.6 the DAG should have copied"
|
|
963
|
+
pass "the DAG copied the new value from Iceberg into Valkey"
|
|
964
|
+
|
|
965
|
+
# Leave no hourly run behind: a DAG that stays unpaused fails on its next run once
|
|
966
|
+
# the table below is gone, and the next smoke run would read that failed run.
|
|
967
|
+
docker exec airflow airflow dags pause feast_materialize >/dev/null 2>&1 \
|
|
968
|
+
|| fail "could not pause the materialise DAG"
|
|
969
|
+
docker exec airflow python -c '
|
|
970
|
+
import boto3
|
|
971
|
+
boto3.client("s3", endpoint_url="http://seaweed:8333").delete_object(
|
|
972
|
+
Bucket="airflow", Key="dags/feast_materialize.py")' >/dev/null 2>&1 \
|
|
973
|
+
|| fail "could not delete the materialise DAG from s3://airflow/dags"
|
|
974
|
+
|
|
975
|
+
# No feast teardown: it leaves the view's row in feature_view_version_history, and
|
|
976
|
+
# a second run of this check would then fail to apply the view again.
|
|
977
|
+
curl -fsS --max-time 10 -X DELETE \
|
|
978
|
+
"http://127.0.0.1:8181/v1/namespaces/demo/tables/driver_stats?purgeRequested=true" >/dev/null 2>&1 || true
|
|
979
|
+
curl -fsS --max-time 10 -X DELETE "http://127.0.0.1:8181/v1/namespaces/demo" >/dev/null 2>&1 || true
|
|
980
|
+
rm -rf "$work"
|
|
883
981
|
}
|
|
884
982
|
|
|
885
983
|
# deps is a one-shot copy into a shared volume, so there is no container to
|
|
@@ -1427,7 +1525,9 @@ om_step() {
|
|
|
1427
1525
|
# Feast keeps no feature values of its own, so liveness proves nothing here.
|
|
1428
1526
|
# This builds a real Iceberg table through the catalog, registers a feature view
|
|
1429
1527
|
# over it, takes a point-in-time join, materializes to Valkey and reads it back.
|
|
1430
|
-
#
|
|
1528
|
+
# It also retrieves the nearest rows from a second repository on PostgreSQL with
|
|
1529
|
+
# pgvector. A failure in any of those is a broken profile even when both
|
|
1530
|
+
# containers are up.
|
|
1431
1531
|
smoke_feast() {
|
|
1432
1532
|
retry 60 5 http_ok "http://127.0.0.1:8890/api/v1/projects" || fail "no HTTP response from the feast UI on :8890"
|
|
1433
1533
|
pass "feast UI answering, registry reachable"
|
|
@@ -1442,17 +1542,16 @@ smoke_feast() {
|
|
|
1442
1542
|
# the published image carries feast[minimal], which has no iceberg or duckdb.
|
|
1443
1543
|
uv venv "$work/.venv" >/dev/null 2>&1 || fail "could not create a venv for the feast client"
|
|
1444
1544
|
VIRTUAL_ENV="$work/.venv" uv pip install -q \
|
|
1445
|
-
"feast[duckdb,iceberg,redis,postgres]==0.66.0" "pyarrow" >/dev/null 2>&1 \
|
|
1446
|
-
|| fail "could not install the feast
|
|
1545
|
+
"feast[duckdb,iceberg,redis,postgres]==0.66.0" "pyarrow" "mcp>=1.24,<2" >/dev/null 2>&1 \
|
|
1546
|
+
|| fail "could not install the feast and mcp clients"
|
|
1447
1547
|
|
|
1448
|
-
# Repository from the guide,
|
|
1449
|
-
#
|
|
1450
|
-
# catalogs, and Iceberg's REST fixture server serves the spec with no prefix,
|
|
1451
|
-
# so a real warehouse gives HTTP 400 "Ambiguous URI empty segment".
|
|
1548
|
+
# Repository from the guide, whose source passes warehouse="" so that PyIceberg
|
|
1549
|
+
# takes the warehouse from PYICEBERG_CATALOG__ODCTL__WAREHOUSE.
|
|
1452
1550
|
cp "$SNIPPETS/feast-iceberg/feature_repo/"* "$work/feature_repo/"
|
|
1453
|
-
# Its own project,
|
|
1454
|
-
#
|
|
1455
|
-
# feature_view_version_history, so
|
|
1551
|
+
# Its own project, so that the guide's Runnable Example can still register demo.
|
|
1552
|
+
# No feast teardown: it deletes every project in the registry, and it leaves each
|
|
1553
|
+
# view's row in feature_view_version_history, so the next apply of that view fails
|
|
1554
|
+
# on its key. feast delete removes one view and its online values, and the view applies again.
|
|
1456
1555
|
sed 's/^project: demo$/project: smoke/' "$work/feature_repo/feature_store.yaml" \
|
|
1457
1556
|
> "$work/feature_store.yaml" && mv "$work/feature_store.yaml" "$work/feature_repo/feature_store.yaml"
|
|
1458
1557
|
|
|
@@ -1503,11 +1602,17 @@ PYONLINE
|
|
|
1503
1602
|
fi
|
|
1504
1603
|
}
|
|
1505
1604
|
|
|
1605
|
+
# A table left by the Runnable Example or by a reader would make the guide's create step fail.
|
|
1606
|
+
local code
|
|
1607
|
+
code=$(curl -sS --max-time 10 -o /dev/null -w '%{http_code}' -X DELETE "http://127.0.0.1:8181/v1/namespaces/demo/tables/driver_stats?purgeRequested=true")
|
|
1608
|
+
case "$code" in 204|404) ;; *) fail "could not delete demo.driver_stats before the guide's create step (HTTP $code)";; esac
|
|
1506
1609
|
run_step "could not create the Iceberg table through the catalog" "$py" build.py
|
|
1507
1610
|
pass "Iceberg table created through the REST catalog on SeaweedFS"
|
|
1508
1611
|
|
|
1612
|
+
# Delete the view, so that materialise starts again and the check reads this run's values.
|
|
1613
|
+
(cd "$work" && "$feast_bin" -c feature_repo delete driver_stats) >/dev/null 2>&1 || true
|
|
1509
1614
|
run_step "feast apply or materialize failed" \
|
|
1510
|
-
env PATH="$work/.venv/bin:$PATH" bash -o pipefail -c "$(snippet feast-iceberg/commands.sh apply-materialize)"
|
|
1615
|
+
env PATH="$work/.venv/bin:$PATH" bash -e -o pipefail -c "$(snippet feast-iceberg/commands.sh apply-materialize)"
|
|
1511
1616
|
pass "feast apply registered the feature view, and materialize wrote to valkey"
|
|
1512
1617
|
|
|
1513
1618
|
run_step "the point-in-time join returned the wrong values" "$py" check.py
|
|
@@ -1516,7 +1621,25 @@ PYONLINE
|
|
|
1516
1621
|
run_step "the online read from valkey returned the wrong values" "$py" online.py
|
|
1517
1622
|
pass "get_online_features read the materialised values from valkey"
|
|
1518
1623
|
|
|
1519
|
-
|
|
1624
|
+
# Vector retrieval, from the vector page: a second repository, with
|
|
1625
|
+
# PostgreSQL and pgvector as its online store. Its own project, because the
|
|
1626
|
+
# Runnable Example registers vector_demo afterwards.
|
|
1627
|
+
mkdir -p "$work/vector_repo"
|
|
1628
|
+
cp "$SNIPPETS/feast-vector-retrieval/vector_repo/"* "$work/vector_repo/"
|
|
1629
|
+
sed 's/^project: vector_demo$/project: vector_smoke/' "$work/vector_repo/feature_store.yaml" \
|
|
1630
|
+
> "$work/feature_store.yaml" && mv "$work/feature_store.yaml" "$work/vector_repo/feature_store.yaml"
|
|
1631
|
+
snippet feast-vector-retrieval/steps.py write > "$work/vector_write.py"
|
|
1632
|
+
snippet feast-vector-retrieval/steps.py retrieve > "$work/vector_retrieve.py"
|
|
1633
|
+
run_step "feast apply failed on the vector repository" \
|
|
1634
|
+
env PATH="$work/.venv/bin:$PATH" bash -o pipefail -c "$(snippet feast-vector-retrieval/commands.sh apply)"
|
|
1635
|
+
run_step "writing the documents to the PostgreSQL online store failed" "$py" vector_write.py
|
|
1636
|
+
local nearest
|
|
1637
|
+
nearest=$(cd "$work" && "$py" vector_retrieve.py 2>"$log") \
|
|
1638
|
+
|| { tail -25 "$log"; fail "retrieve_online_documents_v2 failed"; }
|
|
1639
|
+
[ "$nearest" = "['Feast', 'Iceberg']" ] \
|
|
1640
|
+
|| fail "retrieve_online_documents_v2 returned $nearest, not the two nearest documents"
|
|
1641
|
+
pass "retrieve_online_documents_v2 returned the nearest rows from pgvector"
|
|
1642
|
+
|
|
1520
1643
|
curl -fsS --max-time 10 -X DELETE \
|
|
1521
1644
|
"http://127.0.0.1:8181/v1/namespaces/demo/tables/driver_stats?purgeRequested=true" >/dev/null 2>&1 || true
|
|
1522
1645
|
curl -fsS --max-time 10 -X DELETE "http://127.0.0.1:8181/v1/namespaces/demo" >/dev/null 2>&1 || true
|
|
@@ -1524,6 +1647,31 @@ PYONLINE
|
|
|
1524
1647
|
# case removes $work after that.
|
|
1525
1648
|
}
|
|
1526
1649
|
|
|
1650
|
+
# The checks run after the guide's Runnable Example, so they reuse its push
|
|
1651
|
+
# definitions in feast-serve; the HTTP push then stores a newer value than the
|
|
1652
|
+
# example's Flink push. Read it back until the feature server's cached registry
|
|
1653
|
+
# knows the feature view, then read it through the MCP endpoint.
|
|
1654
|
+
smoke_feast_push_and_mcp() {
|
|
1655
|
+
local work="${TMPDIR:-/tmp}/odctl-feast-smoke"
|
|
1656
|
+
local out
|
|
1657
|
+
out=$(cd "$work" && run_snippet feast-iceberg/commands.sh push-http 2>&1) \
|
|
1658
|
+
|| { echo "$out" | tail -20; fail "the HTTP push to /push failed"; }
|
|
1659
|
+
[ "$out" = "200" ] || { echo "$out" | tail -20; fail "the HTTP push to /push answered '$out', not 200"; }
|
|
1660
|
+
retry 24 5 snippet_says feast-iceberg/commands.sh push-read -F 0.699999988079071 \
|
|
1661
|
+
|| fail "the value pushed over HTTP was not read back from /get-online-features"
|
|
1662
|
+
pass "a value pushed over HTTP was read back from the feature server"
|
|
1663
|
+
|
|
1664
|
+
cp "$SNIPPETS/feast-iceberg/mcp_client.py" "$work/"
|
|
1665
|
+
out=$(cd "$work" && "$work/.venv/bin/python" mcp_client.py 2>&1) \
|
|
1666
|
+
|| { echo "$out" | tail -20; fail "the MCP client failed"; }
|
|
1667
|
+
grep -qx 'get_online_features_get_online_features_post' <<<"$out" \
|
|
1668
|
+
&& grep -qx 'push_push_post' <<<"$out" \
|
|
1669
|
+
|| { echo "$out" | tail -20; fail "the MCP endpoint did not list Feast's tools"; }
|
|
1670
|
+
grep -qxF '[0.699999988079071]' <<<"$out" \
|
|
1671
|
+
|| { echo "$out" | tail -20; fail "the MCP tool did not return the pushed value"; }
|
|
1672
|
+
pass "an MCP client listed Feast's tools and read the pushed value"
|
|
1673
|
+
}
|
|
1674
|
+
|
|
1527
1675
|
# Temporal: a healthy server proves only that it answers. Workflows run in the
|
|
1528
1676
|
# caller's worker, so this runs one with the temporalio SDK from the host: an
|
|
1529
1677
|
# activity that fails once and succeeds on retry, and a workflow that waits for
|
|
@@ -1765,8 +1913,13 @@ case "$PROFILE" in
|
|
|
1765
1913
|
telemetry) smoke_telemetry; run_example telemetry '"42"' ;;
|
|
1766
1914
|
metadata)
|
|
1767
1915
|
smoke_metadata
|
|
1768
|
-
smoke_metadata_example
|
|
1769
|
-
|
|
1916
|
+
smoke_metadata_example ;;
|
|
1917
|
+
# Two How It Works examples that recreate metadata, each in a group of its own
|
|
1918
|
+
# so that they run in parallel with the checks above and with each other. Each
|
|
1919
|
+
# recreates metadata twice, once for the example and once to restore it.
|
|
1920
|
+
metadata:deps)
|
|
1921
|
+
how_it_works_edit metadata-deps .odctl/.env metadata '^1\.4\.3$' ;;
|
|
1922
|
+
metadata:autopilot)
|
|
1770
1923
|
how_it_works_autopilot
|
|
1771
1924
|
retry 60 5 http_ok "http://127.0.0.1:8087/api/v2/monitor/health" \
|
|
1772
1925
|
|| fail "ingestion Airflow did not answer after the package was put back" ;;
|
|
@@ -1774,6 +1927,10 @@ case "$PROFILE" in
|
|
|
1774
1927
|
feast)
|
|
1775
1928
|
smoke_feast
|
|
1776
1929
|
run_example feast-iceberg '0.8999999761581421' "env PATH=${TMPDIR:-/tmp}/odctl-feast-smoke/.venv/bin:$PATH bash"
|
|
1930
|
+
smoke_feast_push_and_mcp
|
|
1931
|
+
run_example feast-vector-retrieval "\['Feast', 'Iceberg'\]" "env PATH=${TMPDIR:-/tmp}/odctl-feast-smoke/.venv/bin:$PATH bash"
|
|
1932
|
+
# The two examples write their repositories into the current folder, as a reader's would.
|
|
1933
|
+
rm -rf feature_repo vector_repo push_definitions.py
|
|
1777
1934
|
rm -rf "${TMPDIR:-/tmp}/odctl-feast-smoke" ;;
|
|
1778
1935
|
evidently) smoke_evidently; run_example evidently-reports 'report pushed to project' "$(evidently_runner)" ;;
|
|
1779
1936
|
*)
|
|
@@ -89,9 +89,13 @@ jobs:
|
|
|
89
89
|
run: |
|
|
90
90
|
chmod +x .github/scripts/smoke.sh
|
|
91
91
|
for profile in ${{ matrix.profile_group }}; do
|
|
92
|
+
# A name such as metadata:deps is a group of its own for the same
|
|
93
|
+
# profile: odctl starts the part before the colon, and smoke.sh
|
|
94
|
+
# gets the whole name to choose its checks.
|
|
95
|
+
base="${profile%%:*}"
|
|
92
96
|
echo "::group::Testing Profile: $profile"
|
|
93
|
-
echo "🚀 Launching $
|
|
94
|
-
odctl up "$
|
|
97
|
+
echo "🚀 Launching $base..."
|
|
98
|
+
odctl up "$base"
|
|
95
99
|
|
|
96
100
|
echo "⏳ Containers:"
|
|
97
101
|
docker ps
|
|
@@ -116,8 +120,8 @@ jobs:
|
|
|
116
120
|
# named teardown did not until 0.4.3, so compose rejected the
|
|
117
121
|
# project with "depends on undefined service" and only the named
|
|
118
122
|
# form could catch it. Naming the profile is also what a user runs.
|
|
119
|
-
echo "🧹 Tearing down $
|
|
120
|
-
yes | odctl down "$
|
|
123
|
+
echo "🧹 Tearing down $base by name..."
|
|
124
|
+
yes | odctl down "$base" --volumes
|
|
121
125
|
|
|
122
126
|
# Sweep afterwards so a failure above cannot leak state into the
|
|
123
127
|
# next profile in the group. This is a no-op when the named
|
|
@@ -55,7 +55,7 @@ Each layer checks one thing:
|
|
|
55
55
|
|
|
56
56
|
Run the unit suite in full once before each commit. While fixing, run only the failed tests (`pytest --lf`) and the tests of the files you changed. Run the strict build when `docs/`, `mkdocs.yml` or an input of `scripts/gen_pages.py` changed: `registry.yml`, a compose file, or a command's `--help` text.
|
|
57
57
|
|
|
58
|
-
A pull request runs pre-commit, the unit tests on each supported Python version, the strict docs build, and the image builds when an image input changed. e2e runs only on a tag and on a manual run, because odctl runs images tagged with its own version, and the images of an unreleased version do not exist yet. To check a profile on your machine, do what the e2e job does: install the CLI with `uv tool install .`, run `odctl init`, start the profile with `odctl up <profile>`, then run `.github/scripts/smoke.sh <profile>`. Run one `odctl up` at a time. When no image definition changed since the last release, you can give the last release's images the new version's tag for the test, and remove those tags afterwards.
|
|
58
|
+
A pull request runs pre-commit, the unit tests on each supported Python version, the strict docs build, and the image builds when an image input changed. e2e runs only on a tag and on a manual run, because odctl runs images tagged with its own version, and the images of an unreleased version do not exist yet. To check a profile on your machine, do what the e2e job does: install the CLI with `uv tool install .`, run `odctl init`, start the profile with `odctl up <profile>`, then run `.github/scripts/smoke.sh <profile>`. For a group name with a colon, such as `metadata:deps`, the e2e job starts the profile before the colon (`odctl up metadata`) and passes the whole name to `smoke.sh` (`smoke.sh metadata:deps`). Run one `odctl up` at a time. When no image definition changed since the last release, you can give the last release's images the new version's tag for the test, and remove those tags afterwards.
|
|
59
59
|
|
|
60
60
|
## Rules
|
|
61
61
|
|
|
@@ -94,6 +94,6 @@ Work on a branch from `main` and open a pull request against `main`. Before you
|
|
|
94
94
|
|
|
95
95
|
The maintainer releases. These steps are written down so that each release is the same:
|
|
96
96
|
|
|
97
|
-
1. Bump `version` in `pyproject.toml`, run `uv lock`, and add the release to `docs/about/changelog.md`, with an Upgrading line for any change to ports, profiles, `.env` keys, volumes or workspace files. When it reaches `main`, the docs deploy as that version, aliased `latest`.
|
|
97
|
+
1. Bump `version` in `pyproject.toml`, run `uv lock`, and add the release to `docs/about/changelog.md`, with an Upgrading line for any change to ports, profiles, `.env` keys, volumes or workspace files. The same commit updates the public roadmap, `docs/about/roadmap.md`: the release moves from Next Releases to Released, the next release becomes the current entry (`key: now`), and every planned entry names all the issues planned for it; `tests/test_roadmap.py` fails otherwise. When it reaches `main`, the docs deploy as that version, aliased `latest`.
|
|
98
98
|
2. Once the `main` pipeline for that commit is green, push tag `vX.Y.Z` on it. A tag on a commit whose checks fail builds no images and cannot be released until it is moved. The tag pipeline builds and pushes the `X.Y.Z` images, then runs e2e for every profile group (about an hour).
|
|
99
99
|
3. Publish the GitHub release with that version's Changelog section as its text, so the two never differ: `python scripts/release_notes.py X.Y.Z > /tmp/notes.md`, then `gh release create vX.Y.Z --title vX.Y.Z --notes-file /tmp/notes.md`. `publish.yml` waits for a green tag pipeline and checks that every `X.Y.Z` image is in ghcr, then uploads to PyPI.
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: odctl
|
|
3
|
-
Version: 1.
|
|
3
|
+
Version: 1.4.0
|
|
4
4
|
Summary: Run an open data stack on your machine, with its services already connected, from one command.
|
|
5
5
|
Project-URL: Homepage, https://github.com/jaehyeon-kim/odctl
|
|
6
6
|
Project-URL: Repository, https://github.com/jaehyeon-kim/odctl
|
|
@@ -94,7 +94,7 @@ Each command has its own section in the CLI reference, with its options and exam
|
|
|
94
94
|
|
|
95
95
|
## Roadmap
|
|
96
96
|
|
|
97
|
-
odctl follows semantic versioning from 1.0, so a later 1.x release does not break a project that pins `odctl>=1.0,<2`. The next releases add
|
|
97
|
+
odctl follows semantic versioning from 1.0, so a later 1.x release does not break a project that pins `odctl>=1.0,<2`. The next releases add a written list of what 1.x keeps stable with JSON output, an MCP server for AI agents, a Langfuse profile, and lineage and alert rules. The [roadmap page](https://jaehyeon.me/odctl/latest/about/roadmap/) shows each step in order.
|
|
98
98
|
|
|
99
99
|
## Related Reading
|
|
100
100
|
|
|
@@ -57,7 +57,7 @@ Each command has its own section in the CLI reference, with its options and exam
|
|
|
57
57
|
|
|
58
58
|
## Roadmap
|
|
59
59
|
|
|
60
|
-
odctl follows semantic versioning from 1.0, so a later 1.x release does not break a project that pins `odctl>=1.0,<2`. The next releases add
|
|
60
|
+
odctl follows semantic versioning from 1.0, so a later 1.x release does not break a project that pins `odctl>=1.0,<2`. The next releases add a written list of what 1.x keeps stable with JSON output, an MCP server for AI agents, a Langfuse profile, and lineage and alert rules. The [roadmap page](https://jaehyeon.me/odctl/latest/about/roadmap/) shows each step in order.
|
|
61
61
|
|
|
62
62
|
## Related Reading
|
|
63
63
|
|
|
@@ -67,6 +67,9 @@ exclude_docs: |
|
|
|
67
67
|
# A link to a heading that does not exist is a warning, which --strict fails.
|
|
68
68
|
validation:
|
|
69
69
|
anchors: warn
|
|
70
|
+
nav:
|
|
71
|
+
omitted_files: warn
|
|
72
|
+
not_found: warn
|
|
70
73
|
|
|
71
74
|
extra_css:
|
|
72
75
|
- stylesheets/extra.css
|
|
@@ -109,6 +112,7 @@ nav:
|
|
|
109
112
|
- Temporal Workers: guides/temporal-workers.md
|
|
110
113
|
- ML and AI:
|
|
111
114
|
- Feast with Iceberg: guides/feast-iceberg.md
|
|
115
|
+
- Feast Vector Retrieval: guides/feast-vector-retrieval.md
|
|
112
116
|
- MLflow: guides/mlflow.md
|
|
113
117
|
- Evidently Reports: guides/evidently-reports.md
|
|
114
118
|
- Hybrid Search: guides/hybrid-search.md
|
|
@@ -369,6 +369,8 @@ Feast: Experiment tracking, model registry and model serving (MLflow), the featu
|
|
|
369
369
|
• Online feature server for get_online_features:
|
|
370
370
|
* External (host): http://127.0.0.1:6566
|
|
371
371
|
* Internal (Docker): http://feast-serve:6566
|
|
372
|
+
• MCP endpoint, for agents that list Feast's tools and read features:
|
|
373
|
+
http://127.0.0.1:6566/mcp
|
|
372
374
|
• Liveness: http://127.0.0.1:6566/health
|
|
373
375
|
```
|
|
374
376
|
|
|
@@ -8,11 +8,8 @@
|
|
|
8
8
|
# half runs in the caller's process and the published image has feast[minimal].
|
|
9
9
|
#
|
|
10
10
|
# On an IcebergSource against the catalog profile, pass warehouse="" and
|
|
11
|
-
# catalog_name="odctl"
|
|
12
|
-
#
|
|
13
|
-
# serves the specification with no prefix, so a real warehouse returns
|
|
14
|
-
# HTTP 400 "Ambiguous URI empty segment". PyIceberg still gets the real
|
|
15
|
-
# warehouse, from the PYICEBERG_CATALOG__ODCTL__* variables in compose-mlops.yml.
|
|
11
|
+
# catalog_name="odctl", so that PyIceberg takes the catalog address and the
|
|
12
|
+
# warehouse from the PYICEBERG_CATALOG__ODCTL__* variables in compose-mlops.yml.
|
|
16
13
|
project: odctl
|
|
17
14
|
provider: local
|
|
18
15
|
|
|
@@ -33,4 +30,12 @@ online_store:
|
|
|
33
30
|
connection_string: "valkey:6379,username=user,password=password"
|
|
34
31
|
key_ttl_seconds: 604800
|
|
35
32
|
|
|
33
|
+
# MCP endpoint at /mcp on the feature server, so an agent can list Feast's tools
|
|
34
|
+
# and read features. http is Streamable HTTP, which the MCP specification of
|
|
35
|
+
# 2025-03-26 put in place of the older HTTP with SSE transport (sse).
|
|
36
|
+
feature_server:
|
|
37
|
+
type: mcp
|
|
38
|
+
mcp_enabled: true
|
|
39
|
+
mcp_transport: http
|
|
40
|
+
|
|
36
41
|
entity_key_serialization_version: 3
|
|
@@ -269,6 +269,8 @@ stacks:
|
|
|
269
269
|
• Online feature server for get_online_features:
|
|
270
270
|
* External (host): http://127.0.0.1:6566
|
|
271
271
|
* Internal (Docker): http://feast-serve:6566
|
|
272
|
+
• MCP endpoint, for agents that list Feast's tools and read features:
|
|
273
|
+
http://127.0.0.1:6566/mcp
|
|
272
274
|
• Liveness: http://127.0.0.1:6566/health
|
|
273
275
|
evidently: |
|
|
274
276
|
• Evidently UI & API (no login, and writes are open):
|
|
@@ -285,8 +285,8 @@ def test_snippet_changes_select_the_groups_that_run_them():
|
|
|
285
285
|
assert "spark-lite spark-full" in selector.select(
|
|
286
286
|
["src/odctl/resources/compose-kafka.yml"], registry
|
|
287
287
|
)
|
|
288
|
-
assert "flink-lite flink-full"
|
|
289
|
-
["src/odctl/resources/compose-mlops.yml"], registry
|
|
288
|
+
assert {"flink-lite flink-full", "airflow"} <= set(
|
|
289
|
+
selector.select(["src/odctl/resources/compose-mlops.yml"], registry)
|
|
290
290
|
)
|
|
291
291
|
assert "flink-lite flink-full" in selector.select(
|
|
292
292
|
["src/odctl/resources/compose-kafka.yml"], registry
|
|
@@ -305,7 +305,7 @@ def test_snippet_changes_select_the_groups_that_run_them():
|
|
|
305
305
|
]
|
|
306
306
|
assert set(
|
|
307
307
|
selector.select(["docs/snippets/feast-iceberg/steps.sql"], registry)
|
|
308
|
-
) == {"feast", "flink-lite flink-full", "mlflow"}
|
|
308
|
+
) == {"feast", "flink-lite flink-full", "mlflow", "airflow"}
|
|
309
309
|
|
|
310
310
|
|
|
311
311
|
def test_pages_keep_within_the_heading_limits():
|
|
@@ -328,3 +328,35 @@ def test_pages_keep_within_the_heading_limits():
|
|
|
328
328
|
f"{page.relative_to(REPO)}: {len(second)} ##, {len(headings)} in all"
|
|
329
329
|
)
|
|
330
330
|
assert not over, over
|
|
331
|
+
|
|
332
|
+
|
|
333
|
+
def test_metadata_example_groups_are_selected_wherever_metadata_is():
|
|
334
|
+
selector = load(
|
|
335
|
+
"select_profiles", REPO / ".github" / "scripts" / "select-profiles.py"
|
|
336
|
+
)
|
|
337
|
+
registry = INTERNAL_RESOURCES_DIR / "registry.yml"
|
|
338
|
+
groups = ["metadata", "metadata:deps", "metadata:autopilot"]
|
|
339
|
+
# The two snippet folders the metadata cases read select exactly the metadata
|
|
340
|
+
# groups whose smoke.sh case runs them; how-it-works also selects mlflow.
|
|
341
|
+
for changed, runs_it in (
|
|
342
|
+
("docs/snippets/openmetadata/commands.sh", {"metadata", "metadata:autopilot"}),
|
|
343
|
+
(
|
|
344
|
+
"docs/snippets/how-it-works/commands.sh",
|
|
345
|
+
{"metadata:deps", "metadata:autopilot", "mlflow"},
|
|
346
|
+
),
|
|
347
|
+
):
|
|
348
|
+
selected = set(selector.select([changed], registry))
|
|
349
|
+
assert runs_it <= selected, changed
|
|
350
|
+
assert not (set(groups) - runs_it) & selected, changed
|
|
351
|
+
assert (
|
|
352
|
+
selector.select(["src/odctl/resources/compose-metadata.yml"], registry)
|
|
353
|
+
== groups
|
|
354
|
+
)
|
|
355
|
+
for group in ("metadata:deps", "metadata:autopilot"):
|
|
356
|
+
assert group not in selector.select(
|
|
357
|
+
["src/odctl/resources/compose-orch.yml"], registry
|
|
358
|
+
)
|
|
359
|
+
# Without a case, smoke.sh falls through to its default and passes silently.
|
|
360
|
+
smoke = (REPO / ".github" / "scripts" / "smoke.sh").read_text()
|
|
361
|
+
for group in groups:
|
|
362
|
+
assert re.search(rf"^ {group}\)", smoke, re.M), group
|
|
@@ -69,3 +69,14 @@ def test_feast_ui_waits_for_a_healthy_feast_serve():
|
|
|
69
69
|
assert services["feast-ui"]["depends_on"] == {
|
|
70
70
|
"feast-serve": {"condition": "service_healthy"}
|
|
71
71
|
}
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def test_the_bundled_repository_serves_mcp_and_defines_no_features():
|
|
75
|
+
repo = get_internal_resources_dir() / "feast"
|
|
76
|
+
config = yaml.safe_load((repo / "feature_store.yaml").read_text())
|
|
77
|
+
assert config["feature_server"] == {
|
|
78
|
+
"type": "mcp",
|
|
79
|
+
"mcp_enabled": True,
|
|
80
|
+
"mcp_transport": "http",
|
|
81
|
+
}
|
|
82
|
+
assert [p.name for p in repo.iterdir()] == ["feature_store.yaml"]
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
"""The files for contributors and assistants name only files that exist.
|
|
2
|
+
|
|
3
|
+
`README.md`, `CLAUDE.md` and `CONTRIBUTING.md` name files in backticks, often as a bare
|
|
4
|
+
name inside a folder's row of the code map. A file moved or renamed without these docs
|
|
5
|
+
leaves the name stale, so every name that looks like a path must be the end of a
|
|
6
|
+
tracked path.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
import re
|
|
10
|
+
import subprocess
|
|
11
|
+
from pathlib import Path
|
|
12
|
+
|
|
13
|
+
ROOT = Path(__file__).resolve().parents[1]
|
|
14
|
+
_DOCS = ["README.md", "CLAUDE.md", "CONTRIBUTING.md"]
|
|
15
|
+
# Names of files a reader or a command creates, not files in the repository.
|
|
16
|
+
_NOT_FILES = {"llms.txt"}
|
|
17
|
+
_SUFFIXES = (
|
|
18
|
+
".py",
|
|
19
|
+
".md",
|
|
20
|
+
".yml",
|
|
21
|
+
".yaml",
|
|
22
|
+
".json",
|
|
23
|
+
".txt",
|
|
24
|
+
".toml",
|
|
25
|
+
".sh",
|
|
26
|
+
".drawio",
|
|
27
|
+
".csv",
|
|
28
|
+
)
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def _tracked_paths() -> set[str]:
|
|
32
|
+
listed = subprocess.run(
|
|
33
|
+
["git", "-C", str(ROOT), "ls-files"], capture_output=True, text=True, check=True
|
|
34
|
+
)
|
|
35
|
+
paths = set()
|
|
36
|
+
for line in listed.stdout.splitlines():
|
|
37
|
+
parts = line.split("/")
|
|
38
|
+
for start in range(len(parts)):
|
|
39
|
+
for end in range(start + 1, len(parts) + 1):
|
|
40
|
+
paths.add("/".join(parts[start:end]))
|
|
41
|
+
return paths
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def _named_paths(text: str) -> set[str]:
|
|
45
|
+
names = set()
|
|
46
|
+
for name in re.findall(r"`([^`\s]+)`", text):
|
|
47
|
+
if re.search(r"[<>*{}$~:=()]|^-|^\.|^http", name) or name in _NOT_FILES:
|
|
48
|
+
continue
|
|
49
|
+
if "/" in name or name.endswith(_SUFFIXES):
|
|
50
|
+
names.add(name.rstrip("/"))
|
|
51
|
+
return names
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def test_named_paths_exist():
|
|
55
|
+
tracked = _tracked_paths()
|
|
56
|
+
missing = [
|
|
57
|
+
f"{doc}: {name}"
|
|
58
|
+
for doc in _DOCS
|
|
59
|
+
for name in sorted(_named_paths((ROOT / doc).read_text()))
|
|
60
|
+
if name not in tracked
|
|
61
|
+
]
|
|
62
|
+
assert missing == []
|