odctl 1.0.2__tar.gz → 1.2.0__tar.gz

{odctl-1.0.2 → odctl-1.2.0}/.github/renovate.json

@@ -33,6 +33,14 @@
       "depNameTemplate": "org.apache.fluss:fluss-client",
       "datasourceTemplate": "maven"
     },
+    {
+      "customType": "regex",
+      "description": "Track SPARK_V in versions.env, so the Spark Kafka connector jars follow the apache/spark image tag.",
+      "managerFilePatterns": ["/^src/odctl/resources/deps/versions\\.env$/"],
+      "matchStrings": ["(?m)^SPARK_V=(?<currentValue>\\S+)\\s*$"],
+      "depNameTemplate": "org.apache.spark:spark-sql-kafka-0-10_2.13",
+      "datasourceTemplate": "maven"
+    },
     {
       "customType": "regex",
       "description": "Track DEB_V in fetch_standalone_plugins.sh, which no built-in manager reads.",
@@ -106,7 +114,7 @@
     },
     {
       "description": "Group Spark and PySpark updates together to prevent Py4J mismatches",
-      "matchPackageNames": ["apache/spark", "pyspark"],
+      "matchPackageNames": ["apache/spark", "pyspark", "org.apache.spark:{/,}**"],
       "groupName": "Spark Ecosystem"
     },
     {

{odctl-1.0.2 → odctl-1.2.0}/.github/scripts/select-profiles.py

@@ -10,7 +10,8 @@ itself, the registry, the smoke script or the images, still runs everything.
 Groups are listed here rather than derived, because the grouping is a decision:
 lite and full variants share a large image, so running them together pays for
 the pull once. Which compose file each group needs comes from registry.yml, so
-the mapping is not repeated.
+the mapping is not repeated, except the profiles a group's case starts from
+another file, listed in ALSO_RUNS.
 
 Usage: select-profiles.py <changed-file>...
        select-profiles.py --all
@@ -56,6 +57,7 @@ GROUPS = [
 RUN_EVERYTHING = (
     "src/odctl/",
     ".github/scripts/smoke.sh",
+    ".github/scripts/spark_avro_producer.py",
     ".github/scripts/select-profiles.py",
     ".github/workflows/",
 )
@@ -66,6 +68,43 @@ COMPOSE_PREFIX = "src/odctl/resources/compose-"
 
 REGISTRY = Path("src/odctl/resources/registry.yml")
 
+# smoke.sh starts some profiles from another group's case: the kafka case starts
+# catalog, the flink case starts kafka-lite and feast, the spark case starts
+# kafka-lite, and the fluss case starts flink-lite. A change to that compose file
+# needs these groups as well as the ones the registry maps to it.
+ALSO_RUNS = {
+    "compose-infra.yml": ["kafka-lite kafka-full"],
+    "compose-kafka.yml": ["flink-lite flink-full", "spark-lite spark-full"],
+    "compose-flink.yml": ["fluss"],
+    "compose-mlops.yml": ["flink-lite flink-full"],
+}
+
+# smoke.sh runs the guide commands in docs/snippets/<guide>/, so a change there
+# needs the groups whose assertions run that guide. A folder missing from this
+# map runs everything, and a test fails until it is added.
+SNIPPET_PREFIX = "docs/snippets/"
+SNIPPET_GROUPS = {
+    "kafka": ["kafka-lite kafka-full"],
+    "kafka-connect": ["kafka-lite kafka-full"],
+    "iceberg-ingestion": ["kafka-lite kafka-full", "flink-lite flink-full"],
+    "flink-sql": ["flink-lite flink-full", "mlflow"],
+    "fluss": ["fluss"],
+    "spark": ["spark-lite spark-full"],
+    "clickhouse": ["ch-lite ch-full"],
+    "trino-metabase": ["trino", "metabase"],
+    "hybrid-search": ["postgres"],
+    "storage": ["storage"],
+    "valkey": ["valkey"],
+    "airflow-dags": ["airflow"],
+    "temporal-workers": ["temporal"],
+    "mlflow": ["mlflow"],
+    "feast-iceberg": ["feast"],
+    "evidently-reports": ["evidently"],
+    "lineage": ["lineage"],
+    "openmetadata": ["metadata"],
+    "telemetry": ["telemetry"],
+}
+
 
 def profile_to_file(registry_path: Path = REGISTRY) -> dict:
     """Map each profile to the compose file the registry says provides it."""
@@ -94,15 +133,22 @@ def select(changed: list, registry_path: Path = REGISTRY) -> list:
     if broad:
         return list(GROUPS)
 
+    snippet_groups = set()
+    for c in changed:
+        if c.startswith(SNIPPET_PREFIX):
+            guide = c[len(SNIPPET_PREFIX) :].split("/")[0]
+            if guide not in SNIPPET_GROUPS:
+                return list(GROUPS)
+            snippet_groups.update(SNIPPET_GROUPS[guide])
+
     mapping = profile_to_file(registry_path)
     touched_files = {Path(c).name for c in changed if is_compose(c)}
-    if not touched_files:
-        return []
 
     selected = []
     for group in GROUPS:
         files = {mapping.get(p) for p in group.split()}
-        if files & touched_files:
+        also = any(group in ALSO_RUNS.get(name, []) for name in touched_files)
+        if files & touched_files or group in snippet_groups or also:
             selected.append(group)
     return selected