datacontract-cli 1.2.3__tar.gz → 1.2.5__tar.gz

{datacontract_cli-1.2.3/datacontract_cli.egg-info → datacontract_cli-1.2.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datacontract-cli
-Version: 1.2.3
+Version: 1.2.5
 Summary: The datacontract CLI is an open source command-line tool for working with Data Contracts. It uses data contract YAML files to lint the data contract, connect to data sources and execute schema and quality tests, detect breaking changes, and export to different formats. The tool is written in Python. It can be used as a standalone CLI tool, in a CI/CD pipeline, or directly as a Python library.
 Author-email: Jochen Christ <jochen.christ@entropy-data.com>, Stefan Negele <stefan.negele@innoq.com>, Simon Harrer <simon.harrer@entropy-data.com>
 License-Expression: MIT
@@ -40,7 +40,7 @@ Requires-Dist: jsonschema<5.0.0,>=4.23.0
 Requires-Dist: pytz>=2024.1
 Requires-Dist: python-multipart<1.0.0,>=0.0.20
 Requires-Dist: rich<16.0,>=13.7
-Requires-Dist: sqlglot<30.18,>=26.6.0
+Requires-Dist: sqlglot<31.0.0,>=26.6.0
 Requires-Dist: python-dotenv<2.0.0,>=1.0.0
 Requires-Dist: boto3<2.0.0,>=1.34.41
 Requires-Dist: Jinja2<4.0.0,>=3.1.5
@@ -104,7 +104,7 @@ Requires-Dist: duckdb-extension-azure<1.6.0,>=1.5.0; extra == "azure"
 Requires-Dist: azure-identity<1.26,>=1.17.1; extra == "azure"
 Requires-Dist: azure-storage-blob<12.31,>=12.24; extra == "azure"
 Provides-Extra: snowflake
-Requires-Dist: snowflake-connector-python[pandas]<4.8,>=3.6; extra == "snowflake"
+Requires-Dist: snowflake-connector-python[pandas]<4.9,>=3.6; extra == "snowflake"
 Requires-Dist: ibis-framework[snowflake]<13.0.0,>=10.0.0; extra == "snowflake"
 Provides-Extra: sqlserver
 Requires-Dist: ibis-framework[mssql]<13.0.0,>=10.0.0; extra == "sqlserver"
@@ -141,7 +141,7 @@ Requires-Dist: pyarrow>=18.1.0; extra == "parquet"
 Provides-Extra: rdf
 Requires-Dist: rdflib==7.6.0; extra == "rdf"
 Provides-Extra: api
-Requires-Dist: fastapi<0.142.0,>=0.115.0; extra == "api"
+Requires-Dist: fastapi<0.143.0,>=0.115.0; extra == "api"
 Requires-Dist: uvicorn<0.55.0,>=0.44.0; extra == "api"
 Provides-Extra: protobuf
 Requires-Dist: proto-schema-parser<3.0,>=1.0; extra == "protobuf"
@@ -161,10 +161,10 @@ Requires-Dist: pre-commit<4.7.0,>=3.7.1; extra == "dev"
 Requires-Dist: pytest; extra == "dev"
 Requires-Dist: pytest-cov; extra == "dev"
 Requires-Dist: pytest-xdist; extra == "dev"
-Requires-Dist: pymssql==2.4.1; extra == "dev"
+Requires-Dist: pymssql==2.4.2; extra == "dev"
 Requires-Dist: psycopg2-binary<3.0,>=2.9; extra == "dev"
 Requires-Dist: mysql-connector-python<26.8.0,>=8.0.30; extra == "dev"
-Requires-Dist: ruff==0.16.9; extra == "dev"
+Requires-Dist: ruff==0.16.10; extra == "dev"
 Requires-Dist: testcontainers[kafka,minio,mssql,mysql,postgres]==4.15.0; extra == "dev"
 Requires-Dist: trino==0.339.0; extra == "dev"
 Dynamic: license-file
@@ -183,6 +183,7 @@ Dynamic: license-file
   <a href="https://datacontract.com/slack" rel="nofollow"><img src="https://img.shields.io/badge/slack-join_chat-white.svg?logo=slack&amp;style=social" alt="Slack Status" data-canonical-src="https://img.shields.io/badge/slack-join_chat-white.svg?logo=slack&amp;style=social" style="max-width: 100%;"></a>
 </p>
 
+**Turn your schemas into data contracts and test your data against them, on Snowflake, Databricks, BigQuery, Kafka, Postgres & 15 more. Open source, built on the Open Data Contract Standard.**
 The `datacontract` CLI is an open-source command-line tool for working with [data contracts](https://datacontract.com).
 It natively supports the [Open Data Contract Standard](https://bitol-io.github.io/open-data-contract-standard/latest/) to lint data contracts, connect to data sources and execute schema and quality tests, and export to different formats. 
 The tool is written in Python. 
@@ -256,7 +257,7 @@ Server: production (type=postgres, host=aws-1-eu-central-2.pooler.supabase.com,
 │ passed │ Check that field order_total has physical type INTEGER            │ orders.order_total      │         │
 │ passed │ Check that field order_total has no missing values                │ orders.order_total      │         │
 ╰────────┴───────────────────────────────────────────────────────────────────┴─────────────────────────┴─────────╯
-🟢 Data contract is valid. Run 30 checks. Took 2.504495 seconds.
+🟢 Data contract is valid. Ran 30 checks. Took 2.504495 seconds.
 ```
 
 Voilà, the CLI tested that the YAML itself is valid, all records comply with the schema, and all quality attributes are met.

{datacontract_cli-1.2.3 → datacontract_cli-1.2.5}/README.md

@@ -12,6 +12,7 @@
   <a href="https://datacontract.com/slack" rel="nofollow"><img src="https://img.shields.io/badge/slack-join_chat-white.svg?logo=slack&amp;style=social" alt="Slack Status" data-canonical-src="https://img.shields.io/badge/slack-join_chat-white.svg?logo=slack&amp;style=social" style="max-width: 100%;"></a>
 </p>
 
+**Turn your schemas into data contracts and test your data against them, on Snowflake, Databricks, BigQuery, Kafka, Postgres & 15 more. Open source, built on the Open Data Contract Standard.**
 The `datacontract` CLI is an open-source command-line tool for working with [data contracts](https://datacontract.com).
 It natively supports the [Open Data Contract Standard](https://bitol-io.github.io/open-data-contract-standard/latest/) to lint data contracts, connect to data sources and execute schema and quality tests, and export to different formats. 
 The tool is written in Python. 
@@ -85,7 +86,7 @@ Server: production (type=postgres, host=aws-1-eu-central-2.pooler.supabase.com,
 │ passed │ Check that field order_total has physical type INTEGER            │ orders.order_total      │         │
 │ passed │ Check that field order_total has no missing values                │ orders.order_total      │         │
 ╰────────┴───────────────────────────────────────────────────────────────────┴─────────────────────────┴─────────╯
-🟢 Data contract is valid. Run 30 checks. Took 2.504495 seconds.
+🟢 Data contract is valid. Ran 30 checks. Took 2.504495 seconds.
 ```
 
 Voilà, the CLI tested that the YAML itself is valid, all records comply with the schema, and all quality attributes are met.

{datacontract_cli-1.2.3 → datacontract_cli-1.2.5}/datacontract/api.py

@@ -275,6 +275,8 @@ tenants. Send them over HTTPS only.
 See [Configuration](https://docs.datacontract.com/configuration) for the full list of options.
 """
 
+# The endpoints are plain `def`, not `async def`: they block on data sources, and FastAPI runs
+# plain functions in a thread pool, so one slow test does not stall every other request.
 app = FastAPI(
     docs_url="/",
     title="Data Contract CLI API",
@@ -449,7 +451,8 @@ def config_from_headers(headers) -> "Config | None":
     → ``DATACONTRACT_SNOWFLAKE_PASSWORD``, ``entropy-data-api-key`` →
     ``ENTROPY_DATA_API_KEY``. Returns None when no config headers are present,
     so env-var-configured deployments behave exactly as before. Unknown
-    ``datacontract-*`` option names are rejected with a 400.
+    ``datacontract-*`` option names are rejected with a 400, and so is the
+    custom quality checks folder.
     """
     known = known_env_names()
     values = {}
@@ -457,6 +460,12 @@ def config_from_headers(headers) -> "Config | None":
         lowered = name.lower()
         env = lowered.upper().replace("-", "_")
         if lowered.startswith(_CONFIG_HEADER_PREFIX) or env in known:
+            if env == "DATACONTRACT_CUSTOM_QUALITY_CHECKS":
+                # A folder on the server's disk is the operator's to choose, never the caller's.
+                raise HTTPException(
+                    status_code=status.HTTP_400_BAD_REQUEST,
+                    detail="The custom quality checks folder is configured by the API server, not per request.",
+                )
             values[env] = value
     if not values:
         return None
@@ -756,7 +765,7 @@ handled. A contract that cannot be parsed is reported the same way, as a failed
     response_model_exclude_none=True,
     response_model_exclude_unset=True,
 )
-async def test(
+def test(
     request: Request,
     body: Annotated[
         str,
@@ -863,7 +872,7 @@ violation — not as an error status. A `schema` that is not an `http(s)` URL is
         },
     },
 )
-async def lint(
+def lint(
     body: Annotated[
         str,
         Body(
@@ -961,7 +970,7 @@ parsed is answered with `422`.
         },
     },
 )
-async def changelog_endpoint(
+def changelog_endpoint(
     body: ChangelogRequest,
     api_key: Annotated[str | None, Depends(api_key_header)] = None,
 ) -> ChangelogResponse:

{datacontract_cli-1.2.3 → datacontract_cli-1.2.5}/datacontract/cli.py

@@ -15,7 +15,7 @@ from rich.markup import escape
 from typer.core import TyperGroup
 from typing_extensions import Annotated
 
-from datacontract.config import Config, set_cli_config
+from datacontract.config import Config, cli_config, set_cli_config
 from datacontract.output.output_format import OutputFormat
 
 console = Console()
@@ -31,6 +31,23 @@ inline_references_option = Annotated[
     ),
 ]
 
+custom_quality_checks_option = Annotated[
+    Optional[Path],
+    typer.Option(
+        metavar="DIR",
+        help="Folder with the custom quality checks the contract's quality rules use, one YAML file per check. "
+        "Defaults to DATACONTRACT_CUSTOM_QUALITY_CHECKS.",
+    ),
+]
+
+
+def config_with_custom_quality_checks(custom_quality_checks: Optional[Path]) -> Config | None:
+    """The --config-file config, with the --custom-quality-checks folder if one was given."""
+    config = cli_config()
+    if custom_quality_checks is None:
+        return config
+    return Config.resolve(config).model_copy(update={"custom_quality_checks": str(custom_quality_checks)})
+
 
 # Order in which top-level commands appear in `datacontract --help` (cf. README.md)
 COMMAND_ORDER = [
@@ -130,6 +147,8 @@ app = typer.Typer(
     cls=OrderedCommandsWithMigrationHints,
     no_args_is_help=True,
     add_completion=False,
+    # the locals of a --debug traceback can hold credentials
+    pretty_exceptions_show_locals=False,
     help="CLI to manage data contracts. Documentation: https://docs.datacontract.com",
     epilog="Read the full documentation at https://docs.datacontract.com",
 )

{datacontract_cli-1.2.3 → datacontract_cli-1.2.5}/datacontract/command_ci.py

@@ -10,14 +10,15 @@ from datacontract.cli import (
     _print_logs,
     _print_publish_failure,
     app,
+    config_with_custom_quality_checks,
     console,
+    custom_quality_checks_option,
     debug_option,
     enable_debug_logging,
     inline_references_option,
     resolve_output_format,
     validate_publish_url,
 )
-from datacontract.config import cli_config
 from datacontract.data_contract import DataContract
 from datacontract.output.ci_output import write_ci_output, write_ci_summary, write_json_results
 from datacontract.output.output_format import OutputFormat
@@ -80,6 +81,7 @@ def ci(
         bool,
         typer.Option(help="SSL verification when publishing the data contract."),
     ] = True,
+    custom_quality_checks: custom_quality_checks_option = None,
     inline_references: inline_references_option = True,
     debug: debug_option = None,
 ):
@@ -104,6 +106,7 @@ def ci(
     # Plain text output for CI logs; --json sends human output to stderr.
     out = Console(stderr=True, no_color=True) if json_output else Console(no_color=True)
 
+    config = config_with_custom_quality_checks(custom_quality_checks)
     results = []
     fail_results = {
         "warning": {"warning", "failed", "error"},
@@ -115,7 +118,7 @@ def ci(
     for location in locations:
         out.print(f"Testing {location}")
         run = DataContract(
-            config=cli_config(),
+            config=config,
             data_contract_file=location,
             schema_location=schema,
             publish_url=publish,

{datacontract_cli-1.2.3 → datacontract_cli-1.2.5}/datacontract/command_edit.py

@@ -74,7 +74,29 @@ def _cli_version() -> str:
         return "unknown"
 
 
-def _generate_index_html(filename: str) -> str:
+# The editor's AI assistant is configured via these environment variables (e.g., in .env).
+# It is enabled when DATACONTRACT_EDITOR_AI_ENDPOINT is set.
+EDITOR_AI_ENV_VARS = {
+    "provider": "DATACONTRACT_EDITOR_AI_PROVIDER",
+    "endpoint": "DATACONTRACT_EDITOR_AI_ENDPOINT",
+    "apiKey": "DATACONTRACT_EDITOR_AI_API_KEY",
+    "model": "DATACONTRACT_EDITOR_AI_MODEL",
+    "authHeader": "DATACONTRACT_EDITOR_AI_AUTH_HEADER",
+}
+
+
+def editor_ai_config() -> dict | None:
+    """Return the editor's AI assistant config from the environment, or None if no endpoint is set."""
+    if not os.getenv(EDITOR_AI_ENV_VARS["endpoint"]):
+        return None
+    ai = {"enabled": True}
+    for key, env_var in EDITOR_AI_ENV_VARS.items():
+        if os.getenv(env_var):
+            ai[key] = os.getenv(env_var)
+    return ai
+
+
+def _generate_index_html(filename: str, ai_config: dict | None = None) -> str:
     """
     Generate the editor page: load the YAML from the local file API, write it back on
     save, and point the editor's test runner at this server's own /test endpoint.
@@ -82,8 +104,10 @@ def _generate_index_html(filename: str) -> str:
     The editor runs in EMBEDDED mode, since the file menu (New, Load Example, Open)
     makes no sense when the editor is bound to a single local file. The filename is
     shown in the header via titlePrefix, and Cancel reverts to the file on disk.
+    The AI assistant is configured if ai_config is given.
     """
     filename_js = json.dumps(filename)
+    ai_option = f"ai: {json.dumps(ai_config)}," if ai_config else ""
     file_api_path = f"/api/files/{quote(filename)}"
     return f"""<!doctype html>
 <html lang="en">
@@ -136,6 +160,7 @@ def _generate_index_html(filename: str) -> str:
               enabled: true,
               dataContractCliApiServerUrl: window.location.origin,
             }},
+            {ai_option}
             onCancel: async () => {{
               try {{
                 const response = await fetch('{file_api_path}');
@@ -213,7 +238,7 @@ def create_app(
 
     @edit_app.get("/", response_class=HTMLResponse, include_in_schema=False)
     def index():
-        return _generate_index_html(filename)
+        return _generate_index_html(filename, editor_ai_config())
 
     @edit_app.get(EDITOR_ASSETS_PATH + "/{asset_path:path}", include_in_schema=False)
     def editor_asset(asset_path: str):
@@ -337,6 +362,9 @@ def edit(
     the data contract tests locally against the servers defined in the data contract.
     Credentials for the data sources must be provided as environment variables, see
     https://docs.datacontract.com/testing
+    The editor's AI assistant is enabled by setting DATACONTRACT_EDITOR_AI_ENDPOINT, together with
+    DATACONTRACT_EDITOR_AI_API_KEY, DATACONTRACT_EDITOR_AI_MODEL, DATACONTRACT_EDITOR_AI_PROVIDER
+    (openai or anthropic), and DATACONTRACT_EDITOR_AI_AUTH_HEADER (bearer, api-key, or x-api-key).
     """
     enable_debug_logging(debug)
 

{datacontract_cli-1.2.3 → datacontract_cli-1.2.5}/datacontract/command_test.py

@@ -10,7 +10,9 @@ from datacontract.cli import (
     _print_logs,
     _print_publish_failure,
     app,
+    config_with_custom_quality_checks,
     console,
+    custom_quality_checks_option,
     debug_option,
     enable_debug_logging,
     inline_references_option,
@@ -200,6 +202,7 @@ def test(
         bool,
         typer.Option(help="SSL verification when publishing the data contract."),
     ] = True,
+    custom_quality_checks: custom_quality_checks_option = None,
     inline_references: inline_references_option = True,
     debug: debug_option = None,
 ):
@@ -231,7 +234,7 @@ def test(
     if server == "all":
         server = None
     run = DataContract(
-        config=cli_config(),
+        config=config_with_custom_quality_checks(custom_quality_checks),
         data_contract_file=location,
         schema_location=schema,
         publish_test_results=publish_test_results,

{datacontract_cli-1.2.3 → datacontract_cli-1.2.5}/datacontract/config/settings.py

@@ -13,6 +13,9 @@ logger = logging.getLogger(__name__)
 
 _ENV_PREFIX = "DATACONTRACT_"
 
+# Used by the drivers that would otherwise wait forever for an unresponsive data source.
+DEFAULT_CONNECT_TIMEOUT = 30
+
 # Deprecated option (field name) -> replacement (field name). Single source for
 # the accessor warnings, the Snowflake connect synonyms, and the generated docs.
 DEPRECATED_OPTIONS = {
@@ -110,7 +113,9 @@ class Config(BaseSettings):
 
     # general
     api_header_authorization: SecretStr | None = None
+    custom_quality_checks: str | None = None
     max_errors: int | None = None
+    connect_timeout: int | None = None
 
     # athena (credentials come from the s3_* options)
     # overrides for the contract's servers block
@@ -521,10 +526,19 @@ class Config(BaseSettings):
     def get_api_header_authorization(self, required: bool = False) -> str | None:
         return self._str_option("api_header_authorization", required)
 
+    # --- custom quality checks ---
+    def get_custom_quality_checks(self) -> str | None:
+        return self._str_option("custom_quality_checks")
+
     # --- max ---
     def get_max_errors(self) -> int | None:
         return self._int_option("max_errors")
 
+    # --- connect timeout ---
+    def get_connect_timeout(self) -> int:
+        """Seconds to wait for a data source to accept a connection, for drivers that wait forever by default."""
+        return self._int_option("connect_timeout") or DEFAULT_CONNECT_TIMEOUT
+
     # --- athena ---
     def get_athena_catalog(self, required: bool = False) -> str | None:
         return self._str_option("athena_catalog", required)

{datacontract_cli-1.2.3 → datacontract_cli-1.2.5}/datacontract/config/variables.py

@@ -231,26 +231,56 @@ _DEFERRED_FIELDS = {
 }
 
 
-def resolve_runtime_variables(value, source: str = "contract", variables: Optional[Mapping[str, str]] = None):
+# Compared with the value a query returns, so a whole reference keeps the type it reads as.
+_TYPED_FIELDS = {("DataQuality", "mustBe"), ("DataQuality", "mustNotBe")}
+_PLAIN_NUMBER = re.compile(r"-?(0|[1-9][0-9]*)(\.[0-9]+)?")
+
+
+def resolve_runtime_variables(
+    value, source: str = "contract", variables: Optional[Mapping[str, str]] = None, typed: bool = False
+):
     """Copy test inputs with variables resolved, recursively, without altering the contract.
 
     Handles property names/types, enum values, logical type options, library
     quality arguments, SLA values, and nested array/map definitions. Dictionary
-    keys are structural identifiers and are never interpolated.
+    keys are structural identifiers and are never interpolated. With ``typed``, a
+    value that is one whole reference becomes a number or boolean if its resolved
+    text is a plain one (``1000``, ``-2.5``, ``true``); ``007`` stays a string.
     """
     if isinstance(value, BaseModel):
         owner = type(value).__name__
         updates = {}
         for field in type(value).model_fields:
-            if field in _DEFERRED_FIELDS:
-                continue
             original = getattr(value, field)
             field_source = f"{source}.{_field_name(field)}"
-            resolved = resolve_runtime_variables(original, field_source, variables)
+            # A custom quality check's arguments are test inputs, like a library metric's.
+            if (
+                field == "implementation"
+                and getattr(value, "engine", None) == "datacontract-cli"
+                and isinstance(original, dict)
+                and "arguments" in original
+            ):
+                arguments = resolve_runtime_variables(original["arguments"], f"{field_source}.arguments", variables)
+                updates[field] = {**original, "arguments": arguments}
+                continue
+            if field in _DEFERRED_FIELDS:
+                continue
+            resolved = resolve_runtime_variables(original, field_source, variables, (owner, field) in _TYPED_FIELDS)
             updates[field] = _checked(original, resolved, owner, field, field_source)
         return value.model_copy(update=updates)
     if isinstance(value, dict):
-        return {key: resolve_runtime_variables(item, f"{source}.{key}", variables) for key, item in value.items()}
+        return {
+            key: resolve_runtime_variables(item, f"{source}.{key}", variables, typed) for key, item in value.items()
+        }
     if isinstance(value, list):
-        return [resolve_runtime_variables(item, f"{source}[{index}]", variables) for index, item in enumerate(value)]
-    return resolve_variables(value, source=source, variables=variables)
+        return [
+            resolve_runtime_variables(item, f"{source}[{index}]", variables, typed) for index, item in enumerate(value)
+        ]
+    resolved = resolve_variables(value, source=source, variables=variables)
+    if not typed or not isinstance(value, str) or not _VARIABLE.fullmatch(value):
+        return resolved
+    if _PLAIN_NUMBER.fullmatch(resolved):
+        return float(resolved) if "." in resolved else int(resolved)
+    if resolved in ("true", "false"):
+        return resolved == "true"
+    return resolved

{datacontract_cli-1.2.3 → datacontract_cli-1.2.5}/datacontract/data_contract.py

@@ -32,6 +32,13 @@ from datacontract.model.run import Check, ResultEnum, Run
 logger = logging.getLogger(__name__)
 
 
+def _warning_checks(warnings: list[DataContractException]) -> list[Check]:
+    return [
+        Check(type=warning.type, result=warning.result, name=warning.name, reason=warning.reason, engine=warning.engine)
+        for warning in warnings
+    ]
+
+
 def _unrunnable_rule_checks(data_contract: OpenDataContractStandard) -> list[Check]:
     """Warnings for quality rules `datacontract test` would not be able to run."""
     checks = []
@@ -150,7 +157,7 @@ class DataContract:
         run = Run.create_run()
         try:
             run.log_info("Linting data contract")
-            data_contract, schema_version = resolve.resolve_data_contract_with_schema_version(
+            data_contract, schema_version, warnings = resolve.resolve_data_contract_with_lint_info(
                 self._data_contract_file,
                 self._data_contract_str,
                 self._data_contract,
@@ -171,6 +178,7 @@ class DataContract:
                     engine="datacontract-cli",
                 )
             )
+            run.checks.extend(_warning_checks(warnings))
             run.checks.extend(_unrunnable_rule_checks(data_contract))
             run.dataContractId = data_contract.id
             run.dataContractVersion = data_contract.version
@@ -221,7 +229,7 @@ class DataContract:
         try:
             run.log_info("Testing data contract")
             run.log_info(self._runtime_info())
-            data_contract = resolve.resolve_data_contract(
+            data_contract, _, warnings = resolve.resolve_data_contract_with_lint_info(
                 self._data_contract_file,
                 self._data_contract_str,
                 self._data_contract,
@@ -230,6 +238,7 @@ class DataContract:
                 config=self._config,
                 configured_host_only=self._untrusted_contract,
             )
+            run.checks.extend(_warning_checks(warnings))
 
             execute_data_contract_test(
                 data_contract,

{datacontract_cli-1.2.3 → datacontract_cli-1.2.5}/datacontract/engines/checks/check_spec.py

@@ -27,6 +27,7 @@ class MetricType(str, Enum):
     MISSING_COUNT = "missing_count"
     DUPLICATE_COUNT = "duplicate_count"
     INVALID_COUNT = "invalid_count"
+    MISSING_REFERENCE_COUNT = "missing_reference_count"
     FIELD_PRESENT = "field_present"
     FIELD_TYPE = "field_type"
     FIELD_PHYSICAL_TYPE = "field_physical_type"
@@ -158,10 +159,14 @@ class CheckSpec:
     expected_schema_property: Optional["SchemaProperty"] = None  # FIELD_TYPE: structural comparison
     expected_physical_type: Optional[str] = None  # FIELD_PHYSICAL_TYPE: contract physicalType
 
-    columns: Optional[List[str]] = None  # DUPLICATE_COUNT across multiple columns
+    columns: Optional[List[str]] = None  # DUPLICATE_COUNT across multiple columns; MISSING_REFERENCE_COUNT keys
+
+    referenced_model: Optional[str] = None  # MISSING_REFERENCE_COUNT
+    referenced_columns: Optional[List[str]] = None  # MISSING_REFERENCE_COUNT, pairwise with `columns`
 
     query: Optional[str] = None  # CUSTOM_SQL (placeholders already substituted)
     dialect: Optional[str] = None  # CUSTOM_SQL input SQL dialect
+    diagnostics: Optional[dict] = None  # CUSTOM_SQL: reported alongside the measured value
 
     seconds: Optional[int] = None  # FRESHNESS / RETENTION threshold in seconds