datacontract-cli 1.2.4__tar.gz → 1.2.5__tar.gz

{datacontract_cli-1.2.4/datacontract_cli.egg-info → datacontract_cli-1.2.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datacontract-cli
-Version: 1.2.4
+Version: 1.2.5
 Summary: The datacontract CLI is an open source command-line tool for working with Data Contracts. It uses data contract YAML files to lint the data contract, connect to data sources and execute schema and quality tests, detect breaking changes, and export to different formats. The tool is written in Python. It can be used as a standalone CLI tool, in a CI/CD pipeline, or directly as a Python library.
 Author-email: Jochen Christ <jochen.christ@entropy-data.com>, Stefan Negele <stefan.negele@innoq.com>, Simon Harrer <simon.harrer@entropy-data.com>
 License-Expression: MIT
@@ -183,6 +183,7 @@ Dynamic: license-file
   <a href="https://datacontract.com/slack" rel="nofollow"><img src="https://img.shields.io/badge/slack-join_chat-white.svg?logo=slack&amp;style=social" alt="Slack Status" data-canonical-src="https://img.shields.io/badge/slack-join_chat-white.svg?logo=slack&amp;style=social" style="max-width: 100%;"></a>
 </p>
 
+**Turn your schemas into data contracts and test your data against them, on Snowflake, Databricks, BigQuery, Kafka, Postgres & 15 more. Open source, built on the Open Data Contract Standard.**
 The `datacontract` CLI is an open-source command-line tool for working with [data contracts](https://datacontract.com).
 It natively supports the [Open Data Contract Standard](https://bitol-io.github.io/open-data-contract-standard/latest/) to lint data contracts, connect to data sources and execute schema and quality tests, and export to different formats. 
 The tool is written in Python. 

{datacontract_cli-1.2.4 → datacontract_cli-1.2.5}/README.md

@@ -12,6 +12,7 @@
   <a href="https://datacontract.com/slack" rel="nofollow"><img src="https://img.shields.io/badge/slack-join_chat-white.svg?logo=slack&amp;style=social" alt="Slack Status" data-canonical-src="https://img.shields.io/badge/slack-join_chat-white.svg?logo=slack&amp;style=social" style="max-width: 100%;"></a>
 </p>
 
+**Turn your schemas into data contracts and test your data against them, on Snowflake, Databricks, BigQuery, Kafka, Postgres & 15 more. Open source, built on the Open Data Contract Standard.**
 The `datacontract` CLI is an open-source command-line tool for working with [data contracts](https://datacontract.com).
 It natively supports the [Open Data Contract Standard](https://bitol-io.github.io/open-data-contract-standard/latest/) to lint data contracts, connect to data sources and execute schema and quality tests, and export to different formats. 
 The tool is written in Python. 

{datacontract_cli-1.2.4 → datacontract_cli-1.2.5}/datacontract/api.py

@@ -275,6 +275,8 @@ tenants. Send them over HTTPS only.
 See [Configuration](https://docs.datacontract.com/configuration) for the full list of options.
 """
 
+# The endpoints are plain `def`, not `async def`: they block on data sources, and FastAPI runs
+# plain functions in a thread pool, so one slow test does not stall every other request.
 app = FastAPI(
     docs_url="/",
     title="Data Contract CLI API",
@@ -763,7 +765,7 @@ handled. A contract that cannot be parsed is reported the same way, as a failed
     response_model_exclude_none=True,
     response_model_exclude_unset=True,
 )
-async def test(
+def test(
     request: Request,
     body: Annotated[
         str,
@@ -870,7 +872,7 @@ violation — not as an error status. A `schema` that is not an `http(s)` URL is
         },
     },
 )
-async def lint(
+def lint(
     body: Annotated[
         str,
         Body(
@@ -968,7 +970,7 @@ parsed is answered with `422`.
         },
     },
 )
-async def changelog_endpoint(
+def changelog_endpoint(
     body: ChangelogRequest,
     api_key: Annotated[str | None, Depends(api_key_header)] = None,
 ) -> ChangelogResponse:

{datacontract_cli-1.2.4 → datacontract_cli-1.2.5}/datacontract/command_edit.py

@@ -74,7 +74,29 @@ def _cli_version() -> str:
         return "unknown"
 
 
-def _generate_index_html(filename: str) -> str:
+# The editor's AI assistant is configured via these environment variables (e.g., in .env).
+# It is enabled when DATACONTRACT_EDITOR_AI_ENDPOINT is set.
+EDITOR_AI_ENV_VARS = {
+    "provider": "DATACONTRACT_EDITOR_AI_PROVIDER",
+    "endpoint": "DATACONTRACT_EDITOR_AI_ENDPOINT",
+    "apiKey": "DATACONTRACT_EDITOR_AI_API_KEY",
+    "model": "DATACONTRACT_EDITOR_AI_MODEL",
+    "authHeader": "DATACONTRACT_EDITOR_AI_AUTH_HEADER",
+}
+
+
+def editor_ai_config() -> dict | None:
+    """Return the editor's AI assistant config from the environment, or None if no endpoint is set."""
+    if not os.getenv(EDITOR_AI_ENV_VARS["endpoint"]):
+        return None
+    ai = {"enabled": True}
+    for key, env_var in EDITOR_AI_ENV_VARS.items():
+        if os.getenv(env_var):
+            ai[key] = os.getenv(env_var)
+    return ai
+
+
+def _generate_index_html(filename: str, ai_config: dict | None = None) -> str:
     """
     Generate the editor page: load the YAML from the local file API, write it back on
     save, and point the editor's test runner at this server's own /test endpoint.
@@ -82,8 +104,10 @@ def _generate_index_html(filename: str) -> str:
     The editor runs in EMBEDDED mode, since the file menu (New, Load Example, Open)
     makes no sense when the editor is bound to a single local file. The filename is
     shown in the header via titlePrefix, and Cancel reverts to the file on disk.
+    The AI assistant is configured if ai_config is given.
     """
     filename_js = json.dumps(filename)
+    ai_option = f"ai: {json.dumps(ai_config)}," if ai_config else ""
     file_api_path = f"/api/files/{quote(filename)}"
     return f"""<!doctype html>
 <html lang="en">
@@ -136,6 +160,7 @@ def _generate_index_html(filename: str) -> str:
               enabled: true,
               dataContractCliApiServerUrl: window.location.origin,
             }},
+            {ai_option}
             onCancel: async () => {{
               try {{
                 const response = await fetch('{file_api_path}');
@@ -213,7 +238,7 @@ def create_app(
 
     @edit_app.get("/", response_class=HTMLResponse, include_in_schema=False)
     def index():
-        return _generate_index_html(filename)
+        return _generate_index_html(filename, editor_ai_config())
 
     @edit_app.get(EDITOR_ASSETS_PATH + "/{asset_path:path}", include_in_schema=False)
     def editor_asset(asset_path: str):
@@ -337,6 +362,9 @@ def edit(
     the data contract tests locally against the servers defined in the data contract.
     Credentials for the data sources must be provided as environment variables, see
     https://docs.datacontract.com/testing
+    The editor's AI assistant is enabled by setting DATACONTRACT_EDITOR_AI_ENDPOINT, together with
+    DATACONTRACT_EDITOR_AI_API_KEY, DATACONTRACT_EDITOR_AI_MODEL, DATACONTRACT_EDITOR_AI_PROVIDER
+    (openai or anthropic), and DATACONTRACT_EDITOR_AI_AUTH_HEADER (bearer, api-key, or x-api-key).
     """
     enable_debug_logging(debug)
 

{datacontract_cli-1.2.4 → datacontract_cli-1.2.5}/datacontract/config/settings.py

@@ -13,6 +13,9 @@ logger = logging.getLogger(__name__)
 
 _ENV_PREFIX = "DATACONTRACT_"
 
+# Used by the drivers that would otherwise wait forever for an unresponsive data source.
+DEFAULT_CONNECT_TIMEOUT = 30
+
 # Deprecated option (field name) -> replacement (field name). Single source for
 # the accessor warnings, the Snowflake connect synonyms, and the generated docs.
 DEPRECATED_OPTIONS = {
@@ -112,6 +115,7 @@ class Config(BaseSettings):
     api_header_authorization: SecretStr | None = None
     custom_quality_checks: str | None = None
     max_errors: int | None = None
+    connect_timeout: int | None = None
 
     # athena (credentials come from the s3_* options)
     # overrides for the contract's servers block
@@ -530,6 +534,11 @@ class Config(BaseSettings):
     def get_max_errors(self) -> int | None:
         return self._int_option("max_errors")
 
+    # --- connect timeout ---
+    def get_connect_timeout(self) -> int:
+        """Seconds to wait for a data source to accept a connection, for drivers that wait forever by default."""
+        return self._int_option("connect_timeout") or DEFAULT_CONNECT_TIMEOUT
+
     # --- athena ---
     def get_athena_catalog(self, required: bool = False) -> str | None:
         return self._str_option("athena_catalog", required)

{datacontract_cli-1.2.4 → datacontract_cli-1.2.5}/datacontract/data_contract.py

@@ -32,6 +32,13 @@ from datacontract.model.run import Check, ResultEnum, Run
 logger = logging.getLogger(__name__)
 
 
+def _warning_checks(warnings: list[DataContractException]) -> list[Check]:
+    return [
+        Check(type=warning.type, result=warning.result, name=warning.name, reason=warning.reason, engine=warning.engine)
+        for warning in warnings
+    ]
+
+
 def _unrunnable_rule_checks(data_contract: OpenDataContractStandard) -> list[Check]:
     """Warnings for quality rules `datacontract test` would not be able to run."""
     checks = []
@@ -150,7 +157,7 @@ class DataContract:
         run = Run.create_run()
         try:
             run.log_info("Linting data contract")
-            data_contract, schema_version = resolve.resolve_data_contract_with_schema_version(
+            data_contract, schema_version, warnings = resolve.resolve_data_contract_with_lint_info(
                 self._data_contract_file,
                 self._data_contract_str,
                 self._data_contract,
@@ -171,6 +178,7 @@ class DataContract:
                     engine="datacontract-cli",
                 )
             )
+            run.checks.extend(_warning_checks(warnings))
             run.checks.extend(_unrunnable_rule_checks(data_contract))
             run.dataContractId = data_contract.id
             run.dataContractVersion = data_contract.version
@@ -221,7 +229,7 @@ class DataContract:
         try:
             run.log_info("Testing data contract")
             run.log_info(self._runtime_info())
-            data_contract = resolve.resolve_data_contract(
+            data_contract, _, warnings = resolve.resolve_data_contract_with_lint_info(
                 self._data_contract_file,
                 self._data_contract_str,
                 self._data_contract,
@@ -230,6 +238,7 @@ class DataContract:
                 config=self._config,
                 configured_host_only=self._untrusted_contract,
             )
+            run.checks.extend(_warning_checks(warnings))
 
             execute_data_contract_test(
                 data_contract,

{datacontract_cli-1.2.4 → datacontract_cli-1.2.5}/datacontract/engines/checks/create_checks.py

@@ -43,8 +43,15 @@ from datacontract.model.server import get_server_type
 logger = logging.getLogger(__name__)
 
 _FILE_SERVER_TYPES = {"local", "s3", "gcs", "azure"}
-# Spark and every server read through DuckDB
-_NESTED_CHECK_SERVER_TYPES = {"dataframe", "databricks", "duckdb", "iceberg", "kafka"} | _FILE_SERVER_TYPES
+_NESTED_CHECK_SERVER_TYPES = {
+    "dataframe",
+    "databricks",
+    "duckdb",
+    "iceberg",
+    "kafka",
+    "trino",
+    "athena",
+} | _FILE_SERVER_TYPES
 
 
 # ---------------------------------------------------------------------------

datacontract_cli-1.2.5/datacontract/engines/ibis/connections/athena_patch.py

@@ -0,0 +1,20 @@
+"""ibis lists ``ArrayFilter`` and ``ArrayMap`` as unsupported on Athena, although Athena engine
+version 3 runs the ``filter()`` / ``transform()`` its Trino compiler emits; checks on array items need them."""
+
+from __future__ import annotations
+
+
+def apply_athena_compatibility_patch(con) -> None:
+    import ibis.expr.operations as ops
+    from ibis.backends.sql.compilers.athena import AthenaCompiler
+    from ibis.backends.sql.compilers.trino import TrinoCompiler
+
+    lambdas = (ops.ArrayFilter, ops.ArrayMap)
+
+    class ArrayLambdaAthenaCompiler(AthenaCompiler):
+        # ibis replaces the visit method of every op still listed here
+        UNSUPPORTED_OPS = tuple(op for op in AthenaCompiler.UNSUPPORTED_OPS if op not in lambdas)
+        visit_ArrayFilter = TrinoCompiler.visit_ArrayFilter
+        visit_ArrayMap = TrinoCompiler.visit_ArrayMap
+
+    con.compiler = ArrayLambdaAthenaCompiler()

{datacontract_cli-1.2.4 → datacontract_cli-1.2.5}/datacontract/engines/ibis/connections/connect.py

@@ -125,6 +125,8 @@ def connect_ibis(
             password=config.get_postgres_password(required=True),
             database=config.get_postgres_database() or server.database,
             schema=config.get_postgres_schema() or server.schema_,
+            # libpq waits forever for a server that accepts the connection but never answers
+            connect_timeout=config.get_connect_timeout(),
         )
 
     if server_type == "redshift":
@@ -141,6 +143,7 @@ def connect_ibis(
             database=database,
             schema=config.get_redshift_schema() or server.schema_,
             client_encoding=CLIENT_ENCODING,
+            connect_timeout=config.get_connect_timeout(),
         )
         if login.password:
             kwargs["password"] = login.password
@@ -617,7 +620,7 @@ def _connect_mysql_via_duckdb(
     _load_extension(con, "mysql", "mysql")
 
     run.log_info(f"Attaching MySQL {host}:{port} via the duckdb mysql extension")
-    _attach_mysql(con, host, port, user, password, database)
+    _attach_mysql(con, host, port, user, password, database, config.get_connect_timeout())
 
     if data_contract.schema_:
         for schema_obj in data_contract.schema_:
@@ -677,7 +680,8 @@ def _connect_sqlserver(ibis, server: Server, config: Config):
             del kwargs[key]  # pyodbc would send a literal DATABASE=None
         else:
             kwargs[key] = "{" + kwargs[key].replace("}", "}}") + "}"
-    con = pyodbc.connect(**kwargs)
+    # the login timeout, so an unreachable server fails instead of blocking the run
+    con = pyodbc.connect(timeout=config.get_connect_timeout(), **kwargs)
     return ibis.mssql.from_connection(con)
 
 
@@ -803,6 +807,8 @@ def _azure_cli_access_token() -> bytes:
 
 
 def _connect_athena(ibis, server: Server, config: Config):
+    from datacontract.engines.ibis.connections.athena_patch import apply_athena_compatibility_patch
+
     # regionName is a contract value, so the variable still wins over it
     credentials = aws_credentials.client_kwargs(aws_credentials.configured_region(server.regionName, config), config)
     schema = config.get_athena_schema() or server.schema_
@@ -839,7 +845,9 @@ def _connect_athena(ibis, server: Server, config: Config):
     # Optional data source / catalog; pyathena defaults it to `awsdatacatalog`.
     if catalog:
         kwargs["catalog_name"] = catalog
-    return ibis.athena.connect(**kwargs)
+    con = ibis.athena.connect(**kwargs)
+    apply_athena_compatibility_patch(con)
+    return con
 
 
 def _connect_trino(ibis, server: Server, config: Config):

{datacontract_cli-1.2.4 → datacontract_cli-1.2.5}/datacontract/engines/ibis/connections/duckdb_connection.py

@@ -1,10 +1,12 @@
 import logging
 import re
+import socket
 from typing import TYPE_CHECKING, Any, List, Optional
 
 from open_data_contract_standard.model import OpenDataContractStandard, SchemaObject, SchemaProperty, Server
 
 from datacontract.config import Config
+from datacontract.config.settings import DEFAULT_CONNECT_TIMEOUT
 from datacontract.engines.ibis.connections.aws_credentials import resolve_aws_credentials
 from datacontract.export.duckdb_type_converter import convert_to_duckdb_csv_type, convert_to_duckdb_json_type
 from datacontract.export.sql_type_converter import convert_to_duckdb
@@ -507,8 +509,17 @@ def _load_extension(con, name: str, extra: str) -> None:
         ) from e
 
 
-def _attach_mysql(con, host: str, port: int, user: str, password: str, database: str | None) -> None:
+def _attach_mysql(
+    con,
+    host: str,
+    port: int,
+    user: str,
+    password: str,
+    database: str | None,
+    connect_timeout: int = DEFAULT_CONNECT_TIMEOUT,
+) -> None:
     """ATTACH MySQL as ``mysqldb``; the password goes in a secret, as DuckDB echoes the ATTACH string in its errors."""
+    _wait_for_mysql_greeting(host, port, connect_timeout)
     parts = [f"host={host}", f"port={port}", f"user={user}"]
     if database:
         parts.append(f"database={database}")
@@ -520,6 +531,38 @@ def _attach_mysql(con, host: str, port: int, user: str, password: str, database:
         con.execute("DROP TEMPORARY SECRET mysqldb_password")
 
 
+def _wait_for_mysql_greeting(host: str, port: int, timeout: int) -> None:
+    """Fail fast on a MySQL server that does not answer.
+
+    DuckDB's mysql extension has no connect timeout and waits forever for a server that
+    accepts the connection but never answers, e.g. on a wrong port. A MySQL server sends
+    its handshake as soon as the connection is open, so waiting for the first byte tells
+    the two apart. ``localhost`` is skipped: the MySQL client connects there over the
+    Unix socket, not TCP.
+    """
+    from datacontract.model.exceptions import DataContractException
+    from datacontract.model.run import ResultEnum
+
+    if host == "localhost":
+        return
+    try:
+        with socket.create_connection((host, port), timeout=timeout) as sock:
+            if sock.recv(1):
+                return
+            reason = f"MySQL server {host}:{port} closed the connection without a handshake."
+    except TimeoutError:
+        reason = f"MySQL server {host}:{port} did not answer within {timeout} seconds."
+    except OSError as e:
+        reason = f"Could not connect to MySQL server {host}:{port}: {e}"
+    raise DataContractException(
+        type="connection",
+        name="Connect to mysql",
+        result=ResultEnum.error,
+        reason=reason,
+        engine="datacontract-cli",
+    )
+
+
 def _sql_literal(value) -> str:
     """Escape a value for a single-quoted duckdb string literal.
 

{datacontract_cli-1.2.4 → datacontract_cli-1.2.5}/datacontract/engines/ibis/native_type.py

@@ -281,7 +281,7 @@ def _read_information_schema(con, server: Server, model: str) -> Optional[dict[s
     query = (
         "SELECT column_name, data_type, character_maximum_length, "
         "numeric_precision, numeric_scale, datetime_precision "
-        f"FROM information_schema.columns WHERE upper(table_name) = upper('{_quote(model)}')"
+        f"FROM INFORMATION_SCHEMA.COLUMNS WHERE upper(table_name) = upper('{_quote(model)}')"
         f"{_schema_filter(server)}"
     )
     return _map_reconstructed(con, query)