datacontract-cli 1.0.7__tar.gz → 1.0.8__tar.gz

{datacontract_cli-1.0.7/datacontract_cli.egg-info → datacontract_cli-1.0.8}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datacontract-cli
-Version: 1.0.7
+Version: 1.0.8
 Summary: The datacontract CLI is an open source command-line tool for working with Data Contracts. It uses data contract YAML files to lint the data contract, connect to data sources and execute schema and quality tests, detect breaking changes, and export to different formats. The tool is written in Python. It can be used as a standalone CLI tool, in a CI/CD pipeline, or directly as a Python library.
 Author-email: Jochen Christ <jochen.christ@innoq.com>, Stefan Negele <stefan.negele@innoq.com>, Simon Harrer <simon.harrer@innoq.com>
 License-Expression: MIT

{datacontract_cli-1.0.7 → datacontract_cli-1.0.8}/datacontract/engines/ibis/connections/connect.py

@@ -115,7 +115,13 @@ def connect_ibis(
         if password:
             kwargs["password"] = password
         # Redshift speaks the postgres wire protocol; ibis has no dedicated backend.
-        return ibis.postgres.connect(**kwargs)
+        con = ibis.postgres.connect(**kwargs)
+        # ibis's postgres schema introspection joins pg_catalog.pg_enum, which
+        # Redshift does not expose; patch it out so model lookups don't fail.
+        from datacontract.engines.ibis.connections.redshift_patch import apply_redshift_compatibility_patch
+
+        apply_redshift_compatibility_patch(con)
+        return con
 
     if server_type == "mysql":
         return _connect_mysql_via_duckdb(ibis, data_contract, server, run, schema_name)

datacontract_cli-1.0.8/datacontract/engines/ibis/connections/redshift_patch.py

@@ -0,0 +1,84 @@
+"""Make ibis's Postgres schema introspection valid on Amazon Redshift.
+
+Redshift has no dedicated ibis backend; it rides the Postgres backend over the
+Postgres wire protocol (see ``connect.py``). ibis builds the column-metadata
+query behind ``con.table(...)`` (``ibis.backends.postgres.Backend.get_schema``)
+with an enum-detection ``CASE`` that joins ``pg_catalog.pg_enum``:
+
+    CASE WHEN EXISTS(
+      SELECT 1 FROM pg_catalog.pg_type t
+      INNER JOIN pg_catalog.pg_enum e ON e.enumtypid = t.oid ...
+    ) THEN 'enum' ELSE pg_catalog.format_type(...) END
+
+PostgreSQL exposes ``pg_catalog.pg_enum``; Redshift does not, so introspecting
+any model fails with:
+
+    relation "pg_catalog.pg_enum" does not exist
+
+Redshift has no enum types, so the ``CASE`` is pointless there. This module
+re-binds ``get_schema`` on a connected backend with the same query minus the
+enum join, reading the column type straight from ``pg_catalog.format_type``.
+``_get_schema_using_query`` (custom-SQL introspection) calls ``self.get_schema``
+internally, so it inherits the fix without a separate override. The patch can be
+removed once a fixed ibis release is available.
+"""
+
+from __future__ import annotations
+
+import logging
+import types
+
+logger = logging.getLogger(__name__)
+
+# ibis's get_schema query without the pg_enum-dependent enum CASE. Every other
+# relation (pg_attribute / pg_class / pg_namespace) and pg_catalog.format_type
+# is supported by Redshift's leader-node catalog.
+_TYPE_INFO_SQL = """\
+SELECT
+  a.attname AS column_name,
+  pg_catalog.format_type(a.atttypid, a.atttypmod) AS data_type,
+  NOT a.attnotnull AS nullable
+FROM pg_catalog.pg_attribute a
+INNER JOIN pg_catalog.pg_class c
+   ON a.attrelid = c.oid
+INNER JOIN pg_catalog.pg_namespace n
+   ON c.relnamespace = n.oid
+WHERE a.attnum > 0
+  AND NOT a.attisdropped
+  AND n.nspname = ANY(%(dbs)s)
+  AND c.relname = %(name)s
+ORDER BY a.attnum ASC"""
+
+
+def apply_redshift_compatibility_patch(con) -> None:
+    """Re-bind Postgres schema introspection to emit Redshift-compatible SQL.
+
+    Best-effort: if the backend internals differ from what we expect (e.g. a
+    future ibis refactor), the patch is skipped and the original method stays in
+    place rather than breaking the connection.
+    """
+    try:
+        con.get_schema = types.MethodType(_get_schema, con)
+    except Exception:  # pragma: no cover - defensive, never block a connection
+        logger.debug("Could not apply Redshift compatibility patch", exc_info=True)
+
+
+def _get_schema(self, name: str, *, catalog=None, database=None):
+    """Drop-in for ``Backend.get_schema`` without the ``pg_enum`` join."""
+    import ibis.common.exceptions as com
+    import ibis.expr.schema as sch
+
+    dbs = [database or self.current_database]
+    if database is None and (temp_table_db := self._session_temp_db) is not None:
+        dbs.append(temp_table_db)
+
+    type_mapper = self.compiler.type_mapper
+    con = self.con
+    params = {"dbs": dbs, "name": name}
+    with con.cursor() as cursor, con.transaction():
+        rows = cursor.execute(_TYPE_INFO_SQL, params, prepare=True).fetchall()
+
+    if not rows:
+        raise com.TableNotFound(name)
+
+    return sch.Schema({col: type_mapper.from_string(typestr, nullable=nullable) for col, typestr, nullable in rows})

{datacontract_cli-1.0.7 → datacontract_cli-1.0.8/datacontract_cli.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datacontract-cli
-Version: 1.0.7
+Version: 1.0.8
 Summary: The datacontract CLI is an open source command-line tool for working with Data Contracts. It uses data contract YAML files to lint the data contract, connect to data sources and execute schema and quality tests, detect breaking changes, and export to different formats. The tool is written in Python. It can be used as a standalone CLI tool, in a CI/CD pipeline, or directly as a Python library.
 Author-email: Jochen Christ <jochen.christ@innoq.com>, Stefan Negele <stefan.negele@innoq.com>, Simon Harrer <simon.harrer@innoq.com>
 License-Expression: MIT

{datacontract_cli-1.0.7 → datacontract_cli-1.0.8}/datacontract_cli.egg-info/SOURCES.txt

@@ -136,6 +136,7 @@ datacontract/engines/ibis/connections/connect.py
 datacontract/engines/ibis/connections/duckdb_connection.py
 datacontract/engines/ibis/connections/kafka.py
 datacontract/engines/ibis/connections/oracle_patch.py
+datacontract/engines/ibis/connections/redshift_patch.py
 datacontract/export/__init__.py
 datacontract/export/avro_exporter.py
 datacontract/export/avro_idl_exporter.py
@@ -317,6 +318,7 @@ tests/test_import_unity_file.py
 tests/test_integration_entropydata.py
 tests/test_lint.py
 tests/test_oracle_patch.py
+tests/test_redshift_patch.py
 tests/test_require_env.py
 tests/test_resolve.py
 tests/test_resolve_definitions.py
@@ -349,6 +351,7 @@ tests/test_test_postgres.py
 tests/test_test_quality.py
 tests/test_test_quality_local_fallback.py
 tests/test_test_quality_percent_severity.py
+tests/test_test_redshift.py
 tests/test_test_s3_csv.py
 tests/test_test_s3_delta.py
 tests/test_test_s3_json.py

{datacontract_cli-1.0.7 → datacontract_cli-1.0.8}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "datacontract-cli"
-version = "1.0.7"
+version = "1.0.8"
 description = "The datacontract CLI is an open source command-line tool for working with Data Contracts. It uses data contract YAML files to lint the data contract, connect to data sources and execute schema and quality tests, detect breaking changes, and export to different formats. The tool is written in Python. It can be used as a standalone CLI tool, in a CI/CD pipeline, or directly as a Python library."
 license = "MIT"
 readme = "README.md"

datacontract_cli-1.0.8/tests/test_redshift_patch.py

@@ -0,0 +1,91 @@
+"""Regression tests for the Redshift Postgres schema-introspection patch.
+
+Redshift rides ibis's Postgres backend, whose ``get_schema`` query joins
+``pg_catalog.pg_enum`` to detect enum columns. Redshift does not expose
+``pg_enum``, so the unpatched query fails with ``relation "pg_catalog.pg_enum"
+does not exist``. These tests assert the patched query never references
+``pg_enum`` and still types columns correctly, without needing a live Redshift.
+"""
+
+import contextlib
+
+import pytest
+
+sc = pytest.importorskip("ibis.backends.sql.compilers")
+
+from datacontract.engines.ibis.connections import redshift_patch  # noqa: E402
+
+
+class _StubCursor:
+    def __init__(self, rows):
+        self._rows = rows
+
+    def execute(self, sql, params=None, prepare=False):
+        self._executed_sql = sql
+        return self
+
+    def fetchall(self):
+        return self._rows
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, *exc):
+        return False
+
+
+class _StubPostgresConnection:
+    """Minimal stand-in exposing what the patched introspection method touches."""
+
+    name = "postgres"
+    compiler = sc.postgres.compiler
+    current_database = "dev"
+    _session_temp_db = None
+
+    def __init__(self, rows):
+        self._cursor = _StubCursor(rows)
+
+    @property
+    def con(self):
+        return self
+
+    def cursor(self):
+        return self._cursor
+
+    @contextlib.contextmanager
+    def transaction(self):
+        yield
+
+
+def test_get_schema_query_omits_pg_enum_and_types_columns():
+    rows = [
+        ("cust_code", "character varying(10)", False),
+        ("cust_state", "character varying(24)", True),
+    ]
+    con = _StubPostgresConnection(rows)
+
+    schema = redshift_patch._get_schema(con, "customer", database="dwh_academic")
+
+    assert "pg_enum" not in con._cursor._executed_sql
+    assert schema["cust_code"].is_string()
+    assert schema["cust_code"].nullable is False
+    assert schema["cust_state"].nullable is True
+
+
+def test_get_schema_raises_table_not_found_when_empty():
+    import ibis.common.exceptions as com
+
+    con = _StubPostgresConnection([])
+    with pytest.raises(com.TableNotFound):
+        redshift_patch._get_schema(con, "missing", database="dwh_academic")
+
+
+def test_apply_patch_rebinds_get_schema():
+    class _Backend:
+        def get_schema(self, *a, **k):  # pragma: no cover - replaced by the patch
+            return "original"
+
+    backend = _Backend()
+    redshift_patch.apply_redshift_compatibility_patch(backend)
+
+    assert backend.get_schema.__func__ is redshift_patch._get_schema

datacontract_cli-1.0.8/tests/test_test_redshift.py

@@ -0,0 +1,86 @@
+"""Smoke test for the Redshift test path against a Postgres testcontainer.
+
+Redshift has no dedicated ibis backend; it rides the Postgres backend over the
+Postgres wire protocol (see ``connect_ibis``), with two compatibility patches on
+top: tables are qualified with the configured ``schema`` (#1338) and the column
+introspection query is rewritten to drop the ``pg_catalog.pg_enum`` join that
+Redshift does not expose (``redshift_patch``).
+
+No real Redshift is reachable in CI, so this exercises the full ``type: redshift``
+code path — connect branch, schema-qualified introspection, the pg_enum-free
+``get_schema``, the batched aggregations, a pattern check, and custom SQL quality
+checks — against a wire-compatible Postgres. It proves the redshift branch and
+the patched introspection produce correct schemas and a passing run end-to-end;
+it cannot catch Redshift-only dialect divergences (POSIX regex, temp-view-backed
+``con.sql`` falling back to ``raw_sql``), which need a live cluster.
+"""
+
+import pytest
+from testcontainers.postgres import PostgresContainer
+
+from datacontract.data_contract import DataContract
+from datacontract.model.exceptions import DataContractException
+from datacontract.model.run import ResultEnum
+
+postgres = PostgresContainer("postgres:16")
+
+
+@pytest.fixture(scope="function", autouse=True)
+def postgres_container(request):
+    postgres.start()
+
+    def remove_container():
+        postgres.stop()
+
+    request.addfinalizer(remove_container)
+
+
+def test_test_redshift_path_via_postgres_wire(postgres_container, monkeypatch):
+    monkeypatch.setenv("DATACONTRACT_REDSHIFT_USERNAME", postgres.username)
+    monkeypatch.setenv("DATACONTRACT_REDSHIFT_PASSWORD", postgres.password)
+    _init_sql("fixtures/postgres/data/data.sql")
+
+    data_contract_str = _setup_datacontract("fixtures/redshift/odcs.yaml")
+    data_contract = DataContract(data_contract_str=data_contract_str)
+
+    run = data_contract.test()
+
+    print(run.pretty())
+    assert run.result == "passed"
+    assert all(check.result == ResultEnum.passed for check in run.checks)
+
+
+def _setup_datacontract(file):
+    with open(file) as data_contract_file:
+        data_contract_str = data_contract_file.read()
+    port = postgres.get_exposed_port(5432)
+    # The fixture uses the default Redshift port 5439 as a placeholder.
+    return data_contract_str.replace("5439", str(port))
+
+
+def _init_sql(file_path):
+    try:
+        import psycopg2
+    except ImportError as e:
+        raise DataContractException(
+            type="schema",
+            result="failed",
+            name="postgres extra missing",
+            reason="Install the extra datacontract-cli[postgres] to use postgres",
+            engine="datacontract",
+            original_exception=e,
+        )
+
+    connection = psycopg2.connect(
+        database=postgres.dbname,
+        user=postgres.username,
+        password=postgres.password,
+        host=postgres.get_container_host_ip(),
+        port=postgres.get_exposed_port(5432),
+    )
+    cursor = connection.cursor()
+    with open(file_path, "r") as sql_file:
+        cursor.execute(sql_file.read())
+    connection.commit()
+    cursor.close()
+    connection.close()