datacontract-cli 0.11.3__tar.gz → 0.11.4__tar.gz

{datacontract_cli-0.11.3/datacontract_cli.egg-info → datacontract_cli-0.11.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datacontract-cli
-Version: 0.11.3
+Version: 0.11.4
 Summary: The datacontract CLI is an open source command-line tool for working with Data Contracts. It uses data contract YAML files to lint the data contract, connect to data sources and execute schema and quality tests, detect breaking changes, and export to different formats. The tool is written in Python. It can be used as a standalone CLI tool, in a CI/CD pipeline, or directly as a Python library.
 Author-email: Jochen Christ <jochen.christ@innoq.com>, Stefan Negele <stefan.negele@innoq.com>, Simon Harrer <simon.harrer@innoq.com>
 License-Expression: MIT
@@ -11,18 +11,15 @@ Classifier: Operating System :: OS Independent
 Requires-Python: <3.13,>=3.10
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: typer<0.20,>=0.15.1
+Requires-Dist: typer<0.22,>=0.15.1
 Requires-Dist: pydantic<2.13.0,>=2.8.2
 Requires-Dist: pyyaml~=6.0.1
 Requires-Dist: requests<2.33,>=2.31
 Requires-Dist: fastjsonschema<2.22.0,>=2.19.1
-Requires-Dist: fastparquet<2026.0.0,>=2024.5.0
-Requires-Dist: numpy<2.0.0,>=1.26.4
+Requires-Dist: pytz>=2024.1
 Requires-Dist: python-multipart<1.0.0,>=0.0.20
 Requires-Dist: rich<15.0,>=13.7
 Requires-Dist: sqlglot<29.0.0,>=26.6.0
-Requires-Dist: duckdb<1.4.0,>=1.0.0
-Requires-Dist: soda-core-duckdb<3.6.0,>=3.3.20
 Requires-Dist: setuptools>=60
 Requires-Dist: python-dotenv<2.0.0,>=1.0.0
 Requires-Dist: boto3<2.0.0,>=1.34.41
@@ -42,14 +39,16 @@ Provides-Extra: databricks
 Requires-Dist: soda-core-spark-df<3.6.0,>=3.3.20; extra == "databricks"
 Requires-Dist: soda-core-spark[databricks]<3.6.0,>=3.3.20; extra == "databricks"
 Requires-Dist: databricks-sql-connector<4.3.0,>=3.7.0; extra == "databricks"
-Requires-Dist: databricks-sdk<0.74.0; extra == "databricks"
-Requires-Dist: pyspark<5.0.0,>=3.5.5; extra == "databricks"
+Requires-Dist: databricks-sdk<0.78.0; extra == "databricks"
+Requires-Dist: pyspark<5.0.0,>=3.5.0; extra == "databricks"
+Requires-Dist: numpy<2.0.0,>=1.26.4; extra == "databricks"
 Provides-Extra: iceberg
 Requires-Dist: pyiceberg==0.10.0; extra == "iceberg"
 Provides-Extra: kafka
 Requires-Dist: datacontract-cli[avro]; extra == "kafka"
 Requires-Dist: soda-core-spark-df<3.6.0,>=3.3.20; extra == "kafka"
-Requires-Dist: pyspark<5.0.0,>=3.5.5; extra == "kafka"
+Requires-Dist: pyspark<5.0.0,>=3.5.0; extra == "kafka"
+Requires-Dist: numpy<2.0.0,>=1.26.4; extra == "kafka"
 Provides-Extra: postgres
 Requires-Dist: soda-core-postgres<3.6.0,>=3.3.20; extra == "postgres"
 Provides-Extra: s3
@@ -70,6 +69,9 @@ Provides-Extra: dbt
 Requires-Dist: dbt-core>=1.8.0; extra == "dbt"
 Provides-Extra: dbml
 Requires-Dist: pydbml>=1.1.1; extra == "dbml"
+Provides-Extra: duckdb
+Requires-Dist: duckdb<1.5.0,>=1.0.0; extra == "duckdb"
+Requires-Dist: soda-core-duckdb<3.6.0,>=3.3.20; extra == "duckdb"
 Provides-Extra: parquet
 Requires-Dist: pyarrow>=18.1.0; extra == "parquet"
 Provides-Extra: rdf
@@ -80,7 +82,7 @@ Requires-Dist: uvicorn==0.40.0; extra == "api"
 Provides-Extra: protobuf
 Requires-Dist: grpcio-tools>=1.53; extra == "protobuf"
 Provides-Extra: all
-Requires-Dist: datacontract-cli[api,athena,bigquery,csv,databricks,dbml,dbt,excel,iceberg,kafka,oracle,parquet,postgres,protobuf,rdf,s3,snowflake,sqlserver,trino]; extra == "all"
+Requires-Dist: datacontract-cli[api,athena,bigquery,csv,databricks,dbml,dbt,duckdb,excel,iceberg,kafka,oracle,parquet,postgres,protobuf,rdf,s3,snowflake,sqlserver,trino]; extra == "all"
 Provides-Extra: dev
 Requires-Dist: datacontract-cli[all]; extra == "dev"
 Requires-Dist: httpx==0.28.1; extra == "dev"
@@ -335,6 +337,7 @@ A list of available extras:
 | Avro Support            | `pip install datacontract-cli[avro]`       |
 | Google BigQuery         | `pip install datacontract-cli[bigquery]`   |
 | Databricks Integration  | `pip install datacontract-cli[databricks]` |
+| DuckDB (local/S3/GCS/Azure file testing) | `pip install datacontract-cli[duckdb]` |
 | Iceberg                 | `pip install datacontract-cli[iceberg]`    |
 | Kafka Integration       | `pip install datacontract-cli[kafka]`      |
 | PostgreSQL Integration  | `pip install datacontract-cli[postgres]`   |

{datacontract_cli-0.11.3 → datacontract_cli-0.11.4}/README.md

@@ -236,6 +236,7 @@ A list of available extras:
 | Avro Support            | `pip install datacontract-cli[avro]`       |
 | Google BigQuery         | `pip install datacontract-cli[bigquery]`   |
 | Databricks Integration  | `pip install datacontract-cli[databricks]` |
+| DuckDB (local/S3/GCS/Azure file testing) | `pip install datacontract-cli[duckdb]` |
 | Iceberg                 | `pip install datacontract-cli[iceberg]`    |
 | Kafka Integration       | `pip install datacontract-cli[kafka]`      |
 | PostgreSQL Integration  | `pip install datacontract-cli[postgres]`   |

{datacontract_cli-0.11.3 → datacontract_cli-0.11.4}/datacontract/api.py

@@ -11,50 +11,192 @@ from fastapi.security.api_key import APIKeyHeader
 from datacontract.data_contract import DataContract, ExportFormat
 from datacontract.model.run import Run
 
-DATA_CONTRACT_EXAMPLE_PAYLOAD = """dataContractSpecification: 1.2.1
-id: urn:datacontract:checkout:orders-latest
-info:
-  title: Orders Latest
-  version: 2.0.0
-  owner: Sales Team
-servers:
-  production:
-    type: s3
-    location: s3://datacontract-example-orders-latest/v2/{model}/*.json
-    format: json
-    delimiter: new_line
-models:
-  orders:
-    description: One record per order. Includes cancelled and deleted orders.
-    type: table
-    fields:
-      order_id:
-        type: string
+DATA_CONTRACT_EXAMPLE_PAYLOAD = """apiVersion: v3.1.0
+kind: DataContract
+id: orders
+name: Orders
+version: 1.0.0
+status: active
+description:
+  purpose: "Provides order and line item data for analytics and reporting"
+  usage: "Used by analytics team for sales analysis and business intelligence"
+  limitations: "Contains only the last 2 years of data"
+  customProperties:
+    - property: "sensitivity"
+      value: "secret"
+      description: "Data contains personally identifiable information"
+  authoritativeDefinitions:
+    - url: "https://entropy-data.com/policies/gdpr-compliance"
+      type: "businessDefinition"
+      description: "GDPR compliance policy for handling customer data"
+schema:
+  - name: orders
+    physicalType: TABLE
+    description: All historic web shop orders since 2020-01-01. Includes successful and cancelled orders.
+    properties:
+      - name: order_id
+        logicalType: string
+        description: The internal order id for every orders. Do not show this to a customer.
+        businessName: Internal Order ID
+        physicalType: UUID
+        examples:
+          - 99e8bb10-3785-4634-9664-8dc79eb69d43
         primaryKey: true
-      order_timestamp:
-        description: The business timestamp in UTC when the order was successfully registered in the source system and the payment was successful.
-        type: timestamp
+        classification: internal
         required: true
+        unique: true
+      - name: customer_id
+        logicalType: string
+        description: A reference to the customer number
+        businessName: Customer Number
+        physicalType: TEXT
         examples:
-          - "2024-09-09T08:30:00Z"
-      order_total:
-        description: Total amount the smallest monetary unit (e.g., cents).
-        type: long
+          - c123456789
         required: true
+        unique: false
+        logicalTypeOptions:
+          minLength: 10
+          maxLength: 10
+        authoritativeDefinitions:
+          - type: definition
+            url: https://example.com/definitions/sales/customer/customer_id
+        tags:
+          - pii:true
+        classification: internal
+        criticalDataElement: true
+      - name: order_total
+        logicalType: integer
+        description: The order total amount in cents, including tax, after discounts.
+          Includes shipping costs.
+        physicalType: INTEGER
         examples:
-          - 9999
+          - "9999"
+        quality:
+          - type: text
+            description: The order_total equals the sum of all related line items.
+        required: true
+        businessName: Order Amount
+      - name: order_timestamp
+        logicalType: timestamp
+        description: The time including timezone when the order payment was successfully
+          confirmed.
+        physicalType: TIMESTAMPTZ
+        businessName: Order Date
+        examples:
+          - "2025-03-01 14:30:00+01"
+      - name: order_status
+        businessName: Status
+        description: The business status of the order
+        logicalType: string
+        physicalType: TEXT
+        examples:
+          - shipped
         quality:
-          - type: sql
-            description: 95% of all order total values are expected to be between 10 and 499 EUR.
-            query: |
-              SELECT quantile_cont(order_total, 0.95) AS percentile_95
-              FROM orders
-            mustBeBetween: [1000, 99900]
-      customer_id:
-        description: Unique identifier for the customer.
-        type: text
-        minLength: 10
-        maxLength: 20
+          - type: library
+            description: Ensure that there are no other status values.
+            metric: invalidValues
+            arguments:
+              validValues:
+                - pending
+                - paid
+                - processing
+                - shipped
+                - delivered
+                - cancelled
+                - refunded
+            mustBe: 0
+    quality:
+      - type: library
+        metric: rowCount
+        mustBeGreaterThan: 100000
+        description: If there are less than 100k rows, something is wrong.
+  - name: line_items
+    physicalType: table
+    description: Details for each item in an order
+    properties:
+      - name: line_item_id
+        logicalType: string
+        description: Unique identifier for the line item
+        physicalType: UUID
+        examples:
+          - 12c9ba21-0c44-4e29-ba72-b8fd01c1be30
+        logicalTypeOptions:
+          format: uuid
+        required: true
+        primaryKey: true
+      - name: sku
+        logicalType: string
+        businessName: Stock Keeping Unit
+        description: Identifier for the purchased product
+        physicalType: TEXT
+        examples:
+          - 111222333
+        required: true
+      - name: price
+        logicalType: integer
+        description: Price in cents for this line item including tax
+        physicalType: INTEGER
+        examples:
+          - 9999
+        required: true
+      - name: order_id
+        required: false
+        primaryKey: false
+        logicalType: string
+        physicalType: UUID
+        relationships:
+          - type: foreignKey
+            to: orders.order_id
+servers:
+  - server: production
+    environment: prod
+    type: postgres
+    host: aws-1-eu-central-2.pooler.supabase.com
+    port: 6543
+    database: postgres
+    schema: dp_orders_v1
+team:
+  name: sales
+  description: This data product is owned by the "Sales" team
+  members:
+    - username: john@example.com
+      name: John Doe
+      role: Owner
+  authoritativeDefinitions:
+    - type: slack
+      url: https://slack.example.com/teams/sales
+roles:
+  - role: analyst_us
+    description: Read access for analytics to US orders
+  - role: analyst_eu
+    description: Read access for analytics to EU orders
+slaProperties:
+  - property: availability
+    value: 99.9%
+    description: Data platform uptime guarantee
+  - property: retention
+    value: "1"
+    unit: year
+    description: Data will be deleted after 1 year
+  - property: freshness
+    value: "24"
+    unit: hours
+    description: Within 24 hours of order placement
+  - property: support
+    value: business hours
+    description: Support only during business hours
+price:
+  priceAmount: 0
+  priceCurrency: USD
+  priceUnit: monthly
+tags:
+  - e-commerce
+  - transactions
+  - pii
+customProperties:
+  - property: dataPlatformRole
+    value: role_orders_v1
+contractCreatedTs: "2025-01-15T10:00:00Z"
 """
 
 app = FastAPI(

{datacontract_cli-0.11.3 → datacontract_cli-0.11.4}/datacontract/data_contract.py

@@ -4,10 +4,9 @@ import typing
 from open_data_contract_standard.model import OpenDataContractStandard, Team
 
 if typing.TYPE_CHECKING:
+    from duckdb.duckdb import DuckDBPyConnection
     from pyspark.sql import SparkSession
 
-from duckdb.duckdb import DuckDBPyConnection
-
 from datacontract.engines.data_contract_test import execute_data_contract_test
 from datacontract.export.exporter import ExportFormat
 from datacontract.export.exporter_factory import exporter_factory
@@ -29,7 +28,7 @@ class DataContract:
         server: str = None,
         publish_url: str = None,
         spark: "SparkSession" = None,
-        duckdb_connection: DuckDBPyConnection = None,
+        duckdb_connection: "DuckDBPyConnection" = None,
         inline_definitions: bool = True,
         ssl_verification: bool = True,
         publish_test_results: bool = False,

{datacontract_cli-0.11.3 → datacontract_cli-0.11.4}/datacontract/engines/data_contract_test.py

@@ -4,12 +4,12 @@ import tempfile
 import typing
 
 import requests
-from duckdb.duckdb import DuckDBPyConnection
 from open_data_contract_standard.model import OpenDataContractStandard, Server
 
 from datacontract.engines.data_contract_checks import create_checks
 
 if typing.TYPE_CHECKING:
+    from duckdb.duckdb import DuckDBPyConnection
     from pyspark.sql import SparkSession
 
 from datacontract.engines.datacontract.check_that_datacontract_contains_valid_servers_configuration import (
@@ -26,7 +26,7 @@ def execute_data_contract_test(
     run: Run,
     server_name: str = None,
     spark: "SparkSession" = None,
-    duckdb_connection: DuckDBPyConnection = None,
+    duckdb_connection: "DuckDBPyConnection" = None,
 ):
     if data_contract.schema_ is None or len(data_contract.schema_) == 0:
         raise DataContractException(

{datacontract_cli-0.11.3 → datacontract_cli-0.11.4}/datacontract/engines/soda/check_soda_execute.py

@@ -6,9 +6,9 @@ from datacontract.engines.soda.connections.athena import to_athena_soda_configur
 from datacontract.engines.soda.connections.oracle import initialize_client_and_create_soda_configuration
 
 if typing.TYPE_CHECKING:
+    from duckdb.duckdb import DuckDBPyConnection
     from pyspark.sql import SparkSession
 
-from duckdb.duckdb import DuckDBPyConnection
 from open_data_contract_standard.model import OpenDataContractStandard, Server
 
 from datacontract.engines.soda.connections.bigquery import to_bigquery_soda_configuration
@@ -29,7 +29,7 @@ def check_soda_execute(
     data_contract: OpenDataContractStandard,
     server: Server,
     spark: "SparkSession" = None,
-    duckdb_connection: DuckDBPyConnection = None,
+    duckdb_connection: "DuckDBPyConnection" = None,
 ):
     from soda.common.config_helper import ConfigHelper
 

{datacontract_cli-0.11.3 → datacontract_cli-0.11.4}/datacontract/engines/soda/connections/duckdb_connection.py

@@ -1,21 +1,32 @@
 import os
 import re
-from typing import Any, List, Optional
+from typing import TYPE_CHECKING, Any, List, Optional
 
-import duckdb
 from open_data_contract_standard.model import OpenDataContractStandard, SchemaObject, SchemaProperty, Server
 
 from datacontract.export.duckdb_type_converter import convert_to_duckdb_csv_type, convert_to_duckdb_json_type
 from datacontract.export.sql_type_converter import convert_to_duckdb
 from datacontract.model.run import Run
 
+if TYPE_CHECKING:
+    import duckdb
+
+
+def _import_duckdb():
+    try:
+        import duckdb
+        return duckdb
+    except ImportError:
+        raise ImportError("duckdb is required for this server type. Install with: pip install datacontract-cli[duckdb]")
+
 
 def get_duckdb_connection(
     data_contract: OpenDataContractStandard,
     server: Server,
     run: Run,
-    duckdb_connection: duckdb.DuckDBPyConnection | None = None,
-) -> duckdb.DuckDBPyConnection:
+    duckdb_connection: "duckdb.DuckDBPyConnection | None" = None,
+) -> "duckdb.DuckDBPyConnection":
+    duckdb = _import_duckdb()
     if duckdb_connection is None:
         con = duckdb.connect(database=":memory:")
     else:
@@ -65,9 +76,9 @@ def get_duckdb_connection(
             elif server.format == "delta":
                 con.sql("update extensions;")  # Make sure we have the latest delta extension
                 con.sql(f"""CREATE VIEW "{model_name}" AS SELECT * FROM delta_scan('{model_path}');""")
-            table_info = con.sql(f"PRAGMA table_info('{model_name}');").fetchdf()
-            if table_info is not None and not table_info.empty:
-                run.log_info(f"DuckDB Table Info: {table_info.to_string(index=False)}")
+            table_info = con.sql(f"PRAGMA table_info('{model_name}');").fetchall()
+            if table_info:
+                run.log_info(f"DuckDB Table Info: {table_info}")
     return con
 
 
@@ -137,7 +148,7 @@ def _get_type(prop: SchemaProperty) -> Optional[str]:
     return None
 
 
-def add_nested_views(con: duckdb.DuckDBPyConnection, model_name: str, properties: List[SchemaProperty] | None):
+def add_nested_views(con: "duckdb.DuckDBPyConnection", model_name: str, properties: List[SchemaProperty] | None):
     model_name = model_name.strip('"')
     if properties is None:
         return

{datacontract_cli-0.11.3 → datacontract_cli-0.11.4}/datacontract/export/spark_exporter.py

@@ -1,4 +1,5 @@
 import json
+import re
 from typing import List, Optional
 
 from open_data_contract_standard.model import OpenDataContractStandard, SchemaObject, SchemaProperty
@@ -119,6 +120,34 @@ def _get_custom_property_value(prop: SchemaProperty, key: str) -> Optional[str]:
     return None
 
 
+def _parse_decimal_precision_scale(physical_type: str) -> tuple[Optional[int], Optional[int]]:
+    """Parse precision and scale from physicalType like 'decimal(10,2)' or 'numeric(18,4)'."""
+    match = re.match(r"(?:decimal|numeric)\s*\(\s*(\d+)\s*,\s*(\d+)\s*\)", physical_type, re.IGNORECASE)
+    if match:
+        return int(match.group(1)), int(match.group(2))
+    return None, None
+
+
+def _get_decimal_type(prop: SchemaProperty) -> types.DecimalType:
+    """Get DecimalType: first from customProperties, then parse from physicalType, else Spark defaults."""
+    # First check customProperties
+    precision_str = _get_custom_property_value(prop, "precision")
+    scale_str = _get_custom_property_value(prop, "scale")
+    if precision_str is not None or scale_str is not None:
+        precision = int(precision_str) if precision_str else types.DecimalType().precision
+        scale = int(scale_str) if scale_str else types.DecimalType().scale
+        return types.DecimalType(precision=precision, scale=scale)
+
+    # Fallback: parse from physicalType
+    if prop.physicalType:
+        precision, scale = _parse_decimal_precision_scale(prop.physicalType)
+        if precision is not None:
+            return types.DecimalType(precision=precision, scale=scale if scale is not None else 0)
+
+    # Use Spark defaults
+    return types.DecimalType()
+
+
 def _logical_type_to_spark_type(logical_type: str) -> types.DataType:
     """Convert a logical type string to a Spark DataType."""
     if logical_type is None:
@@ -216,10 +245,8 @@ def to_spark_data_type(prop: SchemaProperty) -> types.DataType:
     if physical_type:
         if physical_type in ["string", "varchar", "text", "char", "nvarchar"]:
             return types.StringType()
-        if physical_type in ["decimal", "numeric"]:
-            precision = _get_logical_type_option(prop, "precision") or 38
-            scale = _get_logical_type_option(prop, "scale") or 0
-            return types.DecimalType(precision=precision, scale=scale)
+        if physical_type in ["decimal", "numeric"] or physical_type.startswith(("decimal(", "numeric(")):
+            return _get_decimal_type(prop)
         if physical_type in ["integer", "int", "int32"]:
             return types.IntegerType()
         if physical_type in ["long", "bigint", "int64"]:
@@ -244,9 +271,7 @@ def to_spark_data_type(prop: SchemaProperty) -> types.DataType:
         case "string":
             return types.StringType()
         case "number":
-            precision = _get_logical_type_option(prop, "precision") or 38
-            scale = _get_logical_type_option(prop, "scale") or 0
-            return types.DecimalType(precision=precision, scale=scale)
+            return _get_decimal_type(prop)
         case "integer":
             return types.LongType()
         case "boolean":

{datacontract_cli-0.11.3 → datacontract_cli-0.11.4}/datacontract/imports/csv_importer.py

@@ -1,7 +1,6 @@
 import os
 from typing import Dict, List
 
-import duckdb
 from open_data_contract_standard.model import OpenDataContractStandard
 
 from datacontract.imports.importer import Importer
@@ -24,6 +23,11 @@ def import_csv(
     source: str, include_examples: bool = False
 ) -> OpenDataContractStandard:
     """Import a CSV file and create an ODCS data contract."""
+    try:
+        import duckdb
+    except ImportError:
+        raise ImportError("duckdb is required for CSV import. Install with: pip install datacontract-cli[duckdb]")
+
     # use the file name as table name
     table_name = os.path.splitext(os.path.basename(source))[0]