datacontract-cli 0.10.37__py3-none-any.whl → 0.10.40__py3-none-any.whl

datacontract/cli.py

@@ -15,8 +15,8 @@ from datacontract.catalog.catalog import create_data_contract_html, create_index
 from datacontract.data_contract import DataContract, ExportFormat
 from datacontract.imports.importer import ImportFormat, Spec
 from datacontract.init.init_template import get_init_template
-from datacontract.integration.datamesh_manager import (
-    publish_data_contract_to_datamesh_manager,
+from datacontract.integration.entropy_data import (
+    publish_data_contract_to_entropy_data,
 )
 from datacontract.lint.resolve import resolve_data_contract_dict
 from datacontract.model.exceptions import DataContractException
@@ -406,7 +406,7 @@ def publish(
     """
     enable_debug_logging(debug)
 
-    publish_data_contract_to_datamesh_manager(
+    publish_data_contract_to_entropy_data(
         data_contract_dict=resolve_data_contract_dict(location),
         ssl_verification=ssl_verification,
     )

datacontract/data_contract.py

@@ -24,7 +24,7 @@ from datacontract.export.exporter import ExportFormat
 from datacontract.export.exporter_factory import exporter_factory
 from datacontract.imports.importer_factory import importer_factory
 from datacontract.init.init_template import get_init_template
-from datacontract.integration.datamesh_manager import publish_test_results_to_datamesh_manager
+from datacontract.integration.entropy_data import publish_test_results_to_entropy_data
 from datacontract.lint import resolve
 from datacontract.model.data_contract_specification import DataContractSpecification, Info
 from datacontract.model.exceptions import DataContractException
@@ -151,7 +151,7 @@ class DataContract:
         run.finish()
 
         if self._publish_url is not None or self._publish_test_results:
-            publish_test_results_to_datamesh_manager(run, self._publish_url, self._ssl_verification)
+            publish_test_results_to_entropy_data(run, self._publish_url, self._ssl_verification)
 
         return run
 

datacontract/engines/soda/check_soda_execute.py

@@ -3,6 +3,7 @@ import typing
 import uuid
 
 from datacontract.engines.soda.connections.athena import to_athena_soda_configuration
+from datacontract.engines.soda.connections.oracle import initialize_client_and_create_soda_configuration
 
 if typing.TYPE_CHECKING:
     from pyspark.sql import SparkSession
@@ -104,6 +105,10 @@ def check_soda_execute(
         soda_configuration_str = to_sqlserver_soda_configuration(server)
         scan.add_configuration_yaml_str(soda_configuration_str)
         scan.set_data_source_name(server.type)
+    elif server.type == "oracle":
+        soda_configuration_str = initialize_client_and_create_soda_configuration(server)
+        scan.add_configuration_yaml_str(soda_configuration_str)
+        scan.set_data_source_name(server.type)
     elif server.type == "trino":
         soda_configuration_str = to_trino_soda_configuration(server)
         scan.add_configuration_yaml_str(soda_configuration_str)

datacontract/engines/soda/connections/athena.py

@@ -71,7 +71,7 @@ def to_athena_soda_configuration(server):
         data_source["catalog"] = server.catalog
 
     if s3_session_token:
-        data_source["aws_session_token"] = s3_session_token
+        data_source["session_token"] = s3_session_token
 
     soda_configuration = {f"data_source {server.type}": data_source}
 

datacontract/engines/soda/connections/oracle.py

@@ -0,0 +1,50 @@
+import os
+
+import yaml
+
+from datacontract.model.data_contract_specification import Server
+
+
+def initialize_client_and_create_soda_configuration(server: Server) -> str:
+    import oracledb
+    soda_config = to_oracle_soda_configuration(server)
+
+    oracle_client_dir = os.getenv("DATACONTRACT_ORACLE_CLIENT_DIR")
+    if oracle_client_dir is not None:
+        # Soda Core currently does not support thick mode natively, see https://github.com/sodadata/soda-core/issues/2036
+        #   but the oracledb client can be configured accordingly before Soda initializes as a work-around
+        oracledb.init_oracle_client(lib_dir=oracle_client_dir)
+
+    return soda_config
+
+
+def to_oracle_soda_configuration(server: Server) -> str:
+    """Serialize server config to soda configuration.
+
+
+    ### Example:
+        type: oracle
+        host: database-1.us-east-1.rds.amazonaws.com
+        port: '1521'
+        username: simple
+        password: simple_pass
+        connectstring: database-1.us-east-1.rds.amazonaws.com:1521/ORCL (database is equal to service name at oracle)
+        schema: SYSTEM
+    """
+
+    service_name = server.service_name or server.database
+    # with service account key, using an external json file
+    soda_configuration = {
+        f"data_source {server.type}": {
+            "type": "oracle",
+            "host": server.host,
+            "port": str(server.port),
+            "username": os.getenv("DATACONTRACT_ORACLE_USERNAME", ""),
+            "password": os.getenv("DATACONTRACT_ORACLE_PASSWORD", ""),
+            "connectstring": f"{server.host}:{server.port}/{service_name}",
+            "schema": server.schema_,
+        }
+    }
+
+    soda_configuration_str = yaml.dump(soda_configuration)
+    return soda_configuration_str

datacontract/export/odcs_v3_exporter.py

@@ -11,6 +11,7 @@ from open_data_contract_standard.model import (
     Server,
     ServiceLevelAgreementProperty,
     Support,
+    Team,
 )
 
 from datacontract.export.exporter import Exporter
@@ -30,7 +31,7 @@ def to_odcs_v3_yaml(data_contract_spec: DataContractSpecification) -> str:
 
 def to_odcs_v3(data_contract_spec: DataContractSpecification) -> OpenDataContractStandard:
     result = OpenDataContractStandard(
-        apiVersion="v3.0.1",
+        apiVersion="v3.1.0",
         kind="DataContract",
         id=data_contract_spec.id,
         name=data_contract_spec.info.title,
@@ -88,7 +89,10 @@ def to_odcs_v3(data_contract_spec: DataContractSpecification) -> OpenDataContrac
             if server_value.account is not None:
                 server.account = server_value.account
             if server_value.database is not None:
-                server.database = server_value.database
+                if server.type == "oracle":
+                    server.serviceName = server_value.database
+                else:
+                    server.database = server_value.database
             if server_value.schema_ is not None:
                 server.schema_ = server_value.schema_
             if server_value.format is not None:
@@ -127,9 +131,9 @@ def to_odcs_v3(data_contract_spec: DataContractSpecification) -> OpenDataContrac
 
         if len(servers) > 0:
             result.servers = servers
+    if (data_contract_spec.info.owner is not None) and (data_contract_spec.info.owner != ""):
+        result.team = Team(name=data_contract_spec.info.owner)
     custom_properties = []
-    if data_contract_spec.info.owner is not None:
-        custom_properties.append(CustomProperty(property="owner", value=data_contract_spec.info.owner))
     if data_contract_spec.info.model_extra is not None:
         for key, value in data_contract_spec.info.model_extra.items():
             custom_properties.append(CustomProperty(property=key, value=value))
@@ -194,14 +198,10 @@ def to_logical_type(type: str) -> str | None:
         return "integer"
     if type.lower() in ["boolean"]:
         return "boolean"
-    if type.lower() in ["object", "record", "struct"]:
+    if type.lower() in ["object", "record", "struct", "map", "variant"]:
         return "object"
-    if type.lower() in ["bytes"]:
+    if type.lower() in ["bytes", "array"]:
         return "array"
-    if type.lower() in ["array"]:
-        return "array"
-    if type.lower() in ["variant"]:
-        return "variant"
     if type.lower() in ["null"]:
         return None
     return None
@@ -224,6 +224,8 @@ def to_physical_type(config: Dict[str, Any]) -> str | None:
         return config["databricksType"]
     elif "physicalType" in config:
         return config["physicalType"]
+    elif "oracleType" in config:
+        return config["oracleType"]
     return None
 
 

datacontract/export/sql_type_converter.py

@@ -22,6 +22,8 @@ def convert_to_sql_type(field: Field, server_type: str) -> str:
         return convert_type_to_bigquery(field)
     elif server_type == "trino":
         return convert_type_to_trino(field)
+    elif server_type == "oracle":
+        return convert_type_to_oracle(field)
 
     return field.type
 
@@ -390,3 +392,105 @@ def convert_type_to_trino(field: Field) -> None | str:
         return "varbinary"
     if field_type in ["object", "record", "struct"]:
         return "json"
+
+
+def convert_type_to_oracle(field: Field) -> None | str:
+    """Convert from supported datacontract types to equivalent Oracle types
+
+    Oracle returns types WITH precision/scale/length through Soda, so we need to match that.
+    For example:
+    - NUMBER -> NUMBER (base types without precision return without it)
+    - TIMESTAMP -> TIMESTAMP(6) (Oracle default precision)
+    - CHAR -> CHAR (but may need explicit handling)
+
+    For fields that were created with specific Oracle types (like NCHAR, ROWID, BLOB),
+    users should use config.oracleType to override the default mapping.
+
+    Reference: https://docs.oracle.com/en/database/oracle/oracle-database/19/sqlrf/Data-Types.html
+    """
+    # config.oracleType always wins - use it as-is without stripping
+    if field.config and "oracleType" in field.config:
+        return field.config["oracleType"]
+
+    if field.config and "physicalType" in field.config:
+        return field.config["physicalType"]
+
+    field_type = field.type
+    if not field_type:
+        return None
+
+    field_type = field_type.lower()
+
+    # String types - default to NVARCHAR2 for strings
+    if field_type in ["string", "varchar"]:
+        return "NVARCHAR2"
+
+    if field_type == "text":
+        # text could be NVARCHAR2 or NCLOB depending on size
+        if field.config and field.config.get("large"):
+            return "NCLOB"
+        return "NVARCHAR2"
+
+    # Numeric types - NUMBER without precision (Oracle returns just NUMBER)
+    if field_type in ["number", "decimal", "numeric", "int", "integer", "long", "bigint", "smallint"]:
+        return "NUMBER"
+
+    # Float types - BINARY_FLOAT/BINARY_DOUBLE by default
+    if field_type == "float":
+        return "BINARY_FLOAT"
+
+    if field_type in ["double", "double precision"]:
+        return "BINARY_DOUBLE"
+
+    # Boolean - maps to CHAR
+    if field_type == "boolean":
+        return "CHAR"
+
+    # Temporal types - Oracle returns with precision
+    if field_type in ["timestamp_tz", "timestamp with time zone", "timestamptz"]:
+        return "TIMESTAMP(6) WITH TIME ZONE"
+
+    if field_type in ["timestamp_ntz", "timestamp", "timestamp without time zone"]:
+        return "TIMESTAMP(6)"
+
+    if field_type == "date":
+        return "DATE"
+
+    if field_type == "time":
+        # Oracle's INTERVAL DAY TO SECOND has default precision
+        return "INTERVAL DAY(0) TO SECOND(6)"
+
+    # Binary types
+    if field_type in ["bytes", "binary"]:
+        # Default to RAW for bytes
+        return "RAW"
+
+    # LOB types
+    if field_type == "blob":
+        return "BLOB"
+
+    if field_type == "nclob":
+        return "NCLOB"
+
+    if field_type == "clob":
+        return "CLOB"
+
+    # Oracle-specific types
+    if field_type == "bfile":
+        return "BFILE"
+
+    if field_type in ["long raw", "longraw"]:
+        return "LONG RAW"
+
+    if field_type == "rowid":
+        return "ROWID"
+
+    if field_type == "urowid":
+        return "UROWID"
+
+    # Complex/JSON types -> CLOB (emulated)
+    if field_type in ["array", "map", "object", "record", "struct", "variant", "json"]:
+        return "CLOB"
+
+    # Default to CLOB for unknown types
+    return "CLOB"

datacontract/imports/excel_importer.py

@@ -16,6 +16,7 @@ from open_data_contract_standard.model import (
     ServiceLevelAgreementProperty,
     Support,
     Team,
+    TeamMember,
 )
 from openpyxl.cell.cell import Cell
 from openpyxl.workbook.workbook import Workbook
@@ -540,7 +541,7 @@ def import_team(workbook: Workbook) -> Optional[List[Team]]:
             if (not (username or name or role)) or row_idx == team_range[0] - 1:
                 continue
 
-            team_member = Team(
+            team_member = TeamMember(
                 username=username,
                 name=name,
                 description=get_cell_value(row, headers.get("description")),

datacontract/imports/odcs_v3_importer.py

@@ -128,6 +128,7 @@ def import_servers(odcs: OpenDataContractStandard) -> Dict[str, Server] | None:
         server.account = odcs_server.account
         server.database = odcs_server.database
         server.schema_ = odcs_server.schema_
+        server.service_name = odcs_server.serviceName
         server.host = odcs_server.host
         server.port = odcs_server.port
         server.catalog = odcs_server.catalog
@@ -196,6 +197,17 @@ def get_server_type(odcs: OpenDataContractStandard) -> str | None:
     return server.type
 
 
+def get_composite_primary_keys(properties: List[SchemaProperty]) -> list[str]:
+    primary_keys = [
+        (property.name, property.primaryKeyPosition)
+        for property in properties
+        if property.name and property.primaryKey is not None and property.primaryKey
+    ]
+
+    primary_keys.sort(key=lambda x: x[1] or -1)
+    return [name for name, _ in primary_keys]
+
+
 def import_models(odcs: Any) -> Dict[str, Model]:
     custom_type_mappings = get_custom_type_mappings(odcs.customProperties)
 
@@ -213,6 +225,8 @@ def import_models(odcs: Any) -> Dict[str, Model]:
             tags=odcs_schema.tags if odcs_schema.tags is not None else None,
         )
         model.fields = import_fields(odcs_schema.properties, custom_type_mappings, server_type=get_server_type(odcs))
+        if has_composite_primary_key(odcs_properties=odcs_schema.properties):
+            model.primaryKey = get_composite_primary_keys(odcs_schema.properties)
         if odcs_schema.quality is not None:
             model.quality = convert_quality_list(odcs_schema.quality)
         model.title = schema_name
@@ -313,6 +327,8 @@ def import_field_config(odcs_property: SchemaProperty, server_type=None) -> dict
             config["sqlserverType"] = physical_type
         elif server_type == "databricks":
             config["databricksType"] = physical_type
+        elif server_type == "oracle":
+            config["oracleType"] = physical_type
         else:
             config["physicalType"] = physical_type
 

datacontract/integration/entropy_data.py

@@ -0,0 +1,126 @@
+import os
+from urllib.parse import urlparse
+
+import requests
+
+from datacontract.model.run import Run
+
+# used to retrieve the HTML location of the published data contract or test results
+RESPONSE_HEADER_LOCATION_HTML = "location-html"
+
+
+def publish_test_results_to_entropy_data(run: Run, publish_url: str, ssl_verification: bool):
+    try:
+        host = publish_url
+        if publish_url is None:
+            # this url supports Data Mesh Manager and Data Contract Manager
+            host = _get_host()
+            url = "%s/api/test-results" % host
+        else:
+            url = publish_url
+
+        api_key = _get_api_key()
+
+        if run.dataContractId is None:
+            raise Exception("Cannot publish run results for unknown data contract ID")
+
+        headers = {"Content-Type": "application/json", "x-api-key": api_key}
+        request_body = run.model_dump_json()
+        # print("Request Body:", request_body)
+        response = requests.post(
+            url,
+            data=request_body,
+            headers=headers,
+            verify=ssl_verification,
+        )
+        # print("Status Code:", response.status_code)
+        # print("Response Body:", response.text)
+        if response.status_code != 200:
+            display_host = _extract_hostname(host)
+            run.log_error(f"Error publishing test results to {display_host}: {response.text}")
+            return
+        run.log_info("Published test results successfully")
+
+        location_html = response.headers.get(RESPONSE_HEADER_LOCATION_HTML)
+        if location_html is not None and len(location_html) > 0:
+            print(f"🚀 Open {location_html}")
+
+    except Exception as e:
+        run.log_error(f"Failed publishing test results. Error: {str(e)}")
+
+
+def publish_data_contract_to_entropy_data(data_contract_dict: dict, ssl_verification: bool):
+    try:
+        api_key = _get_api_key()
+        host = _get_host()
+        headers = {"Content-Type": "application/json", "x-api-key": api_key}
+        id = data_contract_dict["id"]
+        url = f"{host}/api/datacontracts/{id}"
+        response = requests.put(
+            url=url,
+            json=data_contract_dict,
+            headers=headers,
+            verify=ssl_verification,
+        )
+        if response.status_code != 200:
+            display_host = _extract_hostname(host)
+            print(f"Error publishing data contract to {display_host}: {response.text}")
+            exit(1)
+
+        print("✅ Published data contract successfully")
+
+        location_html = response.headers.get(RESPONSE_HEADER_LOCATION_HTML)
+        if location_html is not None and len(location_html) > 0:
+            print(f"🚀 Open {location_html}")
+
+    except Exception as e:
+        print(f"Failed publishing data contract. Error: {str(e)}")
+
+
+def _get_api_key() -> str:
+    """
+    Get API key from environment variables with fallback priority:
+    1. ENTROPY_DATA_API_KEY
+    2. DATAMESH_MANAGER_API_KEY
+    3. DATACONTRACT_MANAGER_API_KEY
+    """
+    api_key = os.getenv("ENTROPY_DATA_API_KEY")
+    if api_key is None:
+        api_key = os.getenv("DATAMESH_MANAGER_API_KEY")
+    if api_key is None:
+        api_key = os.getenv("DATACONTRACT_MANAGER_API_KEY")
+    if api_key is None:
+        raise Exception(
+            "Cannot publish, as neither ENTROPY_DATA_API_KEY, DATAMESH_MANAGER_API_KEY, nor DATACONTRACT_MANAGER_API_KEY is set"
+        )
+    return api_key
+
+
+def _get_host() -> str:
+    """
+    Get host from environment variables with fallback priority:
+    1. ENTROPY_DATA_HOST
+    2. DATAMESH_MANAGER_HOST
+    3. DATACONTRACT_MANAGER_HOST
+    4. Default: https://api.entropy-data.com
+    """
+    host = os.getenv("ENTROPY_DATA_HOST")
+    if host is None:
+        host = os.getenv("DATAMESH_MANAGER_HOST")
+    if host is None:
+        host = os.getenv("DATACONTRACT_MANAGER_HOST")
+    if host is None:
+        host = "https://api.entropy-data.com"
+    return host
+
+
+def _extract_hostname(url: str) -> str:
+    """
+    Extract the hostname (including subdomains and top-level domain) from a URL.
+
+    Examples:
+    - https://app.entropy-data.com/path -> app.entropy-data.com
+    - http://api.example.com:8080/api -> api.example.com
+    """
+    parsed = urlparse(url)
+    return parsed.netloc.split(":")[0] if parsed.netloc else url

datacontract/lint/resolve.py

@@ -312,9 +312,9 @@ def _resolve_data_contract_from_str(
 
     if schema_location is None:
         if is_open_data_contract_standard(yaml_dict):
-            logging.info("Using ODCS 3.0.2 schema to validate data contract")
+            logging.info("Using ODCS 3.1.0 schema to validate data contract")
             # TODO refactor this to a specific function
-            schema_location = resources.files("datacontract").joinpath("schemas", "odcs-3.0.2.schema.json")
+            schema_location = resources.files("datacontract").joinpath("schemas", "odcs-3.1.0.schema.json")
 
     _validate_json_schema(yaml_dict, schema_location)
 

datacontract/lint/urls.py

@@ -28,10 +28,22 @@ def fetch_resource(url: str):
 def _set_api_key(headers, url):
     hostname = urlparse(url).hostname
 
+    entropy_data_api_key = os.getenv("ENTROPY_DATA_API_KEY")
     datamesh_manager_api_key = os.getenv("DATAMESH_MANAGER_API_KEY")
     datacontract_manager_api_key = os.getenv("DATACONTRACT_MANAGER_API_KEY")
 
-    if hostname == "datamesh-manager.com" or hostname.endswith(".datamesh-manager.com"):
+    if hostname == "entropy-data.com" or hostname.endswith(".entropy-data.com"):
+        if entropy_data_api_key is None or entropy_data_api_key == "":
+            print("Error: Entropy Data API key is not set. Set env variable ENTROPY_DATA_API_KEY.")
+            raise DataContractException(
+                type="lint",
+                name=f"Reading data contract from {url}",
+                reason="Error: Entropy Data API key is not set. Set env variable ENTROPY_DATA_API_KEY.",
+                engine="datacontract",
+                result="error",
+            )
+        headers["x-api-key"] = entropy_data_api_key
+    elif hostname == "datamesh-manager.com" or hostname.endswith(".datamesh-manager.com"):
         if datamesh_manager_api_key is None or datamesh_manager_api_key == "":
             print("Error: Data Mesh Manager API key is not set. Set env variable DATAMESH_MANAGER_API_KEY.")
             raise DataContractException(
@@ -54,7 +66,9 @@ def _set_api_key(headers, url):
             )
         headers["x-api-key"] = datacontract_manager_api_key
 
-    if datamesh_manager_api_key is not None and datamesh_manager_api_key != "":
-        headers["x-api-key"] = datamesh_manager_api_key
     if datacontract_manager_api_key is not None and datacontract_manager_api_key != "":
         headers["x-api-key"] = datacontract_manager_api_key
+    if datamesh_manager_api_key is not None and datamesh_manager_api_key != "":
+        headers["x-api-key"] = datamesh_manager_api_key
+    if entropy_data_api_key is not None and entropy_data_api_key != "":
+        headers["x-api-key"] = entropy_data_api_key