datacontract-cli 1.2.1__tar.gz → 1.2.2__tar.gz

{datacontract_cli-1.2.1/datacontract_cli.egg-info → datacontract_cli-1.2.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datacontract-cli
-Version: 1.2.1
+Version: 1.2.2
 Summary: The datacontract CLI is an open source command-line tool for working with Data Contracts. It uses data contract YAML files to lint the data contract, connect to data sources and execute schema and quality tests, detect breaking changes, and export to different formats. The tool is written in Python. It can be used as a standalone CLI tool, in a CI/CD pipeline, or directly as a Python library.
 Author-email: Jochen Christ <jochen.christ@entropy-data.com>, Stefan Negele <stefan.negele@innoq.com>, Simon Harrer <simon.harrer@entropy-data.com>
 License-Expression: MIT

{datacontract_cli-1.2.1 → datacontract_cli-1.2.2}/datacontract/api.py

@@ -12,6 +12,7 @@ from fastapi.security.api_key import APIKeyHeader
 from pydantic import BaseModel, Field, ValidationError
 
 from datacontract.config import Config, known_env_names
+from datacontract.config.variables import CONTRACT_VARIABLES_ENV, allowed_environment, resolve_variables
 from datacontract.data_contract import DataContract, ExportFormat
 from datacontract.model.breaking import BreakingChangeEntry
 from datacontract.model.changelog import ChangelogEntry
@@ -527,7 +528,10 @@ def _selected_server(body: str, server_name: str | None, config):
 
     try:
         data_contract = resolve.resolve_data_contract(data_contract_str=body, config=config)
-        return get_server(data_contract, server_name)
+        server = get_server(data_contract, server_name)
+        # Guard the server type the test will run against, not a ${VAR:-local} reference to it.
+        variables = allowed_environment(_allowed_contract_variables())
+        return server and server.model_copy(update={"type": resolve_variables(server.type, variables=variables)})
     except Exception:
         return None
 
@@ -540,6 +544,11 @@ ALLOW_LOCAL_FILES_ENV = "DATACONTRACT_CLI_API_ALLOW_LOCAL_FILES"
 _TRUTHY = frozenset({"1", "true", "yes", "on"})
 
 
+def _allowed_contract_variables() -> list[str]:
+    """The variable-name globs a posted contract may resolve; none by default."""
+    return (os.getenv(CONTRACT_VARIABLES_ENV) or "").split(",")
+
+
 def _local_files_allowed() -> bool:
     """Whether the operator opted in to serving data from the server's own disk.
 
@@ -569,7 +578,7 @@ def _reject_local_server_type(body: str, server_name: str | None, config) -> Non
         detail=(
             f"Server type '{server.type}' reads from the file system of the server running this API, "
             f"so it is refused. Use a server type that names a data source, such as s3, gcs, azure, "
-            f"postgres, or snowflake — or set {ALLOW_LOCAL_FILES_ENV}=true if this deployment serves "
+            f"postgres, or snowflake — or start the API server with --allow-local-files if this deployment serves "
             f"its own files on purpose."
         ),
     )
@@ -814,6 +823,7 @@ async def test(
         filter=filter,
         filters=parsed_filters,
         untrusted_contract=untrusted_contract,
+        allowed_variables=_allowed_contract_variables(),
     ).test()
 
 

{datacontract_cli-1.2.1 → datacontract_cli-1.2.2}/datacontract/command_api.py

@@ -1,7 +1,10 @@
+import os
+
 import typer
 from typing_extensions import Annotated
 
 from datacontract.cli import app, debug_option, enable_debug_logging
+from datacontract.config.variables import CONTRACT_VARIABLES_ENV
 
 
 def _get_uvicorn_arguments(port: int, host: str, reload: bool, context: typer.Context) -> dict:
@@ -40,6 +43,20 @@ def api(
             help="Watch the source files and restart the server on changes. For development only; off by default.",
         ),
     ] = False,
+    contract_variables: Annotated[
+        str | None,
+        typer.Option(
+            help="Environment variables a posted data contract may read through ${VAR} references, "
+            "as comma-separated fnmatch globs (e.g. 'TABLE_*,CUTOFF_DATE', or '*' to allow all). Empty by default.",
+        ),
+    ] = None,
+    allow_local_files: Annotated[
+        bool | None,
+        typer.Option(
+            "--allow-local-files/--no-allow-local-files",
+            help="Let a posted data contract read the server's own disk through servers[].type: local. Off by default.",
+        ),
+    ] = None,
     debug: debug_option = None,
 ):
     """
@@ -60,6 +77,14 @@ def api(
     """
     enable_debug_logging(debug)
 
+    # Passed through the environment, which survives uvicorn's --reload respawn.
+    from datacontract.api import ALLOW_LOCAL_FILES_ENV
+
+    if contract_variables is not None:
+        os.environ[CONTRACT_VARIABLES_ENV] = contract_variables
+    if allow_local_files is not None:
+        os.environ[ALLOW_LOCAL_FILES_ENV] = "true" if allow_local_files else "false"
+
     import uvicorn
     from uvicorn.config import LOGGING_CONFIG
 

{datacontract_cli-1.2.1 → datacontract_cli-1.2.2}/datacontract/command_import.py

@@ -909,3 +909,52 @@ def import_athena(
         id=id,
     )
     _write_result(result, output)
+
+
+@import_app.command(
+    name="odata",
+    epilog="Example: datacontract import odata --service-root-url https://example.com/odata/ --entity-set Products --output datacontract.yaml",
+)
+def import_odata(
+    service_root_url: Annotated[str, typer.Option(help="HTTP(S) root URL of the OData service.")],
+    service_document_file: Annotated[
+        Optional[Path],
+        typer.Option(help="Local JSON service document. Ignored when --entity-set is supplied."),
+    ] = None,
+    entity_set: Annotated[
+        Optional[List[str]],
+        typer.Option(
+            help="EntitySet to import (repeat for multiple sets). If omitted, will import all sets from the service document."
+        ),
+    ] = None,
+    metadata_url: Annotated[
+        Optional[str],
+        typer.Option(
+            help="CSDL XML or JSON URL. Defaults to SERVICE_ROOT_URL/$metadata.",
+        ),
+    ] = None,
+    metadata_file: Annotated[
+        Optional[Path],
+        typer.Option(
+            help="Path to a local OData CSDL XML or JSON file. Use either --metadata-file or --metadata-url.",
+        ),
+    ] = None,
+    output: output_option = None,
+    owner: owner_option = None,
+    id: id_option = None,
+    debug: debug_option = None,
+):
+    """Import a data contract from OData 4.x CSDL XML or JSON metadata, using a URL or local file."""
+    enable_debug_logging(debug)
+    result = DataContract.import_from_source(
+        config=cli_config(),
+        format="odata",
+        source=service_root_url,
+        odata_service_document_file=service_document_file,
+        odata_entity_set=entity_set,
+        odata_metadata_url=metadata_url,
+        odata_metadata_file=metadata_file,
+        owner=owner,
+        id=id,
+    )
+    _write_result(result, output)

{datacontract_cli-1.2.1 → datacontract_cli-1.2.2}/datacontract/command_init.py

@@ -28,6 +28,6 @@ def init(
         console.print("File already exists, use --overwrite to overwrite")
         raise typer.Exit(code=1)
     template_str = get_init_template(template)
-    with open(location, "w") as f:
+    with open(location, "w", encoding="utf-8") as f:
         f.write(template_str)
     console.print("📄 data contract written to " + location)

{datacontract_cli-1.2.1 → datacontract_cli-1.2.2}/datacontract/config/settings.py

@@ -356,7 +356,7 @@ class Config(BaseSettings):
 
         path = Path(path)
         try:
-            data = yaml.safe_load(path.read_text())
+            data = yaml.safe_load(path.read_text(encoding="utf-8"))
         except yaml.YAMLError as e:
             raise ValueError(f"Config file {path} is not valid YAML: {e}")
         if data is None:

datacontract_cli-1.2.2/datacontract/config/variables.py

@@ -0,0 +1,256 @@
+"""Variable references in string values, per ODCS v3.2.0 (RFC 0050).
+
+Any string value in a data contract may contain ``${VAR_NAME}`` or, with an
+inline default, ``${VAR_NAME:-default}``. References are resolved from the
+process environment (which includes a loaded ``.env`` file) at the moment a
+value is *used*: when a connection is opened or a quality query is executed.
+The loaded contract itself is never mutated, so ``export`` and ``publish``
+write the references back verbatim.
+
+A reference to an unset or empty variable without a default is an error;
+an empty string is never substituted silently.
+An untrusted contract resolves only the allow-listed part of the environment.
+"""
+
+import fnmatch
+import os
+import re
+from typing import Iterable, Mapping, Optional
+
+from open_data_contract_standard.model import Server
+from pydantic import BaseModel
+
+_VARIABLE = re.compile(r"\$\{([A-Za-z_][A-Za-z0-9_]*)(?::-([^}]*))?\}")
+
+CONTRACT_VARIABLES_ENV = "DATACONTRACT_CLI_API_CONTRACT_VARIABLES"
+
+
+class VariableError(ValueError):
+    """A ``${VAR}`` reference could not be turned into a usable value."""
+
+
+class UnresolvedVariableError(VariableError):
+    """A ``${VAR}`` reference names a variable that is unset and has no default."""
+
+    def __init__(self, name: str, source: str = "", restricted: bool = False):
+        self.name = name
+        where = f" in {source}" if source else ""
+        # "Not listed" must read like "not set", or the message enumerates the host's environment.
+        if restricted:
+            reason = "is not available. Check if the API permits the resolution (--contract-variables needs to be set)."
+        else:
+            reason = f"is not set. Set it in the environment or a .env file, or use ${{{name}:-default}}."
+        super().__init__(f"Variable {name} referenced{where} {reason}")
+
+
+def allowed_environment(patterns: Iterable[str]) -> dict[str, str]:
+    """The environment entries whose names match one of the ``fnmatch`` ``patterns``."""
+    globs = [pattern.strip() for pattern in patterns if pattern and pattern.strip()]
+    return {name: value for name, value in os.environ.items() if any(fnmatch.fnmatchcase(name, glob) for glob in globs)}
+
+
+# The ODCS enums a reference must resolve into, by (model, field); a test keeps them in step with the schema.
+_ENUM_VALUES: dict[tuple[str, str], frozenset[str]] = {
+    ("Server", "type"): frozenset(
+        {
+            "api",
+            "athena",
+            "azure",
+            "bigquery",
+            "btrieve",
+            "clickhouse",
+            "databricks",
+            "denodo",
+            "dremio",
+            "duckdb",
+            "exasol",
+            "fastobjects",
+            "glue",
+            "hana",
+            "cloudsql",
+            "db2",
+            "hive",
+            "iceberg",
+            "impala",
+            "informix",
+            "ingres",
+            "kafka",
+            "kinesis",
+            "local",
+            "mysql",
+            "oracle",
+            "poet",
+            "postgresql",
+            "postgres",
+            "presto",
+            "pubsub",
+            "redshift",
+            "s3",
+            "sftp",
+            "snowflake",
+            "sqlserver",
+            "synapse",
+            "teradata",
+            "trino",
+            "vectorwise",
+            "versant",
+            "vertica",
+            "zen",
+            "custom",
+        }
+    ),
+    ("DataQuality", "metric"): frozenset(
+        {"nullValues", "missingValues", "invalidValues", "duplicateValues", "rowCount"}
+    ),
+    ("DataQuality", "type"): frozenset({"text", "library", "sql", "custom"}),
+    ("DataQuality", "dimension"): frozenset(
+        {"accuracy", "completeness", "conformity", "consistency", "coverage", "timeliness", "uniqueness"}
+    ),
+    ("SchemaObject", "logicalType"): frozenset({"object", "blob"}),
+    ("SchemaProperty", "logicalType"): frozenset(
+        {
+            "string",
+            "date",
+            "timestamp",
+            "time",
+            "number",
+            "integer",
+            "boolean",
+            "object",
+            "array",
+            "map",
+            "vector",
+        }
+    ),
+}
+
+
+class InvalidVariableValueError(VariableError):
+    """A reference resolved outside the field's enum. Never quotes the value: it may be a secret."""
+
+    def __init__(self, reference: str, source: str, allowed: frozenset[str]):
+        super().__init__(
+            f"{source} resolved {reference} into an invalid value. Expected one of: {', '.join(sorted(allowed))}."
+        )
+
+
+def _checked(original, resolved, owner: str, field: str, source: str):
+    """Reject a resolved value the field's ODCS enum does not accept."""
+    allowed = _ENUM_VALUES.get((owner, field))
+    if allowed is None or not contains_variables(original) or resolved in allowed:
+        return resolved
+    raise InvalidVariableValueError(original, source, allowed)
+
+
+def contains_variables(value) -> bool:
+    """True if ``value`` is a string holding at least one ``${VAR}`` reference."""
+    return isinstance(value, str) and _VARIABLE.search(value) is not None
+
+
+def resolve_variables(value, source: str = "", variables: Optional[Mapping[str, str]] = None):
+    """Return ``value`` with every ``${VAR}`` reference replaced.
+
+    Non-string values pass through unchanged. ``source`` names the value in
+    the error message (``"server 'prod' host"``, ``"config file x.yaml"``).
+    ``variables`` replaces the process environment for an untrusted contract.
+    """
+    if not isinstance(value, str):
+        return value
+    environment = os.environ if variables is None else variables
+
+    def replace(match: re.Match) -> str:
+        name, default = match.group(1), match.group(2)
+        env_value = environment.get(name)
+        if env_value:
+            return env_value
+        if default is not None:
+            return default
+        raise UnresolvedVariableError(name, source, restricted=variables is not None)
+
+    return _VARIABLE.sub(replace, value)
+
+
+def resolve_server_variables(server: Server, variables: Optional[Mapping[str, str]] = None) -> Server:
+    """Return a copy of ``server`` with the references in its string fields resolved.
+
+    ``port`` may hold a reference (the schema allows a string there for that
+    reason); a resolved all-digit port becomes an int again. Custom property
+    values are resolved too. The contract's server is not mutated.
+    """
+    updates = {}
+    for field in Server.model_fields:
+        value = getattr(server, field)
+        if contains_variables(value):
+            source = f"server '{server.server}' {_field_name(field)}"
+            updates[field] = _checked(
+                value, resolve_variables(value, source=source, variables=variables), "Server", field, source
+            )
+    port = updates.get("port", server.port)
+    if isinstance(port, str) and port.isdigit():
+        updates["port"] = int(port)
+    if server.customProperties:
+        custom_properties = []
+        changed = False
+        for prop in server.customProperties:
+            if contains_variables(prop.value):
+                changed = True
+                prop = prop.model_copy(
+                    update={
+                        "value": resolve_variables(
+                            prop.value,
+                            source=f"server '{server.server}' custom property '{prop.property}'",
+                            variables=variables,
+                        )
+                    }
+                )
+            custom_properties.append(prop)
+        if changed:
+            updates["customProperties"] = custom_properties
+    return server.model_copy(update=updates) if updates else server
+
+
+def _field_name(field: str) -> str:
+    return "schema" if field == "schema_" else field
+
+
+# These fields document the contract rather than supply test inputs. SQL queries
+# resolve separately, after the engine substitutes its ${model}/${field} tokens;
+# servers resolve separately, after configuration overrides and server selection.
+_DEFERRED_FIELDS = {
+    "authoritativeDefinitions",
+    "businessName",
+    "context",
+    "description",
+    "examples",
+    "implementation",
+    "query",
+    "servers",
+    "synonyms",
+    "transformLogic",
+    "transformSourceObjects",
+}
+
+
+def resolve_runtime_variables(value, source: str = "contract", variables: Optional[Mapping[str, str]] = None):
+    """Copy test inputs with variables resolved, recursively, without altering the contract.
+
+    Handles property names/types, enum values, logical type options, library
+    quality arguments, SLA values, and nested array/map definitions. Dictionary
+    keys are structural identifiers and are never interpolated.
+    """
+    if isinstance(value, BaseModel):
+        owner = type(value).__name__
+        updates = {}
+        for field in type(value).model_fields:
+            if field in _DEFERRED_FIELDS:
+                continue
+            original = getattr(value, field)
+            field_source = f"{source}.{_field_name(field)}"
+            resolved = resolve_runtime_variables(original, field_source, variables)
+            updates[field] = _checked(original, resolved, owner, field, field_source)
+        return value.model_copy(update=updates)
+    if isinstance(value, dict):
+        return {key: resolve_runtime_variables(item, f"{source}.{key}", variables) for key, item in value.items()}
+    if isinstance(value, list):
+        return [resolve_runtime_variables(item, f"{source}[{index}]", variables) for index, item in enumerate(value)]
+    return resolve_variables(value, source=source, variables=variables)

{datacontract_cli-1.2.1 → datacontract_cli-1.2.2}/datacontract/data_contract.py

@@ -11,6 +11,7 @@ if typing.TYPE_CHECKING:
 
 from datacontract.breaking.detector import BreakingChangeDetector
 from datacontract.config import Config
+from datacontract.engines.checks.create_checks import to_schema_name, unexecuted_check_name, unrunnable_reason
 from datacontract.engines.checks.dimensions import default_dimension
 from datacontract.engines.data_contract_test import execute_data_contract_test
 from datacontract.export.exporter import ExportFormat
@@ -31,6 +32,53 @@ from datacontract.model.run import Check, ResultEnum, Run
 logger = logging.getLogger(__name__)
 
 
+def _unrunnable_rule_checks(data_contract: OpenDataContractStandard) -> list[Check]:
+    """Warnings for quality rules `datacontract test` would not be able to run."""
+    checks = []
+
+    def visit(properties, schema_name, prefix=None):
+        for prop in properties or []:
+            name = prop.physicalName or prop.name
+            field = f"{prefix}.{name}" if prefix else name
+            for quality in prop.quality or []:
+                reason = unrunnable_reason(quality, field)
+                if reason is not None:
+                    checks.append(
+                        Check(
+                            type="lint",
+                            result=ResultEnum.warning,
+                            name=unexecuted_check_name(schema_name, field),
+                            reason=reason,
+                            model=schema_name,
+                            field=field,
+                            engine="datacontract-cli",
+                        )
+                    )
+            visit(prop.properties, schema_name, field)
+            # `[]` marks the array hop, as the check keys do.
+            if prop.items is not None:
+                visit(prop.items.properties, schema_name, f"{field}[]")
+
+    for schema_obj in data_contract.schema_ or []:
+        # Named like the checks `test` reports for the same rules.
+        schema_name = to_schema_name(schema_obj, None)
+        for quality in schema_obj.quality or []:
+            reason = unrunnable_reason(quality, None)
+            if reason is not None:
+                checks.append(
+                    Check(
+                        type="lint",
+                        result=ResultEnum.warning,
+                        name=unexecuted_check_name(schema_name, None),
+                        reason=reason,
+                        model=schema_name,
+                        engine="datacontract-cli",
+                    )
+                )
+        visit(schema_obj.properties, schema_name)
+    return checks
+
+
 class DataContract:
     def __init__(
         self,
@@ -58,6 +106,7 @@ class DataContract:
         metadata_only: bool = False,
         dry_run: bool = False,
         untrusted_contract: bool = False,
+        allowed_variables: list[str] | None = None,
         config: "Config | dict[str, str] | None" = None,
     ):
         self._data_contract_file = data_contract_file
@@ -87,6 +136,8 @@ class DataContract:
         # server): the SQL it carries must not reach the host running it, and its
         # authoritativeDefinitions are resolved against the configured host only.
         self._untrusted_contract = untrusted_contract
+        # fnmatch globs of the environment variables an untrusted contract may read.
+        self._allowed_variables = allowed_variables
         self._config = Config.resolve(config)
 
     @classmethod
@@ -120,6 +171,7 @@ class DataContract:
                     engine="datacontract-cli",
                 )
             )
+            run.checks.extend(_unrunnable_rule_checks(data_contract))
             run.dataContractId = data_contract.id
             run.dataContractVersion = data_contract.version
         except DataContractValidationErrors as e:
@@ -197,6 +249,7 @@ class DataContract:
                 dry_run=self._dry_run,
                 config=self._config,
                 untrusted_contract=self._untrusted_contract,
+                allowed_variables=self._allowed_variables,
             )
 
         except DataContractException as e:

datacontract_cli-1.2.2/datacontract/editor_assets/VERSION

@@ -0,0 +1 @@
+0.1.14