datacontract-cli 0.10.21__py3-none-any.whl → 0.10.22__py3-none-any.whl

datacontract/export/odcs_v2_exporter.py

@@ -1,124 +0,0 @@
-from typing import Dict
-
-import yaml
-
-from datacontract.export.exporter import Exporter
-from datacontract.model.data_contract_specification import DataContractSpecification, Field, Model
-
-
-class OdcsV2Exporter(Exporter):
-    def export(self, data_contract, model, server, sql_server_type, export_args) -> dict:
-        return to_odcs_v2_yaml(data_contract)
-
-
-def to_odcs_v2_yaml(data_contract_spec: DataContractSpecification):
-    odcs = {
-        "kind": "DataContract",
-        "apiVersion": "2.3.0",
-        "uuid": data_contract_spec.id,
-        "version": data_contract_spec.info.version,
-        "datasetDomain": data_contract_spec.info.owner,
-        "quantumName": data_contract_spec.info.title,
-        "status": "unknown",
-    }
-
-    if data_contract_spec.info.contact is not None:
-        if data_contract_spec.info.contact.email is not None:
-            odcs["productDl"] = data_contract_spec.info.contact.email
-        if data_contract_spec.info.contact.url is not None:
-            odcs["productFeedbackUrl"] = data_contract_spec.info.contact.url
-
-    if data_contract_spec.terms is not None:
-        odcs["description"] = {
-            "purpose": data_contract_spec.terms.description.strip()
-            if data_contract_spec.terms.description is not None
-            else None,
-            "usage": data_contract_spec.terms.usage.strip() if data_contract_spec.terms.usage is not None else None,
-            "limitations": data_contract_spec.terms.limitations.strip()
-            if data_contract_spec.terms.limitations is not None
-            else None,
-        }
-
-    if data_contract_spec.servicelevels is not None:
-        slas = []
-        if data_contract_spec.servicelevels.availability is not None:
-            slas.append(
-                {
-                    "property": "generalAvailability",
-                    "value": data_contract_spec.servicelevels.availability.description,
-                }
-            )
-        if data_contract_spec.servicelevels.retention is not None:
-            slas.append({"property": "retention", "value": data_contract_spec.servicelevels.retention.period})
-
-        if len(slas) > 0:
-            odcs["slaProperties"] = slas
-
-    odcs["type"] = "tables"  # required, TODO read from models.type?
-    odcs["dataset"] = []
-
-    for model_key, model_value in data_contract_spec.models.items():
-        odcs_table = to_odcs_table(model_key, model_value)
-        odcs["dataset"].append(odcs_table)
-    return yaml.dump(odcs, indent=2, sort_keys=False, allow_unicode=True)
-
-
-def to_odcs_table(model_key, model_value: Model) -> dict:
-    odcs_table = {
-        "table": model_key,
-        "physicalName": model_key,
-        "columns": [],
-    }
-    if model_value.description is not None:
-        odcs_table["description"] = model_value.description
-    columns = to_columns(model_value.fields)
-    if columns:
-        odcs_table["columns"] = columns
-    return odcs_table
-
-
-def to_columns(fields: Dict[str, Field]) -> list:
-    columns = []
-    for field_name, field in fields.items():
-        column = to_column(field_name, field)
-        columns.append(column)
-    return columns
-
-
-def to_column(field_name: str, field: Field) -> dict:
-    column = {"column": field_name}
-    if field.type is not None:
-        column["logicalType"] = field.type
-        column["physicalType"] = field.type
-    if field.description is not None:
-        column["description"] = field.description
-    if field.required is not None:
-        column["isNullable"] = not field.required
-    if field.unique is not None:
-        column["isUnique"] = field.unique
-    if field.classification is not None:
-        column["classification"] = field.classification
-    column["tags"] = []
-    if field.tags is not None:
-        column["tags"].extend(field.tags)
-    if field.pii is not None:
-        column["tags"].append(f"pii:{str(field.pii).lower()}")
-    if field.minLength is not None:
-        column["tags"].append(f"minLength:{field.minLength}")
-    if field.maxLength is not None:
-        column["tags"].append(f"maxLength:{field.maxLength}")
-    if field.pattern is not None:
-        column["tags"].append(f"pattern:{field.pattern}")
-    if field.minimum is not None:
-        column["tags"].append(f"minimum:{field.minimum}")
-    if field.maximum is not None:
-        column["tags"].append(f"maximum:{field.maximum}")
-    if field.exclusiveMinimum is not None:
-        column["tags"].append(f"exclusiveMinimum:{field.exclusiveMinimum}")
-    if field.exclusiveMaximum is not None:
-        column["tags"].append(f"exclusiveMaximum:{field.exclusiveMaximum}")
-    if not column["tags"]:
-        del column["tags"]
-
-    # todo enum
-    return column

datacontract/imports/odcs_v2_importer.py

@@ -1,177 +0,0 @@
-import datetime
-import logging
-from typing import Any, Dict, List
-
-import yaml
-
-from datacontract.imports.importer import Importer
-from datacontract.model.data_contract_specification import (
-    DATACONTRACT_TYPES,
-    Availability,
-    Contact,
-    DataContractSpecification,
-    Field,
-    Info,
-    Model,
-    Retention,
-    ServiceLevel,
-    Terms,
-)
-from datacontract.model.exceptions import DataContractException
-
-
-class OdcsImporter(Importer):
-    def import_source(
-        self, data_contract_specification: DataContractSpecification, source: str, import_args: dict
-    ) -> DataContractSpecification:
-        return import_odcs_v2(data_contract_specification, source)
-
-
-def import_odcs_v2(data_contract_specification: DataContractSpecification, source: str) -> DataContractSpecification:
-    try:
-        with open(source, "r") as file:
-            odcs_contract = yaml.safe_load(file.read())
-
-    except Exception as e:
-        raise DataContractException(
-            type="schema",
-            name="Parse ODCS contract",
-            reason=f"Failed to parse odcs contract from {source}",
-            engine="datacontract",
-            original_exception=e,
-        )
-
-    data_contract_specification.id = odcs_contract["uuid"]
-    data_contract_specification.info = import_info(odcs_contract)
-    data_contract_specification.terms = import_terms(odcs_contract)
-    data_contract_specification.servicelevels = import_servicelevels(odcs_contract)
-    data_contract_specification.models = import_models(odcs_contract)
-
-    return data_contract_specification
-
-
-def import_info(odcs_contract: Dict[str, Any]) -> Info:
-    info = Info(title=odcs_contract.get("quantumName"), version=odcs_contract.get("version"))
-
-    if odcs_contract.get("description").get("purpose") is not None:
-        info.description = odcs_contract.get("description").get("purpose")
-
-    if odcs_contract.get("datasetDomain") is not None:
-        info.owner = odcs_contract.get("datasetDomain")
-
-    if odcs_contract.get("productDl") is not None or odcs_contract.get("productFeedbackUrl") is not None:
-        contact = Contact()
-        if odcs_contract.get("productDl") is not None:
-            contact.name = odcs_contract.get("productDl")
-        if odcs_contract.get("productFeedbackUrl") is not None:
-            contact.url = odcs_contract.get("productFeedbackUrl")
-
-        info.contact = contact
-
-    return info
-
-
-def import_terms(odcs_contract: Dict[str, Any]) -> Terms | None:
-    if (
-        odcs_contract.get("description").get("usage") is not None
-        or odcs_contract.get("description").get("limitations") is not None
-        or odcs_contract.get("price") is not None
-    ):
-        terms = Terms()
-        if odcs_contract.get("description").get("usage") is not None:
-            terms.usage = odcs_contract.get("description").get("usage")
-        if odcs_contract.get("description").get("limitations") is not None:
-            terms.limitations = odcs_contract.get("description").get("limitations")
-        if odcs_contract.get("price") is not None:
-            terms.billing = f"{odcs_contract.get('price').get('priceAmount')} {odcs_contract.get('price').get('priceCurrency')} / {odcs_contract.get('price').get('priceUnit')}"
-
-        return terms
-    else:
-        return None
-
-
-def import_servicelevels(odcs_contract: Dict[str, Any]) -> ServiceLevel:
-    # find the two properties we can map (based on the examples)
-    sla_properties = odcs_contract.get("slaProperties") if odcs_contract.get("slaProperties") is not None else []
-    availability = next((p for p in sla_properties if p["property"] == "generalAvailability"), None)
-    retention = next((p for p in sla_properties if p["property"] == "retention"), None)
-
-    if availability is not None or retention is not None:
-        servicelevel = ServiceLevel()
-
-        if availability is not None:
-            value = availability.get("value")
-            if isinstance(value, datetime.datetime):
-                value = value.isoformat()
-            servicelevel.availability = Availability(description=value)
-
-        if retention is not None:
-            servicelevel.retention = Retention(period=f"{retention.get('value')}{retention.get('unit')}")
-
-        return servicelevel
-    else:
-        return None
-
-
-def import_models(odcs_contract: Dict[str, Any]) -> Dict[str, Model]:
-    custom_type_mappings = get_custom_type_mappings(odcs_contract.get("customProperties"))
-
-    odcs_tables = odcs_contract.get("dataset") if odcs_contract.get("dataset") is not None else []
-    result = {}
-
-    for table in odcs_tables:
-        description = table.get("description") if table.get("description") is not None else ""
-        model = Model(description=" ".join(description.splitlines()), type="table")
-        model.fields = import_fields(table.get("columns"), custom_type_mappings)
-        result[table.get("table")] = model
-
-    return result
-
-
-def import_fields(odcs_columns: Dict[str, Any], custom_type_mappings: Dict[str, str]) -> Dict[str, Field]:
-    logger = logging.getLogger(__name__)
-    result = {}
-
-    for column in odcs_columns:
-        mapped_type = map_type(column.get("logicalType"), custom_type_mappings)
-        if mapped_type is not None:
-            description = column.get("description") if column.get("description") is not None else ""
-            field = Field(
-                description=" ".join(description.splitlines()),
-                type=mapped_type,
-                title=column.get("businessName") if column.get("businessName") is not None else "",
-                required=not column.get("isNullable") if column.get("isNullable") is not None else False,
-                primaryKey=column.get("isPrimary") if column.get("isPrimary") is not None else False,
-                unique=column.get("isUnique") if column.get("isUnique") is not None else False,
-                classification=column.get("classification") if column.get("classification") is not None else "",
-                tags=column.get("tags") if column.get("tags") is not None else [],
-            )
-            result[column["column"]] = field
-        else:
-            logger.info(
-                f"Can't properly map {column.get('column')} to the Datacontract Mapping types, as there is no equivalent or special mapping. Consider introducing a customProperty 'dc_mapping_{column.get('logicalName')}' that defines your expected type as the 'value'"
-            )
-
-    return result
-
-
-def map_type(odcs_type: str, custom_mappings: Dict[str, str]) -> str | None:
-    t = odcs_type.lower()
-    if t in DATACONTRACT_TYPES:
-        return t
-    elif custom_mappings.get(t) is not None:
-        return custom_mappings.get(t)
-    else:
-        return None
-
-
-def get_custom_type_mappings(odcs_custom_properties: List[Any]) -> Dict[str, str]:
-    result = {}
-    if odcs_custom_properties is not None:
-        for prop in odcs_custom_properties:
-            if prop["property"].startswith("dc_mapping_"):
-                odcs_type_name = prop["property"].substring(11)
-                datacontract_type = prop["value"]
-                result[odcs_type_name] = datacontract_type
-
-    return result

datacontract/lint/linters/example_model_linter.py

@@ -1,91 +0,0 @@
-import csv
-import io
-import json
-
-import yaml
-
-from datacontract.model.data_contract_specification import DataContractSpecification, Example
-
-from ..lint import Linter, LinterResult
-
-
-class ExampleModelLinter(Linter):
-    @property
-    def name(self) -> str:
-        return "Example(s) match model"
-
-    @property
-    def id(self) -> str:
-        return "example-model"
-
-    @staticmethod
-    def get_example_headers(example: Example) -> list[str]:
-        if isinstance(example.data, str):
-            match example.type:
-                case "csv":
-                    dialect = csv.Sniffer().sniff(example.data)
-                    data = io.StringIO(example.data)
-                    reader = csv.reader(data, dialect=dialect)
-                    return next(reader)
-                case "yaml":
-                    data = yaml.safe_load(example.data)
-                    return data.keys()
-                case "json":
-                    data = json.loads(example.data)
-                    return data.keys()
-                case _:
-                    # This is checked in lint_implementation, so shouldn't happen.
-                    raise NotImplementedError(f"Unknown type {example.type}")
-        else:
-            # Checked in lint_implementation, shouldn't happen.
-            raise NotImplementedError("Can't lint object examples.")
-
-    def lint_implementation(self, contract: DataContractSpecification) -> LinterResult:
-        """Check whether the example(s) headers match the model.
-
-        This linter checks whether the example's fields match the model
-        fields, and whether all required fields of the model are present in
-        the example.
-        """
-        result = LinterResult()
-        examples = contract.examples
-        models = contract.models
-        examples_with_model = []
-        for index, example in enumerate(examples):
-            if example.model not in models:
-                result = result.with_error(f"Example {index + 1} has non-existent model '{example.model}'")
-            else:
-                examples_with_model.append((index, example, models.get(example.model)))
-        for index, example, model in examples_with_model:
-            if example.type == "custom":
-                result = result.with_warning(
-                    f"Example {index + 1} has type" ' "custom", cannot check model' " conformance"
-                )
-            elif not isinstance(example.data, str):
-                result = result.with_warning(
-                    f"Example {index + 1} is not a " "string example, can only lint string examples for now."
-                )
-            elif model.type == "object":
-                result = result.with_warning(
-                    f"Example {index + 1} uses a "
-                    f"model '{example.model}' with type 'object'. Linting is "
-                    "currently only supported for 'table' models"
-                )
-            else:
-                if example.type in ("csv", "yaml", "json"):
-                    headers = self.get_example_headers(example)
-                    for example_header in headers:
-                        if example_header not in model.fields:
-                            result = result.with_error(
-                                f"Example {index + 1} has field '{example_header}'"
-                                f" that's not contained in model '{example.model}'"
-                            )
-                    for field_name, field_value in model.fields.items():
-                        if field_name not in headers and field_value.required:
-                            result = result.with_error(
-                                f"Example {index + 1} is missing field '{field_name}'"
-                                f" required by model '{example.model}'"
-                            )
-                else:
-                    result = result.with_error(f"Example {index + 1} has unknown type" f"{example.type}")
-        return result