dagster-datacontract 0.2.4__py3-none-any.whl → 0.3.0__py3-none-any.whl

dagster_datacontract/__init__.py

@@ -1,6 +1,3 @@
-import json
-import re
-import textwrap
 from datetime import timedelta
 from typing import Any
 
@@ -10,6 +7,14 @@ from datacontract.data_contract import DataContract
 from datacontract.model.run import ResultEnum
 from loguru import logger
 
+from dagster_datacontract.description import get_description
+from dagster_datacontract.metadata import (
+    get_column_lineage,
+    get_server_information,
+    get_table_column,
+)
+from dagster_datacontract.tags import get_tags
+
 
 class DataContractLoader:
     def __init__(
@@ -24,8 +29,11 @@ class DataContractLoader:
             self.data_contract.get_data_contract_specification()
         )
         self.metadata = self._load_metadata()
-        self.tags = self._load_tags(self.data_contract_specification.tags)
-        self.description = self.load_description()
+        self.tags = get_tags(self.data_contract_specification.tags)
+        self.description = get_description(
+            self.asset_name,
+            self.data_contract_specification,
+        )
         self.owner = self._load_owner()
         self.version = self._load_version()
         self.cron_schedule = self._load_cron_schedule()
@@ -33,76 +41,32 @@ class DataContractLoader:
     def _load_metadata(
         self,
     ) -> dict[str, TableColumnLineage | TableSchema | Any] | None:
-        fields = self.data_contract_specification.models.get(self.asset_name).fields
-
         columns = []
         deps_by_column = {}
 
+        fields = self.data_contract_specification.models.get(self.asset_name).fields
+
         for column_name, column_field in fields.items():
-            nullable = column_field.required if column_field.required else True
-            unique = column_field.unique if column_field.unique else False
-
-            columns.append(
-                dg.TableColumn(
-                    name=column_name,
-                    type=column_field.type,
-                    description=column_field.description,
-                    constraints=dg.TableColumnConstraints(
-                        nullable=nullable,
-                        unique=unique,
-                    ),
-                    tags=self._load_tags(column_field.tags),
-                )
-            )
+            table_column = get_table_column(column_name, column_field)
+            columns.append(table_column)
 
-            lineage = json.loads(column_field.model_dump_json()).get("lineage")
-            if not lineage:
-                deps_by_column[column_name] = []
-            else:
-                lineage_entries = lineage.get("inputFields")
+            table_column_lineage = get_column_lineage(column_field)
+            deps_by_column[column_name] = table_column_lineage
 
-                deps_by_column[column_name] = [
-                    dg.TableColumnDep(
-                        asset_key=dg.AssetKey(lineage_entry["name"]),
-                        column_name=lineage_entry["field"],
-                    )
-                    for lineage_entry in lineage_entries
-                ]
+        server_information = get_server_information(
+            self.data_contract_specification,
+            self.data_contract._server,
+            self.asset_name,
+        )
 
         return {
             "dagster/column_schema": dg.TableSchema(columns=columns),
             "dagster/column_lineage": dg.TableColumnLineage(
                 deps_by_column=deps_by_column
             ),
+            **server_information,
         }
 
-    @staticmethod
-    def _load_tags(
-        tags_list: list[str] | None,
-    ) -> dict[str, str]:
-        """Safely load tags from data contract.
-
-        More information about Dagster tags:
-        https://docs.dagster.io/guides/build/assets/metadata-and-tags/tags
-        """
-        key_pattern = re.compile(r"^[\w.-]{1,63}$")
-        val_pattern = re.compile(r"^[\w.-]{0,63}$")
-
-        tags = {}
-
-        for item in tags_list:
-            if ":" in item:
-                key, val = map(str.strip, item.split(":", 1))
-            else:
-                key, val = item.strip(), ""
-
-            if key_pattern.match(key) and val_pattern.match(val):
-                tags[key] = val
-            else:
-                logger.warning(f"Ignoring invalid tag: {item}")
-
-        return tags
-
     def _load_owner(self) -> list[str] | None:
         owner = self.data_contract_specification.info.owner
 
@@ -120,55 +84,9 @@ class DataContractLoader:
             )
             return cron_schedule
         except AttributeError:
+            logger.warning("'servicelevels.frequency.cron' not found in Data Contract.")
             return None
 
-    def load_description(
-        self, config: dict[str, Any] | None = None, separator: str = "\n"
-    ) -> str | None:
-        """Load and return a formatted description string based on the data contract specification.
-
-        This method composes a description by pulling text from different parts
-        of the data contract specification (e.g., model and info descriptions),
-        joining them using the specified separator.
-
-        Args:
-            config (dict[str, Any] | None, optional): A configuration dictionary
-                specifying the order in which to concatenate the description parts.
-                Defaults to `{"order": ["model", "info"]}`.
-            separator (str, optional): A string used to separate different parts
-                of the description. Defaults to a newline character (`"\n"`).
-
-        Returns:
-            str | None: A single string combining the specified description parts
-            if available, otherwise `None`.
-
-
-        Example:
-            >>> self.load_description()
-            'Model description...\nInfo description...'
-        """
-        default_config = {"order": ["model", "info"]}
-
-        configuration = default_config | (config or {})
-
-        descriptions = {
-            "model": self.data_contract_specification.models.get(
-                self.asset_name
-            ).description,
-            "info": self.data_contract_specification.info.description,
-        }
-
-        parts = []
-        for key in configuration["order"]:
-            desc = descriptions.get(key).replace("\n", f"{separator}\n")
-            if desc:
-                parts.append(textwrap.dedent(desc))
-
-        if parts:
-            return f"{separator}\n".join(parts)
-
-        return None
-
     def load_data_quality_checks(self) -> dg.AssetChecksDefinition:
         """Define and return a data quality check for the specified asset.
 
@@ -190,7 +108,11 @@ class DataContractLoader:
             blocking=True,
         )
         def check_asset():
-            run = self.data_contract.test()
+            data_contract = DataContract(
+                data_contract=self.data_contract_specification,
+                server=self.server_name,
+            )
+            run = data_contract.test()
 
             return dg.AssetCheckResult(
                 passed=run.result == ResultEnum.passed,

dagster_datacontract/description/__init__.py

@@ -0,0 +1,3 @@
+from dagster_datacontract.description.description import get_description
+
+__all__ = ["get_description"]

dagster_datacontract/description/description.py

@@ -0,0 +1,48 @@
+import textwrap
+from typing import Any
+
+from datacontract.data_contract import DataContractSpecification
+
+
+def get_description(
+    asset_name: str,
+    data_contract_specification: DataContractSpecification,
+    config: dict[str, Any] | None = None,
+    separator: str = "\n",
+) -> str | None:
+    """Load and return a formatted description string based on the data contract specification.
+
+    This method composes a description by pulling text from different parts
+    of the data contract specification (e.g., model and info descriptions),
+    joining them using the specified separator.
+
+    Args:
+        config (dict[str, Any] | None, optional): A configuration dictionary
+            specifying the order in which to concatenate the description parts.
+            Defaults to `{"order": ["model", "info"]}`.
+        separator (str, optional): A string used to separate different parts
+            of the description. Defaults to a newline character (`"\n"`).
+
+    Returns:
+        str | None: A single string combining the specified description parts
+        if available, otherwise `None`.
+    """
+    default_config = {"order": ["model", "info"]}
+
+    configuration = default_config | (config or {})
+
+    descriptions = {
+        "model": data_contract_specification.models.get(asset_name).description,
+        "info": data_contract_specification.info.description,
+    }
+
+    parts = []
+    for key in configuration["order"]:
+        desc = descriptions.get(key).replace("\n", f"{separator}\n")
+        if desc:
+            parts.append(textwrap.dedent(desc))
+
+    if parts:
+        return f"{separator}\n".join(parts)
+
+    return None

dagster_datacontract/metadata/__init__.py

@@ -0,0 +1,7 @@
+from dagster_datacontract.metadata.server_information import get_server_information
+from dagster_datacontract.metadata.table_colums import (
+    get_column_lineage,
+    get_table_column,
+)
+
+__all__ = ["get_table_column", "get_column_lineage", "get_server_information"]

dagster_datacontract/metadata/server_information.py

@@ -0,0 +1,75 @@
+import os
+import urllib.parse
+
+from datacontract.data_contract import DataContractSpecification
+
+
+def _normalize_path(path: str) -> str:
+    parsed = urllib.parse.urlparse(path)
+
+    if not parsed.scheme or parsed.scheme == "file":
+        full_path = os.path.abspath(os.path.expanduser(path))
+        return f"file://{full_path}"
+    else:
+        return path
+
+
+def get_server_information(
+    data_contract_specification: DataContractSpecification,
+    server_name: str | None,
+    asset_name: str,
+) -> dict[str, str]:
+    server = data_contract_specification.servers.get(server_name)
+    if not server:
+        return {}
+
+    server_information = {}
+    match server.type:
+        case "azure":
+            server_information["dagster/uri"] = server.location
+        case "databricks":
+            server_information["dagster/uri"] = server.host
+            server_information["dagster/table_name"] = (
+                f"{server.catalog}.{server.schema}.{asset_name}"
+            )
+        case "kafka":
+            server_information["dagster/uri"] = server.host
+        case "kinesis":
+            server_information = {}
+        case "local":
+            server_information["dagster/uri"] = _normalize_path(server.path)
+        case "oracle":
+            server_information["dagster/uri"] = f"{server.host}:{server.port}"
+        case "postgres":
+            server_information["dagster/uri"] = f"{server.host}:{server.port}"
+            server_information["dagster/table_name"] = (
+                f"{server.database}.{server.schema}.{asset_name}"
+            )
+        case "pubsub":
+            server_information = {}
+        case "redshift":
+            server_information["dagster/uri"] = server.endpoint
+            server_information["dagster/table_name"] = (
+                f"{server.database}.{server.schema}.{asset_name}"
+            )
+        case "s3":
+            server_information["dagster/uri"] = server.location
+        case "sftp":
+            server_information["dagster/uri"] = server.location
+        case "snowflake":
+            server_information["dagster/table_name"] = (
+                f"{server.database}.{server.schema}.{asset_name}"
+            )
+        case "sqlserver":
+            server_information["dagster/table_name"] = (
+                f"{server.database}.{server.schema}.{asset_name}"
+            )
+        case "trino":
+            server_information["dagster/uri"] = f"{server.host}:{server.port}"
+            server_information["dagster/table_name"] = (
+                f"{server.catalog}.{server.schema}.{asset_name}"
+            )
+        case _:
+            server_information = {}
+
+    return server_information

dagster_datacontract/metadata/table_colums.py

@@ -0,0 +1,109 @@
+import json
+from typing import Any
+
+import dagster as dg
+from dagster import TableColumnDep
+from datacontract.model.data_contract_specification import Field
+
+from dagster_datacontract.tags import get_tags
+
+
+def get_other_item(name: str, column_field: Field) -> list[str] | None:
+    """Retrieve a list containing a single formatted string representing an attribute of a Field, if it exists.
+
+    Args:
+    name (str): The name of the attribute to fetch from the Field.
+    column_field (Field): The Field instance from which to extract the attribute.
+
+    Returns:
+        list[str] | None: A list with a formatted string (e.g., "format=csv")
+            if the attribute exists and is truthy, otherwise an empty list.
+    """
+    value = getattr(column_field, name, None)
+    return [f"{name}={value}"] if value else []
+
+
+def get_table_column_constraints(column_field: Field) -> dg.TableColumnConstraints:
+    """Convert a Field object to Dagster TableColumnConstraints, including nullability, uniqueness, and other properties.
+
+    Args:
+        column_field (Field): A data contract field specification containing
+            column metadata.
+
+    Returns:
+        dg.TableColumnConstraints: A Dagster representation of the field's
+            column constraints.
+    """
+    nullable = column_field.required if column_field.required else True
+    unique = column_field.unique if column_field.unique else False
+    other = [
+        *(get_other_item("title", column_field)),
+        *(get_other_item("primaryKey", column_field)),
+        *(get_other_item("format", column_field)),
+        *(get_other_item("minLength", column_field)),
+        *(get_other_item("maxLength", column_field)),
+        *(get_other_item("pattern", column_field)),
+        *(get_other_item("minimum", column_field)),
+        *(get_other_item("exclusiveMinimum", column_field)),
+        *(get_other_item("maximum", column_field)),
+        *(get_other_item("exclusiveMaximum", column_field)),
+        *(get_other_item("pii", column_field)),
+        *(get_other_item("classification", column_field)),
+    ]
+
+    return dg.TableColumnConstraints(
+        nullable=nullable,
+        unique=unique,
+        other=other,
+    )
+
+
+def get_table_column(column_name: str, column_field: Field) -> dg.TableColumn:
+    """Create a Dagster TableColumn from a given column name and Field metadata.
+
+    Args:
+        column_name (str): The name of the column.
+        column_field (Field): The Field instance containing metadata such as
+            type, description, constraints, and tags.
+
+    Returns:
+        dg.TableColumn: A Dagster TableColumn object representing the column
+            definition.
+    """
+    return dg.TableColumn(
+        name=column_name,
+        type=column_field.type,
+        description=column_field.description,
+        constraints=get_table_column_constraints(column_field),
+        tags=get_tags(column_field.tags),
+    )
+
+
+def get_column_lineage(column_field: Field) -> list[Any] | list[TableColumnDep | Any]:
+    """Extract column-level lineage information from a Field and return it as a list of TableColumnDep objects.
+
+    The function parses the JSON-serialized Field to retrieve any input lineage
+    defined under the "lineage.inputFields" key. Each lineage entry is converted
+    into a Dagster TableColumnDep representing a dependency on a specific column
+    of another asset.
+
+    Args:
+        column_field (Field): The Field instance that may contain lineage metadata.
+
+    Returns:
+        list[Any] | list[TableColumnDep | Any]: A list of TableColumnDep objects
+            if lineage is defined; otherwise, an empty list.
+    """
+    lineage = json.loads(column_field.model_dump_json()).get("lineage")
+
+    if not lineage:
+        return []
+
+    lineage_entries = lineage.get("inputFields")
+    return [
+        dg.TableColumnDep(
+            asset_key=dg.AssetKey(lineage_entry["name"]),
+            column_name=lineage_entry["field"],
+        )
+        for lineage_entry in lineage_entries
+    ]

dagster_datacontract/tags/__init__.py

@@ -0,0 +1,3 @@
+from dagster_datacontract.tags.tags import get_tags
+
+__all__ = ["get_tags"]

dagster_datacontract/tags/tags.py

@@ -0,0 +1,45 @@
+import re
+
+from loguru import logger
+
+
+def get_tags(
+    tags_list: list[str] | None,
+) -> dict[str, str]:
+    """Parse and validate a list of string tags into a dictionary format.
+
+    Each tag in the input list should be in the form "key:value" or simply "key".
+    - Keys must match the pattern: ^[\w.-]{1,63}$
+    - Values (if provided) must match the pattern: ^[\w.-]{0,63}$
+
+    Invalid tags (those that do not match the expected format) will be ignored,
+    and a warning will be logged.
+
+    More information about Dagster tags:
+    https://docs.dagster.io/guides/build/assets/metadata-and-tags/tags
+
+    Args:
+        tags_list (list[str] | None): A list of tags as strings. Each tag may be
+            formatted as "key:value" or just "key". If None, an empty dict is returned.
+
+    Returns:
+        dict[str, str]: A dictionary of validated tags, where keys are tag names
+        and values are tag values (empty string if not provided).
+    """
+    key_pattern = re.compile(r"^[\w.-]{1,63}$")
+    val_pattern = re.compile(r"^[\w.-]{0,63}$")
+
+    tags = {}
+
+    for item in tags_list:
+        if ":" in item:
+            key, val = map(str.strip, item.split(":", 1))
+        else:
+            key, val = item.strip(), ""
+
+        if key_pattern.match(key) and val_pattern.match(val):
+            tags[key] = val
+        else:
+            logger.warning(f"Ignoring invalid tag: {item}")
+
+    return tags

{dagster_datacontract-0.2.4.dist-info → dagster_datacontract-0.3.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dagster-datacontract
-Version: 0.2.4
+Version: 0.3.0
 Summary: Load metadata and asset check spesifications from data contracts.
 Author-email: Fredrik Bakken <fredrik@dataheim.io>
 Requires-Python: >=3.10.0

dagster_datacontract-0.3.0.dist-info/RECORD

@@ -0,0 +1,13 @@
+dagster_datacontract/__init__.py,sha256=UVpDzZYjmskEDJqEewMD52uWvu183O5RV9hjnP8mYAc,5488
+dagster_datacontract/description/__init__.py,sha256=ulWqPp5jIPvCzaDFZcjLjcDkljJ5j_FRsE0dXhK8Wlc,104
+dagster_datacontract/description/description.py,sha256=FmjgCYDpJ9UHrvAv0sAthfRohDjdG0lL1XcMKK8QMmI,1646
+dagster_datacontract/metadata/__init__.py,sha256=e-xmcWWoAhmKTwosshsxnyrjI1j-UyY6YpdpzA2ggF4,269
+dagster_datacontract/metadata/server_information.py,sha256=m1pv9sMfVjGIjzYVJ9R-KF-ABXCgAGKKH12dgAbm_jQ,2669
+dagster_datacontract/metadata/table_colums.py,sha256=Q7ZCiMReWU4-T2YfBvtt5vvoVXEoUgzK5OPMxQEgzpQ,4013
+dagster_datacontract/tags/__init__.py,sha256=2Ph-M0WbBKUjJWIzM_cEBW3SQZh7Nq8oy5MbD5bt_lc,76
+dagster_datacontract/tags/tags.py,sha256=aZ_HTkc-vjJ_rofT32fT_zrLCt9x1ZGn8XoihhOMhfU,1414
+dagster_datacontract-0.3.0.dist-info/licenses/LICENSE,sha256=9ULsEM1ICzCaGoso40plwO-d_SCQ7nsU6ZA4xgfaRq8,11338
+dagster_datacontract-0.3.0.dist-info/METADATA,sha256=1uuc5HEqV3OYmP7jNe9TZftEcirYwKbRXbnoAOe3pWQ,3029
+dagster_datacontract-0.3.0.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
+dagster_datacontract-0.3.0.dist-info/top_level.txt,sha256=_HUQ6OJ50Q0VZxEkdocTtxk1QkJpztb1QY7A0rcvtCE,21
+dagster_datacontract-0.3.0.dist-info/RECORD,,

dagster_datacontract-0.2.4.dist-info/RECORD

@@ -1,6 +0,0 @@
-dagster_datacontract/__init__.py,sha256=uL2Dgw7FHykepsmGQjU51kuQ7lKlVBLUxx4_nGn_3VU,8225
-dagster_datacontract-0.2.4.dist-info/licenses/LICENSE,sha256=9ULsEM1ICzCaGoso40plwO-d_SCQ7nsU6ZA4xgfaRq8,11338
-dagster_datacontract-0.2.4.dist-info/METADATA,sha256=53-UmbMwaHB2LiAnIP0tQrYKR2rU9Jbcee9fYQAWG7M,3029
-dagster_datacontract-0.2.4.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
-dagster_datacontract-0.2.4.dist-info/top_level.txt,sha256=_HUQ6OJ50Q0VZxEkdocTtxk1QkJpztb1QY7A0rcvtCE,21
-dagster_datacontract-0.2.4.dist-info/RECORD,,