dagster-datacontract 0.2.3__py3-none-any.whl → 0.2.5__py3-none-any.whl

dagster_datacontract/__init__.py

@@ -1,5 +1,3 @@
-import json
-import re
 import textwrap
 from datetime import timedelta
 from typing import Any
@@ -10,6 +8,12 @@ from datacontract.data_contract import DataContract
 from datacontract.model.run import ResultEnum
 from loguru import logger
 
+from dagster_datacontract.metadata import (
+    get_column_lineage,
+    get_table_column,
+)
+from dagster_datacontract.tags import get_tags
+
 
 class DataContractLoader:
     def __init__(
@@ -24,7 +28,7 @@ class DataContractLoader:
             self.data_contract.get_data_contract_specification()
         )
         self.metadata = self._load_metadata()
-        self.tags = self._load_tags()
+        self.tags = get_tags(self.data_contract_specification.tags)
         self.description = self.load_description()
         self.owner = self._load_owner()
         self.version = self._load_version()
@@ -33,33 +37,17 @@ class DataContractLoader:
     def _load_metadata(
         self,
     ) -> dict[str, TableColumnLineage | TableSchema | Any] | None:
-        fields = self.data_contract_specification.models.get(self.asset_name).fields
-
         columns = []
         deps_by_column = {}
 
-        for column_name, column_field in fields.items():
-            columns.append(
-                dg.TableColumn(
-                    name=column_name,
-                    type=column_field.type,
-                    description=column_field.description,
-                )
-            )
+        fields = self.data_contract_specification.models.get(self.asset_name).fields
 
-            lineage = json.loads(column_field.model_dump_json()).get("lineage")
-            if not lineage:
-                deps_by_column[column_name] = []
-            else:
-                lineage_entries = lineage.get("inputFields")
+        for column_name, column_field in fields.items():
+            table_column = get_table_column(column_name, column_field)
+            columns.append(table_column)
 
-                deps_by_column[column_name] = [
-                    dg.TableColumnDep(
-                        asset_key=dg.AssetKey(lineage_entry["name"]),
-                        column_name=lineage_entry["field"],
-                    )
-                    for lineage_entry in lineage_entries
-                ]
+            table_column_lineage = get_column_lineage(column_field)
+            deps_by_column[column_name] = table_column_lineage
 
         return {
             "dagster/column_schema": dg.TableSchema(columns=columns),
@@ -68,30 +56,6 @@ class DataContractLoader:
             ),
         }
 
-    def _load_tags(self) -> dict[str, str]:
-        """Safely load tags from data contract.
-
-        More information about Dagster tags:
-        https://docs.dagster.io/guides/build/assets/metadata-and-tags/tags
-        """
-        key_pattern = re.compile(r"^[\w.-]{1,63}$")
-        val_pattern = re.compile(r"^[\w.-]{0,63}$")
-
-        tags = {}
-
-        for item in self.data_contract_specification.tags:
-            if ":" in item:
-                key, val = map(str.strip, item.split(":", 1))
-            else:
-                key, val = item.strip(), ""
-
-            if key_pattern.match(key) and val_pattern.match(val):
-                tags[key] = val
-            else:
-                logger.warning(f"Ignoring invalid tag: {item}")
-
-        return tags
-
     def _load_owner(self) -> list[str] | None:
         owner = self.data_contract_specification.info.owner
 
@@ -109,10 +73,13 @@ class DataContractLoader:
             )
             return cron_schedule
         except AttributeError:
+            logger.warning("'servicelevels.frequency.cron' not found in Data Contract.")
             return None
 
     def load_description(
-        self, config: dict[str, Any] | None = None, separator: str = "\n"
+        self,
+        config: dict[str, Any] | None = None,
+        separator: str = "\n",
     ) -> str | None:
         """Load and return a formatted description string based on the data contract specification.
 

dagster_datacontract/metadata/__init__.py

@@ -0,0 +1,6 @@
+from dagster_datacontract.metadata.table_colums import (
+    get_column_lineage,
+    get_table_column,
+)
+
+__all__ = ["get_table_column", "get_column_lineage"]

dagster_datacontract/metadata/table_colums.py

@@ -0,0 +1,109 @@
+import json
+from typing import Any
+
+import dagster as dg
+from dagster import TableColumnDep
+from datacontract.model.data_contract_specification import Field
+
+from dagster_datacontract.tags import get_tags
+
+
+def get_other_item(name: str, column_field: Field) -> list[str] | None:
+    """Retrieve a list containing a single formatted string representing an attribute of a Field, if it exists.
+
+    Args:
+    name (str): The name of the attribute to fetch from the Field.
+    column_field (Field): The Field instance from which to extract the attribute.
+
+    Returns:
+        list[str] | None: A list with a formatted string (e.g., "format=csv")
+            if the attribute exists and is truthy, otherwise an empty list.
+    """
+    value = getattr(column_field, name, None)
+    return [f"{name}={value}"] if value else []
+
+
+def get_table_column_constraints(column_field: Field) -> dg.TableColumnConstraints:
+    """Convert a Field object to Dagster TableColumnConstraints, including nullability, uniqueness, and other properties.
+
+    Args:
+        column_field (Field): A data contract field specification containing
+            column metadata.
+
+    Returns:
+        dg.TableColumnConstraints: A Dagster representation of the field's
+            column constraints.
+    """
+    nullable = column_field.required if column_field.required else True
+    unique = column_field.unique if column_field.unique else False
+    other = [
+        *(get_other_item("title", column_field)),
+        *(get_other_item("primaryKey", column_field)),
+        *(get_other_item("format", column_field)),
+        *(get_other_item("minLength", column_field)),
+        *(get_other_item("maxLength", column_field)),
+        *(get_other_item("pattern", column_field)),
+        *(get_other_item("minimum", column_field)),
+        *(get_other_item("exclusiveMinimum", column_field)),
+        *(get_other_item("maximum", column_field)),
+        *(get_other_item("exclusiveMaximum", column_field)),
+        *(get_other_item("pii", column_field)),
+        *(get_other_item("classification", column_field)),
+    ]
+
+    return dg.TableColumnConstraints(
+        nullable=nullable,
+        unique=unique,
+        other=other,
+    )
+
+
+def get_table_column(column_name: str, column_field: Field) -> dg.TableColumn:
+    """Create a Dagster TableColumn from a given column name and Field metadata.
+
+    Args:
+        column_name (str): The name of the column.
+        column_field (Field): The Field instance containing metadata such as
+            type, description, constraints, and tags.
+
+    Returns:
+        dg.TableColumn: A Dagster TableColumn object representing the column
+            definition.
+    """
+    return dg.TableColumn(
+        name=column_name,
+        type=column_field.type,
+        description=column_field.description,
+        constraints=get_table_column_constraints(column_field),
+        tags=get_tags(column_field.tags),
+    )
+
+
+def get_column_lineage(column_field: Field) -> list[Any] | list[TableColumnDep | Any]:
+    """Extract column-level lineage information from a Field and return it as a list of TableColumnDep objects.
+
+    The function parses the JSON-serialized Field to retrieve any input lineage
+    defined under the "lineage.inputFields" key. Each lineage entry is converted
+    into a Dagster TableColumnDep representing a dependency on a specific column
+    of another asset.
+
+    Args:
+        column_field (Field): The Field instance that may contain lineage metadata.
+
+    Returns:
+        list[Any] | list[TableColumnDep | Any]: A list of TableColumnDep objects
+            if lineage is defined; otherwise, an empty list.
+    """
+    lineage = json.loads(column_field.model_dump_json()).get("lineage")
+
+    if not lineage:
+        return []
+
+    lineage_entries = lineage.get("inputFields")
+    return [
+        dg.TableColumnDep(
+            asset_key=dg.AssetKey(lineage_entry["name"]),
+            column_name=lineage_entry["field"],
+        )
+        for lineage_entry in lineage_entries
+    ]

dagster_datacontract/tags/__init__.py

@@ -0,0 +1,3 @@
+from dagster_datacontract.tags.load_tags import get_tags
+
+__all__ = ["get_tags"]

dagster_datacontract/tags/load_tags.py

@@ -0,0 +1,45 @@
+import re
+
+from loguru import logger
+
+
+def get_tags(
+    tags_list: list[str] | None,
+) -> dict[str, str]:
+    """Parse and validate a list of string tags into a dictionary format.
+
+    Each tag in the input list should be in the form "key:value" or simply "key".
+    - Keys must match the pattern: ^[\w.-]{1,63}$
+    - Values (if provided) must match the pattern: ^[\w.-]{0,63}$
+
+    Invalid tags (those that do not match the expected format) will be ignored,
+    and a warning will be logged.
+
+    More information about Dagster tags:
+    https://docs.dagster.io/guides/build/assets/metadata-and-tags/tags
+
+    Args:
+        tags_list (list[str] | None): A list of tags as strings. Each tag may be
+            formatted as "key:value" or just "key". If None, an empty dict is returned.
+
+    Returns:
+        dict[str, str]: A dictionary of validated tags, where keys are tag names
+        and values are tag values (empty string if not provided).
+    """
+    key_pattern = re.compile(r"^[\w.-]{1,63}$")
+    val_pattern = re.compile(r"^[\w.-]{0,63}$")
+
+    tags = {}
+
+    for item in tags_list:
+        if ":" in item:
+            key, val = map(str.strip, item.split(":", 1))
+        else:
+            key, val = item.strip(), ""
+
+        if key_pattern.match(key) and val_pattern.match(val):
+            tags[key] = val
+        else:
+            logger.warning(f"Ignoring invalid tag: {item}")
+
+    return tags

{dagster_datacontract-0.2.3.dist-info → dagster_datacontract-0.2.5.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dagster-datacontract
-Version: 0.2.3
+Version: 0.2.5
 Summary: Load metadata and asset check spesifications from data contracts.
 Author-email: Fredrik Bakken <fredrik@dataheim.io>
 Requires-Python: >=3.10.0

dagster_datacontract-0.2.5.dist-info/RECORD

@@ -0,0 +1,10 @@
+dagster_datacontract/__init__.py,sha256=Be_sIJXm0gaX5lDuj5xk7FbM5hZsEmQTwHcsztmkEoo,6730
+dagster_datacontract/metadata/__init__.py,sha256=c5giTkxAgr-05ivY5hgsdVuA5UEDzdskVyjNczZdjHM,159
+dagster_datacontract/metadata/table_colums.py,sha256=Q7ZCiMReWU4-T2YfBvtt5vvoVXEoUgzK5OPMxQEgzpQ,4013
+dagster_datacontract/tags/__init__.py,sha256=jmHogoOunDs8YvgnXiMTSoCEGrxWbWnQCKH2_x2uQz8,81
+dagster_datacontract/tags/load_tags.py,sha256=aZ_HTkc-vjJ_rofT32fT_zrLCt9x1ZGn8XoihhOMhfU,1414
+dagster_datacontract-0.2.5.dist-info/licenses/LICENSE,sha256=9ULsEM1ICzCaGoso40plwO-d_SCQ7nsU6ZA4xgfaRq8,11338
+dagster_datacontract-0.2.5.dist-info/METADATA,sha256=RGX_hGPEYpNeXxF9clpc8oBoCx1XqX4H6mHwcQetL3I,3029
+dagster_datacontract-0.2.5.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
+dagster_datacontract-0.2.5.dist-info/top_level.txt,sha256=_HUQ6OJ50Q0VZxEkdocTtxk1QkJpztb1QY7A0rcvtCE,21
+dagster_datacontract-0.2.5.dist-info/RECORD,,

dagster_datacontract-0.2.3.dist-info/RECORD

@@ -1,6 +0,0 @@
-dagster_datacontract/__init__.py,sha256=bfHWETu-RpUyE6_Z3mZoZMJWQSLLgVxnWqbbFTKubRw,7779
-dagster_datacontract-0.2.3.dist-info/licenses/LICENSE,sha256=9ULsEM1ICzCaGoso40plwO-d_SCQ7nsU6ZA4xgfaRq8,11338
-dagster_datacontract-0.2.3.dist-info/METADATA,sha256=jXtkDSL9P1xLjsAdRBPRhAfMpdYDQqK9udijD8ZtamU,3029
-dagster_datacontract-0.2.3.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
-dagster_datacontract-0.2.3.dist-info/top_level.txt,sha256=_HUQ6OJ50Q0VZxEkdocTtxk1QkJpztb1QY7A0rcvtCE,21
-dagster_datacontract-0.2.3.dist-info/RECORD,,