dagster-datacontract 0.4.0__py3-none-any.whl → 0.4.1__py3-none-any.whl

dagster_datacontract/__init__.py

@@ -2,7 +2,6 @@ from datetime import timedelta
 from typing import Any
 
 import dagster as dg
-from dagster import TableColumnLineage, TableSchema
 from datacontract.data_contract import DataContract
 from datacontract.model.run import ResultEnum
 from loguru import logger
@@ -16,18 +15,28 @@ from dagster_datacontract.metadata import (
 )
 from dagster_datacontract.owners import get_owner
 from dagster_datacontract.tags import get_tags
-from dagster_datacontract.utils import normalize_path
+from dagster_datacontract.utils import combine_parts, normalize_path
 
 
 class DataContractLoader:
     def __init__(
         self,
         asset_name: str,
-        data_contract: DataContract,
+        data_contract: DataContract | None = None,
+        data_contract_path: str | None = None,
     ):
+        if data_contract is None and data_contract_path is None:
+            raise ValueError(
+                "Either 'data_contract' or 'data_contract_path' must be provided."
+            )
+
         self.asset_name = asset_name
         self.asset_key = dg.AssetKey(path=self.asset_name)
-        self.data_contract = data_contract
+        self.data_contract = (
+            data_contract
+            if data_contract
+            else DataContract(data_contract_file=data_contract_path)
+        )
         self.data_contract_specification = (
             self.data_contract.get_data_contract_specification()
         )
@@ -40,10 +49,18 @@ class DataContractLoader:
         self.owner = get_owner(self.data_contract_specification)
         self.version = self._load_version()
         self.cron_schedule = self._load_cron_schedule()
+        self.asset_spec = dg.AssetSpec(
+            key=asset_name,
+            description=self.description,
+            metadata=self.metadata,
+            code_version=self.version,
+            owners=self.owner,
+            tags=self.tags,
+        )
 
     def _load_metadata(
         self,
-    ) -> dict[str, TableColumnLineage | TableSchema | Any] | None:
+    ) -> dict[str, dg.TableColumnLineage | dg.TableSchema | Any] | None:
         metadata = (
             {
                 "datacontract/path": dg.MetadataValue.url(
@@ -56,19 +73,24 @@ class DataContractLoader:
         columns = []
         deps_by_column = {}
 
-        fields = self.data_contract_specification.models.get(self.asset_name).fields
+        try:
+            fields = self.data_contract_specification.models.get(self.asset_name).fields
 
-        for column_name, column_field in fields.items():
-            table_column = get_table_column(column_name, column_field)
-            columns.append(table_column)
+            for column_name, column_field in fields.items():
+                table_column = get_table_column(column_name, column_field)
+                columns.append(table_column)
 
-            table_column_lineage = get_column_lineage(column_field)
-            deps_by_column[column_name] = table_column_lineage
+                table_column_lineage = get_column_lineage(column_field)
+                deps_by_column[column_name] = table_column_lineage
 
-        metadata["dagster/column_schema"] = dg.TableSchema(columns=columns)
-        metadata["dagster/column_lineage"] = dg.TableColumnLineage(
-            deps_by_column=deps_by_column
-        )
+            metadata["dagster/column_schema"] = dg.TableSchema(columns=columns)
+            metadata["dagster/column_lineage"] = dg.TableColumnLineage(
+                deps_by_column=deps_by_column
+            )
+        except AttributeError as e:
+            logger.warning(
+                f"No field named {self.asset_name} found in data contract.\n{e}"
+            )
 
         server_information = get_server_information(
             self.data_contract_specification,
@@ -158,3 +180,55 @@ class DataContractLoader:
         )
 
         return freshness_checks
+
+    def combine_asset_specs(
+        self,
+        asset_spec: dg.AssetSpec,
+    ) -> dg.AssetSpec:
+        """Merge the given AssetSpec with the current object's attributes to produce a new AssetSpec.
+
+        This method combines metadata, descriptions, code versions, owners, and tags from the
+        provided `asset_spec` and the current instance. Preference is generally given to the
+        current instance's values where appropriate. Fields like dependencies, skippability,
+        group name, automation condition, kinds, and partition definitions are taken directly
+        from the input `asset_spec`.
+
+        Args:
+            asset_spec (dg.AssetSpec): The base asset specification to merge with the current one.
+
+        Returns:
+            dg.AssetSpec: A new AssetSpec instance containing the combined data.
+
+        Notes:
+            - Descriptions are joined with double newlines (`"\n\n"`).
+            - Code versions are joined with an underscore (`"_"`).
+            - Owners are concatenated.
+            - Metadata and tags are merged with the current instance taking precedence.
+        """
+        description = combine_parts(
+            [asset_spec.description, self.description], delimiter="\n\n"
+        )
+        metadata = {
+            **asset_spec.metadata,
+            **self.metadata,
+        }
+        code_version = combine_parts(
+            [asset_spec.code_version, self.version], delimiter="_"
+        )
+        owners = list(asset_spec.owners) + self.owner
+        tags = {**asset_spec.tags, **self.tags}
+
+        return dg.AssetSpec(
+            key=self.asset_name,
+            deps=asset_spec.deps,
+            description=description,
+            metadata=metadata,
+            skippable=asset_spec.skippable,
+            group_name=asset_spec.group_name,
+            code_version=code_version,
+            automation_condition=asset_spec.automation_condition,
+            owners=owners,
+            tags=tags,
+            kinds=asset_spec.kinds,
+            partitions_def=asset_spec.partitions_def,
+        )

dagster_datacontract/utils/__init__.py

@@ -1,28 +1,8 @@
-import os
-import urllib.parse
-
-
-def normalize_path(path: str) -> str:
-    """Normalizes a file path to ensure it is returned in a consistent URI format.
-
-    This function checks if the provided path is a local file path (with no scheme
-    or with the 'file' scheme) and converts it into a fully qualified file URI.
-    If the path already has a non-'file' scheme (e.g., 's3://', 'http://'),
-    it is returned unchanged.
-
-    Parameters:
-        path (str): The input file path. This can be a relative or absolute local path,
-                    a path starting with `~`, or a URI with a supported scheme.
-
-    Returns:
-        str: A normalized path string:
-             - If the input is a local path or has a "file" scheme, returns it in the form "file:///absolute/path".
-             - If the input has another scheme (e.g., "s3://", "http://"), returns it unchanged.
-    """
-    parsed = urllib.parse.urlparse(path)
-
-    if not parsed.scheme or parsed.scheme == "file":
-        full_path = os.path.abspath(os.path.expanduser(path))
-        return f"file://{full_path}"
-    else:
-        return path
+from dagster_datacontract.utils.combine_strings import combine_parts
+from dagster_datacontract.utils.paths import get_absolute_path, normalize_path
+
+__all__ = [
+    "combine_parts",
+    "get_absolute_path",
+    "normalize_path",
+]

dagster_datacontract/utils/combine_strings.py

@@ -0,0 +1,29 @@
+from collections.abc import Iterable
+
+
+def combine_parts(parts: Iterable[str | None], delimiter: str = "_") -> str:
+    """
+    Combine multiple optional strings using a specified delimiter.
+
+    This function takes an iterable of optional strings and joins the non-None,
+    non-empty strings using the given delimiter. None values and empty strings
+    are ignored. If all values are None or empty, the result is an empty string.
+
+    Args:
+        parts (Iterable[Optional[str]]): An iterable of strings or None values to combine.
+        delimiter (str): A string used to separate the non-None parts. Defaults to "_".
+
+    Returns:
+        str: A single combined string of all non-None, non-empty parts separated by the delimiter.
+
+    Examples:
+        >>> combine_parts(["v1", "2023", None])
+        'v1_2023'
+
+        >>> combine_parts([None, None])
+        ''
+
+        >>> combine_parts(["", "alpha", None])
+        'alpha'
+    """
+    return delimiter.join(filter(None, parts))

dagster_datacontract/utils/paths.py

@@ -0,0 +1,51 @@
+import os
+from pathlib import Path
+from urllib.parse import urlparse
+
+import dagster as dg
+
+
+def normalize_path(path: str) -> str:
+    """Normalizes a file path to ensure it is returned in a consistent URI format.
+
+    This function checks if the provided path is a local file path (with no scheme
+    or with the 'file' scheme) and converts it into a fully qualified file URI.
+    If the path already has a non-'file' scheme (e.g., 's3://', 'http://'),
+    it is returned unchanged.
+
+    Parameters:
+        path (str): The input file path. This can be a relative or absolute local path,
+                    a path starting with `~`, or a URI with a supported scheme.
+
+    Returns:
+        str: A normalized path string:
+             - If the input is a local path or has a "file" scheme, returns it in the form "file:///absolute/path".
+             - If the input has another scheme (e.g., "s3://", "http://"), returns it unchanged.
+    """
+    parsed = urlparse(path)
+
+    if not parsed.scheme or parsed.scheme == "file":
+        full_path = os.path.abspath(os.path.expanduser(path))
+        return f"file://{full_path}"
+    else:
+        return path
+
+
+def get_absolute_path(
+    context_path: Path,
+    full_path: str,
+) -> Path:
+    """TODO."""
+    if isinstance(full_path, dg.UrlMetadataValue):
+        full_path = full_path.url
+
+    parsed_path = urlparse(full_path)
+    if parsed_path.scheme == "file":
+        full_path = Path(parsed_path.path)
+    else:
+        full_path = Path(full_path)
+
+    if full_path.is_absolute():
+        return full_path
+
+    return Path(context_path, full_path).absolute()

{dagster_datacontract-0.4.0.dist-info → dagster_datacontract-0.4.1.dist-info}/METADATA

@@ -1,13 +1,14 @@
 Metadata-Version: 2.4
 Name: dagster-datacontract
-Version: 0.4.0
+Version: 0.4.1
 Summary: Load metadata and asset check spesifications from data contracts.
 Author-email: Fredrik Bakken <fredrik@dataheim.io>
-Requires-Python: >=3.10.0
+Requires-Python: >=3.10
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: dagster>=1.10.10
-Requires-Dist: datacontract-cli>=0.10.23
+Requires-Dist: dagster-dg>=0.26.11
+Requires-Dist: datacontract-cli>=0.10.24
 Requires-Dist: loguru>=0.7.3
 Dynamic: license-file
 
@@ -25,7 +26,7 @@ uv add dagster-datacontract
 
 ## Simple Example
 
-The following example can be found inside the [example](https://github.com/dataheim-io/dagster-datacontract/tree/main/example) directory:
+The following example can be found inside the [examples/simple](https://github.com/dataheim-io/dagster-datacontract/tree/main/examples/simple)-directory:
 
 ```python
 from datetime import timedelta

{dagster_datacontract-0.4.0.dist-info → dagster_datacontract-0.4.1.dist-info}/RECORD

@@ -1,4 +1,4 @@
-dagster_datacontract/__init__.py,sha256=nlLGDivblIOJqE5CUA7wzSb_MwSVO_3YwhO3dyVjwgo,5711
+dagster_datacontract/__init__.py,sha256=Tv6_G45c16Yq5Sh9A5Xw9rxkkIiAm9BBxuK79kkvSvE,8585
 dagster_datacontract/description/__init__.py,sha256=ulWqPp5jIPvCzaDFZcjLjcDkljJ5j_FRsE0dXhK8Wlc,104
 dagster_datacontract/description/description.py,sha256=FmjgCYDpJ9UHrvAv0sAthfRohDjdG0lL1XcMKK8QMmI,1646
 dagster_datacontract/metadata/__init__.py,sha256=Gj7Htl3rYRXsE-631yr4LWqL7Tf5bZKFEFjTeglxZek,359
@@ -8,9 +8,11 @@ dagster_datacontract/metadata/table_colums.py,sha256=Q7ZCiMReWU4-T2YfBvtt5vvoVXE
 dagster_datacontract/owners/__init__.py,sha256=c0AhLQRzfw-QPmsF9rPXRyE6VoLmgDRRNgVqG8JUvFs,882
 dagster_datacontract/tags/__init__.py,sha256=2Ph-M0WbBKUjJWIzM_cEBW3SQZh7Nq8oy5MbD5bt_lc,76
 dagster_datacontract/tags/tags.py,sha256=aZ_HTkc-vjJ_rofT32fT_zrLCt9x1ZGn8XoihhOMhfU,1414
-dagster_datacontract/utils/__init__.py,sha256=Zfbuf20Eorf7BD8gSDASiPqgPwWu8Mz03r3aa2zE4NA,1106
-dagster_datacontract-0.4.0.dist-info/licenses/LICENSE,sha256=9ULsEM1ICzCaGoso40plwO-d_SCQ7nsU6ZA4xgfaRq8,11338
-dagster_datacontract-0.4.0.dist-info/METADATA,sha256=RRJoWluvktDqQhXBH7dsQOsctgrUQYIgiuo4QAbxYu4,3029
-dagster_datacontract-0.4.0.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
-dagster_datacontract-0.4.0.dist-info/top_level.txt,sha256=_HUQ6OJ50Q0VZxEkdocTtxk1QkJpztb1QY7A0rcvtCE,21
-dagster_datacontract-0.4.0.dist-info/RECORD,,
+dagster_datacontract/utils/__init__.py,sha256=GSQ2Zry9pKlLhWI5Vjoj3X7iZiEgt-SjqnQRwfplbHM,231
+dagster_datacontract/utils/combine_strings.py,sha256=nWy3unX6yuPi8YmvDTl_mO10K6MbJP8cxQWPOMDj6G4,987
+dagster_datacontract/utils/paths.py,sha256=Rh-l5GSmxZwhIVQ_aBJPHTKU5afEma8hlA5RT31EhbY,1611
+dagster_datacontract-0.4.1.dist-info/licenses/LICENSE,sha256=9ULsEM1ICzCaGoso40plwO-d_SCQ7nsU6ZA4xgfaRq8,11338
+dagster_datacontract-0.4.1.dist-info/METADATA,sha256=KAjnQQkr1UUNBAPcQ44mJcqUqmcEqxHDvr3pU-5YyWU,3078
+dagster_datacontract-0.4.1.dist-info/WHEEL,sha256=pxyMxgL8-pra_rKaQ4drOZAegBVuX-G_4nRHjjgWbmo,91
+dagster_datacontract-0.4.1.dist-info/top_level.txt,sha256=_HUQ6OJ50Q0VZxEkdocTtxk1QkJpztb1QY7A0rcvtCE,21
+dagster_datacontract-0.4.1.dist-info/RECORD,,

{dagster_datacontract-0.4.0.dist-info → dagster_datacontract-0.4.1.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (78.1.0)
+Generator: setuptools (79.0.0)
 Root-Is-Purelib: true
 Tag: py3-none-any