dagster-datacontract 0.3.1__py3-none-any.whl → 0.4.1__py3-none-any.whl

dagster_datacontract/__init__.py

@@ -2,7 +2,6 @@ from datetime import timedelta
 from typing import Any
 
 import dagster as dg
-from dagster import TableColumnLineage, TableSchema
 from datacontract.data_contract import DataContract
 from datacontract.model.run import ResultEnum
 from loguru import logger
@@ -10,22 +9,34 @@ from loguru import logger
 from dagster_datacontract.description import get_description
 from dagster_datacontract.metadata import (
     get_column_lineage,
+    get_links,
     get_server_information,
     get_table_column,
 )
+from dagster_datacontract.owners import get_owner
 from dagster_datacontract.tags import get_tags
-from dagster_datacontract.utils import normalize_path
+from dagster_datacontract.utils import combine_parts, normalize_path
 
 
 class DataContractLoader:
     def __init__(
         self,
         asset_name: str,
-        data_contract: DataContract,
+        data_contract: DataContract | None = None,
+        data_contract_path: str | None = None,
     ):
+        if data_contract is None and data_contract_path is None:
+            raise ValueError(
+                "Either 'data_contract' or 'data_contract_path' must be provided."
+            )
+
         self.asset_name = asset_name
         self.asset_key = dg.AssetKey(path=self.asset_name)
-        self.data_contract = data_contract
+        self.data_contract = (
+            data_contract
+            if data_contract
+            else DataContract(data_contract_file=data_contract_path)
+        )
         self.data_contract_specification = (
             self.data_contract.get_data_contract_specification()
         )
@@ -35,16 +46,24 @@ class DataContractLoader:
             self.asset_name,
             self.data_contract_specification,
         )
-        self.owner = self._load_owner()
+        self.owner = get_owner(self.data_contract_specification)
         self.version = self._load_version()
         self.cron_schedule = self._load_cron_schedule()
+        self.asset_spec = dg.AssetSpec(
+            key=asset_name,
+            description=self.description,
+            metadata=self.metadata,
+            code_version=self.version,
+            owners=self.owner,
+            tags=self.tags,
+        )
 
     def _load_metadata(
         self,
-    ) -> dict[str, TableColumnLineage | TableSchema | Any] | None:
+    ) -> dict[str, dg.TableColumnLineage | dg.TableSchema | Any] | None:
         metadata = (
             {
-                "data contract path": dg.MetadataValue.url(
+                "datacontract/path": dg.MetadataValue.url(
                     normalize_path(self.data_contract._data_contract_file)
                 ),
             }
@@ -54,19 +73,24 @@ class DataContractLoader:
         columns = []
         deps_by_column = {}
 
-        fields = self.data_contract_specification.models.get(self.asset_name).fields
+        try:
+            fields = self.data_contract_specification.models.get(self.asset_name).fields
 
-        for column_name, column_field in fields.items():
-            table_column = get_table_column(column_name, column_field)
-            columns.append(table_column)
+            for column_name, column_field in fields.items():
+                table_column = get_table_column(column_name, column_field)
+                columns.append(table_column)
 
-            table_column_lineage = get_column_lineage(column_field)
-            deps_by_column[column_name] = table_column_lineage
+                table_column_lineage = get_column_lineage(column_field)
+                deps_by_column[column_name] = table_column_lineage
 
-        metadata["dagster/column_schema"] = dg.TableSchema(columns=columns)
-        metadata["dagster/column_lineage"] = dg.TableColumnLineage(
-            deps_by_column=deps_by_column
-        )
+            metadata["dagster/column_schema"] = dg.TableSchema(columns=columns)
+            metadata["dagster/column_lineage"] = dg.TableColumnLineage(
+                deps_by_column=deps_by_column
+            )
+        except AttributeError as e:
+            logger.warning(
+                f"No field named {self.asset_name} found in data contract.\n{e}"
+            )
 
         server_information = get_server_information(
             self.data_contract_specification,
@@ -75,12 +99,10 @@ class DataContractLoader:
         )
         metadata.update(server_information)
 
-        return metadata
-
-    def _load_owner(self) -> list[str] | None:
-        owner = self.data_contract_specification.info.owner
+        links = get_links(self.data_contract_specification.links)
+        metadata.update(links)
 
-        return [f"team:{owner}"] if owner else None
+        return metadata
 
     def _load_version(self) -> str | None:
         version = self.data_contract_specification.info.version
@@ -158,3 +180,55 @@ class DataContractLoader:
         )
 
         return freshness_checks
+
+    def combine_asset_specs(
+        self,
+        asset_spec: dg.AssetSpec,
+    ) -> dg.AssetSpec:
+        """Merge the given AssetSpec with the current object's attributes to produce a new AssetSpec.
+
+        This method combines metadata, descriptions, code versions, owners, and tags from the
+        provided `asset_spec` and the current instance. Preference is generally given to the
+        current instance's values where appropriate. Fields like dependencies, skippability,
+        group name, automation condition, kinds, and partition definitions are taken directly
+        from the input `asset_spec`.
+
+        Args:
+            asset_spec (dg.AssetSpec): The base asset specification to merge with the current one.
+
+        Returns:
+            dg.AssetSpec: A new AssetSpec instance containing the combined data.
+
+        Notes:
+            - Descriptions are joined with double newlines (`"\n\n"`).
+            - Code versions are joined with an underscore (`"_"`).
+            - Owners are concatenated.
+            - Metadata and tags are merged with the current instance taking precedence.
+        """
+        description = combine_parts(
+            [asset_spec.description, self.description], delimiter="\n\n"
+        )
+        metadata = {
+            **asset_spec.metadata,
+            **self.metadata,
+        }
+        code_version = combine_parts(
+            [asset_spec.code_version, self.version], delimiter="_"
+        )
+        owners = list(asset_spec.owners) + self.owner
+        tags = {**asset_spec.tags, **self.tags}
+
+        return dg.AssetSpec(
+            key=self.asset_name,
+            deps=asset_spec.deps,
+            description=description,
+            metadata=metadata,
+            skippable=asset_spec.skippable,
+            group_name=asset_spec.group_name,
+            code_version=code_version,
+            automation_condition=asset_spec.automation_condition,
+            owners=owners,
+            tags=tags,
+            kinds=asset_spec.kinds,
+            partitions_def=asset_spec.partitions_def,
+        )

dagster_datacontract/metadata/__init__.py

@@ -1,7 +1,13 @@
+from dagster_datacontract.metadata.links import get_links
 from dagster_datacontract.metadata.server_information import get_server_information
 from dagster_datacontract.metadata.table_colums import (
     get_column_lineage,
     get_table_column,
 )
 
-__all__ = ["get_table_column", "get_column_lineage", "get_server_information"]
+__all__ = [
+    "get_column_lineage",
+    "get_links",
+    "get_table_column",
+    "get_server_information",
+]

dagster_datacontract/metadata/links.py

@@ -0,0 +1,17 @@
+import dagster as dg
+
+
+def get_links(links: dict[str, str]) -> dict[str, str]:
+    """Return a dictionary with keys prefixed by 'link/' and values as Dagster URL metadata.
+
+    Args:
+        links (dict[str, str]): A dictionary where each key is a name/label and each
+            value is a URL string.
+
+    Returns:
+        dict[str, str]: A dictionary where each key is prefixed with 'link/' and
+            each value is a `MetadataValue.url`.
+    """
+    links = {f"link/{key}": dg.MetadataValue.url(value) for key, value in links.items()}
+
+    return links

dagster_datacontract/metadata/server_information.py

@@ -15,6 +15,8 @@ def get_server_information(
     the specified server by name and constructs a dictionary with keys such as
     "dagster/uri" and "dagster/table_name" depending on the server type.
 
+    Server information can be obtained from: https://datacontract.com/#server-object
+
     Parameters:
         data_contract_specification (DataContractSpecification):
             The data contract specification containing server configurations.
@@ -36,43 +38,71 @@ def get_server_information(
     match server.type:
         case "azure":
             server_information["dagster/uri"] = server.location
+            server_information["azure/storage_account"] = server.storageAccount
+            server_information["file/format"] = server.format
+            server_information["file/delimiter"] = server.delimiter
+        case "bigquery":
+            server_information["bigquery/project"] = server.project
+            server_information["bigquery/dataset"] = server.dataset
         case "databricks":
             server_information["dagster/uri"] = server.host
             server_information["dagster/table_name"] = (
                 f"{server.catalog}.{server.schema}.{asset_name}"
             )
+        case "glue":
+            server_information = {}
         case "kafka":
             server_information["dagster/uri"] = server.host
+            server_information["kafka/topic"] = server.topic
+            server_information["kafka/format"] = server.format
         case "kinesis":
-            server_information = {}
+            server_information["kinesis/stream"] = server.stream
+            server_information["kinesis/region"] = server.region
+            server_information["kinesis/format"] = server.format
         case "local":
             server_information["dagster/uri"] = normalize_path(server.path)
+            server_information["file/format"] = server.format
         case "oracle":
             server_information["dagster/uri"] = f"{server.host}:{server.port}"
+            server_information["oracle/service_name"] = server.serviceName
         case "postgres":
             server_information["dagster/uri"] = f"{server.host}:{server.port}"
             server_information["dagster/table_name"] = (
                 f"{server.database}.{server.schema}.{asset_name}"
             )
         case "pubsub":
-            server_information = {}
+            server_information["pubsub/project"] = server.project
+            server_information["pubsub/topic"] = server.topic
         case "redshift":
             server_information["dagster/uri"] = server.endpoint
             server_information["dagster/table_name"] = (
                 f"{server.database}.{server.schema}.{asset_name}"
             )
+            server_information["redshift/account"] = server.account
+            server_information["redshift/host"] = server.host
+            server_information["redshift/port"] = server.port
+            server_information["redshift/cluster"] = server.clusterIdentifier
         case "s3":
             server_information["dagster/uri"] = server.location
+            server_information["s3/endpoint"] = server.endpointUrl
+            server_information["file/format"] = server.format
+            server_information["file/delimiter"] = server.delimiter
         case "sftp":
             server_information["dagster/uri"] = server.location
+            server_information["file/format"] = server.format
+            server_information["file/delimiter"] = server.delimiter
         case "snowflake":
             server_information["dagster/table_name"] = (
                 f"{server.database}.{server.schema}.{asset_name}"
             )
+            server_information["snowflake/account"] = server.account
         case "sqlserver":
             server_information["dagster/table_name"] = (
                 f"{server.database}.{server.schema}.{asset_name}"
             )
+            server_information["sqlserver/host"] = server.host
+            server_information["sqlserver/port"] = server.port
+            server_information["sqlserver/driver"] = server.driver
         case "trino":
             server_information["dagster/uri"] = f"{server.host}:{server.port}"
             server_information["dagster/table_name"] = (

dagster_datacontract/owners/__init__.py

@@ -0,0 +1,23 @@
+from datacontract.data_contract import DataContractSpecification
+
+
+def get_owner(
+    data_contract_specification: DataContractSpecification,
+    is_team: bool = True,
+) -> list[str] | None:
+    """Return the owner of a data contract, optionally formatted as a team identifier.
+
+    Args:
+        data_contract_specification (DataContractSpecification): The data contract specification containing ownership metadata.
+        is_team (bool, optional): If True, formats the owner as a team identifier (e.g., 'team:owner').
+                                  If False, returns the raw owner string. Defaults to True.
+
+    Returns:
+        list[str] | None: A list containing the owner string, formatted depending on `is_team`, or None if no owner is found.
+    """
+    owner = data_contract_specification.info.owner
+
+    if is_team:
+        return [f"team:{owner}"]
+
+    return [owner]

dagster_datacontract/utils/__init__.py

@@ -1,28 +1,8 @@
-import os
-import urllib.parse
-
-
-def normalize_path(path: str) -> str:
-    """Normalizes a file path to ensure it is returned in a consistent URI format.
-
-    This function checks if the provided path is a local file path (with no scheme
-    or with the 'file' scheme) and converts it into a fully qualified file URI.
-    If the path already has a non-'file' scheme (e.g., 's3://', 'http://'),
-    it is returned unchanged.
-
-    Parameters:
-        path (str): The input file path. This can be a relative or absolute local path,
-                    a path starting with `~`, or a URI with a supported scheme.
-
-    Returns:
-        str: A normalized path string:
-             - If the input is a local path or has a "file" scheme, returns it in the form "file:///absolute/path".
-             - If the input has another scheme (e.g., "s3://", "http://"), returns it unchanged.
-    """
-    parsed = urllib.parse.urlparse(path)
-
-    if not parsed.scheme or parsed.scheme == "file":
-        full_path = os.path.abspath(os.path.expanduser(path))
-        return f"file://{full_path}"
-    else:
-        return path
+from dagster_datacontract.utils.combine_strings import combine_parts
+from dagster_datacontract.utils.paths import get_absolute_path, normalize_path
+
+__all__ = [
+    "combine_parts",
+    "get_absolute_path",
+    "normalize_path",
+]

dagster_datacontract/utils/combine_strings.py

@@ -0,0 +1,29 @@
+from collections.abc import Iterable
+
+
+def combine_parts(parts: Iterable[str | None], delimiter: str = "_") -> str:
+    """
+    Combine multiple optional strings using a specified delimiter.
+
+    This function takes an iterable of optional strings and joins the non-None,
+    non-empty strings using the given delimiter. None values and empty strings
+    are ignored. If all values are None or empty, the result is an empty string.
+
+    Args:
+        parts (Iterable[Optional[str]]): An iterable of strings or None values to combine.
+        delimiter (str): A string used to separate the non-None parts. Defaults to "_".
+
+    Returns:
+        str: A single combined string of all non-None, non-empty parts separated by the delimiter.
+
+    Examples:
+        >>> combine_parts(["v1", "2023", None])
+        'v1_2023'
+
+        >>> combine_parts([None, None])
+        ''
+
+        >>> combine_parts(["", "alpha", None])
+        'alpha'
+    """
+    return delimiter.join(filter(None, parts))

dagster_datacontract/utils/paths.py

@@ -0,0 +1,51 @@
+import os
+from pathlib import Path
+from urllib.parse import urlparse
+
+import dagster as dg
+
+
+def normalize_path(path: str) -> str:
+    """Normalizes a file path to ensure it is returned in a consistent URI format.
+
+    This function checks if the provided path is a local file path (with no scheme
+    or with the 'file' scheme) and converts it into a fully qualified file URI.
+    If the path already has a non-'file' scheme (e.g., 's3://', 'http://'),
+    it is returned unchanged.
+
+    Parameters:
+        path (str): The input file path. This can be a relative or absolute local path,
+                    a path starting with `~`, or a URI with a supported scheme.
+
+    Returns:
+        str: A normalized path string:
+             - If the input is a local path or has a "file" scheme, returns it in the form "file:///absolute/path".
+             - If the input has another scheme (e.g., "s3://", "http://"), returns it unchanged.
+    """
+    parsed = urlparse(path)
+
+    if not parsed.scheme or parsed.scheme == "file":
+        full_path = os.path.abspath(os.path.expanduser(path))
+        return f"file://{full_path}"
+    else:
+        return path
+
+
+def get_absolute_path(
+    context_path: Path,
+    full_path: str,
+) -> Path:
+    """TODO."""
+    if isinstance(full_path, dg.UrlMetadataValue):
+        full_path = full_path.url
+
+    parsed_path = urlparse(full_path)
+    if parsed_path.scheme == "file":
+        full_path = Path(parsed_path.path)
+    else:
+        full_path = Path(full_path)
+
+    if full_path.is_absolute():
+        return full_path
+
+    return Path(context_path, full_path).absolute()

{dagster_datacontract-0.3.1.dist-info → dagster_datacontract-0.4.1.dist-info}/METADATA

@@ -1,13 +1,14 @@
 Metadata-Version: 2.4
 Name: dagster-datacontract
-Version: 0.3.1
+Version: 0.4.1
 Summary: Load metadata and asset check spesifications from data contracts.
 Author-email: Fredrik Bakken <fredrik@dataheim.io>
-Requires-Python: >=3.10.0
+Requires-Python: >=3.10
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: dagster>=1.10.10
-Requires-Dist: datacontract-cli>=0.10.23
+Requires-Dist: dagster-dg>=0.26.11
+Requires-Dist: datacontract-cli>=0.10.24
 Requires-Dist: loguru>=0.7.3
 Dynamic: license-file
 
@@ -25,7 +26,7 @@ uv add dagster-datacontract
 
 ## Simple Example
 
-The following example can be found inside the [example](https://github.com/dataheim-io/dagster-datacontract/tree/main/example) directory:
+The following example can be found inside the [examples/simple](https://github.com/dataheim-io/dagster-datacontract/tree/main/examples/simple)-directory:
 
 ```python
 from datetime import timedelta

dagster_datacontract-0.4.1.dist-info/RECORD

@@ -0,0 +1,18 @@
+dagster_datacontract/__init__.py,sha256=Tv6_G45c16Yq5Sh9A5Xw9rxkkIiAm9BBxuK79kkvSvE,8585
+dagster_datacontract/description/__init__.py,sha256=ulWqPp5jIPvCzaDFZcjLjcDkljJ5j_FRsE0dXhK8Wlc,104
+dagster_datacontract/description/description.py,sha256=FmjgCYDpJ9UHrvAv0sAthfRohDjdG0lL1XcMKK8QMmI,1646
+dagster_datacontract/metadata/__init__.py,sha256=Gj7Htl3rYRXsE-631yr4LWqL7Tf5bZKFEFjTeglxZek,359
+dagster_datacontract/metadata/links.py,sha256=ckSTZLhj6ZETrJMeXDEKVFIoZz871lhL8ZgRtsEIIvs,562
+dagster_datacontract/metadata/server_information.py,sha256=Wd7hPZ8IfeISBe5VThz0pBGkmiPv4Ia2MGNhBJ-aWik,5397
+dagster_datacontract/metadata/table_colums.py,sha256=Q7ZCiMReWU4-T2YfBvtt5vvoVXEoUgzK5OPMxQEgzpQ,4013
+dagster_datacontract/owners/__init__.py,sha256=c0AhLQRzfw-QPmsF9rPXRyE6VoLmgDRRNgVqG8JUvFs,882
+dagster_datacontract/tags/__init__.py,sha256=2Ph-M0WbBKUjJWIzM_cEBW3SQZh7Nq8oy5MbD5bt_lc,76
+dagster_datacontract/tags/tags.py,sha256=aZ_HTkc-vjJ_rofT32fT_zrLCt9x1ZGn8XoihhOMhfU,1414
+dagster_datacontract/utils/__init__.py,sha256=GSQ2Zry9pKlLhWI5Vjoj3X7iZiEgt-SjqnQRwfplbHM,231
+dagster_datacontract/utils/combine_strings.py,sha256=nWy3unX6yuPi8YmvDTl_mO10K6MbJP8cxQWPOMDj6G4,987
+dagster_datacontract/utils/paths.py,sha256=Rh-l5GSmxZwhIVQ_aBJPHTKU5afEma8hlA5RT31EhbY,1611
+dagster_datacontract-0.4.1.dist-info/licenses/LICENSE,sha256=9ULsEM1ICzCaGoso40plwO-d_SCQ7nsU6ZA4xgfaRq8,11338
+dagster_datacontract-0.4.1.dist-info/METADATA,sha256=KAjnQQkr1UUNBAPcQ44mJcqUqmcEqxHDvr3pU-5YyWU,3078
+dagster_datacontract-0.4.1.dist-info/WHEEL,sha256=pxyMxgL8-pra_rKaQ4drOZAegBVuX-G_4nRHjjgWbmo,91
+dagster_datacontract-0.4.1.dist-info/top_level.txt,sha256=_HUQ6OJ50Q0VZxEkdocTtxk1QkJpztb1QY7A0rcvtCE,21
+dagster_datacontract-0.4.1.dist-info/RECORD,,

{dagster_datacontract-0.3.1.dist-info → dagster_datacontract-0.4.1.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (78.1.0)
+Generator: setuptools (79.0.0)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

dagster_datacontract-0.3.1.dist-info/RECORD

@@ -1,14 +0,0 @@
-dagster_datacontract/__init__.py,sha256=Pzq4AblLnDhW0QBmV8ntV5KLD0a7D0-mA2Lw9hjS_V8,5685
-dagster_datacontract/description/__init__.py,sha256=ulWqPp5jIPvCzaDFZcjLjcDkljJ5j_FRsE0dXhK8Wlc,104
-dagster_datacontract/description/description.py,sha256=FmjgCYDpJ9UHrvAv0sAthfRohDjdG0lL1XcMKK8QMmI,1646
-dagster_datacontract/metadata/__init__.py,sha256=e-xmcWWoAhmKTwosshsxnyrjI1j-UyY6YpdpzA2ggF4,269
-dagster_datacontract/metadata/server_information.py,sha256=jk_H8aI5PdGzIeoYThlhhZMSOtBh-6xc8QAFz0BFesU,3512
-dagster_datacontract/metadata/table_colums.py,sha256=Q7ZCiMReWU4-T2YfBvtt5vvoVXEoUgzK5OPMxQEgzpQ,4013
-dagster_datacontract/tags/__init__.py,sha256=2Ph-M0WbBKUjJWIzM_cEBW3SQZh7Nq8oy5MbD5bt_lc,76
-dagster_datacontract/tags/tags.py,sha256=aZ_HTkc-vjJ_rofT32fT_zrLCt9x1ZGn8XoihhOMhfU,1414
-dagster_datacontract/utils/__init__.py,sha256=Zfbuf20Eorf7BD8gSDASiPqgPwWu8Mz03r3aa2zE4NA,1106
-dagster_datacontract-0.3.1.dist-info/licenses/LICENSE,sha256=9ULsEM1ICzCaGoso40plwO-d_SCQ7nsU6ZA4xgfaRq8,11338
-dagster_datacontract-0.3.1.dist-info/METADATA,sha256=P98oLsqB-GCJz4uanki-xtG3UnZCHD0biDMUYIZMEhc,3029
-dagster_datacontract-0.3.1.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
-dagster_datacontract-0.3.1.dist-info/top_level.txt,sha256=_HUQ6OJ50Q0VZxEkdocTtxk1QkJpztb1QY7A0rcvtCE,21
-dagster_datacontract-0.3.1.dist-info/RECORD,,