dagster-datacontract 0.4.0__tar.gz → 0.5.0__tar.gz

{dagster_datacontract-0.4.0 → dagster_datacontract-0.5.0}/PKG-INFO

@@ -1,13 +1,13 @@
 Metadata-Version: 2.4
 Name: dagster-datacontract
-Version: 0.4.0
+Version: 0.5.0
 Summary: Load metadata and asset check spesifications from data contracts.
 Author-email: Fredrik Bakken <fredrik@dataheim.io>
-Requires-Python: >=3.10.0
+Requires-Python: >=3.10
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: dagster>=1.10.10
-Requires-Dist: datacontract-cli>=0.10.23
+Requires-Dist: datacontract-cli>=0.10.24
 Requires-Dist: loguru>=0.7.3
 Dynamic: license-file
 
@@ -23,9 +23,9 @@ Install `dagster-datacontract` using [`uv`](https://github.com/astral-sh/uv):
 uv add dagster-datacontract
 ```
 
-## Simple Example
+## Example
 
-The following example can be found inside the [example](https://github.com/dataheim-io/dagster-datacontract/tree/main/example) directory:
+The following example can be found inside the [examples](https://github.com/dataheim-io/dagster-datacontract/tree/main/examples)-directory:
 
 ```python
 from datetime import timedelta
@@ -39,10 +39,11 @@ from datacontract.data_contract import DataContract
 from dagster_datacontract import DataContractLoader
 
 asset_name = "yellow_taxi_trip_records"
+examples_path = Path(__file__).resolve().parent.parent
 data_contract = DataContractLoader(
     asset_name=asset_name,
     data_contract=DataContract(
-        data_contract_file="./example/datacontract.yml",
+        data_contract_file=str(examples_path / "datacontracts" / "datacontract.yml"),
         server="production",
     ),
 )
@@ -59,7 +60,7 @@ data_contract = DataContractLoader(
 def yellow_taxi_trip_records(
     context: dg.AssetExecutionContext,
 ) -> None:
-    download_path = "./example/data"
+    download_path = examples_path.parent.parent / "data"
     Path(download_path).mkdir(parents=True, exist_ok=True)
 
     url = "https://d37ci6vzurychx.cloudfront.net/trip-data/yellow_tripdata_2025-01.parquet"

{dagster_datacontract-0.4.0 → dagster_datacontract-0.5.0}/README.md

@@ -10,9 +10,9 @@ Install `dagster-datacontract` using [`uv`](https://github.com/astral-sh/uv):
 uv add dagster-datacontract
 ```
 
-## Simple Example
+## Example
 
-The following example can be found inside the [example](https://github.com/dataheim-io/dagster-datacontract/tree/main/example) directory:
+The following example can be found inside the [examples](https://github.com/dataheim-io/dagster-datacontract/tree/main/examples)-directory:
 
 ```python
 from datetime import timedelta
@@ -26,10 +26,11 @@ from datacontract.data_contract import DataContract
 from dagster_datacontract import DataContractLoader
 
 asset_name = "yellow_taxi_trip_records"
+examples_path = Path(__file__).resolve().parent.parent
 data_contract = DataContractLoader(
     asset_name=asset_name,
     data_contract=DataContract(
-        data_contract_file="./example/datacontract.yml",
+        data_contract_file=str(examples_path / "datacontracts" / "datacontract.yml"),
         server="production",
     ),
 )
@@ -46,7 +47,7 @@ data_contract = DataContractLoader(
 def yellow_taxi_trip_records(
     context: dg.AssetExecutionContext,
 ) -> None:
-    download_path = "./example/data"
+    download_path = examples_path.parent.parent / "data"
     Path(download_path).mkdir(parents=True, exist_ok=True)
 
     url = "https://d37ci6vzurychx.cloudfront.net/trip-data/yellow_tripdata_2025-01.parquet"

{dagster_datacontract-0.4.0 → dagster_datacontract-0.5.0}/dagster_datacontract/__init__.py

@@ -2,7 +2,6 @@ from datetime import timedelta
 from typing import Any
 
 import dagster as dg
-from dagster import TableColumnLineage, TableSchema
 from datacontract.data_contract import DataContract
 from datacontract.model.run import ResultEnum
 from loguru import logger
@@ -16,18 +15,28 @@ from dagster_datacontract.metadata import (
 )
 from dagster_datacontract.owners import get_owner
 from dagster_datacontract.tags import get_tags
-from dagster_datacontract.utils import normalize_path
+from dagster_datacontract.utils import combine_parts, normalize_path
 
 
 class DataContractLoader:
     def __init__(
         self,
         asset_name: str,
-        data_contract: DataContract,
+        data_contract: DataContract | None = None,
+        data_contract_path: str | None = None,
     ):
+        if data_contract is None and data_contract_path is None:
+            raise ValueError(
+                "Either 'data_contract' or 'data_contract_path' must be provided."
+            )
+
         self.asset_name = asset_name
         self.asset_key = dg.AssetKey(path=self.asset_name)
-        self.data_contract = data_contract
+        self.data_contract = (
+            data_contract
+            if data_contract
+            else DataContract(data_contract_file=data_contract_path)
+        )
         self.data_contract_specification = (
             self.data_contract.get_data_contract_specification()
         )
@@ -40,10 +49,18 @@ class DataContractLoader:
         self.owner = get_owner(self.data_contract_specification)
         self.version = self._load_version()
         self.cron_schedule = self._load_cron_schedule()
+        self.asset_spec = dg.AssetSpec(
+            key=asset_name,
+            description=self.description,
+            metadata=self.metadata,
+            code_version=self.version,
+            owners=self.owner,
+            tags=self.tags,
+        )
 
     def _load_metadata(
         self,
-    ) -> dict[str, TableColumnLineage | TableSchema | Any] | None:
+    ) -> dict[str, dg.TableColumnLineage | dg.TableSchema | Any] | None:
         metadata = (
             {
                 "datacontract/path": dg.MetadataValue.url(
@@ -56,19 +73,24 @@ class DataContractLoader:
         columns = []
         deps_by_column = {}
 
-        fields = self.data_contract_specification.models.get(self.asset_name).fields
+        try:
+            fields = self.data_contract_specification.models.get(self.asset_name).fields
 
-        for column_name, column_field in fields.items():
-            table_column = get_table_column(column_name, column_field)
-            columns.append(table_column)
+            for column_name, column_field in fields.items():
+                table_column = get_table_column(column_name, column_field)
+                columns.append(table_column)
 
-            table_column_lineage = get_column_lineage(column_field)
-            deps_by_column[column_name] = table_column_lineage
+                table_column_lineage = get_column_lineage(column_field)
+                deps_by_column[column_name] = table_column_lineage
 
-        metadata["dagster/column_schema"] = dg.TableSchema(columns=columns)
-        metadata["dagster/column_lineage"] = dg.TableColumnLineage(
-            deps_by_column=deps_by_column
-        )
+            metadata["dagster/column_schema"] = dg.TableSchema(columns=columns)
+            metadata["dagster/column_lineage"] = dg.TableColumnLineage(
+                deps_by_column=deps_by_column
+            )
+        except AttributeError as e:
+            logger.warning(
+                f"No field named {self.asset_name} found in data contract.\n{e}"
+            )
 
         server_information = get_server_information(
             self.data_contract_specification,
@@ -158,3 +180,55 @@ class DataContractLoader:
         )
 
         return freshness_checks
+
+    def combine_asset_specs(
+        self,
+        asset_spec: dg.AssetSpec,
+    ) -> dg.AssetSpec:
+        """Merge the given AssetSpec with the current object's attributes to produce a new AssetSpec.
+
+        This method combines metadata, descriptions, code versions, owners, and tags from the
+        provided `asset_spec` and the current instance. Preference is generally given to the
+        current instance's values where appropriate. Fields like dependencies, skippability,
+        group name, automation condition, kinds, and partition definitions are taken directly
+        from the input `asset_spec`.
+
+        Args:
+            asset_spec (dg.AssetSpec): The base asset specification to merge with the current one.
+
+        Returns:
+            dg.AssetSpec: A new AssetSpec instance containing the combined data.
+
+        Notes:
+            - Descriptions are joined with double newlines (`"\n\n"`).
+            - Code versions are joined with an underscore (`"_"`).
+            - Owners are concatenated.
+            - Metadata and tags are merged with the current instance taking precedence.
+        """
+        description = combine_parts(
+            [asset_spec.description, self.description], delimiter="\n\n"
+        )
+        metadata = {
+            **asset_spec.metadata,
+            **self.metadata,
+        }
+        code_version = combine_parts(
+            [asset_spec.code_version, self.version], delimiter="_"
+        )
+        owners = list(asset_spec.owners) + self.owner
+        tags = {**asset_spec.tags, **self.tags}
+
+        return dg.AssetSpec(
+            key=self.asset_name,
+            deps=asset_spec.deps,
+            description=description,
+            metadata=metadata,
+            skippable=asset_spec.skippable,
+            group_name=asset_spec.group_name,
+            code_version=code_version,
+            automation_condition=asset_spec.automation_condition,
+            owners=owners,
+            tags=tags,
+            kinds=asset_spec.kinds,
+            partitions_def=asset_spec.partitions_def,
+        )

dagster_datacontract-0.5.0/dagster_datacontract/utils/__init__.py

@@ -0,0 +1,8 @@
+from dagster_datacontract.utils.combine_strings import combine_parts
+from dagster_datacontract.utils.paths import get_absolute_path, normalize_path
+
+__all__ = [
+    "combine_parts",
+    "get_absolute_path",
+    "normalize_path",
+]

dagster_datacontract-0.5.0/dagster_datacontract/utils/combine_strings.py

@@ -0,0 +1,29 @@
+from collections.abc import Iterable
+
+
+def combine_parts(parts: Iterable[str | None], delimiter: str = "_") -> str:
+    """
+    Combine multiple optional strings using a specified delimiter.
+
+    This function takes an iterable of optional strings and joins the non-None,
+    non-empty strings using the given delimiter. None values and empty strings
+    are ignored. If all values are None or empty, the result is an empty string.
+
+    Args:
+        parts (Iterable[Optional[str]]): An iterable of strings or None values to combine.
+        delimiter (str): A string used to separate the non-None parts. Defaults to "_".
+
+    Returns:
+        str: A single combined string of all non-None, non-empty parts separated by the delimiter.
+
+    Examples:
+        >>> combine_parts(["v1", "2023", None])
+        'v1_2023'
+
+        >>> combine_parts([None, None])
+        ''
+
+        >>> combine_parts(["", "alpha", None])
+        'alpha'
+    """
+    return delimiter.join(filter(None, parts))

dagster_datacontract-0.4.0/dagster_datacontract/utils/__init__.py → dagster_datacontract-0.5.0/dagster_datacontract/utils/paths.py

@@ -1,5 +1,8 @@
 import os
-import urllib.parse
+from pathlib import Path
+from urllib.parse import urlparse
+
+import dagster as dg
 
 
 def normalize_path(path: str) -> str:
@@ -19,10 +22,30 @@ def normalize_path(path: str) -> str:
              - If the input is a local path or has a "file" scheme, returns it in the form "file:///absolute/path".
              - If the input has another scheme (e.g., "s3://", "http://"), returns it unchanged.
     """
-    parsed = urllib.parse.urlparse(path)
+    parsed = urlparse(path)
 
     if not parsed.scheme or parsed.scheme == "file":
         full_path = os.path.abspath(os.path.expanduser(path))
         return f"file://{full_path}"
     else:
         return path
+
+
+def get_absolute_path(
+    context_path: Path,
+    full_path: str,
+) -> Path:
+    """TODO."""
+    if isinstance(full_path, dg.UrlMetadataValue):
+        full_path = full_path.url
+
+    parsed_path = urlparse(full_path)
+    if parsed_path.scheme == "file":
+        full_path = Path(parsed_path.path)
+    else:
+        full_path = Path(full_path)
+
+    if full_path.is_absolute():
+        return full_path
+
+    return Path(context_path, full_path).absolute()

{dagster_datacontract-0.4.0 → dagster_datacontract-0.5.0}/dagster_datacontract.egg-info/PKG-INFO

@@ -1,13 +1,13 @@
 Metadata-Version: 2.4
 Name: dagster-datacontract
-Version: 0.4.0
+Version: 0.5.0
 Summary: Load metadata and asset check spesifications from data contracts.
 Author-email: Fredrik Bakken <fredrik@dataheim.io>
-Requires-Python: >=3.10.0
+Requires-Python: >=3.10
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: dagster>=1.10.10
-Requires-Dist: datacontract-cli>=0.10.23
+Requires-Dist: datacontract-cli>=0.10.24
 Requires-Dist: loguru>=0.7.3
 Dynamic: license-file
 
@@ -23,9 +23,9 @@ Install `dagster-datacontract` using [`uv`](https://github.com/astral-sh/uv):
 uv add dagster-datacontract
 ```
 
-## Simple Example
+## Example
 
-The following example can be found inside the [example](https://github.com/dataheim-io/dagster-datacontract/tree/main/example) directory:
+The following example can be found inside the [examples](https://github.com/dataheim-io/dagster-datacontract/tree/main/examples)-directory:
 
 ```python
 from datetime import timedelta
@@ -39,10 +39,11 @@ from datacontract.data_contract import DataContract
 from dagster_datacontract import DataContractLoader
 
 asset_name = "yellow_taxi_trip_records"
+examples_path = Path(__file__).resolve().parent.parent
 data_contract = DataContractLoader(
     asset_name=asset_name,
     data_contract=DataContract(
-        data_contract_file="./example/datacontract.yml",
+        data_contract_file=str(examples_path / "datacontracts" / "datacontract.yml"),
         server="production",
     ),
 )
@@ -59,7 +60,7 @@ data_contract = DataContractLoader(
 def yellow_taxi_trip_records(
     context: dg.AssetExecutionContext,
 ) -> None:
-    download_path = "./example/data"
+    download_path = examples_path.parent.parent / "data"
     Path(download_path).mkdir(parents=True, exist_ok=True)
 
     url = "https://d37ci6vzurychx.cloudfront.net/trip-data/yellow_tripdata_2025-01.parquet"

{dagster_datacontract-0.4.0 → dagster_datacontract-0.5.0}/dagster_datacontract.egg-info/SOURCES.txt

@@ -16,4 +16,6 @@ dagster_datacontract/metadata/table_colums.py
 dagster_datacontract/owners/__init__.py
 dagster_datacontract/tags/__init__.py
 dagster_datacontract/tags/tags.py
-dagster_datacontract/utils/__init__.py
+dagster_datacontract/utils/__init__.py
+dagster_datacontract/utils/combine_strings.py
+dagster_datacontract/utils/paths.py

{dagster_datacontract-0.4.0 → dagster_datacontract-0.5.0}/dagster_datacontract.egg-info/requires.txt

@@ -1,3 +1,3 @@
 dagster>=1.10.10
-datacontract-cli>=0.10.23
+datacontract-cli>=0.10.24
 loguru>=0.7.3

{dagster_datacontract-0.4.0 → dagster_datacontract-0.5.0}/pyproject.toml

@@ -1,30 +1,36 @@
 [project]
 name = "dagster-datacontract"
-version = "0.4.0"
+version = "0.5.0"
 description = "Load metadata and asset check spesifications from data contracts."
 authors = [
     { name = "Fredrik Bakken", email = "fredrik@dataheim.io" }
 ]
 readme = "README.md"
-requires-python = ">=3.10.0"
+requires-python = ">=3.10"
 dependencies = [
     "dagster>=1.10.10",
-    "datacontract-cli>=0.10.23",
+    "datacontract-cli>=0.10.24",
     "loguru>=0.7.3",
 ]
 
 [dependency-groups]
 dev = [
+    "dagster-dg-cli[local]>=1.10.18",
     "dagster-webserver>=1.10.10",
     "polars>=1.27.1",
     "pre-commit>=4.2.0",
-    "ruff>=0.11.5",
+    "ruff>=0.11.6",
 ]
 
 [build-system]
 requires = ["setuptools"]
 build-backend = "setuptools.build_meta"
 
+[tool.setuptools.packages.find]
+where = ["."]
+include = ["dagster_datacontract*"]
+exclude = ["examples*", "tests*"]
+
 [tool.ruff.lint]
 extend-select = [
     "UP", # pyupgrade