dagster-datacontract 0.2.5__py3-none-any.whl → 0.3.0__py3-none-any.whl

dagster_datacontract/__init__.py

@@ -1,4 +1,3 @@
-import textwrap
 from datetime import timedelta
 from typing import Any
 
@@ -8,8 +7,10 @@ from datacontract.data_contract import DataContract
 from datacontract.model.run import ResultEnum
 from loguru import logger
 
+from dagster_datacontract.description import get_description
 from dagster_datacontract.metadata import (
     get_column_lineage,
+    get_server_information,
     get_table_column,
 )
 from dagster_datacontract.tags import get_tags
@@ -29,7 +30,10 @@ class DataContractLoader:
         )
         self.metadata = self._load_metadata()
         self.tags = get_tags(self.data_contract_specification.tags)
-        self.description = self.load_description()
+        self.description = get_description(
+            self.asset_name,
+            self.data_contract_specification,
+        )
         self.owner = self._load_owner()
         self.version = self._load_version()
         self.cron_schedule = self._load_cron_schedule()
@@ -49,11 +53,18 @@ class DataContractLoader:
             table_column_lineage = get_column_lineage(column_field)
             deps_by_column[column_name] = table_column_lineage
 
+        server_information = get_server_information(
+            self.data_contract_specification,
+            self.data_contract._server,
+            self.asset_name,
+        )
+
         return {
             "dagster/column_schema": dg.TableSchema(columns=columns),
             "dagster/column_lineage": dg.TableColumnLineage(
                 deps_by_column=deps_by_column
             ),
+            **server_information,
         }
 
     def _load_owner(self) -> list[str] | None:
@@ -76,55 +87,6 @@ class DataContractLoader:
             logger.warning("'servicelevels.frequency.cron' not found in Data Contract.")
             return None
 
-    def load_description(
-        self,
-        config: dict[str, Any] | None = None,
-        separator: str = "\n",
-    ) -> str | None:
-        """Load and return a formatted description string based on the data contract specification.
-
-        This method composes a description by pulling text from different parts
-        of the data contract specification (e.g., model and info descriptions),
-        joining them using the specified separator.
-
-        Args:
-            config (dict[str, Any] | None, optional): A configuration dictionary
-                specifying the order in which to concatenate the description parts.
-                Defaults to `{"order": ["model", "info"]}`.
-            separator (str, optional): A string used to separate different parts
-                of the description. Defaults to a newline character (`"\n"`).
-
-        Returns:
-            str | None: A single string combining the specified description parts
-            if available, otherwise `None`.
-
-
-        Example:
-            >>> self.load_description()
-            'Model description...\nInfo description...'
-        """
-        default_config = {"order": ["model", "info"]}
-
-        configuration = default_config | (config or {})
-
-        descriptions = {
-            "model": self.data_contract_specification.models.get(
-                self.asset_name
-            ).description,
-            "info": self.data_contract_specification.info.description,
-        }
-
-        parts = []
-        for key in configuration["order"]:
-            desc = descriptions.get(key).replace("\n", f"{separator}\n")
-            if desc:
-                parts.append(textwrap.dedent(desc))
-
-        if parts:
-            return f"{separator}\n".join(parts)
-
-        return None
-
     def load_data_quality_checks(self) -> dg.AssetChecksDefinition:
         """Define and return a data quality check for the specified asset.
 
@@ -146,7 +108,11 @@ class DataContractLoader:
             blocking=True,
         )
         def check_asset():
-            run = self.data_contract.test()
+            data_contract = DataContract(
+                data_contract=self.data_contract_specification,
+                server=self.server_name,
+            )
+            run = data_contract.test()
 
             return dg.AssetCheckResult(
                 passed=run.result == ResultEnum.passed,

dagster_datacontract/description/__init__.py

@@ -0,0 +1,3 @@
+from dagster_datacontract.description.description import get_description
+
+__all__ = ["get_description"]

dagster_datacontract/description/description.py

@@ -0,0 +1,48 @@
+import textwrap
+from typing import Any
+
+from datacontract.data_contract import DataContractSpecification
+
+
+def get_description(
+    asset_name: str,
+    data_contract_specification: DataContractSpecification,
+    config: dict[str, Any] | None = None,
+    separator: str = "\n",
+) -> str | None:
+    """Load and return a formatted description string based on the data contract specification.
+
+    This method composes a description by pulling text from different parts
+    of the data contract specification (e.g., model and info descriptions),
+    joining them using the specified separator.
+
+    Args:
+        config (dict[str, Any] | None, optional): A configuration dictionary
+            specifying the order in which to concatenate the description parts.
+            Defaults to `{"order": ["model", "info"]}`.
+        separator (str, optional): A string used to separate different parts
+            of the description. Defaults to a newline character (`"\n"`).
+
+    Returns:
+        str | None: A single string combining the specified description parts
+        if available, otherwise `None`.
+    """
+    default_config = {"order": ["model", "info"]}
+
+    configuration = default_config | (config or {})
+
+    descriptions = {
+        "model": data_contract_specification.models.get(asset_name).description,
+        "info": data_contract_specification.info.description,
+    }
+
+    parts = []
+    for key in configuration["order"]:
+        desc = descriptions.get(key).replace("\n", f"{separator}\n")
+        if desc:
+            parts.append(textwrap.dedent(desc))
+
+    if parts:
+        return f"{separator}\n".join(parts)
+
+    return None

dagster_datacontract/metadata/__init__.py

@@ -1,6 +1,7 @@
+from dagster_datacontract.metadata.server_information import get_server_information
 from dagster_datacontract.metadata.table_colums import (
     get_column_lineage,
     get_table_column,
 )
 
-__all__ = ["get_table_column", "get_column_lineage"]
+__all__ = ["get_table_column", "get_column_lineage", "get_server_information"]

dagster_datacontract/metadata/server_information.py

@@ -0,0 +1,75 @@
+import os
+import urllib.parse
+
+from datacontract.data_contract import DataContractSpecification
+
+
+def _normalize_path(path: str) -> str:
+    parsed = urllib.parse.urlparse(path)
+
+    if not parsed.scheme or parsed.scheme == "file":
+        full_path = os.path.abspath(os.path.expanduser(path))
+        return f"file://{full_path}"
+    else:
+        return path
+
+
+def get_server_information(
+    data_contract_specification: DataContractSpecification,
+    server_name: str | None,
+    asset_name: str,
+) -> dict[str, str]:
+    server = data_contract_specification.servers.get(server_name)
+    if not server:
+        return {}
+
+    server_information = {}
+    match server.type:
+        case "azure":
+            server_information["dagster/uri"] = server.location
+        case "databricks":
+            server_information["dagster/uri"] = server.host
+            server_information["dagster/table_name"] = (
+                f"{server.catalog}.{server.schema}.{asset_name}"
+            )
+        case "kafka":
+            server_information["dagster/uri"] = server.host
+        case "kinesis":
+            server_information = {}
+        case "local":
+            server_information["dagster/uri"] = _normalize_path(server.path)
+        case "oracle":
+            server_information["dagster/uri"] = f"{server.host}:{server.port}"
+        case "postgres":
+            server_information["dagster/uri"] = f"{server.host}:{server.port}"
+            server_information["dagster/table_name"] = (
+                f"{server.database}.{server.schema}.{asset_name}"
+            )
+        case "pubsub":
+            server_information = {}
+        case "redshift":
+            server_information["dagster/uri"] = server.endpoint
+            server_information["dagster/table_name"] = (
+                f"{server.database}.{server.schema}.{asset_name}"
+            )
+        case "s3":
+            server_information["dagster/uri"] = server.location
+        case "sftp":
+            server_information["dagster/uri"] = server.location
+        case "snowflake":
+            server_information["dagster/table_name"] = (
+                f"{server.database}.{server.schema}.{asset_name}"
+            )
+        case "sqlserver":
+            server_information["dagster/table_name"] = (
+                f"{server.database}.{server.schema}.{asset_name}"
+            )
+        case "trino":
+            server_information["dagster/uri"] = f"{server.host}:{server.port}"
+            server_information["dagster/table_name"] = (
+                f"{server.catalog}.{server.schema}.{asset_name}"
+            )
+        case _:
+            server_information = {}
+
+    return server_information

dagster_datacontract/tags/__init__.py

@@ -1,3 +1,3 @@
-from dagster_datacontract.tags.load_tags import get_tags
+from dagster_datacontract.tags.tags import get_tags
 
 __all__ = ["get_tags"]

{dagster_datacontract-0.2.5.dist-info → dagster_datacontract-0.3.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dagster-datacontract
-Version: 0.2.5
+Version: 0.3.0
 Summary: Load metadata and asset check spesifications from data contracts.
 Author-email: Fredrik Bakken <fredrik@dataheim.io>
 Requires-Python: >=3.10.0

dagster_datacontract-0.3.0.dist-info/RECORD

@@ -0,0 +1,13 @@
+dagster_datacontract/__init__.py,sha256=UVpDzZYjmskEDJqEewMD52uWvu183O5RV9hjnP8mYAc,5488
+dagster_datacontract/description/__init__.py,sha256=ulWqPp5jIPvCzaDFZcjLjcDkljJ5j_FRsE0dXhK8Wlc,104
+dagster_datacontract/description/description.py,sha256=FmjgCYDpJ9UHrvAv0sAthfRohDjdG0lL1XcMKK8QMmI,1646
+dagster_datacontract/metadata/__init__.py,sha256=e-xmcWWoAhmKTwosshsxnyrjI1j-UyY6YpdpzA2ggF4,269
+dagster_datacontract/metadata/server_information.py,sha256=m1pv9sMfVjGIjzYVJ9R-KF-ABXCgAGKKH12dgAbm_jQ,2669
+dagster_datacontract/metadata/table_colums.py,sha256=Q7ZCiMReWU4-T2YfBvtt5vvoVXEoUgzK5OPMxQEgzpQ,4013
+dagster_datacontract/tags/__init__.py,sha256=2Ph-M0WbBKUjJWIzM_cEBW3SQZh7Nq8oy5MbD5bt_lc,76
+dagster_datacontract/tags/tags.py,sha256=aZ_HTkc-vjJ_rofT32fT_zrLCt9x1ZGn8XoihhOMhfU,1414
+dagster_datacontract-0.3.0.dist-info/licenses/LICENSE,sha256=9ULsEM1ICzCaGoso40plwO-d_SCQ7nsU6ZA4xgfaRq8,11338
+dagster_datacontract-0.3.0.dist-info/METADATA,sha256=1uuc5HEqV3OYmP7jNe9TZftEcirYwKbRXbnoAOe3pWQ,3029
+dagster_datacontract-0.3.0.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
+dagster_datacontract-0.3.0.dist-info/top_level.txt,sha256=_HUQ6OJ50Q0VZxEkdocTtxk1QkJpztb1QY7A0rcvtCE,21
+dagster_datacontract-0.3.0.dist-info/RECORD,,

dagster_datacontract-0.2.5.dist-info/RECORD

@@ -1,10 +0,0 @@
-dagster_datacontract/__init__.py,sha256=Be_sIJXm0gaX5lDuj5xk7FbM5hZsEmQTwHcsztmkEoo,6730
-dagster_datacontract/metadata/__init__.py,sha256=c5giTkxAgr-05ivY5hgsdVuA5UEDzdskVyjNczZdjHM,159
-dagster_datacontract/metadata/table_colums.py,sha256=Q7ZCiMReWU4-T2YfBvtt5vvoVXEoUgzK5OPMxQEgzpQ,4013
-dagster_datacontract/tags/__init__.py,sha256=jmHogoOunDs8YvgnXiMTSoCEGrxWbWnQCKH2_x2uQz8,81
-dagster_datacontract/tags/load_tags.py,sha256=aZ_HTkc-vjJ_rofT32fT_zrLCt9x1ZGn8XoihhOMhfU,1414
-dagster_datacontract-0.2.5.dist-info/licenses/LICENSE,sha256=9ULsEM1ICzCaGoso40plwO-d_SCQ7nsU6ZA4xgfaRq8,11338
-dagster_datacontract-0.2.5.dist-info/METADATA,sha256=RGX_hGPEYpNeXxF9clpc8oBoCx1XqX4H6mHwcQetL3I,3029
-dagster_datacontract-0.2.5.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
-dagster_datacontract-0.2.5.dist-info/top_level.txt,sha256=_HUQ6OJ50Q0VZxEkdocTtxk1QkJpztb1QY7A0rcvtCE,21
-dagster_datacontract-0.2.5.dist-info/RECORD,,