dagster-datacontract 0.2.5__py3-none-any.whl → 0.3.1__py3-none-any.whl

dagster_datacontract/__init__.py

@@ -1,4 +1,3 @@
-import textwrap
 from datetime import timedelta
 from typing import Any
 
@@ -8,11 +7,14 @@ from datacontract.data_contract import DataContract
 from datacontract.model.run import ResultEnum
 from loguru import logger
 
+from dagster_datacontract.description import get_description
 from dagster_datacontract.metadata import (
     get_column_lineage,
+    get_server_information,
     get_table_column,
 )
 from dagster_datacontract.tags import get_tags
+from dagster_datacontract.utils import normalize_path
 
 
 class DataContractLoader:
@@ -29,7 +31,10 @@ class DataContractLoader:
         )
         self.metadata = self._load_metadata()
         self.tags = get_tags(self.data_contract_specification.tags)
-        self.description = self.load_description()
+        self.description = get_description(
+            self.asset_name,
+            self.data_contract_specification,
+        )
         self.owner = self._load_owner()
         self.version = self._load_version()
         self.cron_schedule = self._load_cron_schedule()
@@ -37,6 +42,15 @@ class DataContractLoader:
     def _load_metadata(
         self,
     ) -> dict[str, TableColumnLineage | TableSchema | Any] | None:
+        metadata = (
+            {
+                "data contract path": dg.MetadataValue.url(
+                    normalize_path(self.data_contract._data_contract_file)
+                ),
+            }
+            if self.data_contract._data_contract_file
+            else {}
+        )
         columns = []
         deps_by_column = {}
 
@@ -49,12 +63,19 @@ class DataContractLoader:
             table_column_lineage = get_column_lineage(column_field)
             deps_by_column[column_name] = table_column_lineage
 
-        return {
-            "dagster/column_schema": dg.TableSchema(columns=columns),
-            "dagster/column_lineage": dg.TableColumnLineage(
-                deps_by_column=deps_by_column
-            ),
-        }
+        metadata["dagster/column_schema"] = dg.TableSchema(columns=columns)
+        metadata["dagster/column_lineage"] = dg.TableColumnLineage(
+            deps_by_column=deps_by_column
+        )
+
+        server_information = get_server_information(
+            self.data_contract_specification,
+            self.data_contract._server,
+            self.asset_name,
+        )
+        metadata.update(server_information)
+
+        return metadata
 
     def _load_owner(self) -> list[str] | None:
         owner = self.data_contract_specification.info.owner
@@ -76,55 +97,6 @@ class DataContractLoader:
             logger.warning("'servicelevels.frequency.cron' not found in Data Contract.")
             return None
 
-    def load_description(
-        self,
-        config: dict[str, Any] | None = None,
-        separator: str = "\n",
-    ) -> str | None:
-        """Load and return a formatted description string based on the data contract specification.
-
-        This method composes a description by pulling text from different parts
-        of the data contract specification (e.g., model and info descriptions),
-        joining them using the specified separator.
-
-        Args:
-            config (dict[str, Any] | None, optional): A configuration dictionary
-                specifying the order in which to concatenate the description parts.
-                Defaults to `{"order": ["model", "info"]}`.
-            separator (str, optional): A string used to separate different parts
-                of the description. Defaults to a newline character (`"\n"`).
-
-        Returns:
-            str | None: A single string combining the specified description parts
-            if available, otherwise `None`.
-
-
-        Example:
-            >>> self.load_description()
-            'Model description...\nInfo description...'
-        """
-        default_config = {"order": ["model", "info"]}
-
-        configuration = default_config | (config or {})
-
-        descriptions = {
-            "model": self.data_contract_specification.models.get(
-                self.asset_name
-            ).description,
-            "info": self.data_contract_specification.info.description,
-        }
-
-        parts = []
-        for key in configuration["order"]:
-            desc = descriptions.get(key).replace("\n", f"{separator}\n")
-            if desc:
-                parts.append(textwrap.dedent(desc))
-
-        if parts:
-            return f"{separator}\n".join(parts)
-
-        return None
-
     def load_data_quality_checks(self) -> dg.AssetChecksDefinition:
         """Define and return a data quality check for the specified asset.
 

dagster_datacontract/description/__init__.py

@@ -0,0 +1,3 @@
+from dagster_datacontract.description.description import get_description
+
+__all__ = ["get_description"]

dagster_datacontract/description/description.py

@@ -0,0 +1,48 @@
+import textwrap
+from typing import Any
+
+from datacontract.data_contract import DataContractSpecification
+
+
+def get_description(
+    asset_name: str,
+    data_contract_specification: DataContractSpecification,
+    config: dict[str, Any] | None = None,
+    separator: str = "\n",
+) -> str | None:
+    """Load and return a formatted description string based on the data contract specification.
+
+    This method composes a description by pulling text from different parts
+    of the data contract specification (e.g., model and info descriptions),
+    joining them using the specified separator.
+
+    Args:
+        config (dict[str, Any] | None, optional): A configuration dictionary
+            specifying the order in which to concatenate the description parts.
+            Defaults to `{"order": ["model", "info"]}`.
+        separator (str, optional): A string used to separate different parts
+            of the description. Defaults to a newline character (`"\n"`).
+
+    Returns:
+        str | None: A single string combining the specified description parts
+        if available, otherwise `None`.
+    """
+    default_config = {"order": ["model", "info"]}
+
+    configuration = default_config | (config or {})
+
+    descriptions = {
+        "model": data_contract_specification.models.get(asset_name).description,
+        "info": data_contract_specification.info.description,
+    }
+
+    parts = []
+    for key in configuration["order"]:
+        desc = descriptions.get(key).replace("\n", f"{separator}\n")
+        if desc:
+            parts.append(textwrap.dedent(desc))
+
+    if parts:
+        return f"{separator}\n".join(parts)
+
+    return None

dagster_datacontract/metadata/__init__.py

@@ -1,6 +1,7 @@
+from dagster_datacontract.metadata.server_information import get_server_information
 from dagster_datacontract.metadata.table_colums import (
     get_column_lineage,
     get_table_column,
 )
 
-__all__ = ["get_table_column", "get_column_lineage"]
+__all__ = ["get_table_column", "get_column_lineage", "get_server_information"]

dagster_datacontract/metadata/server_information.py

@@ -0,0 +1,84 @@
+from datacontract.data_contract import DataContractSpecification
+
+from dagster_datacontract.utils import normalize_path
+
+
+def get_server_information(
+    data_contract_specification: DataContractSpecification,
+    server_name: str | None,
+    asset_name: str,
+) -> dict[str, str]:
+    """Returns a dictionary containing server-specific information to be used
+    by Dagster for identifying asset locations or connections.
+
+    This function inspects the provided `DataContractSpecification` to locate
+    the specified server by name and constructs a dictionary with keys such as
+    "dagster/uri" and "dagster/table_name" depending on the server type.
+
+    Parameters:
+        data_contract_specification (DataContractSpecification):
+            The data contract specification containing server configurations.
+        server_name (str | None):
+            The name of the server to retrieve information for. If None or not found, returns an empty dict.
+        asset_name (str):
+            The name of the asset, used for constructing fully qualified table names for certain server types.
+
+    Returns:
+        dict[str, str]: A dictionary with keys like "dagster/uri" and/or "dagster/table_name"
+        depending on the server type. Returns an empty dictionary if the server is not found
+        or if the server type is not recognized or unsupported.
+    """
+    server = data_contract_specification.servers.get(server_name)
+    if not server:
+        return {}
+
+    server_information = {}
+    match server.type:
+        case "azure":
+            server_information["dagster/uri"] = server.location
+        case "databricks":
+            server_information["dagster/uri"] = server.host
+            server_information["dagster/table_name"] = (
+                f"{server.catalog}.{server.schema}.{asset_name}"
+            )
+        case "kafka":
+            server_information["dagster/uri"] = server.host
+        case "kinesis":
+            server_information = {}
+        case "local":
+            server_information["dagster/uri"] = normalize_path(server.path)
+        case "oracle":
+            server_information["dagster/uri"] = f"{server.host}:{server.port}"
+        case "postgres":
+            server_information["dagster/uri"] = f"{server.host}:{server.port}"
+            server_information["dagster/table_name"] = (
+                f"{server.database}.{server.schema}.{asset_name}"
+            )
+        case "pubsub":
+            server_information = {}
+        case "redshift":
+            server_information["dagster/uri"] = server.endpoint
+            server_information["dagster/table_name"] = (
+                f"{server.database}.{server.schema}.{asset_name}"
+            )
+        case "s3":
+            server_information["dagster/uri"] = server.location
+        case "sftp":
+            server_information["dagster/uri"] = server.location
+        case "snowflake":
+            server_information["dagster/table_name"] = (
+                f"{server.database}.{server.schema}.{asset_name}"
+            )
+        case "sqlserver":
+            server_information["dagster/table_name"] = (
+                f"{server.database}.{server.schema}.{asset_name}"
+            )
+        case "trino":
+            server_information["dagster/uri"] = f"{server.host}:{server.port}"
+            server_information["dagster/table_name"] = (
+                f"{server.catalog}.{server.schema}.{asset_name}"
+            )
+        case _:
+            server_information = {}
+
+    return server_information

dagster_datacontract/tags/__init__.py

@@ -1,3 +1,3 @@
-from dagster_datacontract.tags.load_tags import get_tags
+from dagster_datacontract.tags.tags import get_tags
 
 __all__ = ["get_tags"]

dagster_datacontract/utils/__init__.py

@@ -0,0 +1,28 @@
+import os
+import urllib.parse
+
+
+def normalize_path(path: str) -> str:
+    """Normalizes a file path to ensure it is returned in a consistent URI format.
+
+    This function checks if the provided path is a local file path (with no scheme
+    or with the 'file' scheme) and converts it into a fully qualified file URI.
+    If the path already has a non-'file' scheme (e.g., 's3://', 'http://'),
+    it is returned unchanged.
+
+    Parameters:
+        path (str): The input file path. This can be a relative or absolute local path,
+                    a path starting with `~`, or a URI with a supported scheme.
+
+    Returns:
+        str: A normalized path string:
+             - If the input is a local path or has a "file" scheme, returns it in the form "file:///absolute/path".
+             - If the input has another scheme (e.g., "s3://", "http://"), returns it unchanged.
+    """
+    parsed = urllib.parse.urlparse(path)
+
+    if not parsed.scheme or parsed.scheme == "file":
+        full_path = os.path.abspath(os.path.expanduser(path))
+        return f"file://{full_path}"
+    else:
+        return path

{dagster_datacontract-0.2.5.dist-info → dagster_datacontract-0.3.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dagster-datacontract
-Version: 0.2.5
+Version: 0.3.1
 Summary: Load metadata and asset check spesifications from data contracts.
 Author-email: Fredrik Bakken <fredrik@dataheim.io>
 Requires-Python: >=3.10.0

dagster_datacontract-0.3.1.dist-info/RECORD

@@ -0,0 +1,14 @@
+dagster_datacontract/__init__.py,sha256=Pzq4AblLnDhW0QBmV8ntV5KLD0a7D0-mA2Lw9hjS_V8,5685
+dagster_datacontract/description/__init__.py,sha256=ulWqPp5jIPvCzaDFZcjLjcDkljJ5j_FRsE0dXhK8Wlc,104
+dagster_datacontract/description/description.py,sha256=FmjgCYDpJ9UHrvAv0sAthfRohDjdG0lL1XcMKK8QMmI,1646
+dagster_datacontract/metadata/__init__.py,sha256=e-xmcWWoAhmKTwosshsxnyrjI1j-UyY6YpdpzA2ggF4,269
+dagster_datacontract/metadata/server_information.py,sha256=jk_H8aI5PdGzIeoYThlhhZMSOtBh-6xc8QAFz0BFesU,3512
+dagster_datacontract/metadata/table_colums.py,sha256=Q7ZCiMReWU4-T2YfBvtt5vvoVXEoUgzK5OPMxQEgzpQ,4013
+dagster_datacontract/tags/__init__.py,sha256=2Ph-M0WbBKUjJWIzM_cEBW3SQZh7Nq8oy5MbD5bt_lc,76
+dagster_datacontract/tags/tags.py,sha256=aZ_HTkc-vjJ_rofT32fT_zrLCt9x1ZGn8XoihhOMhfU,1414
+dagster_datacontract/utils/__init__.py,sha256=Zfbuf20Eorf7BD8gSDASiPqgPwWu8Mz03r3aa2zE4NA,1106
+dagster_datacontract-0.3.1.dist-info/licenses/LICENSE,sha256=9ULsEM1ICzCaGoso40plwO-d_SCQ7nsU6ZA4xgfaRq8,11338
+dagster_datacontract-0.3.1.dist-info/METADATA,sha256=P98oLsqB-GCJz4uanki-xtG3UnZCHD0biDMUYIZMEhc,3029
+dagster_datacontract-0.3.1.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
+dagster_datacontract-0.3.1.dist-info/top_level.txt,sha256=_HUQ6OJ50Q0VZxEkdocTtxk1QkJpztb1QY7A0rcvtCE,21
+dagster_datacontract-0.3.1.dist-info/RECORD,,

dagster_datacontract-0.2.5.dist-info/RECORD

@@ -1,10 +0,0 @@
-dagster_datacontract/__init__.py,sha256=Be_sIJXm0gaX5lDuj5xk7FbM5hZsEmQTwHcsztmkEoo,6730
-dagster_datacontract/metadata/__init__.py,sha256=c5giTkxAgr-05ivY5hgsdVuA5UEDzdskVyjNczZdjHM,159
-dagster_datacontract/metadata/table_colums.py,sha256=Q7ZCiMReWU4-T2YfBvtt5vvoVXEoUgzK5OPMxQEgzpQ,4013
-dagster_datacontract/tags/__init__.py,sha256=jmHogoOunDs8YvgnXiMTSoCEGrxWbWnQCKH2_x2uQz8,81
-dagster_datacontract/tags/load_tags.py,sha256=aZ_HTkc-vjJ_rofT32fT_zrLCt9x1ZGn8XoihhOMhfU,1414
-dagster_datacontract-0.2.5.dist-info/licenses/LICENSE,sha256=9ULsEM1ICzCaGoso40plwO-d_SCQ7nsU6ZA4xgfaRq8,11338
-dagster_datacontract-0.2.5.dist-info/METADATA,sha256=RGX_hGPEYpNeXxF9clpc8oBoCx1XqX4H6mHwcQetL3I,3029
-dagster_datacontract-0.2.5.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
-dagster_datacontract-0.2.5.dist-info/top_level.txt,sha256=_HUQ6OJ50Q0VZxEkdocTtxk1QkJpztb1QY7A0rcvtCE,21
-dagster_datacontract-0.2.5.dist-info/RECORD,,