dagster-datacontract 0.3.0__tar.gz → 0.4.0__tar.gz

{dagster_datacontract-0.3.0 → dagster_datacontract-0.4.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dagster-datacontract
-Version: 0.3.0
+Version: 0.4.0
 Summary: Load metadata and asset check spesifications from data contracts.
 Author-email: Fredrik Bakken <fredrik@dataheim.io>
 Requires-Python: >=3.10.0

{dagster_datacontract-0.3.0 → dagster_datacontract-0.4.0}/dagster_datacontract/__init__.py

@@ -10,10 +10,13 @@ from loguru import logger
 from dagster_datacontract.description import get_description
 from dagster_datacontract.metadata import (
     get_column_lineage,
+    get_links,
     get_server_information,
     get_table_column,
 )
+from dagster_datacontract.owners import get_owner
 from dagster_datacontract.tags import get_tags
+from dagster_datacontract.utils import normalize_path
 
 
 class DataContractLoader:
@@ -34,13 +37,22 @@ class DataContractLoader:
             self.asset_name,
             self.data_contract_specification,
         )
-        self.owner = self._load_owner()
+        self.owner = get_owner(self.data_contract_specification)
         self.version = self._load_version()
         self.cron_schedule = self._load_cron_schedule()
 
     def _load_metadata(
         self,
     ) -> dict[str, TableColumnLineage | TableSchema | Any] | None:
+        metadata = (
+            {
+                "datacontract/path": dg.MetadataValue.url(
+                    normalize_path(self.data_contract._data_contract_file)
+                ),
+            }
+            if self.data_contract._data_contract_file
+            else {}
+        )
         columns = []
         deps_by_column = {}
 
@@ -53,24 +65,22 @@ class DataContractLoader:
             table_column_lineage = get_column_lineage(column_field)
             deps_by_column[column_name] = table_column_lineage
 
+        metadata["dagster/column_schema"] = dg.TableSchema(columns=columns)
+        metadata["dagster/column_lineage"] = dg.TableColumnLineage(
+            deps_by_column=deps_by_column
+        )
+
         server_information = get_server_information(
             self.data_contract_specification,
             self.data_contract._server,
             self.asset_name,
         )
+        metadata.update(server_information)
 
-        return {
-            "dagster/column_schema": dg.TableSchema(columns=columns),
-            "dagster/column_lineage": dg.TableColumnLineage(
-                deps_by_column=deps_by_column
-            ),
-            **server_information,
-        }
+        links = get_links(self.data_contract_specification.links)
+        metadata.update(links)
 
-    def _load_owner(self) -> list[str] | None:
-        owner = self.data_contract_specification.info.owner
-
-        return [f"team:{owner}"] if owner else None
+        return metadata
 
     def _load_version(self) -> str | None:
         version = self.data_contract_specification.info.version
@@ -108,11 +118,7 @@ class DataContractLoader:
             blocking=True,
         )
         def check_asset():
-            data_contract = DataContract(
-                data_contract=self.data_contract_specification,
-                server=self.server_name,
-            )
-            run = data_contract.test()
+            run = self.data_contract.test()
 
             return dg.AssetCheckResult(
                 passed=run.result == ResultEnum.passed,

{dagster_datacontract-0.3.0 → dagster_datacontract-0.4.0}/dagster_datacontract/metadata/__init__.py

@@ -1,7 +1,13 @@
+from dagster_datacontract.metadata.links import get_links
 from dagster_datacontract.metadata.server_information import get_server_information
 from dagster_datacontract.metadata.table_colums import (
     get_column_lineage,
     get_table_column,
 )
 
-__all__ = ["get_table_column", "get_column_lineage", "get_server_information"]
+__all__ = [
+    "get_column_lineage",
+    "get_links",
+    "get_table_column",
+    "get_server_information",
+]

dagster_datacontract-0.4.0/dagster_datacontract/metadata/links.py

@@ -0,0 +1,17 @@
+import dagster as dg
+
+
+def get_links(links: dict[str, str]) -> dict[str, str]:
+    """Return a dictionary with keys prefixed by 'link/' and values as Dagster URL metadata.
+
+    Args:
+        links (dict[str, str]): A dictionary where each key is a name/label and each
+            value is a URL string.
+
+    Returns:
+        dict[str, str]: A dictionary where each key is prefixed with 'link/' and
+            each value is a `MetadataValue.url`.
+    """
+    links = {f"link/{key}": dg.MetadataValue.url(value) for key, value in links.items()}
+
+    return links

dagster_datacontract-0.4.0/dagster_datacontract/metadata/server_information.py

@@ -0,0 +1,114 @@
+from datacontract.data_contract import DataContractSpecification
+
+from dagster_datacontract.utils import normalize_path
+
+
+def get_server_information(
+    data_contract_specification: DataContractSpecification,
+    server_name: str | None,
+    asset_name: str,
+) -> dict[str, str]:
+    """Returns a dictionary containing server-specific information to be used
+    by Dagster for identifying asset locations or connections.
+
+    This function inspects the provided `DataContractSpecification` to locate
+    the specified server by name and constructs a dictionary with keys such as
+    "dagster/uri" and "dagster/table_name" depending on the server type.
+
+    Server information can be obtained from: https://datacontract.com/#server-object
+
+    Parameters:
+        data_contract_specification (DataContractSpecification):
+            The data contract specification containing server configurations.
+        server_name (str | None):
+            The name of the server to retrieve information for. If None or not found, returns an empty dict.
+        asset_name (str):
+            The name of the asset, used for constructing fully qualified table names for certain server types.
+
+    Returns:
+        dict[str, str]: A dictionary with keys like "dagster/uri" and/or "dagster/table_name"
+        depending on the server type. Returns an empty dictionary if the server is not found
+        or if the server type is not recognized or unsupported.
+    """
+    server = data_contract_specification.servers.get(server_name)
+    if not server:
+        return {}
+
+    server_information = {}
+    match server.type:
+        case "azure":
+            server_information["dagster/uri"] = server.location
+            server_information["azure/storage_account"] = server.storageAccount
+            server_information["file/format"] = server.format
+            server_information["file/delimiter"] = server.delimiter
+        case "bigquery":
+            server_information["bigquery/project"] = server.project
+            server_information["bigquery/dataset"] = server.dataset
+        case "databricks":
+            server_information["dagster/uri"] = server.host
+            server_information["dagster/table_name"] = (
+                f"{server.catalog}.{server.schema}.{asset_name}"
+            )
+        case "glue":
+            server_information = {}
+        case "kafka":
+            server_information["dagster/uri"] = server.host
+            server_information["kafka/topic"] = server.topic
+            server_information["kafka/format"] = server.format
+        case "kinesis":
+            server_information["kinesis/stream"] = server.stream
+            server_information["kinesis/region"] = server.region
+            server_information["kinesis/format"] = server.format
+        case "local":
+            server_information["dagster/uri"] = normalize_path(server.path)
+            server_information["file/format"] = server.format
+        case "oracle":
+            server_information["dagster/uri"] = f"{server.host}:{server.port}"
+            server_information["oracle/service_name"] = server.serviceName
+        case "postgres":
+            server_information["dagster/uri"] = f"{server.host}:{server.port}"
+            server_information["dagster/table_name"] = (
+                f"{server.database}.{server.schema}.{asset_name}"
+            )
+        case "pubsub":
+            server_information["pubsub/project"] = server.project
+            server_information["pubsub/topic"] = server.topic
+        case "redshift":
+            server_information["dagster/uri"] = server.endpoint
+            server_information["dagster/table_name"] = (
+                f"{server.database}.{server.schema}.{asset_name}"
+            )
+            server_information["redshift/account"] = server.account
+            server_information["redshift/host"] = server.host
+            server_information["redshift/port"] = server.port
+            server_information["redshift/cluster"] = server.clusterIdentifier
+        case "s3":
+            server_information["dagster/uri"] = server.location
+            server_information["s3/endpoint"] = server.endpointUrl
+            server_information["file/format"] = server.format
+            server_information["file/delimiter"] = server.delimiter
+        case "sftp":
+            server_information["dagster/uri"] = server.location
+            server_information["file/format"] = server.format
+            server_information["file/delimiter"] = server.delimiter
+        case "snowflake":
+            server_information["dagster/table_name"] = (
+                f"{server.database}.{server.schema}.{asset_name}"
+            )
+            server_information["snowflake/account"] = server.account
+        case "sqlserver":
+            server_information["dagster/table_name"] = (
+                f"{server.database}.{server.schema}.{asset_name}"
+            )
+            server_information["sqlserver/host"] = server.host
+            server_information["sqlserver/port"] = server.port
+            server_information["sqlserver/driver"] = server.driver
+        case "trino":
+            server_information["dagster/uri"] = f"{server.host}:{server.port}"
+            server_information["dagster/table_name"] = (
+                f"{server.catalog}.{server.schema}.{asset_name}"
+            )
+        case _:
+            server_information = {}
+
+    return server_information

dagster_datacontract-0.4.0/dagster_datacontract/owners/__init__.py

@@ -0,0 +1,23 @@
+from datacontract.data_contract import DataContractSpecification
+
+
+def get_owner(
+    data_contract_specification: DataContractSpecification,
+    is_team: bool = True,
+) -> list[str] | None:
+    """Return the owner of a data contract, optionally formatted as a team identifier.
+
+    Args:
+        data_contract_specification (DataContractSpecification): The data contract specification containing ownership metadata.
+        is_team (bool, optional): If True, formats the owner as a team identifier (e.g., 'team:owner').
+                                  If False, returns the raw owner string. Defaults to True.
+
+    Returns:
+        list[str] | None: A list containing the owner string, formatted depending on `is_team`, or None if no owner is found.
+    """
+    owner = data_contract_specification.info.owner
+
+    if is_team:
+        return [f"team:{owner}"]
+
+    return [owner]

dagster_datacontract-0.4.0/dagster_datacontract/utils/__init__.py

@@ -0,0 +1,28 @@
+import os
+import urllib.parse
+
+
+def normalize_path(path: str) -> str:
+    """Normalizes a file path to ensure it is returned in a consistent URI format.
+
+    This function checks if the provided path is a local file path (with no scheme
+    or with the 'file' scheme) and converts it into a fully qualified file URI.
+    If the path already has a non-'file' scheme (e.g., 's3://', 'http://'),
+    it is returned unchanged.
+
+    Parameters:
+        path (str): The input file path. This can be a relative or absolute local path,
+                    a path starting with `~`, or a URI with a supported scheme.
+
+    Returns:
+        str: A normalized path string:
+             - If the input is a local path or has a "file" scheme, returns it in the form "file:///absolute/path".
+             - If the input has another scheme (e.g., "s3://", "http://"), returns it unchanged.
+    """
+    parsed = urllib.parse.urlparse(path)
+
+    if not parsed.scheme or parsed.scheme == "file":
+        full_path = os.path.abspath(os.path.expanduser(path))
+        return f"file://{full_path}"
+    else:
+        return path

{dagster_datacontract-0.3.0 → dagster_datacontract-0.4.0}/dagster_datacontract.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dagster-datacontract
-Version: 0.3.0
+Version: 0.4.0
 Summary: Load metadata and asset check spesifications from data contracts.
 Author-email: Fredrik Bakken <fredrik@dataheim.io>
 Requires-Python: >=3.10.0

{dagster_datacontract-0.3.0 → dagster_datacontract-0.4.0}/dagster_datacontract.egg-info/SOURCES.txt

@@ -10,7 +10,10 @@ dagster_datacontract.egg-info/top_level.txt
 dagster_datacontract/description/__init__.py
 dagster_datacontract/description/description.py
 dagster_datacontract/metadata/__init__.py
+dagster_datacontract/metadata/links.py
 dagster_datacontract/metadata/server_information.py
 dagster_datacontract/metadata/table_colums.py
+dagster_datacontract/owners/__init__.py
 dagster_datacontract/tags/__init__.py
-dagster_datacontract/tags/tags.py
+dagster_datacontract/tags/tags.py
+dagster_datacontract/utils/__init__.py

{dagster_datacontract-0.3.0 → dagster_datacontract-0.4.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "dagster-datacontract"
-version = "0.3.0"
+version = "0.4.0"
 description = "Load metadata and asset check spesifications from data contracts."
 authors = [
     { name = "Fredrik Bakken", email = "fredrik@dataheim.io" }

dagster_datacontract-0.3.0/dagster_datacontract/metadata/server_information.py

@@ -1,75 +0,0 @@
-import os
-import urllib.parse
-
-from datacontract.data_contract import DataContractSpecification
-
-
-def _normalize_path(path: str) -> str:
-    parsed = urllib.parse.urlparse(path)
-
-    if not parsed.scheme or parsed.scheme == "file":
-        full_path = os.path.abspath(os.path.expanduser(path))
-        return f"file://{full_path}"
-    else:
-        return path
-
-
-def get_server_information(
-    data_contract_specification: DataContractSpecification,
-    server_name: str | None,
-    asset_name: str,
-) -> dict[str, str]:
-    server = data_contract_specification.servers.get(server_name)
-    if not server:
-        return {}
-
-    server_information = {}
-    match server.type:
-        case "azure":
-            server_information["dagster/uri"] = server.location
-        case "databricks":
-            server_information["dagster/uri"] = server.host
-            server_information["dagster/table_name"] = (
-                f"{server.catalog}.{server.schema}.{asset_name}"
-            )
-        case "kafka":
-            server_information["dagster/uri"] = server.host
-        case "kinesis":
-            server_information = {}
-        case "local":
-            server_information["dagster/uri"] = _normalize_path(server.path)
-        case "oracle":
-            server_information["dagster/uri"] = f"{server.host}:{server.port}"
-        case "postgres":
-            server_information["dagster/uri"] = f"{server.host}:{server.port}"
-            server_information["dagster/table_name"] = (
-                f"{server.database}.{server.schema}.{asset_name}"
-            )
-        case "pubsub":
-            server_information = {}
-        case "redshift":
-            server_information["dagster/uri"] = server.endpoint
-            server_information["dagster/table_name"] = (
-                f"{server.database}.{server.schema}.{asset_name}"
-            )
-        case "s3":
-            server_information["dagster/uri"] = server.location
-        case "sftp":
-            server_information["dagster/uri"] = server.location
-        case "snowflake":
-            server_information["dagster/table_name"] = (
-                f"{server.database}.{server.schema}.{asset_name}"
-            )
-        case "sqlserver":
-            server_information["dagster/table_name"] = (
-                f"{server.database}.{server.schema}.{asset_name}"
-            )
-        case "trino":
-            server_information["dagster/uri"] = f"{server.host}:{server.port}"
-            server_information["dagster/table_name"] = (
-                f"{server.catalog}.{server.schema}.{asset_name}"
-            )
-        case _:
-            server_information = {}
-
-    return server_information