acryl-datahub 0.15.0.1rc11__py3-none-any.whl → 0.15.0.1rc13__py3-none-any.whl

datahub/metadata/com/linkedin/pegasus2avro/common/__init__.py

@@ -19,6 +19,8 @@ from .....schema_classes import CostCostClass
 from .....schema_classes import CostCostDiscriminatorClass
 from .....schema_classes import CostTypeClass
 from .....schema_classes import DataPlatformInstanceClass
+from .....schema_classes import DataTransformClass
+from .....schema_classes import DataTransformLogicClass
 from .....schema_classes import DeprecationClass
 from .....schema_classes import DocumentationClass
 from .....schema_classes import DocumentationAssociationClass
@@ -79,6 +81,8 @@ CostCost = CostCostClass
 CostCostDiscriminator = CostCostDiscriminatorClass
 CostType = CostTypeClass
 DataPlatformInstance = DataPlatformInstanceClass
+DataTransform = DataTransformClass
+DataTransformLogic = DataTransformLogicClass
 Deprecation = DeprecationClass
 Documentation = DocumentationClass
 DocumentationAssociation = DocumentationAssociationClass

datahub/metadata/schema.avsc

@@ -400,6 +400,69 @@
     ],
     "doc": "Institutional memory of an entity. This is a way to link to relevant documentation and provide description of the documentation. Institutional or tribal knowledge is very important for users to leverage the entity."
   },
+  {
+    "type": "record",
+    "Aspect": {
+      "name": "dataTransformLogic"
+    },
+    "name": "DataTransformLogic",
+    "namespace": "com.linkedin.pegasus2avro.common",
+    "fields": [
+      {
+        "type": {
+          "type": "array",
+          "items": {
+            "type": "record",
+            "name": "DataTransform",
+            "namespace": "com.linkedin.pegasus2avro.common",
+            "fields": [
+              {
+                "type": [
+                  "null",
+                  {
+                    "type": "record",
+                    "name": "QueryStatement",
+                    "namespace": "com.linkedin.pegasus2avro.query",
+                    "fields": [
+                      {
+                        "type": "string",
+                        "name": "value",
+                        "doc": "The query text"
+                      },
+                      {
+                        "type": {
+                          "type": "enum",
+                          "symbolDocs": {
+                            "SQL": "A SQL Query"
+                          },
+                          "name": "QueryLanguage",
+                          "namespace": "com.linkedin.pegasus2avro.query",
+                          "symbols": [
+                            "SQL"
+                          ]
+                        },
+                        "name": "language",
+                        "default": "SQL",
+                        "doc": "The language of the Query, e.g. SQL."
+                      }
+                    ],
+                    "doc": "A query statement against one or more data assets."
+                  }
+                ],
+                "name": "queryStatement",
+                "default": null,
+                "doc": "The data transform may be defined by a query statement"
+              }
+            ],
+            "doc": "Information about a transformation. It may be a query,"
+          }
+        },
+        "name": "transforms",
+        "doc": "List of transformations applied"
+      }
+    ],
+    "doc": "Information about a Query against one or more data assets (e.g. Tables or Views)."
+  },
   {
     "type": "record",
     "Aspect": {
@@ -11947,35 +12010,7 @@
     "namespace": "com.linkedin.pegasus2avro.query",
     "fields": [
       {
-        "type": {
-          "type": "record",
-          "name": "QueryStatement",
-          "namespace": "com.linkedin.pegasus2avro.query",
-          "fields": [
-            {
-              "type": "string",
-              "name": "value",
-              "doc": "The query text"
-            },
-            {
-              "type": {
-                "type": "enum",
-                "symbolDocs": {
-                  "SQL": "A SQL Query"
-                },
-                "name": "QueryLanguage",
-                "namespace": "com.linkedin.pegasus2avro.query",
-                "symbols": [
-                  "SQL"
-                ]
-              },
-              "name": "language",
-              "default": "SQL",
-              "doc": "The language of the Query, e.g. SQL."
-            }
-          ],
-          "doc": "A query statement against one or more data assets."
-        },
+        "type": "com.linkedin.pegasus2avro.query.QueryStatement",
         "name": "statement",
         "doc": "The Query Statement."
       },

datahub/metadata/schemas/DataJobKey.avsc

@@ -25,7 +25,8 @@
       "forms",
       "subTypes",
       "incidentsSummary",
-      "testResults"
+      "testResults",
+      "dataTransformLogic"
     ]
   },
   "name": "DataJobKey",

datahub/metadata/schemas/DataTransformLogic.avsc

@@ -0,0 +1,63 @@
+{
+  "type": "record",
+  "Aspect": {
+    "name": "dataTransformLogic"
+  },
+  "name": "DataTransformLogic",
+  "namespace": "com.linkedin.pegasus2avro.common",
+  "fields": [
+    {
+      "type": {
+        "type": "array",
+        "items": {
+          "type": "record",
+          "name": "DataTransform",
+          "namespace": "com.linkedin.pegasus2avro.common",
+          "fields": [
+            {
+              "type": [
+                "null",
+                {
+                  "type": "record",
+                  "name": "QueryStatement",
+                  "namespace": "com.linkedin.pegasus2avro.query",
+                  "fields": [
+                    {
+                      "type": "string",
+                      "name": "value",
+                      "doc": "The query text"
+                    },
+                    {
+                      "type": {
+                        "type": "enum",
+                        "symbolDocs": {
+                          "SQL": "A SQL Query"
+                        },
+                        "name": "QueryLanguage",
+                        "namespace": "com.linkedin.pegasus2avro.query",
+                        "symbols": [
+                          "SQL"
+                        ]
+                      },
+                      "name": "language",
+                      "default": "SQL",
+                      "doc": "The language of the Query, e.g. SQL."
+                    }
+                  ],
+                  "doc": "A query statement against one or more data assets."
+                }
+              ],
+              "name": "queryStatement",
+              "default": null,
+              "doc": "The data transform may be defined by a query statement"
+            }
+          ],
+          "doc": "Information about a transformation. It may be a query,"
+        }
+      },
+      "name": "transforms",
+      "doc": "List of transformations applied"
+    }
+  ],
+  "doc": "Information about a Query against one or more data assets (e.g. Tables or Views)."
+}

datahub/utilities/time.py

@@ -1,6 +1,8 @@
 import time
 from dataclasses import dataclass
-from datetime import datetime, timezone
+from datetime import datetime
+
+from datahub.emitter.mce_builder import make_ts_millis, parse_ts_millis
 
 
 def get_current_time_in_seconds() -> int:
@@ -9,12 +11,15 @@ def get_current_time_in_seconds() -> int:
 
 def ts_millis_to_datetime(ts_millis: int) -> datetime:
     """Converts input timestamp in milliseconds to a datetime object with UTC timezone"""
-    return datetime.fromtimestamp(ts_millis / 1000, tz=timezone.utc)
+    return parse_ts_millis(ts_millis)
 
 
 def datetime_to_ts_millis(dt: datetime) -> int:
     """Converts a datetime object to timestamp in milliseconds"""
-    return int(round(dt.timestamp() * 1000))
+    # TODO: Deprecate these helpers in favor of make_ts_millis and parse_ts_millis.
+    # The other ones support None with a typing overload.
+    # Also possibly move those helpers to this file.
+    return make_ts_millis(dt)
 
 
 @dataclass

datahub/utilities/urns/_urn_base.py

@@ -1,9 +1,10 @@
 import functools
 import urllib.parse
 from abc import abstractmethod
-from typing import ClassVar, Dict, List, Optional, Type, TypeVar
+from typing import ClassVar, Dict, List, Optional, Type
 
 from deprecated import deprecated
+from typing_extensions import Self
 
 from datahub.utilities.urns.error import InvalidUrnError
 
@@ -42,9 +43,6 @@ def _split_entity_id(entity_id: str) -> List[str]:
     return parts
 
 
-_UrnSelf = TypeVar("_UrnSelf", bound="Urn")
-
-
 @functools.total_ordering
 class Urn:
     """
@@ -88,7 +86,7 @@ class Urn:
         return self._entity_ids
 
     @classmethod
-    def from_string(cls: Type[_UrnSelf], urn_str: str) -> "_UrnSelf":
+    def from_string(cls, urn_str: str) -> Self:
         """
         Creates an Urn from its string representation.
 
@@ -174,7 +172,7 @@ class Urn:
 
     @classmethod
     @deprecated(reason="prefer .from_string")
-    def create_from_string(cls: Type[_UrnSelf], urn_str: str) -> "_UrnSelf":
+    def create_from_string(cls, urn_str: str) -> Self:
         return cls.from_string(urn_str)
 
     @deprecated(reason="prefer .entity_ids")
@@ -270,5 +268,5 @@ class _SpecificUrn(Urn):
 
     @classmethod
     @abstractmethod
-    def _parse_ids(cls: Type[_UrnSelf], entity_ids: List[str]) -> _UrnSelf:
+    def _parse_ids(cls, entity_ids: List[str]) -> Self:
         raise NotImplementedError()