acryl-datahub 1.0.0.1rc7__py3-none-any.whl → 1.0.0.2__py3-none-any.whl

datahub/metadata/schemas/MetadataChangeLog.avsc

@@ -187,6 +187,9 @@
         "null",
         {
           "type": "record",
+          "Aspect": {
+            "name": "systemMetadata"
+          },
           "name": "SystemMetadata",
           "namespace": "com.linkedin.pegasus2avro.mxe",
           "fields": [

datahub/metadata/schemas/MetadataChangeProposal.avsc

@@ -187,6 +187,9 @@
         "null",
         {
           "type": "record",
+          "Aspect": {
+            "name": "systemMetadata"
+          },
           "name": "SystemMetadata",
           "namespace": "com.linkedin.pegasus2avro.mxe",
           "fields": [

datahub/metadata/schemas/QueryProperties.avsc

@@ -36,12 +36,14 @@
             "type": {
               "type": "enum",
               "symbolDocs": {
-                "SQL": "A SQL Query"
+                "SQL": "A SQL Query",
+                "UNKNOWN": "Unknown query language"
               },
               "name": "QueryLanguage",
               "namespace": "com.linkedin.pegasus2avro.query",
               "symbols": [
-                "SQL"
+                "SQL",
+                "UNKNOWN"
               ]
             },
             "name": "language",

datahub/metadata/schemas/SystemMetadata.avsc

@@ -0,0 +1,86 @@
+{
+  "type": "record",
+  "Aspect": {
+    "name": "systemMetadata"
+  },
+  "name": "SystemMetadata",
+  "namespace": "com.linkedin.pegasus2avro.mxe",
+  "fields": [
+    {
+      "type": [
+        "long",
+        "null"
+      ],
+      "name": "lastObserved",
+      "default": 0,
+      "doc": "The timestamp the metadata was observed at"
+    },
+    {
+      "type": [
+        "string",
+        "null"
+      ],
+      "name": "runId",
+      "default": "no-run-id-provided",
+      "doc": "The original run id that produced the metadata. Populated in case of batch-ingestion."
+    },
+    {
+      "type": [
+        "string",
+        "null"
+      ],
+      "name": "lastRunId",
+      "default": "no-run-id-provided",
+      "doc": "The last run id that produced the metadata. Populated in case of batch-ingestion."
+    },
+    {
+      "type": [
+        "null",
+        "string"
+      ],
+      "name": "pipelineName",
+      "default": null,
+      "doc": "The ingestion pipeline id that produced the metadata. Populated in case of batch ingestion."
+    },
+    {
+      "type": [
+        "null",
+        "string"
+      ],
+      "name": "registryName",
+      "default": null,
+      "doc": "The model registry name that was used to process this event"
+    },
+    {
+      "type": [
+        "null",
+        "string"
+      ],
+      "name": "registryVersion",
+      "default": null,
+      "doc": "The model registry version that was used to process this event"
+    },
+    {
+      "type": [
+        "null",
+        {
+          "type": "map",
+          "values": "string"
+        }
+      ],
+      "name": "properties",
+      "default": null,
+      "doc": "Additional properties"
+    },
+    {
+      "type": [
+        "null",
+        "string"
+      ],
+      "name": "version",
+      "default": null,
+      "doc": "Aspect version\n   Initial implementation will use the aspect version's number, however stored as\n   a string in the case where a different aspect versioning scheme is later adopted."
+    }
+  ],
+  "doc": "Metadata associated with each metadata change that is processed by the system"
+}

datahub/testing/mcp_diff.py

@@ -189,7 +189,7 @@ class MCPDiff:
         """
         aspect_diffs = [v for d in self.aspect_changes.values() for v in d.values()]
         for aspect_diff in aspect_diffs:
-            for _, old, new in aspect_diff.aspects_changed.keys():
+            for _, old, new in aspect_diff.aspects_changed:
                 golden[old.delta_info.idx] = new.delta_info.original
 
         indices_to_remove = set()

datahub/utilities/file_backed_collections.py

@@ -250,7 +250,7 @@ class FileBackedDict(MutableMapping[str, _VT], Closeable, Generic[_VT]):
                 rowid INTEGER PRIMARY KEY AUTOINCREMENT,
                 key TEXT UNIQUE,
                 value BLOB
-                {"".join(f", {column_name} BLOB" for column_name in self.extra_columns.keys())}
+                {"".join(f", {column_name} BLOB" for column_name in self.extra_columns)}
             )"""
         )
 
@@ -267,7 +267,7 @@ class FileBackedDict(MutableMapping[str, _VT], Closeable, Generic[_VT]):
         if self.indexes_created:
             return
         # The key column will automatically be indexed, but we need indexes for the extra columns.
-        for column_name in self.extra_columns.keys():
+        for column_name in self.extra_columns:
             self._conn.execute(
                 f"CREATE INDEX {self.tablename}_{column_name} ON {self.tablename} ({column_name})"
             )
@@ -305,12 +305,12 @@ class FileBackedDict(MutableMapping[str, _VT], Closeable, Generic[_VT]):
                 f"""INSERT INTO {self.tablename} (
                     key,
                     value
-                    {"".join(f", {column_name}" for column_name in self.extra_columns.keys())}
+                    {"".join(f", {column_name}" for column_name in self.extra_columns)}
                 )
                 VALUES ({", ".join(["?"] * (2 + len(self.extra_columns)))})
                 ON CONFLICT (key) DO UPDATE SET
                     value = excluded.value
-                    {"".join(f", {column_name} = excluded.{column_name}" for column_name in self.extra_columns.keys())}
+                    {"".join(f", {column_name} = excluded.{column_name}" for column_name in self.extra_columns)}
                 """,
                 items_to_write,
             )
@@ -321,7 +321,7 @@ class FileBackedDict(MutableMapping[str, _VT], Closeable, Generic[_VT]):
                         f"""INSERT INTO {self.tablename} (
                             key,
                             value
-                            {"".join(f", {column_name}" for column_name in self.extra_columns.keys())}
+                            {"".join(f", {column_name}" for column_name in self.extra_columns)}
                         )
                         VALUES ({", ".join(["?"] * (2 + len(self.extra_columns)))})""",
                         item,
@@ -330,7 +330,7 @@ class FileBackedDict(MutableMapping[str, _VT], Closeable, Generic[_VT]):
                     self._conn.execute(
                         f"""UPDATE {self.tablename} SET
                             value = ?
-                            {"".join(f", {column_name} = ?" for column_name in self.extra_columns.keys())}
+                            {"".join(f", {column_name} = ?" for column_name in self.extra_columns)}
                         WHERE key = ?""",
                         (*item[1:], item[0]),
                     )

datahub/utilities/hive_schema_to_avro.py

@@ -155,7 +155,7 @@ class HiveColumnToAvroConverter:
 
     @staticmethod
     def _parse_basic_datatype_string(s: str) -> Dict[str, object]:
-        if s in HiveColumnToAvroConverter._PRIVIMITE_HIVE_TYPE_TO_AVRO_TYPE.keys():
+        if s in HiveColumnToAvroConverter._PRIVIMITE_HIVE_TYPE_TO_AVRO_TYPE:
             return {
                 "type": HiveColumnToAvroConverter._PRIVIMITE_HIVE_TYPE_TO_AVRO_TYPE[s],
                 "native_data_type": s,
@@ -218,7 +218,7 @@ class HiveColumnToAvroConverter:
         buf = ""
         level = 0
         for c in s:
-            if c in HiveColumnToAvroConverter._BRACKETS.keys():
+            if c in HiveColumnToAvroConverter._BRACKETS:
                 level += 1
                 buf += c
             elif c in HiveColumnToAvroConverter._BRACKETS.values():

datahub/utilities/ingest_utils.py

@@ -32,10 +32,10 @@ def deploy_source_vars(
     name: Optional[str],
     config: str,
     urn: Optional[str],
-    executor_id: str,
+    executor_id: Optional[str],
     cli_version: Optional[str],
     schedule: Optional[str],
-    time_zone: str,
+    time_zone: Optional[str],
     extra_pip: Optional[str],
     debug: bool = False,
 ) -> dict:

datahub/ingestion/transformer/system_metadata_transformer.py

@@ -1,45 +0,0 @@
-import functools
-from typing import Iterable
-
-from datahub.emitter.mce_builder import get_sys_time
-from datahub.ingestion.api.common import PipelineContext, RecordEnvelope
-from datahub.ingestion.api.transform import Transformer
-from datahub.ingestion.api.workunit import MetadataWorkUnit
-from datahub.ingestion.transformer.auto_helper_transformer import AutoHelperTransformer
-from datahub.metadata.schema_classes import SystemMetadataClass
-
-
-def auto_system_metadata(
-    ctx: PipelineContext,
-    stream: Iterable[MetadataWorkUnit],
-) -> Iterable[MetadataWorkUnit]:
-    if not ctx.pipeline_config:
-        raise ValueError("Pipeline config is required for system metadata")
-    set_system_metadata = ctx.pipeline_config.flags.set_system_metadata
-    set_pipeline_name = ctx.pipeline_config.flags.set_system_metadata_pipeline_name
-
-    for workunit in stream:
-        if set_system_metadata:
-            workunit.metadata.systemMetadata = SystemMetadataClass(
-                lastObserved=get_sys_time(), runId=ctx.run_id
-            )
-            if set_pipeline_name:
-                workunit.metadata.systemMetadata.pipelineName = ctx.pipeline_name
-
-        yield workunit
-
-
-class SystemMetadataTransformer(Transformer):
-    def __init__(self, ctx: PipelineContext):
-        self._inner_transformer = AutoHelperTransformer(
-            functools.partial(auto_system_metadata, ctx)
-        )
-
-    def transform(
-        self, record_envelopes: Iterable[RecordEnvelope]
-    ) -> Iterable[RecordEnvelope]:
-        yield from self._inner_transformer.transform(record_envelopes)
-
-    @classmethod
-    def create(cls, config_dict: dict, ctx: PipelineContext) -> Transformer:
-        raise NotImplementedError(f"{cls.__name__} cannot be created from config")