acryl-datahub 1.0.0rc5__py3-none-any.whl → 1.0.0rc6__py3-none-any.whl

datahub/metadata/schemas/IncidentInfo.avsc

@@ -108,7 +108,88 @@
       ],
       "name": "priority",
       "default": 0,
-      "doc": "A numeric severity or priority for the incident. On the UI we will translate this into something easy to understand."
+      "doc": "A numeric severity or priority for the incident. On the UI we will translate this into something easy to understand.\nCurrently supported: 0 - CRITICAL, 1 - HIGH, 2 - MED, 3 - LOW\n(We probably should have modeled as an enum)"
+    },
+    {
+      "type": [
+        "null",
+        {
+          "type": "array",
+          "items": {
+            "type": "record",
+            "name": "IncidentAssignee",
+            "namespace": "com.linkedin.pegasus2avro.incident",
+            "fields": [
+              {
+                "Searchable": {
+                  "addToFilters": true,
+                  "fieldName": "assignees",
+                  "filterNameOverride": "Assignee"
+                },
+                "java": {
+                  "class": "com.linkedin.pegasus2avro.common.urn.Urn"
+                },
+                "type": "string",
+                "name": "actor",
+                "doc": "The user or group assigned to the incident.",
+                "Urn": "Urn"
+              },
+              {
+                "type": {
+                  "type": "record",
+                  "name": "AuditStamp",
+                  "namespace": "com.linkedin.pegasus2avro.common",
+                  "fields": [
+                    {
+                      "type": "long",
+                      "name": "time",
+                      "doc": "When did the resource/association/sub-resource move into the specific lifecycle stage represented by this AuditEvent."
+                    },
+                    {
+                      "java": {
+                        "class": "com.linkedin.pegasus2avro.common.urn.Urn"
+                      },
+                      "type": "string",
+                      "name": "actor",
+                      "doc": "The entity (e.g. a member URN) which will be credited for moving the resource/association/sub-resource into the specific lifecycle stage. It is also the one used to authorize the change.",
+                      "Urn": "Urn"
+                    },
+                    {
+                      "java": {
+                        "class": "com.linkedin.pegasus2avro.common.urn.Urn"
+                      },
+                      "type": [
+                        "null",
+                        "string"
+                      ],
+                      "name": "impersonator",
+                      "default": null,
+                      "doc": "The entity (e.g. a service URN) which performs the change on behalf of the Actor and must be authorized to act as the Actor.",
+                      "Urn": "Urn"
+                    },
+                    {
+                      "type": [
+                        "null",
+                        "string"
+                      ],
+                      "name": "message",
+                      "default": null,
+                      "doc": "Additional context around how DataHub was informed of the particular change. For example: was the change created by an automated process, or manually."
+                    }
+                  ],
+                  "doc": "Data captured on a resource/association/sub-resource level giving insight into when that resource/association/sub-resource moved into a particular lifecycle stage, and who acted to move it into that specific lifecycle stage."
+                },
+                "name": "assignedAt",
+                "doc": "The time & actor responsible for assiging the assignee."
+              }
+            ],
+            "doc": "The incident assignee type.\nThis is in a record so that we can add additional fields if we need to later (e.g.\nthe type of the assignee."
+          }
+        }
+      ],
+      "name": "assignees",
+      "default": null,
+      "doc": "The parties assigned with resolving the incident"
     },
     {
       "type": {
@@ -135,7 +216,38 @@
               ]
             },
             "name": "state",
-            "doc": "The state of the incident"
+            "doc": "The top-level state of the incident, whether it's active or resolved."
+          },
+          {
+            "Searchable": {
+              "addToFilters": true,
+              "filterNameOverride": "Stage"
+            },
+            "type": [
+              "null",
+              {
+                "type": "enum",
+                "symbolDocs": {
+                  "FIXED": "The incident is in the resolved as completed stage.",
+                  "INVESTIGATION": "The incident root cause is being investigated.",
+                  "NO_ACTION_REQUIRED": "The incident is in the resolved with no action required state, e.g. the\nincident was a false positive, or was expected.",
+                  "TRIAGE": "The impact and priority of the incident is being actively assessed.",
+                  "WORK_IN_PROGRESS": "The incident is in the remediation stage."
+                },
+                "name": "IncidentStage",
+                "namespace": "com.linkedin.pegasus2avro.incident",
+                "symbols": [
+                  "TRIAGE",
+                  "INVESTIGATION",
+                  "WORK_IN_PROGRESS",
+                  "FIXED",
+                  "NO_ACTION_REQUIRED"
+                ]
+              }
+            ],
+            "name": "stage",
+            "default": null,
+            "doc": "The lifecycle stage for the incident - Null means no stage was assigned yet.\nIn the future, we may add CUSTOM here with a customStage string field for user-defined stages."
           },
           {
             "type": [
@@ -153,50 +265,7 @@
                 "fieldType": "COUNT"
               }
             },
-            "type": {
-              "type": "record",
-              "name": "AuditStamp",
-              "namespace": "com.linkedin.pegasus2avro.common",
-              "fields": [
-                {
-                  "type": "long",
-                  "name": "time",
-                  "doc": "When did the resource/association/sub-resource move into the specific lifecycle stage represented by this AuditEvent."
-                },
-                {
-                  "java": {
-                    "class": "com.linkedin.pegasus2avro.common.urn.Urn"
-                  },
-                  "type": "string",
-                  "name": "actor",
-                  "doc": "The entity (e.g. a member URN) which will be credited for moving the resource/association/sub-resource into the specific lifecycle stage. It is also the one used to authorize the change.",
-                  "Urn": "Urn"
-                },
-                {
-                  "java": {
-                    "class": "com.linkedin.pegasus2avro.common.urn.Urn"
-                  },
-                  "type": [
-                    "null",
-                    "string"
-                  ],
-                  "name": "impersonator",
-                  "default": null,
-                  "doc": "The entity (e.g. a service URN) which performs the change on behalf of the Actor and must be authorized to act as the Actor.",
-                  "Urn": "Urn"
-                },
-                {
-                  "type": [
-                    "null",
-                    "string"
-                  ],
-                  "name": "message",
-                  "default": null,
-                  "doc": "Additional context around how DataHub was informed of the particular change. For example: was the change created by an automated process, or manually."
-                }
-              ],
-              "doc": "Data captured on a resource/association/sub-resource level giving insight into when that resource/association/sub-resource moved into a particular lifecycle stage, and who acted to move it into that specific lifecycle stage."
-            },
+            "type": "com.linkedin.pegasus2avro.common.AuditStamp",
             "name": "lastUpdated",
             "doc": "The time at which the request was initially created"
           }
@@ -262,6 +331,21 @@
       "default": null,
       "doc": "The source of an incident, i.e. how it was generated."
     },
+    {
+      "Searchable": {
+        "/time": {
+          "fieldName": "startedAt",
+          "fieldType": "COUNT"
+        }
+      },
+      "type": [
+        "null",
+        "long"
+      ],
+      "name": "startedAt",
+      "default": null,
+      "doc": "The time at which the incident actually started (may be before the date it was raised)."
+    },
     {
       "Searchable": {
         "/time": {

datahub/metadata/schemas/__init__.py

@@ -15,10 +15,10 @@ import pathlib
 def _load_schema(schema_name: str) -> str:
     return (pathlib.Path(__file__).parent / f"{schema_name}.avsc").read_text()
 
-def getMetadataChangeEventSchema() -> str:
-    return _load_schema("MetadataChangeEvent")
-
 def getMetadataChangeProposalSchema() -> str:
     return _load_schema("MetadataChangeProposal")
 
+def getMetadataChangeEventSchema() -> str:
+    return _load_schema("MetadataChangeEvent")
+
 # fmt: on

datahub/sdk/__init__.py

@@ -1,7 +1,7 @@
-import warnings
+import types
 
 import datahub.metadata.schema_classes as models
-from datahub.errors import ExperimentalWarning, SdkUsageError
+from datahub.errors import SdkUsageError
 from datahub.ingestion.graph.config import DatahubClientConfig
 from datahub.metadata.urns import (
     ChartUrn,
@@ -21,13 +21,30 @@ from datahub.sdk.container import Container
 from datahub.sdk.dataset import Dataset
 from datahub.sdk.main_client import DataHubClient
 
-warnings.warn(
-    "The new datahub SDK (e.g. datahub.sdk.*) is experimental. "
-    "Our typical backwards-compatibility and stability guarantees do not apply to this code. "
-    "When it's promoted to stable, the import path will change "
-    "from `from datahub.sdk import ...` to `from datahub import ...`.",
-    ExperimentalWarning,
-    stacklevel=2,
-)
-del warnings
-del ExperimentalWarning
+# We want to print out the warning if people do `from datahub.sdk import X`.
+# But we don't want to print out warnings if they're doing a more direct
+# import like `from datahub.sdk.container import Container`, since that's
+# what our internal code does.
+_vars = {}
+for _name, _value in list(locals().items()):
+    if not _name.startswith("_") and (
+        _name == "models" or not isinstance(_value, types.ModuleType)
+    ):
+        _vars[_name] = _value
+        del locals()[_name]
+
+
+def __getattr__(name):
+    import warnings
+
+    from datahub.errors import ExperimentalWarning
+
+    warnings.warn(
+        "The new datahub SDK (e.g. datahub.sdk.*) is experimental. "
+        "Our typical backwards-compatibility and stability guarantees do not apply to this code. "
+        "When it's promoted to stable, the import path will change "
+        "from `from datahub.sdk import ...` to `from datahub import ...`.",
+        ExperimentalWarning,
+        stacklevel=2,
+    )
+    return _vars[name]

datahub/sdk/_entity.py

@@ -1,5 +1,7 @@
+from __future__ import annotations
+
 import abc
-from typing import List, Optional, Type, Union
+from typing import TYPE_CHECKING, List, Optional, Type, Union
 
 from typing_extensions import Self
 
@@ -10,6 +12,12 @@ from datahub.errors import SdkUsageError
 from datahub.metadata.urns import Urn
 from datahub.utilities.urns._urn_base import _SpecificUrn
 
+if TYPE_CHECKING:
+    from datahub.ingestion.api.workunit import MetadataWorkUnit
+
+
+ExtraAspectsType = Union[None, List[AspectTypeVar]]
+
 
 class Entity:
     __slots__ = ("_urn", "_prev_aspects", "_aspects")
@@ -87,5 +95,14 @@ class Entity:
             )
         return mcps
 
+    def as_workunits(self) -> List[MetadataWorkUnit]:
+        return [mcp.as_workunit() for mcp in self._as_mcps()]
+
+    def _set_extra_aspects(self, extra_aspects: ExtraAspectsType) -> None:
+        # TODO: Add validation to ensure that an "extra aspect" does not conflict
+        # with / get overridden by a standard aspect.
+        for aspect in extra_aspects or []:
+            self._set_aspect(aspect)
+
     def __repr__(self) -> str:
         return f"{self.__class__.__name__}('{self.urn}')"

datahub/sdk/container.py

@@ -16,7 +16,7 @@ from datahub.metadata.urns import (
     ContainerUrn,
     Urn,
 )
-from datahub.sdk._entity import Entity
+from datahub.sdk._entity import Entity, ExtraAspectsType
 from datahub.sdk._shared import (
     DomainInputType,
     HasContainer,
@@ -74,6 +74,7 @@ class Container(
         tags: Optional[TagsInputType] = None,
         terms: Optional[TermsInputType] = None,
         domain: Optional[DomainInputType] = None,
+        extra_aspects: ExtraAspectsType = None,
     ):
         # Hack: while the type annotations say container_key is always a ContainerKey,
         # we allow ContainerUrn to make the graph-based constructor work.
@@ -82,6 +83,7 @@ class Container(
         else:
             urn = ContainerUrn.from_string(container_key.as_urn())
         super().__init__(urn)
+        self._set_extra_aspects(extra_aspects)
 
         # This needs to come first to ensure that the display name is registered.
         self._ensure_container_props(name=display_name)

datahub/sdk/dataset.py

@@ -2,7 +2,7 @@ from __future__ import annotations
 
 import warnings
 from datetime import datetime
-from typing import Dict, List, Optional, Tuple, Type, Union
+from typing import Dict, List, Optional, Sequence, Tuple, Type, Union
 
 from typing_extensions import Self, TypeAlias, assert_never
 
@@ -18,7 +18,7 @@ from datahub.errors import (
 from datahub.ingestion.source.sql.sql_types import resolve_sql_type
 from datahub.metadata.urns import DatasetUrn, SchemaFieldUrn, Urn
 from datahub.sdk._attribution import is_ingestion_attribution
-from datahub.sdk._entity import Entity
+from datahub.sdk._entity import Entity, ExtraAspectsType
 from datahub.sdk._shared import (
     DatasetUrnOrStr,
     DomainInputType,
@@ -47,7 +47,7 @@ SchemaFieldInputType: TypeAlias = Union[
     models.SchemaFieldClass,
 ]
 SchemaFieldsInputType: TypeAlias = Union[
-    List[SchemaFieldInputType],
+    Sequence[SchemaFieldInputType],
     models.SchemaMetadataClass,
 ]
 
@@ -457,6 +457,7 @@ class Dataset(
         terms: Optional[TermsInputType] = None,
         # TODO structured_properties
         domain: Optional[DomainInputType] = None,
+        extra_aspects: ExtraAspectsType = None,
         # Dataset-specific aspects.
         schema: Optional[SchemaFieldsInputType] = None,
         upstreams: Optional[models.UpstreamLineageClass] = None,
@@ -468,6 +469,7 @@ class Dataset(
             env=env,
         )
         super().__init__(urn)
+        self._set_extra_aspects(extra_aspects)
 
         self._set_platform_instance(urn.platform, platform_instance)
 

datahub/sql_parsing/_sqlglot_patch.py

@@ -172,17 +172,9 @@ def _patch_lineage() -> None:
          derived_tables = [
              source.expression.parent
              for source in scope.sources.values()
-@@ -254,6 +257,7 @@ def to_node(
-         if dt.comments and dt.comments[0].startswith("source: ")
-     }
-
-+    c: exp.Column
-     for c in source_columns:
-         table = c.table
-         source = scope.sources.get(table)
 @@ -281,8 +285,21 @@ def to_node(
-             # it means this column's lineage is unknown. This can happen if the definition of a source used in a query
-             # is not passed into the `sources` map.
+             # is unknown. This can happen if the definition of a source used in a query is not
+             # passed into the `sources` map.
              source = source or exp.Placeholder()
 +
 +            subfields = []

datahub/utilities/unified_diff.py

@@ -2,8 +2,12 @@ import logging
 from dataclasses import dataclass
 from typing import List, Tuple
 
+from datahub.cli.env_utils import get_boolean_env_variable
+
+_debug_diff = get_boolean_env_variable("DATAHUB_DEBUG_DIFF_PATCHER")
+
 logger = logging.getLogger(__name__)
-logger.setLevel(logging.INFO)
+logger.setLevel(logging.DEBUG if _debug_diff else logging.INFO)
 
 _LOOKAROUND_LINES = 300