acryl-datahub 1.1.0rc3__py3-none-any.whl → 1.1.0.1__py3-none-any.whl

datahub/metadata/schemas/ApplicationKey.avsc

@@ -0,0 +1,31 @@
+{
+  "type": "record",
+  "Aspect": {
+    "name": "applicationKey",
+    "keyForEntity": "application",
+    "entityCategory": "core",
+    "entityAspects": [
+      "applicationProperties",
+      "ownership",
+      "glossaryTerms",
+      "globalTags",
+      "domains",
+      "institutionalMemory",
+      "status",
+      "structuredProperties",
+      "forms",
+      "testResults",
+      "subTypes"
+    ]
+  },
+  "name": "ApplicationKey",
+  "namespace": "com.linkedin.pegasus2avro.application",
+  "fields": [
+    {
+      "type": "string",
+      "name": "id",
+      "doc": "A unique id for the Application."
+    }
+  ],
+  "doc": "Key for a Query"
+}

datahub/metadata/schemas/ApplicationProperties.avsc

@@ -0,0 +1,72 @@
+{
+  "type": "record",
+  "Aspect": {
+    "name": "applicationProperties"
+  },
+  "name": "ApplicationProperties",
+  "namespace": "com.linkedin.pegasus2avro.application",
+  "fields": [
+    {
+      "Searchable": {
+        "/*": {
+          "fieldType": "TEXT",
+          "queryByDefault": true
+        }
+      },
+      "type": {
+        "type": "map",
+        "values": "string"
+      },
+      "name": "customProperties",
+      "default": {},
+      "doc": "Custom property bag."
+    },
+    {
+      "Searchable": {
+        "fieldType": "KEYWORD"
+      },
+      "java": {
+        "class": "com.linkedin.pegasus2avro.common.url.Url",
+        "coercerClass": "com.linkedin.pegasus2avro.common.url.UrlCoercer"
+      },
+      "type": [
+        "null",
+        "string"
+      ],
+      "name": "externalUrl",
+      "default": null,
+      "doc": "URL where the reference exist"
+    },
+    {
+      "Searchable": {
+        "boostScore": 10.0,
+        "enableAutocomplete": true,
+        "fieldNameAliases": [
+          "_entityName"
+        ],
+        "fieldType": "WORD_GRAM"
+      },
+      "type": [
+        "null",
+        "string"
+      ],
+      "name": "name",
+      "default": null,
+      "doc": "Display name of the Application"
+    },
+    {
+      "Searchable": {
+        "fieldType": "TEXT",
+        "hasValuesFieldName": "hasDescription"
+      },
+      "type": [
+        "null",
+        "string"
+      ],
+      "name": "description",
+      "default": null,
+      "doc": "Documentation of the application"
+    }
+  ],
+  "doc": "The main properties of an Application"
+}

datahub/metadata/schemas/Applications.avsc

@@ -0,0 +1,38 @@
+{
+  "type": "record",
+  "Aspect": {
+    "name": "applications"
+  },
+  "name": "Applications",
+  "namespace": "com.linkedin.pegasus2avro.application",
+  "fields": [
+    {
+      "Relationship": {
+        "/*": {
+          "entityTypes": [
+            "application"
+          ],
+          "name": "AssociatedWith"
+        }
+      },
+      "Searchable": {
+        "/*": {
+          "addToFilters": true,
+          "fieldName": "applications",
+          "fieldType": "URN",
+          "filterNameOverride": "Application",
+          "hasValuesFieldName": "hasApplication"
+        }
+      },
+      "type": {
+        "type": "array",
+        "items": "string"
+      },
+      "name": "applications",
+      "doc": "The Applications attached to an Asset",
+      "Urn": "Urn",
+      "urn_is_array": true
+    }
+  ],
+  "doc": "Links from an Asset to its Applications"
+}

datahub/metadata/schemas/ChartKey.avsc

@@ -13,6 +13,7 @@
       "embed",
       "browsePaths",
       "domains",
+      "applications",
       "container",
       "deprecation",
       "ownership",

datahub/metadata/schemas/ContainerKey.avsc

@@ -18,6 +18,7 @@
       "browsePaths",
       "status",
       "domains",
+      "applications",
       "browsePathsV2",
       "structuredProperties",
       "forms",

datahub/metadata/schemas/DashboardKey.avsc

@@ -6,6 +6,7 @@
     "entityCategory": "_unset_",
     "entityAspects": [
       "domains",
+      "applications",
       "container",
       "deprecation",
       "dashboardUsageStatistics",

datahub/metadata/schemas/DataFlowKey.avsc

@@ -6,6 +6,7 @@
     "entityCategory": "core",
     "entityAspects": [
       "domains",
+      "applications",
       "deprecation",
       "versionInfo",
       "dataFlowInfo",

datahub/metadata/schemas/DataHubIngestionSourceKey.avsc

@@ -5,7 +5,8 @@
     "keyForEntity": "dataHubIngestionSource",
     "entityCategory": "internal",
     "entityAspects": [
-      "dataHubIngestionSourceInfo"
+      "dataHubIngestionSourceInfo",
+      "ownership"
     ]
   },
   "name": "DataHubIngestionSourceKey",

datahub/metadata/schemas/DataJobKey.avsc

@@ -8,6 +8,7 @@
       "datahubIngestionRunSummary",
       "datahubIngestionCheckpoint",
       "domains",
+      "applications",
       "deprecation",
       "versionInfo",
       "dataJobInfo",

datahub/metadata/schemas/DataProductKey.avsc

@@ -9,6 +9,7 @@
       "glossaryTerms",
       "globalTags",
       "domains",
+      "applications",
       "dataProductProperties",
       "institutionalMemory",
       "status",

datahub/metadata/schemas/DataProductProperties.avsc

@@ -65,7 +65,7 @@
       ],
       "name": "description",
       "default": null,
-      "doc": "Documentation of the dataset"
+      "doc": "Documentation of the data product"
     },
     {
       "Relationship": {

datahub/metadata/schemas/DatasetKey.avsc

@@ -11,6 +11,7 @@
       "datasetUsageStatistics",
       "operation",
       "domains",
+      "applications",
       "schemaMetadata",
       "status",
       "container",

datahub/metadata/schemas/ExecutionRequestInput.avsc

@@ -20,6 +20,11 @@
       "doc": "Arguments provided to the task"
     },
     {
+      "Searchable": {
+        "fieldName": "executorId",
+        "fieldType": "KEYWORD",
+        "queryByDefault": false
+      },
       "type": "string",
       "name": "executorId",
       "doc": "Advanced: specify a specific executor to route the request to. If none is provided, a \"default\" executor is used."

datahub/metadata/schemas/GlossaryTermKey.avsc

@@ -12,6 +12,7 @@
       "ownership",
       "deprecation",
       "domains",
+      "applications",
       "status",
       "browsePaths",
       "structuredProperties",

datahub/metadata/schemas/MLFeatureKey.avsc

@@ -8,6 +8,7 @@
       "glossaryTerms",
       "editableMlFeatureProperties",
       "domains",
+      "applications",
       "mlFeatureProperties",
       "ownership",
       "institutionalMemory",

datahub/metadata/schemas/MLFeatureTableKey.avsc

@@ -8,6 +8,7 @@
       "glossaryTerms",
       "editableMlFeatureTableProperties",
       "domains",
+      "applications",
       "mlFeatureTableProperties",
       "ownership",
       "institutionalMemory",

datahub/metadata/schemas/MLModelGroupKey.avsc

@@ -8,6 +8,7 @@
       "glossaryTerms",
       "editableMlModelGroupProperties",
       "domains",
+      "applications",
       "mlModelGroupProperties",
       "ownership",
       "status",

datahub/metadata/schemas/MLModelKey.avsc

@@ -8,6 +8,7 @@
       "glossaryTerms",
       "editableMlModelProperties",
       "domains",
+      "applications",
       "ownership",
       "mlModelProperties",
       "intendedUse",

datahub/metadata/schemas/MLPrimaryKeyKey.avsc

@@ -8,6 +8,7 @@
       "glossaryTerms",
       "editableMlPrimaryKeyProperties",
       "domains",
+      "applications",
       "mlPrimaryKeyProperties",
       "ownership",
       "institutionalMemory",

datahub/metadata/schemas/NotebookKey.avsc

@@ -15,6 +15,7 @@
       "browsePaths",
       "institutionalMemory",
       "domains",
+      "applications",
       "subTypes",
       "dataPlatformInstance",
       "browsePathsV2",

datahub/metadata/schemas/__init__.py

@@ -15,10 +15,10 @@ import pathlib
 def _load_schema(schema_name: str) -> str:
     return (pathlib.Path(__file__).parent / f"{schema_name}.avsc").read_text()
 
-def getMetadataChangeProposalSchema() -> str:
-    return _load_schema("MetadataChangeProposal")
-
 def getMetadataChangeEventSchema() -> str:
     return _load_schema("MetadataChangeEvent")
 
+def getMetadataChangeProposalSchema() -> str:
+    return _load_schema("MetadataChangeProposal")
+
 # fmt: on

datahub/sdk/__init__.py

@@ -18,9 +18,15 @@ from datahub.metadata.urns import (
     SchemaFieldUrn,
     TagUrn,
 )
+from datahub.sdk.chart import Chart
 from datahub.sdk.container import Container
+from datahub.sdk.dashboard import Dashboard
+from datahub.sdk.dataflow import DataFlow
+from datahub.sdk.datajob import DataJob
 from datahub.sdk.dataset import Dataset
 from datahub.sdk.main_client import DataHubClient
+from datahub.sdk.mlmodel import MLModel
+from datahub.sdk.mlmodelgroup import MLModelGroup
 from datahub.sdk.search_filters import Filter, FilterDsl
 
 # We want to print out the warning if people do `from datahub.sdk import X`.

datahub/sdk/_all_entities.py

@@ -1,19 +1,30 @@
 from typing import Dict, List, Type
 
+from datahub.sdk.chart import Chart
 from datahub.sdk.container import Container
+from datahub.sdk.dashboard import Dashboard
+from datahub.sdk.dataflow import DataFlow
+from datahub.sdk.datajob import DataJob
 from datahub.sdk.dataset import Dataset
 from datahub.sdk.entity import Entity
 from datahub.sdk.mlmodel import MLModel
 from datahub.sdk.mlmodelgroup import MLModelGroup
 
+# Base entity classes that don't have circular dependencies
+# Those that do are imported in the EntityClient where needed
 # TODO: Is there a better way to declare this?
 ENTITY_CLASSES_LIST: List[Type[Entity]] = [
     Container,
     Dataset,
     MLModel,
     MLModelGroup,
+    DataFlow,
+    DataJob,
+    Dashboard,
+    Chart,
 ]
 
+# Create the mapping of entity types to classes
 ENTITY_CLASSES: Dict[str, Type[Entity]] = {
     cls.get_urn_type().ENTITY_TYPE: cls for cls in ENTITY_CLASSES_LIST
 }

datahub/sdk/_shared.py

@@ -26,9 +26,12 @@ from datahub.emitter.mce_builder import (
 from datahub.emitter.mcp_builder import ContainerKey
 from datahub.errors import MultipleSubtypesWarning, SdkUsageError
 from datahub.metadata.urns import (
+    ChartUrn,
     ContainerUrn,
     CorpGroupUrn,
     CorpUserUrn,
+    DashboardUrn,
+    DataFlowUrn,
     DataJobUrn,
     DataPlatformInstanceUrn,
     DataPlatformUrn,
@@ -37,6 +40,7 @@ from datahub.metadata.urns import (
     DomainUrn,
     GlossaryTermUrn,
     OwnershipTypeUrn,
+    StructuredPropertyUrn,
     TagUrn,
     Urn,
     VersionSetUrn,
@@ -47,12 +51,21 @@ from datahub.utilities.urns.error import InvalidUrnError
 
 if TYPE_CHECKING:
     from datahub.sdk.container import Container
-
 UrnOrStr: TypeAlias = Union[Urn, str]
+ChartUrnOrStr: TypeAlias = Union[str, ChartUrn]
 DatasetUrnOrStr: TypeAlias = Union[str, DatasetUrn]
 DatajobUrnOrStr: TypeAlias = Union[str, DataJobUrn]
+DataflowUrnOrStr: TypeAlias = Union[str, DataFlowUrn]
+DashboardUrnOrStr: TypeAlias = Union[str, DashboardUrn]
+DataPlatformInstanceUrnOrStr: TypeAlias = Union[str, DataPlatformInstanceUrn]
+DataPlatformUrnOrStr: TypeAlias = Union[str, DataPlatformUrn]
 
 ActorUrn: TypeAlias = Union[CorpUserUrn, CorpGroupUrn]
+StructuredPropertyUrnOrStr: TypeAlias = Union[str, StructuredPropertyUrn]
+StructuredPropertyValueType: TypeAlias = Union[str, float, int]
+StructuredPropertyInputType: TypeAlias = Dict[
+    StructuredPropertyUrnOrStr, Sequence[StructuredPropertyValueType]
+]
 
 TrainingMetricsInputType: TypeAlias = Union[
     List[models.MLMetricClass], Dict[str, Optional[str]]
@@ -716,3 +729,107 @@ class HasVersion(Entity):
                 a for a in version_props.aliases if a.versionTag != alias
             ]
             self._set_aspect(version_props)
+
+
+class HasStructuredProperties(Entity):
+    """
+    Mixin for entities that support structured properties
+    """
+
+    __slots__ = ()
+
+    @property
+    def structured_properties(
+        self,
+    ) -> Optional[List[models.StructuredPropertyValueAssignmentClass]]:
+        """
+        Retrieve structured properties for the entity
+
+        Returns:
+            Optional list of structured property value assignments
+        """
+        sp_aspect = self._get_aspect(models.StructuredPropertiesClass)
+        return sp_aspect.properties if sp_aspect else None
+
+    def _ensure_structured_properties(self) -> models.StructuredPropertiesClass:
+        """
+        Ensure structured properties aspect exists, creating it if necessary
+
+        Returns:
+            StructuredPropertiesClass aspect
+        """
+        return self._setdefault_aspect(models.StructuredPropertiesClass(properties=[]))
+
+    def set_structured_property(
+        self,
+        property_urn: StructuredPropertyUrnOrStr,
+        values: Sequence[StructuredPropertyValueType],
+    ) -> None:
+        """
+        Update an existing structured property or add if it doesn't exist
+
+        Args:
+            property_urn: URN of the structured property
+            values: List of values for the property
+        """
+        # validate property_urn is a valid structured property urn
+        property_urn = StructuredPropertyUrn.from_string(property_urn)
+
+        properties = self._ensure_structured_properties()
+
+        # Find existing property assignment
+        existing_prop = next(
+            (
+                prop
+                for prop in properties.properties
+                if prop.propertyUrn == str(property_urn)
+            ),
+            None,
+        )
+        current_timestamp = make_ts_millis(datetime.now())
+
+        if existing_prop:
+            # Update existing property
+            existing_prop.values = list(values)
+            existing_prop.lastModified = models.AuditStampClass(
+                time=current_timestamp,
+                actor=DEFAULT_ACTOR_URN,
+            )
+        else:
+            # Create new property assignment
+            new_property = models.StructuredPropertyValueAssignmentClass(
+                propertyUrn=str(property_urn),
+                values=list(values),
+                created=models.AuditStampClass(
+                    time=current_timestamp,
+                    actor=DEFAULT_ACTOR_URN,
+                ),
+                lastModified=models.AuditStampClass(
+                    time=current_timestamp,
+                    actor=DEFAULT_ACTOR_URN,
+                ),
+            )
+            add_list_unique(
+                properties.properties,
+                key=lambda prop: prop.propertyUrn,
+                item=new_property,
+            )
+
+        self._set_aspect(properties)
+
+    def remove_structured_property(
+        self, property_urn: StructuredPropertyUrnOrStr
+    ) -> None:
+        """
+        Remove a structured property from the entity
+
+        Args:
+            property_urn: URN of the structured property to remove
+        """
+        remove_list_unique(
+            self._ensure_structured_properties().properties,
+            key=lambda prop: prop.propertyUrn,
+            item=models.StructuredPropertyValueAssignmentClass(
+                propertyUrn=str(property_urn), values=[]
+            ),
+        )