acryl-datahub 0.15.0rc24__py3-none-any.whl → 0.15.0.1__py3-none-any.whl

datahub/metadata/schemas/MLModelProperties.avsc

@@ -37,6 +37,21 @@
       "default": null,
       "doc": "URL where the reference exist"
     },
+    {
+      "Searchable": {
+        "boostScore": 10.0,
+        "enableAutocomplete": true,
+        "fieldType": "WORD_GRAM",
+        "queryByDefault": true
+      },
+      "type": [
+        "null",
+        "string"
+      ],
+      "name": "name",
+      "default": null,
+      "doc": "Display name of the MLModel"
+    },
     {
       "Searchable": {
         "fieldType": "TEXT",
@@ -51,6 +66,7 @@
       "doc": "Documentation of the MLModel"
     },
     {
+      "deprecated": true,
       "type": [
         "null",
         "long"
@@ -59,6 +75,49 @@
       "default": null,
       "doc": "Date when the MLModel was developed"
     },
+    {
+      "type": [
+        "null",
+        {
+          "type": "record",
+          "name": "TimeStamp",
+          "namespace": "com.linkedin.pegasus2avro.common",
+          "fields": [
+            {
+              "type": "long",
+              "name": "time",
+              "doc": "When did the event occur"
+            },
+            {
+              "java": {
+                "class": "com.linkedin.pegasus2avro.common.urn.Urn"
+              },
+              "type": [
+                "null",
+                "string"
+              ],
+              "name": "actor",
+              "default": null,
+              "doc": "Optional: The actor urn involved in the event.",
+              "Urn": "Urn"
+            }
+          ],
+          "doc": "A standard event timestamp"
+        }
+      ],
+      "name": "created",
+      "default": null,
+      "doc": "Audit stamp containing who created this and when"
+    },
+    {
+      "type": [
+        "null",
+        "com.linkedin.pegasus2avro.common.TimeStamp"
+      ],
+      "name": "lastModified",
+      "default": null,
+      "doc": "Date when the MLModel was last modified"
+    },
     {
       "type": [
         "null",
@@ -291,7 +350,8 @@
       "Relationship": {
         "/*": {
           "entityTypes": [
-            "dataJob"
+            "dataJob",
+            "dataProcessInstance"
           ],
           "isLineage": true,
           "name": "TrainedBy"
@@ -306,7 +366,7 @@
       ],
       "name": "trainingJobs",
       "default": null,
-      "doc": "List of jobs (if any) used to train the model",
+      "doc": "List of jobs (if any) used to train the model. Visible in Lineage. Note that ML Models can also be specified as the output of a specific Data Process Instances (runs) via the DataProcessInstanceOutputs aspect.",
       "Urn": "Urn",
       "urn_is_array": true
     },

datahub/metadata/schemas/MLTrainingRunProperties.avsc

@@ -0,0 +1,171 @@
+{
+  "type": "record",
+  "Aspect": {
+    "name": "mlTrainingRunProperties"
+  },
+  "name": "MLTrainingRunProperties",
+  "namespace": "com.linkedin.pegasus2avro.ml.metadata",
+  "fields": [
+    {
+      "Searchable": {
+        "/*": {
+          "fieldType": "TEXT",
+          "queryByDefault": true
+        }
+      },
+      "type": {
+        "type": "map",
+        "values": "string"
+      },
+      "name": "customProperties",
+      "default": {},
+      "doc": "Custom property bag."
+    },
+    {
+      "Searchable": {
+        "fieldType": "KEYWORD"
+      },
+      "java": {
+        "class": "com.linkedin.pegasus2avro.common.url.Url",
+        "coercerClass": "com.linkedin.pegasus2avro.common.url.UrlCoercer"
+      },
+      "type": [
+        "null",
+        "string"
+      ],
+      "name": "externalUrl",
+      "default": null,
+      "doc": "URL where the reference exist"
+    },
+    {
+      "type": [
+        "null",
+        "string"
+      ],
+      "name": "id",
+      "default": null,
+      "doc": "Run Id of the ML Training Run"
+    },
+    {
+      "type": [
+        "null",
+        {
+          "type": "array",
+          "items": "string"
+        }
+      ],
+      "name": "outputUrls",
+      "default": null,
+      "doc": "List of URLs for the Outputs of the ML Training Run"
+    },
+    {
+      "type": [
+        "null",
+        {
+          "type": "array",
+          "items": {
+            "type": "record",
+            "Aspect": {
+              "name": "mlHyperParam"
+            },
+            "name": "MLHyperParam",
+            "namespace": "com.linkedin.pegasus2avro.ml.metadata",
+            "fields": [
+              {
+                "type": "string",
+                "name": "name",
+                "doc": "Name of the MLHyperParam"
+              },
+              {
+                "type": [
+                  "null",
+                  "string"
+                ],
+                "name": "description",
+                "default": null,
+                "doc": "Documentation of the MLHyperParam"
+              },
+              {
+                "type": [
+                  "null",
+                  "string"
+                ],
+                "name": "value",
+                "default": null,
+                "doc": "The value of the MLHyperParam"
+              },
+              {
+                "type": [
+                  "null",
+                  "long"
+                ],
+                "name": "createdAt",
+                "default": null,
+                "doc": "Date when the MLHyperParam was developed"
+              }
+            ],
+            "doc": "Properties associated with an ML Hyper Param"
+          }
+        }
+      ],
+      "name": "hyperParams",
+      "default": null,
+      "doc": "Hyperparameters of the ML Training Run"
+    },
+    {
+      "type": [
+        "null",
+        {
+          "type": "array",
+          "items": {
+            "type": "record",
+            "Aspect": {
+              "name": "mlMetric"
+            },
+            "name": "MLMetric",
+            "namespace": "com.linkedin.pegasus2avro.ml.metadata",
+            "fields": [
+              {
+                "type": "string",
+                "name": "name",
+                "doc": "Name of the mlMetric"
+              },
+              {
+                "type": [
+                  "null",
+                  "string"
+                ],
+                "name": "description",
+                "default": null,
+                "doc": "Documentation of the mlMetric"
+              },
+              {
+                "type": [
+                  "null",
+                  "string"
+                ],
+                "name": "value",
+                "default": null,
+                "doc": "The value of the mlMetric"
+              },
+              {
+                "type": [
+                  "null",
+                  "long"
+                ],
+                "name": "createdAt",
+                "default": null,
+                "doc": "Date when the mlMetric was developed"
+              }
+            ],
+            "doc": "Properties associated with an ML Metric"
+          }
+        }
+      ],
+      "name": "trainingMetrics",
+      "default": null,
+      "doc": "Metrics of the ML Training Run"
+    }
+  ],
+  "doc": "The inputs and outputs of this training run"
+}

datahub/metadata/schemas/MetadataChangeEvent.avsc

@@ -5109,6 +5109,21 @@
                         "default": null,
                         "doc": "URL where the reference exist"
                       },
+                      {
+                        "Searchable": {
+                          "boostScore": 10.0,
+                          "enableAutocomplete": true,
+                          "fieldType": "WORD_GRAM",
+                          "queryByDefault": true
+                        },
+                        "type": [
+                          "null",
+                          "string"
+                        ],
+                        "name": "name",
+                        "default": null,
+                        "doc": "Display name of the MLModel"
+                      },
                       {
                         "Searchable": {
                           "fieldType": "TEXT",
@@ -5123,6 +5138,7 @@
                         "doc": "Documentation of the MLModel"
                       },
                       {
+                        "deprecated": true,
                         "type": [
                           "null",
                           "long"
@@ -5131,6 +5147,24 @@
                         "default": null,
                         "doc": "Date when the MLModel was developed"
                       },
+                      {
+                        "type": [
+                          "null",
+                          "com.linkedin.pegasus2avro.common.TimeStamp"
+                        ],
+                        "name": "created",
+                        "default": null,
+                        "doc": "Audit stamp containing who created this and when"
+                      },
+                      {
+                        "type": [
+                          "null",
+                          "com.linkedin.pegasus2avro.common.TimeStamp"
+                        ],
+                        "name": "lastModified",
+                        "default": null,
+                        "doc": "Date when the MLModel was last modified"
+                      },
                       {
                         "type": [
                           "null",
@@ -5363,7 +5397,8 @@
                         "Relationship": {
                           "/*": {
                             "entityTypes": [
-                              "dataJob"
+                              "dataJob",
+                              "dataProcessInstance"
                             ],
                             "isLineage": true,
                             "name": "TrainedBy"
@@ -5378,7 +5413,7 @@
                         ],
                         "name": "trainingJobs",
                         "default": null,
-                        "doc": "List of jobs (if any) used to train the model",
+                        "doc": "List of jobs (if any) used to train the model. Visible in Lineage. Note that ML Models can also be specified as the output of a specific Data Process Instances (runs) via the DataProcessInstanceOutputs aspect.",
                         "Urn": "Urn",
                         "urn_is_array": true
                       },
@@ -6804,6 +6839,21 @@
                         "default": {},
                         "doc": "Custom property bag."
                       },
+                      {
+                        "Searchable": {
+                          "boostScore": 10.0,
+                          "enableAutocomplete": true,
+                          "fieldType": "WORD_GRAM",
+                          "queryByDefault": true
+                        },
+                        "type": [
+                          "null",
+                          "string"
+                        ],
+                        "name": "name",
+                        "default": null,
+                        "doc": "Display name of the MLModelGroup"
+                      },
                       {
                         "Searchable": {
                           "fieldType": "TEXT",
@@ -6818,6 +6868,7 @@
                         "doc": "Documentation of the MLModelGroup"
                       },
                       {
+                        "deprecated": true,
                         "type": [
                           "null",
                           "long"
@@ -6826,6 +6877,47 @@
                         "default": null,
                         "doc": "Date when the MLModelGroup was developed"
                       },
+                      {
+                        "type": [
+                          "null",
+                          "com.linkedin.pegasus2avro.common.TimeStamp"
+                        ],
+                        "name": "created",
+                        "default": null,
+                        "doc": "Time and Actor who created the MLModelGroup"
+                      },
+                      {
+                        "type": [
+                          "null",
+                          "com.linkedin.pegasus2avro.common.TimeStamp"
+                        ],
+                        "name": "lastModified",
+                        "default": null,
+                        "doc": "Date when the MLModelGroup was last modified"
+                      },
+                      {
+                        "Relationship": {
+                          "/*": {
+                            "entityTypes": [
+                              "dataJob"
+                            ],
+                            "isLineage": true,
+                            "name": "TrainedBy"
+                          }
+                        },
+                        "type": [
+                          "null",
+                          {
+                            "type": "array",
+                            "items": "string"
+                          }
+                        ],
+                        "name": "trainingJobs",
+                        "default": null,
+                        "doc": "List of jobs (if any) used to train the model group. Visible in Lineage.",
+                        "Urn": "Urn",
+                        "urn_is_array": true
+                      },
                       {
                         "type": [
                           "null",

datahub/specific/aspect_helpers/custom_properties.py

@@ -0,0 +1,79 @@
+from abc import abstractmethod
+from typing import Dict, Optional, Tuple
+
+from typing_extensions import Self
+
+from datahub.emitter.mcp_patch_builder import MetadataPatchProposal, PatchPath
+
+
+class HasCustomPropertiesPatch(MetadataPatchProposal):
+    @classmethod
+    @abstractmethod
+    def _custom_properties_location(self) -> Tuple[str, PatchPath]:
+        ...
+
+    def add_custom_property(self, key: str, value: str) -> Self:
+        """Add a custom property to the entity.
+
+        Args:
+            key: The key of the custom property.
+            value: The value of the custom property.
+
+        Returns:
+            The patch builder instance.
+        """
+        aspect_name, path = self._custom_properties_location()
+        self._add_patch(
+            aspect_name,
+            "add",
+            path=(*path, key),
+            value=value,
+        )
+        return self
+
+    def add_custom_properties(
+        self, custom_properties: Optional[Dict[str, str]] = None
+    ) -> Self:
+        if custom_properties is not None:
+            for key, value in custom_properties.items():
+                self.add_custom_property(key, value)
+        return self
+
+    def remove_custom_property(self, key: str) -> Self:
+        """Remove a custom property from the entity.
+
+        Args:
+            key: The key of the custom property to remove.
+
+        Returns:
+            The patch builder instance.
+        """
+        aspect_name, path = self._custom_properties_location()
+        self._add_patch(
+            aspect_name,
+            "remove",
+            path=(*path, key),
+            value={},
+        )
+        return self
+
+    def set_custom_properties(self, custom_properties: Dict[str, str]) -> Self:
+        """Sets the custom properties of the entity.
+
+        This method replaces all existing custom properties with the given dictionary.
+
+        Args:
+            custom_properties: A dictionary containing the custom properties to be set.
+
+        Returns:
+            The patch builder instance.
+        """
+
+        aspect_name, path = self._custom_properties_location()
+        self._add_patch(
+            aspect_name,
+            "add",
+            path=path,
+            value=custom_properties,
+        )
+        return self

datahub/specific/aspect_helpers/ownership.py

@@ -0,0 +1,67 @@
+from typing import List, Optional
+
+from typing_extensions import Self
+
+from datahub.emitter.mcp_patch_builder import MetadataPatchProposal
+from datahub.metadata.schema_classes import (
+    OwnerClass,
+    OwnershipClass,
+    OwnershipTypeClass,
+)
+
+
+class HasOwnershipPatch(MetadataPatchProposal):
+    def add_owner(self, owner: OwnerClass) -> Self:
+        """Add an owner to the entity.
+
+        Args:
+            owner: The Owner object to add.
+
+        Returns:
+            The patch builder instance.
+        """
+        self._add_patch(
+            OwnershipClass.ASPECT_NAME,
+            "add",
+            path=("owners", owner.owner, str(owner.type)),
+            value=owner,
+        )
+        return self
+
+    def remove_owner(
+        self, owner: str, owner_type: Optional[OwnershipTypeClass] = None
+    ) -> Self:
+        """Remove an owner from the entity.
+
+        If owner_type is not provided, the owner will be removed regardless of ownership type.
+
+        Args:
+            owner: The owner to remove.
+            owner_type: The ownership type of the owner (optional).
+
+        Returns:
+            The patch builder instance.
+        """
+        self._add_patch(
+            OwnershipClass.ASPECT_NAME,
+            "remove",
+            path=("owners", owner) + ((str(owner_type),) if owner_type else ()),
+            value=owner,
+        )
+        return self
+
+    def set_owners(self, owners: List[OwnerClass]) -> Self:
+        """Set the owners of the entity.
+
+        This will effectively replace all existing owners with the new list - it doesn't really patch things.
+
+        Args:
+            owners: The list of owners to set.
+
+        Returns:
+            The patch builder instance.
+        """
+        self._add_patch(
+            OwnershipClass.ASPECT_NAME, "add", path=("owners",), value=owners
+        )
+        return self

datahub/specific/aspect_helpers/structured_properties.py

@@ -0,0 +1,72 @@
+from typing import List, Union
+
+from typing_extensions import Self
+
+from datahub.emitter.mcp_patch_builder import MetadataPatchProposal
+from datahub.metadata.schema_classes import (
+    StructuredPropertiesClass,
+    StructuredPropertyValueAssignmentClass,
+)
+from datahub.utilities.urns.structured_properties_urn import (
+    make_structured_property_urn,
+)
+
+
+class HasStructuredPropertiesPatch(MetadataPatchProposal):
+    def set_structured_property(
+        self, key: str, value: Union[str, float, List[Union[str, float]]]
+    ) -> Self:
+        """Add or update a structured property.
+
+        Args:
+            key: the name of the property (either bare or urn form)
+            value: the value of the property (for multi-valued properties, this can be a list)
+
+        Returns:
+            The patch builder instance.
+        """
+        self.remove_structured_property(key)
+        self.add_structured_property(key, value)
+        return self
+
+    def remove_structured_property(self, key: str) -> Self:
+        """Remove a structured property.
+
+        Args:
+            key: the name of the property (either bare or urn form)
+
+        Returns:
+            The patch builder instance.
+        """
+
+        self._add_patch(
+            StructuredPropertiesClass.ASPECT_NAME,
+            "remove",
+            path=("properties", make_structured_property_urn(key)),
+            value={},
+        )
+        return self
+
+    def add_structured_property(
+        self, key: str, value: Union[str, float, List[Union[str, float]]]
+    ) -> Self:
+        """Add a structured property.
+
+        Args:
+            key: the name of the property (either bare or urn form)
+            value: the value of the property (for multi-valued properties, this value will be appended to the list)
+
+        Returns:
+            The patch builder instance.
+        """
+
+        self._add_patch(
+            StructuredPropertiesClass.ASPECT_NAME,
+            "add",
+            path=("properties", make_structured_property_urn(key)),
+            value=StructuredPropertyValueAssignmentClass(
+                propertyUrn=make_structured_property_urn(key),
+                values=value if isinstance(value, list) else [value],
+            ),
+        )
+        return self

datahub/specific/aspect_helpers/tags.py

@@ -0,0 +1,42 @@
+from typing import Union
+
+from typing_extensions import Self
+
+from datahub.emitter.mcp_patch_builder import MetadataPatchProposal
+from datahub.metadata.schema_classes import (
+    GlobalTagsClass as GlobalTags,
+    TagAssociationClass as Tag,
+)
+from datahub.metadata.urns import TagUrn, Urn
+
+
+class HasTagsPatch(MetadataPatchProposal):
+    def add_tag(self, tag: Tag) -> Self:
+        """Adds a tag to the entity.
+
+        Args:
+            tag: The Tag object representing the tag to be added.
+
+        Returns:
+            The patch builder instance.
+        """
+
+        # TODO: Make this support raw strings, in addition to Tag objects.
+        self._add_patch(
+            GlobalTags.ASPECT_NAME, "add", path=("tags", tag.tag), value=tag
+        )
+        return self
+
+    def remove_tag(self, tag: Union[str, Urn]) -> Self:
+        """Removes a tag from the entity.
+
+        Args:
+            tag: The tag to remove, specified as a string or Urn object.
+
+        Returns:
+            The patch builder instance.
+        """
+        if isinstance(tag, str) and not tag.startswith("urn:li:tag:"):
+            tag = TagUrn.create_from_id(tag)
+        self._add_patch(GlobalTags.ASPECT_NAME, "remove", path=("tags", tag), value={})
+        return self