acryl-datahub 0.15.0.1rc17__py3-none-any.whl → 0.15.0.2__py3-none-any.whl

datahub/metadata/_urns/urn_defs.py

@@ -10,7 +10,7 @@
 
 # This file contains classes corresponding to entity URNs.
 
-from typing import ClassVar, List, Optional, Type, TYPE_CHECKING
+from typing import ClassVar, List, Optional, Type, TYPE_CHECKING, Union
 
 import functools
 from deprecated.sphinx import deprecated as _sphinx_deprecated
@@ -213,7 +213,7 @@ class SchemaFieldUrn(_SpecificUrn):
     ENTITY_TYPE: ClassVar[str] = "schemaField"
     _URN_PARTS: ClassVar[int] = 2
 
-    def __init__(self, parent: str, field_path: str, *, _allow_coercion: bool = True) -> None:
+    def __init__(self, parent: Union["Urn", str], field_path: str, *, _allow_coercion: bool = True) -> None:
         if _allow_coercion:
             # Field coercion logic (if any is required).
             field_path = UrnEncoder.encode_string(field_path)
@@ -604,7 +604,7 @@ class DataJobUrn(_SpecificUrn):
     ENTITY_TYPE: ClassVar[str] = "dataJob"
     _URN_PARTS: ClassVar[int] = 2
 
-    def __init__(self, flow: str, job_id: str, *, _allow_coercion: bool = True) -> None:
+    def __init__(self, flow: Union["DataFlowUrn", str], job_id: str, *, _allow_coercion: bool = True) -> None:
         if _allow_coercion:
             # Field coercion logic (if any is required).
             job_id = UrnEncoder.encode_string(job_id)
@@ -1435,10 +1435,10 @@ class DataPlatformInstanceUrn(_SpecificUrn):
     ENTITY_TYPE: ClassVar[str] = "dataPlatformInstance"
     _URN_PARTS: ClassVar[int] = 2
 
-    def __init__(self, platform: str, instance: str, *, _allow_coercion: bool = True) -> None:
+    def __init__(self, platform: Union["DataPlatformUrn", str], instance: str, *, _allow_coercion: bool = True) -> None:
         if _allow_coercion:
             # Field coercion logic (if any is required).
-            platform = DataPlatformUrn(platform).urn()
+            platform = platform.urn() if isinstance(platform, DataPlatformUrn) else DataPlatformUrn(platform).urn()
             instance = UrnEncoder.encode_string(instance)
 
         # Validation logic.
@@ -1678,10 +1678,10 @@ class DatasetUrn(_SpecificUrn):
     ENTITY_TYPE: ClassVar[str] = "dataset"
     _URN_PARTS: ClassVar[int] = 3
 
-    def __init__(self, platform: str, name: str, env: str = "PROD", *, _allow_coercion: bool = True) -> None:
+    def __init__(self, platform: Union["DataPlatformUrn", str], name: str, env: str = "PROD", *, _allow_coercion: bool = True) -> None:
         if _allow_coercion:
             # Field coercion logic (if any is required).
-            platform = DataPlatformUrn(platform).urn()
+            platform = platform.urn() if isinstance(platform, DataPlatformUrn) else DataPlatformUrn(platform).urn()
             name = UrnEncoder.encode_string(name)
             env = env.upper()
             env = UrnEncoder.encode_string(env)
@@ -1771,10 +1771,10 @@ class MlModelUrn(_SpecificUrn):
     ENTITY_TYPE: ClassVar[str] = "mlModel"
     _URN_PARTS: ClassVar[int] = 3
 
-    def __init__(self, platform: str, name: str, env: str = "PROD", *, _allow_coercion: bool = True) -> None:
+    def __init__(self, platform: Union["DataPlatformUrn", str], name: str, env: str = "PROD", *, _allow_coercion: bool = True) -> None:
         if _allow_coercion:
             # Field coercion logic (if any is required).
-            platform = DataPlatformUrn(platform).urn()
+            platform = platform.urn() if isinstance(platform, DataPlatformUrn) else DataPlatformUrn(platform).urn()
             name = UrnEncoder.encode_string(name)
             env = env.upper()
             env = UrnEncoder.encode_string(env)
@@ -1889,10 +1889,10 @@ class MlModelDeploymentUrn(_SpecificUrn):
     ENTITY_TYPE: ClassVar[str] = "mlModelDeployment"
     _URN_PARTS: ClassVar[int] = 3
 
-    def __init__(self, platform: str, name: str, env: str = "PROD", *, _allow_coercion: bool = True) -> None:
+    def __init__(self, platform: Union["DataPlatformUrn", str], name: str, env: str = "PROD", *, _allow_coercion: bool = True) -> None:
         if _allow_coercion:
             # Field coercion logic (if any is required).
-            platform = DataPlatformUrn(platform).urn()
+            platform = platform.urn() if isinstance(platform, DataPlatformUrn) else DataPlatformUrn(platform).urn()
             name = UrnEncoder.encode_string(name)
             env = env.upper()
             env = UrnEncoder.encode_string(env)
@@ -1953,10 +1953,10 @@ class MlFeatureTableUrn(_SpecificUrn):
     ENTITY_TYPE: ClassVar[str] = "mlFeatureTable"
     _URN_PARTS: ClassVar[int] = 2
 
-    def __init__(self, platform: str, name: str, *, _allow_coercion: bool = True) -> None:
+    def __init__(self, platform: Union["DataPlatformUrn", str], name: str, *, _allow_coercion: bool = True) -> None:
         if _allow_coercion:
             # Field coercion logic (if any is required).
-            platform = DataPlatformUrn(platform).urn()
+            platform = platform.urn() if isinstance(platform, DataPlatformUrn) else DataPlatformUrn(platform).urn()
             name = UrnEncoder.encode_string(name)
 
         # Validation logic.
@@ -2225,6 +2225,60 @@ class DataHubIngestionSourceUrn(_SpecificUrn):
     def id(self) -> str:
         return self.entity_ids[0]
 
+if TYPE_CHECKING:
+    from datahub.metadata.schema_classes import VersionSetKeyClass
+
+class VersionSetUrn(_SpecificUrn):
+    ENTITY_TYPE: ClassVar[str] = "versionSet"
+    _URN_PARTS: ClassVar[int] = 2
+
+    def __init__(self, id: str, entity_type: str, *, _allow_coercion: bool = True) -> None:
+        if _allow_coercion:
+            # Field coercion logic (if any is required).
+            id = UrnEncoder.encode_string(id)
+            entity_type = UrnEncoder.encode_string(entity_type)
+
+        # Validation logic.
+        if not id:
+            raise InvalidUrnError("VersionSetUrn id cannot be empty")
+        if UrnEncoder.contains_reserved_char(id):
+            raise InvalidUrnError(f'VersionSetUrn id contains reserved characters')
+        if not entity_type:
+            raise InvalidUrnError("VersionSetUrn entity_type cannot be empty")
+        if UrnEncoder.contains_reserved_char(entity_type):
+            raise InvalidUrnError(f'VersionSetUrn entity_type contains reserved characters')
+
+        super().__init__(self.ENTITY_TYPE, [id, entity_type])
+
+    @classmethod
+    def _parse_ids(cls, entity_ids: List[str]) -> "VersionSetUrn":
+        if len(entity_ids) != cls._URN_PARTS:
+            raise InvalidUrnError(f"VersionSetUrn should have {cls._URN_PARTS} parts, got {len(entity_ids)}: {entity_ids}")
+        return cls(id=entity_ids[0], entity_type=entity_ids[1], _allow_coercion=False)
+
+    @classmethod
+    def underlying_key_aspect_type(cls) -> Type["VersionSetKeyClass"]:
+        from datahub.metadata.schema_classes import VersionSetKeyClass
+
+        return VersionSetKeyClass
+
+    def to_key_aspect(self) -> "VersionSetKeyClass":
+        from datahub.metadata.schema_classes import VersionSetKeyClass
+
+        return VersionSetKeyClass(id=self.id, entityType=self.entity_type)
+
+    @classmethod
+    def from_key_aspect(cls, key_aspect: "VersionSetKeyClass") -> "VersionSetUrn":
+        return cls(id=key_aspect.id, entity_type=key_aspect.entityType)
+
+    @property
+    def id(self) -> str:
+        return self.entity_ids[0]
+
+    @property
+    def entity_type(self) -> str:
+        return self.entity_ids[1]
+
 if TYPE_CHECKING:
     from datahub.metadata.schema_classes import DataHubRetentionKeyClass
 
@@ -2385,10 +2439,10 @@ class MlModelGroupUrn(_SpecificUrn):
     ENTITY_TYPE: ClassVar[str] = "mlModelGroup"
     _URN_PARTS: ClassVar[int] = 3
 
-    def __init__(self, platform: str, name: str, env: str = "PROD", *, _allow_coercion: bool = True) -> None:
+    def __init__(self, platform: Union["DataPlatformUrn", str], name: str, env: str = "PROD", *, _allow_coercion: bool = True) -> None:
         if _allow_coercion:
             # Field coercion logic (if any is required).
-            platform = DataPlatformUrn(platform).urn()
+            platform = platform.urn() if isinstance(platform, DataPlatformUrn) else DataPlatformUrn(platform).urn()
             name = UrnEncoder.encode_string(name)
             env = env.upper()
             env = UrnEncoder.encode_string(env)

datahub/metadata/com/linkedin/pegasus2avro/common/__init__.py

@@ -65,6 +65,7 @@ from .....schema_classes import StatusClass
 from .....schema_classes import SubTypesClass
 from .....schema_classes import TagAssociationClass
 from .....schema_classes import TimeStampClass
+from .....schema_classes import VersionPropertiesClass
 from .....schema_classes import VersionTagClass
 from .....schema_classes import WindowDurationClass
 
@@ -127,6 +128,7 @@ Status = StatusClass
 SubTypes = SubTypesClass
 TagAssociation = TagAssociationClass
 TimeStamp = TimeStampClass
+VersionProperties = VersionPropertiesClass
 VersionTag = VersionTagClass
 WindowDuration = WindowDurationClass
 

datahub/metadata/com/linkedin/pegasus2avro/metadata/key/__init__.py

@@ -57,6 +57,7 @@ from ......schema_classes import SchemaFieldKeyClass
 from ......schema_classes import TagKeyClass
 from ......schema_classes import TelemetryKeyClass
 from ......schema_classes import TestKeyClass
+from ......schema_classes import VersionSetKeyClass
 
 
 AssertionKey = AssertionKeyClass
@@ -109,5 +110,6 @@ SchemaFieldKey = SchemaFieldKeyClass
 TagKey = TagKeyClass
 TelemetryKey = TelemetryKeyClass
 TestKey = TestKeyClass
+VersionSetKey = VersionSetKeyClass
 
 # fmt: on

datahub/metadata/com/linkedin/pegasus2avro/versionset/__init__.py

@@ -0,0 +1,17 @@
+# mypy: ignore-errors
+# flake8: noqa
+
+# This file is autogenerated by /metadata-ingestion/scripts/avro_codegen.py
+# Do not modify manually!
+
+# pylint: skip-file
+# fmt: off
+# isort: skip_file
+from .....schema_classes import VersionSetPropertiesClass
+from .....schema_classes import VersioningSchemeClass
+
+
+VersionSetProperties = VersionSetPropertiesClass
+VersioningScheme = VersioningSchemeClass
+
+# fmt: on

datahub/metadata/schema.avsc

@@ -2021,6 +2021,136 @@
     ],
     "doc": "Aspect used for storing all applicable documentations on assets.\nThis aspect supports multiple documentations from different sources.\nThere is an implicit assumption that there is only one documentation per\n   source.\nFor example, if there are two documentations from the same source, the\n   latest one will overwrite the previous one.\nIf there are two documentations from different sources, both will be\n   stored.\nFuture evolution considerations:\nThe first entity that uses this aspect is Schema Field. We will expand this\n    aspect to other entities eventually.\nThe values of the documentation are not currently searchable. This will be\n    changed once this aspect develops opinion on which documentation entry is\n    the authoritative one.\nEnsuring that there is only one documentation per source is a business\n    rule that is not enforced by the aspect yet. This will currently be enforced by the\n    application that uses this aspect. We will eventually enforce this rule in\n    the aspect using AspectMutators."
   },
+  {
+    "type": "record",
+    "Aspect": {
+      "name": "versionProperties"
+    },
+    "name": "VersionProperties",
+    "namespace": "com.linkedin.pegasus2avro.common",
+    "fields": [
+      {
+        "Relationship": {
+          "entityTypes": [
+            "versionSet"
+          ],
+          "name": "VersionOf"
+        },
+        "Searchable": {
+          "queryByDefault": false
+        },
+        "java": {
+          "class": "com.linkedin.pegasus2avro.common.urn.Urn"
+        },
+        "Urn": "Urn",
+        "entityTypes": [
+          "versionSet"
+        ],
+        "type": "string",
+        "name": "versionSet",
+        "doc": "The linked Version Set entity that ties multiple versioned assets together"
+      },
+      {
+        "Searchable": {
+          "/versionTag": {
+            "fieldName": "version",
+            "queryByDefault": false
+          }
+        },
+        "type": {
+          "type": "record",
+          "name": "VersionTag",
+          "namespace": "com.linkedin.pegasus2avro.common",
+          "fields": [
+            {
+              "type": [
+                "null",
+                "string"
+              ],
+              "name": "versionTag",
+              "default": null
+            },
+            {
+              "type": [
+                "null",
+                "com.linkedin.pegasus2avro.common.MetadataAttribution"
+              ],
+              "name": "metadataAttribution",
+              "default": null
+            }
+          ],
+          "doc": "A resource-defined string representing the resource state for the purpose of concurrency control"
+        },
+        "name": "version",
+        "doc": "Label for this versioned asset, is unique within a version set"
+      },
+      {
+        "Searchable": {
+          "/*/versionTag": {
+            "fieldName": "aliases",
+            "queryByDefault": false
+          }
+        },
+        "type": {
+          "type": "array",
+          "items": "com.linkedin.pegasus2avro.common.VersionTag"
+        },
+        "name": "aliases",
+        "default": [],
+        "doc": "Associated aliases for this versioned asset"
+      },
+      {
+        "type": [
+          "null",
+          "string"
+        ],
+        "name": "comment",
+        "default": null,
+        "doc": "Comment documenting what this version was created for, changes, or represents"
+      },
+      {
+        "Searchable": {
+          "fieldName": "versionSortId",
+          "queryByDefault": false
+        },
+        "type": "string",
+        "name": "sortId",
+        "doc": "Sort identifier that determines where a version lives in the order of the Version Set.\nWhat this looks like depends on the Version Scheme. For sort ids generated by DataHub we use an 8 character string representation."
+      },
+      {
+        "type": [
+          "null",
+          "com.linkedin.pegasus2avro.common.AuditStamp"
+        ],
+        "name": "sourceCreatedTimestamp",
+        "default": null,
+        "doc": "Timestamp reflecting when this asset version was created in the source system."
+      },
+      {
+        "type": [
+          "null",
+          "com.linkedin.pegasus2avro.common.AuditStamp"
+        ],
+        "name": "metadataCreatedTimestamp",
+        "default": null,
+        "doc": "Timestamp reflecting when the metadata for this version was created in DataHub"
+      },
+      {
+        "Searchable": {
+          "fieldType": "BOOLEAN",
+          "queryByDefault": false
+        },
+        "type": [
+          "null",
+          "boolean"
+        ],
+        "name": "isLatest",
+        "default": null,
+        "doc": "Marks whether this version is currently the latest. Set by a side effect and should not be modified by API."
+      }
+    ],
+    "doc": "Properties about a versioned asset i.e. dataset, ML Model, etc."
+  },
   {
     "type": "record",
     "Aspect": {
@@ -6890,6 +7020,51 @@
                           "default": null,
                           "doc": "URL where the reference exist"
                         },
+                        {
+                          "Relationship": {
+                            "/*": {
+                              "entityTypes": [
+                                "dataJob",
+                                "dataProcessInstance"
+                              ],
+                              "isLineage": true,
+                              "name": "TrainedBy"
+                            }
+                          },
+                          "type": [
+                            "null",
+                            {
+                              "type": "array",
+                              "items": "string"
+                            }
+                          ],
+                          "name": "trainingJobs",
+                          "default": null,
+                          "doc": "List of jobs or process instances (if any) used to train the model or group. Visible in Lineage. Note that ML Models can also be specified as the output of a specific Data Process Instances (runs) via the DataProcessInstanceOutputs aspect."
+                        },
+                        {
+                          "Relationship": {
+                            "/*": {
+                              "entityTypes": [
+                                "dataJob",
+                                "dataProcessInstance"
+                              ],
+                              "isLineage": true,
+                              "isUpstream": false,
+                              "name": "UsedBy"
+                            }
+                          },
+                          "type": [
+                            "null",
+                            {
+                              "type": "array",
+                              "items": "string"
+                            }
+                          ],
+                          "name": "downstreamJobs",
+                          "default": null,
+                          "doc": "List of jobs or process instances (if any) that use the model or group."
+                        },
                         {
                           "Searchable": {
                             "boostScore": 10.0,
@@ -6949,22 +7124,7 @@
                         {
                           "type": [
                             "null",
-                            {
-                              "type": "record",
-                              "name": "VersionTag",
-                              "namespace": "com.linkedin.pegasus2avro.common",
-                              "fields": [
-                                {
-                                  "type": [
-                                    "null",
-                                    "string"
-                                  ],
-                                  "name": "versionTag",
-                                  "default": null
-                                }
-                              ],
-                              "doc": "A resource-defined string representing the resource state for the purpose of concurrency control"
-                            }
+                            "com.linkedin.pegasus2avro.common.VersionTag"
                           ],
                           "name": "version",
                           "default": null,
@@ -7174,54 +7334,6 @@
                           "default": null,
                           "doc": "Deployments for the MLModel"
                         },
-                        {
-                          "Relationship": {
-                            "/*": {
-                              "entityTypes": [
-                                "dataJob",
-                                "dataProcessInstance"
-                              ],
-                              "isLineage": true,
-                              "name": "TrainedBy"
-                            }
-                          },
-                          "Urn": "Urn",
-                          "urn_is_array": true,
-                          "type": [
-                            "null",
-                            {
-                              "type": "array",
-                              "items": "string"
-                            }
-                          ],
-                          "name": "trainingJobs",
-                          "default": null,
-                          "doc": "List of jobs (if any) used to train the model. Visible in Lineage. Note that ML Models can also be specified as the output of a specific Data Process Instances (runs) via the DataProcessInstanceOutputs aspect."
-                        },
-                        {
-                          "Relationship": {
-                            "/*": {
-                              "entityTypes": [
-                                "dataJob"
-                              ],
-                              "isLineage": true,
-                              "isUpstream": false,
-                              "name": "UsedBy"
-                            }
-                          },
-                          "Urn": "Urn",
-                          "urn_is_array": true,
-                          "type": [
-                            "null",
-                            {
-                              "type": "array",
-                              "items": "string"
-                            }
-                          ],
-                          "name": "downstreamJobs",
-                          "default": null,
-                          "doc": "List of jobs (if any) that use the model"
-                        },
                         {
                           "Relationship": {
                             "/*": {
@@ -8513,6 +8625,51 @@
                           "default": {},
                           "doc": "Custom property bag."
                         },
+                        {
+                          "Relationship": {
+                            "/*": {
+                              "entityTypes": [
+                                "dataJob",
+                                "dataProcessInstance"
+                              ],
+                              "isLineage": true,
+                              "name": "TrainedBy"
+                            }
+                          },
+                          "type": [
+                            "null",
+                            {
+                              "type": "array",
+                              "items": "string"
+                            }
+                          ],
+                          "name": "trainingJobs",
+                          "default": null,
+                          "doc": "List of jobs or process instances (if any) used to train the model or group. Visible in Lineage. Note that ML Models can also be specified as the output of a specific Data Process Instances (runs) via the DataProcessInstanceOutputs aspect."
+                        },
+                        {
+                          "Relationship": {
+                            "/*": {
+                              "entityTypes": [
+                                "dataJob",
+                                "dataProcessInstance"
+                              ],
+                              "isLineage": true,
+                              "isUpstream": false,
+                              "name": "UsedBy"
+                            }
+                          },
+                          "type": [
+                            "null",
+                            {
+                              "type": "array",
+                              "items": "string"
+                            }
+                          ],
+                          "name": "downstreamJobs",
+                          "default": null,
+                          "doc": "List of jobs or process instances (if any) that use the model or group."
+                        },
                         {
                           "Searchable": {
                             "boostScore": 10.0,
@@ -8569,29 +8726,6 @@
                           "default": null,
                           "doc": "Date when the MLModelGroup was last modified"
                         },
-                        {
-                          "Relationship": {
-                            "/*": {
-                              "entityTypes": [
-                                "dataJob"
-                              ],
-                              "isLineage": true,
-                              "name": "TrainedBy"
-                            }
-                          },
-                          "Urn": "Urn",
-                          "urn_is_array": true,
-                          "type": [
-                            "null",
-                            {
-                              "type": "array",
-                              "items": "string"
-                            }
-                          ],
-                          "name": "trainingJobs",
-                          "default": null,
-                          "doc": "List of jobs (if any) used to train the model group. Visible in Lineage."
-                        },
                         {
                           "type": [
                             "null",
@@ -12565,8 +12699,10 @@
         "Relationship": {
           "/*": {
             "entityTypes": [
-              "dataset"
+              "dataset",
+              "mlModel"
             ],
+            "isLineage": true,
             "name": "Consumes"
           }
         },
@@ -12586,7 +12722,7 @@
           "items": "string"
         },
         "name": "inputs",
-        "doc": "Input datasets to be consumed"
+        "doc": "Input assets consumed"
       }
     ],
     "doc": "Information about the inputs datasets of a Data process"
@@ -12749,6 +12885,8 @@
               "dataset",
               "mlModel"
             ],
+            "isLineage": true,
+            "isUpstream": false,
             "name": "Produces"
           }
         },
@@ -15527,6 +15665,55 @@
     ],
     "doc": "A simple wrapper around a String to persist the client ID for telemetry in DataHub's backend DB"
   },
+  {
+    "type": "record",
+    "Aspect": {
+      "name": "versionSetProperties"
+    },
+    "name": "VersionSetProperties",
+    "namespace": "com.linkedin.pegasus2avro.versionset",
+    "fields": [
+      {
+        "Searchable": {
+          "/*": {
+            "fieldType": "TEXT",
+            "queryByDefault": true
+          }
+        },
+        "type": {
+          "type": "map",
+          "values": "string"
+        },
+        "name": "customProperties",
+        "default": {},
+        "doc": "Custom property bag."
+      },
+      {
+        "Searchable": {
+          "queryByDefault": "false"
+        },
+        "java": {
+          "class": "com.linkedin.pegasus2avro.common.urn.Urn"
+        },
+        "Urn": "Urn",
+        "type": "string",
+        "name": "latest",
+        "doc": "The latest versioned entity linked to in this version set"
+      },
+      {
+        "type": {
+          "type": "enum",
+          "name": "VersioningScheme",
+          "namespace": "com.linkedin.pegasus2avro.versionset",
+          "symbols": [
+            "ALPHANUMERIC_GENERATED_BY_DATAHUB"
+          ]
+        },
+        "name": "versioningScheme",
+        "doc": "What versioning scheme is being utilized for the versioned entities sort criterion. Static once set"
+      }
+    ]
+  },
   {
     "type": "record",
     "Aspect": {
@@ -16263,6 +16450,32 @@
     ],
     "doc": "Key for a DataHub ingestion source"
   },
+  {
+    "type": "record",
+    "Aspect": {
+      "name": "versionSetKey",
+      "keyForEntity": "versionSet",
+      "entityCategory": "core",
+      "entityAspects": [
+        "versionSetProperties"
+      ]
+    },
+    "name": "VersionSetKey",
+    "namespace": "com.linkedin.pegasus2avro.metadata.key",
+    "fields": [
+      {
+        "type": "string",
+        "name": "id",
+        "doc": "ID of the Version Set, generated from platform + asset id / name"
+      },
+      {
+        "type": "string",
+        "name": "entityType",
+        "doc": "Type of entities included in version set, limits to a single entity type between linked versioned entities"
+      }
+    ],
+    "doc": "Key for a Version Set entity"
+  },
   "com.linkedin.pegasus2avro.metadata.key.DataHubRetentionKey",
   {
     "type": "record",