acryl-datahub 0.15.0rc25__py3-none-any.whl → 0.15.0.1__py3-none-any.whl

datahub/metadata/_schema_classes.py

@@ -4053,6 +4053,60 @@ class DataPlatformInstanceClass(_Aspect):
         self._inner_dict['instance'] = value
     
     
+class DataTransformClass(DictWrapper):
+    """Information about a transformation. It may be a query,"""
+    
+    RECORD_SCHEMA = get_schema_type("com.linkedin.pegasus2avro.common.DataTransform")
+    def __init__(self,
+        queryStatement: Union[None, "QueryStatementClass"]=None,
+    ):
+        super().__init__()
+        
+        self.queryStatement = queryStatement
+    
+    def _restore_defaults(self) -> None:
+        self.queryStatement = self.RECORD_SCHEMA.fields_dict["queryStatement"].default
+    
+    
+    @property
+    def queryStatement(self) -> Union[None, "QueryStatementClass"]:
+        """The data transform may be defined by a query statement"""
+        return self._inner_dict.get('queryStatement')  # type: ignore
+    
+    @queryStatement.setter
+    def queryStatement(self, value: Union[None, "QueryStatementClass"]) -> None:
+        self._inner_dict['queryStatement'] = value
+    
+    
+class DataTransformLogicClass(_Aspect):
+    """Information about a Query against one or more data assets (e.g. Tables or Views)."""
+
+
+    ASPECT_NAME = 'dataTransformLogic'
+    ASPECT_INFO = {}
+    RECORD_SCHEMA = get_schema_type("com.linkedin.pegasus2avro.common.DataTransformLogic")
+
+    def __init__(self,
+        transforms: List["DataTransformClass"],
+    ):
+        super().__init__()
+        
+        self.transforms = transforms
+    
+    def _restore_defaults(self) -> None:
+        self.transforms = list()
+    
+    
+    @property
+    def transforms(self) -> List["DataTransformClass"]:
+        """List of transformations applied"""
+        return self._inner_dict.get('transforms')  # type: ignore
+    
+    @transforms.setter
+    def transforms(self, value: List["DataTransformClass"]) -> None:
+        self._inner_dict['transforms'] = value
+    
+    
 class DeprecationClass(_Aspect):
     """Deprecation status of an entity"""
 
@@ -14624,7 +14678,7 @@ class DataJobKeyClass(_Aspect):
 
 
     ASPECT_NAME = 'dataJobKey'
-    ASPECT_INFO = {'keyForEntity': 'dataJob', 'entityCategory': '_unset_', 'entityAspects': ['datahubIngestionRunSummary', 'datahubIngestionCheckpoint', 'domains', 'deprecation', 'versionInfo', 'dataJobInfo', 'dataJobInputOutput', 'editableDataJobProperties', 'ownership', 'status', 'globalTags', 'browsePaths', 'glossaryTerms', 'institutionalMemory', 'dataPlatformInstance', 'browsePathsV2', 'structuredProperties', 'forms', 'subTypes', 'incidentsSummary', 'testResults']}
+    ASPECT_INFO = {'keyForEntity': 'dataJob', 'entityCategory': '_unset_', 'entityAspects': ['datahubIngestionRunSummary', 'datahubIngestionCheckpoint', 'domains', 'deprecation', 'versionInfo', 'dataJobInfo', 'dataJobInputOutput', 'editableDataJobProperties', 'ownership', 'status', 'globalTags', 'browsePaths', 'glossaryTerms', 'institutionalMemory', 'dataPlatformInstance', 'browsePathsV2', 'structuredProperties', 'forms', 'subTypes', 'incidentsSummary', 'testResults', 'dataTransformLogic']}
     RECORD_SCHEMA = get_schema_type("com.linkedin.pegasus2avro.metadata.key.DataJobKey")
 
     def __init__(self,
@@ -14737,7 +14791,7 @@ class DataProcessInstanceKeyClass(_Aspect):
 
 
     ASPECT_NAME = 'dataProcessInstanceKey'
-    ASPECT_INFO = {'keyForEntity': 'dataProcessInstance', 'entityCategory': '_unset_', 'entityAspects': ['dataProcessInstanceInput', 'dataProcessInstanceOutput', 'dataProcessInstanceProperties', 'dataProcessInstanceRelationships', 'dataProcessInstanceRunEvent', 'status', 'testResults'], 'entityDoc': 'DataProcessInstance represents an instance of a datajob/jobflow run'}
+    ASPECT_INFO = {'keyForEntity': 'dataProcessInstance', 'entityCategory': '_unset_', 'entityAspects': ['dataProcessInstanceInput', 'dataProcessInstanceOutput', 'dataProcessInstanceProperties', 'dataProcessInstanceRelationships', 'dataProcessInstanceRunEvent', 'status', 'testResults', 'dataPlatformInstance', 'subTypes', 'container', 'mlTrainingRunProperties'], 'entityDoc': 'DataProcessInstance represents an instance of a datajob/jobflow run'}
     RECORD_SCHEMA = get_schema_type("com.linkedin.pegasus2avro.metadata.key.DataProcessInstanceKey")
 
     def __init__(self,
@@ -17715,8 +17769,12 @@ class MLModelGroupPropertiesClass(_Aspect):
 
     def __init__(self,
         customProperties: Optional[Dict[str, str]]=None,
+        name: Union[None, str]=None,
         description: Union[None, str]=None,
         createdAt: Union[None, int]=None,
+        created: Union[None, "TimeStampClass"]=None,
+        lastModified: Union[None, "TimeStampClass"]=None,
+        trainingJobs: Union[None, List[str]]=None,
         version: Union[None, "VersionTagClass"]=None,
     ):
         super().__init__()
@@ -17726,14 +17784,22 @@ class MLModelGroupPropertiesClass(_Aspect):
             self.customProperties = dict()
         else:
             self.customProperties = customProperties
+        self.name = name
         self.description = description
         self.createdAt = createdAt
+        self.created = created
+        self.lastModified = lastModified
+        self.trainingJobs = trainingJobs
         self.version = version
     
     def _restore_defaults(self) -> None:
         self.customProperties = dict()
+        self.name = self.RECORD_SCHEMA.fields_dict["name"].default
         self.description = self.RECORD_SCHEMA.fields_dict["description"].default
         self.createdAt = self.RECORD_SCHEMA.fields_dict["createdAt"].default
+        self.created = self.RECORD_SCHEMA.fields_dict["created"].default
+        self.lastModified = self.RECORD_SCHEMA.fields_dict["lastModified"].default
+        self.trainingJobs = self.RECORD_SCHEMA.fields_dict["trainingJobs"].default
         self.version = self.RECORD_SCHEMA.fields_dict["version"].default
     
     
@@ -17747,6 +17813,16 @@ class MLModelGroupPropertiesClass(_Aspect):
         self._inner_dict['customProperties'] = value
     
     
+    @property
+    def name(self) -> Union[None, str]:
+        """Display name of the MLModelGroup"""
+        return self._inner_dict.get('name')  # type: ignore
+    
+    @name.setter
+    def name(self, value: Union[None, str]) -> None:
+        self._inner_dict['name'] = value
+    
+    
     @property
     def description(self) -> Union[None, str]:
         """Documentation of the MLModelGroup"""
@@ -17767,6 +17843,36 @@ class MLModelGroupPropertiesClass(_Aspect):
         self._inner_dict['createdAt'] = value
     
     
+    @property
+    def created(self) -> Union[None, "TimeStampClass"]:
+        """Time and Actor who created the MLModelGroup"""
+        return self._inner_dict.get('created')  # type: ignore
+    
+    @created.setter
+    def created(self, value: Union[None, "TimeStampClass"]) -> None:
+        self._inner_dict['created'] = value
+    
+    
+    @property
+    def lastModified(self) -> Union[None, "TimeStampClass"]:
+        """Date when the MLModelGroup was last modified"""
+        return self._inner_dict.get('lastModified')  # type: ignore
+    
+    @lastModified.setter
+    def lastModified(self, value: Union[None, "TimeStampClass"]) -> None:
+        self._inner_dict['lastModified'] = value
+    
+    
+    @property
+    def trainingJobs(self) -> Union[None, List[str]]:
+        """List of jobs (if any) used to train the model group. Visible in Lineage."""
+        return self._inner_dict.get('trainingJobs')  # type: ignore
+    
+    @trainingJobs.setter
+    def trainingJobs(self, value: Union[None, List[str]]) -> None:
+        self._inner_dict['trainingJobs'] = value
+    
+    
     @property
     def version(self) -> Union[None, "VersionTagClass"]:
         """Version of the MLModelGroup"""
@@ -17788,8 +17894,11 @@ class MLModelPropertiesClass(_Aspect):
     def __init__(self,
         customProperties: Optional[Dict[str, str]]=None,
         externalUrl: Union[None, str]=None,
+        name: Union[None, str]=None,
         description: Union[None, str]=None,
         date: Union[None, int]=None,
+        created: Union[None, "TimeStampClass"]=None,
+        lastModified: Union[None, "TimeStampClass"]=None,
         version: Union[None, "VersionTagClass"]=None,
         type: Union[None, str]=None,
         hyperParameters: Union[None, Dict[str, Union[str, int, float, float, bool]]]=None,
@@ -17811,8 +17920,11 @@ class MLModelPropertiesClass(_Aspect):
         else:
             self.customProperties = customProperties
         self.externalUrl = externalUrl
+        self.name = name
         self.description = description
         self.date = date
+        self.created = created
+        self.lastModified = lastModified
         self.version = version
         self.type = type
         self.hyperParameters = hyperParameters
@@ -17833,8 +17945,11 @@ class MLModelPropertiesClass(_Aspect):
     def _restore_defaults(self) -> None:
         self.customProperties = dict()
         self.externalUrl = self.RECORD_SCHEMA.fields_dict["externalUrl"].default
+        self.name = self.RECORD_SCHEMA.fields_dict["name"].default
         self.description = self.RECORD_SCHEMA.fields_dict["description"].default
         self.date = self.RECORD_SCHEMA.fields_dict["date"].default
+        self.created = self.RECORD_SCHEMA.fields_dict["created"].default
+        self.lastModified = self.RECORD_SCHEMA.fields_dict["lastModified"].default
         self.version = self.RECORD_SCHEMA.fields_dict["version"].default
         self.type = self.RECORD_SCHEMA.fields_dict["type"].default
         self.hyperParameters = self.RECORD_SCHEMA.fields_dict["hyperParameters"].default
@@ -17869,6 +17984,16 @@ class MLModelPropertiesClass(_Aspect):
         self._inner_dict['externalUrl'] = value
     
     
+    @property
+    def name(self) -> Union[None, str]:
+        """Display name of the MLModel"""
+        return self._inner_dict.get('name')  # type: ignore
+    
+    @name.setter
+    def name(self, value: Union[None, str]) -> None:
+        self._inner_dict['name'] = value
+    
+    
     @property
     def description(self) -> Union[None, str]:
         """Documentation of the MLModel"""
@@ -17889,6 +18014,26 @@ class MLModelPropertiesClass(_Aspect):
         self._inner_dict['date'] = value
     
     
+    @property
+    def created(self) -> Union[None, "TimeStampClass"]:
+        """Audit stamp containing who created this and when"""
+        return self._inner_dict.get('created')  # type: ignore
+    
+    @created.setter
+    def created(self, value: Union[None, "TimeStampClass"]) -> None:
+        self._inner_dict['created'] = value
+    
+    
+    @property
+    def lastModified(self) -> Union[None, "TimeStampClass"]:
+        """Date when the MLModel was last modified"""
+        return self._inner_dict.get('lastModified')  # type: ignore
+    
+    @lastModified.setter
+    def lastModified(self, value: Union[None, "TimeStampClass"]) -> None:
+        self._inner_dict['lastModified'] = value
+    
+    
     @property
     def version(self) -> Union[None, "VersionTagClass"]:
         """Version of the MLModel"""
@@ -17983,7 +18128,7 @@ class MLModelPropertiesClass(_Aspect):
     
     @property
     def trainingJobs(self) -> Union[None, List[str]]:
-        """List of jobs (if any) used to train the model"""
+        """List of jobs (if any) used to train the model. Visible in Lineage. Note that ML Models can also be specified as the output of a specific Data Process Instances (runs) via the DataProcessInstanceOutputs aspect."""
         return self._inner_dict.get('trainingJobs')  # type: ignore
     
     @trainingJobs.setter
@@ -18096,6 +18241,104 @@ class MLPrimaryKeyPropertiesClass(_Aspect):
         self._inner_dict['sources'] = value
     
     
+class MLTrainingRunPropertiesClass(_Aspect):
+    """The inputs and outputs of this training run"""
+
+
+    ASPECT_NAME = 'mlTrainingRunProperties'
+    ASPECT_INFO = {}
+    RECORD_SCHEMA = get_schema_type("com.linkedin.pegasus2avro.ml.metadata.MLTrainingRunProperties")
+
+    def __init__(self,
+        customProperties: Optional[Dict[str, str]]=None,
+        externalUrl: Union[None, str]=None,
+        id: Union[None, str]=None,
+        outputUrls: Union[None, List[str]]=None,
+        hyperParams: Union[None, List["MLHyperParamClass"]]=None,
+        trainingMetrics: Union[None, List["MLMetricClass"]]=None,
+    ):
+        super().__init__()
+        
+        if customProperties is None:
+            # default: {}
+            self.customProperties = dict()
+        else:
+            self.customProperties = customProperties
+        self.externalUrl = externalUrl
+        self.id = id
+        self.outputUrls = outputUrls
+        self.hyperParams = hyperParams
+        self.trainingMetrics = trainingMetrics
+    
+    def _restore_defaults(self) -> None:
+        self.customProperties = dict()
+        self.externalUrl = self.RECORD_SCHEMA.fields_dict["externalUrl"].default
+        self.id = self.RECORD_SCHEMA.fields_dict["id"].default
+        self.outputUrls = self.RECORD_SCHEMA.fields_dict["outputUrls"].default
+        self.hyperParams = self.RECORD_SCHEMA.fields_dict["hyperParams"].default
+        self.trainingMetrics = self.RECORD_SCHEMA.fields_dict["trainingMetrics"].default
+    
+    
+    @property
+    def customProperties(self) -> Dict[str, str]:
+        """Custom property bag."""
+        return self._inner_dict.get('customProperties')  # type: ignore
+    
+    @customProperties.setter
+    def customProperties(self, value: Dict[str, str]) -> None:
+        self._inner_dict['customProperties'] = value
+    
+    
+    @property
+    def externalUrl(self) -> Union[None, str]:
+        """URL where the reference exist"""
+        return self._inner_dict.get('externalUrl')  # type: ignore
+    
+    @externalUrl.setter
+    def externalUrl(self, value: Union[None, str]) -> None:
+        self._inner_dict['externalUrl'] = value
+    
+    
+    @property
+    def id(self) -> Union[None, str]:
+        """Run Id of the ML Training Run"""
+        return self._inner_dict.get('id')  # type: ignore
+    
+    @id.setter
+    def id(self, value: Union[None, str]) -> None:
+        self._inner_dict['id'] = value
+    
+    
+    @property
+    def outputUrls(self) -> Union[None, List[str]]:
+        """List of URLs for the Outputs of the ML Training Run"""
+        return self._inner_dict.get('outputUrls')  # type: ignore
+    
+    @outputUrls.setter
+    def outputUrls(self, value: Union[None, List[str]]) -> None:
+        self._inner_dict['outputUrls'] = value
+    
+    
+    @property
+    def hyperParams(self) -> Union[None, List["MLHyperParamClass"]]:
+        """Hyperparameters of the ML Training Run"""
+        return self._inner_dict.get('hyperParams')  # type: ignore
+    
+    @hyperParams.setter
+    def hyperParams(self, value: Union[None, List["MLHyperParamClass"]]) -> None:
+        self._inner_dict['hyperParams'] = value
+    
+    
+    @property
+    def trainingMetrics(self) -> Union[None, List["MLMetricClass"]]:
+        """Metrics of the ML Training Run"""
+        return self._inner_dict.get('trainingMetrics')  # type: ignore
+    
+    @trainingMetrics.setter
+    def trainingMetrics(self, value: Union[None, List["MLMetricClass"]]) -> None:
+        self._inner_dict['trainingMetrics'] = value
+    
+    
 class MetricsClass(_Aspect):
     """Metrics to be featured for the MLModel."""
 
@@ -24526,6 +24769,8 @@ __SCHEMA_TYPES = {
     'com.linkedin.pegasus2avro.common.CostCostDiscriminator': CostCostDiscriminatorClass,
     'com.linkedin.pegasus2avro.common.CostType': CostTypeClass,
     'com.linkedin.pegasus2avro.common.DataPlatformInstance': DataPlatformInstanceClass,
+    'com.linkedin.pegasus2avro.common.DataTransform': DataTransformClass,
+    'com.linkedin.pegasus2avro.common.DataTransformLogic': DataTransformLogicClass,
     'com.linkedin.pegasus2avro.common.Deprecation': DeprecationClass,
     'com.linkedin.pegasus2avro.common.Documentation': DocumentationClass,
     'com.linkedin.pegasus2avro.common.DocumentationAssociation': DocumentationAssociationClass,
@@ -24791,6 +25036,7 @@ __SCHEMA_TYPES = {
     'com.linkedin.pegasus2avro.ml.metadata.MLModelGroupProperties': MLModelGroupPropertiesClass,
     'com.linkedin.pegasus2avro.ml.metadata.MLModelProperties': MLModelPropertiesClass,
     'com.linkedin.pegasus2avro.ml.metadata.MLPrimaryKeyProperties': MLPrimaryKeyPropertiesClass,
+    'com.linkedin.pegasus2avro.ml.metadata.MLTrainingRunProperties': MLTrainingRunPropertiesClass,
     'com.linkedin.pegasus2avro.ml.metadata.Metrics': MetricsClass,
     'com.linkedin.pegasus2avro.ml.metadata.QuantitativeAnalyses': QuantitativeAnalysesClass,
     'com.linkedin.pegasus2avro.ml.metadata.SourceCode': SourceCodeClass,
@@ -24992,6 +25238,8 @@ __SCHEMA_TYPES = {
     'CostCostDiscriminator': CostCostDiscriminatorClass,
     'CostType': CostTypeClass,
     'DataPlatformInstance': DataPlatformInstanceClass,
+    'DataTransform': DataTransformClass,
+    'DataTransformLogic': DataTransformLogicClass,
     'Deprecation': DeprecationClass,
     'Documentation': DocumentationClass,
     'DocumentationAssociation': DocumentationAssociationClass,
@@ -25257,6 +25505,7 @@ __SCHEMA_TYPES = {
     'MLModelGroupProperties': MLModelGroupPropertiesClass,
     'MLModelProperties': MLModelPropertiesClass,
     'MLPrimaryKeyProperties': MLPrimaryKeyPropertiesClass,
+    'MLTrainingRunProperties': MLTrainingRunPropertiesClass,
     'Metrics': MetricsClass,
     'QuantitativeAnalyses': QuantitativeAnalysesClass,
     'SourceCode': SourceCodeClass,
@@ -25397,6 +25646,7 @@ ASPECT_CLASSES: List[Type[_Aspect]] = [
     CostClass,
     BrowsePathsClass,
     InstitutionalMemoryClass,
+    DataTransformLogicClass,
     SubTypesClass,
     FormsClass,
     DeprecationClass,
@@ -25473,6 +25723,7 @@ ASPECT_CLASSES: List[Type[_Aspect]] = [
     MetricsClass,
     EditableMLPrimaryKeyPropertiesClass,
     SourceCodeClass,
+    MLTrainingRunPropertiesClass,
     EthicalConsiderationsClass,
     MLPrimaryKeyPropertiesClass,
     MLModelFactorPromptsClass,
@@ -25610,6 +25861,7 @@ class AspectBag(TypedDict, total=False):
     cost: CostClass
     browsePaths: BrowsePathsClass
     institutionalMemory: InstitutionalMemoryClass
+    dataTransformLogic: DataTransformLogicClass
     subTypes: SubTypesClass
     forms: FormsClass
     deprecation: DeprecationClass
@@ -25686,6 +25938,7 @@ class AspectBag(TypedDict, total=False):
     mlModelMetrics: MetricsClass
     editableMlPrimaryKeyProperties: EditableMLPrimaryKeyPropertiesClass
     sourceCode: SourceCodeClass
+    mlTrainingRunProperties: MLTrainingRunPropertiesClass
     mlModelEthicalConsiderations: EthicalConsiderationsClass
     mlPrimaryKeyProperties: MLPrimaryKeyPropertiesClass
     mlModelFactorPrompts: MLModelFactorPromptsClass