acryl-datahub-cloud 0.3.7.8rc3__tar.gz → 0.3.7.9__tar.gz

{acryl_datahub_cloud-0.3.7.8rc3 → acryl_datahub_cloud-0.3.7.9}/PKG-INFO

@@ -1,80 +1,82 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.2
 Name: acryl-datahub-cloud
-Version: 0.3.7.8rc3
+Version: 0.3.7.9
 Requires-Dist: avro-gen3==0.7.16
 Requires-Dist: acryl-datahub
 Provides-Extra: datahub-lineage-features
-Requires-Dist: opensearch-py==2.4.2; extra == "datahub-lineage-features"
 Requires-Dist: duckdb; extra == "datahub-lineage-features"
 Requires-Dist: pandas; extra == "datahub-lineage-features"
-Requires-Dist: pydantic<2; extra == "datahub-lineage-features"
 Requires-Dist: pyarrow; extra == "datahub-lineage-features"
+Requires-Dist: pydantic<2; extra == "datahub-lineage-features"
+Requires-Dist: opensearch-py==2.4.2; extra == "datahub-lineage-features"
 Provides-Extra: datahub-reporting-forms
-Requires-Dist: boto3; extra == "datahub-reporting-forms"
 Requires-Dist: duckdb; extra == "datahub-reporting-forms"
 Requires-Dist: pandas; extra == "datahub-reporting-forms"
-Requires-Dist: pydantic<2; extra == "datahub-reporting-forms"
 Requires-Dist: pyarrow; extra == "datahub-reporting-forms"
+Requires-Dist: pydantic<2; extra == "datahub-reporting-forms"
+Requires-Dist: boto3; extra == "datahub-reporting-forms"
 Provides-Extra: datahub-reporting-extract-graph
-Requires-Dist: boto3; extra == "datahub-reporting-extract-graph"
-Requires-Dist: opensearch-py==2.4.2; extra == "datahub-reporting-extract-graph"
 Requires-Dist: duckdb; extra == "datahub-reporting-extract-graph"
 Requires-Dist: pandas; extra == "datahub-reporting-extract-graph"
-Requires-Dist: pydantic<2; extra == "datahub-reporting-extract-graph"
 Requires-Dist: pyarrow; extra == "datahub-reporting-extract-graph"
+Requires-Dist: pydantic<2; extra == "datahub-reporting-extract-graph"
+Requires-Dist: boto3; extra == "datahub-reporting-extract-graph"
+Requires-Dist: opensearch-py==2.4.2; extra == "datahub-reporting-extract-graph"
 Provides-Extra: datahub-reporting-extract-sql
-Requires-Dist: boto3; extra == "datahub-reporting-extract-sql"
 Requires-Dist: duckdb; extra == "datahub-reporting-extract-sql"
 Requires-Dist: pandas; extra == "datahub-reporting-extract-sql"
-Requires-Dist: pydantic<2; extra == "datahub-reporting-extract-sql"
 Requires-Dist: pyarrow; extra == "datahub-reporting-extract-sql"
+Requires-Dist: pydantic<2; extra == "datahub-reporting-extract-sql"
+Requires-Dist: boto3; extra == "datahub-reporting-extract-sql"
 Provides-Extra: datahub-usage-feature-reporting
-Requires-Dist: boto3; extra == "datahub-usage-feature-reporting"
-Requires-Dist: numpy<2; extra == "datahub-usage-feature-reporting"
-Requires-Dist: opensearch-py==2.4.2; extra == "datahub-usage-feature-reporting"
-Requires-Dist: scipy<=1.14.1; extra == "datahub-usage-feature-reporting"
 Requires-Dist: duckdb; extra == "datahub-usage-feature-reporting"
+Requires-Dist: pydantic<2; extra == "datahub-usage-feature-reporting"
+Requires-Dist: scipy<=1.14.1; extra == "datahub-usage-feature-reporting"
+Requires-Dist: opensearch-py==2.4.2; extra == "datahub-usage-feature-reporting"
+Requires-Dist: elasticsearch==7.13.4; extra == "datahub-usage-feature-reporting"
 Requires-Dist: pandas; extra == "datahub-usage-feature-reporting"
 Requires-Dist: pyarrow; extra == "datahub-usage-feature-reporting"
+Requires-Dist: boto3; extra == "datahub-usage-feature-reporting"
 Requires-Dist: pyarrow<=18.0.0; extra == "datahub-usage-feature-reporting"
+Requires-Dist: numpy<2; extra == "datahub-usage-feature-reporting"
 Requires-Dist: polars<=1.16.0; extra == "datahub-usage-feature-reporting"
-Requires-Dist: elasticsearch==7.13.4; extra == "datahub-usage-feature-reporting"
-Requires-Dist: pydantic<2; extra == "datahub-usage-feature-reporting"
 Provides-Extra: acryl-cs-issues
+Requires-Dist: slack-sdk; extra == "acryl-cs-issues"
 Requires-Dist: jinja2; extra == "acryl-cs-issues"
-Requires-Dist: openai; extra == "acryl-cs-issues"
 Requires-Dist: zenpy; extra == "acryl-cs-issues"
-Requires-Dist: slack-sdk; extra == "acryl-cs-issues"
+Requires-Dist: openai; extra == "acryl-cs-issues"
 Provides-Extra: all
-Requires-Dist: boto3; extra == "all"
-Requires-Dist: numpy<2; extra == "all"
-Requires-Dist: opensearch-py==2.4.2; extra == "all"
-Requires-Dist: scipy<=1.14.1; extra == "all"
-Requires-Dist: slack-sdk; extra == "all"
 Requires-Dist: duckdb; extra == "all"
-Requires-Dist: pandas; extra == "all"
+Requires-Dist: pydantic<2; extra == "all"
+Requires-Dist: scipy<=1.14.1; extra == "all"
+Requires-Dist: opensearch-py==2.4.2; extra == "all"
+Requires-Dist: elasticsearch==7.13.4; extra == "all"
 Requires-Dist: zenpy; extra == "all"
 Requires-Dist: jinja2; extra == "all"
 Requires-Dist: openai; extra == "all"
+Requires-Dist: pandas; extra == "all"
 Requires-Dist: pyarrow; extra == "all"
+Requires-Dist: boto3; extra == "all"
 Requires-Dist: pyarrow<=18.0.0; extra == "all"
+Requires-Dist: slack-sdk; extra == "all"
+Requires-Dist: numpy<2; extra == "all"
 Requires-Dist: polars<=1.16.0; extra == "all"
-Requires-Dist: elasticsearch==7.13.4; extra == "all"
-Requires-Dist: pydantic<2; extra == "all"
 Provides-Extra: dev
-Requires-Dist: boto3; extra == "dev"
-Requires-Dist: numpy<2; extra == "dev"
-Requires-Dist: opensearch-py==2.4.2; extra == "dev"
-Requires-Dist: scipy<=1.14.1; extra == "dev"
-Requires-Dist: slack-sdk; extra == "dev"
 Requires-Dist: duckdb; extra == "dev"
-Requires-Dist: pandas; extra == "dev"
+Requires-Dist: pydantic<2; extra == "dev"
+Requires-Dist: scipy<=1.14.1; extra == "dev"
+Requires-Dist: opensearch-py==2.4.2; extra == "dev"
+Requires-Dist: elasticsearch==7.13.4; extra == "dev"
 Requires-Dist: zenpy; extra == "dev"
+Requires-Dist: acryl-datahub[dev]; extra == "dev"
 Requires-Dist: jinja2; extra == "dev"
-Requires-Dist: openai; extra == "dev"
+Requires-Dist: pandas; extra == "dev"
 Requires-Dist: pyarrow; extra == "dev"
+Requires-Dist: boto3; extra == "dev"
 Requires-Dist: pyarrow<=18.0.0; extra == "dev"
-Requires-Dist: acryl-datahub[dev]; extra == "dev"
+Requires-Dist: slack-sdk; extra == "dev"
+Requires-Dist: openai; extra == "dev"
+Requires-Dist: numpy<2; extra == "dev"
 Requires-Dist: polars<=1.16.0; extra == "dev"
-Requires-Dist: elasticsearch==7.13.4; extra == "dev"
-Requires-Dist: pydantic<2; extra == "dev"
+Dynamic: provides-extra
+Dynamic: requires-dist

{acryl_datahub_cloud-0.3.7.8rc3 → acryl_datahub_cloud-0.3.7.9}/src/acryl_datahub_cloud/_codegen_config.json

@@ -1,6 +1,6 @@
 {
   "name": "acryl-datahub-cloud",
-  "version": "0.3.7.8rc3",
+  "version": "0.3.7.9",
   "install_requires": [
     "avro-gen3==0.7.16",
     "acryl-datahub"

{acryl_datahub_cloud-0.3.7.8rc3 → acryl_datahub_cloud-0.3.7.9}/src/acryl_datahub_cloud/metadata/com/linkedin/pegasus2avro/ml/metadata/__init__.py

@@ -30,6 +30,7 @@ from ......schema_classes import MLModelFactorsClass
 from ......schema_classes import MLModelGroupPropertiesClass
 from ......schema_classes import MLModelPropertiesClass
 from ......schema_classes import MLPrimaryKeyPropertiesClass
+from ......schema_classes import MLTrainingRunPropertiesClass
 from ......schema_classes import MetricsClass
 from ......schema_classes import QuantitativeAnalysesClass
 from ......schema_classes import SourceCodeClass
@@ -61,6 +62,7 @@ MLModelFactors = MLModelFactorsClass
 MLModelGroupProperties = MLModelGroupPropertiesClass
 MLModelProperties = MLModelPropertiesClass
 MLPrimaryKeyProperties = MLPrimaryKeyPropertiesClass
+MLTrainingRunProperties = MLTrainingRunPropertiesClass
 Metrics = MetricsClass
 QuantitativeAnalyses = QuantitativeAnalysesClass
 SourceCode = SourceCodeClass

{acryl_datahub_cloud-0.3.7.8rc3 → acryl_datahub_cloud-0.3.7.9}/src/acryl_datahub_cloud/metadata/schema.avsc

@@ -8799,6 +8799,66 @@
                           "default": null,
                           "doc": "URL where the reference exist"
                         },
+                        {
+                          "Relationship": {
+                            "/*": {
+                              "entityTypes": [
+                                "dataJob",
+                                "dataProcessInstance"
+                              ],
+                              "isLineage": true,
+                              "name": "TrainedBy"
+                            }
+                          },
+                          "type": [
+                            "null",
+                            {
+                              "type": "array",
+                              "items": "string"
+                            }
+                          ],
+                          "name": "trainingJobs",
+                          "default": null,
+                          "doc": "List of jobs or process instances (if any) used to train the model or group. Visible in Lineage. Note that ML Models can also be specified as the output of a specific Data Process Instances (runs) via the DataProcessInstanceOutputs aspect."
+                        },
+                        {
+                          "Relationship": {
+                            "/*": {
+                              "entityTypes": [
+                                "dataJob",
+                                "dataProcessInstance"
+                              ],
+                              "isLineage": true,
+                              "isUpstream": false,
+                              "name": "UsedBy"
+                            }
+                          },
+                          "type": [
+                            "null",
+                            {
+                              "type": "array",
+                              "items": "string"
+                            }
+                          ],
+                          "name": "downstreamJobs",
+                          "default": null,
+                          "doc": "List of jobs or process instances (if any) that use the model or group."
+                        },
+                        {
+                          "Searchable": {
+                            "boostScore": 10.0,
+                            "enableAutocomplete": true,
+                            "fieldType": "WORD_GRAM",
+                            "queryByDefault": true
+                          },
+                          "type": [
+                            "null",
+                            "string"
+                          ],
+                          "name": "name",
+                          "default": null,
+                          "doc": "Display name of the MLModel"
+                        },
                         {
                           "Searchable": {
                             "fieldType": "TEXT",
@@ -8813,6 +8873,7 @@
                           "doc": "Documentation of the MLModel"
                         },
                         {
+                          "deprecated": true,
                           "type": [
                             "null",
                             "long"
@@ -8821,6 +8882,24 @@
                           "default": null,
                           "doc": "Date when the MLModel was developed"
                         },
+                        {
+                          "type": [
+                            "null",
+                            "com.linkedin.pegasus2avro.common.TimeStamp"
+                          ],
+                          "name": "created",
+                          "default": null,
+                          "doc": "Audit stamp containing who created this and when"
+                        },
+                        {
+                          "type": [
+                            "null",
+                            "com.linkedin.pegasus2avro.common.TimeStamp"
+                          ],
+                          "name": "lastModified",
+                          "default": null,
+                          "doc": "Date when the MLModel was last modified"
+                        },
                         {
                           "type": [
                             "null",
@@ -9049,53 +9128,6 @@
                           "default": null,
                           "doc": "Deployments for the MLModel"
                         },
-                        {
-                          "Relationship": {
-                            "/*": {
-                              "entityTypes": [
-                                "dataJob"
-                              ],
-                              "isLineage": true,
-                              "name": "TrainedBy"
-                            }
-                          },
-                          "Urn": "Urn",
-                          "urn_is_array": true,
-                          "type": [
-                            "null",
-                            {
-                              "type": "array",
-                              "items": "string"
-                            }
-                          ],
-                          "name": "trainingJobs",
-                          "default": null,
-                          "doc": "List of jobs (if any) used to train the model"
-                        },
-                        {
-                          "Relationship": {
-                            "/*": {
-                              "entityTypes": [
-                                "dataJob"
-                              ],
-                              "isLineage": true,
-                              "isUpstream": false,
-                              "name": "UsedBy"
-                            }
-                          },
-                          "Urn": "Urn",
-                          "urn_is_array": true,
-                          "type": [
-                            "null",
-                            {
-                              "type": "array",
-                              "items": "string"
-                            }
-                          ],
-                          "name": "downstreamJobs",
-                          "default": null,
-                          "doc": "List of jobs (if any) that use the model"
-                        },
                         {
                           "Relationship": {
                             "/*": {
@@ -10391,6 +10423,66 @@
                           "default": {},
                           "doc": "Custom property bag."
                         },
+                        {
+                          "Relationship": {
+                            "/*": {
+                              "entityTypes": [
+                                "dataJob",
+                                "dataProcessInstance"
+                              ],
+                              "isLineage": true,
+                              "name": "TrainedBy"
+                            }
+                          },
+                          "type": [
+                            "null",
+                            {
+                              "type": "array",
+                              "items": "string"
+                            }
+                          ],
+                          "name": "trainingJobs",
+                          "default": null,
+                          "doc": "List of jobs or process instances (if any) used to train the model or group. Visible in Lineage. Note that ML Models can also be specified as the output of a specific Data Process Instances (runs) via the DataProcessInstanceOutputs aspect."
+                        },
+                        {
+                          "Relationship": {
+                            "/*": {
+                              "entityTypes": [
+                                "dataJob",
+                                "dataProcessInstance"
+                              ],
+                              "isLineage": true,
+                              "isUpstream": false,
+                              "name": "UsedBy"
+                            }
+                          },
+                          "type": [
+                            "null",
+                            {
+                              "type": "array",
+                              "items": "string"
+                            }
+                          ],
+                          "name": "downstreamJobs",
+                          "default": null,
+                          "doc": "List of jobs or process instances (if any) that use the model or group."
+                        },
+                        {
+                          "Searchable": {
+                            "boostScore": 10.0,
+                            "enableAutocomplete": true,
+                            "fieldType": "WORD_GRAM",
+                            "queryByDefault": true
+                          },
+                          "type": [
+                            "null",
+                            "string"
+                          ],
+                          "name": "name",
+                          "default": null,
+                          "doc": "Display name of the MLModelGroup"
+                        },
                         {
                           "Searchable": {
                             "fieldType": "TEXT",
@@ -10405,6 +10497,7 @@
                           "doc": "Documentation of the MLModelGroup"
                         },
                         {
+                          "deprecated": true,
                           "type": [
                             "null",
                             "long"
@@ -10413,6 +10506,24 @@
                           "default": null,
                           "doc": "Date when the MLModelGroup was developed"
                         },
+                        {
+                          "type": [
+                            "null",
+                            "com.linkedin.pegasus2avro.common.TimeStamp"
+                          ],
+                          "name": "created",
+                          "default": null,
+                          "doc": "Time and Actor who created the MLModelGroup"
+                        },
+                        {
+                          "type": [
+                            "null",
+                            "com.linkedin.pegasus2avro.common.TimeStamp"
+                          ],
+                          "name": "lastModified",
+                          "default": null,
+                          "doc": "Date when the MLModelGroup was last modified"
+                        },
                         {
                           "type": [
                             "null",
@@ -18109,7 +18220,8 @@
         "Relationship": {
           "/*": {
             "entityTypes": [
-              "dataset"
+              "dataset",
+              "mlModel"
             ],
             "name": "Produces"
           }
@@ -18438,6 +18550,93 @@
     "doc": "Properties associated with a MLPrimaryKey editable from the UI"
   },
   "com.linkedin.pegasus2avro.ml.metadata.SourceCode",
+  {
+    "type": "record",
+    "Aspect": {
+      "name": "mlTrainingRunProperties"
+    },
+    "name": "MLTrainingRunProperties",
+    "namespace": "com.linkedin.pegasus2avro.ml.metadata",
+    "fields": [
+      {
+        "Searchable": {
+          "/*": {
+            "fieldType": "TEXT",
+            "queryByDefault": true
+          }
+        },
+        "type": {
+          "type": "map",
+          "values": "string"
+        },
+        "name": "customProperties",
+        "default": {},
+        "doc": "Custom property bag."
+      },
+      {
+        "Searchable": {
+          "fieldType": "KEYWORD"
+        },
+        "java": {
+          "class": "com.linkedin.pegasus2avro.common.url.Url",
+          "coercerClass": "com.linkedin.pegasus2avro.common.url.UrlCoercer"
+        },
+        "type": [
+          "null",
+          "string"
+        ],
+        "name": "externalUrl",
+        "default": null,
+        "doc": "URL where the reference exist"
+      },
+      {
+        "type": [
+          "null",
+          "string"
+        ],
+        "name": "id",
+        "default": null,
+        "doc": "Run Id of the ML Training Run"
+      },
+      {
+        "type": [
+          "null",
+          {
+            "type": "array",
+            "items": "string"
+          }
+        ],
+        "name": "outputUrls",
+        "default": null,
+        "doc": "List of URLs for the Outputs of the ML Training Run"
+      },
+      {
+        "type": [
+          "null",
+          {
+            "type": "array",
+            "items": "com.linkedin.pegasus2avro.ml.metadata.MLHyperParam"
+          }
+        ],
+        "name": "hyperParams",
+        "default": null,
+        "doc": "Hyperparameters of the ML Training Run"
+      },
+      {
+        "type": [
+          "null",
+          {
+            "type": "array",
+            "items": "com.linkedin.pegasus2avro.ml.metadata.MLMetric"
+          }
+        ],
+        "name": "trainingMetrics",
+        "default": null,
+        "doc": "Metrics of the ML Training Run"
+      }
+    ],
+    "doc": "The inputs and outputs of this training run"
+  },
   "com.linkedin.pegasus2avro.ml.metadata.EthicalConsiderations",
   "com.linkedin.pegasus2avro.ml.metadata.MLPrimaryKeyProperties",
   "com.linkedin.pegasus2avro.ml.metadata.MLModelFactorPrompts",
@@ -21078,6 +21277,10 @@
         "dataProcessInstanceRunEvent",
         "status",
         "testResults",
+        "dataPlatformInstance",
+        "subTypes",
+        "container",
+        "mlTrainingRunProperties",
         "lineageFeatures"
       ],
       "entityDoc": "DataProcessInstance represents an instance of a datajob/jobflow run"