acryl-datahub-cloud 0.3.8.2rc3__py3-none-any.whl → 0.3.9rc1__py3-none-any.whl

acryl_datahub_cloud/metadata/schemas/MonitorInfo.avsc

@@ -2538,7 +2538,9 @@
                                                                                         "fieldName": "glossaryTerms",
                                                                                         "fieldType": "URN",
                                                                                         "filterNameOverride": "Glossary Term Applied",
-                                                                                        "hasValuesFieldName": "hasGlossaryTerms"
+                                                                                        "hasValuesFieldName": "hasGlossaryTerms",
+                                                                                        "includeSystemModifiedAt": true,
+                                                                                        "systemModifiedAtFieldName": "termsModifiedAt"
                                                                                       },
                                                                                       "java": {
                                                                                         "class": "com.linkedin.pegasus2avro.common.urn.GlossaryTermUrn"
@@ -2926,8 +2928,8 @@
                                                           "type": "enum",
                                                           "symbolDocs": {
                                                             "EXTERNAL": "The assertion was defined and managed externally of DataHub.",
-                                                            "INFERRED": "The assertion was inferred, e.g. from offline AI / ML models.",
-                                                            "NATIVE": "The assertion was defined natively on DataHub by a user."
+                                                            "INFERRED": "The assertion was inferred, e.g. from offline AI / ML models.\nDataHub Cloud only",
+                                                            "NATIVE": "The assertion was defined natively on DataHub by a user.\nDataHub Cloud only"
                                                           },
                                                           "name": "AssertionSourceType",
                                                           "namespace": "com.linkedin.pegasus2avro.assertion",
@@ -3231,7 +3233,7 @@
       ],
       "name": "executorId",
       "default": null,
-      "doc": "Advanced: The executor ID of the remote monitor service, if any."
+      "doc": "Advanced: The executor pool id of the remote monitor service, if any."
     }
   ],
   "doc": "Information about an asset monitor."

acryl_datahub_cloud/metadata/schemas/NotebookKey.avsc

@@ -17,9 +17,9 @@
       "domains",
       "subTypes",
       "dataPlatformInstance",
-      "proposals",
       "browsePathsV2",
       "testResults",
+      "proposals",
       "share",
       "origin",
       "documentation"

acryl_datahub_cloud/metadata/schemas/Operation.avsc

@@ -150,7 +150,7 @@
         "symbolDocs": {
           "ALTER": "Asset was altered",
           "CREATE": "Asset was created",
-          "CUSTOM": "Custom asset operation",
+          "CUSTOM": "Custom asset operation. If this is set, ensure customOperationType is filled out.",
           "DELETE": "Rows were deleted",
           "DROP": "Asset was dropped",
           "INSERT": "Rows were inserted",
@@ -371,7 +371,9 @@
         "fieldName": "lastOperationTime",
         "fieldType": "DATETIME"
       },
-      "TimeseriesField": {},
+      "TimeseriesField": {
+        "fieldType": "DATETIME"
+      },
       "type": "long",
       "name": "lastUpdatedTimestamp",
       "doc": "The time at which the operation occurred. Would be better named 'operationTime'"

acryl_datahub_cloud/metadata/schemas/RemoteExecutorGlobalConfigKey.avsc

@@ -0,0 +1,21 @@
+{
+  "type": "record",
+  "Aspect": {
+    "name": "dataHubRemoteExecutorGlobalConfigKey",
+    "keyForEntity": "dataHubRemoteExecutorGlobalConfig",
+    "entityCategory": "internal",
+    "entityAspects": [
+      "dataHubRemoteExecutorPoolGlobalConfig"
+    ]
+  },
+  "name": "RemoteExecutorGlobalConfigKey",
+  "namespace": "com.linkedin.pegasus2avro.metadata.key",
+  "fields": [
+    {
+      "type": "string",
+      "name": "id",
+      "doc": "The unique identifier for the remote executor global config\nNOTE: since this a singleton, there should be a hardcoded key in the AcrylConstants file"
+    }
+  ],
+  "doc": "Key for the *Singleton* DataHub Remote Executor Global Config"
+}

acryl_datahub_cloud/metadata/schemas/RemoteExecutorPoolGlobalConfig.avsc

@@ -0,0 +1,16 @@
+{
+  "type": "record",
+  "Aspect": {
+    "name": "dataHubRemoteExecutorPoolGlobalConfig"
+  },
+  "name": "RemoteExecutorPoolGlobalConfig",
+  "namespace": "com.linkedin.pegasus2avro.executorglobalconfig",
+  "fields": [
+    {
+      "type": "string",
+      "name": "defaultExecutorPoolId",
+      "doc": "The default pool to use for tasks that require remote executors.\nReferences 'id' in com.linkedin.pegasus2avro.metadata.key.RemoteExecutorPoolKey"
+    }
+  ],
+  "doc": "Global singleton storing configs for remote executor pools."
+}

acryl_datahub_cloud/metadata/schemas/RemoteExecutorPoolInfo.avsc

@@ -14,6 +14,91 @@
       "type": "long",
       "name": "createdAt",
       "doc": "UTC-based timestamp of when this pool was created"
+    },
+    {
+      "java": {
+        "class": "com.linkedin.pegasus2avro.common.urn.Urn"
+      },
+      "type": [
+        "null",
+        "string"
+      ],
+      "name": "creator",
+      "default": null,
+      "doc": "The creator of this pool",
+      "Urn": "Urn"
+    },
+    {
+      "type": [
+        "null",
+        "string"
+      ],
+      "name": "description",
+      "default": null,
+      "doc": "A description for this pool"
+    },
+    {
+      "type": [
+        "null",
+        "string"
+      ],
+      "name": "queueUrl",
+      "default": null,
+      "doc": "The url to the task queue for this pool. I.e., SQS queue url."
+    },
+    {
+      "type": [
+        "null",
+        "boolean"
+      ],
+      "name": "isEmbedded",
+      "default": null,
+      "doc": "Only set true if this is the pool embedded within the DataHub Cloud deployment"
+    },
+    {
+      "type": [
+        "null",
+        {
+          "type": "record",
+          "name": "RemoteExecutorPoolState",
+          "namespace": "com.linkedin.pegasus2avro.executorpool",
+          "fields": [
+            {
+              "type": {
+                "type": "enum",
+                "symbolDocs": {
+                  "PROVISIONING_FAILED": "The pool has failed to be provisioned.",
+                  "PROVISIONING_IN_PROGRESS": "The pool has been picked up by DataHub and is in the process of being provisioned.",
+                  "PROVISIONING_PENDING": "The pool is pending provisioning. Default state on creation.",
+                  "READY": "The pool has been successfully provisioned and is ready to accept tasks."
+                },
+                "name": "RemoteExecutorPoolStatus",
+                "namespace": "com.linkedin.pegasus2avro.executorpool",
+                "symbols": [
+                  "PROVISIONING_PENDING",
+                  "PROVISIONING_IN_PROGRESS",
+                  "PROVISIONING_FAILED",
+                  "READY"
+                ]
+              },
+              "name": "status",
+              "doc": "The status of the remote executor pool"
+            },
+            {
+              "type": [
+                "null",
+                "string"
+              ],
+              "name": "message",
+              "default": null,
+              "doc": "The message associated with the status. I.e., an error message explaining failure."
+            }
+          ]
+        }
+      ],
+      "name": "state",
+      "default": null,
+      "doc": "The status of the remote executor pool"
     }
   ]
 }

acryl_datahub_cloud/metadata/schemas/RemoteExecutorPoolKey.avsc

@@ -16,7 +16,7 @@
         "fieldType": "KEYWORD"
       },
       "type": "string",
-      "name": "name",
+      "name": "id",
       "doc": "The unique identifier for the remote executor pool"
     }
   ],

acryl_datahub_cloud/metadata/schemas/RemoteExecutorStatus.avsc

@@ -11,8 +11,8 @@
         "fieldType": "KEYWORD"
       },
       "type": "string",
-      "name": "poolName",
-      "doc": "References the 'name' defined in RemoteExecutorPoolKey"
+      "name": "executorPoolId",
+      "doc": "References the 'id' defined in RemoteExecutorPoolKey"
     },
     {
       "type": "string",
@@ -41,19 +41,19 @@
       "type": "boolean",
       "name": "executorExpired",
       "default": false,
-      "doc": "Flag indicating whether remote executor status record is stale."
+      "doc": "Flag indicating whether remote executor status record is stale.\nThis means executor no longer sends heartbeats and considered dead."
     },
     {
       "type": "boolean",
       "name": "executorStopped",
       "default": false,
-      "doc": "Flag indicating whether remote executor is stopped."
+      "doc": "Flag indicating whether remote executor is stopped, and properly reported its termination."
     },
     {
       "type": "boolean",
       "name": "executorEmbedded",
       "default": false,
-      "doc": "Flag indicating whether remote executor is embedded executor"
+      "doc": "Flag indicating whether remote executor is embedded into the coordinator pod"
     },
     {
       "type": "boolean",

acryl_datahub_cloud/metadata/schemas/SchemaFieldKey.avsc

@@ -13,9 +13,9 @@
       "schemaFieldAliases",
       "documentation",
       "testResults",
+      "deprecation",
       "schemaFieldProfile",
-      "lineageFeatures",
-      "deprecation"
+      "lineageFeatures"
     ]
   },
   "name": "SchemaFieldKey",

acryl_datahub_cloud/metadata/schemas/SchemaMetadata.avsc

@@ -777,7 +777,9 @@
                                 "fieldName": "glossaryTerms",
                                 "fieldType": "URN",
                                 "filterNameOverride": "Glossary Term Applied",
-                                "hasValuesFieldName": "hasGlossaryTerms"
+                                "hasValuesFieldName": "hasGlossaryTerms",
+                                "includeSystemModifiedAt": true,
+                                "systemModifiedAtFieldName": "termsModifiedAt"
                               },
                               "java": {
                                 "class": "com.linkedin.pegasus2avro.common.urn.GlossaryTermUrn"

acryl_datahub_cloud/metadata/schemas/StructuredPropertyDefinition.avsc

@@ -23,6 +23,13 @@
       "doc": "The display name of the property. This is the name that will be shown in the UI and can be used to look up the property id."
     },
     {
+      "UrnValidation": {
+        "entityTypes": [
+          "dataType"
+        ],
+        "exist": true,
+        "strict": true
+      },
       "java": {
         "class": "com.linkedin.pegasus2avro.common.urn.Urn"
       },
@@ -111,6 +118,13 @@
           "fieldName": "entityTypes"
         }
       },
+      "UrnValidation": {
+        "entityTypes": [
+          "entityType"
+        ],
+        "exist": true,
+        "strict": true
+      },
       "type": {
         "type": "array",
         "items": "string"

{acryl_datahub_cloud-0.3.8.2rc3.dist-info → acryl_datahub_cloud-0.3.9rc1.dist-info}/METADATA

@@ -1,90 +1,90 @@
 Metadata-Version: 2.2
 Name: acryl-datahub-cloud
-Version: 0.3.8.2rc3
+Version: 0.3.9rc1
 Requires-Dist: avro-gen3==0.7.16
 Requires-Dist: acryl-datahub
 Provides-Extra: datahub-lineage-features
+Requires-Dist: pyarrow; extra == "datahub-lineage-features"
 Requires-Dist: opensearch-py==2.4.2; extra == "datahub-lineage-features"
 Requires-Dist: duckdb; extra == "datahub-lineage-features"
-Requires-Dist: pydantic<2; extra == "datahub-lineage-features"
 Requires-Dist: pandas; extra == "datahub-lineage-features"
-Requires-Dist: pyarrow; extra == "datahub-lineage-features"
+Requires-Dist: pydantic<2; extra == "datahub-lineage-features"
 Provides-Extra: datahub-reporting-forms
+Requires-Dist: pyarrow; extra == "datahub-reporting-forms"
+Requires-Dist: termcolor==2.5.0; extra == "datahub-reporting-forms"
 Requires-Dist: duckdb; extra == "datahub-reporting-forms"
-Requires-Dist: pydantic<2; extra == "datahub-reporting-forms"
 Requires-Dist: boto3; extra == "datahub-reporting-forms"
-Requires-Dist: termcolor==2.5.0; extra == "datahub-reporting-forms"
 Requires-Dist: pandas; extra == "datahub-reporting-forms"
-Requires-Dist: pyarrow; extra == "datahub-reporting-forms"
+Requires-Dist: pydantic<2; extra == "datahub-reporting-forms"
 Provides-Extra: datahub-reporting-extract-graph
+Requires-Dist: pyarrow; extra == "datahub-reporting-extract-graph"
 Requires-Dist: opensearch-py==2.4.2; extra == "datahub-reporting-extract-graph"
 Requires-Dist: duckdb; extra == "datahub-reporting-extract-graph"
-Requires-Dist: pydantic<2; extra == "datahub-reporting-extract-graph"
 Requires-Dist: boto3; extra == "datahub-reporting-extract-graph"
 Requires-Dist: pandas; extra == "datahub-reporting-extract-graph"
-Requires-Dist: pyarrow; extra == "datahub-reporting-extract-graph"
+Requires-Dist: pydantic<2; extra == "datahub-reporting-extract-graph"
 Provides-Extra: datahub-reporting-extract-sql
+Requires-Dist: pyarrow; extra == "datahub-reporting-extract-sql"
 Requires-Dist: duckdb; extra == "datahub-reporting-extract-sql"
-Requires-Dist: pydantic<2; extra == "datahub-reporting-extract-sql"
 Requires-Dist: boto3; extra == "datahub-reporting-extract-sql"
 Requires-Dist: pandas; extra == "datahub-reporting-extract-sql"
-Requires-Dist: pyarrow; extra == "datahub-reporting-extract-sql"
+Requires-Dist: pydantic<2; extra == "datahub-reporting-extract-sql"
 Provides-Extra: datahub-usage-reporting
-Requires-Dist: boto3; extra == "datahub-usage-reporting"
-Requires-Dist: pydantic<2; extra == "datahub-usage-reporting"
-Requires-Dist: termcolor==2.5.0; extra == "datahub-usage-reporting"
-Requires-Dist: scipy<=1.14.1; extra == "datahub-usage-reporting"
-Requires-Dist: pyarrow; extra == "datahub-usage-reporting"
 Requires-Dist: opensearch-py==2.4.2; extra == "datahub-usage-reporting"
-Requires-Dist: pyarrow<=18.0.0; extra == "datahub-usage-reporting"
 Requires-Dist: duckdb; extra == "datahub-usage-reporting"
+Requires-Dist: pyarrow<=18.0.0; extra == "datahub-usage-reporting"
+Requires-Dist: pyarrow; extra == "datahub-usage-reporting"
+Requires-Dist: termcolor==2.5.0; extra == "datahub-usage-reporting"
+Requires-Dist: scipy<=1.14.1; extra == "datahub-usage-reporting"
 Requires-Dist: elasticsearch==7.13.4; extra == "datahub-usage-reporting"
-Requires-Dist: numpy<2; extra == "datahub-usage-reporting"
+Requires-Dist: polars==1.23.0; extra == "datahub-usage-reporting"
+Requires-Dist: boto3; extra == "datahub-usage-reporting"
 Requires-Dist: pandas; extra == "datahub-usage-reporting"
-Requires-Dist: polars==1.19.0; extra == "datahub-usage-reporting"
+Requires-Dist: pydantic<2; extra == "datahub-usage-reporting"
+Requires-Dist: numpy<2; extra == "datahub-usage-reporting"
 Provides-Extra: datahub-metadata-sharing
 Requires-Dist: tenacity; extra == "datahub-metadata-sharing"
 Provides-Extra: acryl-cs-issues
-Requires-Dist: zenpy; extra == "acryl-cs-issues"
+Requires-Dist: jinja2; extra == "acryl-cs-issues"
 Requires-Dist: openai; extra == "acryl-cs-issues"
 Requires-Dist: slack-sdk; extra == "acryl-cs-issues"
-Requires-Dist: jinja2; extra == "acryl-cs-issues"
+Requires-Dist: zenpy; extra == "acryl-cs-issues"
 Provides-Extra: all
-Requires-Dist: openai; extra == "all"
+Requires-Dist: tenacity; extra == "all"
+Requires-Dist: duckdb; extra == "all"
+Requires-Dist: zenpy; extra == "all"
+Requires-Dist: pyarrow<=18.0.0; extra == "all"
+Requires-Dist: pyarrow; extra == "all"
 Requires-Dist: termcolor==2.5.0; extra == "all"
 Requires-Dist: scipy<=1.14.1; extra == "all"
-Requires-Dist: pyarrow<=18.0.0; extra == "all"
-Requires-Dist: duckdb; extra == "all"
-Requires-Dist: tenacity; extra == "all"
 Requires-Dist: elasticsearch==7.13.4; extra == "all"
-Requires-Dist: numpy<2; extra == "all"
-Requires-Dist: polars==1.19.0; extra == "all"
-Requires-Dist: zenpy; extra == "all"
 Requires-Dist: boto3; extra == "all"
+Requires-Dist: pandas; extra == "all"
 Requires-Dist: pydantic<2; extra == "all"
-Requires-Dist: pyarrow; extra == "all"
 Requires-Dist: slack-sdk; extra == "all"
+Requires-Dist: openai; extra == "all"
 Requires-Dist: opensearch-py==2.4.2; extra == "all"
 Requires-Dist: jinja2; extra == "all"
-Requires-Dist: pandas; extra == "all"
+Requires-Dist: polars==1.23.0; extra == "all"
+Requires-Dist: numpy<2; extra == "all"
 Provides-Extra: dev
-Requires-Dist: zenpy; extra == "dev"
-Requires-Dist: boto3; extra == "dev"
-Requires-Dist: pydantic<2; extra == "dev"
-Requires-Dist: termcolor==2.5.0; extra == "dev"
+Requires-Dist: acryl-datahub[dev]; extra == "dev"
 Requires-Dist: openai; extra == "dev"
-Requires-Dist: jinja2; extra == "dev"
-Requires-Dist: scipy<=1.14.1; extra == "dev"
-Requires-Dist: pyarrow; extra == "dev"
-Requires-Dist: slack-sdk; extra == "dev"
 Requires-Dist: opensearch-py==2.4.2; extra == "dev"
-Requires-Dist: pyarrow<=18.0.0; extra == "dev"
-Requires-Dist: duckdb; extra == "dev"
 Requires-Dist: tenacity; extra == "dev"
+Requires-Dist: duckdb; extra == "dev"
+Requires-Dist: zenpy; extra == "dev"
+Requires-Dist: pyarrow<=18.0.0; extra == "dev"
+Requires-Dist: jinja2; extra == "dev"
+Requires-Dist: pyarrow; extra == "dev"
+Requires-Dist: termcolor==2.5.0; extra == "dev"
+Requires-Dist: scipy<=1.14.1; extra == "dev"
 Requires-Dist: elasticsearch==7.13.4; extra == "dev"
-Requires-Dist: numpy<2; extra == "dev"
-Requires-Dist: acryl-datahub[dev]; extra == "dev"
+Requires-Dist: polars==1.23.0; extra == "dev"
+Requires-Dist: boto3; extra == "dev"
 Requires-Dist: pandas; extra == "dev"
-Requires-Dist: polars==1.19.0; extra == "dev"
+Requires-Dist: pydantic<2; extra == "dev"
+Requires-Dist: numpy<2; extra == "dev"
+Requires-Dist: slack-sdk; extra == "dev"
 Dynamic: provides-extra
 Dynamic: requires-dist