acryl-datahub-cloud 0.3.7.9rc1__py3-none-any.whl → 0.3.8rc1__py3-none-any.whl

acryl_datahub_cloud/metadata/schemas/PostInfo.avsc

@@ -7,6 +7,7 @@
   "namespace": "com.linkedin.pegasus2avro.post",
   "fields": [
     {
+      "Searchable": {},
       "type": {
         "type": "enum",
         "symbolDocs": {
@@ -205,9 +206,20 @@
           "dataset",
           "schemaField",
           "chart",
+          "container",
           "dashboard",
           "dataFlow",
           "dataJob",
+          "dataProduct",
+          "glossaryTerm",
+          "glossaryNode",
+          "mlModel",
+          "mlFeature",
+          "notebook",
+          "mlFeatureTable",
+          "mlPrimaryKey",
+          "mlModelGroup",
+          "domain",
           "dataProduct"
         ],
         "name": "PostTarget"
@@ -228,9 +240,20 @@
         "dataset",
         "schemaField",
         "chart",
+        "container",
         "dashboard",
         "dataFlow",
         "dataJob",
+        "dataProduct",
+        "glossaryTerm",
+        "glossaryNode",
+        "mlModel",
+        "mlFeature",
+        "notebook",
+        "mlFeatureTable",
+        "mlPrimaryKey",
+        "mlModelGroup",
+        "domain",
         "dataProduct"
       ]
     }

acryl_datahub_cloud/metadata/schemas/RecommendationModule.avsc

@@ -130,6 +130,7 @@
                                         "type": "enum",
                                         "symbolDocs": {
                                           "ANCESTORS_INCL": "Represent the relation: URN field matches any nested parent in addition to the given URN",
+                                          "BETWEEN": "Represent the relation within an inclusive range, e.g. 3 <= ownerCount <= 5. Note that the values in Criterion must have two entries for a lower and upper bound.",
                                           "CONTAIN": "Represent the relation: String field contains value, e.g. name contains Profile",
                                           "DESCENDANTS_INCL": "Represent the relation: URN field any nested children in addition to the given URN",
                                           "END_WITH": "Represent the relation: String field ends with value, e.g. name ends with Event",
@@ -159,6 +160,7 @@
                                           "IN",
                                           "LESS_THAN",
                                           "LESS_THAN_OR_EQUAL_TO",
+                                          "BETWEEN",
                                           "START_WITH",
                                           "DESCENDANTS_INCL",
                                           "ANCESTORS_INCL",

acryl_datahub_cloud/metadata/schemas/RemoteExecutorKey.avsc

@@ -0,0 +1,21 @@
+{
+  "type": "record",
+  "Aspect": {
+    "name": "dataHubRemoteExecutorKey",
+    "keyForEntity": "dataHubRemoteExecutor",
+    "entityCategory": "internal",
+    "entityAspects": [
+      "dataHubRemoteExecutorStatus"
+    ]
+  },
+  "name": "RemoteExecutorKey",
+  "namespace": "com.linkedin.pegasus2avro.metadata.key",
+  "fields": [
+    {
+      "type": "string",
+      "name": "id",
+      "doc": "A unique id for the DataHub Remote Executor."
+    }
+  ],
+  "doc": "Key for an DataHub Remote Executor"
+}

acryl_datahub_cloud/metadata/schemas/RemoteExecutorStatus.avsc

@@ -0,0 +1,80 @@
+{
+  "type": "record",
+  "Aspect": {
+    "name": "dataHubRemoteExecutorStatus"
+  },
+  "name": "RemoteExecutorStatus",
+  "namespace": "com.linkedin.pegasus2avro.executor",
+  "fields": [
+    {
+      "Searchable": {
+        "fieldType": "KEYWORD"
+      },
+      "type": "string",
+      "name": "executorId",
+      "doc": "TODO: rename to pipeline id"
+    },
+    {
+      "type": "string",
+      "name": "executorReleaseVersion",
+      "doc": " Release Version (Tag) embedded into remote executor image"
+    },
+    {
+      "type": "string",
+      "name": "executorAddress",
+      "doc": " IP address of the host where remote executor is running"
+    },
+    {
+      "type": "string",
+      "name": "executorHostname",
+      "doc": "Hostname of the host where remote executor is running"
+    },
+    {
+      "type": "float",
+      "name": "executorUptime",
+      "doc": "Uptime of the remote executor master process"
+    },
+    {
+      "Searchable": {
+        "fieldType": "BOOLEAN"
+      },
+      "type": "boolean",
+      "name": "executorExpired",
+      "default": false,
+      "doc": "Flag indicating whether remote executor status record is stale."
+    },
+    {
+      "type": "boolean",
+      "name": "executorStopped",
+      "default": false,
+      "doc": "Flag indicating whether remote executor is stopped."
+    },
+    {
+      "type": "boolean",
+      "name": "executorEmbedded",
+      "default": false,
+      "doc": "Flag indicating whether remote executor is embedded executor"
+    },
+    {
+      "type": "boolean",
+      "name": "executorInternal",
+      "default": false,
+      "doc": "Flag indicating whether remote executor is Acryl managed executor"
+    },
+    {
+      "type": "boolean",
+      "name": "logDeliveryEnabled",
+      "default": false,
+      "doc": "Flag indicating whether log delivery is enabled by the customer in the given\nremote executor instance."
+    },
+    {
+      "Searchable": {
+        "fieldType": "COUNT",
+        "queryByDefault": false
+      },
+      "type": "int",
+      "name": "reportedAt",
+      "doc": "UTC-based timestamp of the last reported status"
+    }
+  ]
+}

acryl_datahub_cloud/metadata/schemas/SchemaFieldKey.avsc

@@ -14,7 +14,8 @@
       "documentation",
       "testResults",
       "schemaFieldProfile",
-      "lineageFeatures"
+      "lineageFeatures",
+      "deprecation"
     ]
   },
   "name": "SchemaFieldKey",

acryl_datahub_cloud/metadata/schemas/VersionProperties.avsc

@@ -0,0 +1,216 @@
+{
+  "type": "record",
+  "Aspect": {
+    "name": "versionProperties"
+  },
+  "name": "VersionProperties",
+  "namespace": "com.linkedin.pegasus2avro.common",
+  "fields": [
+    {
+      "Relationship": {
+        "entityTypes": [
+          "versionSet"
+        ],
+        "name": "VersionOf"
+      },
+      "Searchable": {
+        "queryByDefault": false
+      },
+      "java": {
+        "class": "com.linkedin.pegasus2avro.common.urn.Urn"
+      },
+      "type": "string",
+      "name": "versionSet",
+      "doc": "The linked Version Set entity that ties multiple versioned assets together",
+      "Urn": "Urn",
+      "entityTypes": [
+        "versionSet"
+      ]
+    },
+    {
+      "Searchable": {
+        "/versionTag": {
+          "fieldName": "version",
+          "queryByDefault": false
+        }
+      },
+      "type": {
+        "type": "record",
+        "name": "VersionTag",
+        "namespace": "com.linkedin.pegasus2avro.common",
+        "fields": [
+          {
+            "type": [
+              "null",
+              "string"
+            ],
+            "name": "versionTag",
+            "default": null
+          },
+          {
+            "type": [
+              "null",
+              {
+                "type": "record",
+                "name": "MetadataAttribution",
+                "namespace": "com.linkedin.pegasus2avro.common",
+                "fields": [
+                  {
+                    "type": "long",
+                    "name": "time",
+                    "doc": "When this metadata was updated."
+                  },
+                  {
+                    "java": {
+                      "class": "com.linkedin.pegasus2avro.common.urn.Urn"
+                    },
+                    "type": "string",
+                    "name": "actor",
+                    "doc": "The entity (e.g. a member URN) responsible for applying the assocated metadata. This can\neither be a user (in case of UI edits) or the datahub system for automation.",
+                    "Urn": "Urn"
+                  },
+                  {
+                    "java": {
+                      "class": "com.linkedin.pegasus2avro.common.urn.Urn"
+                    },
+                    "type": [
+                      "null",
+                      "string"
+                    ],
+                    "name": "source",
+                    "default": null,
+                    "doc": "The DataHub source responsible for applying the associated metadata. This will only be filled out\nwhen a DataHub source is responsible. This includes the specific metadata test urn, the automation urn.",
+                    "Urn": "Urn"
+                  },
+                  {
+                    "type": {
+                      "type": "map",
+                      "values": "string"
+                    },
+                    "name": "sourceDetail",
+                    "default": {},
+                    "doc": "The details associated with why this metadata was applied. For example, this could include\nthe actual regex rule, sql statement, ingestion pipeline ID, etc.\nAlso can include flags like 'propagated'=true or 'inferred'=true."
+                  }
+                ],
+                "doc": "Information about who, why, and how this metadata was applied"
+              }
+            ],
+            "name": "metadataAttribution",
+            "default": null
+          }
+        ],
+        "doc": "A resource-defined string representing the resource state for the purpose of concurrency control"
+      },
+      "name": "version",
+      "doc": "Label for this versioned asset, is unique within a version set"
+    },
+    {
+      "Searchable": {
+        "/*/versionTag": {
+          "fieldName": "aliases",
+          "queryByDefault": false
+        }
+      },
+      "type": {
+        "type": "array",
+        "items": "com.linkedin.pegasus2avro.common.VersionTag"
+      },
+      "name": "aliases",
+      "default": [],
+      "doc": "Associated aliases for this versioned asset"
+    },
+    {
+      "type": [
+        "null",
+        "string"
+      ],
+      "name": "comment",
+      "default": null,
+      "doc": "Comment documenting what this version was created for, changes, or represents"
+    },
+    {
+      "Searchable": {
+        "fieldName": "versionSortId",
+        "queryByDefault": false
+      },
+      "type": "string",
+      "name": "sortId",
+      "doc": "Sort identifier that determines where a version lives in the order of the Version Set.\nWhat this looks like depends on the Version Scheme. For sort ids generated by DataHub we use an 8 character string representation."
+    },
+    {
+      "type": [
+        "null",
+        {
+          "type": "record",
+          "name": "AuditStamp",
+          "namespace": "com.linkedin.pegasus2avro.common",
+          "fields": [
+            {
+              "type": "long",
+              "name": "time",
+              "doc": "When did the resource/association/sub-resource move into the specific lifecycle stage represented by this AuditEvent."
+            },
+            {
+              "java": {
+                "class": "com.linkedin.pegasus2avro.common.urn.Urn"
+              },
+              "type": "string",
+              "name": "actor",
+              "doc": "The entity (e.g. a member URN) which will be credited for moving the resource/association/sub-resource into the specific lifecycle stage. It is also the one used to authorize the change.",
+              "Urn": "Urn"
+            },
+            {
+              "java": {
+                "class": "com.linkedin.pegasus2avro.common.urn.Urn"
+              },
+              "type": [
+                "null",
+                "string"
+              ],
+              "name": "impersonator",
+              "default": null,
+              "doc": "The entity (e.g. a service URN) which performs the change on behalf of the Actor and must be authorized to act as the Actor.",
+              "Urn": "Urn"
+            },
+            {
+              "type": [
+                "null",
+                "string"
+              ],
+              "name": "message",
+              "default": null,
+              "doc": "Additional context around how DataHub was informed of the particular change. For example: was the change created by an automated process, or manually."
+            }
+          ],
+          "doc": "Data captured on a resource/association/sub-resource level giving insight into when that resource/association/sub-resource moved into a particular lifecycle stage, and who acted to move it into that specific lifecycle stage."
+        }
+      ],
+      "name": "sourceCreatedTimestamp",
+      "default": null,
+      "doc": "Timestamp reflecting when this asset version was created in the source system."
+    },
+    {
+      "type": [
+        "null",
+        "com.linkedin.pegasus2avro.common.AuditStamp"
+      ],
+      "name": "metadataCreatedTimestamp",
+      "default": null,
+      "doc": "Timestamp reflecting when the metadata for this version was created in DataHub"
+    },
+    {
+      "Searchable": {
+        "fieldType": "BOOLEAN",
+        "queryByDefault": false
+      },
+      "type": [
+        "null",
+        "boolean"
+      ],
+      "name": "isLatest",
+      "default": null,
+      "doc": "Marks whether this version is currently the latest. Set by a side effect and should not be modified by API."
+    }
+  ],
+  "doc": "Properties about a versioned asset i.e. dataset, ML Model, etc."
+}

acryl_datahub_cloud/metadata/schemas/VersionSetKey.avsc

@@ -0,0 +1,26 @@
+{
+  "type": "record",
+  "Aspect": {
+    "name": "versionSetKey",
+    "keyForEntity": "versionSet",
+    "entityCategory": "core",
+    "entityAspects": [
+      "versionSetProperties"
+    ]
+  },
+  "name": "VersionSetKey",
+  "namespace": "com.linkedin.pegasus2avro.metadata.key",
+  "fields": [
+    {
+      "type": "string",
+      "name": "id",
+      "doc": "ID of the Version Set, generated from platform + asset id / name"
+    },
+    {
+      "type": "string",
+      "name": "entityType",
+      "doc": "Type of entities included in version set, limits to a single entity type between linked versioned entities"
+    }
+  ],
+  "doc": "Key for a Version Set entity"
+}

acryl_datahub_cloud/metadata/schemas/VersionSetProperties.avsc

@@ -0,0 +1,49 @@
+{
+  "type": "record",
+  "Aspect": {
+    "name": "versionSetProperties"
+  },
+  "name": "VersionSetProperties",
+  "namespace": "com.linkedin.pegasus2avro.versionset",
+  "fields": [
+    {
+      "Searchable": {
+        "/*": {
+          "fieldType": "TEXT",
+          "queryByDefault": true
+        }
+      },
+      "type": {
+        "type": "map",
+        "values": "string"
+      },
+      "name": "customProperties",
+      "default": {},
+      "doc": "Custom property bag."
+    },
+    {
+      "Searchable": {
+        "queryByDefault": "false"
+      },
+      "java": {
+        "class": "com.linkedin.pegasus2avro.common.urn.Urn"
+      },
+      "type": "string",
+      "name": "latest",
+      "doc": "The latest versioned entity linked to in this version set",
+      "Urn": "Urn"
+    },
+    {
+      "type": {
+        "type": "enum",
+        "name": "VersioningScheme",
+        "namespace": "com.linkedin.pegasus2avro.versionset",
+        "symbols": [
+          "ALPHANUMERIC_GENERATED_BY_DATAHUB"
+        ]
+      },
+      "name": "versioningScheme",
+      "doc": "What versioning scheme is being utilized for the versioned entities sort criterion. Static once set"
+    }
+  ]
+}

acryl_datahub_cloud/metadata/schemas/__init__.py

@@ -15,10 +15,10 @@ import pathlib
 def _load_schema(schema_name: str) -> str:
     return (pathlib.Path(__file__).parent / f"{schema_name}.avsc").read_text()
 
-def getMetadataChangeProposalSchema() -> str:
-    return _load_schema("MetadataChangeProposal")
-
 def getMetadataChangeEventSchema() -> str:
     return _load_schema("MetadataChangeEvent")
 
+def getMetadataChangeProposalSchema() -> str:
+    return _load_schema("MetadataChangeProposal")
+
 # fmt: on

{acryl_datahub_cloud-0.3.7.9rc1.dist-info → acryl_datahub_cloud-0.3.8rc1.dist-info}/METADATA

@@ -1,80 +1,81 @@
 Metadata-Version: 2.1
 Name: acryl-datahub-cloud
-Version: 0.3.7.9rc1
+Version: 0.3.8rc1
 Requires-Dist: avro-gen3==0.7.16
 Requires-Dist: acryl-datahub
 Provides-Extra: datahub-lineage-features
 Requires-Dist: pyarrow; extra == "datahub-lineage-features"
-Requires-Dist: duckdb; extra == "datahub-lineage-features"
-Requires-Dist: pydantic<2; extra == "datahub-lineage-features"
 Requires-Dist: pandas; extra == "datahub-lineage-features"
+Requires-Dist: duckdb; extra == "datahub-lineage-features"
 Requires-Dist: opensearch-py==2.4.2; extra == "datahub-lineage-features"
+Requires-Dist: pydantic<2; extra == "datahub-lineage-features"
 Provides-Extra: datahub-reporting-forms
 Requires-Dist: pyarrow; extra == "datahub-reporting-forms"
-Requires-Dist: duckdb; extra == "datahub-reporting-forms"
-Requires-Dist: pydantic<2; extra == "datahub-reporting-forms"
 Requires-Dist: pandas; extra == "datahub-reporting-forms"
+Requires-Dist: duckdb; extra == "datahub-reporting-forms"
 Requires-Dist: boto3; extra == "datahub-reporting-forms"
+Requires-Dist: pydantic<2; extra == "datahub-reporting-forms"
 Provides-Extra: datahub-reporting-extract-graph
 Requires-Dist: pyarrow; extra == "datahub-reporting-extract-graph"
-Requires-Dist: duckdb; extra == "datahub-reporting-extract-graph"
-Requires-Dist: pydantic<2; extra == "datahub-reporting-extract-graph"
 Requires-Dist: pandas; extra == "datahub-reporting-extract-graph"
+Requires-Dist: duckdb; extra == "datahub-reporting-extract-graph"
 Requires-Dist: boto3; extra == "datahub-reporting-extract-graph"
 Requires-Dist: opensearch-py==2.4.2; extra == "datahub-reporting-extract-graph"
+Requires-Dist: pydantic<2; extra == "datahub-reporting-extract-graph"
 Provides-Extra: datahub-reporting-extract-sql
 Requires-Dist: pyarrow; extra == "datahub-reporting-extract-sql"
-Requires-Dist: duckdb; extra == "datahub-reporting-extract-sql"
-Requires-Dist: pydantic<2; extra == "datahub-reporting-extract-sql"
 Requires-Dist: pandas; extra == "datahub-reporting-extract-sql"
+Requires-Dist: duckdb; extra == "datahub-reporting-extract-sql"
 Requires-Dist: boto3; extra == "datahub-reporting-extract-sql"
+Requires-Dist: pydantic<2; extra == "datahub-reporting-extract-sql"
 Provides-Extra: datahub-usage-feature-reporting
-Requires-Dist: duckdb; extra == "datahub-usage-feature-reporting"
-Requires-Dist: pydantic<2; extra == "datahub-usage-feature-reporting"
+Requires-Dist: pyarrow; extra == "datahub-usage-feature-reporting"
+Requires-Dist: scipy<=1.14.1; extra == "datahub-usage-feature-reporting"
 Requires-Dist: pandas; extra == "datahub-usage-feature-reporting"
-Requires-Dist: elasticsearch==7.13.4; extra == "datahub-usage-feature-reporting"
+Requires-Dist: duckdb; extra == "datahub-usage-feature-reporting"
 Requires-Dist: numpy<2; extra == "datahub-usage-feature-reporting"
+Requires-Dist: boto3; extra == "datahub-usage-feature-reporting"
 Requires-Dist: opensearch-py==2.4.2; extra == "datahub-usage-feature-reporting"
-Requires-Dist: scipy<=1.14.1; extra == "datahub-usage-feature-reporting"
-Requires-Dist: pyarrow; extra == "datahub-usage-feature-reporting"
+Requires-Dist: pydantic<2; extra == "datahub-usage-feature-reporting"
+Requires-Dist: elasticsearch==7.13.4; extra == "datahub-usage-feature-reporting"
 Requires-Dist: polars<=1.16.0; extra == "datahub-usage-feature-reporting"
 Requires-Dist: pyarrow<=18.0.0; extra == "datahub-usage-feature-reporting"
-Requires-Dist: boto3; extra == "datahub-usage-feature-reporting"
+Provides-Extra: datahub-metadata-sharing
 Provides-Extra: acryl-cs-issues
-Requires-Dist: openai; extra == "acryl-cs-issues"
-Requires-Dist: zenpy; extra == "acryl-cs-issues"
-Requires-Dist: slack-sdk; extra == "acryl-cs-issues"
 Requires-Dist: jinja2; extra == "acryl-cs-issues"
+Requires-Dist: slack-sdk; extra == "acryl-cs-issues"
+Requires-Dist: zenpy; extra == "acryl-cs-issues"
+Requires-Dist: openai; extra == "acryl-cs-issues"
 Provides-Extra: all
-Requires-Dist: openai; extra == "all"
+Requires-Dist: pyarrow; extra == "all"
+Requires-Dist: jinja2; extra == "all"
 Requires-Dist: zenpy; extra == "all"
+Requires-Dist: openai; extra == "all"
+Requires-Dist: scipy<=1.14.1; extra == "all"
+Requires-Dist: pandas; extra == "all"
 Requires-Dist: duckdb; extra == "all"
+Requires-Dist: numpy<2; extra == "all"
+Requires-Dist: boto3; extra == "all"
+Requires-Dist: opensearch-py==2.4.2; extra == "all"
 Requires-Dist: pydantic<2; extra == "all"
-Requires-Dist: pandas; extra == "all"
 Requires-Dist: elasticsearch==7.13.4; extra == "all"
-Requires-Dist: numpy<2; extra == "all"
 Requires-Dist: slack-sdk; extra == "all"
-Requires-Dist: jinja2; extra == "all"
-Requires-Dist: opensearch-py==2.4.2; extra == "all"
-Requires-Dist: scipy<=1.14.1; extra == "all"
-Requires-Dist: pyarrow; extra == "all"
 Requires-Dist: polars<=1.16.0; extra == "all"
 Requires-Dist: pyarrow<=18.0.0; extra == "all"
-Requires-Dist: boto3; extra == "all"
 Provides-Extra: dev
-Requires-Dist: openai; extra == "dev"
+Requires-Dist: pyarrow; extra == "dev"
+Requires-Dist: jinja2; extra == "dev"
 Requires-Dist: zenpy; extra == "dev"
+Requires-Dist: openai; extra == "dev"
+Requires-Dist: scipy<=1.14.1; extra == "dev"
 Requires-Dist: duckdb; extra == "dev"
-Requires-Dist: pydantic<2; extra == "dev"
 Requires-Dist: pandas; extra == "dev"
-Requires-Dist: elasticsearch==7.13.4; extra == "dev"
 Requires-Dist: numpy<2; extra == "dev"
-Requires-Dist: slack-sdk; extra == "dev"
-Requires-Dist: jinja2; extra == "dev"
+Requires-Dist: boto3; extra == "dev"
 Requires-Dist: opensearch-py==2.4.2; extra == "dev"
+Requires-Dist: pydantic<2; extra == "dev"
+Requires-Dist: elasticsearch==7.13.4; extra == "dev"
+Requires-Dist: slack-sdk; extra == "dev"
 Requires-Dist: acryl-datahub[dev]; extra == "dev"
-Requires-Dist: scipy<=1.14.1; extra == "dev"
-Requires-Dist: pyarrow; extra == "dev"
 Requires-Dist: polars<=1.16.0; extra == "dev"
 Requires-Dist: pyarrow<=18.0.0; extra == "dev"
-Requires-Dist: boto3; extra == "dev"