acryl-datahub-cloud 0.3.7.6__tar.gz → 0.3.7.7rc2__tar.gz

{acryl_datahub_cloud-0.3.7.6 → acryl_datahub_cloud-0.3.7.7rc2}/PKG-INFO

@@ -1,77 +1,80 @@
 Metadata-Version: 2.1
 Name: acryl-datahub-cloud
-Version: 0.3.7.6
+Version: 0.3.7.7rc2
 Requires-Dist: avro-gen3==0.7.16
 Requires-Dist: acryl-datahub
 Provides-Extra: datahub-lineage-features
-Requires-Dist: opensearch-py==2.4.2; extra == "datahub-lineage-features"
+Requires-Dist: pydantic<2; extra == "datahub-lineage-features"
 Requires-Dist: pyarrow; extra == "datahub-lineage-features"
 Requires-Dist: duckdb; extra == "datahub-lineage-features"
-Requires-Dist: pydantic<2; extra == "datahub-lineage-features"
 Requires-Dist: pandas; extra == "datahub-lineage-features"
+Requires-Dist: opensearch-py==2.4.2; extra == "datahub-lineage-features"
 Provides-Extra: datahub-reporting-forms
-Requires-Dist: pyarrow; extra == "datahub-reporting-forms"
 Requires-Dist: boto3; extra == "datahub-reporting-forms"
-Requires-Dist: duckdb; extra == "datahub-reporting-forms"
 Requires-Dist: pydantic<2; extra == "datahub-reporting-forms"
+Requires-Dist: pyarrow; extra == "datahub-reporting-forms"
+Requires-Dist: duckdb; extra == "datahub-reporting-forms"
 Requires-Dist: pandas; extra == "datahub-reporting-forms"
 Provides-Extra: datahub-reporting-extract-graph
-Requires-Dist: opensearch-py==2.4.2; extra == "datahub-reporting-extract-graph"
-Requires-Dist: pyarrow; extra == "datahub-reporting-extract-graph"
 Requires-Dist: boto3; extra == "datahub-reporting-extract-graph"
-Requires-Dist: duckdb; extra == "datahub-reporting-extract-graph"
 Requires-Dist: pydantic<2; extra == "datahub-reporting-extract-graph"
+Requires-Dist: pyarrow; extra == "datahub-reporting-extract-graph"
+Requires-Dist: duckdb; extra == "datahub-reporting-extract-graph"
 Requires-Dist: pandas; extra == "datahub-reporting-extract-graph"
+Requires-Dist: opensearch-py==2.4.2; extra == "datahub-reporting-extract-graph"
 Provides-Extra: datahub-reporting-extract-sql
-Requires-Dist: pyarrow; extra == "datahub-reporting-extract-sql"
 Requires-Dist: boto3; extra == "datahub-reporting-extract-sql"
-Requires-Dist: duckdb; extra == "datahub-reporting-extract-sql"
 Requires-Dist: pydantic<2; extra == "datahub-reporting-extract-sql"
+Requires-Dist: pyarrow; extra == "datahub-reporting-extract-sql"
+Requires-Dist: duckdb; extra == "datahub-reporting-extract-sql"
 Requires-Dist: pandas; extra == "datahub-reporting-extract-sql"
 Provides-Extra: datahub-usage-feature-reporting
+Requires-Dist: pyarrow<=18.0.0; extra == "datahub-usage-feature-reporting"
+Requires-Dist: elasticsearch==7.13.4; extra == "datahub-usage-feature-reporting"
+Requires-Dist: numpy<2; extra == "datahub-usage-feature-reporting"
+Requires-Dist: polars<=1.16.0; extra == "datahub-usage-feature-reporting"
 Requires-Dist: pyarrow; extra == "datahub-usage-feature-reporting"
-Requires-Dist: duckdb; extra == "datahub-usage-feature-reporting"
-Requires-Dist: pydantic<2; extra == "datahub-usage-feature-reporting"
 Requires-Dist: pandas; extra == "datahub-usage-feature-reporting"
-Requires-Dist: polars; extra == "datahub-usage-feature-reporting"
+Requires-Dist: scipy<=1.14.1; extra == "datahub-usage-feature-reporting"
 Requires-Dist: opensearch-py==2.4.2; extra == "datahub-usage-feature-reporting"
 Requires-Dist: boto3; extra == "datahub-usage-feature-reporting"
-Requires-Dist: numpy<2; extra == "datahub-usage-feature-reporting"
-Requires-Dist: scipy; extra == "datahub-usage-feature-reporting"
-Requires-Dist: elasticsearch==7.13.4; extra == "datahub-usage-feature-reporting"
+Requires-Dist: duckdb; extra == "datahub-usage-feature-reporting"
+Requires-Dist: pydantic<2; extra == "datahub-usage-feature-reporting"
 Provides-Extra: acryl-cs-issues
+Requires-Dist: openai; extra == "acryl-cs-issues"
 Requires-Dist: zenpy; extra == "acryl-cs-issues"
 Requires-Dist: slack-sdk; extra == "acryl-cs-issues"
 Requires-Dist: jinja2; extra == "acryl-cs-issues"
-Requires-Dist: openai; extra == "acryl-cs-issues"
 Provides-Extra: all
-Requires-Dist: pyarrow; extra == "all"
-Requires-Dist: duckdb; extra == "all"
-Requires-Dist: pydantic<2; extra == "all"
+Requires-Dist: pyarrow<=18.0.0; extra == "all"
+Requires-Dist: elasticsearch==7.13.4; extra == "all"
+Requires-Dist: numpy<2; extra == "all"
 Requires-Dist: slack-sdk; extra == "all"
+Requires-Dist: polars<=1.16.0; extra == "all"
+Requires-Dist: pyarrow; extra == "all"
 Requires-Dist: pandas; extra == "all"
-Requires-Dist: zenpy; extra == "all"
-Requires-Dist: polars; extra == "all"
+Requires-Dist: scipy<=1.14.1; extra == "all"
 Requires-Dist: opensearch-py==2.4.2; extra == "all"
 Requires-Dist: boto3; extra == "all"
 Requires-Dist: jinja2; extra == "all"
-Requires-Dist: numpy<2; extra == "all"
-Requires-Dist: scipy; extra == "all"
-Requires-Dist: elasticsearch==7.13.4; extra == "all"
+Requires-Dist: zenpy; extra == "all"
+Requires-Dist: duckdb; extra == "all"
 Requires-Dist: openai; extra == "all"
+Requires-Dist: pydantic<2; extra == "all"
 Provides-Extra: dev
-Requires-Dist: pyarrow; extra == "dev"
-Requires-Dist: duckdb; extra == "dev"
-Requires-Dist: pydantic<2; extra == "dev"
+Requires-Dist: pyarrow<=18.0.0; extra == "dev"
+Requires-Dist: elasticsearch==7.13.4; extra == "dev"
+Requires-Dist: numpy<2; extra == "dev"
 Requires-Dist: slack-sdk; extra == "dev"
+Requires-Dist: polars<=1.16.0; extra == "dev"
+Requires-Dist: pyarrow; extra == "dev"
 Requires-Dist: pandas; extra == "dev"
-Requires-Dist: acryl-datahub[dev]; extra == "dev"
-Requires-Dist: polars; extra == "dev"
+Requires-Dist: scipy<=1.14.1; extra == "dev"
 Requires-Dist: opensearch-py==2.4.2; extra == "dev"
 Requires-Dist: boto3; extra == "dev"
-Requires-Dist: zenpy; extra == "dev"
+Requires-Dist: acryl-datahub[dev]; extra == "dev"
 Requires-Dist: jinja2; extra == "dev"
-Requires-Dist: numpy<2; extra == "dev"
-Requires-Dist: scipy; extra == "dev"
-Requires-Dist: elasticsearch==7.13.4; extra == "dev"
+Requires-Dist: zenpy; extra == "dev"
+Requires-Dist: duckdb; extra == "dev"
 Requires-Dist: openai; extra == "dev"
+Requires-Dist: pydantic<2; extra == "dev"

{acryl_datahub_cloud-0.3.7.6 → acryl_datahub_cloud-0.3.7.7rc2}/setup.py

@@ -18,7 +18,14 @@ plugins = {
     "datahub-reporting-extract-sql": stats_common | aws_common,
     "datahub-usage-feature-reporting": stats_common
     | aws_common
-    | {"opensearch-py==2.4.2", "polars", "elasticsearch==7.13.4", "numpy<2", "scipy", "pyarrow"},
+    | {
+        "opensearch-py==2.4.2",
+        "polars<=1.16.0",
+        "elasticsearch==7.13.4",
+        "numpy<2",
+        "scipy<=1.14.1",
+        "pyarrow<=18.0.0",
+    },
     "acryl-cs-issues": {"zenpy", "openai", "jinja2", "slack-sdk"},
 }
 

{acryl_datahub_cloud-0.3.7.6 → acryl_datahub_cloud-0.3.7.7rc2}/src/acryl_datahub_cloud/_codegen_config.json

@@ -1,6 +1,6 @@
 {
   "name": "acryl-datahub-cloud",
-  "version": "0.3.7.6",
+  "version": "0.3.7.7rc2",
   "install_requires": [
     "avro-gen3==0.7.16",
     "acryl-datahub"

{acryl_datahub_cloud-0.3.7.6 → acryl_datahub_cloud-0.3.7.7rc2}/src/acryl_datahub_cloud/datahub_usage_reporting/usage_feature_reporter.py

@@ -449,7 +449,6 @@ class DataHubUsageFeatureReportingSource(StatefulIngestionSourceBase):
 
     def queries_entities_batch(self, results: Iterable) -> Iterable[Dict]:
         with PerfTimer() as timer:
-
             for doc in results:
                 if "platform" not in doc["_source"] or not doc["_source"]["platform"]:
                     logger.warning(
@@ -473,9 +472,11 @@ class DataHubUsageFeatureReportingSource(StatefulIngestionSourceBase):
                         )
                     ),
                     "platform": doc["_source"]["platform"],
-                    "removed": doc["_source"]["removed"]
-                    if "removed" in doc["_source"]
-                    else False,
+                    "removed": (
+                        doc["_source"]["removed"]
+                        if "removed" in doc["_source"]
+                        else False
+                    ),
                 }
 
             time_taken = timer.elapsed_seconds()
@@ -587,7 +588,6 @@ class DataHubUsageFeatureReportingSource(StatefulIngestionSourceBase):
 
     def process_batch(self, results: Iterable) -> Iterable[Dict]:
         with PerfTimer() as timer:
-
             for doc in results:
                 match = re.match(platform_regexp, doc["_source"]["urn"])
                 if match:
@@ -754,7 +754,6 @@ class DataHubUsageFeatureReportingSource(StatefulIngestionSourceBase):
         prefix: Optional[str] = None,
         use_exp_cdf: Optional[bool] = None,
     ) -> polars.LazyFrame:
-
         logger.debug(f"Generating rank and percentile for {count_field} field")
         lf = lf.with_columns(
             polars.col(count_field)
@@ -880,7 +879,6 @@ class DataHubUsageFeatureReportingSource(StatefulIngestionSourceBase):
     def load_write_usage(
         self, soft_deleted_entities_df: polars.LazyFrame
     ) -> polars.LazyFrame:
-
         if self.config.streaming_mode:
             wdf = self.load_es_data_to_lf(
                 index="dataset_operationaspect_v1",
@@ -1557,10 +1555,8 @@ class DataHubUsageFeatureReportingSource(StatefulIngestionSourceBase):
             usage_with_top_users_with_ranks.join(
                 write_lf, on="urn", how="full", suffix="_write"
             )
-            .with_columns("write_count")
-            .fill_null(polars.lit(0))
-            .with_columns("totalSqlQueries")
-            .fill_null(polars.lit(0))
+            .with_columns(polars.col("write_count").fill_null(polars.lit(0)))
+            .with_columns(polars.col("totalSqlQueries").fill_null(polars.lit(0)))
         )
 
         # If we get a dataset from the operation aspect index only then we have to use its urn and platform

{acryl_datahub_cloud-0.3.7.6 → acryl_datahub_cloud-0.3.7.7rc2}/src/acryl_datahub_cloud/lineage_features/source.py

@@ -83,16 +83,31 @@ class DataHubLineageFeaturesSource(Source):
         query = {
             "query": {
                 "bool": {
-                    "filter": [
-                        {"term": {"source.entityType": "schemaField"}},
-                        {"term": {"destination.entityType": "schemaField"}},
+                    "should": [
+                        {"term": {"relationshipType": "Consumes"}},
                         {"term": {"relationshipType": "DownstreamOf"}},
+                        {"term": {"relationshipType": "TrainedBy"}},
+                        {"term": {"relationshipType": "UsedBy"}},
+                        {"term": {"relationshipType": "MemberOf"}},
+                        {"term": {"relationshipType": "DerivedFrom"}},
+                        {"term": {"relationshipType": "Produces"}},
+                        {"term": {"relationshipType": "DashboardContainsDashboard"}},
+                        {
+                            "bool": {
+                                "must": [
+                                    {"term": {"relationshipType": "Contains"}},
+                                    {"term": {"source.entityType": "dashboard"}},
+                                    {"term": {"destination.entityType": "chart"}},
+                                ]
+                            }
+                        },
                     ],
-                }
+                },
             },
             "sort": [
                 {"source.urn": {"order": "desc"}},
                 {"destination.urn": {"order": "desc"}},
+                {"relationshipType": {"order": "desc"}},
                 {"lifecycleOwner": {"order": "desc"}},
             ],
         }
@@ -124,7 +139,9 @@ class DataHubLineageFeaturesSource(Source):
         for urn in set(self.upstream_counts.keys()).union(
             self.downstream_counts.keys()
         ):
-            print(urn, self.upstream_counts[urn], self.downstream_counts[urn])
+            logger.debug(
+                f"{urn}: {self.upstream_counts[urn]}, {self.downstream_counts[urn]}"
+            )
             yield MetadataChangeProposalWrapper(
                 entityUrn=urn,
                 aspect=LineageFeaturesClass(

{acryl_datahub_cloud-0.3.7.6 → acryl_datahub_cloud-0.3.7.7rc2}/src/acryl_datahub_cloud/metadata/com/linkedin/pegasus2avro/structured/__init__.py

@@ -13,6 +13,7 @@ from .....schema_classes import StructuredPropertiesClass
 from .....schema_classes import StructuredPropertyDefinitionClass
 from .....schema_classes import StructuredPropertyFilterStatusClass
 from .....schema_classes import StructuredPropertyKeyClass
+from .....schema_classes import StructuredPropertySettingsClass
 from .....schema_classes import StructuredPropertyValueAssignmentClass
 
 
@@ -22,6 +23,7 @@ StructuredProperties = StructuredPropertiesClass
 StructuredPropertyDefinition = StructuredPropertyDefinitionClass
 StructuredPropertyFilterStatus = StructuredPropertyFilterStatusClass
 StructuredPropertyKey = StructuredPropertyKeyClass
+StructuredPropertySettings = StructuredPropertySettingsClass
 StructuredPropertyValueAssignment = StructuredPropertyValueAssignmentClass
 
 # fmt: on

{acryl_datahub_cloud-0.3.7.6 → acryl_datahub_cloud-0.3.7.7rc2}/src/acryl_datahub_cloud/metadata/schema.avsc

@@ -3389,7 +3389,8 @@
         "documentation",
         "testResults",
         "schemaFieldProfile",
-        "lineageFeatures"
+        "lineageFeatures",
+        "deprecation"
       ]
     },
     "name": "SchemaFieldKey",
@@ -6195,6 +6196,7 @@
       "entityCategory": "core",
       "entityAspects": [
         "propertyDefinition",
+        "structuredPropertySettings",
         "institutionalMemory",
         "status",
         "share",
@@ -6536,6 +6538,7 @@
         "Searchable": {
           "fieldType": "KEYWORD"
         },
+        "deprecated": true,
         "type": [
           {
             "type": "enum",
@@ -6554,10 +6557,81 @@
         ],
         "name": "filterStatus",
         "default": "DISABLED",
-        "doc": "The filter status of this structured property - whether it will show up in search filters or not"
+        "doc": "The filter status of this structured property - whether it will show up in search filters or not\nDeprecated! Use structuredPropertySettings.showInSearchFilters instead"
       }
     ]
   },
+  {
+    "type": "record",
+    "Aspect": {
+      "name": "structuredPropertySettings"
+    },
+    "name": "StructuredPropertySettings",
+    "namespace": "com.linkedin.pegasus2avro.structured",
+    "fields": [
+      {
+        "Searchable": {
+          "fieldType": "BOOLEAN"
+        },
+        "type": "boolean",
+        "name": "isHidden",
+        "default": false,
+        "doc": "Whether or not this asset should be hidden in the main application"
+      },
+      {
+        "Searchable": {
+          "fieldType": "BOOLEAN"
+        },
+        "type": "boolean",
+        "name": "showInSearchFilters",
+        "default": false,
+        "doc": "Whether or not this asset should be displayed as a search filter"
+      },
+      {
+        "Searchable": {
+          "fieldType": "BOOLEAN"
+        },
+        "type": "boolean",
+        "name": "showInAssetSummary",
+        "default": false,
+        "doc": "Whether or not this asset should be displayed in the asset sidebar"
+      },
+      {
+        "Searchable": {
+          "fieldType": "BOOLEAN"
+        },
+        "type": "boolean",
+        "name": "showAsAssetBadge",
+        "default": false,
+        "doc": "Whether or not this asset should be displayed as an asset badge on other\nasset's headers"
+      },
+      {
+        "Searchable": {
+          "fieldType": "BOOLEAN"
+        },
+        "type": "boolean",
+        "name": "showInColumnsTable",
+        "default": false,
+        "doc": "Whether or not this asset should be displayed as a column in the schema field table\nin a Dataset's \"Columns\" tab."
+      },
+      {
+        "Searchable": {
+          "/time": {
+            "fieldName": "lastModifiedSettings",
+            "fieldType": "DATETIME"
+          }
+        },
+        "type": [
+          "null",
+          "com.linkedin.pegasus2avro.common.AuditStamp"
+        ],
+        "name": "lastModified",
+        "default": null,
+        "doc": "Last Modified Audit stamp"
+      }
+    ],
+    "doc": "Settings specific to a structured property entity"
+  },
   {
     "type": "record",
     "Aspect": {
@@ -9292,7 +9366,7 @@
                                                                     "fields": [
                                                                       {
                                                                         "Searchable": {
-                                                                          "boostScore": 5.0,
+                                                                          "boostScore": 1.0,
                                                                           "fieldName": "fieldPaths",
                                                                           "fieldType": "TEXT",
                                                                           "queryByDefault": "true"
@@ -10249,7 +10323,7 @@
                                                                             "doc": "The adjustment algorithm to use when determining the threshold for an assertion."
                                                                           },
                                                                           "name": "algorithm",
-                                                                          "doc": "The algorithm to use to adjust assertion values to power the sensitivity control feature"
+                                                                          "doc": "The algorithm to use to adjust assertion values to power the sensitivity control feature\n\nNote that for algorithm \"STDDEV\" which leverages the standard deviation of trailing values,\nthe \"stdDev\" field must be provided inside of AssertionEvaluationContext.pdl by the offline pipeline."
                                                                         },
                                                                         {
                                                                           "type": "string",
@@ -10335,6 +10409,15 @@
                                 "name": "embeddedAssertions",
                                 "default": null,
                                 "doc": "Currently used for Smart Assertions\nAn embedded copy of the assertion used to evaluate which will overwrite the referenced assertion\nif present and if the EmbeddedAssertion's evaluationTimeWindow period is valid"
+                              },
+                              {
+                                "type": [
+                                  "null",
+                                  "float"
+                                ],
+                                "name": "stdDev",
+                                "default": null,
+                                "doc": "The std deviation of the metric values used for training.\nThis is used to determine the final adjusted threshold for the assertion."
                               }
                             ],
                             "doc": "Additional context about assertion being evaluated."
@@ -13383,6 +13466,18 @@
         "type": "string",
         "name": "actor",
         "doc": "The user URN which will be credited for modifying this deprecation content."
+      },
+      {
+        "java": {
+          "class": "com.linkedin.pegasus2avro.common.urn.Urn"
+        },
+        "Urn": "Urn",
+        "type": [
+          "null",
+          "string"
+        ],
+        "name": "replacement",
+        "default": null
       }
     ],
     "doc": "Deprecation status of an entity"
@@ -14080,6 +14175,9 @@
         "doc": "The display name of the ingestion source"
       },
       {
+        "Searchable": {
+          "fieldType": "TEXT_PARTIAL"
+        },
         "type": "string",
         "name": "type",
         "doc": "The type of the source itself, e.g. mysql, bigquery, bigquery-usage. Should match the recipe."
@@ -14144,6 +14242,9 @@
               "doc": "The PyPI version of the datahub CLI to use when executing a recipe"
             },
             {
+              "Searchable": {
+                "fieldName": "sourceExecutorId"
+              },
               "type": [
                 "null",
                 "string"