acryl-datahub-cloud 0.3.10.4rc2__tar.gz → 0.3.11__tar.gz

{acryl_datahub_cloud-0.3.10.4rc2 → acryl_datahub_cloud-0.3.11}/PKG-INFO

@@ -1,92 +1,92 @@
 Metadata-Version: 2.4
 Name: acryl-datahub-cloud
-Version: 0.3.10.4rc2
+Version: 0.3.11
 Requires-Dist: avro-gen3==0.7.16
 Requires-Dist: acryl-datahub
 Provides-Extra: datahub-lineage-features
-Requires-Dist: opensearch-py==2.4.2; extra == "datahub-lineage-features"
-Requires-Dist: pyarrow; extra == "datahub-lineage-features"
 Requires-Dist: pydantic<2; extra == "datahub-lineage-features"
 Requires-Dist: duckdb; extra == "datahub-lineage-features"
+Requires-Dist: opensearch-py==2.4.2; extra == "datahub-lineage-features"
+Requires-Dist: pyarrow; extra == "datahub-lineage-features"
 Requires-Dist: pandas; extra == "datahub-lineage-features"
 Provides-Extra: datahub-reporting-forms
-Requires-Dist: boto3; extra == "datahub-reporting-forms"
-Requires-Dist: pyarrow; extra == "datahub-reporting-forms"
-Requires-Dist: termcolor==2.5.0; extra == "datahub-reporting-forms"
 Requires-Dist: pydantic<2; extra == "datahub-reporting-forms"
 Requires-Dist: duckdb; extra == "datahub-reporting-forms"
+Requires-Dist: pyarrow; extra == "datahub-reporting-forms"
+Requires-Dist: boto3; extra == "datahub-reporting-forms"
+Requires-Dist: termcolor==2.5.0; extra == "datahub-reporting-forms"
 Requires-Dist: pandas; extra == "datahub-reporting-forms"
 Provides-Extra: datahub-reporting-extract-graph
-Requires-Dist: boto3; extra == "datahub-reporting-extract-graph"
-Requires-Dist: opensearch-py==2.4.2; extra == "datahub-reporting-extract-graph"
-Requires-Dist: pyarrow; extra == "datahub-reporting-extract-graph"
 Requires-Dist: pydantic<2; extra == "datahub-reporting-extract-graph"
 Requires-Dist: duckdb; extra == "datahub-reporting-extract-graph"
+Requires-Dist: opensearch-py==2.4.2; extra == "datahub-reporting-extract-graph"
+Requires-Dist: pyarrow; extra == "datahub-reporting-extract-graph"
+Requires-Dist: boto3; extra == "datahub-reporting-extract-graph"
 Requires-Dist: pandas; extra == "datahub-reporting-extract-graph"
 Provides-Extra: datahub-reporting-extract-sql
-Requires-Dist: boto3; extra == "datahub-reporting-extract-sql"
-Requires-Dist: pyarrow; extra == "datahub-reporting-extract-sql"
 Requires-Dist: pydantic<2; extra == "datahub-reporting-extract-sql"
 Requires-Dist: duckdb; extra == "datahub-reporting-extract-sql"
+Requires-Dist: pyarrow; extra == "datahub-reporting-extract-sql"
+Requires-Dist: boto3; extra == "datahub-reporting-extract-sql"
 Requires-Dist: pandas; extra == "datahub-reporting-extract-sql"
 Provides-Extra: datahub-usage-reporting
-Requires-Dist: boto3; extra == "datahub-usage-reporting"
+Requires-Dist: numpy<2; extra == "datahub-usage-reporting"
 Requires-Dist: polars==1.23.0; extra == "datahub-usage-reporting"
-Requires-Dist: elasticsearch==7.13.4; extra == "datahub-usage-reporting"
+Requires-Dist: opensearch-py==2.4.2; extra == "datahub-usage-reporting"
+Requires-Dist: scipy<=1.14.1; extra == "datahub-usage-reporting"
 Requires-Dist: termcolor==2.5.0; extra == "datahub-usage-reporting"
+Requires-Dist: pandas; extra == "datahub-usage-reporting"
 Requires-Dist: pydantic<2; extra == "datahub-usage-reporting"
 Requires-Dist: duckdb; extra == "datahub-usage-reporting"
-Requires-Dist: pandas; extra == "datahub-usage-reporting"
-Requires-Dist: scipy<=1.14.1; extra == "datahub-usage-reporting"
-Requires-Dist: numpy<2; extra == "datahub-usage-reporting"
-Requires-Dist: opensearch-py==2.4.2; extra == "datahub-usage-reporting"
-Requires-Dist: pyarrow; extra == "datahub-usage-reporting"
 Requires-Dist: pyarrow<=18.0.0; extra == "datahub-usage-reporting"
+Requires-Dist: boto3; extra == "datahub-usage-reporting"
+Requires-Dist: elasticsearch==7.13.4; extra == "datahub-usage-reporting"
+Requires-Dist: pyarrow; extra == "datahub-usage-reporting"
 Provides-Extra: datahub-metadata-sharing
 Requires-Dist: tenacity; extra == "datahub-metadata-sharing"
 Provides-Extra: datahub-action-request-owner
 Requires-Dist: tenacity; extra == "datahub-action-request-owner"
 Provides-Extra: acryl-cs-issues
-Requires-Dist: slack-sdk; extra == "acryl-cs-issues"
-Requires-Dist: openai; extra == "acryl-cs-issues"
 Requires-Dist: jinja2; extra == "acryl-cs-issues"
 Requires-Dist: zenpy; extra == "acryl-cs-issues"
+Requires-Dist: openai; extra == "acryl-cs-issues"
+Requires-Dist: slack-sdk; extra == "acryl-cs-issues"
 Provides-Extra: all
-Requires-Dist: boto3; extra == "all"
-Requires-Dist: elasticsearch==7.13.4; extra == "all"
-Requires-Dist: jinja2; extra == "all"
+Requires-Dist: polars==1.23.0; extra == "all"
+Requires-Dist: opensearch-py==2.4.2; extra == "all"
+Requires-Dist: scipy<=1.14.1; extra == "all"
+Requires-Dist: pandas; extra == "all"
 Requires-Dist: pydantic<2; extra == "all"
 Requires-Dist: duckdb; extra == "all"
-Requires-Dist: pandas; extra == "all"
-Requires-Dist: numpy<2; extra == "all"
-Requires-Dist: openai; extra == "all"
-Requires-Dist: opensearch-py==2.4.2; extra == "all"
+Requires-Dist: boto3; extra == "all"
 Requires-Dist: pyarrow<=18.0.0; extra == "all"
-Requires-Dist: polars==1.23.0; extra == "all"
-Requires-Dist: termcolor==2.5.0; extra == "all"
-Requires-Dist: slack-sdk; extra == "all"
-Requires-Dist: scipy<=1.14.1; extra == "all"
+Requires-Dist: elasticsearch==7.13.4; extra == "all"
+Requires-Dist: openai; extra == "all"
+Requires-Dist: pyarrow; extra == "all"
 Requires-Dist: tenacity; extra == "all"
+Requires-Dist: numpy<2; extra == "all"
+Requires-Dist: jinja2; extra == "all"
+Requires-Dist: termcolor==2.5.0; extra == "all"
 Requires-Dist: zenpy; extra == "all"
-Requires-Dist: pyarrow; extra == "all"
+Requires-Dist: slack-sdk; extra == "all"
 Provides-Extra: dev
-Requires-Dist: boto3; extra == "dev"
+Requires-Dist: numpy<2; extra == "dev"
 Requires-Dist: polars==1.23.0; extra == "dev"
-Requires-Dist: elasticsearch==7.13.4; extra == "dev"
+Requires-Dist: opensearch-py==2.4.2; extra == "dev"
+Requires-Dist: scipy<=1.14.1; extra == "dev"
 Requires-Dist: jinja2; extra == "dev"
 Requires-Dist: termcolor==2.5.0; extra == "dev"
-Requires-Dist: slack-sdk; extra == "dev"
-Requires-Dist: pydantic<2; extra == "dev"
-Requires-Dist: duckdb; extra == "dev"
-Requires-Dist: pandas; extra == "dev"
 Requires-Dist: acryl-datahub[dev]; extra == "dev"
+Requires-Dist: pandas; extra == "dev"
 Requires-Dist: tenacity; extra == "dev"
-Requires-Dist: scipy<=1.14.1; extra == "dev"
-Requires-Dist: numpy<2; extra == "dev"
-Requires-Dist: openai; extra == "dev"
-Requires-Dist: opensearch-py==2.4.2; extra == "dev"
+Requires-Dist: pydantic<2; extra == "dev"
+Requires-Dist: duckdb; extra == "dev"
 Requires-Dist: zenpy; extra == "dev"
-Requires-Dist: pyarrow; extra == "dev"
 Requires-Dist: pyarrow<=18.0.0; extra == "dev"
+Requires-Dist: boto3; extra == "dev"
+Requires-Dist: slack-sdk; extra == "dev"
+Requires-Dist: elasticsearch==7.13.4; extra == "dev"
+Requires-Dist: openai; extra == "dev"
+Requires-Dist: pyarrow; extra == "dev"
 Dynamic: provides-extra
 Dynamic: requires-dist

{acryl_datahub_cloud-0.3.10.4rc2 → acryl_datahub_cloud-0.3.11}/README.md

@@ -3,4 +3,3 @@
 This package contains additional DataHub models and sources that are specific to Acryl Cloud.
 
 Installing this package will automatically enable the `acryl-datahub` package to make use of this additional functionality.
-

acryl_datahub_cloud-0.3.11/pyproject.toml

@@ -0,0 +1,40 @@
+[build-system]
+build-backend = "setuptools.build_meta"
+requires = ["setuptools >= 71.1", "wheel"]
+
+[tool.ruff.lint.isort]
+known-first-party = ["datahub"]
+combine-as-imports = true
+
+[tool.ruff]
+target-version = "py310"
+
+[tool.ruff.lint]
+extend-select = [
+    "B",    # flake8-bugbear
+    "C90",  # mccabe complexity
+    "E",    # pycodestyle errors
+    "F",    # pyflakes
+    "G010", # logging.warn -> logging.warning
+    "I",    # isort
+    "TID",  # flake8-tidy-imports
+    "RUF100",  # unused-noqa
+    "SIM",   # flake8-simplify
+]
+extend-ignore = [
+    "E501",  # Handled by formatter
+    "E111",  # Handled by formatter
+    "E114",  # Handled by formatter
+    "E117",  # Handled by formatter
+    #"B019",  # Allow usages of functools.lru_cache
+    #"B008",  # Allow function call in argument defaults
+    #"RUF012", # mutable-class-default; incompatible with pydantic
+    #"RUF015", # unnecessary-iterable-allocation-for-first-element
+    
+    # Can be enabled later if someone wants to fix existing cases or an auto-fix becomes available
+    "SIM103", # Return the condition directly
+    "SIM108", # Uses ternaries instead of if..else
+]
+
+[tool.ruff.lint.flake8-tidy-imports]
+ban-relative-imports = "all"

{acryl_datahub_cloud-0.3.10.4rc2 → acryl_datahub_cloud-0.3.11}/setup.cfg

@@ -41,7 +41,7 @@ skip_glob = src/acryl_datahub_cloud/metadata
 known_local_folder = datahub,acryl_datahub_cloud
 
 [tool:pytest]
-addopts = --cov=src --cov-report='' --cov-config setup.cfg --strict-markers -s -v
+addopts = --strict-markers -s -v
 
 [coverage:report]
 show_missing = true

{acryl_datahub_cloud-0.3.10.4rc2 → acryl_datahub_cloud-0.3.11}/src/acryl_datahub_cloud/_codegen_config.json

@@ -1,6 +1,6 @@
 {
   "name": "acryl-datahub-cloud",
-  "version": "0.3.10.4rc2",
+  "version": "0.3.11",
   "install_requires": [
     "avro-gen3==0.7.16",
     "acryl-datahub"

acryl_datahub_cloud-0.3.11/src/acryl_datahub_cloud/_sdk_extras/__init__.py

@@ -0,0 +1,4 @@
+from acryl_datahub_cloud._sdk_extras.assertion import Assertion
+from acryl_datahub_cloud._sdk_extras.assertions_client import AssertionsClient
+
+__all__ = ["Assertion", "AssertionsClient"]

acryl_datahub_cloud-0.3.11/src/acryl_datahub_cloud/_sdk_extras/assertion.py

@@ -0,0 +1,15 @@
+"""
+This file contains the Assertion class, which is used to represent an assertion in DataHub.
+
+The Assertion class is currently not implemented, this is a placeholder for future implementation.
+"""
+
+from typing import Union
+
+from datahub.metadata.urns import AssertionUrn
+
+
+class Assertion:
+    def __init__(self, urn: Union[str, AssertionUrn]):
+        print(f"The Assertion class is currently not implemented. Urn provided: {urn}")
+        self.urn = AssertionUrn(urn)

acryl_datahub_cloud-0.3.11/src/acryl_datahub_cloud/_sdk_extras/assertions_client.py

@@ -0,0 +1,23 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Union
+
+from acryl_datahub_cloud._sdk_extras.assertion import Assertion
+from datahub.metadata.urns import AssertionUrn
+
+if TYPE_CHECKING:
+    from datahub.sdk.main_client import DataHubClient
+
+
+class AssertionsClient:
+    def __init__(self, client: DataHubClient):
+        self.client = client
+
+    def get_assertions(
+        self, urn: Union[str, list[str], AssertionUrn, list[AssertionUrn]]
+    ) -> list[Assertion]:
+        print(
+            "get_assertions is not implemented, this is a placeholder. Returning empty list."
+        )
+        print(f"urn provided: {urn}")
+        return []

{acryl_datahub_cloud-0.3.10.4rc2 → acryl_datahub_cloud-0.3.11}/src/acryl_datahub_cloud/acryl_cs_issues/acryl_customer.py

@@ -219,6 +219,7 @@ class AcrylCustomer:
         self._emitted_containers: Dict[str, bool] = {}
 
     def _get_owner_from_assignee(self, assignee: ExternalUser) -> str:
+        assert assignee.email, "Assignee must have an email"
         owner_urn_options = [
             urn
             for urn in self.graph.get_urns_by_filter(
@@ -233,7 +234,6 @@ class AcrylCustomer:
             )
         ]
         if not owner_urn_options:
-            assert assignee.email, "Assignee must have an email"
             owner_urn = "urn:li:corpuser:" + assignee.email
             self.graph.emit(
                 MetadataChangeProposalWrapper(

{acryl_datahub_cloud-0.3.10.4rc2 → acryl_datahub_cloud-0.3.11}/src/acryl_datahub_cloud/action_request/action_request_owner_source.py

@@ -2,6 +2,7 @@ import logging
 from typing import Dict, Iterable, List, Optional
 
 from datahub.configuration import ConfigModel
+from datahub.emitter.mcp import MetadataChangeProposalWrapper
 from datahub.ingestion.api.common import PipelineContext
 from datahub.ingestion.api.decorators import (
     SupportStatus,
@@ -14,10 +15,8 @@ from datahub.ingestion.api.workunit import MetadataWorkUnit
 from datahub.metadata.schema_classes import (
     ActionRequestInfoClass,
 )
-from datahub.emitter.mcp import MetadataChangeProposalWrapper
 from datahub.utilities.urns.urn import guess_entity_type
 
-
 logger = logging.getLogger(__name__)
 
 

{acryl_datahub_cloud-0.3.10.4rc2 → acryl_datahub_cloud-0.3.11}/src/acryl_datahub_cloud/datahub_reporting/datahub_dataset.py

@@ -163,7 +163,7 @@ class DataHubBasedS3Dataset:
                     self.schema = row.arrow_schema()
                 else:
                     # hail mary: infer schema from the first row and cast everything to string
-                    self.schema = pa.schema([(key, pa.string()) for key in row.keys()])
+                    self.schema = pa.schema([(key, pa.string()) for key in row])
                     self.stringify_row = True
 
             self._initialize_local_file()
@@ -172,7 +172,7 @@ class DataHubBasedS3Dataset:
                 self.schema,
                 compression=self.config.file_compression,
             )
-        if isinstance(row, BaseModel) or isinstance(row, BaseModelRow):
+        if isinstance(row, (BaseModel, BaseModelRow)):
             # for anything extending BaseModel, we want to use the dict representation
             write_row: Dict[str, Any] = row.dict()
         elif isinstance(row, dict):
@@ -274,11 +274,7 @@ class DataHubBasedS3Dataset:
         self, duckdb_columns: List[Tuple[str, str]]
     ) -> SchemaMetadataClass:
         def get_type_from_dtype(dtype: str) -> SchemaFieldDataTypeClass:
-            if "int" in dtype:
-                return SchemaFieldDataTypeClass(type=NumberTypeClass())
-            elif "float" in dtype:
-                return SchemaFieldDataTypeClass(type=NumberTypeClass())
-            elif "number" in dtype:
+            if "int" in dtype or "float" in dtype or "number" in dtype:
                 return SchemaFieldDataTypeClass(type=NumberTypeClass())
             elif "bool" in dtype:
                 return SchemaFieldDataTypeClass(type=BooleanTypeClass())

{acryl_datahub_cloud-0.3.10.4rc2 → acryl_datahub_cloud-0.3.11}/src/acryl_datahub_cloud/datahub_reporting/datahub_form_reporting.py

@@ -5,6 +5,9 @@ from enum import Enum
 from typing import Any, Callable, Dict, Iterable, List, Optional
 
 import pandas as pd
+from pydantic import BaseModel
+
+from acryl_datahub_cloud.elasticsearch.graph_service import BaseModelRow
 from datahub.emitter.mcp import MetadataChangeProposalWrapper
 from datahub.ingestion.graph.client import DataHubGraph
 from datahub.ingestion.graph.filters import RawSearchFilterRule
@@ -16,9 +19,6 @@ from datahub.metadata.schema_classes import (
     FormStateClass,
     FormTypeClass,
 )
-from pydantic import BaseModel
-
-from acryl_datahub_cloud.elasticsearch.graph_service import BaseModelRow
 
 logger = logging.getLogger(__name__)
 
@@ -257,6 +257,7 @@ class DataHubFormReportingData(FormData):
                 for prompt_id, response_time in zip(
                     search_row.completedFormsCompletedPromptIds,
                     search_row.completedFormsCompletedPromptResponseTimes,
+                    strict=False,
                 )
                 if prompt_id in form_prompts
             }
@@ -289,9 +290,10 @@ class DataHubFormReportingData(FormData):
         on_asset_scanned: Optional[Callable[[str], Any]] = None,
         on_form_scanned: Optional[Callable[[str], Any]] = None,
     ) -> Iterable[FormReportingRow]:
-        extra_fields = [f for f in self.DataHubDatasetSearchRow.__fields__.keys()]
+        extra_fields = [f for f in self.DataHubDatasetSearchRow.__fields__]
+        # TODO: Replace with the new search/filter SDK.
         result = self.graph.get_results_by_filter(
-            extra_or_filters=self.get_form_existence_or_filters(),
+            extra_or_filters=[{"and": self.get_form_existence_or_filters()}],
             extra_source_fields=extra_fields,
             skip_cache=True,
         )
@@ -388,6 +390,7 @@ class DataHubFormReportingData(FormData):
                     for (p, p_response_time) in zip(
                         search_row.incompleteFormsCompletedPromptIds,
                         search_row.incompleteFormsCompletedPromptResponseTimes,
+                        strict=False,
                     )
                     if p in form_prompts
                 ]:
@@ -485,6 +488,7 @@ class DataHubFormReportingData(FormData):
                     for (p, p_response_time) in zip(
                         search_row.completedFormsCompletedPromptIds,
                         search_row.completedFormsCompletedPromptResponseTimes,
+                        strict=False,
                     )
                     if p in form_prompts
                 ]:

{acryl_datahub_cloud-0.3.10.4rc2 → acryl_datahub_cloud-0.3.11}/src/acryl_datahub_cloud/datahub_usage_reporting/usage_feature_reporter.py

@@ -395,18 +395,10 @@ class DataHubUsageFeatureReportingSource(StatefulIngestionSourceBase):
                     "last_modified_at": (
                         doc["_source"]["lastModifiedAt"]
                         if "lastModifiedAt" in doc["_source"]
-                        else (
-                            doc["_source"]["lastModifiedAt"]
-                            if "lastModifiedAt" in doc["_source"]
-                            else None
-                        )
+                        else (doc["_source"].get("lastModifiedAt", None))
                     ),
                     "platform": doc["_source"]["platform"],
-                    "removed": (
-                        doc["_source"]["removed"]
-                        if "removed" in doc["_source"]
-                        else False
-                    ),
+                    "removed": (doc["_source"].get("removed", False)),
                 }
 
             time_taken = timer.elapsed_seconds()
@@ -509,11 +501,7 @@ class DataHubUsageFeatureReportingSource(StatefulIngestionSourceBase):
                         "eventGranularity": doc["_source"].get("eventGranularity"),
                         "totalSqlQueries": doc["_source"].get("totalSqlQueries", 0),
                         "uniqueUserCount": doc["_source"].get("uniqueUserCount", 0),
-                        "userCounts": (
-                            doc["_source"]["event"]["userCounts"]
-                            if "userCounts" in doc["_source"]["event"]
-                            else None
-                        ),
+                        "userCounts": (doc["_source"]["event"].get("userCounts", None)),
                         "platform": platform,
                     }
                 except KeyError as e:
@@ -525,7 +513,7 @@ class DataHubUsageFeatureReportingSource(StatefulIngestionSourceBase):
             time_taken = timer.elapsed_seconds()
             logger.info(f"DatasetUsage processing took {time_taken:.3f} seconds")
 
-    def search_score(
+    def search_score(  # noqa: C901
         self, urn: str, last_update_time: int, usage_percentile: int
     ) -> SearchRankingMultipliers:
         usage_search_score_multiplier = 1.0
@@ -622,10 +610,10 @@ class DataHubUsageFeatureReportingSource(StatefulIngestionSourceBase):
                         [endpoint],
                         http_auth=(user, password),
                         use_ssl=(
-                            True
-                            if self.config.search_index
-                            and self.config.search_index.use_ssl
-                            else False
+                            bool(
+                                self.config.search_index
+                                and self.config.search_index.use_ssl
+                            )
                         ),
                     )
 
@@ -639,10 +627,10 @@ class DataHubUsageFeatureReportingSource(StatefulIngestionSourceBase):
                         [endpoint],
                         http_auth=(user, password),
                         use_ssl=(
-                            True
-                            if self.config.search_index
-                            and self.config.search_index.use_ssl
-                            else False
+                            bool(
+                                self.config.search_index
+                                and self.config.search_index.use_ssl
+                            )
                         ),
                     )
 
@@ -737,7 +725,7 @@ class DataHubUsageFeatureReportingSource(StatefulIngestionSourceBase):
                 polars.Duration(): pa.duration("ns"),
             }
 
-            if polars_dtype in [type(key) for key in type_mapping.keys()]:
+            if polars_dtype in [type(key) for key in type_mapping]:
                 return type_mapping[polars_dtype]
             elif polars_dtype == polars.Categorical:
                 return pa.dictionary(index_type=pa.int32(), value_type=pa.string())
@@ -1006,12 +994,9 @@ class DataHubUsageFeatureReportingSource(StatefulIngestionSourceBase):
     def generate_mcp_from_lazyframe(
         self, lazy_frame: polars.LazyFrame
     ) -> Iterable[MetadataWorkUnit]:
-        num = 0
         for row in lazy_frame.collect(
             streaming=self.config.experimental_full_streaming
         ).to_struct():
-            num += 1
-
             if "siblings" in row and row["siblings"]:
                 logger.info(f"Siblings found for urn: {row['urn']} -> row['siblings']")
 
@@ -1101,10 +1086,7 @@ class DataHubUsageFeatureReportingSource(StatefulIngestionSourceBase):
     def generate_query_usage_mcp_from_lazyframe(
         self, lazy_frame: polars.LazyFrame
     ) -> Iterable[MetadataWorkUnit]:
-        num = 0
         for row in lazy_frame.collect().iter_rows(named=True):
-            num += 1
-
             query_usage_features = QueryUsageFeaturesClass(
                 queryCountLast30Days=int(row.get("totalSqlQueries", 0) or 0),
                 queryCountTotal=None,  # This is not implemented
@@ -1287,7 +1269,7 @@ class DataHubUsageFeatureReportingSource(StatefulIngestionSourceBase):
                 .is_not_null()
                 # We only want to downrank datasets that have a search score multiplier greater than 1. 1 is the minimum score of a dataset
                 .and_(polars.col("combinedSearchRankingMultiplier").ne(1))
-            )  # noqa: E712
+            )
             .filter(polars.col("removed") == False)  # noqa: E712
             .drop(["removed"])
             .drop(["last_modified_at"])