acryl-datahub 1.0.0.3rc11__py3-none-any.whl → 1.0.0.3rc12__py3-none-any.whl

{acryl_datahub-1.0.0.3rc11.dist-info → acryl_datahub-1.0.0.3rc12.dist-info}/RECORD

@@ -1,7 +1,7 @@
-acryl_datahub-1.0.0.3rc11.dist-info/licenses/LICENSE,sha256=9xNHpsD0uYF5ONzXsKDCuHHB-xbiCrSbueWXqrTNsxk,11365
+acryl_datahub-1.0.0.3rc12.dist-info/licenses/LICENSE,sha256=9xNHpsD0uYF5ONzXsKDCuHHB-xbiCrSbueWXqrTNsxk,11365
 datahub/__init__.py,sha256=aq_i5lVREmoLfYIqcx_pEQicO855YlhD19tWc1eZZNI,59
 datahub/__main__.py,sha256=pegIvQ9hzK7IhqVeUi1MeADSZ2QlP-D3K0OQdEg55RU,106
-datahub/_version.py,sha256=twce5d4IphdpniEHSuyITBxTGY0-8pqvE-ojb42fqFs,324
+datahub/_version.py,sha256=GsdkGKv5g7BK04mNrjmP0H1a3Wlz4nl0YfMuvCp1f_c,324
 datahub/entrypoints.py,sha256=AQN5MzCe6q3LKI4SS6WmwN56kgjF6AC1ld7yELWVP2w,8953
 datahub/errors.py,sha256=p5rFAdAGVCk4Lqolol1YvthceadUSwpaCxLXRcyCCFQ,676
 datahub/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -171,7 +171,7 @@ datahub/ingestion/glossary/classifier.py,sha256=daLxnVv_JlfB_jBOxH5LrU_xQRndrsGo
 datahub/ingestion/glossary/classifier_registry.py,sha256=yFOYLQhDgCLqXYMG3L1BquXafeLcZDcmp8meyw6k9ts,307
 datahub/ingestion/glossary/datahub_classifier.py,sha256=O7wm6gQT1Jf2QSKdWjJQbS5oSzJwplXzfza26Gdq5Mg,7555
 datahub/ingestion/graph/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-datahub/ingestion/graph/client.py,sha256=Mtnidbtkd14r-bp8gN1rlBFiOSTJI7SvstBdCRp9uYU,70314
+datahub/ingestion/graph/client.py,sha256=a4cQPgElfW-IuBV_78pN5gM6rJe-PN7yK77jFana924,70296
 datahub/ingestion/graph/config.py,sha256=sPBwa7_v10B4sR0mW7ssQli3ljb6hdZd2Xeaq4Jhs6Y,1087
 datahub/ingestion/graph/connections.py,sha256=9462L0ZWGKURyypAln25eMPhK3pcufBar9tNDoqspXs,741
 datahub/ingestion/graph/entity_versioning.py,sha256=nrcNz0Qm6kpE6oTu_mrYUQDx14KPspBTc6R9SyFUY6c,6901
@@ -307,7 +307,7 @@ datahub/ingestion/source/dremio/dremio_source.py,sha256=_52Z0ifntbhYNwlrMs6jZ59C
 datahub/ingestion/source/dremio/dremio_sql_queries.py,sha256=W0rcXawlwJOHNYr5o73rilMijtFOO3cVkn6pY-JLc6o,8186
 datahub/ingestion/source/dynamodb/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 datahub/ingestion/source/dynamodb/data_reader.py,sha256=vC77KpcP8LJN0g8wsPRDVw4sebv0ZWIP3tJkEIHaomA,3120
-datahub/ingestion/source/dynamodb/dynamodb.py,sha256=Z1cwomrAgTfBfSdVtG-Tfbn_--2STY6wTsm6VoACVA4,22625
+datahub/ingestion/source/dynamodb/dynamodb.py,sha256=vM3Ia5rZidqOcdPPigpuo6-7Ipoof8eF3RwxJ3SX2Ck,22771
 datahub/ingestion/source/fivetran/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 datahub/ingestion/source/fivetran/config.py,sha256=BP3KRfAQ6H5qyEeJNu9vNfZNwLoyj4Tl2kXiLVR5DNM,9027
 datahub/ingestion/source/fivetran/data_classes.py,sha256=ecdUJH5BEze0yv-uFpKWPNaNmV1gORDA2XMFk0zhcBw,595
@@ -474,7 +474,7 @@ datahub/ingestion/source/sql/cockroachdb.py,sha256=XaD7eae34plU9ISRC6PzYX9q6RdT2
 datahub/ingestion/source/sql/druid.py,sha256=_tzgTa5jhPUXk6WCmS7p10feCwJm6yUFcOgMZA-OcE8,2922
 datahub/ingestion/source/sql/hana.py,sha256=0PIvcX0Rz59NyR7Ag5Bv1MBV_UbJwxl9UAopo_xe_CA,1342
 datahub/ingestion/source/sql/hive.py,sha256=voQl6QjHUXPdx7LPONuHiFavW9nRKMjHZx7o3vJQG7A,31034
-datahub/ingestion/source/sql/hive_metastore.py,sha256=HW0zoHKarBYb8oVCy5fHvPOn-pTo25LctW_AusmH0hQ,36252
+datahub/ingestion/source/sql/hive_metastore.py,sha256=ZzRPmieCf_9LOq9NeSPZhDn2txQ6t-xrskgE-0e5g0k,36232
 datahub/ingestion/source/sql/mariadb.py,sha256=Hm102kmfs_1rd4lsTYhzVMZq5S3B6cyfvpHSzJjqvMw,737
 datahub/ingestion/source/sql/mysql.py,sha256=nDWK4YbqomcJgnit9b8geUGrp_3eix4bt0_k94o7g-0,3350
 datahub/ingestion/source/sql/oracle.py,sha256=ftnrk3iiEelyv9PBHPYbairuP1WgxZbi1gu6YdqY69E,29965
@@ -995,7 +995,7 @@ datahub/utilities/sample_data.py,sha256=DMwN3yAqJPrF_rZF6CU0_iLSNBFfsAQ-no0KQhgw
 datahub/utilities/search_utils.py,sha256=BdZfixXrB6lcD_ec6pCotGtVKQ4ekTtmaZDFGyeZfBg,9683
 datahub/utilities/sentinels.py,sha256=p299J2YR62K2Mbuy-2lrGH4BnF9dkRqGo59Wub0hYV4,417
 datahub/utilities/serialized_lru_cache.py,sha256=R9enyN5BlatknX4KFq9euYc2yiE1rRMucPbAHg09Goc,3247
-datahub/utilities/server_config_util.py,sha256=m5n_9rhORXJaqCaSiTSLT-omhFP8W6puKtbhwdhJqvA,13039
+datahub/utilities/server_config_util.py,sha256=ZHB4JtQN-z9J-9QQnXw-S-sb8g24PP92uOeSN1LP1tg,12032
 datahub/utilities/sql_formatter.py,sha256=tYXIsKjKmpKh0JXGxeAPrHkUWYd1SwJNLjUZsfQP2h0,1016
 datahub/utilities/sqlalchemy_query_combiner.py,sha256=oxW20uXz8hV1Zb4fLXvTQ7c3LjACBsrF58TR2_-RSps,14982
 datahub/utilities/sqlalchemy_type_converter.py,sha256=H4S4xnnyPozDBHFhBh4rjjoXa5novFzYIUBJy2KSrVc,9805
@@ -1050,8 +1050,8 @@ datahub_provider/operators/datahub_assertion_operator.py,sha256=uvTQ-jk2F0sbqqxp
 datahub_provider/operators/datahub_assertion_sensor.py,sha256=lCBj_3x1cf5GMNpHdfkpHuyHfVxsm6ff5x2Z5iizcAo,140
 datahub_provider/operators/datahub_operation_operator.py,sha256=aevDp2FzX7FxGlXrR0khoHNbxbhKR2qPEX5e8O2Jyzw,174
 datahub_provider/operators/datahub_operation_sensor.py,sha256=8fcdVBCEPgqy1etTXgLoiHoJrRt_nzFZQMdSzHqSG7M,168
-acryl_datahub-1.0.0.3rc11.dist-info/METADATA,sha256=ExpPg4Ny4XSUtMdD_mf4VyfzT-uL0S8aeqWSk0-lJzY,180030
-acryl_datahub-1.0.0.3rc11.dist-info/WHEEL,sha256=wXxTzcEDnjrTwFYjLPcsW_7_XihufBwmpiBeiXNBGEA,91
-acryl_datahub-1.0.0.3rc11.dist-info/entry_points.txt,sha256=o3mDeJXSKhsy7XLkuogihraiabBdLn9HaizYXPrxmk0,9710
-acryl_datahub-1.0.0.3rc11.dist-info/top_level.txt,sha256=iLjSrLK5ox1YVYcglRUkcvfZPvKlobBWx7CTUXx8_GI,25
-acryl_datahub-1.0.0.3rc11.dist-info/RECORD,,
+acryl_datahub-1.0.0.3rc12.dist-info/METADATA,sha256=FXy_GoGEMFkaahaoHBzthO6IV2K3vDf21hu91Ef_nHo,180030
+acryl_datahub-1.0.0.3rc12.dist-info/WHEEL,sha256=wXxTzcEDnjrTwFYjLPcsW_7_XihufBwmpiBeiXNBGEA,91
+acryl_datahub-1.0.0.3rc12.dist-info/entry_points.txt,sha256=o3mDeJXSKhsy7XLkuogihraiabBdLn9HaizYXPrxmk0,9710
+acryl_datahub-1.0.0.3rc12.dist-info/top_level.txt,sha256=iLjSrLK5ox1YVYcglRUkcvfZPvKlobBWx7CTUXx8_GI,25
+acryl_datahub-1.0.0.3rc12.dist-info/RECORD,,

datahub/_version.py

@@ -1,6 +1,6 @@
 # Published at https://pypi.org/project/acryl-datahub/.
 __package_name__ = "acryl-datahub"
-__version__ = "1.0.0.3rc11"
+__version__ = "1.0.0.3rc12"
 
 
 def is_dev_mode() -> bool:

datahub/ingestion/graph/client.py

@@ -361,7 +361,7 @@ class DataHubGraph(DatahubRestEmitter, EntityVersioningAPI):
         )
 
     def get_config(self) -> Dict[str, Any]:
-        return self._get_generic(f"{self.config.server}/config")
+        return self.get_server_config().config
 
     def get_ownership(self, entity_urn: str) -> Optional[OwnershipClass]:
         return self.get_aspect(entity_urn=entity_urn, aspect_type=OwnershipClass)

datahub/ingestion/source/dynamodb/dynamodb.py

@@ -474,6 +474,7 @@ class DynamoDBSource(StatefulIngestionSourceBase):
             dataset_properties.customProperties["schema.downsampled"] = "True"
             dataset_properties.customProperties["schema.totalFields"] = f"{schema_size}"
         # append each schema field, schema will be sorted by count descending and delimited_name ascending and sliced to only include MAX_SCHEMA_SIZE items
+        primary_keys = []
         for schema_field in sorted(
             table_fields,
             key=lambda x: (
@@ -484,22 +485,23 @@ class DynamoDBSource(StatefulIngestionSourceBase):
             field_path = schema_field["delimited_name"]
             native_data_type = self.get_native_type(schema_field["type"], table_name)
             type = self.get_field_type(schema_field["type"], table_name)
-            description = None
             nullable = True
             if field_path in primary_key_dict:
-                description = (
+                # primary key should not be nullable
+                type_key = (
                     "Partition Key"
                     if primary_key_dict.get(field_path) == "HASH"
                     else "Sort Key"
                 )
-                # primary key should not be nullable
+                dataset_properties.customProperties[type_key] = field_path
                 nullable = False
+                primary_keys.append(field_path)
 
             field = SchemaField(
                 fieldPath=field_path,
                 nativeDataType=native_data_type,
                 type=type,
-                description=description,
+                description=None,
                 nullable=nullable,
                 recursive=False,
             )
@@ -513,6 +515,7 @@ class DynamoDBSource(StatefulIngestionSourceBase):
             hash="",
             platformSchema=SchemalessClass(),
             fields=canonical_schema,
+            primaryKeys=primary_keys,
         )
         return schema_metadata
 

datahub/ingestion/source/sql/hive_metastore.py

@@ -67,10 +67,10 @@ TableKey = namedtuple("TableKey", ["schema", "table"])
 
 
 class HiveMetastoreConfigMode(StrEnum):
-    hive: str = "hive"
-    presto: str = "presto"
-    presto_on_hive: str = "presto-on-hive"
-    trino: str = "trino"
+    hive = "hive"
+    presto = "presto"
+    presto_on_hive = "presto-on-hive"
+    trino = "trino"
 
 
 @dataclass

datahub/utilities/server_config_util.py

@@ -44,15 +44,8 @@ class ServiceFeature(Enum):
 
 
 _REQUIRED_VERSION_OPENAPI_TRACING = {
-    "acryl": (
-        0,
-        3,
-        11,
-        0,
-    ),  # Requires v0.3.11.0 or higher for acryl versions
-    "cloud": (0, 3, 11, 0),  # Special case for '-cloud' suffix
-    "any_suffix": (0, 3, 11, 0),  # Generic requirement for any other suffix
-    "none": (1, 0, 1, 0),  # Requirement for versions without suffix
+    "cloud": (0, 3, 11, 0),
+    "core": (1, 0, 1, 0),
 }
 
 
@@ -259,7 +252,7 @@ class RestServiceConfig:
         Check if DataHub Cloud is enabled.
 
         Returns:
-            True if the server environment is not 'oss'
+            True if the server environment is not 'core'
         """
         datahub_config = self.config.get("datahub") or {}
         server_env = datahub_config.get("serverEnv")
@@ -268,17 +261,12 @@ class RestServiceConfig:
         if not server_env:
             return False
 
-        return server_env != "oss"
+        return server_env != "core"
 
     def supports_feature(self, feature: ServiceFeature) -> bool:
         """
-        Determines whether a specific feature is supported based on service version.
-
-        Version categorization follows these rules:
-        1. Has '-acryl' suffix (highest priority)
-        2. Has a specific known suffix (e.g. '-other')
-        3. Has some other suffix (catchall for any suffix)
-        4. No suffix
+        Determines whether a specific feature is supported based on service version
+        and whether this is a cloud deployment or not.
 
         Args:
             feature: Feature enum value to check
@@ -286,32 +274,7 @@ class RestServiceConfig:
         Returns:
             Boolean indicating whether the feature is supported
         """
-        version = self.service_version
-        if not version:
-            return False
-
-        # Determine the suffix category
-        suffix_category = "none"  # Default: no suffix
-
-        if "-" in version:
-            suffix = version.split("-", 1)[1]
-
-            if suffix == "acryl":
-                suffix_category = "acryl"
-            elif suffix == "cloud":  # Example of a specific override
-                suffix_category = "cloud"
-            else:
-                suffix_category = "any_suffix"  # Catchall for any other suffix
-
-        # Define feature requirements based on version scheme
-        # This can be expanded to include more features
-        feature_requirements = {
-            ServiceFeature.OPEN_API_SDK: _REQUIRED_VERSION_OPENAPI_TRACING,
-            ServiceFeature.API_TRACING: _REQUIRED_VERSION_OPENAPI_TRACING,
-            # Additional features can be defined here
-        }
-
-        # Special handling for features that rely on config flags instead of version
+        # Special handling for features that rely on config flags
         config_based_features = {
             ServiceFeature.NO_CODE: lambda: self.is_no_code_enabled,
             ServiceFeature.STATEFUL_INGESTION: lambda: self.config.get(
@@ -327,27 +290,36 @@ class RestServiceConfig:
             ServiceFeature.CLI_TELEMETRY: lambda: (
                 self.config.get("telemetry") or {}
             ).get("enabledCli", None),
-            # Add more config-based feature checks as needed
+            ServiceFeature.DATAHUB_CLOUD: lambda: self.is_datahub_cloud,
         }
 
         # Check if this is a config-based feature
         if feature in config_based_features:
             return config_based_features[feature]()
 
+        # For environment-based features, determine requirements based on cloud vs. non-cloud
+        deployment_type = "cloud" if self.is_datahub_cloud else "core"
+
+        # Define feature requirements
+        feature_requirements = {
+            ServiceFeature.OPEN_API_SDK: _REQUIRED_VERSION_OPENAPI_TRACING,
+            ServiceFeature.API_TRACING: _REQUIRED_VERSION_OPENAPI_TRACING,
+            # Additional features can be defined here
+        }
+
         # Check if the feature exists in our requirements dictionary
         if feature not in feature_requirements:
             # Unknown feature, assume not supported
             return False
 
-        # Get version requirements for this feature and version category
+        # Get version requirements for this feature and deployment type
         feature_reqs = feature_requirements[feature]
-        requirements = feature_reqs.get(suffix_category)
+        requirements = feature_reqs.get(deployment_type)
 
         if not requirements:
-            # Fallback to the no-suffix requirements if specific requirements aren't defined
-            requirements = feature_reqs.get(
-                "none", (99, 99, 99, 99)
-            )  # Very high version if none defined
+            # If no specific requirements defined for this deployment type,
+            # assume feature is not supported
+            return False
 
         # Check if the current version meets the requirements
         req_major, req_minor, req_patch, req_build = requirements