tencent-wedata-feature-engineering-dev 0.1.41__tar.gz → 0.1.43__tar.gz

{tencent-wedata-feature-engineering-dev-0.1.41 → tencent-wedata-feature-engineering-dev-0.1.43}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: tencent-wedata-feature-engineering-dev
-Version: 0.1.41
+Version: 0.1.43
 Summary: Wedata Feature Engineering Library Development
 Home-page: 
 Author: meahqian

{tencent-wedata-feature-engineering-dev-0.1.41 → tencent-wedata-feature-engineering-dev-0.1.43}/tencent_wedata_feature_engineering_dev.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: tencent-wedata-feature-engineering-dev
-Version: 0.1.41
+Version: 0.1.43
 Summary: Wedata Feature Engineering Library Development
 Home-page: 
 Author: meahqian

{tencent-wedata-feature-engineering-dev-0.1.41 → tencent-wedata-feature-engineering-dev-0.1.43}/wedata/feature_store/constants/constants.py

@@ -43,7 +43,7 @@ ML_MODEL = "MLmodel"
 FEATURE_LOOKUP_CLIENT_PIP_PACKAGE = "tencent-wedata-feature-engineering-dev"
 
 # 特征查找版本号
-FEATURE_LOOKUP_CLIENT_MAJOR_VERSION = "0.1.41"
+FEATURE_LOOKUP_CLIENT_MAJOR_VERSION = "0.1.43"
 
 # 特征存储内部数据目录
 FEATURE_STORE_INTERNAL_DATA_DIR = "_wedata_internal/"

{tencent-wedata-feature-engineering-dev-0.1.41 → tencent-wedata-feature-engineering-dev-0.1.43}/wedata/feature_store/feast_client/feast_client.py

@@ -7,7 +7,8 @@ Feast客户端，用于与Feast服务器交互
 import json
 import os
 import re
-from typing import List, Dict, Optional, Any
+from datetime import datetime
+from typing import List, Dict, Optional, Any, Tuple
 try:
     # pyspark 3.5.0 以后
     from pyspark.errors import IllegalArgumentException
@@ -19,6 +20,7 @@ import pytz
 from feast import FeatureStore, RepoConfig, FeatureView
 from pyspark.sql import DataFrame, SparkSession
 from wedata.feature_store.common.store_config.redis import RedisStoreConfig
+from wedata.feature_store.utils import env_utils
 from feast import Entity, FeatureService
 from feast.infra.offline_stores.contrib.spark_offline_store.spark_source import SparkSource
 from feast.infra.online_stores.redis import RedisOnlineStore
@@ -36,8 +38,8 @@ TEMP_FILE_PATH = "/tmp/feast_data/"
 class FeastClient:
 
     def __init__(self, offline_store: SparkSession, online_store_config: RedisStoreConfig = None):
-        project_id = os.getenv("WEDATA_PROJECT_ID", "")
-        remote_path = os.getenv("FEAST_REMOTE_ADDRESS", "")
+        project_id = env_utils.get_project_id()
+        remote_path = env_utils.get_feast_remote_url()
         if offline_store is None or not isinstance(offline_store, SparkSession):
             raise ValueError("offline_store must be provided SparkSession instance")
 
@@ -97,6 +99,7 @@ class FeastClient:
             tags=tags,
             description=description
         )
+        # 确保feature在增量服务时获取的数据时间范围时正确的。
         self._apply_feature_view(table_name, entities, feature_view)
 
     def _apply_feature_view(self, table_name, entities, feature_view: FeatureView):
@@ -160,11 +163,10 @@ class FeastClient:
             raise ValueError(f"Table {table_name} not found in Feast")
 
         if self._client.config.online_store.type == "redis":
-            print("redis table_view join_keys:", table_view.join_keys)
             redis_online_store = RedisOnlineStore()
             redis_online_store.delete_table(self._client.config, table_view)
             table_view.online = False
-            table_view.materialization_intervals = []
+            table_view.update_materialization_intervals(get_materialization_default_time())
             self._client.apply(table_view)
         else:
             raise ValueError(f"Unsupported online store type: {self._client.config.online_store.type}")
@@ -361,6 +363,7 @@ def _create_table_to_feature_view(
         source=resources,
     )
     feature_view.online = False
+    feature_view.update_materialization_intervals([(datetime(1990, 1, 1), datetime(1990, 1, 1))])
     return feature_view
 
 
@@ -465,18 +468,22 @@ def translate_table_name_to_feast(table_name: str):
         raise ValueError(f"Invalid table name: {table_name}")
 
 
-if __name__ == '__main__':
-    import datetime
-    FeastClient = FeastClient()
-    FeastClient.client.registry.delete_data_source(name="xxxxx")
-    FeastClient.client.registry.delete_entity("xxxxx", )
-    FeastClient.client.registry.delete_feature_view()
-    FeastClient.client.registry.get_feature_view()
-    FeastClient.client.registry.delete_feature_service()
-    FeastClient.client.get_historical_features()
-    feature_view = FeastClient.client.get_feature_view(name="xxxxx")
-    feature_view.source.get_table_query_string()
-    feast_table_name = "xxx"
-    from wedata.feature_store.utils.common_utils import build_full_table_name
-    feast_table_name = translate_table_name_to_feast(build_full_table_name(table_name, database_name))
-    FeastClient.client.materialize(start_date=datetime.datetime(2021,1,1), end_date=datetime.datetime.now(), feature_views=[feast_table_name])
+def get_materialization_default_time() -> List[Tuple[datetime, datetime]]:
+    return [(datetime(1990, 1, 1), datetime(1990, 1, 1))]
+
+
+# if __name__ == '__main__':
+#     import datetime
+#     FeastClient = FeastClient()
+#     FeastClient.client.registry.delete_data_source(name="xxxxx")
+#     FeastClient.client.registry.delete_entity("xxxxx", )
+#     FeastClient.client.registry.delete_feature_view()
+#     FeastClient.client.registry.get_feature_view()
+#     FeastClient.client.registry.delete_feature_service()
+#     FeastClient.client.get_historical_features()
+#     feature_view = FeastClient.client.get_feature_view(name="xxxxx")
+#     feature_view.source.get_table_query_string()
+#     feast_table_name = "xxx"
+#     from wedata.feature_store.utils.common_utils import build_full_table_name
+#     feast_table_name = translate_table_name_to_feast(build_full_table_name(table_name, database_name))
+#     FeastClient.client.materialize(start_date=datetime.datetime(2021,1,1), end_date=datetime.datetime.now(), feature_views=[feast_table_name])

{tencent-wedata-feature-engineering-dev-0.1.41 → tencent-wedata-feature-engineering-dev-0.1.43}/wedata/feature_store/feature_table_client/feature_table_client.py

@@ -881,27 +881,22 @@ class FeatureTableClient:
             online_feature_config.DB = online_config.db
 
         offline_feature_config = OfflineFeatureConfiguration()
-        offline_feature_config.DatabaseName = database_name if database_name else os.environ.get(
-            "WEDATA_DEFAULT_FEATURE_STORE_DATABASE")
+        offline_feature_config.DatabaseName = env_utils.get_database_name(database_name)
         offline_feature_config.TableName = table_name
 
         offline_feature_config.PrimaryKeys, offline_feature_config.TimestampColumn = self._get_table_primary_keys_and_timestamp_key(
             full_table_name)
 
         offline_feature_config.DatasourceName = data_source_name
-        offline_feature_config.DatasourceType = "DLC" if os.environ.get("DLC_REGION") else "EMR"
-        offline_feature_config.EngineName = os.environ.get("KERNEL_ENGINE")
+        offline_feature_config.DatasourceType = env_utils.get_engine_type()
+        offline_feature_config.EngineName = env_utils.get_engine_name()
 
         api_requests = CreateOnlineFeatureTableRequest()
         api_requests.OfflineFeatureConfiguration = offline_feature_config
         api_requests.OnlineFeatureConfiguration = online_feature_config
         api_requests.TaskSchedulerConfiguration = cycle_obj
-        api_requests.ProjectId = os.environ.get("WEDATA_PROJECT_ID")
-        if offline_feature_config.DatasourceType == "DLC":
-            region = os.environ.get("DLC_REGION")
-        else:
-            region = os.environ.get("EMR_REGION")
-
+        api_requests.ProjectId = env_utils.get_project_id()
+        region = env_utils.get_region()
         if not os.environ.get("RESOURCE_GROUP_ID", ""):
             res_group_item = _get_default_resource_group(
                 api_requests.ProjectId, cloud_secret_id, cloud_secret_key, region)

{tencent-wedata-feature-engineering-dev-0.1.41 → tencent-wedata-feature-engineering-dev-0.1.43}/wedata/feature_store/spark_client/spark_client.py

@@ -30,14 +30,12 @@ class SparkClient:
             str: 当前catalog名称，如果未设置则返回None
         """
         _, ok, _ = check_package_version("pyspark", "3.4.0", ">=")
-        print(f"pyspark >= 3.4.0  {ok}")
         if ok:
             return unsanitize_identifier(self._spark.catalog.currentCatalog())
         else:
             catalog = self._spark.sql("SELECT current_catalog()").first()[0]
             return unsanitize_identifier(catalog)
 
-
     def get_current_database(self):
         """
         获取Spark上下文中当前设置的数据库名称
@@ -57,7 +55,6 @@ class SparkClient:
     def createDataFrame(self, data, schema) -> DataFrame:
         return self._spark.createDataFrame(data, schema)
 
-
     def read_table(self, table_name):
         """读取Spark表数据
 

{tencent-wedata-feature-engineering-dev-0.1.41 → tencent-wedata-feature-engineering-dev-0.1.43}/wedata/feature_store/utils/common_utils.py

@@ -348,5 +348,4 @@ def check_spark_table_exists(spark_client: SparkSession, full_table_name: str) -
             query = f"SHOW TABLES IN {split[1]} LIKE '{split[2]}'"
         else:
             query = f"SHOW TABLES LIKE '{full_table_name}'"
-        logging.debug("check table sql: =======", query)
         return spark_client.sql(query).count() > 0

{tencent-wedata-feature-engineering-dev-0.1.41 → tencent-wedata-feature-engineering-dev-0.1.43}/wedata/feature_store/utils/env_utils.py

@@ -38,10 +38,10 @@ def get_region() -> str:
     获取当前地域
     """
     region_dlc = os.environ.get("DLC_REGION")
-    region_emr = os.environ.get("EMR_REGION")
+    region_emr = os.environ.get("KERNEL_REGION")
     region = region_dlc if region_dlc else region_emr
     if not region:
-        raise EnvironmentError("environment variable DLC_REGION or EMR_REGION is not set, "
+        raise EnvironmentError("environment variable DLC_REGION or KERNEL_REGION is not set, "
                                "please check environment configuration")
     return region
 
@@ -72,10 +72,14 @@ def get_engine_name() -> str:
     """
     获取引擎名称
     """
-    engine_name = os.environ.get("KERNEL_ENGINE")
-    if engine_name:
-        return engine_name
-    raise EnvironmentError("environment variable KERNEL_ENGINE is not set, please check environment configuration")
+    return _get_variable("KERNEL_ENGINE_NAME")
+
+
+def get_engine_id() -> str:
+    """
+    获取引擎ID
+    """
+    return _get_variable("KERNEL_ENGINE")
 
 
 def get_engine_type() -> str:
@@ -84,3 +88,18 @@ def get_engine_type() -> str:
     """
     return "DLC" if os.environ.get("DLC_REGION") else "EMR"
 
+
+def get_feast_remote_url() -> str:
+    """
+    获取Feast远程URL
+    """
+    return _get_variable("KERNEL_FEAST_REMOTE_ADDRESS")
+
+
+def _get_variable(variable_key: str, is_raise: bool = True, default_value: str = None) -> str:
+    val = os.environ.get(variable_key, default_value)
+    if not val:
+        if is_raise:
+            raise EnvironmentError(f"environment variable {variable_key} is not set, "
+                                   f"please check environment configuration")
+    return val