tencent-wedata-feature-engineering-dev 0.1.40__tar.gz → 0.1.42__tar.gz

{tencent-wedata-feature-engineering-dev-0.1.40 → tencent-wedata-feature-engineering-dev-0.1.42}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: tencent-wedata-feature-engineering-dev
-Version: 0.1.40
+Version: 0.1.42
 Summary: Wedata Feature Engineering Library Development
 Home-page: 
 Author: meahqian

{tencent-wedata-feature-engineering-dev-0.1.40 → tencent-wedata-feature-engineering-dev-0.1.42}/tencent_wedata_feature_engineering_dev.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: tencent-wedata-feature-engineering-dev
-Version: 0.1.40
+Version: 0.1.42
 Summary: Wedata Feature Engineering Library Development
 Home-page: 
 Author: meahqian

{tencent-wedata-feature-engineering-dev-0.1.40 → tencent-wedata-feature-engineering-dev-0.1.42}/wedata/feature_store/constants/constants.py

@@ -43,7 +43,7 @@ ML_MODEL = "MLmodel"
 FEATURE_LOOKUP_CLIENT_PIP_PACKAGE = "tencent-wedata-feature-engineering-dev"
 
 # 特征查找版本号
-FEATURE_LOOKUP_CLIENT_MAJOR_VERSION = "0.1.40"
+FEATURE_LOOKUP_CLIENT_MAJOR_VERSION = "0.1.42"
 
 # 特征存储内部数据目录
 FEATURE_STORE_INTERNAL_DATA_DIR = "_wedata_internal/"

{tencent-wedata-feature-engineering-dev-0.1.40 → tencent-wedata-feature-engineering-dev-0.1.42}/wedata/feature_store/feast_client/feast_client.py

@@ -7,7 +7,8 @@ Feast客户端，用于与Feast服务器交互
 import json
 import os
 import re
-from typing import List, Dict, Optional, Any
+from datetime import datetime
+from typing import List, Dict, Optional, Any, Tuple
 try:
     # pyspark 3.5.0 以后
     from pyspark.errors import IllegalArgumentException
@@ -97,6 +98,7 @@ class FeastClient:
             tags=tags,
             description=description
         )
+        # 确保feature在增量服务时获取的数据时间范围时正确的。
         self._apply_feature_view(table_name, entities, feature_view)
 
     def _apply_feature_view(self, table_name, entities, feature_view: FeatureView):
@@ -160,11 +162,10 @@ class FeastClient:
             raise ValueError(f"Table {table_name} not found in Feast")
 
         if self._client.config.online_store.type == "redis":
-            print("redis table_view join_keys:", table_view.join_keys)
             redis_online_store = RedisOnlineStore()
             redis_online_store.delete_table(self._client.config, table_view)
             table_view.online = False
-            table_view.materialization_intervals = []
+            table_view.update_materialization_intervals(get_materialization_default_time())
             self._client.apply(table_view)
         else:
             raise ValueError(f"Unsupported online store type: {self._client.config.online_store.type}")
@@ -361,6 +362,7 @@ def _create_table_to_feature_view(
         source=resources,
     )
     feature_view.online = False
+    feature_view.update_materialization_intervals([(datetime(1990, 1, 1), datetime(1990, 1, 1))])
     return feature_view
 
 
@@ -465,18 +467,22 @@ def translate_table_name_to_feast(table_name: str):
         raise ValueError(f"Invalid table name: {table_name}")
 
 
-if __name__ == '__main__':
-    import datetime
-    FeastClient = FeastClient()
-    FeastClient.client.registry.delete_data_source(name="xxxxx")
-    FeastClient.client.registry.delete_entity("xxxxx", )
-    FeastClient.client.registry.delete_feature_view()
-    FeastClient.client.registry.get_feature_view()
-    FeastClient.client.registry.delete_feature_service()
-    FeastClient.client.get_historical_features()
-    feature_view = FeastClient.client.get_feature_view(name="xxxxx")
-    feature_view.source.get_table_query_string()
-    feast_table_name = "xxx"
-    from wedata.feature_store.utils.common_utils import build_full_table_name
-    feast_table_name = translate_table_name_to_feast(build_full_table_name(table_name, database_name))
-    FeastClient.client.materialize(start_date=datetime.datetime(2021,1,1), end_date=datetime.datetime.now(), feature_views=[feast_table_name])
+def get_materialization_default_time() -> List[Tuple[datetime, datetime]]:
+    return [(datetime(1990, 1, 1), datetime(1990, 1, 1))]
+
+
+# if __name__ == '__main__':
+#     import datetime
+#     FeastClient = FeastClient()
+#     FeastClient.client.registry.delete_data_source(name="xxxxx")
+#     FeastClient.client.registry.delete_entity("xxxxx", )
+#     FeastClient.client.registry.delete_feature_view()
+#     FeastClient.client.registry.get_feature_view()
+#     FeastClient.client.registry.delete_feature_service()
+#     FeastClient.client.get_historical_features()
+#     feature_view = FeastClient.client.get_feature_view(name="xxxxx")
+#     feature_view.source.get_table_query_string()
+#     feast_table_name = "xxx"
+#     from wedata.feature_store.utils.common_utils import build_full_table_name
+#     feast_table_name = translate_table_name_to_feast(build_full_table_name(table_name, database_name))
+#     FeastClient.client.materialize(start_date=datetime.datetime(2021,1,1), end_date=datetime.datetime.now(), feature_views=[feast_table_name])

{tencent-wedata-feature-engineering-dev-0.1.40 → tencent-wedata-feature-engineering-dev-0.1.42}/wedata/feature_store/feature_table_client/feature_table_client.py

@@ -881,27 +881,22 @@ class FeatureTableClient:
             online_feature_config.DB = online_config.db
 
         offline_feature_config = OfflineFeatureConfiguration()
-        offline_feature_config.DatabaseName = database_name if database_name else os.environ.get(
-            "WEDATA_DEFAULT_FEATURE_STORE_DATABASE")
+        offline_feature_config.DatabaseName = env_utils.get_database_name(database_name)
         offline_feature_config.TableName = table_name
 
         offline_feature_config.PrimaryKeys, offline_feature_config.TimestampColumn = self._get_table_primary_keys_and_timestamp_key(
             full_table_name)
 
         offline_feature_config.DatasourceName = data_source_name
-        offline_feature_config.DatasourceType = "DLC" if os.environ.get("DLC_REGION") else "EMR"
-        offline_feature_config.EngineName = os.environ.get("KERNEL_ENGINE")
+        offline_feature_config.DatasourceType = env_utils.get_engine_type()
+        offline_feature_config.EngineName = env_utils.get_engine_name()
 
         api_requests = CreateOnlineFeatureTableRequest()
         api_requests.OfflineFeatureConfiguration = offline_feature_config
         api_requests.OnlineFeatureConfiguration = online_feature_config
         api_requests.TaskSchedulerConfiguration = cycle_obj
-        api_requests.ProjectId = os.environ.get("WEDATA_PROJECT_ID")
-        if offline_feature_config.DatasourceType == "DLC":
-            region = os.environ.get("DLC_REGION")
-        else:
-            region = os.environ.get("EMR_REGION")
-
+        api_requests.ProjectId = env_utils.get_project_id()
+        region = env_utils.get_region()
         if not os.environ.get("RESOURCE_GROUP_ID", ""):
             res_group_item = _get_default_resource_group(
                 api_requests.ProjectId, cloud_secret_id, cloud_secret_key, region)
@@ -962,7 +957,7 @@ def _refresh_table(project_id: str, secret_id: str, secret_key: str, region: str
     request.DatasourceName = data_source_name
     request.DatasourceType = data_source_type
     request.EngineName = engine_name
-    request.Action = action
+    request.ActionName = action
     request.IsTry = is_try
     resp = client.RefreshFeatureTable(request)
     return resp

{tencent-wedata-feature-engineering-dev-0.1.40 → tencent-wedata-feature-engineering-dev-0.1.42}/wedata/feature_store/spark_client/spark_client.py

@@ -30,7 +30,6 @@ class SparkClient:
             str: 当前catalog名称，如果未设置则返回None
         """
         _, ok, _ = check_package_version("pyspark", "3.4.0", ">=")
-        print(f"pyspark >= 3.4.0  {ok}")
         if ok:
             return unsanitize_identifier(self._spark.catalog.currentCatalog())
         else:

{tencent-wedata-feature-engineering-dev-0.1.40 → tencent-wedata-feature-engineering-dev-0.1.42}/wedata/feature_store/utils/common_utils.py

@@ -348,5 +348,4 @@ def check_spark_table_exists(spark_client: SparkSession, full_table_name: str) -
             query = f"SHOW TABLES IN {split[1]} LIKE '{split[2]}'"
         else:
             query = f"SHOW TABLES LIKE '{full_table_name}'"
-        logging.debug("check table sql: =======", query)
         return spark_client.sql(query).count() > 0

{tencent-wedata-feature-engineering-dev-0.1.40 → tencent-wedata-feature-engineering-dev-0.1.42}/wedata/feature_store/utils/env_utils.py

@@ -72,9 +72,19 @@ def get_engine_name() -> str:
     """
     获取引擎名称
     """
-    engine_name = os.environ.get("KERNEL_ENGINE")
+    engine_name = os.environ.get("KERNEL_ENGINE_NAME")
     if engine_name:
         return engine_name
+    raise EnvironmentError("environment variable KERNEL_ENGINE_NAME is not set, please check environment configuration")
+
+
+def get_engine_id() -> str:
+    """
+    获取引擎ID
+    """
+    engine_id = os.environ.get("KERNEL_ENGINE")
+    if engine_id:
+        return engine_id
     raise EnvironmentError("environment variable KERNEL_ENGINE is not set, please check environment configuration")