tencent-wedata-feature-engineering-dev 0.1.40__py3-none-any.whl → 0.1.42__py3-none-any.whl

{tencent_wedata_feature_engineering_dev-0.1.40.dist-info → tencent_wedata_feature_engineering_dev-0.1.42.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: tencent-wedata-feature-engineering-dev
-Version: 0.1.40
+Version: 0.1.42
 Summary: Wedata Feature Engineering Library Development
 Home-page: 
 Author: meahqian

{tencent_wedata_feature_engineering_dev-0.1.40.dist-info → tencent_wedata_feature_engineering_dev-0.1.42.dist-info}/RECORD

@@ -12,7 +12,7 @@ wedata/feature_store/common/protos/feature_store_pb2.py,sha256=oMIUGGeGNP84g_nFq
 wedata/feature_store/common/store_config/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 wedata/feature_store/common/store_config/redis.py,sha256=9R5npM2s1u0o9IakmpbRsFdJC0vNar_uvA62OLWuXBs,1145
 wedata/feature_store/constants/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-wedata/feature_store/constants/constants.py,sha256=kzelGSFVIj2druYso9jFjRSmOaWrj7A4EE83TRbw9xs,1959
+wedata/feature_store/constants/constants.py,sha256=d4xnvPZeVKMFi8cYp6Ya9GdoUl9NQzyH4hIaXOgTWd4,1959
 wedata/feature_store/constants/engine_types.py,sha256=42mI-kNDDtoA4_I3iqDe4FkF2M2l_Bt4Q1V6WUB-_k0,921
 wedata/feature_store/entities/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 wedata/feature_store/entities/column_info.py,sha256=-AR6EKHwgoqIkRHFyguxVEtnYt6fvusWHkEjF4kvS0A,5141
@@ -30,16 +30,16 @@ wedata/feature_store/entities/on_demand_column_info.py,sha256=a44ep-f3FOruWNXl3c
 wedata/feature_store/entities/source_data_column_info.py,sha256=FyBmBPUSvc2S2OPFTvsQf2AdS-KFGkYBmd4yL_Vur8M,702
 wedata/feature_store/entities/training_set.py,sha256=ylt1h6Z_xU8hKYvnvd80CeewTGSN68-_kvFpoliwH7s,5679
 wedata/feature_store/feast_client/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-wedata/feature_store/feast_client/feast_client.py,sha256=nRk2XMmXaZOp-3PMYi-_ScxS4hXbJ_awDylu07T8L_s,20343
+wedata/feature_store/feast_client/feast_client.py,sha256=TT1ESVYbP9kT5LyfYEYfnsM2mvOiJBug3527Y73lg3o,20694
 wedata/feature_store/feature_table_client/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-wedata/feature_store/feature_table_client/feature_table_client.py,sha256=9NNhsYdLdlzeuOFDQW384EuDP-RHQAqbbCbOYUJKua4,41402
+wedata/feature_store/feature_table_client/feature_table_client.py,sha256=wLawB-97ZJ4og5Y8f0VfRYsuygXY6DzMeorTTzdA9mM,41176
 wedata/feature_store/spark_client/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-wedata/feature_store/spark_client/spark_client.py,sha256=B_6f_VVSdhft1SsoAohXCG08XnjokR3FgXql8cROtsI,11840
+wedata/feature_store/spark_client/spark_client.py,sha256=ACWaRaDChqr2qAM8st6e0BP-LSEBSIj2xZAElX2nxHE,11799
 wedata/feature_store/training_set_client/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 wedata/feature_store/training_set_client/training_set_client.py,sha256=teaWM-xDgp2TwnadovUm0i4A26roTozgRefIZaHORko,23376
 wedata/feature_store/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-wedata/feature_store/utils/common_utils.py,sha256=BRIU4Qb8uvhe5E3fwBMY_Q7t7XTt-aNqEfhUsHjQnqQ,12371
-wedata/feature_store/utils/env_utils.py,sha256=72-KFPZ_L4SNthEZ-0ZH6ynNlmj0AboH5JnM1BVQoP8,2288
+wedata/feature_store/utils/common_utils.py,sha256=vkpoXxZTd6S-2MgdyTQZ6P_ckdqNSK50ECuVBG2BwfI,12314
+wedata/feature_store/utils/env_utils.py,sha256=ffDhQVW96vZxWurzOhYfZQk_dQ1LHC-y8tl4PQlp9Tg,2570
 wedata/feature_store/utils/feature_lookup_utils.py,sha256=mNV6RhBdpv1iTZduCA9YwXwkeJCwU5MFQ1MkFeD9IhY,22003
 wedata/feature_store/utils/feature_spec_utils.py,sha256=j8t-zel2_r8Q9m88BmFKkHMdkGNIduWJB-28OZDASRY,11613
 wedata/feature_store/utils/feature_utils.py,sha256=KKq28bVB_lCuhnR9Hk6JegJBOVgcelWlvrRM-F9onkA,2796
@@ -58,7 +58,7 @@ wedata/tempo/ml.py,sha256=WtGa2szn6PditvZsTZoxo7wFDe4k1SRoMZ-jgNGIjvE,4323
 wedata/tempo/resample.py,sha256=h81RVVmCl4ect-YKE-KZZHPDi1rGI3sh-YIb-Btz0ck,9698
 wedata/tempo/tsdf.py,sha256=S4lZfxhSRFiezYoYS6gvGsl1mZA3zp-MWEKFHYZpDg0,70968
 wedata/tempo/utils.py,sha256=I9I6l2DMwUoY213L04Yc1UR_zTWgSkj1BVo4ZwzQd4Y,7977
-tencent_wedata_feature_engineering_dev-0.1.40.dist-info/METADATA,sha256=xzoUceLzG0JG5B-c11dHf9tEB3Spdz9OF0kPlKSMHZk,582
-tencent_wedata_feature_engineering_dev-0.1.40.dist-info/WHEEL,sha256=tZoeGjtWxWRfdplE7E3d45VPlLNQnvbKiYnx7gwAy8A,92
-tencent_wedata_feature_engineering_dev-0.1.40.dist-info/top_level.txt,sha256=Xa0v1rh__RvfVTVDirW5r5UBKg7ZO_iuTeXfp8MNo2A,7
-tencent_wedata_feature_engineering_dev-0.1.40.dist-info/RECORD,,
+tencent_wedata_feature_engineering_dev-0.1.42.dist-info/METADATA,sha256=92FGxlFyakPGdx3bwUjkpmImzCl5eB_KxiYMU2yqc_8,582
+tencent_wedata_feature_engineering_dev-0.1.42.dist-info/WHEEL,sha256=tZoeGjtWxWRfdplE7E3d45VPlLNQnvbKiYnx7gwAy8A,92
+tencent_wedata_feature_engineering_dev-0.1.42.dist-info/top_level.txt,sha256=Xa0v1rh__RvfVTVDirW5r5UBKg7ZO_iuTeXfp8MNo2A,7
+tencent_wedata_feature_engineering_dev-0.1.42.dist-info/RECORD,,

wedata/feature_store/constants/constants.py

@@ -43,7 +43,7 @@ ML_MODEL = "MLmodel"
 FEATURE_LOOKUP_CLIENT_PIP_PACKAGE = "tencent-wedata-feature-engineering-dev"
 
 # 特征查找版本号
-FEATURE_LOOKUP_CLIENT_MAJOR_VERSION = "0.1.40"
+FEATURE_LOOKUP_CLIENT_MAJOR_VERSION = "0.1.42"
 
 # 特征存储内部数据目录
 FEATURE_STORE_INTERNAL_DATA_DIR = "_wedata_internal/"

wedata/feature_store/feast_client/feast_client.py

@@ -7,7 +7,8 @@ Feast客户端，用于与Feast服务器交互
 import json
 import os
 import re
-from typing import List, Dict, Optional, Any
+from datetime import datetime
+from typing import List, Dict, Optional, Any, Tuple
 try:
     # pyspark 3.5.0 以后
     from pyspark.errors import IllegalArgumentException
@@ -97,6 +98,7 @@ class FeastClient:
             tags=tags,
             description=description
         )
+        # 确保feature在增量服务时获取的数据时间范围时正确的。
         self._apply_feature_view(table_name, entities, feature_view)
 
     def _apply_feature_view(self, table_name, entities, feature_view: FeatureView):
@@ -160,11 +162,10 @@ class FeastClient:
             raise ValueError(f"Table {table_name} not found in Feast")
 
         if self._client.config.online_store.type == "redis":
-            print("redis table_view join_keys:", table_view.join_keys)
             redis_online_store = RedisOnlineStore()
             redis_online_store.delete_table(self._client.config, table_view)
             table_view.online = False
-            table_view.materialization_intervals = []
+            table_view.update_materialization_intervals(get_materialization_default_time())
             self._client.apply(table_view)
         else:
             raise ValueError(f"Unsupported online store type: {self._client.config.online_store.type}")
@@ -361,6 +362,7 @@ def _create_table_to_feature_view(
         source=resources,
     )
     feature_view.online = False
+    feature_view.update_materialization_intervals([(datetime(1990, 1, 1), datetime(1990, 1, 1))])
     return feature_view
 
 
@@ -465,18 +467,22 @@ def translate_table_name_to_feast(table_name: str):
         raise ValueError(f"Invalid table name: {table_name}")
 
 
-if __name__ == '__main__':
-    import datetime
-    FeastClient = FeastClient()
-    FeastClient.client.registry.delete_data_source(name="xxxxx")
-    FeastClient.client.registry.delete_entity("xxxxx", )
-    FeastClient.client.registry.delete_feature_view()
-    FeastClient.client.registry.get_feature_view()
-    FeastClient.client.registry.delete_feature_service()
-    FeastClient.client.get_historical_features()
-    feature_view = FeastClient.client.get_feature_view(name="xxxxx")
-    feature_view.source.get_table_query_string()
-    feast_table_name = "xxx"
-    from wedata.feature_store.utils.common_utils import build_full_table_name
-    feast_table_name = translate_table_name_to_feast(build_full_table_name(table_name, database_name))
-    FeastClient.client.materialize(start_date=datetime.datetime(2021,1,1), end_date=datetime.datetime.now(), feature_views=[feast_table_name])
+def get_materialization_default_time() -> List[Tuple[datetime, datetime]]:
+    return [(datetime(1990, 1, 1), datetime(1990, 1, 1))]
+
+
+# if __name__ == '__main__':
+#     import datetime
+#     FeastClient = FeastClient()
+#     FeastClient.client.registry.delete_data_source(name="xxxxx")
+#     FeastClient.client.registry.delete_entity("xxxxx", )
+#     FeastClient.client.registry.delete_feature_view()
+#     FeastClient.client.registry.get_feature_view()
+#     FeastClient.client.registry.delete_feature_service()
+#     FeastClient.client.get_historical_features()
+#     feature_view = FeastClient.client.get_feature_view(name="xxxxx")
+#     feature_view.source.get_table_query_string()
+#     feast_table_name = "xxx"
+#     from wedata.feature_store.utils.common_utils import build_full_table_name
+#     feast_table_name = translate_table_name_to_feast(build_full_table_name(table_name, database_name))
+#     FeastClient.client.materialize(start_date=datetime.datetime(2021,1,1), end_date=datetime.datetime.now(), feature_views=[feast_table_name])

wedata/feature_store/feature_table_client/feature_table_client.py

@@ -881,27 +881,22 @@ class FeatureTableClient:
             online_feature_config.DB = online_config.db
 
         offline_feature_config = OfflineFeatureConfiguration()
-        offline_feature_config.DatabaseName = database_name if database_name else os.environ.get(
-            "WEDATA_DEFAULT_FEATURE_STORE_DATABASE")
+        offline_feature_config.DatabaseName = env_utils.get_database_name(database_name)
         offline_feature_config.TableName = table_name
 
         offline_feature_config.PrimaryKeys, offline_feature_config.TimestampColumn = self._get_table_primary_keys_and_timestamp_key(
             full_table_name)
 
         offline_feature_config.DatasourceName = data_source_name
-        offline_feature_config.DatasourceType = "DLC" if os.environ.get("DLC_REGION") else "EMR"
-        offline_feature_config.EngineName = os.environ.get("KERNEL_ENGINE")
+        offline_feature_config.DatasourceType = env_utils.get_engine_type()
+        offline_feature_config.EngineName = env_utils.get_engine_name()
 
         api_requests = CreateOnlineFeatureTableRequest()
         api_requests.OfflineFeatureConfiguration = offline_feature_config
         api_requests.OnlineFeatureConfiguration = online_feature_config
         api_requests.TaskSchedulerConfiguration = cycle_obj
-        api_requests.ProjectId = os.environ.get("WEDATA_PROJECT_ID")
-        if offline_feature_config.DatasourceType == "DLC":
-            region = os.environ.get("DLC_REGION")
-        else:
-            region = os.environ.get("EMR_REGION")
-
+        api_requests.ProjectId = env_utils.get_project_id()
+        region = env_utils.get_region()
         if not os.environ.get("RESOURCE_GROUP_ID", ""):
             res_group_item = _get_default_resource_group(
                 api_requests.ProjectId, cloud_secret_id, cloud_secret_key, region)
@@ -962,7 +957,7 @@ def _refresh_table(project_id: str, secret_id: str, secret_key: str, region: str
     request.DatasourceName = data_source_name
     request.DatasourceType = data_source_type
     request.EngineName = engine_name
-    request.Action = action
+    request.ActionName = action
     request.IsTry = is_try
     resp = client.RefreshFeatureTable(request)
     return resp

wedata/feature_store/spark_client/spark_client.py

@@ -30,7 +30,6 @@ class SparkClient:
             str: 当前catalog名称，如果未设置则返回None
         """
         _, ok, _ = check_package_version("pyspark", "3.4.0", ">=")
-        print(f"pyspark >= 3.4.0  {ok}")
         if ok:
             return unsanitize_identifier(self._spark.catalog.currentCatalog())
         else:

wedata/feature_store/utils/common_utils.py

@@ -348,5 +348,4 @@ def check_spark_table_exists(spark_client: SparkSession, full_table_name: str) -
             query = f"SHOW TABLES IN {split[1]} LIKE '{split[2]}'"
         else:
             query = f"SHOW TABLES LIKE '{full_table_name}'"
-        logging.debug("check table sql: =======", query)
         return spark_client.sql(query).count() > 0

wedata/feature_store/utils/env_utils.py

@@ -72,9 +72,19 @@ def get_engine_name() -> str:
     """
     获取引擎名称
     """
-    engine_name = os.environ.get("KERNEL_ENGINE")
+    engine_name = os.environ.get("KERNEL_ENGINE_NAME")
     if engine_name:
         return engine_name
+    raise EnvironmentError("environment variable KERNEL_ENGINE_NAME is not set, please check environment configuration")
+
+
+def get_engine_id() -> str:
+    """
+    获取引擎ID
+    """
+    engine_id = os.environ.get("KERNEL_ENGINE")
+    if engine_id:
+        return engine_id
     raise EnvironmentError("environment variable KERNEL_ENGINE is not set, please check environment configuration")