tencent-wedata-feature-engineering-dev 0.1.42__py3-none-any.whl → 0.1.43__py3-none-any.whl

{tencent_wedata_feature_engineering_dev-0.1.42.dist-info → tencent_wedata_feature_engineering_dev-0.1.43.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: tencent-wedata-feature-engineering-dev
-Version: 0.1.42
+Version: 0.1.43
 Summary: Wedata Feature Engineering Library Development
 Home-page: 
 Author: meahqian

{tencent_wedata_feature_engineering_dev-0.1.42.dist-info → tencent_wedata_feature_engineering_dev-0.1.43.dist-info}/RECORD

@@ -12,7 +12,7 @@ wedata/feature_store/common/protos/feature_store_pb2.py,sha256=oMIUGGeGNP84g_nFq
 wedata/feature_store/common/store_config/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 wedata/feature_store/common/store_config/redis.py,sha256=9R5npM2s1u0o9IakmpbRsFdJC0vNar_uvA62OLWuXBs,1145
 wedata/feature_store/constants/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-wedata/feature_store/constants/constants.py,sha256=d4xnvPZeVKMFi8cYp6Ya9GdoUl9NQzyH4hIaXOgTWd4,1959
+wedata/feature_store/constants/constants.py,sha256=QX8I8g_DApln3vbdLMpP7IhlyYo0ZwnGve1RNX5CY6w,1959
 wedata/feature_store/constants/engine_types.py,sha256=42mI-kNDDtoA4_I3iqDe4FkF2M2l_Bt4Q1V6WUB-_k0,921
 wedata/feature_store/entities/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 wedata/feature_store/entities/column_info.py,sha256=-AR6EKHwgoqIkRHFyguxVEtnYt6fvusWHkEjF4kvS0A,5141
@@ -30,16 +30,16 @@ wedata/feature_store/entities/on_demand_column_info.py,sha256=a44ep-f3FOruWNXl3c
 wedata/feature_store/entities/source_data_column_info.py,sha256=FyBmBPUSvc2S2OPFTvsQf2AdS-KFGkYBmd4yL_Vur8M,702
 wedata/feature_store/entities/training_set.py,sha256=ylt1h6Z_xU8hKYvnvd80CeewTGSN68-_kvFpoliwH7s,5679
 wedata/feature_store/feast_client/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-wedata/feature_store/feast_client/feast_client.py,sha256=TT1ESVYbP9kT5LyfYEYfnsM2mvOiJBug3527Y73lg3o,20694
+wedata/feature_store/feast_client/feast_client.py,sha256=ZjhOHlY99pkgPJXIxYzezK4c2hEYxz9FURYtVcyEIj8,20730
 wedata/feature_store/feature_table_client/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 wedata/feature_store/feature_table_client/feature_table_client.py,sha256=wLawB-97ZJ4og5Y8f0VfRYsuygXY6DzMeorTTzdA9mM,41176
 wedata/feature_store/spark_client/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-wedata/feature_store/spark_client/spark_client.py,sha256=ACWaRaDChqr2qAM8st6e0BP-LSEBSIj2xZAElX2nxHE,11799
+wedata/feature_store/spark_client/spark_client.py,sha256=aTenEqfZoJYMrph98qjNHZ-H4dgNKnMaH14st8bCVRQ,11797
 wedata/feature_store/training_set_client/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 wedata/feature_store/training_set_client/training_set_client.py,sha256=teaWM-xDgp2TwnadovUm0i4A26roTozgRefIZaHORko,23376
 wedata/feature_store/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 wedata/feature_store/utils/common_utils.py,sha256=vkpoXxZTd6S-2MgdyTQZ6P_ckdqNSK50ECuVBG2BwfI,12314
-wedata/feature_store/utils/env_utils.py,sha256=ffDhQVW96vZxWurzOhYfZQk_dQ1LHC-y8tl4PQlp9Tg,2570
+wedata/feature_store/utils/env_utils.py,sha256=kMCl6gyqDxjN5IZ7-wZMR0R4YY-Evh_7NHxugSYPWQc,2736
 wedata/feature_store/utils/feature_lookup_utils.py,sha256=mNV6RhBdpv1iTZduCA9YwXwkeJCwU5MFQ1MkFeD9IhY,22003
 wedata/feature_store/utils/feature_spec_utils.py,sha256=j8t-zel2_r8Q9m88BmFKkHMdkGNIduWJB-28OZDASRY,11613
 wedata/feature_store/utils/feature_utils.py,sha256=KKq28bVB_lCuhnR9Hk6JegJBOVgcelWlvrRM-F9onkA,2796
@@ -58,7 +58,7 @@ wedata/tempo/ml.py,sha256=WtGa2szn6PditvZsTZoxo7wFDe4k1SRoMZ-jgNGIjvE,4323
 wedata/tempo/resample.py,sha256=h81RVVmCl4ect-YKE-KZZHPDi1rGI3sh-YIb-Btz0ck,9698
 wedata/tempo/tsdf.py,sha256=S4lZfxhSRFiezYoYS6gvGsl1mZA3zp-MWEKFHYZpDg0,70968
 wedata/tempo/utils.py,sha256=I9I6l2DMwUoY213L04Yc1UR_zTWgSkj1BVo4ZwzQd4Y,7977
-tencent_wedata_feature_engineering_dev-0.1.42.dist-info/METADATA,sha256=92FGxlFyakPGdx3bwUjkpmImzCl5eB_KxiYMU2yqc_8,582
-tencent_wedata_feature_engineering_dev-0.1.42.dist-info/WHEEL,sha256=tZoeGjtWxWRfdplE7E3d45VPlLNQnvbKiYnx7gwAy8A,92
-tencent_wedata_feature_engineering_dev-0.1.42.dist-info/top_level.txt,sha256=Xa0v1rh__RvfVTVDirW5r5UBKg7ZO_iuTeXfp8MNo2A,7
-tencent_wedata_feature_engineering_dev-0.1.42.dist-info/RECORD,,
+tencent_wedata_feature_engineering_dev-0.1.43.dist-info/METADATA,sha256=kO9LqVAfns7FFPTPlhPt3cdnd6IIROvXEiMglhFKVp4,582
+tencent_wedata_feature_engineering_dev-0.1.43.dist-info/WHEEL,sha256=tZoeGjtWxWRfdplE7E3d45VPlLNQnvbKiYnx7gwAy8A,92
+tencent_wedata_feature_engineering_dev-0.1.43.dist-info/top_level.txt,sha256=Xa0v1rh__RvfVTVDirW5r5UBKg7ZO_iuTeXfp8MNo2A,7
+tencent_wedata_feature_engineering_dev-0.1.43.dist-info/RECORD,,

wedata/feature_store/constants/constants.py

@@ -43,7 +43,7 @@ ML_MODEL = "MLmodel"
 FEATURE_LOOKUP_CLIENT_PIP_PACKAGE = "tencent-wedata-feature-engineering-dev"
 
 # 特征查找版本号
-FEATURE_LOOKUP_CLIENT_MAJOR_VERSION = "0.1.42"
+FEATURE_LOOKUP_CLIENT_MAJOR_VERSION = "0.1.43"
 
 # 特征存储内部数据目录
 FEATURE_STORE_INTERNAL_DATA_DIR = "_wedata_internal/"

wedata/feature_store/feast_client/feast_client.py

@@ -20,6 +20,7 @@ import pytz
 from feast import FeatureStore, RepoConfig, FeatureView
 from pyspark.sql import DataFrame, SparkSession
 from wedata.feature_store.common.store_config.redis import RedisStoreConfig
+from wedata.feature_store.utils import env_utils
 from feast import Entity, FeatureService
 from feast.infra.offline_stores.contrib.spark_offline_store.spark_source import SparkSource
 from feast.infra.online_stores.redis import RedisOnlineStore
@@ -37,8 +38,8 @@ TEMP_FILE_PATH = "/tmp/feast_data/"
 class FeastClient:
 
     def __init__(self, offline_store: SparkSession, online_store_config: RedisStoreConfig = None):
-        project_id = os.getenv("WEDATA_PROJECT_ID", "")
-        remote_path = os.getenv("FEAST_REMOTE_ADDRESS", "")
+        project_id = env_utils.get_project_id()
+        remote_path = env_utils.get_feast_remote_url()
         if offline_store is None or not isinstance(offline_store, SparkSession):
             raise ValueError("offline_store must be provided SparkSession instance")
 

wedata/feature_store/spark_client/spark_client.py

@@ -36,7 +36,6 @@ class SparkClient:
             catalog = self._spark.sql("SELECT current_catalog()").first()[0]
             return unsanitize_identifier(catalog)
 
-
     def get_current_database(self):
         """
         获取Spark上下文中当前设置的数据库名称
@@ -56,7 +55,6 @@ class SparkClient:
     def createDataFrame(self, data, schema) -> DataFrame:
         return self._spark.createDataFrame(data, schema)
 
-
     def read_table(self, table_name):
         """读取Spark表数据
 

wedata/feature_store/utils/env_utils.py

@@ -38,10 +38,10 @@ def get_region() -> str:
     获取当前地域
     """
     region_dlc = os.environ.get("DLC_REGION")
-    region_emr = os.environ.get("EMR_REGION")
+    region_emr = os.environ.get("KERNEL_REGION")
     region = region_dlc if region_dlc else region_emr
     if not region:
-        raise EnvironmentError("environment variable DLC_REGION or EMR_REGION is not set, "
+        raise EnvironmentError("environment variable DLC_REGION or KERNEL_REGION is not set, "
                                "please check environment configuration")
     return region
 
@@ -72,20 +72,14 @@ def get_engine_name() -> str:
     """
     获取引擎名称
     """
-    engine_name = os.environ.get("KERNEL_ENGINE_NAME")
-    if engine_name:
-        return engine_name
-    raise EnvironmentError("environment variable KERNEL_ENGINE_NAME is not set, please check environment configuration")
+    return _get_variable("KERNEL_ENGINE_NAME")
 
 
 def get_engine_id() -> str:
     """
     获取引擎ID
     """
-    engine_id = os.environ.get("KERNEL_ENGINE")
-    if engine_id:
-        return engine_id
-    raise EnvironmentError("environment variable KERNEL_ENGINE is not set, please check environment configuration")
+    return _get_variable("KERNEL_ENGINE")
 
 
 def get_engine_type() -> str:
@@ -94,3 +88,18 @@ def get_engine_type() -> str:
     """
     return "DLC" if os.environ.get("DLC_REGION") else "EMR"
 
+
+def get_feast_remote_url() -> str:
+    """
+    获取Feast远程URL
+    """
+    return _get_variable("KERNEL_FEAST_REMOTE_ADDRESS")
+
+
+def _get_variable(variable_key: str, is_raise: bool = True, default_value: str = None) -> str:
+    val = os.environ.get(variable_key, default_value)
+    if not val:
+        if is_raise:
+            raise EnvironmentError(f"environment variable {variable_key} is not set, "
+                                   f"please check environment configuration")
+    return val