tencent-wedata-feature-engineering-dev 0.1.42__tar.gz → 0.1.43__tar.gz

{tencent-wedata-feature-engineering-dev-0.1.42 → tencent-wedata-feature-engineering-dev-0.1.43}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: tencent-wedata-feature-engineering-dev
-Version: 0.1.42
+Version: 0.1.43
 Summary: Wedata Feature Engineering Library Development
 Home-page: 
 Author: meahqian

{tencent-wedata-feature-engineering-dev-0.1.42 → tencent-wedata-feature-engineering-dev-0.1.43}/tencent_wedata_feature_engineering_dev.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: tencent-wedata-feature-engineering-dev
-Version: 0.1.42
+Version: 0.1.43
 Summary: Wedata Feature Engineering Library Development
 Home-page: 
 Author: meahqian

{tencent-wedata-feature-engineering-dev-0.1.42 → tencent-wedata-feature-engineering-dev-0.1.43}/wedata/feature_store/constants/constants.py

@@ -43,7 +43,7 @@ ML_MODEL = "MLmodel"
 FEATURE_LOOKUP_CLIENT_PIP_PACKAGE = "tencent-wedata-feature-engineering-dev"
 
 # 特征查找版本号
-FEATURE_LOOKUP_CLIENT_MAJOR_VERSION = "0.1.42"
+FEATURE_LOOKUP_CLIENT_MAJOR_VERSION = "0.1.43"
 
 # 特征存储内部数据目录
 FEATURE_STORE_INTERNAL_DATA_DIR = "_wedata_internal/"

{tencent-wedata-feature-engineering-dev-0.1.42 → tencent-wedata-feature-engineering-dev-0.1.43}/wedata/feature_store/feast_client/feast_client.py

@@ -20,6 +20,7 @@ import pytz
 from feast import FeatureStore, RepoConfig, FeatureView
 from pyspark.sql import DataFrame, SparkSession
 from wedata.feature_store.common.store_config.redis import RedisStoreConfig
+from wedata.feature_store.utils import env_utils
 from feast import Entity, FeatureService
 from feast.infra.offline_stores.contrib.spark_offline_store.spark_source import SparkSource
 from feast.infra.online_stores.redis import RedisOnlineStore
@@ -37,8 +38,8 @@ TEMP_FILE_PATH = "/tmp/feast_data/"
 class FeastClient:
 
     def __init__(self, offline_store: SparkSession, online_store_config: RedisStoreConfig = None):
-        project_id = os.getenv("WEDATA_PROJECT_ID", "")
-        remote_path = os.getenv("FEAST_REMOTE_ADDRESS", "")
+        project_id = env_utils.get_project_id()
+        remote_path = env_utils.get_feast_remote_url()
         if offline_store is None or not isinstance(offline_store, SparkSession):
             raise ValueError("offline_store must be provided SparkSession instance")
 

{tencent-wedata-feature-engineering-dev-0.1.42 → tencent-wedata-feature-engineering-dev-0.1.43}/wedata/feature_store/spark_client/spark_client.py

@@ -36,7 +36,6 @@ class SparkClient:
             catalog = self._spark.sql("SELECT current_catalog()").first()[0]
             return unsanitize_identifier(catalog)
 
-
     def get_current_database(self):
         """
         获取Spark上下文中当前设置的数据库名称
@@ -56,7 +55,6 @@ class SparkClient:
     def createDataFrame(self, data, schema) -> DataFrame:
         return self._spark.createDataFrame(data, schema)
 
-
     def read_table(self, table_name):
         """读取Spark表数据
 

{tencent-wedata-feature-engineering-dev-0.1.42 → tencent-wedata-feature-engineering-dev-0.1.43}/wedata/feature_store/utils/env_utils.py

@@ -38,10 +38,10 @@ def get_region() -> str:
     获取当前地域
     """
     region_dlc = os.environ.get("DLC_REGION")
-    region_emr = os.environ.get("EMR_REGION")
+    region_emr = os.environ.get("KERNEL_REGION")
     region = region_dlc if region_dlc else region_emr
     if not region:
-        raise EnvironmentError("environment variable DLC_REGION or EMR_REGION is not set, "
+        raise EnvironmentError("environment variable DLC_REGION or KERNEL_REGION is not set, "
                                "please check environment configuration")
     return region
 
@@ -72,20 +72,14 @@ def get_engine_name() -> str:
     """
     获取引擎名称
     """
-    engine_name = os.environ.get("KERNEL_ENGINE_NAME")
-    if engine_name:
-        return engine_name
-    raise EnvironmentError("environment variable KERNEL_ENGINE_NAME is not set, please check environment configuration")
+    return _get_variable("KERNEL_ENGINE_NAME")
 
 
 def get_engine_id() -> str:
     """
     获取引擎ID
     """
-    engine_id = os.environ.get("KERNEL_ENGINE")
-    if engine_id:
-        return engine_id
-    raise EnvironmentError("environment variable KERNEL_ENGINE is not set, please check environment configuration")
+    return _get_variable("KERNEL_ENGINE")
 
 
 def get_engine_type() -> str:
@@ -94,3 +88,18 @@ def get_engine_type() -> str:
     """
     return "DLC" if os.environ.get("DLC_REGION") else "EMR"
 
+
+def get_feast_remote_url() -> str:
+    """
+    获取Feast远程URL
+    """
+    return _get_variable("KERNEL_FEAST_REMOTE_ADDRESS")
+
+
+def _get_variable(variable_key: str, is_raise: bool = True, default_value: str = None) -> str:
+    val = os.environ.get(variable_key, default_value)
+    if not val:
+        if is_raise:
+            raise EnvironmentError(f"environment variable {variable_key} is not set, "
+                                   f"please check environment configuration")
+    return val