tencent-wedata-feature-engineering-dev 0.1.49__py3-none-any.whl → 0.2.0__py3-none-any.whl

{tencent_wedata_feature_engineering_dev-0.1.49.dist-info → tencent_wedata_feature_engineering_dev-0.2.0.dist-info}/METADATA

@@ -1,7 +1,7 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.4
 Name: tencent-wedata-feature-engineering-dev
-Version: 0.1.49
-Summary: Wedata Feature Engineering Library Development
+Version: 0.2.0
+Summary: Wedata Feature Engineering Library
 Home-page: 
 Author: meahqian
 Author-email: 
@@ -12,8 +12,10 @@ Classifier: Operating System :: OS Independent
 Requires-Python: >=3.7
 Description-Content-Type: text/markdown
 Requires-Dist: pandas>=1.0.0
-Requires-Dist: feast[redis]==0.49.0
-Requires-Dist: grpcio==1.74.0
-Requires-Dist: tencentcloud-sdk-python
-Requires-Dist: ipython
-
+Dynamic: author
+Dynamic: classifier
+Dynamic: description-content-type
+Dynamic: license
+Dynamic: requires-dist
+Dynamic: requires-python
+Dynamic: summary

tencent_wedata_feature_engineering_dev-0.2.0.dist-info/RECORD

@@ -0,0 +1,46 @@
+wedata/__init__.py,sha256=GYxqkkgH0oH4QtNiOCZHuGkc0sSH1LgEqmhSX6sB4So,200
+wedata/feature_store/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+wedata/feature_store/client.py,sha256=NkjwOdmynA9UkAQfIl9VHj6STPGPAF4xvF9zA9czUGU,14767
+wedata/feature_store/mlflow_model.py,sha256=OCUuccOoO0NXWSzIPoGeL03Ha1Q3aQTJW2RlJrTCmzc,554
+wedata/feature_store/common/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+wedata/feature_store/common/protos/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+wedata/feature_store/common/protos/feature_store_pb2.py,sha256=oMIUGGeGNP84g_nFqOQwTXjV1GiU2ehSOy7CyFu2__g,4207
+wedata/feature_store/constants/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+wedata/feature_store/constants/constants.py,sha256=JzuahjljVFF-ySioQTBfBOMTCuKqN6fkSDC3Fvuxvy0,1774
+wedata/feature_store/entities/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+wedata/feature_store/entities/column_info.py,sha256=-AR6EKHwgoqIkRHFyguxVEtnYt6fvusWHkEjF4kvS0A,5141
+wedata/feature_store/entities/environment_variables.py,sha256=ZEFml5H9MQuzBKM074mUrFYu-Sga4Knmxqiwpke2WGc,1679
+wedata/feature_store/entities/feature.py,sha256=wX8fTBlJq3GYdj9rrBDCY3kFgcVBBAiOOZdxEhnQkNQ,1241
+wedata/feature_store/entities/feature_column_info.py,sha256=ZAS_I-MDg2ofCv3nwYvGCQrrpEljzrh_L1D-gqOV_mM,2407
+wedata/feature_store/entities/feature_function.py,sha256=R17INrCE-U_Uj9KLbFz69aYlOkTETTwQHMMo470F4lQ,1865
+wedata/feature_store/entities/feature_lookup.py,sha256=YjYz8kLq42doFbgPzpmm1r3GPhPYkLsIss4H71x-KAo,8009
+wedata/feature_store/entities/feature_spec.py,sha256=3tLLgZ95QL5qI01DQW5BxKVi4Bm5Qm31b2cFvO2DZh0,21767
+wedata/feature_store/entities/feature_spec_constants.py,sha256=YWDBfRiNDe6fUJFUBo3V4WYg2xsljoPAE-ZejfFZCgM,785
+wedata/feature_store/entities/feature_table.py,sha256=dHZHSDPD4HJ2XanLVIrVTkaCYUeqZ6eWEpA0d3YO71g,4010
+wedata/feature_store/entities/feature_table_info.py,sha256=yJ1P3AYaPiHW6ehCbMWhndzguHJqJKWfeFwYjwTLt2U,1481
+wedata/feature_store/entities/function_info.py,sha256=XRoobE5d9dz9NDjhLmUrEIHUNBiG44WXmFVWWzFJwtg,2942
+wedata/feature_store/entities/on_demand_column_info.py,sha256=a44ep-f3FOruWNXl3c8v7733rNuoKXJaHTv1aqF905s,1739
+wedata/feature_store/entities/source_data_column_info.py,sha256=FyBmBPUSvc2S2OPFTvsQf2AdS-KFGkYBmd4yL_Vur8M,702
+wedata/feature_store/entities/training_set.py,sha256=ylt1h6Z_xU8hKYvnvd80CeewTGSN68-_kvFpoliwH7s,5679
+wedata/feature_store/feature_table_client/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+wedata/feature_store/feature_table_client/feature_table_client.py,sha256=foBPizFye0ITj1kZfVDlbMwcNJ7SJwMf1Wl9iLxG3Qc,17554
+wedata/feature_store/spark_client/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+wedata/feature_store/spark_client/spark_client.py,sha256=wLF7NhQ_hZ1c_7opGO81z52XV5TT8PElRQeONJhuhss,10430
+wedata/feature_store/training_set_client/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+wedata/feature_store/training_set_client/training_set_client.py,sha256=SQ7m3JUo1HF74SV4OohOE2DRYMDMA2pJCeZ3L0ONb_w,23744
+wedata/feature_store/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+wedata/feature_store/utils/common_utils.py,sha256=a7msUfGpJz19EDNZ_L_W_-kCIHy9GrOotF77LW6Vg1Y,10491
+wedata/feature_store/utils/feature_lookup_utils.py,sha256=da6ULwf5D-FRVpZoNyag1rroBfq_XPSH4a3uEMB_8io,22372
+wedata/feature_store/utils/feature_spec_utils.py,sha256=GFwc-WT6nN1tnal5F2c0bgVDRhH-YW58S0GaHBPZEsQ,11624
+wedata/feature_store/utils/feature_utils.py,sha256=KKq28bVB_lCuhnR9Hk6JegJBOVgcelWlvrRM-F9onkA,2796
+wedata/feature_store/utils/on_demand_utils.py,sha256=pazZRG5c0Se08MV_inBddIeX4Q9xlVN_H9SC_WK3xzs,4260
+wedata/feature_store/utils/schema_utils.py,sha256=y6EYY1pUxjVg6MP4C7avdW8ZEBBaDo1YTV2CmPF4i8o,4491
+wedata/feature_store/utils/signature_utils.py,sha256=SZFufd19m0jmGnOLmAl3JPKZC-qHq-wQezh6G7HOMfc,7773
+wedata/feature_store/utils/topological_sort.py,sha256=ebzKxmxeCLk9seB1zR0ASCGXsZsa-DjxJeTc4KUadtg,6475
+wedata/feature_store/utils/training_set_utils.py,sha256=MYsPZS1d9HKswHgjgxD8K7H9N3dWPyyTTx20Mkp4PVU,22497
+wedata/feature_store/utils/uc_utils.py,sha256=KwkiymVx_8AxODUL1RQCIMGU28WM98_i6BmIuQqrZ6o,11431
+wedata/feature_store/utils/validation_utils.py,sha256=lJe6HCg5v5CZxH_pvT-vpGhCpo66LT2erXraHE2T0iI,2584
+tencent_wedata_feature_engineering_dev-0.2.0.dist-info/METADATA,sha256=X-U-df9gDfIyIr16cz8C-xIyfsC1JqdFiK5e6FAZtdI,592
+tencent_wedata_feature_engineering_dev-0.2.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+tencent_wedata_feature_engineering_dev-0.2.0.dist-info/top_level.txt,sha256=Xa0v1rh__RvfVTVDirW5r5UBKg7ZO_iuTeXfp8MNo2A,7
+tencent_wedata_feature_engineering_dev-0.2.0.dist-info/RECORD,,

{tencent_wedata_feature_engineering_dev-0.1.49.dist-info → tencent_wedata_feature_engineering_dev-0.2.0.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: bdist_wheel (0.45.1)
+Generator: setuptools (80.9.0)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

wedata/feature_store/client.py

@@ -4,6 +4,7 @@ Wedata FeatureStoreClient Python实现
 
 from __future__ import annotations
 
+import os
 from types import ModuleType
 from typing import Union, List, Dict, Optional, Any
 from pyspark.sql import DataFrame, SparkSession
@@ -13,9 +14,6 @@ import mlflow
 from wedata.feature_store.constants.constants import FEATURE_STORE_CLIENT
 
 from wedata.feature_store.constants.constants import APPEND, DEFAULT_WRITE_STREAM_TRIGGER
-from wedata.feature_store.constants.engine_types import EngineTypes
-from wedata.feature_store.common.store_config.redis import RedisStoreConfig
-from wedata.feature_store.cloud_sdk_client.models import TaskSchedulerConfiguration
 from wedata.feature_store.entities.feature_function import FeatureFunction
 from wedata.feature_store.entities.feature_lookup import FeatureLookup
 from wedata.feature_store.entities.feature_table import FeatureTable
@@ -30,7 +28,7 @@ from wedata.feature_store.utils.feature_utils import format_feature_lookups_and_
 class FeatureStoreClient:
     """特征存储统一客户端，提供特征全生命周期管理能力"""
 
-    def __init__(self, spark: Optional[SparkSession] = None, cloud_secret_id: str = None, cloud_secret_key: str = None):
+    def __init__(self, spark: Optional[SparkSession] = None):
         """
         :param spark: 已初始化的SparkSession对象
         """
@@ -38,20 +36,19 @@ class FeatureStoreClient:
             spark = SparkSession.builder.getOrCreate()
         self._spark = spark
         self._spark_client = SparkClient(spark)
-        self._feature_table_client = FeatureTableClient(spark, cloud_secret_id=cloud_secret_id,
-                                                        cloud_secret_key=cloud_secret_key)
+        self._feature_table_client = FeatureTableClient(spark)
         self._training_set_client = TrainingSetClient(self._spark_client)
+        self._init_catalog()
 
     def create_table(
             self,
             name: str,
             primary_keys: Union[str, List[str]],
-            timestamp_key: [str],
-            engine_type: [EngineTypes],
-            data_source_name: [str],
             database_name: Optional[str] = None,
+            location: Optional[str] = None,
             df: Optional[DataFrame] = None,
             *,
+            timestamp_keys: Union[str, List[str], None] = None,
             partition_columns: Union[str, List[str], None] = None,
             schema: Optional[StructType] = None,
             description: Optional[str] = None,
@@ -63,11 +60,10 @@ class FeatureStoreClient:
         Args:
             name: 特征表全称（格式：<table>）
             primary_keys: 主键列名（支持复合主键）
-            timestamp_key: 时间戳键（用于时态特征）
-            engine_type: 引擎类型  wedata.feature_store.constants.engine_types.EngineTypes
-            data_source_name: 数据源名称
             database_name: 数据库名
+            location: cos存储位置
             df: 初始数据（可选，用于推断schema）
+            timestamp_keys: 时间戳键（用于时态特征）
             partition_columns: 分区列（优化存储查询）
             schema: 表结构定义（可选，当不提供df时必需）
             description: 业务描述
@@ -83,56 +79,43 @@ class FeatureStoreClient:
         return self._feature_table_client.create_table(
             name=name,
             primary_keys=primary_keys,
-            engine_type=engine_type,
             database_name=database_name,
-            data_source_name=data_source_name,
+            location=location,
             df=df,
-            timestamp_key=timestamp_key,
+            timestamp_keys=timestamp_keys,
             partition_columns=partition_columns,
             schema=schema,
             description=description,
             tags=tags
         )
 
-    def register_table(self, name: str, timestamp_key: str, engine_type: EngineTypes, data_source_name: [str],
-                       database_name: Optional[str] = None,
-                       primary_keys: Union[str, List[str]] = None) -> DataFrame:
+    def register_table(self, name: str, database_name: Optional[str] = None) -> DataFrame:
         """
-        将普通的表注册为特征表，并返回特征表数据
+        读取特征表数据
 
         Args:
             name: 特征表名称
             database_name: 特征库名称
-            timestamp_key: 时间戳键 (用于后续离在线特征同步)
-            engine_type: 引擎类型  wedata.feature_store.constants.engine_types.EngineTypes
-            data_source_name: 数据源名称
-            primary_keys: 主键列名（支持复合主键）（仅当engine_type为EngineTypes.HIVE_ENGINE时有效）
+
         Returns:
             DataFrame: 包含特征表数据的DataFrame对象
         """
 
-        return self._feature_table_client.register_table(name, database_name, timestamp_key=timestamp_key,
-                                                         engine_type=engine_type, primary_keys=primary_keys, data_source_name=data_source_name)
+        return self._feature_table_client.register_table(name, database_name)
 
-    def read_table(self, name: str, database_name: Optional[str] = None, is_online: bool = False,
-                   online_config: Optional[RedisStoreConfig] = None,
-                   entity_row: Optional[List[Dict[str, Any]]] = None) -> DataFrame:
+    def read_table(self, name: str, database_name: Optional[str] = None) -> DataFrame:
         """
         读取特征表数据
 
         Args:
             name: 特征表名称
             database_name: 特征库名称
-            is_online: 是否读取在线特征表（默认不读取）
-            online_config: 在线特征表配置（仅当is_online为True时有效）
-            entity_row: 实体行数据（仅当is_online为True时有效）
-                       [{primary_key1: [value1, value2]}, {primary_key2: [value1, value2]}]
+
         Returns:
             DataFrame: 包含特征表数据的DataFrame对象
         """
 
-        return self._feature_table_client.read_table(name=name, database_name=database_name, is_online=is_online,
-                                                     online_config=online_config, entity_row=entity_row)
+        return self._feature_table_client.read_table(name, database_name)
 
     def get_table(self, name: str, database_name: Optional[str] = None) -> FeatureTable:
         """
@@ -154,6 +137,7 @@ class FeatureStoreClient:
         Args:
             name: 要删除的特征表名称
             database_name: database name
+
         Returns:
             None
         """
@@ -288,7 +272,7 @@ class FeatureStoreClient:
         )
 
     def score_batch(
-            self, model_uri: str, df: DataFrame, result_type: str = "double", timestamp_key: str = None
+        self, model_uri: str, df: DataFrame, result_type: str = "double"
     ) -> DataFrame:
         """
         Evaluate the model on the provided :class:`DataFrame <pyspark.sql.DataFrame>`.
@@ -399,65 +383,17 @@ class FeatureStoreClient:
             df=df,
             result_type=result_type,
             client_name=FEATURE_STORE_CLIENT,
-            timestamp_key=timestamp_key,
         )
 
-    def publish_table(self, table_name: str, data_source_name: str, cloud_secret_id: str, cloud_secret_key: str,
-                      database_name: Optional[str] = None,
-                      is_cycle: bool = False, cycle_obj: TaskSchedulerConfiguration = None,
-                      is_use_default_online: bool = True, online_config: RedisStoreConfig = None):
-        """
-        Publish an offline feature table to an online feature table.
-        
-        This method synchronizes the offline feature table data to online storage
-        for low-latency feature serving in real-time applications.
-        
-        Args:
-            table_name: Name of the offline feature table
-            data_source_name: Name of the data source
-            cloud_secret_id: Cloud secret ID for authentication
-            cloud_secret_key: Cloud secret key for authentication
-            database_name: Database name (optional)
-            is_cycle: Whether to enable periodic publishing (default: False)
-            cycle_obj: Periodic task configuration object (required if is_cycle is True)
-            is_use_default_online: Whether to use default online storage configuration (default: True)
-            online_config: Custom online storage configuration (only effective when is_use_default_online is False)
-        
-        Returns:
-            None
-
-        """
-        return self._feature_table_client.publish_table(table_name=table_name, database_name=database_name,
-                                                        data_source_name=data_source_name,
-                                                        cloud_secret_key=cloud_secret_key,
-                                                        cloud_secret_id=cloud_secret_id,
-                                                        is_cycle=is_cycle, cycle_obj=cycle_obj,
-                                                        is_use_default_online=is_use_default_online,
-                                                        online_config=online_config)
-
-    def drop_online_table(self, table_name: str, online_config: RedisStoreConfig, database_name: Optional[str] = None):
-        """
-        Drop an online feature table.
-        :param table_name: Name of the offline feature table
-        :param database_name: Database name (optional)
-        :param online_config: Custom online storage configuration (only effective when is_use_default_online is False)
-        :return:
-        """
-        self._feature_table_client.drop_online_table(table_name=table_name, database_name=database_name, online_config=online_config)
-
-    def create_feature_spec(
-            self, name: str,
-            features: List[Union[FeatureLookup, FeatureFunction]],
-            exclude_columns: List[str]):
-
-        """
-        创建特征配置文件
-        :arg name: 特征配置文件名称
-        :arg features: 特征列表，可以是FeatureLookup(特征查找)或FeatureFunction(特征函数)
-        :arg exclude_columns: 需要从最终特征集中排除的列名列表
-        """
-        return self._training_set_client.create_feature_spec(name, features, self._spark_client, exclude_columns)
-
     @property
     def spark(self):
         return self._spark
+
+    def _init_catalog(self):
+        """关联catalog"""
+        qcloud_region = os.getenv("QCLOUD_REGION") or os.getenv("REGION") or os.getenv("KERNEL_COS_REGION") \
+                        or os.getenv("NOTEBOOK_COS_REGION")
+        if qcloud_region:
+            mlflow.set_registry_uri(f"tclake:{qcloud_region}")
+
+

wedata/feature_store/constants/constants.py

@@ -43,7 +43,7 @@ ML_MODEL = "MLmodel"
 FEATURE_LOOKUP_CLIENT_PIP_PACKAGE = "tencent-wedata-feature-engineering-dev"
 
 # 特征查找版本号
-FEATURE_LOOKUP_CLIENT_MAJOR_VERSION = "0.1.49"
+FEATURE_LOOKUP_CLIENT_MAJOR_VERSION = "0.2.0"
 
 # 特征存储内部数据目录
 FEATURE_STORE_INTERNAL_DATA_DIR = "_wedata_internal/"
@@ -53,7 +53,4 @@ WEDATA_DEFAULT_FEATURE_STORE_DATABASE = "WEDATA_DEFAULT_FEATURE_STORE_DATABASE"
 FEATURE_TABLE_KEY = "wedata.feature_table"
 FEATURE_TABLE_VALUE = "true"
 
-FEATURE_TABLE_PROJECT = "wedata.feature_project_id"
-FEATURE_TABLE_TIMESTAMP = "timestampKeys"
-FEATURE_TABLE_BACKUP_PRIMARY_KEY = "primaryKeys"    # 备用标识，主键
-FEATURE_DLC_TABLE_PRIMARY_KEY = "dlc.ao.data.govern.sorted.keys"
+FEATURE_TABLE_PROJECT = "wedata.feature_project_id"

wedata/feature_store/entities/feature_lookup.py

@@ -4,7 +4,6 @@ import logging
 from typing import Dict, List, Optional, Union
 
 from wedata.feature_store.utils import common_utils
-from wedata.feature_store.common.store_config.redis import RedisStoreConfig
 
 _logger = logging.getLogger(__name__)
 
@@ -20,8 +19,6 @@ class FeatureLookup:
 
     - table_name：特征表的名称。
     - lookup_key：用于在特征表和训练集之间进行联接的键。lookup_key必须是训练集中的列。lookup_key的类型和顺序必须与特征表的主键匹配。
-    - is_online：如果为True，则会使用在线特征表。如果为False，则会使用离线特征表。默认值为False。
-    - online_config：如果is_online为True，则会使用此配置来配置在线特征表。默认值为None。
     - feature_names：要从特征表中查找的特征的名称。如果您的模型需要主键作为特征，则可以将它们声明为独立的FeatureLookups。
     - rename_outputs：如果提供，则会将特征重命名为 :meth:`create_training_set() <databricks.feature_engineering.client.FeatureEngineeringClient.create_training_set>`返回的 :class:`TrainingSet <databricks.ml_features.training_set.TrainingSet>` 中的特征。
     - timestamp_lookup_key：用于在特征表和训练集之间进行联接的时间戳键。timestamp_lookup_key必须是训练集中的列。timestamp_lookup_key的类型必须与特征表的时间戳键的类型匹配。
@@ -49,8 +46,6 @@ class FeatureLookup:
         table_name: str,
         lookup_key: Union[str, List[str]],
         *,
-        is_online: bool = False,
-        online_config: RedisStoreConfig = None,
         feature_names: Union[str, List[str], None] = None,
         rename_outputs: Optional[Dict[str, str]] = None,
         timestamp_lookup_key: Optional[str] = None,
@@ -109,8 +104,6 @@ class FeatureLookup:
         self._lookup_key = copy.copy(lookup_key)
         self._timestamp_lookup_key = copy.copy(timestamp_lookup_key)
         self._lookback_window = copy.copy(lookback_window)
-        self._is_online = is_online
-        self._online_config = online_config
 
         self._rename_outputs = {}
         if rename_outputs is not None:
@@ -156,16 +149,6 @@ class FeatureLookup:
         """A lookback window applied only for point-in-time lookups."""
         return self._lookback_window
 
-    @property
-    def is_online(self):
-        """Whether to use online feature tables."""
-        return self._is_online
-
-    @property
-    def online_config(self):
-        """The online feature table configuration."""
-        return self._online_config
-
     def _get_feature_names(self):
         return self._feature_names
 

wedata/feature_store/entities/feature_spec.py

@@ -138,14 +138,14 @@ class FeatureSpec:
 
         # function_infos should not be duplicated
         common_utils.validate_strings_unique(
-            [function_info.full_name for function_info in self.function_infos],
+            [function_info.udf_name for function_info in self.function_infos],
             "Internal Error: Expect all udf_names in function_infos to be unique. Found duplicates {}",
         )
 
         # Unique UDF names in function_infos must match those in column_infos.
         # No version check is required as both fields were added simultaneously in FeatureSpec v5.
         unique_udf_names = set(
-            [function_info.full_name for function_info in self.function_infos]
+            [function_info.udf_name for function_info in self.function_infos]
         )
         unique_column_udf_names = set(
             [odci.udf_name for odci in self.on_demand_column_infos]

wedata/feature_store/entities/feature_table.py

@@ -75,15 +75,11 @@ class FeatureTable:
             格式化的字符串，包含表名、ID、描述、主键、分区列、特征数量、
             时间戳键、创建时间、数据源数量和标签数量等信息
         """
-        if self.description and len(self.description) > 50:
-            desc = self.description[:50] + "..."
-        else:
-            desc = self.description
         return (
             f"FeatureTable(\n"
             f"  name='{self.name}',\n"
             f"  table_id='{self.table_id}',\n"
-            f"  description='{desc}',\n"
+            f"  description='{self.description[:50]}{'...' if len(self.description) > 50 else ''}',\n"
             f"  primary_keys={self.primary_keys},\n"
             f"  partition_columns={self.partition_columns},\n"
             f"  features={len(self.features)},\n"

wedata/feature_store/entities/function_info.py

@@ -1,6 +1,9 @@
-
+from collections import defaultdict
 from typing import List, Optional
 
+from pyspark.sql import Column, DataFrame
+from pyspark.sql.functions import isnull, when
+from pyspark.sql.types import StringType, StructField, StructType
 from wedata.feature_store.common.protos import feature_store_pb2
 
 class FunctionParameterInfo():