tencent-wedata-feature-engineering-dev 0.1.50__tar.gz → 0.2.0__tar.gz

{tencent-wedata-feature-engineering-dev-0.1.50 → tencent_wedata_feature_engineering_dev-0.2.0}/PKG-INFO

@@ -1,7 +1,7 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.4
 Name: tencent-wedata-feature-engineering-dev
-Version: 0.1.50
-Summary: Wedata Feature Engineering Library Development
+Version: 0.2.0
+Summary: Wedata Feature Engineering Library
 Home-page: 
 Author: meahqian
 Author-email: 
@@ -11,3 +11,11 @@ Classifier: License :: OSI Approved :: Apache Software License
 Classifier: Operating System :: OS Independent
 Requires-Python: >=3.7
 Description-Content-Type: text/markdown
+Requires-Dist: pandas>=1.0.0
+Dynamic: author
+Dynamic: classifier
+Dynamic: description-content-type
+Dynamic: license
+Dynamic: requires-dist
+Dynamic: requires-python
+Dynamic: summary

{tencent-wedata-feature-engineering-dev-0.1.50 → tencent_wedata_feature_engineering_dev-0.2.0}/setup.py

@@ -11,16 +11,12 @@ setup(
     version=version["__version__"],
     packages=find_packages(include=['wedata', 'wedata.*']),
     install_requires=[
-        'pandas>=1.0.0',
-        'feast[redis]==0.49.0',
-        'grpcio==1.74.0',
-        'tencentcloud-sdk-python',
-        'ipython'
+        'pandas>=1.0.0'
     ],
     python_requires='>=3.7',
     author="meahqian",
     author_email="",
-    description="Wedata Feature Engineering Library Development",
+    description="Wedata Feature Engineering Library",
     long_description=open("README.md").read(),
     long_description_content_type="text/markdown",
     license="Apache 2.0",

{tencent-wedata-feature-engineering-dev-0.1.50 → tencent_wedata_feature_engineering_dev-0.2.0}/tencent_wedata_feature_engineering_dev.egg-info/PKG-INFO

@@ -1,7 +1,7 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.4
 Name: tencent-wedata-feature-engineering-dev
-Version: 0.1.50
-Summary: Wedata Feature Engineering Library Development
+Version: 0.2.0
+Summary: Wedata Feature Engineering Library
 Home-page: 
 Author: meahqian
 Author-email: 
@@ -11,3 +11,11 @@ Classifier: License :: OSI Approved :: Apache Software License
 Classifier: Operating System :: OS Independent
 Requires-Python: >=3.7
 Description-Content-Type: text/markdown
+Requires-Dist: pandas>=1.0.0
+Dynamic: author
+Dynamic: classifier
+Dynamic: description-content-type
+Dynamic: license
+Dynamic: requires-dist
+Dynamic: requires-python
+Dynamic: summary

{tencent-wedata-feature-engineering-dev-0.1.50 → tencent_wedata_feature_engineering_dev-0.2.0}/tencent_wedata_feature_engineering_dev.egg-info/SOURCES.txt

@@ -1,4 +1,5 @@
 README.md
+setup.cfg
 setup.py
 tencent_wedata_feature_engineering_dev.egg-info/PKG-INFO
 tencent_wedata_feature_engineering_dev.egg-info/SOURCES.txt
@@ -9,20 +10,11 @@ wedata/__init__.py
 wedata/feature_store/__init__.py
 wedata/feature_store/client.py
 wedata/feature_store/mlflow_model.py
-wedata/feature_store/cloud_sdk_client/__init__.py
-wedata/feature_store/cloud_sdk_client/client.py
-wedata/feature_store/cloud_sdk_client/models.py
-wedata/feature_store/cloud_sdk_client/utils.py
 wedata/feature_store/common/__init__.py
-wedata/feature_store/common/log/__init__.py
-wedata/feature_store/common/log/logger.py
 wedata/feature_store/common/protos/__init__.py
 wedata/feature_store/common/protos/feature_store_pb2.py
-wedata/feature_store/common/store_config/__init__.py
-wedata/feature_store/common/store_config/redis.py
 wedata/feature_store/constants/__init__.py
 wedata/feature_store/constants/constants.py
-wedata/feature_store/constants/engine_types.py
 wedata/feature_store/entities/__init__.py
 wedata/feature_store/entities/column_info.py
 wedata/feature_store/entities/environment_variables.py
@@ -38,8 +30,6 @@ wedata/feature_store/entities/function_info.py
 wedata/feature_store/entities/on_demand_column_info.py
 wedata/feature_store/entities/source_data_column_info.py
 wedata/feature_store/entities/training_set.py
-wedata/feature_store/feast_client/__init__.py
-wedata/feature_store/feast_client/feast_client.py
 wedata/feature_store/feature_table_client/__init__.py
 wedata/feature_store/feature_table_client/feature_table_client.py
 wedata/feature_store/spark_client/__init__.py
@@ -48,7 +38,6 @@ wedata/feature_store/training_set_client/__init__.py
 wedata/feature_store/training_set_client/training_set_client.py
 wedata/feature_store/utils/__init__.py
 wedata/feature_store/utils/common_utils.py
-wedata/feature_store/utils/env_utils.py
 wedata/feature_store/utils/feature_lookup_utils.py
 wedata/feature_store/utils/feature_spec_utils.py
 wedata/feature_store/utils/feature_utils.py
@@ -58,12 +47,4 @@ wedata/feature_store/utils/signature_utils.py
 wedata/feature_store/utils/topological_sort.py
 wedata/feature_store/utils/training_set_utils.py
 wedata/feature_store/utils/uc_utils.py
-wedata/feature_store/utils/validation_utils.py
-wedata/tempo/__init__.py
-wedata/tempo/interpol.py
-wedata/tempo/intervals.py
-wedata/tempo/io.py
-wedata/tempo/ml.py
-wedata/tempo/resample.py
-wedata/tempo/tsdf.py
-wedata/tempo/utils.py
+wedata/feature_store/utils/validation_utils.py

tencent_wedata_feature_engineering_dev-0.2.0/tencent_wedata_feature_engineering_dev.egg-info/requires.txt

@@ -0,0 +1 @@
+pandas>=1.0.0

{tencent-wedata-feature-engineering-dev-0.1.50 → tencent_wedata_feature_engineering_dev-0.2.0}/wedata/feature_store/client.py

@@ -4,6 +4,7 @@ Wedata FeatureStoreClient Python实现
 
 from __future__ import annotations
 
+import os
 from types import ModuleType
 from typing import Union, List, Dict, Optional, Any
 from pyspark.sql import DataFrame, SparkSession
@@ -13,9 +14,6 @@ import mlflow
 from wedata.feature_store.constants.constants import FEATURE_STORE_CLIENT
 
 from wedata.feature_store.constants.constants import APPEND, DEFAULT_WRITE_STREAM_TRIGGER
-from wedata.feature_store.constants.engine_types import EngineTypes
-from wedata.feature_store.common.store_config.redis import RedisStoreConfig
-from wedata.feature_store.cloud_sdk_client.models import TaskSchedulerConfiguration
 from wedata.feature_store.entities.feature_function import FeatureFunction
 from wedata.feature_store.entities.feature_lookup import FeatureLookup
 from wedata.feature_store.entities.feature_table import FeatureTable
@@ -30,7 +28,7 @@ from wedata.feature_store.utils.feature_utils import format_feature_lookups_and_
 class FeatureStoreClient:
     """特征存储统一客户端，提供特征全生命周期管理能力"""
 
-    def __init__(self, spark: Optional[SparkSession] = None, cloud_secret_id: str = None, cloud_secret_key: str = None):
+    def __init__(self, spark: Optional[SparkSession] = None):
         """
         :param spark: 已初始化的SparkSession对象
         """
@@ -38,20 +36,19 @@ class FeatureStoreClient:
             spark = SparkSession.builder.getOrCreate()
         self._spark = spark
         self._spark_client = SparkClient(spark)
-        self._feature_table_client = FeatureTableClient(spark, cloud_secret_id=cloud_secret_id,
-                                                        cloud_secret_key=cloud_secret_key)
+        self._feature_table_client = FeatureTableClient(spark)
         self._training_set_client = TrainingSetClient(self._spark_client)
+        self._init_catalog()
 
     def create_table(
             self,
             name: str,
             primary_keys: Union[str, List[str]],
-            timestamp_key: [str],
-            engine_type: [EngineTypes],
-            data_source_name: [str],
             database_name: Optional[str] = None,
+            location: Optional[str] = None,
             df: Optional[DataFrame] = None,
             *,
+            timestamp_keys: Union[str, List[str], None] = None,
             partition_columns: Union[str, List[str], None] = None,
             schema: Optional[StructType] = None,
             description: Optional[str] = None,
@@ -63,11 +60,10 @@ class FeatureStoreClient:
         Args:
             name: 特征表全称（格式：<table>）
             primary_keys: 主键列名（支持复合主键）
-            timestamp_key: 时间戳键（用于时态特征）
-            engine_type: 引擎类型  wedata.feature_store.constants.engine_types.EngineTypes
-            data_source_name: 数据源名称
             database_name: 数据库名
+            location: cos存储位置
             df: 初始数据（可选，用于推断schema）
+            timestamp_keys: 时间戳键（用于时态特征）
             partition_columns: 分区列（优化存储查询）
             schema: 表结构定义（可选，当不提供df时必需）
             description: 业务描述
@@ -83,56 +79,43 @@ class FeatureStoreClient:
         return self._feature_table_client.create_table(
             name=name,
             primary_keys=primary_keys,
-            engine_type=engine_type,
             database_name=database_name,
-            data_source_name=data_source_name,
+            location=location,
             df=df,
-            timestamp_key=timestamp_key,
+            timestamp_keys=timestamp_keys,
             partition_columns=partition_columns,
             schema=schema,
             description=description,
             tags=tags
         )
 
-    def register_table(self, name: str, timestamp_key: str, engine_type: EngineTypes, data_source_name: [str],
-                       database_name: Optional[str] = None,
-                       primary_keys: Union[str, List[str]] = None) -> DataFrame:
+    def register_table(self, name: str, database_name: Optional[str] = None) -> DataFrame:
         """
-        将普通的表注册为特征表，并返回特征表数据
+        读取特征表数据
 
         Args:
             name: 特征表名称
             database_name: 特征库名称
-            timestamp_key: 时间戳键 (用于后续离在线特征同步)
-            engine_type: 引擎类型  wedata.feature_store.constants.engine_types.EngineTypes
-            data_source_name: 数据源名称
-            primary_keys: 主键列名（支持复合主键）（仅当engine_type为EngineTypes.HIVE_ENGINE时有效）
+
         Returns:
             DataFrame: 包含特征表数据的DataFrame对象
         """
 
-        return self._feature_table_client.register_table(name, database_name, timestamp_key=timestamp_key,
-                                                         engine_type=engine_type, primary_keys=primary_keys, data_source_name=data_source_name)
+        return self._feature_table_client.register_table(name, database_name)
 
-    def read_table(self, name: str, database_name: Optional[str] = None, is_online: bool = False,
-                   online_config: Optional[RedisStoreConfig] = None,
-                   entity_row: Optional[List[Dict[str, Any]]] = None) -> DataFrame:
+    def read_table(self, name: str, database_name: Optional[str] = None) -> DataFrame:
         """
         读取特征表数据
 
         Args:
             name: 特征表名称
             database_name: 特征库名称
-            is_online: 是否读取在线特征表（默认不读取）
-            online_config: 在线特征表配置（仅当is_online为True时有效）
-            entity_row: 实体行数据（仅当is_online为True时有效）
-                       [{primary_key1: [value1, value2]}, {primary_key2: [value1, value2]}]
+
         Returns:
             DataFrame: 包含特征表数据的DataFrame对象
         """
 
-        return self._feature_table_client.read_table(name=name, database_name=database_name, is_online=is_online,
-                                                     online_config=online_config, entity_row=entity_row)
+        return self._feature_table_client.read_table(name, database_name)
 
     def get_table(self, name: str, database_name: Optional[str] = None) -> FeatureTable:
         """
@@ -154,6 +137,7 @@ class FeatureStoreClient:
         Args:
             name: 要删除的特征表名称
             database_name: database name
+
         Returns:
             None
         """
@@ -288,7 +272,7 @@ class FeatureStoreClient:
         )
 
     def score_batch(
-            self, model_uri: str, df: DataFrame, result_type: str = "double", timestamp_key: str = None
+        self, model_uri: str, df: DataFrame, result_type: str = "double"
     ) -> DataFrame:
         """
         Evaluate the model on the provided :class:`DataFrame <pyspark.sql.DataFrame>`.
@@ -399,65 +383,17 @@ class FeatureStoreClient:
             df=df,
             result_type=result_type,
             client_name=FEATURE_STORE_CLIENT,
-            timestamp_key=timestamp_key,
         )
 
-    def publish_table(self, table_name: str, data_source_name: str, cloud_secret_id: str, cloud_secret_key: str,
-                      database_name: Optional[str] = None,
-                      is_cycle: bool = False, cycle_obj: TaskSchedulerConfiguration = None,
-                      is_use_default_online: bool = True, online_config: RedisStoreConfig = None):
-        """
-        Publish an offline feature table to an online feature table.
-        
-        This method synchronizes the offline feature table data to online storage
-        for low-latency feature serving in real-time applications.
-        
-        Args:
-            table_name: Name of the offline feature table
-            data_source_name: Name of the data source
-            cloud_secret_id: Cloud secret ID for authentication
-            cloud_secret_key: Cloud secret key for authentication
-            database_name: Database name (optional)
-            is_cycle: Whether to enable periodic publishing (default: False)
-            cycle_obj: Periodic task configuration object (required if is_cycle is True)
-            is_use_default_online: Whether to use default online storage configuration (default: True)
-            online_config: Custom online storage configuration (only effective when is_use_default_online is False)
-        
-        Returns:
-            None
-
-        """
-        return self._feature_table_client.publish_table(table_name=table_name, database_name=database_name,
-                                                        data_source_name=data_source_name,
-                                                        cloud_secret_key=cloud_secret_key,
-                                                        cloud_secret_id=cloud_secret_id,
-                                                        is_cycle=is_cycle, cycle_obj=cycle_obj,
-                                                        is_use_default_online=is_use_default_online,
-                                                        online_config=online_config)
-
-    def drop_online_table(self, table_name: str, online_config: RedisStoreConfig, database_name: Optional[str] = None):
-        """
-        Drop an online feature table.
-        :param table_name: Name of the offline feature table
-        :param database_name: Database name (optional)
-        :param online_config: Custom online storage configuration (only effective when is_use_default_online is False)
-        :return:
-        """
-        self._feature_table_client.drop_online_table(table_name=table_name, database_name=database_name, online_config=online_config)
-
-    def create_feature_spec(
-            self, name: str,
-            features: List[Union[FeatureLookup, FeatureFunction]],
-            exclude_columns: List[str]):
-
-        """
-        创建特征配置文件
-        :arg name: 特征配置文件名称
-        :arg features: 特征列表，可以是FeatureLookup(特征查找)或FeatureFunction(特征函数)
-        :arg exclude_columns: 需要从最终特征集中排除的列名列表
-        """
-        return self._training_set_client.create_feature_spec(name, features, self._spark_client, exclude_columns)
-
     @property
     def spark(self):
         return self._spark
+
+    def _init_catalog(self):
+        """关联catalog"""
+        qcloud_region = os.getenv("QCLOUD_REGION") or os.getenv("REGION") or os.getenv("KERNEL_COS_REGION") \
+                        or os.getenv("NOTEBOOK_COS_REGION")
+        if qcloud_region:
+            mlflow.set_registry_uri(f"tclake:{qcloud_region}")
+
+

{tencent-wedata-feature-engineering-dev-0.1.50 → tencent_wedata_feature_engineering_dev-0.2.0}/wedata/feature_store/constants/constants.py

@@ -43,7 +43,7 @@ ML_MODEL = "MLmodel"
 FEATURE_LOOKUP_CLIENT_PIP_PACKAGE = "tencent-wedata-feature-engineering-dev"
 
 # 特征查找版本号
-FEATURE_LOOKUP_CLIENT_MAJOR_VERSION = "0.1.50"
+FEATURE_LOOKUP_CLIENT_MAJOR_VERSION = "0.2.0"
 
 # 特征存储内部数据目录
 FEATURE_STORE_INTERNAL_DATA_DIR = "_wedata_internal/"
@@ -53,7 +53,4 @@ WEDATA_DEFAULT_FEATURE_STORE_DATABASE = "WEDATA_DEFAULT_FEATURE_STORE_DATABASE"
 FEATURE_TABLE_KEY = "wedata.feature_table"
 FEATURE_TABLE_VALUE = "true"
 
-FEATURE_TABLE_PROJECT = "wedata.feature_project_id"
-FEATURE_TABLE_TIMESTAMP = "timestampKeys"
-FEATURE_TABLE_BACKUP_PRIMARY_KEY = "primaryKeys"    # 备用标识，主键
-FEATURE_DLC_TABLE_PRIMARY_KEY = "dlc.ao.data.govern.sorted.keys"
+FEATURE_TABLE_PROJECT = "wedata.feature_project_id"

{tencent-wedata-feature-engineering-dev-0.1.50 → tencent_wedata_feature_engineering_dev-0.2.0}/wedata/feature_store/entities/feature_lookup.py

@@ -4,7 +4,6 @@ import logging
 from typing import Dict, List, Optional, Union
 
 from wedata.feature_store.utils import common_utils
-from wedata.feature_store.common.store_config.redis import RedisStoreConfig
 
 _logger = logging.getLogger(__name__)
 
@@ -20,8 +19,6 @@ class FeatureLookup:
 
     - table_name：特征表的名称。
     - lookup_key：用于在特征表和训练集之间进行联接的键。lookup_key必须是训练集中的列。lookup_key的类型和顺序必须与特征表的主键匹配。
-    - is_online：如果为True，则会使用在线特征表。如果为False，则会使用离线特征表。默认值为False。
-    - online_config：如果is_online为True，则会使用此配置来配置在线特征表。默认值为None。
     - feature_names：要从特征表中查找的特征的名称。如果您的模型需要主键作为特征，则可以将它们声明为独立的FeatureLookups。
     - rename_outputs：如果提供，则会将特征重命名为 :meth:`create_training_set() <databricks.feature_engineering.client.FeatureEngineeringClient.create_training_set>`返回的 :class:`TrainingSet <databricks.ml_features.training_set.TrainingSet>` 中的特征。
     - timestamp_lookup_key：用于在特征表和训练集之间进行联接的时间戳键。timestamp_lookup_key必须是训练集中的列。timestamp_lookup_key的类型必须与特征表的时间戳键的类型匹配。
@@ -49,8 +46,6 @@ class FeatureLookup:
         table_name: str,
         lookup_key: Union[str, List[str]],
         *,
-        is_online: bool = False,
-        online_config: RedisStoreConfig = None,
         feature_names: Union[str, List[str], None] = None,
         rename_outputs: Optional[Dict[str, str]] = None,
         timestamp_lookup_key: Optional[str] = None,
@@ -109,8 +104,6 @@ class FeatureLookup:
         self._lookup_key = copy.copy(lookup_key)
         self._timestamp_lookup_key = copy.copy(timestamp_lookup_key)
         self._lookback_window = copy.copy(lookback_window)
-        self._is_online = is_online
-        self._online_config = online_config
 
         self._rename_outputs = {}
         if rename_outputs is not None:
@@ -156,16 +149,6 @@ class FeatureLookup:
         """A lookback window applied only for point-in-time lookups."""
         return self._lookback_window
 
-    @property
-    def is_online(self):
-        """Whether to use online feature tables."""
-        return self._is_online
-
-    @property
-    def online_config(self):
-        """The online feature table configuration."""
-        return self._online_config
-
     def _get_feature_names(self):
         return self._feature_names
 

{tencent-wedata-feature-engineering-dev-0.1.50 → tencent_wedata_feature_engineering_dev-0.2.0}/wedata/feature_store/entities/feature_spec.py

@@ -138,14 +138,14 @@ class FeatureSpec:
 
         # function_infos should not be duplicated
         common_utils.validate_strings_unique(
-            [function_info.full_name for function_info in self.function_infos],
+            [function_info.udf_name for function_info in self.function_infos],
             "Internal Error: Expect all udf_names in function_infos to be unique. Found duplicates {}",
         )
 
         # Unique UDF names in function_infos must match those in column_infos.
         # No version check is required as both fields were added simultaneously in FeatureSpec v5.
         unique_udf_names = set(
-            [function_info.full_name for function_info in self.function_infos]
+            [function_info.udf_name for function_info in self.function_infos]
         )
         unique_column_udf_names = set(
             [odci.udf_name for odci in self.on_demand_column_infos]

{tencent-wedata-feature-engineering-dev-0.1.50 → tencent_wedata_feature_engineering_dev-0.2.0}/wedata/feature_store/entities/feature_table.py

@@ -75,15 +75,11 @@ class FeatureTable:
             格式化的字符串，包含表名、ID、描述、主键、分区列、特征数量、
             时间戳键、创建时间、数据源数量和标签数量等信息
         """
-        if self.description and len(self.description) > 50:
-            desc = self.description[:50] + "..."
-        else:
-            desc = self.description
         return (
             f"FeatureTable(\n"
             f"  name='{self.name}',\n"
             f"  table_id='{self.table_id}',\n"
-            f"  description='{desc}',\n"
+            f"  description='{self.description[:50]}{'...' if len(self.description) > 50 else ''}',\n"
             f"  primary_keys={self.primary_keys},\n"
             f"  partition_columns={self.partition_columns},\n"
             f"  features={len(self.features)},\n"

{tencent-wedata-feature-engineering-dev-0.1.50 → tencent_wedata_feature_engineering_dev-0.2.0}/wedata/feature_store/entities/function_info.py

@@ -1,6 +1,9 @@
-
+from collections import defaultdict
 from typing import List, Optional
 
+from pyspark.sql import Column, DataFrame
+from pyspark.sql.functions import isnull, when
+from pyspark.sql.types import StringType, StructField, StructType
 from wedata.feature_store.common.protos import feature_store_pb2
 
 class FunctionParameterInfo():