wedata-feature-engineering 0.1.0__tar.gz

wedata_feature_engineering-0.1.0/PKG-INFO

@@ -0,0 +1,13 @@
+Metadata-Version: 2.1
+Name: wedata_feature_engineering
+Version: 0.1.0
+Summary: Wedata Feature Engineering Library
+Home-page: 
+Author: meahqian
+Author-email: 
+License: Apache 2.0
+Classifier: Programming Language :: Python :: 3
+Classifier: License :: OSI Approved :: Apache Software License
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.7
+Description-Content-Type: text/markdown

wedata_feature_engineering-0.1.0/feature_store/__init__.py

@@ -0,0 +1,6 @@
+"""
+WeData Feature Engineering
+A toolkit for automated feature engineering
+"""
+
+__version__ = "0.1.0"

wedata_feature_engineering-0.1.0/feature_store/client.py

@@ -0,0 +1,169 @@
+"""
+Wedata FeatureStoreClient Python实现
+"""
+
+from __future__ import annotations
+from typing import Union, List, Dict, Optional, Any
+from pyspark.sql import DataFrame, SparkSession
+from pyspark.sql.streaming import StreamingQuery
+from pyspark.sql.types import StructType
+
+from feature_store.constants.constants import APPEND, DEFAULT_WRITE_STREAM_TRIGGER
+from feature_store.entities.feature_function import FeatureFunction
+from feature_store.entities.feature_lookup import FeatureLookup
+from feature_store.entities.training_set import TrainingSet
+from feature_store.feature_table_client.feature_table_client import FeatureTableClient
+from feature_store.spark_client.spark_client import SparkClient
+from feature_store.training_set_client.training_set_client import TrainingSetClient
+from feature_store.utils.feature_utils import format_feature_lookups_and_functions
+
+
+class FeatureStoreClient:
+    """特征存储统一客户端，提供特征全生命周期管理能力"""
+
+    def __init__(self, spark: SparkSession):
+        """
+        :param spark: 已初始化的SparkSession对象
+        """
+        self._spark = spark
+        self._spark_client = SparkClient(spark)
+        self._feature_table_client = FeatureTableClient(spark)
+
+    def create_table(
+            self,
+            name: str,
+            primary_keys: Union[str, List[str]],
+            df: Optional[DataFrame] = None,
+            *,
+            timestamp_keys: Union[str, List[str], None] = None,
+            partition_columns: Union[str, List[str], None] = None,
+            schema: Optional[StructType] = None,
+            description: Optional[str] = None,
+            tags: Optional[Dict[str, str]] = None
+    ):
+        """
+        创建特征表（支持批流数据写入）
+
+        Args:
+            name: 特征表全称（格式：<table>）
+            primary_keys: 主键列名（支持复合主键）
+            df: 初始数据（可选，用于推断schema）
+            timestamp_keys: 时间戳键（用于时态特征）
+            partition_columns: 分区列（优化存储查询）
+            description: 业务描述
+            tags: 业务标签
+
+        Returns:
+            FeatureTable实例
+
+        Raises:
+            ValueError: 当schema与数据不匹配时
+        """
+
+        return self._feature_table_client.create_table(
+            name=name,
+            primary_keys=primary_keys,
+            df=df,
+            timestamp_keys=timestamp_keys,
+            partition_columns=partition_columns,
+            schema=schema,
+            description=description,
+            tags=tags
+        )
+
+
+    def read_table(self, name: str) -> DataFrame:
+        """
+        读取特征表数据
+
+        Args:
+            name: 特征表名称
+
+        Returns:
+            DataFrame: 包含特征表数据的DataFrame对象
+        """
+        return self._feature_table_client.read_table(name)
+
+
+    def drop_table(self, name: str) -> None:
+        """
+        删除特征表
+
+        Args:
+            name: 要删除的特征表名称
+
+        Returns:
+            None
+        """
+        return self._feature_table_client.drop_table(name)
+
+
+    def create_training_set(
+            self,
+            df: DataFrame,
+            feature_lookups: List[Union[FeatureLookup, FeatureFunction]],
+            label: Union[str, List[str], None],
+            exclude_columns: Optional[List[str]] = None,
+            **kwargs,
+    ) -> TrainingSet:
+
+        """
+        创建训练集
+
+        Args:
+            df: 基础数据
+            feature_lookups: 特征查询列表
+            label: 标签列名
+            exclude_columns: 排除列名
+
+        Returns:
+            TrainingSet实例
+        """
+
+        if exclude_columns is None:
+            exclude_columns = []
+
+        features = feature_lookups
+        del feature_lookups
+
+        features = format_feature_lookups_and_functions(self._spark_client, features)
+        # 创建TrainingSetClient实例
+        training_set_client = TrainingSetClient(self._spark_client)
+        return training_set_client.create_training_set_from_feature_lookups(
+            df=df,
+            feature_lookups=features,
+            label=label,
+            exclude_columns=exclude_columns,
+            **kwargs
+        )
+
+    def write_table(
+            self,
+            name: str,
+            df: DataFrame,
+            mode: str = APPEND,
+            checkpoint_location: Optional[str] = None,
+            trigger: Dict[str, Any] = DEFAULT_WRITE_STREAM_TRIGGER,
+    ) -> Optional[StreamingQuery]:
+
+        """
+        写入数据到特征表（支持批处理和流式处理）
+
+        Args:
+            name: 特征表名称
+            df: 要写入的数据DataFrame
+            mode: 写入模式（默认追加）
+            checkpoint_location: 流式处理的检查点位置（可选）
+            trigger: 流式处理触发器配置（默认使用系统预设）
+
+        Returns:
+            如果是流式写入返回StreamingQuery对象，否则返回None
+        """
+
+        return self._feature_table_client.write_table(
+            name=name,
+            df=df,
+            mode=mode,
+            checkpoint_location=checkpoint_location,
+            trigger=trigger,
+        )

wedata_feature_engineering-0.1.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

wedata_feature_engineering-0.1.0/setup.py

@@ -0,0 +1,25 @@
+from setuptools import setup, find_packages
+
+setup(
+    name="wedata_feature_engineering",  # 包名称
+    version="0.1.0",                   # 版本号
+    packages=find_packages(exclude=["tests*"]),  # 自动发现所有包，排除tests目录
+    install_requires=[                 # 依赖包
+        'pyspark>=3.0.0',
+        'delta-spark>=1.0.0',
+        'pandas>=1.0.0'               # 新增常用数据处理依赖
+    ],
+    python_requires='>=3.7',           # Python版本要求
+    author="meahqian",                 # 作者
+    author_email="",                   # 新增作者邮箱(建议添加)
+    description="Wedata Feature Engineering Library",  # 描述
+    long_description=open("README.md").read(),  # 新增详细描述(需README.md文件)
+    long_description_content_type="text/markdown",  # 描述内容类型
+    license="Apache 2.0",              # 许可证
+    url="",                            # 新增项目URL(建议添加)
+    classifiers=[                      # 新增分类器
+        "Programming Language :: Python :: 3",
+        "License :: OSI Approved :: Apache Software License",
+        "Operating System :: OS Independent",
+    ],
+)

wedata_feature_engineering-0.1.0/wedata_feature_engineering.egg-info/PKG-INFO

@@ -0,0 +1,13 @@
+Metadata-Version: 2.1
+Name: wedata-feature-engineering
+Version: 0.1.0
+Summary: Wedata Feature Engineering Library
+Home-page: 
+Author: meahqian
+Author-email: 
+License: Apache 2.0
+Classifier: Programming Language :: Python :: 3
+Classifier: License :: OSI Approved :: Apache Software License
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.7
+Description-Content-Type: text/markdown

wedata_feature_engineering-0.1.0/wedata_feature_engineering.egg-info/SOURCES.txt

@@ -0,0 +1,9 @@
+README.md
+setup.py
+feature_store/__init__.py
+feature_store/client.py
+wedata_feature_engineering.egg-info/PKG-INFO
+wedata_feature_engineering.egg-info/SOURCES.txt
+wedata_feature_engineering.egg-info/dependency_links.txt
+wedata_feature_engineering.egg-info/requires.txt
+wedata_feature_engineering.egg-info/top_level.txt

wedata_feature_engineering-0.1.0/wedata_feature_engineering.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

wedata_feature_engineering-0.1.0/wedata_feature_engineering.egg-info/requires.txt

@@ -0,0 +1,3 @@
+pyspark>=3.0.0
+delta-spark>=1.0.0
+pandas>=1.0.0

wedata_feature_engineering-0.1.0/wedata_feature_engineering.egg-info/top_level.txt

@@ -0,0 +1 @@
+feature_store