staran 0.3.0__py3-none-any.whl → 0.4.0__py3-none-any.whl

staran/features/manager.py

@@ -0,0 +1,155 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+"""
+特征管理器
+负责特征工程的核心管理功能，基于新的引擎架构
+"""
+
+from typing import Optional, Dict, Any, List, Callable
+from datetime import datetime
+from ..engines import BaseEngine, create_engine, DatabaseType
+
+
+class FeatureManager:
+    """
+    特征管理器 - 使用引擎架构的核心特征管理
+    """
+    
+    def __init__(self, database_name: str, engine_type: str = "spark", 
+                 sql_executor: Optional[Callable] = None):
+        """
+        初始化特征管理器
+        
+        Args:
+            database_name: 数据库名称
+            engine_type: 引擎类型 ('spark', 'hive', 'turing')
+            sql_executor: SQL执行器函数 (可选，仅用于非turing引擎)
+        """
+        self.database_name = database_name
+        self.engine_type = engine_type
+        
+        # 创建数据库引擎
+        self.engine = create_engine(
+            engine_type=engine_type,
+            database_name=database_name,
+            sql_executor=sql_executor
+        )
+    
+    # 委托给引擎的方法
+    def execute_sql(self, sql: str, description: str = "") -> Any:
+        """执行SQL语句"""
+        return self.engine.execute_sql(sql, description)
+    
+    def get_full_table_name(self, table_name: str) -> str:
+        """获取完整的表名（包含数据库名）"""
+        return self.engine.get_full_table_name(table_name)
+    
+    def generate_table_name(self, base_name: str, year: int, month: int, 
+                          suffix: str = "raw") -> str:
+        """
+        生成标准化的表名
+        格式: {base_name}_{yyyy}_{MM}_{suffix}
+        """
+        return self.engine.generate_table_name(base_name, year, month, suffix)
+    
+    def create_table(self, table_name: str, select_sql: str, 
+                    execute: bool = False, **kwargs) -> Dict[str, Any]:
+        """创建表"""
+        return self.engine.create_table(table_name, select_sql, execute, **kwargs)
+    
+    def drop_table(self, table_name: str, execute: bool = False) -> Dict[str, Any]:
+        """删除表"""
+        return self.engine.drop_table(table_name, execute)
+    
+    def download_table_data(self, table_name: str, output_path: str, 
+                          **kwargs) -> Dict[str, Any]:
+        """下载表数据"""
+        return self.engine.download_table_data(table_name, output_path, **kwargs)
+    
+    def download_query_result(self, sql: str, output_path: str, 
+                            **kwargs) -> Dict[str, Any]:
+        """下载查询结果"""
+        return self.engine.download_query_result(sql, output_path, **kwargs)
+    
+    def get_execution_history(self) -> List[Dict]:
+        """获取SQL执行历史"""
+        return self.engine.get_execution_history()
+    
+    def clear_history(self):
+        """清空执行历史"""
+        self.engine.clear_history()
+    
+    def __str__(self):
+        return f"FeatureManager(engine={self.engine})"
+
+
+class FeatureTableManager:
+    """
+    特征表管理器
+    负责特征表的创建、删除、管理等操作
+    """
+    
+    def __init__(self, feature_manager: FeatureManager):
+        """
+        初始化表管理器
+        
+        Args:
+            feature_manager: 特征管理器实例
+        """
+        self.feature_manager = feature_manager
+        self.created_tables = []
+    
+    def create_feature_table(self, base_name: str, year: int, month: int, 
+                           version: int, sql: str, execute: bool = False,
+                           **kwargs) -> str:
+        """
+        创建特征表
+        
+        Args:
+            base_name: 基础表名
+            year: 年份
+            month: 月份
+            version: 版本号
+            sql: 创建表的SQL
+            execute: 是否立即执行
+            **kwargs: 传递给引擎的其他参数
+            
+        Returns:
+            创建的表名
+        """
+        table_name = self.feature_manager.generate_table_name(base_name, year, month)
+        
+        result = self.feature_manager.create_table(table_name, sql, execute, **kwargs)
+        
+        if execute and result.get('status') == 'success':
+            self.created_tables.append(table_name)
+        
+        return table_name
+    
+    def drop_feature_table(self, table_name: str, execute: bool = False) -> str:
+        """
+        删除特征表
+        
+        Args:
+            table_name: 表名
+            execute: 是否立即执行
+            
+        Returns:
+            删除表的SQL
+        """
+        result = self.feature_manager.drop_table(table_name, execute)
+        
+        if execute and result.get('status') == 'success':
+            if table_name in self.created_tables:
+                self.created_tables.remove(table_name)
+        
+        return result.get('sql', '')
+    
+    def get_created_tables(self) -> List[str]:
+        """获取已创建的表列表"""
+        return self.created_tables.copy()
+    
+    def table_exists(self, table_name: str) -> bool:
+        """检查表是否存在（简单检查，实际需要查询数据库）"""
+        return table_name in self.created_tables

staran/features/schema.py

@@ -0,0 +1,193 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+"""
+表结构定义模块
+定义数据库表的字段、类型和分析特性
+"""
+
+from enum import Enum
+from typing import Dict, List, Optional, Union
+from dataclasses import dataclass
+
+
+class FieldType(Enum):
+    """字段类型枚举"""
+    STRING = "string"
+    INTEGER = "int"
+    BIGINT = "bigint" 
+    DECIMAL = "decimal"
+    DOUBLE = "double"
+    FLOAT = "float"
+    DATE = "date"
+    TIMESTAMP = "timestamp"
+    BOOLEAN = "boolean"
+
+
+@dataclass
+class Field:
+    """字段定义"""
+    name: str
+    field_type: FieldType
+    is_primary_key: bool = False
+    is_date_field: bool = False
+    aggregatable: bool = False
+    nullable: bool = True
+    comment: str = ""
+    
+    def __post_init__(self):
+        """初始化后处理"""
+        # 数值类型默认可聚合
+        if self.field_type in [FieldType.INTEGER, FieldType.BIGINT, 
+                              FieldType.DECIMAL, FieldType.DOUBLE, FieldType.FLOAT]:
+            if not hasattr(self, '_aggregatable_set'):
+                self.aggregatable = True
+                
+    def set_aggregatable(self, aggregatable: bool):
+        """设置是否可聚合"""
+        self.aggregatable = aggregatable
+        self._aggregatable_set = True
+        return self
+
+
+class TableSchema:
+    """表结构定义类"""
+    
+    def __init__(self, table_name: str, comment: str = ""):
+        """
+        初始化表结构
+        
+        Args:
+            table_name: 表名
+            comment: 表注释
+        """
+        self.table_name = table_name
+        self.comment = comment
+        self.fields: Dict[str, Field] = {}
+        self.primary_key: Optional[str] = None
+        self.date_field: Optional[str] = None
+        self.is_monthly_unique: bool = False
+        
+    def add_field(self, name: str, field_type: Union[str, FieldType], 
+                  aggregatable: bool = None, nullable: bool = True, 
+                  comment: str = "") -> 'TableSchema':
+        """
+        添加字段
+        
+        Args:
+            name: 字段名
+            field_type: 字段类型
+            aggregatable: 是否可聚合（None时自动判断）
+            nullable: 是否可空
+            comment: 字段注释
+            
+        Returns:
+            self: 支持链式调用
+        """
+        if isinstance(field_type, str):
+            field_type = FieldType(field_type.lower())
+            
+        field = Field(
+            name=name,
+            field_type=field_type,
+            nullable=nullable,
+            comment=comment
+        )
+        
+        if aggregatable is not None:
+            field.set_aggregatable(aggregatable)
+            
+        self.fields[name] = field
+        return self
+        
+    def add_primary_key(self, name: str, field_type: Union[str, FieldType],
+                       comment: str = "主键") -> 'TableSchema':
+        """添加主键字段"""
+        if isinstance(field_type, str):
+            field_type = FieldType(field_type.lower())
+            
+        field = Field(
+            name=name,
+            field_type=field_type,
+            is_primary_key=True,
+            nullable=False,
+            comment=comment
+        )
+        field.set_aggregatable(False)
+        
+        self.fields[name] = field
+        self.primary_key = name
+        return self
+        
+    def add_date_field(self, name: str, field_type: Union[str, FieldType] = FieldType.DATE,
+                      comment: str = "日期字段") -> 'TableSchema':
+        """添加日期字段"""
+        if isinstance(field_type, str):
+            field_type = FieldType(field_type.lower())
+            
+        field = Field(
+            name=name,
+            field_type=field_type,
+            is_date_field=True,
+            nullable=False,
+            comment=comment
+        )
+        field.set_aggregatable(False)
+        
+        self.fields[name] = field
+        self.date_field = name
+        return self
+        
+    def set_monthly_unique(self, is_unique: bool = True) -> 'TableSchema':
+        """设置是否为每人每月唯一数据"""
+        self.is_monthly_unique = is_unique
+        return self
+        
+    def get_aggregatable_fields(self) -> List[Field]:
+        """获取可聚合字段列表"""
+        return [field for field in self.fields.values() if field.aggregatable]
+        
+    def get_non_aggregatable_fields(self) -> List[Field]:
+        """获取不可聚合字段列表（用于原始拷贝）"""
+        return [field for field in self.fields.values() 
+                if not field.aggregatable and not field.is_primary_key and not field.is_date_field]
+        
+    def validate(self) -> bool:
+        """验证表结构"""
+        if not self.primary_key:
+            raise ValueError("表必须定义主键")
+            
+        if not self.date_field:
+            raise ValueError("表必须定义日期字段")
+            
+        if self.primary_key not in self.fields:
+            raise ValueError(f"主键字段 {self.primary_key} 不存在")
+            
+        if self.date_field not in self.fields:
+            raise ValueError(f"日期字段 {self.date_field} 不存在")
+            
+        return True
+        
+    def __str__(self) -> str:
+        """字符串表示"""
+        lines = [f"Table: {self.table_name}"]
+        if self.comment:
+            lines.append(f"Comment: {self.comment}")
+            
+        lines.append(f"Primary Key: {self.primary_key}")
+        lines.append(f"Date Field: {self.date_field}")
+        lines.append(f"Monthly Unique: {self.is_monthly_unique}")
+        lines.append("Fields:")
+        
+        for field in self.fields.values():
+            flag_str = ""
+            if field.is_primary_key:
+                flag_str += "[PK]"
+            if field.is_date_field:
+                flag_str += "[DATE]"
+            if field.aggregatable:
+                flag_str += "[AGG]"
+                
+            lines.append(f"  {field.name}: {field.field_type.value} {flag_str}")
+            
+        return "\n".join(lines)

staran/schemas/__init__.py

@@ -0,0 +1,28 @@
+"""
+Staran Schemas模块 - 数据表结构定义与文档生成
+
+提供标准化的表结构定义、字段管理和文档生成功能。
+支持根据表结构生成Markdown和PDF文档供业务方使用。
+
+主要功能:
+- 表结构标准化定义
+- 业务字段含义管理  
+- 文档自动生成 (MD/PDF)
+- 多业务领域支持
+"""
+
+from .document_generator import SchemaDocumentGenerator
+from .aum import *
+
+__all__ = [
+    'SchemaDocumentGenerator',
+    # AUM业务表
+    'AUMBehaviorSchema',
+    'AUMAssetAvgSchema', 
+    'AUMAssetConfigSchema',
+    'AUMMonthlyStatSchema',
+    'get_aum_schemas',
+    'export_aum_docs'
+]
+
+__version__ = "0.3.0"