staran 0.6.1__py3-none-any.whl → 1.0.0__py3-none-any.whl

staran/features/manager.py

@@ -1,155 +0,0 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-
-"""
-特征管理器
-负责特征工程的核心管理功能，基于新的引擎架构
-"""
-
-from typing import Optional, Dict, Any, List, Callable
-from datetime import datetime
-from ..engines import BaseEngine, create_engine, DatabaseType
-
-
-class FeatureManager:
-    """
-    特征管理器 - 使用引擎架构的核心特征管理
-    """
-    
-    def __init__(self, database_name: str, engine_type: str = "spark", 
-                 sql_executor: Optional[Callable] = None):
-        """
-        初始化特征管理器
-        
-        Args:
-            database_name: 数据库名称
-            engine_type: 引擎类型 ('spark', 'hive', 'turing')
-            sql_executor: SQL执行器函数 (可选，仅用于非turing引擎)
-        """
-        self.database_name = database_name
-        self.engine_type = engine_type
-        
-        # 创建数据库引擎
-        self.engine = create_engine(
-            engine_type=engine_type,
-            database_name=database_name,
-            sql_executor=sql_executor
-        )
-    
-    # 委托给引擎的方法
-    def execute_sql(self, sql: str, description: str = "") -> Any:
-        """执行SQL语句"""
-        return self.engine.execute_sql(sql, description)
-    
-    def get_full_table_name(self, table_name: str) -> str:
-        """获取完整的表名（包含数据库名）"""
-        return self.engine.get_full_table_name(table_name)
-    
-    def generate_table_name(self, base_name: str, year: int, month: int, 
-                          suffix: str = "raw") -> str:
-        """
-        生成标准化的表名
-        格式: {base_name}_{yyyy}_{MM}_{suffix}
-        """
-        return self.engine.generate_table_name(base_name, year, month, suffix)
-    
-    def create_table(self, table_name: str, select_sql: str, 
-                    execute: bool = False, **kwargs) -> Dict[str, Any]:
-        """创建表"""
-        return self.engine.create_table(table_name, select_sql, execute, **kwargs)
-    
-    def drop_table(self, table_name: str, execute: bool = False) -> Dict[str, Any]:
-        """删除表"""
-        return self.engine.drop_table(table_name, execute)
-    
-    def download_table_data(self, table_name: str, output_path: str, 
-                          **kwargs) -> Dict[str, Any]:
-        """下载表数据"""
-        return self.engine.download_table_data(table_name, output_path, **kwargs)
-    
-    def download_query_result(self, sql: str, output_path: str, 
-                            **kwargs) -> Dict[str, Any]:
-        """下载查询结果"""
-        return self.engine.download_query_result(sql, output_path, **kwargs)
-    
-    def get_execution_history(self) -> List[Dict]:
-        """获取SQL执行历史"""
-        return self.engine.get_execution_history()
-    
-    def clear_history(self):
-        """清空执行历史"""
-        self.engine.clear_history()
-    
-    def __str__(self):
-        return f"FeatureManager(engine={self.engine})"
-
-
-class FeatureTableManager:
-    """
-    特征表管理器
-    负责特征表的创建、删除、管理等操作
-    """
-    
-    def __init__(self, feature_manager: FeatureManager):
-        """
-        初始化表管理器
-        
-        Args:
-            feature_manager: 特征管理器实例
-        """
-        self.feature_manager = feature_manager
-        self.created_tables = []
-    
-    def create_feature_table(self, base_name: str, year: int, month: int, 
-                           version: int, sql: str, execute: bool = False,
-                           **kwargs) -> str:
-        """
-        创建特征表
-        
-        Args:
-            base_name: 基础表名
-            year: 年份
-            month: 月份
-            version: 版本号
-            sql: 创建表的SQL
-            execute: 是否立即执行
-            **kwargs: 传递给引擎的其他参数
-            
-        Returns:
-            创建的表名
-        """
-        table_name = self.feature_manager.generate_table_name(base_name, year, month)
-        
-        result = self.feature_manager.create_table(table_name, sql, execute, **kwargs)
-        
-        if execute and result.get('status') == 'success':
-            self.created_tables.append(table_name)
-        
-        return table_name
-    
-    def drop_feature_table(self, table_name: str, execute: bool = False) -> str:
-        """
-        删除特征表
-        
-        Args:
-            table_name: 表名
-            execute: 是否立即执行
-            
-        Returns:
-            删除表的SQL
-        """
-        result = self.feature_manager.drop_table(table_name, execute)
-        
-        if execute and result.get('status') == 'success':
-            if table_name in self.created_tables:
-                self.created_tables.remove(table_name)
-        
-        return result.get('sql', '')
-    
-    def get_created_tables(self) -> List[str]:
-        """获取已创建的表列表"""
-        return self.created_tables.copy()
-    
-    def table_exists(self, table_name: str) -> bool:
-        """检查表是否存在（简单检查，实际需要查询数据库）"""
-        return table_name in self.created_tables

staran/features/schema.py

@@ -1,193 +0,0 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-
-"""
-表结构定义模块
-定义数据库表的字段、类型和分析特性
-"""
-
-from enum import Enum
-from typing import Dict, List, Optional, Union
-from dataclasses import dataclass
-
-
-class FieldType(Enum):
-    """字段类型枚举"""
-    STRING = "string"
-    INTEGER = "int"
-    BIGINT = "bigint" 
-    DECIMAL = "decimal"
-    DOUBLE = "double"
-    FLOAT = "float"
-    DATE = "date"
-    TIMESTAMP = "timestamp"
-    BOOLEAN = "boolean"
-
-
-@dataclass
-class Field:
-    """字段定义"""
-    name: str
-    field_type: FieldType
-    is_primary_key: bool = False
-    is_date_field: bool = False
-    aggregatable: bool = False
-    nullable: bool = True
-    comment: str = ""
-    
-    def __post_init__(self):
-        """初始化后处理"""
-        # 数值类型默认可聚合
-        if self.field_type in [FieldType.INTEGER, FieldType.BIGINT, 
-                              FieldType.DECIMAL, FieldType.DOUBLE, FieldType.FLOAT]:
-            if not hasattr(self, '_aggregatable_set'):
-                self.aggregatable = True
-                
-    def set_aggregatable(self, aggregatable: bool):
-        """设置是否可聚合"""
-        self.aggregatable = aggregatable
-        self._aggregatable_set = True
-        return self
-
-
-class TableSchema:
-    """表结构定义类"""
-    
-    def __init__(self, table_name: str, comment: str = ""):
-        """
-        初始化表结构
-        
-        Args:
-            table_name: 表名
-            comment: 表注释
-        """
-        self.table_name = table_name
-        self.comment = comment
-        self.fields: Dict[str, Field] = {}
-        self.primary_key: Optional[str] = None
-        self.date_field: Optional[str] = None
-        self.is_monthly_unique: bool = False
-        
-    def add_field(self, name: str, field_type: Union[str, FieldType], 
-                  aggregatable: bool = None, nullable: bool = True, 
-                  comment: str = "") -> 'TableSchema':
-        """
-        添加字段
-        
-        Args:
-            name: 字段名
-            field_type: 字段类型
-            aggregatable: 是否可聚合（None时自动判断）
-            nullable: 是否可空
-            comment: 字段注释
-            
-        Returns:
-            self: 支持链式调用
-        """
-        if isinstance(field_type, str):
-            field_type = FieldType(field_type.lower())
-            
-        field = Field(
-            name=name,
-            field_type=field_type,
-            nullable=nullable,
-            comment=comment
-        )
-        
-        if aggregatable is not None:
-            field.set_aggregatable(aggregatable)
-            
-        self.fields[name] = field
-        return self
-        
-    def add_primary_key(self, name: str, field_type: Union[str, FieldType],
-                       comment: str = "主键") -> 'TableSchema':
-        """添加主键字段"""
-        if isinstance(field_type, str):
-            field_type = FieldType(field_type.lower())
-            
-        field = Field(
-            name=name,
-            field_type=field_type,
-            is_primary_key=True,
-            nullable=False,
-            comment=comment
-        )
-        field.set_aggregatable(False)
-        
-        self.fields[name] = field
-        self.primary_key = name
-        return self
-        
-    def add_date_field(self, name: str, field_type: Union[str, FieldType] = FieldType.DATE,
-                      comment: str = "日期字段") -> 'TableSchema':
-        """添加日期字段"""
-        if isinstance(field_type, str):
-            field_type = FieldType(field_type.lower())
-            
-        field = Field(
-            name=name,
-            field_type=field_type,
-            is_date_field=True,
-            nullable=False,
-            comment=comment
-        )
-        field.set_aggregatable(False)
-        
-        self.fields[name] = field
-        self.date_field = name
-        return self
-        
-    def set_monthly_unique(self, is_unique: bool = True) -> 'TableSchema':
-        """设置是否为每人每月唯一数据"""
-        self.is_monthly_unique = is_unique
-        return self
-        
-    def get_aggregatable_fields(self) -> List[Field]:
-        """获取可聚合字段列表"""
-        return [field for field in self.fields.values() if field.aggregatable]
-        
-    def get_non_aggregatable_fields(self) -> List[Field]:
-        """获取不可聚合字段列表（用于原始拷贝）"""
-        return [field for field in self.fields.values() 
-                if not field.aggregatable and not field.is_primary_key and not field.is_date_field]
-        
-    def validate(self) -> bool:
-        """验证表结构"""
-        if not self.primary_key:
-            raise ValueError("表必须定义主键")
-            
-        if not self.date_field:
-            raise ValueError("表必须定义日期字段")
-            
-        if self.primary_key not in self.fields:
-            raise ValueError(f"主键字段 {self.primary_key} 不存在")
-            
-        if self.date_field not in self.fields:
-            raise ValueError(f"日期字段 {self.date_field} 不存在")
-            
-        return True
-        
-    def __str__(self) -> str:
-        """字符串表示"""
-        lines = [f"Table: {self.table_name}"]
-        if self.comment:
-            lines.append(f"Comment: {self.comment}")
-            
-        lines.append(f"Primary Key: {self.primary_key}")
-        lines.append(f"Date Field: {self.date_field}")
-        lines.append(f"Monthly Unique: {self.is_monthly_unique}")
-        lines.append("Fields:")
-        
-        for field in self.fields.values():
-            flag_str = ""
-            if field.is_primary_key:
-                flag_str += "[PK]"
-            if field.is_date_field:
-                flag_str += "[DATE]"
-            if field.aggregatable:
-                flag_str += "[AGG]"
-                
-            lines.append(f"  {field.name}: {field.field_type.value} {flag_str}")
-            
-        return "\n".join(lines)

staran/models/__init__.py

@@ -1,72 +0,0 @@
-"""
-staran.models v0.6.0 - 新疆工行代发长尾客户模型管理
-
-专门针对新疆工行代发长尾客户的两个核心模型：
-1. 代发长尾客户提升3k预测模型 (daifa_longtail_upgrade_3k)
-2. 代发长尾客户防流失1.5k预测模型 (daifa_longtail_churn_1_5k)
-
-主要功能：
-- 模型配置管理
-- SQL驱动的目标变量定义
-- 模型注册和版本控制
-- 新疆工行特定配置
-"""
-
-from .config import ModelConfig, create_model_config
-from .target import TargetDefinition, create_target_definition
-from .registry import ModelRegistry, register_model, save_model_registry
-from .daifa_models import (
-    create_daifa_longtail_upgrade_model,
-    create_daifa_longtail_churn_model,
-    get_available_daifa_models,
-    create_both_daifa_models
-)
-
-# 便捷函数
-def create_xinjiang_icbc_models(output_dir: str = "./xinjiang_models") -> dict:
-    """为新疆工行创建两个代发长尾客户模型"""
-    return create_both_daifa_models(output_dir)
-
-def list_available_models() -> list:
-    """列出所有可用的代发长尾客户模型"""
-    return get_available_daifa_models()
-
-def get_model_summary() -> dict:
-    """获取模型概述信息"""
-    return {
-        "version": "0.6.0",
-        "bank": "新疆工行",
-        "business_domain": "代发长尾客户",
-        "models": [
-            {
-                "name": "daifa_longtail_upgrade_3k",
-                "description": "预测下个月代发长尾客户资产提升3k的概率",
-                "target_amount": 3000,
-                "model_type": "binary_classification"
-            },
-            {
-                "name": "daifa_longtail_churn_1_5k", 
-                "description": "预测下个月代发长尾客户流失1.5k资产的风险",
-                "target_amount": 1500,
-                "model_type": "binary_classification"
-            }
-        ]
-    }
-
-__all__ = [
-    # 核心组件
-    'ModelConfig', 'TargetDefinition', 'ModelRegistry',
-    
-    # 创建函数
-    'create_model_config', 'create_target_definition', 'register_model',
-    
-    # 代发长尾模型
-    'create_daifa_longtail_upgrade_model', 'create_daifa_longtail_churn_model',
-    'create_both_daifa_models', 'get_available_daifa_models',
-    
-    # 便捷函数
-    'create_xinjiang_icbc_models', 'list_available_models', 'get_model_summary',
-    'save_model_registry'
-]
-
-__version__ = "0.6.0"

staran/models/config.py

@@ -1,271 +0,0 @@
-"""
-模型配置管理模块
-
-定义模型的核心配置信息，包括模型类型、参数、特征配置等
-"""
-
-from enum import Enum
-from typing import Dict, Any, List, Optional
-from dataclasses import dataclass, field
-from datetime import datetime
-
-
-class ModelType(Enum):
-    """模型类型枚举"""
-    CLASSIFICATION = "classification"
-    REGRESSION = "regression" 
-    CLUSTERING = "clustering"
-    TIME_SERIES = "time_series"
-    ANOMALY_DETECTION = "anomaly_detection"
-    RECOMMENDATION = "recommendation"
-
-
-class ModelAlgorithm(Enum):
-    """模型算法枚举"""
-    # 分类算法
-    LOGISTIC_REGRESSION = "logistic_regression"
-    RANDOM_FOREST = "random_forest"
-    GRADIENT_BOOSTING = "gradient_boosting"
-    SVM = "svm"
-    NEURAL_NETWORK = "neural_network"
-    
-    # 回归算法
-    LINEAR_REGRESSION = "linear_regression"
-    RIDGE_REGRESSION = "ridge_regression"
-    LASSO_REGRESSION = "lasso_regression"
-    
-    # 聚类算法
-    KMEANS = "kmeans"
-    DBSCAN = "dbscan"
-    HIERARCHICAL = "hierarchical"
-    
-    # 时间序列
-    ARIMA = "arima"
-    LSTM = "lstm"
-    PROPHET = "prophet"
-
-
-@dataclass
-class FeatureConfig:
-    """特征配置"""
-    schema_name: str                    # 使用的schema名称 (如 'aum')
-    table_types: List[str]              # 使用的表类型列表 (如 ['behavior', 'asset_avg'])
-    feature_selection: bool = True      # 是否启用特征选择
-    feature_engineering: bool = True    # 是否启用特征工程
-    scaling: bool = True                # 是否启用特征缩放
-    encoding: Dict[str, str] = field(default_factory=dict)  # 编码配置
-
-
-@dataclass 
-class ModelConfig:
-    """模型配置类"""
-    # 基本信息
-    name: str                           # 模型名称
-    model_type: ModelType               # 模型类型
-    algorithm: ModelAlgorithm           # 使用的算法
-    version: str = "1.0.0"              # 模型版本
-    
-    # 特征配置
-    feature_config: FeatureConfig = None
-    
-    # 模型参数
-    hyperparameters: Dict[str, Any] = field(default_factory=dict)
-    
-    # 训练配置
-    training_config: Dict[str, Any] = field(default_factory=lambda: {
-        'test_size': 0.2,
-        'random_state': 42,
-        'cross_validation': True,
-        'cv_folds': 5
-    })
-    
-    # 评估配置
-    evaluation_metrics: List[str] = field(default_factory=list)
-    
-    # 银行特定配置
-    bank_code: str = "generic"          # 银行代码
-    business_domain: str = "generic"    # 业务领域
-    
-    # 元数据
-    description: str = ""               # 模型描述
-    created_at: datetime = field(default_factory=datetime.now)
-    created_by: str = "system"          # 创建者
-    tags: List[str] = field(default_factory=list)
-    
-    # 部署配置
-    deployment_config: Dict[str, Any] = field(default_factory=dict)
-    
-    def __post_init__(self):
-        """初始化后处理"""
-        if self.feature_config is None:
-            self.feature_config = FeatureConfig(
-                schema_name="generic",
-                table_types=["base"]
-            )
-        
-        # 根据模型类型设置默认评估指标
-        if not self.evaluation_metrics:
-            self.evaluation_metrics = self._get_default_metrics()
-    
-    def _get_default_metrics(self) -> List[str]:
-        """根据模型类型获取默认评估指标"""
-        if self.model_type == ModelType.CLASSIFICATION:
-            return ['accuracy', 'precision', 'recall', 'f1_score', 'auc']
-        elif self.model_type == ModelType.REGRESSION:
-            return ['mae', 'mse', 'rmse', 'r2_score']
-        elif self.model_type == ModelType.CLUSTERING:
-            return ['silhouette_score', 'calinski_harabasz_score']
-        else:
-            return ['custom_metric']
-    
-    def to_dict(self) -> Dict[str, Any]:
-        """转换为字典格式"""
-        return {
-            'name': self.name,
-            'model_type': self.model_type.value,
-            'algorithm': self.algorithm.value,
-            'version': self.version,
-            'feature_config': {
-                'schema_name': self.feature_config.schema_name,
-                'table_types': self.feature_config.table_types,
-                'feature_selection': self.feature_config.feature_selection,
-                'feature_engineering': self.feature_config.feature_engineering,
-                'scaling': self.feature_config.scaling,
-                'encoding': self.feature_config.encoding
-            },
-            'hyperparameters': self.hyperparameters,
-            'training_config': self.training_config,
-            'evaluation_metrics': self.evaluation_metrics,
-            'bank_code': self.bank_code,
-            'business_domain': self.business_domain,
-            'description': self.description,
-            'created_at': self.created_at.isoformat(),
-            'created_by': self.created_by,
-            'tags': self.tags,
-            'deployment_config': self.deployment_config
-        }
-    
-    @classmethod
-    def from_dict(cls, data: Dict[str, Any]) -> 'ModelConfig':
-        """从字典创建ModelConfig实例"""
-        feature_config_data = data.get('feature_config', {})
-        feature_config = FeatureConfig(
-            schema_name=feature_config_data.get('schema_name', 'generic'),
-            table_types=feature_config_data.get('table_types', ['base']),
-            feature_selection=feature_config_data.get('feature_selection', True),
-            feature_engineering=feature_config_data.get('feature_engineering', True),
-            scaling=feature_config_data.get('scaling', True),
-            encoding=feature_config_data.get('encoding', {})
-        )
-        
-        return cls(
-            name=data['name'],
-            model_type=ModelType(data['model_type']),
-            algorithm=ModelAlgorithm(data['algorithm']),
-            version=data.get('version', '1.0.0'),
-            feature_config=feature_config,
-            hyperparameters=data.get('hyperparameters', {}),
-            training_config=data.get('training_config', {}),
-            evaluation_metrics=data.get('evaluation_metrics', []),
-            bank_code=data.get('bank_code', 'generic'),
-            business_domain=data.get('business_domain', 'generic'),
-            description=data.get('description', ''),
-            created_by=data.get('created_by', 'system'),
-            tags=data.get('tags', []),
-            deployment_config=data.get('deployment_config', {})
-        )
-
-
-def create_model_config(
-    name: str,
-    model_type: str,
-    algorithm: str,
-    schema_name: str = "generic",
-    table_types: List[str] = None,
-    bank_code: str = "generic",
-    **kwargs
-) -> ModelConfig:
-    """
-    创建模型配置的便捷函数
-    
-    Args:
-        name: 模型名称
-        model_type: 模型类型 
-        algorithm: 算法名称
-        schema_name: 使用的schema名称
-        table_types: 使用的表类型列表
-        bank_code: 银行代码
-        **kwargs: 其他配置参数
-        
-    Returns:
-        ModelConfig实例
-    """
-    if table_types is None:
-        table_types = ["base"]
-    
-    feature_config = FeatureConfig(
-        schema_name=schema_name,
-        table_types=table_types
-    )
-    
-    return ModelConfig(
-        name=name,
-        model_type=ModelType(model_type),
-        algorithm=ModelAlgorithm(algorithm),
-        feature_config=feature_config,
-        bank_code=bank_code,
-        **kwargs
-    )
-
-
-# 预定义的模型配置模板
-PRESET_CONFIGS = {
-    "aum_longtail_classification": {
-        "model_type": "classification",
-        "algorithm": "random_forest",
-        "schema_name": "aum",
-        "table_types": ["behavior", "asset_avg", "asset_config", "monthly_stat"],
-        "hyperparameters": {
-            "n_estimators": 100,
-            "max_depth": 10,
-            "random_state": 42
-        },
-        "description": "AUM长尾客户分类模型"
-    },
-    
-    "customer_value_regression": {
-        "model_type": "regression", 
-        "algorithm": "gradient_boosting",
-        "schema_name": "aum",
-        "table_types": ["behavior", "asset_avg"],
-        "hyperparameters": {
-            "n_estimators": 150,
-            "learning_rate": 0.1,
-            "max_depth": 8
-        },
-        "description": "客户价值预测回归模型"
-    }
-}
-
-
-def create_preset_config(preset_name: str, **overrides) -> ModelConfig:
-    """
-    基于预设模板创建模型配置
-    
-    Args:
-        preset_name: 预设模板名称
-        **overrides: 覆盖的配置参数
-        
-    Returns:
-        ModelConfig实例
-    """
-    if preset_name not in PRESET_CONFIGS:
-        raise ValueError(f"未知的预设配置: {preset_name}")
-    
-    config = PRESET_CONFIGS[preset_name].copy()
-    config.update(overrides)
-    
-    return create_model_config(
-        name=preset_name,
-        **config
-    )