staran 0.6.0__py3-none-any.whl → 1.0.0__py3-none-any.whl

staran/engines/turing.py

@@ -1,439 +0,0 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-
-"""
-图灵平台引擎
-继承Spark引擎，重写执行和下载方法以使用turingPythonLib
-"""
-
-from typing import Dict, Any, Optional, List, Callable
-import sys
-import os
-from datetime import datetime
-from .spark import SparkEngine
-
-# 尝试导入turingPythonLib（在图灵平台环境中）
-try:
-    sys.path.append("/nfsHome/")
-    import turingPythonLib as tp
-    TURINGLIB_AVAILABLE = True
-except ImportError:
-    tp = None
-    TURINGLIB_AVAILABLE = False
-
-
-class TuringEngine(SparkEngine):
-    """
-    图灵平台引擎
-    继承Spark引擎，使用turingPythonLib进行SQL执行和数据下载
-    """
-    
-    def __init__(self, database_name: str, sql_executor: Optional[Callable] = None):
-        # 不使用传入的sql_executor，因为我们使用turingPythonLib
-        super().__init__(database_name, None)
-        
-        # 检查turingPythonLib是否可用
-        if not TURINGLIB_AVAILABLE:
-            print("⚠️  警告: turingPythonLib不可用，将使用模拟模式")
-    
-    def get_engine_name(self) -> str:
-        return "Turing Platform (Spark)"
-    
-    # ==================== 重写SQL执行方法 ====================
-    
-    def execute_sql(self, sql: str, description: str = "") -> Any:
-        """
-        使用turingPythonLib执行SQL
-        
-        Args:
-            sql: SQL语句
-            description: 执行描述
-            
-        Returns:
-            执行结果
-        """
-        if TURINGLIB_AVAILABLE:
-            try:
-                # 使用turingPythonLib执行SQL
-                result = tp.execute_sql(sql)
-                
-                self.execution_history.append({
-                    'sql': sql,
-                    'description': description,
-                    'timestamp': datetime.now(),
-                    'result': result,
-                    'platform': 'turingPythonLib'
-                })
-                
-                return result
-                
-            except Exception as e:
-                error_result = {
-                    'status': 'error',
-                    'message': f"执行SQL失败: {str(e)}",
-                    'error': str(e)
-                }
-                
-                self.execution_history.append({
-                    'sql': sql,
-                    'description': description,
-                    'timestamp': datetime.now(),
-                    'result': error_result,
-                    'platform': 'turingPythonLib'
-                })
-                
-                return error_result
-        else:
-            # 模拟模式
-            print(f"模拟执行SQL: {description or 'SQL语句'}")
-            print(f"  {sql[:100]}...")
-            
-            mock_result = {
-                'status': 'simulated',
-                'message': '模拟执行成功',
-                'sql': sql[:100] + '...'
-            }
-            
-            self.execution_history.append({
-                'sql': sql,
-                'description': description,
-                'timestamp': datetime.now(),
-                'result': mock_result,
-                'platform': 'simulation'
-            })
-            
-            return mock_result
-    
-    def create_table(self, table_name: str, select_sql: str, 
-                    execute: bool = False, mode: str = "cluster",
-                    spark_resource: Optional[Dict[str, str]] = None) -> Dict[str, Any]:
-        """
-        使用turingPythonLib创建表
-        
-        Args:
-            table_name: 表名
-            select_sql: 选择SQL
-            execute: 是否立即执行
-            mode: 运行模式 ('local' 或 'cluster')
-            spark_resource: Spark资源配置
-            
-        Returns:
-            创建结果
-        """
-        full_table_name = self.get_full_table_name(table_name)
-        
-        result = {
-            'table_name': table_name,
-            'full_table_name': full_table_name,
-            'select_sql': select_sql,
-            'executed': execute,
-            'mode': mode
-        }
-        
-        if execute:
-            if TURINGLIB_AVAILABLE:
-                # 构建turingPythonLib参数
-                params = {
-                    'create_mode': 'by_select',
-                    'table_name': full_table_name,
-                    'sql': select_sql,
-                    'run_mode': mode
-                }
-                
-                # 如果是集群模式且提供了资源配置
-                if mode == 'cluster' and spark_resource:
-                    params['spark_resource'] = spark_resource
-                elif mode == 'cluster':
-                    # 使用默认资源配置
-                    params['spark_resource'] = {
-                        'num_executors': '4',
-                        'driver_cores': '2',
-                        'driver_memory': '4G',
-                        'executor_cores': '2',
-                        'executor_memory': '4G'
-                    }
-                
-                try:
-                    tp_result = tp.create_hive_table(params)
-                    
-                    result.update({
-                        'status': 'success',
-                        'message': f"成功创建表: {full_table_name}",
-                        'turinglib_result': tp_result,
-                        'params': params
-                    })
-                    
-                except Exception as e:
-                    result.update({
-                        'status': 'error',
-                        'message': f"创建表失败: {str(e)}",
-                        'error': str(e),
-                        'params': params
-                    })
-            else:
-                # 模拟模式
-                result.update({
-                    'status': 'simulated',
-                    'message': f"模拟创建表: {full_table_name}",
-                    'simulated': True
-                })
-        else:
-            result['status'] = 'prepared'
-        
-        return result
-    
-    # ==================== 重写数据下载方法 ====================
-    
-    def download_table_data(self, table_name: str, output_path: str,
-                          source: str = "hadoop", mode: str = "cluster",
-                          columns: str = "*", condition: str = "",
-                          overwrite_path: str = "yes",
-                          spark_resource: Optional[Dict[str, str]] = None,
-                          **kwargs) -> Dict[str, Any]:
-        """
-        使用turingPythonLib下载表数据
-        
-        Args:
-            table_name: 要下载的表名
-            output_path: 输出路径，必须以 'file:///nfsHome/' 开头
-            source: 数据源类型 ('hadoop' 或 'mppdb')
-            mode: 运行模式 ('local' 或 'cluster')
-            columns: 要选择的列，默认为 "*"
-            condition: WHERE条件
-            overwrite_path: 是否覆盖路径 ('yes' 或 'no')
-            spark_resource: 集群模式下的资源配置
-            **kwargs: 其他参数
-            
-        Returns:
-            下载结果
-        """
-        # 验证输出路径
-        if not output_path.startswith('file:///nfsHome/'):
-            raise ValueError("输出路径必须以 'file:///nfsHome/' 开头")
-        
-        full_table_name = self.get_full_table_name(table_name)
-        
-        # 构建下载SQL
-        sql = f"SELECT {columns} FROM {full_table_name}"
-        if condition.strip():
-            if not condition.upper().strip().startswith('WHERE'):
-                condition = f"WHERE {condition}"
-            sql += f" {condition}"
-        
-        # 构建下载参数
-        params = {
-            'sql': sql,
-            'source': source,
-            'outputPath': output_path,
-            'overwrite_path': overwrite_path,
-            'mode': mode
-        }
-        
-        # 如果是集群模式且提供了资源配置
-        if mode == 'cluster' and spark_resource:
-            params['spark_resource'] = spark_resource
-        elif mode == 'cluster':
-            # 使用默认资源配置
-            params['spark_resource'] = {
-                'num_executors': '4',
-                'driver_cores': '2', 
-                'driver_memory': '4G',
-                'executor_cores': '2',
-                'executor_memory': '4G'
-            }
-        
-        try:
-            if TURINGLIB_AVAILABLE:
-                # 使用真实的turingPythonLib
-                tp_result = tp.download(params)
-                
-                # 判断下载是否成功
-                if isinstance(tp_result, dict) and tp_result.get('success') == '0':
-                    return {
-                        'status': 'success',
-                        'message': f'数据已下载到: {output_path}',
-                        'table_name': table_name,
-                        'output_path': output_path,
-                        'turinglib_result': tp_result,
-                        'params': params
-                    }
-                else:
-                    return {
-                        'status': 'error',
-                        'message': f"下载失败: {tp_result.get('data', '未知错误')}",
-                        'table_name': table_name,
-                        'turinglib_result': tp_result,
-                        'params': params
-                    }
-            else:
-                # 模拟模式
-                return {
-                    'status': 'simulated',
-                    'message': f'模拟下载到: {output_path}',
-                    'table_name': table_name,
-                    'output_path': output_path,
-                    'turinglib_result': {'success': '0', 'message': '模拟下载成功'},
-                    'params': params,
-                    'simulated': True
-                }
-                
-        except Exception as e:
-            return {
-                'status': 'error',
-                'message': f"下载异常: {str(e)}",
-                'table_name': table_name,
-                'error': str(e),
-                'params': params
-            }
-    
-    def download_query_result(self, sql: str, output_path: str,
-                            source: str = "hadoop", mode: str = "cluster",
-                            overwrite_path: str = "yes",
-                            spark_resource: Optional[Dict[str, str]] = None,
-                            **kwargs) -> Dict[str, Any]:
-        """
-        直接下载查询结果，使用turingPythonLib
-        
-        Args:
-            sql: 查询SQL
-            output_path: 输出路径
-            source: 数据源类型
-            mode: 运行模式
-            overwrite_path: 是否覆盖路径
-            spark_resource: 资源配置
-            **kwargs: 其他参数
-            
-        Returns:
-            下载结果
-        """
-        # 验证输出路径
-        if not output_path.startswith('file:///nfsHome/'):
-            raise ValueError("输出路径必须以 'file:///nfsHome/' 开头")
-        
-        # 构建下载参数
-        params = {
-            'sql': sql,
-            'source': source,
-            'outputPath': output_path,
-            'overwrite_path': overwrite_path,
-            'mode': mode
-        }
-        
-        # 如果是集群模式且提供了资源配置
-        if mode == 'cluster' and spark_resource:
-            params['spark_resource'] = spark_resource
-        elif mode == 'cluster':
-            params['spark_resource'] = {
-                'num_executors': '4',
-                'driver_cores': '2', 
-                'driver_memory': '4G',
-                'executor_cores': '2',
-                'executor_memory': '4G'
-            }
-        
-        try:
-            if TURINGLIB_AVAILABLE:
-                tp_result = tp.download(params)
-                
-                if isinstance(tp_result, dict) and tp_result.get('success') == '0':
-                    return {
-                        'status': 'success',
-                        'message': f'查询结果已下载到: {output_path}',
-                        'output_path': output_path,
-                        'turinglib_result': tp_result,
-                        'params': params
-                    }
-                else:
-                    return {
-                        'status': 'error',
-                        'message': f"下载失败: {tp_result.get('data', '未知错误')}",
-                        'turinglib_result': tp_result,
-                        'params': params
-                    }
-            else:
-                return {
-                    'status': 'simulated',
-                    'message': f'模拟下载查询结果到: {output_path}',
-                    'output_path': output_path,
-                    'turinglib_result': {'success': '0', 'message': '模拟下载成功'},
-                    'params': params,
-                    'simulated': True
-                }
-                
-        except Exception as e:
-            return {
-                'status': 'error',
-                'message': f"下载查询结果失败: {str(e)}",
-                'error': str(e),
-                'params': params
-            }
-    
-    # ==================== 图灵平台特有方法 ====================
-    
-    def install_python_packages(self, packages: List[str]) -> Dict[str, Any]:
-        """
-        安装Python包（使用turingPythonLib）
-        
-        Args:
-            packages: 要安装的包列表
-            
-        Returns:
-            安装结果
-        """
-        results = []
-        
-        for package in packages:
-            try:
-                if TURINGLIB_AVAILABLE:
-                    tp.pip_install(package)
-                    results.append({
-                        'package': package,
-                        'status': 'success',
-                        'message': f'成功安装 {package}'
-                    })
-                else:
-                    results.append({
-                        'package': package,
-                        'status': 'simulated',
-                        'message': f'模拟安装 {package} (turingPythonLib不可用)'
-                    })
-            except Exception as e:
-                results.append({
-                    'package': package,
-                    'status': 'error',
-                    'error': str(e),
-                    'message': f'安装 {package} 失败'
-                })
-        
-        return {
-            'total_packages': len(packages),
-            'successful_installs': len([r for r in results if r['status'] == 'success']),
-            'results': results
-        }
-    
-    def get_platform_info(self) -> Dict[str, Any]:
-        """获取图灵平台信息"""
-        return {
-            'engine_name': self.get_engine_name(),
-            'engine_type': self.get_engine_type().value,
-            'turinglib_available': TURINGLIB_AVAILABLE,
-            'nfs_home_exists': os.path.exists('/nfsHome'),
-            'database_name': self.database_name,
-            'current_working_dir': os.getcwd(),
-            'python_path': sys.path[:3]  # 只显示前3个路径
-        }
-
-
-# 便捷创建函数
-def create_turing_engine(database_name: str, **kwargs) -> TuringEngine:
-    """
-    便捷函数：创建图灵引擎实例
-    
-    Args:
-        database_name: 数据库名称
-        **kwargs: 其他参数
-        
-    Returns:
-        图灵引擎实例
-    """
-    return TuringEngine(database_name, **kwargs)

staran/examples/__init__.py

@@ -1,8 +0,0 @@
-"""
-Staran Examples Module
-包含各种完整的特征工程示例
-"""
-
-from .aum_longtail import AUMLongtailExample, create_aum_example, run_aum_example
-
-__all__ = ['AUMLongtailExample', 'create_aum_example', 'run_aum_example']

staran/examples/aum_longtail.py

@@ -1,250 +0,0 @@
-"""
-AUM代发长尾模型示例
-基于Staran v0.3.0架构，使用schemas模块的预定义表结构
-"""
-
-from typing import Dict, Optional
-from ..engines import create_turing_engine
-from ..features import FeatureManager, FeatureConfig, FeatureType
-from ..tools import Date
-from ..schemas.aum import get_aum_schemas
-
-
-class AUMLongtailExample:
-    """AUM代发长尾模型示例类"""
-    
-    def __init__(self, database: str = "dwegdata03000"):
-        """
-        初始化AUM长尾模型示例
-        
-        Args:
-            database: 数据库名称，默认为dwegdata03000
-        """
-        self.database = database
-        self.engine = create_turing_engine(database)
-        self.schemas = get_aum_schemas()  # 从schemas模块获取预定义的表结构
-        
-    def run(self, feature_date: Optional[str] = None, output_path: str = "file:///nfsHome/aum_longtail") -> Dict:
-        """
-        运行完整的AUM长尾模型特征工程
-        
-        Args:
-            feature_date: 特征日期，格式为YYYYMM，默认为当前月
-            output_path: 输出路径，默认为file:///nfsHome/aum_longtail
-            
-        Returns:
-            包含所有结果的字典
-        """
-        if feature_date is None:
-            feature_date = Date.today().format_compact()[:6]
-        
-        print(f"🚀 开始AUM长尾模型特征工程 - {feature_date}")
-        print("="*60)
-        
-        results = {}
-        
-        # 步骤1: 生成A表特征（行为特征表）- 只生成原始拷贝和聚合特征
-        print("📊 步骤1: 生成客户行为特征...")
-        results['behavior'] = self._generate_behavior_features('behavior', feature_date)
-        
-        # 步骤2: 生成B表特征（资产平均值表）- 完整特征
-        print("💰 步骤2: 生成资产平均值特征...")
-        results['asset_avg'] = self._generate_full_features('asset_avg', feature_date)
-        
-        # 步骤3: 生成C表特征（资产配置表）- 完整特征  
-        print("📈 步骤3: 生成资产配置特征...")
-        results['asset_config'] = self._generate_full_features('asset_config', feature_date)
-        
-        # 步骤4: 生成D表特征（月度统计表）- 完整特征
-        print("📋 步骤4: 生成月度统计特征...")
-        results['monthly_stat'] = self._generate_full_features('monthly_stat', feature_date)
-        
-        # 步骤5: 导出特征表
-        print("💾 步骤5: 导出特征表...")
-        results['exports'] = self._export_features(feature_date, output_path)
-        
-        print("="*60)
-        print("✅ AUM长尾模型特征工程完成！")
-        return results
-    
-    def _generate_behavior_features(self, table_type: str, feature_date: str) -> Dict:
-        """生成行为特征（A表）- 只生成原始拷贝和聚合特征"""
-        schema = self.schemas[table_type]
-        manager = FeatureManager(self.engine, self.database)
-        
-        # A表特征配置：只启用原始拷贝和聚合
-        config = FeatureConfig()
-        config.enable_feature(FeatureType.RAW_COPY)
-        config.enable_feature(FeatureType.AGGREGATION)
-        config.disable_feature(FeatureType.MOM)  # 不生成环比
-        config.disable_feature(FeatureType.YOY)  # 不生成同比
-        
-        print(f"   🔧 生成{schema.table_name}的特征...")
-        result = manager.generate_features(
-            schema=schema,
-            config=config,
-            feature_date=feature_date
-        )
-        
-        feature_count = manager.count_features(schema, config)
-        print(f"   ✅ A表特征生成完成: {feature_count}个特征")
-        return result
-    
-    def _generate_full_features(self, table_type: str, feature_date: str) -> Dict:
-        """生成完整特征（B、C、D表）- 聚合+5个月环比+1年同比"""
-        schema = self.schemas[table_type]
-        manager = FeatureManager(self.engine, self.database)
-        
-        # B、C、D表特征配置：完整特征集
-        config = FeatureConfig()
-        config.enable_feature(FeatureType.AGGREGATION)
-        config.enable_feature(FeatureType.MOM, mom_windows=[5])    # 5个月环比
-        config.enable_feature(FeatureType.YOY, yoy_windows=[12])  # 1年同比
-        
-        print(f"   🔧 生成{schema.table_name}的特征...")
-        result = manager.generate_features(
-            schema=schema,
-            config=config,
-            feature_date=feature_date
-        )
-        
-        feature_count = manager.count_features(schema, config)
-        print(f"   ✅ {table_type}表特征生成完成: {feature_count}个特征")
-        return result
-    
-    def _export_features(self, feature_date: str, output_path: str) -> Dict:
-        """导出所有特征表到指定路径"""
-        file_prefixes = {
-            'behavior': 'aum_behavior_features',
-            'asset_avg': 'aum_asset_avg_features',
-            'asset_config': 'aum_asset_config_features',
-            'monthly_stat': 'monthly_stat_features'
-        }
-        
-        results = {}
-        for table_type, file_prefix in file_prefixes.items():
-            print(f"   💾 导出{table_type}表...")
-            
-            # 构建特征表名
-            table_name = f"{self.schemas[table_type].table_name}_{feature_date}_f001"
-            
-            result = self.engine.download_table_data(
-                table_name=f"{self.database}.{table_name}",
-                output_path=f"{output_path}/{file_prefix}_{feature_date}.parquet",
-                mode="cluster"
-            )
-            
-            results[table_type] = result
-            print(f"   ✅ 导出 {table_type}: {result.get('status', 'unknown')}")
-        
-        return results
-    
-    def get_summary(self) -> Dict:
-        """获取示例摘要信息"""
-        summary = {
-            'database': self.database,
-            'tables': {},
-            'total_features': 0
-        }
-        
-        for table_type, schema in self.schemas.items():
-            try:
-                manager = FeatureManager(self.engine, self.database)
-                
-                if table_type == 'behavior':
-                    # A表只有原始拷贝和聚合特征
-                    config = FeatureConfig()
-                    config.enable_feature(FeatureType.RAW_COPY)
-                    config.enable_feature(FeatureType.AGGREGATION)
-                    config.disable_feature(FeatureType.MOM)
-                    config.disable_feature(FeatureType.YOY)
-                else:
-                    # B、C、D表包含完整特征：聚合+5个月MoM+1年YoY
-                    config = FeatureConfig()
-                    config.enable_feature(FeatureType.AGGREGATION, mom_windows=[5], yoy_windows=[12])
-                
-                feature_count = manager.count_features(schema, config)
-                summary['tables'][table_type] = {
-                    'table_name': schema.table_name,
-                    'field_count': len(schema.fields),
-                    'feature_count': feature_count,
-                    'features': {
-                        'total': feature_count,
-                        'aggregation': len(schema.fields),  # 估算
-                        'mom': len(schema.fields) * 5 if table_type != 'behavior' else 0,
-                        'yoy': len(schema.fields) * 1 if table_type != 'behavior' else 0
-                    }
-                }
-                summary['total_features'] += feature_count
-            except Exception as e:
-                # 在模拟模式下返回预估数量
-                base_fields = len(schema.fields)
-                if table_type == 'behavior':
-                    estimated_features = base_fields * 2  # 原始拷贝 + 聚合
-                    agg_count = base_fields
-                    mom_count = 0
-                    yoy_count = 0
-                else:
-                    estimated_features = base_fields * 8  # 聚合 + MoM + YoY 组合
-                    agg_count = base_fields
-                    mom_count = base_fields * 5
-                    yoy_count = base_fields * 1
-                
-                summary['tables'][table_type] = {
-                    'table_name': schema.table_name,
-                    'field_count': base_fields,
-                    'feature_count': estimated_features,
-                    'mode': 'estimated',
-                    'features': {
-                        'total': estimated_features,
-                        'aggregation': agg_count,
-                        'mom': mom_count,
-                        'yoy': yoy_count
-                    }
-                }
-                summary['total_features'] += estimated_features
-        
-        return summary
-
-
-# 简化的API函数
-def create_aum_example(database: str = "dwegdata03000") -> AUMLongtailExample:
-    """
-    一键创建AUM长尾模型示例
-    
-    Args:
-        database: 数据库名称，默认为dwegdata03000
-        
-    Returns:
-        AUMLongtailExample实例
-    """
-    return AUMLongtailExample(database)
-
-
-def run_aum_example(feature_date: Optional[str] = None, 
-                   database: str = "dwegdata03000",
-                   output_path: str = "file:///nfsHome/aum_longtail") -> Dict:
-    """
-    一键运行AUM长尾模型特征工程
-    
-    Args:
-        feature_date: 特征日期，格式为YYYYMM，默认为当前月
-        database: 数据库名称，默认为dwegdata03000
-        output_path: 输出路径，默认为file:///nfsHome/aum_longtail
-        
-    Returns:
-        包含所有结果的字典
-        
-    Example:
-        >>> results = run_aum_example('202507')
-        >>> print(f"生成特征数: {len(results)}")
-    """
-    example = create_aum_example(database)
-    return example.run(feature_date, output_path)
-
-
-__all__ = [
-    'AUMLongtailExample',
-    'create_aum_example', 
-    'run_aum_example'
-]