alpha-dc-common 0.1.23__tar.gz

alpha_dc_common-0.1.23/.gitignore

@@ -0,0 +1,90 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+pip-wheel-metadata/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+.python-version
+
+# PEP 582; __pypackages__
+__pypackages__/
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# uv lock file
+uv.lock
+
+# Log files
+logs/
+*.log
+
+# Data directories (runtime generated data)
+data/
+
+# Project specific files
+services/worker_service/data_fetchers/StockClassifyUse_stock.xls

alpha_dc_common-0.1.23/PKG-INFO

@@ -0,0 +1,6 @@
+Metadata-Version: 2.4
+Name: alpha-dc-common
+Version: 0.1.23
+Summary: Shared schemas and utils for the datacenter project.
+Author-email: Hang GuangLiang <hanguangliang@alphaaidig.com>
+Requires-Dist: pydantic>=2.0

alpha_dc_common-0.1.23/pyproject.toml

@@ -0,0 +1,20 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "alpha-dc-common"
+version = "0.1.23"
+description = "Shared schemas and utils for the datacenter project."
+authors = [
+    {name = "Hang GuangLiang", email = "hanguangliang@alphaaidig.com"},
+]
+dependencies = [
+    "pydantic>=2.0",
+]
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/dc_common"]
+
+[tool.uv]
+index-url = "https://pypi.tuna.tsinghua.edu.cn/simple"

alpha_dc_common-0.1.23/pyproject.toml.backup

@@ -0,0 +1,20 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "dc_common"
+version = "0.1.23"
+description = "Shared schemas and utils for the datacenter project."
+authors = [
+    {name = "Hang GuangLiang", email = "hanguangliang@alphaaidig.com"},
+]
+dependencies = [
+    "pydantic>=2.0",
+]
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/dc_common"]
+
+[tool.uv]
+index-url = "https://pypi.tuna.tsinghua.edu.cn/simple"

alpha_dc_common-0.1.23/src/dc_common/__init__.py

@@ -0,0 +1,15 @@
+"""
+DataCenter Common - 公共库
+
+提供：
+- 数据源（Tushare、本地缓存等）
+- 工具函数
+- 核心组件
+"""
+from .data_sources import BaseDataSource, DataSourceStatus, DailyQuoteSource
+
+__all__ = [
+    'BaseDataSource',
+    'DataSourceStatus',
+    'DailyQuoteSource',
+]

alpha_dc_common-0.1.23/src/dc_common/core/logger.py

@@ -0,0 +1,48 @@
+import logging
+import sys
+import os
+from pathlib import Path
+
+def setup_logger(name: str = "datacenter") -> logging.Logger:
+    """设置并返回配置好的logger实例"""
+    
+    # 区分开发和生产环境
+    if os.environ.get('ENVIRONMENT') == 'production':
+        # 生产环境使用标准日志目录
+        log_dir = Path("/var/log/datacenter")
+        log_dir.mkdir(exist_ok=True, mode=0o755)
+    else:
+        # 开发环境使用相对路径
+        log_dir = Path("logs")
+        log_dir.mkdir(exist_ok=True)
+    
+    # 创建logger
+    logger = logging.getLogger(name)
+    logger.setLevel(logging.INFO)
+    
+    # 清除现有的handlers
+    logger.handlers.clear()
+    
+    # 创建formatter
+    formatter = logging.Formatter(
+        '%(asctime)s - %(name)s - %(levelname)s - %(message)s'
+    )
+    
+    # 文件handler
+    file_handler = logging.FileHandler(log_dir / "app.log", encoding="utf-8")
+    file_handler.setLevel(logging.INFO)
+    file_handler.setFormatter(formatter)
+    
+    # 控制台handler
+    console_handler = logging.StreamHandler(sys.stdout)
+    console_handler.setLevel(logging.INFO)
+    console_handler.setFormatter(formatter)
+    
+    # 添加handlers
+    logger.addHandler(file_handler)
+    logger.addHandler(console_handler)
+    
+    return logger
+
+# 创建默认logger实例
+logger = setup_logger()

alpha_dc_common-0.1.23/src/dc_common/data_sources/__init__.py

@@ -0,0 +1,16 @@
+"""
+数据源模块 - 提供各类数据源的实现
+
+本模块提供统一的数据源接口，支持：
+- Tushare 数据源
+- 本地缓存管理
+- 数据质量验证
+"""
+from .base_source import BaseDataSource, DataSourceStatus
+from .daily_quote_source import DailyQuoteSource
+
+__all__ = [
+    'BaseDataSource',
+    'DataSourceStatus',
+    'DailyQuoteSource'
+]

alpha_dc_common-0.1.23/src/dc_common/data_sources/base_source.py

@@ -0,0 +1,176 @@
+"""
+数据源基类定义
+"""
+from abc import ABC, abstractmethod
+from datetime import datetime, time
+from typing import Dict, Any, Optional
+import pandas as pd
+import logging
+from pathlib import Path
+from enum import Enum
+
+
+class DataSourceStatus(Enum):
+    """数据源状态"""
+    PENDING = "pending"        # 等待中
+    READY = "ready"           # 数据就绪
+    UPDATING = "updating"     # 更新中
+    FAILED = "failed"         # 更新失败
+    NOT_NEEDED = "not_needed" # 该日不需要更新
+
+
+class BaseDataSource(ABC):
+    """数据源基类
+
+    每个数据源有独立的调度策略和更新时间
+    """
+
+    def __init__(self, data_dir: str, logger: Optional[logging.Logger] = None):
+        self.data_dir = Path(data_dir)
+        self.data_dir.mkdir(parents=True, exist_ok=True)
+        self.logger = logger or logging.getLogger(__name__)
+
+    @property
+    @abstractmethod
+    def source_name(self) -> str:
+        """数据源唯一标识"""
+        pass
+
+    @property
+    @abstractmethod
+    def display_name(self) -> str:
+        """显示名称"""
+        pass
+
+    @property
+    @abstractmethod
+    def update_time(self) -> str:
+        """
+        期望更新时间
+        例如: "17:00", "09:30"
+        """
+        pass
+
+    @property
+    def update_delay_days(self) -> int:
+        """
+        更新延迟天数
+        0 = 当天更新
+        1 = 次日更新
+        """
+        return 0
+
+    @property
+    def priority(self) -> int:
+        """
+        优先级（数字越小越优先）
+        基础数据优先级高，衍生数据优先级低
+        """
+        return 100
+
+    @abstractmethod
+    def fetch_data(self, trade_date: str) -> pd.DataFrame:
+        """
+        获取数据
+
+        Args:
+            trade_date: 交易日期 (YYYYMMDD)
+
+        Returns:
+            数据 DataFrame
+        """
+        pass
+
+    @abstractmethod
+    def validate_data(self, df: pd.DataFrame) -> bool:
+        """
+        验证数据质量
+
+        Returns:
+            True=数据有效, False=数据无效
+        """
+        pass
+
+    def is_ready(self, trade_date: str) -> DataSourceStatus:
+        """
+        检查数据是否就绪
+
+        Args:
+            trade_date: 交易日期
+
+        Returns:
+            数据源状态
+        """
+        # 1. 检查本地是否已有数据
+        if self._has_local_data(trade_date):
+            return DataSourceStatus.READY
+
+        # 2. 检查是否到了更新时间
+        if not self._is_update_time():
+            return DataSourceStatus.PENDING
+
+        return DataSourceStatus.READY
+
+    def update(self, trade_date: str) -> Dict[str, Any]:
+        """
+        更新数据
+
+        Args:
+            trade_date: 交易日期
+
+        Returns:
+            更新结果字典
+        """
+        result = {
+            'source': self.source_name,
+            'trade_date': trade_date,
+            'status': 'unknown',
+            'rows': 0,
+            'message': ''
+        }
+
+        try:
+            self.logger.info(f"🔄 开始更新 {self.display_name}: {trade_date}")
+
+            # 1. 获取数据
+            df = self.fetch_data(trade_date)
+
+            # 2. 验证数据
+            if not self.validate_data(df):
+                result['status'] = 'failed'
+                result['message'] = '数据验证失败'
+                return result
+
+            # 3. 保存数据
+            self._save_data(df, trade_date)
+
+            result['status'] = 'success'
+            result['rows'] = len(df)
+            result['message'] = f'成功更新 {len(df)} 条数据'
+
+            self.logger.info(f"✅ {self.display_name} 更新完成: {len(df)} 条")
+
+        except Exception as e:
+            result['status'] = 'failed'
+            result['message'] = str(e)
+            self.logger.error(f"❌ {self.display_name} 更新失败: {e}")
+
+        return result
+
+    def _is_update_time(self) -> bool:
+        """检查是否到了更新时间"""
+        try:
+            hour, minute = map(int, self.update_time.split(':'))
+            target_time = time(hour, minute)
+            now = datetime.now().time()
+            return now >= target_time
+        except:
+            return True
+
+    def _has_local_data(self, trade_date: str) -> bool:
+        """检查本地是否已有数据（子类覆盖）"""
+        return False
+
+    def _save_data(self, df: pd.DataFrame, trade_date: str):
+        """保存数据（子类实现）"""
+        pass

alpha_dc_common-0.1.23/src/dc_common/data_sources/daily_quote_source.py

@@ -0,0 +1,195 @@
+"""
+日线行情数据源
+"""
+import os
+import pandas as pd
+from pathlib import Path
+from typing import Optional
+import logging
+
+from .base_source import BaseDataSource, DataSourceStatus
+
+
+class DailyQuoteSource(BaseDataSource):
+    """日线行情数据源
+
+    从 Tushare 获取日线行情数据
+    更新时间：每天 17:00
+    """
+
+    def __init__(
+        self,
+        data_dir: str,
+        tushare_token: Optional[str] = None,
+        logger: Optional[logging.Logger] = None
+    ):
+        super().__init__(data_dir, logger)
+
+        self.token = tushare_token or os.getenv('TUSHARE_TOKEN')
+        if not self.token:
+            raise ValueError("Tushare token 未配置，请设置 TUSHARE_TOKEN 环境变量")
+
+        # 缓存文件直接存储在 data_dir 下（不再添加 raw 子目录）
+        self.cache_file = self.data_dir / 'daily_quotes.parquet'
+        self.raw_cache_file = self.data_dir / 'daily_quotes_raw.parquet'
+
+        # 延迟初始化 Tushare（避免导入时立即连接）
+        self._pro = None
+
+    @property
+    def pro(self):
+        """延迟初始化 Tushare Pro API"""
+        if self._pro is None:
+            import tushare as ts
+            ts.set_token(self.token)
+            self._pro = ts.pro_api()
+        return self._pro
+
+    @property
+    def source_name(self) -> str:
+        return "daily_quotes"
+
+    @property
+    def display_name(self) -> str:
+        return "日线行情"
+
+    @property
+    def update_time(self) -> str:
+        return "17:00"
+
+    @property
+    def update_delay_days(self) -> int:
+        return 0
+
+    @property
+    def priority(self) -> int:
+        return 10
+
+    def fetch_data(self, trade_date: str) -> pd.DataFrame:
+        """从 Tushare 获取日线数据"""
+        self.logger.info(f"从 Tushare 获取 {trade_date} 的日线数据...")
+
+        df = self.pro.daily(
+            trade_date=trade_date
+        )
+
+        if df.empty:
+            self.logger.warning(f"{trade_date} 无数据（可能是非交易日）")
+        else:
+            self.logger.info(f"获取到 {len(df)} 条数据")
+
+        return df
+
+    def validate_data(self, df: pd.DataFrame) -> bool:
+        """验证数据质量"""
+        if df.empty:
+            return True  # 空数据也是有效的（可能是非交易日）
+
+        required_fields = ['ts_code', 'trade_date', 'close']
+        return all(field in df.columns for field in required_fields)
+
+    def _has_local_data(self, trade_date: str) -> bool:
+        """检查本地是否已有数据"""
+        if not self.raw_cache_file.exists():
+            return False
+
+        try:
+            df = pd.read_parquet(self.raw_cache_file)
+            return trade_date in df['trade_date'].values
+        except Exception as e:
+            self.logger.warning(f"检查本地数据失败: {e}")
+            return False
+
+    def _save_data(self, df: pd.DataFrame, trade_date: str):
+        """保存数据（追加模式）"""
+        # 确保目录存在
+        self.raw_cache_file.parent.mkdir(parents=True, exist_ok=True)
+        self.cache_file.parent.mkdir(parents=True, exist_ok=True)
+
+        # 保存原始长格式数据
+        if self.raw_cache_file.exists():
+            existing_df = pd.read_parquet(self.raw_cache_file)
+
+            # 移除旧的同日期数据（如果有）
+            existing_df = existing_df[existing_df['trade_date'] != trade_date]
+
+            # 合并数据
+            combined_df = pd.concat([existing_df, df], ignore_index=True)
+
+            # 去重（基于 trade_date 和 ts_code，保留后出现的）
+            combined_df = combined_df.drop_duplicates(
+                subset=['trade_date', 'ts_code'],
+                keep='last'
+            )
+        else:
+            combined_df = df
+
+        combined_df.to_parquet(self.raw_cache_file, compression='snappy')
+        self.logger.info(f"原始数据已保存: {self.raw_cache_file}")
+
+        # 转换并保存宽表格式（用于因子计算）
+        self._save_wide_format(combined_df)
+
+    def _save_wide_format(self, df: pd.DataFrame):
+        """转换为宽表格式并保存（增量追加）"""
+        if df.empty:
+            return
+
+        # 排序
+        df = df.sort_values(['trade_date', 'ts_code'])
+
+        # 如果文件已存在，进行增量追加
+        if self.cache_file.exists():
+            try:
+                # 读取已有数据
+                existing_df = pd.read_parquet(self.cache_file)
+
+                # 移除与新数据重复的日期
+                existing_dates = existing_df['trade_date'].unique()
+                new_dates = df['trade_date'].unique()
+
+                # 只保留新数据中没有的旧日期
+                dates_to_keep = set(existing_dates) - set(new_dates)
+                if dates_to_keep:
+                    existing_df = existing_df[existing_df['trade_date'].isin(dates_to_keep)]
+                else:
+                    existing_df = pd.DataFrame()
+
+                # 合并数据
+                if not existing_df.empty:
+                    combined_df = pd.concat([existing_df, df], ignore_index=True)
+                else:
+                    combined_df = df
+
+                # 去重（基于 trade_date 和 ts_code，保留后出现的）
+                combined_df = combined_df.drop_duplicates(
+                    subset=['trade_date', 'ts_code'],
+                    keep='last'
+                )
+
+                # 排序
+                combined_df = combined_df.sort_values(['trade_date', 'ts_code'])
+
+                # 保存
+                combined_df.to_parquet(self.cache_file, compression='snappy', index=False)
+
+                old_len = len(existing_df) if dates_to_keep else 0
+                new_len = len(combined_df)
+                added_rows = new_len - old_len
+
+                self.logger.info(
+                    f"宽表数据已增量更新: {self.cache_file} "
+                    f"(原有: {old_len}, 新增: {added_rows}, 总计: {new_len})"
+                )
+            except Exception as e:
+                self.logger.warning(f"增量更新失败: {e}，执行覆盖保存")
+                df.to_parquet(self.cache_file, compression='snappy', index=False)
+                self.logger.info(f"宽表数据已保存: {self.cache_file}")
+        else:
+            # 首次保存（先去重）
+            df = df.drop_duplicates(subset=['trade_date', 'ts_code'], keep='last')
+            df.to_parquet(self.cache_file, compression='snappy', index=False)
+            self.logger.info(f"宽表数据已保存: {self.cache_file}")
+
+        self.logger.info(f"数据范围: {df['trade_date'].min()} ~ {df['trade_date'].max()}")
+        self.logger.info(f"股票数量: {df['ts_code'].nunique()}")

alpha_dc_common-0.1.23/src/dc_common/schemas/__init__.py

@@ -0,0 +1,46 @@
+"""
+数据中心通用schemas模块
+"""
+from .base import BaseResponse, PaginatedResponse, PaginationInfo
+from .results import ResultType, BaseResult, TableResult, PageResult, DictResult, SingleResult
+from .a_stock import AStock
+from .hk_stock import HKStock
+from .index_basic import IndexBasic
+from .index_company import IndexCompany
+from .margin_account import MarginAccount
+from .margin_analysis import MarginAnalysis
+from .margin_detail import MarginDetail
+
+from .hs_industry import HSIndustry, HSIndustryCategory
+from .hs_industry_company import HSIndustryCompany
+from .sw_industry import SWIndustry
+from .sw_industry_company import SWIndustryCompany
+from .index_daily import IndexDaily
+from .sw_index_daily import SWIndexDaily
+
+__all__ = [
+    "BaseResponse",
+    "PaginatedResponse",
+    "PaginationInfo",
+    "ResultType",
+    "BaseResult",
+    "TableResult",
+    "PageResult",
+    "DictResult",
+    "SingleResult",
+    "AStock",
+    "HKStock",
+    "IndexBasic",
+    "IndexCompany",
+    "MarginAccount",
+    "MarginAnalysis",
+    "MarginDetail",
+
+    "HSIndustry",
+    "HSIndustryCategory",
+    "HSIndustryCompany",
+    "SWIndustry",
+    "SWIndustryCompany",
+    "IndexDaily",
+    "SWIndexDaily",
+]

alpha_dc_common-0.1.23/src/dc_common/schemas/a_stock.py

@@ -0,0 +1,24 @@
+"""
+A股相关的Pydantic模型
+"""
+from pydantic import BaseModel, Field
+from typing import Optional
+from datetime import datetime
+
+class AStockBase(BaseModel):
+    stock_code: str = Field(..., description="股票代码")
+    stock_name: str = Field(..., description="股票名称")
+    exchange: str = Field(..., description="交易所(SH/SZ/BJ)")
+    list_status: Optional[str] = Field(None, description="上市状态(L上市/D退市/P暂停上市)")
+    list_date: Optional[datetime] = Field(None, description="上市日期")
+    curr_type: Optional[str] = Field(None, description="交易货币(CNY)")
+    market: Optional[str] = Field(None, description="市场类型(主板/创业板/科创板/北交所)")
+    ric_code: Optional[str] = Field(None, description="路透代码")
+
+class AStock(AStockBase):
+    id: int
+    created_at: datetime
+    updated_at: datetime
+
+    class Config:
+        from_attributes = True