rquote 0.3.9__py3-none-any.whl → 0.4.1__py3-none-any.whl

rquote/__init__.py

@@ -35,6 +35,11 @@ from .plots import PlotUtils
 from . import config
 from . import exceptions
 from .cache import MemoryCache, Cache
+# 尝试导入持久化缓存（可选依赖）
+try:
+    from .cache import PersistentCache
+except ImportError:
+    PersistentCache = None
 from .utils.http import HTTPClient
 
 
@@ -93,5 +98,6 @@ __all__ = [
     'exceptions',
     'MemoryCache',
     'Cache',
+    'PersistentCache',
     'HTTPClient',
 ]

rquote/cache/__init__.py

@@ -5,5 +5,10 @@
 from .base import Cache
 from .memory import MemoryCache
 
-__all__ = ['Cache', 'MemoryCache']
+# 尝试导入持久化缓存（可选依赖）
+try:
+    from .persistent import PersistentCache
+    __all__ = ['Cache', 'MemoryCache', 'PersistentCache']
+except ImportError:
+    __all__ = ['Cache', 'MemoryCache']
 

rquote/cache/persistent.py

@@ -0,0 +1,507 @@
+# -*- coding: utf-8 -*-
+"""
+持久化缓存实现
+"""
+import os
+import time
+from pathlib import Path
+from typing import Optional, Any, Tuple
+import pandas as pd
+from .base import Cache
+
+# 导入日志
+try:
+    from ..utils.logging import logger
+except ImportError:
+    import logging
+    logger = logging.getLogger(__name__)
+
+# 尝试导入 duckdb（可选依赖）
+try:
+    import duckdb
+    DUCKDB_AVAILABLE = True
+except ImportError:
+    DUCKDB_AVAILABLE = False
+    duckdb = None
+
+
+class PersistentCache(Cache):
+    """持久化缓存实现，使用 duckdb 或文件系统存储数据"""
+    
+    def __init__(self, db_path: Optional[str] = None, use_duckdb: bool = True, ttl: Optional[int] = None):
+        """
+        初始化持久化缓存
+        
+        Args:
+            db_path: 数据库文件路径，默认为 ~/.rquote/cache.db
+            use_duckdb: 是否使用 duckdb（如果可用），否则使用 pickle 文件
+            ttl: 默认过期时间（秒），None 表示不过期
+        """
+        self.use_duckdb = use_duckdb and DUCKDB_AVAILABLE
+        self.ttl = ttl
+        
+        if db_path is None:
+            # 默认路径：~/.rquote/cache.db 或 ~/.rquote/cache.pkl
+            home = Path.home()
+            cache_dir = home / '.rquote'
+            cache_dir.mkdir(exist_ok=True)
+            if self.use_duckdb:
+                db_path = str(cache_dir / 'cache.db')
+            else:
+                db_path = str(cache_dir / 'cache.pkl')
+        
+        self.db_path = db_path
+        
+        if self.use_duckdb:
+            self._init_duckdb()
+        else:
+            self._init_pickle()
+    
+    def _init_duckdb(self):
+        """初始化 duckdb 数据库"""
+        self.conn = duckdb.connect(self.db_path)
+        # 创建缓存表
+        self.conn.execute("""
+            CREATE TABLE IF NOT EXISTS cache_data (
+                cache_key TEXT PRIMARY KEY,
+                symbol TEXT NOT NULL,
+                name TEXT,
+                data BLOB,
+                earliest_date TEXT,
+                latest_date TEXT,
+                freq TEXT,
+                fq TEXT,
+                updated_at TIMESTAMP,
+                expire_at TIMESTAMP
+            )
+        """)
+        self.conn.execute("""
+            CREATE INDEX IF NOT EXISTS idx_symbol_freq_fq 
+            ON cache_data(symbol, freq, fq)
+        """)
+    
+    def _init_pickle(self):
+        """初始化 pickle 存储"""
+        import pickle
+        self.pickle = pickle
+        if os.path.exists(self.db_path):
+            try:
+                with open(self.db_path, 'rb') as f:
+                    self._cache_data = self.pickle.load(f)
+            except:
+                self._cache_data = {}
+        else:
+            self._cache_data = {}
+    
+    def _save_pickle(self):
+        """保存 pickle 数据"""
+        import pickle
+        with open(self.db_path, 'wb') as f:
+            self.pickle.dump(self._cache_data, f)
+    
+    def _extract_key_parts(self, key: str) -> Tuple[str, str, str, str, str]:
+        """
+        从完整 key 中提取各部分
+        
+        Args:
+            key: 完整 key，格式如 "symbol:sdate:edate:freq:days:fq"
+        
+        Returns:
+            (symbol, sdate, edate, freq, fq)
+        """
+        parts = key.split(':')
+        if len(parts) >= 6:
+            return parts[0], parts[1], parts[2], parts[3], parts[5]
+        elif len(parts) >= 4:
+            return parts[0], parts[1] if len(parts) > 1 else '', parts[2] if len(parts) > 2 else '', parts[3], parts[4] if len(parts) > 4 else 'qfq'
+        else:
+            return parts[0] if parts else '', '', '', 'day', 'qfq'
+    
+    def _get_base_key(self, symbol: str, freq: str, fq: str) -> str:
+        """生成基础 key（不包含日期）"""
+        return f"{symbol}:{freq}:{fq}"
+    
+    def _parse_date(self, date_str: str) -> Optional[pd.Timestamp]:
+        """解析日期字符串"""
+        if not date_str:
+            return None
+        try:
+            return pd.to_datetime(date_str)
+        except:
+            return None
+    
+    def _get_dataframe_date_range(self, df: pd.DataFrame) -> Tuple[Optional[pd.Timestamp], Optional[pd.Timestamp]]:
+        """获取 DataFrame 的日期范围"""
+        if df.empty:
+            return None, None
+        
+        # 如果索引不是 DatetimeIndex，尝试转换
+        if not isinstance(df.index, pd.DatetimeIndex):
+            try:
+                # 尝试转换为 DatetimeIndex
+                index = pd.to_datetime(df.index)
+                if len(index) > 0:
+                    return index.min(), index.max()
+            except (ValueError, TypeError):
+                pass
+            return None, None
+        
+        return df.index.min(), df.index.max()
+    
+    def _filter_dataframe_by_date(self, df: pd.DataFrame, sdate: Optional[str] = None, 
+                                   edate: Optional[str] = None) -> pd.DataFrame:
+        """根据日期范围过滤 DataFrame"""
+        if df.empty:
+            return df
+        
+        if not isinstance(df.index, pd.DatetimeIndex):
+            return df
+        
+        start_date = self._parse_date(sdate) if sdate else None
+        end_date = self._parse_date(edate) if edate else None
+        
+        if start_date is not None and end_date is not None:
+            mask = (df.index >= start_date) & (df.index <= end_date)
+            return df[mask]
+        elif start_date is not None:
+            return df[df.index >= start_date]
+        elif end_date is not None:
+            return df[df.index <= end_date]
+        else:
+            return df
+    
+    def _merge_dataframes(self, df1: pd.DataFrame, df2: pd.DataFrame) -> pd.DataFrame:
+        """合并两个 DataFrame，去重并排序"""
+        if df1.empty:
+            return df2
+        if df2.empty:
+            return df1
+        
+        # 合并并去重
+        combined = pd.concat([df1, df2])
+        combined = combined[~combined.index.duplicated(keep='last')]
+        combined = combined.sort_index()
+        return combined
+    
+    def get(self, key: str, sdate: Optional[str] = None, edate: Optional[str] = None) -> Optional[Any]:
+        """
+        获取缓存数据
+        
+        Args:
+            key: 缓存 key，可以是完整格式 "symbol:sdate:edate:freq:days:fq" 
+                 或 base_key 格式 "symbol:freq:fq"
+            sdate: 开始日期（可选，如果 key 是 base_key 格式则必须提供）
+            edate: 结束日期（可选，如果 key 是 base_key 格式则必须提供）
+        
+        Returns:
+            (symbol, name, DataFrame) 或 None
+        """
+        # 判断 key 格式：如果是 base_key 格式（只有3部分），使用参数中的日期
+        parts = key.split(':')
+        if len(parts) == 3:
+            # base_key 格式：symbol:freq:fq
+            symbol, freq, fq = parts
+            base_key = key
+            # 使用参数中的日期，如果没有则使用空字符串
+            sdate = sdate or ''
+            edate = edate or ''
+        else:
+            # 完整 key 格式：symbol:sdate:edate:freq:days:fq
+            symbol, sdate_from_key, edate_from_key, freq, fq = self._extract_key_parts(key)
+            base_key = self._get_base_key(symbol, freq, fq)
+            # 优先使用参数中的日期，如果没有则使用 key 中的日期
+            sdate = sdate if sdate is not None else sdate_from_key
+            edate = edate if edate is not None else edate_from_key
+        
+        logger.info(f"[CACHE GET] key={key}, base_key={base_key}, sdate={sdate}, edate={edate}")
+        
+        if self.use_duckdb:
+            result = self._get_duckdb(base_key, symbol, sdate, edate, freq, fq)
+        else:
+            result = self._get_pickle(base_key, symbol, sdate, edate, freq, fq)
+        
+        if result:
+            _, _, df = result
+            logger.info(f"[CACHE HIT] key={key}, 返回数据行数={len(df)}, 日期范围={df.index.min()} 到 {df.index.max()}")
+        else:
+            logger.info(f"[CACHE MISS] key={key}, 缓存中无数据")
+        
+        return result
+    
+    def _get_duckdb(self, base_key: str, symbol: str, sdate: str, edate: str, 
+                    freq: str, fq: str) -> Optional[Tuple[str, str, pd.DataFrame]]:
+        """从 duckdb 获取数据"""
+        result = self.conn.execute("""
+            SELECT name, data, earliest_date, latest_date, expire_at
+            FROM cache_data
+            WHERE cache_key = ?
+        """, [base_key]).fetchone()
+        
+        if not result:
+            return None
+        
+        name, data_blob, earliest_date, latest_date, expire_at = result
+        
+        # 检查过期
+        if self.ttl and expire_at:
+            expire_ts = pd.to_datetime(expire_at)
+            if pd.Timestamp.now() > expire_ts:
+                self.delete(base_key)
+                return None
+        
+        # 反序列化 DataFrame
+        import pickle
+        df = pickle.loads(data_blob)
+        
+        # 获取缓存数据的日期范围
+        cached_earliest = self._parse_date(earliest_date)
+        cached_latest = self._parse_date(latest_date)
+        
+        # 如果请求的日期范围完全在缓存范围内，直接返回过滤后的数据
+        request_sdate = self._parse_date(sdate) if sdate else None
+        request_edate = self._parse_date(edate) if edate else None
+        
+        # 检查是否有重叠
+        if request_edate and cached_earliest and request_edate < cached_earliest:
+            # 请求的结束日期早于缓存的最早日期，无重叠
+            return None
+        if request_sdate and cached_latest and request_sdate > cached_latest:
+            # 请求的开始日期晚于缓存的最晚日期，无重叠
+            return None
+        
+        # 有重叠，返回缓存中可用的部分数据
+        # 计算实际可用的日期范围
+        actual_sdate = max(request_sdate, cached_earliest) if request_sdate and cached_earliest else (request_sdate or cached_earliest)
+        actual_edate = min(request_edate, cached_latest) if request_edate and cached_latest else (request_edate or cached_latest)
+        
+        # 过滤数据
+        filtered_df = self._filter_dataframe_by_date(
+            df, 
+            actual_sdate.strftime('%Y-%m-%d') if actual_sdate else None,
+            actual_edate.strftime('%Y-%m-%d') if actual_edate else None
+        )
+        
+        if filtered_df.empty:
+            return None
+        
+        # 确保索引是 DatetimeIndex
+        if not isinstance(filtered_df.index, pd.DatetimeIndex):
+            try:
+                filtered_df.index = pd.to_datetime(filtered_df.index)
+            except (ValueError, TypeError):
+                pass  # 如果转换失败，保持原样
+        
+        return (symbol, name, filtered_df)
+    
+    def _get_pickle(self, base_key: str, symbol: str, sdate: str, edate: str,
+                    freq: str, fq: str) -> Optional[Tuple[str, str, pd.DataFrame]]:
+        """从 pickle 文件获取数据"""
+        if base_key not in self._cache_data:
+            return None
+        
+        cache_entry = self._cache_data[base_key]
+        
+        # 检查过期
+        if self.ttl and 'expire_at' in cache_entry:
+            expire_ts = cache_entry['expire_at']
+            if pd.Timestamp.now() > expire_ts:
+                del self._cache_data[base_key]
+                self._save_pickle()
+                return None
+        
+        df = cache_entry['data']
+        name = cache_entry.get('name', '')
+        earliest_date = cache_entry.get('earliest_date')
+        latest_date = cache_entry.get('latest_date')
+        
+        # 获取缓存数据的日期范围
+        cached_earliest = self._parse_date(earliest_date)
+        cached_latest = self._parse_date(latest_date)
+        
+        # 如果请求的日期范围完全在缓存范围内，直接返回过滤后的数据
+        request_sdate = self._parse_date(sdate) if sdate else None
+        request_edate = self._parse_date(edate) if edate else None
+        
+        # 检查是否有重叠
+        if request_edate and cached_earliest and request_edate < cached_earliest:
+            # 请求的结束日期早于缓存的最早日期，无重叠
+            return None
+        if request_sdate and cached_latest and request_sdate > cached_latest:
+            # 请求的开始日期晚于缓存的最晚日期，无重叠
+            return None
+        
+        # 有重叠，返回缓存中可用的部分数据
+        # 计算实际可用的日期范围
+        actual_sdate = max(request_sdate, cached_earliest) if request_sdate and cached_earliest else (request_sdate or cached_earliest)
+        actual_edate = min(request_edate, cached_latest) if request_edate and cached_latest else (request_edate or cached_latest)
+        
+        # 过滤数据
+        filtered_df = self._filter_dataframe_by_date(
+            df,
+            actual_sdate.strftime('%Y-%m-%d') if actual_sdate else None,
+            actual_edate.strftime('%Y-%m-%d') if actual_edate else None
+        )
+        
+        if filtered_df.empty:
+            return None
+        
+        # 确保索引是 DatetimeIndex
+        if not isinstance(filtered_df.index, pd.DatetimeIndex):
+            try:
+                filtered_df.index = pd.to_datetime(filtered_df.index)
+            except (ValueError, TypeError):
+                pass  # 如果转换失败，保持原样
+        
+        return (symbol, name, filtered_df)
+    
+    def put(self, key: str, value: Any, ttl: Optional[int] = None) -> None:
+        """
+        存储缓存数据
+        
+        Args:
+            key: 缓存 key，可以是完整格式 "symbol:sdate:edate:freq:days:fq" 
+                 或 base_key 格式 "symbol:freq:fq"（推荐使用 base_key）
+            value: (symbol, name, DataFrame) 元组
+            ttl: 过期时间（秒）
+        """
+        if not isinstance(value, tuple) or len(value) != 3:
+            return
+        
+        symbol, name, df = value
+        if not isinstance(df, pd.DataFrame) or df.empty:
+            return
+        
+        logger.info(f"[CACHE PUT] key={key}, 数据行数={len(df)}, 日期范围={df.index.min() if not df.empty else 'N/A'} 到 {df.index.max() if not df.empty else 'N/A'}")
+        
+        # 确保索引是 DatetimeIndex（用于正确获取日期范围）
+        if not isinstance(df.index, pd.DatetimeIndex):
+            try:
+                df.index = pd.to_datetime(df.index)
+            except (ValueError, TypeError):
+                pass  # 如果转换失败，继续处理（_get_dataframe_date_range 会处理）
+        
+        # 判断 key 格式：如果是 base_key 格式（只有3部分），直接使用
+        parts = key.split(':')
+        if len(parts) == 3:
+            # base_key 格式：symbol:freq:fq
+            base_key = key
+            freq, fq = parts[1], parts[2]
+        else:
+            # 完整 key 格式：symbol:sdate:edate:freq:days:fq
+            _, _, _, freq, fq = self._extract_key_parts(key)
+            base_key = self._get_base_key(symbol, freq, fq)
+        
+        # 尝试从基础 key 获取完整数据并合并
+        existing = self._get_raw(base_key)
+        if existing:
+            _, existing_name, existing_df = existing
+            # 使用新数据的 name（如果有）
+            if not name:
+                name = existing_name
+            # 合并数据
+            df = self._merge_dataframes(existing_df, df)
+            # 合并后再次确保索引是 DatetimeIndex
+            if not isinstance(df.index, pd.DatetimeIndex):
+                try:
+                    df.index = pd.to_datetime(df.index)
+                except (ValueError, TypeError):
+                    pass
+        
+        # 获取日期范围
+        earliest_date, latest_date = self._get_dataframe_date_range(df)
+        earliest_str = earliest_date.strftime('%Y-%m-%d') if earliest_date else None
+        latest_str = latest_date.strftime('%Y-%m-%d') if latest_date else None
+        
+        # 计算过期时间
+        expire_at = None
+        if ttl or self.ttl:
+            expire_seconds = (ttl or self.ttl)
+            expire_at = pd.Timestamp.now() + pd.Timedelta(seconds=expire_seconds)
+        
+        if self.use_duckdb:
+            self._put_duckdb(base_key, symbol, name, df, earliest_str, latest_str, freq, fq, expire_at)
+        else:
+            self._put_pickle(base_key, symbol, name, df, earliest_str, latest_str, freq, fq, expire_at)
+        
+        logger.info(f"[CACHE PUT] 存储完成, base_key={base_key}, 日期范围={earliest_str} 到 {latest_str}")
+    
+    def _get_raw(self, base_key: str) -> Optional[Tuple[str, str, pd.DataFrame]]:
+        """获取原始数据（不进行日期过滤）"""
+        if self.use_duckdb:
+            result = self.conn.execute("""
+                SELECT name, data
+                FROM cache_data
+                WHERE cache_key = ?
+            """, [base_key]).fetchone()
+            
+            if not result:
+                return None
+            
+            import pickle
+            df = pickle.loads(result[1])
+            return (base_key.split(':')[0], result[0], df)
+        else:
+            if base_key not in self._cache_data:
+                return None
+            cache_entry = self._cache_data[base_key]
+            return (base_key.split(':')[0], cache_entry.get('name', ''), cache_entry['data'])
+    
+    def _put_duckdb(self, base_key: str, symbol: str, name: str, df: pd.DataFrame,
+                     earliest_date: Optional[str], latest_date: Optional[str],
+                     freq: str, fq: str, expire_at: Optional[pd.Timestamp]):
+        """存储到 duckdb"""
+        import pickle
+        data_blob = pickle.dumps(df)
+        
+        self.conn.execute("""
+            INSERT OR REPLACE INTO cache_data 
+            (cache_key, symbol, name, data, earliest_date, latest_date, freq, fq, updated_at, expire_at)
+            VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+        """, [base_key, symbol, name, data_blob, earliest_date, latest_date, freq, fq,
+              pd.Timestamp.now(), expire_at])
+        self.conn.commit()
+    
+    def _put_pickle(self, base_key: str, symbol: str, name: str, df: pd.DataFrame,
+                    earliest_date: Optional[str], latest_date: Optional[str],
+                    freq: str, fq: str, expire_at: Optional[pd.Timestamp]):
+        """存储到 pickle 文件"""
+        self._cache_data[base_key] = {
+            'symbol': symbol,
+            'name': name,
+            'data': df,
+            'earliest_date': earliest_date,
+            'latest_date': latest_date,
+            'freq': freq,
+            'fq': fq,
+            'updated_at': pd.Timestamp.now(),
+            'expire_at': expire_at
+        }
+        self._save_pickle()
+    
+    def delete(self, key: str) -> None:
+        """删除缓存"""
+        symbol, _, _, freq, fq = self._extract_key_parts(key)
+        base_key = self._get_base_key(symbol, freq, fq)
+        
+        if self.use_duckdb:
+            self.conn.execute("DELETE FROM cache_data WHERE cache_key = ?", [base_key])
+            self.conn.commit()
+        else:
+            if base_key in self._cache_data:
+                del self._cache_data[base_key]
+                self._save_pickle()
+    
+    def clear(self) -> None:
+        """清空所有缓存"""
+        if self.use_duckdb:
+            self.conn.execute("DELETE FROM cache_data")
+            self.conn.commit()
+        else:
+            self._cache_data.clear()
+            self._save_pickle()
+    
+    def close(self):
+        """关闭连接"""
+        if self.use_duckdb:
+            self.conn.close()
+

rquote/markets/base.py

@@ -5,9 +5,23 @@
 from abc import ABC, abstractmethod
 from typing import Tuple, Optional
 import pandas as pd
+from datetime import datetime, timedelta
 from ..cache import Cache
 from ..data_sources.base import DataSource
 
+# 导入日志
+try:
+    from ..utils.logging import logger
+except ImportError:
+    import logging
+    logger = logging.getLogger(__name__)
+
+# 尝试导入持久化缓存（可选依赖）
+try:
+    from ..cache.persistent import PersistentCache
+except ImportError:
+    PersistentCache = None
+
 
 class Market(ABC):
     """市场基类"""
@@ -23,21 +37,77 @@ class Market(ABC):
         self.data_source = data_source
         self.cache = cache
     
-    @abstractmethod
     def get_price(self, symbol: str, sdate: str = '', edate: str = '', 
                   freq: str = 'day', days: int = 320, fq: str = 'qfq') -> Tuple[str, str, pd.DataFrame]:
-        """获取价格数据"""
-        pass
+        """
+        获取价格数据（模板方法，统一处理缓存逻辑）
+        
+        子类可以重写此方法以处理特殊情况，但建议调用 super().get_price() 来使用缓存功能
+        或者实现 _fetch_price_data 方法，让基类自动处理缓存
+        """
+        symbol = self.normalize_symbol(symbol)
+        cache_key = f"{symbol}:{sdate}:{edate}:{freq}:{days}:{fq}"
+        
+        # 如果是持久化缓存且是日级别数据，使用智能扩展逻辑
+        if PersistentCache and isinstance(self.cache, PersistentCache) and freq == 'day':
+            return self._get_price_with_persistent_cache(
+                symbol, sdate, edate, freq, days, fq,
+                lambda s, sd, ed, f, d, fq_param: self._fetch_price_data(s, sd, ed, f, d, fq_param)
+            )
+        
+        # 普通缓存逻辑
+        cached = self._get_cached(cache_key)
+        if cached:
+            return cached
+        
+        # 从数据源获取
+        result = self._fetch_price_data(symbol, sdate, edate, freq, days, fq)
+        self._put_cache(cache_key, result)
+        return result
+    
+    def _fetch_price_data(self, symbol: str, sdate: str = '', edate: str = '', 
+                          freq: str = 'day', days: int = 320, fq: str = 'qfq') -> Tuple[str, str, pd.DataFrame]:
+        """
+        从数据源获取价格数据（子类需要实现）
+        
+        这个方法只负责获取数据，不处理缓存。缓存逻辑由 get_price 统一处理。
+        
+        Args:
+            symbol: 股票代码（已标准化）
+            sdate: 开始日期
+            edate: 结束日期
+            freq: 频率
+            days: 天数
+            fq: 复权方式
+        
+        Returns:
+            (symbol, name, DataFrame)
+        """
+        # 默认实现：子类应该重写此方法
+        return (symbol, '', pd.DataFrame())
     
     @abstractmethod
     def normalize_symbol(self, symbol: str) -> str:
         """标准化股票代码"""
         pass
     
-    def _get_cached(self, key: str) -> Optional[Tuple[str, str, pd.DataFrame]]:
+    def _get_cached(self, key: str, sdate: str = '', edate: str = '') -> Optional[Tuple[str, str, pd.DataFrame]]:
         """从缓存获取数据"""
         if self.cache:
-            cached = self.cache.get(key)
+            # 如果是 PersistentCache，使用 base_key + 日期参数的方式
+            if PersistentCache and isinstance(self.cache, PersistentCache):
+                # 从完整 key 中提取 base_key
+                parts = key.split(':')
+                if len(parts) >= 3:
+                    symbol = parts[0]
+                    freq = parts[3] if len(parts) > 3 else 'day'
+                    fq = parts[5] if len(parts) > 5 else 'qfq'
+                    base_key = f"{symbol}:{freq}:{fq}"
+                    cached = self.cache.get(base_key, sdate=sdate, edate=edate)
+                else:
+                    cached = self.cache.get(key)
+            else:
+                cached = self.cache.get(key)
             if cached:
                 return cached
         return None
@@ -45,5 +115,128 @@ class Market(ABC):
     def _put_cache(self, key: str, value: Tuple[str, str, pd.DataFrame]) -> None:
         """存入缓存"""
         if self.cache:
-            self.cache.put(key, value)
+            # 如果是 PersistentCache，使用 base_key 存储
+            if PersistentCache and isinstance(self.cache, PersistentCache):
+                # 从完整 key 中提取 base_key
+                parts = key.split(':')
+                if len(parts) >= 3:
+                    symbol = parts[0]
+                    freq = parts[3] if len(parts) > 3 else 'day'
+                    fq = parts[5] if len(parts) > 5 else 'qfq'
+                    base_key = f"{symbol}:{freq}:{fq}"
+                    self.cache.put(base_key, value)
+                else:
+                    self.cache.put(key, value)
+            else:
+                self.cache.put(key, value)
+    
+    def _get_price_with_persistent_cache(self, symbol: str, sdate: str, edate: str,
+                                          freq: str, days: int, fq: str,
+                                          fetch_func) -> Tuple[str, str, pd.DataFrame]:
+        """
+        使用持久化缓存的智能扩展逻辑
+        
+        当请求的 edate 不在缓存中时，从缓存的最新日期向前扩展到 edate
+        当请求的 sdate 不在缓存中时，从缓存的最早日期向后扩展到 sdate
+        """
+        cache_key = f"{symbol}:{sdate}:{edate}:{freq}:{days}:{fq}"
+        
+        logger.info(f"[PRICE GET] symbol={symbol}, sdate={sdate}, edate={edate}, freq={freq}, cache_key={cache_key}")
+        
+        # 尝试从缓存获取（传入日期参数，PersistentCache 会使用 base_key + 日期参数）
+        cached = self._get_cached(cache_key, sdate=sdate, edate=edate)
+        if cached:
+            _, name, cached_df = cached
+            logger.info(f"[PRICE CACHE HIT] symbol={symbol}, 缓存数据行数={len(cached_df)}, 日期范围={cached_df.index.min() if not cached_df.empty else 'N/A'} 到 {cached_df.index.max() if not cached_df.empty else 'N/A'}")
+            
+            # 检查是否需要扩展
+            if cached_df.empty or not isinstance(cached_df.index, pd.DatetimeIndex):
+                # 缓存为空或索引不是日期，直接获取新数据
+                logger.info(f"[PRICE FETCH] 缓存数据无效，从网络获取 symbol={symbol}, sdate={sdate}, edate={edate}")
+                result = fetch_func(symbol, sdate, edate, freq, days, fq)
+                self._put_cache(cache_key, result)
+                return result
+            
+            cached_earliest = cached_df.index.min()
+            cached_latest = cached_df.index.max()
+            request_sdate = pd.to_datetime(sdate) if sdate else None
+            request_edate = pd.to_datetime(edate) if edate else None
+            
+            need_extend_forward = False  # 需要向前扩展（更新日期）
+            need_extend_backward = False  # 需要向后扩展（更早日期）
+            extend_sdate = sdate
+            extend_edate = edate
+            
+            # 检查是否需要向前扩展
+            if request_edate and request_edate > cached_latest:
+                need_extend_forward = True
+                # 从缓存的最新日期+1天开始，扩展到请求的 edate
+                extend_sdate = (cached_latest + pd.Timedelta(days=1)).strftime('%Y-%m-%d')
+                extend_edate = edate
+            
+            # 检查是否需要向后扩展
+            if request_sdate and request_sdate < cached_earliest:
+                need_extend_backward = True
+                # 从请求的 sdate 开始，扩展到缓存的最早日期-1天
+                extend_sdate = sdate
+                extend_edate = (cached_earliest - pd.Timedelta(days=1)).strftime('%Y-%m-%d')
+            
+            # 如果需要扩展，获取缺失的数据
+            if need_extend_forward or need_extend_backward:
+                logger.info(f"[PRICE EXTEND] 需要扩展数据, symbol={symbol}, extend_sdate={extend_sdate}, extend_edate={extend_edate}, need_forward={need_extend_forward}, need_backward={need_extend_backward}")
+                # 获取扩展的数据
+                extended_result = fetch_func(symbol, extend_sdate, extend_edate, freq, days, fq)
+                _, _, extended_df = extended_result
+                logger.info(f"[PRICE FETCH] 从网络获取扩展数据, 数据行数={len(extended_df)}")
+                
+                if not extended_df.empty:
+                    # 确保两个 DataFrame 的索引都是 DatetimeIndex
+                    if not isinstance(cached_df.index, pd.DatetimeIndex):
+                        try:
+                            cached_df.index = pd.to_datetime(cached_df.index)
+                        except (ValueError, TypeError):
+                            pass
+                    if not isinstance(extended_df.index, pd.DatetimeIndex):
+                        try:
+                            extended_df.index = pd.to_datetime(extended_df.index)
+                        except (ValueError, TypeError):
+                            pass
+                    
+                    # 合并数据
+                    merged_df = pd.concat([cached_df, extended_df])
+                    merged_df = merged_df[~merged_df.index.duplicated(keep='last')]
+                    merged_df = merged_df.sort_index()
+                    
+                    # 过滤到请求的日期范围
+                    if request_sdate or request_edate:
+                        if request_sdate and request_edate:
+                            mask = (merged_df.index >= request_sdate) & (merged_df.index <= request_edate)
+                        elif request_sdate:
+                            mask = merged_df.index >= request_sdate
+                        else:
+                            mask = merged_df.index <= request_edate
+                        merged_df = merged_df[mask]
+                    
+                    result = (symbol, name, merged_df)
+                    # 更新缓存（使用原始 key，PersistentCache 会智能合并）
+                    self._put_cache(cache_key, result)
+                    return result
+            
+            # 不需要扩展，直接返回缓存的数据
+            # 注意：PersistentCache.get() 已经根据请求的日期范围进行了过滤，
+            # 返回的数据已经是过滤后的，不需要再次过滤
+            logger.info(f"[PRICE RETURN] 直接返回缓存数据, symbol={symbol}, 数据行数={len(cached_df)}")
+            return (symbol, name, cached_df)
+        
+        # 缓存未命中，直接获取
+        if fetch_func:
+            logger.info(f"[PRICE FETCH] 缓存未命中，从网络获取 symbol={symbol}, sdate={sdate}, edate={edate}")
+            result = fetch_func(symbol, sdate, edate, freq, days, fq)
+            _, _, df = result
+            logger.info(f"[PRICE FETCH] 网络获取完成, 数据行数={len(df)}, 准备存储到缓存")
+            self._put_cache(cache_key, result)
+            return result
+        else:
+            # 如果没有提供 fetch_func，返回空数据
+            return (symbol, '', pd.DataFrame())
 

rquote/markets/cn_stock.py

@@ -27,21 +27,20 @@ class CNStockMarket(Market):
         """获取A股价格数据"""
         symbol = self.normalize_symbol(symbol)
         
-        # 检查缓存
-        cache_key = f"{symbol}:{sdate}:{edate}:{freq}:{days}:{fq}"
-        cached = self._get_cached(cache_key)
-        if cached:
-            return cached
-        
-        # 特殊处理BK（板块）代码
+        # 特殊处理BK（板块）代码（不使用缓存）
         if symbol[:2] == 'BK':
             return self._get_bk_price(symbol)
         
-        # 特殊处理PT代码
+        # 特殊处理PT代码（不使用缓存）
         if symbol[:2] == 'pt':
             return self._get_pt_price(symbol, sdate, edate, freq, days, fq)
         
-        # 使用数据源获取数据
+        # 使用基类的缓存逻辑
+        return super().get_price(symbol, sdate, edate, freq, days, fq)
+    
+    def _fetch_price_data(self, symbol: str, sdate: str = '', edate: str = '', 
+                          freq: str = 'day', days: int = 320, fq: str = 'qfq') -> Tuple[str, str, pd.DataFrame]:
+        """从数据源获取A股价格数据"""
         try:
             raw_data = self.data_source.fetch_kline(
                 symbol, freq=freq, sdate=sdate, edate=edate, days=days, fq=fq
@@ -51,9 +50,7 @@ class CNStockMarket(Market):
             parser = KlineParser()
             name, df = parser.parse_tencent_kline(raw_data, symbol)
             
-            result = (symbol, name, df)
-            self._put_cache(cache_key, result)
-            return result
+            return (symbol, name, df)
         except (DataSourceError, ParseError) as e:
             logger.warning(f'Failed to fetch {symbol} using new architecture: {e}')
             # 降级到旧方法

rquote/markets/future.py

@@ -25,18 +25,19 @@ class FutureMarket(Market):
         """获取期货价格数据"""
         symbol = self.normalize_symbol(symbol)
         
-        # 特殊处理BTC
+        # 特殊处理BTC（不使用缓存）
         if symbol[2:5].lower() == 'btc':
             if freq in ('min', '1min', 'minute'):
                 return self._get_btc_minute_price(symbol)
             else:
                 return self._get_btc_price(symbol)
         
-        cache_key = f"{symbol}:{sdate}:{edate}:{freq}:{days}"
-        cached = self._get_cached(cache_key)
-        if cached:
-            return cached
-        
+        # 使用基类的缓存逻辑
+        return super().get_price(symbol, sdate, edate, freq, days, fq)
+    
+    def _fetch_price_data(self, symbol: str, sdate: str = '', edate: str = '', 
+                          freq: str = 'day', days: int = 320, fq: str = 'qfq') -> Tuple[str, str, pd.DataFrame]:
+        """从数据源获取期货价格数据"""
         future_code = symbol[2:]  # 去掉'fu'前缀
         
         try:
@@ -44,9 +45,7 @@ class FutureMarket(Market):
             parser = KlineParser()
             df = parser.parse_sina_future_kline(raw_data, freq=freq)
             
-            result = (symbol, future_code, df)
-            self._put_cache(cache_key, result)
-            return result
+            return (symbol, future_code, df)
         except (DataSourceError, ParseError) as e:
             logger.warning(f'Failed to fetch {symbol} using new architecture, falling back: {e}')
             return self._get_price_fallback(symbol, future_code, freq)

rquote/markets/hk_stock.py

@@ -19,16 +19,9 @@ class HKStockMarket(Market):
             return 'hk' + symbol
         return symbol
     
-    def get_price(self, symbol: str, sdate: str = '', edate: str = '', 
-                  freq: str = 'day', days: int = 320, fq: str = 'qfq') -> Tuple[str, str, pd.DataFrame]:
-        """获取港股价格数据"""
-        symbol = self.normalize_symbol(symbol)
-        
-        cache_key = f"{symbol}:{sdate}:{edate}:{freq}:{days}:{fq}"
-        cached = self._get_cached(cache_key)
-        if cached:
-            return cached
-        
+    def _fetch_price_data(self, symbol: str, sdate: str = '', edate: str = '', 
+                          freq: str = 'day', days: int = 320, fq: str = 'qfq') -> Tuple[str, str, pd.DataFrame]:
+        """从数据源获取港股价格数据"""
         try:
             raw_data = self.data_source.fetch_kline(
                 symbol, freq=freq, sdate=sdate, edate=edate, days=days, fq=fq
@@ -37,9 +30,7 @@ class HKStockMarket(Market):
             parser = KlineParser()
             name, df = parser.parse_tencent_kline(raw_data, symbol, fq=fq)
             
-            result = (symbol, name, df)
-            self._put_cache(cache_key, result)
-            return result
+            return (symbol, name, df)
         except (DataSourceError, ParseError) as e:
             logger.warning(f'Failed to fetch {symbol}: {e}')
             raise

rquote/markets/us_stock.py

@@ -25,15 +25,16 @@ class USStockMarket(Market):
         """获取美股价格数据"""
         symbol = self.normalize_symbol(symbol)
         
-        cache_key = f"{symbol}:{sdate}:{edate}:{freq}:{days}:{fq}"
-        cached = self._get_cached(cache_key)
-        if cached:
-            return cached
-        
-        # 特殊处理分钟数据
+        # 特殊处理分钟数据（不使用缓存）
         if freq in ('min', '1min', 'minute'):
             return self._get_minute_data(symbol)
         
+        # 使用基类的缓存逻辑
+        return super().get_price(symbol, sdate, edate, freq, days, fq)
+    
+    def _fetch_price_data(self, symbol: str, sdate: str = '', edate: str = '', 
+                          freq: str = 'day', days: int = 320, fq: str = 'qfq') -> Tuple[str, str, pd.DataFrame]:
+        """从数据源获取美股价格数据"""
         try:
             raw_data = self.data_source.fetch_kline(
                 symbol, freq=freq, sdate=sdate, edate=edate, days=days, fq=fq
@@ -42,9 +43,7 @@ class USStockMarket(Market):
             parser = KlineParser()
             name, df = parser.parse_tencent_kline(raw_data, symbol)
             
-            result = (symbol, name, df)
-            self._put_cache(cache_key, result)
-            return result
+            return (symbol, name, df)
         except (DataSourceError, ParseError) as e:
             logger.warning(f'Failed to fetch {symbol}: {e}')
             raise

{rquote-0.3.9.dist-info → rquote-0.4.1.dist-info}/METADATA

@@ -1,14 +1,16 @@
 Metadata-Version: 2.4
 Name: rquote
-Version: 0.3.9
+Version: 0.4.1
 Summary: Mostly day quotes of cn/hk/us/fund/future markets, side with quote list fetch
-Requires-Python: >=3.6.1
+Requires-Python: >=3.9.0
 Description-Content-Type: text/markdown
 Requires-Dist: build>=0.9.0
 Requires-Dist: httpx>=0.20.0
 Requires-Dist: pandas>=1.0.0
 Requires-Dist: setuptools>=42
 Requires-Dist: twine>=3.8.0
+Provides-Extra: persistent
+Requires-Dist: duckdb>=0.9.0; extra == "persistent"
 
 # rquote
 
@@ -16,7 +18,7 @@ Requires-Dist: twine>=3.8.0
 
 ## 版本信息
 
-当前版本：**0.3.5**
+当前版本：**0.4.1**
 
 ## 主要特性
 
@@ -60,17 +62,67 @@ sid, name, df = get_price('sz000001', sdate='2024-01-01', edate='2024-02-01')
 
 ### 使用缓存
 
+#### 内存缓存（MemoryCache）
+
 ```python
 from rquote import get_price, MemoryCache
 
 # 创建缓存实例
 cache = MemoryCache(ttl=3600)  # 缓存1小时
 
-# 使用缓存（通过dd参数，向后兼容）
-cache_dict = {}
-sid, name, df = get_price('sh000001', dd=cache_dict)
+# 使用缓存（通过dd参数传递MemoryCache实例）
+sid, name, df = get_price('sh000001', dd=cache)
+
+# 注意：MemoryCache 是内存缓存，数据仅在当前进程运行期间有效
+# 脚本运行结束后，缓存数据会丢失
+```
+
+**缓存生命周期说明：**
+- `MemoryCache` 是纯内存缓存，数据存储在进程内存中
+- 缓存数据仅在当前脚本运行期间有效
+- 脚本运行结束后，所有缓存数据会丢失
+
+#### 持久化缓存（PersistentCache）
+
+持久化缓存支持跨进程/跨运行的缓存持久化，数据会保存到本地文件。
+
+**安装可选依赖：**
+```bash
+pip install rquote[persistent]
+# 或
+uv pip install "rquote[persistent]"
+```
+
+**使用持久化缓存：**
+```python
+from rquote import get_price, PersistentCache
+
+# 创建持久化缓存实例
+# 默认使用 duckdb（如果已安装），否则使用 pickle 文件
+cache = PersistentCache(ttl=86400)  # 缓存24小时，默认路径：~/.rquote/cache.db
+
+# 或指定自定义路径
+cache = PersistentCache(db_path='./my_cache.db', use_duckdb=True)
+
+# 使用缓存
+sid, name, df = get_price('sh000001', dd=cache)
+
+# 持久化缓存支持智能扩展：
+# - 当请求的结束日期不在缓存中时，会自动从缓存的最新日期向前扩展
+# - 当请求的开始日期不在缓存中时，会自动从缓存的最早日期向后扩展
+# - 数据会自动合并，避免重复请求
+
+# 关闭缓存（可选，程序退出时会自动保存）
+cache.close()
 ```
 
+**持久化缓存特性：**
+- ✅ 跨进程/跨运行持久化：数据保存在本地文件，下次运行仍可使用
+- ✅ 智能数据合并：相同股票的数据会自动合并，key 不包含日期范围
+- ✅ 智能扩展：当请求的日期范围超出缓存时，自动扩展并合并数据
+- ✅ 支持 TTL：可设置缓存过期时间
+- ✅ 可选 duckdb：如果安装了 duckdb，使用 duckdb 存储（性能更好），否则使用 pickle 文件
+
 ## 主要功能
 
 ### 历史价格数据获取
@@ -147,13 +199,13 @@ stocks = get_cn_stock_list(money_min=5e8)
 
 #### `get_hk_stocks_500()`
 
-获取港股前500只股票列表
+获取港股前500只股票列表(按当日成交额排序)
 
 ```python
 from rquote import get_hk_stocks_500
 
 stocks = get_hk_stocks_500()
-# 返回格式: [[code, name, price, turnover, ...], ...]
+# 返回格式: [[code, name, price, -, -, -, -, volume, turnover, ...], ...]
 ```
 
 #### `get_us_stocks(k=100)`

{rquote-0.3.9.dist-info → rquote-0.4.1.dist-info}/RECORD

@@ -1,4 +1,4 @@
-rquote/__init__.py,sha256=-U5Uq4eT3Hhl2EkVmBgr5TAfU-ZfFzpOaGeJafnhyos,2208
+rquote/__init__.py,sha256=HMXqZ_wfGoRqw1V3xm2MyBGYKB9ooGWIRnk60bisLZo,2370
 rquote/config.py,sha256=noep_VzY_nJehnkPQb4mkwzpeYLwkU1riqofQJ6Hhw0,1108
 rquote/exceptions.py,sha256=lJH2GC5dDhMoW_OtlBc03wlUn684-7jNPyF1NjmfVIE,569
 rquote/plots.py,sha256=UQn4sjhIzVwagfhUDM738b2HHjKo4tRdU2UCs_1-FbY,2341
@@ -8,9 +8,10 @@ rquote/api/lists.py,sha256=fRebS02Fi0qe6KpWBA-9W1UG0It6__DmRlNimtMa7L8,5331
 rquote/api/price.py,sha256=I5lZl6cUQRlE4AtzNbR-uGZt1ho9vgP1cgNFDjaigMA,3575
 rquote/api/stock_info.py,sha256=912ICdIBr8z2lKWDbq3gG0E94czTPvbx9aXsKUi-QkE,1537
 rquote/api/tick.py,sha256=nEcjuAjtBHUaD8KPRLg643piVa21PhKDQvkVWNwvvME,1431
-rquote/cache/__init__.py,sha256=IXGSRpvSgBlcM6twLuJEOEockbb09_VqORXdQpfwpCA,138
+rquote/cache/__init__.py,sha256=S393I5Wmp0QooaRka9n7bvDUdEbg3jUhm6u815T86rM,317
 rquote/cache/base.py,sha256=orzG4Yo-6gzVG027j1-LTZPT718JohnCdLDnOLoLUQ4,515
 rquote/cache/memory.py,sha256=7z4keb3q91pzI4ASQWy1MU8T5nbWLCEUjJcStv_3hvk,1933
+rquote/cache/persistent.py,sha256=ZffpM1a_ngRTjvHvPwk3XoorH87TSCBwtrXtvO0yqBo,19751
 rquote/data_sources/__init__.py,sha256=WCe1aam4677jM5G6wP4a-dQFTeBzcU5PJqsKieAVMBo,215
 rquote/data_sources/base.py,sha256=JuKsTMxH7y8yRxHg3JbLzQwXPr43rS4pnwc5625u2U4,443
 rquote/data_sources/sina.py,sha256=T_3Dl0Mwlhx8CKRJll_UKobYecRWltGaIOiGkpHS43Q,3300
@@ -18,12 +19,12 @@ rquote/data_sources/tencent.py,sha256=ayt1O85pheLwzX3z5c6Qij1NrmUywcsz6YcSVzdDoM
 rquote/factors/__init__.py,sha256=_ZbH2XxYtXwCJpvRVdNvGncoPSpMqrtlYmf1_fMGIjM,116
 rquote/factors/technical.py,sha256=dPDs3pDEDRV9iQJBrSoKpGFLQMjOqyoBdN2rUntpOUU,4235
 rquote/markets/__init__.py,sha256=k4F8cZgb-phqemMqhZXFPdOKsR4P--DD3d5i21vKhbg,365
-rquote/markets/base.py,sha256=DjvxRcJqwUsBTxnsE28Gd-zJLFsCGwdQpezLRAZ_9sQ,1347
-rquote/markets/cn_stock.py,sha256=fyF7jJHFUrI5jwuqBKHXpsIE51H4kbyc3q-uuviPLGk,8224
+rquote/markets/base.py,sha256=nHBMzQRkuDUrsx9GvB_QiMh2deMUjTiUZsIRYPJpB_8,11206
+rquote/markets/cn_stock.py,sha256=nu2ebTE4a6FAJkvpMN0FEPuqwom_hqTRjnUg96cQGKc,8320
 rquote/markets/factory.py,sha256=4Txpuok0LBOLT_vAiIU-NslwVnYF7sKHCdlacAboxpo,2875
-rquote/markets/future.py,sha256=7AqViPp0S9OQZsaU2hkJzh4My6gYFqLo1OUW2mVMSDo,7215
-rquote/markets/hk_stock.py,sha256=NlWaXQgXttpcQVFZjflcEkMTmXMxeP2C6Y7OGG50u7E,1452
-rquote/markets/us_stock.py,sha256=17mTg50g3ImOnGM4Re1MRSyvbD2mgFW6wjtMh86IEXA,2465
+rquote/markets/future.py,sha256=yGMyUu9Fv75jbzPbvW6_36otEeebSij7vnzow_zyEn8,7358
+rquote/markets/hk_stock.py,sha256=AhRJpWp027ACew9ogxkVCJXbqbYQ1AkbFwDJccXbvAs,1183
+rquote/markets/us_stock.py,sha256=GT4IxCMTgxb0JSkDa6acZ6PpHLhK6lrskI0ftiYxGCA,2603
 rquote/parsers/__init__.py,sha256=q4g-FgpzxKBPfhJiQH3B5MEeZWUIXlyre-vAnOnfYmA,110
 rquote/parsers/kline.py,sha256=g6k8W76-4hpYsuBgvwmb5G6ZkzHOJDX-JrVVXYksw4c,4020
 rquote/utils/__init__.py,sha256=-ZHABqFHQeJrCCsgnqEYWR57jl7GduCKn2V3hpFi-pE,348
@@ -32,7 +33,7 @@ rquote/utils/helpers.py,sha256=V07n9BtRS8bEJH023Kca78-unk7iD3B9hn2UjELetYs,354
 rquote/utils/http.py,sha256=X0Alhnu0CNqyQeOt6ivUWmh2XwrWxXd2lSpQOKDdnzw,3249
 rquote/utils/logging.py,sha256=cbeRH4ODazn7iyQmGoEBT2lH5LX4Ca3zDfs_20J1T28,566
 rquote/utils/web.py,sha256=I8_pcThW6VUvahuRHdtp32iZwr85hEt1hB6TgznMy_U,3854
-rquote-0.3.9.dist-info/METADATA,sha256=gFYIx3AMMzbJgDx9BlXn-Xn3T9PgkTuYkQTXthyjeIA,11262
-rquote-0.3.9.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-rquote-0.3.9.dist-info/top_level.txt,sha256=CehAiaZx7Fo8HGoV2zd5GhILUW1jQEN8YS-cWMlrK9Y,7
-rquote-0.3.9.dist-info/RECORD,,
+rquote-0.4.1.dist-info/METADATA,sha256=Cp9oRn7ohg_2XmhczYDNR2JJMX0MK2aDFjqhP5kJWVA,13259
+rquote-0.4.1.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+rquote-0.4.1.dist-info/top_level.txt,sha256=CehAiaZx7Fo8HGoV2zd5GhILUW1jQEN8YS-cWMlrK9Y,7
+rquote-0.4.1.dist-info/RECORD,,