litequant 3.0.0__py3-none-any.whl

litequant/LiteQuantClient.py

@@ -0,0 +1,884 @@
+# -*- coding: utf-8 -*-
+"""
+LiteQuant 3.0 Client - 独立简洁版
+
+完整独立实现，自动管理 Data Ticket 生命周期：
+- 获取/续租/释放 Data Ticket
+- 直连 Redis 下载数据
+- 本地 Parquet 缓存管理
+- 一键更新和读取数据
+
+使用示例：
+    import litequant as lq
+    
+    client = lq.LiteQuantClient(
+        api_token='your_api_token',
+        save_path='D:/LiteQuant/'
+    )
+    
+    client.UpdateAllCategory(update_method='full')
+    df = client.GetCategory("cn_stock_pivot#open")
+    
+    # 使用 with 语句（自动关闭）
+    with lq.LiteQuantClient(api_token='xxx', save_path='D:/data/') as client:
+        df = client.GetCategory("cn_stock_pivot#open")
+"""
+
+import hashlib
+import io
+import json
+import logging
+import os
+import time
+import threading
+from typing import Any, Dict, List, Optional, Literal
+from dataclasses import dataclass
+from datetime import datetime, timedelta
+
+import pandas as pd
+import redis
+import requests
+from tqdm import tqdm
+
+from .ParquetManager import ParquetDataManager
+from .exceptions import (
+    LiteQuantError,
+    AuthError,
+    CategoryNotFoundError,
+    InvalidCategoryError,
+    RemoteDataError,
+    SerializationError,
+    MetaError,
+    ProtocolError,
+    TicketExpiredError,
+    QuotaExceededError,
+    PermissionDeniedError,
+    NetworkError,
+    DataConnectionError,
+    ServiceUnavailableError,
+    SyncError,
+    APIError,
+)
+from .log import GetLogger
+
+
+# ============ 数据模型 ============
+
+@dataclass(repr=False)
+class DataTicket:
+    """Data Ticket data returned by the service."""
+    ticket_id: str
+    session_id: str
+    redis_host: str
+    redis_port: int
+    redis_password: str
+    expires_at: datetime
+    datasets: List[str]
+    key_patterns: List[str]
+    redis_username: Optional[str] = None
+    redis_ssl: bool = False
+    
+    @property
+    def is_expired(self) -> bool:
+        return datetime.now() >= self.expires_at
+    
+    @property
+    def ttl_seconds(self) -> float:
+        delta = self.expires_at - datetime.now()
+        return max(0, delta.total_seconds())
+
+    def __repr__(self) -> str:
+        return "DataTicket(<hidden>)"
+
+
+# ============ 核心客户端 ============
+
+class LiteQuantClient:
+    """
+    LiteQuant 3.0 简洁版客户端
+    
+    独立完整实现，自动管理所有底层连接和数据缓存。
+    
+    Args:
+        api_token: API Token（认证凭据）
+        save_path: 本地数据存储路径
+        api_url: API 地址（可选，默认从环境变量 LITEQUANT_API_URL 读取）
+        log_level: 日志级别
+    """
+    
+    # 常量配置
+    DEFAULT_API_URL = os.environ.get('LITEQUANT_API_URL', 'https://www.litequant.pro')
+    PROTOCOL_VERSION = 1
+    DATA_FORMAT = "parquet"
+    
+    # Ticket 配置
+    HEARTBEAT_INTERVAL = 30  # 心跳间隔（秒）
+    TICKET_REFRESH_THRESHOLD = 30  # 过期前30秒续租
+    
+    # Redis Key 前缀
+    META_CATEGORIES_KEY = "litequant:meta:categories"
+    META_CATEGORY_KEY_PREFIX = "litequant:meta:category:"
+    META_PARTITION_KEY_PREFIX = "litequant:meta:partition:"
+    
+    PUBLIC_ERROR_MESSAGES = {
+        "AUTH_INVALID": "API 凭证无效或已过期，请检查后重试",
+        "ACCOUNT_UNAVAILABLE": "账号不可用，请联系管理员处理",
+        "SUBSCRIPTION_UNAVAILABLE": "套餐不可用或已过期，请续费后重试",
+        "PERMISSION_DENIED": "权限不足：当前账号没有该数据权限",
+        "CONNECTION_LIMIT": "连接数已达上限，请关闭其他连接后重试",
+        "CONNECTION_INTERRUPTED": "连接中断，请重新初始化客户端",
+        "REQUEST_INVALID": "请求参数无效，请检查输入后重试",
+        "SERVICE_UNAVAILABLE": "服务暂时不可用，请稍后重试",
+        "DATA_UNAVAILABLE": "数据暂时不可用，请稍后重试",
+        "DATA_VERIFY_FAILED": "数据校验失败，请重新同步",
+    }
+    LEGACY_ERROR_CODE_MAP = {
+        "Unauthorized": "AUTH_INVALID",
+        "UserDisabled": "ACCOUNT_UNAVAILABLE",
+        "SubscriptionInvalid": "SUBSCRIPTION_UNAVAILABLE",
+        "SubscriptionExpired": "SUBSCRIPTION_UNAVAILABLE",
+        "NoDatasetAccess": "PERMISSION_DENIED",
+        "ConnectionLimitExceeded": "CONNECTION_LIMIT",
+        "InvalidRequest": "REQUEST_INVALID",
+        "MissingParameters": "REQUEST_INVALID",
+        "TicketNotFound": "CONNECTION_INTERRUPTED",
+        "TicketExpired": "CONNECTION_INTERRUPTED",
+        "SessionNotFound": "CONNECTION_INTERRUPTED",
+        "SessionExpired": "CONNECTION_INTERRUPTED",
+        "RenewFailed": "CONNECTION_INTERRUPTED",
+        "InternalError": "SERVICE_UNAVAILABLE",
+    }
+    ERROR_CLASS_MAP = {
+        "AUTH_INVALID": AuthError,
+        "ACCOUNT_UNAVAILABLE": PermissionDeniedError,
+        "SUBSCRIPTION_UNAVAILABLE": PermissionDeniedError,
+        "PERMISSION_DENIED": PermissionDeniedError,
+        "CONNECTION_LIMIT": QuotaExceededError,
+        "CONNECTION_INTERRUPTED": DataConnectionError,
+        "REQUEST_INVALID": APIError,
+        "SERVICE_UNAVAILABLE": ServiceUnavailableError,
+        "DATA_UNAVAILABLE": RemoteDataError,
+        "DATA_VERIFY_FAILED": SerializationError,
+    }
+    
+    def __init__(
+        self,
+        api_token: str,
+        save_path: str,
+        api_url: str = None,
+        log_level: int = logging.INFO,
+        display_errors: bool = True,
+    ):
+        self.api_token = api_token
+        self.api_url = (api_url or os.environ.get('LITEQUANT_API_URL', self.DEFAULT_API_URL)).rstrip("/")
+        self.save_path = os.path.abspath(save_path)
+        self.display_errors = display_errors
+        
+        # 确保路径存在
+        os.makedirs(self.save_path, exist_ok=True)
+        
+        # 日志（使用 token 前8位作为标识）
+        self.logger = GetLogger("litequant", level=log_level)
+        self.logger.info(f"初始化 LiteQuantClient")
+        
+        # 数据管理器
+        self.LQ_db = ParquetDataManager(self.save_path, logger=self.logger)
+        
+        # Ticket 和连接
+        self._ticket: Optional[DataTicket] = None
+        self._redis_client: Optional[redis.Redis] = None
+        
+        # 心跳线程
+        self._stop_heartbeat = threading.Event()
+        self._heartbeat_thread: Optional[threading.Thread] = None
+        
+        # 状态
+        self._remote_categories: List[str] = []
+        self._remote_categories_set = set()
+        
+        # 初始化连接
+        try:
+            self._init_connection()
+            self._start_heartbeat()
+        except LiteQuantError as e:
+            err = self._public_error_clone(e)
+            self._display_error(err)
+            raise err from None
+        except Exception as e:
+            err = ServiceUnavailableError()
+            self._display_error(err)
+            raise err from None
+        
+        # 启动心跳
+        
+        self.logger.info("LiteQuantClient 初始化完成")
+    
+    # ============ 连接管理 ============
+    
+    def _display_error(self, error: LiteQuantError) -> None:
+        """Print a privacy-safe user message to the configured SDK logger."""
+        if self.display_errors:
+            self.logger.error(error.user_message)
+
+    def _public_error_clone(self, error: LiteQuantError) -> LiteQuantError:
+        """Return a fresh privacy-safe exception without traceback/cause state."""
+        return error.__class__(
+            code=error.code,
+            user_message=error.user_message,
+            retryable=error.retryable,
+        )
+
+    def _as_litequant_error(self, error: Exception) -> LiteQuantError:
+        if isinstance(error, LiteQuantError):
+            return self._public_error_clone(error)
+        return ServiceUnavailableError()
+
+    def _api_post_json(self, endpoint: str, payload: dict, timeout: int) -> dict:
+        url = f"{self.api_url}{endpoint}"
+        headers = {
+            "Authorization": f"Bearer {self.api_token}",
+            "Content-Type": "application/json",
+        }
+        try:
+            response = requests.post(url, headers=headers, json=payload, timeout=timeout)
+        except (requests.exceptions.Timeout, requests.exceptions.ConnectionError):
+            raise NetworkError() from None
+        except requests.exceptions.RequestException:
+            raise ServiceUnavailableError() from None
+
+        return self._parse_api_response(response)
+
+    def _parse_api_response(self, response: Any) -> dict:
+        status_code = getattr(response, "status_code", 200)
+        ok = getattr(response, "ok", 200 <= status_code < 300)
+        try:
+            data = response.json()
+        except ValueError:
+            public_code = self._public_code_from_status(status_code)
+            raise self._build_public_error(public_code) from None
+
+        if not isinstance(data, dict):
+            raise ServiceUnavailableError(detail="invalid response shape")
+
+        if ok and data.get("success", True):
+            return data
+
+        public_code = self._normalize_error_code(data.get("error"), status_code)
+        raise self._build_public_error(public_code) from None
+
+    def _normalize_error_code(self, raw_code: Any, status_code: int = None) -> str:
+        if raw_code:
+            raw_code = str(raw_code)
+            public_code = self.LEGACY_ERROR_CODE_MAP.get(raw_code, raw_code)
+            if public_code in self.PUBLIC_ERROR_MESSAGES:
+                return public_code
+        return self._public_code_from_status(status_code)
+
+    def _public_code_from_status(self, status_code: int = None) -> str:
+        if status_code == 401:
+            return "AUTH_INVALID"
+        if status_code == 403:
+            return "PERMISSION_DENIED"
+        if status_code == 429:
+            return "CONNECTION_LIMIT"
+        if status_code == 400:
+            return "REQUEST_INVALID"
+        if status_code and status_code >= 500:
+            return "SERVICE_UNAVAILABLE"
+        return "SERVICE_UNAVAILABLE"
+
+    def _build_public_error(self, public_code: str, detail: Any = None) -> LiteQuantError:
+        error_cls = self.ERROR_CLASS_MAP.get(public_code, ServiceUnavailableError)
+        return error_cls(
+            code=public_code,
+            user_message=self.PUBLIC_ERROR_MESSAGES.get(public_code, self.PUBLIC_ERROR_MESSAGES["SERVICE_UNAVAILABLE"]),
+            retryable=public_code in {
+                "CONNECTION_LIMIT",
+                "CONNECTION_INTERRUPTED",
+                "SERVICE_UNAVAILABLE",
+                "DATA_UNAVAILABLE",
+                "DATA_VERIFY_FAILED",
+            },
+        )
+
+    def _init_connection(self):
+        """初始化连接（获取 Ticket + 连接 Redis）"""
+        self._fetch_ticket()
+        self._connect_redis()
+        self._refresh_categories()
+    
+    def _fetch_ticket(self) -> DataTicket:
+        """从 Django API 获取 Data Ticket"""
+        try:
+            data = self._api_post_json(
+                "/api/v1/data/ticket/",
+                {"client_version": "3.0.0"},
+                timeout=30,
+            )
+            
+            redis_info = data["redis"]
+            scope = data.get("scope", {})
+            redis_username = redis_info.get("username")
+            redis_ssl = bool(redis_info.get("ssl", False))
+            
+            self._ticket = DataTicket(
+                ticket_id=data["ticket"],
+                session_id=data["session_id"],
+                redis_host=redis_info["host"],
+                redis_port=redis_info["port"],
+                redis_password=redis_info["password"],
+                expires_at=datetime.now() + timedelta(seconds=data["expires_in"]),
+                datasets=scope.get("datasets", []),
+                key_patterns=scope.get("key_patterns", []),
+                redis_username=redis_username,
+                redis_ssl=redis_ssl,
+            )
+            
+            self.logger.debug("连接凭证已获取")
+            return self._ticket
+            
+        except LiteQuantError:
+            raise
+        except (KeyError, TypeError, ValueError):
+            raise ServiceUnavailableError() from None
+    
+    def _connect_redis(self):
+        """使用 Ticket 连接 Redis"""
+        if not self._ticket:
+            raise DataConnectionError()
+        
+        try:
+            redis_kwargs = {
+                "host": self._ticket.redis_host,
+                "port": self._ticket.redis_port,
+                "password": self._ticket.redis_password,
+                "socket_timeout": 20,
+                "socket_connect_timeout": 5,
+                "decode_responses": False,
+            }
+            if self._ticket.redis_username:
+                redis_kwargs["username"] = self._ticket.redis_username
+                redis_kwargs["ssl"] = self._ticket.redis_ssl
+            else:
+                # Aliyun Tair can use password formatted as "username:password".
+                redis_kwargs["db"] = 0
+
+            for attempt in range(6):
+                self._redis_client = redis.Redis(**redis_kwargs)
+                try:
+                    if not self._redis_client.ping():
+                        raise DataConnectionError()
+                    break
+                except redis.AuthenticationError:
+                    try:
+                        self._redis_client.close()
+                    except Exception:
+                        pass
+                    self._redis_client = None
+                    if attempt >= 5:
+                        raise
+                    self.logger.warning(f"数据连接暂未就绪，正在重试 {attempt + 1}/5")
+                    time.sleep(3)
+            else:
+                raise DataConnectionError()
+            
+            self.logger.info("数据连接已建立")
+            
+        except redis.AuthenticationError:
+            raise DataConnectionError() from None
+        except Exception as e:
+            if isinstance(e, LiteQuantError):
+                raise self._public_error_clone(e) from None
+            raise DataConnectionError() from None
+    
+    def _disconnect_server(self):
+        """通知服务器断开连接"""
+        if not self._ticket:
+            return
+        
+        try:
+            requests.post(
+                f"{self.api_url}/api/v1/data/disconnect/",
+                headers={
+                    "Authorization": f"Bearer {self.api_token}",
+                    "Content-Type": "application/json"
+                },
+                json={
+                    "ticket": self._ticket.ticket_id,
+                    "session_id": self._ticket.session_id
+                },
+                timeout=10
+            )
+        except Exception:
+            pass
+    
+    def _ensure_connection(self):
+        """确保连接有效（过期时自动刷新）"""
+        if not self._ticket or self._ticket.is_expired:
+            self.logger.info("连接已过期，正在重新连接...")
+            self._fetch_ticket()
+            self._connect_redis()
+        elif self._ticket.ttl_seconds < self.TICKET_REFRESH_THRESHOLD:
+            self._renew_ticket()
+    
+    def _renew_ticket(self):
+        """续租 Ticket"""
+        if not self._ticket:
+            raise DataConnectionError()
+        try:
+            data = self._api_post_json(
+                "/api/v1/data/heartbeat/",
+                {
+                    "ticket": self._ticket.ticket_id,
+                    "session_id": self._ticket.session_id,
+                },
+                timeout=10,
+            )
+            self._ticket.expires_at = datetime.now() + timedelta(seconds=data.get("expires_in", 60))
+            self.logger.debug("连接已保持")
+        except DataConnectionError:
+            self.logger.warning("连接已中断，正在重新连接...")
+            self._fetch_ticket()
+            self._connect_redis()
+        except LiteQuantError:
+            raise
+        except Exception:
+            raise ServiceUnavailableError() from None
+    
+    def _start_heartbeat(self):
+        """启动后台心跳线程"""
+        self._stop_heartbeat.clear()
+        self._heartbeat_thread = threading.Thread(target=self._heartbeat_loop, daemon=True)
+        self._heartbeat_thread.start()
+        self.logger.debug("心跳线程已启动")
+    
+    def _heartbeat_loop(self):
+        """心跳循环"""
+        while not self._stop_heartbeat.wait(self.HEARTBEAT_INTERVAL):
+            try:
+                if self._ticket and self._ticket.ttl_seconds < self.TICKET_REFRESH_THRESHOLD:
+                    self._renew_ticket()
+            except LiteQuantError as e:
+                self.logger.warning(e.user_message)
+            except Exception:
+                self.logger.warning(ServiceUnavailableError().user_message)
+    
+    # ============ 核心用户接口 ============
+    
+    def UpdateAllCategory(self, update_method: Literal['full', 'incremental'] = 'full') -> None:
+        """更新所有已授权的数据类别。"""
+        try:
+            return self._update_all_category(update_method)
+        except LiteQuantError as e:
+            err = self._public_error_clone(e)
+            self._display_error(err)
+            raise err from None
+        except Exception as e:
+            err = ServiceUnavailableError()
+            self._display_error(err)
+            raise err from None
+
+    def _update_all_category(self, update_method: Literal['full', 'incremental'] = 'full') -> None:
+        """
+        更新所有授权的数据类别
+        
+        Args:
+            update_method: 更新方式
+                - 'full' - 全量更新，扫描所有数据（首次使用或需要完整数据时）
+                - 'incremental' - 增量更新，只扫描最近两个月的数据（日常使用）
+        """
+        if update_method not in ('full', 'incremental'):
+            raise ValueError("update_method must be 'full' or 'incremental'")
+
+        self.logger.info(f"开始更新数据 [方式: {update_method}]")
+        
+        # 计算增量更新的日期范围（最近两个月）
+        incremental_start_date = None
+        if update_method == 'incremental':
+            incremental_start_date = datetime.now() - timedelta(days=60)
+            self.logger.info(f"增量更新起始日期: {incremental_start_date.strftime('%Y-%m-%d')}")
+        
+        # 确保连接有效
+        self._ensure_connection()
+        
+        # 刷新类别列表
+        self._refresh_categories()
+        
+        if not self._remote_categories:
+            self.logger.warning("没有可更新的数据类别")
+            return
+        
+        self.logger.info(f"发现 {len(self._remote_categories)} 个类别")
+        
+        # 同步所有类别
+        failures = []
+        for category in self._remote_categories:
+            try:
+                self.logger.info(f"正在同步: {category}")
+                self._sync_category(category, incremental_start_date=incremental_start_date)
+            except Exception as e:
+                err = self._as_litequant_error(e)
+                failures.append((category, err))
+                self.logger.error(f"{category}: {err.user_message}")
+        
+        if failures:
+            raise SyncError(user_message=f"部分数据更新失败：{len(failures)} 个数据类别未完成")
+        
+        self.logger.info("数据更新完成")
+    
+    def GetCategory(
+        self,
+        category: str,
+        start_date: Optional[str] = None,
+        end_date: Optional[str] = None,
+    ) -> pd.DataFrame:
+        """读取指定数据类别。"""
+        try:
+            return self._get_category(category, start_date, end_date)
+        except LiteQuantError as e:
+            err = self._public_error_clone(e)
+            self._display_error(err)
+            raise err from None
+        except Exception as e:
+            err = ServiceUnavailableError()
+            self._display_error(err)
+            raise err from None
+
+    def _get_category(
+        self,
+        category: str,
+        start_date: Optional[str] = None,
+        end_date: Optional[str] = None,
+    ) -> pd.DataFrame:
+        """
+        读取数据类别
+        
+        Args:
+            category: 数据类别名称，如 "cn_stock_pivot#open"
+            start_date: 开始日期（可选）
+            end_date: 结束日期（可选）
+        
+        Returns:
+            DataFrame
+        """
+        self.logger.debug(f"读取数据: {category}")
+        
+        # 检查本地是否存在
+        local_categories = self.LQ_db._get_all_categories()
+        
+        if category not in local_categories:
+            self.logger.info(f"本地不存在 {category}，从远程同步...")
+            self._ensure_connection()
+            
+            if category not in self._remote_categories_set:
+                self._refresh_categories()
+            
+            if category not in self._remote_categories_set:
+                raise CategoryNotFoundError(f"数据类别不存在: {category}")
+            
+            self._sync_category(category)
+        
+        # 读取本地数据
+        category_type = self._validate_category_name(category)
+        
+        if category_type == "pivot":
+            return self.LQ_db.read_pivot_category(category, start_date, end_date)
+        else:
+            return self.LQ_db.read_unstack_category(category)
+    
+    def ListCategories(self) -> List[str]:
+        """列出所有可用的数据类别（远程）"""
+        try:
+            return self._list_categories()
+        except LiteQuantError as e:
+            err = self._public_error_clone(e)
+            self._display_error(err)
+            raise err from None
+        except Exception as e:
+            err = ServiceUnavailableError()
+            self._display_error(err)
+            raise err from None
+
+    def _list_categories(self) -> List[str]:
+        """列出所有可用的数据类别（远程）"""
+        self._ensure_connection()
+        self._refresh_categories()
+        return list(self._remote_categories)
+    
+    def ListLocalCategories(self) -> List[str]:
+        """列出本地已存储的数据类别"""
+        return self.LQ_db._get_all_categories()
+    
+    def close(self):
+        """关闭客户端，释放资源"""
+        self.logger.info("关闭 LiteQuantClient...")
+        
+        # 停止心跳
+        self._stop_heartbeat.set()
+        if self._heartbeat_thread and self._heartbeat_thread.is_alive():
+            self._heartbeat_thread.join(timeout=5)
+        
+        # 关闭 Redis 连接
+        if self._redis_client:
+            try:
+                self._redis_client.close()
+            except Exception:
+                pass
+            self._redis_client = None
+        
+        # 通知服务器断开
+        if self._ticket:
+            self._disconnect_server()
+        
+        self.logger.info("LiteQuantClient 已关闭")
+    
+    def __enter__(self):
+        return self
+    
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        self.close()
+        return False
+    
+    # ============ 内部方法 ============
+    
+    def _refresh_categories(self):
+        """刷新类别列表，只保留授权的数据集"""
+        categories = self._json_get_required(self.META_CATEGORIES_KEY)
+        if not isinstance(categories, list):
+            raise MetaError("categories 必须是列表")
+        
+        all_categories = [str(x) for x in categories]
+        
+        # 过滤：只保留授权数据集的类别
+        if self._ticket and self._ticket.datasets:
+            authorized_datasets = set(self._ticket.datasets)
+            filtered_categories = [
+                cat for cat in all_categories
+                if any(cat.startswith(f"{ds}#") for ds in authorized_datasets)
+            ]
+            self.logger.debug("过滤类别: %s -> %s", len(all_categories), len(filtered_categories))
+        else:
+            filtered_categories = all_categories
+        
+        self._remote_categories = sorted(filtered_categories)
+        self._remote_categories_set = set(self._remote_categories)
+        self.logger.info(f"加载了 {len(self._remote_categories)} 个类别")
+    
+    def _sync_category(self, category: str, incremental_start_date: datetime = None):
+        """同步单个类别"""
+        category_type = self._validate_category_name(category)
+        
+        if category_type == "pivot":
+            self._sync_pivot_category(category, incremental_start_date)
+        else:
+            self._sync_unstack_category(category)
+    
+    def _sync_pivot_category(self, category: str, incremental_start_date: datetime = None):
+        """同步 Pivot 类别"""
+        if isinstance(incremental_start_date, str):
+            incremental_start_date = datetime.strptime(incremental_start_date, "%Y-%m-%d")
+
+        meta = self._get_category_meta(category)
+        self.LQ_db.set_category_meta(category, meta, merge=True)
+        partitions = meta.get("partition_keys", [])
+        partition_mode = meta.get("partition_mode", "monthly")
+        if partition_mode not in ("daily", "monthly"):
+            raise ProtocolError()
+        
+        if not partitions:
+            self.logger.info(f"{category}: 没有分区")
+            return
+        
+        # 如果指定了增量更新日期，过滤分区
+        filtered_partitions = partitions
+        if incremental_start_date is not None:
+            filtered_partitions = self._filter_partitions_by_date(partitions, incremental_start_date)
+            self.logger.info(f"{category}: 全量分区 {len(partitions)} 个，增量分区 {len(filtered_partitions)} 个")
+        
+        for partition in tqdm(filtered_partitions, desc=f"同步 {category}"):
+            partition_meta = self._get_partition_meta(category, partition)
+            remote_hash = partition_meta.get("hash")
+            
+            # 下载数据
+            df = self._download_partition(category, partition, partition_meta)
+            self.LQ_db.update_pivot_category(
+                df=df,
+                category=category,
+                strf="%Y-%m-%d",
+                partition_mode=partition_mode,
+            )
+            
+            # 保存 hash
+            if remote_hash:
+                self._write_local_hash(category, partition, remote_hash)
+    
+    def _sync_unstack_category(self, category: str):
+        """同步 Unstack 类别（unstack类型通常不分区，直接全量同步）"""
+        partition = "full"
+        category_meta = self._get_category_meta(category)
+        self.LQ_db.set_category_meta(category, category_meta, merge=True)
+        partition_meta = self._get_partition_meta(category, partition)
+        remote_hash = partition_meta.get("hash")
+        duplicate_keys = (
+            partition_meta.get("duplicate_keys")
+            or category_meta.get("duplicate_keys")
+            or partition_meta.get("key_columns")
+            or partition_meta.get("dedupe_keys")
+            or category_meta.get("key_columns")
+            or category_meta.get("dedupe_keys")
+        )
+        
+        df = self._download_partition(category, partition, partition_meta)
+        if not isinstance(duplicate_keys, (list, tuple, str)):
+            duplicate_keys = None
+        self.LQ_db.update_unstack_category(category=category, sub_df=df, duplicate_keys=duplicate_keys)
+        
+        if remote_hash:
+            self._write_local_hash(category, partition, remote_hash)
+    
+    def _download_partition(self, category: str, partition: str, partition_meta: dict) -> pd.DataFrame:
+        """下载分区数据"""
+        data_key = partition_meta.get("data_key")
+        if not data_key:
+            raise MetaError()
+        
+        # 获取数据（带重试）
+        payload = self._redis_get_with_retry(data_key)
+        if payload is None:
+            raise RemoteDataError()
+        
+        # 校验 hash
+        remote_hash = partition_meta.get("hash")
+        if remote_hash:
+            local_hash = "sha256:" + hashlib.sha256(payload).hexdigest()
+            if local_hash != remote_hash:
+                raise SerializationError()
+        
+        # 解析 Parquet
+        try:
+            return pd.read_parquet(io.BytesIO(payload))
+        except Exception:
+            raise SerializationError() from None
+    
+    def _redis_get_with_retry(self, key: str, max_retries: int = 2) -> bytes:
+        """带重试的 Redis GET"""
+        for attempt in range(max_retries + 1):
+            try:
+                self._ensure_connection()
+                return self._redis_client.get(key)
+            except redis.AuthenticationError:
+                if attempt < max_retries:
+                    self._fetch_ticket()
+                    self._connect_redis()
+                else:
+                    raise DataConnectionError()
+            except Exception as e:
+                if isinstance(e, LiteQuantError):
+                    raise self._public_error_clone(e) from None
+                if attempt < max_retries:
+                    time.sleep(0.5)
+                else:
+                    raise RemoteDataError() from None
+    
+    def _json_get(self, key: str):
+        """获取 JSON 数据"""
+        data = self._redis_get_with_retry(key)
+        if data is None:
+            return None
+        try:
+            if isinstance(data, bytes):
+                data = data.decode("utf-8")
+            return json.loads(data)
+        except Exception:
+            raise SerializationError() from None
+    
+    def _json_get_required(self, key: str):
+        """获取必需的 JSON 数据"""
+        obj = self._json_get(key)
+        if obj is None:
+            raise MetaError()
+        return obj
+    
+    def _get_category_meta(self, category: str) -> dict:
+        """获取类别元数据"""
+        key = f"{self.META_CATEGORY_KEY_PREFIX}{category}"
+        meta = self._json_get_required(key)
+        self._validate_protocol(meta)
+        return meta
+    
+    def _get_partition_meta(self, category: str, partition: str) -> dict:
+        """获取分区元数据"""
+        key = f"{self.META_PARTITION_KEY_PREFIX}{category}:{partition}"
+        meta = self._json_get_required(key)
+        self._validate_protocol(meta)
+        return meta
+    
+    def _validate_protocol(self, meta: dict):
+        """验证协议版本"""
+        if meta.get("protocol_version") != self.PROTOCOL_VERSION:
+            raise ProtocolError()
+        if meta.get("data_format") != self.DATA_FORMAT:
+            raise ProtocolError()
+    
+    def _validate_category_name(self, category: str) -> str:
+        """验证类别名称"""
+        if "pivot#" in category:
+            return "pivot"
+        if "unstack#" in category:
+            return "unstack"
+        raise InvalidCategoryError()
+    
+    def _get_hash_path(self, category: str, partition: str) -> str:
+        """获取 hash 文件路径"""
+        hash_dir = os.path.join(self.save_path, "_litequant_meta_hash")
+        os.makedirs(hash_dir, exist_ok=True)
+        safe_name = f"{category}__{partition}".replace("/", "__")
+        return os.path.join(hash_dir, f"{safe_name}.txt")
+    
+    def _check_local_hash(self, category: str, partition: str, remote_hash: str) -> bool:
+        """检查本地 hash 是否匹配"""
+        if not remote_hash:
+            return False
+        path = self._get_hash_path(category, partition)
+        if not os.path.exists(path):
+            return False
+        with open(path, "r", encoding="utf-8") as f:
+            local_hash = f.read().strip()
+        return local_hash == remote_hash
+    
+    def _write_local_hash(self, category: str, partition: str, hash_value: str):
+        """写入本地 hash"""
+        path = self._get_hash_path(category, partition)
+        with open(path, "w", encoding="utf-8") as f:
+            f.write(hash_value)
+    
+    def _filter_partitions_by_date(self, partitions: List[str], start_date: datetime) -> List[str]:
+        """
+        根据日期过滤分区
+        
+        支持的分区格式:
+        - YYYY-MM-DD (日分区)
+        - YYYY-MM (月分区) 
+        - YYYY (年分区)
+        """
+        filtered = []
+        start_date_str = start_date.strftime("%Y-%m-%d")
+        start_month_str = start_date.strftime("%Y-%m")
+        start_year_str = start_date.strftime("%Y")
+        
+        for partition in partitions:
+            # 尝试匹配不同格式的分区
+            if len(partition) == 10 and partition.count('-') == 2:  # YYYY-MM-DD
+                if partition >= start_date_str:
+                    filtered.append(partition)
+            elif len(partition) == 7 and partition.count('-') == 1:  # YYYY-MM
+                if partition >= start_month_str:
+                    filtered.append(partition)
+            elif len(partition) == 4 and partition.isdigit():  # YYYY
+                if partition >= start_year_str:
+                    filtered.append(partition)
+            else:
+                # 无法识别的格式，默认包含
+                filtered.append(partition)
+        
+        return filtered