litequant 3.0.0__py3-none-any.whl

litequant/ParquetManager.py

@@ -0,0 +1,1379 @@
+# -*- coding: utf-8 -*-
+
+import datetime
+import json
+import os
+import re
+import shutil
+import tempfile
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from typing import Callable, Iterable, List, Optional, Tuple
+
+import pandas as pd
+from tqdm import tqdm
+
+from .exceptions import CategoryNotFoundError, LocalStorageError
+from .log import GetLogger
+
+
+def DFSaveToFile(df: pd.DataFrame, filepath: str, file_type: Optional[str] = None, debug: bool = False) -> None:
+    """
+    原子写入文件：
+    1. 写入临时文件
+    2. os.replace 原子替换
+    """
+    filepath = os.path.abspath(filepath)
+    directory = os.path.dirname(filepath)
+    os.makedirs(directory, exist_ok=True)
+
+    if file_type is None:
+        file_type = filepath.split(".")[-1].lower()
+
+    fd, temp_filepath = tempfile.mkstemp(
+        prefix=".tmp_",
+        suffix=f".{file_type}",
+        dir=directory,
+    )
+    os.close(fd)
+
+    try:
+        if file_type == "pickle":
+            df.to_pickle(temp_filepath)
+        elif file_type == "parquet":
+            df.to_parquet(temp_filepath)
+        else:
+            raise ValueError(f"file_type must be 'pickle' or 'parquet', got {file_type}")
+
+        os.replace(temp_filepath, filepath)
+
+        if debug:
+            print(f"File saved successfully: {filepath}")
+
+    except Exception:
+        if os.path.exists(temp_filepath):
+            os.remove(temp_filepath)
+        raise LocalStorageError() from None
+
+
+PIVOT_MONTHLY_SUBDIR = "data_pivot_monthly_partition"
+PIVOT_DAILY_SUBDIR = "data_pivot_daily_partition"
+UNSTACK_SUBDIR = "data_unstack"
+CATEGORY_META_SUBDIR = "category_metadata"
+
+
+class ParquetDataManager:
+    def __init__(self, root_dir: str, max_workers: int = 8, logger=None):
+        self.root_dir = os.path.abspath(root_dir)
+        self.max_workers = max_workers
+        self.logger = logger or GetLogger("litequant.parquet")
+        os.makedirs(self.root_dir, exist_ok=True)
+
+    def _get_typed_category_path(self, category: str, category_type: str, raise_err: bool = False) -> str:
+        mapping = {
+            "pivot_monthly": PIVOT_MONTHLY_SUBDIR,
+            "pivot_daily": PIVOT_DAILY_SUBDIR,
+            "unstack": UNSTACK_SUBDIR,
+        }
+        subdir = mapping.get(category_type, PIVOT_MONTHLY_SUBDIR)
+        dir_path = os.path.join(self.root_dir, subdir, category)
+        if raise_err and not os.path.exists(dir_path):
+            raise CategoryNotFoundError(f"Directory not exists: {dir_path}")
+        return dir_path
+
+    def _get_category_path(self, category: str, raise_err: bool = False) -> str:
+        """自动探测 category 所在的子目录（daily -> monthly -> unstack）"""
+        for subdir in [PIVOT_DAILY_SUBDIR, PIVOT_MONTHLY_SUBDIR, UNSTACK_SUBDIR]:
+            dir_path = os.path.join(self.root_dir, subdir, category)
+            if os.path.exists(dir_path):
+                return dir_path
+        dir_path = os.path.join(self.root_dir, PIVOT_MONTHLY_SUBDIR, category)
+        if raise_err and not os.path.exists(dir_path):
+            raise CategoryNotFoundError(f"Directory not exists: {dir_path}")
+        return dir_path
+
+    def _get_all_categories(self) -> List[str]:
+        categories = []
+        for subdir in [PIVOT_MONTHLY_SUBDIR, PIVOT_DAILY_SUBDIR, UNSTACK_SUBDIR]:
+            path = os.path.join(self.root_dir, subdir)
+            if os.path.exists(path):
+                categories.extend([
+                    x for x in os.listdir(path)
+                    if os.path.isdir(os.path.join(path, x))
+                ])
+        return sorted(categories)
+
+    def _if_exists_unstack_category(self, category: str) -> bool:
+        return os.path.exists(self._get_unstack_file_path(category))
+
+    def _if_exists_pivot_category(self, category: str) -> bool:
+        for category_type in ("pivot_monthly", "pivot_daily"):
+            category_path = self._get_typed_category_path(category, category_type, raise_err=False)
+            if not os.path.exists(category_path):
+                continue
+            parquet_files = [
+                x for x in os.listdir(category_path)
+                if x.endswith(".parquet") and (
+                    re.search(r"=\d{4}-\d{2}\.parquet$", x) or
+                    re.search(r"=\d{4}-\d{2}-\d{2}\.parquet$", x)
+                )
+            ]
+            if len(parquet_files) > 0:
+                return True
+        return False
+
+    @staticmethod
+    def ExtractYearMonth(input_string: str) -> Tuple[Optional[str], Optional[str]]:
+        match = re.search(r"=(\d{4})-(\d{1,2})\.parquet$", input_string)
+        if match:
+            year = match.group(1)
+            month = f"{int(match.group(2)):02d}"
+            return year, month
+        return None, None
+
+    @staticmethod
+    def ExtractYearMonthDay(input_string: str) -> Tuple[Optional[str], Optional[str], Optional[str]]:
+        match = re.search(r"=(?:\(')?(\d{4})-(\d{2})-(\d{2})(?:',\))?\.parquet$", input_string)
+        if match:
+            return match.group(1), match.group(2), match.group(3)
+        return None, None, None
+
+    def _get_pivot_year_month(self, category: str) -> List[Tuple[str, str]]:
+        directory = self._get_typed_category_path(category, "pivot_monthly", raise_err=True)
+        result = []
+        for x in os.listdir(directory):
+            year, month = self.ExtractYearMonth(x)
+            if year is not None:
+                result.append((year, month))
+        return sorted(result)
+
+    def _get_pivot_dates(self, category: str) -> List[str]:
+        directory = self._get_typed_category_path(category, "pivot_daily", raise_err=True)
+        result = []
+        for x in os.listdir(directory):
+            y, m, d = self.ExtractYearMonthDay(x)
+            if y is not None:
+                result.append(f"{y}-{m}-{d}")
+        return sorted(result)
+
+    def _get_pivot_file_path(self, category: str, year: int, month: int) -> str:
+        directory = self._get_typed_category_path(category, "pivot_monthly")
+        return os.path.join(directory, f"{category}={int(year):04d}-{int(month):02d}.parquet")
+
+    def _get_pivot_daily_file_path(self, category: str, date_str: str) -> str:
+        directory = self._get_typed_category_path(category, "pivot_daily")
+        return os.path.join(directory, f"{category}={date_str}.parquet")
+
+    def _get_unstack_file_path(self, category: str) -> str:
+        directory = self._get_typed_category_path(category, "unstack")
+        return os.path.join(directory, f"{category}.parquet")
+
+    @staticmethod
+    def _infer_category_type(category: str) -> str:
+        if "pivot#" in category:
+            return "pivot"
+        if "unstack#" in category:
+            return "unstack"
+        return "unknown"
+
+    @staticmethod
+    def _safe_meta_filename(category: str) -> str:
+        safe_name = re.sub(r'[<>:"/\\|?*]', "_", category)
+        return f"{safe_name}.json"
+
+    def _get_category_meta_dir(self) -> str:
+        return os.path.join(self.root_dir, CATEGORY_META_SUBDIR)
+
+    def _get_category_meta_file_path(self, category: str) -> str:
+        return os.path.join(self._get_category_meta_dir(), self._safe_meta_filename(category))
+
+    @staticmethod
+    def _write_json_atomic(obj: dict, file_path: str) -> None:
+        directory = os.path.dirname(os.path.abspath(file_path))
+        os.makedirs(directory, exist_ok=True)
+
+        fd, temp_filepath = tempfile.mkstemp(
+            prefix=".tmp_",
+            suffix=".json",
+            dir=directory,
+        )
+        os.close(fd)
+
+        try:
+            with open(temp_filepath, "w", encoding="utf-8") as f:
+                json.dump(obj, f, ensure_ascii=False, sort_keys=True, indent=2)
+                f.write("\n")
+            os.replace(temp_filepath, file_path)
+        except Exception:
+            if os.path.exists(temp_filepath):
+                os.remove(temp_filepath)
+            raise LocalStorageError() from None
+
+    def get_category_meta(self, category: str, default: Optional[dict] = None) -> dict:
+        file_path = self._get_category_meta_file_path(category)
+        if not os.path.exists(file_path):
+            return dict(default or {})
+
+        try:
+            with open(file_path, "r", encoding="utf-8") as f:
+                meta = json.load(f)
+        except Exception:
+            raise LocalStorageError() from None
+
+        if not isinstance(meta, dict):
+            raise LocalStorageError() from None
+        return meta
+
+    def set_category_meta(self, category: str, meta: Optional[dict], merge: bool = True) -> dict:
+        if meta is None:
+            meta = {}
+        if not isinstance(meta, dict):
+            raise LocalStorageError() from None
+
+        base = self.get_category_meta(category, default={}) if merge else {}
+        merged = dict(base)
+        merged.update(meta)
+        merged["category"] = category
+        merged.setdefault("category_type", self._infer_category_type(category))
+
+        if merged["category_type"] == "pivot":
+            mode = self._first_meta_value(merged, ("partition_mode", "storage.partition_mode"))
+            merged["partition_mode"] = mode if mode in ("daily", "monthly") else "monthly"
+        elif merged["category_type"] == "unstack":
+            mode = self._first_meta_value(merged, ("partition_mode", "storage.partition_mode"))
+            merged["partition_mode"] = mode if mode == "full" else "full"
+
+        self._write_json_atomic(merged, self._get_category_meta_file_path(category))
+        return merged
+
+    def ensure_category_meta(self, category: str, **fields) -> dict:
+        meta = self.get_category_meta(category, default={})
+        next_meta = dict(meta)
+        next_meta.update({k: v for k, v in fields.items() if v is not None})
+        return self.set_category_meta(category, next_meta, merge=True)
+
+    @staticmethod
+    def _get_nested_value(meta: dict, path: str, default=None):
+        current = meta
+        for part in path.split("."):
+            if not isinstance(current, dict) or part not in current:
+                return default
+            current = current[part]
+        return current
+
+    @staticmethod
+    def _first_meta_value(meta: dict, paths: Iterable[str], default=None):
+        for path in paths:
+            value = ParquetDataManager._get_nested_value(meta, path, default=None)
+            if value is not None:
+                return value
+        return default
+
+    def get_metadata_field(self, category: str, path: str, default=None):
+        meta = self.get_category_meta(category, default={})
+        return self._get_nested_value(meta, path, default=default)
+
+    def get_first_metadata_field(self, category: str, paths: Iterable[str], default=None):
+        meta = self.get_category_meta(category, default={})
+        return self._first_meta_value(meta, paths, default=default)
+
+    def _resolve_partition_mode(self, category: str, explicit_partition_mode: Optional[str] = None) -> Optional[str]:
+        if explicit_partition_mode in ("daily", "monthly", "full"):
+            return explicit_partition_mode
+        meta = self.get_category_meta(category, default={})
+        mode = self._first_meta_value(meta, ("partition_mode", "storage.partition_mode"))
+        if mode in ("daily", "monthly", "full"):
+            return mode
+        return None
+
+    def _resolve_calendar_category(self, category: str) -> Optional[str]:
+        meta = self.get_category_meta(category, default={})
+        value = self._first_meta_value(
+            meta,
+            (
+                "calendar_category",
+                "time.calendar_category",
+                "market.calendar.calendar_category",
+                "references.calendar_category",
+            ),
+        )
+        return str(value) if value else None
+
+    def _resolve_duplicate_keys(self, category: str) -> List[str]:
+        meta = self.get_category_meta(category, default={})
+        value = self._first_meta_value(
+            meta,
+            (
+                "duplicate_keys",
+                "schema_contract.primary_key",
+            ),
+            default=[],
+        )
+        if not isinstance(value, list):
+            return []
+        return [str(x).strip() for x in value if str(x).strip()]
+
+    def _resolve_date_column(self, category: str, df: Optional[pd.DataFrame] = None) -> Optional[str]:
+        meta = self.get_category_meta(category, default={})
+        value = self._first_meta_value(
+            meta,
+            (
+                "date_column",
+                "time.date_column",
+                "time.timestamp_column",
+            ),
+        )
+        if value and (df is None or str(value) in df.columns):
+            return str(value)
+        if df is not None and not df.empty:
+            return next(
+                (col for col in ("trade_date", "cal_date", "date", "timestamp") if col in df.columns),
+                df.columns[0],
+            )
+        return str(value) if value else None
+
+    def _resolve_is_open_column(self, category: str, df: Optional[pd.DataFrame] = None) -> Optional[str]:
+        meta = self.get_category_meta(category, default={})
+        value = self._first_meta_value(
+            meta,
+            (
+                "is_open_column",
+                "market.calendar.is_open_column",
+            ),
+        )
+        if value and (df is None or str(value) in df.columns):
+            return str(value)
+        if df is not None and "is_open" in df.columns:
+            return "is_open"
+        return str(value) if value else None
+
+    @staticmethod
+    def _normalize_date_columns(df: pd.DataFrame) -> pd.DataFrame:
+        if df.empty:
+            return df.copy()
+        out = df.copy()
+        out.columns = pd.to_datetime(out.columns)
+        out.sort_index(axis=0, inplace=True)
+        out.sort_index(axis=1, inplace=True)
+        return out
+
+    @staticmethod
+    def _format_date_columns(df: pd.DataFrame, date_format: str = "%Y-%m-%d") -> pd.DataFrame:
+        if df.empty:
+            return df.copy()
+        out = df.copy()
+        out.columns = pd.to_datetime(out.columns).strftime(date_format)
+        return out
+
+    @staticmethod
+    def _drop_all_null_rows(df: pd.DataFrame) -> pd.DataFrame:
+        if df.empty:
+            return df
+        return df[df.notnull().sum(axis=1) > 0]
+
+    @staticmethod
+    def _drop_duplicate_index_columns(df: pd.DataFrame) -> pd.DataFrame:
+        if df.empty:
+            return df
+
+        out = df
+        if out.index.duplicated().any():
+            out = out[~out.index.duplicated(keep="last")]
+        if out.columns.duplicated().any():
+            out = out.loc[:, ~out.columns.duplicated(keep="last")]
+        return out
+
+    def update_pivot_category(
+        self,
+        df: pd.DataFrame,
+        category: str,
+        strf: str = "%Y-%m-%d",
+        partition_mode: Optional[str] = None,
+        replace_partition: bool = False,
+    ) -> None:
+        partition_mode = self._resolve_partition_mode(category, explicit_partition_mode=partition_mode)
+        if partition_mode not in ("monthly", "daily"):
+            partition_mode = "monthly"
+
+        self.ensure_category_meta(
+            category,
+            category_type="pivot",
+            partition_mode=partition_mode,
+        )
+
+        if df is None or df.empty:
+            self.logger.info("%s: input df is empty, skip update.", category)
+            return
+
+        df = df.copy()
+        df.columns = pd.to_datetime(df.columns)
+        df = df.dropna(axis=1, how="all")
+
+        if df.empty:
+            self.logger.info("%s: all columns are empty after dropna, skip update.", category)
+            return
+
+        df = self._drop_duplicate_index_columns(df)
+
+        if partition_mode == "daily":
+            os.makedirs(self._get_typed_category_path(category, "pivot_daily"), exist_ok=True)
+            grouped = df.T.groupby([df.columns.strftime("%Y-%m-%d")])
+            tasks = [
+                [
+                    self._get_pivot_daily_file_path(category, date_key[0] if isinstance(date_key, tuple) else date_key),
+                    sub_df.T,
+                    strf,
+                    replace_partition,
+                ]
+                for date_key, sub_df in grouped
+            ]
+        else:
+            os.makedirs(self._get_typed_category_path(category, "pivot_monthly"), exist_ok=True)
+            grouped = df.T.groupby([df.columns.year, df.columns.month])
+            tasks = [
+                [self._get_pivot_file_path(category, int(year), int(month)), sub_df.T, strf, replace_partition]
+                for (year, month), sub_df in grouped
+            ]
+
+        if not tasks:
+            return
+
+        if len(tasks) == 1:
+            ParquetDataManager.UpdatePivotParquet(*tasks[0])
+        else:
+            self.RunConcurrently(
+                tasks=tasks,
+                worker_function=ParquetDataManager.UpdatePivotParquet,
+                max_workers=self.max_workers,
+                task_desc=f"Updating {category}",
+            )
+
+    @staticmethod
+    def UpdatePivotParquet(
+        file_path: str,
+        sub_df: pd.DataFrame,
+        date_format: str = "%Y-%m-%d",
+        replace_partition: bool = False,
+    ) -> None:
+        if sub_df is None or sub_df.empty:
+            return
+
+        sub_df = sub_df.copy()
+        sub_df = ParquetDataManager._drop_all_null_rows(sub_df)
+        if sub_df.empty:
+            if replace_partition and os.path.exists(file_path):
+                os.remove(file_path)
+            return
+
+        try:
+            sub_df = ParquetDataManager._normalize_date_columns(sub_df)
+            sub_df = ParquetDataManager._drop_duplicate_index_columns(sub_df)
+        except Exception:
+            raise LocalStorageError() from None
+
+        if replace_partition:
+            write_df = ParquetDataManager._format_date_columns(sub_df, date_format=date_format)
+            DFSaveToFile(write_df, file_path, debug=False)
+            return
+
+        if not os.path.exists(file_path):
+            write_df = ParquetDataManager._format_date_columns(sub_df, date_format=date_format)
+            DFSaveToFile(write_df, file_path, debug=False)
+            return
+
+        try:
+            existing_df = pd.read_parquet(file_path)
+            existing_df = ParquetDataManager._normalize_date_columns(existing_df)
+            existing_df = ParquetDataManager._drop_duplicate_index_columns(existing_df)
+        except Exception:
+            raise LocalStorageError() from None
+
+        old_index = existing_df.index
+        old_columns = existing_df.columns
+
+        common_index = old_index.union(sub_df.index)
+        common_columns = old_columns.union(sub_df.columns)
+
+        existing_df = existing_df.reindex(index=common_index, columns=common_columns)
+        sub_df = sub_df.reindex(index=common_index, columns=common_columns)
+
+        has_new_index = len(common_index.difference(old_index)) > 0
+        has_new_columns = len(common_columns.difference(old_columns)) > 0
+
+        candidate_mask = sub_df.notna()
+        candidate_count = int(candidate_mask.to_numpy().sum())
+
+        if candidate_count == 0 and not has_new_index and not has_new_columns:
+            return
+
+        diff_mask = candidate_mask & (existing_df.isna() | existing_df.ne(sub_df))
+        diff_count = int(diff_mask.to_numpy().sum())
+
+        if diff_count == 0 and not has_new_index and not has_new_columns:
+            return
+
+        with pd.option_context("future.no_silent_downcasting", True):
+            merged_df = existing_df.where(~candidate_mask, sub_df)
+        merged_df = merged_df.infer_objects(copy=False)
+        merged_df = ParquetDataManager._drop_all_null_rows(merged_df)
+        merged_df = ParquetDataManager._drop_duplicate_index_columns(merged_df)
+        merged_df.sort_index(axis=0, inplace=True)
+        merged_df.sort_index(axis=1, inplace=True)
+
+        write_df = ParquetDataManager._format_date_columns(merged_df, date_format=date_format)
+        DFSaveToFile(write_df, file_path, debug=False)
+
+    @staticmethod
+    def _normalize_date_strings(values: Iterable) -> List[str]:
+        if values is None:
+            return []
+        dates = pd.to_datetime(list(values), errors="coerce")
+        if len(dates) == 0:
+            return []
+        valid_dates = pd.Series(dates).dropna()
+        if valid_dates.empty:
+            return []
+        return sorted(valid_dates.dt.strftime("%Y-%m-%d").unique().tolist())
+
+    @staticmethod
+    def _filter_open_dates(df: pd.DataFrame, is_open_col: Optional[str]) -> pd.DataFrame:
+        if not is_open_col or is_open_col not in df.columns:
+            return df
+        open_flag = df[is_open_col]
+        if pd.api.types.is_numeric_dtype(open_flag):
+            return df[open_flag.fillna(0).astype(int) == 1]
+        return df[open_flag.astype(str).str.strip().isin(("1", "True", "true", "\u4ea4\u6613"))]
+
+    @staticmethod
+    def _us_juneteenth_observed_date(year: int) -> datetime.date:
+        holiday = datetime.date(year, 6, 19)
+        if holiday.weekday() == 5:
+            return datetime.date(year, 6, 18)
+        if holiday.weekday() == 6:
+            return datetime.date(year, 6, 20)
+        return holiday
+
+    @staticmethod
+    def _filter_stock_calendar_dates(calendar_category: str, dates: List[str]) -> List[str]:
+        if not dates or "_stock_unstack#trade_date" not in calendar_category:
+            return dates
+
+        result = []
+        for date_str in dates:
+            try:
+                date_obj = datetime.datetime.strptime(date_str, "%Y-%m-%d").date()
+            except ValueError:
+                continue
+            if date_obj.weekday() >= 5:
+                continue
+            if (
+                calendar_category.startswith("us_stock_")
+                and date_obj.year >= 2022
+                and date_obj == ParquetDataManager._us_juneteenth_observed_date(date_obj.year)
+            ):
+                continue
+            result.append(date_str)
+        return result
+
+    def get_calendar_trade_dates(self, calendar_category: str) -> List[str]:
+        file_path = self._get_unstack_file_path(calendar_category)
+        if not os.path.exists(file_path):
+            raise LocalStorageError() from None
+
+        try:
+            df = pd.read_parquet(file_path)
+        except Exception:
+            raise LocalStorageError() from None
+
+        if df.empty:
+            return []
+
+        date_col = self._resolve_date_column(calendar_category, df=df)
+        if not date_col or date_col not in df.columns:
+            date_col = next(
+                (col for col in ("trade_date", "cal_date", "date") if col in df.columns),
+                df.columns[0],
+            )
+
+        is_open_col = self._resolve_is_open_column(calendar_category, df=df)
+        work_df = self._filter_open_dates(df, is_open_col)
+        dates = self._normalize_date_strings(work_df[date_col])
+        return self._filter_stock_calendar_dates(calendar_category, dates)
+
+    def _get_expected_trade_dates(
+        self,
+        category: str,
+        start_date: Optional[str],
+        end_date: Optional[str],
+    ) -> Tuple[Optional[str], List[str]]:
+        calendar_category = self._resolve_calendar_category(category)
+        if not calendar_category:
+            return None, []
+
+        trade_dates = self.get_calendar_trade_dates(str(calendar_category))
+        if start_date:
+            start = pd.to_datetime(start_date).strftime("%Y-%m-%d")
+            trade_dates = [x for x in trade_dates if x >= start]
+        if end_date:
+            end = pd.to_datetime(end_date).strftime("%Y-%m-%d")
+            trade_dates = [x for x in trade_dates if x <= end]
+        return str(calendar_category), trade_dates
+
+    def _missing_trade_dates_from_actual(
+        self,
+        category: str,
+        start_date: Optional[str],
+        end_date: Optional[str],
+        actual_dates: Iterable,
+    ) -> Tuple[Optional[str], List[str]]:
+        calendar_category, expected_dates = self._get_expected_trade_dates(category, start_date, end_date)
+        if not calendar_category or not expected_dates:
+            return calendar_category, []
+
+        actual_set = set(self._normalize_date_strings(actual_dates))
+        missing_dates = [x for x in expected_dates if x not in actual_set]
+        return calendar_category, missing_dates
+
+    def _raise_if_pivot_missing_trade_dates(
+        self,
+        category: str,
+        start_date: Optional[str],
+        end_date: Optional[str],
+        actual_dates: Iterable,
+    ) -> None:
+        _, missing_dates = self._missing_trade_dates_from_actual(
+            category=category,
+            start_date=start_date,
+            end_date=end_date,
+            actual_dates=actual_dates,
+        )
+        if missing_dates:
+            raise LocalStorageError() from None
+
+    def check_pivot_missing_trade_dates(
+        self,
+        category: str,
+        start_date: Optional[str] = None,
+        end_date: Optional[str] = None,
+    ) -> List[str]:
+        df = self.read_pivot_category(
+            category=category,
+            start_date=start_date,
+            end_date=end_date,
+            raise_err=False,
+            validate_calendar=False,
+            show_progress=False,
+        )
+        if df.empty:
+            if start_date and end_date:
+                _, missing_dates = self._missing_trade_dates_from_actual(
+                    category=category,
+                    start_date=start_date,
+                    end_date=end_date,
+                    actual_dates=[],
+                )
+                return missing_dates
+            return []
+
+        actual_dates = self._normalize_date_strings(df.columns)
+        effective_start = start_date or (actual_dates[0] if actual_dates else None)
+        effective_end = end_date or (actual_dates[-1] if actual_dates else None)
+        _, missing_dates = self._missing_trade_dates_from_actual(
+            category=category,
+            start_date=effective_start,
+            end_date=effective_end,
+            actual_dates=actual_dates,
+        )
+        return missing_dates
+
+    def read_pivot_category(
+        self,
+        category: str,
+        start_date: Optional[str] = None,
+        end_date: Optional[str] = None,
+        column_type: str = "str",
+        raise_err: bool = True,
+        partition_mode: Optional[str] = None,
+        show_progress: bool = True,
+        validate_calendar: bool = True,
+    ) -> pd.DataFrame:
+        requested_start_date = start_date
+        requested_end_date = end_date
+        partition_mode = self._resolve_partition_mode(category, explicit_partition_mode=partition_mode)
+
+        # auto detect partition_mode by subdir existence
+        if partition_mode is None:
+            if os.path.exists(self._get_typed_category_path(category, "pivot_daily", raise_err=False)):
+                partition_mode = "daily"
+            else:
+                partition_mode = "monthly"
+
+        if partition_mode == "daily":
+            directory = self._get_typed_category_path(category, "pivot_daily", raise_err=False)
+            if not os.path.exists(directory):
+                if validate_calendar:
+                    validation_start = requested_start_date or start_date
+                    validation_end = requested_end_date or end_date
+                    self._raise_if_pivot_missing_trade_dates(category, validation_start, validation_end, [])
+                if raise_err:
+                    raise CategoryNotFoundError() from None
+                return pd.DataFrame()
+
+            exists_parquets = [f for f in os.listdir(directory) if f.endswith(".parquet")]
+            part_file_map = {}
+            for name in exists_parquets:
+                y, m, d = self.ExtractYearMonthDay(name)
+                if y is not None:
+                    part_file_map[f"{y}-{m}-{d}"] = name
+            exists_parts = sorted(part_file_map)
+
+            if not exists_parts:
+                self.logger.info("数据不存在 %s", category)
+                return pd.DataFrame()
+
+            if start_date is None:
+                start_date = exists_parts[0]
+            if end_date is None:
+                end_date = exists_parts[-1]
+
+            read_parts = [p for p in exists_parts if start_date <= p <= end_date]
+
+            if not read_parts:
+                self.logger.info("数据不存在 %s: [%s, %s]", category, start_date, end_date)
+                if validate_calendar:
+                    validation_start = requested_start_date or start_date
+                    validation_end = requested_end_date or end_date
+                    self._raise_if_pivot_missing_trade_dates(category, validation_start, validation_end, [])
+                return pd.DataFrame()
+
+            tasks = [[os.path.join(directory, part_file_map[p])] for p in read_parts]
+        else:
+            directory = self._get_typed_category_path(category, "pivot_monthly", raise_err=False)
+            if not os.path.exists(directory):
+                if validate_calendar:
+                    validation_start = requested_start_date or start_date
+                    validation_end = requested_end_date or end_date
+                    self._raise_if_pivot_missing_trade_dates(category, validation_start, validation_end, [])
+                if raise_err:
+                    raise CategoryNotFoundError() from None
+                return pd.DataFrame()
+
+            exists_parquets = [f for f in os.listdir(directory) if f.endswith(".parquet")]
+            date_pattern = re.compile(r"(\d{4})-(\d{2})\.parquet$")
+            exists_ms = []
+            for name in exists_parquets:
+                match = date_pattern.search(name)
+                if match:
+                    exists_ms.append(f"{match.group(1)}-{match.group(2)}")
+            exists_ms = sorted(set(exists_ms))
+
+            if not exists_ms:
+                self.logger.info("数据不存在 %s", category)
+                return pd.DataFrame()
+
+            if start_date is None:
+                start_date = "2000-01-01"
+            if end_date is None:
+                end_date = (pd.Timestamp(exists_ms[-1]) + pd.offsets.MonthEnd(0)).strftime("%Y-%m-%d")
+
+            start_month = pd.to_datetime(start_date).replace(day=1)
+            end_month = pd.to_datetime(end_date).replace(day=1)
+            target_ms = pd.date_range(start=start_month, end=end_month, freq="MS").strftime("%Y-%m").tolist()
+            read_ms = sorted(set(target_ms) & set(exists_ms))
+
+            if not read_ms:
+                self.logger.info("数据不存在 %s: [%s, %s]", category, start_date, end_date)
+                if validate_calendar:
+                    validation_start = requested_start_date or start_date
+                    validation_end = requested_end_date or end_date
+                    self._raise_if_pivot_missing_trade_dates(category, validation_start, validation_end, [])
+                return pd.DataFrame()
+
+            tasks = [[os.path.join(directory, f"{category}={ms}.parquet")] for ms in read_ms]
+
+        results = self.RunConcurrently(
+            tasks=tasks,
+            worker_function=ParquetDataManager.ReadParquet,
+            max_workers=self.max_workers,
+            task_desc=f"Reading {category}" if show_progress else None,
+        )
+
+        valid_results = [x for x in results if isinstance(x, pd.DataFrame) and not x.empty]
+        if not valid_results:
+            if validate_calendar:
+                validation_start = requested_start_date or start_date
+                validation_end = requested_end_date or end_date
+                self._raise_if_pivot_missing_trade_dates(category, validation_start, validation_end, [])
+            return pd.DataFrame()
+
+        dfs = pd.concat(valid_results, axis=1)
+        dfs = self._drop_duplicate_index_columns(dfs)
+
+        if column_type == "datetime":
+            dfs.columns = pd.to_datetime(dfs.columns)
+            dfs.sort_index(axis=0, inplace=True)
+            dfs.sort_index(axis=1, inplace=True)
+            start = pd.to_datetime(start_date)
+            end = pd.to_datetime(end_date)
+            result_df = dfs.loc[:, (dfs.columns >= start) & (dfs.columns <= end)]
+            if validate_calendar:
+                actual_dates = self._normalize_date_strings(result_df.columns)
+                validation_start = requested_start_date or (actual_dates[0] if actual_dates else start_date)
+                validation_end = requested_end_date or (actual_dates[-1] if actual_dates else end_date)
+                self._raise_if_pivot_missing_trade_dates(category, validation_start, validation_end, result_df.columns)
+            return result_df
+
+        dfs.columns = pd.to_datetime(dfs.columns).strftime("%Y-%m-%d")
+        dfs.sort_index(axis=0, inplace=True)
+        dfs.sort_index(axis=1, inplace=True)
+
+        start = pd.to_datetime(start_date).strftime("%Y-%m-%d")
+        end = pd.to_datetime(end_date).strftime("%Y-%m-%d")
+        result_df = dfs.loc[:, (dfs.columns >= start) & (dfs.columns <= end)]
+        if validate_calendar:
+            actual_dates = self._normalize_date_strings(result_df.columns)
+            validation_start = requested_start_date or (actual_dates[0] if actual_dates else start_date)
+            validation_end = requested_end_date or (actual_dates[-1] if actual_dates else end_date)
+            self._raise_if_pivot_missing_trade_dates(category, validation_start, validation_end, result_df.columns)
+        return result_df
+
+    def ReadPivotCategoryLastNDays(
+        self,
+        category: str,
+        n_days: int = 20,
+        column_type: str = "datetime",
+        raise_err: bool = True,
+    ) -> pd.DataFrame:
+        latest_date = self.GetCategoryLatestDate(category)
+        if latest_date is None:
+            return pd.DataFrame()
+
+        start_date = (pd.Timestamp(latest_date) - pd.Timedelta(days=n_days * 3)).strftime("%Y-%m-%d")
+        df = self.read_pivot_category(
+            category=category,
+            start_date=start_date,
+            end_date=pd.Timestamp(latest_date).strftime("%Y-%m-%d"),
+            column_type=column_type,
+            raise_err=raise_err,
+        )
+        if df.empty:
+            return df
+        return df.iloc[:, -n_days:]
+
+    @staticmethod
+    def ReadParquet(file_path: str) -> pd.DataFrame:
+        return pd.read_parquet(file_path)
+
+    def update_unstack_category(
+        self,
+        category: str,
+        sub_df: pd.DataFrame,
+        duplicate_keys: Optional[Iterable[str]] = None,
+        key_columns: Optional[Iterable[str]] = None,
+    ) -> None:
+        if duplicate_keys is None:
+            duplicate_keys = key_columns
+        if duplicate_keys is None:
+            duplicate_keys = self._resolve_duplicate_keys(category)
+
+        resolved_key_columns = (
+            self.ResolveUnstackKeyColumns(sub_df, key_columns=duplicate_keys)
+            if sub_df is not None
+            else None
+        )
+        self.ensure_category_meta(
+            category,
+            category_type="unstack",
+            partition_mode="full",
+            duplicate_keys=resolved_key_columns,
+        )
+
+        if sub_df is None or sub_df.empty:
+            self.logger.info("%s: sub_df is empty, skip update.", category)
+            return
+
+        os.makedirs(self._get_typed_category_path(category, "unstack"), exist_ok=True)
+        file_path = self._get_unstack_file_path(category)
+        sub_df = self.NormalizeUnstackDf(sub_df, key_columns=resolved_key_columns)
+
+        if os.path.exists(file_path):
+            existing_df = pd.read_parquet(file_path)
+            if existing_df.equals(sub_df):
+                self.logger.info("%s: 无需更新，DataFrames identical.", file_path)
+                return
+
+            new_df = self.UpdateUnstackDf(existing_df, sub_df, duplicate_keys=resolved_key_columns)
+            DFSaveToFile(new_df, file_path, debug=False)
+            self.logger.info("%s: 更新 Unstack，Pre=%s After=%s", file_path, existing_df.shape, new_df.shape)
+        else:
+            DFSaveToFile(sub_df, file_path, debug=False)
+            self.logger.info("%s: 初始化 Unstack Shape=%s", file_path, sub_df.shape)
+
+    def read_unstack_category(self, category: str, raise_err: bool = True) -> pd.DataFrame:
+        file_path = self._get_unstack_file_path(category)
+        if raise_err:
+            return pd.read_parquet(file_path)
+
+        try:
+            return pd.read_parquet(file_path)
+        except Exception:
+            self.logger.warning("数据暂时不可用，请稍后重试")
+            return pd.DataFrame()
+
+    @staticmethod
+    def ResolveUnstackKeyColumns(
+        df: pd.DataFrame,
+        key_columns: Optional[Iterable[str]] = None,
+    ) -> List[str]:
+        if key_columns is None:
+            columns_name = df.columns.name
+            if columns_name is not None and str(columns_name).startswith("keys:"):
+                key_part = str(columns_name)[5:]
+                key_columns = [x.strip() for x in key_part.split(",") if x.strip()]
+
+        if key_columns is None:
+            return list(df.columns)
+
+        if isinstance(key_columns, str):
+            key_columns = [x.strip() for x in key_columns.split(",") if x.strip()]
+        else:
+            key_columns = [str(x).strip() for x in key_columns if str(x).strip()]
+
+        missing = [col for col in key_columns if col not in df.columns]
+        if missing:
+            raise LocalStorageError()
+
+        return key_columns
+
+    @staticmethod
+    def NormalizeUnstackDf(
+        df: pd.DataFrame,
+        key_columns: Optional[Iterable[str]] = None,
+    ) -> pd.DataFrame:
+        if df is None or df.empty:
+            return df
+
+        out = df.copy()
+        resolved_keys = ParquetDataManager.ResolveUnstackKeyColumns(out, key_columns=key_columns)
+        if resolved_keys:
+            out = out.drop_duplicates(subset=resolved_keys, keep="last")
+        else:
+            out = out.drop_duplicates(keep="last")
+        return out
+
+    @staticmethod
+    def UpdateUnstackDf(
+        original_df: pd.DataFrame,
+        new_data: pd.DataFrame,
+        duplicate_keys: Optional[Iterable[str]] = None,
+        key_columns: Optional[Iterable[str]] = None,
+    ) -> pd.DataFrame:
+        """
+        合并 unstack 数据，支持指定去重键
+        
+        去重键指定方式：
+        - duplicate_keys / key_columns 显式传入时优先
+        - columns.name 以 "keys:" 开头，如 "keys:股票代码,计入日期"
+        - 否则按所有列去重
+        """
+        updated_df = pd.concat([original_df, new_data], axis=0)
+        if duplicate_keys is None:
+            duplicate_keys = key_columns
+        key_columns = duplicate_keys
+        key_columns = ParquetDataManager.ResolveUnstackKeyColumns(updated_df, key_columns=key_columns)
+
+        if len(key_columns) > 0:
+            updated_df = updated_df.drop_duplicates(subset=key_columns, keep="last")
+        else:
+            updated_df = updated_df.drop_duplicates(keep="last")
+
+        return updated_df
+
+    @staticmethod
+    def RunConcurrently(
+        tasks: List[list],
+        worker_function: Callable,
+        max_workers: int = 5,
+        task_desc: Optional[str] = "Processing tasks",
+        fail_fast: bool = True,
+    ) -> List:
+        if tasks is None or len(tasks) == 0:
+            return []
+
+        results = [None] * len(tasks)
+        errors = []
+
+        with ThreadPoolExecutor(max_workers=max_workers) as executor:
+            future_to_idx = {
+                executor.submit(worker_function, *task): idx
+                for idx, task in enumerate(tasks)
+            }
+
+            with tqdm(total=len(future_to_idx), desc=task_desc, disable=task_desc is None) as pbar:
+                for future in as_completed(future_to_idx):
+                    idx = future_to_idx[future]
+                    try:
+                        results[idx] = future.result()
+                    except Exception as e:
+                        errors.append((idx, e))
+                        if fail_fast:
+                            raise
+                    finally:
+                        pbar.update(1)
+
+        if errors and fail_fast:
+            raise errors[0][1]
+
+        return results
+
+    def GetCategoryLatestDate(self, category: str, partition_mode: Optional[str] = None):
+        if not self._if_exists_pivot_category(category):
+            return None
+
+        partition_mode = self._resolve_partition_mode(category, explicit_partition_mode=partition_mode)
+
+        # auto detect partition_mode by subdir existence
+        if partition_mode is None:
+            if os.path.exists(self._get_typed_category_path(category, "pivot_daily", raise_err=False)):
+                partition_mode = "daily"
+            else:
+                partition_mode = "monthly"
+
+        if partition_mode == "daily":
+            date_list = self._get_pivot_dates(category)
+            if len(date_list) == 0:
+                return None
+
+            for latest_date in reversed(date_list):
+                file_path = self._get_pivot_daily_file_path(category, latest_date)
+                if not os.path.exists(file_path):
+                    continue
+
+                df = pd.read_parquet(file_path)
+                if df.empty or len(df.columns) == 0:
+                    continue
+
+                cols = pd.to_datetime(df.columns)
+                return cols.max()
+        else:
+            ym_list = self._get_pivot_year_month(category)
+            if len(ym_list) == 0:
+                return None
+
+            for latest_y, latest_m in reversed(ym_list):
+                file_path = self._get_pivot_file_path(category, int(latest_y), int(latest_m))
+                if not os.path.exists(file_path):
+                    continue
+
+                df = pd.read_parquet(file_path)
+                if df.empty or len(df.columns) == 0:
+                    continue
+
+                cols = pd.to_datetime(df.columns)
+                return cols.max()
+
+        return None
+
+    def get_download_days(self, category: str, template_date: List, iloc: Optional[int] = None) -> List:
+        """
+        获取需要下载的日期列表。
+        
+        Args:
+            category: 数据类别名称
+            template_date: 模板日期列表
+            iloc: 可选，从末尾删除的列数
+            
+        Returns:
+            [exists_df, update_dates]: 已存在的数据框和需要更新的日期列表
+        """
+        category_path = self._get_category_path(category, raise_err=False)
+
+        if os.path.exists(category_path):
+            exists_df = self.read_pivot_category(
+                category=category,
+                start_date=min(template_date),
+                end_date=max(template_date),
+            )
+        else:
+            exists_df = pd.DataFrame()
+
+        if iloc:
+            exists_df = exists_df.iloc[:, :-iloc]
+
+        existing_dates = set(pd.to_datetime(exists_df.columns)) if not exists_df.empty else set()
+        template_dates = set(pd.to_datetime(template_date))
+        update_dates = sorted(template_dates - existing_dates)
+
+        return [exists_df, update_dates]
+
+    def GetCategoryDateCoverage(self, category: str, partition_mode: Optional[str] = None):
+        if not self._if_exists_pivot_category(category):
+            return {
+                "min_date": None,
+                "max_date": None,
+                "partition_count": 0,
+            }
+
+        partition_mode = self._resolve_partition_mode(category, explicit_partition_mode=partition_mode)
+
+        # auto detect partition_mode by subdir existence
+        if partition_mode is None:
+            if os.path.exists(self._get_typed_category_path(category, "pivot_daily", raise_err=False)):
+                partition_mode = "daily"
+            else:
+                partition_mode = "monthly"
+
+        if partition_mode == "daily":
+            date_list = self._get_pivot_dates(category)
+            if len(date_list) == 0:
+                return {
+                    "min_date": None,
+                    "max_date": None,
+                    "partition_count": 0,
+                }
+
+            first_fp = self._get_pivot_daily_file_path(category, date_list[0])
+            last_fp = self._get_pivot_daily_file_path(category, date_list[-1])
+
+            first_df = pd.read_parquet(first_fp)
+            last_df = pd.read_parquet(last_fp)
+
+            min_date = pd.to_datetime(first_df.columns).min() if len(first_df.columns) > 0 else None
+            max_date = pd.to_datetime(last_df.columns).max() if len(last_df.columns) > 0 else None
+
+            return {
+                "min_date": min_date,
+                "max_date": max_date,
+                "partition_count": len(date_list),
+            }
+        else:
+            ym_list = self._get_pivot_year_month(category)
+            if len(ym_list) == 0:
+                return {
+                    "min_date": None,
+                    "max_date": None,
+                    "partition_count": 0,
+                }
+
+            first_y, first_m = ym_list[0]
+            last_y, last_m = ym_list[-1]
+
+            first_fp = self._get_pivot_file_path(category, int(first_y), int(first_m))
+            last_fp = self._get_pivot_file_path(category, int(last_y), int(last_m))
+
+            first_df = pd.read_parquet(first_fp)
+            last_df = pd.read_parquet(last_fp)
+
+            min_date = pd.to_datetime(first_df.columns).min() if len(first_df.columns) > 0 else None
+            max_date = pd.to_datetime(last_df.columns).max() if len(last_df.columns) > 0 else None
+
+            return {
+                "min_date": min_date,
+                "max_date": max_date,
+                "partition_count": len(ym_list),
+            }
+
+    def DeleteCategory(self, category: str) -> None:
+        category_path = self._get_category_path(category, raise_err=False)
+        if os.path.exists(category_path):
+            shutil.rmtree(category_path)
+            self.logger.info("Deleted category: %s", category_path)
+        else:
+            self.logger.info("Category not exists: %s", category_path)
+
+    def _get_local_trade_dates_until_today(self) -> List[str]:
+        today = pd.Timestamp(datetime.datetime.now().date())
+        for category in (
+                "cn_stock_unstack#trade_date",
+                "trade_date",
+                "us_stock_unstack#trade_date",
+                "us_trade_date",
+        ):
+            file_path = self._get_unstack_file_path(category)
+            if not os.path.exists(file_path):
+                continue
+            try:
+                df = pd.read_parquet(file_path)
+            except Exception:
+                continue
+            if df.empty:
+                continue
+
+            date_col = next(
+                (col for col in ("trade_date", "cal_date", "date") if col in df.columns),
+                df.columns[0],
+            )
+            work_df = df.copy()
+            if "is_open" in work_df.columns:
+                open_flag = work_df["is_open"]
+                if pd.api.types.is_numeric_dtype(open_flag):
+                    work_df = work_df[open_flag.fillna(0).astype(int) == 1]
+                else:
+                    work_df = work_df[open_flag.astype(str).str.strip().isin(("1", "True", "true", "交易"))]
+
+            dates = pd.to_datetime(work_df[date_col].astype(str), errors="coerce")
+            dates = dates[(dates.notna()) & (dates <= today)]
+            if dates.empty:
+                continue
+            return sorted(dates.dt.strftime("%Y-%m-%d").unique().tolist())
+        return []
+
+    def CheckPivotCategoriesDates(
+            self,
+            exclude_category_keywords: Optional[Iterable[str]] = None,
+    ) -> pd.DataFrame:
+        pivot_categories = [x for x in self._get_all_categories() if "pivot#" in x]
+        if exclude_category_keywords:
+            exclude_keywords = tuple(exclude_category_keywords)
+            pivot_categories = [
+                x for x in pivot_categories
+                if not any(keyword in x for keyword in exclude_keywords)
+            ]
+
+        date_list = []
+        data_nums = []
+        file_sizes = []
+
+        with tqdm(pivot_categories, desc="检查本地 Pivot 最新日期") as pbar:
+            for category in pbar:
+                pbar.set_postfix_str(f"last_process: {category}")
+                df = self.read_pivot_category(
+                    category,
+                    start_date=(datetime.datetime.now() - datetime.timedelta(days=30)).strftime("%Y-%m-%d"),
+                    show_progress=False,
+                    validate_calendar=False,
+                )
+
+                if df.empty:
+                    date_list.append(pd.NaT)
+                    data_nums.append(0)
+                else:
+                    if df.columns.duplicated().sum() > 0:
+                        raise LocalStorageError(f"{category} columns duplicated")
+                    if df.index.duplicated().sum() > 0:
+                        raise LocalStorageError(f"{category} index duplicated")
+
+                    latest_col = pd.to_datetime(df.columns).max()
+                    date_list.append(latest_col)
+                    data_nums.append(len(df.iloc[:, -1]))
+
+                category_dir = self._get_category_path(category)
+                total_size_bytes = sum(
+                    os.path.getsize(os.path.join(category_dir, x))
+                    for x in os.listdir(category_dir)
+                    if os.path.isfile(os.path.join(category_dir, x))
+                )
+                file_sizes.append(f"{round(total_size_bytes / 1024 / 1024, 2)} MB")
+
+        check_df = pd.DataFrame({
+            "category": pivot_categories,
+            "Latest Date": date_list,
+            "数据量": data_nums,
+            "储存大小": file_sizes,
+        })
+
+        check_df.columns.name = "本地Pivot"
+        check_df.set_index("category", inplace=True)
+        check_df.sort_values("数据量", inplace=True)
+        return check_df
+
+    def ShowCategoriesDateSummary(
+            self,
+            exclude_pivot_category_keywords: Optional[Iterable[str]] = None,
+    ) -> None:
+        """
+        显示本地 Parquet 数据的日期汇总（包括 Pivot 和 Unstack 数据）
+        """
+        import datetime
+        
+        print("\n" + "="*100)
+        print(f"本地 Parquet 数据日期汇总")
+        print("="*100)
+        
+        today = datetime.datetime.now().strftime("%Y-%m-%d")
+        yesterday = (datetime.datetime.now() - datetime.timedelta(days=1)).strftime("%Y-%m-%d")
+        trade_dates = self._get_local_trade_dates_until_today()
+        latest_expected = trade_dates[-1] if trade_dates else today
+        previous_expected = trade_dates[-2] if len(trade_dates) >= 2 else yesterday
+        freshness_label = "交易日" if trade_dates else "自然日"
+        
+        # ==================== Pivot 数据 ====================
+        pivot_df = self.CheckPivotCategoriesDates(
+            exclude_category_keywords=exclude_pivot_category_keywords,
+        )
+        
+        if not pivot_df.empty:
+            print(f"\n📊 Pivot 数据 ({len(pivot_df)} 个):")
+            print("-"*100)
+            
+            for category, row in pivot_df.iterrows():
+                latest_date = row['Latest Date']
+                data_count = row['数据量']
+                storage_size = row['储存大小']
+                
+                if pd.isna(latest_date):
+                    status = "❌ 无数据"
+                else:
+                    latest_date_str = pd.to_datetime(latest_date).strftime("%Y-%m-%d")
+                    if latest_date_str >= latest_expected:
+                        status = f"✅ 最新{freshness_label} ({latest_date_str})"
+                    elif latest_date_str == previous_expected:
+                        status = f"✅ 上一{freshness_label} ({latest_date_str})"
+                    else:
+                        try:
+                            if trade_dates:
+                                latest_ts = pd.to_datetime(latest_date).normalize()
+                                days_behind = sum(pd.to_datetime(x) > latest_ts for x in trade_dates)
+                                status = f"⚠️  滞后 {days_behind} 个交易日 ({latest_date_str})"
+                            else:
+                                days_behind = (datetime.datetime.now() - pd.to_datetime(latest_date)).days
+                                status = f"⚠️  滞后 {days_behind} 天 ({latest_date_str})"
+                        except:
+                            status = f"⚠️  {latest_date_str}"
+                
+                print(f"  {category:<50} | {status:<25} | 数据量: {data_count:<6} | {storage_size}")
+            
+            # Pivot 统计
+            latest_ok = sum(1 for _, row in pivot_df.iterrows() if pd.notna(row['Latest Date']) and pd.to_datetime(row['Latest Date']).strftime("%Y-%m-%d") >= latest_expected)
+            yesterday_ok = sum(1 for _, row in pivot_df.iterrows() if pd.notna(row['Latest Date']) and pd.to_datetime(row['Latest Date']).strftime("%Y-%m-%d") == previous_expected)
+            outdated = sum(1 for _, row in pivot_df.iterrows() if pd.notna(row['Latest Date']) and pd.to_datetime(row['Latest Date']).strftime("%Y-%m-%d") < previous_expected)
+            
+            print("-"*100)
+            print(f"Pivot 数据新鲜度({freshness_label}): ✅ 最新 {latest_ok} 个 | ✅ 上一日 {yesterday_ok} 个 | ⚠️ 滞后 {outdated} 个")
+        else:
+            print("\n📊 Pivot 数据: 未找到")
+        
+        # ==================== Unstack 数据 ====================
+        unstack_categories = [x for x in self._get_all_categories() if "unstack#" in x]
+        
+        if unstack_categories:
+            print(f"\n📋 Unstack 数据 ({len(unstack_categories)} 个):")
+            print("-"*100)
+            
+            unstack_info = []
+            for category in unstack_categories:
+                file_path = self._get_unstack_file_path(category)
+                if os.path.exists(file_path):
+                    file_size_bytes = os.path.getsize(file_path)
+                    file_size_mb = round(file_size_bytes / 1024 / 1024, 2)
+                    mod_time = datetime.datetime.fromtimestamp(os.path.getmtime(file_path)).strftime("%Y-%m-%d %H:%M")
+                    
+                    # 尝试读取获取行数
+                    try:
+                        df = pd.read_parquet(file_path)
+                        row_count = len(df)
+                    except:
+                        row_count = -1
+                    
+                    unstack_info.append({
+                        'category': category,
+                        'rows': row_count,
+                        'size': f"{file_size_mb} MB",
+                        'modified': mod_time
+                    })
+                    print(f"  {category:<50} | 行数: {row_count:<8} | {file_size_mb:>8} MB | 更新于: {mod_time}")
+                else:
+                    print(f"  {category:<50} | ❌ 文件不存在")
+        else:
+            print("\n📋 Unstack 数据: 未找到")
+        
+        print("="*100 + "\n")