mdbq 3.6.8__py3-none-any.whl → 3.6.10__py3-none-any.whl

mdbq/mysql/s_query.py

@@ -12,6 +12,7 @@ from sqlalchemy import create_engine
 import os
 import calendar
 from mdbq.dataframe import converter
+from decimal import Decimal
 
 warnings.filterwarnings('ignore')
 """
@@ -49,9 +50,14 @@ class QueryDatas:
             return columns
 
     def data_to_df(self, db_name, table_name, start_date, end_date, projection: dict=[]):
-
-        start_date = pd.to_datetime(start_date).strftime('%Y-%m-%d')
-        end_date = pd.to_datetime(end_date).strftime('%Y-%m-%d')
+        if start_date:
+            start_date = pd.to_datetime(start_date).strftime('%Y-%m-%d')
+        else:
+            start_date = '1970-01-01'
+        if end_date:
+            end_date = pd.to_datetime(end_date).strftime('%Y-%m-%d')
+        else:
+            end_date = datetime.datetime.today().strftime('%Y-%m-%d')
         df = pd.DataFrame()  # 初始化df
 
         if self.check_infos(db_name, table_name) == False:
@@ -97,6 +103,9 @@ class QueryDatas:
             rows = cursor.fetchall()  # 获取查询结果
             columns = [desc[0] for desc in cursor.description]
             df = pd.DataFrame(rows, columns=columns)  # 转为 df
+            # 使用applymap将每个Decimal转换为float
+            df_float = df.applymap(lambda x: float(x) if isinstance(x, Decimal) else x)
+
             if 'id' in df.columns.tolist():
                 df.pop('id')  # 默认不返回 id 列
             if len(df) == 0:

mdbq/redis/getredis.py

@@ -12,6 +12,7 @@ import logging
 from logging.handlers import RotatingFileHandler
 import getpass
 import platform
+from decimal import Decimal
 
 if platform.system() == 'Windows':
     D_PATH = os.path.join(f'C:\\Users\\{getpass.getuser()}\\Downloads')
@@ -57,6 +58,9 @@ logger.addHandler(file_handler)
 
 
 class RedisData(object):
+    """
+    存储 string
+    """
     def __init__(self, redis_engine, download, cache_ttl: int):
         self.redis_engine = redis_engine  # Redis 数据处理引擎
         self.download = download  # MySQL 数据处理引擎
@@ -267,6 +271,304 @@ class RedisData(object):
             temp_df[col] = temp_df[col].dt.strftime("%Y-%m-%d")
         return temp_df.to_json(orient="records", force_ascii=False)
 
+class RedisDataHash(object):
+    """
+    存储 hash
+    Redis缓存与MySQL数据联合查询处理器
+
+    功能特性：
+    - 支持带年份分表的MySQL数据查询
+    - 多级缓存策略（内存缓存+Redis缓存）
+    - 异步缓存更新机制
+    - 自动处理日期范围和数据类型转换
+    """
+
+    def __init__(self, redis_engine, download, cache_ttl: int):
+        self.redis_engine = redis_engine
+        self.download = download
+        self.cache_ttl = cache_ttl * 60  # 转换为秒存储
+
+    def get_from_mysql(
+            self,
+            db_name: str,
+            table_name: str,
+            set_year: bool,
+            start_date,
+            end_date
+    ) -> pd.DataFrame:
+        dfs = []
+        if set_year:
+            current_year = datetime.datetime.today().year
+            for year in range(2024, current_year + 1):
+                df = self._fetch_table_data(
+                    db_name, f"{table_name}_{year}", start_date, end_date
+                )
+                if df is not None:
+                    dfs.append(df)
+        else:
+            df = self._fetch_table_data(db_name, table_name, start_date, end_date)
+            if df is not None:
+                dfs.append(df)
+
+        combined_df = pd.concat(dfs, ignore_index=True) if dfs else pd.DataFrame()
+        if combined_df.empty:
+            logger.warn(f"warning: {db_name}.{table_name} 未读取到数据")
+        else:
+            combined_df = self._convert_date_columns(combined_df)
+        return combined_df
+
+    def get_from_redis(
+            self,
+            db_name: str,
+            table_name: str,
+            set_year: bool,
+            start_date,
+            end_date
+    ) -> pd.DataFrame:
+        start_dt = pd.to_datetime(start_date).floor('D')
+        end_dt = pd.to_datetime(end_date).floor('D')
+        cache_key = self._generate_cache_key(db_name, table_name, set_year)
+
+        try:
+            ttl = self.redis_engine.ttl(cache_key)
+            if ttl < 60:
+                cache_data = self._fetch_redis_data(cache_key)
+                self._trigger_async_cache_update(
+                    cache_key, db_name, table_name, set_year, start_date, end_date, cache_data
+                )
+                return self.get_from_mysql(db_name, table_name, set_year, start_date, end_date)
+
+            # 生成月份范围
+            start_month = start_dt.to_period('M')
+            end_month = end_dt.to_period('M')
+            months = pd.period_range(start_month, end_month, freq='M').strftime("%Y%m").tolist()
+            cache_data = self._fetch_redis_data(cache_key, months)
+
+            if cache_data.empty:
+                self._trigger_async_cache_update(
+                    cache_key, db_name, table_name, set_year, start_date, end_date, cache_data
+                )
+                return self.get_from_mysql(db_name, table_name, set_year, start_date, end_date)
+
+            filtered_df = self._filter_by_date_range(cache_data, start_dt, end_dt)
+            if not filtered_df.empty:
+                if '日期' in filtered_df.columns.tolist():
+                    exsit_min_date = filtered_df['日期'].min()
+                    if exsit_min_date <= start_dt:
+                        return filtered_df
+                else:
+                    return filtered_df
+
+            self._trigger_async_cache_update(
+                cache_key, db_name, table_name, set_year, start_date, end_date, cache_data
+            )
+            return self.get_from_mysql(db_name, table_name, set_year, start_date, end_date)
+
+        except Exception as e:
+            logger.error(f"Redis 连接异常: {e}，直接访问 MySQL")
+            return self.get_from_mysql(db_name, table_name, set_year, start_date, end_date)
+
+    def set_redis(
+            self,
+            cache_key: str,
+            db_name: str,
+            table_name: str,
+            set_year: bool,
+            start_date,
+            end_date,
+            existing_data: pd.DataFrame
+    ) -> None:
+        try:
+            new_data = self.get_from_mysql(db_name, table_name, set_year, start_date, end_date)
+            if new_data.empty:
+                return
+
+            combined_data = self._merge_data(new_data, existing_data)
+
+            if not combined_data.empty:
+                if '日期' not in combined_data.columns.tolist():
+                    serialized_data = self._serialize_data(combined_data)
+                    self.redis_engine.hset(cache_key, "all", serialized_data)
+                    self.redis_engine.expire(cache_key, self.cache_ttl)
+                else:
+                    # 按月分片存储
+                    combined_data['month'] = combined_data['日期'].dt.to_period('M').dt.strftime("%Y%m")
+                    for month_str, group in combined_data.groupby('month'):
+                        group = group.drop(columns=['month'])
+                        serialized_data = self._serialize_data(group)
+                        self.redis_engine.hset(cache_key, month_str, serialized_data)
+                    self.redis_engine.expire(cache_key, self.cache_ttl)
+                logger.info(f"缓存更新 {cache_key} | 数据量: {len(combined_data)}")
+        except Exception as e:
+            logger.error(f"缓存更新失败: {cache_key} - {str(e)}")
+
+    def _fetch_table_data(
+            self,
+            db_name: str,
+            table_name: str,
+            start_date,
+            end_date
+    ) -> pd.DataFrame:
+        try:
+            return self.download.data_to_df(
+                db_name=db_name,
+                table_name=table_name,
+                start_date=start_date,
+                end_date=end_date,
+                projection={}
+            )
+        except Exception as e:
+            logger.error(f"MySQL 查询异常 {db_name}.{table_name}: {e}")
+            return pd.DataFrame()
+
+    def _fetch_redis_data(self, cache_key: str, months: list = None) -> pd.DataFrame:
+        try:
+            if months is not None:
+                fields = months.copy()
+                fields.append('all')
+                data_list = self.redis_engine.hmget(cache_key, fields)
+                dfs = []
+                for data, field in zip(data_list, fields):
+                    if data:
+                        df = pd.DataFrame(json.loads(data.decode("utf-8")))
+                        df = self._convert_date_columns(df)
+                        dfs.append(df)
+                return pd.concat(dfs, ignore_index=True) if dfs else pd.DataFrame()
+            else:
+                data_dict = self.redis_engine.hgetall(cache_key)
+                dfs = []
+                for field, data in data_dict.items():
+                    try:
+                        df = pd.DataFrame(json.loads(data.decode("utf-8")))
+                        df = self._convert_date_columns(df)
+                        dfs.append(df)
+                    except Exception as e:
+                        logger.error(f"Redis 数据解析失败 {cache_key} 字段 {field}: {e}")
+                return pd.concat(dfs, ignore_index=True) if dfs else pd.DataFrame()
+        except Exception as e:
+            logger.error(f"Redis 数据获取失败 {cache_key}: {e}")
+            return pd.DataFrame()
+
+    def _convert_date_columns(self, df: pd.DataFrame) -> pd.DataFrame:
+        if "日期" in df.columns:
+            df["日期"] = pd.to_datetime(df["日期"], format="%Y-%m-%d", errors="coerce")
+        return df
+
+    def _generate_cache_key(self, db_name: str, table_name: str, set_year: bool) -> str:
+        return f"{db_name}:{table_name}_haveyear" if set_year else f"{db_name}:{table_name}"
+
+    def _filter_by_date_range(
+            self,
+            df: pd.DataFrame,
+            start_dt: datetime.datetime,
+            end_dt: datetime.datetime
+    ) -> pd.DataFrame:
+        if "日期" not in df.columns:
+            return df
+        date_mask = (df["日期"] >= start_dt) & (df["日期"] <= end_dt)
+        return df[date_mask].copy()
+
+    def _trigger_async_cache_update(
+            self,
+            cache_key: str,
+            db_name: str,
+            table_name: str,
+            set_year: bool,
+            start_date: str,
+            end_date: str,
+            existing_data: pd.DataFrame
+    ):
+        thread = threading.Thread(
+            target=self.set_redis,
+            args=(cache_key, db_name, table_name, set_year, start_date, end_date, existing_data),
+            daemon=True
+        )
+        thread.start()
+
+    def _merge_data(self, new_data: pd.DataFrame, existing_data: pd.DataFrame) -> pd.DataFrame:
+        if existing_data.empty or "日期" not in existing_data.columns:
+            return new_data
+        new_data["日期"] = pd.to_datetime(new_data["日期"])
+        existing_data["日期"] = pd.to_datetime(existing_data["日期"])
+
+        new_min = new_data["日期"].min()
+        new_max = new_data["日期"].max()
+
+        valid_historical = existing_data[
+            (existing_data["日期"] < new_min) | (existing_data["日期"] > new_max)
+            ]
+        merged_data = pd.concat([new_data, valid_historical], ignore_index=True)
+        merged_data.sort_values(['日期'], ascending=[False], ignore_index=True, inplace=True)
+        return merged_data
+
+    def _serialize_data(self, df: pd.DataFrame) -> bytes:
+        if df.empty:
+            return json.dumps([], ensure_ascii=False).encode("utf-8")
+        temp_df = df.copy()
+
+        date_cols = temp_df.select_dtypes(include=["datetime64[ns]"]).columns
+        for col in date_cols:
+            if temp_df[col].isna().all():
+                temp_df[col] = temp_df[col].astype(object)
+            temp_df[col] = (
+                temp_df[col]
+                .dt.strftime("%Y-%m-%d")
+                .where(temp_df[col].notna(), None)
+            )
+
+        def safe_null_convert(series):
+            if series.isna().all():
+                return series.astype(object).where(pd.notnull(series), None)
+            return series.where(pd.notnull(series), None)
+
+        temp_df = temp_df.apply(safe_null_convert)
+
+        def decimal_serializer(obj):
+            if obj is None:
+                return None
+            if isinstance(obj, Decimal):
+                return round(float(obj), 6)
+            elif isinstance(obj, pd.Timestamp):
+                return obj.strftime("%Y-%m-%d %H:%M:%S")
+            elif isinstance(obj, np.generic):
+                return obj.item()
+            elif isinstance(obj, (datetime.date, datetime.datetime)):
+                return obj.isoformat()
+            elif isinstance(obj, (list, tuple, set)):
+                return [decimal_serializer(item) for item in obj]
+            elif isinstance(obj, dict):
+                return {decimal_serializer(k): decimal_serializer(v) for k, v in obj.items()}
+            elif isinstance(obj, bytes):
+                return obj.decode("utf-8", errors="replace")
+            elif isinstance(obj, pd.Series):
+                return obj.to_list()
+            else:
+                try:
+                    json.dumps(obj)
+                    return obj
+                except TypeError:
+                    logger.error(f"无法序列化类型 {type(obj)}: {str(obj)}")
+                    raise
+
+        try:
+            data_records = temp_df.to_dict(orient="records")
+        except Exception as e:
+            logger.error(f"数据转换字典失败: {str(e)}")
+            raise
+
+        if not data_records:
+            return json.dumps([], ensure_ascii=False).encode("utf-8")
+
+        try:
+            return json.dumps(
+                data_records,
+                ensure_ascii=False,
+                default=decimal_serializer
+            ).encode("utf-8")
+        except TypeError as e:
+            logger.error(f"序列化失败，请检查未处理的数据类型: {str(e)}")
+            raise
+
 
 if __name__ == '__main__':
     # # ****************************************************