mdbq 3.6.3__tar.gz → 3.6.4__tar.gz

{mdbq-3.6.3 → mdbq-3.6.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: mdbq
-Version: 3.6.3
+Version: 3.6.4
 Home-page: https://pypi.org/project/mdbq
 Author: xigua, 
 Author-email: 2587125111@qq.com

mdbq-3.6.4/mdbq/redis/getredis.py

@@ -0,0 +1,291 @@
+# -*- coding: UTF-8 –*-
+import os.path
+import redis
+import socket
+from mdbq.mysql import s_query
+from mdbq.config import myconfig
+import pandas as pd
+import json
+import datetime
+import threading
+import logging
+
+
+if socket.gethostname() == 'company' or socket.gethostname() == 'Mac2.local':
+    conf = myconfig.main()
+    conf_data = conf['Windows']['company']['mysql']['local']
+    username, password, host, port = conf_data['username'], conf_data['password'], conf_data['host'], conf_data['port']
+    redis_password = conf['Windows']['company']['redis']['local']['password']
+elif socket.gethostname() == 'MacBookPro':
+    conf = myconfig.main()
+    conf_data = conf['Windows']['xigua_lx']['mysql']['local']
+    username, password, host, port = conf_data['username'], conf_data['password'], conf_data['host'], conf_data['port']
+    redis_password = conf['Windows']['company']['redis']['local']['password']
+else:
+    conf = myconfig.main()
+    conf_data = conf['Windows']['xigua_lx']['mysql']['local']
+    username, password, host, port = conf_data['username'], conf_data['password'], conf_data['host'], conf_data['port']
+    redis_password = conf['Windows']['company']['redis']['local']['password']  # redis 使用本地数据，全部机子相同
+
+logging.basicConfig(level=logging.INFO, format='%(asctime)s | %(levelname)s | %(message)s')
+
+# 获取当前模块的日志记录器
+logger = logging.getLogger(__name__)
+
+# 创建一个文件处理器，用于将日志写入文件
+file_handler = logging.FileHandler(os.path.join('/Users/xigua/Downloads', 'redis.log'))
+file_handler.setLevel(logging.INFO)  # 设置文件处理器的日志级别
+
+# 创建一个日志格式器，并设置给文件处理器
+formatter = logging.Formatter('[%(asctime)s] %(levelname)s %(message)s', datefmt='%Y-%m-%d %H:%M:%S')
+file_handler.setFormatter(formatter)
+
+# 将文件处理器添加到日志记录器
+logger.addHandler(file_handler)
+
+
+class RedisData(object):
+    def __init__(self, redis_engine, download, cache_ttl: int):
+        self.redis_engine = redis_engine  # Redis 数据处理引擎
+        self.download = download  # MySQL 数据处理引擎
+        self.cache_ttl = cache_ttl * 60  # 缓存过期时间（秒）
+
+    def get_from_mysql(
+            self,
+            db_name: str,
+            table_name: str,
+            set_year: bool,
+            start_date,
+            end_date
+    ) -> pd.DataFrame:
+        """
+        从 MySQL 读取数据并返回 DataFrame
+
+        Args:
+            set_year: 表名是否包含年份后缀
+        """
+        dfs = []
+        if set_year:
+            current_year = datetime.datetime.today().year
+            for year in range(2024, current_year + 1):
+                df = self._fetch_table_data(
+                    db_name, f"{table_name}_{year}", start_date, end_date
+                )
+                if df is not None:
+                    dfs.append(df)
+        else:
+            df = self._fetch_table_data(db_name, table_name, start_date, end_date)
+            if df is not None:
+                dfs.append(df)
+
+        combined_df = pd.concat(dfs, ignore_index=True) if dfs else pd.DataFrame()
+        if combined_df.empty:
+            logger.info(f"警告: {db_name}.{table_name} 未读取到数据")
+        else:
+            combined_df = self._convert_date_columns(combined_df)
+        return combined_df
+
+    def get_from_redis(
+            self,
+            db_name: str,
+            table_name: str,
+            set_year: bool,
+            start_date,
+            end_date
+    ) -> pd.DataFrame:
+        """
+        从 Redis 获取数据，若缓存过期/不完整则触发异步更新
+        """
+        start_dt = pd.to_datetime(start_date)
+        end_dt = pd.to_datetime(end_date)
+        cache_key = self._generate_cache_key(db_name, table_name, set_year)
+
+        # 尝试获取缓存元数据
+        try:
+            ttl = self.redis_engine.ttl(cache_key)
+            cache_data = self._fetch_redis_data(cache_key)
+        except Exception as e:
+            logger.info(f"Redis 连接异常: {e}，直接访问 MySQL")
+            return self.get_from_mysql(db_name, table_name, set_year, start_date, end_date)
+
+        # 缓存失效处理逻辑
+        if ttl < 60 or cache_data.empty:
+            self._trigger_async_cache_update(
+                cache_key, db_name, table_name, set_year, start_date, end_date, cache_data
+            )
+            return self.get_from_mysql(db_name, table_name, set_year, start_date, end_date)
+
+        # 处理有效缓存数据
+        filtered_df = self._filter_by_date_range(cache_data, start_dt, end_dt)
+        if not filtered_df.empty:
+            return filtered_df
+
+        # 缓存数据不满足查询范围要求
+        self._trigger_async_cache_update(
+            cache_key, db_name, table_name, set_year, start_date, end_date, cache_data
+        )
+        return self.get_from_mysql(db_name, table_name, set_year, start_date, end_date)
+
+    def set_redis(
+            self,
+            cache_key: str,
+            db_name: str,
+            table_name: str,
+            set_year: bool,
+            start_date,
+            end_date,
+            existing_data: pd.DataFrame
+    ) -> pd.DataFrame:
+        """
+        异步更新 Redis 缓存，合并新旧数据
+        """
+        try:
+            # 从 MySQL 获取新数据
+            new_data = self.get_from_mysql(db_name, table_name, set_year, start_date, end_date)
+            if new_data.empty:
+                return pd.DataFrame()
+
+            # 合并历史数据
+            combined_data = self._merge_data(new_data, existing_data)
+
+            # 序列化并存储到 Redis
+            serialized_data = self._serialize_data(combined_data)
+            self.redis_engine.set(cache_key, serialized_data)
+            self.redis_engine.expire(cache_key, self.cache_ttl)
+
+            logger.info(f"缓存更新 {cache_key} | 数据量: {len(combined_data)}")
+            return combined_data
+
+        except Exception as e:
+            logger.info(f"缓存更新失败: {cache_key} - {str(e)}")
+            return pd.DataFrame()
+
+    # Helper Methods ------------------------------------------------
+
+    def _fetch_table_data(
+            self,
+            db_name: str,
+            table_name: str,
+            start_date,
+            end_date
+    ) -> pd.DataFrame:
+        """封装 MySQL 数据获取逻辑"""
+        try:
+            return self.download.data_to_df(
+                db_name=db_name,
+                table_name=table_name,
+                start_date=start_date,
+                end_date=end_date,
+                projection={}
+            )
+        except Exception as e:
+            logger.info(f"MySQL 查询异常 {db_name}.{table_name}: {e}")
+            return pd.DataFrame()
+
+    def _fetch_redis_data(self, cache_key: str) -> pd.DataFrame:
+        """从 Redis 获取并解析数据（自动转换日期列）"""
+        try:
+            data = self.redis_engine.get(cache_key)
+            if not data:
+                return pd.DataFrame()
+            # 反序列化数据
+            df = pd.DataFrame(json.loads(data.decode("utf-8")))
+            return self._convert_date_columns(df)
+        except Exception as e:
+            logger.info(f"Redis 数据解析失败 {cache_key}: {e}")
+            return pd.DataFrame()
+
+    def _convert_date_columns(self, df: pd.DataFrame) -> pd.DataFrame:
+        """统一处理日期列转换"""
+        if "日期" in df.columns:
+            df["日期"] = pd.to_datetime(df["日期"], format="%Y-%m-%d", errors="coerce")
+        return df
+
+    def _generate_cache_key(self, db_name: str, table_name: str, set_year: bool) -> str:
+        """生成标准化的缓存键"""
+        return f"{db_name}:{table_name}_haveyear" if set_year else f"{db_name}:{table_name}"
+
+    def _filter_by_date_range(
+            self,
+            df: pd.DataFrame,
+            start_dt: datetime.datetime,
+            end_dt: datetime.datetime
+    ) -> pd.DataFrame:
+        """按日期范围筛选数据"""
+        if "日期" not in df.columns:
+            return df
+        date_mask = (df["日期"] >= start_dt) & (df["日期"] <= end_dt)
+        return df[date_mask].copy()
+
+    def _trigger_async_cache_update(
+            self,
+            cache_key: str,
+            db_name: str,
+            table_name: str,
+            set_year: bool,
+            start_date: str,
+            end_date: str,
+            existing_data: pd.DataFrame
+    ):
+        """启动异步缓存更新线程"""
+        thread = threading.Thread(
+            target=self.set_redis,
+            args=(cache_key, db_name, table_name, set_year, start_date, end_date, existing_data),
+            daemon=True
+        )
+        thread.start()
+
+    def _merge_data(self, new_data: pd.DataFrame, existing_data: pd.DataFrame) -> pd.DataFrame:
+        """合并新旧数据集"""
+        if existing_data.empty or "日期" not in existing_data.columns:
+            return new_data
+
+        new_min = new_data["日期"].min()
+        new_max = new_data["日期"].max()
+        valid_historical = existing_data[
+            (existing_data["日期"] < new_min) | (existing_data["日期"] > new_max)
+            ]
+        return pd.concat([new_data, valid_historical], ignore_index=True).drop_duplicates(subset=["日期"])
+
+    def _serialize_data(self, df: pd.DataFrame) -> str:
+        """序列化 DataFrame 并处理日期类型"""
+        temp_df = df.copy()
+        date_cols = temp_df.select_dtypes(include=["datetime64[ns]"]).columns
+        for col in date_cols:
+            temp_df[col] = temp_df[col].dt.strftime("%Y-%m-%d")
+        return temp_df.to_json(orient="records", force_ascii=False)
+
+
+if __name__ == '__main__':
+    # # ****************************************************
+    # # 这一部分在外部定义，只需要定义一次，开始
+    # redis_config = {
+    #     'host': '127.0.0.1',
+    #     'port': 6379,  # 默认Redis端口
+    #     'db': 0,  # 默认Redis数据库索引
+    #     # 'username': 'default',
+    #     'password': redis_password,
+    # }
+    # # redis 实例化
+    # r = redis.Redis(**redis_config)
+    # # mysql 实例化
+    # d = s_query.QueryDatas(username=username, password=password, host=host, port=port)
+    # # 将两个库的实例化对象传给 RedisData 类，并实例化数据处理引擎
+    # m = RedisData(redis_engin=r, download=d)
+    # # ****************************************************
+    #
+    # # 以下为动态获取数据库数据
+    # db_name = '聚合数据'
+    # table_name = '多店推广场景_按日聚合'
+    # set_year = False
+    # df = m.get_from_redis(
+    #     db_name=db_name,
+    #     table_name=table_name,
+    #     set_year=set_year,
+    #     start_date='2025-01-01',
+    #     end_date='2025-01-31'
+    # )
+    # logger.info(df)
+    #
+
+    logger.info(socket.gethostname())

mdbq-3.6.3/mdbq/redis/getredis.py → mdbq-3.6.4/mdbq/redis/getredis_bak20250131.py

@@ -14,6 +14,11 @@ if socket.gethostname() == 'company' or socket.gethostname() == 'Mac2.local':
     conf_data = conf['Windows']['company']['mysql']['local']
     username, password, host, port = conf_data['username'], conf_data['password'], conf_data['host'], conf_data['port']
     redis_password = conf['Windows']['company']['redis']['local']['password']
+# elif socket.gethostname() == 'MacBook-Pro.local':
+#     conf = myconfig.main()
+#     conf_data = conf['Windows']['xigua_lx']['mysql']['local']
+#     username, password, host, port = conf_data['username'], conf_data['password'], conf_data['host'], conf_data['port']
+#     redis_password = conf['Windows']['company']['redis']['local']['password']
 else:
     conf = myconfig.main()
     conf_data = conf['Windows']['xigua_lx']['mysql']['local']
@@ -101,7 +106,7 @@ class RedisData(object):
             # Bug: 如果外部请求日期小于 table 最小日期，每次都要从 mysql 获取数据，即使 redis 缓存了数据
             if start_date < min_date:  # 外部请求日期小于 redis 数据最小日期
                 # 3. redis 有数据但数据不完整时
-                print(f'{start_date} -- {min_date} 数据日期不对劲需要更新')
+                print(f'{start_date} -- {min_date} 数据日期需要更新')
                 thread = threading.Thread(target=self.set_redis, args=(my_key, _db_name, _table_name, _set_year, start_date, end_date, _df))
                 thread.start()
                 # _df = self.set_redis(my_key=my_key, _db_name=_db_name, _table_name=_table_name, _set_year=_set_year, start_date=start_date, end_date=end_date)
@@ -112,6 +117,82 @@ class RedisData(object):
         return _df
 
     def set_redis(self, my_key, _db_name, _table_name, _set_year, start_date, end_date, before_df):
+        """
+        从MySQL读取数据并存储到Redis（异步执行）
+
+        Args:
+            my_key: Redis存储键名
+            _db_name: 数据库名称
+            _table_name: 数据表名称
+            _set_year: 数据集年份
+            start_date: 查询开始日期
+            end_date: 查询结束日期
+            before_df: 合并用的历史数据
+
+        Returns:
+            pd.DataFrame: 处理后的数据集（含历史数据合并）
+        """
+        # 异常处理容器
+        datetime_cols = []
+
+        try:
+            # 从MySQL获取数据
+            _df = self.get_from_mysql(
+                _db_name=_db_name,
+                _table_name=_table_name,
+                start_date=start_date,
+                end_date=end_date,
+                _set_year=_set_year
+            )
+
+            # 日期列处理（当新旧数据都存在日期列时）
+            if '日期' in _df.columns and '日期' in before_df.columns:
+                # 获取当前数据时间范围
+                _min_date, _max_date = _df['日期'].min(), _df['日期'].max()
+
+                # 筛选需要保留的历史数据
+                mask = (before_df['日期'] < _min_date) | (before_df['日期'] > _max_date)
+                valid_history = before_df[mask]
+
+                # 合并数据
+                _df = pd.concat([_df, valid_history], ignore_index=True, axis=0)
+                _df.drop_duplicates(subset='日期', keep='first', inplace=True)  # 可选去重
+
+            # 预处理时间类型转换
+            datetime_cols = _df.select_dtypes(include=['datetime64[ns]']).columns.tolist()
+            if datetime_cols:
+                _df[datetime_cols] = _df[datetime_cols].astype(str)
+
+            # 空数据检查
+            if _df.empty:
+                print(f'Warning: {_table_name} 空数据集，跳过Redis存储')
+                return pd.DataFrame()
+
+            # Redis存储操作
+            self.redis_engin.set(my_key, _df.to_json(orient='records', force_ascii=False))
+            self.redis_engin.expire(my_key, self.minute * 60)
+
+            # 恢复时间类型（返回用）
+            if datetime_cols:
+                _df[datetime_cols] = _df[datetime_cols].apply(pd.to_datetime, errors='coerce')
+
+            # 记录操作日志
+            print(f'{datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")}'
+                  f' | 刷新Redis {_db_name}:{_table_name}'
+                  f' | 数据量：{len(_df)}行')
+
+        except Exception as e:
+            print(f'Error: {_table_name} 数据处理失败 - {str(e)}')
+            _df = pd.DataFrame()
+
+        finally:
+            # 确保返回前恢复时间类型
+            if datetime_cols and not _df.empty:
+                _df[datetime_cols] = _df[datetime_cols].apply(pd.to_datetime, errors='ignore')
+
+        return _df
+
+    def set_redis_bak(self, my_key, _db_name, _table_name, _set_year, start_date, end_date, before_df):
         """
         从 mysql 读取数据并存储 redis
         由于这个函数是异步执行的，从页面段首次加载数据时，可能返回空，等待异步执行结束后会正常返回数据
@@ -149,34 +230,36 @@ class RedisData(object):
         print(f'{now}: 刷新 redis -> {_db_name}:{_table_name}')
         return _df
 
-
 if __name__ == '__main__':
-    # ****************************************************
-    # 这一部分在外部定义，只需要定义一次，开始
-    redis_config = {
-        'host': '127.0.0.1',
-        'port': 6379,  # 默认Redis端口
-        'db': 0,  # 默认Redis数据库索引
-        # 'username': 'default',
-        'password': redis_password,
-    }
-    # redis 实例化
-    r = redis.Redis(**redis_config)
-    # mysql 实例化
-    d = s_query.QueryDatas(username=username, password=password, host=host, port=port)
-    # 将两个库的实例化对象传给 RedisData 类，并实例化数据处理引擎
-    m = RedisData(redis_engin=r, download=d)
-    # ****************************************************
-
-    # 以下为动态获取数据库数据
-    db_name = '聚合数据'
-    table_name = '多店推广场景_按日聚合'
-    set_year = False
-    df = m.get_from_redis(
-        _db_name=db_name,
-        _table_name=table_name,
-        _set_year=set_year,
-        start_date='2025-01-01',
-        end_date='2025-01-31'
-    )
-    print(df)
+    # # ****************************************************
+    # # 这一部分在外部定义，只需要定义一次，开始
+    # redis_config = {
+    #     'host': '127.0.0.1',
+    #     'port': 6379,  # 默认Redis端口
+    #     'db': 0,  # 默认Redis数据库索引
+    #     # 'username': 'default',
+    #     'password': redis_password,
+    # }
+    # # redis 实例化
+    # r = redis.Redis(**redis_config)
+    # # mysql 实例化
+    # d = s_query.QueryDatas(username=username, password=password, host=host, port=port)
+    # # 将两个库的实例化对象传给 RedisData 类，并实例化数据处理引擎
+    # m = RedisData(redis_engin=r, download=d)
+    # # ****************************************************
+    #
+    # # 以下为动态获取数据库数据
+    # db_name = '聚合数据'
+    # table_name = '多店推广场景_按日聚合'
+    # set_year = False
+    # df = m.get_from_redis(
+    #     _db_name=db_name,
+    #     _table_name=table_name,
+    #     _set_year=set_year,
+    #     start_date='2025-01-01',
+    #     end_date='2025-01-31'
+    # )
+    # print(df)
+    #
+
+    print(socket.gethostname())

mdbq-3.6.4/mdbq/redis/getredis_deepseek.py

@@ -0,0 +1,235 @@
+# -*- coding: UTF-8 –*-
+import redis
+import socket
+from mdbq.mysql import s_query
+from mdbq.config import myconfig
+import pandas as pd
+import json
+import datetime
+import threading
+
+
+if socket.gethostname() == 'company' or socket.gethostname() == 'Mac2.local':
+    conf = myconfig.main()
+    conf_data = conf['Windows']['company']['mysql']['local']
+    username, password, host, port = conf_data['username'], conf_data['password'], conf_data['host'], conf_data['port']
+    redis_password = conf['Windows']['company']['redis']['local']['password']
+# elif socket.gethostname() == 'MacBook-Pro.local':
+#     conf = myconfig.main()
+#     conf_data = conf['Windows']['xigua_lx']['mysql']['local']
+#     username, password, host, port = conf_data['username'], conf_data['password'], conf_data['host'], conf_data['port']
+#     redis_password = conf['Windows']['company']['redis']['local']['password']
+else:
+    conf = myconfig.main()
+    conf_data = conf['Windows']['xigua_lx']['mysql']['local']
+    username, password, host, port = conf_data['username'], conf_data['password'], conf_data['host'], conf_data['port']
+    redis_password = conf['Windows']['company']['redis']['local']['password']  # redis 使用本地数据，全部机子相同
+
+
+class RedisData:
+    def __init__(self, redis_engine, download, cache_ttl: int = 60):
+        """
+        初始化Redis数据处理对象
+
+        :param redis_engine: Redis连接引擎
+        :param download: 数据库下载处理器
+        :param cache_ttl: 缓存过期时间（分钟）
+        """
+        self.redis_engine = redis_engine
+        self.download = download
+        self.cache_ttl = cache_ttl * 60  # 转换为秒
+        self.lock = threading.Lock()  # 线程锁
+
+    def _handle_datetime_columns(self, df: pd.DataFrame) -> pd.DataFrame:
+        """统一处理日期列转换"""
+        if '日期' in df.columns:
+            df['日期'] = pd.to_datetime(df['日期'], errors='coerce', format='%Y-%m-%d')
+        return df
+
+    def get_from_mysql(self, db_name: str, table_name: str,
+                       start_date, end_date,
+                       set_year: bool) -> pd.DataFrame:
+        """
+        从MySQL获取数据
+
+        :param set_year: 是否按年份分表
+        """
+        try:
+            if set_year:
+                current_year = datetime.datetime.now().year
+                dfs = []
+                # 动态获取需要查询的年份范围
+                min_year = min(2024, pd.to_datetime(start_date).year)  # 根据实际需求调整
+                for year in range(min_year, current_year + 1):
+                    table = f"{table_name}_{year}"
+                    df = self.download.data_to_df(
+                        db_name=db_name,
+                        table_name=table,
+                        start_date=start_date,
+                        end_date=end_date,
+                        projection={}
+                    )
+                    if not df.empty:
+                        dfs.append(df)
+                _df = pd.concat(dfs, ignore_index=True) if dfs else pd.DataFrame()
+            else:
+                _df = self.download.data_to_df(
+                    db_name=db_name,
+                    table_name=table_name,
+                    start_date=start_date,
+                    end_date=end_date,
+                    projection={}
+                )
+
+            if _df.empty:
+                print(f"空数据 - {db_name}.{table_name}")
+                return pd.DataFrame()
+
+            return self._handle_datetime_columns(_df)
+
+        except Exception as e:
+            print(f"MySQL查询失败: {str(e)}")
+            return pd.DataFrame()
+
+    def get_from_redis(self, db_name: str, table_name: str,
+                       start_date, end_date,
+                       set_year: bool) -> pd.DataFrame:
+        """
+        从Redis获取数据（带自动缓存更新）
+        """
+        start_dt = pd.to_datetime(start_date)
+        end_dt = pd.to_datetime(end_date)
+
+        # 生成统一缓存键
+        cache_key = f"{db_name}:{table_name}{'_year' if set_year else ''}"
+
+        try:
+            # 检查缓存状态
+            with self.lock:  # 保证线程安全
+                ttl = self.redis_engine.ttl(cache_key)
+
+                if ttl < 300:  # 剩余时间小于5分钟时触发更新
+                    print(f"异步缓存更新: {cache_key}")
+                    threading.Thread(
+                        target=self._update_cache,
+                        args=(cache_key, db_name, table_name,
+                              start_date, end_date, set_year),
+                        daemon=True
+                    ).start()
+
+            # 获取缓存数据
+            cached_data = self.redis_engine.get(cache_key)
+            if not cached_data:
+                return self._fallback_to_mysql(db_name, table_name,
+                                               start_date, end_date, set_year)
+            json_str = cached_data.decode('utf-8')
+            _df = pd.read_json(json_str, orient='records')
+            _df = self._handle_datetime_columns(_df)
+
+            # 数据范围校验
+            if '日期' in _df.columns:
+                cache_min = _df['日期'].min()
+                cache_max = _df['日期'].max()
+
+                # 请求范围超出缓存范围时需要更新
+                if start_dt < cache_min or end_dt > cache_max:
+                    print(f"请求范围超出缓存 {start_dt.strftime('%Y-%m-%d ')} - {end_dt.strftime('%Y-%m-%d ')}")
+                    self._update_cache(cache_key, db_name, table_name,
+                                       start_date, end_date, set_year, _df)
+                    return self._fallback_to_mysql(db_name, table_name,
+                                                   start_date, end_date, set_year)
+
+                return _df[(start_dt <= _df['日期']) & (_df['日期'] <= end_dt)]
+            return _df
+
+        except Exception as e:
+            print(f"Redis操作失败: {str(e)}")
+            return self._fallback_to_mysql(db_name, table_name,
+                                           start_date, end_date, set_year)
+
+    def _update_cache(self, cache_key: str, db_name: str, table_name: str,
+                      start_date: str, end_date: str, set_year: bool,
+                      existing_df: pd.DataFrame = None) -> None:
+        """缓存更新核心逻辑"""
+        try:
+            # 获取最新数据
+            new_data = self.get_from_mysql(
+                db_name=db_name,
+                table_name=table_name,
+                start_date=start_date,
+                end_date=end_date,
+                set_year=set_year
+            )
+
+            # 合并历史数据
+            if existing_df is not None and not new_data.empty:
+                combined = pd.concat([existing_df, new_data], ignore_index=True)
+                combined = combined.drop_duplicates(subset='日期', keep='last')
+            else:
+                combined = new_data
+
+            if not combined.empty:
+                # 转换日期类型为字符串
+                temp_df = combined.copy()
+                datetime_cols = temp_df.select_dtypes(include=['datetime64[ns]']).columns
+                temp_df[datetime_cols] = temp_df[datetime_cols].astype(str)
+
+                # 存储到Redis
+                with self.lock:
+                    self.redis_engine.set(
+                        cache_key,
+                        temp_df.to_json(orient='records', force_ascii=False),
+                        ex=self.cache_ttl
+                    )
+                print(f"缓存更新成功: {cache_key} | 记录数: {len(combined)}")
+
+        except Exception as e:
+            print(f"缓存更新失败: {str(e)}")
+
+    def _fallback_to_mysql(self, db_name: str, table_name: str,
+                           start_date: str, end_date: str,
+                           set_year: bool) -> pd.DataFrame:
+        """降级到直接MySQL查询"""
+        print(f"降级到MySQL查询: {db_name}.{table_name}")
+        return self.get_from_mysql(
+            db_name=db_name,
+            table_name=table_name,
+            start_date=start_date,
+            end_date=end_date,
+            set_year=set_year
+        )
+
+
+if __name__ == '__main__':
+    # # ****************************************************
+    # # 这一部分在外部定义，只需要定义一次，开始
+    # redis_config = {
+    #     'host': '127.0.0.1',
+    #     'port': 6379,  # 默认Redis端口
+    #     'db': 0,  # 默认Redis数据库索引
+    #     # 'username': 'default',
+    #     'password': redis_password,
+    # }
+    # # redis 实例化
+    # r = redis.Redis(**redis_config)
+    # # mysql 实例化
+    # d = s_query.QueryDatas(username=username, password=password, host=host, port=port)
+    # # 将两个库的实例化对象传给 RedisData 类，并实例化数据处理引擎
+    # m = RedisData(redis_engin=r, download=d)
+    # # ****************************************************
+    #
+    # # 以下为动态获取数据库数据
+    # db_name = '聚合数据'
+    # table_name = '多店推广场景_按日聚合'
+    # set_year = False
+    # df = m.get_from_redis(
+    #     _db_name=db_name,
+    #     _table_name=table_name,
+    #     _set_year=set_year,
+    #     start_date='2025-01-01',
+    #     end_date='2025-01-31'
+    # )
+    # print(df)
+    #
+
+    print(socket.gethostname())

{mdbq-3.6.3 → mdbq-3.6.4}/mdbq.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: mdbq
-Version: 3.6.3
+Version: 3.6.4
 Home-page: https://pypi.org/project/mdbq
 Author: xigua, 
 Author-email: 2587125111@qq.com

{mdbq-3.6.3 → mdbq-3.6.4}/mdbq.egg-info/SOURCES.txt

@@ -40,5 +40,7 @@ mdbq/pbix/refresh_all.py
 mdbq/pbix/refresh_all_old.py
 mdbq/redis/__init__.py
 mdbq/redis/getredis.py
+mdbq/redis/getredis_bak20250131.py
+mdbq/redis/getredis_deepseek.py
 mdbq/spider/__init__.py
 mdbq/spider/aikucun.py

{mdbq-3.6.3 → mdbq-3.6.4}/setup.py

@@ -3,7 +3,7 @@
 from setuptools import setup, find_packages
 
 setup(name='mdbq',
-      version='3.6.3',
+      version='3.6.4',
       author='xigua, ',
       author_email="2587125111@qq.com",
       url='https://pypi.org/project/mdbq',