mdbq 3.6.2__py3-none-any.whl → 3.6.4__py3-none-any.whl

mdbq/redis/getredis.py

@@ -1,4 +1,5 @@
 # -*- coding: UTF-8 –*-
+import os.path
 import redis
 import socket
 from mdbq.mysql import s_query
@@ -7,6 +8,7 @@ import pandas as pd
 import json
 import datetime
 import threading
+import logging
 
 
 if socket.gethostname() == 'company' or socket.gethostname() == 'Mac2.local':
@@ -14,166 +16,276 @@ if socket.gethostname() == 'company' or socket.gethostname() == 'Mac2.local':
     conf_data = conf['Windows']['company']['mysql']['local']
     username, password, host, port = conf_data['username'], conf_data['password'], conf_data['host'], conf_data['port']
     redis_password = conf['Windows']['company']['redis']['local']['password']
+elif socket.gethostname() == 'MacBookPro':
+    conf = myconfig.main()
+    conf_data = conf['Windows']['xigua_lx']['mysql']['local']
+    username, password, host, port = conf_data['username'], conf_data['password'], conf_data['host'], conf_data['port']
+    redis_password = conf['Windows']['company']['redis']['local']['password']
 else:
     conf = myconfig.main()
     conf_data = conf['Windows']['xigua_lx']['mysql']['local']
     username, password, host, port = conf_data['username'], conf_data['password'], conf_data['host'], conf_data['port']
     redis_password = conf['Windows']['company']['redis']['local']['password']  # redis 使用本地数据，全部机子相同
 
+logging.basicConfig(level=logging.INFO, format='%(asctime)s | %(levelname)s | %(message)s')
+
+# 获取当前模块的日志记录器
+logger = logging.getLogger(__name__)
+
+# 创建一个文件处理器，用于将日志写入文件
+file_handler = logging.FileHandler(os.path.join('/Users/xigua/Downloads', 'redis.log'))
+file_handler.setLevel(logging.INFO)  # 设置文件处理器的日志级别
+
+# 创建一个日志格式器，并设置给文件处理器
+formatter = logging.Formatter('[%(asctime)s] %(levelname)s %(message)s', datefmt='%Y-%m-%d %H:%M:%S')
+file_handler.setFormatter(formatter)
+
+# 将文件处理器添加到日志记录器
+logger.addHandler(file_handler)
 
 
 class RedisData(object):
-    def __init__(self, redis_engin, download):
-        self.redis_engin = redis_engin  # redis 数据处理引擎
-        self.download = download  # mysql 数据处理引擎
-        self.minute = 60  # 缓存过期时间: 分钟
+    def __init__(self, redis_engine, download, cache_ttl: int):
+        self.redis_engine = redis_engine  # Redis 数据处理引擎
+        self.download = download  # MySQL 数据处理引擎
+        self.cache_ttl = cache_ttl * 60  # 缓存过期时间（秒）
 
-    def get_from_mysql(self, _db_name, _table_name, _set_year, start_date, end_date):
+    def get_from_mysql(
+            self,
+            db_name: str,
+            table_name: str,
+            set_year: bool,
+            start_date,
+            end_date
+    ) -> pd.DataFrame:
         """
-        _set_year: _table_name 中是否含有年份
+        从 MySQL 读取数据并返回 DataFrame
+
+        Args:
+            set_year: 表名是否包含年份后缀
         """
-        if _set_year:
-            __res = []
-            for year in range(2024, datetime.datetime.today().year + 1):
-                _df = self.download.data_to_df(
-                    db_name=_db_name,
-                    table_name=f'{_table_name}_{year}',
-                    start_date=start_date,
-                    end_date=end_date,
-                    projection={},
+        dfs = []
+        if set_year:
+            current_year = datetime.datetime.today().year
+            for year in range(2024, current_year + 1):
+                df = self._fetch_table_data(
+                    db_name, f"{table_name}_{year}", start_date, end_date
                 )
-                __res.append(_df)
-            _df = pd.concat(__res, ignore_index=True)
+                if df is not None:
+                    dfs.append(df)
         else:
-            _df = self.download.data_to_df(
-                db_name=_db_name,
-                table_name=_table_name,
-                start_date=start_date,
-                end_date=end_date,
-                projection={},
-            )
-        if len(_df) == 0:
-            print(f'{_db_name} - {_table_name}: mysql读取的数据不能为空')
-            return pd.DataFrame()
-        if '日期' in _df.columns.tolist():
-            _df['日期'] = pd.to_datetime(_df['日期'], format='%Y-%m-%d', errors='ignore')  # 转换日期列
-        return _df
+            df = self._fetch_table_data(db_name, table_name, start_date, end_date)
+            if df is not None:
+                dfs.append(df)
 
-    def get_from_redis(self, _db_name, _table_name, _set_year, start_date, end_date):
+        combined_df = pd.concat(dfs, ignore_index=True) if dfs else pd.DataFrame()
+        if combined_df.empty:
+            logger.info(f"警告: {db_name}.{table_name} 未读取到数据")
+        else:
+            combined_df = self._convert_date_columns(combined_df)
+        return combined_df
+
+    def get_from_redis(
+            self,
+            db_name: str,
+            table_name: str,
+            set_year: bool,
+            start_date,
+            end_date
+    ) -> pd.DataFrame:
         """
-        _set_year: _table_name 中是否含有年份
-        _col_list: 如果不传就取 table 的所有列
-        对于日期: 最终传出的是日期格式，但如果存入 redis ，需要先格式化为 str，避免日期变整数形式
+        从 Redis 获取数据，若缓存过期/不完整则触发异步更新
         """
-        start_date = pd.to_datetime(start_date)
-        end_date = pd.to_datetime(end_date)
-        if _set_year:
-            my_key = f'{_db_name}:{_table_name}_haveyear'
-        else:
-            my_key = f'{_db_name}:{_table_name}'
-        # ttl 对于不存在的键，它返回 -2；而对于没有设置过期时间的键，它返回 -1
+        start_dt = pd.to_datetime(start_date)
+        end_dt = pd.to_datetime(end_date)
+        cache_key = self._generate_cache_key(db_name, table_name, set_year)
+
+        # 尝试获取缓存元数据
         try:
-            ttl_result = self.redis_engin.ttl(my_key)
+            ttl = self.redis_engine.ttl(cache_key)
+            cache_data = self._fetch_redis_data(cache_key)
         except Exception as e:
-            # redis 连接失败, 则绕过 redis 直接从 mysql 获取数据
-            print('redis 连接失败, 绕过 redis 直接从 mysql 获取数据')
-            _df = self.get_from_mysql(_db_name=_db_name, _table_name=_table_name, start_date=start_date, end_date=end_date, _set_year=_set_year)
-            return _df
-        _df = pd.DataFrame()
-
-        if ttl_result < 60:
-            # 1. redis 没有该数据时
-            print(f'数据不存在或过期')
-            thread = threading.Thread(target=self.set_redis, args=(my_key, _db_name, _table_name, _set_year, start_date, end_date, _df))
-            thread.start()
-            # _df = self.set_redis(my_key=my_key, _db_name=_db_name, _table_name=_table_name, _set_year=_set_year, start_date=start_date, end_date=end_date)
-            _df = self.get_from_mysql(_db_name=_db_name, _table_name=_table_name, start_date=start_date, end_date=end_date, _set_year=_set_year)
-            return _df
-        # 2. redis 有数据时
-        json_string = self.redis_engin.get(my_key)
-        data_dict = json.loads(json_string.decode('utf-8'))
-        _df = pd.DataFrame(data_dict)
-
-        if '日期' in _df.columns.tolist():
-            _df['日期'] = pd.to_datetime(_df['日期'], format='%Y-%m-%d', errors='ignore')  # 转换日期列
-            min_date = _df['日期'].min()
-            max_date = _df['日期'].max()
-            # Bug: 如果外部请求日期小于 table 最小日期，每次都要从 mysql 获取数据，即使 redis 缓存了数据
-            if start_date < min_date:  # 外部请求日期小于 redis 数据最小日期
-                # 3. redis 有数据但数据不完整时
-                print(f'{start_date} -- {min_date} 数据日期不对劲需要更新')
-                thread = threading.Thread(target=self.set_redis, args=(my_key, _db_name, _table_name, _set_year, start_date, end_date, _df))
-                thread.start()
-                # _df = self.set_redis(my_key=my_key, _db_name=_db_name, _table_name=_table_name, _set_year=_set_year, start_date=start_date, end_date=end_date)
-                _df = self.get_from_mysql(_db_name=_db_name, _table_name=_table_name, start_date=start_date, end_date=end_date, _set_year=_set_year)
-                return _df
-            _df = _df[(_df['日期'] >= start_date) & (_df['日期'] <= end_date)]
-
-        return _df
-
-    def set_redis(self, my_key, _db_name, _table_name, _set_year, start_date, end_date, before_df):
+            logger.info(f"Redis 连接异常: {e}，直接访问 MySQL")
+            return self.get_from_mysql(db_name, table_name, set_year, start_date, end_date)
+
+        # 缓存失效处理逻辑
+        if ttl < 60 or cache_data.empty:
+            self._trigger_async_cache_update(
+                cache_key, db_name, table_name, set_year, start_date, end_date, cache_data
+            )
+            return self.get_from_mysql(db_name, table_name, set_year, start_date, end_date)
+
+        # 处理有效缓存数据
+        filtered_df = self._filter_by_date_range(cache_data, start_dt, end_dt)
+        if not filtered_df.empty:
+            return filtered_df
+
+        # 缓存数据不满足查询范围要求
+        self._trigger_async_cache_update(
+            cache_key, db_name, table_name, set_year, start_date, end_date, cache_data
+        )
+        return self.get_from_mysql(db_name, table_name, set_year, start_date, end_date)
+
+    def set_redis(
+            self,
+            cache_key: str,
+            db_name: str,
+            table_name: str,
+            set_year: bool,
+            start_date,
+            end_date,
+            existing_data: pd.DataFrame
+    ) -> pd.DataFrame:
         """
-        从 mysql 读取数据并存储 redis
-        由于这个函数是异步执行的，从页面段首次加载数据时，可能返回空，等待异步执行结束后会正常返回数据
+        异步更新 Redis 缓存，合并新旧数据
         """
-        _df = self.get_from_mysql(
-            _db_name=_db_name,
-            _table_name=_table_name,
-            start_date=start_date,
-            end_date=end_date,
-            _set_year=_set_year
+        try:
+            # 从 MySQL 获取新数据
+            new_data = self.get_from_mysql(db_name, table_name, set_year, start_date, end_date)
+            if new_data.empty:
+                return pd.DataFrame()
+
+            # 合并历史数据
+            combined_data = self._merge_data(new_data, existing_data)
+
+            # 序列化并存储到 Redis
+            serialized_data = self._serialize_data(combined_data)
+            self.redis_engine.set(cache_key, serialized_data)
+            self.redis_engine.expire(cache_key, self.cache_ttl)
+
+            logger.info(f"缓存更新 {cache_key} | 数据量: {len(combined_data)}")
+            return combined_data
+
+        except Exception as e:
+            logger.info(f"缓存更新失败: {cache_key} - {str(e)}")
+            return pd.DataFrame()
+
+    # Helper Methods ------------------------------------------------
+
+    def _fetch_table_data(
+            self,
+            db_name: str,
+            table_name: str,
+            start_date,
+            end_date
+    ) -> pd.DataFrame:
+        """封装 MySQL 数据获取逻辑"""
+        try:
+            return self.download.data_to_df(
+                db_name=db_name,
+                table_name=table_name,
+                start_date=start_date,
+                end_date=end_date,
+                projection={}
+            )
+        except Exception as e:
+            logger.info(f"MySQL 查询异常 {db_name}.{table_name}: {e}")
+            return pd.DataFrame()
+
+    def _fetch_redis_data(self, cache_key: str) -> pd.DataFrame:
+        """从 Redis 获取并解析数据（自动转换日期列）"""
+        try:
+            data = self.redis_engine.get(cache_key)
+            if not data:
+                return pd.DataFrame()
+            # 反序列化数据
+            df = pd.DataFrame(json.loads(data.decode("utf-8")))
+            return self._convert_date_columns(df)
+        except Exception as e:
+            logger.info(f"Redis 数据解析失败 {cache_key}: {e}")
+            return pd.DataFrame()
+
+    def _convert_date_columns(self, df: pd.DataFrame) -> pd.DataFrame:
+        """统一处理日期列转换"""
+        if "日期" in df.columns:
+            df["日期"] = pd.to_datetime(df["日期"], format="%Y-%m-%d", errors="coerce")
+        return df
+
+    def _generate_cache_key(self, db_name: str, table_name: str, set_year: bool) -> str:
+        """生成标准化的缓存键"""
+        return f"{db_name}:{table_name}_haveyear" if set_year else f"{db_name}:{table_name}"
+
+    def _filter_by_date_range(
+            self,
+            df: pd.DataFrame,
+            start_dt: datetime.datetime,
+            end_dt: datetime.datetime
+    ) -> pd.DataFrame:
+        """按日期范围筛选数据"""
+        if "日期" not in df.columns:
+            return df
+        date_mask = (df["日期"] >= start_dt) & (df["日期"] <= end_dt)
+        return df[date_mask].copy()
+
+    def _trigger_async_cache_update(
+            self,
+            cache_key: str,
+            db_name: str,
+            table_name: str,
+            set_year: bool,
+            start_date: str,
+            end_date: str,
+            existing_data: pd.DataFrame
+    ):
+        """启动异步缓存更新线程"""
+        thread = threading.Thread(
+            target=self.set_redis,
+            args=(cache_key, db_name, table_name, set_year, start_date, end_date, existing_data),
+            daemon=True
         )
-        if '日期' in _df.columns.tolist():
-            _min_date = _df['日期'].min()
-            _max_date = _df['日期'].max()
-        if '日期' in before_df.columns.tolist():
-            # 移除 redis 指定范围的数据，再合并新数据
-            before_df1 = before_df[(before_df['日期'] < _min_date)]
-            before_df2 = before_df[(before_df['日期'] > _max_date)]
-            _df = pd.concat([_df, before_df1, before_df2], ignore_index=True, axis=0)
-        # if '日期' in _df.columns.tolist():
-        #     _df['日期'] = _df['日期'].astype('str')
-        for col in _df.columns.tolist():
-            # 存入 redis ，需要先格式化为 str，避免日期变整数形式
-            if _df[col].dtype == 'datetime64[ns]':
-                _df[col] = _df[col].astype('str')
-        jsondata = _df.to_json(orient='records', force_ascii=False)
-        self.redis_engin.set(my_key, jsondata)
-        self.redis_engin.expire(my_key, self.minute * 60)  # 设置缓存过期时间: 分钟
-        if '日期' in _df.columns.tolist():
-            _df['日期'] = pd.to_datetime(_df['日期'], format='%Y-%m-%d', errors='ignore')  # 转换日期列
-        now = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
-        print(f'{now}: 刷新 redis -> {_db_name}:{_table_name}')
-        return _df
+        thread.start()
+
+    def _merge_data(self, new_data: pd.DataFrame, existing_data: pd.DataFrame) -> pd.DataFrame:
+        """合并新旧数据集"""
+        if existing_data.empty or "日期" not in existing_data.columns:
+            return new_data
+
+        new_min = new_data["日期"].min()
+        new_max = new_data["日期"].max()
+        valid_historical = existing_data[
+            (existing_data["日期"] < new_min) | (existing_data["日期"] > new_max)
+            ]
+        return pd.concat([new_data, valid_historical], ignore_index=True).drop_duplicates(subset=["日期"])
+
+    def _serialize_data(self, df: pd.DataFrame) -> str:
+        """序列化 DataFrame 并处理日期类型"""
+        temp_df = df.copy()
+        date_cols = temp_df.select_dtypes(include=["datetime64[ns]"]).columns
+        for col in date_cols:
+            temp_df[col] = temp_df[col].dt.strftime("%Y-%m-%d")
+        return temp_df.to_json(orient="records", force_ascii=False)
 
 
 if __name__ == '__main__':
-    # ****************************************************
-    # 这一部分在外部定义，只需要定义一次，开始
-    redis_config = {
-        'host': '127.0.0.1',
-        'port': 6379,  # 默认Redis端口
-        'db': 0,  # 默认Redis数据库索引
-        # 'username': 'default',
-        'password': redis_password,
-    }
-    # redis 实例化
-    r = redis.Redis(**redis_config)
-    # mysql 实例化
-    d = s_query.QueryDatas(username=username, password=password, host=host, port=port)
-    # 将两个库的实例化对象传给 RedisData 类，并实例化数据处理引擎
-    m = RedisData(redis_engin=r, download=d)
-    # ****************************************************
-
-    # 以下为动态获取数据库数据
-    db_name = '聚合数据'
-    table_name = '多店推广场景_按日聚合'
-    set_year = False
-    df = m.get_from_redis(
-        _db_name=db_name,
-        _table_name=table_name,
-        _set_year=set_year,
-        start_date='2025-01-01',
-        end_date='2025-01-31'
-    )
-    print(df)
+    # # ****************************************************
+    # # 这一部分在外部定义，只需要定义一次，开始
+    # redis_config = {
+    #     'host': '127.0.0.1',
+    #     'port': 6379,  # 默认Redis端口
+    #     'db': 0,  # 默认Redis数据库索引
+    #     # 'username': 'default',
+    #     'password': redis_password,
+    # }
+    # # redis 实例化
+    # r = redis.Redis(**redis_config)
+    # # mysql 实例化
+    # d = s_query.QueryDatas(username=username, password=password, host=host, port=port)
+    # # 将两个库的实例化对象传给 RedisData 类，并实例化数据处理引擎
+    # m = RedisData(redis_engin=r, download=d)
+    # # ****************************************************
+    #
+    # # 以下为动态获取数据库数据
+    # db_name = '聚合数据'
+    # table_name = '多店推广场景_按日聚合'
+    # set_year = False
+    # df = m.get_from_redis(
+    #     db_name=db_name,
+    #     table_name=table_name,
+    #     set_year=set_year,
+    #     start_date='2025-01-01',
+    #     end_date='2025-01-31'
+    # )
+    # logger.info(df)
+    #
+
+    logger.info(socket.gethostname())

mdbq/redis/getredis_bak20250131.py

@@ -0,0 +1,265 @@
+# -*- coding: UTF-8 –*-
+import redis
+import socket
+from mdbq.mysql import s_query
+from mdbq.config import myconfig
+import pandas as pd
+import json
+import datetime
+import threading
+
+
+if socket.gethostname() == 'company' or socket.gethostname() == 'Mac2.local':
+    conf = myconfig.main()
+    conf_data = conf['Windows']['company']['mysql']['local']
+    username, password, host, port = conf_data['username'], conf_data['password'], conf_data['host'], conf_data['port']
+    redis_password = conf['Windows']['company']['redis']['local']['password']
+# elif socket.gethostname() == 'MacBook-Pro.local':
+#     conf = myconfig.main()
+#     conf_data = conf['Windows']['xigua_lx']['mysql']['local']
+#     username, password, host, port = conf_data['username'], conf_data['password'], conf_data['host'], conf_data['port']
+#     redis_password = conf['Windows']['company']['redis']['local']['password']
+else:
+    conf = myconfig.main()
+    conf_data = conf['Windows']['xigua_lx']['mysql']['local']
+    username, password, host, port = conf_data['username'], conf_data['password'], conf_data['host'], conf_data['port']
+    redis_password = conf['Windows']['company']['redis']['local']['password']  # redis 使用本地数据，全部机子相同
+
+
+
+class RedisData(object):
+    def __init__(self, redis_engin, download):
+        self.redis_engin = redis_engin  # redis 数据处理引擎
+        self.download = download  # mysql 数据处理引擎
+        self.minute = 60  # 缓存过期时间: 分钟
+
+    def get_from_mysql(self, _db_name, _table_name, _set_year, start_date, end_date):
+        """
+        _set_year: _table_name 中是否含有年份
+        """
+        if _set_year:
+            __res = []
+            for year in range(2024, datetime.datetime.today().year + 1):
+                _df = self.download.data_to_df(
+                    db_name=_db_name,
+                    table_name=f'{_table_name}_{year}',
+                    start_date=start_date,
+                    end_date=end_date,
+                    projection={},
+                )
+                __res.append(_df)
+            _df = pd.concat(__res, ignore_index=True)
+        else:
+            _df = self.download.data_to_df(
+                db_name=_db_name,
+                table_name=_table_name,
+                start_date=start_date,
+                end_date=end_date,
+                projection={},
+            )
+        if len(_df) == 0:
+            print(f'{_db_name} - {_table_name}: mysql读取的数据不能为空')
+            return pd.DataFrame()
+        if '日期' in _df.columns.tolist():
+            _df['日期'] = pd.to_datetime(_df['日期'], format='%Y-%m-%d', errors='ignore')  # 转换日期列
+        return _df
+
+    def get_from_redis(self, _db_name, _table_name, _set_year, start_date, end_date):
+        """
+        _set_year: _table_name 中是否含有年份
+        _col_list: 如果不传就取 table 的所有列
+        对于日期: 最终传出的是日期格式，但如果存入 redis ，需要先格式化为 str，避免日期变整数形式
+        """
+        start_date = pd.to_datetime(start_date)
+        end_date = pd.to_datetime(end_date)
+        if _set_year:
+            my_key = f'{_db_name}:{_table_name}_haveyear'
+        else:
+            my_key = f'{_db_name}:{_table_name}'
+        # ttl 对于不存在的键，它返回 -2；而对于没有设置过期时间的键，它返回 -1
+        try:
+            ttl_result = self.redis_engin.ttl(my_key)
+        except Exception as e:
+            # redis 连接失败, 则绕过 redis 直接从 mysql 获取数据
+            print('redis 连接失败, 绕过 redis 直接从 mysql 获取数据')
+            _df = self.get_from_mysql(_db_name=_db_name, _table_name=_table_name, start_date=start_date, end_date=end_date, _set_year=_set_year)
+            return _df
+        _df = pd.DataFrame()
+
+        if ttl_result < 60:
+            # 1. redis 没有该数据时
+            print(f'数据不存在或过期')
+            thread = threading.Thread(target=self.set_redis, args=(my_key, _db_name, _table_name, _set_year, start_date, end_date, _df))
+            thread.start()
+            # _df = self.set_redis(my_key=my_key, _db_name=_db_name, _table_name=_table_name, _set_year=_set_year, start_date=start_date, end_date=end_date)
+            _df = self.get_from_mysql(_db_name=_db_name, _table_name=_table_name, start_date=start_date, end_date=end_date, _set_year=_set_year)
+            return _df
+        # 2. redis 有数据时
+        json_string = self.redis_engin.get(my_key)
+        data_dict = json.loads(json_string.decode('utf-8'))
+        _df = pd.DataFrame(data_dict)
+
+        if '日期' in _df.columns.tolist():
+            _df['日期'] = pd.to_datetime(_df['日期'], format='%Y-%m-%d', errors='ignore')  # 转换日期列
+            min_date = _df['日期'].min()
+            max_date = _df['日期'].max()
+            # Bug: 如果外部请求日期小于 table 最小日期，每次都要从 mysql 获取数据，即使 redis 缓存了数据
+            if start_date < min_date:  # 外部请求日期小于 redis 数据最小日期
+                # 3. redis 有数据但数据不完整时
+                print(f'{start_date} -- {min_date} 数据日期需要更新')
+                thread = threading.Thread(target=self.set_redis, args=(my_key, _db_name, _table_name, _set_year, start_date, end_date, _df))
+                thread.start()
+                # _df = self.set_redis(my_key=my_key, _db_name=_db_name, _table_name=_table_name, _set_year=_set_year, start_date=start_date, end_date=end_date)
+                _df = self.get_from_mysql(_db_name=_db_name, _table_name=_table_name, start_date=start_date, end_date=end_date, _set_year=_set_year)
+                return _df
+            _df = _df[(_df['日期'] >= start_date) & (_df['日期'] <= end_date)]
+
+        return _df
+
+    def set_redis(self, my_key, _db_name, _table_name, _set_year, start_date, end_date, before_df):
+        """
+        从MySQL读取数据并存储到Redis（异步执行）
+
+        Args:
+            my_key: Redis存储键名
+            _db_name: 数据库名称
+            _table_name: 数据表名称
+            _set_year: 数据集年份
+            start_date: 查询开始日期
+            end_date: 查询结束日期
+            before_df: 合并用的历史数据
+
+        Returns:
+            pd.DataFrame: 处理后的数据集（含历史数据合并）
+        """
+        # 异常处理容器
+        datetime_cols = []
+
+        try:
+            # 从MySQL获取数据
+            _df = self.get_from_mysql(
+                _db_name=_db_name,
+                _table_name=_table_name,
+                start_date=start_date,
+                end_date=end_date,
+                _set_year=_set_year
+            )
+
+            # 日期列处理（当新旧数据都存在日期列时）
+            if '日期' in _df.columns and '日期' in before_df.columns:
+                # 获取当前数据时间范围
+                _min_date, _max_date = _df['日期'].min(), _df['日期'].max()
+
+                # 筛选需要保留的历史数据
+                mask = (before_df['日期'] < _min_date) | (before_df['日期'] > _max_date)
+                valid_history = before_df[mask]
+
+                # 合并数据
+                _df = pd.concat([_df, valid_history], ignore_index=True, axis=0)
+                _df.drop_duplicates(subset='日期', keep='first', inplace=True)  # 可选去重
+
+            # 预处理时间类型转换
+            datetime_cols = _df.select_dtypes(include=['datetime64[ns]']).columns.tolist()
+            if datetime_cols:
+                _df[datetime_cols] = _df[datetime_cols].astype(str)
+
+            # 空数据检查
+            if _df.empty:
+                print(f'Warning: {_table_name} 空数据集，跳过Redis存储')
+                return pd.DataFrame()
+
+            # Redis存储操作
+            self.redis_engin.set(my_key, _df.to_json(orient='records', force_ascii=False))
+            self.redis_engin.expire(my_key, self.minute * 60)
+
+            # 恢复时间类型（返回用）
+            if datetime_cols:
+                _df[datetime_cols] = _df[datetime_cols].apply(pd.to_datetime, errors='coerce')
+
+            # 记录操作日志
+            print(f'{datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")}'
+                  f' | 刷新Redis {_db_name}:{_table_name}'
+                  f' | 数据量：{len(_df)}行')
+
+        except Exception as e:
+            print(f'Error: {_table_name} 数据处理失败 - {str(e)}')
+            _df = pd.DataFrame()
+
+        finally:
+            # 确保返回前恢复时间类型
+            if datetime_cols and not _df.empty:
+                _df[datetime_cols] = _df[datetime_cols].apply(pd.to_datetime, errors='ignore')
+
+        return _df
+
+    def set_redis_bak(self, my_key, _db_name, _table_name, _set_year, start_date, end_date, before_df):
+        """
+        从 mysql 读取数据并存储 redis
+        由于这个函数是异步执行的，从页面段首次加载数据时，可能返回空，等待异步执行结束后会正常返回数据
+        """
+        _df = self.get_from_mysql(
+            _db_name=_db_name,
+            _table_name=_table_name,
+            start_date=start_date,
+            end_date=end_date,
+            _set_year=_set_year
+        )
+        if '日期' in _df.columns.tolist():
+            _min_date = _df['日期'].min()
+            _max_date = _df['日期'].max()
+        if '日期' in before_df.columns.tolist():
+            # 移除 redis 指定范围的数据，再合并新数据
+            before_df1 = before_df[(before_df['日期'] < _min_date)]
+            before_df2 = before_df[(before_df['日期'] > _max_date)]
+            _df = pd.concat([_df, before_df1, before_df2], ignore_index=True, axis=0)
+        # if '日期' in _df.columns.tolist():
+        #     _df['日期'] = _df['日期'].astype('str')
+        for col in _df.columns.tolist():
+            # 存入 redis ，需要先格式化为 str，避免日期变整数形式
+            if _df[col].dtype == 'datetime64[ns]':
+                _df[col] = _df[col].astype('str')
+        if len(_df) == 0:
+            print(f'{_table_name}: 写入 redis 的数据不能为空')
+            return pd.DataFrame()
+        jsondata = _df.to_json(orient='records', force_ascii=False)
+        self.redis_engin.set(my_key, jsondata)
+        self.redis_engin.expire(my_key, self.minute * 60)  # 设置缓存过期时间: 分钟
+        if '日期' in _df.columns.tolist():
+            _df['日期'] = pd.to_datetime(_df['日期'], format='%Y-%m-%d', errors='ignore')  # 转换日期列
+        now = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
+        print(f'{now}: 刷新 redis -> {_db_name}:{_table_name}')
+        return _df
+
+if __name__ == '__main__':
+    # # ****************************************************
+    # # 这一部分在外部定义，只需要定义一次，开始
+    # redis_config = {
+    #     'host': '127.0.0.1',
+    #     'port': 6379,  # 默认Redis端口
+    #     'db': 0,  # 默认Redis数据库索引
+    #     # 'username': 'default',
+    #     'password': redis_password,
+    # }
+    # # redis 实例化
+    # r = redis.Redis(**redis_config)
+    # # mysql 实例化
+    # d = s_query.QueryDatas(username=username, password=password, host=host, port=port)
+    # # 将两个库的实例化对象传给 RedisData 类，并实例化数据处理引擎
+    # m = RedisData(redis_engin=r, download=d)
+    # # ****************************************************
+    #
+    # # 以下为动态获取数据库数据
+    # db_name = '聚合数据'
+    # table_name = '多店推广场景_按日聚合'
+    # set_year = False
+    # df = m.get_from_redis(
+    #     _db_name=db_name,
+    #     _table_name=table_name,
+    #     _set_year=set_year,
+    #     start_date='2025-01-01',
+    #     end_date='2025-01-31'
+    # )
+    # print(df)
+    #
+
+    print(socket.gethostname())

mdbq/redis/getredis_deepseek.py

@@ -0,0 +1,235 @@
+# -*- coding: UTF-8 –*-
+import redis
+import socket
+from mdbq.mysql import s_query
+from mdbq.config import myconfig
+import pandas as pd
+import json
+import datetime
+import threading
+
+
+if socket.gethostname() == 'company' or socket.gethostname() == 'Mac2.local':
+    conf = myconfig.main()
+    conf_data = conf['Windows']['company']['mysql']['local']
+    username, password, host, port = conf_data['username'], conf_data['password'], conf_data['host'], conf_data['port']
+    redis_password = conf['Windows']['company']['redis']['local']['password']
+# elif socket.gethostname() == 'MacBook-Pro.local':
+#     conf = myconfig.main()
+#     conf_data = conf['Windows']['xigua_lx']['mysql']['local']
+#     username, password, host, port = conf_data['username'], conf_data['password'], conf_data['host'], conf_data['port']
+#     redis_password = conf['Windows']['company']['redis']['local']['password']
+else:
+    conf = myconfig.main()
+    conf_data = conf['Windows']['xigua_lx']['mysql']['local']
+    username, password, host, port = conf_data['username'], conf_data['password'], conf_data['host'], conf_data['port']
+    redis_password = conf['Windows']['company']['redis']['local']['password']  # redis 使用本地数据，全部机子相同
+
+
+class RedisData:
+    def __init__(self, redis_engine, download, cache_ttl: int = 60):
+        """
+        初始化Redis数据处理对象
+
+        :param redis_engine: Redis连接引擎
+        :param download: 数据库下载处理器
+        :param cache_ttl: 缓存过期时间（分钟）
+        """
+        self.redis_engine = redis_engine
+        self.download = download
+        self.cache_ttl = cache_ttl * 60  # 转换为秒
+        self.lock = threading.Lock()  # 线程锁
+
+    def _handle_datetime_columns(self, df: pd.DataFrame) -> pd.DataFrame:
+        """统一处理日期列转换"""
+        if '日期' in df.columns:
+            df['日期'] = pd.to_datetime(df['日期'], errors='coerce', format='%Y-%m-%d')
+        return df
+
+    def get_from_mysql(self, db_name: str, table_name: str,
+                       start_date, end_date,
+                       set_year: bool) -> pd.DataFrame:
+        """
+        从MySQL获取数据
+
+        :param set_year: 是否按年份分表
+        """
+        try:
+            if set_year:
+                current_year = datetime.datetime.now().year
+                dfs = []
+                # 动态获取需要查询的年份范围
+                min_year = min(2024, pd.to_datetime(start_date).year)  # 根据实际需求调整
+                for year in range(min_year, current_year + 1):
+                    table = f"{table_name}_{year}"
+                    df = self.download.data_to_df(
+                        db_name=db_name,
+                        table_name=table,
+                        start_date=start_date,
+                        end_date=end_date,
+                        projection={}
+                    )
+                    if not df.empty:
+                        dfs.append(df)
+                _df = pd.concat(dfs, ignore_index=True) if dfs else pd.DataFrame()
+            else:
+                _df = self.download.data_to_df(
+                    db_name=db_name,
+                    table_name=table_name,
+                    start_date=start_date,
+                    end_date=end_date,
+                    projection={}
+                )
+
+            if _df.empty:
+                print(f"空数据 - {db_name}.{table_name}")
+                return pd.DataFrame()
+
+            return self._handle_datetime_columns(_df)
+
+        except Exception as e:
+            print(f"MySQL查询失败: {str(e)}")
+            return pd.DataFrame()
+
+    def get_from_redis(self, db_name: str, table_name: str,
+                       start_date, end_date,
+                       set_year: bool) -> pd.DataFrame:
+        """
+        从Redis获取数据（带自动缓存更新）
+        """
+        start_dt = pd.to_datetime(start_date)
+        end_dt = pd.to_datetime(end_date)
+
+        # 生成统一缓存键
+        cache_key = f"{db_name}:{table_name}{'_year' if set_year else ''}"
+
+        try:
+            # 检查缓存状态
+            with self.lock:  # 保证线程安全
+                ttl = self.redis_engine.ttl(cache_key)
+
+                if ttl < 300:  # 剩余时间小于5分钟时触发更新
+                    print(f"异步缓存更新: {cache_key}")
+                    threading.Thread(
+                        target=self._update_cache,
+                        args=(cache_key, db_name, table_name,
+                              start_date, end_date, set_year),
+                        daemon=True
+                    ).start()
+
+            # 获取缓存数据
+            cached_data = self.redis_engine.get(cache_key)
+            if not cached_data:
+                return self._fallback_to_mysql(db_name, table_name,
+                                               start_date, end_date, set_year)
+            json_str = cached_data.decode('utf-8')
+            _df = pd.read_json(json_str, orient='records')
+            _df = self._handle_datetime_columns(_df)
+
+            # 数据范围校验
+            if '日期' in _df.columns:
+                cache_min = _df['日期'].min()
+                cache_max = _df['日期'].max()
+
+                # 请求范围超出缓存范围时需要更新
+                if start_dt < cache_min or end_dt > cache_max:
+                    print(f"请求范围超出缓存 {start_dt.strftime('%Y-%m-%d ')} - {end_dt.strftime('%Y-%m-%d ')}")
+                    self._update_cache(cache_key, db_name, table_name,
+                                       start_date, end_date, set_year, _df)
+                    return self._fallback_to_mysql(db_name, table_name,
+                                                   start_date, end_date, set_year)
+
+                return _df[(start_dt <= _df['日期']) & (_df['日期'] <= end_dt)]
+            return _df
+
+        except Exception as e:
+            print(f"Redis操作失败: {str(e)}")
+            return self._fallback_to_mysql(db_name, table_name,
+                                           start_date, end_date, set_year)
+
+    def _update_cache(self, cache_key: str, db_name: str, table_name: str,
+                      start_date: str, end_date: str, set_year: bool,
+                      existing_df: pd.DataFrame = None) -> None:
+        """缓存更新核心逻辑"""
+        try:
+            # 获取最新数据
+            new_data = self.get_from_mysql(
+                db_name=db_name,
+                table_name=table_name,
+                start_date=start_date,
+                end_date=end_date,
+                set_year=set_year
+            )
+
+            # 合并历史数据
+            if existing_df is not None and not new_data.empty:
+                combined = pd.concat([existing_df, new_data], ignore_index=True)
+                combined = combined.drop_duplicates(subset='日期', keep='last')
+            else:
+                combined = new_data
+
+            if not combined.empty:
+                # 转换日期类型为字符串
+                temp_df = combined.copy()
+                datetime_cols = temp_df.select_dtypes(include=['datetime64[ns]']).columns
+                temp_df[datetime_cols] = temp_df[datetime_cols].astype(str)
+
+                # 存储到Redis
+                with self.lock:
+                    self.redis_engine.set(
+                        cache_key,
+                        temp_df.to_json(orient='records', force_ascii=False),
+                        ex=self.cache_ttl
+                    )
+                print(f"缓存更新成功: {cache_key} | 记录数: {len(combined)}")
+
+        except Exception as e:
+            print(f"缓存更新失败: {str(e)}")
+
+    def _fallback_to_mysql(self, db_name: str, table_name: str,
+                           start_date: str, end_date: str,
+                           set_year: bool) -> pd.DataFrame:
+        """降级到直接MySQL查询"""
+        print(f"降级到MySQL查询: {db_name}.{table_name}")
+        return self.get_from_mysql(
+            db_name=db_name,
+            table_name=table_name,
+            start_date=start_date,
+            end_date=end_date,
+            set_year=set_year
+        )
+
+
+if __name__ == '__main__':
+    # # ****************************************************
+    # # 这一部分在外部定义，只需要定义一次，开始
+    # redis_config = {
+    #     'host': '127.0.0.1',
+    #     'port': 6379,  # 默认Redis端口
+    #     'db': 0,  # 默认Redis数据库索引
+    #     # 'username': 'default',
+    #     'password': redis_password,
+    # }
+    # # redis 实例化
+    # r = redis.Redis(**redis_config)
+    # # mysql 实例化
+    # d = s_query.QueryDatas(username=username, password=password, host=host, port=port)
+    # # 将两个库的实例化对象传给 RedisData 类，并实例化数据处理引擎
+    # m = RedisData(redis_engin=r, download=d)
+    # # ****************************************************
+    #
+    # # 以下为动态获取数据库数据
+    # db_name = '聚合数据'
+    # table_name = '多店推广场景_按日聚合'
+    # set_year = False
+    # df = m.get_from_redis(
+    #     _db_name=db_name,
+    #     _table_name=table_name,
+    #     _set_year=set_year,
+    #     start_date='2025-01-01',
+    #     end_date='2025-01-31'
+    # )
+    # print(df)
+    #
+
+    print(socket.gethostname())

{mdbq-3.6.2.dist-info → mdbq-3.6.4.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: mdbq
-Version: 3.6.2
+Version: 3.6.4
 Home-page: https://pypi.org/project/mdbq
 Author: xigua, 
 Author-email: 2587125111@qq.com

{mdbq-3.6.2.dist-info → mdbq-3.6.4.dist-info}/RECORD

@@ -33,10 +33,12 @@ mdbq/pbix/pbix_refresh.py,sha256=JUjKW3bNEyoMVfVfo77UhguvS5AWkixvVhDbw4_MHco,239
 mdbq/pbix/refresh_all.py,sha256=OBT9EewSZ0aRS9vL_FflVn74d4l2G00wzHiikCC4TC0,5926
 mdbq/pbix/refresh_all_old.py,sha256=_pq3WSQ728GPtEG5pfsZI2uTJhU8D6ra-htIk1JXYzw,7192
 mdbq/redis/__init__.py,sha256=YtgBlVSMDphtpwYX248wGge1x-Ex_mMufz4-8W0XRmA,12
-mdbq/redis/getredis.py,sha256=zpkdy624Drj2ujvOcldKS4PSN8bfKKmIIRkB0KqL7x0,8657
+mdbq/redis/getredis.py,sha256=5aLu2_S5hvv48uVQSjp9wxHBE8WOiAkpSpugXcy9FXM,11184
+mdbq/redis/getredis_bak20250131.py,sha256=DQazRyKVnaDziP9JEIofAJF8dw_PKyLEgwEznlTnGDw,12284
+mdbq/redis/getredis_deepseek.py,sha256=bQ6VfiTYkQ5cYK6MYJPKgwbdrwsOLBLrV-ObblKaurA,9653
 mdbq/spider/__init__.py,sha256=RBMFXGy_jd1HXZhngB2T2XTvJqki8P_Fr-pBcwijnew,18
 mdbq/spider/aikucun.py,sha256=v7VO5gtEXR6_4Q6ujbTyu1FHu7TXHcwSQ6hIO249YH0,22208
-mdbq-3.6.2.dist-info/METADATA,sha256=EtY3g6bo6ZkYkc_47W3VJO6toGHWZXIFVaBgXZtjX_U,243
-mdbq-3.6.2.dist-info/WHEEL,sha256=eOLhNAGa2EW3wWl_TU484h7q1UNgy0JXjjoqKoxAAQc,92
-mdbq-3.6.2.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
-mdbq-3.6.2.dist-info/RECORD,,
+mdbq-3.6.4.dist-info/METADATA,sha256=PSbZGIuI23xbX4wJsaqFJltQAJ4ugR_gaD1SPTnljMQ,243
+mdbq-3.6.4.dist-info/WHEEL,sha256=cpQTJ5IWu9CdaPViMhC9YzF8gZuS5-vlfoFihTBC86A,91
+mdbq-3.6.4.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
+mdbq-3.6.4.dist-info/RECORD,,

{mdbq-3.6.2.dist-info → mdbq-3.6.4.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: bdist_wheel (0.44.0)
+Generator: setuptools (70.1.0)
 Root-Is-Purelib: true
 Tag: py3-none-any