mdbq 3.6.7__py3-none-any.whl → 3.6.9__py3-none-any.whl

mdbq/redis/getredis_bak20250131.py

@@ -1,265 +0,0 @@
-# -*- coding: UTF-8 –*-
-import redis
-import socket
-from mdbq.mysql import s_query
-from mdbq.config import myconfig
-import pandas as pd
-import json
-import datetime
-import threading
-
-
-if socket.gethostname() == 'company' or socket.gethostname() == 'Mac2.local':
-    conf = myconfig.main()
-    conf_data = conf['Windows']['company']['mysql']['local']
-    username, password, host, port = conf_data['username'], conf_data['password'], conf_data['host'], conf_data['port']
-    redis_password = conf['Windows']['company']['redis']['local']['password']
-# elif socket.gethostname() == 'MacBook-Pro.local':
-#     conf = myconfig.main()
-#     conf_data = conf['Windows']['xigua_lx']['mysql']['local']
-#     username, password, host, port = conf_data['username'], conf_data['password'], conf_data['host'], conf_data['port']
-#     redis_password = conf['Windows']['company']['redis']['local']['password']
-else:
-    conf = myconfig.main()
-    conf_data = conf['Windows']['xigua_lx']['mysql']['local']
-    username, password, host, port = conf_data['username'], conf_data['password'], conf_data['host'], conf_data['port']
-    redis_password = conf['Windows']['company']['redis']['local']['password']  # redis 使用本地数据，全部机子相同
-
-
-
-class RedisData(object):
-    def __init__(self, redis_engin, download):
-        self.redis_engin = redis_engin  # redis 数据处理引擎
-        self.download = download  # mysql 数据处理引擎
-        self.minute = 60  # 缓存过期时间: 分钟
-
-    def get_from_mysql(self, _db_name, _table_name, _set_year, start_date, end_date):
-        """
-        _set_year: _table_name 中是否含有年份
-        """
-        if _set_year:
-            __res = []
-            for year in range(2024, datetime.datetime.today().year + 1):
-                _df = self.download.data_to_df(
-                    db_name=_db_name,
-                    table_name=f'{_table_name}_{year}',
-                    start_date=start_date,
-                    end_date=end_date,
-                    projection={},
-                )
-                __res.append(_df)
-            _df = pd.concat(__res, ignore_index=True)
-        else:
-            _df = self.download.data_to_df(
-                db_name=_db_name,
-                table_name=_table_name,
-                start_date=start_date,
-                end_date=end_date,
-                projection={},
-            )
-        if len(_df) == 0:
-            print(f'{_db_name} - {_table_name}: mysql读取的数据不能为空')
-            return pd.DataFrame()
-        if '日期' in _df.columns.tolist():
-            _df['日期'] = pd.to_datetime(_df['日期'], format='%Y-%m-%d', errors='ignore')  # 转换日期列
-        return _df
-
-    def get_from_redis(self, _db_name, _table_name, _set_year, start_date, end_date):
-        """
-        _set_year: _table_name 中是否含有年份
-        _col_list: 如果不传就取 table 的所有列
-        对于日期: 最终传出的是日期格式，但如果存入 redis ，需要先格式化为 str，避免日期变整数形式
-        """
-        start_date = pd.to_datetime(start_date)
-        end_date = pd.to_datetime(end_date)
-        if _set_year:
-            my_key = f'{_db_name}:{_table_name}_haveyear'
-        else:
-            my_key = f'{_db_name}:{_table_name}'
-        # ttl 对于不存在的键，它返回 -2；而对于没有设置过期时间的键，它返回 -1
-        try:
-            ttl_result = self.redis_engin.ttl(my_key)
-        except Exception as e:
-            # redis 连接失败, 则绕过 redis 直接从 mysql 获取数据
-            print('redis 连接失败, 绕过 redis 直接从 mysql 获取数据')
-            _df = self.get_from_mysql(_db_name=_db_name, _table_name=_table_name, start_date=start_date, end_date=end_date, _set_year=_set_year)
-            return _df
-        _df = pd.DataFrame()
-
-        if ttl_result < 60:
-            # 1. redis 没有该数据时
-            print(f'数据不存在或过期')
-            thread = threading.Thread(target=self.set_redis, args=(my_key, _db_name, _table_name, _set_year, start_date, end_date, _df))
-            thread.start()
-            # _df = self.set_redis(my_key=my_key, _db_name=_db_name, _table_name=_table_name, _set_year=_set_year, start_date=start_date, end_date=end_date)
-            _df = self.get_from_mysql(_db_name=_db_name, _table_name=_table_name, start_date=start_date, end_date=end_date, _set_year=_set_year)
-            return _df
-        # 2. redis 有数据时
-        json_string = self.redis_engin.get(my_key)
-        data_dict = json.loads(json_string.decode('utf-8'))
-        _df = pd.DataFrame(data_dict)
-
-        if '日期' in _df.columns.tolist():
-            _df['日期'] = pd.to_datetime(_df['日期'], format='%Y-%m-%d', errors='ignore')  # 转换日期列
-            min_date = _df['日期'].min()
-            max_date = _df['日期'].max()
-            # Bug: 如果外部请求日期小于 table 最小日期，每次都要从 mysql 获取数据，即使 redis 缓存了数据
-            if start_date < min_date:  # 外部请求日期小于 redis 数据最小日期
-                # 3. redis 有数据但数据不完整时
-                print(f'{start_date} -- {min_date} 数据日期需要更新')
-                thread = threading.Thread(target=self.set_redis, args=(my_key, _db_name, _table_name, _set_year, start_date, end_date, _df))
-                thread.start()
-                # _df = self.set_redis(my_key=my_key, _db_name=_db_name, _table_name=_table_name, _set_year=_set_year, start_date=start_date, end_date=end_date)
-                _df = self.get_from_mysql(_db_name=_db_name, _table_name=_table_name, start_date=start_date, end_date=end_date, _set_year=_set_year)
-                return _df
-            _df = _df[(_df['日期'] >= start_date) & (_df['日期'] <= end_date)]
-
-        return _df
-
-    def set_redis(self, my_key, _db_name, _table_name, _set_year, start_date, end_date, before_df):
-        """
-        从MySQL读取数据并存储到Redis（异步执行）
-
-        Args:
-            my_key: Redis存储键名
-            _db_name: 数据库名称
-            _table_name: 数据表名称
-            _set_year: 数据集年份
-            start_date: 查询开始日期
-            end_date: 查询结束日期
-            before_df: 合并用的历史数据
-
-        Returns:
-            pd.DataFrame: 处理后的数据集（含历史数据合并）
-        """
-        # 异常处理容器
-        datetime_cols = []
-
-        try:
-            # 从MySQL获取数据
-            _df = self.get_from_mysql(
-                _db_name=_db_name,
-                _table_name=_table_name,
-                start_date=start_date,
-                end_date=end_date,
-                _set_year=_set_year
-            )
-
-            # 日期列处理（当新旧数据都存在日期列时）
-            if '日期' in _df.columns and '日期' in before_df.columns:
-                # 获取当前数据时间范围
-                _min_date, _max_date = _df['日期'].min(), _df['日期'].max()
-
-                # 筛选需要保留的历史数据
-                mask = (before_df['日期'] < _min_date) | (before_df['日期'] > _max_date)
-                valid_history = before_df[mask]
-
-                # 合并数据
-                _df = pd.concat([_df, valid_history], ignore_index=True, axis=0)
-                _df.drop_duplicates(subset='日期', keep='first', inplace=True)  # 可选去重
-
-            # 预处理时间类型转换
-            datetime_cols = _df.select_dtypes(include=['datetime64[ns]']).columns.tolist()
-            if datetime_cols:
-                _df[datetime_cols] = _df[datetime_cols].astype(str)
-
-            # 空数据检查
-            if _df.empty:
-                print(f'Warning: {_table_name} 空数据集，跳过Redis存储')
-                return pd.DataFrame()
-
-            # Redis存储操作
-            self.redis_engin.set(my_key, _df.to_json(orient='records', force_ascii=False))
-            self.redis_engin.expire(my_key, self.minute * 60)
-
-            # 恢复时间类型（返回用）
-            if datetime_cols:
-                _df[datetime_cols] = _df[datetime_cols].apply(pd.to_datetime, errors='coerce')
-
-            # 记录操作日志
-            print(f'{datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")}'
-                  f' | 刷新Redis {_db_name}:{_table_name}'
-                  f' | 数据量：{len(_df)}行')
-
-        except Exception as e:
-            print(f'Error: {_table_name} 数据处理失败 - {str(e)}')
-            _df = pd.DataFrame()
-
-        finally:
-            # 确保返回前恢复时间类型
-            if datetime_cols and not _df.empty:
-                _df[datetime_cols] = _df[datetime_cols].apply(pd.to_datetime, errors='ignore')
-
-        return _df
-
-    def set_redis_bak(self, my_key, _db_name, _table_name, _set_year, start_date, end_date, before_df):
-        """
-        从 mysql 读取数据并存储 redis
-        由于这个函数是异步执行的，从页面段首次加载数据时，可能返回空，等待异步执行结束后会正常返回数据
-        """
-        _df = self.get_from_mysql(
-            _db_name=_db_name,
-            _table_name=_table_name,
-            start_date=start_date,
-            end_date=end_date,
-            _set_year=_set_year
-        )
-        if '日期' in _df.columns.tolist():
-            _min_date = _df['日期'].min()
-            _max_date = _df['日期'].max()
-        if '日期' in before_df.columns.tolist():
-            # 移除 redis 指定范围的数据，再合并新数据
-            before_df1 = before_df[(before_df['日期'] < _min_date)]
-            before_df2 = before_df[(before_df['日期'] > _max_date)]
-            _df = pd.concat([_df, before_df1, before_df2], ignore_index=True, axis=0)
-        # if '日期' in _df.columns.tolist():
-        #     _df['日期'] = _df['日期'].astype('str')
-        for col in _df.columns.tolist():
-            # 存入 redis ，需要先格式化为 str，避免日期变整数形式
-            if _df[col].dtype == 'datetime64[ns]':
-                _df[col] = _df[col].astype('str')
-        if len(_df) == 0:
-            print(f'{_table_name}: 写入 redis 的数据不能为空')
-            return pd.DataFrame()
-        jsondata = _df.to_json(orient='records', force_ascii=False)
-        self.redis_engin.set(my_key, jsondata)
-        self.redis_engin.expire(my_key, self.minute * 60)  # 设置缓存过期时间: 分钟
-        if '日期' in _df.columns.tolist():
-            _df['日期'] = pd.to_datetime(_df['日期'], format='%Y-%m-%d', errors='ignore')  # 转换日期列
-        now = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
-        print(f'{now}: 刷新 redis -> {_db_name}:{_table_name}')
-        return _df
-
-if __name__ == '__main__':
-    # # ****************************************************
-    # # 这一部分在外部定义，只需要定义一次，开始
-    # redis_config = {
-    #     'host': '127.0.0.1',
-    #     'port': 6379,  # 默认Redis端口
-    #     'db': 0,  # 默认Redis数据库索引
-    #     # 'username': 'default',
-    #     'password': redis_password,
-    # }
-    # # redis 实例化
-    # r = redis.Redis(**redis_config)
-    # # mysql 实例化
-    # d = s_query.QueryDatas(username=username, password=password, host=host, port=port)
-    # # 将两个库的实例化对象传给 RedisData 类，并实例化数据处理引擎
-    # m = RedisData(redis_engin=r, download=d)
-    # # ****************************************************
-    #
-    # # 以下为动态获取数据库数据
-    # db_name = '聚合数据'
-    # table_name = '多店推广场景_按日聚合'
-    # set_year = False
-    # df = m.get_from_redis(
-    #     _db_name=db_name,
-    #     _table_name=table_name,
-    #     _set_year=set_year,
-    #     start_date='2025-01-01',
-    #     end_date='2025-01-31'
-    # )
-    # print(df)
-    #
-
-    print(socket.gethostname())

mdbq/redis/getredis_deepseek.py

@@ -1,235 +0,0 @@
-# -*- coding: UTF-8 –*-
-import redis
-import socket
-from mdbq.mysql import s_query
-from mdbq.config import myconfig
-import pandas as pd
-import json
-import datetime
-import threading
-
-
-if socket.gethostname() == 'company' or socket.gethostname() == 'Mac2.local':
-    conf = myconfig.main()
-    conf_data = conf['Windows']['company']['mysql']['local']
-    username, password, host, port = conf_data['username'], conf_data['password'], conf_data['host'], conf_data['port']
-    redis_password = conf['Windows']['company']['redis']['local']['password']
-# elif socket.gethostname() == 'MacBook-Pro.local':
-#     conf = myconfig.main()
-#     conf_data = conf['Windows']['xigua_lx']['mysql']['local']
-#     username, password, host, port = conf_data['username'], conf_data['password'], conf_data['host'], conf_data['port']
-#     redis_password = conf['Windows']['company']['redis']['local']['password']
-else:
-    conf = myconfig.main()
-    conf_data = conf['Windows']['xigua_lx']['mysql']['local']
-    username, password, host, port = conf_data['username'], conf_data['password'], conf_data['host'], conf_data['port']
-    redis_password = conf['Windows']['company']['redis']['local']['password']  # redis 使用本地数据，全部机子相同
-
-
-class RedisData:
-    def __init__(self, redis_engine, download, cache_ttl: int = 60):
-        """
-        初始化Redis数据处理对象
-
-        :param redis_engine: Redis连接引擎
-        :param download: 数据库下载处理器
-        :param cache_ttl: 缓存过期时间（分钟）
-        """
-        self.redis_engine = redis_engine
-        self.download = download
-        self.cache_ttl = cache_ttl * 60  # 转换为秒
-        self.lock = threading.Lock()  # 线程锁
-
-    def _handle_datetime_columns(self, df: pd.DataFrame) -> pd.DataFrame:
-        """统一处理日期列转换"""
-        if '日期' in df.columns:
-            df['日期'] = pd.to_datetime(df['日期'], errors='coerce', format='%Y-%m-%d')
-        return df
-
-    def get_from_mysql(self, db_name: str, table_name: str,
-                       start_date, end_date,
-                       set_year: bool) -> pd.DataFrame:
-        """
-        从MySQL获取数据
-
-        :param set_year: 是否按年份分表
-        """
-        try:
-            if set_year:
-                current_year = datetime.datetime.now().year
-                dfs = []
-                # 动态获取需要查询的年份范围
-                min_year = min(2024, pd.to_datetime(start_date).year)  # 根据实际需求调整
-                for year in range(min_year, current_year + 1):
-                    table = f"{table_name}_{year}"
-                    df = self.download.data_to_df(
-                        db_name=db_name,
-                        table_name=table,
-                        start_date=start_date,
-                        end_date=end_date,
-                        projection={}
-                    )
-                    if not df.empty:
-                        dfs.append(df)
-                _df = pd.concat(dfs, ignore_index=True) if dfs else pd.DataFrame()
-            else:
-                _df = self.download.data_to_df(
-                    db_name=db_name,
-                    table_name=table_name,
-                    start_date=start_date,
-                    end_date=end_date,
-                    projection={}
-                )
-
-            if _df.empty:
-                print(f"空数据 - {db_name}.{table_name}")
-                return pd.DataFrame()
-
-            return self._handle_datetime_columns(_df)
-
-        except Exception as e:
-            print(f"MySQL查询失败: {str(e)}")
-            return pd.DataFrame()
-
-    def get_from_redis(self, db_name: str, table_name: str,
-                       start_date, end_date,
-                       set_year: bool) -> pd.DataFrame:
-        """
-        从Redis获取数据（带自动缓存更新）
-        """
-        start_dt = pd.to_datetime(start_date)
-        end_dt = pd.to_datetime(end_date)
-
-        # 生成统一缓存键
-        cache_key = f"{db_name}:{table_name}{'_year' if set_year else ''}"
-
-        try:
-            # 检查缓存状态
-            with self.lock:  # 保证线程安全
-                ttl = self.redis_engine.ttl(cache_key)
-
-                if ttl < 300:  # 剩余时间小于5分钟时触发更新
-                    print(f"异步缓存更新: {cache_key}")
-                    threading.Thread(
-                        target=self._update_cache,
-                        args=(cache_key, db_name, table_name,
-                              start_date, end_date, set_year),
-                        daemon=True
-                    ).start()
-
-            # 获取缓存数据
-            cached_data = self.redis_engine.get(cache_key)
-            if not cached_data:
-                return self._fallback_to_mysql(db_name, table_name,
-                                               start_date, end_date, set_year)
-            json_str = cached_data.decode('utf-8')
-            _df = pd.read_json(json_str, orient='records')
-            _df = self._handle_datetime_columns(_df)
-
-            # 数据范围校验
-            if '日期' in _df.columns:
-                cache_min = _df['日期'].min()
-                cache_max = _df['日期'].max()
-
-                # 请求范围超出缓存范围时需要更新
-                if start_dt < cache_min or end_dt > cache_max:
-                    print(f"请求范围超出缓存 {start_dt.strftime('%Y-%m-%d ')} - {end_dt.strftime('%Y-%m-%d ')}")
-                    self._update_cache(cache_key, db_name, table_name,
-                                       start_date, end_date, set_year, _df)
-                    return self._fallback_to_mysql(db_name, table_name,
-                                                   start_date, end_date, set_year)
-
-                return _df[(start_dt <= _df['日期']) & (_df['日期'] <= end_dt)]
-            return _df
-
-        except Exception as e:
-            print(f"Redis操作失败: {str(e)}")
-            return self._fallback_to_mysql(db_name, table_name,
-                                           start_date, end_date, set_year)
-
-    def _update_cache(self, cache_key: str, db_name: str, table_name: str,
-                      start_date: str, end_date: str, set_year: bool,
-                      existing_df: pd.DataFrame = None) -> None:
-        """缓存更新核心逻辑"""
-        try:
-            # 获取最新数据
-            new_data = self.get_from_mysql(
-                db_name=db_name,
-                table_name=table_name,
-                start_date=start_date,
-                end_date=end_date,
-                set_year=set_year
-            )
-
-            # 合并历史数据
-            if existing_df is not None and not new_data.empty:
-                combined = pd.concat([existing_df, new_data], ignore_index=True)
-                combined = combined.drop_duplicates(subset='日期', keep='last')
-            else:
-                combined = new_data
-
-            if not combined.empty:
-                # 转换日期类型为字符串
-                temp_df = combined.copy()
-                datetime_cols = temp_df.select_dtypes(include=['datetime64[ns]']).columns
-                temp_df[datetime_cols] = temp_df[datetime_cols].astype(str)
-
-                # 存储到Redis
-                with self.lock:
-                    self.redis_engine.set(
-                        cache_key,
-                        temp_df.to_json(orient='records', force_ascii=False),
-                        ex=self.cache_ttl
-                    )
-                print(f"缓存更新成功: {cache_key} | 记录数: {len(combined)}")
-
-        except Exception as e:
-            print(f"缓存更新失败: {str(e)}")
-
-    def _fallback_to_mysql(self, db_name: str, table_name: str,
-                           start_date: str, end_date: str,
-                           set_year: bool) -> pd.DataFrame:
-        """降级到直接MySQL查询"""
-        print(f"降级到MySQL查询: {db_name}.{table_name}")
-        return self.get_from_mysql(
-            db_name=db_name,
-            table_name=table_name,
-            start_date=start_date,
-            end_date=end_date,
-            set_year=set_year
-        )
-
-
-if __name__ == '__main__':
-    # # ****************************************************
-    # # 这一部分在外部定义，只需要定义一次，开始
-    # redis_config = {
-    #     'host': '127.0.0.1',
-    #     'port': 6379,  # 默认Redis端口
-    #     'db': 0,  # 默认Redis数据库索引
-    #     # 'username': 'default',
-    #     'password': redis_password,
-    # }
-    # # redis 实例化
-    # r = redis.Redis(**redis_config)
-    # # mysql 实例化
-    # d = s_query.QueryDatas(username=username, password=password, host=host, port=port)
-    # # 将两个库的实例化对象传给 RedisData 类，并实例化数据处理引擎
-    # m = RedisData(redis_engin=r, download=d)
-    # # ****************************************************
-    #
-    # # 以下为动态获取数据库数据
-    # db_name = '聚合数据'
-    # table_name = '多店推广场景_按日聚合'
-    # set_year = False
-    # df = m.get_from_redis(
-    #     _db_name=db_name,
-    #     _table_name=table_name,
-    #     _set_year=set_year,
-    #     start_date='2025-01-01',
-    #     end_date='2025-01-31'
-    # )
-    # print(df)
-    #
-
-    print(socket.gethostname())