mdbq 3.5.5__py3-none-any.whl → 3.5.7__py3-none-any.whl

mdbq/mysql/s_query.py

@@ -100,7 +100,7 @@ class QueryDatas:
             if 'id' in df.columns.tolist():
                 df.pop('id')  # 默认不返回 id 列
             if len(df) == 0:
-                print(f'database: {db_name}, table: {table_name} 查询的数据为空1')
+                print(f's_query.py -> data_to_df -> database: {db_name}, table: {table_name} 查询的数据为空1')
             connection.close()
             return df
 

mdbq/redis/__init__.py

@@ -0,0 +1,4 @@
+
+
+
+# 缓存

mdbq/redis/getredis.py

@@ -0,0 +1,176 @@
+# -*- coding: UTF-8 –*-
+import redis
+import socket
+from mdbq.mysql import s_query
+from mdbq.config import myconfig
+import pandas as pd
+import json
+import datetime
+import threading
+
+
+if socket.gethostname() == 'company' or socket.gethostname() == 'Mac2.local':
+    conf = myconfig.main()
+    conf_data = conf['Windows']['company']['mysql']['local']
+    username, password, host, port = conf_data['username'], conf_data['password'], conf_data['host'], conf_data['port']
+    redis_password = conf['Windows']['company']['redis']['local']['password']
+else:
+    conf = myconfig.main()
+    conf_data = conf['Windows']['xigua_lx']['mysql']['local']
+    username, password, host, port = conf_data['username'], conf_data['password'], conf_data['host'], conf_data['port']
+    redis_password = conf['Windows']['company']['redis']['local']['password']  # redis 使用本地数据，全部机子相同
+
+
+
+class RedisData(object):
+    def __init__(self, redis_engin, download):
+        self.redis_engin = redis_engin  # redis 数据处理引擎
+        self.download = download  # mysql 数据处理引擎
+        self.minute = 60  # 缓存过期时间: 分钟
+
+    def get_from_mysql(self, _db_name, _table_name, _set_year, start_date, end_date):
+        """
+        _set_year: _table_name 中是否含有年份
+        """
+        if _set_year:
+            __res = []
+            for year in range(2024, datetime.datetime.today().year + 1):
+                _df = self.download.data_to_df(
+                    db_name=_db_name,
+                    table_name=f'{_table_name}_{year}',
+                    start_date=start_date,
+                    end_date=end_date,
+                    projection={},
+                )
+                __res.append(_df)
+            _df = pd.concat(__res, ignore_index=True)
+        else:
+            _df = self.download.data_to_df(
+                db_name=_db_name,
+                table_name=_table_name,
+                start_date=start_date,
+                end_date=end_date,
+                projection={},
+            )
+        if len(_df) == 0:
+            print(f'{_db_name} - {_table_name}: mysql读取的数据不能为空')
+            return pd.DataFrame()
+        if '日期' in _df.columns.tolist():
+            _df['日期'] = pd.to_datetime(_df['日期'], format='%Y-%m-%d', errors='ignore')  # 转换日期列
+        return _df
+
+    def get_from_redis(self, _db_name, _table_name, _set_year, start_date, end_date):
+        """
+        _set_year: _table_name 中是否含有年份
+        _col_list: 如果不传就取 table 的所有列
+        对于日期: 最终传出的是日期格式，但如果存入 redis ，需要先格式化为 str，避免日期变整数形式
+        """
+        start_date = pd.to_datetime(start_date)
+        end_date = pd.to_datetime(end_date)
+        if _set_year:
+            my_key = f'{_db_name}:{_table_name}_haveyear'
+        else:
+            my_key = f'{_db_name}:{_table_name}'
+        # ttl 对于不存在的键，它返回 -2；而对于没有设置过期时间的键，它返回 -1
+        try:
+            ttl_result = self.redis_engin.ttl(my_key)
+        except Exception as e:
+            # redis 连接失败, 则绕过 redis 直接从 mysql 获取数据
+            print('redis 连接失败, 绕过 redis 直接从 mysql 获取数据')
+            _df = self.get_from_mysql(_db_name=_db_name, _table_name=_table_name, start_date=start_date, end_date=end_date, _set_year=_set_year)
+            return _df
+        _df = pd.DataFrame()
+
+        if ttl_result < 60:
+            # 1. redis 没有该数据时
+            print(f'数据不存在或过期')
+            thread = threading.Thread(target=self.set_redis, args=(my_key, _db_name, _table_name, _set_year, start_date, end_date, _df))
+            thread.start()
+            # _df = self.set_redis(my_key=my_key, _db_name=_db_name, _table_name=_table_name, _set_year=_set_year, start_date=start_date, end_date=end_date)
+            _df = self.get_from_mysql(_db_name=_db_name, _table_name=_table_name, start_date=start_date, end_date=end_date, _set_year=_set_year)
+            return _df
+        # 2. redis 有数据时
+        json_string = self.redis_engin.get(my_key)
+        data_dict = json.loads(json_string.decode('utf-8'))
+        _df = pd.DataFrame(data_dict)
+
+        if '日期' in _df.columns.tolist():
+            _df['日期'] = pd.to_datetime(_df['日期'], format='%Y-%m-%d', errors='ignore')  # 转换日期列
+            min_date = _df['日期'].min()
+            max_date = _df['日期'].max()
+            # Bug: 如果外部请求日期小于 table 最小日期，每次都要从 mysql 获取数据，即使 redis 缓存了数据
+            if start_date < min_date:  # 外部请求日期小于 redis 数据最小日期
+                # 3. redis 有数据但数据不完整时
+                print(f'数据日期不对劲需要更新')
+                thread = threading.Thread(target=self.set_redis, args=(my_key, _db_name, _table_name, _set_year, start_date, end_date, _df))
+                thread.start()
+                # _df = self.set_redis(my_key=my_key, _db_name=_db_name, _table_name=_table_name, _set_year=_set_year, start_date=start_date, end_date=end_date)
+                _df = self.get_from_mysql(_db_name=_db_name, _table_name=_table_name, start_date=start_date, end_date=end_date, _set_year=_set_year)
+                return _df
+            _df = _df[(_df['日期'] >= start_date) & (_df['日期'] <= end_date)]
+
+        return _df
+
+    def set_redis(self, my_key, _db_name, _table_name, _set_year, start_date, end_date, before_df):
+        """
+        从 mysql 读取数据并存储 redis
+        由于这个函数是异步执行的，从页面段首次加载数据时，可能返回空，等待异步执行结束后会正常返回数据
+        """
+        _df = self.get_from_mysql(
+            _db_name=_db_name,
+            _table_name=_table_name,
+            start_date=start_date,
+            end_date=end_date,
+            _set_year=_set_year
+        )
+        if '日期' in _df.columns.tolist():
+            _min_date = _df['日期'].min()
+            _max_date = _df['日期'].max()
+        if '日期' in before_df.columns.tolist():
+            # 移除 redis 指定范围的数据，再合并新数据
+            before_df1 = before_df[(before_df['日期'] < _min_date)]
+            before_df2 = before_df[(before_df['日期'] > _max_date)]
+            _df = pd.concat([_df, before_df1, before_df2], ignore_index=True, axis=0)
+        if '日期' in _df.columns.tolist():
+            # 存入 redis ，需要先格式化为 str，避免日期变整数形式
+            _df['日期'] = _df['日期'].astype('str')
+        jsondata = _df.to_json(orient='records', force_ascii=False)
+        self.redis_engin.set(my_key, jsondata)
+        self.redis_engin.expire(my_key, self.minute * 60)  # 设置缓存过期时间: 分钟
+        if '日期' in _df.columns.tolist():
+            _df['日期'] = pd.to_datetime(_df['日期'], format='%Y-%m-%d', errors='ignore')  # 转换日期列
+        now = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
+        print(f'{now}: 刷新 redis -> {_db_name}:{_table_name}')
+        return _df
+
+
+if __name__ == '__main__':
+    # ****************************************************
+    # 这一部分在外部定义，只需要定义一次，开始
+    redis_config = {
+        'host': '127.0.0.1',
+        'port': 6379,  # 默认Redis端口
+        'db': 0,  # 默认Redis数据库索引
+        # 'username': 'default',
+        'password': redis_password,
+    }
+    # redis 实例化
+    r = redis.Redis(**redis_config)
+    # mysql 实例化
+    d = s_query.QueryDatas(username=username, password=password, host=host, port=port)
+    # 将两个库的实例化对象传给 RedisData 类，并实例化数据处理引擎
+    m = RedisData(redis_engin=r, download=d)
+    # ****************************************************
+
+    # 以下为动态获取数据库数据
+    db_name = '聚合数据'
+    table_name = '多店推广场景_按日聚合'
+    set_year = False
+    df = m.get_from_redis(
+        _db_name=db_name,
+        _table_name=table_name,
+        _set_year=set_year,
+        start_date='2025-01-01',
+        end_date='2025-01-31'
+    )
+    print(df)

mdbq/spider/aikucun.py

@@ -483,7 +483,7 @@ class AikuCunNew:
 
 if __name__ == '__main__':
     get_cookie_aikucun()  # 登录并获取 cookies
-    akucun(date_num=5, headless=True)  # 下载数据
+    akucun(date_num=10, headless=True)  # 下载数据
 
     # a = AikuCunNew(shop_name='aikucun')
     # a.akc()

{mdbq-3.5.5.dist-info → mdbq-3.5.7.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: mdbq
-Version: 3.5.5
+Version: 3.5.7
 Home-page: https://pypi.org/project/mdbq
 Author: xigua, 
 Author-email: 2587125111@qq.com

{mdbq-3.5.5.dist-info → mdbq-3.5.7.dist-info}/RECORD

@@ -20,7 +20,7 @@ mdbq/mongo/mongo.py,sha256=M9DUeUCMPDngkwn9-ui0uTiFrvfNU1kLs22s5SmoNm0,31899
 mdbq/mysql/__init__.py,sha256=A_DPJyAoEvTSFojiI2e94zP0FKtCkkwKP1kYUCSyQzo,11
 mdbq/mysql/mysql.py,sha256=r5YkS1WnV9dGtEHFcwaekjtUBgFcvkdmwif-m52CyHI,99560
 mdbq/mysql/recheck_mysql.py,sha256=ppBTfBLgkRWirMVZ31e_ZPULiGPJU7K3PP9G6QBZ3QI,8605
-mdbq/mysql/s_query.py,sha256=6L5Cp90zq13noZHjzSA5mqms_hD01c8GO1_NfbYDu6w,9252
+mdbq/mysql/s_query.py,sha256=Z0C3lQQcGtnizyVHa62HYIAqZ8R3KeSHP-N0O7Qb5eU,9280
 mdbq/mysql/year_month_day.py,sha256=VgewoE2pJxK7ErjfviL_SMTN77ki8GVbTUcao3vFUCE,1523
 mdbq/other/__init__.py,sha256=jso1oHcy6cJEfa7udS_9uO5X6kZLoPBF8l3wCYmr5dM,18
 mdbq/other/download_sku_picture.py,sha256=GdphR7Q3psXXVuZoyJ4u_6OWn_rWlcbT0iJ-1zPT6O0,45368
@@ -32,9 +32,11 @@ mdbq/pbix/__init__.py,sha256=Trtfaynu9RjoTyLLYBN2xdRxTvm_zhCniUkVTAYwcjo,24
 mdbq/pbix/pbix_refresh.py,sha256=JUjKW3bNEyoMVfVfo77UhguvS5AWkixvVhDbw4_MHco,2396
 mdbq/pbix/refresh_all.py,sha256=OBT9EewSZ0aRS9vL_FflVn74d4l2G00wzHiikCC4TC0,5926
 mdbq/pbix/refresh_all_old.py,sha256=_pq3WSQ728GPtEG5pfsZI2uTJhU8D6ra-htIk1JXYzw,7192
+mdbq/redis/__init__.py,sha256=YtgBlVSMDphtpwYX248wGge1x-Ex_mMufz4-8W0XRmA,12
+mdbq/redis/getredis.py,sha256=PlWAGMC-WqdZtyvtjjj-i0i8AiBsNP6zgAUb5Fdkark,8481
 mdbq/spider/__init__.py,sha256=RBMFXGy_jd1HXZhngB2T2XTvJqki8P_Fr-pBcwijnew,18
-mdbq/spider/aikucun.py,sha256=zOacjrJ3MvToyuugA68xB-oN6RKj8K3GxMKudnln9EA,22207
-mdbq-3.5.5.dist-info/METADATA,sha256=vh6KUSs4_g68EQt2S8VwB93eEhuEfNQ_i4aobYARvuQ,243
-mdbq-3.5.5.dist-info/WHEEL,sha256=cpQTJ5IWu9CdaPViMhC9YzF8gZuS5-vlfoFihTBC86A,91
-mdbq-3.5.5.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
-mdbq-3.5.5.dist-info/RECORD,,
+mdbq/spider/aikucun.py,sha256=v7VO5gtEXR6_4Q6ujbTyu1FHu7TXHcwSQ6hIO249YH0,22208
+mdbq-3.5.7.dist-info/METADATA,sha256=2vbVtGiIZ0FKagUbtistkxCiI5m3Iwbh8c-kTpWEkSk,243
+mdbq-3.5.7.dist-info/WHEEL,sha256=eOLhNAGa2EW3wWl_TU484h7q1UNgy0JXjjoqKoxAAQc,92
+mdbq-3.5.7.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
+mdbq-3.5.7.dist-info/RECORD,,

{mdbq-3.5.5.dist-info → mdbq-3.5.7.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (70.1.0)
+Generator: bdist_wheel (0.44.0)
 Root-Is-Purelib: true
 Tag: py3-none-any