mdbq 3.6.12__py3-none-any.whl → 3.6.14__py3-none-any.whl

mdbq/mysql/s_query.py

@@ -13,11 +13,13 @@ import os
 import calendar
 from mdbq.dataframe import converter
 from decimal import Decimal
+import logging
 
 warnings.filterwarnings('ignore')
 """
 程序专门用来下载数据库数据, 并返回 df, 不做清洗数据操作;
 """
+logger = logging.getLogger(__name__)
 
 
 class QueryDatas:
@@ -44,86 +46,83 @@ class QueryDatas:
         connection = pymysql.connect(**self.config)  # 重新连接数据库
         with connection.cursor() as cursor:
             sql = f"SELECT 更新时间 FROM {table_name} WHERE {condition}"
-            # print(sql)
+            # logger.info(sql)
             cursor.execute(sql)
             columns = cursor.fetchall()
             return columns
 
-    def data_to_df(self, db_name, table_name, start_date, end_date, projection: dict=[]):
+    def data_to_df(self, db_name, table_name, start_date, end_date, projection: dict = None):
         """
-        projection = {'日期': 1, '场景名字': 1,}
+        从数据库表获取数据到DataFrame，支持列筛选和日期范围过滤
+        Args:
+            db_name: 数据库名
+            table_name: 表名
+            start_date: 起始日期（包含）
+            end_date: 结束日期（包含）
+            projection: 列筛选字典，e.g. {'日期': 1, '场景名字': 1}
         """
-        if start_date:
-            start_date = pd.to_datetime(start_date).strftime('%Y-%m-%d')
-        else:
-            start_date = '1970-01-01'
-        if end_date:
-            end_date = pd.to_datetime(end_date).strftime('%Y-%m-%d')
-        else:
-            end_date = datetime.datetime.today().strftime('%Y-%m-%d')
-        df = pd.DataFrame()  # 初始化df
-
-        if self.check_infos(db_name, table_name) == False:
+        # 初始化默认参数
+        projection = projection or {}
+        df = pd.DataFrame()
+        # 日期处理
+        start_date = pd.to_datetime(start_date or '1970-01-01').strftime('%Y-%m-%d')
+        end_date = pd.to_datetime(end_date or datetime.datetime.today()).strftime('%Y-%m-%d')
+
+        # 前置检查
+        if not self.check_infos(db_name, table_name):
             return df
 
-        self.config.update({'database': db_name})
-        connection = pymysql.connect(**self.config)  # 重新连接数据库
+        # 配置数据库连接
+        self.config['database'] = db_name
+        connection = None
 
-        with connection.cursor() as cursor:
-            # 3. 获取数据表的所有列信息
-            sql = 'SELECT `COLUMN_NAME` FROM information_schema.columns WHERE table_schema = %s AND table_name = %s'
-            cursor.execute(sql, (db_name, {table_name}))
-            columns = cursor.fetchall()
-            cols_exist = [col['COLUMN_NAME'] for col in columns]  # 数据表的所有列, 返回 list
+        try:
+            connection = pymysql.connect(**self.config)
+            with connection.cursor() as cursor:
+                # 获取表结构（排除id列）
+                cursor.execute(
+                    """SELECT COLUMN_NAME 
+                    FROM information_schema.columns 
+                    WHERE table_schema = %s AND table_name = %s""",
+                    (db_name, table_name)
+                )
+                cols_exist = {col['COLUMN_NAME'] for col in cursor.fetchall()} - {'id'}
+
+                # 处理列选择
+                selected_columns = []
+                if projection:
+                    selected_columns = [k for k, v in projection.items() if v and k in cols_exist]
+                    if not selected_columns:
+                        logger.info("Warning: Projection 参数不匹配任何数据库字段")
+                        return df
+                else:
+                    selected_columns = list(cols_exist)
+                # 构建基础SQL
+                quoted_columns = [f'`{col}`' for col in selected_columns]
+                base_sql = f"SELECT {', '.join(quoted_columns)} FROM `{db_name}`.`{table_name}`"
+
+                # 添加日期条件
+                if '日期' in cols_exist:
+                    base_sql += f" WHERE 日期 BETWEEN '{start_date}' AND '{end_date}'"
+
+                # 执行查询
+                cursor.execute(base_sql)
+                result = cursor.fetchall()
+
+                # 处理结果集
+                if result:
+                    df = pd.DataFrame(result, columns=[desc[0] for desc in cursor.description])
+                    # 类型转换优化
+                    decimal_cols = [col for col in df.columns if df[col].apply(lambda x: isinstance(x, Decimal)).any()]
+                    df[decimal_cols] = df[decimal_cols].astype(float)
 
-            # 4. 构建 SQL 查询语句
-            if projection:  # 获取指定列
-                columns_in = []
-                for key, value in projection.items():
-                    if value == 1 and key in cols_exist:
-                        columns_in.append(key)  # 提取值为 1 的键并清理不在数据表的键
-                columns_in = [f"`{item}`" for item in columns_in]
-                if not columns_in:
-                    print(f'传递的参数 projection，在数据库中没有找到匹配的列，请检查 projection： {projection}')
-                    return df
-                columns_in = ', '.join(columns_in)
-                if '日期' in cols_exist:  # 不论是否指定, 只要数据表有日期，则执行
-                    sql = (f"SELECT {columns_in} FROM `{db_name}`.`{table_name}` "
-                           f"WHERE {'日期'} BETWEEN '{start_date}' AND '{end_date}'")
-                else:  # 数据表没有日期列时，返回指定列的所有数据
-                    sql = f"SELECT {columns_in} FROM `{db_name}`.`{table_name}`"
-            else:  # 没有指定获取列时
-                if '日期' in cols_exist:  # 但数据表有日期，仍然执行
-                    cols_exist = [f"`{item}`" for item in cols_exist]
-                    columns_in = ', '.join(cols_exist)
-                    sql = (f"SELECT {columns_in} FROM `{db_name}`.`{table_name}` "
-                           f"WHERE {'日期'} BETWEEN '{start_date}' AND '{end_date}'")
-                else:  # 没有指定获取列，且数据表也没有日期列，则返回全部列的全部数据
-                    all_col = ', '.join([f"`{item}`" for item in cols_exist if item != 'id'])
-                    sql = f"SELECT %s FROM `%s`.`%s`" % (all_col, db_name, table_name)
-            # print(sql)
-            cursor.execute(sql)
-            rows = cursor.fetchall()  # 获取查询结果
-            columns = [desc[0] for desc in cursor.description]
-            df = pd.DataFrame(rows, columns=columns)  # 转为 df
-            # 使用applymap将每个Decimal转换为float
-            df_float = df.applymap(lambda x: float(x) if isinstance(x, Decimal) else x)
-
-            if 'id' in df.columns.tolist():
-                df.pop('id')  # 默认不返回 id 列
-            if len(df) == 0:
-                print(f's_query.py -> data_to_df -> database: {db_name}, table: {table_name} 查询的数据为空1')
-            connection.close()
-            return df
+        except Exception as e:
+            logger.info(f"Database operation failed: {str(e)}")
+        finally:
+            if connection:
+                connection.close()
 
-        # if len(df) == 0:
-        #     print(f'database: {db_name}, table: {table_name} 查询的数据为空2')
-        #     return pd.DataFrame()
-        # cv = converter.DataFrameConverter()
-        # df = cv.convert_df_cols(df)
-        # if 'id' in df.columns.tolist():
-        #     df.pop('id')  # 默认不返回 id 列
-        # return df
+        return df
 
     def columns_to_list(self, db_name, table_name,  columns_name) -> list:
         """
@@ -179,7 +178,7 @@ class QueryDatas:
                 cursor.execute(f"SHOW DATABASES LIKE '{db_name}'")  # 检查数据库是否存在
                 database_exists = cursor.fetchone()
                 if not database_exists:
-                    print(f"Database <{db_name}>: 数据库不存在")
+                    logger.info(f"Database <{db_name}>: 数据库不存在")
                     return False
         finally:
             connection.close()  # 这里要断开连接
@@ -192,11 +191,11 @@ class QueryDatas:
                 sql = f"SHOW TABLES LIKE '{table_name}'"
                 cursor.execute(sql)
                 if not cursor.fetchone():
-                    print(f'{db_name} -> <{table_name}>: 表不存在')
+                    logger.info(f'{db_name} -> <{table_name}>: 表不存在')
                     return False
                 return True
         except Exception as e:
-            print(e)
+            logger.info(e)
             return False
         finally:
             connection.close()  # 断开连接
@@ -209,4 +208,4 @@ if __name__ == '__main__':
 
     q = QueryDatas(username, password, host, port)
     res = q.columns_to_list(db_name='视频数据', table_name='bilibili视频', columns_name=['视频bv号', '下载进度'])
-    print(res)
+    logger.info(res)

mdbq/redis/getredis.py

@@ -45,7 +45,6 @@ logging.basicConfig(level=logging.INFO, format='%(asctime)s | %(levelname)s | %(
 logger = logging.getLogger(__name__)
 
 # 创建一个文件处理器，用于将日志写入文件
-# file_handler = logging.FileHandler(os.path.join(D_PATH, 'logfile', 'redis.log'))
 if not os.path.isdir(os.path.join(D_PATH, 'logfile')):
     os.makedirs(os.path.join(D_PATH, 'logfile'))
 log_file = os.path.join(D_PATH, 'logfile', 'redis.log')
@@ -358,6 +357,7 @@ class RedisDataHash(object):
                 return self.get_from_mysql(db_name, table_name, set_year, start_date, end_date, projection)
 
             filtered_df = self._filter_by_date_range(cache_data, start_dt, end_dt)
+
             if not filtered_df.empty:
                 if '日期' in filtered_df.columns.tolist():
                     exsit_min_date = filtered_df['日期'].min()

{mdbq-3.6.12.dist-info → mdbq-3.6.14.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: mdbq
-Version: 3.6.12
+Version: 3.6.14
 Home-page: https://pypi.org/project/mdbq
 Author: xigua, 
 Author-email: 2587125111@qq.com

{mdbq-3.6.12.dist-info → mdbq-3.6.14.dist-info}/RECORD

@@ -3,8 +3,8 @@ mdbq/__version__.py,sha256=y9Mp_8x0BCZSHsdLT_q5tX9wZwd5QgqrSIENLrb6vXA,62
 mdbq/aggregation/__init__.py,sha256=EeDqX2Aml6SPx8363J-v1lz0EcZtgwIBYyCJV6CcEDU,40
 mdbq/aggregation/aggregation.py,sha256=-yzApnlqSN2L0E1YMu5ml-W827qpKQvWPCOI7jj2kzY,80264
 mdbq/aggregation/datashow.py,sha256=1AYSIDkdUx-4as1Ax2rPj0cExM9d-qFMrFYLAaPHNuk,54962
-mdbq/aggregation/optimize_data.py,sha256=RXIv7cACCgYyehAxMjUYi_S7rVyjIwXKWMaM3nduGtA,3068
-mdbq/aggregation/query_data.py,sha256=KrQqe9qzrKAs3s6QNoQZuY8_U36I852dFPiIcv1QIi0,193075
+mdbq/aggregation/optimize_data.py,sha256=87Dc2RQTF-wqp8Gct_e0S_xwiNlzsEmM82mwuHPZKlc,3152
+mdbq/aggregation/query_data.py,sha256=7g1Z2qyU8dgnOHdpN5ltr7BX8ZFRrbVmFpk6s9045Pw,193363
 mdbq/bdup/__init__.py,sha256=AkhsGk81SkG1c8FqDH5tRq-8MZmFobVbN60DTyukYTY,28
 mdbq/bdup/bdup.py,sha256=LAV0TgnQpc-LB-YuJthxb0U42_VkPidzQzAagan46lU,4234
 mdbq/config/__init__.py,sha256=jso1oHcy6cJEfa7udS_9uO5X6kZLoPBF8l3wCYmr5dM,18
@@ -18,10 +18,10 @@ mdbq/log/mylogger.py,sha256=oaT7Bp-Hb9jZt52seP3ISUuxVcI19s4UiqTeouScBO0,3258
 mdbq/mongo/__init__.py,sha256=SILt7xMtQIQl_m-ik9WLtJSXIVf424iYgCfE_tnQFbw,13
 mdbq/mongo/mongo.py,sha256=M9DUeUCMPDngkwn9-ui0uTiFrvfNU1kLs22s5SmoNm0,31899
 mdbq/mysql/__init__.py,sha256=A_DPJyAoEvTSFojiI2e94zP0FKtCkkwKP1kYUCSyQzo,11
-mdbq/mysql/mysql.py,sha256=_jFo2_OC1BNm5wEmoYiBG_TcuNNA2xUWKNhMBfgDiAM,99699
+mdbq/mysql/mysql.py,sha256=1HoOvSZKPHc3Bz1YI_A_5x3JmGvjUN4suCM9oHCF-t4,100465
 mdbq/mysql/mysql_bak.py,sha256=_jFo2_OC1BNm5wEmoYiBG_TcuNNA2xUWKNhMBfgDiAM,99699
 mdbq/mysql/recheck_mysql.py,sha256=ppBTfBLgkRWirMVZ31e_ZPULiGPJU7K3PP9G6QBZ3QI,8605
-mdbq/mysql/s_query.py,sha256=rUyemVsjFM7OYG_o-DYGy18aRCbMsDlpzNGNS8Un7dg,9722
+mdbq/mysql/s_query.py,sha256=CL2Ayo2sL11RbLnh9nE-GXA-NpA815-rrlFo24TipKY,8792
 mdbq/mysql/year_month_day.py,sha256=VgewoE2pJxK7ErjfviL_SMTN77ki8GVbTUcao3vFUCE,1523
 mdbq/other/__init__.py,sha256=jso1oHcy6cJEfa7udS_9uO5X6kZLoPBF8l3wCYmr5dM,18
 mdbq/other/download_sku_picture.py,sha256=GdphR7Q3psXXVuZoyJ4u_6OWn_rWlcbT0iJ-1zPT6O0,45368
@@ -34,11 +34,11 @@ mdbq/pbix/pbix_refresh.py,sha256=JUjKW3bNEyoMVfVfo77UhguvS5AWkixvVhDbw4_MHco,239
 mdbq/pbix/refresh_all.py,sha256=OBT9EewSZ0aRS9vL_FflVn74d4l2G00wzHiikCC4TC0,5926
 mdbq/pbix/refresh_all_old.py,sha256=_pq3WSQ728GPtEG5pfsZI2uTJhU8D6ra-htIk1JXYzw,7192
 mdbq/redis/__init__.py,sha256=YtgBlVSMDphtpwYX248wGge1x-Ex_mMufz4-8W0XRmA,12
-mdbq/redis/getredis.py,sha256=DKahNJeO3W3RZ-u6LsVbbGLi-CK-dZ8y3UV9dxk8YM8,26720
+mdbq/redis/getredis.py,sha256=4rYk9lMRvvlpY7cV3VNQcSnbDWlZIsZZ-tSq--YqfSQ,26638
 mdbq/redis/getredis_优化hash.py,sha256=q7omKJCPw_6Zr_r6WwTv4RGSXzZzpLPkIaqJ22svJhE,29104
 mdbq/spider/__init__.py,sha256=RBMFXGy_jd1HXZhngB2T2XTvJqki8P_Fr-pBcwijnew,18
 mdbq/spider/aikucun.py,sha256=v7VO5gtEXR6_4Q6ujbTyu1FHu7TXHcwSQ6hIO249YH0,22208
-mdbq-3.6.12.dist-info/METADATA,sha256=La5gbnCv5fzKJ69TGAZ6HZPgeF_bc8-jJnM0D1r06Fs,244
-mdbq-3.6.12.dist-info/WHEEL,sha256=cpQTJ5IWu9CdaPViMhC9YzF8gZuS5-vlfoFihTBC86A,91
-mdbq-3.6.12.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
-mdbq-3.6.12.dist-info/RECORD,,
+mdbq-3.6.14.dist-info/METADATA,sha256=6nLgVL5HWxyW9-EBob6LnpGnfncM3l8rOL4nKQtqtbg,244
+mdbq-3.6.14.dist-info/WHEEL,sha256=cpQTJ5IWu9CdaPViMhC9YzF8gZuS5-vlfoFihTBC86A,91
+mdbq-3.6.14.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
+mdbq-3.6.14.dist-info/RECORD,,