mdbq 3.7.4__py3-none-any.whl → 3.7.5__py3-none-any.whl

mdbq/mysql/s_query.py

@@ -1,211 +0,0 @@
-# -*- coding:utf-8 -*-
-import datetime
-import platform
-import re
-import time
-from functools import wraps
-import warnings
-import pymysql
-import numpy as np
-import pandas as pd
-from sqlalchemy import create_engine
-import os
-import calendar
-from mdbq.dataframe import converter
-from decimal import Decimal
-import logging
-
-warnings.filterwarnings('ignore')
-"""
-程序专门用来下载数据库数据, 并返回 df, 不做清洗数据操作;
-"""
-logger = logging.getLogger(__name__)
-
-
-class QueryDatas:
-    def __init__(self, username: str, password: str, host: str, port: int, charset: str = 'utf8mb4'):
-        self.username = username
-        self.password = password
-        self.host = host
-        self.port = port
-        self.config = {
-            'host': self.host,
-            'port': int(self.port),
-            'user': self.username,
-            'password': self.password,
-            'charset': charset,  # utf8mb4 支持存储四字节的UTF-8字符集
-            'cursorclass': pymysql.cursors.DictCursor,
-        }
-
-    def check_condition(self, db_name, table_name, condition):
-        """ 按指定条件查询数据库，并返回 """
-        if self.check_infos(db_name, table_name) == False:
-            return
-
-        self.config.update({'database': db_name})
-        connection = pymysql.connect(**self.config)  # 重新连接数据库
-        with connection.cursor() as cursor:
-            sql = f"SELECT 更新时间 FROM {table_name} WHERE {condition}"
-            # logger.info(sql)
-            cursor.execute(sql)
-            columns = cursor.fetchall()
-            return columns
-
-    def data_to_df(self, db_name, table_name, start_date, end_date, projection: dict = None):
-        """
-        从数据库表获取数据到DataFrame，支持列筛选和日期范围过滤
-        Args:
-            db_name: 数据库名
-            table_name: 表名
-            start_date: 起始日期（包含）
-            end_date: 结束日期（包含）
-            projection: 列筛选字典，e.g. {'日期': 1, '场景名字': 1}
-        """
-        # 初始化默认参数
-        projection = projection or {}
-        df = pd.DataFrame()
-        # 日期处理
-        start_date = pd.to_datetime(start_date or '1970-01-01').strftime('%Y-%m-%d')
-        end_date = pd.to_datetime(end_date or datetime.datetime.today()).strftime('%Y-%m-%d')
-
-        # 前置检查
-        if not self.check_infos(db_name, table_name):
-            return df
-
-        # 配置数据库连接
-        self.config['database'] = db_name
-        connection = None
-
-        try:
-            connection = pymysql.connect(**self.config)
-            with connection.cursor() as cursor:
-                # 获取表结构（排除id列）
-                cursor.execute(
-                    """SELECT COLUMN_NAME 
-                    FROM information_schema.columns 
-                    WHERE table_schema = %s AND table_name = %s""",
-                    (db_name, table_name)
-                )
-                cols_exist = {col['COLUMN_NAME'] for col in cursor.fetchall()} - {'id'}
-
-                # 处理列选择
-                selected_columns = []
-                if projection:
-                    selected_columns = [k for k, v in projection.items() if v and k in cols_exist]
-                    if not selected_columns:
-                        logger.info("Warning: Projection 参数不匹配任何数据库字段")
-                        return df
-                else:
-                    selected_columns = list(cols_exist)
-                # 构建基础SQL
-                quoted_columns = [f'`{col}`' for col in selected_columns]
-                base_sql = f"SELECT {', '.join(quoted_columns)} FROM `{db_name}`.`{table_name}`"
-
-                # 添加日期条件
-                if '日期' in cols_exist:
-                    base_sql += f" WHERE 日期 BETWEEN '{start_date}' AND '{end_date}'"
-
-                # 执行查询
-                cursor.execute(base_sql)
-                result = cursor.fetchall()
-
-                # 处理结果集
-                if result:
-                    df = pd.DataFrame(result, columns=[desc[0] for desc in cursor.description])
-                    # 类型转换优化
-                    decimal_cols = [col for col in df.columns if df[col].apply(lambda x: isinstance(x, Decimal)).any()]
-                    df[decimal_cols] = df[decimal_cols].astype(float)
-
-        except Exception as e:
-            logger.error(f"Database operation failed: {str(e)}")
-        finally:
-            if connection:
-                connection.close()
-
-        return df
-
-    def columns_to_list(self, db_name, table_name,  columns_name) -> list:
-        """
-        获取数据表的指定列, 返回列表
-        [{'视频bv号': 'BV1Dm4y1S7BU', '下载进度': 1}, {'视频bv号': 'BV1ov411c7US', '下载进度': 1}]
-        """
-        if self.check_infos(db_name, table_name) == False:  # 检查传入的数据库和数据表是否存在
-            return []
-
-        self.config.update({'database': db_name})
-        connection = pymysql.connect(**self.config)  # 重新连接数据库
-        with connection.cursor() as cursor:
-            # 3. 获取数据表的所有列信息
-            sql = 'SELECT COLUMN_NAME FROM information_schema.columns WHERE table_schema = %s AND table_name = %s'
-            cursor.execute(sql, (db_name, {table_name}))
-            columns = cursor.fetchall()
-            cols_exist = [col['COLUMN_NAME'] for col in columns]  # 数据表的所有列, 返回 list
-            columns_name = [item for item in columns_name if item in cols_exist]
-            if len(columns_name) == 0:
-                return []
-            columns_in = ', '.join(columns_name)
-            sql = (f"SELECT {columns_in} FROM {db_name}.{table_name} ")
-            cursor.execute(sql)
-            column_values = cursor.fetchall()  # 返回指定列，结果是[dict, dict, dict, ...]
-            # column_values = [item[column_name] for item in column_values]  # 提取字典的值, 组成列表
-        connection.close()
-        return column_values
-
-    def dtypes_to_list(self, db_name, table_name) -> list:
-        """
-        获取数据表的指定列, 返回列表
-        [{'视频bv号': 'BV1Dm4y1S7BU', '下载进度': 1}, {'视频bv号': 'BV1ov411c7US', '下载进度': 1}]
-        """
-        if self.check_infos(db_name, table_name) == False:  # 检查传入的数据库和数据表是否存在
-            return []
-
-        self.config.update({'database': db_name})
-        connection = pymysql.connect(**self.config)  # 重新连接数据库
-        with connection.cursor() as cursor:
-            # 3. 获取数据表的所有列信息
-            sql = 'SELECT COLUMN_NAME, COLUMN_TYPE FROM information_schema.columns WHERE table_schema = %s AND table_name = %s'
-            cursor.execute(sql, (db_name, {table_name}))
-            column_name_and_type = cursor.fetchall()
-        connection.close()
-        return column_name_and_type
-
-    def check_infos(self, db_name, table_name) -> bool:
-        """ 检查数据库、数据表是否存在 """
-        connection = pymysql.connect(**self.config)  # 连接数据库
-        try:
-            with connection.cursor() as cursor:
-                # 1. 检查数据库是否存在
-                cursor.execute(f"SHOW DATABASES LIKE '{db_name}'")  # 检查数据库是否存在
-                database_exists = cursor.fetchone()
-                if not database_exists:
-                    logger.info(f"Database <{db_name}>: 数据库不存在")
-                    return False
-        finally:
-            connection.close()  # 这里要断开连接
-
-        self.config.update({'database': db_name})  # 添加更新 config 字段
-        connection = pymysql.connect(**self.config)  # 重新连接数据库
-        try:
-            with connection.cursor() as cursor:
-                # 2. 查询表是否存在
-                sql = f"SHOW TABLES LIKE '{table_name}'"
-                cursor.execute(sql)
-                if not cursor.fetchone():
-                    logger.info(f'{db_name} -> <{table_name}>: 表不存在')
-                    return False
-                return True
-        except Exception as e:
-            logger.error(e)
-            return False
-        finally:
-            connection.close()  # 断开连接
-
-
-if __name__ == '__main__':
-    conf = ConfigTxt()
-    data = conf.config_datas['Windows']['xigua_lx']['mysql']['remoto']
-    username, password, host, port = data['username'], data['password'], data['host'], data['port']
-
-    q = QueryDatas(username, password, host, port)
-    res = q.columns_to_list(db_name='视频数据', table_name='bilibili视频', columns_name=['视频bv号', '下载进度'])
-    logger.info(res)

mdbq/mysql/year_month_day.py

@@ -1,38 +0,0 @@
-# -*- coding:utf-8 -*-
-import warnings
-import pandas as pd
-import calendar
-
-warnings.filterwarnings('ignore')
-
-
-def year_month_day(start_date, end_date):
-    """
-    使用date_range函数和DataFrame来获取从start_date至end_date之间的所有年月日
-    calendar.monthrange： 获取当月第一个工作日的星期值(0,6) 以及当月天数
-    返回值: [{'起始日期': '2025-05-01', '结束日期': '2025-05-31'}, {'起始日期': '2025-06-01', '结束日期': '2025-06-30'}]
-    """
-    # 替换年月日中的日, 以便即使传入当月日期也有返回值
-    try:
-        start_date = f'{pd.to_datetime(start_date).year}-{pd.to_datetime(start_date).month}-01'
-    except Exception as e:
-        print(e)
-        return []
-    # 使用pandas的date_range创建一个日期范围，频率为'MS'代表每月开始
-    date_range = pd.date_range(start=start_date, end=end_date, freq='MS')
-    # 转换格式
-    year_months = date_range.strftime('%Y-%m').drop_duplicates().sort_values()
-
-    results = []
-    for year_month in year_months:
-        year = re.findall(r'(\d{4})', year_month)[0]
-        month = re.findall(r'\d{4}-(\d{2})', year_month)[0]
-        s, d = calendar.monthrange(int(year), int(month))
-        results.append({'起始日期': f'{year_month}-01', '结束日期': f'{year_month}-{d}'})
-
-    return results  # start_date至end_date之间的所有年月日
-
-
-if __name__ == '__main__':
-    results = year_month_day(start_date='2025-05-01', end_date='2025-08-01')
-    print(results)

mdbq/other/__init__.py

@@ -1,4 +0,0 @@
-
-
-
-# 配置文件