mdbq 3.8.8__py3-none-any.whl → 3.8.10__py3-none-any.whl

mdbq/__version__.py

@@ -1 +1 @@
-VERSION = '3.8.8'
+VERSION = '3.8.10'

mdbq/mysql/mysql.py

@@ -1,6 +1,5 @@
 # -*- coding:utf-8 -*-
 import datetime
-import platform
 import re
 import time
 from functools import wraps
@@ -10,8 +9,8 @@ import numpy as np
 import pandas as pd
 from sqlalchemy import create_engine
 import os
-import calendar
 import logging
+from mdbq.other import otk
 
 warnings.filterwarnings('ignore')
 """
@@ -21,44 +20,6 @@ warnings.filterwarnings('ignore')
 logger = logging.getLogger(__name__)
 
 
-def is_valid_date(date_string):
-    """
-    判断是否是日期格式, 且允许没有前导零, 且允许带时间
-    纯日期格式： 返回 1
-    日期+时间： 返回 2
-    """
-    date_pattern = r"^(\d{4})-(0?[1-9]|1[0-2])-(0?[1-9]|[12]\d|3[01])$"
-    match = re.match(date_pattern, str(date_string))  # 判断纯日期格式：2024-11-09
-    if match is None:
-        date_pattern = r".*\d+:\d+:\d+$"
-        match = re.match(date_pattern, date_string)  # 判断日期+时间：2024-11-09 00:36:45
-        if match is not None:
-            return 2
-    else:
-        return 1
-
-
-def is_integer(int_str):
-    """ 判断是否整数, 允许包含千分位分隔符, 允许科学计数法 """
-    # 如果是科学计数法
-    match = re.findall(r'^[-+]?(\d+)\.(\d+)[eE][-+]?(\d+)$', str(int_str))
-    if match:
-        if len(match[0]) == 3:
-            if int(match[0][0]) == 0:  # 0 开头
-                if int(match[0][2]) > 10:  # 转换后整数长度超过 10 位
-                    return False
-            else:  # 不是 0 开头
-                if len(match[0][0]) + int(match[0][2]) > 10:  # 转换后整数长度超过 10 位
-                    return False
-            if int(match[0][2]) >= len(match[0][1]):
-                return True
-            else:
-                return False
-    # 如果是普通数字, 且允许千分符
-    __pattern = r'^[-+]?\d{1,3}(,\d{3}){0,3}$|^[-+]?\d{1,9}$'
-    return re.match(__pattern, str(int_str)) is not None
-
-
 def count_decimal_places(num_str):
     """ 计算小数位数, 允许科学计数法 """
     match = re.match(r'^[-+]?\d+(\.\d+)?([eE][-+]?\d+)?$', str(num_str))
@@ -144,8 +105,8 @@ class MysqlUpload:
             result3 = re.findall(r'同比$|环比$', k, re.IGNORECASE)
             result4 = re.findall(r'花费$|消耗$|金额$', k, re.IGNORECASE)
 
-            date_type = is_valid_date(v)  # 判断日期时间
-            int_num = is_integer(v)  # 判断整数
+            date_type = otk.is_valid_date(v)  # 判断日期时间
+            int_num = otk.is_integer(v)  # 判断整数
             count_int, count_float = count_decimal_places(v)  # 判断小数，返回小数位数
             if result1:  # 京东sku/spu商品信息
                 __res_dict.update({k: 'varchar(100)'})
@@ -204,13 +165,7 @@ class MysqlUpload:
             database_exists = cursor.fetchone()
             if not database_exists:
                 # 如果数据库不存在，则新建
-                if '8.138.27' in str(self.host) or platform.system() == "Linux":  # 阿里云 mysql 低版本不支持 0900
-                    sql = f"CREATE DATABASE `{db_name}` COLLATE utf8mb4_unicode_ci"
-                    self.config.update({'charset': 'utf8mb4_unicode_ci'})
-                if '192.168.1.100' in str(self.host):
-                    sql = f"CREATE DATABASE `{db_name}`"
-                else:
-                    sql = f"CREATE DATABASE `{db_name}` COLLATE utf8mb4_0900_ai_ci"
+                sql = f"CREATE DATABASE `{db_name}` COLLATE utf8mb4_0900_ai_ci"
                 cursor.execute(sql)
                 connection.commit()
                 logger.info(f"创建Database: {db_name}")
@@ -369,13 +324,8 @@ class MysqlUpload:
             database_exists = cursor.fetchone()
             if not database_exists:
                 # 如果数据库不存在，则新建
-                if '8.138.27' in str(self.host) or platform.system() == "Linux":  # 阿里云 mysql 低版本不支持 0900
-                    sql = f"CREATE DATABASE `{db_name}` COLLATE utf8mb4_unicode_ci"
-                    self.config.update({'charset': 'utf8mb4_unicode_ci'})
-                if '192.168.1.100' in str(self.host):
-                    sql = f"CREATE DATABASE `{db_name}`"
-                else:
-                    sql = f"CREATE DATABASE `{db_name}` COLLATE utf8mb4_0900_ai_ci"
+
+                sql = f"CREATE DATABASE `{db_name}` COLLATE utf8mb4_0900_ai_ci"
                 cursor.execute(sql)
                 connection.commit()
                 logger.info(f"创建Database: {db_name}")
@@ -580,13 +530,7 @@ class MysqlUpload:
             database_exists = cursor.fetchone()
             if not database_exists:
                 # 如果数据库不存在，则新建
-                if '8.138.27' in str(self.host) or platform.system() == "Linux":  # 阿里云 mysql 低版本不支持 0900
-                    sql = f"CREATE DATABASE `{db_name}` COLLATE utf8mb4_unicode_ci"
-                    self.config.update({'charset': 'utf8mb4_unicode_ci'})
-                if '192.168.1.100' in str(self.host):
-                    sql = f"CREATE DATABASE `{db_name}`"
-                else:
-                    sql = f"CREATE DATABASE `{db_name}` COLLATE utf8mb4_0900_ai_ci"
+                sql = f"CREATE DATABASE `{db_name}` COLLATE utf8mb4_0900_ai_ci"
                 cursor.execute(sql)
                 connection.commit()
                 logger.info(f"创建Database: {db_name}")
@@ -770,8 +714,8 @@ class MysqlUpload:
             result3 = re.findall(r'同比$|环比$', k, re.IGNORECASE)
             result4 = re.findall(r'花费$|消耗$|金额$', k, re.IGNORECASE)
 
-            date_type = is_valid_date(v)  # 判断日期时间
-            int_num = is_integer(v)  # 判断整数
+            date_type = otk.is_valid_date(v)  # 判断日期时间
+            int_num = otk.is_integer(v)  # 判断整数
             count_int, count_float = count_decimal_places(v)  # 判断小数，返回小数位数
             if result1:  # 京东sku/spu商品信息
                 __res_dict.update({k: 'varchar(100)'})
@@ -806,44 +750,9 @@ class MysqlUpload:
             new_dict_data.update({k: v})
         return __res_dict, new_dict_data
 
-    def cover_df(self, df):
-        """ 清理 df 的值和列名 """
-        df.replace([np.inf, -np.inf], '0', inplace=True)  # 清理一些非法值
-        # df.replace(to_replace=['\\N', '-', '--', '', 'nan', 'NAN'], value='0', regex=False, inplace=True)  # 替换掉特殊字符
-        df.replace(to_replace=['\\N', '', 'nan', 'NAN'], value='0', regex=False, inplace=True)  # 替换掉特殊字符
-        # df.replace(to_replace=[','], value='', regex=True, inplace=True)
-        df.replace(to_replace=['="'], value='', regex=True, inplace=True)  # ="和"不可以放在一起清洗, 因为有: id=86785565
-        df.replace(to_replace=['"'], value='', regex=True, inplace=True)
-        cols = df.columns.tolist()
-        for col in cols:
-            if col == 'id':
-                df.pop('id')
-                continue
-            df[col] = df[col].apply(lambda x: float(re.sub(r'%$', '', str(x))) / 100 if (
-                    str(x) != '' and str(x).endswith('%')) and not re.findall('[\\u4e00-\\u9fa5]', str(x)) else '0.0' if str(x) == '0%' else x)
-            try:
-                # 不能直接使用 int() ，对于大数，可能转为uint64，导致数据库入库可能异常
-                df[col] = df[col].apply(
-                    lambda x: np.int64(str(x)) if '_' not in str(x) and '.' not in str(x) else x)  # 不含小数点尝试转整数
-            except:
-                pass
-            try:
-                if df[col].dtype == 'object':  # 有些列没有被 pandas 识别数据类型，会没有 dtype 属性
-                    df[col] = df[col].apply(lambda x: float(x) if '.' in str(x) and '_' not in str(x) else x)
-            except:
-                pass
-            new_col = col.lower()
-            new_col = re.sub(r'[()\-，,&~^、 （）\"\'“”=·/。》《><！!`]', '_', new_col, re.IGNORECASE)
-            new_col = new_col.replace('）', '')
-            new_col = re.sub(r'_{2,}', '_', new_col)
-            new_col = re.sub(r'_+$', '', new_col)
-            df.rename(columns={col: new_col}, inplace=True)
-        df.fillna(0, inplace=True)
-        return df
-
     def convert_df_dtypes(self, df: pd.DataFrame):
         """ 清理 df 的值和列名，并转换数据类型 """
-        df = self.cover_df(df=df)  # 清理 df 的值和列名
+        df = otk.cover_df(df=df)  # 清理 df 的值和列名
         [pd.to_numeric(df[col], errors='ignore') for col in df.columns.tolist()]
         dtypes = df.dtypes.to_dict()
         __res_dict = {}
@@ -949,13 +858,7 @@ class MysqlUpload:
             database_exists = cursor.fetchone()
             if not database_exists:
                 # 如果数据库不存在，则新建
-                # if '8.138.27' in str(self.host) or platform.system() == "Linux":  # 阿里云 mysql 低版本不支持 0900
-                #     sql = f"CREATE DATABASE `{db_name}` COLLATE utf8mb4_unicode_ci"
-                #     self.config.update({'charset': 'utf8mb4_unicode_ci'})
-                if '192.168.1.100' in str(self.host):
-                    sql = f"CREATE DATABASE `{db_name}`"
-                else:
-                    sql = f"CREATE DATABASE `{db_name}` COLLATE utf8mb4_0900_ai_ci"
+                sql = f"CREATE DATABASE `{db_name}` COLLATE utf8mb4_0900_ai_ci"
                 cursor.execute(sql)
                 connection.commit()
                 logger.info(f"创建Database: {db_name}")
@@ -1705,31 +1608,5 @@ class OptimizeDatas:
             self.connection.close()
 
 
-def year_month_day_bak(start_date, end_date):
-    """
-    使用date_range函数和DataFrame来获取从start_date至end_date之间的所有年月日
-    calendar.monthrange： 获取当月第一个工作日的星期值(0,6) 以及当月天数
-    """
-    # 替换年月日中的日, 以便即使传入当月日期也有返回值
-    try:
-        start_date = f'{pd.to_datetime(start_date).year}-{pd.to_datetime(start_date).month}-01'
-    except Exception as e:
-        logger.error(e)
-        return []
-    # 使用pandas的date_range创建一个日期范围，频率为'MS'代表每月开始
-    date_range = pd.date_range(start=start_date, end=end_date, freq='MS')
-    # 转换格式
-    year_months = date_range.strftime('%Y-%m').drop_duplicates().sort_values()
-
-    results = []
-    for year_month in year_months:
-        year = re.findall(r'(\d{4})', year_month)[0]
-        month = re.findall(r'\d{4}-(\d{2})', year_month)[0]
-        s, d = calendar.monthrange(int(year), int(month))
-        results.append({'起始日期': f'{year_month}-01', '结束日期': f'{year_month}-{d}'})
-
-    return results  # start_date至end_date之间的所有年月日
-
-
 if __name__ == '__main__':
     pass

mdbq/mysql/s_query.py

@@ -1,16 +1,12 @@
 # -*- coding:utf-8 -*-
 import datetime
-import platform
 import re
 import time
-from functools import wraps
 import warnings
 import pymysql
 import numpy as np
 import pandas as pd
-from sqlalchemy import create_engine
 import os
-import calendar
 from decimal import Decimal
 import logging
 

mdbq/other/otk.py

@@ -0,0 +1,151 @@
+import re
+import calendar
+import datetime
+from dateutil.relativedelta import relativedelta
+import pandas as pd
+import numpy as np
+
+
+def first_and_last_day(date):
+    """
+    返回指定日期当月的第一天和最后一天
+    """
+    date = pd.to_datetime(date)  # n 月以前的今天
+    _, _lastDay = calendar.monthrange(date.year, date.month)  # 返回月的第一天的星期和当月总天数
+    _firstDay = datetime.date(date.year, date.month, day=1)
+    _lastDay = datetime.date(date.year, date.month, day=_lastDay)
+    return _firstDay, _lastDay
+
+
+def get_day_of_month(num: int, fm=None):
+    """
+    num: 获取n月以前的第一天和最后一天, num=0时, 返回当月第一天和最后一天
+    fm: 日期输出格式
+    """
+    if not fm:
+        fm ='%Y%m%d'
+    _today = datetime.date.today()
+    months_ago = _today - relativedelta(months=num)  # n 月以前的今天
+    _, _lastDay = calendar.monthrange(months_ago.year, months_ago.month)  # 返回月的第一天的星期和当月总天数
+    _firstDay = datetime.date(months_ago.year, months_ago.month, day=1).strftime(fm)
+    _lastDay = datetime.date(months_ago.year, months_ago.month, day=_lastDay).strftime(fm)
+    return _firstDay, _lastDay
+
+
+def dates_between(start_date, end_date, fm=None) -> list:
+    """
+    获取两个日期之间的所有日期， 返回 list
+    fm: 日期输出格式
+    """
+    if not fm:
+        fm ='%Y-%m-%d'
+    start_date = pd.to_datetime(start_date)
+    end_date = pd.to_datetime(end_date)
+    dates = []
+    current_date = start_date
+    while current_date <= end_date:
+        dates.append(current_date.strftime(fm))
+        current_date += datetime.timedelta(days=1)
+    return dates
+
+
+def cover_df(df):
+    df.replace([np.inf, -np.inf], '0', inplace=True)  # 清理一些非法值
+    df.replace(to_replace=['\\N', '-', '--', '', 'nan', 'NAN'], value='0', regex=False, inplace=True)  # 替换掉特殊字符
+    df.replace(to_replace=[','], value='', regex=True, inplace=True)
+    df.replace(to_replace=['="'], value='', regex=True, inplace=True)  # ="和"不可以放在一起清洗, 因为有: id=86785565
+    df.replace(to_replace=['"'], value='', regex=True, inplace=True)
+    cols = df.columns.tolist()
+    for col in cols:
+        if col == 'id':
+            df.pop('id')
+            continue
+        # df[col] = df[col].apply(
+        #     lambda x: float(float((str(x).rstrip("%"))) / 100) if re.findall(r'^\d+\.?\d*%$', str(x)) else x)
+        # df[col] = df[col].apply(lambda x:
+        #                         float(re.sub(r'%$', '', str(x))) / 100
+        #                         if (str(x) != '' and str(x).endswith('%')) and not re.findall(
+        #     '[\\u4e00-\\u9fa5]', str(x)) else '0.0' if str(x) == '0%' else x)
+        df[col] = df[col].apply(
+            lambda x: float(str(x).rstrip("%")) / 100
+            if (
+                    re.fullmatch(r'^\d+\.?\d*%$', str(x))  # 匹配数字加%格式
+                    and not re.search(r'[\u4e00-\u9fa5]', str(x))  # 排除含中文的情况
+            )
+            else (
+                '0.0' if str(x) == '0%' else x  # 处理 "0%"
+            )
+        )
+
+        try:
+            # 不能直接使用 int() ，对于大数，可能转为uint64，导致数据库入库可能异常
+            df[col] = df[col].apply(
+                lambda x: np.int64(str(x)) if '_' not in str(x) and '.' not in str(x) else x)  # 不含小数点尝试转整数
+        except:
+            pass
+        try:
+            if df[col].dtype == 'object':  # 有些列没有被 pandas 识别数据类型，会没有 dtype 属性
+                df[col] = df[col].apply(lambda x: float(x) if '.' in str(x) and '_' not in str(x) else x)
+        except:
+            pass
+        new_col = col.lower()
+        new_col = re.sub(r'[()\-，,&~^、 （）\"\'“”=·/。》《><！!`]', '_', new_col, re.IGNORECASE)
+        new_col = new_col.replace('）', '')
+        new_col = re.sub(r'_{2,}', '_', new_col)
+        new_col = re.sub(r'_+$', '', new_col)
+        df.rename(columns={col: new_col}, inplace=True)
+    df.fillna(0, inplace=True)
+    return df
+
+
+def translate_keys(original_dict:dict, translation_dict:dict) -> dict:
+    """
+    original_dict键名翻译, 若键存在则返回翻译值，否则返回原键
+    """
+    return {translation_dict.get(k, k): v for k, v in original_dict.items()}
+
+
+def is_valid_date(date_string):
+    """
+    mysql调用
+    判断是否是日期格式, 且允许没有前导零, 且允许带时间
+    纯日期格式： 返回 1
+    日期+时间： 返回 2
+    """
+    date_pattern = r"^(\d{4})-(0?[1-9]|1[0-2])-(0?[1-9]|[12]\d|3[01])$"
+    match = re.match(date_pattern, str(date_string))  # 判断纯日期格式：2024-11-09
+    if match is None:
+        date_pattern = r".*\d+:\d+:\d+$"
+        match = re.match(date_pattern, date_string)  # 判断日期+时间：2024-11-09 00:36:45
+        if match is not None:
+            return 2
+    else:
+        return 1
+
+
+def is_integer(int_str):
+    """
+    mysql调用
+    判断是否整数, 允许包含千分位分隔符, 允许科学计数法
+    """
+    # 如果是科学计数法
+    match = re.findall(r'^[-+]?(\d+)\.(\d+)[eE][-+]?(\d+)$', str(int_str))
+    if match:
+        if len(match[0]) == 3:
+            if int(match[0][0]) == 0:  # 0 开头
+                if int(match[0][2]) > 10:  # 转换后整数长度超过 10 位
+                    return False
+            else:  # 不是 0 开头
+                if len(match[0][0]) + int(match[0][2]) > 10:  # 转换后整数长度超过 10 位
+                    return False
+            if int(match[0][2]) >= len(match[0][1]):
+                return True
+            else:
+                return False
+    # 如果是普通数字, 且允许千分符
+    __pattern = r'^[-+]?\d{1,3}(,\d{3}){0,3}$|^[-+]?\d{1,9}$'
+    return re.match(__pattern, str(int_str)) is not None
+
+
+if __name__ == '__main__':
+    pass

mdbq/spider/aikucun.py

@@ -21,6 +21,7 @@ from mdbq.mysql import mysql
 from mdbq.mysql import s_query
 from mdbq.config import config
 from mdbq.other import ua_sj
+from mdbq.other import otk
 
 dir_path = os.path.expanduser("~")
 config_file = os.path.join(dir_path, 'spd.txt')
@@ -47,18 +48,6 @@ def keep_connect(_db_name, _config, max_try: int=10):
     return None
 
 
-def dates_between(start_date, end_date) -> list:
-    """ 获取两个日期之间的所有日期， 返回 list   """
-    start_date = pd.to_datetime(start_date)
-    end_date = pd.to_datetime(end_date)
-    dates = []
-    current_date = start_date
-    while current_date <= end_date:
-        dates.append(current_date.strftime('%Y-%m-%d'))
-        current_date += datetime.timedelta(days=1)
-    return dates
-
-
 class AikuCun:
     def __init__(self):
         self.url = 'https://gray-merc.aikucun.com/index.html'
@@ -204,7 +193,7 @@ class AikuCun:
             self.start_date = start_date
         if end_date:
             self.end_date = end_date
-        date_list = dates_between(start_date=self.start_date, end_date=self.end_date)
+        date_list = otk.dates_between(start_date=self.start_date, end_date=self.end_date)
 
         df = download.data_to_df(
             db_name=self.db_name,

{mdbq-3.8.8.dist-info → mdbq-3.8.10.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: mdbq
-Version: 3.8.8
+Version: 3.8.10
 Home-page: https://pypi.org/project/mdbq
 Author: xigua, 
 Author-email: 2587125111@qq.com

{mdbq-3.8.8.dist-info → mdbq-3.8.10.dist-info}/RECORD

@@ -1,25 +1,17 @@
 mdbq/__init__.py,sha256=Il5Q9ATdX8yXqVxtP_nYqUhExzxPC_qk_WXQ_4h0exg,16
-mdbq/__version__.py,sha256=wmmK6wPunxlLTIhSLvjwGzadK_0XfwOYNMA_w0QkRF4,17
+mdbq/__version__.py,sha256=30BOEMWMMdvugdYm1n90xiBvxiQzusLf7XtVO4-Zjr8,18
 mdbq/aggregation/__init__.py,sha256=EeDqX2Aml6SPx8363J-v1lz0EcZtgwIBYyCJV6CcEDU,40
 mdbq/aggregation/query_data.py,sha256=-4HWC1HZmgqUAuvcRiHZU4FLtI70nRq_Hp3eXVZTyH8,185843
-mdbq/bdup/__init__.py,sha256=AkhsGk81SkG1c8FqDH5tRq-8MZmFobVbN60DTyukYTY,28
-mdbq/bdup/bdup.py,sha256=hJs815hGFwm_X5bP2i9XugG2w2ZY_F0n3-Q0hVpIPPw,4892
 mdbq/config/__init__.py,sha256=jso1oHcy6cJEfa7udS_9uO5X6kZLoPBF8l3wCYmr5dM,18
 mdbq/config/config.py,sha256=eaTfrfXQ65xLqjr5I8-HkZd_jEY1JkGinEgv3TSLeoQ,3170
-mdbq/config/set_support.py,sha256=7C7NFy7Em_uC7lig54qQlIlKG_AJeMCskxzK87anGkM,462
-mdbq/dataframe/__init__.py,sha256=2HtCN8AdRj53teXDqzysC1h8aPL-mMFy561ESmhehGQ,22
-mdbq/dataframe/converter.py,sha256=lETYhT7KXlWzWwqguqhk6vI6kj4rnOBEW1lhqKy2Abc,5035
 mdbq/log/__init__.py,sha256=Mpbrav0s0ifLL7lVDAuePEi1hJKiSHhxcv1byBKDl5E,15
-mdbq/log/mylogger.py,sha256=oaT7Bp-Hb9jZt52seP3ISUuxVcI19s4UiqTeouScBO0,3258
 mdbq/log/spider_logging.py,sha256=KX9TTUn9naZNBACCEFhyTktnWhr5JaSNQLppLGyrm9Y,1645
-mdbq/mongo/__init__.py,sha256=SILt7xMtQIQl_m-ik9WLtJSXIVf424iYgCfE_tnQFbw,13
 mdbq/mysql/__init__.py,sha256=A_DPJyAoEvTSFojiI2e94zP0FKtCkkwKP1kYUCSyQzo,11
-mdbq/mysql/mysql.py,sha256=tR6l4Zzn9j6zKaFcy0Ktw2oL8OoX3QB6jDoDp1l2fiM,95474
-mdbq/mysql/s_query.py,sha256=09Dp7DrVXui6dAI6zFDfrsUOdjPblF_oYUpgqbZMhXg,8757
-mdbq/mysql/year_month_day.py,sha256=VgewoE2pJxK7ErjfviL_SMTN77ki8GVbTUcao3vFUCE,1523
+mdbq/mysql/mysql.py,sha256=2cPuqX4zq2b6ghFWxTylr52DPZGE2WNrCdFV0RcF6LY,89048
+mdbq/mysql/s_query.py,sha256=X055aLRAgxVvueXx4NbfNjp6MyBI02_XBb1pTKw09L0,8660
 mdbq/other/__init__.py,sha256=jso1oHcy6cJEfa7udS_9uO5X6kZLoPBF8l3wCYmr5dM,18
 mdbq/other/download_sku_picture.py,sha256=YU8DxKMXbdeE1OOKEA848WVp62jYHw5O4tXTjUdq9H0,44832
-mdbq/other/porxy.py,sha256=UHfgEyXugogvXgsG68a7QouUCKaohTKKkI4RN-kYSdQ,4961
+mdbq/other/otk.py,sha256=amIFeLDNUJpSi0U6hXbnqXeGTbYL-8-5U5yAATzSM3Y,5947
 mdbq/other/pov_city.py,sha256=AEOmCOzOwyjHi9LLZWPKi6DUuSC-_M163664I52u9qw,21050
 mdbq/other/ua_sj.py,sha256=JuVYzc_5QZ9s_oQSrTHVKkQv4S_7-CWx4oIKOARn_9U,22178
 mdbq/pbix/__init__.py,sha256=Trtfaynu9RjoTyLLYBN2xdRxTvm_zhCniUkVTAYwcjo,24
@@ -28,8 +20,8 @@ mdbq/pbix/refresh_all.py,sha256=OBT9EewSZ0aRS9vL_FflVn74d4l2G00wzHiikCC4TC0,5926
 mdbq/redis/__init__.py,sha256=YtgBlVSMDphtpwYX248wGge1x-Ex_mMufz4-8W0XRmA,12
 mdbq/redis/getredis.py,sha256=Uk8-cOWT0JU1qRyIVqdbYokSLvkDIAfcokmYj1ebw8k,24104
 mdbq/spider/__init__.py,sha256=RBMFXGy_jd1HXZhngB2T2XTvJqki8P_Fr-pBcwijnew,18
-mdbq/spider/aikucun.py,sha256=bUjjPjNoW3EL6H89nnBdFEwnWgGuEB2CENuBxcvx0Kw,20284
-mdbq-3.8.8.dist-info/METADATA,sha256=gNb3isnfJokgROYlgUClnTfmR7AT-GfhxCkCyfLasrI,363
-mdbq-3.8.8.dist-info/WHEEL,sha256=jB7zZ3N9hIM9adW7qlTAyycLYW9npaWKLRzaoVcLKcM,91
-mdbq-3.8.8.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
-mdbq-3.8.8.dist-info/RECORD,,
+mdbq/spider/aikucun.py,sha256=YLRTDgOKPGDyNB-z5dPOJhBoTzM6Rmbjy1Qng_KyJQc,19906
+mdbq-3.8.10.dist-info/METADATA,sha256=bYHqcx9saoyDq-BQSXUtUQWTcDNICdZeF5jaKP6Dlyc,364
+mdbq-3.8.10.dist-info/WHEEL,sha256=jB7zZ3N9hIM9adW7qlTAyycLYW9npaWKLRzaoVcLKcM,91
+mdbq-3.8.10.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
+mdbq-3.8.10.dist-info/RECORD,,

mdbq/bdup/__init__.py

@@ -1,5 +0,0 @@
-
-
-
-# 百度云数据处理
-

mdbq/bdup/bdup.py

@@ -1,111 +0,0 @@
-# -*- coding: UTF-8 –*-
-import os
-import platform
-import subprocess
-from concurrent.futures import ThreadPoolExecutor
-from bypy import ByPy
-
-
-class BaiDu:
-    """
-    如果通过调用命令行终端运行, 云端路径必须使用linux格式，不要使用windows格式,否则在windows系统里面会上传失败(无法在云端创建文件)
-    """
-    def __init__(self):
-        self.local_path = None
-        self.remote_path = None
-        self.skip:list = []
-        self.delete_remote_files:list = []
-        self.bp = ByPy()
-        self.count = 0
-        self.total = 0
-
-    def upload_dir(self, local_path, remote_path):
-        """
-        上传整个文件夹，执行完后删除指定文件, 指定 self.delete_remote_files
-        如果通过调用命令行终端运行, 《云端路径!!》必须使用linux格式，不要使用反斜杆,否则在windows系统里面会上传失败
-        """
-        self.local_path = local_path
-        self.remote_path = remote_path.replace('\\', '/')
-        if not os.path.exists(self.local_path):
-            print(f'{self.local_path}: 本地目录不存在，没有什么可传的')
-            return
-
-        if platform.system() == 'Windows':
-            self.bp.upload(localpath=self.local_path, remotepath=self.remote_path.replace('\\', '/'))  # 上传文件到百度云
-        else:
-            command = f'bypy upload "{self.local_path}" "{self.remote_path}" --on-dup skip'  # 相同文件跳过
-            try:
-                subprocess.run(command, shell=True)
-            except Exception as e:
-                print(e)
-        self.delete_files()  # 最好是在内部执行删除, 避免路径异常
-
-    def upload_file(self, local_path, remote_path, processes=False):
-        """
-        上传文件夹，按单个文件上传，可以跳过指定文件/文件夹, 指定 self.skip
-        《云端路径!!》必须使用linux格式
-        """
-        if not isinstance(self.skip, list):
-            raise TypeError('skip must be a list')
-        self.skip += ['.DS_Store', '.localized', 'desktop.ini', '$RECYCLE.BIN', 'Icon']
-        self.local_path = local_path
-        self.remote_path = remote_path.replace('\\', '/')
-        if not os.path.exists(self.local_path):
-            print(f'{self.local_path}: 本地目录不存在，没有什么可传的')
-            return
-
-        local_files = os.listdir(self.local_path)
-
-        local_file_list = []
-        for file in local_files:
-            if file in self.skip:  # 跳过指定文件/文件夹
-                continue
-            local_p = os.path.join(self.local_path, file)
-            if os.path.isfile(local_p):
-                rt_path = os.path.join(self.remote_path, file).replace('\\', '/')
-                self.total += 1
-                local_file_list.append({local_p: rt_path})
-            elif os.path.isdir(local_p):
-                for root, dirs, files in os.walk(local_p, topdown=False):
-                    for name in files:
-                        if name in self.skip:  # 从子文件夹内跳过指定文件
-                            continue
-                        lc_path = os.path.join(root, name)
-                        rt_path = lc_path.replace(self.local_path, self.remote_path).replace('\\', '/')
-                        self.total += 1
-                        local_file_list.append({lc_path: rt_path})
-        if processes:
-            # 不指定 max_workers 参数，默认值是 os.cpu_count() * 5
-            with ThreadPoolExecutor() as executor:
-                    executor.map(self.up_one_file, local_file_list)
-        else:
-            for item in local_file_list:
-                self.up_one_file(file_dict=item)
-
-    def up_one_file(self, file_dict:dict):
-        if not isinstance(file_dict, dict):
-            raise TypeError('file_dict must be a dict')
-        for k, v in file_dict.items():
-            self.count += 1
-            print(f'上传: {self.count}/{self.total}  {k}')
-            self.bp.upload(localpath=k, remotepath=v)  # 上传文件到百度云
-
-    def delete_files(self):
-        """ 移除云端文件，位于 self.remote_path 文件夹下的子文件 """
-        self.delete_remote_files += ['.DS_Store', '.localized', 'desktop.ini', '$RECYCLE.BIN', 'Icon']
-        for delete_file in self.delete_remote_files:
-            self.bp.remove(remotepath=f'{self.remote_path.replace('\\', '/')}/{delete_file}')  # 移除文件
-
-    def download_dir(self, local_path, remote_path):
-        """ 下载文件夹到本地 """
-        self.local_path = local_path
-        self.remote_path = remote_path.replace('\\', '/')
-        if not os.path.exists(self.local_path):
-            os.mkdir(self.local_path)
-
-        self.bp.download(localpath=f'{self.local_path}', remotepath=f'{self.remote_path.replace('\\', '/')}')
-
-
-if __name__ == '__main__':
-    bp = ByPy()
-    bp.list()

mdbq/config/set_support.py

@@ -1,20 +0,0 @@
-# -*- coding: UTF-8 –*-
-import platform
-import getpass
-import os
-import sys
-
-"""
-专门用来设置 support 文件夹路径
-support 文件夹包含很多配置类文件，是程序必不可少的依赖
-"""
-
-
-class SetSupport:
-    def __init__(self, dirname):
-        self.dirname = os.path.join(os.path.realpath(os.path.dirname(sys.argv[0])), dirname)
-
-
-if __name__ == '__main__':
-    s = SetSupport(dirname='support').dirname
-    print(s)

mdbq/dataframe/__init__.py

@@ -1,4 +0,0 @@
-
-
-
-# dataframe 优化

mdbq/dataframe/converter.py

@@ -1,107 +0,0 @@
-# -*- coding:utf-8 -*-
-import pandas as pd
-import numpy as np
-from decimal import Decimal
-import re
-
-
-class DataFrameConverter(object):
-    def __init__(self, df=pd.DataFrame({})):
-        self.df = df
-
-    def convert_df_cols(self, df=pd.DataFrame({})):
-        """
-        清理 dataframe 非法值
-        对数据类型进行转换(尝试将 object 类型转为 int 或 float)
-        """
-        if len(df) == 0:
-            df = self.df
-            if len(df) == 0:
-                return
-
-        def find_longest_decimal_value(number_list):
-            # 取列表中小数位数最长的值
-            longest_value = None
-            max_decimals = 0
-            for num in number_list:
-                decimal_places = len(str(num).split('.')[1])
-                if decimal_places > max_decimals:
-                    max_decimals = decimal_places
-                    longest_value = num
-            return longest_value
-
-        # dtypes = df.dtypes.apply(str).to_dict()  # 将 dataframe 数据类型转为字典形式
-        df.replace([np.inf, -np.inf], '0', inplace=True)  # 清理一些非法值
-        # df.replace(to_replace=['\\N', '-', '--', '', 'nan', 'NAN'], value='0', regex=False, inplace=True)  # 替换掉特殊字符
-        df.replace(to_replace=['\\N', '', 'nan', 'NAN'], value='0', regex=False, inplace=True)  # 替换掉特殊字符
-        # df.replace(to_replace=[','], value='', regex=True, inplace=True)
-        df.replace(to_replace=['="'], value='', regex=True, inplace=True)  # ="和"不可以放在一起清洗, 因为有: id=86785565
-        df.replace(to_replace=['"'], value='', regex=True, inplace=True)
-        cols = df.columns.tolist()
-
-        df.reset_index(inplace=True, drop=True)  # 重置索引，避免下面的 df.loc[0, col] 会出错
-
-        for col in cols:
-            if col.lower() == 'id':
-                df.pop(col)  # 等待插入的 df 不能包含 id 列，否则可能跟现有 id 主键冲突
-                continue
-
-            try:
-                # 百分比在某些数据库中不兼容, 转换百分比为小数, # 转百分比的列不能含有中文或特殊字符
-                df[col] = df[col].apply(
-                    lambda x: float(float((str(x).rstrip("%"))) / 100) if re.findall(r'^\d+\.?\d*%$', str(x)) else x)
-            except Exception as e:
-                print(f'留意错误信息: 位于列 -> {col} -> {e}')
-
-            if (col.endswith('占比') or col.endswith('率') or col.endswith('同比')
-                    or col.endswith('环比') or col.lower().endswith('roi')
-                    or col.endswith('产出比')):
-                df = df.astype({col: 'float64'}, errors='raise')
-
-            # 尝试转换合适的数据类型
-            if df[col].dtype == 'object':
-                #  "_"符号会被错误识别
-                try:
-                    # 不能直接使用 int() ，对于大数，可能转为uint64，导致数据库入库可能异常
-                    df[col] = df[col].apply(
-                        lambda x: np.int64(str(x)) if '_' not in str(x) and '.' not in str(x) else x)  # 不含小数点尝试转整数
-                    # df[col] = df[col].apply(lambda x: int(x) if '_' not in str(x) and '.' not in str(x) else x)  # 不含小数点尝试转整数
-                except:
-                    pass
-                if df[col].dtype == 'object':
-                    try:
-                        df[col] = df[col].apply(lambda x: float(x) if '.' in str(x) and '_' not in str(x) else x)
-                    except:
-                        pass
-            if df[col].dtype == 'float' or df[col].dtype == 'float64':  # 对于小数类型, 保留 6 位小数
-                df[col] = df[col].fillna(0.0).apply(lambda x: round(x, 6))
-
-            # 转换日期样式的列为日期类型
-            value = df.loc[0, col]
-            if value:
-                res = re.match(r'\d{4}-\d{2}-\d{2}|\d{4}-\d{2}-\d{2} |\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}'
-                               r'|\d{4}/\d{1}/\d{1}|\d{4}/\d{1}/\d{2}|\d{4}/\d{2}/\d{1}|\d{4}/\d{2}/\d{2}', str(value))
-                if res:
-                    try:
-                        df[col] = df[col].apply(lambda x: pd.to_datetime(x))
-                    except:
-                        pass
-            new_col = col.lower()
-            new_col = re.sub(r'[()\-，,&~^、 （）\"\'“”=·/。》《><！!`]', '_', new_col, re.IGNORECASE)
-            new_col = new_col.replace('）', '')
-            new_col = re.sub(r'_{2,}', '_', new_col)
-            new_col = re.sub(r'_+$', '', new_col)
-            df.rename(columns={col: new_col}, inplace=True)
-        df.fillna(0, inplace=True)
-        return df
-
-
-if __name__ == '__main__':
-    # df = pd.DataFrame(np.random.randn(5, 3), columns=['a', 'b', 'c'])
-    # converter = DataFrameConverter()
-    # df = converter.convert_df_cols(df)
-    # print(df['a'].dtype)
-    # print(df)
-    pattern = '1540%'
-    pattern = re.findall(r'^\d+\.?\d*%$', pattern)
-    print(pattern)

mdbq/log/mylogger.py

@@ -1,66 +0,0 @@
-import logging
-from logging import Logger
-from logging import handlers
-
-
-class MyLogger(Logger):
-    """
-    从Logger类中继承，实例化一个日志器
-    """
-    def __init__(self, logger_name, level='INFO', is_stream_handler=True, file=None, debug_file=None,
-                 max_bytes=False, back_count=10, when=None):
-        """
-        :param logger_name: 日志器的名字
-        :param level: 日志级别  # DEBUG  INFO  WARNING  ERROR  CRITICAL
-        :param is_stream_handler: 默认True输出到控制台
-        :param file: 传入文件名，默认None不输出到 file
-        param debug_file: 传入文件名，记录详细debug时使用，默认None不输出， 尽量不要和file同时使用，会重复写
-        :param when: 按周期分割日志，默认不分割，除非指定其他值
-        :param max_bytes: 按文件大小分割日志
-        :param back_count: 保留日志的数量， 值从0开始
-        """
-        # 设置日志器名字、级别
-        super().__init__(logger_name, level)
-
-        # 定义日志格式, 使用Formatter类实例化一个日志类
-        fmt_stream = "%(asctime)s %(levelname)s %(name)s: %(message)s"
-        fmt_file = "%(asctime)s %(name)s: %(message)s"
-        fmt_debug_file = "%(asctime)s %(levelname)s %(name)s %(funcName)s: %(message)s"
-        formatter_stream = logging.Formatter(fmt_stream, datefmt="%Y-%m-%d %H:%M:%S")
-        formatter_file = logging.Formatter(fmt_file, datefmt="%Y-%m-%d %H:%M:%S")
-        formatter_debug_file = logging.Formatter(fmt_debug_file, datefmt="%Y-%m-%d %H:%M:%S")
-
-        # 创建一个handler，默认输出到控制台，如果设置为False，日志将不输出到控制台
-        if is_stream_handler:
-            stream_handler = logging.StreamHandler()  # 设置渠道当中的日志格式
-            stream_handler.setFormatter(formatter_stream)  # 将渠道与实例日志器绑定
-            self.addHandler(stream_handler)
-
-        # 创建一个handler，输出到文件file
-        if file:
-            file_handle = logging.FileHandler(file, mode='a', encoding='utf-8')
-            file_handle.setFormatter(formatter_file)
-            self.addHandler(file_handle)
-
-        # 创建一个handler，输出到文件file，记录详细的debug信息
-        if debug_file:
-            debug_file_handle = logging.FileHandler(debug_file, mode='a', encoding='utf-8')
-            debug_file_handle.setFormatter(formatter_debug_file)
-            self.addHandler(debug_file_handle)
-
-        # 创建一个handler，按日志文件大小分割
-        if max_bytes:
-            formatter_ = logging.Formatter(fmt='%(asctime)s %(name)s: %(message)s', datefmt="%Y-%m-%d %H:%M:%S")
-            formatter_time = handlers.RotatingFileHandler(filename='日志_分割.txt', encoding='utf-8',
-                                                          maxBytes=max_bytes, backupCount=back_count)
-            formatter_time.setLevel(level)
-            formatter_time.setFormatter(formatter_)
-            self.addHandler(formatter_time)
-
-        # 创建一个handler，按指定周期分割日志
-        if when:
-            pass
-
-
-if __name__ == '__main__':
-    pass

mdbq/mongo/__init__.py

@@ -1,4 +0,0 @@
-
-
-
-# mongodb

mdbq/mysql/year_month_day.py

@@ -1,38 +0,0 @@
-# -*- coding:utf-8 -*-
-import warnings
-import pandas as pd
-import calendar
-
-warnings.filterwarnings('ignore')
-
-
-def year_month_day(start_date, end_date):
-    """
-    使用date_range函数和DataFrame来获取从start_date至end_date之间的所有年月日
-    calendar.monthrange： 获取当月第一个工作日的星期值(0,6) 以及当月天数
-    返回值: [{'起始日期': '2025-05-01', '结束日期': '2025-05-31'}, {'起始日期': '2025-06-01', '结束日期': '2025-06-30'}]
-    """
-    # 替换年月日中的日, 以便即使传入当月日期也有返回值
-    try:
-        start_date = f'{pd.to_datetime(start_date).year}-{pd.to_datetime(start_date).month}-01'
-    except Exception as e:
-        print(e)
-        return []
-    # 使用pandas的date_range创建一个日期范围，频率为'MS'代表每月开始
-    date_range = pd.date_range(start=start_date, end=end_date, freq='MS')
-    # 转换格式
-    year_months = date_range.strftime('%Y-%m').drop_duplicates().sort_values()
-
-    results = []
-    for year_month in year_months:
-        year = re.findall(r'(\d{4})', year_month)[0]
-        month = re.findall(r'\d{4}-(\d{2})', year_month)[0]
-        s, d = calendar.monthrange(int(year), int(month))
-        results.append({'起始日期': f'{year_month}-01', '结束日期': f'{year_month}-{d}'})
-
-    return results  # start_date至end_date之间的所有年月日
-
-
-if __name__ == '__main__':
-    results = year_month_day(start_date='2025-05-01', end_date='2025-08-01')
-    print(results)

mdbq/other/porxy.py

@@ -1,115 +0,0 @@
-import requests
-import kdl
-import warnings
-import os
-import requests
-import datetime
-import re
-import time
-import socket
-warnings.filterwarnings('ignore')
-"""
-需要传入 订单的 secret_id 和 secret_key
-"""
-
-
-class MyProxy(object):
-
-    def __init__(self, secret_id, secret_key):
-        self.secret_id = secret_id
-        self.secret_key = secret_key
-        self.cookie_path = 'cookies'
-        if not os.path.exists(self.cookie_path):
-            os.mkdir(self.cookie_path)
-
-    def get_proxy(self):
-        """
-        从代理网站获取代理ip， 默认参数是文件位置，不需要修改
-        """
-        secret_id = self.secret_id
-        secret_key = self.secret_key
-        cookie_path = self.cookie_path
-        headers = {
-            "User-Agent": 'Mozilla/5.0'
-        }
-        auth = kdl.Auth(secret_id=secret_id, secret_key=secret_key)
-        client = kdl.Client(auth)
-
-        def ip_address():
-            try:
-                _response = requests.get("https://api.ipify.org/?format=json")
-                _ip = _response.json()["ip"]
-            except:
-                _ip = ''
-            return str(_ip)
-
-        myip_path = f'{cookie_path}/本机ip_{socket.gethostname()}.txt'  # 将本机地址保存本地, 下次直接使用, 避免获取失败
-        if os.path.exists(myip_path):
-            file_timestamp = os.path.getmtime(myip_path)
-            file_date = datetime.datetime.fromtimestamp(file_timestamp).strftime('%Y-%m-%d')
-            today_date = datetime.datetime.today().strftime('%Y-%m-%d')
-            if file_date == today_date:
-                with open(myip_path) as m:
-                    my_ip = m.read().strip()
-            else:
-                my_ip = ip_address()
-                with open(f'{cookie_path}/本机ip_{socket.gethostname()}.txt', 'w') as f:
-                    f.write(my_ip)
-        else:
-            my_ip = ip_address()
-            with open(f'{cookie_path}/本机ip_{socket.gethostname()}.txt', 'w') as f:
-                f.write(my_ip)
-        try:
-            ip_whitelist = client.get_ip_whitelist()  # 检查ip白名单, 如果这句报错，就直接设置白名单
-            if my_ip not in ip_whitelist:
-                ip_whitelist.append(my_ip)
-                client.set_ip_whitelist(ip_whitelist)  # 添加本机到白名单
-        except Exception as e:
-            print(e)
-            client.set_ip_whitelist(my_ip)  # 设置本机到白名单，会清空其他ip
-
-        if not os.path.isfile(f'{cookie_path}/secret_token_{socket.gethostname()}.txt'):  # 如果本地没有密钥令牌则创建
-            secret_token = client.get_secret_token()
-            with open(f'{cookie_path}/secret_token_{socket.gethostname()}.txt', 'w') as f:
-                f.write(secret_token)
-        else:
-            with open(f'{cookie_path}/secret_token_{socket.gethostname()}.txt', 'r') as f:
-                secret_token = f.read()
-        data = f'secret_id={secret_id}&secret_token={secret_token}'  # 检查密钥令牌的有效时长
-        token_expire = requests.post(
-            'https://dev.kdlapi.com/api/check_secret_token',
-            data, headers=headers).json()['data']['expire']
-        if token_expire < 300:  # token_expire 密钥令牌距离过期的剩余时长（单位：秒），不足5分钟则重新创建令牌
-            secret_token = client.get_secret_token()
-            with open(f'{cookie_path}/secret_token_{socket.gethostname()}.txt', 'w') as f:
-                f.write(secret_token)
-        # api地址
-        proxy_url = (f'https://dev.kdlapi.com/api/getdps/?'
-                     f'secret_id={secret_id}'
-                     f'&signature={secret_token}'
-                     f'&num=1&pt=1&format=text&sep=1&f_loc=1&f_citycode=1&area=440100')
-        # expire_time = client.get_order_expire_time()  # 账户有效期
-        _proxy = requests.get(proxy_url, headers=headers).text  # 通过api地址获取代理ip
-        ip_times = client.get_dps_valid_time(proxy=_proxy).values()  # ip有效时间
-        for t in ip_times:
-            if str(t) != '0':
-                ip_times = t
-        balance = client.get_ip_balance(sign_type='hmacsha1')  # 可用ip余额
-        d_time = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
-        ip_proxy = re.findall(r'\d+\.\d+\.\d+\.\d+:\d+', _proxy)[0]
-        city_proxy = re.findall(r'\d+\.\d+\.\d+\.\d+:\d+,([\u4e00-\u9fa5]+),', _proxy)[0]
-        ip_port = ip_proxy.split(':')
-        content = (f'{d_time} 中转IP：{ip_port[0]}, '
-                   f'端口：{ip_port[1]}, '
-                   f'出口地址：{city_proxy}, '
-                   f'ip时长：{ip_times}秒, '
-                   f'可用ip余额：{balance}, '
-                   )
-        # print(content)
-        with open(f'{cookie_path}/代理ip地址.txt', 'a', encoding='utf-8') as f:
-            f.write(content)
-        return ip_proxy
-
-
-if __name__ == '__main__':
-    cookie_path = 'cookies'