mdbq 3.3.5__py3-none-any.whl → 3.3.8__py3-none-any.whl

mdbq/aggregation/query_data.py

@@ -2088,6 +2088,8 @@ class MysqlDatasQuery:
         }
         min_date = df['日期'].min()
         max_date = df['日期'].max()
+        now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+        print(f'{now} 正在更新: mysql ({host}:{port}) {db_name}/{table_name} -> {min_date}~{max_date}')
         new_dict = {
             '日期': '',
             '店铺名称': '',
@@ -2121,23 +2123,24 @@ class MysqlDatasQuery:
             '预售定金支付金额': '',
             '预售预估总金额': '',
         }
-        now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
-        print(f'{now} 正在更新: mysql ({host}:{port}) {db_name}/{table_name} -> {min_date}~{max_date}')
+        _results = []
         for dict_data in df.to_dict(orient='records'):
             new_dict.update(dict_data)
-            m_engine.dict_to_mysql(
+            _results.append(new_dict)
+        if _results:
+            m_engine.insert_many_dict(
                 db_name=db_name,
                 table_name=table_name,
-                dict_data=new_dict,
+                dict_data_list=_results,
                 unique_main_key=None,
                 icm_update=['场次id'],  # 唯一组合键
                 main_key=None,  # 指定索引列, 通常用日期列，默认会设置日期为索引
                 set_typ=set_typ,  # 指定数据类型
             )
-            company_engine.dict_to_mysql(
+            company_engine.insert_many_dict(
                 db_name=db_name,
                 table_name=table_name,
-                dict_data=new_dict,
+                dict_data_list=_results,
                 unique_main_key=None,
                 icm_update=['场次id'],  # 唯一组合键
                 main_key=None,  # 指定索引列, 通常用日期列，默认会设置日期为索引
@@ -2564,40 +2567,6 @@ class MysqlDatasQuery:
         max_date = df['日期'].max()
         now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
         print(f'{now} 正在更新: mysql ({host}:{port}) {db_name}/{table_name} -> {min_date}~{max_date}')
-        # new_dict = {
-        #     '日期': '',
-        #     '店铺名称': '',
-        #     '序号': '',
-        #     '商品名称': '',
-        #     'spu_id': '',
-        #     '商品款号': '',
-        #     '一级类目名称': '',
-        #     '二级类目名称': '',
-        #     '三级类目名称': '',
-        #     '访客量': '',
-        #     '浏览量': '',
-        #     '下单gmv': '',
-        #     '成交gmv': '',
-        #     '支付人数_成交': '',
-        # }
-        # for dict_data in df.to_dict(orient='records'):
-        #     new_dict.update(dict_data)
-        #     m_engine.dict_to_mysql(
-        #         db_name=db_name,
-        #         table_name=table_name,
-        #         dict_data=new_dict,
-        #         icm_update=['日期', '店铺名称', 'spu_id', '商品款号'],
-        #         unique_main_key=None,
-        #         set_typ=set_typ,
-        #     )
-        #     company_engine.dict_to_mysql(
-        #         db_name=db_name,
-        #         table_name=table_name,
-        #         dict_data=new_dict,
-        #         icm_update=['日期', '店铺名称', 'spu_id', '商品款号'],
-        #         unique_main_key=None,
-        #         set_typ=set_typ,
-        #     )
         m_engine.df_to_mysql(
             df=df,
             db_name=db_name,

mdbq/mongo/mongo.py

@@ -8,11 +8,23 @@ import pandas as pd
 import numpy as np
 import pymongo
 from functools import wraps
+import socket
+import platform
 from concurrent.futures import ThreadPoolExecutor
-from mdbq.config import get_myconf
+from mdbq.config import myconfig
 from mdbq.dataframe import converter
 
 warnings.filterwarnings('ignore')
+if socket.gethostname() == 'company' or socket.gethostname() == 'Mac2.local':
+    conf = myconfig.main()
+    conf_data = conf['Windows']['xigua_lx']['mysql']['remoto']
+    username, password, host, port = conf_data['username'], conf_data['password'], conf_data['host'], conf_data[
+        'port']
+else:
+    conf = myconfig.main()
+    conf_data = conf['Windows']['company']['mysql']['remoto']
+    username, password, host, port = conf_data['username'], conf_data['password'], conf_data['host'], conf_data[
+        'port']
 
 
 def rename_col(username, password, host, db_name, collection_name, old_name, new_name, port: int = 27017,):
@@ -679,30 +691,21 @@ class OptimizeDatas:
 
 
 def upload_one_dir():
-    username, password, host, port = get_myconf.select_config_values(target_service='home_lx', database='mongodb')
+    if socket.gethostname() == 'company' or socket.gethostname() == 'Mac2.local':
+        conf = myconfig.main()
+        conf_data = conf['Windows']['xigua_lx']['mysql']['remoto']
+        username, password, host, port = conf_data['username'], conf_data['password'], conf_data['host'], conf_data[
+            'port']
+    else:
+        conf = myconfig.main()
+        conf_data = conf['Windows']['company']['mysql']['remoto']
+        username, password, host, port = conf_data['username'], conf_data['password'], conf_data['host'], conf_data[
+            'port']
+
     p = UploadMongo(username=username, password=password, host=host, port=port, drop_duplicates=False)
     now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S ")
     print(f'{now}数据处理中...')
 
-    p.db_name = ''
-    p.collection_name = f''
-    path = os.path.join('C:\\同步空间', 'BaiduSyncdisk', '原始文件2', r'京东报表', 'JD流量来源')
-
-    for root, dirs, files in os.walk(path, topdown=False):
-        for name in files:
-            if '按天_' not in name:
-                continue
-            if name.endswith('.csv') and 'baidu' not in name:
-                df = pd.read_csv(os.path.join(root, name), encoding='utf-8_sig', header=0, na_filter=False)
-                for col in df.columns.tolist():
-                    if '日期' in col:
-                        df[col] = df[col].apply(lambda x: pd.to_datetime(x) if x else pd.to_datetime('2099-01-01'))
-                p.df_to_mongo(df=df)
-    if p.client:
-        p.client.close()
-    now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S ")
-    print(f'{now}数据完成!')
-
 
 def main():
     pass
@@ -710,7 +713,6 @@ def main():
 
 if __name__ == '__main__':
     # main()
-    username, password, host, port = get_myconf.select_config_values(target_service='home_lx', database='mongodb')
     print(username, password, host, port)
 
     # for db_name in [

mdbq/mysql/mysql.py

@@ -344,6 +344,221 @@ class MysqlUpload:
                     connection.rollback()
             connection.commit()
 
+    @try_except
+    def insert_many_dict(self, db_name, table_name, dict_data_list, icm_update=None, main_key=None, unique_main_key=None, index_length=100, set_typ=None, allow_not_null=False, cut_data=None):
+        """
+        插入字典数据
+        dict_data： 字典
+        main_key： 指定索引列, 通常用日期列，默认会设置日期为索引
+        unique_main_key： 指定唯一索引列
+        index_length: 索引长度
+        icm_update: 增量更正，指定后 main_key 只用于检查/创建列，不能更新数据
+        set_typ: {}
+        allow_not_null: 创建允许插入空值的列，正常情况下不允许空值
+        """
+        if not self.config:
+            return
+        if icm_update:
+            if main_key or unique_main_key:
+                print(f'icm_update/unique_main_key/unique_main_key 参数不能同时设定')
+                return
+        if not main_key:
+            main_key = []
+        if not unique_main_key:
+            unique_main_key = []
+
+        if not dict_data_list:
+            print(f'dict_data_list 不能为空 ')
+            return
+        dict_data = dict_data_list[0]
+        if cut_data:
+            if '日期' in dict_data.keys():
+                try:
+                    __y = pd.to_datetime(dict_data['日期']).strftime('%Y')
+                    __y_m = pd.to_datetime(dict_data['日期']).strftime('%Y-%m')
+                    if str(cut_data).lower() == 'year':
+                        table_name = f'{table_name}_{__y}'
+                    elif str(cut_data).lower() == 'month':
+                        table_name = f'{table_name}_{__y_m}'
+                    else:
+                        print(f'参数不正确，cut_data应为 year 或 month ')
+                except Exception as e:
+                    print(f'{table_name} 将数据按年/月保存(cut_data)，但在转换日期时报错 -> {e}')
+
+        # connection = pymysql.connect(**self.config)  # 连接数据库
+        connection = self.keep_connect(_db_name=db_name, _config=self.config, max_try=5)
+        if not connection:
+            return
+        with connection.cursor() as cursor:
+            cursor.execute(f"SHOW DATABASES LIKE '{db_name}'")  # 检查数据库是否存在
+            database_exists = cursor.fetchone()
+            if not database_exists:
+                # 如果数据库不存在，则新建
+                if '8.138.27' in str(self.host) or platform.system() == "Linux":  # 阿里云 mysql 低版本不支持 0900
+                    sql = f"CREATE DATABASE `{db_name}` COLLATE utf8mb4_unicode_ci"
+                    self.config.update({'charset': 'utf8mb4_unicode_ci'})
+                if '192.168.1.100' in str(self.host):
+                    sql = f"CREATE DATABASE `{db_name}`"
+                else:
+                    sql = f"CREATE DATABASE `{db_name}` COLLATE utf8mb4_0900_ai_ci"
+                cursor.execute(sql)
+                connection.commit()
+                print(f"创建Database: {db_name}")
+
+        self.config.update({'database': db_name})  # 添加更新 config 字段
+        # connection = pymysql.connect(**self.config)  # 重新连接数据库
+        connection = self.keep_connect(_db_name=db_name, _config=self.config, max_try=5)
+        if not connection:
+            return
+        with connection.cursor() as cursor:
+            # 1. 查询表, 不存在则创建一个空表
+            sql = "SHOW TABLES LIKE %s;"  # 有特殊字符不需转义
+            cursor.execute(sql, (table_name))
+            if not cursor.fetchone():
+                sql = f"CREATE TABLE IF NOT EXISTS `{table_name}` (id INT AUTO_INCREMENT PRIMARY KEY);"
+                cursor.execute(sql)
+                print(f'创建 mysql 表: {table_name}')
+
+            # 根据 dict_data 的值添加指定的数据类型
+            dtypes, dict_data = self.cover_dict_dtypes(dict_data=dict_data)  # {'店铺名称': 'varchar(100)',...}
+            if set_typ:
+                # 更新自定义的列数据类型
+                for k, v in dtypes.items():
+                    # 确保传进来的 set_typ 键存在于实际的 df 列才 update
+                    [dtypes.update({k: inside_v}) for inside_k, inside_v in set_typ.items() if k == inside_k]
+
+            # 检查列
+            sql = "SELECT COLUMN_NAME FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_SCHEMA = %s AND TABLE_NAME = %s;"
+            cursor.execute(sql, (db_name, table_name))
+            col_exist = [item['COLUMN_NAME'] for item in cursor.fetchall()]  # 已存在的所有列
+            col_not_exist = [col for col in dict_data.keys() if col not in col_exist]  # 不存在的列
+            # 不存在则新建列
+            if col_not_exist:  # 数据表中不存在的列
+                for col in col_not_exist:
+                    #  创建列，需转义
+                    if allow_not_null:
+                        sql = f"ALTER TABLE `{table_name}` ADD COLUMN `{col}` {dtypes[col]};"
+                    else:
+                        sql = f"ALTER TABLE `{table_name}` ADD COLUMN `{col}` {dtypes[col]} NOT NULL;"
+                    # sql = f"ALTER TABLE `{table_name}` ADD COLUMN `{col}` {dtypes[col]} NOT NULL;"
+                    # print(sql)
+                    cursor.execute(sql)
+                    print(f"添加列: {col}({dtypes[col]})")  # 添加列并指定数据类型
+
+                    if col in main_key or col == '日期':
+                        sql = f"CREATE INDEX index_name ON `{table_name}`(`{col}`);"
+                        print(f"设置为索引: {col}({dtypes[col]})")
+                        cursor.execute(sql)
+                    if col in unique_main_key:
+                        if dtypes[col] == 'mediumtext':
+                            sql = f"ALTER TABLE `{table_name}` ADD UNIQUE (`{col}`({index_length}))"
+                        else:
+                            sql = f"ALTER TABLE `{table_name}` ADD UNIQUE (`{col}`)"
+                        cursor.execute(sql)
+                    # if col in main_key or col in unique_main_key:
+                    #     sql = f"SHOW INDEXES FROM `{table_name}` WHERE `Column_name` = %s"
+                    #     cursor.execute(sql, (col))
+                    #     result = cursor.fetchone()  # 检查索引是否存在
+                    #     if not result:
+                    #         if col in main_key:
+                    #             sql = f"CREATE INDEX index_name ON `{table_name}`(`{col}`);"
+                    #             print(f"设置为索引: {col}({dtypes[col]})")
+                    #             cursor.execute(sql)
+                    #         elif col in unique_main_key:
+                    #             if dtypes[col] == 'mediumtext':
+                    #                 sql = f"CREATE INDEX UNIQUE index_name ON `{table_name}` (`{col}`({index_length}));"
+                    #             else:
+                    #                 sql = f"CREATE INDEX UNIQUE index_name ON `{table_name}` (`{col}`);"
+                    #             print(f"设置唯一索引: {col}({dtypes[col]})")
+                    #             print(sql)
+                    #             cursor.execute(sql)
+            connection.commit()  # 提交事务
+            """"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""
+            """"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""
+            # 处理插入的数据
+            for dict_data in dict_data_list:
+                dtypes, dict_data = self.cover_dict_dtypes(dict_data=dict_data)  # {'店铺名称': 'varchar(100)',...}
+                if icm_update:
+                    """ 使用增量更新: 需确保 icm_update['主键'] 传进来的列组合是数据表中唯一，值不会发生变化且不会重复，否则可能产生覆盖 """
+                    sql = 'SELECT COLUMN_NAME FROM information_schema.columns WHERE table_schema = %s AND table_name = %s'
+                    cursor.execute(sql, (db_name, {table_name}))
+                    columns = cursor.fetchall()
+                    cols_exist = [col['COLUMN_NAME'] for col in columns]  # 数据表的所有列, 返回 list
+                    update_col = [item for item in cols_exist if item not in icm_update and item != 'id']  # 除了主键外的其他列
+
+                    # unique_keys 示例: `日期`, `余额`
+                    unique_keys = ', '.join(f"`{item}`" for item in update_col)  # 列名需要转义
+                    condition = []
+                    for up_col in icm_update:
+                        condition += [f'`{up_col}` = "{dict_data[up_col]}"']
+                    condition = ' AND '.join(condition)  # condition值示例: `品销宝余额` = '2930.73' AND `短信剩余` = '67471'
+                    sql = f"SELECT {unique_keys} FROM `{table_name}` WHERE {condition}"
+                    # print(sql)
+                    # sql = f"SELECT {unique_keys} FROM `{table_name}` WHERE `创建时间` = '2014-09-19 14:32:33'"
+                    cursor.execute(sql)
+                    results = cursor.fetchall()  # results 是数据库取出的数据
+                    if results:  # 有数据返回，再进行增量检查
+                        for result in results:  # results 是数据库数据, dict_data 是传进来的数据
+                            change_col = []  # 发生变化的列名
+                            change_values = []  # 发生变化的数据
+                            for col in update_col:
+                                # 因为 mysql 里面有 decimal 数据类型，要移除末尾的 0 再做比较（df 默认将 5.00 小数截断为 5.0）
+                                df_value = str(dict_data[col])
+                                mysql_value = str(result[col])
+                                if '.' in df_value:
+                                    df_value = re.sub(r'0+$', '', df_value)
+                                    df_value = re.sub(r'\.$', '', df_value)
+                                if '.' in mysql_value:
+                                    mysql_value = re.sub(r'0+$', '', mysql_value)
+                                    mysql_value = re.sub(r'\.$', '', mysql_value)
+                                if df_value != mysql_value:  # 传进来的数据和数据库比较, 有变化
+                                    # print(f'{dict_data['日期']}{dict_data['商品id']}{col} 列的值有变化，{str(dict_data[col])}  !=  {str(result[col])}')
+                                    change_values += [f"`{col}` = \"{str(dict_data[col])}\""]
+                                    change_col.append(col)
+                            not_change_col = [item for item in update_col if item not in change_col]
+                            # change_values 是 df 传进来且和数据库对比后，发生了变化的数据，值示例： [`品销宝余额` = '9999.0', `短信剩余` = '888']
+                            if change_values:  # change_values 有数据返回，表示值需要更新
+                                if not_change_col:
+                                    not_change_values = [f'`{col}` = "{str(dict_data[col])}"' for col in not_change_col]
+                                    not_change_values = ' AND '.join(
+                                        not_change_values)  # 示例: `短信剩余` = '888' AND `test1` = '93'
+                                    # print(change_values, not_change_values)
+                                    condition += f' AND {not_change_values}'  # 重新构建完整的查询条件，将未发生变化的列加进查询条件
+                                change_values = ', '.join(f"{item}" for item in change_values)  # 注意这里 item 外面没有反引号
+                                sql = "UPDATE `%s` SET %s WHERE %s" % (table_name, change_values, condition)
+                                # print(sql)
+                                cursor.execute(sql)
+                    else:  # 没有数据返回，则直接插入数据
+                        cols = ', '.join(f"`{item}`" for item in dict_data.keys())  # 列名需要转义
+                        # data.update({item: f"{data[item]}" for item in data.keys()})  # 全部值转字符, 不是必须的
+                        values = ', '.join([f'"{item}"' for item in dict_data.values()])  # 值要加引号
+                        sql = f"INSERT INTO `{table_name}` ({cols}) VALUES ({values});"
+                        cursor.execute(sql)
+                    connection.commit()  # 提交数据库
+                    connection.close()
+                    return
+
+                # 构建 keys
+                keys_data = ', '.join([f'`{str(item)}`' for item in dict_data.keys()])
+                # 构建 values
+                values_data = ', '.join(f'"{str(item)}"' for item in dict_data.values())
+                # 构建其他键值，重复时要更新的其他键
+                if main_key:
+                    for col in main_key:
+                        del dict_data[col]
+                if unique_main_key:
+                    for col in unique_main_key:
+                        del dict_data[col]
+                # 涉及列名务必使用反引号
+                update_datas = ', '.join([f'`{k}` = VALUES(`{k}`)' for k, v in dict_data.items()])
+
+                # 构建 sql
+                sql = f"INSERT INTO %s (%s) VALUES (%s) ON DUPLICATE KEY UPDATE %s" % (table_name, keys_data, values_data, update_datas)
+                # print(sql)
+                cursor.execute(sql)
+                connection.commit()  # 提交数据库
+        connection.close()
+
     @try_except
     def dict_to_mysql(self, db_name, table_name, dict_data, icm_update=None, main_key=None, unique_main_key=None, index_length=100, set_typ=None, allow_not_null=False, cut_data=None):
         """
@@ -1086,7 +1301,6 @@ class MysqlUpload:
                         print(f'{now} 写入本地文件: ({self.host}:{self.port}) {db_name}/{table_name} -> {os.path.join(save_path, filename)}')
         connection.close()
 
-
     def read_mysql(self, table_name, start_date, end_date, db_name='远程数据源', date_name='日期'):
         """ 读取指定数据表，可指定日期范围，返回结果: df """
         start_date = pd.to_datetime(start_date).strftime('%Y-%m-%d')

mdbq/mysql/recheck_mysql.py

@@ -156,7 +156,7 @@ class ReCheckMysql:
 
 
 def recheck_csv():
-    path = '/Users/xigua/数据中心/原始文件2/推广报表34324234'
+    path = ''
     for root, dirs, files in os.walk(path, topdown=False):
         for name in files:
             if '~' in name or 'baidu' in name or 'Ds_' in name or 'xunlei' in name:

mdbq/mysql/s_query.py

@@ -11,7 +11,6 @@ import pandas as pd
 from sqlalchemy import create_engine
 import os
 import calendar
-from mdbq.config import get_myconf
 from mdbq.dataframe import converter
 
 warnings.filterwarnings('ignore')

mdbq/spider/aikucun.py

@@ -17,8 +17,6 @@ from selenium.webdriver.support import expected_conditions as EC
 from selenium.webdriver.chrome.service import Service
 from mdbq.config import set_support
 from selenium.webdriver.common.keys import Keys
-from mdbq.aggregation import aggregation
-from mdbq.clean import data_clean
 from mdbq.other import ua_sj
 from mdbq.mysql import mysql
 from mdbq.config import myconfig
@@ -46,7 +44,7 @@ else:
     D_PATH = str(pathlib.Path(f'/Users/{getpass.getuser()}/Downloads'))
     Share_Path = str(pathlib.Path('/Volumes/时尚事业部/01.运营部/天猫报表'))  # 共享文件根目录
     Source_Path = str(pathlib.Path(Data_Path, '原始文件2'))
-upload_path = os.path.join(D_PATH, '数据上传中心')  # 此目录位于下载文件夹
+upload_path = os.path.join(D_PATH, '数据上传中心', '爱库存')  # 此目录位于下载文件夹
 
 m_engine = mysql.MysqlUpload(username='', password='', host='', port=0, charset='utf8mb4')
 company_engine = mysql.MysqlUpload(username='', password='', host='', port=0, charset='utf8mb4')
@@ -403,53 +401,29 @@ class AikuCun:
                         '成交gmv': '',
                         '支付人数_成交': '',
                     }
+                    _results = []
                     for dict_data in df.to_dict(orient='records'):
                         new_dict.update(dict_data)
                         new_dict.update({'更新时间': datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')})
-                        m_engine.dict_to_mysql(
+                        _results.append(new_dict)
+                    if _results:
+                        m_engine.insert_many_dict(
                             db_name='爱库存2',
                             table_name='商品spu榜单',
-                            dict_data=new_dict,
+                            dict_data_list=_results,
                             icm_update=['日期', '店铺名称', 'spu_id', '商品款号'],
                             unique_main_key=None,
                             set_typ=set_typ,
                         )
-                        company_engine.dict_to_mysql(
+                        company_engine.insert_many_dict(
                             db_name='爱库存2',
                             table_name='商品spu榜单',
-                            dict_data=new_dict,
+                            dict_data_list=_results,
                             icm_update=['日期', '店铺名称', 'spu_id', '商品款号'],
                             unique_main_key=None,
                             set_typ=set_typ,
                         )
 
-                    # m_engine.df_to_mysql(
-                    #     df=df,
-                    #     db_name='爱库存2',
-                    #     table_name='商品spu榜单',
-                    #     icm_update=[],  # 增量更新, 在聚合数据中使用，其他不要用
-                    #     move_insert=False,  # 先删除，再插入
-                    #     df_sql=True,  # 值为 True 时使用 df.to_sql 函数上传整个表, 不会排重
-                    #     drop_duplicates=False,  # 值为 True 时检查重复数据再插入，反之直接上传，会比较慢
-                    #     count=None,
-                    #     filename=None,  # 用来追踪处理进度
-                    #     reset_id=False,  # 是否重置自增列
-                    #     set_typ=set_typ,
-                    # )
-                    # company_engine.df_to_mysql(
-                    #     df=df,
-                    #     db_name='爱库存2',
-                    #     table_name='商品spu榜单',
-                    #     icm_update=[],  # 增量更新, 在聚合数据中使用，其他不要用
-                    #     move_insert=False,  # 先删除，再插入
-                    #     df_sql=True,  # 值为 True 时使用 df.to_sql 函数上传整个表, 不会排重
-                    #     drop_duplicates=False,  # 值为 True 时检查重复数据再插入，反之直接上传，会比较慢
-                    #     count=None,
-                    #     filename=None,  # 用来追踪处理进度
-                    #     reset_id=False,  # 是否重置自增列
-                    #     set_typ=set_typ,
-                    # )
-
                     new_name = f'爱库存_商品榜单_spu_{date}_{date}.csv'
                     df.to_csv(os.path.join(root, new_name), encoding='utf-8_sig', index=False)
                     os.remove(os.path.join(root, name))
@@ -458,26 +432,6 @@ class AikuCun:
 def akucun(headless=True, date_num=10):
     akc = AikuCun()
     akc.get_data(shop_name='aikucun', date_num=date_num, headless=headless)  # 获取最近 N 天数据，0表示今天
-    # akc.clean_data()
-
-    # # 新版 数据分类
-    # dp = aggregation.DatabaseUpdate(path=upload_path)
-    # dp.new_unzip(is_move=True)
-    # dp.cleaning(is_move=False, is_except=['临时文件'])  # 清洗数据, 存入 self.datas, 不需要立即移除文件，仍保留文件到原始文件中
-    # # 将 self.datas 更新至数据库
-    # dp.upload_df(service_databases=[
-    #     # {'home_lx': 'mongodb'},
-    #     # {'home_lx': 'mysql'},
-    #     {'company': 'mysql'},
-    #     # {'nas': 'mysql'},
-    # ])
-    # # 数据分类
-    # c = data_clean.DataClean(path=upload_path, source_path=Source_Path)
-    # c.set_up_to_mogo = False  # 不再使用 data_clean 更新数据库，改为 aggregation.py
-    # c.set_up_to_mysql = False  # 不再使用 data_clean 更新数据库，改为 aggregation.py
-    # c.new_unzip(is_move=True, )  # 解压文件
-    # c.change_and_sort(is_except=['临时文件'])
-    # c.move_all(is_except=['临时文件'])  # 移到文件到原始文件夹
 
 
 class AikuCunNew:
@@ -529,7 +483,7 @@ class AikuCunNew:
 
 if __name__ == '__main__':
     get_cookie_aikucun()  # 登录并获取 cookies
-    akucun(date_num=10, headless=True)  # 下载数据
+    akucun(date_num=5, headless=True)  # 下载数据
 
     # a = AikuCunNew(shop_name='aikucun')
     # a.akc()

{mdbq-3.3.5.dist-info → mdbq-3.3.8.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: mdbq
-Version: 3.3.5
+Version: 3.3.8
 Home-page: https://pypi.org/project/mdbq
 Author: xigua, 
 Author-email: 2587125111@qq.com

{mdbq-3.3.5.dist-info → mdbq-3.3.8.dist-info}/RECORD

@@ -2,33 +2,24 @@ mdbq/__init__.py,sha256=Il5Q9ATdX8yXqVxtP_nYqUhExzxPC_qk_WXQ_4h0exg,16
 mdbq/__version__.py,sha256=y9Mp_8x0BCZSHsdLT_q5tX9wZwd5QgqrSIENLrb6vXA,62
 mdbq/aggregation/__init__.py,sha256=EeDqX2Aml6SPx8363J-v1lz0EcZtgwIBYyCJV6CcEDU,40
 mdbq/aggregation/aggregation.py,sha256=-yzApnlqSN2L0E1YMu5ml-W827qpKQvWPCOI7jj2kzY,80264
-mdbq/aggregation/df_types.py,sha256=U9i3q2eRPTDY8qAPTw7irzu-Tlg4CIySW9uYro81wdk,8125
-mdbq/aggregation/mysql_types.py,sha256=YTGyrF9vcRgfkQbpT-e-JdJ7c7VF1dDHgyx9YZRES8w,10934
 mdbq/aggregation/optimize_data.py,sha256=RXIv7cACCgYyehAxMjUYi_S7rVyjIwXKWMaM3nduGtA,3068
-mdbq/aggregation/query_data.py,sha256=_5mnSFHV6xAFs_1YF_H2zMOdJeMavgga4lZQ_qpqxPQ,167637
+mdbq/aggregation/query_data.py,sha256=CE5oKG9FNZ2I3kNuJtvcbGFFJqyKvO9T15XeXWCIc8s,166441
 mdbq/bdup/__init__.py,sha256=AkhsGk81SkG1c8FqDH5tRq-8MZmFobVbN60DTyukYTY,28
 mdbq/bdup/bdup.py,sha256=LAV0TgnQpc-LB-YuJthxb0U42_VkPidzQzAagan46lU,4234
-mdbq/clean/__init__.py,sha256=A1d6x3L27j4NtLgiFV5TANwEkLuaDfPHDQNrPBbNWtU,41
-mdbq/clean/clean_upload.py,sha256=yMAb6tV9XHhFJbRrCOeaPfszApJ9y5M4-hQGuBSXNqE,67799
-mdbq/clean/data_clean.py,sha256=ucfslhqXVZoH2QaXHSAWDky0GhIvH9f4GeNaHg4SrFE,104790
-mdbq/company/__init__.py,sha256=qz8F_GsP_pMB5PblgJAUAMjasuZbOEp3qQOCB39E8f0,21
-mdbq/company/copysh.py,sha256=eFu6focRqm2Njn_XN1KW2ZYJiTv6EYgsdBCLokobyxQ,21572
 mdbq/config/__init__.py,sha256=jso1oHcy6cJEfa7udS_9uO5X6kZLoPBF8l3wCYmr5dM,18
-mdbq/config/get_myconf.py,sha256=cmNvsyoNa0RbZ9FOTjSd3jyyGwkxjUo0phvdHbGlrms,6010
 mdbq/config/myconfig.py,sha256=EGymTlAimtHIDJ9egCtOehBEPOj6rea504kvsEZu64o,854
 mdbq/config/products.py,sha256=Ab6eaAUMUtjRL8z9NvYukyCjp3nAi4OYISY_IdPhAJ0,6279
 mdbq/config/set_support.py,sha256=xkZCX6y9Bq1ppBpJAofld4B2YtchA7fl0eT3dx3CrSI,777
-mdbq/config/update_conf.py,sha256=taL3ZqKgiVWwUrDFuaYhim9a72Hm4BHRhhDscJTziR8,4535
 mdbq/dataframe/__init__.py,sha256=2HtCN8AdRj53teXDqzysC1h8aPL-mMFy561ESmhehGQ,22
 mdbq/dataframe/converter.py,sha256=lETYhT7KXlWzWwqguqhk6vI6kj4rnOBEW1lhqKy2Abc,5035
 mdbq/log/__init__.py,sha256=Mpbrav0s0ifLL7lVDAuePEi1hJKiSHhxcv1byBKDl5E,15
 mdbq/log/mylogger.py,sha256=oaT7Bp-Hb9jZt52seP3ISUuxVcI19s4UiqTeouScBO0,3258
 mdbq/mongo/__init__.py,sha256=SILt7xMtQIQl_m-ik9WLtJSXIVf424iYgCfE_tnQFbw,13
-mdbq/mongo/mongo.py,sha256=v9qvrp6p1ZRWuPpbSilqveiE0FEcZF7U5xUPI0RN4xs,31880
+mdbq/mongo/mongo.py,sha256=M9DUeUCMPDngkwn9-ui0uTiFrvfNU1kLs22s5SmoNm0,31899
 mdbq/mysql/__init__.py,sha256=A_DPJyAoEvTSFojiI2e94zP0FKtCkkwKP1kYUCSyQzo,11
-mdbq/mysql/mysql.py,sha256=ZK6E-idQWrURtoimc0uG8B1tnhtSFQXDJHfu8sWeJg4,85675
-mdbq/mysql/recheck_mysql.py,sha256=rgTpvDMWYTyEn7UQdlig-pdXDluTgiU8JG6lkMh8DV0,8665
-mdbq/mysql/s_query.py,sha256=MbIprZ4yJDAZ9AahZPzl7hqS695Vs0P-AJNwAtA_EEc,9287
+mdbq/mysql/mysql.py,sha256=Ekqg-svOi2kbPIBBoEtL9Eg4-LJqyqoLprSSUy17MWU,99360
+mdbq/mysql/recheck_mysql.py,sha256=ppBTfBLgkRWirMVZ31e_ZPULiGPJU7K3PP9G6QBZ3QI,8605
+mdbq/mysql/s_query.py,sha256=6L5Cp90zq13noZHjzSA5mqms_hD01c8GO1_NfbYDu6w,9252
 mdbq/mysql/year_month_day.py,sha256=VgewoE2pJxK7ErjfviL_SMTN77ki8GVbTUcao3vFUCE,1523
 mdbq/other/__init__.py,sha256=jso1oHcy6cJEfa7udS_9uO5X6kZLoPBF8l3wCYmr5dM,18
 mdbq/other/download_sku_picture.py,sha256=Z2Crtwsv69nMGN8uw7qea_1HghRwil3vfnczBM6Ab80,45879
@@ -40,11 +31,9 @@ mdbq/pbix/__init__.py,sha256=Trtfaynu9RjoTyLLYBN2xdRxTvm_zhCniUkVTAYwcjo,24
 mdbq/pbix/pbix_refresh.py,sha256=JUjKW3bNEyoMVfVfo77UhguvS5AWkixvVhDbw4_MHco,2396
 mdbq/pbix/refresh_all.py,sha256=OBT9EewSZ0aRS9vL_FflVn74d4l2G00wzHiikCC4TC0,5926
 mdbq/pbix/refresh_all_old.py,sha256=_pq3WSQ728GPtEG5pfsZI2uTJhU8D6ra-htIk1JXYzw,7192
-mdbq/req_post/__init__.py,sha256=jso1oHcy6cJEfa7udS_9uO5X6kZLoPBF8l3wCYmr5dM,18
-mdbq/req_post/req_tb.py,sha256=qg7pet73IgKGmCwxaeUyImJIoeK_pBQT9BBKD7fkBNg,36160
 mdbq/spider/__init__.py,sha256=RBMFXGy_jd1HXZhngB2T2XTvJqki8P_Fr-pBcwijnew,18
-mdbq/spider/aikucun.py,sha256=BKVa0xbTkyhIH5kQgOdyPDtwFPScbMNAFYup_-fFF9Y,24809
-mdbq-3.3.5.dist-info/METADATA,sha256=ndDUyzoqoEH--Mhw_0Z7xXEPTwWuLyBkT8jsfUx8AxA,243
-mdbq-3.3.5.dist-info/WHEEL,sha256=eOLhNAGa2EW3wWl_TU484h7q1UNgy0JXjjoqKoxAAQc,92
-mdbq-3.3.5.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
-mdbq-3.3.5.dist-info/RECORD,,
+mdbq/spider/aikucun.py,sha256=zOacjrJ3MvToyuugA68xB-oN6RKj8K3GxMKudnln9EA,22207
+mdbq-3.3.8.dist-info/METADATA,sha256=MAs4cq1MyNeUOHWp9orMnoBXaLmXGK_jryKf_-G_134,243
+mdbq-3.3.8.dist-info/WHEEL,sha256=eOLhNAGa2EW3wWl_TU484h7q1UNgy0JXjjoqKoxAAQc,92
+mdbq-3.3.8.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
+mdbq-3.3.8.dist-info/RECORD,,