mdbq 2.8.9__py3-none-any.whl → 2.9.2__py3-none-any.whl

mdbq/clean/clean_upload.py

@@ -128,6 +128,11 @@ class DataClean:
                 '数据库名': '生意参谋3',
                 '集合名称': '直播分场次效果',
             },
+            {
+                '文件简称': 'crm_客户列表_',  # 文件名中包含的字符
+                '数据库名': '生意参谋3',
+                '集合名称': 'crm成交客户',
+            },
         ]
         for root, dirs, files in os.walk(path, topdown=False):
             for name in files:
@@ -180,6 +185,8 @@ class DataClean:
                     # new_name = f'py_xg_{os.path.splitext(name)[0]}.csv'
                     # self.save_to_csv(df, root, new_name, encoding='utf-8_sig')
                     # os.remove(os.path.join(root, name))
+                elif name.endswith('.csv') and 'crm_客户列表' in name:
+                    df = pd.read_csv(os.path.join(root, name), encoding='utf-8_sig', header=0, na_filter=False)
 
                 # 将数据传入 self.datas 等待更新进数据库
                 if not db_name or not collection_name:
@@ -209,6 +216,11 @@ class DataClean:
                 '数据库名': '达摩盘3',
                 '集合名称': 'dmp人群报表',
             },
+            {
+                '文件简称': '货品洞察_全店单品',  # 文件名中包含的字符
+                '数据库名': '达摩盘3',
+                '集合名称': '货品洞察_全店单品',
+            },
         ]
         for root, dirs, files in os.walk(path, topdown=False):
             for name in files:
@@ -239,6 +251,8 @@ class DataClean:
                     df = pd.read_csv(os.path.join(root, name), encoding='utf-8_sig', header=0, na_filter=False)
                 elif name.endswith('.csv') and 'dmp人群报表_' in name:
                     df = pd.read_csv(os.path.join(root, name), encoding='utf-8_sig', header=0, na_filter=False)
+                elif name.endswith('.csv') and '货品洞察_全店单品' in name:
+                    df = pd.read_csv(os.path.join(root, name), encoding='utf-8_sig', header=0, na_filter=False)
 
                 # 将数据传入 self.datas 等待更新进数据库
                 if not db_name or not collection_name:
@@ -831,6 +845,9 @@ class DataClean:
                 elif name.endswith('.csv') and 'dmp人群报表_' in name:
                     t_path = os.path.join(self.source_path, '达摩盘', 'dmp人群报表')
                     bib(t_path, _as_month=True)
+                elif name.endswith('.csv') and '货品洞察_全店单品' in name:
+                    t_path = os.path.join(self.source_path, '达摩盘', '货品洞察')
+                    bib(t_path, _as_month=True)
 
     # @try_except
     def move_sjy(self, path=None, is_except=[]):
@@ -1283,7 +1300,11 @@ def main(is_mysql=False, is_company=False):
     cn.sp_scene_clean(is_except=['except'])  # 商品素材
     cn.upload_df()  # 上传数据库
 
-    if not is_company:
+    if is_company:  # 公司移除所有文件
+        files = os.listdir(upload_path)
+        for file in files:
+            os.remove(os.path.join(upload_path, file))
+    else:  # 其他主机则进行文件分类
         cn.move_sycm(is_except=['临时文件', ])  # 生意参谋，移到文件到原始文件夹
         cn.move_dmp(is_except=['临时文件', ])  # 达摩盘
         cn.move_sjy(is_except=['临时文件',])  # 生意经，移到文件到原始文件夹

mdbq/company/copysh.py

@@ -304,7 +304,7 @@ class TbFiles:
     def refresh_excel(self):
         # 刷新共享位置的指定文件/文件夹
         if platform.system() == 'Windows' and socket.gethostname() == 'company':
-            excel_path = os.path.join(self.share_path, 'EXCEL报表')
+            excel_path = os.path.join(Share_Path, 'EXCEL报表')
             files = os.listdir(excel_path)
             files = [f'{excel_path}\\{item}' for item in files if item.endswith('.xlsx') or item.endswith('.xls')]
             r = refresh_all.RefreshAll()

mdbq/dataframe/converter.py

@@ -52,7 +52,9 @@ class DataFrameConverter(object):
             except Exception as e:
                 print(f'留意错误信息: 位于列 -> {col} -> {e}')
 
-            if col.endswith('占比') or col.endswith('率'):
+            if (col.endswith('占比') or col.endswith('率') or col.endswith('同比')
+                    or col.endswith('环比') or col.lower().endswith('roi')
+                    or col.endswith('产出比')):
                 df = df.astype({col: float}, errors='raise')
 
             # 尝试转换合适的数据类型

mdbq/mysql/mysql.py

@@ -41,6 +41,63 @@ warnings.filterwarnings('ignore')
 """
 
 
+def is_valid_date(date_string):
+    """ 判断是否是日期格式, 且允许没有前导零, 且允许带时间 """
+    date_pattern = r"^(\d{4})-(0?[1-9]|1[0-2])-(0?[1-9]|[12]\d|3[01])$"
+    match = re.match(date_pattern, str(date_string))  # 判断纯日期格式：2024-11-09
+    if match is None:
+        date_pattern = r".*\d+:\d+:\d+$"
+        match = re.match(date_pattern, date_string)  # 判断日期+时间：2024-11-09 00:36:45
+        if match is not None:
+            return 2
+    else:
+        return 1
+
+
+def is_integer(int_str):
+    """ 判断是否整数, 允许包含千分位分隔符, 允许科学计数法 """
+    # 如果是科学计数法
+    match = re.findall(r'^[-+]?(\d+)\.(\d+)[eE][-+]?(\d+)$', str(int_str))
+    if match:
+        if len(match[0]) == 3:
+            if int(match[0][0]) == 0:  # 0 开头
+                if int(match[0][2]) > 10:  # 转换后整数长度超过 10 位
+                    return False
+            else:  # 不是 0 开头
+                if len(match[0][0]) + int(match[0][2]) > 10:  # 转换后整数长度超过 10 位
+                    return False
+            if int(match[0][2]) >= len(match[0][1]):
+                return True
+            else:
+                return False
+    # 如果是普通数字, 且允许千分符
+    __pattern = r'^[-+]?\d{1,3}(,\d{3}){0,3}$|^[-+]?\d{1,9}$'
+    return re.match(__pattern, str(int_str)) is not None
+
+
+def count_decimal_places(num_str):
+    """ 计算小数位数, 允许科学计数法 """
+    match = re.match(r'^[-+]?\d+(\.\d+)?([eE][-+]?\d+)?$', str(num_str))
+    if match:
+        # 如果是科学计数法
+        match = re.findall(r'(\d+)\.(\d+)[eE][-+]?(\d+)$', str(num_str))
+        if match:
+            if len(match[0]) == 3:
+                if int(match[0][2]) < len(match[0][1]):
+                    # count_int 清除整数部分开头的 0 并计算整数位数
+                    count_int = len(re.sub('^0+', '', str(match[0][0]))) + int(match[0][2])
+                    # 计算小数位数
+                    count_float = len(match[0][1]) - int(match[0][2])
+                    return count_int, count_float
+        # 如果是普通小数
+        match = re.findall(r'(\d+)\.(\d+)$', str(num_str))
+        if match:
+            count_int = len(re.sub('^0+', '', str(match[0][0])))
+            count_float = len(match[0][1])
+            return count_int, count_float  # 计算小数位数
+    return 0, 0
+
+
 class MysqlUpload:
     def __init__(self, username: str, password: str, host: str, port: int, charset: str = 'utf8mb4'):
         self.username = username
@@ -68,6 +125,127 @@ class MysqlUpload:
 
         return wrapper
 
+    def dict_to_mysql(self, db_name, table_name, dict_data, main_key=None, unique_main_key=None, index_length=100):
+        """
+        插入字典数据
+        dict_data： 字典
+        main_key： 指定索引列
+        unique_main_key： 指定唯一索引列
+        index_length: 索引长度
+        """
+
+        if not main_key:
+            main_key = []
+        if not unique_main_key:
+            unique_main_key = []
+        connection = pymysql.connect(**self.config)  # 连接数据库
+        with connection.cursor() as cursor:
+            cursor.execute(f"SHOW DATABASES LIKE '{db_name}'")  # 检查数据库是否存在
+            database_exists = cursor.fetchone()
+            if not database_exists:
+                # 如果数据库不存在，则新建
+                if '8.138.27' in str(self.host) or platform.system() == "Linux":  # 阿里云 mysql 低版本不支持 0900
+                    sql = f"CREATE DATABASE `{db_name}` COLLATE utf8mb4_unicode_ci"
+                    self.config.update({'charset': 'utf8mb4_unicode_ci'})
+                if '192.168.1.100' in str(self.host):
+                    sql = f"CREATE DATABASE `{db_name}`"
+                else:
+                    sql = f"CREATE DATABASE `{db_name}` COLLATE utf8mb4_0900_ai_ci"
+                cursor.execute(sql)
+                connection.commit()
+                print(f"创建Database: {db_name}")
+
+        self.config.update({'database': db_name})  # 添加更新 config 字段
+        connection = pymysql.connect(**self.config)  # 重新连接数据库
+        with connection.cursor() as cursor:
+            # 1. 查询表, 不存在则创建一个空表
+            sql = "SHOW TABLES LIKE %s;"  # 有特殊字符不需转义
+            cursor.execute(sql, (table_name))
+            if not cursor.fetchone():
+                sql = f"CREATE TABLE IF NOT EXISTS `{table_name}` (id INT AUTO_INCREMENT PRIMARY KEY);"
+                cursor.execute(sql)
+                print(f'创建 mysql 表: {table_name}')
+
+            # 根据 dict_data 的值添加指定的数据类型
+            dtypes = self.cover_dict_dtypes(dict_data=dict_data)  # {'店铺名称': 'mediumtext',...}
+            # 检查列
+            sql = "SELECT COLUMN_NAME FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_SCHEMA = %s AND TABLE_NAME = %s;"
+            cursor.execute(sql, (db_name, table_name))
+            col_exist = [item['COLUMN_NAME'] for item in cursor.fetchall()]  # 已存在的所有列
+            col_not_exist = [col for col in dict_data.keys() if col not in col_exist]  # 不存在的列
+            # 不存在则新建列
+            if col_not_exist:  # 数据表中不存在的列
+                for col in col_not_exist:
+                    #  创建列，需转义
+                    sql = f"ALTER TABLE `{table_name}` ADD COLUMN `{col}` {dtypes[col]} NOT NULL;"
+                    cursor.execute(sql)
+                    print(f"添加列: {col}({dtypes[col]})")  # 添加列并指定数据类型
+
+                    if col in main_key or col in unique_main_key:
+                        sql = f"SHOW INDEXES FROM `{table_name}` WHERE `Column_name` = %s"
+                        cursor.execute(sql, (col))
+                        result = cursor.fetchone()  # 检查索引是否存在
+                        if not result:
+                            if col in main_key:
+                                cursor.execute(f"CREATE INDEX index_name ON `{table_name}`(`{col}`({index_length}))")
+                                print(f"创建索引列: {col}({dtypes[col]})")  # 添加列并指定数据类型
+                            elif col in unique_main_key:
+                                cursor.execute(f"CREATE UNIQUE INDEX index_name ON `{table_name}`(`{col}`({index_length}))")
+                                print(f"创建唯一索引: {col}({dtypes[col]})")  # 添加列并指定数据类型
+            connection.commit()  # 提交事务
+
+            keys_data = ', '.join([f'`{str(item)}`' for item in dict_data.keys()])
+            values_data = ', '.join(f'"{str(item)}"' for item in dict_data.values())
+            del dict_data['客户id']
+            update_datas = ', '.join([f'{k} = VALUES({k})' for k, v in dict_data.items()])
+            # print(update_datas)
+            sql = f"INSERT INTO %s (%s) VALUES (%s) ON DUPLICATE KEY UPDATE %s" % (table_name, keys_data, values_data, update_datas)
+            # print(sql)
+            cursor.execute(sql)
+            connection.commit()  # 提交事务
+        connection.close()
+
+
+    def cover_dict_dtypes(self, dict_data):
+        if not dict_data:
+            print(f'mysql.py -> MysqlUpload -> cover_dict_dtypes -> 传入的字典不能为空')
+            return
+        __res_dict = {}
+        for k, v in dict_data.items():
+            result1 = re.findall(r'商品编码|[商品宝贝]_?id|s[kp]u_?id|货号|款号|文件大小', str(k), re.IGNORECASE)
+            result2 = re.findall(r'占比|投产|产出|同比|环比|roi$', str(k), re.IGNORECASE)
+            date_type = is_valid_date(str(v))  # 判断日期时间
+            int_num = is_integer(str(v))  # 判断整数
+            count_int, count_float = count_decimal_places(str(v))  # 判断小数，返回小数位数
+            if result1:  # 京东sku/spu商品信息
+                __res_dict.update({k: 'mediumtext'})
+            elif result2:
+                __res_dict.update({k: 'decimal(10,2)'})
+            elif str(v) == '' or str(v).lower() == 'nan' or str(v).lower() == 'null':
+                v = 0
+                dict_data.update({k: v})
+                __res_dict.update({k: 'mediumtext'})
+            elif date_type == 1:
+                __res_dict.update({k: 'DATE'})
+            elif date_type == 2:
+                __res_dict.update({k: 'DATETIME'})
+            elif int_num:
+                __res_dict.update({k: 'INT'})
+            elif count_float > 0:
+                if count_int + count_float > 10:
+                    __res_dict.update({k: 'mediumtext'})
+                elif count_float >= 6:
+                    __res_dict.update({k: 'decimal(12,6)'})
+                elif count_float >= 4:
+                    __res_dict.update({k: 'decimal(10,4)'})
+                else:
+                    __res_dict.update({k: 'decimal(10,2)'})
+            else:
+                __res_dict.update({k: 'mediumtext'})
+        return __res_dict
+
+
+
     @try_except
     def df_to_mysql(self, df, table_name, db_name='远程数据源', icm_update=[], service_database={'xigua_lx': 'mysql'}, move_insert=False, df_sql=False, drop_duplicates=False, filename=None, count=None, json_path=None, reset_id=False):
         """

{mdbq-2.8.9.dist-info → mdbq-2.9.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: mdbq
-Version: 2.8.9
+Version: 2.9.2
 Home-page: https://pypi.org/project/mdbq
 Author: xigua, 
 Author-email: 2587125111@qq.com

{mdbq-2.8.9.dist-info → mdbq-2.9.2.dist-info}/RECORD

@@ -9,10 +9,10 @@ mdbq/aggregation/query_data.py,sha256=iRgPljgOPE7dzhaaVxRXOEOOKQTmWg6sGsDplNLTvQ
 mdbq/bdup/__init__.py,sha256=AkhsGk81SkG1c8FqDH5tRq-8MZmFobVbN60DTyukYTY,28
 mdbq/bdup/bdup.py,sha256=LAV0TgnQpc-LB-YuJthxb0U42_VkPidzQzAagan46lU,4234
 mdbq/clean/__init__.py,sha256=A1d6x3L27j4NtLgiFV5TANwEkLuaDfPHDQNrPBbNWtU,41
-mdbq/clean/clean_upload.py,sha256=FJxoEX-2QKuFhrF1ecl_LdZ1uFnVPx4HigNcXdErB28,66561
+mdbq/clean/clean_upload.py,sha256=yMAb6tV9XHhFJbRrCOeaPfszApJ9y5M4-hQGuBSXNqE,67799
 mdbq/clean/data_clean.py,sha256=ucfslhqXVZoH2QaXHSAWDky0GhIvH9f4GeNaHg4SrFE,104790
 mdbq/company/__init__.py,sha256=qz8F_GsP_pMB5PblgJAUAMjasuZbOEp3qQOCB39E8f0,21
-mdbq/company/copysh.py,sha256=UD5BLBe9uMfqjdslyY7-TtGXuJI5jsol-w4kIQFDfQk,21577
+mdbq/company/copysh.py,sha256=eFu6focRqm2Njn_XN1KW2ZYJiTv6EYgsdBCLokobyxQ,21572
 mdbq/company/copysh_bak.py,sha256=NvlXCBZBcO2GIT5nLRYYqhOyHWM1-1RE7DHvgbj6jmQ,19723
 mdbq/company/home_sh.py,sha256=42CZ2tZIXHLl2mOl2gk2fZnjH2IHh1VJ1s3qHABjonY,18021
 mdbq/config/__init__.py,sha256=jso1oHcy6cJEfa7udS_9uO5X6kZLoPBF8l3wCYmr5dM,18
@@ -22,13 +22,13 @@ mdbq/config/products.py,sha256=Sj4FSb2dZcMKp6ox-FJdIR87QLgMN_TJ7Z6KAWMTWyw,6214
 mdbq/config/set_support.py,sha256=xkZCX6y9Bq1ppBpJAofld4B2YtchA7fl0eT3dx3CrSI,777
 mdbq/config/update_conf.py,sha256=taL3ZqKgiVWwUrDFuaYhim9a72Hm4BHRhhDscJTziR8,4535
 mdbq/dataframe/__init__.py,sha256=2HtCN8AdRj53teXDqzysC1h8aPL-mMFy561ESmhehGQ,22
-mdbq/dataframe/converter.py,sha256=X5Aubm9Z4_bhslcu1-XZzT8X6UpoAW5BFs30RfgfRmE,4460
+mdbq/dataframe/converter.py,sha256=s-5EzHU9HlcSHlKgRd3autGODaKn523av1N0gh_56YY,4614
 mdbq/log/__init__.py,sha256=Mpbrav0s0ifLL7lVDAuePEi1hJKiSHhxcv1byBKDl5E,15
 mdbq/log/mylogger.py,sha256=oaT7Bp-Hb9jZt52seP3ISUuxVcI19s4UiqTeouScBO0,3258
 mdbq/mongo/__init__.py,sha256=SILt7xMtQIQl_m-ik9WLtJSXIVf424iYgCfE_tnQFbw,13
 mdbq/mongo/mongo.py,sha256=v9qvrp6p1ZRWuPpbSilqveiE0FEcZF7U5xUPI0RN4xs,31880
 mdbq/mysql/__init__.py,sha256=A_DPJyAoEvTSFojiI2e94zP0FKtCkkwKP1kYUCSyQzo,11
-mdbq/mysql/mysql.py,sha256=apcj0WDdbrHr7UzO2kjcesDxDUlWxG4KcIpI1mBuwMk,46152
+mdbq/mysql/mysql.py,sha256=jq2_lQFTnQR6N6QdSqxKqjennazITh5TdTg6j7oerYE,55006
 mdbq/mysql/recheck_mysql.py,sha256=jHQSlQy0PlQ_EYICQv_2nairUX3t6OIwPtSELKIpjkY,8702
 mdbq/mysql/s_query.py,sha256=bgNNIqYLDCHjD5KTFcm6x4u74selpAGs5ouJYuqX86k,8447
 mdbq/mysql/year_month_day.py,sha256=VgewoE2pJxK7ErjfviL_SMTN77ki8GVbTUcao3vFUCE,1523
@@ -45,7 +45,7 @@ mdbq/req_post/__init__.py,sha256=jso1oHcy6cJEfa7udS_9uO5X6kZLoPBF8l3wCYmr5dM,18
 mdbq/req_post/req_tb.py,sha256=PexWSCPJNM6Tv0ol4lAWIhlOwsAr_frnjtcdSHCFiek,36179
 mdbq/spider/__init__.py,sha256=RBMFXGy_jd1HXZhngB2T2XTvJqki8P_Fr-pBcwijnew,18
 mdbq/spider/aikucun.py,sha256=jHrdGWBJQaSywx7V-U4YuM6vWkwC5SR5tTOOdB3YU_c,17306
-mdbq-2.8.9.dist-info/METADATA,sha256=2_jXKvIIyj0iXk8T4uADVdvpKKF_kxFoWlN75PLoQlo,243
-mdbq-2.8.9.dist-info/WHEEL,sha256=cpQTJ5IWu9CdaPViMhC9YzF8gZuS5-vlfoFihTBC86A,91
-mdbq-2.8.9.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
-mdbq-2.8.9.dist-info/RECORD,,
+mdbq-2.9.2.dist-info/METADATA,sha256=3_u-jNgpFq2qX6uv2ufoYgPirGomwoz9IZkYmRp1BIc,243
+mdbq-2.9.2.dist-info/WHEEL,sha256=cpQTJ5IWu9CdaPViMhC9YzF8gZuS5-vlfoFihTBC86A,91
+mdbq-2.9.2.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
+mdbq-2.9.2.dist-info/RECORD,,