mdbq 1.1.0__tar.gz → 1.1.2__tar.gz

{mdbq-1.1.0 → mdbq-1.1.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: mdbq
-Version: 1.1.0
+Version: 1.1.2
 Home-page: https://pypi.org/project/mdbsql
 Author: xigua, 
 Author-email: 2587125111@qq.com

{mdbq-1.1.0 → mdbq-1.1.2}/mdbq/aggregation/aggregation.py

@@ -874,12 +874,16 @@ def upload(path, db_name, collection_name):
         collection_name=collection_name,
     )
     # print(dtypes)
+    count = 0
+    for root, dirs, files in os.walk(path, topdown=False):
+        for name in files:
+            count += 1
+    i = 0  # 用来统计当前处理文件进度
     for root, dirs, files in os.walk(path, topdown=False):
         for name in files:
             if '~$' in name or '.DS' in name or '.localized' in name or 'baidu' in name:
                 continue
             if name.endswith('.csv'):
-                # print(name)
                 try:
                     df = pd.read_csv(os.path.join(root, name), encoding='utf-8_sig', header=0, na_filter=False)
                     if len(df) == 0:
@@ -896,10 +900,11 @@ def upload(path, db_name, collection_name):
                         df = df.astype(dtypes)
 
                     # d.df_to_mongo(df=df, db_name=db_name, collection_name=collection_name)
-                    m.df_to_mysql(df=df, db_name=db_name, table_name=collection_name)
+                    m.df_to_mysql(df=df, db_name=db_name, table_name=collection_name, filename=name, count=f'{i}/{count}')
                     # nas.df_to_mysql(df=df, db_name=db_name, table_name=collection_name)
                 except Exception as e:
                     print(name, e)
+            i += 1
     if d.client:
         d.client.close()  # 必须手动关闭数据库连接
 
@@ -920,7 +925,7 @@ if __name__ == '__main__':
     # username, password, host, port = get_myconf.select_config_values(target_service='nas', database='mysql')
     # print(username, password, host, port)
     upload(
-        path='/Users/xigua/数据中心/原始文件2/属性设置/商品素材',
-        db_name = '属性设置2',
-        collection_name = '商品素材导出',
+        path='/Users/xigua/数据中心/原始文件2/生意经/宝贝指标',
+        db_name = '生意经2',
+        collection_name = '宝贝指标',
     )

{mdbq-1.1.0 → mdbq-1.1.2}/mdbq/aggregation/query_data.py

@@ -373,6 +373,7 @@ class GroupBy:
             print(f'<{table_name}>: Groupby 类尚未配置，数据为空')
             return pd.DataFrame({})
 
+    @try_except
     def performance(self, bb_tg=True):
          # print(self.data_tgyj)
         tg, syj, idbm, pic, cost = (

{mdbq-1.1.0 → mdbq-1.1.2}/mdbq/config/get_myconf.py

@@ -62,7 +62,7 @@ def select_config_values(target_service, database, path=None):
             if socket.gethostname() == 'xigua_lx':
                 # 本机自身运行使用 127.0.0.1
                 options = ['username_mysql_lx_nw', 'password_mysql_lx_nw', 'host_bd',  'port_mysql_lx_nw',]
-            elif socket.gethostname() == 'xigua1' or socket.gethostname() == 'macbook pro':
+            elif socket.gethostname() == 'xigua1' or socket.gethostname() == 'MacBook-Pro':
                 # 内网地址：正在运行的是 家里笔记本或者台式机，或者 macb    ook pro
                 options = ['username_mysql_lx_nw', 'password_mysql_lx_nw', 'host_mysql_lx_nw',  'port_mysql_lx_nw',]
             else:
@@ -126,6 +126,6 @@ def main():
 
 if __name__ == '__main__':
     # main()
-    r, d, s, g = select_config_values(target_service='home_lx', database='mongodb')
+    r, d, s, g = select_config_values(target_service='home_lx', database='mysql')
     print(r, d, s, g, type(r), type(d), type(s), type(g))
     print(f'本机: {platform.system()} // {socket.gethostname()}')

{mdbq-1.1.0 → mdbq-1.1.2}/mdbq/mysql/mysql.py

@@ -35,14 +35,16 @@ class MysqlUpload:
             'charset': charset,  # utf8mb4 支持存储四字节的UTF-8字符集
             'cursorclass': pymysql.cursors.DictCursor,
         }
+        self.filename = None
 
-    def df_to_mysql(self, df, table_name, db_name='远程数据源', drop_duplicates=False):
+    def df_to_mysql(self, df, table_name, db_name='远程数据源', drop_duplicates=False, filename=None, count=None):
         """
         将 df 写入数据库
         db_name: 数据库名称
         table_name: 集合/表名称
         drop_duplicates：仅限于聚合数据使用，其他情况不要设置
         """
+        self.filename = filename
         if isinstance(df, pd.DataFrame):
             if len(df) == 0:
                 print(f'{db_name}: {table_name} 传入的 df 数据长度为0')
@@ -95,7 +97,7 @@ class MysqlUpload:
                 for col in col_not_exist:
                     try:
                         #  创建列，需转义
-                        sql = f"ALTER TABLE `{table_name}` ADD COLUMN `{col}` {dtypes[col]} DEFAULT NULL;"
+                        sql = f"ALTER TABLE `{table_name}` ADD COLUMN `{col}` {dtypes[col]};"
                         cursor.execute(sql)
                         print(f"添加列: {col}({dtypes[col]})")  # 添加列并指定数据类型
 
@@ -105,8 +107,8 @@ class MysqlUpload:
                             result = cursor.fetchone()  # 检查索引是否存在
                             if not result:
                                 cursor.execute(f"CREATE INDEX index_name ON `{table_name}`(`{col}`)")
-                    except:
-                        pass
+                    except Exception as e:
+                        print(f'{self.filename}: {e}')
             connection.commit()  # 提交事务
 
             # 返回这些结果的目的是等添加完列再写 json 文件才能读到 types 信息
@@ -128,7 +130,7 @@ class MysqlUpload:
 
             # 5. 更新插入数据
             now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S ")
-            print(f'{now}正在更新 mysql ({self.host}:{self.port}) {db_name}/{table_name}')
+            print(f'{now}正在更新 mysql ({self.host}:{self.port}) {db_name}/{table_name}, {count},{self.filename}')
             datas = df.to_dict(orient='records')
             for data in datas:
                 try:
@@ -143,11 +145,14 @@ class MysqlUpload:
 
                     sql = f"SELECT {cols} FROM `{table_name}` WHERE {condition}"
                     cursor.execute(sql)
-                    result = cursor.fetchall()  # 获取查询结果, 如果有结果返回 list，没有则返回空元组 tuple
+                    result = cursor.fetchall()  # 获取查询结果, 有结果返回 list 表示数据已存在(不重复插入)，没有则返回空 tuple
                     if not result:  # 数据不存在则插入
                         sql = f"INSERT INTO `{table_name}` ({cols}) VALUES ({values});"
                         cursor.execute(sql)
+                    else:
+                        print(f'重复数据不插入: {condition}')
                 except Exception as e:
+                    print(f'{self.filename}:')
                     print(f'mysql -> df_to_mysql 报错: {e}')
         connection.commit()  # 提交事务
 
@@ -183,6 +188,18 @@ class MysqlUpload:
 
     def convert_dtype_to_sql(self, df, col, dtype):
         """ 按照以下规则转换DataFrame列的数据类型为 MYSQL 专有的数据类型 """
+
+        def find_longest_decimal_value(number_list):
+            # 取列表中小数位数最长的值
+            longest_value = None
+            max_decimals = 0
+            for num in number_list:
+                decimal_places = len(str(num).split('.')[1])
+                if decimal_places > max_decimals:
+                    max_decimals = decimal_places
+                    longest_value = num
+            return longest_value
+
         # 最优先处理 ID 类型, 在 mysql 里面, 有些列数字过长不能存储为 int 类型
         if 'id' in col or 'ID' in col or 'Id' in col or '摘要' in col or '商家编码' in col or '单号' in col or '款号' in col:
             return 'mediumtext'
@@ -207,9 +224,9 @@ class MysqlUpload:
                 return 'mediumtext'
             return 'INT'
         elif dtype == 'float64':
-            # step = len(str(max(df[col].tolist())))  # 数字长度包含小数点
-            int_step = len(str(max(df[col].tolist())).split('.')[0])   # 整数位数长度
-            f_step = len(str(max(df[col].tolist())).split('.')[1])  # 小数位数长度
+            res = find_longest_decimal_value(df[col].tolist())  # 取小数位数最长的值
+            int_step = len(str(res).split('.')[0])   # 整数位数长度
+            f_step = len(str(res).split('.')[1])  # 小数位数长度
             if int_step >= 12:
                 return 'mediumtext'  # mysql 中不要使用 float 和 double 类型，会影响计算结果
             elif int_step >= 8 and f_step >= 0:
@@ -679,24 +696,22 @@ def download_datas(table_name, save_path, start_date):
 
 def one_file_to_mysql(file, db_name, table_name, target_service, database):
     username, password, host, port = get_myconf.select_config_values(target_service=target_service, database=database)
-
+    filename = os.path.basename(file)
     df = pd.read_csv(file, encoding='utf-8_sig', header=0, na_filter=False)
     m = MysqlUpload(username=username, password=password, host=host, port=port)
-    m.df_to_mysql(df=df, db_name=db_name, table_name=table_name)
+    m.df_to_mysql(df=df, db_name=db_name, table_name=table_name, filename=filename)
 
 
 if __name__ == '__main__':
-    username, password, host, port = get_myconf.select_config_values(target_service='company', database='mysql')
-    print(username, password, host, port)
+    # username, password, host, port = get_myconf.select_config_values(target_service='home_lx', database='mysql')
+    # print(username, password, host, port)
 
-    file = '/Users/xigua/Downloads/余额查询.csv'
+    file = '/Users/xigua/数据中心/原始文件2/生意经/宝贝指标/2024-08/baobeitrans-2024-08-22.csv'
     one_file_to_mysql(
         file=file,
-        db_name='test',
-        table_name='余额查询',
-        target_service='company',
+        db_name='生意经2',
+        table_name='宝贝指标',
+        target_service='home_lx',
         database='mysql'
     )
 
-
-

{mdbq-1.1.0 → mdbq-1.1.2}/mdbq.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: mdbq
-Version: 1.1.0
+Version: 1.1.2
 Home-page: https://pypi.org/project/mdbsql
 Author: xigua, 
 Author-email: 2587125111@qq.com

{mdbq-1.1.0 → mdbq-1.1.2}/setup.py

@@ -3,7 +3,7 @@
 from setuptools import setup, find_packages
 
 setup(name='mdbq',
-      version='1.1.0',
+      version='1.1.2',
       author='xigua, ',
       author_email="2587125111@qq.com",
       url='https://pypi.org/project/mdbsql',