mdbq 1.1.1__tar.gz → 1.1.2__tar.gz

{mdbq-1.1.1 → mdbq-1.1.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: mdbq
-Version: 1.1.1
+Version: 1.1.2
 Home-page: https://pypi.org/project/mdbsql
 Author: xigua, 
 Author-email: 2587125111@qq.com

{mdbq-1.1.1 → mdbq-1.1.2}/mdbq/aggregation/aggregation.py

@@ -874,12 +874,16 @@ def upload(path, db_name, collection_name):
         collection_name=collection_name,
     )
     # print(dtypes)
+    count = 0
+    for root, dirs, files in os.walk(path, topdown=False):
+        for name in files:
+            count += 1
+    i = 0  # 用来统计当前处理文件进度
     for root, dirs, files in os.walk(path, topdown=False):
         for name in files:
             if '~$' in name or '.DS' in name or '.localized' in name or 'baidu' in name:
                 continue
             if name.endswith('.csv'):
-                # print(name)
                 try:
                     df = pd.read_csv(os.path.join(root, name), encoding='utf-8_sig', header=0, na_filter=False)
                     if len(df) == 0:
@@ -896,10 +900,11 @@ def upload(path, db_name, collection_name):
                         df = df.astype(dtypes)
 
                     # d.df_to_mongo(df=df, db_name=db_name, collection_name=collection_name)
-                    m.df_to_mysql(df=df, db_name=db_name, table_name=collection_name)
+                    m.df_to_mysql(df=df, db_name=db_name, table_name=collection_name, filename=name, count=f'{i}/{count}')
                     # nas.df_to_mysql(df=df, db_name=db_name, table_name=collection_name)
                 except Exception as e:
                     print(name, e)
+            i += 1
     if d.client:
         d.client.close()  # 必须手动关闭数据库连接
 
@@ -920,7 +925,7 @@ if __name__ == '__main__':
     # username, password, host, port = get_myconf.select_config_values(target_service='nas', database='mysql')
     # print(username, password, host, port)
     upload(
-        path='/Users/xigua/数据中心/原始文件2/属性设置/商品素材',
-        db_name = '属性设置2',
-        collection_name = '商品素材导出',
+        path='/Users/xigua/数据中心/原始文件2/生意经/宝贝指标',
+        db_name = '生意经2',
+        collection_name = '宝贝指标',
     )

{mdbq-1.1.1 → mdbq-1.1.2}/mdbq/config/get_myconf.py

@@ -62,7 +62,7 @@ def select_config_values(target_service, database, path=None):
             if socket.gethostname() == 'xigua_lx':
                 # 本机自身运行使用 127.0.0.1
                 options = ['username_mysql_lx_nw', 'password_mysql_lx_nw', 'host_bd',  'port_mysql_lx_nw',]
-            elif socket.gethostname() == 'xigua1' or socket.gethostname() == 'macbook pro':
+            elif socket.gethostname() == 'xigua1' or socket.gethostname() == 'MacBook-Pro':
                 # 内网地址：正在运行的是 家里笔记本或者台式机，或者 macb    ook pro
                 options = ['username_mysql_lx_nw', 'password_mysql_lx_nw', 'host_mysql_lx_nw',  'port_mysql_lx_nw',]
             else:
@@ -126,6 +126,6 @@ def main():
 
 if __name__ == '__main__':
     # main()
-    r, d, s, g = select_config_values(target_service='home_lx', database='mongodb')
+    r, d, s, g = select_config_values(target_service='home_lx', database='mysql')
     print(r, d, s, g, type(r), type(d), type(s), type(g))
     print(f'本机: {platform.system()} // {socket.gethostname()}')

{mdbq-1.1.1 → mdbq-1.1.2}/mdbq/mysql/mysql.py

@@ -35,14 +35,16 @@ class MysqlUpload:
             'charset': charset,  # utf8mb4 支持存储四字节的UTF-8字符集
             'cursorclass': pymysql.cursors.DictCursor,
         }
+        self.filename = None
 
-    def df_to_mysql(self, df, table_name, db_name='远程数据源', drop_duplicates=False):
+    def df_to_mysql(self, df, table_name, db_name='远程数据源', drop_duplicates=False, filename=None, count=None):
         """
         将 df 写入数据库
         db_name: 数据库名称
         table_name: 集合/表名称
         drop_duplicates：仅限于聚合数据使用，其他情况不要设置
         """
+        self.filename = filename
         if isinstance(df, pd.DataFrame):
             if len(df) == 0:
                 print(f'{db_name}: {table_name} 传入的 df 数据长度为0')
@@ -95,7 +97,7 @@ class MysqlUpload:
                 for col in col_not_exist:
                     try:
                         #  创建列，需转义
-                        sql = f"ALTER TABLE `{table_name}` ADD COLUMN `{col}` {dtypes[col]} DEFAULT NULL;"
+                        sql = f"ALTER TABLE `{table_name}` ADD COLUMN `{col}` {dtypes[col]};"
                         cursor.execute(sql)
                         print(f"添加列: {col}({dtypes[col]})")  # 添加列并指定数据类型
 
@@ -105,8 +107,8 @@ class MysqlUpload:
                             result = cursor.fetchone()  # 检查索引是否存在
                             if not result:
                                 cursor.execute(f"CREATE INDEX index_name ON `{table_name}`(`{col}`)")
-                    except:
-                        pass
+                    except Exception as e:
+                        print(f'{self.filename}: {e}')
             connection.commit()  # 提交事务
 
             # 返回这些结果的目的是等添加完列再写 json 文件才能读到 types 信息
@@ -128,7 +130,7 @@ class MysqlUpload:
 
             # 5. 更新插入数据
             now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S ")
-            print(f'{now}正在更新 mysql ({self.host}:{self.port}) {db_name}/{table_name}')
+            print(f'{now}正在更新 mysql ({self.host}:{self.port}) {db_name}/{table_name}, {count},{self.filename}')
             datas = df.to_dict(orient='records')
             for data in datas:
                 try:
@@ -143,11 +145,14 @@ class MysqlUpload:
 
                     sql = f"SELECT {cols} FROM `{table_name}` WHERE {condition}"
                     cursor.execute(sql)
-                    result = cursor.fetchall()  # 获取查询结果, 如果有结果返回 list，没有则返回空元组 tuple
+                    result = cursor.fetchall()  # 获取查询结果, 有结果返回 list 表示数据已存在(不重复插入)，没有则返回空 tuple
                     if not result:  # 数据不存在则插入
                         sql = f"INSERT INTO `{table_name}` ({cols}) VALUES ({values});"
                         cursor.execute(sql)
+                    else:
+                        print(f'重复数据不插入: {condition}')
                 except Exception as e:
+                    print(f'{self.filename}:')
                     print(f'mysql -> df_to_mysql 报错: {e}')
         connection.commit()  # 提交事务
 
@@ -691,22 +696,22 @@ def download_datas(table_name, save_path, start_date):
 
 def one_file_to_mysql(file, db_name, table_name, target_service, database):
     username, password, host, port = get_myconf.select_config_values(target_service=target_service, database=database)
-
+    filename = os.path.basename(file)
     df = pd.read_csv(file, encoding='utf-8_sig', header=0, na_filter=False)
     m = MysqlUpload(username=username, password=password, host=host, port=port)
-    m.df_to_mysql(df=df, db_name=db_name, table_name=table_name)
+    m.df_to_mysql(df=df, db_name=db_name, table_name=table_name, filename=filename)
 
 
 if __name__ == '__main__':
-    username, password, host, port = get_myconf.select_config_values(target_service='company', database='mysql')
-    print(username, password, host, port)
+    # username, password, host, port = get_myconf.select_config_values(target_service='home_lx', database='mysql')
+    # print(username, password, host, port)
 
-    file = '/Users/xigua/数据中心/原始文件2/京东报表/JD商品明细spu/2024-08/spu_2024-08-18_全部渠道_商品明细.csv'
+    file = '/Users/xigua/数据中心/原始文件2/生意经/宝贝指标/2024-08/baobeitrans-2024-08-22.csv'
     one_file_to_mysql(
         file=file,
-        db_name='test',
-        table_name='余额查询',
-        target_service='company',
+        db_name='生意经2',
+        table_name='宝贝指标',
+        target_service='home_lx',
         database='mysql'
     )
 

{mdbq-1.1.1 → mdbq-1.1.2}/mdbq.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: mdbq
-Version: 1.1.1
+Version: 1.1.2
 Home-page: https://pypi.org/project/mdbsql
 Author: xigua, 
 Author-email: 2587125111@qq.com

{mdbq-1.1.1 → mdbq-1.1.2}/setup.py

@@ -3,7 +3,7 @@
 from setuptools import setup, find_packages
 
 setup(name='mdbq',
-      version='1.1.1',
+      version='1.1.2',
       author='xigua, ',
       author_email="2587125111@qq.com",
       url='https://pypi.org/project/mdbsql',