mdbq 3.2.17__py3-none-any.whl → 3.2.19__py3-none-any.whl

mdbq/aggregation/query_data.py

@@ -2191,8 +2191,8 @@ class MysqlDatasQuery:
         }
         if not self.update_service:
             return
-        min_date = df['日期'].min()
-        max_date = df['日期'].max()
+        min_date = df['日期'].min().strftime('%Y-%m-%d')
+        max_date = df['日期'].max().strftime('%Y-%m-%d')
         now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
         print(f'{now} 正在更新: mysql ({host}:{port}) {db_name}/{table_name} -> {min_date}~{max_date}')
         m_engine.df_to_mysql(
@@ -3255,6 +3255,8 @@ def main(days=150, months=3):
         "生意参谋3",
         "生意经3",
         "达摩盘3",
+        '人群画像2',
+        '人群画像3',
     ]
     # 使用 ThreadPoolExecutor 来并行运行
     with concurrent.futures.ThreadPoolExecutor() as executor:
@@ -3297,10 +3299,10 @@ def main(days=150, months=3):
 
 
 if __name__ == '__main__':
-    # main(
-    #     days=150,  # 清理聚合数据的日期长度
-    #     months=3  # 生成聚合数据的长度
-    # )
+    main(
+        days=150,  # 清理聚合数据的日期长度
+        months=3  # 生成聚合数据的长度
+    )
 
     # query_list = [query1, query2]
     # # 使用 ThreadPoolExecutor 来并行运行
@@ -3314,10 +3316,10 @@ if __name__ == '__main__':
     #             ),
     #         }
 
-    # 3. 清理聚合数据
-    optimize_data.op_data(
-        db_name_lists=['聚合数据'],
-        days=100,  # 清理聚合数据的日期长度
-        is_mongo=False,
-        is_mysql=True,
-    )
+    # # 3. 清理聚合数据
+    # optimize_data.op_data(
+    #     db_name_lists=['聚合数据'],
+    #     days=30,  # 清理聚合数据的日期长度
+    #     is_mongo=False,
+    #     is_mysql=True,
+    # )

mdbq/mysql/mysql.py

@@ -180,7 +180,7 @@ class MysqlUpload:
         __res_dict.update({'数据主体': 'longblob'})
         return __res_dict, new_dict_data
 
-    # @try_except
+    @try_except
     def doc_to_sql(self, db_name, table_name, dict_data, set_typ={}, remove_by_key=None, allow_not_null=False, filename=None, reset_id=False):
         """
         db_name:
@@ -299,18 +299,24 @@ class MysqlUpload:
                     cursor.execute(sql)
                     result = cursor.fetchall()  # 复合主键数
                     if len(result) <= 1:  # 如果存在复合主键，则不能直接删除 id 键，其他主键可能不是唯一，会报错
-                        cursor.execute(f"SHOW COLUMNS FROM {table_name} LIKE 'id'")
+                        column_name = 'id'
+                        sql = (f'SELECT COLUMN_NAME FROM INFORMATION_SCHEMA.COLUMNS '
+                               f'WHERE TABLE_SCHEMA = %s AND TABLE_NAME = %s AND COLUMN_NAME = %s')
+                        # cursor.execute(f"SHOW COLUMNS FROM `{table_name}` LIKE 'id'")
+                        cursor.execute(sql, (db_name, table_name, column_name))
                         result = cursor.fetchone()
                         if result:
-                            cursor.execute(f"ALTER TABLE {table_name} DROP COLUMN id;")  # 删除 id  列
+                            # cursor.execute(f"ALTER TABLE `{table_name}` DROP COLUMN id;")  # 删除 id  列
+                            sql = f"ALTER TABLE `{table_name}` DROP COLUMN {column_name}"  # 删除 id  列
+                            cursor.execute(sql)
                         cursor.execute(
-                            f"ALTER TABLE {table_name} ADD column id INT AUTO_INCREMENT PRIMARY KEY FIRST;")
-                        cursor.execute(f"ALTER TABLE {table_name} AUTO_INCREMENT = 1")  # 设置自增从 1 开始
+                            f"ALTER TABLE `{table_name}` ADD column id INT AUTO_INCREMENT PRIMARY KEY FIRST;")
+                        cursor.execute(f"ALTER TABLE `{table_name}` AUTO_INCREMENT = 1")  # 设置自增从 1 开始
                         # print(f'重置自增id')
                     else:
-                        print(f'{table_name} 当前表存在复合主键: {result}, 无法重置自增id')
+                        print(f'{table_name} 存在复合主键: 存在复合主键: {[item['PrimaryKey'] for item in result]}, 无法重置自增id')
                 except Exception as e:
-                    print(f'{e}')
+                    print(f'333 `{table_name}` {e}')
                     connection.rollback()
             connection.commit()
 
@@ -678,10 +684,10 @@ class MysqlUpload:
         self.filename = filename
         if isinstance(df, pd.DataFrame):
             if len(df) == 0:
-                print(f'{db_name}: {table_name} 传入的 df 数据长度为0, {self.filename}')
+                print(f'{db_name}: `{table_name}` 传入的 df 数据长度为0, {self.filename}')
                 return
         else:
-            print(f'{db_name}: {table_name} 传入的 df 不是有效的 dataframe 结构, {self.filename}')
+            print(f'{db_name}: `{table_name}` 传入的 df 不是有效的 dataframe 结构, {self.filename}')
             return
         if not db_name or db_name == 'None':
             print(f'{db_name} 不能为 None')
@@ -780,7 +786,6 @@ class MysqlUpload:
                     chunksize=1000
                 )
                 if reset_id:
-                    pass
                     # 6. 重置自增列
                     try:
                         # 查询所有复合主键
@@ -791,18 +796,24 @@ class MysqlUpload:
                         cursor.execute(sql)
                         result = cursor.fetchall()  # 复合主键数
                         if len(result) <= 1:  # 如果存在复合主键，则不能直接删除 id 键，其他主键可能不是唯一，会报错
-                            cursor.execute(f"SHOW COLUMNS FROM {table_name} LIKE 'id'")
+                            column_name = 'id'
+                            sql = (f'SELECT COLUMN_NAME FROM INFORMATION_SCHEMA.COLUMNS '
+                                   f'WHERE TABLE_SCHEMA = %s AND TABLE_NAME = %s AND COLUMN_NAME = %s')
+                            # cursor.execute(f"SHOW COLUMNS FROM `{table_name}` LIKE 'id'")
+                            cursor.execute(sql, (db_name, table_name, column_name))
                             result = cursor.fetchone()
                             if result:
-                                cursor.execute(f"ALTER TABLE {table_name} DROP COLUMN id;")  # 删除 id  列
+                                # cursor.execute(f"ALTER TABLE `{table_name}` DROP COLUMN id;")  # 删除 id  列
+                                sql = f"ALTER TABLE `{table_name}` DROP COLUMN {column_name}"  # 删除 id  列
+                                cursor.execute(sql)
                             cursor.execute(
-                                f"ALTER TABLE {table_name} ADD column id INT AUTO_INCREMENT PRIMARY KEY FIRST;")
-                            cursor.execute(f"ALTER TABLE {table_name} AUTO_INCREMENT = 1")  # 设置自增从 1 开始
+                                f"ALTER TABLE `{table_name}` ADD column id INT AUTO_INCREMENT PRIMARY KEY FIRST;")
+                            cursor.execute(f"ALTER TABLE `{table_name}` AUTO_INCREMENT = 1")  # 设置自增从 1 开始
                             # print(f'重置自增id')
                         else:
-                            print(f'{table_name} 当前表存在复合主键: {result}, 无法重置自增id')
+                            print(f'{table_name} 存在复合主键: {[item['PrimaryKey'] for item in result]}, 无法重置自增id')
                     except Exception as e:
-                        print(f'{e}')
+                        print(f'333 `{table_name}` {e}')
                         connection.rollback()
                 connection.commit()  # 提交事务
                 connection.close()
@@ -832,29 +843,34 @@ class MysqlUpload:
                     chunksize=1000
                 )
                 # 6. 重置自增列
-                if reset_id:
-                    try:
-                        # 查询所有复合主键
-                        sql = (
-                            f"SELECT `COLUMN_NAME` AS `PrimaryKey` FROM `information_schema`.`COLUMNS` "
-                            f"WHERE `TABLE_SCHEMA` = '{db_name}'AND `TABLE_NAME` = '{table_name}' AND `COLUMN_KEY` = 'PRI';"
-                        )
-                        cursor.execute(sql)
-                        result = cursor.fetchall()  # 复合主键数
-                        if len(result) <= 1:  # 如果存在复合主键，则不能直接删除 id 键，其他主键可能不是唯一，会报错
-                            cursor.execute(f"SHOW COLUMNS FROM {table_name} LIKE 'id'")
-                            result = cursor.fetchone()
-                            if result:
-                                cursor.execute(f"ALTER TABLE {table_name} DROP COLUMN id;")  # 删除 id  列
-                            cursor.execute(
-                                f"ALTER TABLE {table_name} ADD column id INT AUTO_INCREMENT PRIMARY KEY FIRST;")
-                            cursor.execute(f"ALTER TABLE {table_name} AUTO_INCREMENT = 1")  # 设置自增从 1 开始
-                            # print(f'重置自增id')
-                        else:
-                            print(f'{table_name} 当前表存在复合主键: {result}, 无法重置自增id')
-                    except Exception as e:
-                        print(f'{e}')
-                        connection.rollback()
+                try:
+                    # 查询所有复合主键
+                    sql = (
+                        f"SELECT `COLUMN_NAME` AS `PrimaryKey` FROM `information_schema`.`COLUMNS` "
+                        f"WHERE `TABLE_SCHEMA` = '{db_name}'AND `TABLE_NAME` = '{table_name}' AND `COLUMN_KEY` = 'PRI';"
+                    )
+                    cursor.execute(sql)
+                    result = cursor.fetchall()  # 复合主键数
+                    if len(result) <= 1:  # 如果存在复合主键，则不能直接删除 id 键，其他主键可能不是唯一，会报错
+                        column_name = 'id'
+                        sql = (f'SELECT COLUMN_NAME FROM INFORMATION_SCHEMA.COLUMNS '
+                               f'WHERE TABLE_SCHEMA = %s AND TABLE_NAME = %s AND COLUMN_NAME = %s')
+                        # cursor.execute(f"SHOW COLUMNS FROM `{table_name}` LIKE 'id'")
+                        cursor.execute(sql, (db_name, table_name, column_name))
+                        result = cursor.fetchone()
+                        if result:
+                            # cursor.execute(f"ALTER TABLE `{table_name}` DROP COLUMN id;")  # 删除 id  列
+                            sql = f"ALTER TABLE `{table_name}` DROP COLUMN {column_name}"  # 删除 id  列
+                            cursor.execute(sql)
+                        cursor.execute(
+                            f"ALTER TABLE `{table_name}` ADD column id INT AUTO_INCREMENT PRIMARY KEY FIRST;")
+                        cursor.execute(f"ALTER TABLE `{table_name}` AUTO_INCREMENT = 1")  # 设置自增从 1 开始
+                        # print(f'重置自增id')
+                    else:
+                        print(f'{table_name} 存在复合主键: {[item['PrimaryKey'] for item in result]}, 无法重置自增id')
+                except Exception as e:
+                    print(f'333 `{table_name}` {e}')
+                    connection.rollback()
                 connection.close()
                 return
 
@@ -948,8 +964,8 @@ class MysqlUpload:
                     # print(f'mysql -> df_to_mysql 报错: {e}, {self.filename}')
                     # breakpoint()
 
-            # 6. 重置自增列
             if reset_id:
+                # 6. 重置自增列
                 try:
                     # 查询所有复合主键
                     sql = (
@@ -959,18 +975,24 @@ class MysqlUpload:
                     cursor.execute(sql)
                     result = cursor.fetchall()  # 复合主键数
                     if len(result) <= 1:  # 如果存在复合主键，则不能直接删除 id 键，其他主键可能不是唯一，会报错
-                        cursor.execute(f"SHOW COLUMNS FROM {table_name} LIKE 'id'")
+                        column_name = 'id'
+                        sql = (f'SELECT COLUMN_NAME FROM INFORMATION_SCHEMA.COLUMNS '
+                               f'WHERE TABLE_SCHEMA = %s AND TABLE_NAME = %s AND COLUMN_NAME = %s')
+                        # cursor.execute(f"SHOW COLUMNS FROM `{table_name}` LIKE 'id'")
+                        cursor.execute(sql, (db_name, table_name, column_name))
                         result = cursor.fetchone()
                         if result:
-                            cursor.execute(f"ALTER TABLE {table_name} DROP COLUMN id;")  # 删除 id  列
+                            # cursor.execute(f"ALTER TABLE `{table_name}` DROP COLUMN id;")  # 删除 id  列
+                            sql = f"ALTER TABLE `{table_name}` DROP COLUMN {column_name}"  # 删除 id  列
+                            cursor.execute(sql)
                         cursor.execute(
-                            f"ALTER TABLE {table_name} ADD column id INT AUTO_INCREMENT PRIMARY KEY FIRST;")
-                        cursor.execute(f"ALTER TABLE {table_name} AUTO_INCREMENT = 1")  # 设置自增从 1 开始
+                            f"ALTER TABLE `{table_name}` ADD column id INT AUTO_INCREMENT PRIMARY KEY FIRST;")
+                        cursor.execute(f"ALTER TABLE `{table_name}` AUTO_INCREMENT = 1")  # 设置自增从 1 开始
                         # print(f'重置自增id')
                     else:
-                        print(f'{table_name} 当前表存在复合主键: {result}, 无法重置自增id')
+                        print(f'{table_name} 存在复合主键: {[item['PrimaryKey'] for item in result]}, 无法重置自增id')
                 except Exception as e:
-                    print(f'{e}')
+                    print(f'333 `{table_name}` {e}')
                     connection.rollback()
         connection.commit()  # 提交事务
         connection.close()
@@ -1055,19 +1077,19 @@ class MysqlUpload:
                 columns = [desc[0] for desc in cursor.description]
                 df = pd.DataFrame(rows, columns=columns)  # 转为 df
         except Exception as e:
-            print(f'{e} {db_name} -> {table_name} 表不存在')
+            print(f'{e} {db_name} -> `{table_name}` 表不存在')
             return df
         finally:
             connection.close()
 
         if len(df) == 0:
-            print(f'database: {db_name}, table: {table_name} 查询的数据为空')
+            print(f'database: {db_name}, table: `{table_name}` 查询的数据为空')
         else:
             now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
             cost_time = int(time.time() - before_time)
             if cost_time < 1:
                 cost_time = round(time.time() - before_time, 2)
-            print(f'{now} mysql ({self.host}) 表: {table_name} 获取数据长度: {len(df)}, 用时: {cost_time} 秒')
+            print(f'{now} mysql ({self.host}) 表: `{table_name}` 获取数据长度: {len(df)}, 用时: {cost_time} 秒')
         return df
 
     def upload_pandas(self, update_path, db_name, days=None):
@@ -1263,18 +1285,24 @@ class OptimizeDatas:
                         cursor.execute(sql)
                         result = cursor.fetchall()  # 复合主键数
                         if len(result) <= 1:  # 如果存在复合主键，则不能直接删除 id 键，其他主键可能不是唯一，会报错
-                            cursor.execute(f"SHOW COLUMNS FROM {table_name} LIKE 'id'")
+                            column_name = 'id'
+                            sql = (f'SELECT COLUMN_NAME FROM INFORMATION_SCHEMA.COLUMNS '
+                                   f'WHERE TABLE_SCHEMA = %s AND TABLE_NAME = %s AND COLUMN_NAME = %s')
+                            # cursor.execute(f"SHOW COLUMNS FROM `{table_name}` LIKE 'id'")
+                            cursor.execute(sql, (self.db_name, table_name, column_name))
                             result = cursor.fetchone()
                             if result:
-                                cursor.execute(f"ALTER TABLE {table_name} DROP COLUMN id;")  # 删除 id  列
+                                # cursor.execute(f"ALTER TABLE `{table_name}` DROP COLUMN id;")  # 删除 id  列
+                                sql = f"ALTER TABLE `{table_name}` DROP COLUMN {column_name}"  # 删除 id  列
+                                cursor.execute(sql)
                             cursor.execute(
-                                f"ALTER TABLE {table_name} ADD column id INT AUTO_INCREMENT PRIMARY KEY FIRST;")
-                            cursor.execute(f"ALTER TABLE {table_name} AUTO_INCREMENT = 1")  # 设置自增从 1 开始
+                                f"ALTER TABLE `{table_name}` ADD column id INT AUTO_INCREMENT PRIMARY KEY FIRST;")
+                            cursor.execute(f"ALTER TABLE `{table_name}` AUTO_INCREMENT = 1")  # 设置自增从 1 开始
                             # print(f'重置自增id')
                         else:
-                            print(f'{table_name} 当前表存在复合主键: {result}, 无法重置自增id')
+                            print(f'{table_name} 存在复合主键: {[item['PrimaryKey'] for item in result]}, 无法重置自增id')
                     except Exception as e:
-                        print(f'{e}')
+                        print(f'333 `{table_name}` {e}')
                         self.connection.rollback()
                 self.connection.close()
         now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
@@ -1312,7 +1340,7 @@ class OptimizeDatas:
                 sql = f"DELETE FROM `{table_name}` WHERE id IN ({placeholders})"
                 cursor.execute(sql, duplicate_id)
                 now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
-                print(f"{now} {table_name} -> {date.strftime('%Y-%m-%d')} before: {len(datas)}, remove: {cursor.rowcount}")
+                print(f"{now} `{table_name}` -> {date.strftime('%Y-%m-%d')} before: {len(datas)}, remove: {cursor.rowcount}")
             self.connection.commit()  # 提交事务
         except Exception as e:
             print(f'{self.db_name}/{table_name}, {e}')
@@ -1350,7 +1378,7 @@ class OptimizeDatas:
                 sql = f"DELETE FROM `{table_name}` WHERE id IN ({placeholders})"
                 cursor.execute(sql, duplicate_id)
                 now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
-                print(f"{now} {table_name} -> before: {len(datas)}, "
+                print(f"{now} `{table_name}` -> before: {len(datas)}, "
                       f"remove: {cursor.rowcount}")
             self.connection.commit()  # 提交事务
         except Exception as e:

{mdbq-3.2.17.dist-info → mdbq-3.2.19.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: mdbq
-Version: 3.2.17
+Version: 3.2.19
 Home-page: https://pypi.org/project/mdbq
 Author: xigua, 
 Author-email: 2587125111@qq.com

{mdbq-3.2.17.dist-info → mdbq-3.2.19.dist-info}/RECORD

@@ -5,7 +5,7 @@ mdbq/aggregation/aggregation.py,sha256=-yzApnlqSN2L0E1YMu5ml-W827qpKQvWPCOI7jj2k
 mdbq/aggregation/df_types.py,sha256=U9i3q2eRPTDY8qAPTw7irzu-Tlg4CIySW9uYro81wdk,8125
 mdbq/aggregation/mysql_types.py,sha256=YTGyrF9vcRgfkQbpT-e-JdJ7c7VF1dDHgyx9YZRES8w,10934
 mdbq/aggregation/optimize_data.py,sha256=RXIv7cACCgYyehAxMjUYi_S7rVyjIwXKWMaM3nduGtA,3068
-mdbq/aggregation/query_data.py,sha256=-l29T7LNZJ7U6QHfa0ZyZVz7N35n23xNZb7XlCDOspQ,148348
+mdbq/aggregation/query_data.py,sha256=Is5RUwDb_ODjU3WH2S-0b-lV1Jux2xgfE7kwNNvPx4Q,148447
 mdbq/aggregation/query_data_bak.py,sha256=r1FU0C4zjXln7oVSrRkElh4Ehl-9mYhGcq57jLbViUA,104071
 mdbq/aggregation/query_data_bak20241124.py,sha256=oY95ZK3qt3Wx9pdZKZ5cvDh45Yi5yGj1kl8G6riumHA,144513
 mdbq/bdup/__init__.py,sha256=AkhsGk81SkG1c8FqDH5tRq-8MZmFobVbN60DTyukYTY,28
@@ -28,7 +28,7 @@ mdbq/log/mylogger.py,sha256=oaT7Bp-Hb9jZt52seP3ISUuxVcI19s4UiqTeouScBO0,3258
 mdbq/mongo/__init__.py,sha256=SILt7xMtQIQl_m-ik9WLtJSXIVf424iYgCfE_tnQFbw,13
 mdbq/mongo/mongo.py,sha256=v9qvrp6p1ZRWuPpbSilqveiE0FEcZF7U5xUPI0RN4xs,31880
 mdbq/mysql/__init__.py,sha256=A_DPJyAoEvTSFojiI2e94zP0FKtCkkwKP1kYUCSyQzo,11
-mdbq/mysql/mysql.py,sha256=ZG6BMfoXg6YGnHqv7GfwPwd7RXjoetCAFqPnbdHWqOM,79507
+mdbq/mysql/mysql.py,sha256=4Ef1h6ucw_E8Ej_MOBa5e_IUP9PCzcq9_F0EmXelr7U,82054
 mdbq/mysql/recheck_mysql.py,sha256=rgTpvDMWYTyEn7UQdlig-pdXDluTgiU8JG6lkMh8DV0,8665
 mdbq/mysql/s_query.py,sha256=MbIprZ4yJDAZ9AahZPzl7hqS695Vs0P-AJNwAtA_EEc,9287
 mdbq/mysql/year_month_day.py,sha256=VgewoE2pJxK7ErjfviL_SMTN77ki8GVbTUcao3vFUCE,1523
@@ -46,7 +46,7 @@ mdbq/req_post/__init__.py,sha256=jso1oHcy6cJEfa7udS_9uO5X6kZLoPBF8l3wCYmr5dM,18
 mdbq/req_post/req_tb.py,sha256=qg7pet73IgKGmCwxaeUyImJIoeK_pBQT9BBKD7fkBNg,36160
 mdbq/spider/__init__.py,sha256=RBMFXGy_jd1HXZhngB2T2XTvJqki8P_Fr-pBcwijnew,18
 mdbq/spider/aikucun.py,sha256=yG0wqvKKqAsx_OQXAy-wAJF_DPITL3r_S-9txb5bnHk,22436
-mdbq-3.2.17.dist-info/METADATA,sha256=lxqDb9y53ZzJ5OeiUfsV8sdJptpkycbMtQN5HIqO5mM,244
-mdbq-3.2.17.dist-info/WHEEL,sha256=eOLhNAGa2EW3wWl_TU484h7q1UNgy0JXjjoqKoxAAQc,92
-mdbq-3.2.17.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
-mdbq-3.2.17.dist-info/RECORD,,
+mdbq-3.2.19.dist-info/METADATA,sha256=k853SFwdrs-X-mw5US3F9rK4G0ps5IKA2XWSHJJPWws,244
+mdbq-3.2.19.dist-info/WHEEL,sha256=cpQTJ5IWu9CdaPViMhC9YzF8gZuS5-vlfoFihTBC86A,91
+mdbq-3.2.19.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
+mdbq-3.2.19.dist-info/RECORD,,

{mdbq-3.2.17.dist-info → mdbq-3.2.19.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: bdist_wheel (0.44.0)
+Generator: setuptools (70.1.0)
 Root-Is-Purelib: true
 Tag: py3-none-any