mdbq 2.4.2__py3-none-any.whl → 2.4.4__py3-none-any.whl

mdbq/aggregation/aggregation.py

@@ -1180,6 +1180,7 @@ def upload_dir(path, db_name, collection_name, dbs={'mysql': True, 'mongodb': Tr
                     #     continue
                     cv = converter.DataFrameConverter()
                     df = cv.convert_df_cols(df=df)  # 清理列名和 df 中的非法字符
+
                     try:
                         df = df.astype(dtypes)  # 按本地文件更新 df 的数据类型, 可能因为字段不同产生异常
                     except Exception as e:
@@ -1298,7 +1299,7 @@ def test2():
 if __name__ == '__main__':
     username, password, host, port = get_myconf.select_config_values(target_service='nas', database='mysql')
     print(username, password, host, port)
-    # file_dir(one_file=False, target_service='company')
+    file_dir(one_file=False, target_service='company')
     # one_file_to_mysql(
     #     file='/Users/xigua/Downloads/爱库存_商品榜单_spu_2024-10-17_2024-10-17.csv',
     #     db_name='爱库存2',

mdbq/dataframe/converter.py

@@ -40,6 +40,11 @@ class DataFrameConverter(object):
         df.reset_index(inplace=True, drop=True)  # 重置索引，避免下面的 df.loc[0, col] 会出错
 
         for col in cols:
+            if col.lower() == 'id':
+                df.pop(col)  # 等待插入的 df 不能包含 id 列，否则可能跟现有 id 主键冲突
+                continue
+            if col.endswith('占比') or col.endswith('率'):
+                df = df.astype({col: float}, errors='raise')
             # 百分比在某些数据库中不兼容, 转换百分比为小数
             df[col] = df[col].apply(lambda x: float(float((str(x).rstrip("%"))) / 100) if str(x).endswith('%') and '~' not in str(x) else x)
             # 尝试转换合适的数据类型

mdbq/mysql/mysql.py

@@ -68,7 +68,7 @@ class MysqlUpload:
 
         return wrapper
 
-    @try_except
+    # @try_except
     def df_to_mysql(self, df, table_name, db_name='远程数据源', icm_update=[], service_database={'home_lx': 'mysql'}, move_insert=False, df_sql=False, drop_duplicates=False, filename=None, count=None, json_path=None, reset_id=False):
         """
         将 df 写入数据库
@@ -167,6 +167,7 @@ class MysqlUpload:
                     index=False,
                     chunksize=1000
                 )
+                # 6. 重置自增列
                 try:
                     cursor.execute(f"SHOW COLUMNS FROM {table_name} LIKE 'id'")
                     result = cursor.fetchone()
@@ -385,6 +386,8 @@ class MysqlUpload:
             return 'mediumtext'
         if '文件大小' in col:  # bw 程序
             return 'mediumtext'
+        if col.endswith('占比') and (df[col].dtype == 'float' or df[col].dtype == 'int'):
+            return 'decimal(10,4)'
         elif dtype == 'datetime64[ns]':
             return 'DATETIME'  # 使用 DATE 后续排重可能会引发不能排重
         elif dtype == 'int32':

{mdbq-2.4.2.dist-info → mdbq-2.4.4.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: mdbq
-Version: 2.4.2
+Version: 2.4.4
 Home-page: https://pypi.org/project/mdbsql
 Author: xigua, 
 Author-email: 2587125111@qq.com

{mdbq-2.4.2.dist-info → mdbq-2.4.4.dist-info}/RECORD

@@ -1,7 +1,7 @@
 mdbq/__init__.py,sha256=Il5Q9ATdX8yXqVxtP_nYqUhExzxPC_qk_WXQ_4h0exg,16
 mdbq/__version__.py,sha256=y9Mp_8x0BCZSHsdLT_q5tX9wZwd5QgqrSIENLrb6vXA,62
 mdbq/aggregation/__init__.py,sha256=EeDqX2Aml6SPx8363J-v1lz0EcZtgwIBYyCJV6CcEDU,40
-mdbq/aggregation/aggregation.py,sha256=CS_gMBwPKQo7uId8BrsYNTjtCZKzRVV7gT4PfE2Q46k,76014
+mdbq/aggregation/aggregation.py,sha256=kHKCBRMfpv0F1dq4n75LqOt44K4lRWQCt8MeM6n6hnM,76013
 mdbq/aggregation/df_types.py,sha256=U9i3q2eRPTDY8qAPTw7irzu-Tlg4CIySW9uYro81wdk,8125
 mdbq/aggregation/mysql_types.py,sha256=DQYROALDiwjJzjhaJfIIdnsrNs11i5BORlj_v6bp67Y,11062
 mdbq/aggregation/optimize_data.py,sha256=Wis40oL04M7E1pkvgNPjyVFAUe-zgjimjIVAikxYY8Y,4418
@@ -19,13 +19,13 @@ mdbq/config/products.py,sha256=hN9UMkM6j76HYMulTYdtr3mOhh9QdpvvrLH14a_mbFY,5980
 mdbq/config/set_support.py,sha256=xkZCX6y9Bq1ppBpJAofld4B2YtchA7fl0eT3dx3CrSI,777
 mdbq/config/update_conf.py,sha256=taL3ZqKgiVWwUrDFuaYhim9a72Hm4BHRhhDscJTziR8,4535
 mdbq/dataframe/__init__.py,sha256=2HtCN8AdRj53teXDqzysC1h8aPL-mMFy561ESmhehGQ,22
-mdbq/dataframe/converter.py,sha256=ljCLblx7CMygQtH7eeC-XznBT3hKDG2huSwQ0xuRjEQ,3813
+mdbq/dataframe/converter.py,sha256=xKPv0TOOyCaeUqJ3W3GGQiM6o0mEcnY8v4WMEwTTVVE,4107
 mdbq/log/__init__.py,sha256=Mpbrav0s0ifLL7lVDAuePEi1hJKiSHhxcv1byBKDl5E,15
 mdbq/log/mylogger.py,sha256=oaT7Bp-Hb9jZt52seP3ISUuxVcI19s4UiqTeouScBO0,3258
 mdbq/mongo/__init__.py,sha256=SILt7xMtQIQl_m-ik9WLtJSXIVf424iYgCfE_tnQFbw,13
 mdbq/mongo/mongo.py,sha256=v9qvrp6p1ZRWuPpbSilqveiE0FEcZF7U5xUPI0RN4xs,31880
 mdbq/mysql/__init__.py,sha256=A_DPJyAoEvTSFojiI2e94zP0FKtCkkwKP1kYUCSyQzo,11
-mdbq/mysql/mysql.py,sha256=KJfbe17lcPTfcKdGufW6b5ZHiYHiPSHFvugPNTm8n2M,46645
+mdbq/mysql/mysql.py,sha256=FaKpsoVOKIITJBHEt6RpV4bKAwYIWCzkPjEY-uL4uYA,46811
 mdbq/mysql/s_query.py,sha256=37GGHzRpycfUjsYEoQgDpdEs9JwjW-LxFXnGwwP2b2Q,8403
 mdbq/mysql/year_month_day.py,sha256=VgewoE2pJxK7ErjfviL_SMTN77ki8GVbTUcao3vFUCE,1523
 mdbq/other/__init__.py,sha256=jso1oHcy6cJEfa7udS_9uO5X6kZLoPBF8l3wCYmr5dM,18
@@ -41,7 +41,7 @@ mdbq/req_post/__init__.py,sha256=jso1oHcy6cJEfa7udS_9uO5X6kZLoPBF8l3wCYmr5dM,18
 mdbq/req_post/req_tb.py,sha256=PexWSCPJNM6Tv0ol4lAWIhlOwsAr_frnjtcdSHCFiek,36179
 mdbq/spider/__init__.py,sha256=RBMFXGy_jd1HXZhngB2T2XTvJqki8P_Fr-pBcwijnew,18
 mdbq/spider/aikucun.py,sha256=Olq7IJP9itM4wuNxZeHOG-Q3i8wWyB4hY8TUGGwCvQ0,14104
-mdbq-2.4.2.dist-info/METADATA,sha256=eWoyYl2-G_Dxp_d_FEyYMYrm5YCGLzHsi-dcT98B9kQ,245
-mdbq-2.4.2.dist-info/WHEEL,sha256=eOLhNAGa2EW3wWl_TU484h7q1UNgy0JXjjoqKoxAAQc,92
-mdbq-2.4.2.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
-mdbq-2.4.2.dist-info/RECORD,,
+mdbq-2.4.4.dist-info/METADATA,sha256=5JUAtgippW56SKj0ik_57kiLLQazGdmxiX2ffwc2o2I,245
+mdbq-2.4.4.dist-info/WHEEL,sha256=eOLhNAGa2EW3wWl_TU484h7q1UNgy0JXjjoqKoxAAQc,92
+mdbq-2.4.4.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
+mdbq-2.4.4.dist-info/RECORD,,