mdbq 1.2.2__py3-none-any.whl → 1.2.3__py3-none-any.whl

mdbq/aggregation/aggregation.py

@@ -438,6 +438,7 @@ class DatabaseUpdate:
                         new_name = f'未分类_{date1}_全部渠道_商品明细.csv'
                     df.rename(columns={'商品ID': '商品id'}, inplace=True)
                     df.insert(loc=0, column='日期', value=date1)
+                    df['最近上架时间'].loc[0] = df['最近上架时间'].loc[1]  # 填充这一列, 避免上传 mysql 日期类型报错
                     if 'sku' in new_name:
                         db_name = '京东数据2'
                         collection_name = 'sku_商品明细'
@@ -981,7 +982,27 @@ def file_dir(one_file=True):
     df.to_csv(os.path.join(support_file, filename), encoding='utf-8_sig', index=False, header=True)
 
 
+def test():
+    path = '/Users/xigua/数据中心/原始文件2/京东报表/JD商品明细spu'
+    for root, dirs, files in os.walk(path, topdown=False):
+        for name in files:
+            if name.endswith('.csv') and 'baidu' not in name and '~' not in name:
+                df = pd.read_csv(os.path.join(root, name), encoding='utf-8_sig', header=0, na_filter=False)
+                df['最近上架时间'].loc[0] = df['最近上架时间'].loc[1]
+                # print(df[['日期', '最近上架时间']])
+                df.to_csv(os.path.join(root, name), encoding='utf-8_sig', index=False, header=True)
+        #         break
+        # break
+
+
 if __name__ == '__main__':
     # username, password, host, port = get_myconf.select_config_values(target_service='nas', database='mysql')
     # print(username, password, host, port)
-    file_dir(one_file=True)
+    file_dir(one_file=False)
+    # one_file_to_mysql(
+    #     file='',
+    #     db_name='京东数据2',
+    #     table_name='商品词下排名',
+    #     target_service='home_lx',
+    #     database='mysql'
+    # )

mdbq/aggregation/mysql_types.py

@@ -123,6 +123,7 @@ class DataTypes:
                 sort_keys=True,  # 默认为False。如果为True，则字典的输出将按键排序。
                 indent=4,
             )
+        print(f'已更新 json 文件: {self.json_file}')
         time.sleep(1)
 
     def load_dtypes(self, db_name, table_name, cl='mysql', ):
@@ -236,5 +237,6 @@ def mysql_all_dtypes(db_name=None, table_name=None, path=None):
     # print(d.datas)
     d.as_json_file()
 
+
 if __name__ == '__main__':
     mysql_all_dtypes()  # 更新 mysql 中所有数据库的 dtypes 信息到本地 json

mdbq/mysql/mysql.py

@@ -21,6 +21,7 @@ from mdbq.aggregation import mysql_types
 warnings.filterwarnings('ignore')
 """
 建表规范:
+尽可能手动建表，再上传数据
 1. 先建 json 表，再批量上传数据;（非常重要）
     在初创数据表时, 如果有不同类报表，新版和旧版都要取一个文件，先创建数据表，再导其他数据;
     例如有的报表转化率是0%，数据类型会被识别为2位小数: decimal(10, 2)，正常值应类似 0.43%，应保留4个小数, 创建类型为 decimal(10, 4)
@@ -132,6 +133,7 @@ class MysqlUpload:
                         print(f'{self.filename}: {e}')
             connection.commit()  # 提交事务
 
+            # print(cl, db_n, tb_n)
             # 返回这些结果的目的是等添加完列再写 json 文件才能读到 types 信息
             if cl and db_n and tb_n:
                 mysql_types.mysql_all_dtypes(db_name=db_name, table_name=table_name)  # 更新一个表的 dtypes
@@ -208,6 +210,7 @@ class MysqlUpload:
             col_not_exist = cols
         # 对文件不存在的列信息进行数据类型转换(按指定规则)
         dtypes.update({col: self.convert_dtype_to_sql(df=df, col=col, dtype=df[col].dtype) for col in col_not_exist})
+        # print(dtypes)
         # 至此 df 中全部列类型已经转换完成
         # 返回结果, 示例: {'上市年份': 'mediumtext', '商品id': 'mediumtext', '平台': 'mediumtext'}
         return dtypes, cl, db_n, tb_n  # 返回这些结果的目的是等添加完列再写 json 文件才能读到 types 信息

{mdbq-1.2.2.dist-info → mdbq-1.2.3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: mdbq
-Version: 1.2.2
+Version: 1.2.3
 Home-page: https://pypi.org/project/mdbsql
 Author: xigua, 
 Author-email: 2587125111@qq.com

{mdbq-1.2.2.dist-info → mdbq-1.2.3.dist-info}/RECORD

@@ -1,9 +1,9 @@
 mdbq/__init__.py,sha256=Il5Q9ATdX8yXqVxtP_nYqUhExzxPC_qk_WXQ_4h0exg,16
 mdbq/__version__.py,sha256=y9Mp_8x0BCZSHsdLT_q5tX9wZwd5QgqrSIENLrb6vXA,62
 mdbq/aggregation/__init__.py,sha256=EeDqX2Aml6SPx8363J-v1lz0EcZtgwIBYyCJV6CcEDU,40
-mdbq/aggregation/aggregation.py,sha256=-cyucSkFCW9WHfhourcVH2_u4syRsR7VmbICWE82Brg,56259
+mdbq/aggregation/aggregation.py,sha256=-TGvWaLUMfPJHeQ3ZsL6WaN2bkA9DHyXWLM-WEMa26Y,57255
 mdbq/aggregation/df_types.py,sha256=rHLIgv82PJSFmDvXkZyOJAffXkFyyMyFO23w9tUt8EQ,7525
-mdbq/aggregation/mysql_types.py,sha256=kzUAGM4FYp77tA_dvHjZNGyPoUAOU1WY5QD0uanh9I4,10418
+mdbq/aggregation/mysql_types.py,sha256=h0BCAY98nzkCwoLCQczPz5cLY1jkU4w84Ovkg7dEu4Y,10477
 mdbq/aggregation/optimize_data.py,sha256=jLAWtxPUuhpo4XTVrhKtT4xK3grs7r73ePQfLhxlu1I,779
 mdbq/aggregation/query_data.py,sha256=sdMWj0st_VFDfBHmqTrY05k-0yoagdnaiNMoB0otEuk,25255
 mdbq/bdup/__init__.py,sha256=AkhsGk81SkG1c8FqDH5tRq-8MZmFobVbN60DTyukYTY,28
@@ -25,7 +25,7 @@ mdbq/mongo/__init__.py,sha256=SILt7xMtQIQl_m-ik9WLtJSXIVf424iYgCfE_tnQFbw,13
 mdbq/mongo/mongo.py,sha256=v9qvrp6p1ZRWuPpbSilqveiE0FEcZF7U5xUPI0RN4xs,31880
 mdbq/mysql/__init__.py,sha256=A_DPJyAoEvTSFojiI2e94zP0FKtCkkwKP1kYUCSyQzo,11
 mdbq/mysql/data_types_即将删除.py,sha256=sjBBDKr9674LdjM5N_dwyJACdZPbdB8Beli59jGdgnQ,10378
-mdbq/mysql/mysql.py,sha256=Urzkj_qq0-uxE0Z2xzVBzxpVySgn9HN-5zcNYrKh6G8,36340
+mdbq/mysql/mysql.py,sha256=Hj2ymZY15lsPQKCPOA9JMUOwOi8gTBq-LnEsFKW5-BY,36440
 mdbq/mysql/s_query.py,sha256=4c24SwbqtnO33o8CgWlTQ_j8sZYl5BRIQkaD9CI-vTY,7901
 mdbq/mysql/year_month_day.py,sha256=VgewoE2pJxK7ErjfviL_SMTN77ki8GVbTUcao3vFUCE,1523
 mdbq/other/__init__.py,sha256=jso1oHcy6cJEfa7udS_9uO5X6kZLoPBF8l3wCYmr5dM,18
@@ -36,7 +36,7 @@ mdbq/pbix/__init__.py,sha256=Trtfaynu9RjoTyLLYBN2xdRxTvm_zhCniUkVTAYwcjo,24
 mdbq/pbix/pbix_refresh.py,sha256=JUjKW3bNEyoMVfVfo77UhguvS5AWkixvVhDbw4_MHco,2396
 mdbq/pbix/refresh_all.py,sha256=tgy762608HMaXWynbOURIf2UVMuSPybzrDXQnOOcnZU,6102
 mdbq/spider/__init__.py,sha256=RBMFXGy_jd1HXZhngB2T2XTvJqki8P_Fr-pBcwijnew,18
-mdbq-1.2.2.dist-info/METADATA,sha256=8J7jj1T-AEP4LVtu84LnG_IvmoXV8JkXodk_F-iJr8I,245
-mdbq-1.2.2.dist-info/WHEEL,sha256=cpQTJ5IWu9CdaPViMhC9YzF8gZuS5-vlfoFihTBC86A,91
-mdbq-1.2.2.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
-mdbq-1.2.2.dist-info/RECORD,,
+mdbq-1.2.3.dist-info/METADATA,sha256=6nMtIPxpdSxm_Ws6LOo_EaoOkmfll2lYwrdLDsQ0RSU,245
+mdbq-1.2.3.dist-info/WHEEL,sha256=eOLhNAGa2EW3wWl_TU484h7q1UNgy0JXjjoqKoxAAQc,92
+mdbq-1.2.3.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
+mdbq-1.2.3.dist-info/RECORD,,

{mdbq-1.2.2.dist-info → mdbq-1.2.3.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (70.1.0)
+Generator: bdist_wheel (0.44.0)
 Root-Is-Purelib: true
 Tag: py3-none-any