mdbq 1.4.4__py3-none-any.whl → 1.4.6__py3-none-any.whl

mdbq/aggregation/aggregation.py

@@ -594,15 +594,15 @@ class DatabaseUpdate:
                     if is_move:
                         os.remove(os.path.join(root, name))
 
-        df = self.date_table()  # 创建一个日期表
-        self.datas.append(
-            {
-                '数据库名': '聚合数据',
-                '集合名称': '日期表',
-                '数据主体': df,
-                '文件名': '日期表文件名',
-            }
-        )
+        # df = self.date_table()  # 创建一个日期表
+        # self.datas.append(
+        #     {
+        #         '数据库名': '聚合数据',
+        #         '集合名称': '日期表',
+        #         '数据主体': df,
+        #         '文件名': '日期表文件名',
+        #     }
+        # )
 
     def upload_df(self, service_databases=[{}], path=None, system_name=None):
         """
@@ -633,6 +633,8 @@ class DatabaseUpdate:
                             is_file_dtype=True,  # 默认本地文件优先: True
                         )
                         d.df_to_mongo(df=df, db_name=db_name, collection_name=collection_name)
+                    if d.client:
+                        d.client.close()
 
                 elif database == 'mysql':
                     username, password, host, port = get_myconf.select_config_values(
@@ -658,7 +660,7 @@ class DatabaseUpdate:
                             db_name=db_name,
                             table_name=collection_name,
                             df_sql=False,  # 值为 True 时使用 df.to_sql 函数上传整个表, 不会排重
-                            drop_dup=False,  # 值为 True 时检查重复数据再插入，反之直接上传，会比较慢
+                            drop_duplicates=True,  # 值为 True 时检查重复数据再插入，反之直接上传，会比较慢
                             filename=rt_filename,  # 用来追踪处理进度
                             system_name=system_name,  # 用来追踪处理进度
                         )
@@ -782,7 +784,7 @@ class DatabaseUpdate:
             encod = chardet.detect(f1).get('encoding')
         return encod
 
-    def date_table(self):
+    def date_table(self, service_databases=[{}]):
         """
         生成 pbix使用的日期表
         """
@@ -807,7 +809,29 @@ class DatabaseUpdate:
         df['索引'] = p
         df['月索引'] = mon
         df.sort_values('日期', ascending=False, ignore_index=True, inplace=True)
-        return df
+
+        for service_database in service_databases:
+            for service_name, database in service_database.items():
+                username, password, host, port = get_myconf.select_config_values(
+                    target_service=service_name,
+                    database=database,
+                )
+                m = mysql.MysqlUpload(
+                    username=username,
+                    password=password,
+                    host=host,
+                    port=port,
+                )
+                m.df_to_mysql(
+                    df=df,
+                    db_name='聚合数据',
+                    table_name='日期表',
+                    df_sql=False,  # 值为 True 时使用 df.to_sql 函数上传整个表, 不会排重
+                    drop_duplicates=True,  # 值为 True 时检查重复数据再插入，反之直接上传，会比较慢
+                    filename=None,  # 用来追踪处理进度
+                    system_name=service_name,  # 用来追踪处理进度
+                )
+        # return df
 
 
 def upload_dir(path, db_name, collection_name, dbs={'mysql': True, 'mongodb': True}, json_path=None, system_name=None):
@@ -888,10 +912,10 @@ def upload_dir(path, db_name, collection_name, dbs={'mysql': True, 'mongodb': Tr
 
                     if dbs['mongodb']:
                         d.df_to_mongo(df=df, db_name=db_name, collection_name=collection_name)
-                    if dbs['mysql']:  # drop_dup: 值为 True 时检查重复数据再插入
+                    if dbs['mysql']:  # drop_duplicates: 值为 True 时检查重复数据再插入
                         m.df_to_mysql(df=df, db_name=db_name, table_name=collection_name,
-                                      drop_dup=False, filename=name, count=f'{i}/{count}')
-                        # nas.df_to_mysql(df=df, db_name=db_name, table_name=collection_name)
+                                      drop_duplicates=True, filename=name, count=f'{i}/{count}')
+                        # nas.df_to_mysql(df=df, db_name=db_name, table_name=collection_name, drop_duplicates=True,)
                 except Exception as e:
                     print(name, e)
             i += 1
@@ -909,7 +933,7 @@ def one_file_to_mysql(file, db_name, table_name, target_service, database):
     filename = os.path.basename(file)
     df = pd.read_csv(file, encoding='utf-8_sig', header=0, na_filter=False, float_precision='high')
     m = mysql.MysqlUpload(username=username, password=password, host=host, port=port)
-    m.df_to_mysql(df=df, db_name=db_name, table_name=table_name, filename=filename, df_sql=True)
+    m.df_to_mysql(df=df, db_name=db_name, table_name=table_name, filename=filename, df_sql=True, drop_duplicates=False,)
 
 
 def file_dir(one_file=True):
@@ -1013,7 +1037,7 @@ if __name__ == '__main__':
     # username, password, host, port = get_myconf.select_config_values(target_service='company', database='mysql')
     # m = mysql.MysqlUpload(username=username, password=password, host=host, port=port)
     # m.df_to_mysql(df=df, db_name='test', table_name='增量更新测试',
-    #               drop_dup=False,
+    #               drop_duplicates=True,
     #               # icm_update=['日期', '推广费余额'],
     #               system_name='company',
     #               )

mdbq/aggregation/query_data.py

@@ -526,7 +526,7 @@ def data_aggregation_one(service_databases=[{}], months=1, system_name=None,):
                     df=df,
                     db_name=db_name,
                     table_name=table_name,
-                    drop_dup=False,
+                    drop_duplicates=False,
                     icm_update=unique_key_list,
                     system_name=system_name,
                     service_databases=service_databases,
@@ -596,7 +596,7 @@ def data_aggregation(service_databases=[{}], months=1, system_name=None,):
                     df=df,
                     db_name=db_name,
                     table_name=table_name,
-                    drop_dup=False,
+                    drop_duplicates=False,
                     icm_update=unique_key_list,
                     system_name=system_name,
                     service_databases=service_databases,
@@ -606,7 +606,7 @@ def data_aggregation(service_databases=[{}], months=1, system_name=None,):
                 df=res,
                 db_name='聚合数据',
                 table_name='_全店商品销售',
-                drop_dup=False,
+                drop_duplicates=False,
                 icm_update=['日期', '商品id'],  # 设置唯一主键
                 system_name = system_name,
                 service_databases=service_databases,
@@ -616,7 +616,7 @@ def data_aggregation(service_databases=[{}], months=1, system_name=None,):
                 df=res,
                 db_name='聚合数据',
                 table_name='_推广商品销售',
-                drop_dup=False,
+                drop_duplicates=False,
                 icm_update=['日期', '商品id'],  # 设置唯一主键
                 system_name=system_name,
                 service_databases=service_databases,
@@ -627,6 +627,6 @@ def data_aggregation(service_databases=[{}], months=1, system_name=None,):
 
 
 if __name__ == '__main__':
-    # data_aggregation(service_databases=[{'company': 'mysql'}], months=1, system_name='company')
-    data_aggregation_one(service_databases=[{'company': 'mysql'}], months=10, system_name='company')
+    data_aggregation(service_databases=[{'home_lx': 'mysql'}], months=1, system_name='home_lx')
+    # data_aggregation_one(service_databases=[{'company': 'mysql'}], months=10, system_name='company')
     # optimize_data.op_data(service_databases=[{'company': 'mysql'}], days=3650)  # 立即启动对聚合数据的清理工作

mdbq/company/copysh.py

@@ -347,6 +347,7 @@ def main():
             dp.new_unzip(is_move=True)
             dp.cleaning(is_move=True)  # 公司台式机需要移除自身下载的文件
             dp.upload_df(service_databases=[{'company': 'mysql'}], system_name='company')
+            dp.date_table(service_databases=[{'company': 'mysql'}])  # 因为日期表不受 days 参数控制，因此单独更新日期表
 
             # 此操作用于修改 .copysh_conf 文件，将 ch_record 改为 false (更新完成)
             w = update_conf.UpdateConf()

mdbq/mysql/mysql.py

@@ -57,17 +57,19 @@ class MysqlUpload:
         }
         self.filename = None
 
-    def df_to_mysql(self, df, table_name, db_name='远程数据源', icm_update=[], icm_up=[], service_databases=[{'home_lx': 'mysql'}], df_sql=False, drop_dup=False, drop_duplicates=False, filename=None, count=None, json_path=None, system_name=None):
+    def df_to_mysql(self, df, table_name, db_name='远程数据源', icm_update=[], service_databases=[{'home_lx': 'mysql'}], df_sql=False, drop_duplicates=False, filename=None, count=None, json_path=None, system_name=None):
         """
         将 df 写入数据库
         db_name: 数据库名称
         table_name: 集合/表名称
-        df_sql: 这是一个临时参数, 值为 True 时使用 df.to_sql 函数上传整个表, 不会排重
-        drop_duplicates：值为 True 时（仅限于聚合数据使用），其他情况不要设置此参数
-        drop_dup: 值为 True 时检查重复数据再插入，反之直接上传，数据量大时会比较慢
-        filename: 传这个参数是方便定位产生错误的文件
-        icm_update: 增量更新, 在聚合数据中使用，原始文件不要使用，设置此参数时需将 drop_dup 改为 False
+        df_sql: 这是一个临时参数, 值为 True 时使用 df.to_sql 函数上传整个表, 不会排重，初创表大量上传数据的时候使用
+        drop_duplicates: 值为 True 时检查重复数据再插入，反之直接上传，数据量大时会比较慢
+        icm_update: 增量更新, 在聚合数据中使用，原始文件不要使用，设置此参数时需将 drop_duplicates 改为 False
                 使用增量更新: 必须确保 icm_update 传进来的列必须是数据表中唯一主键，值不会发生变化，不会重复，否则可能产生错乱覆盖情况
+        filename: 用来追踪处理进度，传这个参数是方便定位产生错误的文件
+        system_name: 同样是用来追踪处理进度
+        service_databases: 这个参数是用来设置更新哪台服务器的 types 信息到本地 json 文件
+        json_path: 这个参数同样也是是用来设置更新 json 文件
         """
         self.filename = filename
         if isinstance(df, pd.DataFrame):
@@ -81,19 +83,6 @@ class MysqlUpload:
         cv = converter.DataFrameConverter()
         df = cv.convert_df_cols(df=df)  # 清理 dataframe 非法值
 
-        # if df_sql:
-        #     now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S ")
-        #     print(f'{now}正在更新 mysql ({self.host}:{self.port}) {db_name}/{table_name}, {count},{self.filename}')
-        #     engine = create_engine(f"mysql+pymysql://{self.username}:{self.password}@{self.host}:{self.port}/{db_name}")  # 创建数据库引擎
-        #     df.to_sql(
-        #         name=table_name,
-        #         con=engine,
-        #         if_exists='append',
-        #         index=False,
-        #         chunksize=1000,
-        #         dtype={'京东价': 'INT'},
-        #     )
-        #     return
         connection = pymysql.connect(**self.config)  # 连接数据库
         with connection.cursor() as cursor:
             cursor.execute(f"SHOW DATABASES LIKE '{db_name}'")  # 检查数据库是否存在
@@ -173,14 +162,14 @@ class MysqlUpload:
             elif cl:
                 mysql_types.mysql_all_dtypes(service_databases=service_databases, system_name=system_name)  # 更新所有数据库所有数据表的 dtypes 信息到本地 json
 
-            # 4. 移除指定日期范围内的数据，仅限于聚合数据使用，其他情况不要设置
-            if drop_duplicates and '日期' in df.columns.tolist():
-                dates = df['日期'].values.tolist()
-                start_date = pd.to_datetime(min(dates)).strftime('%Y-%m-%d')
-                end_date = (pd.to_datetime(max(dates)) + datetime.timedelta(days=1)).strftime('%Y-%m-%d')
-                sql = f"DELETE FROM `{table_name}` WHERE {'日期'} BETWEEN '%s' AND '%s'" % (start_date, end_date)
-                cursor.execute(sql)
-                connection.commit()
+            # # 4. 移除指定日期范围内的数据，仅限于聚合数据使用，其他情况不要设置
+            # if drop_duplicates and '日期' in df.columns.tolist():
+            #     dates = df['日期'].values.tolist()
+            #     start_date = pd.to_datetime(min(dates)).strftime('%Y-%m-%d')
+            #     end_date = (pd.to_datetime(max(dates)) + datetime.timedelta(days=1)).strftime('%Y-%m-%d')
+            #     sql = f"DELETE FROM `{table_name}` WHERE {'日期'} BETWEEN '%s' AND '%s'" % (start_date, end_date)
+            #     cursor.execute(sql)
+            #     connection.commit()
 
             # 5. 更新插入数据
             now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S ")
@@ -200,7 +189,7 @@ class MysqlUpload:
                     condition = ' AND '.join(condition)  # 构建查询条件
                     # print(condition)
 
-                    if drop_dup:  # 查重插入
+                    if drop_duplicates:  # 查重插入
                         sql = "SELECT %s FROM %s WHERE %s" % (cols, table_name, condition)
                         # sql = f"SELECT {cols} FROM `{table_name}` WHERE `创建时间` = '2014-09-19 14:32:33'"
                         # print(sql)
@@ -211,7 +200,7 @@ class MysqlUpload:
                             cursor.execute(sql)
                         # else:
                         #     print(f'重复数据不插入: {condition[:50]}...')
-                    elif icm_update:  # 增量更新
+                    elif icm_update:  # 增量更新, 专门用于聚合数据，其他库不要调用
                         """ 使用增量更新: 需确保 icm_update['主键'] 传进来的列必须是数据表中唯一主键，值不会发生变化且不会重复，否则可能产生覆盖情况 """
                         sql = 'SELECT COLUMN_NAME FROM information_schema.columns WHERE table_schema = %s AND table_name = %s'
                         cursor.execute(sql, (db_name, {table_name}))
@@ -261,26 +250,6 @@ class MysqlUpload:
                         else:  # 没有数据返回，则直接插入数据
                             sql = f"INSERT INTO `{table_name}` ({cols}) VALUES ({values});"
                             cursor.execute(sql)
-                    # elif icm_up:
-                    #     sql = 'SELECT COLUMN_NAME FROM information_schema.columns WHERE table_schema = %s AND table_name = %s'
-                    #     cursor.execute(sql, (db_name, {table_name}))
-                    #     columns = cursor.fetchall()
-                    #     cols_exist = [col['COLUMN_NAME'] for col in columns]  # 数据表的所有列, 返回 list
-                    #     cols_exist = [item for item in cols_exist if item != 'id']
-                    #     update_col = [item for item in cols_exist if item not in icm_up]  # 除了主键外的其他列
-                    #
-                    #     unique_keys = ', '.join([f"`{item}`" for item in cols_exist])
-                    #     unique_keys_values = ', '.join([f"'{data[item]}'" for item in cols_exist])
-                    #
-                    #     change_values = []
-                    #     for col in update_col:
-                    #         change_values += [f"`{col}` = '{str(data[col])}'"]
-                    #     change_values = ', '.join(f"{item}" for item in change_values)  # 注意这里 item 外面没有反引号
-                    #     # print(change_values)
-                    #     sql = f"INSERT INTO `{table_name}` ({unique_keys}) VALUES ({unique_keys_values}) ON DUPLICATE KEY UPDATE {change_values};"
-                    #     print(sql)
-                    #     # cursor.execute(sql)
-
                     else:
                         sql = f"INSERT INTO `{table_name}` ({cols}) VALUES (%s);" % (values)
                         cursor.execute(sql)
@@ -834,4 +803,17 @@ def download_datas_bak(table_name, save_path, start_date):
 
 if __name__ == '__main__':
     username, password, host, port = get_myconf.select_config_values(target_service='home_lx', database='mysql')
-    print(username, password, host, port)
+    # print(username, password, host, port)
+    path = '/Users/xigua/Downloads/人群洞察.csv'
+    df = pd.read_csv(path, encoding='utf-8_sig', header=0, na_filter=False)
+    # print(df)
+    m = MysqlUpload(username=username, password=password, host=host, port=port)
+    m.df_to_mysql(
+        df=df,
+        db_name='test',
+        table_name='测试数据',
+        drop_duplicates=True,
+        # system_name=system_name,
+        # service_databases=service_databases,
+    )
+

{mdbq-1.4.4.dist-info → mdbq-1.4.6.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: mdbq
-Version: 1.4.4
+Version: 1.4.6
 Home-page: https://pypi.org/project/mdbsql
 Author: xigua, 
 Author-email: 2587125111@qq.com

{mdbq-1.4.4.dist-info → mdbq-1.4.6.dist-info}/RECORD

@@ -1,17 +1,17 @@
 mdbq/__init__.py,sha256=Il5Q9ATdX8yXqVxtP_nYqUhExzxPC_qk_WXQ_4h0exg,16
 mdbq/__version__.py,sha256=y9Mp_8x0BCZSHsdLT_q5tX9wZwd5QgqrSIENLrb6vXA,62
 mdbq/aggregation/__init__.py,sha256=EeDqX2Aml6SPx8363J-v1lz0EcZtgwIBYyCJV6CcEDU,40
-mdbq/aggregation/aggregation.py,sha256=xm9H2bD9GjKQd-sHQ5WIeskzagzaHwYMuokZnhGdFY8,58303
+mdbq/aggregation/aggregation.py,sha256=uGLS4Zn8oX0TmFxo2G_bDuA7hok4afiZajJZDLxmSNg,59528
 mdbq/aggregation/df_types.py,sha256=oQJS2IBU3_IO6GMgbssHuC2yCjNnbta0QPGrFOwNLnU,7591
 mdbq/aggregation/mysql_types.py,sha256=3j_SUTi7qYJyY3JtV_lRLIyczIczmRPE_WaIrXiZ6Rw,11425
 mdbq/aggregation/optimize_data.py,sha256=u2Kl_MFtZueXJ57ycy4H2OhXD431RctUYJYCl637uT0,4176
-mdbq/aggregation/query_data.py,sha256=2SFeTkgsNgQVjNVDi1K2zTsNSw9dwL19Xj7Peu7sfRg,29828
+mdbq/aggregation/query_data.py,sha256=vqAf-FRSnA8umAfhKZn-21AWjmXozcQT5Ec0gwEThzo,29856
 mdbq/bdup/__init__.py,sha256=AkhsGk81SkG1c8FqDH5tRq-8MZmFobVbN60DTyukYTY,28
 mdbq/bdup/bdup.py,sha256=LAV0TgnQpc-LB-YuJthxb0U42_VkPidzQzAagan46lU,4234
 mdbq/clean/__init__.py,sha256=A1d6x3L27j4NtLgiFV5TANwEkLuaDfPHDQNrPBbNWtU,41
 mdbq/clean/data_clean.py,sha256=BIzc1XCJjJaZyPT6DCRXRCCRwBaeC5_lER0aqYF1P3M,87778
 mdbq/company/__init__.py,sha256=qz8F_GsP_pMB5PblgJAUAMjasuZbOEp3qQOCB39E8f0,21
-mdbq/company/copysh.py,sha256=XBnSalNHJPCBkswyiDpHhhVPqxRMOGcbAncWytrhCyQ,17039
+mdbq/company/copysh.py,sha256=RWlQqfpqDiRO1QaVCbPhgAYSopnMoOgcnFhRi0TNFnM,17181
 mdbq/config/__init__.py,sha256=jso1oHcy6cJEfa7udS_9uO5X6kZLoPBF8l3wCYmr5dM,18
 mdbq/config/get_myconf.py,sha256=bp6bVARZVm3ANj1pmM9hLB8Ao539TUWeM9xxeSsBpzw,5994
 mdbq/config/products.py,sha256=9gqXJMsw8KKuD4Xs6krNgcF7AuWDvV7clI6wVi3QjcA,4260
@@ -24,7 +24,7 @@ mdbq/log/mylogger.py,sha256=oaT7Bp-Hb9jZt52seP3ISUuxVcI19s4UiqTeouScBO0,3258
 mdbq/mongo/__init__.py,sha256=SILt7xMtQIQl_m-ik9WLtJSXIVf424iYgCfE_tnQFbw,13
 mdbq/mongo/mongo.py,sha256=v9qvrp6p1ZRWuPpbSilqveiE0FEcZF7U5xUPI0RN4xs,31880
 mdbq/mysql/__init__.py,sha256=A_DPJyAoEvTSFojiI2e94zP0FKtCkkwKP1kYUCSyQzo,11
-mdbq/mysql/mysql.py,sha256=PXgE5mjLzzFDzor7DmcudEYz0pwHZRiiFomdkBmGj7U,44497
+mdbq/mysql/mysql.py,sha256=flsp6ol0SpEfZpPTyvjYpcU67CNb26rUavAXxw5Y4VE,43123
 mdbq/mysql/s_query.py,sha256=a33aYhW6gAnspIZfQ7l23ePln9-MD1f_ukypr5M0jd8,8018
 mdbq/mysql/year_month_day.py,sha256=VgewoE2pJxK7ErjfviL_SMTN77ki8GVbTUcao3vFUCE,1523
 mdbq/other/__init__.py,sha256=jso1oHcy6cJEfa7udS_9uO5X6kZLoPBF8l3wCYmr5dM,18
@@ -35,7 +35,7 @@ mdbq/pbix/__init__.py,sha256=Trtfaynu9RjoTyLLYBN2xdRxTvm_zhCniUkVTAYwcjo,24
 mdbq/pbix/pbix_refresh.py,sha256=JUjKW3bNEyoMVfVfo77UhguvS5AWkixvVhDbw4_MHco,2396
 mdbq/pbix/refresh_all.py,sha256=tgy762608HMaXWynbOURIf2UVMuSPybzrDXQnOOcnZU,6102
 mdbq/spider/__init__.py,sha256=RBMFXGy_jd1HXZhngB2T2XTvJqki8P_Fr-pBcwijnew,18
-mdbq-1.4.4.dist-info/METADATA,sha256=k_rFWJWIrV84IBHNX4pfTrN9L3uoFh31MlFapUllfgs,245
-mdbq-1.4.4.dist-info/WHEEL,sha256=eOLhNAGa2EW3wWl_TU484h7q1UNgy0JXjjoqKoxAAQc,92
-mdbq-1.4.4.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
-mdbq-1.4.4.dist-info/RECORD,,
+mdbq-1.4.6.dist-info/METADATA,sha256=vQjJtP-PX-XLQgL2rZMIMnpav-BKqdoRoHomvHN0brA,245
+mdbq-1.4.6.dist-info/WHEEL,sha256=cpQTJ5IWu9CdaPViMhC9YzF8gZuS5-vlfoFihTBC86A,91
+mdbq-1.4.6.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
+mdbq-1.4.6.dist-info/RECORD,,

{mdbq-1.4.4.dist-info → mdbq-1.4.6.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: bdist_wheel (0.44.0)
+Generator: setuptools (70.1.0)
 Root-Is-Purelib: true
 Tag: py3-none-any