PyPI - mdbq - Versions diffs - 1.3.8__py3-none-any.whl → 1.4.0__py3-none-any.whl - Mend

mdbq 1.3.8py3-none-any.whl → 1.4.0py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (8) hide show

mdbq/aggregation/aggregation.py +7 -2
mdbq/aggregation/optimize_data.py +49 -10
mdbq/company/copysh.py +36 -19
mdbq/mysql/mysql.py +2 -2
{mdbq-1.3.8.dist-info → mdbq-1.4.0.dist-info}/METADATA +1 -1
{mdbq-1.3.8.dist-info → mdbq-1.4.0.dist-info}/RECORD +8 -8
{mdbq-1.3.8.dist-info → mdbq-1.4.0.dist-info}/WHEEL +0 -0
{mdbq-1.3.8.dist-info → mdbq-1.4.0.dist-info}/top_level.txt +0 -0

mdbq/aggregation/aggregation.py CHANGED Viewed

@@ -549,6 +549,7 @@ class DatabaseUpdate:
                             '数据库名': db_name,
                             '集合名称': collection_name,
                             '数据主体': df,
+                            '文件名': name,
                         }
                     )
@@ -585,6 +586,7 @@ class DatabaseUpdate:
                                 '数据库名': db_name,
                                 '集合名称': collection_name,
                                 '数据主体': df,
+                                '文件名': name,
                             }
                         )
                     if is_move:
@@ -596,6 +598,7 @@ class DatabaseUpdate:
                 '数据库名': '聚合数据',
                 '集合名称': '日期表',
                 '数据主体': df,
+                '文件名': '日期表文件名',
             }
         )
@@ -641,7 +644,7 @@ class DatabaseUpdate:
                         port=port,
                     )
                     for data in self.datas:
-                        df, db_name, collection_name = data['数据主体'], data['数据库名'], data['集合名称']
+                        df, db_name, collection_name, rt_filename = data['数据主体'], data['数据库名'], data['集合名称'], data['文件名']
                         df_to_json.get_df_types(
                             df=df,
                             db_name=db_name,
@@ -653,7 +656,9 @@ class DatabaseUpdate:
                             db_name=db_name,
                             table_name=collection_name,
                             df_sql=False,  # 值为 True 时使用 df.to_sql 函数上传整个表, 不会排重
-                            drop_dup=True  # 值为 True 时检查重复数据再插入，反之直接上传
+                            drop_dup=False,  # 值为 True 时检查重复数据再插入，反之直接上传，会比较慢
+                            filename=rt_filename,  # 用来追踪处理进度
+                            system_name=system_name,  # 用来追踪处理进度
                         )
                 df_to_json.as_json_file()  # 写入 json 文件, 包含数据的 dtypes 信息

mdbq/aggregation/optimize_data.py CHANGED Viewed

@@ -6,16 +6,55 @@ from mdbq.config import get_myconf
 """
-def op_data(service_databases, days: int = 63):
-    for service_database in service_databases:
-        for service_name, database in service_database.items():
-            username, password, host, port = get_myconf.select_config_values(target_service=service_name, database=database)
-            s = mysql.OptimizeDatas(username=username, password=password, host=host, port=port)
-            s.db_name_lists = [
-                '聚合数据',
-            ]
-            s.days = days
-            s.optimize_list()
+def op_data(db_name_lists, service_databases=None, days: int = 63):
+    """ service_databases 这个参数暂时没有用 """
+    # for service_database in service_databases:
+    #     for service_name, database in service_database.items():
+    #         username, password, host, port = get_myconf.select_config_values(target_service=service_name, database=database)
+    #         s = mysql.OptimizeDatas(username=username, password=password, host=host, port=port)
+    #         s.db_name_lists = [
+    #             '聚合数据',
+    #         ]
+    #         s.days = days
+    #         s.optimize_list()
+    if socket.gethostname() == 'xigua_lx' or socket.gethostname() == 'xigua1' or socket.gethostname() == 'Mac2.local':
+        # mongodb
+        username, password, host, port = get_myconf.select_config_values(
+            target_service='home_lx',
+            database='mongodb',
+        )
+        m = mongo.OptimizeDatas(username=username, password=password, host=host, port=port)
+        m.db_name_lists = db_name_lists
+        m.days = days
+        m.optimize_list()
+        if m.client:
+            m.client.close()
+            print(f'已关闭 mongodb 连接')
+        if socket.gethostname() == 'xigua_lx':
+            restart_mongodb()  # mongodb 太占内存了, 重启服务， 释放内存
+        # Mysql
+        username, password, host, port = get_myconf.select_config_values(
+            target_service='home_lx',
+            database='mysql',
+        )
+        s = mysql.OptimizeDatas(username=username, password=password, host=host, port=port)
+        s.db_name_lists = db_name_lists
+        s.days = days
+        s.optimize_list()
+    elif socket.gethostname() == 'company':
+        # Mysql
+        username, password, host, port = get_myconf.select_config_values(
+            target_service='company',
+            database='mysql',
+        )
+        s = mysql.OptimizeDatas(username=username, password=password, host=host, port=port)
+        s.db_name_lists = db_name_lists
+        s.days = days
+        s.optimize_list()
 if __name__ == '__main__':

mdbq/company/copysh.py CHANGED Viewed

@@ -293,27 +293,44 @@ def op_data(days: int =3650):
     # 清理数据库， 除了 聚合数据
     if socket.gethostname() == 'company':  # 公司台式机自身运行
-        # Mysql
-        username, password, host, port = get_myconf.select_config_values(
-            target_service='company',
-            database='mysql',
+        # # Mysql
+        # username, password, host, port = get_myconf.select_config_values(
+        #     target_service='company',
+        #     database='mysql',
+        # )
+        # s = mysql.OptimizeDatas(username=username, password=password, host=host, port=port)
+        # s.db_name_lists = [
+        #     '京东数据2',
+        #     '推广数据2',
+        #     '市场数据2',
+        #     '生意参谋2',
+        #     '生意经2',
+        #     '属性设置2',
+        #     # '聚合数据',  # 不在这里清理聚合数据, 还未开始聚合呢
+        # ]
+        # s.days = days
+        # s.optimize_list()
+        # 清理所有非聚合数据的库
+        optimize_data.op_data(
+            db_name_lists=[
+                '京东数据2',
+                '推广数据2',
+                '市场数据2',
+                '生意参谋2',
+                '生意经2',
+                '属性设置2',
+                # '聚合数据',  # 不在这里清理聚合数据, 还未开始聚合呢
+            ],
+            days=3650,
         )
-        s = mysql.OptimizeDatas(username=username, password=password, host=host, port=port)
-        s.db_name_lists = [
-            '京东数据2',
-            '推广数据2',
-            '市场数据2',
-            '生意参谋2',
-            '生意经2',
-            '属性设置2',
-            # '聚合数据',  # 不在这里清理聚合数据, 还未开始聚合呢
-        ]
-        s.days = days
-        s.optimize_list()
-    # 聚合数据，并清理聚合数据
-    query_data.data_aggregation(service_databases=[{'company': 'mysql'}], months=1, system_name='company')
+        # 数据聚合
+        query_data.data_aggregation(service_databases=[{'home_lx': 'mysql'}], months=1, system_name='home_lx')
+        time.sleep(60)
+        # 清理聚合数据
+        optimize_data.op_data(db_name_lists=['聚合数据'], days=3650, )
 def main():
@@ -329,7 +346,7 @@ def main():
             dp = aggregation.DatabaseUpdate(path=d_path)
             dp.new_unzip(is_move=True)
             dp.cleaning(is_move=True)  # 公司台式机需要移除自身下载的文件
-            dp.upload_df(service_databases=[{'company': 'mysql'}])
+            dp.upload_df(service_databases=[{'company': 'mysql'}], system_name='company')
             # 此操作用于修改 .copysh_conf 文件，将 ch_record 改为 false (更新完成)
             w = update_conf.UpdateConf()

mdbq/mysql/mysql.py CHANGED Viewed

@@ -57,14 +57,14 @@ class MysqlUpload:
         }
         self.filename = None
-    def df_to_mysql(self, df, table_name, db_name='远程数据源', icm_update=[], icm_up=[], df_sql=False, drop_dup=True, drop_duplicates=False, filename=None, count=None, json_path=None, system_name=None):
+    def df_to_mysql(self, df, table_name, db_name='远程数据源', icm_update=[], icm_up=[], df_sql=False, drop_dup=False, drop_duplicates=False, filename=None, count=None, json_path=None, system_name=None):
         """
         将 df 写入数据库
         db_name: 数据库名称
         table_name: 集合/表名称
         df_sql: 这是一个临时参数, 值为 True 时使用 df.to_sql 函数上传整个表, 不会排重
         drop_duplicates：值为 True 时（仅限于聚合数据使用），其他情况不要设置此参数
-        drop_dup: 值为 True 时检查重复数据再插入，反之直接上传
+        drop_dup: 值为 True 时检查重复数据再插入，反之直接上传，数据量大时会比较慢
         filename: 传这个参数是方便定位产生错误的文件
         icm_update: 增量更新, 在聚合数据中使用，原始文件不要使用，设置此参数时需将 drop_dup 改为 False
                 使用增量更新: 必须确保 icm_update 传进来的列必须是数据表中唯一主键，值不会发生变化，不会重复，否则可能产生错乱覆盖情况

{mdbq-1.3.8.dist-info → mdbq-1.4.0.dist-info}/METADATA RENAMED Viewed

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: mdbq
-Version: 1.3.8
+Version: 1.4.0
 Home-page: https://pypi.org/project/mdbsql
 Author: xigua,
 Author-email: 2587125111@qq.com

{mdbq-1.3.8.dist-info → mdbq-1.4.0.dist-info}/RECORD RENAMED Viewed

@@ -1,17 +1,17 @@
 mdbq/__init__.py,sha256=Il5Q9ATdX8yXqVxtP_nYqUhExzxPC_qk_WXQ_4h0exg,16
 mdbq/__version__.py,sha256=y9Mp_8x0BCZSHsdLT_q5tX9wZwd5QgqrSIENLrb6vXA,62
 mdbq/aggregation/__init__.py,sha256=EeDqX2Aml6SPx8363J-v1lz0EcZtgwIBYyCJV6CcEDU,40
-mdbq/aggregation/aggregation.py,sha256=BcwU5cKMw-tJ8ibD3N9rX_N7tFkcQFW0nPHGuS2bMzI,57756
+mdbq/aggregation/aggregation.py,sha256=cEKhmlqvrChXlxyxEq5qNMAgL_3hp1xtt9heOxfu9Z4,58113
 mdbq/aggregation/df_types.py,sha256=oQJS2IBU3_IO6GMgbssHuC2yCjNnbta0QPGrFOwNLnU,7591
 mdbq/aggregation/mysql_types.py,sha256=jTQaCrDy9hj36GIImshXANyQNFAvVKJTybkzJSh-qJ8,10671
-mdbq/aggregation/optimize_data.py,sha256=jLAWtxPUuhpo4XTVrhKtT4xK3grs7r73ePQfLhxlu1I,779
+mdbq/aggregation/optimize_data.py,sha256=uv-KHSGazEpbUyCu2G3kjrdq0h0QLZY7BhIkcbqQqek,2360
 mdbq/aggregation/query_data.py,sha256=qSCN3Y20Ken4cmwG63kMnvz9jg115PB9qvo-hF9Aq34,27504
 mdbq/bdup/__init__.py,sha256=AkhsGk81SkG1c8FqDH5tRq-8MZmFobVbN60DTyukYTY,28
 mdbq/bdup/bdup.py,sha256=LAV0TgnQpc-LB-YuJthxb0U42_VkPidzQzAagan46lU,4234
 mdbq/clean/__init__.py,sha256=A1d6x3L27j4NtLgiFV5TANwEkLuaDfPHDQNrPBbNWtU,41
 mdbq/clean/data_clean.py,sha256=TnG0t6Ted9miMoBdGanpTmj6AUa0YFSyEBvmgN3V7sw,87558
 mdbq/company/__init__.py,sha256=qz8F_GsP_pMB5PblgJAUAMjasuZbOEp3qQOCB39E8f0,21
-mdbq/company/copysh.py,sha256=Uqkv9uJMe1HPgDI1kE_H4hJfG4T2MZjkb_SKnqo9zxs,16418
+mdbq/company/copysh.py,sha256=NJWEnxOZhbnqbHPQfCxBNAcl-LLLEk4Gn9QP35XlA0c,17041
 mdbq/config/__init__.py,sha256=jso1oHcy6cJEfa7udS_9uO5X6kZLoPBF8l3wCYmr5dM,18
 mdbq/config/get_myconf.py,sha256=bp6bVARZVm3ANj1pmM9hLB8Ao539TUWeM9xxeSsBpzw,5994
 mdbq/config/products.py,sha256=9gqXJMsw8KKuD4Xs6krNgcF7AuWDvV7clI6wVi3QjcA,4260
@@ -24,7 +24,7 @@ mdbq/log/mylogger.py,sha256=oaT7Bp-Hb9jZt52seP3ISUuxVcI19s4UiqTeouScBO0,3258
 mdbq/mongo/__init__.py,sha256=SILt7xMtQIQl_m-ik9WLtJSXIVf424iYgCfE_tnQFbw,13
 mdbq/mongo/mongo.py,sha256=v9qvrp6p1ZRWuPpbSilqveiE0FEcZF7U5xUPI0RN4xs,31880
 mdbq/mysql/__init__.py,sha256=A_DPJyAoEvTSFojiI2e94zP0FKtCkkwKP1kYUCSyQzo,11
-mdbq/mysql/mysql.py,sha256=SF29RXZHa6bTK27vPk25qBay89d4qKPvoWwJCRmcxC0,44313
+mdbq/mysql/mysql.py,sha256=kkkbFJHox0diPpDadwHicRQcDUxdH1h8MsMF1MNNBRI,44344
 mdbq/mysql/s_query.py,sha256=a33aYhW6gAnspIZfQ7l23ePln9-MD1f_ukypr5M0jd8,8018
 mdbq/mysql/year_month_day.py,sha256=VgewoE2pJxK7ErjfviL_SMTN77ki8GVbTUcao3vFUCE,1523
 mdbq/other/__init__.py,sha256=jso1oHcy6cJEfa7udS_9uO5X6kZLoPBF8l3wCYmr5dM,18
@@ -35,7 +35,7 @@ mdbq/pbix/__init__.py,sha256=Trtfaynu9RjoTyLLYBN2xdRxTvm_zhCniUkVTAYwcjo,24
 mdbq/pbix/pbix_refresh.py,sha256=JUjKW3bNEyoMVfVfo77UhguvS5AWkixvVhDbw4_MHco,2396
 mdbq/pbix/refresh_all.py,sha256=tgy762608HMaXWynbOURIf2UVMuSPybzrDXQnOOcnZU,6102
 mdbq/spider/__init__.py,sha256=RBMFXGy_jd1HXZhngB2T2XTvJqki8P_Fr-pBcwijnew,18
-mdbq-1.3.8.dist-info/METADATA,sha256=CM88kpobEabq8EMpIdtxhP1vHTUlktRtlQcCMJlsPEc,245
-mdbq-1.3.8.dist-info/WHEEL,sha256=cpQTJ5IWu9CdaPViMhC9YzF8gZuS5-vlfoFihTBC86A,91
-mdbq-1.3.8.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
-mdbq-1.3.8.dist-info/RECORD,,
+mdbq-1.4.0.dist-info/METADATA,sha256=2azUUrdXUdVs0FNaexwkZcRDo4_MnvwLYtX4MvSIGlU,245
+mdbq-1.4.0.dist-info/WHEEL,sha256=cpQTJ5IWu9CdaPViMhC9YzF8gZuS5-vlfoFihTBC86A,91
+mdbq-1.4.0.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
+mdbq-1.4.0.dist-info/RECORD,,

{mdbq-1.3.8.dist-info → mdbq-1.4.0.dist-info}/WHEEL RENAMED Viewed

File without changes

{mdbq-1.3.8.dist-info → mdbq-1.4.0.dist-info}/top_level.txt RENAMED Viewed

File without changes

mdbq 1.3.8__py3-none-any.whl → 1.4.0__py3-none-any.whl

mdbq 1.3.8py3-none-any.whl → 1.4.0py3-none-any.whl