mdbq 1.3.9__py3-none-any.whl → 1.4.1__py3-none-any.whl

mdbq/aggregation/optimize_data.py

@@ -1,21 +1,111 @@
 # -*- coding: UTF-8 –*-
+from mdbq.mongo import mongo
 from mdbq.mysql import mysql
 from mdbq.config import get_myconf
+import socket
+import subprocess
+import psutil
+import time
+import platform
 """
 对指定数据库所有冗余数据进行清理
 """
 
 
-def op_data(service_databases, days: int = 63):
-    for service_database in service_databases:
-        for service_name, database in service_database.items():
-            username, password, host, port = get_myconf.select_config_values(target_service=service_name, database=database)
-            s = mysql.OptimizeDatas(username=username, password=password, host=host, port=port)
-            s.db_name_lists = [
-                '聚合数据',
-            ]
-            s.days = days
-            s.optimize_list()
+def restart_mongodb():
+    """
+    检查服务, 并重启, 只能操作本机
+    """
+
+    def get_pid(program_name):
+        # macos 系统中，使用psutil.process_iter()遍历系统中所有运行的进程
+        for process in psutil.process_iter(['name', 'pid']):
+            if process.info['name'] == program_name:
+                return process.info['pid']
+        return None
+
+    if platform.system() == 'Windows':
+        print(f'即将重启 mongodb 服务')
+        time.sleep(60)
+        stop_command = f'net stop MongoDB'
+        subprocess.call(stop_command, shell=True)  # 停止MongoDB服务
+
+        time.sleep(30)
+        start_command = f'net start MongoDB'
+        subprocess.call(start_command, shell=True)  # 启动MongoDB服务
+        time.sleep(30)
+
+    elif platform.system() == 'Darwin':
+        print(f'即将重启 mongodb 服务')
+        time.sleep(60)
+        result = get_pid('mongod')  # 获取进程号
+        if result:  # 有 pid, 重启服务
+            command = f'kill {result}'
+            subprocess.call(command, shell=True)
+            time.sleep(10)
+            command = f'mongod --config /usr/local/mongodb/mongod.conf'
+            subprocess.call(command, shell=True)
+            # print('已重启服务')
+        else:  # 没有启动, 则启动服务
+            command = f'mongod --config /usr/local/mongodb/mongod.conf'
+            subprocess.call(command, shell=True)
+
+    elif platform.system() == 'Linux':
+        print(f'即将重启 mongodb 服务')
+        time.sleep(60)
+        command = f'service mongod restart'
+        subprocess.call(command, shell=True)
+
+
+def op_data(db_name_lists, service_databases=None, days: int = 63):
+    """ service_databases 这个参数暂时没有用 """
+    # for service_database in service_databases:
+    #     for service_name, database in service_database.items():
+    #         username, password, host, port = get_myconf.select_config_values(target_service=service_name, database=database)
+    #         s = mysql.OptimizeDatas(username=username, password=password, host=host, port=port)
+    #         s.db_name_lists = [
+    #             '聚合数据',
+    #         ]
+    #         s.days = days
+    #         s.optimize_list()
+
+    if socket.gethostname() == 'xigua_lx' or socket.gethostname() == 'xigua1' or socket.gethostname() == 'Mac2.local':
+        # mongodb
+        username, password, host, port = get_myconf.select_config_values(
+            target_service='home_lx',
+            database='mongodb',
+        )
+        m = mongo.OptimizeDatas(username=username, password=password, host=host, port=port)
+        m.db_name_lists = db_name_lists
+        m.days = days
+        m.optimize_list()
+        if m.client:
+            m.client.close()
+            print(f'已关闭 mongodb 连接')
+
+        if socket.gethostname() == 'xigua_lx':
+            restart_mongodb()  # mongodb 太占内存了, 重启服务， 释放内存
+
+        # Mysql
+        username, password, host, port = get_myconf.select_config_values(
+            target_service='home_lx',
+            database='mysql',
+        )
+        s = mysql.OptimizeDatas(username=username, password=password, host=host, port=port)
+        s.db_name_lists = db_name_lists
+        s.days = days
+        s.optimize_list()
+
+    elif socket.gethostname() == 'company':
+        # Mysql
+        username, password, host, port = get_myconf.select_config_values(
+            target_service='company',
+            database='mysql',
+        )
+        s = mysql.OptimizeDatas(username=username, password=password, host=host, port=port)
+        s.db_name_lists = db_name_lists
+        s.days = days
+        s.optimize_list()
 
 
 if __name__ == '__main__':

mdbq/company/copysh.py

@@ -293,27 +293,44 @@ def op_data(days: int =3650):
 
     # 清理数据库， 除了 聚合数据
     if socket.gethostname() == 'company':  # 公司台式机自身运行
-        # Mysql
-        username, password, host, port = get_myconf.select_config_values(
-            target_service='company',
-            database='mysql',
+        # # Mysql
+        # username, password, host, port = get_myconf.select_config_values(
+        #     target_service='company',
+        #     database='mysql',
+        # )
+        # s = mysql.OptimizeDatas(username=username, password=password, host=host, port=port)
+        # s.db_name_lists = [
+        #     '京东数据2',
+        #     '推广数据2',
+        #     '市场数据2',
+        #     '生意参谋2',
+        #     '生意经2',
+        #     '属性设置2',
+        #     # '聚合数据',  # 不在这里清理聚合数据, 还未开始聚合呢
+        # ]
+        # s.days = days
+        # s.optimize_list()
+
+        # 清理所有非聚合数据的库
+        optimize_data.op_data(
+            db_name_lists=[
+                '京东数据2',
+                '推广数据2',
+                '市场数据2',
+                '生意参谋2',
+                '生意经2',
+                '属性设置2',
+                # '聚合数据',  # 不在这里清理聚合数据, 还未开始聚合呢
+            ],
+            days=3650,
         )
-        s = mysql.OptimizeDatas(username=username, password=password, host=host, port=port)
-        s.db_name_lists = [
-            '京东数据2',
-            '推广数据2',
-            '市场数据2',
-            '生意参谋2',
-            '生意经2',
-            '属性设置2',
-            # '聚合数据',  # 不在这里清理聚合数据, 还未开始聚合呢
-        ]
-        s.days = days
-        s.optimize_list()
-
-    # 聚合数据，并清理聚合数据
-    query_data.data_aggregation(service_databases=[{'company': 'mysql'}], months=1, system_name='company')
 
+        # 数据聚合
+        query_data.data_aggregation(service_databases=[{'home_lx': 'mysql'}], months=1, system_name='home_lx')
+        time.sleep(60)
+
+        # 清理聚合数据
+        optimize_data.op_data(db_name_lists=['聚合数据'], days=3650, )
 
 
 def main():

{mdbq-1.3.9.dist-info → mdbq-1.4.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: mdbq
-Version: 1.3.9
+Version: 1.4.1
 Home-page: https://pypi.org/project/mdbsql
 Author: xigua, 
 Author-email: 2587125111@qq.com

{mdbq-1.3.9.dist-info → mdbq-1.4.1.dist-info}/RECORD

@@ -4,14 +4,14 @@ mdbq/aggregation/__init__.py,sha256=EeDqX2Aml6SPx8363J-v1lz0EcZtgwIBYyCJV6CcEDU,
 mdbq/aggregation/aggregation.py,sha256=cEKhmlqvrChXlxyxEq5qNMAgL_3hp1xtt9heOxfu9Z4,58113
 mdbq/aggregation/df_types.py,sha256=oQJS2IBU3_IO6GMgbssHuC2yCjNnbta0QPGrFOwNLnU,7591
 mdbq/aggregation/mysql_types.py,sha256=jTQaCrDy9hj36GIImshXANyQNFAvVKJTybkzJSh-qJ8,10671
-mdbq/aggregation/optimize_data.py,sha256=jLAWtxPUuhpo4XTVrhKtT4xK3grs7r73ePQfLhxlu1I,779
+mdbq/aggregation/optimize_data.py,sha256=MnGEO4csKYHJ8MzzfsOf-r7yQbjO4ugthwdI-g6PRr4,4144
 mdbq/aggregation/query_data.py,sha256=qSCN3Y20Ken4cmwG63kMnvz9jg115PB9qvo-hF9Aq34,27504
 mdbq/bdup/__init__.py,sha256=AkhsGk81SkG1c8FqDH5tRq-8MZmFobVbN60DTyukYTY,28
 mdbq/bdup/bdup.py,sha256=LAV0TgnQpc-LB-YuJthxb0U42_VkPidzQzAagan46lU,4234
 mdbq/clean/__init__.py,sha256=A1d6x3L27j4NtLgiFV5TANwEkLuaDfPHDQNrPBbNWtU,41
 mdbq/clean/data_clean.py,sha256=TnG0t6Ted9miMoBdGanpTmj6AUa0YFSyEBvmgN3V7sw,87558
 mdbq/company/__init__.py,sha256=qz8F_GsP_pMB5PblgJAUAMjasuZbOEp3qQOCB39E8f0,21
-mdbq/company/copysh.py,sha256=hcR3WcXmPAJVN1UEqLNm5FjJALiafPzwCZrne2i8YM0,16441
+mdbq/company/copysh.py,sha256=NJWEnxOZhbnqbHPQfCxBNAcl-LLLEk4Gn9QP35XlA0c,17041
 mdbq/config/__init__.py,sha256=jso1oHcy6cJEfa7udS_9uO5X6kZLoPBF8l3wCYmr5dM,18
 mdbq/config/get_myconf.py,sha256=bp6bVARZVm3ANj1pmM9hLB8Ao539TUWeM9xxeSsBpzw,5994
 mdbq/config/products.py,sha256=9gqXJMsw8KKuD4Xs6krNgcF7AuWDvV7clI6wVi3QjcA,4260
@@ -35,7 +35,7 @@ mdbq/pbix/__init__.py,sha256=Trtfaynu9RjoTyLLYBN2xdRxTvm_zhCniUkVTAYwcjo,24
 mdbq/pbix/pbix_refresh.py,sha256=JUjKW3bNEyoMVfVfo77UhguvS5AWkixvVhDbw4_MHco,2396
 mdbq/pbix/refresh_all.py,sha256=tgy762608HMaXWynbOURIf2UVMuSPybzrDXQnOOcnZU,6102
 mdbq/spider/__init__.py,sha256=RBMFXGy_jd1HXZhngB2T2XTvJqki8P_Fr-pBcwijnew,18
-mdbq-1.3.9.dist-info/METADATA,sha256=mxi5Z2e11Ajiwi3M8jyrnMF9P-6XajAu5cqCOyLwsck,245
-mdbq-1.3.9.dist-info/WHEEL,sha256=cpQTJ5IWu9CdaPViMhC9YzF8gZuS5-vlfoFihTBC86A,91
-mdbq-1.3.9.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
-mdbq-1.3.9.dist-info/RECORD,,
+mdbq-1.4.1.dist-info/METADATA,sha256=6CQIPEW8CigYNwESE70LBIjNpEAGVSQ1wR1HRkHA8Kk,245
+mdbq-1.4.1.dist-info/WHEEL,sha256=cpQTJ5IWu9CdaPViMhC9YzF8gZuS5-vlfoFihTBC86A,91
+mdbq-1.4.1.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
+mdbq-1.4.1.dist-info/RECORD,,