mns-scheduler 1.1.6.5__tar.gz → 1.1.6.7__tar.gz

{mns_scheduler-1.1.6.5 → mns_scheduler-1.1.6.7}/PKG-INFO

@@ -1,3 +1,3 @@
 Metadata-Version: 2.1
 Name: mns-scheduler
-Version: 1.1.6.5
+Version: 1.1.6.7

{mns_scheduler-1.1.6.5 → mns_scheduler-1.1.6.7}/mns_scheduler/concept/clean/ths_concept_clean_api.py

@@ -77,7 +77,12 @@ def update_one_concept_relevance_industry(concept_code):
     ths_stock_concept_detail_df['second_relevance_industry'] = second_relevance_industry
 
     mongodb_util.save_mongo(ths_concept_list_one_df, db_name_constant.THS_CONCEPT_LIST)
-    ths_stock_concept_detail_df['valid'] = True
+
+    ths_stock_concept_detail_df['grade'] = 1
+    # 详细标识
+    ths_stock_concept_detail_df['remark'] = ''
+    # 简单标识
+    ths_stock_concept_detail_df['remark_flag'] = ''
     mongodb_util.save_mongo(ths_stock_concept_detail_df, db_name_constant.THS_STOCK_CONCEPT_DETAIL)
 
 

{mns_scheduler-1.1.6.5 → mns_scheduler-1.1.6.7}/mns_scheduler/concept/ths/common/ths_concept_sync_common_api.py

@@ -135,7 +135,11 @@ def save_ths_concept_detail(new_concept_symbol_df,
     query_detail = {"concept_code": int(concept_code)}
     exist_concept_detail = mongodb_util.find_query_data(db_name_constant.THS_STOCK_CONCEPT_DETAIL, query_detail)
     if exist_concept_detail is None or exist_concept_detail.shape[0] == 0:
-        new_concept_symbol_df['valid'] = True
+        new_concept_symbol_df['grade'] = 1
+        # 详细标识
+        new_concept_symbol_df['remark'] = ''
+        # 简单标识
+        new_concept_symbol_df['remark_flag'] = ''
         mongodb_util.save_mongo(new_concept_symbol_df, db_name_constant.THS_STOCK_CONCEPT_DETAIL)
         # 保存到当日新增概念列表
         new_concept_symbol_df['concept_type'] = 'ths'
@@ -145,7 +149,11 @@ def save_ths_concept_detail(new_concept_symbol_df,
         new_concept_symbol_df = new_concept_symbol_df.loc[~(
             new_concept_symbol_df['symbol'].isin(exist_concept_detail_symbol_list))]
         if new_concept_symbol_df.shape[0] > 0:
-            new_concept_symbol_df['valid'] = True
+            new_concept_symbol_df['grade'] = 1
+            # 详细标识
+            new_concept_symbol_df['remark'] = ''
+            # 简单标识
+            new_concept_symbol_df['remark_flag'] = ''
             mongodb_util.save_mongo(new_concept_symbol_df, db_name_constant.THS_STOCK_CONCEPT_DETAIL)
             # 保存到当日新增概念列表
             new_concept_symbol_df['concept_type'] = 'ths'

mns_scheduler-1.1.6.7/mns_scheduler/db/script/sync/local_mongo_util.py

@@ -0,0 +1,231 @@
+import sys
+import os
+
+file_path = os.path.abspath(__file__)
+end = file_path.index('mns') + 14
+project_path = file_path[0:end]
+sys.path.append(project_path)
+import pandas as pd
+import pymongo
+from mns_common.utils.async_fun import async_fun
+from loguru import logger
+import warnings
+
+warnings.filterwarnings("ignore")
+
+
+class LocalMongodbUtil:
+    def __init__(self, port):
+        self.port = port
+
+    def get_db(self):
+        client = pymongo.MongoClient("mongodb://127.0.0.1:" + '27017' + "/patience")
+        return client.patience
+
+    def group(self, query, coll_name):
+        db = self.get_db()
+        collection = db[coll_name]
+        data = collection.aggregate(query)
+        return pd.DataFrame(list(data))
+
+    def remove_data(self, query, coll_name):
+        db = self.get_db()
+        collection = db[coll_name]
+        return collection.delete_many(query)
+
+    def exist_data_query(self, coll_name, query):
+        db = self.get_db()
+        collection = db[coll_name]
+        return collection.count(query, limit=1) > 0
+
+    def find_one(self, coll_name, _id):
+        db = self.get_db()
+        collection = db[coll_name]
+        return collection.find_one({'_id': _id})
+
+    def find_one_query(self, coll_name, query):
+        db = self.get_db()
+        collection = db[coll_name]
+        return pd.DataFrame(collection.find_one(query), index=[0])
+
+    def find_all_data(self, coll_name):
+        db = self.get_db()
+        collection = db[coll_name]
+        rows = collection.find({})
+        df = pd.DataFrame([basic for basic in rows])
+        return df
+
+    def find_query_data(self, coll_name, query):
+        db = self.get_db()
+        collection = db[coll_name]
+        rows = collection.find(query)
+        df = pd.DataFrame(list(rows))
+        return df
+
+    def find_query_data_choose_field(self, coll_name, query, query_field):
+        db = self.get_db()
+        collection = db[coll_name]
+        rows = collection.find(query, query_field)
+        df = pd.DataFrame(list(rows))
+        return df
+
+    def find_query_data_list(self, coll_name, query):
+        db = self.get_db()
+        collection = db[coll_name]
+        rows = collection.find(query)
+        return list(rows)
+
+    def find_query_data_list(self, coll_name, query):
+        db = self.get_db()
+        collection = db[coll_name]
+        rows = collection.find(query)
+        return list(rows)
+
+    def remove_all_data(self, database):
+        db = self.get_db()
+        collection = db[database]
+        query = {"_id": {"$ne": "null"}}
+        return collection.delete_many(query)
+
+    def drop_collection(self, database):
+        db = self.get_db()
+        collection = db[database]
+        collection.drop()
+
+    def ascend_query(self, query, coll_name, field, num):
+        db = self.get_db()
+        collection = db[coll_name]
+        return pd.DataFrame(list(collection.find(query).sort(field, 1).skip(0).limit(num)));
+
+    def descend_query(self, query, coll_name, field, num):
+        db = self.get_db()
+        collection = db[coll_name]
+        return pd.DataFrame(list(collection.find(query).sort(field, -1).skip(0).limit(num)));
+
+    def count(self, query, coll_name):
+        db = self.get_db()
+        collection = db[coll_name]
+        return collection.count_documents(query)
+
+    def query_max(self, query, coll_name, field, num):
+        db = self.get_db()
+        collection = db[coll_name]
+        return pd.DataFrame(list(collection.find(query).sort(field, -1).skip(0).limit(num)));
+
+    def query_min(self, query, coll_name, field):
+        db = self.get_db()
+        collection = db[coll_name]
+        return pd.DataFrame(list(collection.find(query).sort(field, 1).skip(0).limit(1)));
+
+    def insert_mongo(self, df, database):
+        db = self.get_db()
+        if df is None or len(df) == 0:
+            return
+        collection = db[database]
+        # 格式转换
+        try:
+            df = df.drop_duplicates()
+            # df = df.T.drop_duplicates().T
+            records = df.to_dict('records')
+            collection.insert_many(records)
+        except BaseException as e:
+            logger.error("插入数据异常:{}", e)
+
+    def insert_mongo_json(self, json, database):
+        db = self.get_db()
+        collection = db[database]
+        # 格式转换
+        try:
+            collection.insert_many(json)
+        except BaseException as e:
+            logger.error("插入数据异常:{}", e)
+
+    def save_mongo_json(self, json, database):
+        db = self.get_db()
+        collection = db[database]
+        for record in json:
+            try:
+                collection.save(record)
+            except BaseException as e:
+                logger.error("保存数据出现异常:{}", e)
+
+    def save_mongo(self, df, database):
+        db = self.get_db()
+        if df is None or len(df) == 0:
+            return
+        collection = db[database]
+        # df = df.T.drop_duplicates().T
+        # 格式转换
+        records = df.to_dict('records')
+        for record in records:
+            try:
+                collection.save(record)
+            except BaseException as e:
+                logger.error("保存数据出现异常:{},{}", record, e)
+
+    def save_mongo_no_catch_exception(self, df, database):
+        db = self.get_db()
+        if df is None or len(df) == 0:
+            return
+        collection = db[database]
+        # df = df.T.drop_duplicates().T
+        # 格式转换
+        records = df.to_dict('records')
+        for record in records:
+            collection.save(record)
+
+    def update_one(self, df, database):
+        db = self.get_db()
+        condition = {'_id': list(df['_id'])[0]}
+        if len(df) == 0:
+            return
+        collection = db[database]
+        collection.update(condition, df)
+
+    def update_many(self, query, new_values, database):
+        db = self.get_db()
+        collection = db[database]
+        x = collection.update_many(query, new_values)
+        return x
+
+    @async_fun
+    def update_one_query(self, query, new_values, database):
+        db = self.get_db()
+        collection = db[database]
+        x = collection.update(query, new_values)
+        return x
+
+    def distinct_field(self, database, field, query):
+        db = self.get_db()
+        collection = db[database]
+        return collection.distinct(field, query)
+
+    def create_index(self, database, index):
+        db = self.get_db()
+        collection = db[database]
+        collection.create_index(
+            index)
+
+    def aggregate(self, pipeline, database):
+        db = self.get_db()
+        collection = db[database]
+        data = collection.aggregate(pipeline)
+        return pd.DataFrame(list(data))
+
+    def get_col_keys(self, database):
+        db = self.get_db()
+        collection = db[database]
+        keys = collection.find_one().keys()
+        return keys
+
+    # 分页查询 descend 是否降序
+    def find_page_skip_data(self, coll_name, page_query, page, page_number, field, descend):
+        db = self.get_db()
+        collection = db[coll_name]
+        if descend:
+            sort_tag = -1
+        else:
+            sort_tag = 1
+        rows = collection.find(page_query).sort(field, sort_tag).skip((page - 1) * page_number).limit(page_number)
+        df = pd.DataFrame(list(rows))
+        return df

mns_scheduler-1.1.6.7/mns_scheduler/db/script/sync/remote_data_sync_to_local.py

@@ -0,0 +1,52 @@
+import sys
+import os
+
+file_path = os.path.abspath(__file__)
+end = file_path.index('mns') + 16
+project_path = file_path[0:end]
+sys.path.append(project_path)
+import mns_common.utils.data_frame_util as data_frame_util
+from mns_scheduler.db.script.sync.remote_mongo_util import RemoteMongodbUtil
+from mns_scheduler.db.script.sync.local_mongo_util import LocalMongodbUtil
+from loguru import logger
+
+remote_mongodb_util = RemoteMongodbUtil('27017')
+local_mongodb_util = LocalMongodbUtil('27017')
+
+col_list = [
+    'company_remark_info',
+    'company_holding_info',
+    'industry_concept_remark',
+    'trade_date_list',
+    'company_info',
+    'de_list_stock',
+    'kpl_best_choose_index',
+    'kpl_best_choose_index_detail',
+    'realtime_quotes_now_zt_new_kc_open',
+    'industry_concept_remark',
+    'self_black_stock',
+    'self_choose_plate',
+    'self_choose_stock',
+    'stock_account_info',
+    'ths_concept_list',
+    'stock_zt_pool',
+    'ths_stock_concept_detail'
+]
+
+
+def remote_data():
+    for col in col_list:
+        try:
+            col_df = remote_mongodb_util.find_all_data(col)
+            if data_frame_util.is_not_empty(col_df):
+                result = local_mongodb_util.remove_all_data(col)
+                if result.acknowledged:
+                    local_mongodb_util.save_mongo(col_df, col)
+
+                logger.info("同步集合完成:{}", col)
+        except BaseException as e:
+            logger.error("同步失败:{},{}", e, col)
+
+
+if __name__ == '__main__':
+    remote_data()

mns_scheduler-1.1.6.7/mns_scheduler/db/script/sync/remote_mongo_util.py

@@ -0,0 +1,306 @@
+import sys
+import os
+
+file_path = os.path.abspath(__file__)
+end = file_path.index('mns') + 14
+project_path = file_path[0:end]
+sys.path.append(project_path)
+import pandas as pd
+import pymongo
+from mns_common.utils.async_fun import async_fun
+from loguru import logger
+import warnings
+
+warnings.filterwarnings("ignore")
+
+
+class RemoteMongodbUtil:
+    def __init__(self, port):
+        self.port = port
+
+    def get_db(self):
+        client = pymongo.MongoClient("mongodb://100.87.2.149:" + '27017' + "/patience")
+        return client.patience
+
+    def group(self, query, coll_name):
+        db = self.get_db()
+        collection = db[coll_name]
+        data = collection.aggregate(query)
+        return pd.DataFrame(list(data))
+
+    def remove_data(self, query, coll_name):
+        db = self.get_db()
+        collection = db[coll_name]
+        return collection.delete_many(query)
+
+    def exist_data_query(self, coll_name, query):
+        db = self.get_db()
+        collection = db[coll_name]
+        return collection.count(query, limit=1) > 0
+
+    def find_one(self, coll_name, _id):
+        db = self.get_db()
+        collection = db[coll_name]
+        return collection.find_one({'_id': _id})
+
+    def find_one_query(self, coll_name, query):
+        db = self.get_db()
+        collection = db[coll_name]
+        return pd.DataFrame(collection.find_one(query), index=[0])
+
+    def find_all_data(self, coll_name):
+        db = self.get_db()
+        collection = db[coll_name]
+        rows = collection.find({})
+        df = pd.DataFrame([basic for basic in rows])
+        return df
+
+    def find_query_data(self, coll_name, query):
+        db = self.get_db()
+        collection = db[coll_name]
+        rows = collection.find(query)
+        df = pd.DataFrame(list(rows))
+        return df
+
+    def find_query_data_choose_field(self, coll_name, query, query_field):
+        db = self.get_db()
+        collection = db[coll_name]
+        rows = collection.find(query, query_field)
+        df = pd.DataFrame(list(rows))
+        return df
+
+    def find_query_data_list(self, coll_name, query):
+        db = self.get_db()
+        collection = db[coll_name]
+        rows = collection.find(query)
+        return list(rows)
+
+    def find_query_data_list(self, coll_name, query):
+        db = self.get_db()
+        collection = db[coll_name]
+        rows = collection.find(query)
+        return list(rows)
+
+    def remove_all_data(self, database):
+        db = self.get_db()
+        collection = db[database]
+        query = {"_id": {"$ne": "null"}}
+        collection.delete_many(query)
+
+    def drop_collection(self, database):
+        db = self.get_db()
+        collection = db[database]
+        collection.drop()
+
+    def ascend_query(self, query, coll_name, field, num):
+        db = self.get_db()
+        collection = db[coll_name]
+        return pd.DataFrame(list(collection.find(query).sort(field, 1).skip(0).limit(num)));
+
+    def descend_query(self, query, coll_name, field, num):
+        db = self.get_db()
+        collection = db[coll_name]
+        return pd.DataFrame(list(collection.find(query).sort(field, -1).skip(0).limit(num)));
+
+    def count(self, query, coll_name):
+        db = self.get_db()
+        collection = db[coll_name]
+        return collection.count_documents(query)
+
+    def query_max(self, query, coll_name, field, num):
+        db = self.get_db()
+        collection = db[coll_name]
+        return pd.DataFrame(list(collection.find(query).sort(field, -1).skip(0).limit(num)));
+
+    def query_min(self, query, coll_name, field):
+        db = self.get_db()
+        collection = db[coll_name]
+        return pd.DataFrame(list(collection.find(query).sort(field, 1).skip(0).limit(1)));
+
+    def insert_mongo(self, df, database):
+        db = self.get_db()
+        if df is None or len(df) == 0:
+            return
+        collection = db[database]
+        # 格式转换
+        try:
+            df = df.drop_duplicates()
+            # df = df.T.drop_duplicates().T
+            records = df.to_dict('records')
+            collection.insert_many(records)
+        except BaseException as e:
+            logger.error("插入数据异常:{}", e)
+
+    def insert_mongo_json(self, json, database):
+        db = self.get_db()
+        collection = db[database]
+        # 格式转换
+        try:
+            collection.insert_many(json)
+        except BaseException as e:
+            logger.error("插入数据异常:{}", e)
+
+    def save_mongo_json(self, json, database):
+        db = self.get_db()
+        collection = db[database]
+        for record in json:
+            try:
+                collection.save(record)
+            except BaseException as e:
+                logger.error("保存数据出现异常:{}", e)
+
+    def save_mongo(self, df, database):
+        db = self.get_db()
+        if df is None or len(df) == 0:
+            return
+        collection = db[database]
+        # df = df.T.drop_duplicates().T
+        # 格式转换
+        records = df.to_dict('records')
+        for record in records:
+            try:
+                collection.save(record)
+            except BaseException as e:
+                logger.error("保存数据出现异常:{},{}", record, e)
+
+    def save_mongo_no_catch_exception(self, df, database):
+        db = self.get_db()
+        if df is None or len(df) == 0:
+            return
+        collection = db[database]
+        # df = df.T.drop_duplicates().T
+        # 格式转换
+        records = df.to_dict('records')
+        for record in records:
+            collection.save(record)
+
+    def update_one(self, df, database):
+        db = self.get_db()
+        condition = {'_id': list(df['_id'])[0]}
+        if len(df) == 0:
+            return
+        collection = db[database]
+        collection.update(condition, df)
+
+    def update_many(self, query, new_values, database):
+        db = self.get_db()
+        collection = db[database]
+        x = collection.update_many(query, new_values)
+        return x
+
+    @async_fun
+    def update_one_query(self, query, new_values, database):
+        db = self.get_db()
+        collection = db[database]
+        x = collection.update(query, new_values)
+        return x
+
+    def distinct_field(self, database, field, query):
+        db = self.get_db()
+        collection = db[database]
+        return collection.distinct(field, query)
+
+    def create_index(self, database, index):
+        db = self.get_db()
+        collection = db[database]
+        collection.create_index(
+            index)
+
+    def aggregate(self, pipeline, database):
+        db = self.get_db()
+        collection = db[database]
+        data = collection.aggregate(pipeline)
+        return pd.DataFrame(list(data))
+
+    def get_col_keys(self, database):
+        db = self.get_db()
+        collection = db[database]
+        keys = collection.find_one().keys()
+        return keys
+
+    # 分页查询 descend 是否降序
+    def find_page_skip_data(self, coll_name, page_query, page, page_number, field, descend):
+        db = self.get_db()
+        collection = db[coll_name]
+        if descend:
+            sort_tag = -1
+        else:
+            sort_tag = 1
+        rows = collection.find(page_query).sort(field, sort_tag).skip((page - 1) * page_number).limit(page_number)
+        df = pd.DataFrame(list(rows))
+        return df
+
+
+# if __name__ == '__main__':
+#     symbol = '002992'
+#     query = {'symbol': symbol,
+#              '$and': [{'str_day': {'$gte': '2022-07-06'}}, {'str_day': {'$lte': '2022-11-06'}}]}
+#     mongodb_util = MongodbUtil('27017')
+#     # num = mongodb_util.count(query, 'stock_zt_pool')
+#     # print(num)
+#     key = mongodb_util.get_col_keys('stock_zt_pool')
+#     print(key)
+#
+#     # num = mongodb_util.count(query, 'stock_zt_pool')
+#     # print(num)
+#
+#     pipeline = [
+#         {'$match': {
+#             "classification": {'$in': ["K", "C"]},
+#             "str_day": {'$gte': "2022-03-16"}}},
+#         {'$group': {'_id': "$flow_mv_level", 'count': {'$sum': 1}}}
+#     ]
+#     result = mongodb_util.aggregate(pipeline, 'realtime_quotes_now_zt_new_kc_open')
+#
+#     result = result.sort_values(by=['_id'], ascending=True)
+#     print(result)
+from io import StringIO
+import re
+
+if __name__ == '__main__':
+    mongodb_util = RemoteMongodbUtil('27017')
+    #
+    # kpl_best_choose_index_df = mongodb_util.find_page_skip_data('kpl_best_choose_index', {"index_class": "sub_index"},
+    #                                                             1, 100, 'create_time', True)
+    key_word = '高速连接'
+    EXCLUDE_INFO_KEY = '股东人数'
+    # query = {
+    #     "$or": [{'question': {"$regex": re.compile(key_word, re.IGNORECASE)}},
+    #             {'answer_content': {"$regex": re.compile(key_word, re.IGNORECASE)}}],
+    #     "$and": [{'question': {"$not": re.compile(EXCLUDE_INFO_KEY, re.IGNORECASE)}},
+    #              {'answer_content': {"$not": re.compile(EXCLUDE_INFO_KEY, re.IGNORECASE)}}],
+    # }
+    #
+    # pipeline = [
+    #     {'$match': query},
+    #     {'$group': {'_id': "$symbol", 'count': {'$sum': 1}}}
+    # ]
+    # result = mongodb_util.aggregate(pipeline, 'stock_interactive_question')
+    #
+    # result = result.sort_values(by=['_id'], ascending=True)
+    # print(result)
+    #
+    # # ths_new_concept = mongodb_util.find_all_data('ths_new_concept')
+    # key = mongodb_util.get_col_keys('company_info')
+    # print(key)
+
+    # mongodb_util.create_index('realtime_quotes_now_open', [("number", 1)])
+    # mongodb_util.create_index('realtime_quotes_now_open', [("symbol", 1), ("number", 1)])
+    # mongodb_util.create_index('realtime_quotes_now_open', [("str_day", 1)])
+    # update_query = {"str_day": "2023-06-30"}
+    # mongodb_util.update_many(update_query, {"$set": {"number": 1}}, "realtime_quotes_now_open")
+    # query = {"symbol": "000617"}
+    # company_info_base = mongodb_util.find_query_data('company_info_base', query)
+    # ths_stock_concept_detail = mongodb_util.find_query_data('ths_stock_concept_detail', query)
+    # ths_stock_concept_detail = ths_stock_concept_detail[[
+    #     'concept_code',
+    #     'concept_name',
+    #     'str_now_time',
+    #     'concept_create_day']]
+    # # 去除空格
+    # ths_stock_concept_detail['concept_name'] = ths_stock_concept_detail['concept_name'].str.replace(' ', '')
+    # company_info_base.loc[:, 'ths_concept_list_info'] = ths_stock_concept_detail.to_string(index=False)
+    # for company_one in company_info_base.itertuples():
+    #     ths_concept_list_info = company_one.ths_concept_list_info
+    #     ths_concept_list_info_df = pd.read_csv(StringIO(ths_concept_list_info), delim_whitespace=True)
+    #     print(ths_concept_list_info_df)

mns_scheduler-1.1.6.7/mns_scheduler/zt/script/__init__.py

@@ -0,0 +1,7 @@
+import sys
+import os
+
+file_path = os.path.abspath(__file__)
+end = file_path.index('mns') + 16
+project_path = file_path[0:end]
+sys.path.append(project_path)

{mns_scheduler-1.1.6.5 → mns_scheduler-1.1.6.7}/mns_scheduler/zt/zt_pool/em_zt_pool_sync_api.py

@@ -122,7 +122,7 @@ def save_zt_info(str_day):
 
 
 if __name__ == '__main__':
-    save_zt_info('2024-10-08')
+    save_zt_info('2024-10-18')
 # from datetime import datetime
 #
 # if __name__ == '__main__':

{mns_scheduler-1.1.6.5 → mns_scheduler-1.1.6.7}/mns_scheduler.egg-info/PKG-INFO

@@ -1,3 +1,3 @@
 Metadata-Version: 2.1
 Name: mns-scheduler
-Version: 1.1.6.5
+Version: 1.1.6.7

{mns_scheduler-1.1.6.5 → mns_scheduler-1.1.6.7}/mns_scheduler.egg-info/SOURCES.txt

@@ -42,6 +42,10 @@ mns_scheduler/db/db_status.py
 mns_scheduler/db/real_time_task_check.py
 mns_scheduler/db/script/__init__.py
 mns_scheduler/db/script/col_move_script.py
+mns_scheduler/db/script/sync/__init__.py
+mns_scheduler/db/script/sync/local_mongo_util.py
+mns_scheduler/db/script/sync/remote_data_sync_to_local.py
+mns_scheduler/db/script/sync/remote_mongo_util.py
 mns_scheduler/debt/__init__.py
 mns_scheduler/debt/kzz_bond_info_sync.py
 mns_scheduler/dt/__init__.py

{mns_scheduler-1.1.6.5 → mns_scheduler-1.1.6.7}/setup.py

@@ -2,7 +2,7 @@ from setuptools import setup, find_packages
 
 setup(
     name='mns-scheduler',
-    version='1.1.6.5',
+    version='1.1.6.7',
     packages=find_packages(),
     install_requires=[],  # 如果有依赖项，可以在这里列出
 )