mns-scheduler 1.3.1.7__tar.gz → 1.4.3.2__tar.gz

mns_scheduler-1.4.3.2/PKG-INFO

@@ -0,0 +1,3 @@
+Metadata-Version: 2.1
+Name: mns_scheduler
+Version: 1.4.3.2

{mns-scheduler-1.3.1.7 → mns_scheduler-1.4.3.2}/mns_scheduler/company_info/announce/company_announce_sync_service.py

@@ -17,21 +17,25 @@ mongodb_util = MongodbUtil('27017')
 # 同步最新公告
 # eq-f1001 业绩预告 eq-f1002 重大事项   eq-f1003 股份变动公告
 
-def sync_company_announce():
-    page_size = 10
+def sync_company_announce(symbol_list):
+    page_size = 100
     announce_type_list = ['all', 'eq-f1003', 'eq-f1001', 'eq-f1002']
     for announce_type_one in announce_type_list:
         try:
-            get_company_announce(announce_type_one, page_size)
+            get_company_announce(announce_type_one, page_size, symbol_list)
         except BaseException as e:
             logger.error("更新公告出现异常:{}", e)
     logger.info("同步到公告信息完成")
 
 
-def get_company_announce(announce_type, page_size):
+def get_company_announce(announce_type, page_size, symbol_list):
     company_all_df = company_common_service_new_api.get_company_all_info_info()
     de_list_company = company_common_service_new_api.get_de_list_company()
     company_all_df = company_all_df.loc[~(company_all_df['symbol'].isin(de_list_company))]
+
+    if symbol_list is not None:
+        company_all_df = company_all_df.loc[(company_all_df['symbol'].isin(symbol_list))]
+
     for stock_one in company_all_df.itertuples():
         try:
             symbol = stock_one.symbol
@@ -51,10 +55,11 @@ def get_company_announce(announce_type, page_size):
             ths_company_announce_result['_id'] = ths_company_announce_result['guid'] + '_' + \
                                                  ths_company_announce_result['seq'] + "_" + announce_type
             mongodb_util.save_mongo(ths_company_announce_result, db_name_constant.COMPANY_ANNOUNCE_INFO)
+            logger.info("更新公告完成:{},{}", symbol, stock_one.name)
 
         except BaseException as e:
             logger.error("更新公告出现异常:{}", e)
 
 
 if __name__ == '__main__':
-    sync_company_announce()
+    sync_company_announce(None)

{mns-scheduler-1.3.1.7 → mns_scheduler-1.4.3.2}/mns_scheduler/company_info/base/sync_company_base_info_api.py

@@ -34,67 +34,6 @@ result_lock = threading.Lock()
 result = []
 
 
-# 计算实际流通比例
-def calculate_circulation_ratio(symbol):
-    query = {"symbol": symbol}
-    stock_gdfx_free_top_10 = mongodb_util.descend_query(query, 'stock_gdfx_free_top_10', "period", 10)
-    if stock_gdfx_free_top_10.shape[0] == 0:
-        mv_circulation_ratio = 1
-    else:
-        # 排除香港结算公司 大于5%减持不用发公告  香港中央结算    HKSCC
-        stock_gdfx_free_top_10['is_hk'] = stock_gdfx_free_top_10['shareholder_name'].apply(
-            lambda shareholder_name: "HK" if shareholder_name.startswith('香港中央结算') or shareholder_name.startswith(
-                'HKSCC') else "A")
-
-        # 持股大于5%      # 排除香港结算公司 大于5%减持不用发公告  香港中央结算    HKSCC
-        stock_gdfx_free_top_10 = stock_gdfx_free_top_10.loc[
-            (stock_gdfx_free_top_10['circulation_ratio'] >= 5) & (stock_gdfx_free_top_10['is_hk'] == 'A')]
-
-        circulation_ratio = sum(stock_gdfx_free_top_10['circulation_ratio'])
-        mv_circulation_ratio = round((100 - circulation_ratio) / 100, 2)
-        # 防止错误数据
-        if mv_circulation_ratio < 0:
-            mv_circulation_ratio = 1
-    return mv_circulation_ratio
-
-
-def get_east_money_stock_info():
-    all_real_time_quotes = em_stock_info_api.get_a_stock_info()
-    all_real_time_quotes = all_real_time_quotes[['symbol',
-                                                 'name',
-                                                 "now_price",
-                                                 'total_mv',
-                                                 'flow_mv',
-                                                 'pe_ttm',
-                                                 'sz_sh',
-                                                 'area',
-                                                 'pb',
-                                                 'list_date',
-                                                 'ROE',
-                                                 'total_share',
-                                                 'flow_share',
-                                                 'industry',
-                                                 'amount',
-                                                 "hk_stock_code",
-                                                 "hk_stock_name",
-                                                 'concept']]
-
-    return all_real_time_quotes
-
-
-def create_index():
-    mongodb_util.create_index('company_info',
-                              [("classification", 1)])
-    mongodb_util.create_index('company_info',
-                              [("industry", 1)])
-    mongodb_util.create_index('company_info',
-                              [("flow_mv", 1)])
-    mongodb_util.create_index('company_info',
-                              [("list_date", 1)])
-    mongodb_util.create_index('company_info',
-                              [("symbol", 1)])
-
-
 # 同步公司基本信息
 
 def sync_company_base_info(symbol_list):
@@ -102,6 +41,8 @@ def sync_company_base_info(symbol_list):
     result = []
     create_index()
     east_money_stock_info = get_east_money_stock_info()
+    east_money_stock_info = east_money_stock_info.sort_values(by=['list_date'], ascending=False)
+
     de_listed_stock_list = company_common_service_api.get_de_list_company()
     east_money_stock_info = east_money_stock_info.loc[~(
         east_money_stock_info['symbol'].isin(de_listed_stock_list))]
@@ -145,6 +86,30 @@ def sync_company_base_info(symbol_list):
     single_thread_sync_company_info(fail_df, kpl_real_time_quotes, exist_company_df)
 
 
+def get_east_money_stock_info():
+    all_real_time_quotes = em_stock_info_api.get_a_stock_info()
+    all_real_time_quotes = all_real_time_quotes[['symbol',
+                                                 'name',
+                                                 "now_price",
+                                                 'total_mv',
+                                                 'flow_mv',
+                                                 'pe_ttm',
+                                                 'sz_sh',
+                                                 'area',
+                                                 'pb',
+                                                 'list_date',
+                                                 'ROE',
+                                                 'total_share',
+                                                 'flow_share',
+                                                 'industry',
+                                                 'amount',
+                                                 "hk_stock_code",
+                                                 "hk_stock_name",
+                                                 'concept']]
+
+    return all_real_time_quotes
+
+
 def single_thread_sync_company_info(east_money_stock_info,
                                     kpl_real_time_quotes, exist_company_df):
     global result
@@ -201,12 +166,23 @@ def single_thread_sync_company_info(east_money_stock_info,
             company_info_type['total_mv_sp'] = company_one.total_mv_sp
             company_info_type['flow_mv_level'] = company_one.flow_mv_level
             company_info_type['classification'] = company_one.classification
-            company_info_type['mv_circulation_ratio'] = calculate_circulation_ratio(company_one.symbol)
-            # 获取同花顺最新概念
-            company_info_type = ths_concept_common_service_api.set_ths_concept(company_one.symbol, company_info_type)
+
             now_date = datetime.now()
             str_now_date = now_date.strftime('%Y-%m-%d %H:%M:%S')
+
+            now_str_day = now_date.strftime('%Y-%m-%d')
             company_info_type['sync_date'] = str_now_date
+
+            result_dict = calculate_circulation_ratio(company_one.symbol, now_str_day)
+
+            company_info_type['mv_circulation_ratio'] = result_dict['mv_circulation_ratio']
+            company_info_type['qfii_type'] = result_dict['qfii_type']
+            company_info_type['qfii_number'] = result_dict['qfii_number']
+            company_info_type['share_holder_sync_day'] = result_dict['share_holder_sync_day']
+
+            # 获取同花顺最新概念
+            company_info_type = ths_concept_common_service_api.set_ths_concept(company_one.symbol, company_info_type)
+
             fix_symbol_industry_df = company_constant_data.get_fix_symbol_industry()
             if company_one.symbol in list(fix_symbol_industry_df['symbol']):
                 # fix sw_industry
@@ -257,6 +233,113 @@ def single_thread_sync_company_info(east_money_stock_info,
         result = fail_list
 
 
+# 计算实际流通比例
+def calculate_circulation_ratio(symbol, now_str_day):
+    query = {"symbol": symbol}
+    stock_gdfx_free_top_1 = mongodb_util.descend_query(query, 'stock_gdfx_free_top_10', "period", 1)
+    if stock_gdfx_free_top_1.shape[0] == 0:
+        mv_circulation_ratio = 1
+        qfii_number = 0
+        qfii_type = 'A股'
+        share_holder_sync_day = now_str_day
+    else:
+        period_time = list(stock_gdfx_free_top_1['period'])[0]
+
+        query_free = {'symbol': symbol, 'period': period_time}
+        stock_gdfx_free_top_10 = mongodb_util.find_query_data('stock_gdfx_free_top_10', query_free)
+
+        stock_gdfx_free_top_10['shares_number_str'] = stock_gdfx_free_top_10['shares_number'].astype(str)
+
+        stock_gdfx_free_top_10['id_key'] = stock_gdfx_free_top_10['symbol'] + '_' + stock_gdfx_free_top_10[
+            'period'] + '_' + stock_gdfx_free_top_10.shares_number_str
+
+        stock_gdfx_free_top_10.drop_duplicates('id_key', keep='last', inplace=True)
+
+        # 排除香港结算公司 大于5%减持不用发公告  香港中央结算    HKSCC
+        stock_gdfx_free_top_10['is_hk'] = stock_gdfx_free_top_10['shareholder_name'].apply(
+            lambda shareholder_name: "HK" if shareholder_name.startswith('香港中央结算') or shareholder_name.startswith(
+                'HKSCC') else "A")
+
+        # 持股大于5% 减持需要发公告
+        # 排除香港结算公司不发公共 小于5%减持不用发公告
+        # 香港中央结算    HKSCC
+        stock_free_top_greater_than_5 = stock_gdfx_free_top_10.loc[
+            (stock_gdfx_free_top_10['circulation_ratio'] >= 5) & (stock_gdfx_free_top_10['is_hk'] == 'A')]
+
+        stock_free_qfii = stock_gdfx_free_top_10.loc[stock_gdfx_free_top_10['shareholder_nature'] == 'QFII']
+
+        share_holder_sync_day = list(stock_gdfx_free_top_10['create_day'])[0]
+
+        # qfii 数量
+        qfii_number = stock_free_qfii.shape[0]
+        # qfii 类型
+        qfii_type = set_qfii_type(qfii_number, stock_free_qfii.copy())
+
+        circulation_ratio = sum(stock_free_top_greater_than_5['circulation_ratio'])
+        mv_circulation_ratio = round((100 - circulation_ratio) / 100, 2)
+        # 防止错误数据
+        if mv_circulation_ratio < 0:
+            mv_circulation_ratio = 1
+
+    result_dict = {
+        'mv_circulation_ratio': mv_circulation_ratio,
+        'qfii_type': qfii_type,
+        'qfii_number': qfii_number,
+        'share_holder_sync_day': share_holder_sync_day
+
+    }
+    return result_dict
+
+
+def set_qfii_type(qfii_number, stock_free_qfii):
+    if qfii_number > 0:
+        stock_free_qfii['new_change'] = stock_free_qfii['change']
+        stock_free_qfii.loc[stock_free_qfii['change_ratio'] == 0, 'new_change'] = 0
+        stock_free_qfii.loc[stock_free_qfii['change'] == '新进', 'new_change'] = \
+            stock_free_qfii['shares_number']
+        stock_free_qfii['new_change'] = stock_free_qfii['new_change'].astype(float)
+
+        stock_free_qfii_new_in = stock_free_qfii.loc[stock_free_qfii['change'] == '新进']
+        if data_frame_util.is_not_empty(stock_free_qfii_new_in):
+            qfii_type = 1
+            return qfii_type
+
+        stock_free_qfii_add = stock_free_qfii.loc[
+            (~stock_free_qfii['change'].isin(['不变', '新进'])) & (stock_free_qfii['new_change'] > 0)]
+
+        if data_frame_util.is_not_empty(stock_free_qfii_add):
+            qfii_type = 2
+            return qfii_type
+
+        stock_free_qfii_not_change = stock_free_qfii.loc[stock_free_qfii['change'] == '不变']
+
+        if data_frame_util.is_not_empty(stock_free_qfii_not_change):
+            qfii_type = 3
+            return qfii_type
+
+        stock_free_qfii_reduce = stock_free_qfii.loc[
+            (~stock_free_qfii['change'].isin(['不变', '新进'])) & (stock_free_qfii['new_change'] < 0)]
+
+        if data_frame_util.is_not_empty(stock_free_qfii_reduce):
+            qfii_type = 4
+            return qfii_type
+    else:
+        return 0
+
+
+def create_index():
+    mongodb_util.create_index('company_info',
+                              [("classification", 1)])
+    mongodb_util.create_index('company_info',
+                              [("industry", 1)])
+    mongodb_util.create_index('company_info',
+                              [("flow_mv", 1)])
+    mongodb_util.create_index('company_info',
+                              [("list_date", 1)])
+    mongodb_util.create_index('company_info',
+                              [("symbol", 1)])
+
+
 def set_kpl_data(kpl_real_time_quotes_one, company_info_type, company_one):
     if data_frame_util.is_not_empty(kpl_real_time_quotes_one):
         company_info_type['kpl_plate_name'] = list(kpl_real_time_quotes_one['plate_name_list'])[0]
@@ -395,17 +478,20 @@ def save_sw_data(company_info_type):
 def get_recent_year_income(symbol, company_info_type, exist_company_df):
     now_date = datetime.now()
     hour = now_date.hour
-    if hour <= 15:
-        exist_company_one_df = exist_company_df.loc[exist_company_df['_id'] == symbol]
-        if data_frame_util.is_not_empty(exist_company_one_df):
-            company_info_type['operate_profit'] = list(exist_company_one_df['operate_profit'])[0]
-            company_info_type['total_operate_income'] = list(exist_company_one_df['total_operate_income'])[0]
-            company_info_type['operate_date_name'] = ''
-        else:
-            company_info_type['operate_profit'] = 0
-            company_info_type['total_operate_income'] = 0
-            company_info_type['operate_date_name'] = ''
-        return company_info_type
+    # if hour <= 15:
+    #     exist_company_one_df = exist_company_df.loc[exist_company_df['_id'] == symbol]
+    #     if data_frame_util.is_not_empty(exist_company_one_df):
+    #         company_info_type['operate_profit'] = list(exist_company_one_df['operate_profit'])[0]
+    #         company_info_type['total_operate_income'] = list(exist_company_one_df['total_operate_income'])[0]
+    #         if 'operate_date_name' in exist_company_one_df:
+    #             company_info_type['operate_date_name'] = list(exist_company_one_df['total_operate_income'])[0]
+    #         else:
+    #             company_info_type['operate_date_name'] = '暂无年报'
+    #     else:
+    #         company_info_type['operate_profit'] = 0
+    #         company_info_type['total_operate_income'] = 0
+    #         company_info_type['operate_date_name'] = '暂无年报'
+    #     return company_info_type
     query = {'symbol': symbol, "REPORT_TYPE": "年报"}
     em_stock_profit = mongodb_util.descend_query(query, db_name_constant.EM_STOCK_PROFIT, 'REPORT_DATE', 1)
     if data_frame_util.is_not_empty(em_stock_profit):
@@ -420,7 +506,7 @@ def get_recent_year_income(symbol, company_info_type, exist_company_df):
     else:
         company_info_type['operate_profit'] = 0
         company_info_type['total_operate_income'] = 0
-        company_info_type['operate_date_name'] = ''
+        company_info_type['operate_date_name'] = '暂无年报'
     company_info_type['operate_profit'] = round(
         company_info_type['operate_profit'] / common_service_fun_api.HUNDRED_MILLION, 2)
     company_info_type['total_operate_income'] = round(
@@ -441,5 +527,5 @@ if __name__ == '__main__':
     # query = {"total_operate_income": 0}
     # un_report_company_info = mongodb_util.find_query_data(db_name_constant.COMPANY_INFO, query)
     # symbol_list = list(un_report_company_info['symbol'])
-    sync_company_base_info(['600753'])
-    # group_by_industry()
+    sync_company_base_info(['920009'])
+    sync_company_base_info(None)

{mns-scheduler-1.3.1.7 → mns_scheduler-1.4.3.2}/mns_scheduler/company_info/base/sync_company_hold_info_api.py

@@ -27,11 +27,8 @@ def sync_company_hold_info(symbol):
         now_date = datetime.now()
         sync_str_date = now_date.strftime('%Y-%m-%d %H:%M:%S')
         if data_frame_util.is_not_empty(company_hold_info_df):
-            query = {'symbol': symbol}
-            result = mongodb_util.remove_data(query, db_name_constant.COMPANY_HOLDING_INFO)
-            if result.acknowledged:
-                company_hold_info_df['sync_str_date'] = sync_str_date
-                mongodb_util.insert_mongo(company_hold_info_df, db_name_constant.COMPANY_HOLDING_INFO)
+            company_hold_info_df['sync_str_date'] = sync_str_date
+            mongodb_util.insert_mongo(company_hold_info_df, db_name_constant.COMPANY_HOLDING_INFO)
     except BaseException as e:
         logger.error("同步公司控股子公司信息:{},{}", symbol, e)
 

mns_scheduler-1.4.3.2/mns_scheduler/company_info/base/sync_company_product_area_industry.py

@@ -0,0 +1,161 @@
+import os
+import sys
+
+file_path = os.path.abspath(__file__)
+end = file_path.index('mns') + 17
+project_path = file_path[0:end]
+sys.path.append(project_path)
+import mns_common.component.em.em_stock_info_api as em_stock_info_api
+import mns_common.component.common_service_fun_api as common_service_fun_api
+import mns_common.api.ths.company.company_product_area_industry_index_query as company_product_area_industry_index_query
+from loguru import logger
+from mns_common.db.MongodbUtil import MongodbUtil
+import mns_common.constant.db_name_constant as db_name_constant
+import mns_common.utils.data_frame_util as data_frame_util
+import pandas as pd
+from datetime import datetime
+
+mongodb_util = MongodbUtil('27017')
+
+
+def sync_company_product_area_industry_task(symbol):
+    now_date = datetime.now()
+    now_year = now_date.year
+    now_month = now_date.month
+
+    if now_month in [1, 2, 3, 4]:
+        period_time_year = str(now_year - 1) + "-12-31"
+        sync_company_product_area_industry(symbol, period_time_year)
+
+    if now_month in [4, 5, 6]:
+        period_time_one = str(now_year) + "-03-31"
+        sync_company_product_area_industry(symbol, period_time_one)
+
+    elif now_month in [7, 8, 9]:
+        period_time_two = str(now_year) + "-06-30"
+        sync_company_product_area_industry(symbol, period_time_two)
+
+    elif now_month in [10, 11, 12]:
+        period_time_three = str(now_year) + "-09-30"
+        sync_company_product_area_industry(symbol, period_time_three)
+
+
+def sync_company_product_area_industry(symbol, date):
+    real_time_quotes_all_stocks = em_stock_info_api.get_a_stock_info()
+    real_time_quotes_all_stocks = common_service_fun_api.classify_symbol(real_time_quotes_all_stocks)
+    if symbol is not None:
+        real_time_quotes_all_stocks = real_time_quotes_all_stocks.loc[real_time_quotes_all_stocks['symbol'] == symbol]
+    for stock_one in real_time_quotes_all_stocks.itertuples():
+        try:
+            symbol = stock_one.symbol
+
+            classification = stock_one.classification
+            if classification in ['H', 'K']:
+                market = '17'
+            elif classification in ['S', 'C']:
+                market = '33'
+            elif classification in ['X']:
+                market = '151'
+
+            company_product_area_industry_list = company_product_area_industry_index_query.company_product_area_industry(
+                symbol, market, date)
+            for company_one in company_product_area_industry_list:
+                try:
+                    analysis_type = company_one['analysis_type']
+                    time_operate_index_item_list = company_one['time_operate_index_item_list']
+                    time_operate_index_item_df = pd.DataFrame(time_operate_index_item_list)
+                    if data_frame_util.is_empty(time_operate_index_item_df):
+                        continue
+                    time_operate_index_item_df['symbol'] = symbol
+                    time_operate_index_item_df['analysis_type'] = analysis_type
+
+                    time_operate_index_item_df['_id'] = symbol + '_' + time_operate_index_item_df[
+                        'time'] + '_' + analysis_type
+                    handle_industry_area_product(time_operate_index_item_df, symbol)
+                except BaseException as e:
+                    logger.error("同步经营数据异常:{},{}", symbol, e)
+
+            logger.info("同步经营数据完成:{}", stock_one.symbol)
+        except BaseException as e:
+            logger.error("同步经营数据:{},{}", stock_one.symbol, e)
+
+
+def handle_industry_area_product(time_operate_index_item_df, symbol):
+    if data_frame_util.is_empty(time_operate_index_item_df):
+        return None
+
+    for business_one in time_operate_index_item_df.itertuples():
+        time = business_one.time
+        analysis_type = business_one.analysis_type
+
+        product_index_item_list = business_one.product_index_item_list
+        for product_one in product_index_item_list:
+            try:
+                # 初始化数据
+                income_amount = 0
+                income_percent = 0
+                cost_amount = 0
+                cost_percent = 0
+                gross_profit_amount = 0
+                gross_profit_percent = 0
+                gross_profit_rate_amount = 0
+                gross_profit_rate_percent = 0
+
+                product_name = product_one['product_name']
+                index_analysis_list = product_one['index_analysis_list']
+                for index_one in index_analysis_list:
+                    try:
+                        index_id = index_one['index_id']
+                        if index_id == 'income':
+                            income_amount = index_one['index_value']
+                            income_percent = index_one['account']
+                        elif index_id == 'cost':
+                            cost_amount = index_one['index_value']
+                            cost_percent = index_one['account']
+                        elif index_id == 'gross_profit':
+                            gross_profit_amount = index_one['index_value']
+                            gross_profit_percent = index_one['account']
+
+                        elif index_id == 'gross_profit_rate':
+                            gross_profit_rate_amount = index_one['index_value']
+                            gross_profit_rate_percent = index_one['account']
+                    except BaseException as e:
+                        logger.error("同步经营数据异常:{},{}", symbol, e)
+
+                id_key = symbol + '_' + time + '_' + analysis_type + '_' + product_name
+                result_dict = {
+                    '_id': id_key,
+                    'symbol': symbol,
+                    'time': time,
+                    'analysis_type': analysis_type,
+                    'product_name': product_name,
+
+                    'income_amount': income_amount,
+                    'income_percent': income_percent,
+
+                    'cost_amount': cost_amount,
+                    'cost_percent': cost_percent,
+
+                    'gross_profit_amount': gross_profit_amount,
+                    'gross_profit_percent': gross_profit_percent,
+
+                    'gross_profit_rate_amount': gross_profit_rate_amount,
+                    'gross_profit_rate_percent': gross_profit_rate_percent,
+                }
+                result_dict_df = pd.DataFrame(result_dict, index=[1])
+                mongodb_util.save_mongo(result_dict_df, db_name_constant.COMPANY_BUSINESS_INFO)
+            except BaseException as e:
+                logger.error("同步经营数据异常:{},{}", symbol, e)
+
+
+if __name__ == '__main__':
+    sync_company_product_area_industry('300211', '2025-09-30')
+    # sync_company_product_area_industry('002323')
+    # sync_company_product_area_industry('300901')
+    # sync_company_product_area_industry('603225')
+    # sync_company_product_area_industry('688039')
+    # sync_company_product_area_industry('600849')
+    # sync_company_product_area_industry('000508')
+    # sync_company_product_area_industry('810011')
+
+    sync_company_product_area_industry(None, None)

{mns-scheduler-1.3.1.7 → mns_scheduler-1.4.3.2}/mns_scheduler/company_info/clean/company_info_clean_api.py

@@ -47,7 +47,7 @@ def clean_company_info(symbol):
     company_info['company_type'] = company_info['business_nature']
 
     # 将list_date列中的所有NaN值设置为99990909
-    company_info.fillna({'list_date': 20990909.0}, inplace=True)
+    company_info.fillna({'list_date': 19890604}, inplace=True)
 
     # 将日期数值转换为日期时间格式
     company_info['list_date_01'] = pd.to_datetime(company_info['list_date'], format='%Y%m%d')

{mns-scheduler-1.3.1.7 → mns_scheduler-1.4.3.2}/mns_scheduler/company_info/constant/company_constant_data.py

@@ -443,6 +443,9 @@ def filed_sort(company_info):
         "final_controller_name",
         "final_controller_rate",
         "mv_circulation_ratio",
+        'qfii_type',
+        'qfii_number',
+        'share_holder_sync_day',
         "flow_mv_sp",
         "total_mv_sp",
         "now_price",

mns_scheduler-1.4.3.2/mns_scheduler/company_info/em_stock_info/sync_em_stock_info_sync.py

@@ -0,0 +1,80 @@
+import sys
+import os
+
+file_path = os.path.abspath(__file__)
+end = file_path.index('mns') + 17
+project_path = file_path[0:end]
+sys.path.append(project_path)
+
+from mns_common.db.MongodbUtil import MongodbUtil
+from loguru import logger
+import mns_common.api.em.real_time.east_money_debt_api as east_money_debt_api
+import mns_common.api.em.real_time.east_money_etf_api as east_money_etf_api
+import mns_common.api.em.real_time.east_money_stock_a_v2_api as east_money_stock_a_v2_api
+from datetime import datetime
+import mns_common.constant.extra_income_db_name as extra_income_db_name
+import mns_common.api.em.real_time.east_money_stock_hk_api as east_money_stock_hk_api
+import mns_common.component.cookie.cookie_info_service as cookie_info_service
+import mns_common.api.em.real_time.east_money_stock_us_api as east_money_stock_us_api
+
+mongodb_util = MongodbUtil('27017')
+
+
+def sync_all_em_stock_info():
+    now_date = datetime.now()
+    str_now_date = now_date.strftime('%Y-%m-%d %H:%M:%S')
+
+    logger.info("同步东方财富a,etf,kzz,us,hk,信息开始")
+
+    try:
+        # 这里需要使用详情接口 获取全量数据
+        em_a_stock_info_df = east_money_stock_a_v2_api.get_stock_real_time_quotes(60)
+        em_a_stock_info_df['list_date'] = em_a_stock_info_df['list_date'].fillna(19890604)
+        em_a_stock_info_df['_id'] = em_a_stock_info_df['symbol']
+        em_a_stock_info_df['sync_time'] = str_now_date
+        mongodb_util.save_mongo(em_a_stock_info_df, extra_income_db_name.EM_A_STOCK_INFO)
+    except BaseException as e:
+        logger.error("同步东方财富A股信息异常:{}", e)
+    try:
+        em_etf_info = east_money_etf_api.get_etf_real_time_quotes(30, 6)
+        em_etf_info['sync_time'] = str_now_date
+        mongodb_util.save_mongo(em_etf_info, extra_income_db_name.EM_ETF_INFO)
+    except BaseException as e:
+        logger.error("同步东方财富ETF信息异常:{}", e)
+
+    try:
+        em_kzz_info = east_money_debt_api.get_kzz_real_time_quotes(30, 6)
+        em_kzz_info['_id'] = em_kzz_info['symbol']
+        em_kzz_info['sync_time'] = str_now_date
+        mongodb_util.save_mongo(em_kzz_info, extra_income_db_name.EM_KZZ_INFO)
+    except BaseException as e:
+        logger.error("同步东方财富可转债信息异常:{}", e)
+
+    # em_cookie = cookie_info_service.get_em_cookie()
+    # try:
+    #     em_hk_stock_info = east_money_stock_hk_api.get_hk_real_time_quotes(em_cookie, proxies)
+    #     em_hk_stock_info['_id'] = em_hk_stock_info['symbol']
+    #     mongodb_util.save_mongo(em_hk_stock_info, extra_income_db_name.EM_HK_STOCK_INFO)
+    # except BaseException as e:
+    #     logger.error("同步东方财富港股信息异常:{}", e)
+    #
+    # try:
+    #     em_cookie = cookie_info_service.get_em_cookie()
+    #     em_us_stock_info = east_money_stock_us_api.get_us_stock_real_time_quotes(em_cookie, proxies)
+    #     em_us_stock_info['_id'] = em_us_stock_info['symbol']
+    #     mongodb_util.save_mongo(em_us_stock_info, extra_income_db_name.EM_US_STOCK_INFO)
+    #     em_us_etf_info = em_us_stock_info.loc[(em_us_stock_info['amount'] != 0) & (em_us_stock_info['total_mv'] == 0)]
+    #     em_us_etf_info = em_us_etf_info.sort_values(by=['amount'], ascending=False)
+    #     mongodb_util.save_mongo(em_us_etf_info, extra_income_db_name.EM_US_ETF_INFO)
+    #
+    # except BaseException as e:
+    #     logger.error("同步东方财富美股信息异常:{}", e)
+    logger.info("同步东方财富a,etf,kzz,us,hk,信息完成")
+
+
+if __name__ == '__main__':
+    sync_all_em_stock_info()
+    # em_cookie = cookie_info_service.get_em_cookie()
+    # em_us_stock_info = east_money_stock_us_api.get_us_stock_real_time_quotes(em_cookie, None)
+    # em_us_stock_info['_id'] = em_us_stock_info['symbol']
+    # mongodb_util.save_mongo(em_us_stock_info, db_name_constant.EM_US_STOCK_INFO)

{mns-scheduler-1.3.1.7 → mns_scheduler-1.4.3.2}/mns_scheduler/concept/clean/ths_concept_clean_api.py

@@ -7,7 +7,7 @@ project_path = file_path[0:end]
 sys.path.append(project_path)
 from mns_common.db.MongodbUtil import MongodbUtil
 import pandas as pd
-import mns_common.utils.data_frame_util as data_frame_util
+import mns_common.api.ths.concept.web.ths_concept_index_web as ths_concept_index_web
 import time
 from loguru import logger
 import mns_common.component.company.company_common_service_api as company_common_service_api
@@ -107,10 +107,7 @@ def update_null_name():
         exist_url = concept_one.url
 
         if name == '':
-            concept_name_df = ths_concept_index_app.get_new_concept_from_app_search(concept_code)
-            if data_frame_util.is_empty(concept_name_df):
-                continue
-            concept_name = list(concept_name_df['concept_name'])[0]
+            concept_name = ths_concept_index_web.get_concept_name(concept_code)
             query_concept = {"symbol": concept_code}
             new_values = {'$set': {"name": concept_name}}
             mongodb_util.update_one_query(query_concept, new_values, 'ths_concept_list')
@@ -131,7 +128,7 @@ def update_null_name():
 
 
 if __name__ == '__main__':
-    update_one_concept_relevance_industry(886095)
+    # update_one_concept_relevance_industry(886095)
     update_null_name()
     logger.info("开始")
     update_ths_concept_info()