mns-scheduler 1.1.8.4__py3-none-any.whl → 1.4.5.7__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- mns_scheduler/__init__.py +1 -3
- mns_scheduler/auto_da_ban/auto_da_ban_service.py +89 -0
- mns_scheduler/company_info/clean/__init__.py +1 -1
- mns_scheduler/company_info/clean/company_info_clean_api.py +62 -21
- mns_scheduler/company_info/common/company_common_query_service.py +45 -0
- mns_scheduler/company_info/constant/__init__.py +1 -1
- mns_scheduler/company_info/constant/company_constant_data.py +335 -224
- mns_scheduler/company_info/de_list_stock/__init__.py +1 -1
- mns_scheduler/company_info/de_list_stock/de_list_stock_service.py +1 -1
- mns_scheduler/company_info/em_stock_info/sync_em_stock_info_sync.py +134 -0
- mns_scheduler/company_info/sync/company_info_set_service.py +208 -0
- mns_scheduler/company_info/sync/sync_company_info_task.py +203 -0
- mns_scheduler/company_info/task/company_announce_info_task.py +77 -0
- mns_scheduler/company_info/task/company_base_info_task.py +64 -0
- mns_scheduler/company_info/task/company_business_info_task.py +177 -0
- mns_scheduler/company_info/task/company_hold_info_task.py +66 -0
- mns_scheduler/company_info/task/company_industry_info_task.py +167 -0
- mns_scheduler/company_info/task/company_total_task.py +69 -0
- mns_scheduler/concept/clean/kpl_concept_clean_api.py +1 -1
- mns_scheduler/concept/clean/ths_concept_clean_api.py +20 -4
- mns_scheduler/concept/ths/common/ths_concept_sync_common_api.py +23 -18
- mns_scheduler/concept/ths/common/ths_concept_update_common_api.py +4 -1
- mns_scheduler/concept/ths/detaill/ths_concept_detail_api.py +7 -7
- mns_scheduler/concept/ths/sync_new_index/sync_ths_concept_new_index_api.py +9 -4
- mns_scheduler/concept/ths/update_concept_info/sync_one_symbol_all_concepts_api.py +2 -2
- mns_scheduler/db/col_move_service.py +3 -3
- mns_scheduler/db/script/__init__.py +1 -1
- mns_scheduler/db/script/col_move_script.py +1 -1
- mns_scheduler/db/script/db_move/__init__.py +7 -0
- mns_scheduler/db/script/db_move/col_move_one_service.py +34 -0
- mns_scheduler/db/script/sync/__init__.py +1 -1
- mns_scheduler/db/script/sync/remote_data_sync_to_local.py +65 -4
- mns_scheduler/db/script/sync/sync_hui_ce_test_data.py +80 -0
- mns_scheduler/db/script/sync/sync_hui_ce_test_data_01.py +69 -0
- mns_scheduler/db/script/update/__init__.py +7 -0
- mns_scheduler/db/script/update/update_col_field.py +36 -0
- mns_scheduler/finance/__init__.py +1 -1
- mns_scheduler/finance/em/__init__.py +7 -0
- mns_scheduler/finance/{em_financial_asset_liability_sync_service_api.py → em/em_financial_asset_liability_sync_service_api.py} +2 -2
- mns_scheduler/finance/{em_financial_profit_sync_service_api.py → em/em_financial_profit_sync_service_api.py} +27 -26
- mns_scheduler/finance/{finance_common_api.py → em/finance_common_api.py} +3 -3
- mns_scheduler/finance/{sync_financial_report_service_api.py → sync_financial_report_service_task.py} +80 -27
- mns_scheduler/finance/xue_qiu/__init__.py +7 -0
- mns_scheduler/finance/xue_qiu/down_load_xueqiu_report_api.py +77 -0
- mns_scheduler/finance/xue_qiu/sync_xue_qiu_fiance_data.py +161 -0
- mns_scheduler/hk/__init__.py +1 -1
- mns_scheduler/hk/hk_company_info_sync_service_api.py +4 -4
- mns_scheduler/hk/hk_industry_info_sync_service_api.py +3 -5
- mns_scheduler/industry/__init__.py +7 -0
- mns_scheduler/industry/ths/__init__.py +7 -0
- mns_scheduler/industry/ths/ths_industry_index_service.py +58 -0
- mns_scheduler/industry/ths/ths_industry_sync_service.py +68 -0
- mns_scheduler/irm/__init__.py +1 -1
- mns_scheduler/irm/api/__init__.py +1 -1
- mns_scheduler/irm/api/sh_stock_sns_sse_info_api.py +8 -58
- mns_scheduler/irm/api/sz_stock_sns_sse_info_api.py +15 -26
- mns_scheduler/irm/stock_irm_cninfo_service.py +43 -31
- mns_scheduler/irm/stock_question_id_service.py +169 -0
- mns_scheduler/k_line/clean/daily/__init__.py +1 -1
- mns_scheduler/k_line/clean/daily/daily_k_line_clean_common_service.py +52 -6
- mns_scheduler/k_line/clean/daily/daily_k_line_service.py +7 -2
- mns_scheduler/k_line/clean/k_line_info_clean_impl.py +3 -2
- mns_scheduler/k_line/clean/k_line_info_clean_task.py +42 -15
- mns_scheduler/k_line/clean/week_month/__init__.py +1 -1
- mns_scheduler/k_line/clean/week_month/normal_week_month_k_line_service.py +124 -26
- mns_scheduler/k_line/clean/week_month/sub_new_week_month_k_line_service.py +2 -2
- mns_scheduler/k_line/common/__init__.py +7 -0
- mns_scheduler/k_line/common/k_line_common_api.py +188 -0
- mns_scheduler/k_line/hot_stocks/__init__.py +1 -1
- mns_scheduler/k_line/hot_stocks/recent_hot_stocks_clean_service.py +1 -1
- mns_scheduler/k_line/{sync → month_week_daily}/bfq_k_line_sync.py +14 -29
- mns_scheduler/k_line/{sync → month_week_daily}/daily_week_month_line_sync.py +11 -12
- mns_scheduler/k_line/sync_status/__init__.py +7 -0
- mns_scheduler/k_line/sync_status/k_line_sync_status_check.py +54 -0
- mns_scheduler/k_line/test/__init__.py +1 -1
- mns_scheduler/k_line/test/k_line_info_clean_his_data.py +14 -3
- mns_scheduler/k_line/year_quarter/__init__.py +7 -0
- mns_scheduler/k_line/year_quarter/year_quarter_line_sync.py +76 -0
- mns_scheduler/kpl/selection/symbol/sync_best_choose_symbol.py +1 -2
- mns_scheduler/kpl/selection/symbol/sync_kpl_concept_symbol_choose_reason_api.py +108 -0
- mns_scheduler/kpl/selection/total/sync_kpl_best_total_sync_api.py +5 -0
- mns_scheduler/lhb/__init__.py +1 -1
- mns_scheduler/lhb/stock_lhb_sync_service.py +1 -1
- mns_scheduler/open/__init__.py +1 -1
- mns_scheduler/open/sync_one_day_open_data_to_db_service.py +2 -3
- mns_scheduler/risk/__init__.py +1 -1
- mns_scheduler/risk/compliance/undisclosed_annual_report_api.py +8 -2
- mns_scheduler/risk/financial/annual_report_audit_check_api.py +13 -3
- mns_scheduler/risk/financial/net_assets_check_api.py +21 -18
- mns_scheduler/risk/financial/profit_income_check_api.py +7 -2
- mns_scheduler/risk/financial_report_risk_check_api.py +1 -1
- mns_scheduler/risk/major_violations/register_and_investigate_stock_sync_api.py +1 -1
- mns_scheduler/risk/self/wei_pan_stock_api.py +1 -1
- mns_scheduler/risk/test/__init__.py +1 -1
- mns_scheduler/risk/test/fix_blask_list.py +4 -6
- mns_scheduler/risk/transactions/transactions_check_api.py +22 -4
- mns_scheduler/self_choose/__init__.py +1 -1
- mns_scheduler/self_choose/ths_self_choose_service.py +99 -43
- mns_scheduler/trade/auto_login/trader_auto_service.py +7 -4
- mns_scheduler/trade/auto_sell_service_api.py +4 -4
- mns_scheduler/trade/balance/__init__.py +7 -0
- mns_scheduler/trade/balance/ths_account_balance_service.py +7 -0
- mns_scheduler/trade/sync_position_api.py +39 -6
- mns_scheduler/trade/task/trader_task_service.py +38 -10
- mns_scheduler/trade/tfp/__init__.py +7 -0
- mns_scheduler/trade/tfp/stock_tfp_info_sync.py +56 -0
- mns_scheduler/zb/stock_zb_pool_sync.py +1 -16
- mns_scheduler/zt/high_chg/sync_high_chg_pool_service.py +2 -2
- mns_scheduler/zt/high_chg/sync_high_chg_real_time_quotes_service.py +1 -1
- mns_scheduler/zt/script/__init__.py +1 -1
- mns_scheduler/zt/script/fix_error_deal_day.py +41 -0
- mns_scheduler/zt/script/kcx_high_chg_open_his_data_handle.py +2 -2
- mns_scheduler/zt/script/sync_high_chg_pool_his_data.py +2 -2
- mns_scheduler/zt/script/sync_now_higt_chg_zt.py +8 -7
- mns_scheduler/zt/zt_pool/em_zt_pool_sync_api.py +260 -83
- mns_scheduler/zt/zt_pool/ths_zt_pool_sync_api.py +33 -90
- mns_scheduler/zt/zt_pool/update_null_zt_reason_api.py +27 -13
- mns_scheduler/zz_task/compensation/__init__.py +0 -0
- mns_scheduler/zz_task/compensation/compensate_task.py +161 -0
- mns_scheduler/zz_task/compensation/compensate_task_one_day.py +142 -0
- mns_scheduler/zz_task/data_sync_task.py +197 -103
- {mns_scheduler-1.1.8.4.dist-info → mns_scheduler-1.4.5.7.dist-info}/METADATA +1 -1
- mns_scheduler-1.4.5.7.dist-info/RECORD +176 -0
- {mns_scheduler-1.1.8.4.dist-info → mns_scheduler-1.4.5.7.dist-info}/WHEEL +1 -1
- mns_scheduler/2014-2015-test/2014_2015_chg_statistics.py +0 -87
- mns_scheduler/big_deal/ths_big_deal_sync.py +0 -98
- mns_scheduler/company_info/base/sync_company_base_info_api.py +0 -439
- mns_scheduler/company_info/base/sync_company_hold_info_api.py +0 -40
- mns_scheduler/company_info/remark/company_remark_info_sync.py +0 -46
- mns_scheduler/db/real_time_task_check.py +0 -84
- mns_scheduler/debt/kzz_bond_info_sync.py +0 -33
- mns_scheduler-1.1.8.4.dist-info/RECORD +0 -142
- /mns_scheduler/{2014-2015-test → auto_da_ban}/__init__.py +0 -0
- /mns_scheduler/company_info/{base → common}/__init__.py +0 -0
- /mns_scheduler/{big_deal → company_info/em_stock_info}/__init__.py +0 -0
- /mns_scheduler/company_info/{remark → sync}/__init__.py +0 -0
- /mns_scheduler/{debt → company_info/task}/__init__.py +0 -0
- /mns_scheduler/k_line/{sync → month_week_daily}/__init__.py +0 -0
- {mns_scheduler-1.1.8.4.dist-info → mns_scheduler-1.4.5.7.dist-info}/top_level.txt +0 -0
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
import sys
|
|
2
|
+
import os
|
|
3
|
+
|
|
4
|
+
file_path = os.path.abspath(__file__)
|
|
5
|
+
end = file_path.index('mns') + 17
|
|
6
|
+
project_path = file_path[0:end]
|
|
7
|
+
sys.path.append(project_path)
|
|
8
|
+
import mns_common.api.ths.company.ths_company_info_web as ths_company_info_web
|
|
9
|
+
from mns_common.db.MongodbUtil import MongodbUtil
|
|
10
|
+
import mns_common.utils.data_frame_util as data_frame_util
|
|
11
|
+
import mns_common.constant.db_name_constant as db_name_constant
|
|
12
|
+
from loguru import logger
|
|
13
|
+
from mns_scheduler.company_info.common.company_common_query_service import get_company_info
|
|
14
|
+
import time
|
|
15
|
+
from datetime import datetime
|
|
16
|
+
|
|
17
|
+
mongodb_util = MongodbUtil('27017')
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def sync_company_base_info_task(symbol_list):
|
|
21
|
+
all_company_info_df = get_company_info()
|
|
22
|
+
if len(symbol_list) > 0:
|
|
23
|
+
all_company_info_df = all_company_info_df.loc[all_company_info_df['symbol'].isin(symbol_list)]
|
|
24
|
+
fail_list = []
|
|
25
|
+
for stock_one in all_company_info_df.itertuples():
|
|
26
|
+
try:
|
|
27
|
+
sync_one_symbol_base_info(stock_one.symbol)
|
|
28
|
+
time.sleep(0.5)
|
|
29
|
+
except BaseException as e:
|
|
30
|
+
time.sleep(3)
|
|
31
|
+
logger.error("同步公司基础信息发生异常:{},{}", stock_one.symbol, e)
|
|
32
|
+
fail_list.append(stock_one.symbol)
|
|
33
|
+
sync_number = 1
|
|
34
|
+
while len(fail_list) > 0 and sync_number < 10:
|
|
35
|
+
for symbol in fail_list:
|
|
36
|
+
try:
|
|
37
|
+
sync_one_symbol_base_info(symbol)
|
|
38
|
+
time.sleep(5)
|
|
39
|
+
except BaseException as e:
|
|
40
|
+
time.sleep(10)
|
|
41
|
+
logger.error("同步公司基础信息发生异常:{},{}", symbol, e)
|
|
42
|
+
sync_number = sync_number + 1
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def sync_one_symbol_base_info(symbol):
|
|
46
|
+
company_remark_info = ths_company_info_web.get_company_info(symbol)
|
|
47
|
+
company_remark_info['_id'] = symbol
|
|
48
|
+
company_remark_info['symbol'] = symbol
|
|
49
|
+
company_remark_info['remark'] = ''
|
|
50
|
+
|
|
51
|
+
now_date = datetime.now()
|
|
52
|
+
sync_str_date = now_date.strftime('%Y-%m-%d %H:%M:%S')
|
|
53
|
+
|
|
54
|
+
company_remark_info['sync_str_date'] = sync_str_date
|
|
55
|
+
|
|
56
|
+
exist_company_remark_df = mongodb_util.find_query_data(db_name_constant.COMPANY_BASE_INFO,
|
|
57
|
+
query={"symbol": symbol})
|
|
58
|
+
if data_frame_util.is_not_empty(exist_company_remark_df):
|
|
59
|
+
company_remark_info['remark'] = list(exist_company_remark_df['remark'])[0]
|
|
60
|
+
mongodb_util.save_mongo(company_remark_info, db_name_constant.COMPANY_BASE_INFO)
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
if __name__ == '__main__':
|
|
64
|
+
sync_company_base_info_task([])
|
|
@@ -0,0 +1,177 @@
|
|
|
1
|
+
import os
|
|
2
|
+
import sys
|
|
3
|
+
|
|
4
|
+
file_path = os.path.abspath(__file__)
|
|
5
|
+
end = file_path.index('mns') + 17
|
|
6
|
+
project_path = file_path[0:end]
|
|
7
|
+
sys.path.append(project_path)
|
|
8
|
+
|
|
9
|
+
import mns_common.component.common_service_fun_api as common_service_fun_api
|
|
10
|
+
import mns_common.api.ths.company.company_product_area_industry_index_query as company_product_area_industry_index_query
|
|
11
|
+
from loguru import logger
|
|
12
|
+
from mns_common.db.MongodbUtil import MongodbUtil
|
|
13
|
+
import mns_common.constant.db_name_constant as db_name_constant
|
|
14
|
+
import mns_common.utils.data_frame_util as data_frame_util
|
|
15
|
+
import pandas as pd
|
|
16
|
+
from datetime import datetime
|
|
17
|
+
from mns_scheduler.company_info.common.company_common_query_service import get_company_info
|
|
18
|
+
|
|
19
|
+
mongodb_util = MongodbUtil('27017')
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def sync_company_business_task(symbol_list):
|
|
23
|
+
now_date = datetime.now()
|
|
24
|
+
now_year = now_date.year
|
|
25
|
+
now_month = now_date.month
|
|
26
|
+
|
|
27
|
+
if now_month in [1, 2, 3, 4]:
|
|
28
|
+
period_time_year = str(now_year - 1) + "-12-31"
|
|
29
|
+
sync_company_product_area_industry(symbol_list, period_time_year)
|
|
30
|
+
|
|
31
|
+
if now_month in [4, 5, 6]:
|
|
32
|
+
period_time_one = str(now_year) + "-03-31"
|
|
33
|
+
sync_company_product_area_industry(symbol_list, period_time_one)
|
|
34
|
+
|
|
35
|
+
elif now_month in [7, 8, 9]:
|
|
36
|
+
period_time_two = str(now_year) + "-06-30"
|
|
37
|
+
sync_company_product_area_industry(symbol_list, period_time_two)
|
|
38
|
+
|
|
39
|
+
elif now_month in [10, 11, 12]:
|
|
40
|
+
period_time_three = str(now_year) + "-09-30"
|
|
41
|
+
sync_company_product_area_industry(symbol_list, period_time_three)
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def sync_company_product_area_industry(symbol_list, date):
|
|
45
|
+
all_company_info_df = get_company_info()
|
|
46
|
+
|
|
47
|
+
all_company_info_df = common_service_fun_api.classify_symbol(all_company_info_df)
|
|
48
|
+
if len(symbol_list) > 0:
|
|
49
|
+
all_company_info_df = all_company_info_df.loc[all_company_info_df['symbol'].isin(symbol_list)]
|
|
50
|
+
for stock_one in all_company_info_df.itertuples():
|
|
51
|
+
try:
|
|
52
|
+
symbol = stock_one.symbol
|
|
53
|
+
|
|
54
|
+
classification = stock_one.classification
|
|
55
|
+
if classification in ['H', 'K']:
|
|
56
|
+
market = '17'
|
|
57
|
+
elif classification in ['S', 'C']:
|
|
58
|
+
market = '33'
|
|
59
|
+
elif classification in ['X']:
|
|
60
|
+
market = '151'
|
|
61
|
+
|
|
62
|
+
query_exist = {'symbol': symbol, 'time': date}
|
|
63
|
+
exist_company_business_info_df = mongodb_util.find_query_data(db_name_constant.COMPANY_BUSINESS_INFO,
|
|
64
|
+
query_exist)
|
|
65
|
+
if data_frame_util.is_empty(exist_company_business_info_df):
|
|
66
|
+
exist_all = False
|
|
67
|
+
else:
|
|
68
|
+
exist_all = (
|
|
69
|
+
(exist_company_business_info_df.loc[
|
|
70
|
+
exist_company_business_info_df['analysis_type'] == 'area'].shape[0] > 0)
|
|
71
|
+
and (exist_company_business_info_df.loc[
|
|
72
|
+
exist_company_business_info_df['analysis_type'] == 'industry'].shape[0] > 0)
|
|
73
|
+
and (exist_company_business_info_df.loc[
|
|
74
|
+
exist_company_business_info_df['analysis_type'] == 'product'].shape[
|
|
75
|
+
0] > 0))
|
|
76
|
+
if exist_all:
|
|
77
|
+
continue
|
|
78
|
+
company_product_area_industry_list = company_product_area_industry_index_query.company_product_area_industry(
|
|
79
|
+
symbol, market, date)
|
|
80
|
+
for company_one in company_product_area_industry_list:
|
|
81
|
+
try:
|
|
82
|
+
analysis_type = company_one['analysis_type']
|
|
83
|
+
time_operate_index_item_list = company_one['time_operate_index_item_list']
|
|
84
|
+
time_operate_index_item_df = pd.DataFrame(time_operate_index_item_list)
|
|
85
|
+
if data_frame_util.is_empty(time_operate_index_item_df):
|
|
86
|
+
continue
|
|
87
|
+
time_operate_index_item_df['symbol'] = symbol
|
|
88
|
+
time_operate_index_item_df['analysis_type'] = analysis_type
|
|
89
|
+
|
|
90
|
+
time_operate_index_item_df['_id'] = symbol + '_' + time_operate_index_item_df[
|
|
91
|
+
'time'] + '_' + analysis_type
|
|
92
|
+
handle_industry_area_product(time_operate_index_item_df, symbol)
|
|
93
|
+
except BaseException as e:
|
|
94
|
+
logger.error("同步经营数据异常:{},{}", symbol, e)
|
|
95
|
+
except BaseException as e:
|
|
96
|
+
logger.error("同步经营数据异常:{},{}", stock_one.symbol, e)
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
def handle_industry_area_product(time_operate_index_item_df, symbol):
|
|
100
|
+
if data_frame_util.is_empty(time_operate_index_item_df):
|
|
101
|
+
return None
|
|
102
|
+
|
|
103
|
+
for business_one in time_operate_index_item_df.itertuples():
|
|
104
|
+
time = business_one.time
|
|
105
|
+
analysis_type = business_one.analysis_type
|
|
106
|
+
|
|
107
|
+
product_index_item_list = business_one.product_index_item_list
|
|
108
|
+
for product_one in product_index_item_list:
|
|
109
|
+
try:
|
|
110
|
+
# 初始化数据
|
|
111
|
+
income_amount = 0
|
|
112
|
+
income_percent = 0
|
|
113
|
+
cost_amount = 0
|
|
114
|
+
cost_percent = 0
|
|
115
|
+
gross_profit_amount = 0
|
|
116
|
+
gross_profit_percent = 0
|
|
117
|
+
gross_profit_rate_amount = 0
|
|
118
|
+
gross_profit_rate_percent = 0
|
|
119
|
+
|
|
120
|
+
product_name = product_one['product_name']
|
|
121
|
+
index_analysis_list = product_one['index_analysis_list']
|
|
122
|
+
for index_one in index_analysis_list:
|
|
123
|
+
try:
|
|
124
|
+
index_id = index_one['index_id']
|
|
125
|
+
if index_id == 'income':
|
|
126
|
+
income_amount = index_one['index_value']
|
|
127
|
+
income_percent = index_one['account']
|
|
128
|
+
elif index_id == 'cost':
|
|
129
|
+
cost_amount = index_one['index_value']
|
|
130
|
+
cost_percent = index_one['account']
|
|
131
|
+
elif index_id == 'gross_profit':
|
|
132
|
+
gross_profit_amount = index_one['index_value']
|
|
133
|
+
gross_profit_percent = index_one['account']
|
|
134
|
+
|
|
135
|
+
elif index_id == 'gross_profit_rate':
|
|
136
|
+
gross_profit_rate_amount = index_one['index_value']
|
|
137
|
+
gross_profit_rate_percent = index_one['account']
|
|
138
|
+
except BaseException as e:
|
|
139
|
+
logger.error("同步经营数据异常:{},{}", symbol, e)
|
|
140
|
+
|
|
141
|
+
id_key = symbol + '_' + time + '_' + analysis_type + '_' + product_name
|
|
142
|
+
result_dict = {
|
|
143
|
+
'_id': id_key,
|
|
144
|
+
'symbol': symbol,
|
|
145
|
+
'time': time,
|
|
146
|
+
'analysis_type': analysis_type,
|
|
147
|
+
'product_name': product_name,
|
|
148
|
+
|
|
149
|
+
'income_amount': income_amount,
|
|
150
|
+
'income_percent': income_percent,
|
|
151
|
+
|
|
152
|
+
'cost_amount': cost_amount,
|
|
153
|
+
'cost_percent': cost_percent,
|
|
154
|
+
|
|
155
|
+
'gross_profit_amount': gross_profit_amount,
|
|
156
|
+
'gross_profit_percent': gross_profit_percent,
|
|
157
|
+
|
|
158
|
+
'gross_profit_rate_amount': gross_profit_rate_amount,
|
|
159
|
+
'gross_profit_rate_percent': gross_profit_rate_percent,
|
|
160
|
+
}
|
|
161
|
+
result_dict_df = pd.DataFrame(result_dict, index=[1])
|
|
162
|
+
mongodb_util.save_mongo(result_dict_df, db_name_constant.COMPANY_BUSINESS_INFO)
|
|
163
|
+
except BaseException as e:
|
|
164
|
+
logger.error("同步经营数据异常:{},{}", symbol, e)
|
|
165
|
+
|
|
166
|
+
|
|
167
|
+
if __name__ == '__main__':
|
|
168
|
+
sync_company_product_area_industry('300211', '2024-12-31')
|
|
169
|
+
# sync_company_product_area_industry('002323')
|
|
170
|
+
# sync_company_product_area_industry('300901')
|
|
171
|
+
# sync_company_product_area_industry('603225')
|
|
172
|
+
# sync_company_product_area_industry('688039')
|
|
173
|
+
# sync_company_product_area_industry('600849')
|
|
174
|
+
# sync_company_product_area_industry('000508')
|
|
175
|
+
# sync_company_product_area_industry('810011')
|
|
176
|
+
|
|
177
|
+
sync_company_product_area_industry([], None)
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
import sys
|
|
2
|
+
import os
|
|
3
|
+
|
|
4
|
+
file_path = os.path.abspath(__file__)
|
|
5
|
+
end = file_path.index('mns') + 16
|
|
6
|
+
project_path = file_path[0:end]
|
|
7
|
+
sys.path.append(project_path)
|
|
8
|
+
import mns_common.api.ths.company.ths_company_info_api as ths_company_info_api
|
|
9
|
+
import mns_common.component.cookie.cookie_info_service as cookie_info_service
|
|
10
|
+
import mns_common.utils.data_frame_util as data_frame_util
|
|
11
|
+
from mns_common.db.MongodbUtil import MongodbUtil
|
|
12
|
+
import mns_common.constant.db_name_constant as db_name_constant
|
|
13
|
+
from datetime import datetime
|
|
14
|
+
from loguru import logger
|
|
15
|
+
from mns_scheduler.company_info.common.company_common_query_service import get_company_info
|
|
16
|
+
import time
|
|
17
|
+
|
|
18
|
+
mongodb_util = MongodbUtil('27017')
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
# 同步公司控股子公司信息任务
|
|
22
|
+
def sync_one_company_hold_info(symbol):
|
|
23
|
+
try:
|
|
24
|
+
ths_cookie = cookie_info_service.get_ths_cookie()
|
|
25
|
+
company_hold_info_df = ths_company_info_api.get_company_hold_info(symbol, ths_cookie)
|
|
26
|
+
|
|
27
|
+
now_date = datetime.now()
|
|
28
|
+
sync_str_date = now_date.strftime('%Y-%m-%d %H:%M:%S')
|
|
29
|
+
|
|
30
|
+
if data_frame_util.is_not_empty(company_hold_info_df):
|
|
31
|
+
query_exist = {'symbol': symbol}
|
|
32
|
+
exist_company_holding_info_df = mongodb_util.find_query_data(db_name_constant.COMPANY_HOLDING_INFO,
|
|
33
|
+
query_exist)
|
|
34
|
+
if data_frame_util.is_not_empty(exist_company_holding_info_df):
|
|
35
|
+
# 作废不在参股控股的子公司
|
|
36
|
+
invalid_symbol_df = exist_company_holding_info_df.loc[
|
|
37
|
+
~exist_company_holding_info_df['symbol'].isin(list(company_hold_info_df['symbol']))]
|
|
38
|
+
invalid_symbol_df['valid'] = False
|
|
39
|
+
|
|
40
|
+
company_hold_info_df['valid'] = True
|
|
41
|
+
company_hold_info_df['_id'] = company_hold_info_df['symbol'] + '_' + company_hold_info_df[
|
|
42
|
+
'holding_company']
|
|
43
|
+
company_hold_info_df['sync_str_date'] = sync_str_date
|
|
44
|
+
mongodb_util.save_mongo(company_hold_info_df, db_name_constant.COMPANY_HOLDING_INFO)
|
|
45
|
+
else:
|
|
46
|
+
logger.warning("同步控股子公司为空:{}", symbol)
|
|
47
|
+
|
|
48
|
+
except BaseException as e:
|
|
49
|
+
logger.error("同步公司控股子公司信息:{},{}", symbol, e)
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def sync_all_company_hold_info_task(symbol_list):
|
|
53
|
+
all_company_info_df = get_company_info()
|
|
54
|
+
if len(symbol_list) > 0:
|
|
55
|
+
all_company_info_df = all_company_info_df.loc[all_company_info_df['symbol'].isin(symbol_list)]
|
|
56
|
+
for stock_one in all_company_info_df.itertuples():
|
|
57
|
+
try:
|
|
58
|
+
sync_one_company_hold_info(stock_one.symbol)
|
|
59
|
+
time.sleep(1)
|
|
60
|
+
except BaseException as e:
|
|
61
|
+
logger.error("同步控股子公司信息异常:{},{}", stock_one.symbol, e)
|
|
62
|
+
time.sleep(5)
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
if __name__ == '__main__':
|
|
66
|
+
sync_all_company_hold_info_task([])
|
|
@@ -0,0 +1,167 @@
|
|
|
1
|
+
import sys
|
|
2
|
+
import os
|
|
3
|
+
|
|
4
|
+
file_path = os.path.abspath(__file__)
|
|
5
|
+
end = file_path.index('mns') + 16
|
|
6
|
+
project_path = file_path[0:end]
|
|
7
|
+
sys.path.append(project_path)
|
|
8
|
+
import mns_common.api.ths.company.ths_company_info_web as ths_company_info_web
|
|
9
|
+
from mns_common.db.MongodbUtil import MongodbUtil
|
|
10
|
+
import mns_common.constant.db_name_constant as db_name_constant
|
|
11
|
+
from loguru import logger
|
|
12
|
+
from mns_scheduler.company_info.common.company_common_query_service import get_company_info
|
|
13
|
+
import time
|
|
14
|
+
import mns_common.utils.data_frame_util as data_frame_util
|
|
15
|
+
import mns_common.component.common_service_fun_api as common_service_fun_api
|
|
16
|
+
|
|
17
|
+
mongodb_util = MongodbUtil('27017')
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def sync_company_industry_info_task(symbol_list):
|
|
21
|
+
all_company_info_df = get_company_info()
|
|
22
|
+
if len(symbol_list) > 0:
|
|
23
|
+
all_company_info_df = all_company_info_df.loc[all_company_info_df['symbol'].isin(symbol_list)]
|
|
24
|
+
|
|
25
|
+
all_company_info_df = common_service_fun_api.classify_symbol(all_company_info_df)
|
|
26
|
+
fail_list = []
|
|
27
|
+
for stock_one in all_company_info_df.itertuples():
|
|
28
|
+
try:
|
|
29
|
+
tag = sync_one_company_industry_info(stock_one.symbol, stock_one.classification)
|
|
30
|
+
if bool(1 - tag):
|
|
31
|
+
fail_list.append(stock_one.symbol)
|
|
32
|
+
time.sleep(0.5)
|
|
33
|
+
except BaseException as e:
|
|
34
|
+
time.sleep(1)
|
|
35
|
+
logger.error("同步公司行业信息发生异常:{},{}", stock_one.symbol, e)
|
|
36
|
+
fail_list.append(stock_one.symbol)
|
|
37
|
+
sync_number = 1
|
|
38
|
+
while len(fail_list) > 0 and sync_number < 10:
|
|
39
|
+
for symbol in fail_list:
|
|
40
|
+
try:
|
|
41
|
+
company_info_one_df = all_company_info_df.loc[all_company_info_df['symbol'] == symbol]
|
|
42
|
+
classification = list(company_info_one_df['classification'])[0]
|
|
43
|
+
tag = sync_one_company_industry_info(symbol, classification)
|
|
44
|
+
if tag and symbol in fail_list:
|
|
45
|
+
fail_list.remove(symbol)
|
|
46
|
+
time.sleep(2)
|
|
47
|
+
except BaseException as e:
|
|
48
|
+
time.sleep(3)
|
|
49
|
+
logger.error("同步公司行业信息发生异常:{},{}", symbol, e)
|
|
50
|
+
sync_number = sync_number + 1
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def sync_one_company_industry_info(symbol, classification):
|
|
54
|
+
if classification in ['H', 'K']:
|
|
55
|
+
market_id = '17'
|
|
56
|
+
elif classification in ['S', 'C']:
|
|
57
|
+
market_id = '33'
|
|
58
|
+
elif classification in ['X']:
|
|
59
|
+
market_id = '151'
|
|
60
|
+
|
|
61
|
+
company_industry_info = ths_company_info_web.get_company_info_detail(symbol, market_id)
|
|
62
|
+
if data_frame_util.is_empty(company_industry_info):
|
|
63
|
+
return False
|
|
64
|
+
|
|
65
|
+
company_industry_info['first_industry_code'] = company_industry_info['hycode'].apply(
|
|
66
|
+
lambda x: x[1:3] + '0000')
|
|
67
|
+
company_industry_info['second_industry_code'] = company_industry_info['hy2code'].apply(
|
|
68
|
+
lambda x: x[1:5] + '00')
|
|
69
|
+
company_industry_info['third_industry_code'] = company_industry_info['hy3code'].apply(
|
|
70
|
+
lambda x: x[1:7])
|
|
71
|
+
|
|
72
|
+
company_industry_info['first_sw_industry'] = company_industry_info['hy']
|
|
73
|
+
company_industry_info['second_sw_industry'] = company_industry_info['hy2']
|
|
74
|
+
company_industry_info['third_sw_industry'] = company_industry_info['hy3']
|
|
75
|
+
del company_industry_info['hy']
|
|
76
|
+
del company_industry_info['hy2']
|
|
77
|
+
del company_industry_info['hy3']
|
|
78
|
+
del company_industry_info['hycode']
|
|
79
|
+
del company_industry_info['hy2code']
|
|
80
|
+
del company_industry_info['hy3code']
|
|
81
|
+
|
|
82
|
+
company_industry_info['_id'] = symbol
|
|
83
|
+
company_industry_info['symbol'] = symbol
|
|
84
|
+
mongodb_util.save_mongo(company_industry_info, db_name_constant.COMPANY_INDUSTRY_INFO)
|
|
85
|
+
# 保存股票申万行业原始信息
|
|
86
|
+
save_sw_data(company_industry_info)
|
|
87
|
+
return True
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
# 保存申万行业分类
|
|
91
|
+
def save_sw_data(company_industry_info):
|
|
92
|
+
first_sw_info = company_industry_info[[
|
|
93
|
+
'first_sw_industry',
|
|
94
|
+
'first_industry_code'
|
|
95
|
+
]].copy()
|
|
96
|
+
|
|
97
|
+
first_sw_info.loc[:, "industry_code"] = first_sw_info['first_industry_code']
|
|
98
|
+
first_sw_info.loc[:, "_id"] = first_sw_info['first_industry_code']
|
|
99
|
+
first_sw_info.loc[:, "second_sw_industry"] = 0
|
|
100
|
+
first_sw_info.loc[:, "third_sw_industry"] = 0
|
|
101
|
+
first_sw_info.loc[:, "second_industry_code"] = 0
|
|
102
|
+
first_sw_info.loc[:, "third_industry_code"] = 0
|
|
103
|
+
first_sw_info = first_sw_info[[
|
|
104
|
+
"_id",
|
|
105
|
+
"industry_code",
|
|
106
|
+
'first_industry_code',
|
|
107
|
+
'first_sw_industry',
|
|
108
|
+
'second_industry_code',
|
|
109
|
+
'second_sw_industry',
|
|
110
|
+
'third_industry_code',
|
|
111
|
+
'third_sw_industry'
|
|
112
|
+
]]
|
|
113
|
+
mongodb_util.save_mongo(first_sw_info, 'sw_industry')
|
|
114
|
+
|
|
115
|
+
second_sw_info = company_industry_info[[
|
|
116
|
+
'first_industry_code',
|
|
117
|
+
'first_sw_industry',
|
|
118
|
+
'second_sw_industry',
|
|
119
|
+
'second_industry_code',
|
|
120
|
+
]].copy()
|
|
121
|
+
|
|
122
|
+
second_sw_info.loc[:, "industry_code"] = second_sw_info['second_industry_code']
|
|
123
|
+
second_sw_info.loc[:, "_id"] = second_sw_info['industry_code']
|
|
124
|
+
|
|
125
|
+
second_sw_info.loc[:, "third_sw_industry"] = 0
|
|
126
|
+
second_sw_info.loc[:, "third_sw_industry"] = 0
|
|
127
|
+
second_sw_info.loc[:, "third_industry_code"] = 0
|
|
128
|
+
second_sw_info = second_sw_info[[
|
|
129
|
+
"_id",
|
|
130
|
+
"industry_code",
|
|
131
|
+
'first_industry_code',
|
|
132
|
+
'first_sw_industry',
|
|
133
|
+
'second_industry_code',
|
|
134
|
+
'second_sw_industry',
|
|
135
|
+
'third_industry_code',
|
|
136
|
+
'third_sw_industry'
|
|
137
|
+
]]
|
|
138
|
+
mongodb_util.save_mongo(second_sw_info, 'sw_industry')
|
|
139
|
+
|
|
140
|
+
third_sw_info = company_industry_info[[
|
|
141
|
+
'first_industry_code',
|
|
142
|
+
'first_sw_industry',
|
|
143
|
+
'second_industry_code',
|
|
144
|
+
'second_sw_industry',
|
|
145
|
+
'third_industry_code',
|
|
146
|
+
'third_sw_industry'
|
|
147
|
+
]].copy()
|
|
148
|
+
|
|
149
|
+
third_sw_info.loc[:, "industry_code"] = third_sw_info['third_industry_code']
|
|
150
|
+
|
|
151
|
+
third_sw_info.loc[:, "_id"] = third_sw_info['industry_code']
|
|
152
|
+
|
|
153
|
+
third_sw_info = third_sw_info[[
|
|
154
|
+
"_id",
|
|
155
|
+
"industry_code",
|
|
156
|
+
'first_industry_code',
|
|
157
|
+
'first_sw_industry',
|
|
158
|
+
'second_industry_code',
|
|
159
|
+
'second_sw_industry',
|
|
160
|
+
'third_industry_code',
|
|
161
|
+
'third_sw_industry'
|
|
162
|
+
]]
|
|
163
|
+
mongodb_util.save_mongo(third_sw_info, 'sw_industry')
|
|
164
|
+
|
|
165
|
+
|
|
166
|
+
if __name__ == '__main__':
|
|
167
|
+
sync_company_industry_info_task(['688795'])
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
import sys
|
|
2
|
+
import os
|
|
3
|
+
|
|
4
|
+
file_path = os.path.abspath(__file__)
|
|
5
|
+
end = file_path.index('mns') + 16
|
|
6
|
+
project_path = file_path[0:end]
|
|
7
|
+
sys.path.append(project_path)
|
|
8
|
+
import mns_scheduler.company_info.task.company_industry_info_task as company_industry_info_task
|
|
9
|
+
import mns_scheduler.company_info.task.company_business_info_task as company_business_info_task
|
|
10
|
+
import mns_scheduler.company_info.task.company_base_info_task as company_base_info_task
|
|
11
|
+
import mns_scheduler.company_info.task.company_hold_info_task as company_hold_info_task
|
|
12
|
+
import mns_scheduler.company_info.task.company_announce_info_task as company_announce_info_task
|
|
13
|
+
from mns_scheduler.company_info.common.company_common_query_service import get_company_info
|
|
14
|
+
from datetime import datetime
|
|
15
|
+
import mns_common.utils.data_frame_util as data_frame_util
|
|
16
|
+
from loguru import logger
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def sync_all_company_task():
|
|
20
|
+
# 同步公司行业信息
|
|
21
|
+
company_industry_info_task.sync_company_industry_info_task([])
|
|
22
|
+
logger.info("[全量]同步公司[行业]信息完成")
|
|
23
|
+
# 同步公司业务信息
|
|
24
|
+
company_business_info_task.sync_company_business_task([])
|
|
25
|
+
logger.info("[全量]同步公司[业务]信息完成")
|
|
26
|
+
# 同步公司基本信息
|
|
27
|
+
company_base_info_task.sync_company_base_info_task([])
|
|
28
|
+
logger.info("[全量]同步公司[基本]信息完成")
|
|
29
|
+
# 同步公司控股信息
|
|
30
|
+
company_hold_info_task.sync_all_company_hold_info_task([])
|
|
31
|
+
logger.info("[全量]同步公司[控股]信息完成")
|
|
32
|
+
# 同步公司公告信息
|
|
33
|
+
company_announce_info_task.sync_company_announce_task([])
|
|
34
|
+
logger.info("[全量]同步公司[公告]信息完成")
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def sync_new_stock_company_task():
|
|
38
|
+
all_company_info_df = get_company_info()
|
|
39
|
+
|
|
40
|
+
now_date = datetime.now()
|
|
41
|
+
str_day = now_date.strftime('%Y-%m-%d')
|
|
42
|
+
# 去掉横线并转换为整数
|
|
43
|
+
now_day = int(str_day.replace('-', ''))
|
|
44
|
+
|
|
45
|
+
new_company_info_df = all_company_info_df.loc[
|
|
46
|
+
(all_company_info_df['list_date'] == 19890604) | (all_company_info_df['list_date'] >= now_day)]
|
|
47
|
+
if data_frame_util.is_empty(new_company_info_df):
|
|
48
|
+
return
|
|
49
|
+
symbol_list = list(new_company_info_df['symbol'])
|
|
50
|
+
# 同步公司行业信息
|
|
51
|
+
company_industry_info_task.sync_company_industry_info_task(symbol_list)
|
|
52
|
+
logger.info("[新股]同步公司[行业]信息完成")
|
|
53
|
+
# 同步公司业务信息
|
|
54
|
+
company_business_info_task.sync_company_business_task(symbol_list)
|
|
55
|
+
logger.info("[新股]同步公司[业务]信息完成")
|
|
56
|
+
# 同步公司基本信息
|
|
57
|
+
company_base_info_task.sync_company_base_info_task(symbol_list)
|
|
58
|
+
logger.info("[新股]同步公司[基本]信息完成")
|
|
59
|
+
# 同步公司控股信息
|
|
60
|
+
company_hold_info_task.sync_all_company_hold_info_task(symbol_list)
|
|
61
|
+
logger.info("[新股]同步公司[控股]信息完成")
|
|
62
|
+
# 同步公司公告信息
|
|
63
|
+
company_announce_info_task.sync_company_announce_task(symbol_list)
|
|
64
|
+
logger.info("[新股]同步公司[公告]信息完成")
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
if __name__ == '__main__':
|
|
68
|
+
sync_new_stock_company_task()
|
|
69
|
+
sync_all_company_task()
|
|
@@ -7,12 +7,13 @@ project_path = file_path[0:end]
|
|
|
7
7
|
sys.path.append(project_path)
|
|
8
8
|
from mns_common.db.MongodbUtil import MongodbUtil
|
|
9
9
|
import pandas as pd
|
|
10
|
-
import
|
|
10
|
+
import mns_common.api.ths.concept.web.ths_concept_index_web as ths_concept_index_web
|
|
11
11
|
import time
|
|
12
12
|
from loguru import logger
|
|
13
13
|
import mns_common.component.company.company_common_service_api as company_common_service_api
|
|
14
14
|
import mns_common.constant.db_name_constant as db_name_constant
|
|
15
15
|
import mns_common.component.concept.ths_concept_common_service_api as ths_concept_common_service_api
|
|
16
|
+
import mns_common.api.ths.concept.app.ths_concept_index_app as ths_concept_index_app
|
|
16
17
|
|
|
17
18
|
mongodb_util = MongodbUtil('27017')
|
|
18
19
|
|
|
@@ -76,9 +77,22 @@ def update_one_concept_relevance_industry(concept_code):
|
|
|
76
77
|
ths_stock_concept_detail_df['first_relevance_industry'] = first_relevance_industry
|
|
77
78
|
ths_stock_concept_detail_df['second_relevance_industry'] = second_relevance_industry
|
|
78
79
|
|
|
79
|
-
|
|
80
|
+
new_values_list = {"$set": {"first_relevance_industry": first_relevance_industry,
|
|
81
|
+
"second_relevance_industry": second_relevance_industry,
|
|
82
|
+
"first_relevance_industry_number": first_relevance_industry_number,
|
|
83
|
+
"second_relevance_industry_number": second_relevance_industry_number,
|
|
84
|
+
"concept_count": concept_count
|
|
85
|
+
}}
|
|
80
86
|
|
|
81
|
-
|
|
87
|
+
update_query = {"symbol": concept_code}
|
|
88
|
+
mongodb_util.update_one_query(update_query, new_values_list, db_name_constant.THS_CONCEPT_LIST)
|
|
89
|
+
|
|
90
|
+
new_values_detail = {"$set": {"first_relevance_industry": first_relevance_industry,
|
|
91
|
+
"second_relevance_industry": second_relevance_industry
|
|
92
|
+
|
|
93
|
+
}}
|
|
94
|
+
update_query_detail = {"concept_code": concept_code}
|
|
95
|
+
mongodb_util.update_one_query(update_query_detail, new_values_detail, db_name_constant.THS_STOCK_CONCEPT_DETAIL)
|
|
82
96
|
|
|
83
97
|
|
|
84
98
|
# 更新空名字
|
|
@@ -93,7 +107,7 @@ def update_null_name():
|
|
|
93
107
|
exist_url = concept_one.url
|
|
94
108
|
|
|
95
109
|
if name == '':
|
|
96
|
-
concept_name =
|
|
110
|
+
concept_name = ths_concept_index_web.get_concept_name(concept_code)
|
|
97
111
|
query_concept = {"symbol": concept_code}
|
|
98
112
|
new_values = {'$set': {"name": concept_name}}
|
|
99
113
|
mongodb_util.update_one_query(query_concept, new_values, 'ths_concept_list')
|
|
@@ -114,6 +128,8 @@ def update_null_name():
|
|
|
114
128
|
|
|
115
129
|
|
|
116
130
|
if __name__ == '__main__':
|
|
131
|
+
# update_one_concept_relevance_industry(886095)
|
|
132
|
+
update_null_name()
|
|
117
133
|
logger.info("开始")
|
|
118
134
|
update_ths_concept_info()
|
|
119
135
|
logger.info("结束")
|