mns-scheduler 1.1.8.4__py3-none-any.whl → 1.4.3.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- mns_scheduler/__init__.py +1 -3
- mns_scheduler/company_info/announce/company_announce_sync_service.py +65 -0
- mns_scheduler/company_info/base/__init__.py +1 -1
- mns_scheduler/company_info/base/sync_company_base_info_api.py +171 -79
- mns_scheduler/company_info/base/sync_company_hold_info_api.py +3 -6
- mns_scheduler/company_info/base/sync_company_product_area_industry.py +161 -0
- mns_scheduler/company_info/clean/__init__.py +1 -1
- mns_scheduler/company_info/clean/company_info_clean_api.py +29 -9
- mns_scheduler/company_info/constant/__init__.py +1 -1
- mns_scheduler/company_info/constant/company_constant_data.py +285 -184
- mns_scheduler/company_info/de_list_stock/__init__.py +1 -1
- mns_scheduler/company_info/de_list_stock/de_list_stock_service.py +1 -1
- mns_scheduler/company_info/em_stock_info/__init__.py +7 -0
- mns_scheduler/company_info/em_stock_info/sync_em_stock_info_sync.py +80 -0
- mns_scheduler/company_info/remark/__init__.py +1 -1
- mns_scheduler/company_info/remark/company_remark_info_sync.py +3 -3
- mns_scheduler/concept/clean/kpl_concept_clean_api.py +1 -1
- mns_scheduler/concept/clean/ths_concept_clean_api.py +20 -4
- mns_scheduler/concept/ths/common/ths_concept_sync_common_api.py +21 -16
- mns_scheduler/concept/ths/common/ths_concept_update_common_api.py +4 -1
- mns_scheduler/concept/ths/detaill/ths_concept_detail_api.py +7 -7
- mns_scheduler/concept/ths/sync_new_index/sync_ths_concept_new_index_api.py +9 -4
- mns_scheduler/concept/ths/update_concept_info/sync_one_symbol_all_concepts_api.py +3 -3
- mns_scheduler/db/col_move_service.py +3 -3
- mns_scheduler/db/script/__init__.py +1 -1
- mns_scheduler/db/script/col_move_script.py +1 -1
- mns_scheduler/db/script/db_move/__init__.py +7 -0
- mns_scheduler/db/script/db_move/col_move_one_service.py +34 -0
- mns_scheduler/db/script/sync/__init__.py +1 -1
- mns_scheduler/db/script/sync/remote_data_sync_to_local.py +57 -4
- mns_scheduler/db/script/sync/sync_hui_ce_test_data.py +80 -0
- mns_scheduler/db/script/sync/sync_hui_ce_test_data_01.py +69 -0
- mns_scheduler/db/script/update/__init__.py +7 -0
- mns_scheduler/db/script/update/update_col_field.py +36 -0
- mns_scheduler/finance/__init__.py +1 -1
- mns_scheduler/finance/{em_financial_asset_liability_sync_service_api.py → em/em_financial_asset_liability_sync_service_api.py} +2 -2
- mns_scheduler/finance/{em_financial_profit_sync_service_api.py → em/em_financial_profit_sync_service_api.py} +27 -26
- mns_scheduler/finance/{finance_common_api.py → em/finance_common_api.py} +3 -3
- mns_scheduler/finance/{sync_financial_report_service_api.py → sync_financial_report_service_task.py} +80 -27
- mns_scheduler/finance/xue_qiu/down_load_xueqiu_report_api.py +77 -0
- mns_scheduler/finance/xue_qiu/sync_xue_qiu_fiance_data.py +161 -0
- mns_scheduler/hk/__init__.py +1 -1
- mns_scheduler/hk/hk_company_info_sync_service_api.py +4 -4
- mns_scheduler/hk/hk_industry_info_sync_service_api.py +3 -5
- mns_scheduler/industry/__init__.py +7 -0
- mns_scheduler/industry/ths/__init__.py +7 -0
- mns_scheduler/industry/ths/ths_industry_index_service.py +58 -0
- mns_scheduler/industry/ths/ths_industry_sync_service.py +68 -0
- mns_scheduler/irm/__init__.py +1 -1
- mns_scheduler/irm/api/__init__.py +1 -1
- mns_scheduler/irm/api/sh_stock_sns_sse_info_api.py +1 -1
- mns_scheduler/irm/api/sz_stock_sns_sse_info_api.py +1 -1
- mns_scheduler/irm/stock_irm_cninfo_service.py +12 -8
- mns_scheduler/k_line/clean/daily/__init__.py +1 -1
- mns_scheduler/k_line/clean/daily/daily_k_line_clean_common_service.py +52 -6
- mns_scheduler/k_line/clean/daily/daily_k_line_service.py +7 -2
- mns_scheduler/k_line/clean/k_line_info_clean_impl.py +3 -2
- mns_scheduler/k_line/clean/k_line_info_clean_task.py +42 -15
- mns_scheduler/k_line/clean/week_month/__init__.py +1 -1
- mns_scheduler/k_line/clean/week_month/normal_week_month_k_line_service.py +124 -26
- mns_scheduler/k_line/clean/week_month/sub_new_week_month_k_line_service.py +2 -2
- mns_scheduler/k_line/common/__init__.py +7 -0
- mns_scheduler/k_line/common/k_line_common_api.py +188 -0
- mns_scheduler/k_line/hot_stocks/__init__.py +1 -1
- mns_scheduler/k_line/hot_stocks/recent_hot_stocks_clean_service.py +1 -1
- mns_scheduler/k_line/{sync → month_week_daily}/bfq_k_line_sync.py +14 -29
- mns_scheduler/k_line/{sync → month_week_daily}/daily_week_month_line_sync.py +11 -12
- mns_scheduler/k_line/sync_status/__init__.py +7 -0
- mns_scheduler/k_line/sync_status/k_line_sync_status_check.py +54 -0
- mns_scheduler/k_line/test/__init__.py +1 -1
- mns_scheduler/k_line/test/k_line_info_clean_his_data.py +14 -3
- mns_scheduler/k_line/year_quarter/__init__.py +7 -0
- mns_scheduler/k_line/year_quarter/year_quarter_line_sync.py +76 -0
- mns_scheduler/kpl/selection/symbol/sync_best_choose_symbol.py +1 -2
- mns_scheduler/kpl/selection/symbol/sync_kpl_concept_symbol_choose_reason_api.py +108 -0
- mns_scheduler/kpl/selection/total/sync_kpl_best_total_sync_api.py +5 -0
- mns_scheduler/lhb/__init__.py +1 -1
- mns_scheduler/lhb/stock_lhb_sync_service.py +1 -1
- mns_scheduler/open/__init__.py +1 -1
- mns_scheduler/open/sync_one_day_open_data_to_db_service.py +2 -3
- mns_scheduler/risk/__init__.py +1 -1
- mns_scheduler/risk/compliance/undisclosed_annual_report_api.py +8 -2
- mns_scheduler/risk/financial/annual_report_audit_check_api.py +13 -3
- mns_scheduler/risk/financial/net_assets_check_api.py +21 -18
- mns_scheduler/risk/financial/profit_income_check_api.py +7 -2
- mns_scheduler/risk/financial_report_risk_check_api.py +1 -1
- mns_scheduler/risk/major_violations/register_and_investigate_stock_sync_api.py +1 -1
- mns_scheduler/risk/self/wei_pan_stock_api.py +1 -1
- mns_scheduler/risk/test/__init__.py +1 -1
- mns_scheduler/risk/test/fix_blask_list.py +4 -6
- mns_scheduler/risk/transactions/transactions_check_api.py +22 -4
- mns_scheduler/self_choose/__init__.py +1 -1
- mns_scheduler/self_choose/ths_self_choose_service.py +60 -32
- mns_scheduler/trade/auto_login/trader_auto_service.py +6 -4
- mns_scheduler/trade/auto_sell_service_api.py +4 -4
- mns_scheduler/trade/balance/__init__.py +7 -0
- mns_scheduler/trade/balance/ths_account_balance_service.py +7 -0
- mns_scheduler/trade/sync_position_api.py +39 -6
- mns_scheduler/trade/task/trader_task_service.py +26 -9
- mns_scheduler/trade/tfp/__init__.py +7 -0
- mns_scheduler/trade/tfp/stock_tfp_info_sync.py +56 -0
- mns_scheduler/zb/stock_zb_pool_sync.py +1 -16
- mns_scheduler/zt/high_chg/sync_high_chg_pool_service.py +2 -2
- mns_scheduler/zt/high_chg/sync_high_chg_real_time_quotes_service.py +1 -1
- mns_scheduler/zt/script/__init__.py +1 -1
- mns_scheduler/zt/script/fix_error_deal_day.py +41 -0
- mns_scheduler/zt/script/kcx_high_chg_open_his_data_handle.py +2 -2
- mns_scheduler/zt/script/sync_high_chg_pool_his_data.py +2 -2
- mns_scheduler/zt/script/sync_now_higt_chg_zt.py +8 -7
- mns_scheduler/zt/zt_pool/em_zt_pool_sync_api.py +256 -55
- mns_scheduler/zt/zt_pool/ths_zt_pool_sync_api.py +33 -90
- mns_scheduler/zt/zt_pool/update_null_zt_reason_api.py +24 -13
- mns_scheduler/zz_task/compensation/__init__.py +0 -0
- mns_scheduler/zz_task/compensation/compensate_task.py +161 -0
- mns_scheduler/zz_task/compensation/compensate_task_one_day.py +142 -0
- mns_scheduler/zz_task/data_sync_task.py +177 -91
- {mns_scheduler-1.1.8.4.dist-info → mns_scheduler-1.4.3.2.dist-info}/METADATA +1 -1
- mns_scheduler-1.4.3.2.dist-info/RECORD +169 -0
- {mns_scheduler-1.1.8.4.dist-info → mns_scheduler-1.4.3.2.dist-info}/WHEEL +1 -1
- mns_scheduler/2014-2015-test/2014_2015_chg_statistics.py +0 -87
- mns_scheduler/big_deal/ths_big_deal_sync.py +0 -98
- mns_scheduler/db/real_time_task_check.py +0 -84
- mns_scheduler/debt/kzz_bond_info_sync.py +0 -33
- mns_scheduler-1.1.8.4.dist-info/RECORD +0 -142
- /mns_scheduler/{big_deal → company_info/announce}/__init__.py +0 -0
- /mns_scheduler/{2014-2015-test → finance/em}/__init__.py +0 -0
- /mns_scheduler/{debt → finance/xue_qiu}/__init__.py +0 -0
- /mns_scheduler/k_line/{sync → month_week_daily}/__init__.py +0 -0
- {mns_scheduler-1.1.8.4.dist-info → mns_scheduler-1.4.3.2.dist-info}/top_level.txt +0 -0
|
@@ -0,0 +1,161 @@
|
|
|
1
|
+
import sys
|
|
2
|
+
import os
|
|
3
|
+
|
|
4
|
+
file_path = os.path.abspath(__file__)
|
|
5
|
+
end = file_path.index('mns') + 16
|
|
6
|
+
project_path = file_path[0:end]
|
|
7
|
+
sys.path.append(project_path)
|
|
8
|
+
|
|
9
|
+
from mns_common.db.MongodbUtil import MongodbUtil
|
|
10
|
+
import mns_common.constant.extra_income_db_name as extra_income_db_name
|
|
11
|
+
import mns_scheduler.finance.xue_qiu.down_load_xueqiu_report_api as down_load_xueqiu_report_api
|
|
12
|
+
import mns_common.component.common_service_fun_api as common_service_fun_api
|
|
13
|
+
import pandas as pd
|
|
14
|
+
from loguru import logger
|
|
15
|
+
import time
|
|
16
|
+
import mns_common.utils.data_frame_util as data_frame_util
|
|
17
|
+
import mns_common.component.cookie.cookie_info_service as cookie_info_service
|
|
18
|
+
from datetime import datetime
|
|
19
|
+
import mns_common.component.em.em_stock_info_api as em_stock_info_api
|
|
20
|
+
|
|
21
|
+
mongodb_util_27017 = MongodbUtil('27017')
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
# report_type income 利润表
|
|
25
|
+
# cash_flow 现金流量
|
|
26
|
+
# balance 资产负债
|
|
27
|
+
# 同步所有股票 报表
|
|
28
|
+
def sync_all_stocks_report():
|
|
29
|
+
em_a_stock_info_df = em_stock_info_api.get_a_stock_info()
|
|
30
|
+
em_a_stock_info_df = common_service_fun_api.add_pre_prefix(em_a_stock_info_df)
|
|
31
|
+
# 或等效写法 df['A'].str[0:6]
|
|
32
|
+
|
|
33
|
+
fail_list = []
|
|
34
|
+
|
|
35
|
+
xue_qiu_cookie = cookie_info_service.get_xue_qiu_cookie()
|
|
36
|
+
report_type_list = ['income', 'balance', 'cash_flow']
|
|
37
|
+
for stock_one in em_a_stock_info_df.itertuples():
|
|
38
|
+
fail_list = save_one_symbol_data(stock_one, report_type_list, xue_qiu_cookie, True, fail_list, '', False)
|
|
39
|
+
|
|
40
|
+
handle_number = 0
|
|
41
|
+
# 处理失败的
|
|
42
|
+
while len(fail_list) > 0:
|
|
43
|
+
fail_df = em_a_stock_info_df.loc[em_a_stock_info_df['symbol'].isin(fail_list)]
|
|
44
|
+
for fail_one in fail_df.itertuples():
|
|
45
|
+
fail_list = save_one_symbol_data(fail_one, report_type_list, xue_qiu_cookie, True, fail_list, '', False)
|
|
46
|
+
handle_number = handle_number + 1
|
|
47
|
+
if handle_number > 10:
|
|
48
|
+
break
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def save_one_symbol_data(stock_one, report_type_list, xue_qiu_cookie, save_tag, fail_list, report_name, check_exist):
|
|
52
|
+
try:
|
|
53
|
+
symbol_prefix = stock_one.symbol_prefix
|
|
54
|
+
name = stock_one.name
|
|
55
|
+
symbol = stock_one.symbol
|
|
56
|
+
for report_type in report_type_list:
|
|
57
|
+
if report_type == 'income':
|
|
58
|
+
col_name = extra_income_db_name.XUE_QIU_LRB_INCOME
|
|
59
|
+
elif report_type == 'balance':
|
|
60
|
+
col_name = extra_income_db_name.XUE_QIU_ASSET_DEBT
|
|
61
|
+
elif report_type == 'cash_flow':
|
|
62
|
+
col_name = extra_income_db_name.XUE_QIU_CASH_FLOW
|
|
63
|
+
if check_exist:
|
|
64
|
+
query_exist = {'symbol': symbol, 'report_name': report_name}
|
|
65
|
+
# 存在数据 不在同步
|
|
66
|
+
if mongodb_util_27017.exist_data_query(col_name, query_exist):
|
|
67
|
+
continue
|
|
68
|
+
|
|
69
|
+
index_create = [('symbol', 1), ('report_date', 1)]
|
|
70
|
+
mongodb_util_27017.create_index(col_name, index_create)
|
|
71
|
+
|
|
72
|
+
index_create_01 = [('symbol', 1), ('sync_time', 1)]
|
|
73
|
+
mongodb_util_27017.create_index(col_name, index_create_01)
|
|
74
|
+
|
|
75
|
+
if check_exist:
|
|
76
|
+
# 季度同步只同步一条数据
|
|
77
|
+
result_df = down_load_xueqiu_report_api.get_xue_qiu_report(symbol_prefix, report_type, xue_qiu_cookie,
|
|
78
|
+
1,
|
|
79
|
+
'all')
|
|
80
|
+
else:
|
|
81
|
+
result_df = down_load_xueqiu_report_api.get_xue_qiu_report(symbol_prefix, report_type, xue_qiu_cookie,
|
|
82
|
+
200,
|
|
83
|
+
'all')
|
|
84
|
+
|
|
85
|
+
now_date = datetime.now()
|
|
86
|
+
sync_time = now_date.strftime('%Y-%m-%d %H:%M:%S')
|
|
87
|
+
|
|
88
|
+
if data_frame_util.is_empty(result_df):
|
|
89
|
+
logger.error("财务信息为空,代码:{}:{}", symbol, name)
|
|
90
|
+
continue
|
|
91
|
+
else:
|
|
92
|
+
# 季度同步check
|
|
93
|
+
if check_exist:
|
|
94
|
+
result_df = result_df.loc[result_df['report_name'] == report_name]
|
|
95
|
+
if data_frame_util.is_empty(result_df):
|
|
96
|
+
continue
|
|
97
|
+
|
|
98
|
+
result_df['sync_time'] = sync_time
|
|
99
|
+
time.sleep(0.5)
|
|
100
|
+
# 1. 将毫秒时间戳转为 datetime
|
|
101
|
+
result_df['report_date'] = pd.to_datetime(result_df['report_date'], unit='ms')
|
|
102
|
+
|
|
103
|
+
# 2. 格式化为 '%Y-%m-%d' 字符串
|
|
104
|
+
result_df['report_date'] = result_df['report_date'].dt.strftime('%Y-%m-%d')
|
|
105
|
+
|
|
106
|
+
result_df['_id'] = symbol + '_' + result_df['report_date']
|
|
107
|
+
result_df['symbol'] = symbol
|
|
108
|
+
|
|
109
|
+
# 1. 将毫秒时间戳转为 datetime
|
|
110
|
+
result_df['ctime'] = pd.to_datetime(result_df['ctime'], unit='ms')
|
|
111
|
+
|
|
112
|
+
# 2. 格式化为 '%Y-%m-%d' 字符串
|
|
113
|
+
result_df['ctime'] = result_df['ctime'].dt.strftime('%Y-%m-%d')
|
|
114
|
+
result_df.loc[result_df['report_name'].str.contains('年报'), 'period'] = 4
|
|
115
|
+
result_df.loc[result_df['report_name'].str.contains('一季报'), 'period'] = 1
|
|
116
|
+
result_df.loc[result_df['report_name'].str.contains('中报'), 'period'] = 2
|
|
117
|
+
result_df.loc[result_df['report_name'].str.contains('三季报'), 'period'] = 3
|
|
118
|
+
result_df['year'] = result_df['report_name'].str[:4]
|
|
119
|
+
if save_tag:
|
|
120
|
+
mongodb_util_27017.save_mongo(result_df, col_name)
|
|
121
|
+
else:
|
|
122
|
+
mongodb_util_27017.insert_mongo(result_df, col_name)
|
|
123
|
+
|
|
124
|
+
if symbol in fail_list:
|
|
125
|
+
fail_list.remove(symbol)
|
|
126
|
+
logger.info("同步财务数据完成:{}:{}", symbol, name, report_name)
|
|
127
|
+
except BaseException as e:
|
|
128
|
+
logger.error("同步错误:{},异常信息:{}", symbol, e)
|
|
129
|
+
fail_list.append(symbol)
|
|
130
|
+
return fail_list
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
def sync_xue_qiu_very_period_report(report_name, symbol):
|
|
134
|
+
em_a_stock_info_df = em_stock_info_api.get_a_stock_info()
|
|
135
|
+
if symbol is not None:
|
|
136
|
+
em_a_stock_info_df = em_a_stock_info_df.loc[em_a_stock_info_df['symbol'] == symbol]
|
|
137
|
+
em_a_stock_info_df = common_service_fun_api.add_pre_prefix(em_a_stock_info_df)
|
|
138
|
+
# 或等效写法 df['A'].str[0:6]
|
|
139
|
+
|
|
140
|
+
fail_list = []
|
|
141
|
+
|
|
142
|
+
xue_qiu_cookie = cookie_info_service.get_xue_qiu_cookie()
|
|
143
|
+
report_type_list = ['income', 'balance', 'cash_flow']
|
|
144
|
+
for stock_one in em_a_stock_info_df.itertuples():
|
|
145
|
+
fail_list = save_one_symbol_data(stock_one, report_type_list, xue_qiu_cookie, False, fail_list, report_name,
|
|
146
|
+
True)
|
|
147
|
+
|
|
148
|
+
handle_number = 0
|
|
149
|
+
# 处理失败的
|
|
150
|
+
while len(fail_list) > 0:
|
|
151
|
+
fail_df = em_a_stock_info_df.loc[em_a_stock_info_df['symbol'].isin(fail_list)]
|
|
152
|
+
for fail_one in fail_df.itertuples():
|
|
153
|
+
fail_list = save_one_symbol_data(fail_one, report_type_list, xue_qiu_cookie, False, fail_list, report_name,
|
|
154
|
+
True)
|
|
155
|
+
handle_number = handle_number + 1
|
|
156
|
+
if handle_number > 10:
|
|
157
|
+
break
|
|
158
|
+
|
|
159
|
+
|
|
160
|
+
if __name__ == '__main__':
|
|
161
|
+
sync_all_stocks_report()
|
mns_scheduler/hk/__init__.py
CHANGED
|
@@ -2,10 +2,10 @@ import sys
|
|
|
2
2
|
import os
|
|
3
3
|
|
|
4
4
|
file_path = os.path.abspath(__file__)
|
|
5
|
-
end = file_path.index('mns') +
|
|
5
|
+
end = file_path.index('mns') + 17
|
|
6
6
|
project_path = file_path[0:end]
|
|
7
7
|
sys.path.append(project_path)
|
|
8
|
-
import mns_common.
|
|
8
|
+
import mns_common.component.em.em_stock_info_api as em_stock_info_api
|
|
9
9
|
import akshare as ak
|
|
10
10
|
import mns_common.constant.db_name_constant as db_name_constant
|
|
11
11
|
from mns_common.db.MongodbUtil import MongodbUtil
|
|
@@ -48,8 +48,7 @@ def get_ths_cookie():
|
|
|
48
48
|
|
|
49
49
|
# https://quote.eastmoney.com/center/gridlist.html#hk_stocks
|
|
50
50
|
def sync_hk_company_info():
|
|
51
|
-
|
|
52
|
-
hk_real_time_df = east_money_stock_hk_api.hk_real_time_quotes(cookie)
|
|
51
|
+
hk_real_time_df = em_stock_info_api.get_hk_stock_info()
|
|
53
52
|
|
|
54
53
|
hk_real_time_df = hk_real_time_df[[
|
|
55
54
|
"symbol",
|
|
@@ -100,4 +99,5 @@ def sync_hk_company_info():
|
|
|
100
99
|
|
|
101
100
|
|
|
102
101
|
if __name__ == '__main__':
|
|
102
|
+
get_hk_ggt_component()
|
|
103
103
|
sync_hk_company_info()
|
|
@@ -2,16 +2,15 @@ import sys
|
|
|
2
2
|
import os
|
|
3
3
|
|
|
4
4
|
file_path = os.path.abspath(__file__)
|
|
5
|
-
end = file_path.index('mns') +
|
|
5
|
+
end = file_path.index('mns') + 17
|
|
6
6
|
project_path = file_path[0:end]
|
|
7
7
|
sys.path.append(project_path)
|
|
8
8
|
|
|
9
|
-
import mns_common.
|
|
9
|
+
import mns_common.component.em.em_stock_info_api as em_stock_info_api
|
|
10
10
|
import akshare as ak
|
|
11
11
|
import mns_common.constant.db_name_constant as db_name_constant
|
|
12
12
|
from mns_common.db.MongodbUtil import MongodbUtil
|
|
13
13
|
from functools import lru_cache
|
|
14
|
-
import mns_common.component.common_service_fun_api as common_service_fun_api
|
|
15
14
|
import mns_common.component.zt.zt_common_service_api as zt_common_service_api
|
|
16
15
|
|
|
17
16
|
mongodb_util = MongodbUtil('27017')
|
|
@@ -39,8 +38,7 @@ def get_em_cookie():
|
|
|
39
38
|
|
|
40
39
|
|
|
41
40
|
def sync_hk_company_industry():
|
|
42
|
-
|
|
43
|
-
hk_real_time_df = east_money_stock_hk_api.hk_real_time_quotes(cookie)
|
|
41
|
+
hk_real_time_df = em_stock_info_api.get_hk_stock_info()
|
|
44
42
|
|
|
45
43
|
hk_real_time_df = hk_real_time_df[[
|
|
46
44
|
"symbol",
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
import sys
|
|
2
|
+
import os
|
|
3
|
+
|
|
4
|
+
file_path = os.path.abspath(__file__)
|
|
5
|
+
end = file_path.index('mns') + 17
|
|
6
|
+
project_path = file_path[0:end]
|
|
7
|
+
sys.path.append(project_path)
|
|
8
|
+
import mns_common.api.ths.concept.app.ths_concept_index_app as ths_concept_index_app
|
|
9
|
+
from datetime import datetime
|
|
10
|
+
import mns_common.component.trade_date.trade_date_common_service_api as trade_date_common_service_api
|
|
11
|
+
import mns_common.utils.date_handle_util as date_handle_util
|
|
12
|
+
import mns_common.utils.data_frame_util as data_frame_util
|
|
13
|
+
import pandas as pd
|
|
14
|
+
import mns_common.component.common_service_fun_api as common_service_fun_api
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
# 通过api 获取ths行业和指数
|
|
18
|
+
def get_ths_index_by_api(query_type):
|
|
19
|
+
now_date = datetime.now()
|
|
20
|
+
hour = now_date.hour
|
|
21
|
+
minute = now_date.minute
|
|
22
|
+
now_str_day = now_date.strftime('%Y-%m-%d')
|
|
23
|
+
|
|
24
|
+
is_trade_day = trade_date_common_service_api.is_trade_day(now_str_day)
|
|
25
|
+
|
|
26
|
+
if bool(1 - is_trade_day):
|
|
27
|
+
last_trade_day = trade_date_common_service_api.get_before_trade_date(now_str_day, 1)
|
|
28
|
+
begin_time = date_handle_util.no_slash_date(last_trade_day) + '093000'
|
|
29
|
+
end_time = date_handle_util.no_slash_date(last_trade_day) + '150000'
|
|
30
|
+
else:
|
|
31
|
+
if hour < 9 or (hour == 9 and minute <= 25):
|
|
32
|
+
last_trade_day = trade_date_common_service_api.get_before_trade_date(now_str_day, 2)
|
|
33
|
+
begin_time = date_handle_util.no_slash_date(last_trade_day) + '093000'
|
|
34
|
+
end_time = date_handle_util.no_slash_date(last_trade_day) + '150000'
|
|
35
|
+
else:
|
|
36
|
+
begin_time = date_handle_util.no_slash_date(now_str_day) + '093000'
|
|
37
|
+
if hour == 9:
|
|
38
|
+
hour = '0' + str(hour)
|
|
39
|
+
end_time = date_handle_util.no_slash_date(now_str_day) + str(hour) + str(minute) + '00'
|
|
40
|
+
elif (hour == 11 and minute >= 30) or (hour == 12):
|
|
41
|
+
end_time = date_handle_util.no_slash_date(now_str_day) + '113000'
|
|
42
|
+
elif hour >= 15:
|
|
43
|
+
end_time = date_handle_util.no_slash_date(now_str_day) + '150000'
|
|
44
|
+
else:
|
|
45
|
+
end_time = date_handle_util.no_slash_date(now_str_day) + str(hour) + str(minute) + '00'
|
|
46
|
+
|
|
47
|
+
df = ths_concept_index_app.get_ths_concept_his_info(begin_time, end_time, 500, query_type)
|
|
48
|
+
if data_frame_util.is_empty(df):
|
|
49
|
+
return pd.DataFrame()
|
|
50
|
+
df['turnover'] = round(df['turnover'] / common_service_fun_api.HUNDRED_MILLION, 1)
|
|
51
|
+
df['net_inflow_of_main_force'] = round(df['net_inflow_of_main_force'] / common_service_fun_api.TEN_THOUSAND, 1)
|
|
52
|
+
df.fillna('', inplace=True)
|
|
53
|
+
return df
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
if __name__ == '__main__':
|
|
57
|
+
df_industry = get_ths_index_by_api(1)
|
|
58
|
+
print(df_industry)
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
import sys
|
|
2
|
+
import os
|
|
3
|
+
|
|
4
|
+
file_path = os.path.abspath(__file__)
|
|
5
|
+
end = file_path.index('mns') + 17
|
|
6
|
+
project_path = file_path[0:end]
|
|
7
|
+
sys.path.append(project_path)
|
|
8
|
+
import mns_scheduler.industry.ths.ths_industry_index_service as ths_industry_index_service
|
|
9
|
+
from mns_common.db.MongodbUtil import MongodbUtil
|
|
10
|
+
import time
|
|
11
|
+
from datetime import datetime
|
|
12
|
+
from loguru import logger
|
|
13
|
+
import mns_common.utils.data_frame_util as data_frame_util
|
|
14
|
+
import mns_common.api.ths.concept.app.ths_concept_detail_app as ths_concept_detail_app
|
|
15
|
+
|
|
16
|
+
mongodb_util = MongodbUtil('27017')
|
|
17
|
+
import mns_common.constant.db_name_constant as db_name_constant
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
# 同步同花顺行业指数
|
|
21
|
+
def sync_ths_industry_index():
|
|
22
|
+
ths_industry_index_df = ths_industry_index_service.get_ths_index_by_api(1)
|
|
23
|
+
if data_frame_util.is_empty(ths_industry_index_df):
|
|
24
|
+
return None
|
|
25
|
+
ths_industry_index_df['_id'] = ths_industry_index_df['block_code']
|
|
26
|
+
ths_industry_index_df = ths_industry_index_df[[
|
|
27
|
+
'_id',
|
|
28
|
+
'turnover',
|
|
29
|
+
'block_market',
|
|
30
|
+
'block_code',
|
|
31
|
+
'block_name',
|
|
32
|
+
'net_inflow_of_main_force',
|
|
33
|
+
'chg'
|
|
34
|
+
]]
|
|
35
|
+
now_date = datetime.now()
|
|
36
|
+
str_now_date = now_date.strftime('%Y-%m-%d %H:%M:%S')
|
|
37
|
+
ths_industry_index_df['str_now_date'] = str_now_date
|
|
38
|
+
mongodb_util.save_mongo(ths_industry_index_df, db_name_constant.THS_INDUSTRY_LIST)
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def sync_ths_industry_detail():
|
|
42
|
+
ths_industry_list_df = mongodb_util.find_all_data(db_name_constant.THS_INDUSTRY_LIST)
|
|
43
|
+
for industry_one in ths_industry_list_df.itertuples():
|
|
44
|
+
try:
|
|
45
|
+
time.sleep(1)
|
|
46
|
+
now_date = datetime.now()
|
|
47
|
+
str_now_date = now_date.strftime('%Y-%m-%d %H:%M:%S')
|
|
48
|
+
|
|
49
|
+
ths_industry_symbol_detail_df = ths_concept_detail_app.get_ths_concept_detail_by_app(
|
|
50
|
+
industry_one.block_code)
|
|
51
|
+
ths_industry_symbol_detail_df = ths_industry_symbol_detail_df.rename(
|
|
52
|
+
columns={"concept_code": 'ths_industry_code',
|
|
53
|
+
"concept_name": 'ths_industry_name',
|
|
54
|
+
})
|
|
55
|
+
if data_frame_util.is_empty(ths_industry_symbol_detail_df):
|
|
56
|
+
continue
|
|
57
|
+
ths_industry_symbol_detail_df['str_now_date'] = str_now_date
|
|
58
|
+
ths_industry_symbol_detail_df['_id'] = ths_industry_symbol_detail_df['symbol']
|
|
59
|
+
mongodb_util.save_mongo(ths_industry_symbol_detail_df, db_name_constant.THS_STOCK_INDUSTRY_DETAIL)
|
|
60
|
+
logger.info("同步ths行业股票详情:{}", industry_one.block_name)
|
|
61
|
+
|
|
62
|
+
except BaseException as e:
|
|
63
|
+
logger.error("同步ths行业股票详情异常:{}", e)
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
if __name__ == '__main__':
|
|
67
|
+
sync_ths_industry_index()
|
|
68
|
+
sync_ths_industry_detail()
|
mns_scheduler/irm/__init__.py
CHANGED
|
@@ -2,13 +2,13 @@ import sys
|
|
|
2
2
|
import os
|
|
3
3
|
|
|
4
4
|
file_path = os.path.abspath(__file__)
|
|
5
|
-
end = file_path.index('mns') +
|
|
5
|
+
end = file_path.index('mns') + 17
|
|
6
6
|
project_path = file_path[0:end]
|
|
7
7
|
sys.path.append(project_path)
|
|
8
8
|
import akshare as ak
|
|
9
9
|
import mns_common.utils.data_frame_util as data_frame_util
|
|
10
10
|
import pandas as pd
|
|
11
|
-
import mns_common.
|
|
11
|
+
import mns_common.component.em.em_stock_info_api as em_stock_info_api
|
|
12
12
|
import mns_common.component.company.company_common_service_api as company_common_service_api
|
|
13
13
|
from loguru import logger
|
|
14
14
|
from datetime import datetime
|
|
@@ -119,18 +119,18 @@ def save_sh_stock_uid():
|
|
|
119
119
|
|
|
120
120
|
|
|
121
121
|
# 同步所有互动问题
|
|
122
|
-
def
|
|
122
|
+
def sync_symbols_interactive_questions(symbol_list):
|
|
123
123
|
# 同步互动易映射
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
real_time_quotes_all_stocks =
|
|
124
|
+
if len(symbol_list) == 0:
|
|
125
|
+
save_sh_stock_uid()
|
|
126
|
+
real_time_quotes_all_stocks = em_stock_info_api.get_a_stock_info()
|
|
127
127
|
de_list_company_symbols = company_common_service_api.get_de_list_company()
|
|
128
128
|
real_time_quotes_all_stocks = real_time_quotes_all_stocks.loc[
|
|
129
129
|
~(real_time_quotes_all_stocks['symbol'].isin(de_list_company_symbols))]
|
|
130
130
|
real_time_quotes_all_stocks = common_service_fun_api.classify_symbol(real_time_quotes_all_stocks)
|
|
131
131
|
real_time_quotes_all_stocks = real_time_quotes_all_stocks.sort_values(by=['chg'], ascending=False)
|
|
132
132
|
fail_symbol_list = []
|
|
133
|
-
if symbol_list
|
|
133
|
+
if len(symbol_list) != 0:
|
|
134
134
|
real_time_quotes_all_stocks = real_time_quotes_all_stocks.loc[
|
|
135
135
|
real_time_quotes_all_stocks['symbol'].isin(symbol_list)]
|
|
136
136
|
for stock_one in real_time_quotes_all_stocks.itertuples():
|
|
@@ -149,6 +149,8 @@ def sync_all_interactive_questions(symbol_list):
|
|
|
149
149
|
|
|
150
150
|
else:
|
|
151
151
|
continue
|
|
152
|
+
if data_frame_util.is_empty(stock_irm_cninfo_df):
|
|
153
|
+
continue
|
|
152
154
|
stock_irm_cninfo_df['sync_time'] = str_now_date
|
|
153
155
|
stock_irm_cninfo_df['str_day'] = str_day
|
|
154
156
|
stock_irm_cninfo_df.drop_duplicates('_id', keep='last', inplace=True)
|
|
@@ -191,6 +193,8 @@ def save_new_data(stock_irm_cninfo_df):
|
|
|
191
193
|
|
|
192
194
|
|
|
193
195
|
if __name__ == '__main__':
|
|
196
|
+
sync_symbols_interactive_questions([])
|
|
197
|
+
get_stock_irm_cninfo_sh_api('688778')
|
|
194
198
|
fail_symbol_list_01 = ['000638', '002886', '688778', '688766', '688733', '688778', '688793', '688787']
|
|
195
199
|
# get_stock_irm_cninfo_sh_api('603633')
|
|
196
|
-
|
|
200
|
+
# sync_symbols_interactive_questions(None)
|
|
@@ -2,7 +2,7 @@ import sys
|
|
|
2
2
|
import os
|
|
3
3
|
|
|
4
4
|
file_path = os.path.abspath(__file__)
|
|
5
|
-
end = file_path.index('mns') +
|
|
5
|
+
end = file_path.index('mns') + 17
|
|
6
6
|
project_path = file_path[0:end]
|
|
7
7
|
sys.path.append(project_path)
|
|
8
8
|
import pandas as pd
|
|
@@ -85,6 +85,12 @@ def init_day_line_data(k_line_info, stock_qfq_daily):
|
|
|
85
85
|
k_line_info.loc[:, 'avg_thirty_last'] = 0
|
|
86
86
|
k_line_info.loc[:, 'avg_sixty_last'] = 0
|
|
87
87
|
|
|
88
|
+
k_line_info.loc[:, 'slope_five_last'] = 0
|
|
89
|
+
k_line_info.loc[:, 'slope_ten_last'] = 0
|
|
90
|
+
k_line_info.loc[:, 'slope_twenty_last'] = 0
|
|
91
|
+
k_line_info.loc[:, 'slope_thirty_last'] = 0
|
|
92
|
+
k_line_info.loc[:, 'slope_sixty_last'] = 0
|
|
93
|
+
|
|
88
94
|
k_line_info.loc[:, 'std_amount_ten'] = 0
|
|
89
95
|
k_line_info.loc[:, 'mean_amount_ten'] = 0
|
|
90
96
|
k_line_info.loc[:, 'std_amount_thirty'] = 0
|
|
@@ -200,24 +206,31 @@ def calculate_exchange_and_k_line_avg_param(stock_qfq_daily):
|
|
|
200
206
|
|
|
201
207
|
# 收盘价格与均线差值
|
|
202
208
|
stock_qfq_daily['close_difference_five'] = round(
|
|
203
|
-
100 * (stock_qfq_daily['close'] - stock_qfq_daily['avg_five']) / stock_qfq_daily['
|
|
209
|
+
100 * (stock_qfq_daily['close'] - stock_qfq_daily['avg_five']) / stock_qfq_daily['avg_five'],
|
|
204
210
|
2)
|
|
205
211
|
|
|
206
212
|
stock_qfq_daily['close_difference_ten'] = round(
|
|
207
|
-
100 * (stock_qfq_daily['close'] - stock_qfq_daily['avg_ten']) / stock_qfq_daily['
|
|
213
|
+
100 * (stock_qfq_daily['close'] - stock_qfq_daily['avg_ten']) / stock_qfq_daily['avg_ten'],
|
|
208
214
|
2)
|
|
209
215
|
|
|
210
216
|
stock_qfq_daily['close_difference_twenty'] = round(
|
|
211
|
-
100 * (stock_qfq_daily['close'] - stock_qfq_daily['avg_twenty']) / stock_qfq_daily['
|
|
217
|
+
100 * (stock_qfq_daily['close'] - stock_qfq_daily['avg_twenty']) / stock_qfq_daily['avg_twenty'],
|
|
212
218
|
2)
|
|
213
219
|
|
|
214
220
|
stock_qfq_daily['close_difference_thirty'] = round(
|
|
215
|
-
100 * (stock_qfq_daily['close'] - stock_qfq_daily['avg_thirty']) / stock_qfq_daily['
|
|
221
|
+
100 * (stock_qfq_daily['close'] - stock_qfq_daily['avg_thirty']) / stock_qfq_daily['avg_thirty'],
|
|
216
222
|
2)
|
|
217
223
|
|
|
218
224
|
stock_qfq_daily['close_difference_sixty'] = round(
|
|
219
|
-
100 * (stock_qfq_daily['close'] - stock_qfq_daily['avg_sixty']) / stock_qfq_daily['
|
|
225
|
+
100 * (stock_qfq_daily['close'] - stock_qfq_daily['avg_sixty']) / stock_qfq_daily['avg_sixty'],
|
|
220
226
|
2)
|
|
227
|
+
if (('slope_' + 'five' not in stock_qfq_daily.columns)
|
|
228
|
+
or ('slope_' + 'ten' not in stock_qfq_daily.columns) \
|
|
229
|
+
or ('slope_' + 'twenty' not in stock_qfq_daily.columns) \
|
|
230
|
+
or ('slope_' + 'thirty' not in stock_qfq_daily.columns) \
|
|
231
|
+
or ('slope_' + 'sixty' not in stock_qfq_daily.columns)):
|
|
232
|
+
# 计算均线斜率
|
|
233
|
+
stock_qfq_daily = calculate_slope(stock_qfq_daily, windows=['five', 'ten', 'twenty', 'thirty', 'sixty'])
|
|
221
234
|
|
|
222
235
|
stock_qfq_daily = stock_qfq_daily[[
|
|
223
236
|
"symbol",
|
|
@@ -255,6 +268,11 @@ def calculate_exchange_and_k_line_avg_param(stock_qfq_daily):
|
|
|
255
268
|
"avg_twenty",
|
|
256
269
|
'avg_thirty',
|
|
257
270
|
'avg_sixty',
|
|
271
|
+
"slope_five",
|
|
272
|
+
"slope_ten",
|
|
273
|
+
"slope_twenty",
|
|
274
|
+
'slope_thirty',
|
|
275
|
+
'slope_sixty',
|
|
258
276
|
"classification",
|
|
259
277
|
"_id",
|
|
260
278
|
"date"
|
|
@@ -265,6 +283,34 @@ def calculate_exchange_and_k_line_avg_param(stock_qfq_daily):
|
|
|
265
283
|
return stock_qfq_daily
|
|
266
284
|
|
|
267
285
|
|
|
286
|
+
def fix_avg_slope_name(k_line_info, stock_qfq_daily):
|
|
287
|
+
stock_qfq_daily_one = stock_qfq_daily.iloc[0:1]
|
|
288
|
+
k_line_info['slope_five_last'] = stock_qfq_daily_one['slope_five']
|
|
289
|
+
k_line_info['slope_ten_last'] = stock_qfq_daily_one['slope_ten']
|
|
290
|
+
k_line_info['slope_twenty_last'] = stock_qfq_daily_one['slope_twenty']
|
|
291
|
+
k_line_info['slope_thirty_last'] = stock_qfq_daily_one['slope_thirty']
|
|
292
|
+
k_line_info['slope_sixty_last'] = stock_qfq_daily_one['slope_sixty']
|
|
293
|
+
return k_line_info
|
|
294
|
+
|
|
295
|
+
|
|
296
|
+
# 计算均线
|
|
297
|
+
def calculate_moving_averages(data, windows=['five', 'ten', 'twenty', 'thirty', 'sixty']):
|
|
298
|
+
for window in windows:
|
|
299
|
+
data[f'avg_{window}', window] = data['close'].rolling(window=window).mean()
|
|
300
|
+
return data
|
|
301
|
+
|
|
302
|
+
|
|
303
|
+
# 计算均线斜率
|
|
304
|
+
def calculate_slope(data, windows=['five', 'ten', 'twenty', 'thirty', 'sixty']):
|
|
305
|
+
for window in windows:
|
|
306
|
+
if 'slope_' + window in data.columns:
|
|
307
|
+
continue
|
|
308
|
+
data[f'slope_{window}'] = data[f'avg_{window}'].diff() / data[f'avg_{window}'].shift(1)
|
|
309
|
+
data[f'slope_{window}'] = data[f'slope_{window}'] * 100
|
|
310
|
+
data[f'slope_{window}'] = round(data[f'slope_{window}'], 2)
|
|
311
|
+
return data
|
|
312
|
+
|
|
313
|
+
|
|
268
314
|
# 设置当天k线形态 下一个交易日判断当前交易日k线形态
|
|
269
315
|
def set_k_line_patterns(stock_qfq_daily_one):
|
|
270
316
|
open = list(stock_qfq_daily_one['open'])[0]
|
|
@@ -2,7 +2,7 @@ import sys
|
|
|
2
2
|
import os
|
|
3
3
|
|
|
4
4
|
file_path = os.path.abspath(__file__)
|
|
5
|
-
end = file_path.index('mns') +
|
|
5
|
+
end = file_path.index('mns') + 17
|
|
6
6
|
project_path = file_path[0:end]
|
|
7
7
|
sys.path.append(project_path)
|
|
8
8
|
from mns_common.db.MongodbUtil import MongodbUtil
|
|
@@ -71,6 +71,9 @@ def handle_day_line_sub_new(k_line_info, str_day, symbol, deal_days):
|
|
|
71
71
|
# 排除最近有三板以上的股票 todo
|
|
72
72
|
# 计算最近热门大涨的股票
|
|
73
73
|
recent_hot_stocks_clean_service.calculate_recent_hot_stocks(stock_qfq_daily, symbol, str_day)
|
|
74
|
+
# 修改 avg name
|
|
75
|
+
k_line_info = daily_k_line_clean_common_service.fix_avg_slope_name(k_line_info, stock_qfq_daily)
|
|
76
|
+
|
|
74
77
|
return k_line_info
|
|
75
78
|
|
|
76
79
|
|
|
@@ -107,9 +110,11 @@ def handle_day_line_normal(k_line_info, str_day, symbol, deal_days):
|
|
|
107
110
|
# 排除最近有三板以上的股票 todo
|
|
108
111
|
# 计算最近热门大涨的股票
|
|
109
112
|
recent_hot_stocks_clean_service.calculate_recent_hot_stocks(stock_qfq_daily, symbol, str_day)
|
|
110
|
-
|
|
113
|
+
# 修改 avg name
|
|
114
|
+
k_line_info = daily_k_line_clean_common_service.fix_avg_slope_name(k_line_info, stock_qfq_daily)
|
|
111
115
|
return k_line_info
|
|
112
116
|
|
|
117
|
+
|
|
113
118
|
# if __name__ == '__main__':
|
|
114
119
|
# query1 = {"symbol": '301596', 'date': {"$lte": date_handle_util.no_slash_date('2024-05-31')}}
|
|
115
120
|
# stock_qfq_daily_301596 = mongodb_util.descend_query(query1, 'stock_qfq_daily', 'date', 15)
|
|
@@ -16,7 +16,7 @@ mongodb_util = MongodbUtil('27017')
|
|
|
16
16
|
|
|
17
17
|
|
|
18
18
|
# 日线 周线 月线 成交量 筹码信息
|
|
19
|
-
def calculate_k_line_info(str_day, symbol, diff_days):
|
|
19
|
+
def calculate_k_line_info(str_day, symbol, diff_days, stock_qfq_year_df):
|
|
20
20
|
k_line_info = pd.DataFrame([[
|
|
21
21
|
str_day,
|
|
22
22
|
symbol, diff_days]],
|
|
@@ -27,7 +27,8 @@ def calculate_k_line_info(str_day, symbol, diff_days):
|
|
|
27
27
|
# 交易天数
|
|
28
28
|
deal_days = k_line_common_service_api.get_deal_days(str_day, symbol)
|
|
29
29
|
# 处理周线 月线
|
|
30
|
-
k_line_info = week_month_k_line_service.handle_month_week_line(k_line_info, str_day, symbol,
|
|
30
|
+
k_line_info = week_month_k_line_service.handle_month_week_line(k_line_info, str_day, symbol,
|
|
31
|
+
deal_days, stock_qfq_year_df)
|
|
31
32
|
# 处理日线
|
|
32
33
|
k_line_info = daily_k_line_service.handle_day_line(k_line_info, str_day, symbol, deal_days)
|
|
33
34
|
return k_line_info
|