mns-scheduler 1.4.4.5__py3-none-any.whl → 1.4.6.4__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of mns-scheduler might be problematic. Click here for more details.
- mns_scheduler/auto_da_ban/auto_da_ban_service.py +19 -17
- mns_scheduler/company_info/constant/company_constant_data.py +1 -2
- mns_scheduler/company_info/em_stock_info/clean_em_us_hk_stock_info.py +30 -0
- mns_scheduler/company_info/em_stock_info/sync_em_stock_info_sync.py +99 -39
- mns_scheduler/db/script/sync/remote_data_sync_to_local.py +5 -2
- mns_scheduler/irm/stock_irm_cninfo_service.py +2 -0
- mns_scheduler/irm/stock_question_id_service.py +11 -1
- mns_scheduler/k_line/clean/k_line_info_clean_task.py +1 -0
- mns_scheduler/kpl/theme/__init__.py +7 -0
- mns_scheduler/kpl/theme/kpl_theme_sync_service.py +231 -0
- mns_scheduler/self_choose/ths_self_choose_service.py +53 -25
- mns_scheduler/trade/tfp/stock_tfp_info_sync.py +1 -1
- mns_scheduler/zt/zt_pool/em_zt_pool_sync_api.py +26 -199
- mns_scheduler/zt/zt_pool/update_null_zt_reason_api.py +49 -31
- mns_scheduler/zz_task/compensation/compensate_task.py +1 -1
- mns_scheduler/zz_task/compensation/compensate_task_one_day.py +6 -6
- mns_scheduler/zz_task/data_sync_task.py +28 -1
- {mns_scheduler-1.4.4.5.dist-info → mns_scheduler-1.4.6.4.dist-info}/METADATA +1 -1
- {mns_scheduler-1.4.4.5.dist-info → mns_scheduler-1.4.6.4.dist-info}/RECORD +21 -18
- {mns_scheduler-1.4.4.5.dist-info → mns_scheduler-1.4.6.4.dist-info}/WHEEL +0 -0
- {mns_scheduler-1.4.4.5.dist-info → mns_scheduler-1.4.6.4.dist-info}/top_level.txt +0 -0
|
@@ -15,6 +15,8 @@ from loguru import logger
|
|
|
15
15
|
from mns_common.utils.async_fun import async_fun
|
|
16
16
|
import mns_common.component.common_service_fun_api as common_service_fun_api
|
|
17
17
|
import mns_common.utils.date_handle_util as date_handle_util
|
|
18
|
+
import pandas as pd
|
|
19
|
+
|
|
18
20
|
mongodb_util = MongodbUtil('27017')
|
|
19
21
|
from datetime import datetime
|
|
20
22
|
|
|
@@ -46,32 +48,32 @@ def auto_da_ban():
|
|
|
46
48
|
symbol = stock_one.symbol
|
|
47
49
|
buy_price = stock_one.zt_price
|
|
48
50
|
buy_volume = stock_one.buy_volume
|
|
49
|
-
|
|
50
|
-
time.sleep(0.5)
|
|
51
|
+
xia_dan_to_qmt(symbol, buy_price, buy_volume, now_str_day)
|
|
51
52
|
except BaseException as e:
|
|
52
|
-
logger.error("
|
|
53
|
+
logger.error("隔夜委托出现异常:{},{}", symbol, e)
|
|
53
54
|
|
|
54
55
|
|
|
55
|
-
def
|
|
56
|
-
|
|
56
|
+
def xia_dan_to_qmt(symbol, buy_price, buy_volume, str_day):
|
|
57
|
+
symbol_prefix = common_service_fun_api.add_after_prefix_one(symbol)
|
|
57
58
|
|
|
58
|
-
|
|
59
|
+
deal_service_api.trade_buy(symbol_prefix, buy_price, buy_volume, 'qmt')
|
|
59
60
|
|
|
60
61
|
# 异步更新信息
|
|
61
|
-
handle_async_msg(
|
|
62
|
+
handle_async_msg(str_day, symbol, symbol_prefix)
|
|
62
63
|
|
|
63
64
|
|
|
64
65
|
@async_fun
|
|
65
|
-
def handle_async_msg(
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
if
|
|
66
|
+
def handle_async_msg(str_day, symbol, symbol_prefix):
|
|
67
|
+
order_list = deal_service_api.query_orders('qmt')
|
|
68
|
+
order_list_df = pd.DataFrame(order_list)
|
|
69
|
+
# order_status ==57 费单
|
|
70
|
+
success_order_df = order_list_df.loc[
|
|
71
|
+
(order_list_df['stock_code'] == symbol_prefix) & (order_list_df['order_status'] != 57)]
|
|
72
|
+
# 多委托几个
|
|
73
|
+
time.sleep(1)
|
|
74
|
+
# 委托成功 更新状态
|
|
75
|
+
if data_frame_util.is_not_empty(success_order_df):
|
|
76
|
+
logger.info("隔夜委托成功:{}", symbol)
|
|
75
77
|
query = {"str_day": str_day, "symbol": symbol}
|
|
76
78
|
new_values = {"$set": {"valid": False}}
|
|
77
79
|
mongodb_util.update_many(query, new_values, db_name_constant.OVER_NIGHT_DA_BAN)
|
|
@@ -16,8 +16,7 @@ mongodb_util = MongodbUtil('27017')
|
|
|
16
16
|
def get_fix_symbol_industry():
|
|
17
17
|
return pd.DataFrame([
|
|
18
18
|
['688480', '赛恩斯', '760103', '环境治理'],
|
|
19
|
-
|
|
20
|
-
['688480', '赛恩斯', '640704', '自动化设备'],
|
|
19
|
+
|
|
21
20
|
['603260', '合盛硅业', '220316', '有机硅'],
|
|
22
21
|
['300559', '佳发教育', '461102', '培训教育'],
|
|
23
22
|
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
import sys
|
|
2
|
+
import os
|
|
3
|
+
|
|
4
|
+
file_path = os.path.abspath(__file__)
|
|
5
|
+
end = file_path.index('mns') + 16
|
|
6
|
+
project_path = file_path[0:end]
|
|
7
|
+
sys.path.append(project_path)
|
|
8
|
+
|
|
9
|
+
from mns_common.db.MongodbUtil import MongodbUtil
|
|
10
|
+
from loguru import logger
|
|
11
|
+
from datetime import datetime
|
|
12
|
+
import mns_common.constant.extra_income_db_name as extra_income_db_name
|
|
13
|
+
|
|
14
|
+
mongodb_util = MongodbUtil('27017')
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def clean_us_stock_concept_industry():
|
|
18
|
+
em_us_stock_info_df = mongodb_util.find_all_data(extra_income_db_name.EM_US_STOCK_INFO)
|
|
19
|
+
return em_us_stock_info_df
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
if __name__ == '__main__':
|
|
23
|
+
em_us_stock_info_test_df = clean_us_stock_concept_industry()
|
|
24
|
+
em_us_stock_info_test_industry_df = em_us_stock_info_test_df.loc[em_us_stock_info_test_df['industry'] != '-']
|
|
25
|
+
|
|
26
|
+
em_us_stock_info_test_no_industry_df = em_us_stock_info_test_df.loc[em_us_stock_info_test_df['industry'] == '-']
|
|
27
|
+
em_us_stock_info_test_no_industry_df = em_us_stock_info_test_no_industry_df.sort_values(by=['flow_mv'],
|
|
28
|
+
ascending=False)
|
|
29
|
+
|
|
30
|
+
print(em_us_stock_info_test_industry_df)
|
|
@@ -8,69 +8,129 @@ sys.path.append(project_path)
|
|
|
8
8
|
|
|
9
9
|
from mns_common.db.MongodbUtil import MongodbUtil
|
|
10
10
|
from loguru import logger
|
|
11
|
-
import mns_common.api.em.real_time.east_money_debt_api as east_money_debt_api
|
|
12
|
-
import mns_common.api.em.real_time.east_money_etf_api as east_money_etf_api
|
|
13
|
-
import mns_common.api.em.real_time.east_money_stock_a_v2_api as east_money_stock_a_v2_api
|
|
14
11
|
from datetime import datetime
|
|
15
12
|
import mns_common.constant.extra_income_db_name as extra_income_db_name
|
|
16
|
-
import mns_common.api.em.real_time.east_money_stock_hk_api as east_money_stock_hk_api
|
|
17
13
|
import mns_common.component.cookie.cookie_info_service as cookie_info_service
|
|
18
14
|
import mns_common.api.em.real_time.east_money_stock_us_api as east_money_stock_us_api
|
|
15
|
+
import mns_common.api.em.real_time.east_money_debt_api as east_money_debt_api
|
|
16
|
+
import mns_common.api.em.real_time.east_money_etf_api as east_money_etf_api
|
|
17
|
+
import mns_common.api.em.real_time.east_money_stock_hk_api as east_money_stock_hk_api
|
|
18
|
+
import mns_common.api.em.real_time.east_money_stock_hk_gtt_api as east_money_stock_hk_gtt_api
|
|
19
|
+
import mns_common.api.em.real_time.east_money_stock_a_v2_api as east_money_stock_a_v2_api
|
|
19
20
|
|
|
20
21
|
mongodb_util = MongodbUtil('27017')
|
|
21
22
|
|
|
22
23
|
|
|
23
24
|
def sync_all_em_stock_info():
|
|
24
|
-
now_date = datetime.now()
|
|
25
|
-
str_now_date = now_date.strftime('%Y-%m-%d %H:%M:%S')
|
|
26
|
-
|
|
27
25
|
logger.info("同步东方财富a,etf,kzz,us,hk,信息开始")
|
|
26
|
+
# 同步东方财富A股股票信息
|
|
27
|
+
sync_stock_info()
|
|
28
|
+
# 同步东方财富A股可转债信息
|
|
29
|
+
sync_kzz_info()
|
|
30
|
+
# 同步东方财富A股ETF信息
|
|
31
|
+
sync_etf_info()
|
|
32
|
+
# 同步东方财富港股信息
|
|
33
|
+
sync_hk_stock_info()
|
|
34
|
+
# 同步东方财富港股通信息
|
|
35
|
+
sync_hk_ggt_stock_info()
|
|
36
|
+
# 同步东方财富美股信息
|
|
37
|
+
sync_us_stock_info()
|
|
28
38
|
|
|
39
|
+
logger.info("同步东方财富a,etf,kzz,us,hk,信息完成")
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
# 同步东方财富A股可转债信息
|
|
43
|
+
def sync_kzz_info():
|
|
44
|
+
now_date = datetime.now()
|
|
45
|
+
str_now_date = now_date.strftime('%Y-%m-%d %H:%M:%S')
|
|
46
|
+
logger.error("同步东方财富可转债信息")
|
|
29
47
|
try:
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
mongodb_util.save_mongo(em_a_stock_info_df, extra_income_db_name.EM_A_STOCK_INFO)
|
|
48
|
+
em_kzz_info = east_money_debt_api.get_kzz_real_time_quotes(30, 6)
|
|
49
|
+
em_kzz_info['_id'] = em_kzz_info['symbol']
|
|
50
|
+
em_kzz_info['sync_time'] = str_now_date
|
|
51
|
+
em_kzz_info = em_kzz_info.fillna(0)
|
|
52
|
+
mongodb_util.save_mongo(em_kzz_info, extra_income_db_name.EM_KZZ_INFO)
|
|
36
53
|
except BaseException as e:
|
|
37
|
-
logger.error("
|
|
54
|
+
logger.error("同步东方财富可转债信息异常:{}", e)
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
# 同步东方财富A股ETF信息
|
|
58
|
+
def sync_etf_info():
|
|
59
|
+
now_date = datetime.now()
|
|
60
|
+
str_now_date = now_date.strftime('%Y-%m-%d %H:%M:%S')
|
|
61
|
+
logger.error("同步东方财富ETF信息")
|
|
38
62
|
try:
|
|
39
63
|
em_etf_info = east_money_etf_api.get_etf_real_time_quotes(30, 6)
|
|
40
64
|
em_etf_info['_id'] = em_etf_info['symbol']
|
|
41
65
|
em_etf_info['sync_time'] = str_now_date
|
|
66
|
+
em_etf_info = em_etf_info.fillna(0)
|
|
42
67
|
mongodb_util.save_mongo(em_etf_info, extra_income_db_name.EM_ETF_INFO)
|
|
43
68
|
except BaseException as e:
|
|
44
69
|
logger.error("同步东方财富ETF信息异常:{}", e)
|
|
45
70
|
|
|
71
|
+
|
|
72
|
+
# 同步东方财富A股股票信息
|
|
73
|
+
def sync_stock_info():
|
|
74
|
+
now_date = datetime.now()
|
|
75
|
+
str_now_date = now_date.strftime('%Y-%m-%d %H:%M:%S')
|
|
76
|
+
logger.error("同步东方财富股票信息")
|
|
46
77
|
try:
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
78
|
+
em_stock_info = east_money_stock_a_v2_api.get_stock_real_time_quotes(60)
|
|
79
|
+
em_stock_info['_id'] = em_stock_info['symbol']
|
|
80
|
+
em_stock_info['sync_time'] = str_now_date
|
|
81
|
+
em_stock_info = em_stock_info.fillna(0)
|
|
82
|
+
mongodb_util.save_mongo(em_stock_info, extra_income_db_name.EM_A_STOCK_INFO)
|
|
51
83
|
except BaseException as e:
|
|
52
|
-
logger.error("
|
|
84
|
+
logger.error("同步东方财富ETF信息异常:{}", e)
|
|
53
85
|
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
86
|
+
|
|
87
|
+
# 同步东方财富港股信息
|
|
88
|
+
def sync_hk_stock_info():
|
|
89
|
+
now_date = datetime.now()
|
|
90
|
+
str_now_date = now_date.strftime('%Y-%m-%d %H:%M:%S')
|
|
91
|
+
logger.error("同步东方财富港股信息")
|
|
92
|
+
|
|
93
|
+
em_cookie = cookie_info_service.get_em_cookie()
|
|
94
|
+
|
|
95
|
+
try:
|
|
96
|
+
em_hk_info = east_money_stock_hk_api.get_hk_real_time_quotes(30, em_cookie)
|
|
97
|
+
em_hk_info['_id'] = em_hk_info['symbol']
|
|
98
|
+
em_hk_info['sync_time'] = str_now_date
|
|
99
|
+
em_hk_info = em_hk_info.fillna(0)
|
|
100
|
+
mongodb_util.save_mongo(em_hk_info, extra_income_db_name.EM_HK_STOCK_INFO)
|
|
101
|
+
except BaseException as e:
|
|
102
|
+
logger.error("同步东方财富ETF信息异常:{}", e)
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
# 同步东方财富港股通信息
|
|
106
|
+
def sync_hk_ggt_stock_info():
|
|
107
|
+
now_date = datetime.now()
|
|
108
|
+
str_now_date = now_date.strftime('%Y-%m-%d %H:%M:%S')
|
|
109
|
+
logger.error("同步东方财富港股信息")
|
|
110
|
+
em_cookie = cookie_info_service.get_em_cookie()
|
|
111
|
+
|
|
112
|
+
try:
|
|
113
|
+
em_hk_gtt_info = east_money_stock_hk_gtt_api.get_ggt_real_time_quotes(em_cookie, 30, 6)
|
|
114
|
+
em_hk_gtt_info['_id'] = em_hk_gtt_info['symbol']
|
|
115
|
+
em_hk_gtt_info['sync_time'] = str_now_date
|
|
116
|
+
em_hk_gtt_info = em_hk_gtt_info.fillna(0)
|
|
117
|
+
mongodb_util.save_mongo(em_hk_gtt_info, extra_income_db_name.EM_HK_GGT_STOCK_INFO)
|
|
118
|
+
except BaseException as e:
|
|
119
|
+
logger.error("同步东方财富ETF信息异常:{}", e)
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
# 同步东方财富美股信息 todo 增加稳定接口
|
|
123
|
+
def sync_us_stock_info():
|
|
124
|
+
now_date = datetime.now()
|
|
125
|
+
str_now_date = now_date.strftime('%Y-%m-%d %H:%M:%S')
|
|
126
|
+
logger.error("同步东方财富美股信息")
|
|
127
|
+
em_cookie = cookie_info_service.get_em_cookie()
|
|
128
|
+
us_stock_info = east_money_stock_us_api.get_us_real_time_quotes(30, em_cookie)
|
|
129
|
+
us_stock_info['_id'] = us_stock_info['symbol']
|
|
130
|
+
us_stock_info['sync_time'] = str_now_date
|
|
131
|
+
|
|
132
|
+
us_stock_info = us_stock_info.fillna(0)
|
|
133
|
+
mongodb_util.save_mongo(us_stock_info, extra_income_db_name.US_STOCK_INFO_EM)
|
|
74
134
|
|
|
75
135
|
|
|
76
136
|
if __name__ == '__main__':
|
|
@@ -96,8 +96,11 @@ def sync_open_data():
|
|
|
96
96
|
|
|
97
97
|
|
|
98
98
|
if __name__ == '__main__':
|
|
99
|
-
|
|
100
|
-
|
|
99
|
+
sync_zt_data('2025-12-21')
|
|
100
|
+
sync_zt_data('2025-12-23')
|
|
101
|
+
sync_zt_data('2025-12-24')
|
|
102
|
+
sync_zt_data('2025-12-25')
|
|
103
|
+
sync_zt_data('2025-12-26')
|
|
101
104
|
remote_data()
|
|
102
105
|
# sync_zt_data('2025-08-26')
|
|
103
106
|
# sync_zt_data('2025-08-25')
|
|
@@ -143,6 +143,8 @@ def sync_symbols_interactive_questions(symbol_list):
|
|
|
143
143
|
if len(symbol_list) != 0:
|
|
144
144
|
real_time_quotes_all_stocks = real_time_quotes_all_stocks.loc[
|
|
145
145
|
real_time_quotes_all_stocks['symbol'].isin(symbol_list)]
|
|
146
|
+
|
|
147
|
+
real_time_quotes_all_stocks = real_time_quotes_all_stocks.reset_index(drop=True)
|
|
146
148
|
for stock_one in real_time_quotes_all_stocks.itertuples():
|
|
147
149
|
try:
|
|
148
150
|
now_date = datetime.now()
|
|
@@ -20,6 +20,7 @@ import mns_common.component.em.em_stock_info_api as em_stock_info_api
|
|
|
20
20
|
import mns_common.component.company.company_common_service_api as company_common_service_api
|
|
21
21
|
from loguru import logger
|
|
22
22
|
import mns_common.utils.data_frame_util as data_frame_util
|
|
23
|
+
from datetime import datetime, timedelta
|
|
23
24
|
|
|
24
25
|
mongodb_util = MongodbUtil('27017')
|
|
25
26
|
from tqdm import tqdm
|
|
@@ -137,7 +138,15 @@ def sync_sz_stock_uid(symbol_list):
|
|
|
137
138
|
if len(symbol_list) != 0:
|
|
138
139
|
real_time_quotes_all_stocks = real_time_quotes_all_stocks.loc[
|
|
139
140
|
real_time_quotes_all_stocks['symbol'].isin(symbol_list)]
|
|
140
|
-
|
|
141
|
+
else:
|
|
142
|
+
# 获取当前时间
|
|
143
|
+
now_date = datetime.now()
|
|
144
|
+
# 计算前7天的时间(timedelta 用于表示时间间隔)
|
|
145
|
+
seven_days_ago = now_date - timedelta(days=30)
|
|
146
|
+
str_day_number = int(seven_days_ago.strftime('%Y%m%d'))
|
|
147
|
+
real_time_quotes_all_stocks = real_time_quotes_all_stocks.loc[
|
|
148
|
+
(real_time_quotes_all_stocks['list_date'] >= str_day_number) | (
|
|
149
|
+
real_time_quotes_all_stocks['list_date'] == 19890604)]
|
|
141
150
|
for stock_one in real_time_quotes_all_stocks.itertuples():
|
|
142
151
|
try:
|
|
143
152
|
symbol = stock_one.symbol
|
|
@@ -155,5 +164,6 @@ if __name__ == '__main__':
|
|
|
155
164
|
sh_symbol_org_id = get_one_sh_symbol_org_id('600000')
|
|
156
165
|
print(sz_symbol_org_id)
|
|
157
166
|
print(sh_symbol_org_id)
|
|
167
|
+
sync_sz_stock_uid([])
|
|
158
168
|
# sync_sz_stock_uid([])
|
|
159
169
|
# sync_sh_stock_uid([])
|
|
@@ -116,6 +116,7 @@ def multi_threaded_k_line_sync(str_day):
|
|
|
116
116
|
# 将list_date列中的所有NaN值设置为19890604
|
|
117
117
|
real_time_quotes_now['list_date'].fillna(19890604, inplace=True)
|
|
118
118
|
real_time_quotes_now['list_date'] = real_time_quotes_now['list_date'].replace(99990909, 19890604)
|
|
119
|
+
real_time_quotes_now['list_date'] = real_time_quotes_now['list_date'].replace(0, 19890604)
|
|
119
120
|
|
|
120
121
|
# 将日期数值转换为日期时间格式
|
|
121
122
|
real_time_quotes_now['list_date_01'] = pd.to_datetime(real_time_quotes_now['list_date'], format='%Y%m%d')
|
|
@@ -0,0 +1,231 @@
|
|
|
1
|
+
import sys
|
|
2
|
+
import os
|
|
3
|
+
|
|
4
|
+
file_path = os.path.abspath(__file__)
|
|
5
|
+
end = file_path.index('mns') + 16
|
|
6
|
+
project_path = file_path[0:end]
|
|
7
|
+
sys.path.append(project_path)
|
|
8
|
+
|
|
9
|
+
import mns_common.api.kpl.theme.kpl_theme_api as kpl_theme_api
|
|
10
|
+
from mns_common.db.MongodbUtil import MongodbUtil
|
|
11
|
+
from datetime import datetime
|
|
12
|
+
import pandas as pd
|
|
13
|
+
import mns_common.constant.db_name_constant as db_name_constant
|
|
14
|
+
from loguru import logger
|
|
15
|
+
import mns_common.component.cookie.cookie_info_service as cookie_info_service
|
|
16
|
+
import mns_common.utils.data_frame_util as data_frame_util
|
|
17
|
+
import mns_common.api.msg.push_msg_api as push_msg_api
|
|
18
|
+
|
|
19
|
+
mongodb_util = MongodbUtil('27017')
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
# 获取最大id
|
|
23
|
+
def query_max_theme_id():
|
|
24
|
+
kpl_theme_max_one_df = mongodb_util.descend_query({}, db_name_constant.KPL_THEME_LIST, 'theme_id', 1)
|
|
25
|
+
if data_frame_util.is_empty(kpl_theme_max_one_df):
|
|
26
|
+
return 1
|
|
27
|
+
else:
|
|
28
|
+
return list(kpl_theme_max_one_df['theme_id'])[0]
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def sync_new_kpl_theme_info():
|
|
32
|
+
exist_max_theme_id = query_max_theme_id() + 1
|
|
33
|
+
max_sync_theme_id = exist_max_theme_id + 10
|
|
34
|
+
|
|
35
|
+
while exist_max_theme_id <= max_sync_theme_id:
|
|
36
|
+
try:
|
|
37
|
+
kpl_token = cookie_info_service.get_kpl_cookie()
|
|
38
|
+
|
|
39
|
+
json_data = kpl_theme_api.kpl_theme_index(exist_max_theme_id, kpl_token)
|
|
40
|
+
tables = json_data.get('Table')
|
|
41
|
+
if tables is not None and len(tables) > 0:
|
|
42
|
+
theme_name = json_data.get('Name')
|
|
43
|
+
|
|
44
|
+
title = '新增开盘啦题材:' + theme_name
|
|
45
|
+
push_msg_api.push_msg_to_wechat(title, title)
|
|
46
|
+
|
|
47
|
+
# 保存到主线列表
|
|
48
|
+
sync_kpl_theme_list(json_data, exist_max_theme_id)
|
|
49
|
+
# 保存到主线详细列表
|
|
50
|
+
sync_kpl_theme_detail(json_data, exist_max_theme_id)
|
|
51
|
+
exist_max_theme_id = exist_max_theme_id + 1
|
|
52
|
+
except BaseException as e:
|
|
53
|
+
logger.error("更新新题材信息异常:{},{}", str(exist_max_theme_id), e)
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def update_all_kpl_theme_info():
|
|
57
|
+
theme_id = 0
|
|
58
|
+
while theme_id < 2000:
|
|
59
|
+
try:
|
|
60
|
+
kpl_token = cookie_info_service.get_kpl_cookie()
|
|
61
|
+
json_data = kpl_theme_api.kpl_theme_index(theme_id, kpl_token)
|
|
62
|
+
# 保存到主线列表
|
|
63
|
+
sync_kpl_theme_list(json_data, theme_id)
|
|
64
|
+
# 保存到主线详细列表
|
|
65
|
+
sync_kpl_theme_detail(json_data, theme_id)
|
|
66
|
+
theme_id = theme_id + 1
|
|
67
|
+
except BaseException as e:
|
|
68
|
+
logger.error("出现异常:{}", e)
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def sync_kpl_theme_list(json_data, theme_id):
|
|
72
|
+
tables = json_data.get('Table')
|
|
73
|
+
if tables is None or len(tables) == 0:
|
|
74
|
+
return None
|
|
75
|
+
|
|
76
|
+
theme_name = json_data.get('Name')
|
|
77
|
+
|
|
78
|
+
logger.info("同步题材:{},{}", str(theme_id), theme_name)
|
|
79
|
+
brief_intro = json_data.get('BriefIntro')
|
|
80
|
+
introduction = json_data.get('Introduction')
|
|
81
|
+
kpl_create_time = covert_time(json_data.get('CreateTime'))
|
|
82
|
+
kpl_update_time = covert_time(json_data.get('UpdateTime'))
|
|
83
|
+
now_date = datetime.now()
|
|
84
|
+
sync_time = now_date.strftime('%Y-%m-%d %H:%M:%S')
|
|
85
|
+
|
|
86
|
+
theme_dict = {
|
|
87
|
+
'_id': theme_id,
|
|
88
|
+
'theme_id': theme_id,
|
|
89
|
+
'theme_name': theme_name,
|
|
90
|
+
'brief_intro': brief_intro,
|
|
91
|
+
'introduction': introduction,
|
|
92
|
+
'kpl_create_time': kpl_create_time,
|
|
93
|
+
'kpl_update_time': kpl_update_time,
|
|
94
|
+
'sync_time': sync_time,
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
ths_concept_one_df = pd.DataFrame(theme_dict, index=[1])
|
|
98
|
+
mongodb_util.save_mongo(ths_concept_one_df, db_name_constant.KPL_THEME_LIST)
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
def sync_kpl_theme_detail(json_data, theme_id):
|
|
102
|
+
tables = json_data.get('Table')
|
|
103
|
+
theme_name = json_data.get('Name')
|
|
104
|
+
if tables is None or len(tables) == 0:
|
|
105
|
+
return None
|
|
106
|
+
for table in tables:
|
|
107
|
+
save_theme_l1(table, theme_name, theme_id)
|
|
108
|
+
|
|
109
|
+
return tables
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
def save_theme_l1(table, theme_name, theme_id):
|
|
113
|
+
theme_l1 = table.get('Level1')
|
|
114
|
+
|
|
115
|
+
first_shelve_time = covert_time(theme_l1.get('FirstShelveTime'))
|
|
116
|
+
theme_l1_id = theme_l1.get('ID')
|
|
117
|
+
is_new = theme_l1.get('IsNew')
|
|
118
|
+
theme_l1_name = theme_l1.get('Name')
|
|
119
|
+
update_cache_time = covert_time(theme_l1.get('UpdateCacheTime'))
|
|
120
|
+
zs_code = theme_l1.get('ZSCode')
|
|
121
|
+
now_date = datetime.now()
|
|
122
|
+
sync_time = now_date.strftime('%Y-%m-%d %H:%M:%S')
|
|
123
|
+
theme_dict = {
|
|
124
|
+
'_id': str(theme_l1_id),
|
|
125
|
+
'theme_l1_id': str(theme_l1_id),
|
|
126
|
+
'theme_l1_name': str(theme_l1_name),
|
|
127
|
+
'theme_id': str(theme_id),
|
|
128
|
+
'theme_name': str(theme_name),
|
|
129
|
+
'first_shelve_time': first_shelve_time,
|
|
130
|
+
'update_cache_time': update_cache_time,
|
|
131
|
+
'zs_code': zs_code,
|
|
132
|
+
'is_new': is_new,
|
|
133
|
+
'sync_time': sync_time,
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
ths_concept_one_df = pd.DataFrame(theme_dict, index=[1])
|
|
137
|
+
mongodb_util.save_mongo(ths_concept_one_df, db_name_constant.KPL_THEME_LIST_L1)
|
|
138
|
+
stock_list = theme_l1.get('Stocks')
|
|
139
|
+
if len(stock_list) != 0:
|
|
140
|
+
save_theme_stocks(stock_list, theme_name, theme_id, theme_l1_name, theme_l1_id, '无', 0)
|
|
141
|
+
else:
|
|
142
|
+
theme_l2_list = table.get('Level2')
|
|
143
|
+
save_theme_l2(theme_l2_list, theme_name, theme_id, theme_l1_name, theme_l1_id)
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
def save_theme_l2(theme_l2_list, theme_name, theme_id, theme_l1_name, theme_l1_id):
|
|
147
|
+
for theme_l2_one in theme_l2_list:
|
|
148
|
+
|
|
149
|
+
first_shelve_time = covert_time(theme_l2_one.get('FirstShelveTime'))
|
|
150
|
+
theme_l2_id = theme_l2_one.get('ID')
|
|
151
|
+
is_new = theme_l2_one.get('IsNew')
|
|
152
|
+
theme_l2_name = theme_l2_one.get('Name')
|
|
153
|
+
update_cache_time = covert_time(theme_l2_one.get('UpdateCacheTime'))
|
|
154
|
+
zs_code = theme_l2_one.get('ZSCode')
|
|
155
|
+
now_date = datetime.now()
|
|
156
|
+
sync_time = now_date.strftime('%Y-%m-%d %H:%M:%S')
|
|
157
|
+
theme_dict = {
|
|
158
|
+
'_id': str(theme_l2_id),
|
|
159
|
+
'theme_l2_id': str(theme_l2_id),
|
|
160
|
+
'theme_l2_name': str(theme_l2_name),
|
|
161
|
+
|
|
162
|
+
'theme_l1_id': str(theme_l1_id),
|
|
163
|
+
'theme_l1_name': str(theme_l1_name),
|
|
164
|
+
'theme_id': str(theme_id),
|
|
165
|
+
'theme_name': str(theme_name),
|
|
166
|
+
'first_shelve_time': first_shelve_time,
|
|
167
|
+
'update_cache_time': update_cache_time,
|
|
168
|
+
'zs_code': zs_code,
|
|
169
|
+
'is_new': is_new,
|
|
170
|
+
'sync_time': sync_time,
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
ths_concept_one_df = pd.DataFrame(theme_dict, index=[1])
|
|
174
|
+
mongodb_util.save_mongo(ths_concept_one_df, db_name_constant.KPL_THEME_LIST_L2)
|
|
175
|
+
stock_list = theme_l2_one.get('Stocks')
|
|
176
|
+
if len(stock_list) != 0:
|
|
177
|
+
save_theme_stocks(stock_list, theme_name, theme_id, theme_l1_name, theme_l1_id, theme_l2_name, theme_l2_id)
|
|
178
|
+
|
|
179
|
+
|
|
180
|
+
def save_theme_stocks(stock_list, theme_name, theme_id, theme_l1_name, theme_l1_id, theme_l2_name, theme_l2_id):
|
|
181
|
+
stock_list_df = pd.DataFrame(stock_list)
|
|
182
|
+
stock_list_df = stock_list_df.rename(columns={
|
|
183
|
+
"StockID": "symbol",
|
|
184
|
+
"IsZz": "is_zz",
|
|
185
|
+
"IsHot": "is_hot",
|
|
186
|
+
"Reason": "reason",
|
|
187
|
+
"FirstShelveTime": "first_shelve_time",
|
|
188
|
+
"UpdateCacheTime": "update_cache_time",
|
|
189
|
+
"IsNew": "is_new",
|
|
190
|
+
"prod_name": "prod_name",
|
|
191
|
+
"Hot": "hot",
|
|
192
|
+
})
|
|
193
|
+
stock_list_df["first_shelve_time"] = stock_list_df["first_shelve_time"].apply(covert_time)
|
|
194
|
+
stock_list_df["update_cache_time"] = stock_list_df["update_cache_time"].apply(covert_time)
|
|
195
|
+
|
|
196
|
+
stock_list_df['theme_id'] = str(theme_id)
|
|
197
|
+
stock_list_df['theme_name'] = theme_name
|
|
198
|
+
|
|
199
|
+
stock_list_df['theme_l1_id'] = str(theme_l1_id)
|
|
200
|
+
stock_list_df['theme_l1_name'] = theme_l1_name
|
|
201
|
+
|
|
202
|
+
stock_list_df['theme_l2_name'] = theme_l2_name
|
|
203
|
+
stock_list_df['theme_l2_id'] = str(theme_l2_id)
|
|
204
|
+
|
|
205
|
+
stock_list_df['_id'] = (stock_list_df['symbol']
|
|
206
|
+
+ '_'
|
|
207
|
+
+ stock_list_df['theme_id']
|
|
208
|
+
+ '_'
|
|
209
|
+
+ stock_list_df['theme_l1_id']
|
|
210
|
+
+ '_'
|
|
211
|
+
+ stock_list_df['theme_l2_id'])
|
|
212
|
+
now_date = datetime.now()
|
|
213
|
+
sync_time = now_date.strftime('%Y-%m-%d %H:%M:%S')
|
|
214
|
+
stock_list_df['sync_time'] = sync_time
|
|
215
|
+
mongodb_util.save_mongo(stock_list_df, db_name_constant.KPL_THEME_DETAILS)
|
|
216
|
+
|
|
217
|
+
|
|
218
|
+
def covert_time(timestamp_str):
|
|
219
|
+
timestamp = int(timestamp_str)
|
|
220
|
+
if timestamp == 0:
|
|
221
|
+
return "1989-06-04"
|
|
222
|
+
else:
|
|
223
|
+
# 3. 转换为本地时区的datetime对象
|
|
224
|
+
dt_local = datetime.fromtimestamp(timestamp)
|
|
225
|
+
# 4. 格式化为「年月日 时分秒」的字符串(格式可自定义)
|
|
226
|
+
datetime_str = dt_local.strftime('%Y-%m-%d %H:%M:%S')
|
|
227
|
+
return datetime_str
|
|
228
|
+
|
|
229
|
+
|
|
230
|
+
if __name__ == '__main__':
|
|
231
|
+
update_all_kpl_theme_info()
|