mns-scheduler 1.0.5.5__py3-none-any.whl → 1.0.5.8__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of mns-scheduler might be problematic. Click here for more details.
- mns_scheduler/company_info/company_info_sync_api.py +24 -2
- mns_scheduler/irm/api/__init__.py +7 -0
- mns_scheduler/irm/api/sh_stock_sns_sse_info_api.py +168 -0
- mns_scheduler/irm/api/sz_stock_sns_sse_info_api.py +202 -0
- mns_scheduler/irm/stock_irm_cninfo_service.py +124 -5
- mns_scheduler/zz_task/data_sync_task.py +10 -0
- {mns_scheduler-1.0.5.5.dist-info → mns_scheduler-1.0.5.8.dist-info}/METADATA +1 -1
- {mns_scheduler-1.0.5.5.dist-info → mns_scheduler-1.0.5.8.dist-info}/RECORD +10 -7
- {mns_scheduler-1.0.5.5.dist-info → mns_scheduler-1.0.5.8.dist-info}/WHEEL +0 -0
- {mns_scheduler-1.0.5.5.dist-info → mns_scheduler-1.0.5.8.dist-info}/top_level.txt +0 -0
|
@@ -19,8 +19,8 @@ import mns_common.api.kpl.symbol.kpl_real_time_quotes_api as kpl_real_time_quote
|
|
|
19
19
|
import mns_common.utils.data_frame_util as data_frame_util
|
|
20
20
|
import mns_common.api.kpl.constant.kpl_constant as kpl_constant
|
|
21
21
|
import mns_common.component.company.company_common_service_api as company_common_service_api
|
|
22
|
-
import mns_common.constant.db_name_constant as db_name_constant
|
|
23
22
|
import mns_common.component.k_line.common.k_line_common_service_api as k_line_common_service_api
|
|
23
|
+
import mns_common.constant.db_name_constant as db_name_constant
|
|
24
24
|
|
|
25
25
|
mongodb_util = MongodbUtil('27017')
|
|
26
26
|
# 分页大小
|
|
@@ -105,7 +105,9 @@ def filed_sort(company_info):
|
|
|
105
105
|
"ths_concept_list_info",
|
|
106
106
|
"kpl_plate_name",
|
|
107
107
|
"kpl_most_relative_name",
|
|
108
|
-
"kpl_plate_list_info"
|
|
108
|
+
"kpl_plate_list_info",
|
|
109
|
+
'operate_profit',
|
|
110
|
+
'total_operate_income'
|
|
109
111
|
]]
|
|
110
112
|
|
|
111
113
|
|
|
@@ -332,6 +334,9 @@ def single_thread_sync_company_info(east_money_stock_info,
|
|
|
332
334
|
str_day = now_date.strftime('%Y-%m-%d')
|
|
333
335
|
deal_days = k_line_common_service_api.get_deal_days(str_day, company_one.symbol)
|
|
334
336
|
company_info_type['deal_days'] = deal_days
|
|
337
|
+
# 设置年报信息
|
|
338
|
+
company_info_type = get_recent_year_income(company_one.symbol, company_info_type)
|
|
339
|
+
|
|
335
340
|
try:
|
|
336
341
|
if data_frame_util.is_not_empty(kpl_real_time_quotes):
|
|
337
342
|
kpl_real_time_quotes_one = kpl_real_time_quotes.loc[
|
|
@@ -480,6 +485,23 @@ def save_sw_data(company_info_type):
|
|
|
480
485
|
mongodb_util.save_mongo(third_sw_info, 'sw_industry')
|
|
481
486
|
|
|
482
487
|
|
|
488
|
+
# 获取最近年报收入
|
|
489
|
+
def get_recent_year_income(symbol, company_info_type):
|
|
490
|
+
query = {'symbol': symbol, "REPORT_TYPE": "年报"}
|
|
491
|
+
em_stock_profit = mongodb_util.descend_query(query, db_name_constant.EM_STOCK_PROFIT, 'REPORT_DATE', 1)
|
|
492
|
+
if data_frame_util.is_not_empty(em_stock_profit):
|
|
493
|
+
company_info_type['operate_profit'] = list(em_stock_profit['OPERATE_PROFIT'])[0]
|
|
494
|
+
company_info_type['total_operate_income'] = list(em_stock_profit['TOTAL_OPERATE_INCOME'])[0]
|
|
495
|
+
else:
|
|
496
|
+
company_info_type['operate_profit'] = 0
|
|
497
|
+
company_info_type['total_operate_income'] = 0
|
|
498
|
+
company_info_type['operate_profit'] = round(
|
|
499
|
+
company_info_type['operate_profit'] / common_service_fun_api.HUNDRED_MILLION, 2)
|
|
500
|
+
company_info_type['total_operate_income'] = round(
|
|
501
|
+
company_info_type['total_operate_income'] / common_service_fun_api.HUNDRED_MILLION, 2)
|
|
502
|
+
return company_info_type
|
|
503
|
+
|
|
504
|
+
|
|
483
505
|
if __name__ == '__main__':
|
|
484
506
|
# while True:
|
|
485
507
|
# df = ths_stock_api.get_company_info_detail('000001')
|
|
@@ -0,0 +1,168 @@
|
|
|
1
|
+
import sys
|
|
2
|
+
import os
|
|
3
|
+
|
|
4
|
+
file_path = os.path.abspath(__file__)
|
|
5
|
+
end = file_path.index('mns') + 16
|
|
6
|
+
project_path = file_path[0:end]
|
|
7
|
+
sys.path.append(project_path)
|
|
8
|
+
# !/usr/bin/env python
|
|
9
|
+
# -*- coding:utf-8 -*-
|
|
10
|
+
"""
|
|
11
|
+
Date: 2023/8/4 19:20
|
|
12
|
+
Desc: 上证e互动-提问与回答
|
|
13
|
+
https://sns.sseinfo.com/
|
|
14
|
+
"""
|
|
15
|
+
import warnings
|
|
16
|
+
from functools import lru_cache
|
|
17
|
+
|
|
18
|
+
import pandas as pd
|
|
19
|
+
import requests
|
|
20
|
+
from bs4 import BeautifulSoup
|
|
21
|
+
|
|
22
|
+
from mns_common.db.MongodbUtil import MongodbUtil
|
|
23
|
+
import mns_common.constant.db_name_constant as db_name_constant
|
|
24
|
+
|
|
25
|
+
mongodb_util = MongodbUtil('27017')
|
|
26
|
+
from tqdm import tqdm
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
@lru_cache()
|
|
30
|
+
def sync_stock_uid() -> pd.DataFrame:
|
|
31
|
+
"""
|
|
32
|
+
上证e互动-代码ID映射
|
|
33
|
+
https://sns.sseinfo.com/list/company.do
|
|
34
|
+
:return: 代码ID映射
|
|
35
|
+
:rtype: str
|
|
36
|
+
"""
|
|
37
|
+
url = "https://sns.sseinfo.com/allcompany.do"
|
|
38
|
+
data = {
|
|
39
|
+
"code": "0",
|
|
40
|
+
"order": "2",
|
|
41
|
+
"areaId": "0",
|
|
42
|
+
"page": "1",
|
|
43
|
+
}
|
|
44
|
+
uid_list = list()
|
|
45
|
+
code_list = list()
|
|
46
|
+
for page in tqdm(range(1, 74), leave=False):
|
|
47
|
+
data.update({"page": page})
|
|
48
|
+
r = requests.post(url, data=data)
|
|
49
|
+
data_json = r.json()
|
|
50
|
+
soup = BeautifulSoup(data_json["content"], "lxml")
|
|
51
|
+
soup.find_all("a", attrs={"rel": "tag"})
|
|
52
|
+
uid_list.extend(
|
|
53
|
+
[item["uid"] for item in soup.find_all("a", attrs={"rel": "tag"})]
|
|
54
|
+
)
|
|
55
|
+
code_list.extend(
|
|
56
|
+
[
|
|
57
|
+
item.find("img")["src"].split("?")[0].split("/")[-1].split(".")[0]
|
|
58
|
+
for item in soup.find_all("a", attrs={"rel": "tag"})
|
|
59
|
+
]
|
|
60
|
+
)
|
|
61
|
+
code_uid_df = pd.DataFrame()
|
|
62
|
+
code_uid_df['symbol'] = code_list
|
|
63
|
+
code_uid_df['uid'] = uid_list
|
|
64
|
+
code_uid_df['_id'] = uid_list
|
|
65
|
+
return code_uid_df
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
@lru_cache()
|
|
69
|
+
def get_stock_uid() -> pd.DataFrame:
|
|
70
|
+
return mongodb_util.find_all_data(db_name_constant.SSE_INFO_UID)
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def stock_sns_sse_info(symbol: str = "603119") -> pd.DataFrame:
|
|
74
|
+
"""
|
|
75
|
+
上证e互动-提问与回答
|
|
76
|
+
https://sns.sseinfo.com/company.do?uid=65
|
|
77
|
+
:param symbol: 股票代码
|
|
78
|
+
:type symbol: str
|
|
79
|
+
:return: 提问与回答
|
|
80
|
+
:rtype: str
|
|
81
|
+
"""
|
|
82
|
+
|
|
83
|
+
stock_uid_df = get_stock_uid()
|
|
84
|
+
stock_uid_df = stock_uid_df.loc[stock_uid_df['symbol'] == symbol]
|
|
85
|
+
uid = list(stock_uid_df['uid'])[0]
|
|
86
|
+
url = "https://sns.sseinfo.com/ajax/userfeeds.do"
|
|
87
|
+
params = {
|
|
88
|
+
"typeCode": "company",
|
|
89
|
+
"type": "11",
|
|
90
|
+
"pageSize": "100",
|
|
91
|
+
"uid": uid,
|
|
92
|
+
"page": "1",
|
|
93
|
+
}
|
|
94
|
+
big_df = pd.DataFrame()
|
|
95
|
+
page = 1
|
|
96
|
+
warnings.warn("正在下载中")
|
|
97
|
+
params.update({"page": page})
|
|
98
|
+
r = requests.post(url, params=params)
|
|
99
|
+
if len(r.text) < 300:
|
|
100
|
+
return None
|
|
101
|
+
r = requests.post(url, params=params)
|
|
102
|
+
soup = BeautifulSoup(r.text, "lxml")
|
|
103
|
+
content_list = [
|
|
104
|
+
item.get_text().strip()
|
|
105
|
+
for item in soup.find_all("div", attrs={"class": "m_feed_txt"})
|
|
106
|
+
]
|
|
107
|
+
date_list = [
|
|
108
|
+
item.get_text().strip().split("\n")[0]
|
|
109
|
+
for item in soup.find_all("div", attrs={"class": "m_feed_from"})
|
|
110
|
+
]
|
|
111
|
+
source_list = [
|
|
112
|
+
item.get_text().strip().split("\n")[2]
|
|
113
|
+
for item in soup.find_all("div", attrs={"class": "m_feed_from"})
|
|
114
|
+
]
|
|
115
|
+
q_list = [
|
|
116
|
+
item.split(")")[1]
|
|
117
|
+
for index, item in enumerate(content_list)
|
|
118
|
+
if index % 2 == 0
|
|
119
|
+
]
|
|
120
|
+
stock_name = [
|
|
121
|
+
item.split("(")[0].strip(":")
|
|
122
|
+
for index, item in enumerate(content_list)
|
|
123
|
+
if index % 2 == 0
|
|
124
|
+
]
|
|
125
|
+
stock_code = [
|
|
126
|
+
item.split("(")[1].split(")")[0]
|
|
127
|
+
for index, item in enumerate(content_list)
|
|
128
|
+
if index % 2 == 0
|
|
129
|
+
]
|
|
130
|
+
a_list = [item for index, item in enumerate(content_list) if index % 2 != 0]
|
|
131
|
+
d_q_list = [item for index, item in enumerate(date_list) if index % 2 == 0]
|
|
132
|
+
d_a_list = [item for index, item in enumerate(date_list) if index % 2 != 0]
|
|
133
|
+
s_q_list = [item for index, item in enumerate(source_list) if index % 2 == 0]
|
|
134
|
+
s_a_list = [item for index, item in enumerate(source_list) if index % 2 != 0]
|
|
135
|
+
author_name = [
|
|
136
|
+
item["title"] for item in soup.find_all("a", attrs={"rel": "face"})
|
|
137
|
+
]
|
|
138
|
+
temp_df = pd.DataFrame(
|
|
139
|
+
[
|
|
140
|
+
stock_code,
|
|
141
|
+
stock_name,
|
|
142
|
+
q_list,
|
|
143
|
+
a_list,
|
|
144
|
+
d_q_list,
|
|
145
|
+
d_a_list,
|
|
146
|
+
s_q_list,
|
|
147
|
+
s_a_list,
|
|
148
|
+
author_name,
|
|
149
|
+
]
|
|
150
|
+
).T
|
|
151
|
+
temp_df.columns = [
|
|
152
|
+
"股票代码",
|
|
153
|
+
"公司简称",
|
|
154
|
+
"问题",
|
|
155
|
+
"回答",
|
|
156
|
+
"问题时间",
|
|
157
|
+
"回答时间",
|
|
158
|
+
"问题来源",
|
|
159
|
+
"回答来源",
|
|
160
|
+
"用户名",
|
|
161
|
+
]
|
|
162
|
+
big_df = pd.concat([big_df, temp_df], ignore_index=True)
|
|
163
|
+
return big_df
|
|
164
|
+
|
|
165
|
+
|
|
166
|
+
if __name__ == "__main__":
|
|
167
|
+
stock_sns_sse_info_df = stock_sns_sse_info(symbol="688787")
|
|
168
|
+
print(stock_sns_sse_info_df)
|
|
@@ -0,0 +1,202 @@
|
|
|
1
|
+
import sys
|
|
2
|
+
import os
|
|
3
|
+
|
|
4
|
+
file_path = os.path.abspath(__file__)
|
|
5
|
+
end = file_path.index('mns') + 16
|
|
6
|
+
project_path = file_path[0:end]
|
|
7
|
+
sys.path.append(project_path)
|
|
8
|
+
# !/usr/bin/env python
|
|
9
|
+
# -*- coding:utf-8 -*-
|
|
10
|
+
"""
|
|
11
|
+
Date: 2024/5/12 22:20
|
|
12
|
+
Desc: 互动易-提问与回答
|
|
13
|
+
https://irm.cninfo.com.cn/
|
|
14
|
+
"""
|
|
15
|
+
|
|
16
|
+
import pandas as pd
|
|
17
|
+
import requests
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def _fetch_org_id(symbol: str = "000001") -> str:
|
|
21
|
+
"""
|
|
22
|
+
股票-互动易-组织代码
|
|
23
|
+
https://irm.cninfo.com.cn/
|
|
24
|
+
:return: 组织代码
|
|
25
|
+
:rtype: str
|
|
26
|
+
"""
|
|
27
|
+
url = "https://irm.cninfo.com.cn/newircs/index/queryKeyboardInfo"
|
|
28
|
+
params = {"_t": "1691144074"}
|
|
29
|
+
data = {"keyWord": symbol}
|
|
30
|
+
r = requests.post(url, params=params, data=data)
|
|
31
|
+
data_json = r.json()
|
|
32
|
+
org_id = data_json["data"][0]["secid"]
|
|
33
|
+
return org_id
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
# 获取股票所有问题
|
|
37
|
+
|
|
38
|
+
def stock_irm_cninfo(symbol: str = "002594") -> pd.DataFrame:
|
|
39
|
+
"""
|
|
40
|
+
互动易-提问
|
|
41
|
+
https://irm.cninfo.com.cn/ircs/question/questionDetail?questionId=1515236357817618432
|
|
42
|
+
:param symbol: 股票代码
|
|
43
|
+
:type symbol: str
|
|
44
|
+
:return: 提问
|
|
45
|
+
:rtype: str
|
|
46
|
+
"""
|
|
47
|
+
url = "https://irm.cninfo.com.cn/newircs/company/question"
|
|
48
|
+
params = {
|
|
49
|
+
"_t": "1691142650",
|
|
50
|
+
"stockcode": symbol,
|
|
51
|
+
"orgId": _fetch_org_id(symbol),
|
|
52
|
+
"pageSize": "100",
|
|
53
|
+
"pageNum": "1",
|
|
54
|
+
"keyWord": "",
|
|
55
|
+
"startDay": "",
|
|
56
|
+
"endDay": "",
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
big_df = pd.DataFrame()
|
|
60
|
+
page = 1
|
|
61
|
+
params.update({"pageNum": page})
|
|
62
|
+
r = requests.post(url, params=params)
|
|
63
|
+
data_json = r.json()
|
|
64
|
+
temp_df = pd.DataFrame(data_json["rows"])
|
|
65
|
+
big_df = pd.concat(objs=[big_df, temp_df], ignore_index=True)
|
|
66
|
+
big_df.rename(
|
|
67
|
+
columns={
|
|
68
|
+
"indexId": "问题编号",
|
|
69
|
+
"contentType": "-",
|
|
70
|
+
"trade": "行业",
|
|
71
|
+
"mainContent": "问题",
|
|
72
|
+
"attachmentUrl": "-",
|
|
73
|
+
"boardType": "行业代码",
|
|
74
|
+
"filetype": "-",
|
|
75
|
+
"pubDate": "提问时间",
|
|
76
|
+
"stockCode": "股票代码",
|
|
77
|
+
"companyShortName": "公司简称",
|
|
78
|
+
"author": "提问者编号",
|
|
79
|
+
"authorName": "提问者",
|
|
80
|
+
"authorLogo": "-",
|
|
81
|
+
"pubClient": "来源",
|
|
82
|
+
"attachedId": "回答ID",
|
|
83
|
+
"attachedContent": "回答内容",
|
|
84
|
+
"attachedAuthor": "回答者",
|
|
85
|
+
"attachedPubDate": "-",
|
|
86
|
+
"updateDate": "更新时间",
|
|
87
|
+
"isPraise": "-",
|
|
88
|
+
"isFavorite": "-",
|
|
89
|
+
"isForward": "-",
|
|
90
|
+
"praiseCount": "-",
|
|
91
|
+
"qaStatus": "-",
|
|
92
|
+
"rights": "-",
|
|
93
|
+
"topStatus": "-",
|
|
94
|
+
"companyLogo": "-",
|
|
95
|
+
"favoriteCount": "-",
|
|
96
|
+
"forwardCount": "-",
|
|
97
|
+
},
|
|
98
|
+
inplace=True,
|
|
99
|
+
)
|
|
100
|
+
big_df = big_df[
|
|
101
|
+
[
|
|
102
|
+
"股票代码",
|
|
103
|
+
"公司简称",
|
|
104
|
+
"行业",
|
|
105
|
+
"行业代码",
|
|
106
|
+
"问题",
|
|
107
|
+
"提问者",
|
|
108
|
+
"来源",
|
|
109
|
+
"提问时间",
|
|
110
|
+
"更新时间",
|
|
111
|
+
"提问者编号",
|
|
112
|
+
"问题编号",
|
|
113
|
+
"回答ID",
|
|
114
|
+
"回答内容",
|
|
115
|
+
"回答者",
|
|
116
|
+
]
|
|
117
|
+
]
|
|
118
|
+
big_df["行业"] = [item[0] for item in big_df["行业"]]
|
|
119
|
+
big_df["行业代码"] = [item[0] for item in big_df["行业代码"]]
|
|
120
|
+
big_df["提问时间"] = (
|
|
121
|
+
pd.to_datetime(big_df["提问时间"], unit="ms", errors="coerce")
|
|
122
|
+
.dt.tz_localize("UTC")
|
|
123
|
+
.dt.tz_convert("Asia/Shanghai")
|
|
124
|
+
.dt.strftime("%Y-%m-%d %H:%M:%S")
|
|
125
|
+
)
|
|
126
|
+
big_df["更新时间"] = (
|
|
127
|
+
pd.to_datetime(big_df["更新时间"], unit="ms", errors="coerce")
|
|
128
|
+
.dt.tz_localize("UTC")
|
|
129
|
+
.dt.tz_convert("Asia/Shanghai")
|
|
130
|
+
.dt.strftime("%Y-%m-%d %H:%M:%S")
|
|
131
|
+
)
|
|
132
|
+
big_df["来源"] = big_df["来源"].map(
|
|
133
|
+
{
|
|
134
|
+
"2": "APP",
|
|
135
|
+
"5": "公众号",
|
|
136
|
+
"4": "网站",
|
|
137
|
+
}
|
|
138
|
+
)
|
|
139
|
+
big_df["来源"] = big_df["来源"].fillna("网站")
|
|
140
|
+
return big_df
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
def stock_irm_ans_cninfo(symbol: str = "1513586704097333248") -> pd.DataFrame:
|
|
144
|
+
"""
|
|
145
|
+
互动易-回答
|
|
146
|
+
https://irm.cninfo.com.cn/ircs/question/questionDetail?questionId=1515236357817618432
|
|
147
|
+
:param symbol: 提问者编号; 通过 ak.stock_irm_cninfo 来获取具体的提问者编号
|
|
148
|
+
:type symbol: str
|
|
149
|
+
:return: 回答
|
|
150
|
+
:rtype: str
|
|
151
|
+
"""
|
|
152
|
+
url = "https://irm.cninfo.com.cn/newircs/question/getQuestionDetail"
|
|
153
|
+
params = {"questionId": symbol, "_t": "1691146921"}
|
|
154
|
+
r = requests.get(url, params=params)
|
|
155
|
+
data_json = r.json()
|
|
156
|
+
temp_df = pd.DataFrame.from_dict(data_json["data"], orient="index").T
|
|
157
|
+
if "replyDate" not in temp_df.columns:
|
|
158
|
+
return pd.DataFrame()
|
|
159
|
+
temp_df.rename(
|
|
160
|
+
columns={
|
|
161
|
+
"questionContent": "问题",
|
|
162
|
+
"questioner": "提问者",
|
|
163
|
+
"questionDate": "提问时间",
|
|
164
|
+
"replyDate": "回答时间",
|
|
165
|
+
"replyContent": "回答内容",
|
|
166
|
+
"stockCode": "股票代码",
|
|
167
|
+
"shortName": "公司简称",
|
|
168
|
+
},
|
|
169
|
+
inplace=True,
|
|
170
|
+
)
|
|
171
|
+
temp_df = temp_df[
|
|
172
|
+
[
|
|
173
|
+
"股票代码",
|
|
174
|
+
"公司简称",
|
|
175
|
+
"问题",
|
|
176
|
+
"回答内容",
|
|
177
|
+
"提问者",
|
|
178
|
+
"提问时间",
|
|
179
|
+
"回答时间",
|
|
180
|
+
]
|
|
181
|
+
]
|
|
182
|
+
temp_df["提问时间"] = (
|
|
183
|
+
pd.to_datetime(temp_df["提问时间"], unit="ms", errors="coerce")
|
|
184
|
+
.dt.tz_localize("UTC")
|
|
185
|
+
.dt.tz_convert("Asia/Shanghai")
|
|
186
|
+
.dt.strftime("%Y-%m-%d %H:%M:%S")
|
|
187
|
+
)
|
|
188
|
+
temp_df["回答时间"] = (
|
|
189
|
+
pd.to_datetime(temp_df["回答时间"], unit="ms", errors="coerce")
|
|
190
|
+
.dt.tz_localize("UTC")
|
|
191
|
+
.dt.tz_convert("Asia/Shanghai")
|
|
192
|
+
.dt.strftime("%Y-%m-%d %H:%M:%S")
|
|
193
|
+
)
|
|
194
|
+
return temp_df
|
|
195
|
+
|
|
196
|
+
|
|
197
|
+
if __name__ == "__main__":
|
|
198
|
+
stock_irm_cninfo_df = stock_irm_cninfo(symbol="002594")
|
|
199
|
+
print(stock_irm_cninfo_df)
|
|
200
|
+
|
|
201
|
+
stock_irm_ans_cninfo_df = stock_irm_ans_cninfo(symbol="1495108801386602496")
|
|
202
|
+
print(stock_irm_ans_cninfo_df)
|
|
@@ -8,13 +8,30 @@ sys.path.append(project_path)
|
|
|
8
8
|
import akshare as ak
|
|
9
9
|
import mns_common.utils.data_frame_util as data_frame_util
|
|
10
10
|
import pandas as pd
|
|
11
|
+
import mns_common.api.em.east_money_stock_api as east_money_stock_api
|
|
12
|
+
import mns_common.component.company.company_common_service_api as company_common_service_api
|
|
11
13
|
from loguru import logger
|
|
14
|
+
from datetime import datetime
|
|
12
15
|
|
|
16
|
+
import mns_common.utils.date_handle_util as date_handle_util
|
|
13
17
|
|
|
14
|
-
|
|
15
|
-
|
|
18
|
+
import mns_common.constant.db_name_constant as db_name_constant
|
|
19
|
+
import mns_common.component.common_service_fun_api as common_service_fun_api
|
|
20
|
+
import mns_scheduler.irm.api.sh_stock_sns_sse_info_api as sh_stock_sns_sse_info_api
|
|
21
|
+
import mns_scheduler.irm.api.sz_stock_sns_sse_info_api as sz_stock_sns_sse_info_api
|
|
22
|
+
|
|
23
|
+
from mns_common.db.MongodbUtil import MongodbUtil
|
|
24
|
+
|
|
25
|
+
mongodb_util = MongodbUtil('27017')
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
# 获取股票提问 互动易-提问 深交所
|
|
29
|
+
def get_stock_irm_cninfo_sz_api(symbol):
|
|
16
30
|
try:
|
|
17
|
-
|
|
31
|
+
# 获取一页
|
|
32
|
+
stock_irm_cninfo_df = sz_stock_sns_sse_info_api.stock_irm_cninfo(symbol)
|
|
33
|
+
# 获取全页
|
|
34
|
+
# stock_irm_cninfo_df = ak.stock_irm_cninfo(symbol)
|
|
18
35
|
except Exception as e:
|
|
19
36
|
logger.error("获取提问者异常:{},{}", symbol, e)
|
|
20
37
|
return pd.DataFrame()
|
|
@@ -28,15 +45,117 @@ def get_stock_irm_cninfo(symbol):
|
|
|
28
45
|
'提问者': "questioner",
|
|
29
46
|
'来源': "source",
|
|
30
47
|
'提问时间': "question_time",
|
|
31
|
-
'更新时间': "
|
|
48
|
+
'更新时间': "answer_time",
|
|
32
49
|
"提问者编号": "questioner_no",
|
|
33
50
|
"问题编号": "question_no",
|
|
34
51
|
"回答ID": "answer_id",
|
|
35
52
|
"回答内容": "answer_content",
|
|
36
53
|
"回答者": "answer"
|
|
37
54
|
})
|
|
55
|
+
stock_irm_cninfo_df['_id'] = stock_irm_cninfo_df['symbol'] + '_' + stock_irm_cninfo_df['question_no']
|
|
56
|
+
stock_irm_cninfo_df = stock_irm_cninfo_df[[
|
|
57
|
+
'_id',
|
|
58
|
+
"symbol",
|
|
59
|
+
"name",
|
|
60
|
+
"question",
|
|
61
|
+
"answer_content",
|
|
62
|
+
"question_time",
|
|
63
|
+
"answer_time",
|
|
64
|
+
"questioner",
|
|
65
|
+
"source"]]
|
|
38
66
|
return stock_irm_cninfo_df
|
|
39
67
|
|
|
40
68
|
|
|
69
|
+
# 获取股票提问 互动易-提问 上交所
|
|
70
|
+
def get_stock_irm_cninfo_sh_api(symbol):
|
|
71
|
+
try:
|
|
72
|
+
# 获取一页
|
|
73
|
+
stock_sns_sse_info_df = sh_stock_sns_sse_info_api.stock_sns_sse_info(symbol)
|
|
74
|
+
# 获取全页
|
|
75
|
+
# stock_sns_sse_info_df = ak.stock_sns_sseinfo(symbol)
|
|
76
|
+
except Exception as e:
|
|
77
|
+
logger.error("获取提问者异常:{},{}", symbol, e)
|
|
78
|
+
return pd.DataFrame()
|
|
79
|
+
if data_frame_util.is_empty(stock_sns_sse_info_df):
|
|
80
|
+
return pd.DataFrame()
|
|
81
|
+
|
|
82
|
+
stock_sns_sse_info_df = stock_sns_sse_info_df.rename(columns={"股票代码": "symbol",
|
|
83
|
+
"公司简称": "name",
|
|
84
|
+
"问题": "question",
|
|
85
|
+
"回答": "answer_content",
|
|
86
|
+
'问题时间': "question_time",
|
|
87
|
+
'回答时间': "answer_time",
|
|
88
|
+
'用户名': "questioner",
|
|
89
|
+
'问题来源': "source"
|
|
90
|
+
})
|
|
91
|
+
stock_sns_sse_info_df['question_time'] = stock_sns_sse_info_df['question_time'].apply(replace_date_format)
|
|
92
|
+
stock_sns_sse_info_df['answer_time'] = stock_sns_sse_info_df['answer_time'].apply(replace_date_format)
|
|
93
|
+
stock_sns_sse_info_df['_id'] = stock_sns_sse_info_df['symbol'] + '_' + stock_sns_sse_info_df['question_time']
|
|
94
|
+
stock_sns_sse_info_df = stock_sns_sse_info_df[[
|
|
95
|
+
"_id",
|
|
96
|
+
"symbol",
|
|
97
|
+
"name",
|
|
98
|
+
"question",
|
|
99
|
+
"answer_content",
|
|
100
|
+
"question_time",
|
|
101
|
+
"answer_time",
|
|
102
|
+
"questioner",
|
|
103
|
+
"source"]]
|
|
104
|
+
|
|
105
|
+
return stock_sns_sse_info_df
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
def replace_date_format(date_str):
|
|
109
|
+
now_date = datetime.now()
|
|
110
|
+
str_day = now_date.strftime('%Y-%m-%d')
|
|
111
|
+
last_day = date_handle_util.add_date(str_day, -1)
|
|
112
|
+
|
|
113
|
+
return date_str.replace('年', '-').replace('月', '-').replace('日', '').replace('昨天', last_day)
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
def save_sh_stock_uid():
|
|
117
|
+
code_uid_df = sh_stock_sns_sse_info_api.sync_stock_uid()
|
|
118
|
+
mongodb_util.save_mongo(code_uid_df, db_name_constant.SSE_INFO_UID)
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
# 同步所有互动问题
|
|
122
|
+
def sync_all_interactive_questions(symbol_list):
|
|
123
|
+
# 同步互动易映射
|
|
124
|
+
save_sh_stock_uid()
|
|
125
|
+
|
|
126
|
+
real_time_quotes_all_stocks = east_money_stock_api.get_real_time_quotes_all_stocks()
|
|
127
|
+
de_list_company_symbols = company_common_service_api.get_de_list_company()
|
|
128
|
+
real_time_quotes_all_stocks = real_time_quotes_all_stocks.loc[
|
|
129
|
+
~(real_time_quotes_all_stocks['symbol'].isin(de_list_company_symbols))]
|
|
130
|
+
real_time_quotes_all_stocks = common_service_fun_api.classify_symbol(real_time_quotes_all_stocks)
|
|
131
|
+
real_time_quotes_all_stocks = real_time_quotes_all_stocks.sort_values(by=['chg'], ascending=False)
|
|
132
|
+
if symbol_list is not None:
|
|
133
|
+
real_time_quotes_all_stocks = real_time_quotes_all_stocks.loc[
|
|
134
|
+
real_time_quotes_all_stocks['symbol'].isin(symbol_list)]
|
|
135
|
+
for stock_one in real_time_quotes_all_stocks.itertuples():
|
|
136
|
+
try:
|
|
137
|
+
now_date = datetime.now()
|
|
138
|
+
str_day = now_date.strftime('%Y-%m-%d')
|
|
139
|
+
str_now_date = now_date.strftime('%Y-%m-%d %H:%M:%S')
|
|
140
|
+
classification = stock_one.classification
|
|
141
|
+
|
|
142
|
+
if classification in ['S', 'C']:
|
|
143
|
+
stock_irm_cninfo_df = get_stock_irm_cninfo_sz_api(stock_one.symbol)
|
|
144
|
+
elif classification in ['K', 'H']:
|
|
145
|
+
stock_irm_cninfo_df = get_stock_irm_cninfo_sh_api(stock_one.symbol)
|
|
146
|
+
else:
|
|
147
|
+
continue
|
|
148
|
+
stock_irm_cninfo_df['sync_time'] = str_now_date
|
|
149
|
+
stock_irm_cninfo_df['str_day'] = str_day
|
|
150
|
+
stock_irm_cninfo_df.drop_duplicates('_id', keep='last', inplace=True)
|
|
151
|
+
stock_irm_cninfo_df.fillna("", inplace=True)
|
|
152
|
+
mongodb_util.save_mongo(stock_irm_cninfo_df, db_name_constant.STOCK_INTERACTIVE_QUESTION)
|
|
153
|
+
logger.info("完成同步互动回答到:{}", stock_one.symbol)
|
|
154
|
+
except Exception as e:
|
|
155
|
+
logger.error("同步互动问题出现异常:{},{}", stock_one.symbol, e)
|
|
156
|
+
|
|
157
|
+
|
|
41
158
|
if __name__ == '__main__':
|
|
42
|
-
|
|
159
|
+
fail_symbol_list = ['000638', '002886', '688778', '688766', '688733', '688778', '688793', '688787']
|
|
160
|
+
# get_stock_irm_cninfo_sh_api('603633')
|
|
161
|
+
sync_all_interactive_questions(None)
|
|
@@ -41,6 +41,7 @@ import mns_scheduler.concept.ths.common.ths_concept_update_common_api as ths_con
|
|
|
41
41
|
import mns_scheduler.trade.sync_position_api as sync_position_api
|
|
42
42
|
import mns_scheduler.concept.clean.kpl_concept_clean_api as kpl_concept_clean_api
|
|
43
43
|
import mns_scheduler.company_info.de_list_stock_service as de_list_stock_service
|
|
44
|
+
import mns_scheduler.irm.stock_irm_cninfo_service as stock_irm_cninfo_service
|
|
44
45
|
|
|
45
46
|
|
|
46
47
|
# 同步交易日期任务完成
|
|
@@ -322,6 +323,12 @@ def sync_new_high_risk_stocks():
|
|
|
322
323
|
register_and_investigate_stock_sync_api.sync_new_high_risk_stocks()
|
|
323
324
|
|
|
324
325
|
|
|
326
|
+
# 同步互动回答
|
|
327
|
+
def sync_all_interactive_questions():
|
|
328
|
+
logger.info('同步互动回答')
|
|
329
|
+
stock_irm_cninfo_service.sync_all_interactive_questions(None)
|
|
330
|
+
|
|
331
|
+
|
|
325
332
|
# # 定义BlockingScheduler
|
|
326
333
|
blockingScheduler = BlockingScheduler()
|
|
327
334
|
# sync_trade_date 同步交易日期
|
|
@@ -397,6 +404,9 @@ blockingScheduler.add_job(sync_position, 'cron', hour='0,08', minute='10')
|
|
|
397
404
|
# 同步被立案调查的股票
|
|
398
405
|
blockingScheduler.add_job(sync_new_high_risk_stocks, 'cron', hour='0,09,12,16', minute='20')
|
|
399
406
|
|
|
407
|
+
# 同步互动回答
|
|
408
|
+
blockingScheduler.add_job(sync_all_interactive_questions, 'cron', hour='08,12,17', minute='05')
|
|
409
|
+
|
|
400
410
|
print('定时任务启动成功')
|
|
401
411
|
blockingScheduler.start()
|
|
402
412
|
#
|
|
@@ -12,7 +12,7 @@ mns_scheduler/big_deal/__init__.py,sha256=QWBdZwBCvQw8aS4hnL9_pg3U3ZiNLUXzlImyy9
|
|
|
12
12
|
mns_scheduler/big_deal/ths_big_deal_sync.py,sha256=wezGJWFRnKYBaPP9PVXLqMbHENOXgvJtw3HSGCSlX5c,4555
|
|
13
13
|
mns_scheduler/company_info/__init__.py,sha256=QWBdZwBCvQw8aS4hnL9_pg3U3ZiNLUXzlImyy9WhUcI,163
|
|
14
14
|
mns_scheduler/company_info/company_constant_data.py,sha256=g1SSGgNT5QsS-zUNW_zrXD3pZXgSkyjsFLZgWeh7daU,14306
|
|
15
|
-
mns_scheduler/company_info/company_info_sync_api.py,sha256=
|
|
15
|
+
mns_scheduler/company_info/company_info_sync_api.py,sha256=8Qz-QrccEskHkqB7uKAocFd-6DNsoxIFFuNayukVojM,22727
|
|
16
16
|
mns_scheduler/company_info/de_list_stock_service.py,sha256=GCp6hlvO-SuH1oIpEsYZwEnGUOa6fXb2D7CqAUYXKQA,1993
|
|
17
17
|
mns_scheduler/concept/__init__.py,sha256=QWBdZwBCvQw8aS4hnL9_pg3U3ZiNLUXzlImyy9WhUcI,163
|
|
18
18
|
mns_scheduler/concept/clean/__init__.py,sha256=QWBdZwBCvQw8aS4hnL9_pg3U3ZiNLUXzlImyy9WhUcI,163
|
|
@@ -44,7 +44,10 @@ mns_scheduler/finance/test/fix_blask_list.py,sha256=sXmNcnfnjKHBylc27ysCCb9G4HYl
|
|
|
44
44
|
mns_scheduler/hk/__init__.py,sha256=wEg73KlZo-dU0yKGwpA1C2y6LZm4IBb94tNda1tqLeg,163
|
|
45
45
|
mns_scheduler/hk/hk_company_info_sync_service_api.py,sha256=JEfIl_up36b1UpDxfPhosP6_i2Lo17Ma_GHLytoMjWI,3661
|
|
46
46
|
mns_scheduler/irm/__init__.py,sha256=wEg73KlZo-dU0yKGwpA1C2y6LZm4IBb94tNda1tqLeg,163
|
|
47
|
-
mns_scheduler/irm/stock_irm_cninfo_service.py,sha256=
|
|
47
|
+
mns_scheduler/irm/stock_irm_cninfo_service.py,sha256=W2PrmpCdzEofuW7GobzGxzN01hP3MYM0uvKnRv_yvYE,7808
|
|
48
|
+
mns_scheduler/irm/api/__init__.py,sha256=wEg73KlZo-dU0yKGwpA1C2y6LZm4IBb94tNda1tqLeg,163
|
|
49
|
+
mns_scheduler/irm/api/sh_stock_sns_sse_info_api.py,sha256=iXy_FukyyQ_ZcU1WcT_DSpZ8-aEtDrTEcyiiIlFzXho,4931
|
|
50
|
+
mns_scheduler/irm/api/sz_stock_sns_sse_info_api.py,sha256=6iZpdQ1pARchL3kV-oEVPDc9Ja0ciQBjBntVmxseeNE,6199
|
|
48
51
|
mns_scheduler/k_line/__init__.py,sha256=ffZXFCLFdIwOsbxnw__u1MbQYh9yz7Bs8UMP6VF0X2M,161
|
|
49
52
|
mns_scheduler/k_line/clean/__init__.py,sha256=QWBdZwBCvQw8aS4hnL9_pg3U3ZiNLUXzlImyy9WhUcI,163
|
|
50
53
|
mns_scheduler/k_line/clean/k_line_info_clean_impl.py,sha256=2xk7gzLX728PRTKUgx2sXoyPKDu-xGAvNpefgeBBd2w,22262
|
|
@@ -82,9 +85,9 @@ mns_scheduler/zt/today_high_chg_pool_sync_api.py,sha256=KfEp2Mu8dZKB8DBai4m0UpZw
|
|
|
82
85
|
mns_scheduler/zt/zt_five_boards_sync_api.py,sha256=HfjPHKD99fU9c37kSenEX2_qNvFAjQGgy8ERuacSxwk,10916
|
|
83
86
|
mns_scheduler/zt/zt_pool_sync_api.py,sha256=RrVAbU1u-HTqXF9BSwNlzIxMHrUgjNaLpDKFgXC1XuY,7604
|
|
84
87
|
mns_scheduler/zz_task/__init__.py,sha256=QWBdZwBCvQw8aS4hnL9_pg3U3ZiNLUXzlImyy9WhUcI,163
|
|
85
|
-
mns_scheduler/zz_task/data_sync_task.py,sha256=
|
|
88
|
+
mns_scheduler/zz_task/data_sync_task.py,sha256=oC8J9dk1bk-WOskgFt0nnh5zGpCNdhkij4O9Gpi2qbU,16633
|
|
86
89
|
mns_scheduler/zz_task/sync_realtime_quotes_task.py,sha256=DN3bq2XCDZC-PHlbD2NTog48bR44EruIEc2QVGKg7Tk,932
|
|
87
|
-
mns_scheduler-1.0.5.
|
|
88
|
-
mns_scheduler-1.0.5.
|
|
89
|
-
mns_scheduler-1.0.5.
|
|
90
|
-
mns_scheduler-1.0.5.
|
|
90
|
+
mns_scheduler-1.0.5.8.dist-info/METADATA,sha256=iABzQ9-jM6Ns-qWu7xlGWKo4Mh8bZuvVlWVvw7ZADA8,64
|
|
91
|
+
mns_scheduler-1.0.5.8.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
|
|
92
|
+
mns_scheduler-1.0.5.8.dist-info/top_level.txt,sha256=PXQDFBGR1pWmsUbH5yiLAh71P5HZODTRED0zJ8CCgOc,14
|
|
93
|
+
mns_scheduler-1.0.5.8.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|