mns-scheduler 1.3.1.2__py3-none-any.whl → 1.3.1.6__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of mns-scheduler might be problematic. Click here for more details.
- mns_scheduler/baidu/__init__.py +1 -1
- mns_scheduler/baidu/baidu_yun_pan_handle_service.py +2 -2
- mns_scheduler/company_info/announce/__init__.py +1 -1
- mns_scheduler/company_info/announce/company_announce_sync_service.py +1 -1
- mns_scheduler/company_info/base/__init__.py +1 -1
- mns_scheduler/company_info/base/sync_company_hold_info_api.py +1 -1
- mns_scheduler/company_info/clean/__init__.py +1 -1
- mns_scheduler/company_info/clean/company_info_clean_api.py +1 -1
- mns_scheduler/company_info/constant/__init__.py +1 -1
- mns_scheduler/company_info/de_list_stock/__init__.py +1 -1
- mns_scheduler/company_info/de_list_stock/de_list_stock_service.py +1 -1
- mns_scheduler/company_info/em_stock_info/__init__.py +1 -1
- mns_scheduler/company_info/em_stock_info/sync_em_stock_info_sync.py +8 -3
- mns_scheduler/company_info/remark/__init__.py +1 -1
- mns_scheduler/company_info/remark/company_remark_info_sync.py +1 -1
- mns_scheduler/concept/clean/kpl_concept_clean_api.py +1 -1
- mns_scheduler/db/script/__init__.py +1 -1
- mns_scheduler/db/script/col_move_script.py +1 -1
- mns_scheduler/db/script/db_move/__init__.py +1 -1
- mns_scheduler/db/script/db_move/col_move_one_service.py +1 -1
- mns_scheduler/db/script/sync/__init__.py +1 -1
- mns_scheduler/db/script/sync/remote_data_sync_to_local.py +1 -1
- mns_scheduler/db/script/sync/sync_hui_ce_test_data.py +1 -1
- mns_scheduler/db/script/sync/sync_hui_ce_test_data_01.py +1 -1
- mns_scheduler/db/script/update/__init__.py +1 -1
- mns_scheduler/db/script/update/update_col_field.py +1 -1
- mns_scheduler/debt/__init__.py +1 -1
- mns_scheduler/debt/kzz_bond_info_sync.py +1 -1
- mns_scheduler/extraIncome/__init__.py +1 -1
- mns_scheduler/extraIncome/a_stock/__init__.py +1 -1
- mns_scheduler/extraIncome/a_stock/one_minute/__init__.py +1 -1
- mns_scheduler/extraIncome/a_stock/one_minute/common/__init__.py +1 -1
- mns_scheduler/extraIncome/a_stock/one_minute/common/symbol_handle_util.py +1 -1
- mns_scheduler/extraIncome/a_stock/one_minute/etf/__init__.py +1 -1
- mns_scheduler/extraIncome/a_stock/one_minute/etf/etf_one_minute_sync_task.py +1 -1
- mns_scheduler/extraIncome/a_stock/one_minute/index/__init__.py +1 -1
- mns_scheduler/extraIncome/a_stock/one_minute/index/main_index_sync_task.py +1 -1
- mns_scheduler/extraIncome/a_stock/one_minute/kzz/__init__.py +1 -1
- mns_scheduler/extraIncome/a_stock/one_minute/kzz/kzz_one_minute_sync_task.py +1 -1
- mns_scheduler/extraIncome/a_stock/one_minute/one_minute_sync_task.py +1 -1
- mns_scheduler/extraIncome/a_stock/one_minute/stock/__init__.py +1 -1
- mns_scheduler/extraIncome/a_stock/one_minute/stock/stock_one_minute_sync_task.py +1 -1
- mns_scheduler/extraIncome/a_stock/one_minute/upload/__init__.py +1 -1
- mns_scheduler/extraIncome/a_stock/one_minute/upload/etf_upload_to_baidu_task.py +95 -0
- mns_scheduler/extraIncome/a_stock/one_minute/upload/index_upload_to_baidu_task.py +87 -0
- mns_scheduler/extraIncome/a_stock/one_minute/upload/kzz_upload_to_baidu_task.py +95 -0
- mns_scheduler/extraIncome/a_stock/one_minute/upload/{upload_to_baidu_task.py → stock_upload_to_baidu_task.py} +4 -2
- mns_scheduler/extraIncome/hk/__init__.py +1 -1
- mns_scheduler/extraIncome/hk/hk_stock_qfq_daily_k_line.py +1 -1
- mns_scheduler/extraIncome/us/__init__.py +1 -1
- mns_scheduler/extraIncome/us/daily/__init__.py +1 -1
- mns_scheduler/extraIncome/us/daily/us_stock_qfq_daily_k_line.py +1 -1
- mns_scheduler/extraIncome/us/one_minute/__init__.py +1 -1
- mns_scheduler/extraIncome/us/one_minute/api/__init__.py +0 -7
- mns_scheduler/extraIncome/us/one_minute/api/alpha_vantage_api.py +2 -2
- mns_scheduler/extraIncome/us/one_minute/api/em_us_one_minute_api.py +139 -0
- mns_scheduler/extraIncome/us/one_minute/api/stock_etf_info_api.py +68 -0
- mns_scheduler/extraIncome/us/one_minute/api/y_finance_api.py +25 -9
- mns_scheduler/extraIncome/us/one_minute/etf/__init__.py +1 -1
- mns_scheduler/extraIncome/us/one_minute/stock/__init__.py +1 -1
- mns_scheduler/extraIncome/us/one_minute/stock/down_load/__init__.py +1 -1
- mns_scheduler/extraIncome/us/one_minute/stock/down_load/etf/__init__.py +5 -1
- mns_scheduler/extraIncome/us/one_minute/stock/down_load/etf/down_load_ETF_his_2024.py +1 -1
- mns_scheduler/extraIncome/us/one_minute/stock/down_load/etf/handle_down_load_fail_ETF.py +1 -1
- mns_scheduler/extraIncome/us/one_minute/stock/{his → down_load/stock}/__init__.py +1 -1
- mns_scheduler/extraIncome/us/one_minute/stock/down_load/stock/down_load_stock_his_01.py +203 -0
- mns_scheduler/extraIncome/us/one_minute/stock/down_load/{down_load_stock_his_2024_02.py → stock/down_load_stock_his_02.py} +65 -16
- mns_scheduler/extraIncome/us/one_minute/stock/down_load/{down_load_stock_his_2024_01.py → stock/down_load_stock_his_2025.py} +10 -11
- mns_scheduler/extraIncome/us/one_minute/stock/now/__init__.py +7 -0
- mns_scheduler/extraIncome/us/one_minute/stock/now/us_etf_one_minute_sync.py +52 -0
- mns_scheduler/extraIncome/us/one_minute/stock/now/us_stock_one_minute_sync.py +141 -0
- mns_scheduler/finance/__init__.py +1 -1
- mns_scheduler/finance/em_financial_asset_liability_sync_service_api.py +1 -1
- mns_scheduler/finance/em_financial_profit_sync_service_api.py +1 -1
- mns_scheduler/finance/finance_common_api.py +1 -1
- mns_scheduler/finance/sync_financial_report_service_api.py +1 -1
- mns_scheduler/hk/__init__.py +1 -1
- mns_scheduler/hk/hk_company_info_sync_service_api.py +1 -1
- mns_scheduler/hk/hk_industry_info_sync_service_api.py +1 -1
- mns_scheduler/industry/__init__.py +1 -1
- mns_scheduler/industry/ths/__init__.py +1 -1
- mns_scheduler/industry/ths/ths_industry_index_service.py +1 -1
- mns_scheduler/industry/ths/ths_industry_sync_service.py +1 -1
- mns_scheduler/irm/__init__.py +1 -1
- mns_scheduler/irm/api/__init__.py +1 -1
- mns_scheduler/irm/api/sh_stock_sns_sse_info_api.py +1 -1
- mns_scheduler/irm/api/sz_stock_sns_sse_info_api.py +1 -1
- mns_scheduler/irm/stock_irm_cninfo_service.py +1 -1
- mns_scheduler/k_line/clean/daily/__init__.py +1 -1
- mns_scheduler/k_line/clean/daily/daily_k_line_clean_common_service.py +1 -1
- mns_scheduler/k_line/clean/daily/daily_k_line_service.py +1 -1
- mns_scheduler/k_line/clean/week_month/__init__.py +1 -1
- mns_scheduler/k_line/clean/week_month/normal_week_month_k_line_service.py +1 -1
- mns_scheduler/k_line/clean/week_month/sub_new_week_month_k_line_service.py +1 -1
- mns_scheduler/k_line/hot_stocks/__init__.py +1 -1
- mns_scheduler/k_line/hot_stocks/recent_hot_stocks_clean_service.py +1 -1
- mns_scheduler/k_line/test/__init__.py +1 -1
- mns_scheduler/k_line/test/k_line_info_clean_his_data.py +1 -1
- mns_scheduler/k_line/year_quarter/__init__.py +1 -1
- mns_scheduler/k_line/year_quarter/year_quarter_line_sync.py +1 -1
- mns_scheduler/lhb/__init__.py +1 -1
- mns_scheduler/lhb/stock_lhb_sync_service.py +1 -1
- mns_scheduler/open/__init__.py +1 -1
- mns_scheduler/open/sync_one_day_open_data_to_db_service.py +1 -1
- mns_scheduler/risk/__init__.py +1 -1
- mns_scheduler/risk/financial/annual_report_audit_check_api.py +1 -1
- mns_scheduler/risk/financial/profit_income_check_api.py +1 -1
- mns_scheduler/risk/financial_report_risk_check_api.py +1 -1
- mns_scheduler/risk/major_violations/register_and_investigate_stock_sync_api.py +1 -1
- mns_scheduler/risk/self/wei_pan_stock_api.py +1 -1
- mns_scheduler/risk/test/__init__.py +1 -1
- mns_scheduler/risk/test/fix_blask_list.py +1 -1
- mns_scheduler/risk/transactions/transactions_check_api.py +1 -1
- mns_scheduler/self_choose/__init__.py +1 -1
- mns_scheduler/self_choose/ths_self_choose_service.py +1 -1
- mns_scheduler/trade/auto_login/trader_auto_service.py +1 -1
- mns_scheduler/trade/auto_sell_service_api.py +2 -2
- mns_scheduler/trade/balance/__init__.py +1 -1
- mns_scheduler/trade/balance/ths_account_balance_service.py +1 -1
- mns_scheduler/trade/sync_position_api.py +1 -1
- mns_scheduler/trade/task/trader_task_service.py +1 -1
- mns_scheduler/trade/tfp/__init__.py +1 -1
- mns_scheduler/trade/tfp/stock_tfp_info_sync.py +1 -1
- mns_scheduler/us/__init__.py +1 -1
- mns_scheduler/us/baidu_yun_pan_handle_service.py +1 -1
- mns_scheduler/us/us_company_info_sync_service_api.py +1 -1
- mns_scheduler/zt/script/__init__.py +1 -1
- mns_scheduler/zt/script/fix_error_deal_day.py +1 -1
- mns_scheduler/zt/script/kcx_high_chg_open_his_data_handle.py +1 -1
- mns_scheduler/zt/script/sync_high_chg_pool_his_data.py +1 -1
- mns_scheduler/zt/script/sync_now_higt_chg_zt.py +1 -1
- mns_scheduler/zt/zt_pool/em_zt_pool_sync_api.py +1 -1
- mns_scheduler/zt/zt_pool/update_null_zt_reason_api.py +1 -1
- mns_scheduler/zz_task/data_sync_task.py +13 -3
- {mns_scheduler-1.3.1.2.dist-info → mns_scheduler-1.3.1.6.dist-info}/METADATA +1 -1
- mns_scheduler-1.3.1.6.dist-info/RECORD +210 -0
- mns_scheduler/extraIncome/us/one_minute/stock/his/us_stock_one_minute_his.py +0 -199
- mns_scheduler/extraIncome/us/one_minute/stock/his/us_stock_one_minute_his_2024.py +0 -212
- mns_scheduler/extraIncome/us/one_minute/stock/us_stock_one_minute_task.py +0 -26
- mns_scheduler-1.3.1.2.dist-info/RECORD +0 -204
- {mns_scheduler-1.3.1.2.dist-info → mns_scheduler-1.3.1.6.dist-info}/WHEEL +0 -0
- {mns_scheduler-1.3.1.2.dist-info → mns_scheduler-1.3.1.6.dist-info}/top_level.txt +0 -0
|
@@ -0,0 +1,203 @@
|
|
|
1
|
+
import sys
|
|
2
|
+
import os
|
|
3
|
+
|
|
4
|
+
file_path = os.path.abspath(__file__)
|
|
5
|
+
end = file_path.index('mns') + 17
|
|
6
|
+
project_path = file_path[0:end]
|
|
7
|
+
sys.path.append(project_path)
|
|
8
|
+
from loguru import logger
|
|
9
|
+
import time
|
|
10
|
+
import mns_common.utils.data_frame_util as data_frame_util
|
|
11
|
+
from mns_common.db.MongodbUtil import MongodbUtil
|
|
12
|
+
from mns_common.db.v2.MongodbUtilV2 import MongodbUtilV2
|
|
13
|
+
import mns_common.constant.extra_income_db_name as extra_income_db_name
|
|
14
|
+
import mns_scheduler.extraIncome.us.one_minute.api.alpha_vantage_api as alpha_vantage_api
|
|
15
|
+
import pandas as pd
|
|
16
|
+
from pathlib import Path
|
|
17
|
+
|
|
18
|
+
mongodb_util_27017 = MongodbUtil('27017')
|
|
19
|
+
mongodbUtilV2_27019 = MongodbUtilV2('27019', extra_income_db_name.EXTRA_INCOME)
|
|
20
|
+
from datetime import datetime
|
|
21
|
+
import mns_scheduler.extraIncome.us.one_minute.api.stock_etf_info_api as stock_etf_info_api
|
|
22
|
+
|
|
23
|
+
import threading
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
# 定义一个带超时的函数调用
|
|
27
|
+
def call_with_timeout(func, *args, timeout=60, **kwargs):
|
|
28
|
+
# 用于存储函数执行结果
|
|
29
|
+
result = None
|
|
30
|
+
exception = None
|
|
31
|
+
|
|
32
|
+
# 定义一个线程目标函数
|
|
33
|
+
def target():
|
|
34
|
+
nonlocal result, exception
|
|
35
|
+
try:
|
|
36
|
+
result = func(*args, **kwargs)
|
|
37
|
+
except Exception as e:
|
|
38
|
+
exception = e
|
|
39
|
+
|
|
40
|
+
# 创建线程并启动
|
|
41
|
+
thread = threading.Thread(target=target)
|
|
42
|
+
thread.start()
|
|
43
|
+
|
|
44
|
+
# 等待线程完成,最多等待 timeout 秒
|
|
45
|
+
thread.join(timeout)
|
|
46
|
+
|
|
47
|
+
# 如果线程仍然存活,说明函数超时了
|
|
48
|
+
if thread.is_alive():
|
|
49
|
+
raise TimeoutError(f"Function exceeded timeout of {timeout} seconds")
|
|
50
|
+
|
|
51
|
+
# 如果函数抛出了异常,重新抛出
|
|
52
|
+
if exception is not None:
|
|
53
|
+
raise exception
|
|
54
|
+
return result
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def sync_one_minute_data(symbol, now_month):
|
|
58
|
+
df = alpha_vantage_api.sync_one_minute_data(symbol, now_month)
|
|
59
|
+
return df
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def sync_us_stock_one_minute(now_year, now_month):
|
|
63
|
+
real_time_quotes_all_us_stocks = stock_etf_info_api.get_us_stock_info()
|
|
64
|
+
|
|
65
|
+
path = r'F:\us_stock\one_minute\{}'.format(now_year)
|
|
66
|
+
if not os.path.exists(path):
|
|
67
|
+
os.makedirs(path)
|
|
68
|
+
|
|
69
|
+
path = path + '\{}'.format(now_month)
|
|
70
|
+
if not os.path.exists(path):
|
|
71
|
+
os.makedirs(path)
|
|
72
|
+
stock_name_list = find_exist_file(path)
|
|
73
|
+
real_time_quotes_all_us_stocks = real_time_quotes_all_us_stocks.loc[
|
|
74
|
+
~(real_time_quotes_all_us_stocks['symbol'].isin(stock_name_list))]
|
|
75
|
+
real_time_quotes_all_us_stocks.dropna(subset=['list_date'], inplace=True)
|
|
76
|
+
for stock_one in real_time_quotes_all_us_stocks.itertuples():
|
|
77
|
+
flow_mv = stock_one.flow_mv
|
|
78
|
+
symbol = stock_one.symbo
|
|
79
|
+
try:
|
|
80
|
+
symbol = stock_one.symbol
|
|
81
|
+
# simple_symbol = int(stock_one.simple_symbol)
|
|
82
|
+
# code = str(simple_symbol) + '.' + symbol
|
|
83
|
+
list_date = str(stock_one.list_date)
|
|
84
|
+
list_date_year = int(list_date[0:4])
|
|
85
|
+
list_month = int(list_date[4:6])
|
|
86
|
+
now_month_int = int(now_month[5:7])
|
|
87
|
+
if (list_date_year > now_year) or ((list_date_year == now_year) and (list_month > now_month_int)):
|
|
88
|
+
continue
|
|
89
|
+
now_date = datetime.now()
|
|
90
|
+
if net_work_check(now_date):
|
|
91
|
+
# 休眠 6分钟
|
|
92
|
+
time.sleep(5 * 60)
|
|
93
|
+
|
|
94
|
+
df = call_with_timeout(sync_one_minute_data,
|
|
95
|
+
symbol,
|
|
96
|
+
now_month,
|
|
97
|
+
timeout=60)
|
|
98
|
+
df['time'] = df['time'].dt.strftime('%Y-%m-%d %H:%M:%S')
|
|
99
|
+
df['str_day'] = df['time'].str.slice(0, 10)
|
|
100
|
+
df['minute'] = df['time'].str.slice(11, 19)
|
|
101
|
+
df['_id'] = symbol + "_" + df['time']
|
|
102
|
+
df['symbol'] = symbol
|
|
103
|
+
df_export_df = df.copy()
|
|
104
|
+
export_original_data(df_export_df, symbol, now_year, now_month)
|
|
105
|
+
except BaseException as e:
|
|
106
|
+
time.sleep(1)
|
|
107
|
+
fail_dict = {
|
|
108
|
+
'_id': symbol + '_' + now_month,
|
|
109
|
+
'type': "stock",
|
|
110
|
+
'path': path,
|
|
111
|
+
'symbol': symbol,
|
|
112
|
+
'now_year': now_year,
|
|
113
|
+
'now_month': now_month,
|
|
114
|
+
'flow_mv': flow_mv,
|
|
115
|
+
}
|
|
116
|
+
fail_df = pd.DataFrame(fail_dict, index=[1])
|
|
117
|
+
|
|
118
|
+
mongodb_util_27017.save_mongo(fail_df, 'us_stock_one_minute_down_load_fail')
|
|
119
|
+
logger.error("同步股票分钟数据出现异常:,{},{},{}", e, symbol, now_month)
|
|
120
|
+
logger.info("同步股票分钟票数据完成:{},{}", stock_one.symbol, stock_one.name)
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
def export_original_data(df, symbol, year, now_month):
|
|
124
|
+
path = r'F:\us_stock\one_minute\{}'.format(year)
|
|
125
|
+
if not os.path.exists(path):
|
|
126
|
+
os.makedirs(path)
|
|
127
|
+
|
|
128
|
+
path = path + '\{}'.format(now_month)
|
|
129
|
+
if not os.path.exists(path):
|
|
130
|
+
os.makedirs(path)
|
|
131
|
+
|
|
132
|
+
file_name = path + '\{}.csv'.format(symbol)
|
|
133
|
+
if data_frame_util.is_not_empty(df):
|
|
134
|
+
df = df.dropna(subset=['_id'])
|
|
135
|
+
del df['str_day']
|
|
136
|
+
del df['minute']
|
|
137
|
+
del df['_id']
|
|
138
|
+
del df['symbol']
|
|
139
|
+
df.to_csv(file_name, index=False, encoding='utf-8')
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
def net_work_check(now_date):
|
|
143
|
+
hour = now_date.hour
|
|
144
|
+
minute = now_date.minute
|
|
145
|
+
if hour == 7 and minute == 34:
|
|
146
|
+
return True
|
|
147
|
+
elif hour == 9 and minute == 59:
|
|
148
|
+
return True
|
|
149
|
+
elif hour == 10 and minute == 29:
|
|
150
|
+
return True
|
|
151
|
+
elif hour == 10 and minute == 59:
|
|
152
|
+
return True
|
|
153
|
+
elif hour == 12 and minute == 49:
|
|
154
|
+
return True
|
|
155
|
+
elif hour == 13 and minute == 28:
|
|
156
|
+
return True
|
|
157
|
+
elif hour == 13 and minute == 58:
|
|
158
|
+
return True
|
|
159
|
+
elif hour == 14 and minute == 28:
|
|
160
|
+
return True
|
|
161
|
+
elif hour == 15 and minute == 1:
|
|
162
|
+
return True
|
|
163
|
+
else:
|
|
164
|
+
return False
|
|
165
|
+
|
|
166
|
+
|
|
167
|
+
def sync_by_year(begin_year):
|
|
168
|
+
begin_month = 10
|
|
169
|
+
while begin_month > 0:
|
|
170
|
+
if begin_month < 10:
|
|
171
|
+
str_month = '0' + str(begin_month)
|
|
172
|
+
else:
|
|
173
|
+
str_month = str(begin_month)
|
|
174
|
+
str_month = str(begin_year) + '-' + str_month
|
|
175
|
+
sync_us_stock_one_minute(begin_year, str_month)
|
|
176
|
+
begin_month = begin_month - 1
|
|
177
|
+
logger.error("同步完成月份:{}", str_month)
|
|
178
|
+
|
|
179
|
+
|
|
180
|
+
def find_exist_file(folder_path):
|
|
181
|
+
if not os.path.exists(folder_path):
|
|
182
|
+
logger.error("错误:目录不存在:{}", folder_path)
|
|
183
|
+
else:
|
|
184
|
+
folder_path = Path(folder_path)
|
|
185
|
+
stock_names = [f.stem for f in folder_path.glob("*.csv")]
|
|
186
|
+
return stock_names
|
|
187
|
+
|
|
188
|
+
|
|
189
|
+
if __name__ == '__main__':
|
|
190
|
+
# k_line_df = query_k_line('TSLA')
|
|
191
|
+
# sync_by_year(2024)
|
|
192
|
+
# sync_by_year(2022)
|
|
193
|
+
sync_by_year(2020)
|
|
194
|
+
sync_by_year(2018)
|
|
195
|
+
sync_by_year(2016)
|
|
196
|
+
sync_by_year(2014)
|
|
197
|
+
sync_by_year(2012)
|
|
198
|
+
sync_by_year(2010)
|
|
199
|
+
sync_by_year(2008)
|
|
200
|
+
sync_by_year(2006)
|
|
201
|
+
sync_by_year(2004)
|
|
202
|
+
sync_by_year(2002)
|
|
203
|
+
sync_by_year(2000)
|
|
@@ -2,31 +2,65 @@ import sys
|
|
|
2
2
|
import os
|
|
3
3
|
|
|
4
4
|
file_path = os.path.abspath(__file__)
|
|
5
|
-
end = file_path.index('mns') +
|
|
5
|
+
end = file_path.index('mns') + 17
|
|
6
6
|
project_path = file_path[0:end]
|
|
7
7
|
sys.path.append(project_path)
|
|
8
|
-
import mns_common.component.em.em_stock_info_api as em_stock_info_api
|
|
9
8
|
from loguru import logger
|
|
10
9
|
import time
|
|
11
10
|
import mns_common.utils.data_frame_util as data_frame_util
|
|
12
11
|
from mns_common.db.MongodbUtil import MongodbUtil
|
|
13
12
|
from mns_common.db.v2.MongodbUtilV2 import MongodbUtilV2
|
|
14
|
-
import mns_scheduler.extraIncome.a_stock.one_minute.common.db_create_index as db_create_index
|
|
15
13
|
import mns_common.constant.extra_income_db_name as extra_income_db_name
|
|
16
14
|
import mns_scheduler.extraIncome.us.one_minute.api.alpha_vantage_api as alpha_vantage_api
|
|
17
15
|
import pandas as pd
|
|
18
|
-
from functools import lru_cache
|
|
19
|
-
import glob
|
|
20
16
|
from pathlib import Path
|
|
17
|
+
|
|
21
18
|
mongodb_util_27017 = MongodbUtil('27017')
|
|
22
19
|
mongodbUtilV2_27019 = MongodbUtilV2('27019', extra_income_db_name.EXTRA_INCOME)
|
|
23
20
|
from datetime import datetime
|
|
21
|
+
import mns_scheduler.extraIncome.us.one_minute.api.stock_etf_info_api as stock_etf_info_api
|
|
22
|
+
|
|
23
|
+
import threading
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
# 定义一个带超时的函数调用
|
|
27
|
+
def call_with_timeout(func, *args, timeout=60, **kwargs):
|
|
28
|
+
# 用于存储函数执行结果
|
|
29
|
+
result = None
|
|
30
|
+
exception = None
|
|
31
|
+
|
|
32
|
+
# 定义一个线程目标函数
|
|
33
|
+
def target():
|
|
34
|
+
nonlocal result, exception
|
|
35
|
+
try:
|
|
36
|
+
result = func(*args, **kwargs)
|
|
37
|
+
except Exception as e:
|
|
38
|
+
exception = e
|
|
39
|
+
|
|
40
|
+
# 创建线程并启动
|
|
41
|
+
thread = threading.Thread(target=target)
|
|
42
|
+
thread.start()
|
|
43
|
+
|
|
44
|
+
# 等待线程完成,最多等待 timeout 秒
|
|
45
|
+
thread.join(timeout)
|
|
46
|
+
|
|
47
|
+
# 如果线程仍然存活,说明函数超时了
|
|
48
|
+
if thread.is_alive():
|
|
49
|
+
raise TimeoutError(f"Function exceeded timeout of {timeout} seconds")
|
|
50
|
+
|
|
51
|
+
# 如果函数抛出了异常,重新抛出
|
|
52
|
+
if exception is not None:
|
|
53
|
+
raise exception
|
|
54
|
+
return result
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def sync_one_minute_data(symbol, now_month):
|
|
58
|
+
df = alpha_vantage_api.sync_one_minute_data(symbol, now_month)
|
|
59
|
+
return df
|
|
24
60
|
|
|
25
61
|
|
|
26
62
|
def sync_us_stock_one_minute(now_year, now_month):
|
|
27
|
-
|
|
28
|
-
real_time_quotes_all_us_stocks = real_time_quotes_all_us.loc[real_time_quotes_all_us['flow_mv'] != 0]
|
|
29
|
-
real_time_quotes_all_us_stocks = real_time_quotes_all_us_stocks.sort_values(by=['amount'], ascending=False)
|
|
63
|
+
real_time_quotes_all_us_stocks = stock_etf_info_api.get_us_stock_info()
|
|
30
64
|
|
|
31
65
|
path = r'F:\us_stock\one_minute\{}'.format(now_year)
|
|
32
66
|
if not os.path.exists(path):
|
|
@@ -40,9 +74,10 @@ def sync_us_stock_one_minute(now_year, now_month):
|
|
|
40
74
|
~(real_time_quotes_all_us_stocks['symbol'].isin(stock_name_list))]
|
|
41
75
|
real_time_quotes_all_us_stocks.dropna(subset=['list_date'], inplace=True)
|
|
42
76
|
for stock_one in real_time_quotes_all_us_stocks.itertuples():
|
|
43
|
-
|
|
77
|
+
flow_mv = stock_one.flow_mv
|
|
78
|
+
symbol = stock_one.symbo
|
|
44
79
|
try:
|
|
45
|
-
|
|
80
|
+
|
|
46
81
|
# simple_symbol = int(stock_one.simple_symbol)
|
|
47
82
|
# code = str(simple_symbol) + '.' + symbol
|
|
48
83
|
list_date = str(stock_one.list_date)
|
|
@@ -56,7 +91,10 @@ def sync_us_stock_one_minute(now_year, now_month):
|
|
|
56
91
|
# 休眠 6分钟
|
|
57
92
|
time.sleep(5 * 60)
|
|
58
93
|
|
|
59
|
-
df =
|
|
94
|
+
df = call_with_timeout(sync_one_minute_data,
|
|
95
|
+
symbol,
|
|
96
|
+
now_month,
|
|
97
|
+
timeout=60)
|
|
60
98
|
df['time'] = df['time'].dt.strftime('%Y-%m-%d %H:%M:%S')
|
|
61
99
|
df['str_day'] = df['time'].str.slice(0, 10)
|
|
62
100
|
df['minute'] = df['time'].str.slice(11, 19)
|
|
@@ -72,7 +110,8 @@ def sync_us_stock_one_minute(now_year, now_month):
|
|
|
72
110
|
'path': path,
|
|
73
111
|
'symbol': symbol,
|
|
74
112
|
'now_year': now_year,
|
|
75
|
-
'now_month': now_month
|
|
113
|
+
'now_month': now_month,
|
|
114
|
+
'flow_mv': flow_mv,
|
|
76
115
|
}
|
|
77
116
|
fail_df = pd.DataFrame(fail_dict, index=[1])
|
|
78
117
|
|
|
@@ -126,8 +165,8 @@ def net_work_check(now_date):
|
|
|
126
165
|
|
|
127
166
|
|
|
128
167
|
def sync_by_year(begin_year):
|
|
129
|
-
begin_month =
|
|
130
|
-
while begin_month >
|
|
168
|
+
begin_month = 10
|
|
169
|
+
while begin_month > 0:
|
|
131
170
|
if begin_month < 10:
|
|
132
171
|
str_month = '0' + str(begin_month)
|
|
133
172
|
else:
|
|
@@ -147,7 +186,17 @@ def find_exist_file(folder_path):
|
|
|
147
186
|
return stock_names
|
|
148
187
|
|
|
149
188
|
|
|
150
|
-
|
|
151
189
|
if __name__ == '__main__':
|
|
152
190
|
# k_line_df = query_k_line('TSLA')
|
|
153
|
-
sync_by_year(
|
|
191
|
+
# sync_by_year(2023)
|
|
192
|
+
sync_by_year(2021)
|
|
193
|
+
sync_by_year(2019)
|
|
194
|
+
sync_by_year(2017)
|
|
195
|
+
sync_by_year(2015)
|
|
196
|
+
sync_by_year(2013)
|
|
197
|
+
sync_by_year(2011)
|
|
198
|
+
sync_by_year(2009)
|
|
199
|
+
sync_by_year(2007)
|
|
200
|
+
sync_by_year(2005)
|
|
201
|
+
sync_by_year(2003)
|
|
202
|
+
sync_by_year(2001)
|
|
@@ -2,21 +2,19 @@ import sys
|
|
|
2
2
|
import os
|
|
3
3
|
|
|
4
4
|
file_path = os.path.abspath(__file__)
|
|
5
|
-
end = file_path.index('mns') +
|
|
5
|
+
end = file_path.index('mns') + 17
|
|
6
6
|
project_path = file_path[0:end]
|
|
7
7
|
sys.path.append(project_path)
|
|
8
|
-
import mns_common.component.em.em_stock_info_api as em_stock_info_api
|
|
9
8
|
from loguru import logger
|
|
10
9
|
import time
|
|
11
10
|
import mns_common.utils.data_frame_util as data_frame_util
|
|
12
11
|
from mns_common.db.MongodbUtil import MongodbUtil
|
|
13
12
|
from mns_common.db.v2.MongodbUtilV2 import MongodbUtilV2
|
|
14
|
-
import mns_scheduler.extraIncome.a_stock.one_minute.common.db_create_index as db_create_index
|
|
15
13
|
import mns_common.constant.extra_income_db_name as extra_income_db_name
|
|
16
14
|
import mns_scheduler.extraIncome.us.one_minute.api.alpha_vantage_api as alpha_vantage_api
|
|
17
15
|
import pandas as pd
|
|
18
16
|
from pathlib import Path
|
|
19
|
-
|
|
17
|
+
import mns_scheduler.extraIncome.us.one_minute.api.stock_etf_info_api as stock_etf_info
|
|
20
18
|
|
|
21
19
|
mongodb_util_27017 = MongodbUtil('27017')
|
|
22
20
|
mongodbUtilV2_27019 = MongodbUtilV2('27019', extra_income_db_name.EXTRA_INCOME)
|
|
@@ -24,8 +22,7 @@ from datetime import datetime
|
|
|
24
22
|
|
|
25
23
|
|
|
26
24
|
def sync_us_stock_one_minute(now_year, now_month):
|
|
27
|
-
|
|
28
|
-
real_time_quotes_all_us_stocks = real_time_quotes_all_us.loc[real_time_quotes_all_us['flow_mv'] != 0]
|
|
25
|
+
real_time_quotes_all_us_stocks = stock_etf_info.get_us_stock_info()
|
|
29
26
|
real_time_quotes_all_us_stocks = real_time_quotes_all_us_stocks.sort_values(by=['amount'], ascending=False)
|
|
30
27
|
|
|
31
28
|
path = r'F:\us_stock\one_minute\{}'.format(now_year)
|
|
@@ -40,10 +37,11 @@ def sync_us_stock_one_minute(now_year, now_month):
|
|
|
40
37
|
~(real_time_quotes_all_us_stocks['symbol'].isin(stock_name_list))]
|
|
41
38
|
real_time_quotes_all_us_stocks.dropna(subset=['list_date'], inplace=True)
|
|
42
39
|
for stock_one in real_time_quotes_all_us_stocks.itertuples():
|
|
43
|
-
|
|
40
|
+
symbol = stock_one.symbol
|
|
41
|
+
flow_mv = stock_one.flow_mv
|
|
44
42
|
try:
|
|
45
43
|
|
|
46
|
-
|
|
44
|
+
|
|
47
45
|
# simple_symbol = int(stock_one.simple_symbol)
|
|
48
46
|
# code = str(simple_symbol) + '.' + symbol
|
|
49
47
|
list_date = str(stock_one.list_date)
|
|
@@ -73,7 +71,8 @@ def sync_us_stock_one_minute(now_year, now_month):
|
|
|
73
71
|
'path': path,
|
|
74
72
|
'symbol': symbol,
|
|
75
73
|
'now_year': now_year,
|
|
76
|
-
'now_month': now_month
|
|
74
|
+
'now_month': now_month,
|
|
75
|
+
'flow_mv': flow_mv,
|
|
77
76
|
}
|
|
78
77
|
fail_df = pd.DataFrame(fail_dict, index=[1])
|
|
79
78
|
|
|
@@ -119,7 +118,7 @@ def net_work_check(now_date):
|
|
|
119
118
|
|
|
120
119
|
|
|
121
120
|
def sync_by_year(begin_year):
|
|
122
|
-
begin_month =
|
|
121
|
+
begin_month = 4
|
|
123
122
|
while begin_month > 0:
|
|
124
123
|
if begin_month < 10:
|
|
125
124
|
str_month = '0' + str(begin_month)
|
|
@@ -142,4 +141,4 @@ def find_exist_file(folder_path):
|
|
|
142
141
|
|
|
143
142
|
if __name__ == '__main__':
|
|
144
143
|
# k_line_df = query_k_line('TSLA')
|
|
145
|
-
sync_by_year(
|
|
144
|
+
sync_by_year(2025)
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
import sys
|
|
2
|
+
import os
|
|
3
|
+
|
|
4
|
+
file_path = os.path.abspath(__file__)
|
|
5
|
+
end = file_path.index('mns') + 17
|
|
6
|
+
project_path = file_path[0:end]
|
|
7
|
+
sys.path.append(project_path)
|
|
8
|
+
from loguru import logger
|
|
9
|
+
from mns_common.db.MongodbUtil import MongodbUtil
|
|
10
|
+
from mns_common.db.v2.MongodbUtilV2 import MongodbUtilV2
|
|
11
|
+
import mns_scheduler.extraIncome.a_stock.one_minute.common.db_create_index as db_create_index
|
|
12
|
+
import mns_common.constant.extra_income_db_name as extra_income_db_name
|
|
13
|
+
import mns_scheduler.extraIncome.us.one_minute.api.stock_etf_info_api as stock_etf_info_api
|
|
14
|
+
import mns_scheduler.extraIncome.us.one_minute.api.y_finance_api as y_finance_api
|
|
15
|
+
|
|
16
|
+
from datetime import datetime, timedelta
|
|
17
|
+
|
|
18
|
+
mongodb_util_27017 = MongodbUtil('27017')
|
|
19
|
+
mongodbUtilV2_27019 = MongodbUtilV2('27019', extra_income_db_name.EXTRA_INCOME)
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def sync_us_etf_one_minute():
|
|
23
|
+
us_stock_df = stock_etf_info_api.get_us_etf_info()
|
|
24
|
+
|
|
25
|
+
# 获取当前日期
|
|
26
|
+
current_date = datetime.now()
|
|
27
|
+
year = current_date.year
|
|
28
|
+
col_name = extra_income_db_name.US_ETF_MINUTE_K_LINE_BFQ + "_" + str(year)
|
|
29
|
+
|
|
30
|
+
# 创建索引
|
|
31
|
+
db_create_index.create_index(mongodbUtilV2_27019, col_name)
|
|
32
|
+
|
|
33
|
+
str_day = current_date.strftime('%Y-%m-%d')
|
|
34
|
+
# 计算七天前的日期 todo 修改时间
|
|
35
|
+
seven_days_ago = current_date - timedelta(days=8)
|
|
36
|
+
|
|
37
|
+
str_seven_days_ago = seven_days_ago.strftime('%Y-%m-%d')
|
|
38
|
+
|
|
39
|
+
for stock_one in us_stock_df.itertuples():
|
|
40
|
+
symbol = stock_one.symbol
|
|
41
|
+
name = stock_one.name
|
|
42
|
+
try:
|
|
43
|
+
us_one_minute_df = y_finance_api.get_us_one_minute(symbol, str_seven_days_ago, str_day, )
|
|
44
|
+
us_one_minute_df['_id'] = symbol + '_' + us_one_minute_df['time']
|
|
45
|
+
mongodbUtilV2_27019.insert_mongo(us_one_minute_df, col_name)
|
|
46
|
+
logger.info("同步美国ETF分钟数据完成:{},{}", symbol, name)
|
|
47
|
+
except BaseException as e:
|
|
48
|
+
logger.error("同步美国股票ETF数据出现异常:{},{},{}", symbol, name, e)
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
if __name__ == '__main__':
|
|
52
|
+
sync_us_etf_one_minute()
|
|
@@ -0,0 +1,141 @@
|
|
|
1
|
+
import os
|
|
2
|
+
import sys
|
|
3
|
+
|
|
4
|
+
file_path = os.path.abspath(__file__)
|
|
5
|
+
end = file_path.index('mns') + 17
|
|
6
|
+
project_path = file_path[0:end]
|
|
7
|
+
sys.path.append(project_path)
|
|
8
|
+
from loguru import logger
|
|
9
|
+
import mns_common.utils.data_frame_util as data_frame_util
|
|
10
|
+
from mns_common.db.MongodbUtil import MongodbUtil
|
|
11
|
+
from mns_common.db.v2.MongodbUtilV2 import MongodbUtilV2
|
|
12
|
+
import mns_scheduler.extraIncome.a_stock.one_minute.common.db_create_index as db_create_index
|
|
13
|
+
import mns_common.constant.extra_income_db_name as extra_income_db_name
|
|
14
|
+
import mns_scheduler.extraIncome.us.one_minute.api.stock_etf_info_api as stock_etf_info_api
|
|
15
|
+
import mns_scheduler.extraIncome.us.one_minute.api.y_finance_api as y_finance_api
|
|
16
|
+
import pandas as pd
|
|
17
|
+
from datetime import datetime, timedelta
|
|
18
|
+
import numpy as np
|
|
19
|
+
|
|
20
|
+
mongodb_util_27017 = MongodbUtil('27017')
|
|
21
|
+
mongodbUtilV2_27019 = MongodbUtilV2('27019', extra_income_db_name.EXTRA_INCOME)
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def sync_us_stock_one_minute():
|
|
25
|
+
us_stock_df = stock_etf_info_api.get_us_stock_info()
|
|
26
|
+
us_stock_df = us_stock_df.loc[us_stock_df['amount'] != 0]
|
|
27
|
+
|
|
28
|
+
# 获取当前日期
|
|
29
|
+
current_date = datetime.now()
|
|
30
|
+
year = current_date.year
|
|
31
|
+
col_name = extra_income_db_name.US_STOCK_MINUTE_K_LINE_BFQ + "_" + str(year)
|
|
32
|
+
|
|
33
|
+
# 创建索引
|
|
34
|
+
db_create_index.create_index(mongodbUtilV2_27019, col_name)
|
|
35
|
+
|
|
36
|
+
str_day = current_date.strftime('%Y-%m-%d')
|
|
37
|
+
# 计算七天前的日期 todo 修改时间
|
|
38
|
+
seven_days_ago = current_date - timedelta(days=7)
|
|
39
|
+
|
|
40
|
+
str_seven_days_ago = seven_days_ago.strftime('%Y-%m-%d')
|
|
41
|
+
|
|
42
|
+
save_one_minute_data(us_stock_df, str_day, str_seven_days_ago, col_name, True)
|
|
43
|
+
handle_fail_stocks()
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def save_one_minute_data(us_stock_df, str_day, str_seven_days_ago, col_name, save_flag):
|
|
47
|
+
col_name_status = extra_income_db_name.US_STOCK_MINUTE_K_LINE_BFQ + "_status"
|
|
48
|
+
for stock_one in us_stock_df.itertuples():
|
|
49
|
+
|
|
50
|
+
symbol = stock_one.symbol
|
|
51
|
+
name = stock_one.name
|
|
52
|
+
try:
|
|
53
|
+
list_date = stock_one.list_date
|
|
54
|
+
if mongodbUtilV2_27019.exist_data_query(col_name_status, {"str_day": str_day,
|
|
55
|
+
'symbol': symbol, 'status': 'success'}):
|
|
56
|
+
continue
|
|
57
|
+
|
|
58
|
+
if not np.isnan(list_date):
|
|
59
|
+
list_date = str(int(list_date))
|
|
60
|
+
|
|
61
|
+
date_obj = datetime.strptime(list_date, '%Y%m%d')
|
|
62
|
+
# 格式化为 "YYYY-MM-DD" 或其他格式
|
|
63
|
+
list_date_str = date_obj.strftime('%Y-%m-%d')
|
|
64
|
+
|
|
65
|
+
if list_date_str > str_day:
|
|
66
|
+
continue
|
|
67
|
+
elif list_date_str > str_seven_days_ago:
|
|
68
|
+
str_seven_days_ago = list_date_str
|
|
69
|
+
|
|
70
|
+
us_one_minute_df = y_finance_api.get_us_one_minute(symbol, str_seven_days_ago, str_day)
|
|
71
|
+
us_one_minute_df = us_one_minute_df.fillna(0)
|
|
72
|
+
|
|
73
|
+
us_one_minute_df['_id'] = symbol + '_' + us_one_minute_df['time']
|
|
74
|
+
us_one_minute_df['symbol'] = symbol
|
|
75
|
+
if save_flag:
|
|
76
|
+
mongodbUtilV2_27019.insert_mongo(us_one_minute_df, col_name)
|
|
77
|
+
else:
|
|
78
|
+
mongodbUtilV2_27019.save_mongo(us_one_minute_df, col_name)
|
|
79
|
+
|
|
80
|
+
result_dict = {
|
|
81
|
+
'_id': symbol + "_" + str_day,
|
|
82
|
+
"str_day": str_day,
|
|
83
|
+
'symbol': symbol,
|
|
84
|
+
'status': 'success',
|
|
85
|
+
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
result_dict_df = pd.DataFrame(result_dict, index=[1])
|
|
89
|
+
mongodbUtilV2_27019.save_mongo(result_dict_df, col_name_status)
|
|
90
|
+
logger.info("同步美股分钟数据完成:{},{}", symbol, name)
|
|
91
|
+
except BaseException as e:
|
|
92
|
+
result_dict = {
|
|
93
|
+
'_id': symbol + "_" + str_day,
|
|
94
|
+
"str_day": str_day,
|
|
95
|
+
'symbol': symbol,
|
|
96
|
+
'status': 'fail',
|
|
97
|
+
}
|
|
98
|
+
result_dict_df = pd.DataFrame(result_dict, index=[1])
|
|
99
|
+
mongodbUtilV2_27019.save_mongo(result_dict_df, col_name_status)
|
|
100
|
+
|
|
101
|
+
logger.error("同步美股分钟数据出现异常:{},{},{}", symbol, name, e)
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def handle_fail_stocks():
|
|
105
|
+
us_stock_df = stock_etf_info_api.get_us_stock_info()
|
|
106
|
+
|
|
107
|
+
# 获取当前日期
|
|
108
|
+
current_date = datetime.now()
|
|
109
|
+
str_current_date = current_date.strftime('%Y-%m-%d')
|
|
110
|
+
|
|
111
|
+
query = {'status': 'fail', 'str_day': str_current_date}
|
|
112
|
+
col_name_status = extra_income_db_name.US_STOCK_MINUTE_K_LINE_BFQ + "_status"
|
|
113
|
+
fail_df = mongodbUtilV2_27019.find_query_data(col_name_status, query)
|
|
114
|
+
if data_frame_util.is_empty(fail_df):
|
|
115
|
+
return
|
|
116
|
+
else:
|
|
117
|
+
fail_us_df = us_stock_df.loc[us_stock_df['symbol'].isin(fail_df['symbol'])]
|
|
118
|
+
year = current_date.year
|
|
119
|
+
col_name = extra_income_db_name.US_STOCK_MINUTE_K_LINE_BFQ + "_" + str(year)
|
|
120
|
+
|
|
121
|
+
# 循环7天,每天处理一次
|
|
122
|
+
for days_ago in range(7):
|
|
123
|
+
try:
|
|
124
|
+
# 计算当前处理日期(从今天往前推days_ago天)
|
|
125
|
+
target_date = current_date - timedelta(days=days_ago)
|
|
126
|
+
str_target_date = target_date.strftime('%Y-%m-%d')
|
|
127
|
+
|
|
128
|
+
# 计算前一天日期
|
|
129
|
+
previous_date = target_date - timedelta(days=1)
|
|
130
|
+
str_previous_date = previous_date.strftime('%Y-%m-%d')
|
|
131
|
+
|
|
132
|
+
# 调用保存函数
|
|
133
|
+
save_one_minute_data(fail_us_df, str_target_date, str_previous_date, col_name, False)
|
|
134
|
+
except BaseException as e:
|
|
135
|
+
|
|
136
|
+
logger.error("同步美股分钟数据补偿任务出现异常:{}", e)
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
if __name__ == '__main__':
|
|
140
|
+
sync_us_stock_one_minute()
|
|
141
|
+
|
mns_scheduler/hk/__init__.py
CHANGED
|
@@ -2,7 +2,7 @@ import sys
|
|
|
2
2
|
import os
|
|
3
3
|
|
|
4
4
|
file_path = os.path.abspath(__file__)
|
|
5
|
-
end = file_path.index('mns') +
|
|
5
|
+
end = file_path.index('mns') + 17
|
|
6
6
|
project_path = file_path[0:end]
|
|
7
7
|
sys.path.append(project_path)
|
|
8
8
|
import mns_common.component.em.em_stock_info_api as em_stock_info_api
|