mns-scheduler 1.3.1.6__tar.gz → 1.3.1.7__tar.gz

{mns-scheduler-1.3.1.6 → mns-scheduler-1.3.1.7}/PKG-INFO

@@ -1,3 +1,3 @@
 Metadata-Version: 2.1
 Name: mns-scheduler
-Version: 1.3.1.6
+Version: 1.3.1.7

mns-scheduler-1.3.1.7/mns_scheduler/extraIncome/us/one_minute/stock/down_load/fail/__init__.py

@@ -0,0 +1,7 @@
+import sys
+import os
+
+file_path = os.path.abspath(__file__)
+end = file_path.index('mns') + 16
+project_path = file_path[0:end]
+sys.path.append(project_path)

mns-scheduler-1.3.1.7/mns_scheduler/extraIncome/us/one_minute/stock/down_load/fail/handle_fail_data.py

@@ -0,0 +1,161 @@
+import sys
+import os
+
+file_path = os.path.abspath(__file__)
+end = file_path.index('mns') + 16
+project_path = file_path[0:end]
+sys.path.append(project_path)
+import mns_scheduler.extraIncome.us.one_minute.stock.down_load.rename.rename_stock as rename_stock
+from loguru import logger
+import time
+import mns_common.utils.data_frame_util as data_frame_util
+from mns_common.db.MongodbUtil import MongodbUtil
+from mns_common.db.v2.MongodbUtilV2 import MongodbUtilV2
+import mns_common.constant.extra_income_db_name as extra_income_db_name
+import mns_scheduler.extraIncome.us.one_minute.api.alpha_vantage_api as alpha_vantage_api
+import pandas as pd
+
+mongodb_util_27017 = MongodbUtil('27017')
+mongodbUtilV2_27019 = MongodbUtilV2('27019', extra_income_db_name.EXTRA_INCOME)
+import threading
+
+
+def handle_fail_sync():
+    begin_year = 2025
+    end_year = 2020
+    while begin_year > end_year:
+        query = {"now_year": {"$in": [begin_year, str(begin_year)]}}
+        us_fail_df = mongodb_util_27017.find_query_data('us_stock_one_minute_down_load_fail', query)
+        us_fail_df = us_fail_df.sort_values(by=['now_month'], ascending=False)
+        fix_name_df = rename_stock.read_csv_df()
+        for stock_one in us_fail_df.itertuples():
+            symbol = stock_one.symbol
+            now_month = stock_one.now_month
+            path = stock_one.path
+            fix_name_df_one = fix_name_df.loc[fix_name_df['current_code'] == symbol]
+            try:
+
+                if data_frame_util.is_not_empty(fix_name_df_one):
+                    current_code = list(fix_name_df_one['current_code'])[0]
+                    former_code = list(fix_name_df_one['former_code'])[0]
+
+                    change_date = list(fix_name_df_one['change_date'])[0]
+
+                    changed = list(fix_name_df_one['changed'])[0]
+                    if changed:
+                        change_month = change_date.strftime('%Y-%m')
+                        if change_month == now_month:
+                            df_01 = call_with_timeout(sync_one_minute_data,
+                                                      current_code,
+                                                      now_month,
+                                                      timeout=60)
+                            df_02 = call_with_timeout(sync_one_minute_data,
+                                                      former_code,
+                                                      now_month,
+                                                      timeout=60)
+
+                            export_original_data(df_01, current_code, str(begin_year), now_month)
+                            export_original_data(df_02, former_code, str(begin_year), now_month)
+                        elif change_month > now_month:
+                            df_02 = call_with_timeout(sync_one_minute_data,
+                                                      former_code,
+                                                      now_month,
+                                                      timeout=60)
+                            export_original_data(df_02, former_code, str(begin_year), now_month)
+
+                        elif change_month < now_month:
+                            df_01 = call_with_timeout(sync_one_minute_data,
+                                                      current_code,
+                                                      now_month,
+                                                      timeout=60)
+                            export_original_data(df_01, current_code, str(begin_year), now_month)
+                    else:
+                        df = call_with_timeout(sync_one_minute_data,
+                                               symbol,
+                                               now_month,
+                                               timeout=60)
+                        export_original_data(df, symbol, str(begin_year), now_month)
+                else:
+                    df = call_with_timeout(sync_one_minute_data,
+                                           symbol,
+                                           now_month,
+                                           timeout=60)
+                    export_original_data(df, symbol, str(begin_year), now_month)
+                remove_query = {'symbol': symbol, 'now_month': now_month}
+                mongodb_util_27017.remove_data(remove_query, 'us_stock_one_minute_down_load_fail')
+                logger.info("同步股票分钟数据完成:,{},{}", symbol, now_month)
+            except BaseException as e:
+                time.sleep(1)
+                fail_dict = {
+                    '_id': symbol + '_' + now_month,
+                    'type': "stock",
+                    'path': path,
+                    'symbol': symbol,
+                    'now_year': begin_year,
+                    'now_month': now_month,
+                }
+                fail_df = pd.DataFrame(fail_dict, index=[1])
+
+                mongodb_util_27017.save_mongo(fail_df, 'us_stock_one_minute_down_load_fail')
+                logger.error("同步股票分钟数据出现异常:,{},{},{}", e, symbol, now_month)
+
+        begin_year = begin_year - 1
+
+
+def export_original_data(df, symbol, year, now_month):
+    path = r'F:\us_stock\one_minute\{}'.format(year)
+    if not os.path.exists(path):
+        os.makedirs(path)
+
+    path = path + '\{}'.format(now_month)
+    if not os.path.exists(path):
+        os.makedirs(path)
+
+    file_name = path + '\{}.csv'.format(symbol)
+    if data_frame_util.is_not_empty(df):
+        # df = df.dropna(subset=['_id'])
+        # del df['str_day']
+        # del df['minute']
+        # del df['_id']
+        # del df['symbol']
+        df.to_csv(file_name, index=False, encoding='utf-8')
+
+
+# 定义一个带超时的函数调用
+def call_with_timeout(func, *args, timeout=60, **kwargs):
+    # 用于存储函数执行结果
+    result = None
+    exception = None
+
+    # 定义一个线程目标函数
+    def target():
+        nonlocal result, exception
+        try:
+            result = func(*args, **kwargs)
+        except Exception as e:
+            exception = e
+
+    # 创建线程并启动
+    thread = threading.Thread(target=target)
+    thread.start()
+
+    # 等待线程完成，最多等待 timeout 秒
+    thread.join(timeout)
+
+    # 如果线程仍然存活，说明函数超时了
+    if thread.is_alive():
+        raise TimeoutError(f"Function exceeded timeout of {timeout} seconds")
+
+    # 如果函数抛出了异常，重新抛出
+    if exception is not None:
+        raise exception
+    return result
+
+
+def sync_one_minute_data(symbol, now_month):
+    df = alpha_vantage_api.sync_one_minute_data(symbol, now_month)
+    return df
+
+
+if __name__ == '__main__':
+    handle_fail_sync()

mns-scheduler-1.3.1.7/mns_scheduler/extraIncome/us/one_minute/stock/down_load/rename/__init__.py

@@ -0,0 +1,7 @@
+import sys
+import os
+
+file_path = os.path.abspath(__file__)
+end = file_path.index('mns') + 16
+project_path = file_path[0:end]
+sys.path.append(project_path)

mns-scheduler-1.3.1.7/mns_scheduler/extraIncome/us/one_minute/stock/down_load/rename/rename_stock.py

@@ -0,0 +1,18 @@
+import sys
+import os
+
+file_path = os.path.abspath(__file__)
+end = file_path.index('mns') + 16
+project_path = file_path[0:end]
+sys.path.append(project_path)
+import pandas as pd
+
+
+def read_csv_df():
+    data = pd.read_excel(
+        fr"D:\mns\mns-scheduler\mns_scheduler\extraIncome\us\one_minute\stock\down_load\rename\stock_code_changes.xlsx")
+    return data
+
+
+if __name__ == '__main__':
+    read_csv_df()

{mns-scheduler-1.3.1.6 → mns-scheduler-1.3.1.7}/mns_scheduler/extraIncome/us/one_minute/stock/down_load/stock/down_load_stock_his_01.py

@@ -75,7 +75,7 @@ def sync_us_stock_one_minute(now_year, now_month):
     real_time_quotes_all_us_stocks.dropna(subset=['list_date'], inplace=True)
     for stock_one in real_time_quotes_all_us_stocks.itertuples():
         flow_mv = stock_one.flow_mv
-        symbol = stock_one.symbo
+        symbol = stock_one.symbol
         try:
             symbol = stock_one.symbol
             # simple_symbol = int(stock_one.simple_symbol)
@@ -165,7 +165,7 @@ def net_work_check(now_date):
 
 
 def sync_by_year(begin_year):
-    begin_month = 10
+    begin_month = 12
     while begin_month > 0:
         if begin_month < 10:
             str_month = '0' + str(begin_month)
@@ -190,7 +190,7 @@ if __name__ == '__main__':
     # k_line_df = query_k_line('TSLA')
     # sync_by_year(2024)
     # sync_by_year(2022)
-    sync_by_year(2020)
+    # sync_by_year(2020)
     sync_by_year(2018)
     sync_by_year(2016)
     sync_by_year(2014)

{mns-scheduler-1.3.1.6 → mns-scheduler-1.3.1.7}/mns_scheduler/extraIncome/us/one_minute/stock/down_load/stock/down_load_stock_his_02.py

@@ -75,7 +75,7 @@ def sync_us_stock_one_minute(now_year, now_month):
     real_time_quotes_all_us_stocks.dropna(subset=['list_date'], inplace=True)
     for stock_one in real_time_quotes_all_us_stocks.itertuples():
         flow_mv = stock_one.flow_mv
-        symbol = stock_one.symbo
+        symbol = stock_one.symbol
         try:
 
             # simple_symbol = int(stock_one.simple_symbol)
@@ -165,7 +165,7 @@ def net_work_check(now_date):
 
 
 def sync_by_year(begin_year):
-    begin_month = 10
+    begin_month = 12
     while begin_month > 0:
         if begin_month < 10:
             str_month = '0' + str(begin_month)

{mns-scheduler-1.3.1.6 → mns-scheduler-1.3.1.7}/mns_scheduler/zt/zt_pool/em_zt_pool_sync_api.py

@@ -64,7 +64,8 @@ def save_zt_info(str_day):
     stock_em_zt_pool_df_data = common_service_fun_api.symbol_amount_simple(stock_em_zt_pool_df_data.copy())
 
     stock_em_zt_pool_df_data = company_common_service_api.amendment_industry(stock_em_zt_pool_df_data.copy())
-
+    # 主线标记 复盘用
+    stock_em_zt_pool_df_data['main_line'] = '无'
     # 上个交易交易日涨停股票
     last_trade_day_zt_df = zt_common_service_api.get_last_trade_day_zt(str_day)
 
@@ -129,8 +130,6 @@ def save_zt_info(str_day):
 
     stock_em_zt_pool_df_data['str_day'] = str_day
     stock_em_zt_pool_df_data['_id'] = stock_em_zt_pool_df_data['symbol'] + "_" + str_day
-    # 主线标记 复盘用
-    stock_em_zt_pool_df_data['main_line'] = '无'
 
     stock_em_zt_pool_df_data = stock_em_zt_pool_df_data[ZT_FIELD]
 
@@ -152,16 +151,19 @@ def save_zt_info(str_day):
 
 # 设置连板数目
 def set_connected_boards_numbers(stock_em_zt_pool_df_data, symbol, last_trade_day_zt_df):
-    connected_boards_df = last_trade_day_zt_df.loc[
+    connected_boards_df_copy = last_trade_day_zt_df.loc[
         last_trade_day_zt_df['symbol'].isin(stock_em_zt_pool_df_data['symbol'])]
-
+    connected_boards_df = connected_boards_df_copy.copy()
     connected_boards_df['connected_boards_numbers'] = connected_boards_df['connected_boards_numbers'] + 1
 
     connected_boards_df_one = connected_boards_df.loc[connected_boards_df['symbol'] == symbol]
     if data_frame_util.is_not_empty(connected_boards_df_one):
-        connected_boards_df_one.loc[connected_boards_df_one['symbol'] == symbol, 'connected_boards_numbers'] = \
+        stock_em_zt_pool_df_data.loc[stock_em_zt_pool_df_data['symbol'] == symbol, 'connected_boards_numbers'] = \
             list(connected_boards_df_one['connected_boards_numbers'])[0]
 
+        stock_em_zt_pool_df_data.loc[stock_em_zt_pool_df_data['symbol'] == symbol, 'main_line'] = \
+            list(connected_boards_df_one['main_line'])[0]
+
     return stock_em_zt_pool_df_data
 
 
@@ -220,7 +222,7 @@ def sync_miss_zt_data(stock_em_zt_pool_df_data, str_day):
 
 
 if __name__ == '__main__':
-    save_zt_info('2025-05-08')
+    save_zt_info('2025-05-13')
 # from datetime import datetime
 #
 # if __name__ == '__main__':

{mns-scheduler-1.3.1.6 → mns-scheduler-1.3.1.7}/mns_scheduler.egg-info/PKG-INFO

@@ -1,3 +1,3 @@
 Metadata-Version: 2.1
 Name: mns-scheduler
-Version: 1.3.1.6
+Version: 1.3.1.7

{mns-scheduler-1.3.1.6 → mns-scheduler-1.3.1.7}/mns_scheduler.egg-info/SOURCES.txt

@@ -96,6 +96,10 @@ mns_scheduler/extraIncome/us/one_minute/stock/down_load/__init__.py
 mns_scheduler/extraIncome/us/one_minute/stock/down_load/etf/__init__.py
 mns_scheduler/extraIncome/us/one_minute/stock/down_load/etf/down_load_ETF_his_2024.py
 mns_scheduler/extraIncome/us/one_minute/stock/down_load/etf/handle_down_load_fail_ETF.py
+mns_scheduler/extraIncome/us/one_minute/stock/down_load/fail/__init__.py
+mns_scheduler/extraIncome/us/one_minute/stock/down_load/fail/handle_fail_data.py
+mns_scheduler/extraIncome/us/one_minute/stock/down_load/rename/__init__.py
+mns_scheduler/extraIncome/us/one_minute/stock/down_load/rename/rename_stock.py
 mns_scheduler/extraIncome/us/one_minute/stock/down_load/stock/__init__.py
 mns_scheduler/extraIncome/us/one_minute/stock/down_load/stock/down_load_stock_his_01.py
 mns_scheduler/extraIncome/us/one_minute/stock/down_load/stock/down_load_stock_his_02.py

{mns-scheduler-1.3.1.6 → mns-scheduler-1.3.1.7}/setup.py

@@ -2,7 +2,7 @@ from setuptools import setup, find_packages
 
 setup(
     name='mns-scheduler',
-    version='1.3.1.6',
+    version='1.3.1.7',
     packages=find_packages(),
     install_requires=[],  # 如果有依赖项，可以在这里列出
 )