mns-scheduler 1.2.6.6__tar.gz → 1.2.6.8__tar.gz

{mns_scheduler-1.2.6.6 → mns_scheduler-1.2.6.8}/PKG-INFO

@@ -1,3 +1,3 @@
 Metadata-Version: 2.1
 Name: mns-scheduler
-Version: 1.2.6.6
+Version: 1.2.6.8

{mns_scheduler-1.2.6.6 → mns_scheduler-1.2.6.8}/mns_scheduler/db/real_time_task_check.py

@@ -63,7 +63,7 @@ def run_check_real_time_data_sync_status():
         return False
 
     # 关闭上午定时同步
-    if hour == 11 and minute == 31:
+    if (hour == 11 and minute == 31) or( hour == 9 and minute == 27):
         all_cmd_processes = cmd_util.get_all_process()
         if data_frame_util.is_empty(all_cmd_processes):
             return False
@@ -79,7 +79,7 @@ def run_check_real_time_data_sync_status():
                 logger.error("关闭实时行情任务异常:{}", e)
 
     # 重开下午定时同步
-    if hour == 12 and minute == 59:
+    if (hour == 12 and minute == 59) or (hour == 9 and minute == 29):
         all_cmd_processes = cmd_util.get_all_process()
         if data_frame_util.is_empty(all_cmd_processes):
             return False

mns_scheduler-1.2.6.8/mns_scheduler/db/script/db_move/col_move_one_service.py

@@ -0,0 +1,34 @@
+import sys
+import os
+
+file_path = os.path.abspath(__file__)
+end = file_path.index('mns') + 16
+project_path = file_path[0:end]
+sys.path.append(project_path)
+from loguru import logger
+
+
+def db_export(db, col):
+    cmd = 'F:/mongo/bin/mongodump.exe --host ' + db + ' -d patience -c ' + col + ' -o D:/back'
+    os.system(cmd)
+    logger.info("export finished:{}", col)
+
+
+def db_import(db, col):
+    cmd = 'F:/mongo/bin/mongorestore.exe --host ' + db + ' -d patience -c ' + col + ' D:/back/patience/' + col + '.bson'
+    os.system(cmd)
+
+    path = 'D:\\back\\patience\\' + col + '.bson'
+    cmd_del = 'del /F /S /Q ' + path
+    os.system(cmd_del)
+
+    logger.info("import finished:{}", col)
+
+
+def handle_one_col(col_name):
+    db_export('127.0.0.1:27017', col_name)
+    db_import('127.0.0.1:27019', col_name)
+
+
+if __name__ == '__main__':
+    handle_one_col('one_minute_k_line_bfq_h')

{mns_scheduler-1.2.6.6 → mns_scheduler-1.2.6.8}/mns_scheduler/db/script/sync/remote_data_sync_to_local.py

@@ -71,4 +71,5 @@ def sync_open_data():
 
 
 if __name__ == '__main__':
-    remote_data()
+    call_auction_data_df = local_mongodb_util.find_all_data('call_auction_data')
+    remote_mongodb_util.insert_mongo(call_auction_data_df, 'call_auction_data_01')

mns_scheduler-1.2.6.8/mns_scheduler/db/script/sync/sync_hui_ce_test_data_01.py

@@ -0,0 +1,69 @@
+import sys
+import os
+
+file_path = os.path.abspath(__file__)
+end = file_path.index('mns') + 16
+project_path = file_path[0:end]
+sys.path.append(project_path)
+
+import mns_common.utils.data_frame_util as data_frame_util
+from mns_scheduler.db.script.sync.remote_mongo_util import RemoteMongodbUtil
+from mns_scheduler.db.script.sync.local_mongo_util import LocalMongodbUtil
+from loguru import logger
+import numpy as np
+
+remote_mongodb_util = RemoteMongodbUtil('27017')
+local_mongodb_util = LocalMongodbUtil('27017')
+
+
+def sync_real_time_data(str_day, min_number, max_number):
+    db_name = 'realtime_quotes_now_' + str_day
+    while min_number <= max_number:
+        query = {'number': min_number}
+        data_df = remote_mongodb_util.find_query_data(db_name, query)
+        local_mongodb_util.insert_mongo(data_df, db_name)
+        min_number = min_number + 1
+        logger.info(min_number)
+
+
+def sync_k_line(str_day):
+    query = {"str_day": str_day}
+    db_name = 'k_line_info'
+    data_df = remote_mongodb_util.find_query_data(db_name, query)
+    try:
+        local_mongodb_util.save_mongo(data_df, db_name)
+    except BaseException as e:
+        pass
+
+
+if __name__ == '__main__':
+
+    str_day_01 = '2025-02-18'
+    # sync_k_line(str_day_01)
+    # sync_k_line(str_day_01)
+    # sync_k_line('2024-11-13')
+    # sync_k_line('2024-11-12')
+    # sync_k_line('2024-11-15') 232
+    sync_real_time_data(str_day_01, 436, 2000)
+    # sync_real_time_data('2024-11-15', 10, 1010)
+
+    sync_k_line('2024-12-20')
+    sync_k_line('2024-12-19')
+    sync_k_line('2024-12-18')
+    sync_k_line('2024-12-17')
+    sync_k_line('2024-12-16')
+
+    sync_k_line('2024-12-13')
+    sync_k_line('2024-12-12')
+    sync_k_line('2024-12-11')
+    sync_k_line('2024-12-10')
+    sync_k_line('2024-12-09')
+
+    sync_k_line('2024-12-06')
+    sync_k_line('2024-12-05')
+    sync_k_line('2024-12-04')
+    sync_k_line('2024-12-03')
+    sync_k_line('2024-12-02')
+
+    # sync_real_time_data('2024-12-20', 1000, 2000)
+    # sync_real_time_data('2024-11-13', 10, 1010)

mns_scheduler-1.2.6.8/mns_scheduler/extraIncome/stock/sync_stock_one_minute_now_api.py

@@ -0,0 +1,165 @@
+import sys
+import os
+
+file_path = os.path.abspath(__file__)
+end = file_path.index('mns') + 16
+project_path = file_path[0:end]
+sys.path.append(project_path)
+from mns_common.db.MongodbUtil import MongodbUtil
+from mns_common.db.v2.MongodbUtilV2 import MongodbUtilV2
+import mns_common.api.em.east_money_stock_api as east_money_stock_api
+import pandas as pd
+from loguru import logger
+import mns_common.component.common_service_fun_api as common_service_fun_api
+import mns_common.utils.data_frame_util as data_frame_util
+import time
+import mns_common.api.k_line.stock_minute_data_api as stock_minute_data_api
+import mns_common.constant.db_name_constant as db_name_constant
+
+mongodb_util_27017 = MongodbUtil('27017')
+mongodb_util_27019 = MongodbUtil('27019')
+mongodbUtilV2 = MongodbUtilV2('27017', 'extraIncome')
+
+
+def create_index(db_util, db_name):
+    index_create = [('symbol', 1), ('time', 1)]
+    db_util.create_index(db_name, index_create)
+    index_create_01 = [('time', 1)]
+    db_util.create_index(db_name, index_create_01)
+    index_create_02 = [('symbol', 1)]
+    db_util.create_index(db_name, index_create_02)
+
+
+def get_minute_data(symbol, freq, start_date, end_date):
+    # 获取浦发银行60000.SH的历史分钟数据
+    df = stock_minute_data_api.get_minute_data(symbol, start_date, end_date, freq,
+                                               '')
+    return df
+
+
+# 287
+
+def sync_all_stock(trade_date_list_df):
+    de_list_stock_df = mongodb_util_27017.find_all_data('de_list_stock')
+    de_list_stock_df = common_service_fun_api.classify_symbol(de_list_stock_df)
+    de_list_stock_df = de_list_stock_df.loc[
+        de_list_stock_df['classification'].isin(['K', 'C', 'S', 'H', 'X'])]
+    # 对 classification 为 S 或 K 的数据，symbol 列加上 '.SH'，其他加上 '.SZ'
+
+    de_list_stock_df['symbol'] = de_list_stock_df.apply(
+        lambda row: row['symbol'] + '.SZ' if row['classification'] in ['S', 'C']
+        else row['symbol'] + '.BJ' if row['classification'] in ['X']
+        else row['symbol'] + '.SH',
+        axis=1
+    )
+
+    real_time_quotes_all_stocks_df = east_money_stock_api.get_real_time_quotes_all_stocks()
+    # 假设数字格式为 YYYYMMDD
+    real_time_quotes_all_stocks_df['list_date'] = pd.to_datetime(real_time_quotes_all_stocks_df['list_date'],
+                                                                 format='%Y%m%d')
+
+    # 将日期格式化为字符串（YYYY-MM-DD）
+    real_time_quotes_all_stocks_df['list_date'] = real_time_quotes_all_stocks_df['list_date'].dt.strftime('%Y-%m-%d')
+
+    real_time_quotes_all_stocks_df = common_service_fun_api.classify_symbol(real_time_quotes_all_stocks_df)
+
+    # 对 classification 为 S 或 K 的数据，symbol 列加上 '.SH'，其他加上 '.SZ'
+    real_time_quotes_all_stocks_df['symbol'] = real_time_quotes_all_stocks_df.apply(
+        lambda row: row['symbol'] + '.SZ' if row['classification'] in ['S', 'C']
+        else row['symbol'] + '.BJ' if row['classification'] in ['X']
+        else row['symbol'] + '.SH',
+        axis=1
+    )
+
+    real_time_quotes_all_stocks_df = real_time_quotes_all_stocks_df.sort_values(by=['chg'], ascending=True)
+
+    real_time_quotes_all_stocks_df['number'] = real_time_quotes_all_stocks_df['chg'].rank(method='first').astype(int)
+
+    for stock_one in real_time_quotes_all_stocks_df.itertuples():
+        classification = stock_one.classification
+        if classification == 'X':
+            db_name = db_name_constant.ONE_MINUTE_K_LINE_BFQ_BJ
+        elif classification == 'S':
+            db_name = db_name_constant.ONE_MINUTE_K_LINE_BFQ_S
+        elif classification == 'H':
+            db_name = 'one_minute_k_line_bfq_h'
+        elif classification == 'K':
+            db_name = 'one_minute_k_line_bfq_k'
+        elif classification == 'C':
+            db_name = 'one_minute_k_line_bfq_c'
+        db_name = db_name + '_2025'
+        create_index(mongodbUtilV2, db_name)
+        trade_date_list_df_copy = trade_date_list_df.copy()
+
+        list_date = stock_one.list_date
+        trade_date_list_df_copy = trade_date_list_df_copy.loc[trade_date_list_df_copy['_id'] >= list_date]
+
+        symbol = stock_one.symbol
+
+        de_list_stock_df_one = de_list_stock_df.loc[de_list_stock_df['symbol'] == symbol]
+
+        if data_frame_util.is_not_empty(de_list_stock_df_one):
+            de_list_date = list(de_list_stock_df_one['de_list_date'])[0]
+            trade_date_list_df_copy = trade_date_list_df_copy.loc[trade_date_list_df_copy['_id'] <= de_list_date]
+
+        for i in range(0, len(trade_date_list_df_copy), 28):
+            try:
+                new_df = trade_date_list_df_copy.iloc[i:i + 28]
+                first_df = new_df.iloc[0]  # 第一个 DataFrame
+                last_df = new_df.iloc[-1]  # 最后一个 DataFrame
+                begin_date = first_df.trade_date + ' 09:30:00'
+                end_date = last_df.trade_date + ' 15:00:00'
+
+                trade_date_list = list(new_df['trade_date'])
+                if classification in ['K', 'H']:
+                    symbol_quest = '1.' + symbol[0:6]
+                else:
+                    symbol_quest = '0.' + symbol[0:6]
+
+                df = get_minute_data(symbol_quest, '1', begin_date, end_date)
+                if data_frame_util.is_not_empty(df):
+                    df['symbol'] = symbol
+                    df = df[[
+                        'symbol',
+                        "time",
+                        "open",
+                        "close",
+                        "high",
+                        "low",
+                        "amount",
+                        "volume",
+                    ]]
+                    df['str_day'] = df['time'].str[:10]  #
+                    df = df.loc[df['str_day'].isin(trade_date_list)]
+                    if data_frame_util.is_not_empty(df):
+                        df['time_tick'] = df['time'].str[11:19]
+                        df = df.loc[df['time_tick'] <= '15:00:00']
+                        del df['time_tick']
+                        del df['str_day']
+                        df['_id'] = df['symbol'] + '_' + df['time']
+                        mongodbUtilV2.insert_mongo(df, db_name)
+            except BaseException as e:
+                time.sleep(2)
+                first_df = new_df.iloc[0]  # 第一个 DataFrame
+                last_df = new_df.iloc[-1]  # 最后一个 DataFrame
+                begin_date = first_df.trade_date + ' 09:20:00'
+                end_date = last_df.trade_date + ' 15:00:00'
+                fail_dict = {'begin_date': begin_date,
+                             'end_date': end_date,
+                             'symbol': symbol,
+                             'db_name': db_name
+                             }
+                fail_df = pd.DataFrame(fail_dict, index=[1])
+                mongodbUtilV2.insert_mongo(fail_df, 'one_minute_k_line_bfq_fail_2025_03')
+
+                logger.error("同步数据出现异常:{},{},{},{}", e, symbol, begin_date, end_date)
+        logger.info("同步完数据:{},{}", stock_one.symbol, stock_one.name)
+
+
+if __name__ == '__main__':
+    # get_minute_data('833284.BJ', '1min', '2025-02-28 09:30:00', '2025-02-28 15:00:00')
+
+    query_trade = {"$and": [{"trade_date": {"$gte": "2025-03-01"}}, {"trade_date": {"$lte": "2025-03-07"}}]}
+    trade_date_list_df_all = mongodb_util_27017.find_query_data('trade_date_list', query_trade)
+
+    sync_all_stock(trade_date_list_df_all)

{mns_scheduler-1.2.6.6 → mns_scheduler-1.2.6.8}/mns_scheduler/k_line/sync/bfq_k_line_sync.py

@@ -22,7 +22,12 @@ def sync_bfq_k_line_data(period='daily',
                          hq_col='stock_hfq_daily',
                          end_date='22220101',
                          symbol=None):
-    stock_hfq_df = k_line_api.stock_zh_a_hist(symbol=symbol, period=period,
+    # 检查symbol是否以'6'开头
+    if symbol.startswith('6'):
+        symbol_a = '1.' + symbol
+    else:
+        symbol_a = '0.' + symbol
+    stock_hfq_df = k_line_api.stock_zh_a_hist(symbol=symbol_a, period=period,
                                               start_date=date_handle_util.no_slash_date('1990-12-19'),
                                               end_date=date_handle_util.no_slash_date(end_date),
                                               adjust=hq)

{mns_scheduler-1.2.6.6 → mns_scheduler-1.2.6.8}/mns_scheduler/k_line/sync/daily_week_month_line_sync.py

@@ -20,7 +20,12 @@ mongodb_util = MongodbUtil('27017')
 # 多线程接口容易拉爆
 
 def save_one_symbol(symbol, period, end_date, hq, hq_col, real_time_quotes_now):
-    stock_hfq_df = k_line_api.stock_zh_a_hist(symbol=symbol, period=period,
+    # 检查symbol是否以'6'开头
+    if symbol.startswith('6'):
+        symbol_a = '1.' + symbol
+    else:
+        symbol_a = '0.' + symbol
+    stock_hfq_df = k_line_api.stock_zh_a_hist(symbol=symbol_a, period=period,
                                               start_date=date_handle_util.no_slash_date('1990-12-19'),
                                               end_date=date_handle_util.no_slash_date(end_date),
                                               adjust=hq)

mns_scheduler-1.2.6.8/mns_scheduler/self_choose/__init__.py

@@ -0,0 +1,7 @@
+import sys
+import os
+
+file_path = os.path.abspath(__file__)
+end = file_path.index('mns') + 16
+project_path = file_path[0:end]
+sys.path.append(project_path)

mns_scheduler-1.2.6.8/mns_scheduler/trade/balance/__init__.py

@@ -0,0 +1,7 @@
+import sys
+import os
+
+file_path = os.path.abspath(__file__)
+end = file_path.index('mns') + 16
+project_path = file_path[0:end]
+sys.path.append(project_path)

mns_scheduler-1.2.6.8/mns_scheduler/trade/balance/ths_account_balance_service.py

@@ -0,0 +1,7 @@
+import sys
+import os
+
+file_path = os.path.abspath(__file__)
+end = file_path.index('mns') + 16
+project_path = file_path[0:end]
+sys.path.append(project_path)

mns_scheduler-1.2.6.8/mns_scheduler/trade/tfp/__init__.py

@@ -0,0 +1,7 @@
+import sys
+import os
+
+file_path = os.path.abspath(__file__)
+end = file_path.index('mns') + 16
+project_path = file_path[0:end]
+sys.path.append(project_path)

mns_scheduler-1.2.6.8/mns_scheduler/us/__init__.py

@@ -0,0 +1,7 @@
+import sys
+import os
+
+file_path = os.path.abspath(__file__)
+end = file_path.index('mns') + 16
+project_path = file_path[0:end]
+sys.path.append(project_path)

mns_scheduler-1.2.6.8/mns_scheduler/zt/script/__init__.py

@@ -0,0 +1,7 @@
+import sys
+import os
+
+file_path = os.path.abspath(__file__)
+end = file_path.index('mns') + 16
+project_path = file_path[0:end]
+sys.path.append(project_path)

{mns_scheduler-1.2.6.6 → mns_scheduler-1.2.6.8}/mns_scheduler/zt/zt_pool/ths_zt_pool_sync_api.py

@@ -154,7 +154,12 @@ def get_bfq_daily_line(ths_zt_pool_df, str_day):
     for zt_one in ths_zt_pool_df.itertuples():
         try:
             symbol = zt_one.symbol
-            bfq_daily_line_df = k_line_api.stock_zh_a_hist(symbol,
+            # 检查symbol是否以'6'开头
+            if symbol.startswith('6'):
+                symbol_a = '1.' + symbol
+            else:
+                symbol_a = '0.' + symbol
+            bfq_daily_line_df = k_line_api.stock_zh_a_hist(symbol_a,
                                                            'daily',
                                                            date_handle_util.no_slash_date(str_day),
                                                            date_handle_util.no_slash_date(str_day),

{mns_scheduler-1.2.6.6 → mns_scheduler-1.2.6.8}/mns_scheduler/zz_task/data_sync_task.py

@@ -448,7 +448,7 @@ blockingScheduler.add_job(db_status_check, 'interval', seconds=30, max_instances
 blockingScheduler.add_job(sync_daily_data_info, 'cron', hour='15,20', minute='26')
 
 # 开盘前同步当天交易需要的k线数据
-blockingScheduler.add_job(sync_today_trade_k_line_info, 'cron', hour='08', minute='30')
+blockingScheduler.add_job(sync_today_trade_k_line_info, 'cron', hour='07', minute='40')
 
 # 同步十大流通股东信息
 blockingScheduler.add_job(sync_stock_gdfx_free_top_10_one_day, 'cron', hour='08,22', minute='23')
@@ -552,4 +552,5 @@ print('定时任务启动成功')
 blockingScheduler.start()
 #
 # if __name__ == '__main__':
+#
 #     sync_daily_data_info()

{mns_scheduler-1.2.6.6 → mns_scheduler-1.2.6.8}/mns_scheduler.egg-info/PKG-INFO

@@ -1,3 +1,3 @@
 Metadata-Version: 2.1
 Name: mns-scheduler
-Version: 1.2.6.6
+Version: 1.2.6.8

{mns_scheduler-1.2.6.6 → mns_scheduler-1.2.6.8}/mns_scheduler.egg-info/SOURCES.txt

@@ -42,17 +42,26 @@ mns_scheduler/db/db_status.py
 mns_scheduler/db/real_time_task_check.py
 mns_scheduler/db/script/__init__.py
 mns_scheduler/db/script/col_move_script.py
+mns_scheduler/db/script/db_move/__init__.py
+mns_scheduler/db/script/db_move/col_move_one_service.py
 mns_scheduler/db/script/sync/__init__.py
 mns_scheduler/db/script/sync/local_mongo_util.py
 mns_scheduler/db/script/sync/remote_data_sync_to_local.py
 mns_scheduler/db/script/sync/remote_mongo_util.py
 mns_scheduler/db/script/sync/sync_hui_ce_test_data.py
+mns_scheduler/db/script/sync/sync_hui_ce_test_data_01.py
 mns_scheduler/db/script/update/__init__.py
 mns_scheduler/db/script/update/update_col_field.py
 mns_scheduler/debt/__init__.py
 mns_scheduler/debt/kzz_bond_info_sync.py
 mns_scheduler/dt/__init__.py
 mns_scheduler/dt/stock_dt_pool_sync.py
+mns_scheduler/extraIncome/__init__.py
+mns_scheduler/extraIncome/etf/__init__.py
+mns_scheduler/extraIncome/kzz/__init__.py
+mns_scheduler/extraIncome/stock/__init__.py
+mns_scheduler/extraIncome/stock/one_minute_k_line_sync.py
+mns_scheduler/extraIncome/stock/sync_stock_one_minute_now_api.py
 mns_scheduler/finance/__init__.py
 mns_scheduler/finance/em_financial_asset_liability_sync_service_api.py
 mns_scheduler/finance/em_financial_profit_sync_service_api.py

{mns_scheduler-1.2.6.6 → mns_scheduler-1.2.6.8}/setup.py

@@ -2,7 +2,7 @@ from setuptools import setup, find_packages
 
 setup(
     name='mns-scheduler',
-    version='1.2.6.6',
+    version='1.2.6.8',
     packages=find_packages(),
     install_requires=[],  # 如果有依赖项，可以在这里列出
 )