mns-scheduler 1.3.0.3__py3-none-any.whl → 1.3.0.7__py3-none-any.whl

mns_scheduler/db/script/sync/sync_hui_ce_test_data.py

@@ -6,11 +6,9 @@ end = file_path.index('mns') + 16
 project_path = file_path[0:end]
 sys.path.append(project_path)
 
-import mns_common.utils.data_frame_util as data_frame_util
 from mns_scheduler.db.script.sync.remote_mongo_util import RemoteMongodbUtil
 from mns_scheduler.db.script.sync.local_mongo_util import LocalMongodbUtil
 from loguru import logger
-import numpy as np
 
 remote_mongodb_util = RemoteMongodbUtil('27017')
 local_mongodb_util = LocalMongodbUtil('27017')
@@ -33,37 +31,39 @@ def sync_k_line(str_day):
     try:
         local_mongodb_util.save_mongo(data_df, db_name)
     except BaseException as e:
+        logger.error("出现异常:{}", e)
         pass
 
 
 if __name__ == '__main__':
 
-    str_day_01 = '2025-02-18'
+    str_day_01 = '2025-02-27'
+    sync_real_time_data(str_day_01, 1, 2000)
     sync_k_line(str_day_01)
     # sync_k_line(str_day_01)
     # sync_k_line('2024-11-13')
     # sync_k_line('2024-11-12')
     # sync_k_line('2024-11-15') 232
-    sync_real_time_data(str_day_01, 1, 2000)
-    # sync_real_time_data('2024-11-15', 10, 1010)
 
-    sync_k_line('2024-12-20')
-    sync_k_line('2024-12-19')
-    sync_k_line('2024-12-18')
-    sync_k_line('2024-12-17')
-    sync_k_line('2024-12-16')
-
-    sync_k_line('2024-12-13')
-    sync_k_line('2024-12-12')
-    sync_k_line('2024-12-11')
-    sync_k_line('2024-12-10')
-    sync_k_line('2024-12-09')
+    # sync_real_time_data('2024-11-15', 10, 1010)
 
-    sync_k_line('2024-12-06')
-    sync_k_line('2024-12-05')
-    sync_k_line('2024-12-04')
-    sync_k_line('2024-12-03')
-    sync_k_line('2024-12-02')
+    # sync_k_line('2024-12-20')
+    # sync_k_line('2024-12-19')
+    # sync_k_line('2024-12-18')
+    # sync_k_line('2024-12-17')
+    # sync_k_line('2024-12-16')
+    #
+    # sync_k_line('2024-12-13')
+    # sync_k_line('2024-12-12')
+    # sync_k_line('2024-12-11')
+    # sync_k_line('2024-12-10')
+    # sync_k_line('2024-12-09')
+    #
+    # sync_k_line('2024-12-06')
+    # sync_k_line('2024-12-05')
+    # sync_k_line('2024-12-04')
+    # sync_k_line('2024-12-03')
+    # sync_k_line('2024-12-02')
 
     # sync_real_time_data('2024-12-20', 1000, 2000)
     # sync_real_time_data('2024-11-13', 10, 1010)

mns_scheduler/db/script/sync/sync_hui_ce_test_data_01.py

@@ -37,33 +37,32 @@ def sync_k_line(str_day):
 
 
 if __name__ == '__main__':
-
-    str_day_01 = '2025-02-18'
-    # sync_k_line(str_day_01)
+    str_day_01 = '2025-02-27'
+    sync_k_line(str_day_01)
     # sync_k_line(str_day_01)
     # sync_k_line('2024-11-13')
     # sync_k_line('2024-11-12')
     # sync_k_line('2024-11-15') 232
-    sync_real_time_data(str_day_01, 436, 2000)
+    # sync_real_time_data(str_day_01, 1, 2000)
     # sync_real_time_data('2024-11-15', 10, 1010)
-
-    sync_k_line('2024-12-20')
-    sync_k_line('2024-12-19')
-    sync_k_line('2024-12-18')
-    sync_k_line('2024-12-17')
-    sync_k_line('2024-12-16')
-
-    sync_k_line('2024-12-13')
-    sync_k_line('2024-12-12')
-    sync_k_line('2024-12-11')
-    sync_k_line('2024-12-10')
-    sync_k_line('2024-12-09')
-
-    sync_k_line('2024-12-06')
-    sync_k_line('2024-12-05')
-    sync_k_line('2024-12-04')
-    sync_k_line('2024-12-03')
-    sync_k_line('2024-12-02')
+    #
+    # sync_k_line('2024-12-20')
+    # sync_k_line('2024-12-19')
+    # sync_k_line('2024-12-18')
+    # sync_k_line('2024-12-17')
+    # sync_k_line('2024-12-16')
+    #
+    # sync_k_line('2024-12-13')
+    # sync_k_line('2024-12-12')
+    # sync_k_line('2024-12-11')
+    # sync_k_line('2024-12-10')
+    # sync_k_line('2024-12-09')
+    #
+    # sync_k_line('2024-12-06')
+    # sync_k_line('2024-12-05')
+    # sync_k_line('2024-12-04')
+    # sync_k_line('2024-12-03')
+    # sync_k_line('2024-12-02')
 
     # sync_real_time_data('2024-12-20', 1000, 2000)
     # sync_real_time_data('2024-11-13', 10, 1010)

mns_scheduler/extraIncome/hk/hk_stock_qfq_daily_k_line.py

@@ -0,0 +1,102 @@
+import os
+import sys
+
+file_path = os.path.abspath(__file__)
+end = file_path.index('mns') + 16
+project_path = file_path[0:end]
+sys.path.append(project_path)
+
+from mns_common.db.MongodbUtil import MongodbUtil
+from mns_common.db.v2.MongodbUtilV2 import MongodbUtilV2
+import mns_common.constant.extra_income_db_name as extra_income_db_name
+import mns_common.constant.db_name_constant as db_name_constant
+import akshare as ak
+from loguru import logger
+import mns_scheduler.baidu.baidu_yun_pan_handle_service as baidu_yun_pan_handle_service
+import mns_scheduler.hk.hk_company_info_sync_service_api as hk_company_info_sync_service_api
+
+mongodb_util_27017 = MongodbUtil('27017')
+mongodbUtilV2_27019 = MongodbUtilV2('27019', extra_income_db_name.EXTRA_INCOME)
+
+
+def hk_stock_daily_qfq_sync():
+    hk_stock_path = '/港股/qfq/'
+    hk_stock_exist_df = baidu_yun_pan_handle_service.get_file_folder(hk_stock_path)
+    hk_stock_exist_df['symbol'] = hk_stock_exist_df['name'].astype(str).str[:5]
+
+    hk_stock_info_df = mongodb_util_27017.find_all_data(db_name_constant.EM_HK_STOCK_INFO)
+
+
+    hk_stock_info_df = hk_stock_info_df.sort_values(by=['amount'], ascending=False)
+    del hk_stock_info_df['_id']
+    # 上传港股列表
+    baidu_yun_pan_handle_service.upload_to_baidu('港股列表', hk_stock_path, hk_stock_info_df)
+    hk_ggt_component_df = hk_company_info_sync_service_api.get_hk_ggt_component()
+    # 上传港股通列表
+    hk_stock_info_ggt_df = hk_stock_info_df.loc[hk_stock_info_df['symbol'].isin(hk_ggt_component_df['symbol'])]
+    baidu_yun_pan_handle_service.upload_to_baidu('港股通列表', hk_stock_path, hk_stock_info_ggt_df)
+    hk_stock_info_df = hk_stock_info_df.loc[~(hk_stock_info_df['symbol'].isin(hk_stock_exist_df['symbol']))]
+    fail_list = []
+
+    for hk_stock_one in hk_stock_info_df.itertuples():
+        symbol = hk_stock_one.symbol
+        name = hk_stock_one.name
+        try:
+            save_one_symbol(symbol, hk_stock_path)
+
+        except BaseException as e:
+            logger.error("同步出现异常:{},{},{}", e, symbol, name)
+            fail_list.append(symbol)
+    # 处理失败的
+    for symbol_fail in fail_list:
+        try:
+            save_one_symbol(symbol_fail, hk_stock_path)
+        except BaseException as e:
+            logger.error("同步出现异常:{},{},{}", e, symbol, name)
+
+
+def save_one_symbol(symbol, hk_stock_path):
+    hk_stock_k_line_df = hk_stock_k_line_api(symbol, k_line_period='daily', start_date='18000101',
+                                             end_date='22220101', fq='qfq')
+
+    hk_stock_k_line_df["date"] = hk_stock_k_line_df["date"].astype(str)
+
+    hk_stock_k_line_df['_id'] = hk_stock_k_line_df['date'] + '_' + symbol
+    hk_stock_k_line_df['symbol'] = symbol
+
+    query = {'symbol': symbol}
+    if mongodbUtilV2_27019.remove_data(query, extra_income_db_name.HK_STOCK_DAILY_QFQ_K_LINE).acknowledged > 0:
+        mongodbUtilV2_27019.insert_mongo(hk_stock_k_line_df, extra_income_db_name.HK_STOCK_DAILY_QFQ_K_LINE)
+        del hk_stock_k_line_df['_id']
+        del hk_stock_k_line_df['symbol']
+        # 上传列表
+        baidu_yun_pan_handle_service.upload_to_baidu(symbol, hk_stock_path, hk_stock_k_line_df)
+
+
+def hk_stock_k_line_api(symbol='00001', k_line_period='daily', start_date='18000101',
+                        end_date='22220101', fq='qfq'):
+    stock_hk_hist_df = ak.stock_hk_hist(symbol=symbol,
+                                        period=k_line_period,
+                                        start_date=start_date,
+                                        end_date=end_date,
+                                        adjust=fq)
+    stock_hk_hist_df = stock_hk_hist_df.rename(columns={
+        "日期": "date",
+        "涨跌额": "change_price",
+        "涨跌幅": "chg",
+        "开盘": "open",
+        "最高": "high",
+        "最低": "low",
+        "收盘": "close",
+        "成交量": "volume",
+        "成交额": "amount",
+        "振幅": "pct_chg",
+        "换手率": "exchange"
+    })
+
+    return stock_hk_hist_df
+
+
+if __name__ == '__main__':
+    # us_stock_k_line_api()
+    hk_stock_daily_qfq_sync()

mns_scheduler/extraIncome/us/__init__.py

@@ -0,0 +1,7 @@
+import sys
+import os
+
+file_path = os.path.abspath(__file__)
+end = file_path.index('mns') + 16
+project_path = file_path[0:end]
+sys.path.append(project_path)

mns_scheduler/extraIncome/us/us_stock_qfq_daily_k_line.py

@@ -0,0 +1,95 @@
+import os
+import sys
+
+file_path = os.path.abspath(__file__)
+end = file_path.index('mns') + 16
+project_path = file_path[0:end]
+sys.path.append(project_path)
+
+from mns_common.db.MongodbUtil import MongodbUtil
+from mns_common.db.v2.MongodbUtilV2 import MongodbUtilV2
+import mns_common.constant.extra_income_db_name as extra_income_db_name
+import mns_common.constant.db_name_constant as db_name_constant
+import akshare as ak
+from loguru import logger
+import mns_scheduler.baidu.baidu_yun_pan_handle_service as baidu_yun_pan_handle_service
+
+mongodb_util_27017 = MongodbUtil('27017')
+mongodbUtilV2_27019 = MongodbUtilV2('27019', extra_income_db_name.EXTRA_INCOME)
+
+
+def us_stock_daily_qfq_sync():
+    us_stock_path = '/美股/qfq/'
+    em_us_stock_info_df = mongodb_util_27017.find_all_data(db_name_constant.EM_US_STOCK_INFO)
+    # 将列 A 转为字符串，并提取前三位
+    em_us_stock_info_df["simple_symbol"] = em_us_stock_info_df["simple_symbol"].astype(str).str[:3]
+    em_us_stock_info_df = em_us_stock_info_df.sort_values(by=['amount'], ascending=False)
+    # 上传列表
+    baidu_yun_pan_handle_service.upload_to_baidu('美股列表', us_stock_path, em_us_stock_info_df)
+    fail_list = []
+    for us_stock_one in em_us_stock_info_df.itertuples():
+        symbol = us_stock_one.symbol
+        name = us_stock_one.name
+        try:
+            save_one_symbol(us_stock_one, us_stock_path, symbol)
+        except BaseException as e:
+            logger.error("同步出现异常:{},{},{}", e, symbol, name)
+            fail_list.append(symbol)
+    fail_stock_df = em_us_stock_info_df.loc[em_us_stock_info_df['symbol'].isin(fail_list)]
+
+    for us_stock_fail_one in fail_stock_df.itertuples():
+        symbol = us_stock_fail_one.symbol
+        name = us_stock_fail_one.name
+        try:
+            save_one_symbol(us_stock_fail_one, us_stock_path, symbol)
+        except BaseException as e:
+            logger.error("同步出现异常:{},{},{}", e, symbol, name)
+            fail_list.append(symbol)
+
+
+def save_one_symbol(us_stock_one, us_stock_path, symbol):
+    simple_symbol = us_stock_one.simple_symbol
+    code = simple_symbol + '.' + symbol
+
+    us_stock_k_line_df = us_stock_k_line_api(code, k_line_period='daily', start_date='18000101',
+                                             end_date='22220101', fq='qfq')
+    us_stock_k_line_df['_id'] = us_stock_k_line_df['date'] + '_' + symbol
+    us_stock_k_line_df['symbol'] = symbol
+
+    query = {'symbol': symbol}
+    if mongodbUtilV2_27019.remove_data(query, extra_income_db_name.US_STOCK_DAILY_QFQ_K_LINE).acknowledged > 0:
+        mongodbUtilV2_27019.insert_mongo(us_stock_k_line_df, extra_income_db_name.US_STOCK_DAILY_QFQ_K_LINE)
+        del us_stock_k_line_df['_id']
+        del us_stock_k_line_df['symbol']
+
+        # 上传列表
+        baidu_yun_pan_handle_service.upload_to_baidu(symbol, us_stock_path, us_stock_k_line_df)
+
+
+def us_stock_k_line_api(symbol='106.GE', k_line_period='daily', start_date='18000101',
+                        end_date='22220101', fq='hfq'):
+    stock_us_hist_df = ak.stock_us_hist(symbol=symbol,
+                                        period=k_line_period,
+                                        start_date=start_date,
+                                        end_date=end_date,
+                                        adjust=fq)
+    stock_us_hist_df = stock_us_hist_df.rename(columns={
+        "日期": "date",
+        "涨跌额": "change_price",
+        "涨跌幅": "chg",
+        "开盘": "open",
+        "最高": "high",
+        "最低": "low",
+        "收盘": "close",
+        "成交量": "volume",
+        "成交额": "amount",
+        "振幅": "pct_chg",
+        "换手率": "exchange"
+    })
+
+    return stock_us_hist_df
+
+
+if __name__ == '__main__':
+    # us_stock_k_line_api()
+    us_stock_daily_qfq_sync()

mns_scheduler/hk/hk_company_info_sync_service_api.py

@@ -99,4 +99,5 @@ def sync_hk_company_info():
 
 
 if __name__ == '__main__':
+    get_hk_ggt_component()
     sync_hk_company_info()

mns_scheduler/zz_task/data_sync_task.py

@@ -63,6 +63,9 @@ import mns_scheduler.industry.ths.ths_industry_sync_service as ths_industry_sync
 import mns_scheduler.k_line.year_quarter.year_quarter_line_sync as year_quarter_line_sync
 import mns_common.component.task.real_time_data_sync_check as real_time_data_sync_check
 import mns_scheduler.extraIncome.one_minute.one_minute_sync_task as one_minute_sync_task
+import mns_scheduler.extraIncome.hk.hk_stock_qfq_daily_k_line as hk_stock_qfq_daily_k_line
+import mns_scheduler.extraIncome.us.us_stock_qfq_daily_k_line as us_stock_qfq_daily_k_line
+import mns_scheduler.extraIncome.one_minute.upload.upload_to_baidu_task as upload_to_baidu_task
 
 
 # 同步交易日期任务完成
@@ -443,12 +446,30 @@ def sync_all_em_stock_info():
     sync_em_stock_info_sync.sync_all_em_stock_info()
 
 
+# 同步外快收入数据
+def sync_extra_income_data():
+    # 上传一分钟数据到百度网盘
+    upload_to_baidu_task.upload_stock_to_baidu()
+    # hk前复权k线数据
+    hk_stock_qfq_daily_k_line.hk_stock_daily_qfq_sync()
+    # us前复权k线数据
+    us_stock_qfq_daily_k_line.us_stock_daily_qfq_sync()
+
+
 # # 定义BlockingScheduler
 blockingScheduler = BlockingScheduler()
 
 #  同步东方财富a,etf,kzz,us,hk信息
 blockingScheduler.add_job(sync_all_em_stock_info, 'cron', hour='07', minute='31')
 blockingScheduler.add_job(sync_all_em_stock_info, 'cron', hour='15', minute='20')
+# 同步外快收入数据
+blockingScheduler.add_job(
+    sync_extra_income_data,
+    'cron',
+    day_of_week='sat,sun',  # 周六和周日
+    hour='07,18',  # 16点和21点
+    minute='37'  # 37分
+)
 
 # 开盘前同步当天交易需要的k线数据
 blockingScheduler.add_job(sync_today_trade_k_line_info, 'cron', hour='07', minute='50')

{mns_scheduler-1.3.0.3.dist-info → mns_scheduler-1.3.0.7.dist-info}/METADATA

@@ -1,4 +1,4 @@
 Metadata-Version: 2.1
 Name: mns-scheduler
-Version: 1.3.0.3
+Version: 1.3.0.7
 

{mns_scheduler-1.3.0.3.dist-info → mns_scheduler-1.3.0.7.dist-info}/RECORD

@@ -45,8 +45,8 @@ mns_scheduler/db/script/sync/__init__.py,sha256=wEg73KlZo-dU0yKGwpA1C2y6LZm4IBb9
 mns_scheduler/db/script/sync/local_mongo_util.py,sha256=MFG-S7aUBLKe4tkhKNklUzpiZef2n078YXd39dfOMy0,7540
 mns_scheduler/db/script/sync/remote_data_sync_to_local.py,sha256=gLJORdtt0YospxUwNIS5gusV9NtV7xmP1dm-M08neZ4,2789
 mns_scheduler/db/script/sync/remote_mongo_util.py,sha256=-BCR2zeQ9z0zeZg6wO0aCS4bGnsGIohFRH7QR8XXJSo,10966
-mns_scheduler/db/script/sync/sync_hui_ce_test_data.py,sha256=p0NrTWGsOVfecTBuSW96LKa_3eKevN_bll7RA1PuWcM,2090
-mns_scheduler/db/script/sync/sync_hui_ce_test_data_01.py,sha256=cQIxTpfi-wWVpY8aMQrcvig2KG_VykUHBeEmxYIx5y4,2094
+mns_scheduler/db/script/sync/sync_hui_ce_test_data.py,sha256=e4n9kjY-eonjRS1yU5n9D7B2tOP846dNTyvvtLP_zpA,2096
+mns_scheduler/db/script/sync/sync_hui_ce_test_data_01.py,sha256=ywJxst91X2hvE5lvtco_18BpXoP7gKndTEvZ1o4B0jk,2135
 mns_scheduler/db/script/update/__init__.py,sha256=wEg73KlZo-dU0yKGwpA1C2y6LZm4IBb94tNda1tqLeg,163
 mns_scheduler/db/script/update/update_col_field.py,sha256=2XeiKodVgR19IgJKMXZmM7MOLgeUyli15qo58gtYHY8,1274
 mns_scheduler/debt/__init__.py,sha256=wEg73KlZo-dU0yKGwpA1C2y6LZm4IBb94tNda1tqLeg,163
@@ -54,6 +54,8 @@ mns_scheduler/debt/kzz_bond_info_sync.py,sha256=Y4K-cYl3g8IeP_coB-3OoGsvgurcsvg_
 mns_scheduler/dt/__init__.py,sha256=QWBdZwBCvQw8aS4hnL9_pg3U3ZiNLUXzlImyy9WhUcI,163
 mns_scheduler/dt/stock_dt_pool_sync.py,sha256=5ivRUOnFtOapZniwTbujf1lVq3y4btm2Cmd5R6JJAVo,3466
 mns_scheduler/extraIncome/__init__.py,sha256=wEg73KlZo-dU0yKGwpA1C2y6LZm4IBb94tNda1tqLeg,163
+mns_scheduler/extraIncome/hk/__init__.py,sha256=wEg73KlZo-dU0yKGwpA1C2y6LZm4IBb94tNda1tqLeg,163
+mns_scheduler/extraIncome/hk/hk_stock_qfq_daily_k_line.py,sha256=myy7X09gwyPqKp8Dv_nbITok-oKUXFA5USBJSwDyCTY,4236
 mns_scheduler/extraIncome/one_minute/__init__.py,sha256=wEg73KlZo-dU0yKGwpA1C2y6LZm4IBb94tNda1tqLeg,163
 mns_scheduler/extraIncome/one_minute/one_minute_sync_task.py,sha256=qw09DyhYRqWO4U48T3CiIikhCBTOEZ7Y1nkAoBKWgKw,1557
 mns_scheduler/extraIncome/one_minute/common/__init__.py,sha256=wEg73KlZo-dU0yKGwpA1C2y6LZm4IBb94tNda1tqLeg,163
@@ -69,18 +71,15 @@ mns_scheduler/extraIncome/one_minute/stock/__init__.py,sha256=wEg73KlZo-dU0yKGwp
 mns_scheduler/extraIncome/one_minute/stock/stock_one_minute_sync_task.py,sha256=iB3RWD-Pp9e0an46xoPbn-YkgIlVFxZwQYyQ0Cfad7E,4576
 mns_scheduler/extraIncome/one_minute/upload/__init__.py,sha256=wEg73KlZo-dU0yKGwpA1C2y6LZm4IBb94tNda1tqLeg,163
 mns_scheduler/extraIncome/one_minute/upload/upload_to_baidu_task.py,sha256=_AjUYUSqnnq8IST0C0e1cjREBLoI4OWRI-MnPN3avr4,3772
-mns_scheduler/extraIncome/temp/__init__.py,sha256=wEg73KlZo-dU0yKGwpA1C2y6LZm4IBb94tNda1tqLeg,163
-mns_scheduler/extraIncome/temp/tu_share_data_etf_sync.py,sha256=XBsLl4o1Ras1zUQBkJ2BAgWtPebqAf9VUu_kLEisGmQ,4464
-mns_scheduler/extraIncome/temp/tu_share_data_kzz_sync.py,sha256=A2Aa4TB2mgTHiDlW9_UpB0mdRCR_1sOTaPZKs-IBbXc,4850
-mns_scheduler/extraIncome/temp/tu_share_data_stock_sync.py,sha256=XOEYxjq0zvhUi9EE9T9LCIlv2B8EXVOrj2h35stNLQM,6232
-mns_scheduler/extraIncome/temp/tu_share_zhi_shu_sync_api.py,sha256=sAvfsIObHWsapgJP8o2YTL4D1XZiWa2tGguM6B6bgyQ,4169
+mns_scheduler/extraIncome/us/__init__.py,sha256=wEg73KlZo-dU0yKGwpA1C2y6LZm4IBb94tNda1tqLeg,163
+mns_scheduler/extraIncome/us/us_stock_qfq_daily_k_line.py,sha256=6X0w1rYA93PNF_4uYyPi9Uqf1moxte54VB71FqADqQ4,3926
 mns_scheduler/finance/__init__.py,sha256=wEg73KlZo-dU0yKGwpA1C2y6LZm4IBb94tNda1tqLeg,163
 mns_scheduler/finance/em_financial_asset_liability_sync_service_api.py,sha256=kEZQZkxB7RF7UPH4DmHoRWfEKgI61ZN8BcNOzmBUoV0,19417
 mns_scheduler/finance/em_financial_profit_sync_service_api.py,sha256=BMOS0UIUTdDlbsKJlqEJkei1Uhz_PF2n3xgUvzZHlac,14514
 mns_scheduler/finance/finance_common_api.py,sha256=CUC_6GPpf_sLn1ZrT476qNBzxgJWDaxIi8LxdNaKFkE,2453
 mns_scheduler/finance/sync_financial_report_service_api.py,sha256=7ttWeBJOhA76CIz-K0K0jxN5z2-0-_c6-AWEFGFyMIQ,6632
 mns_scheduler/hk/__init__.py,sha256=wEg73KlZo-dU0yKGwpA1C2y6LZm4IBb94tNda1tqLeg,163
-mns_scheduler/hk/hk_company_info_sync_service_api.py,sha256=vIxsl41rJPVvPVw2BTE7b0VZHFiuBecc3i3Gn0TKrFs,3696
+mns_scheduler/hk/hk_company_info_sync_service_api.py,sha256=z8dFxIiGc9aILs-8WKducch_uXKkLS6mBuPWt2zzNyg,3724
 mns_scheduler/hk/hk_industry_info_sync_service_api.py,sha256=Pf2uRGqjhibsBoOFJlUqj2WxujmIPysTAVOeffj3iWM,2275
 mns_scheduler/industry/__init__.py,sha256=wEg73KlZo-dU0yKGwpA1C2y6LZm4IBb94tNda1tqLeg,163
 mns_scheduler/industry/ths/__init__.py,sha256=wEg73KlZo-dU0yKGwpA1C2y6LZm4IBb94tNda1tqLeg,163
@@ -180,8 +179,8 @@ mns_scheduler/zt/zt_pool/em_zt_pool_sync_api.py,sha256=CAB1XCclO92dRd4K0ZGCNggB3
 mns_scheduler/zt/zt_pool/ths_zt_pool_sync_api.py,sha256=u0IvwPuI2hnjTOrwwe8EhBAMv8NbQTENpRbb-_5lDlM,9803
 mns_scheduler/zt/zt_pool/update_null_zt_reason_api.py,sha256=1uoiR2Uw46kDfjkvNg2US5rd_4OIkYO3872gIJOufUY,2135
 mns_scheduler/zz_task/__init__.py,sha256=QWBdZwBCvQw8aS4hnL9_pg3U3ZiNLUXzlImyy9WhUcI,163
-mns_scheduler/zz_task/data_sync_task.py,sha256=ls25q6bINT5jaBukjM8A4EllBoHa8_37j3zadCXOpgE,23490
-mns_scheduler-1.3.0.3.dist-info/METADATA,sha256=UYtPRiAgduV8TYS3H8hawsO81t9HliWmRltkSAWhalQ,64
-mns_scheduler-1.3.0.3.dist-info/WHEEL,sha256=tZoeGjtWxWRfdplE7E3d45VPlLNQnvbKiYnx7gwAy8A,92
-mns_scheduler-1.3.0.3.dist-info/top_level.txt,sha256=PXQDFBGR1pWmsUbH5yiLAh71P5HZODTRED0zJ8CCgOc,14
-mns_scheduler-1.3.0.3.dist-info/RECORD,,
+mns_scheduler/zz_task/data_sync_task.py,sha256=sLFnSRh3NYuMHQsSupnRGHZ2Dda-Veh-RQp0jSlDrXU,24310
+mns_scheduler-1.3.0.7.dist-info/METADATA,sha256=mvw79fFfBZr_wnZJaCc-RFKh_Q_etn0sbtjG-xpCGUM,64
+mns_scheduler-1.3.0.7.dist-info/WHEEL,sha256=tZoeGjtWxWRfdplE7E3d45VPlLNQnvbKiYnx7gwAy8A,92
+mns_scheduler-1.3.0.7.dist-info/top_level.txt,sha256=PXQDFBGR1pWmsUbH5yiLAh71P5HZODTRED0zJ8CCgOc,14
+mns_scheduler-1.3.0.7.dist-info/RECORD,,

mns_scheduler/extraIncome/temp/tu_share_data_etf_sync.py

@@ -1,103 +0,0 @@
-import sys
-import os
-
-file_path = os.path.abspath(__file__)
-end = file_path.index('mns') + 16
-project_path = file_path[0:end]
-sys.path.append(project_path)
-import mns_common.api.em.east_money_etf_api as east_money_etf_api
-import pandas as pd
-from loguru import logger
-import time
-import mns_common.utils.data_frame_util as data_frame_util
-from mns_common.db.MongodbUtil import MongodbUtil
-import tushare as ts
-from mns_common.db.v2.MongodbUtilV2 import MongodbUtilV2
-import mns_scheduler.extraIncome.one_minute.common.db_create_index as db_create_index
-import mns_common.constant.extra_income_db_name as extra_income_db_name
-
-mongodb_util_27017 = MongodbUtil('27017')
-mongodbUtilV2_27019 = MongodbUtilV2('27019', extra_income_db_name.EXTRA_INCOME)
-pro = ts.pro_api('782213d20640249f1dbae50a7f56b22684b8e915a61e435e015579a1')
-
-
-def get_minute_data(symbol, freq, start_date, end_date):
-    # 获取浦发银行60000.SH的历史分钟数据
-    df = pro.stk_mins(ts_code=symbol, freq=freq, start_date=start_date, end_date=end_date)
-    return df
-
-
-def sync_etf_one_minute(trade_date_list_df):
-    etf_real_time_quotes_df = east_money_etf_api.get_etf_real_time_quotes()
-    etf_real_time_quotes_df = classify_symbol(etf_real_time_quotes_df)
-    etf_real_time_quotes_df['symbol'] = etf_real_time_quotes_df.apply(
-        lambda row: row['symbol'] + '.SZ' if row['classification'] in ['S', 'C']
-        else row['symbol'] + '.BJ' if row['classification'] in ['X']
-        else row['symbol'] + '.SH',
-        axis=1
-    )
-
-    db_name = extra_income_db_name.ONE_MINUTE_K_LINE_BFQ_ETF
-    db_create_index.create_index(mongodbUtilV2_27019, db_name)
-    for stock_one in etf_real_time_quotes_df.itertuples():
-        trade_date_list_df_copy = trade_date_list_df.copy()
-        symbol = stock_one.symbol
-        for i in range(0, len(trade_date_list_df_copy), 28):
-            try:
-                new_df = trade_date_list_df_copy.iloc[i:i + 28]
-                first_df = new_df.iloc[0]  # 第一个 DataFrame
-                last_df = new_df.iloc[-1]  # 最后一个 DataFrame
-                begin_date = first_df.trade_date + ' 09:20:00'
-                end_date = last_df.trade_date + ' 15:00:00'
-
-                df = get_minute_data(symbol, '1min', begin_date, end_date)
-                if data_frame_util.is_not_empty(df):
-                    df = df.rename(columns={
-                        "trade_time": "time",
-                        "ts_code": "symbol",
-                        "vol": "volume",
-                    })
-                    df['time_tick'] = df['time'].str[11:19]
-                    df = df.loc[df['time_tick'] <= '15:00:00']
-                    del df['time_tick']
-                    df['_id'] = df['symbol'] + '_' + df['time']
-                    mongodbUtilV2_27019.insert_mongo(df, db_name)
-            except BaseException as e:
-                time.sleep(2)
-                first_df = new_df.iloc[0]  # 第一个 DataFrame
-                last_df = new_df.iloc[-1]  # 最后一个 DataFrame
-                begin_date = first_df.trade_date + ' 09:20:00'
-                end_date = last_df.trade_date + ' 15:00:00'
-                fail_dict = {'begin_date': begin_date,
-                             'end_date': end_date,
-                             'symbol': symbol,
-                             'db_name': db_name
-                             }
-                fail_df = pd.DataFrame(fail_dict, index=[1])
-                mongodbUtilV2_27019.insert_mongo(fail_df, db_name + '_fail')
-
-                logger.error("同步数据出现异常:{},{},{},{}", e, symbol, begin_date, end_date)
-        logger.info("同步完数据:{},{}", stock_one.symbol, stock_one.name)
-
-    return etf_real_time_quotes_df
-
-
-def classify_symbol(debt_real_time_quotes_df):
-    debt_real_time_quotes_df['classification'] = debt_real_time_quotes_df['market'].apply(
-        lambda market: classify_symbol_one(market))
-    return debt_real_time_quotes_df
-
-
-# 单个股票分类
-def classify_symbol_one(market):
-    if market == 0:
-        return 'S'
-    else:
-        return 'H'
-
-
-if __name__ == '__main__':
-    query_trade = {"$and": [{"trade_date": {"$gte": "2025-03-08"}}, {"trade_date": {"$lte": "2025-03-16"}}]}
-    trade_date_list_df_all = mongodb_util_27017.find_query_data('trade_date_list', query_trade)
-
-    sync_etf_one_minute(trade_date_list_df_all)

mns_scheduler/extraIncome/temp/tu_share_data_kzz_sync.py

@@ -1,113 +0,0 @@
-import sys
-import os
-
-file_path = os.path.abspath(__file__)
-end = file_path.index('mns') + 16
-project_path = file_path[0:end]
-sys.path.append(project_path)
-
-import mns_common.api.em.east_money_debt_api as east_money_debt_api
-import pandas as pd
-from loguru import logger
-import time
-import mns_common.utils.data_frame_util as data_frame_util
-from mns_common.db.MongodbUtil import MongodbUtil
-import tushare as ts
-from mns_common.db.v2.MongodbUtilV2 import MongodbUtilV2
-import mns_scheduler.extraIncome.one_minute.common.db_create_index as db_create_index
-import mns_common.constant.extra_income_db_name as extra_income_db_name
-mongodb_util_27017 = MongodbUtil('27017')
-mongodbUtilV2_27019 = MongodbUtilV2('27019', extra_income_db_name.EXTRA_INCOME)
-pro = ts.pro_api('782213d20640249f1dbae50a7f56b22684b8e915a61e435e015579a1')
-
-
-def get_minute_data(symbol, freq, start_date, end_date):
-    # 获取浦发银行60000.SH的历史分钟数据
-    df = pro.stk_mins(ts_code=symbol, freq=freq, start_date=start_date, end_date=end_date)
-    return df
-
-
-def sync_kzz_one_minute(trade_date_list_df):
-    debt_real_time_quotes = east_money_debt_api.get_debt_real_time_quotes()
-    debt_real_time_quotes = classify_symbol(debt_real_time_quotes)
-    debt_real_time_quotes['symbol'] = debt_real_time_quotes.apply(
-        lambda row: row['symbol'] + '.SZ' if row['classification'] in ['S', 'C']
-        else row['symbol'] + '.BJ' if row['classification'] in ['X']
-        else row['symbol'] + '.SH',
-        axis=1
-    )
-    # 假设数字格式为 YYYYMMDD
-    # debt_real_time_quotes['list_date'] = pd.to_datetime(debt_real_time_quotes['list_date'],
-    #                                                     format='%Y%m%d').dt.strftime('%Y-%m-%d')
-
-    db_name = extra_income_db_name.ONE_MINUTE_K_LINE_BFQ_KZZ
-    db_create_index.create_index(mongodbUtilV2_27019, db_name)
-
-    for stock_one in debt_real_time_quotes.itertuples():
-
-        trade_date_list_df_copy = trade_date_list_df.copy()
-
-        # list_date = stock_one.list_date
-        # trade_date_list_df_copy = trade_date_list_df_copy.loc[trade_date_list_df_copy['_id'] >= list_date]
-
-        symbol = stock_one.symbol
-
-        for i in range(0, len(trade_date_list_df_copy), 28):
-            try:
-                new_df = trade_date_list_df_copy.iloc[i:i + 28]
-                first_df = new_df.iloc[0]  # 第一个 DataFrame
-                last_df = new_df.iloc[-1]  # 最后一个 DataFrame
-                begin_date = first_df.trade_date + ' 09:20:00'
-                end_date = last_df.trade_date + ' 15:00:00'
-
-                df = get_minute_data(symbol, '1min', begin_date, end_date)
-                if data_frame_util.is_not_empty(df):
-                    df = df.rename(columns={
-                        "trade_time": "time",
-                        "ts_code": "symbol",
-                        "vol": "volume",
-                    })
-                    df['time_tick'] = df['time'].str[11:19]
-                    df = df.loc[df['time_tick'] <= '15:00:00']
-                    del df['time_tick']
-                    df['_id'] = df['symbol'] + '_' + df['time']
-                    mongodbUtilV2_27019.insert_mongo(df, db_name)
-            except BaseException as e:
-                time.sleep(2)
-                first_df = new_df.iloc[0]  # 第一个 DataFrame
-                last_df = new_df.iloc[-1]  # 最后一个 DataFrame
-                begin_date = first_df.trade_date + ' 09:20:00'
-                end_date = last_df.trade_date + ' 15:00:00'
-                fail_dict = {'begin_date': begin_date,
-                             'end_date': end_date,
-                             'symbol': symbol,
-                             'db_name': db_name
-                             }
-                fail_df = pd.DataFrame(fail_dict, index=[1])
-                mongodbUtilV2_27019.insert_mongo(fail_df, db_name + '_fail')
-
-                logger.error("同步数据出现异常:{},{},{},{}", e, symbol, begin_date, end_date)
-        logger.info("同步完数据:{},{}", stock_one.symbol, stock_one.name)
-
-    return debt_real_time_quotes
-
-
-def classify_symbol(debt_real_time_quotes_df):
-    debt_real_time_quotes_df['classification'] = debt_real_time_quotes_df['market'].apply(
-        lambda market: classify_symbol_one(market))
-    return debt_real_time_quotes_df
-
-
-# 单个股票分类
-def classify_symbol_one(market):
-    if market == 0:
-        return 'S'
-    else:
-        return 'H'
-
-
-if __name__ == '__main__':
-    query_trade = {"$and": [{"trade_date": {"$gte": "2025-03-08"}}, {"trade_date": {"$lte": "2025-03-16"}}]}
-    trade_date_list_df_all = mongodb_util_27017.find_query_data('trade_date_list', query_trade)
-
-    sync_kzz_one_minute(trade_date_list_df_all)

mns_scheduler/extraIncome/temp/tu_share_data_stock_sync.py

@@ -1,137 +0,0 @@
-import sys
-import os
-
-file_path = os.path.abspath(__file__)
-end = file_path.index('mns') + 16
-project_path = file_path[0:end]
-sys.path.append(project_path)
-
-import tushare as ts
-from mns_common.db.v2.MongodbUtilV2 import MongodbUtilV2
-from mns_common.db.MongodbUtil import MongodbUtil
-import mns_common.component.common_service_fun_api as common_service_fun_api
-
-import pandas as pd
-from loguru import logger
-import mns_common.utils.data_frame_util as data_frame_util
-import mns_common.component.em.em_stock_info_api as em_stock_info_api
-import time
-
-mongodb_util_27017 = MongodbUtil('27017')
-mongodbUtilV2 = MongodbUtilV2('27019', 'extraIncome')
-
-pro = ts.pro_api('782213d20640249f1dbae50a7f56b22684b8e915a61e435e015579a1')
-
-
-def get_minute_data(symbol, freq, start_date, end_date):
-    # 获取浦发银行60000.SH的历史分钟数据
-    df = pro.stk_mins(ts_code=symbol, freq=freq, start_date=start_date, end_date=end_date)
-    return df
-
-
-def sync_all_stock(trade_date_list_df):
-    de_list_stock_df = mongodb_util_27017.find_all_data('de_list_stock')
-    de_list_stock_df = common_service_fun_api.classify_symbol(de_list_stock_df)
-    de_list_stock_df = de_list_stock_df.loc[
-        de_list_stock_df['classification'].isin(['K', 'C', 'S', 'H', 'X'])]
-    # 对 classification 为 S 或 K 的数据，symbol 列加上 '.SH'，其他加上 '.SZ'
-
-    de_list_stock_df['symbol'] = de_list_stock_df.apply(
-        lambda row: row['symbol'] + '.SZ' if row['classification'] in ['S', 'C']
-        else row['symbol'] + '.BJ' if row['classification'] in ['X']
-        else row['symbol'] + '.SH',
-        axis=1
-    )
-
-    real_time_quotes_all_stocks_df = em_stock_info_api.get_a_stock_info()
-
-    # 假设数字格式为 YYYYMMDD
-    real_time_quotes_all_stocks_df['list_date'] = pd.to_datetime(real_time_quotes_all_stocks_df['list_date'],
-                                                                 format='%Y%m%d')
-
-    # 将日期格式化为字符串（YYYY-MM-DD）
-    real_time_quotes_all_stocks_df['list_date'] = real_time_quotes_all_stocks_df['list_date'].dt.strftime('%Y-%m-%d')
-    real_time_quotes_all_stocks_df = common_service_fun_api.classify_symbol(real_time_quotes_all_stocks_df)
-
-    # 对 classification 为 S 或 K 的数据，symbol 列加上 '.SH'，其他加上 '.SZ'
-    real_time_quotes_all_stocks_df['symbol'] = real_time_quotes_all_stocks_df.apply(
-        lambda row: row['symbol'] + '.SZ' if row['classification'] in ['S', 'C']
-        else row['symbol'] + '.BJ' if row['classification'] in ['X']
-        else row['symbol'] + '.SH',
-        axis=1
-    )
-
-    real_time_quotes_all_stocks_df['number'] = real_time_quotes_all_stocks_df['chg'].rank(method='first').astype(int)
-
-    for stock_one in real_time_quotes_all_stocks_df.itertuples():
-        classification = stock_one.classification
-        if classification == 'X':
-            db_name = 'one_minute_k_line_bfq_bj'
-        elif classification == 'S':
-            db_name = 'one_minute_k_line_bfq_s'
-
-        elif classification == 'H':
-            db_name = 'one_minute_k_line_bfq_h'
-        elif classification == 'K':
-            db_name = 'one_minute_k_line_bfq_k'
-        elif classification == 'C':
-            db_name = 'one_minute_k_line_bfq_c'
-
-        trade_date_list_df_copy = trade_date_list_df.copy()
-
-        list_date = stock_one.list_date
-        trade_date_list_df_copy = trade_date_list_df_copy.loc[trade_date_list_df_copy['_id'] >= list_date]
-
-        symbol = stock_one.symbol
-
-        de_list_stock_df_one = de_list_stock_df.loc[de_list_stock_df['symbol'] == symbol]
-
-        if data_frame_util.is_not_empty(de_list_stock_df_one):
-            de_list_date = list(de_list_stock_df_one['de_list_date'])[0]
-            trade_date_list_df_copy = trade_date_list_df_copy.loc[trade_date_list_df_copy['_id'] <= de_list_date]
-
-        for i in range(0, len(trade_date_list_df_copy), 28):
-            try:
-                new_df = trade_date_list_df_copy.iloc[i:i + 28]
-                first_df = new_df.iloc[0]  # 第一个 DataFrame
-                last_df = new_df.iloc[-1]  # 最后一个 DataFrame
-                begin_date = first_df.trade_date + ' 09:20:00'
-                end_date = last_df.trade_date + ' 15:00:00'
-
-                df = get_minute_data(symbol, '1min', begin_date, end_date)
-                if data_frame_util.is_not_empty(df):
-                    df = df.rename(columns={
-                        "trade_time": "time",
-                        "ts_code": "symbol",
-                        "vol": "volume",
-                    })
-                    df['time_tick'] = df['time'].str[11:19]
-                    df = df.loc[df['time_tick'] <= '15:00:00']
-                    del df['time_tick']
-                    df['_id'] = df['symbol'] + '_' + df['time']
-                    mongodbUtilV2.insert_mongo(df, db_name)
-            except BaseException as e:
-                time.sleep(2)
-                first_df = new_df.iloc[0]  # 第一个 DataFrame
-                last_df = new_df.iloc[-1]  # 最后一个 DataFrame
-                begin_date = first_df.trade_date + ' 09:20:00'
-                end_date = last_df.trade_date + ' 15:00:00'
-                fail_dict = {'begin_date': begin_date,
-                             'end_date': end_date,
-                             'symbol': symbol,
-                             'db_name': db_name
-                             }
-                fail_df = pd.DataFrame(fail_dict, index=[1])
-                mongodbUtilV2.insert_mongo(fail_df, 'one_minute_k_line_bfq_fail')
-
-                logger.error("同步数据出现异常:{},{},{},{}", e, symbol, begin_date, end_date)
-        logger.info("同步完数据:{},{}", stock_one.symbol, stock_one.name)
-
-
-if __name__ == '__main__':
-    # get_minute_data('833284.BJ', '1min', '2025-02-28 09:30:00', '2025-02-28 15:00:00')
-
-    query_trade = {"$and": [{"trade_date": {"$gte": "2025-03-08"}}, {"trade_date": {"$lte": "2025-03-18"}}]}
-    trade_date_list_df_all = mongodb_util_27017.find_query_data('trade_date_list', query_trade)
-
-    sync_all_stock(trade_date_list_df_all)

mns_scheduler/extraIncome/temp/tu_share_zhi_shu_sync_api.py

@@ -1,107 +0,0 @@
-import sys
-import os
-
-file_path = os.path.abspath(__file__)
-end = file_path.index('mns') + 16
-project_path = file_path[0:end]
-sys.path.append(project_path)
-
-import sys
-import os
-
-file_path = os.path.abspath(__file__)
-end = file_path.index('mns') + 16
-project_path = file_path[0:end]
-sys.path.append(project_path)
-import pandas as pd
-from loguru import logger
-import time
-import mns_common.utils.data_frame_util as data_frame_util
-from mns_common.db.MongodbUtil import MongodbUtil
-import tushare as ts
-from mns_common.db.v2.MongodbUtilV2 import MongodbUtilV2
-import mns_scheduler.extraIncome.one_minute.common.db_create_index as db_create_index
-
-mongodb_util_27017 = MongodbUtil('27017')
-mongodbUtilV2_27019 = MongodbUtilV2('27019', 'extraIncome')
-
-pro = ts.pro_api('782213d20640249f1dbae50a7f56b22684b8e915a61e435e015579a1')
-
-
-def get_minute_data(symbol, freq, start_date, end_date):
-    # 获取浦发银行60000.SH的历史分钟数据
-    df = pro.stk_mins(ts_code=symbol, freq=freq, start_date=start_date, end_date=end_date)
-    return df
-
-
-def sync_zhi_shu_one_minute(symbol_list_param, trade_date_list_df):
-    db_name = 'one_minute_k_line_bfq_main_index'
-    db_create_index.create_index(mongodbUtilV2_27019, db_name)
-    for symbol in symbol_list_param:
-
-        trade_date_list_df_copy = trade_date_list_df.copy()
-
-        for i in range(0, len(trade_date_list_df_copy), 28):
-            try:
-                new_df = trade_date_list_df_copy.iloc[i:i + 28]
-                first_df = new_df.iloc[0]  # 第一个 DataFrame
-                last_df = new_df.iloc[-1]  # 最后一个 DataFrame
-                begin_date = first_df.trade_date + ' 09:20:00'
-                end_date = last_df.trade_date + ' 15:00:00'
-
-                df = get_minute_data(symbol, '1min', begin_date, end_date)
-                if data_frame_util.is_not_empty(df):
-                    df = df.rename(columns={
-                        "trade_time": "time",
-                        "ts_code": "symbol",
-                        "vol": "volume",
-                    })
-                    df['time_tick'] = df['time'].str[11:19]
-                    df = df.loc[df['time_tick'] <= '15:00:00']
-                    del df['time_tick']
-                    df['_id'] = df['symbol'] + '_' + df['time']
-                    mongodbUtilV2_27019.insert_mongo(df, db_name)
-            except BaseException as e:
-                time.sleep(2)
-                first_df = new_df.iloc[0]  # 第一个 DataFrame
-                last_df = new_df.iloc[-1]  # 最后一个 DataFrame
-                begin_date = first_df.trade_date + ' 09:20:00'
-                end_date = last_df.trade_date + ' 15:00:00'
-                fail_dict = {'begin_date': begin_date,
-                             'end_date': end_date,
-                             'symbol': symbol,
-                             'db_name': db_name
-                             }
-                fail_df = pd.DataFrame(fail_dict, index=[1])
-                mongodbUtilV2_27019.insert_mongo(fail_df, db_name + '_fail')
-
-                logger.error("同步数据出现异常:{},{},{},{}", e, symbol, begin_date, end_date)
-        logger.info("同步完数据:{}", symbol)
-
-
-def classify_symbol(debt_real_time_quotes_df):
-    debt_real_time_quotes_df['classification'] = debt_real_time_quotes_df['market'].apply(
-        lambda market: classify_symbol_one(market))
-    return debt_real_time_quotes_df
-
-
-# 单个股票分类
-def classify_symbol_one(market):
-    if market == 0:
-        return 'S'
-    else:
-        return 'H'
-
-
-if __name__ == '__main__':
-    # 文件路径
-    file_path = r"H:\data\1min\指数\指数列表.xlsx"
-
-    df = pd.read_excel(file_path)
-    df = df.rename(columns={
-        "指数代码": "symbol"
-    })
-    new_symbol_list = df['symbol']
-    query_trade = {"$and": [{"trade_date": {"$gte": "2025-03-08"}}, {"trade_date": {"$lte": "2025-03-16"}}]}
-    trade_date_list_df_all = mongodb_util_27017.find_query_data('trade_date_list', query_trade)
-    sync_zhi_shu_one_minute(new_symbol_list, trade_date_list_df_all)