mns-scheduler 1.3.0.3__tar.gz → 1.3.0.6__tar.gz

{mns-scheduler-1.3.0.3 → mns-scheduler-1.3.0.6}/PKG-INFO

@@ -1,3 +1,3 @@
 Metadata-Version: 2.1
 Name: mns-scheduler
-Version: 1.3.0.3
+Version: 1.3.0.6

mns-scheduler-1.3.0.6/mns_scheduler/extraIncome/hk/hk_stock_qfq_daily_k_line.py

@@ -0,0 +1,97 @@
+import os
+import sys
+
+file_path = os.path.abspath(__file__)
+end = file_path.index('mns') + 16
+project_path = file_path[0:end]
+sys.path.append(project_path)
+
+from mns_common.db.MongodbUtil import MongodbUtil
+from mns_common.db.v2.MongodbUtilV2 import MongodbUtilV2
+import mns_common.constant.extra_income_db_name as extra_income_db_name
+import mns_common.constant.db_name_constant as db_name_constant
+import akshare as ak
+from loguru import logger
+import mns_scheduler.baidu.baidu_yun_pan_handle_service as baidu_yun_pan_handle_service
+import mns_scheduler.hk.hk_company_info_sync_service_api as hk_company_info_sync_service_api
+
+mongodb_util_27017 = MongodbUtil('27017')
+mongodbUtilV2_27019 = MongodbUtilV2('27019', extra_income_db_name.EXTRA_INCOME)
+
+
+def hk_stock_daily_qfq_sync():
+    hk_stock_path = '/港股/qfq/'
+    hk_stock_info_df = mongodb_util_27017.find_all_data(db_name_constant.EM_HK_STOCK_INFO)
+    hk_stock_info_df = hk_stock_info_df.sort_values(by=['amount'], ascending=False)
+    del hk_stock_info_df['_id']
+    # 上传港股列表
+    baidu_yun_pan_handle_service.upload_to_baidu('港股列表', hk_stock_path, hk_stock_info_df)
+    hk_ggt_component_df = hk_company_info_sync_service_api.get_hk_ggt_component()
+    # 上传港股通列表
+    hk_stock_info_ggt_df = hk_stock_info_df.loc[hk_stock_info_df['symbol'].isin(hk_ggt_component_df['symbol'])]
+    baidu_yun_pan_handle_service.upload_to_baidu('港股通列表', hk_stock_path, hk_stock_info_ggt_df)
+
+    fail_list = []
+
+    for hk_stock_one in hk_stock_info_df.itertuples():
+        symbol = hk_stock_one.symbol
+        name = hk_stock_one.name
+        try:
+            save_one_symbol(symbol, hk_stock_path)
+
+        except BaseException as e:
+            logger.error("同步出现异常:{},{},{}", e, symbol, name)
+            fail_list.append(symbol)
+    # 处理失败的
+    for symbol_fail in fail_list:
+        try:
+            save_one_symbol(symbol_fail, hk_stock_path)
+        except BaseException as e:
+            logger.error("同步出现异常:{},{},{}", e, symbol, name)
+
+
+def save_one_symbol(symbol, hk_stock_path):
+    hk_stock_k_line_df = hk_stock_k_line_api(symbol, k_line_period='daily', start_date='18000101',
+                                             end_date='22220101', fq='qfq')
+
+    hk_stock_k_line_df["date"] = hk_stock_k_line_df["date"].astype(str)
+
+    hk_stock_k_line_df['_id'] = hk_stock_k_line_df['date'] + '_' + symbol
+    hk_stock_k_line_df['symbol'] = symbol
+
+    query = {'symbol': symbol}
+    if mongodbUtilV2_27019.remove_data(query, extra_income_db_name.HK_STOCK_DAILY_QFQ_K_LINE).acknowledged > 0:
+        mongodbUtilV2_27019.insert_mongo(hk_stock_k_line_df, extra_income_db_name.HK_STOCK_DAILY_QFQ_K_LINE)
+        del hk_stock_k_line_df['_id']
+        del hk_stock_k_line_df['symbol']
+        # 上传列表
+        baidu_yun_pan_handle_service.upload_to_baidu(symbol, hk_stock_path, hk_stock_k_line_df)
+
+
+def hk_stock_k_line_api(symbol='00001', k_line_period='daily', start_date='18000101',
+                        end_date='22220101', fq='qfq'):
+    stock_hk_hist_df = ak.stock_hk_hist(symbol=symbol,
+                                        period=k_line_period,
+                                        start_date=start_date,
+                                        end_date=end_date,
+                                        adjust=fq)
+    stock_hk_hist_df = stock_hk_hist_df.rename(columns={
+        "日期": "date",
+        "涨跌额": "change_price",
+        "涨跌幅": "chg",
+        "开盘": "open",
+        "最高": "high",
+        "最低": "low",
+        "收盘": "close",
+        "成交量": "volume",
+        "成交额": "amount",
+        "振幅": "pct_chg",
+        "换手率": "exchange"
+    })
+
+    return stock_hk_hist_df
+
+
+if __name__ == '__main__':
+    # us_stock_k_line_api()
+    hk_stock_daily_qfq_sync()

mns-scheduler-1.3.0.6/mns_scheduler/extraIncome/us/us_stock_qfq_daily_k_line.py

@@ -0,0 +1,95 @@
+import os
+import sys
+
+file_path = os.path.abspath(__file__)
+end = file_path.index('mns') + 16
+project_path = file_path[0:end]
+sys.path.append(project_path)
+
+from mns_common.db.MongodbUtil import MongodbUtil
+from mns_common.db.v2.MongodbUtilV2 import MongodbUtilV2
+import mns_common.constant.extra_income_db_name as extra_income_db_name
+import mns_common.constant.db_name_constant as db_name_constant
+import akshare as ak
+from loguru import logger
+import mns_scheduler.baidu.baidu_yun_pan_handle_service as baidu_yun_pan_handle_service
+
+mongodb_util_27017 = MongodbUtil('27017')
+mongodbUtilV2_27019 = MongodbUtilV2('27019', extra_income_db_name.EXTRA_INCOME)
+
+
+def us_stock_daily_qfq_sync():
+    us_stock_path = '/美股/qfq/'
+    em_us_stock_info_df = mongodb_util_27017.find_all_data(db_name_constant.EM_US_STOCK_INFO)
+    # 将列 A 转为字符串，并提取前三位
+    em_us_stock_info_df["simple_symbol"] = em_us_stock_info_df["simple_symbol"].astype(str).str[:3]
+    em_us_stock_info_df = em_us_stock_info_df.sort_values(by=['amount'], ascending=False)
+    # 上传列表
+    baidu_yun_pan_handle_service.upload_to_baidu('美股列表', us_stock_path, em_us_stock_info_df)
+    fail_list = []
+    for us_stock_one in em_us_stock_info_df.itertuples():
+        symbol = us_stock_one.symbol
+        name = us_stock_one.name
+        try:
+            save_one_symbol(us_stock_one, us_stock_path, symbol)
+        except BaseException as e:
+            logger.error("同步出现异常:{},{},{}", e, symbol, name)
+            fail_list.append(symbol)
+    fail_stock_df = em_us_stock_info_df.loc[em_us_stock_info_df['symbol'].isin(fail_list)]
+
+    for us_stock_fail_one in fail_stock_df.itertuples():
+        symbol = us_stock_fail_one.symbol
+        name = us_stock_fail_one.name
+        try:
+            save_one_symbol(us_stock_fail_one, us_stock_path, symbol)
+        except BaseException as e:
+            logger.error("同步出现异常:{},{},{}", e, symbol, name)
+            fail_list.append(symbol)
+
+
+def save_one_symbol(us_stock_one, us_stock_path, symbol):
+    simple_symbol = us_stock_one.simple_symbol
+    code = simple_symbol + '.' + symbol
+
+    us_stock_k_line_df = us_stock_k_line_api(code, k_line_period='daily', start_date='18000101',
+                                             end_date='22220101', fq='qfq')
+    us_stock_k_line_df['_id'] = us_stock_k_line_df['date'] + '_' + symbol
+    us_stock_k_line_df['symbol'] = symbol
+
+    query = {'symbol': symbol}
+    if mongodbUtilV2_27019.remove_data(query, extra_income_db_name.US_STOCK_DAILY_QFQ_K_LINE).acknowledged > 0:
+        mongodbUtilV2_27019.insert_mongo(us_stock_k_line_df, extra_income_db_name.US_STOCK_DAILY_QFQ_K_LINE)
+        del us_stock_k_line_df['_id']
+        del us_stock_k_line_df['symbol']
+
+        # 上传列表
+        baidu_yun_pan_handle_service.upload_to_baidu(symbol, us_stock_path, us_stock_k_line_df)
+
+
+def us_stock_k_line_api(symbol='106.GE', k_line_period='daily', start_date='18000101',
+                        end_date='22220101', fq='hfq'):
+    stock_us_hist_df = ak.stock_us_hist(symbol=symbol,
+                                        period=k_line_period,
+                                        start_date=start_date,
+                                        end_date=end_date,
+                                        adjust=fq)
+    stock_us_hist_df = stock_us_hist_df.rename(columns={
+        "日期": "date",
+        "涨跌额": "change_price",
+        "涨跌幅": "chg",
+        "开盘": "open",
+        "最高": "high",
+        "最低": "low",
+        "收盘": "close",
+        "成交量": "volume",
+        "成交额": "amount",
+        "振幅": "pct_chg",
+        "换手率": "exchange"
+    })
+
+    return stock_us_hist_df
+
+
+if __name__ == '__main__':
+    # us_stock_k_line_api()
+    us_stock_daily_qfq_sync()

{mns-scheduler-1.3.0.3 → mns-scheduler-1.3.0.6}/mns_scheduler/hk/hk_company_info_sync_service_api.py

@@ -99,4 +99,5 @@ def sync_hk_company_info():
 
 
 if __name__ == '__main__':
+    get_hk_ggt_component()
     sync_hk_company_info()

mns-scheduler-1.3.0.6/mns_scheduler/zt/script/__init__.py

@@ -0,0 +1,7 @@
+import sys
+import os
+
+file_path = os.path.abspath(__file__)
+end = file_path.index('mns') + 16
+project_path = file_path[0:end]
+sys.path.append(project_path)

{mns-scheduler-1.3.0.3 → mns-scheduler-1.3.0.6}/mns_scheduler/zz_task/data_sync_task.py

@@ -63,6 +63,9 @@ import mns_scheduler.industry.ths.ths_industry_sync_service as ths_industry_sync
 import mns_scheduler.k_line.year_quarter.year_quarter_line_sync as year_quarter_line_sync
 import mns_common.component.task.real_time_data_sync_check as real_time_data_sync_check
 import mns_scheduler.extraIncome.one_minute.one_minute_sync_task as one_minute_sync_task
+import mns_scheduler.extraIncome.hk.hk_stock_qfq_daily_k_line as hk_stock_qfq_daily_k_line
+import mns_scheduler.extraIncome.us.us_stock_qfq_daily_k_line as us_stock_qfq_daily_k_line
+import mns_scheduler.extraIncome.one_minute.upload.upload_to_baidu_task as upload_to_baidu_task
 
 
 # 同步交易日期任务完成
@@ -443,12 +446,30 @@ def sync_all_em_stock_info():
     sync_em_stock_info_sync.sync_all_em_stock_info()
 
 
+# 同步外快收入数据
+def sync_extra_income_data():
+    # 上传一分钟数据到百度网盘
+    upload_to_baidu_task.upload_stock_to_baidu()
+    # hk前复权k线数据
+    hk_stock_qfq_daily_k_line.hk_stock_daily_qfq_sync()
+    # us前复权k线数据
+    us_stock_qfq_daily_k_line.us_stock_daily_qfq_sync()
+
+
 # # 定义BlockingScheduler
 blockingScheduler = BlockingScheduler()
 
 #  同步东方财富a,etf,kzz,us,hk信息
 blockingScheduler.add_job(sync_all_em_stock_info, 'cron', hour='07', minute='31')
 blockingScheduler.add_job(sync_all_em_stock_info, 'cron', hour='15', minute='20')
+# 同步外快收入数据
+blockingScheduler.add_job(
+    sync_extra_income_data,
+    'cron',
+    day_of_week='sat,sun',  # 周六和周日
+    hour='07,18',  # 16点和21点
+    minute='37'  # 37分
+)
 
 # 开盘前同步当天交易需要的k线数据
 blockingScheduler.add_job(sync_today_trade_k_line_info, 'cron', hour='07', minute='50')

{mns-scheduler-1.3.0.3 → mns-scheduler-1.3.0.6}/mns_scheduler.egg-info/PKG-INFO

@@ -1,3 +1,3 @@
 Metadata-Version: 2.1
 Name: mns-scheduler
-Version: 1.3.0.3
+Version: 1.3.0.6

{mns-scheduler-1.3.0.3 → mns-scheduler-1.3.0.6}/mns_scheduler.egg-info/SOURCES.txt

@@ -60,6 +60,8 @@ mns_scheduler/debt/kzz_bond_info_sync.py
 mns_scheduler/dt/__init__.py
 mns_scheduler/dt/stock_dt_pool_sync.py
 mns_scheduler/extraIncome/__init__.py
+mns_scheduler/extraIncome/hk/__init__.py
+mns_scheduler/extraIncome/hk/hk_stock_qfq_daily_k_line.py
 mns_scheduler/extraIncome/one_minute/__init__.py
 mns_scheduler/extraIncome/one_minute/one_minute_sync_task.py
 mns_scheduler/extraIncome/one_minute/common/__init__.py
@@ -75,11 +77,8 @@ mns_scheduler/extraIncome/one_minute/stock/__init__.py
 mns_scheduler/extraIncome/one_minute/stock/stock_one_minute_sync_task.py
 mns_scheduler/extraIncome/one_minute/upload/__init__.py
 mns_scheduler/extraIncome/one_minute/upload/upload_to_baidu_task.py
-mns_scheduler/extraIncome/temp/__init__.py
-mns_scheduler/extraIncome/temp/tu_share_data_etf_sync.py
-mns_scheduler/extraIncome/temp/tu_share_data_kzz_sync.py
-mns_scheduler/extraIncome/temp/tu_share_data_stock_sync.py
-mns_scheduler/extraIncome/temp/tu_share_zhi_shu_sync_api.py
+mns_scheduler/extraIncome/us/__init__.py
+mns_scheduler/extraIncome/us/us_stock_qfq_daily_k_line.py
 mns_scheduler/finance/__init__.py
 mns_scheduler/finance/em_financial_asset_liability_sync_service_api.py
 mns_scheduler/finance/em_financial_profit_sync_service_api.py

{mns-scheduler-1.3.0.3 → mns-scheduler-1.3.0.6}/setup.py

@@ -2,7 +2,7 @@ from setuptools import setup, find_packages
 
 setup(
     name='mns-scheduler',
-    version='1.3.0.3',
+    version='1.3.0.6',
     packages=find_packages(),
     install_requires=[],  # 如果有依赖项，可以在这里列出
 )

mns-scheduler-1.3.0.3/mns_scheduler/extraIncome/temp/tu_share_data_etf_sync.py

@@ -1,103 +0,0 @@
-import sys
-import os
-
-file_path = os.path.abspath(__file__)
-end = file_path.index('mns') + 16
-project_path = file_path[0:end]
-sys.path.append(project_path)
-import mns_common.api.em.east_money_etf_api as east_money_etf_api
-import pandas as pd
-from loguru import logger
-import time
-import mns_common.utils.data_frame_util as data_frame_util
-from mns_common.db.MongodbUtil import MongodbUtil
-import tushare as ts
-from mns_common.db.v2.MongodbUtilV2 import MongodbUtilV2
-import mns_scheduler.extraIncome.one_minute.common.db_create_index as db_create_index
-import mns_common.constant.extra_income_db_name as extra_income_db_name
-
-mongodb_util_27017 = MongodbUtil('27017')
-mongodbUtilV2_27019 = MongodbUtilV2('27019', extra_income_db_name.EXTRA_INCOME)
-pro = ts.pro_api('782213d20640249f1dbae50a7f56b22684b8e915a61e435e015579a1')
-
-
-def get_minute_data(symbol, freq, start_date, end_date):
-    # 获取浦发银行60000.SH的历史分钟数据
-    df = pro.stk_mins(ts_code=symbol, freq=freq, start_date=start_date, end_date=end_date)
-    return df
-
-
-def sync_etf_one_minute(trade_date_list_df):
-    etf_real_time_quotes_df = east_money_etf_api.get_etf_real_time_quotes()
-    etf_real_time_quotes_df = classify_symbol(etf_real_time_quotes_df)
-    etf_real_time_quotes_df['symbol'] = etf_real_time_quotes_df.apply(
-        lambda row: row['symbol'] + '.SZ' if row['classification'] in ['S', 'C']
-        else row['symbol'] + '.BJ' if row['classification'] in ['X']
-        else row['symbol'] + '.SH',
-        axis=1
-    )
-
-    db_name = extra_income_db_name.ONE_MINUTE_K_LINE_BFQ_ETF
-    db_create_index.create_index(mongodbUtilV2_27019, db_name)
-    for stock_one in etf_real_time_quotes_df.itertuples():
-        trade_date_list_df_copy = trade_date_list_df.copy()
-        symbol = stock_one.symbol
-        for i in range(0, len(trade_date_list_df_copy), 28):
-            try:
-                new_df = trade_date_list_df_copy.iloc[i:i + 28]
-                first_df = new_df.iloc[0]  # 第一个 DataFrame
-                last_df = new_df.iloc[-1]  # 最后一个 DataFrame
-                begin_date = first_df.trade_date + ' 09:20:00'
-                end_date = last_df.trade_date + ' 15:00:00'
-
-                df = get_minute_data(symbol, '1min', begin_date, end_date)
-                if data_frame_util.is_not_empty(df):
-                    df = df.rename(columns={
-                        "trade_time": "time",
-                        "ts_code": "symbol",
-                        "vol": "volume",
-                    })
-                    df['time_tick'] = df['time'].str[11:19]
-                    df = df.loc[df['time_tick'] <= '15:00:00']
-                    del df['time_tick']
-                    df['_id'] = df['symbol'] + '_' + df['time']
-                    mongodbUtilV2_27019.insert_mongo(df, db_name)
-            except BaseException as e:
-                time.sleep(2)
-                first_df = new_df.iloc[0]  # 第一个 DataFrame
-                last_df = new_df.iloc[-1]  # 最后一个 DataFrame
-                begin_date = first_df.trade_date + ' 09:20:00'
-                end_date = last_df.trade_date + ' 15:00:00'
-                fail_dict = {'begin_date': begin_date,
-                             'end_date': end_date,
-                             'symbol': symbol,
-                             'db_name': db_name
-                             }
-                fail_df = pd.DataFrame(fail_dict, index=[1])
-                mongodbUtilV2_27019.insert_mongo(fail_df, db_name + '_fail')
-
-                logger.error("同步数据出现异常:{},{},{},{}", e, symbol, begin_date, end_date)
-        logger.info("同步完数据:{},{}", stock_one.symbol, stock_one.name)
-
-    return etf_real_time_quotes_df
-
-
-def classify_symbol(debt_real_time_quotes_df):
-    debt_real_time_quotes_df['classification'] = debt_real_time_quotes_df['market'].apply(
-        lambda market: classify_symbol_one(market))
-    return debt_real_time_quotes_df
-
-
-# 单个股票分类
-def classify_symbol_one(market):
-    if market == 0:
-        return 'S'
-    else:
-        return 'H'
-
-
-if __name__ == '__main__':
-    query_trade = {"$and": [{"trade_date": {"$gte": "2025-03-08"}}, {"trade_date": {"$lte": "2025-03-16"}}]}
-    trade_date_list_df_all = mongodb_util_27017.find_query_data('trade_date_list', query_trade)
-
-    sync_etf_one_minute(trade_date_list_df_all)

mns-scheduler-1.3.0.3/mns_scheduler/extraIncome/temp/tu_share_data_kzz_sync.py

@@ -1,113 +0,0 @@
-import sys
-import os
-
-file_path = os.path.abspath(__file__)
-end = file_path.index('mns') + 16
-project_path = file_path[0:end]
-sys.path.append(project_path)
-
-import mns_common.api.em.east_money_debt_api as east_money_debt_api
-import pandas as pd
-from loguru import logger
-import time
-import mns_common.utils.data_frame_util as data_frame_util
-from mns_common.db.MongodbUtil import MongodbUtil
-import tushare as ts
-from mns_common.db.v2.MongodbUtilV2 import MongodbUtilV2
-import mns_scheduler.extraIncome.one_minute.common.db_create_index as db_create_index
-import mns_common.constant.extra_income_db_name as extra_income_db_name
-mongodb_util_27017 = MongodbUtil('27017')
-mongodbUtilV2_27019 = MongodbUtilV2('27019', extra_income_db_name.EXTRA_INCOME)
-pro = ts.pro_api('782213d20640249f1dbae50a7f56b22684b8e915a61e435e015579a1')
-
-
-def get_minute_data(symbol, freq, start_date, end_date):
-    # 获取浦发银行60000.SH的历史分钟数据
-    df = pro.stk_mins(ts_code=symbol, freq=freq, start_date=start_date, end_date=end_date)
-    return df
-
-
-def sync_kzz_one_minute(trade_date_list_df):
-    debt_real_time_quotes = east_money_debt_api.get_debt_real_time_quotes()
-    debt_real_time_quotes = classify_symbol(debt_real_time_quotes)
-    debt_real_time_quotes['symbol'] = debt_real_time_quotes.apply(
-        lambda row: row['symbol'] + '.SZ' if row['classification'] in ['S', 'C']
-        else row['symbol'] + '.BJ' if row['classification'] in ['X']
-        else row['symbol'] + '.SH',
-        axis=1
-    )
-    # 假设数字格式为 YYYYMMDD
-    # debt_real_time_quotes['list_date'] = pd.to_datetime(debt_real_time_quotes['list_date'],
-    #                                                     format='%Y%m%d').dt.strftime('%Y-%m-%d')
-
-    db_name = extra_income_db_name.ONE_MINUTE_K_LINE_BFQ_KZZ
-    db_create_index.create_index(mongodbUtilV2_27019, db_name)
-
-    for stock_one in debt_real_time_quotes.itertuples():
-
-        trade_date_list_df_copy = trade_date_list_df.copy()
-
-        # list_date = stock_one.list_date
-        # trade_date_list_df_copy = trade_date_list_df_copy.loc[trade_date_list_df_copy['_id'] >= list_date]
-
-        symbol = stock_one.symbol
-
-        for i in range(0, len(trade_date_list_df_copy), 28):
-            try:
-                new_df = trade_date_list_df_copy.iloc[i:i + 28]
-                first_df = new_df.iloc[0]  # 第一个 DataFrame
-                last_df = new_df.iloc[-1]  # 最后一个 DataFrame
-                begin_date = first_df.trade_date + ' 09:20:00'
-                end_date = last_df.trade_date + ' 15:00:00'
-
-                df = get_minute_data(symbol, '1min', begin_date, end_date)
-                if data_frame_util.is_not_empty(df):
-                    df = df.rename(columns={
-                        "trade_time": "time",
-                        "ts_code": "symbol",
-                        "vol": "volume",
-                    })
-                    df['time_tick'] = df['time'].str[11:19]
-                    df = df.loc[df['time_tick'] <= '15:00:00']
-                    del df['time_tick']
-                    df['_id'] = df['symbol'] + '_' + df['time']
-                    mongodbUtilV2_27019.insert_mongo(df, db_name)
-            except BaseException as e:
-                time.sleep(2)
-                first_df = new_df.iloc[0]  # 第一个 DataFrame
-                last_df = new_df.iloc[-1]  # 最后一个 DataFrame
-                begin_date = first_df.trade_date + ' 09:20:00'
-                end_date = last_df.trade_date + ' 15:00:00'
-                fail_dict = {'begin_date': begin_date,
-                             'end_date': end_date,
-                             'symbol': symbol,
-                             'db_name': db_name
-                             }
-                fail_df = pd.DataFrame(fail_dict, index=[1])
-                mongodbUtilV2_27019.insert_mongo(fail_df, db_name + '_fail')
-
-                logger.error("同步数据出现异常:{},{},{},{}", e, symbol, begin_date, end_date)
-        logger.info("同步完数据:{},{}", stock_one.symbol, stock_one.name)
-
-    return debt_real_time_quotes
-
-
-def classify_symbol(debt_real_time_quotes_df):
-    debt_real_time_quotes_df['classification'] = debt_real_time_quotes_df['market'].apply(
-        lambda market: classify_symbol_one(market))
-    return debt_real_time_quotes_df
-
-
-# 单个股票分类
-def classify_symbol_one(market):
-    if market == 0:
-        return 'S'
-    else:
-        return 'H'
-
-
-if __name__ == '__main__':
-    query_trade = {"$and": [{"trade_date": {"$gte": "2025-03-08"}}, {"trade_date": {"$lte": "2025-03-16"}}]}
-    trade_date_list_df_all = mongodb_util_27017.find_query_data('trade_date_list', query_trade)
-
-    sync_kzz_one_minute(trade_date_list_df_all)

mns-scheduler-1.3.0.3/mns_scheduler/extraIncome/temp/tu_share_data_stock_sync.py

@@ -1,137 +0,0 @@
-import sys
-import os
-
-file_path = os.path.abspath(__file__)
-end = file_path.index('mns') + 16
-project_path = file_path[0:end]
-sys.path.append(project_path)
-
-import tushare as ts
-from mns_common.db.v2.MongodbUtilV2 import MongodbUtilV2
-from mns_common.db.MongodbUtil import MongodbUtil
-import mns_common.component.common_service_fun_api as common_service_fun_api
-
-import pandas as pd
-from loguru import logger
-import mns_common.utils.data_frame_util as data_frame_util
-import mns_common.component.em.em_stock_info_api as em_stock_info_api
-import time
-
-mongodb_util_27017 = MongodbUtil('27017')
-mongodbUtilV2 = MongodbUtilV2('27019', 'extraIncome')
-
-pro = ts.pro_api('782213d20640249f1dbae50a7f56b22684b8e915a61e435e015579a1')
-
-
-def get_minute_data(symbol, freq, start_date, end_date):
-    # 获取浦发银行60000.SH的历史分钟数据
-    df = pro.stk_mins(ts_code=symbol, freq=freq, start_date=start_date, end_date=end_date)
-    return df
-
-
-def sync_all_stock(trade_date_list_df):
-    de_list_stock_df = mongodb_util_27017.find_all_data('de_list_stock')
-    de_list_stock_df = common_service_fun_api.classify_symbol(de_list_stock_df)
-    de_list_stock_df = de_list_stock_df.loc[
-        de_list_stock_df['classification'].isin(['K', 'C', 'S', 'H', 'X'])]
-    # 对 classification 为 S 或 K 的数据，symbol 列加上 '.SH'，其他加上 '.SZ'
-
-    de_list_stock_df['symbol'] = de_list_stock_df.apply(
-        lambda row: row['symbol'] + '.SZ' if row['classification'] in ['S', 'C']
-        else row['symbol'] + '.BJ' if row['classification'] in ['X']
-        else row['symbol'] + '.SH',
-        axis=1
-    )
-
-    real_time_quotes_all_stocks_df = em_stock_info_api.get_a_stock_info()
-
-    # 假设数字格式为 YYYYMMDD
-    real_time_quotes_all_stocks_df['list_date'] = pd.to_datetime(real_time_quotes_all_stocks_df['list_date'],
-                                                                 format='%Y%m%d')
-
-    # 将日期格式化为字符串（YYYY-MM-DD）
-    real_time_quotes_all_stocks_df['list_date'] = real_time_quotes_all_stocks_df['list_date'].dt.strftime('%Y-%m-%d')
-    real_time_quotes_all_stocks_df = common_service_fun_api.classify_symbol(real_time_quotes_all_stocks_df)
-
-    # 对 classification 为 S 或 K 的数据，symbol 列加上 '.SH'，其他加上 '.SZ'
-    real_time_quotes_all_stocks_df['symbol'] = real_time_quotes_all_stocks_df.apply(
-        lambda row: row['symbol'] + '.SZ' if row['classification'] in ['S', 'C']
-        else row['symbol'] + '.BJ' if row['classification'] in ['X']
-        else row['symbol'] + '.SH',
-        axis=1
-    )
-
-    real_time_quotes_all_stocks_df['number'] = real_time_quotes_all_stocks_df['chg'].rank(method='first').astype(int)
-
-    for stock_one in real_time_quotes_all_stocks_df.itertuples():
-        classification = stock_one.classification
-        if classification == 'X':
-            db_name = 'one_minute_k_line_bfq_bj'
-        elif classification == 'S':
-            db_name = 'one_minute_k_line_bfq_s'
-
-        elif classification == 'H':
-            db_name = 'one_minute_k_line_bfq_h'
-        elif classification == 'K':
-            db_name = 'one_minute_k_line_bfq_k'
-        elif classification == 'C':
-            db_name = 'one_minute_k_line_bfq_c'
-
-        trade_date_list_df_copy = trade_date_list_df.copy()
-
-        list_date = stock_one.list_date
-        trade_date_list_df_copy = trade_date_list_df_copy.loc[trade_date_list_df_copy['_id'] >= list_date]
-
-        symbol = stock_one.symbol
-
-        de_list_stock_df_one = de_list_stock_df.loc[de_list_stock_df['symbol'] == symbol]
-
-        if data_frame_util.is_not_empty(de_list_stock_df_one):
-            de_list_date = list(de_list_stock_df_one['de_list_date'])[0]
-            trade_date_list_df_copy = trade_date_list_df_copy.loc[trade_date_list_df_copy['_id'] <= de_list_date]
-
-        for i in range(0, len(trade_date_list_df_copy), 28):
-            try:
-                new_df = trade_date_list_df_copy.iloc[i:i + 28]
-                first_df = new_df.iloc[0]  # 第一个 DataFrame
-                last_df = new_df.iloc[-1]  # 最后一个 DataFrame
-                begin_date = first_df.trade_date + ' 09:20:00'
-                end_date = last_df.trade_date + ' 15:00:00'
-
-                df = get_minute_data(symbol, '1min', begin_date, end_date)
-                if data_frame_util.is_not_empty(df):
-                    df = df.rename(columns={
-                        "trade_time": "time",
-                        "ts_code": "symbol",
-                        "vol": "volume",
-                    })
-                    df['time_tick'] = df['time'].str[11:19]
-                    df = df.loc[df['time_tick'] <= '15:00:00']
-                    del df['time_tick']
-                    df['_id'] = df['symbol'] + '_' + df['time']
-                    mongodbUtilV2.insert_mongo(df, db_name)
-            except BaseException as e:
-                time.sleep(2)
-                first_df = new_df.iloc[0]  # 第一个 DataFrame
-                last_df = new_df.iloc[-1]  # 最后一个 DataFrame
-                begin_date = first_df.trade_date + ' 09:20:00'
-                end_date = last_df.trade_date + ' 15:00:00'
-                fail_dict = {'begin_date': begin_date,
-                             'end_date': end_date,
-                             'symbol': symbol,
-                             'db_name': db_name
-                             }
-                fail_df = pd.DataFrame(fail_dict, index=[1])
-                mongodbUtilV2.insert_mongo(fail_df, 'one_minute_k_line_bfq_fail')
-
-                logger.error("同步数据出现异常:{},{},{},{}", e, symbol, begin_date, end_date)
-        logger.info("同步完数据:{},{}", stock_one.symbol, stock_one.name)
-
-
-if __name__ == '__main__':
-    # get_minute_data('833284.BJ', '1min', '2025-02-28 09:30:00', '2025-02-28 15:00:00')
-
-    query_trade = {"$and": [{"trade_date": {"$gte": "2025-03-08"}}, {"trade_date": {"$lte": "2025-03-18"}}]}
-    trade_date_list_df_all = mongodb_util_27017.find_query_data('trade_date_list', query_trade)
-
-    sync_all_stock(trade_date_list_df_all)