mns-scheduler 1.2.9.6__py3-none-any.whl → 1.4.2.5__py3-none-any.whl

mns_scheduler/extraIncome/temp/tu_share_data_stock_sync.py

@@ -1,137 +0,0 @@
-import sys
-import os
-
-file_path = os.path.abspath(__file__)
-end = file_path.index('mns') + 16
-project_path = file_path[0:end]
-sys.path.append(project_path)
-
-import tushare as ts
-from mns_common.db.v2.MongodbUtilV2 import MongodbUtilV2
-from mns_common.db.MongodbUtil import MongodbUtil
-import mns_common.component.common_service_fun_api as common_service_fun_api
-
-import pandas as pd
-from loguru import logger
-import mns_common.utils.data_frame_util as data_frame_util
-import mns_common.component.em.em_stock_info_api as em_stock_info_api
-import time
-
-mongodb_util_27017 = MongodbUtil('27017')
-mongodbUtilV2 = MongodbUtilV2('27019', 'extraIncome')
-
-pro = ts.pro_api('782213d20640249f1dbae50a7f56b22684b8e915a61e435e015579a1')
-
-
-def get_minute_data(symbol, freq, start_date, end_date):
-    # 获取浦发银行60000.SH的历史分钟数据
-    df = pro.stk_mins(ts_code=symbol, freq=freq, start_date=start_date, end_date=end_date)
-    return df
-
-
-def sync_all_stock(trade_date_list_df):
-    de_list_stock_df = mongodb_util_27017.find_all_data('de_list_stock')
-    de_list_stock_df = common_service_fun_api.classify_symbol(de_list_stock_df)
-    de_list_stock_df = de_list_stock_df.loc[
-        de_list_stock_df['classification'].isin(['K', 'C', 'S', 'H', 'X'])]
-    # 对 classification 为 S 或 K 的数据，symbol 列加上 '.SH'，其他加上 '.SZ'
-
-    de_list_stock_df['symbol'] = de_list_stock_df.apply(
-        lambda row: row['symbol'] + '.SZ' if row['classification'] in ['S', 'C']
-        else row['symbol'] + '.BJ' if row['classification'] in ['X']
-        else row['symbol'] + '.SH',
-        axis=1
-    )
-
-    real_time_quotes_all_stocks_df = em_stock_info_api.get_a_stock_info()
-
-    # 假设数字格式为 YYYYMMDD
-    real_time_quotes_all_stocks_df['list_date'] = pd.to_datetime(real_time_quotes_all_stocks_df['list_date'],
-                                                                 format='%Y%m%d')
-
-    # 将日期格式化为字符串（YYYY-MM-DD）
-    real_time_quotes_all_stocks_df['list_date'] = real_time_quotes_all_stocks_df['list_date'].dt.strftime('%Y-%m-%d')
-    real_time_quotes_all_stocks_df = common_service_fun_api.classify_symbol(real_time_quotes_all_stocks_df)
-
-    # 对 classification 为 S 或 K 的数据，symbol 列加上 '.SH'，其他加上 '.SZ'
-    real_time_quotes_all_stocks_df['symbol'] = real_time_quotes_all_stocks_df.apply(
-        lambda row: row['symbol'] + '.SZ' if row['classification'] in ['S', 'C']
-        else row['symbol'] + '.BJ' if row['classification'] in ['X']
-        else row['symbol'] + '.SH',
-        axis=1
-    )
-
-    real_time_quotes_all_stocks_df['number'] = real_time_quotes_all_stocks_df['chg'].rank(method='first').astype(int)
-
-    for stock_one in real_time_quotes_all_stocks_df.itertuples():
-        classification = stock_one.classification
-        if classification == 'X':
-            db_name = 'one_minute_k_line_bfq_bj'
-        elif classification == 'S':
-            db_name = 'one_minute_k_line_bfq_s'
-
-        elif classification == 'H':
-            db_name = 'one_minute_k_line_bfq_h'
-        elif classification == 'K':
-            db_name = 'one_minute_k_line_bfq_k'
-        elif classification == 'C':
-            db_name = 'one_minute_k_line_bfq_c'
-
-        trade_date_list_df_copy = trade_date_list_df.copy()
-
-        list_date = stock_one.list_date
-        trade_date_list_df_copy = trade_date_list_df_copy.loc[trade_date_list_df_copy['_id'] >= list_date]
-
-        symbol = stock_one.symbol
-
-        de_list_stock_df_one = de_list_stock_df.loc[de_list_stock_df['symbol'] == symbol]
-
-        if data_frame_util.is_not_empty(de_list_stock_df_one):
-            de_list_date = list(de_list_stock_df_one['de_list_date'])[0]
-            trade_date_list_df_copy = trade_date_list_df_copy.loc[trade_date_list_df_copy['_id'] <= de_list_date]
-
-        for i in range(0, len(trade_date_list_df_copy), 28):
-            try:
-                new_df = trade_date_list_df_copy.iloc[i:i + 28]
-                first_df = new_df.iloc[0]  # 第一个 DataFrame
-                last_df = new_df.iloc[-1]  # 最后一个 DataFrame
-                begin_date = first_df.trade_date + ' 09:20:00'
-                end_date = last_df.trade_date + ' 15:00:00'
-
-                df = get_minute_data(symbol, '1min', begin_date, end_date)
-                if data_frame_util.is_not_empty(df):
-                    df = df.rename(columns={
-                        "trade_time": "time",
-                        "ts_code": "symbol",
-                        "vol": "volume",
-                    })
-                    df['time_tick'] = df['time'].str[11:19]
-                    df = df.loc[df['time_tick'] <= '15:00:00']
-                    del df['time_tick']
-                    df['_id'] = df['symbol'] + '_' + df['time']
-                    mongodbUtilV2.insert_mongo(df, db_name)
-            except BaseException as e:
-                time.sleep(2)
-                first_df = new_df.iloc[0]  # 第一个 DataFrame
-                last_df = new_df.iloc[-1]  # 最后一个 DataFrame
-                begin_date = first_df.trade_date + ' 09:20:00'
-                end_date = last_df.trade_date + ' 15:00:00'
-                fail_dict = {'begin_date': begin_date,
-                             'end_date': end_date,
-                             'symbol': symbol,
-                             'db_name': db_name
-                             }
-                fail_df = pd.DataFrame(fail_dict, index=[1])
-                mongodbUtilV2.insert_mongo(fail_df, 'one_minute_k_line_bfq_fail')
-
-                logger.error("同步数据出现异常:{},{},{},{}", e, symbol, begin_date, end_date)
-        logger.info("同步完数据:{},{}", stock_one.symbol, stock_one.name)
-
-
-if __name__ == '__main__':
-    # get_minute_data('833284.BJ', '1min', '2025-02-28 09:30:00', '2025-02-28 15:00:00')
-
-    query_trade = {"$and": [{"trade_date": {"$gte": "2025-03-08"}}, {"trade_date": {"$lte": "2025-03-18"}}]}
-    trade_date_list_df_all = mongodb_util_27017.find_query_data('trade_date_list', query_trade)
-
-    sync_all_stock(trade_date_list_df_all)

mns_scheduler/extraIncome/temp/tu_share_zhi_shu_sync_api.py

@@ -1,107 +0,0 @@
-import sys
-import os
-
-file_path = os.path.abspath(__file__)
-end = file_path.index('mns') + 16
-project_path = file_path[0:end]
-sys.path.append(project_path)
-
-import sys
-import os
-
-file_path = os.path.abspath(__file__)
-end = file_path.index('mns') + 16
-project_path = file_path[0:end]
-sys.path.append(project_path)
-import pandas as pd
-from loguru import logger
-import time
-import mns_common.utils.data_frame_util as data_frame_util
-from mns_common.db.MongodbUtil import MongodbUtil
-import tushare as ts
-from mns_common.db.v2.MongodbUtilV2 import MongodbUtilV2
-import mns_scheduler.extraIncome.one_minute.common.db_create_index as db_create_index
-
-mongodb_util_27017 = MongodbUtil('27017')
-mongodbUtilV2_27019 = MongodbUtilV2('27019', 'extraIncome')
-
-pro = ts.pro_api('782213d20640249f1dbae50a7f56b22684b8e915a61e435e015579a1')
-
-
-def get_minute_data(symbol, freq, start_date, end_date):
-    # 获取浦发银行60000.SH的历史分钟数据
-    df = pro.stk_mins(ts_code=symbol, freq=freq, start_date=start_date, end_date=end_date)
-    return df
-
-
-def sync_zhi_shu_one_minute(symbol_list_param, trade_date_list_df):
-    db_name = 'one_minute_k_line_bfq_main_index'
-    db_create_index.create_index(mongodbUtilV2_27019, db_name)
-    for symbol in symbol_list_param:
-
-        trade_date_list_df_copy = trade_date_list_df.copy()
-
-        for i in range(0, len(trade_date_list_df_copy), 28):
-            try:
-                new_df = trade_date_list_df_copy.iloc[i:i + 28]
-                first_df = new_df.iloc[0]  # 第一个 DataFrame
-                last_df = new_df.iloc[-1]  # 最后一个 DataFrame
-                begin_date = first_df.trade_date + ' 09:20:00'
-                end_date = last_df.trade_date + ' 15:00:00'
-
-                df = get_minute_data(symbol, '1min', begin_date, end_date)
-                if data_frame_util.is_not_empty(df):
-                    df = df.rename(columns={
-                        "trade_time": "time",
-                        "ts_code": "symbol",
-                        "vol": "volume",
-                    })
-                    df['time_tick'] = df['time'].str[11:19]
-                    df = df.loc[df['time_tick'] <= '15:00:00']
-                    del df['time_tick']
-                    df['_id'] = df['symbol'] + '_' + df['time']
-                    mongodbUtilV2_27019.insert_mongo(df, db_name)
-            except BaseException as e:
-                time.sleep(2)
-                first_df = new_df.iloc[0]  # 第一个 DataFrame
-                last_df = new_df.iloc[-1]  # 最后一个 DataFrame
-                begin_date = first_df.trade_date + ' 09:20:00'
-                end_date = last_df.trade_date + ' 15:00:00'
-                fail_dict = {'begin_date': begin_date,
-                             'end_date': end_date,
-                             'symbol': symbol,
-                             'db_name': db_name
-                             }
-                fail_df = pd.DataFrame(fail_dict, index=[1])
-                mongodbUtilV2_27019.insert_mongo(fail_df, db_name + '_fail')
-
-                logger.error("同步数据出现异常:{},{},{},{}", e, symbol, begin_date, end_date)
-        logger.info("同步完数据:{}", symbol)
-
-
-def classify_symbol(debt_real_time_quotes_df):
-    debt_real_time_quotes_df['classification'] = debt_real_time_quotes_df['market'].apply(
-        lambda market: classify_symbol_one(market))
-    return debt_real_time_quotes_df
-
-
-# 单个股票分类
-def classify_symbol_one(market):
-    if market == 0:
-        return 'S'
-    else:
-        return 'H'
-
-
-if __name__ == '__main__':
-    # 文件路径
-    file_path = r"H:\data\1min\指数\指数列表.xlsx"
-
-    df = pd.read_excel(file_path)
-    df = df.rename(columns={
-        "指数代码": "symbol"
-    })
-    new_symbol_list = df['symbol']
-    query_trade = {"$and": [{"trade_date": {"$gte": "2025-03-08"}}, {"trade_date": {"$lte": "2025-03-16"}}]}
-    trade_date_list_df_all = mongodb_util_27017.find_query_data('trade_date_list', query_trade)
-    sync_zhi_shu_one_minute(new_symbol_list, trade_date_list_df_all)

mns_scheduler/us/__init__.py

@@ -1,7 +0,0 @@
-import sys
-import os
-
-file_path = os.path.abspath(__file__)
-end = file_path.index('mns') + 16
-project_path = file_path[0:end]
-sys.path.append(project_path)

mns_scheduler/us/baidu_yun_pan_handle_service.py

@@ -1,131 +0,0 @@
-import sys
-import os
-
-file_path = os.path.abspath(__file__)
-end = file_path.index('mns') + 16
-project_path = file_path[0:end]
-sys.path.append(project_path)
-from bypy import ByPy
-from mns_common.db.MongodbUtil import MongodbUtil
-import tempfile
-from loguru import logger
-import akshare as ak
-
-mongodb_util = MongodbUtil('27017')
-
-import subprocess
-
-
-def get_file_list(path):
-    """
-    获取百度网盘指定路径下的文件列表
-    :param path: 百度网盘中的路径，例如 '/我的资源'
-    :return: 文件列表
-    """
-    try:
-        # 调用 bypy list 命令
-        result = subprocess.run(['bypy', 'list', path], capture_output=True, text=True, check=True)
-
-        # 输出结果
-        if result.returncode == 0:
-            file_list = result.stdout.splitlines()  # 按行分割结果
-            return file_list
-        else:
-            logger.error("获取文件路径异常:{}", result.stderr)
-            return []
-    except subprocess.CalledProcessError as e:
-        logger.error("获取文件路径异常:{}", e)
-        return []
-
-
-def upload_to_baidu(file_name, folder_name, data_df):
-    bp = ByPy()
-    file_name = file_name + '.csv'
-    with tempfile.NamedTemporaryFile(mode='w', suffix='.csv', delete=False) as temp_file:
-        data_df.to_csv(temp_file, index=False)
-        temp_file_path = temp_file.name  # 获取临时文件的路径
-
-    # 上传临时文件到百度云
-    remote_path = f'/{folder_name}/{file_name}'
-    result = bp.upload(temp_file_path, remote_path)
-    if result == 0:
-        logger.info("上传成功:{}", file_name)
-    else:
-        logger.error("上传失败:{}", file_name)
-    return result
-
-
-def mkdir_baidu_new_folder(remote_path):
-    bp = ByPy()
-    try:
-        # 调用 mkdir 方法创建文件夹
-        result = bp.mkdir(remote_path)
-
-        if result == 0:
-            logger.info("成功创建文件夹:{}", remote_path)
-        else:
-            logger.error("创建文件夹失败:{}", result)
-
-    except Exception as e:
-        logger.error("创建文件夹失败:{}", e)
-
-
-def del_baidu_old_folder(remote_path):
-    bp = ByPy()
-    try:
-        # 调用 mkdir 方法创建文件夹
-        result = bp.delete(remote_path)
-
-        if result == 0:
-            logger.info("成功删除文件夹:{}", remote_path)
-        else:
-            logger.error("删除文件夹失败:{}", result)
-
-    except Exception as e:
-        logger.error("删除文件夹失败:{}", e)
-
-
-if __name__ == '__main__':
-    folder_name1 = '/美股/不复权日线'
-    mkdir_baidu_new_folder(folder_name1)
-    # get_file_list(folder_name1)
-    stock_us_spot_em_df = ak.stock_us_spot_em()
-    stock_us_spot_em_df = stock_us_spot_em_df.rename(columns={
-        "序号": "index",
-        "代码": "symbol",
-        "名称": "name",
-        "涨跌额": "change_price",
-        "涨跌幅": "chg",
-        "开盘价": "open",
-        "最高价": "high",
-        "最低价": "low",
-        "最新价": "now_price",
-        "昨收价": "last_price",
-        "总市值": "total_mv",
-        "市盈率": "pe",
-        "成交量": "volume",
-        "成交额": "amount",
-        "振幅": "pct_chg",
-        "换手率": "exchange"
-    })
-    stock_us_spot_em_df = stock_us_spot_em_df.sort_values(by=['amount'], ascending=False)
-    stock_us_spot_em_df = stock_us_spot_em_df.fillna(0)
-    stock_us_spot_em_df = stock_us_spot_em_df.loc[stock_us_spot_em_df['total_mv'] != 0]
-    for stock_one in stock_us_spot_em_df.itertuples():
-        try:
-            symbol = stock_one.symbol
-            name = stock_one.name
-            query = {'symbol': symbol, 'amount': {"$gt": 0}}
-            us_stock_bfq_daily_df_one = mongodb_util.find_query_data('us_stock_bfq_daily', query)
-            del us_stock_bfq_daily_df_one['_id']
-            del us_stock_bfq_daily_df_one['name']
-            file_name_one = name + '_' + symbol
-            upload_to_baidu(file_name_one, folder_name1, us_stock_bfq_daily_df_one)
-
-        except BaseException as e:
-            logger.error("同步数据发生异常:{}", e)
-
-# data_df = mongodb_util.find_query_data('us_stock_bfq_daily', query={'name': file_name1})
-# upload_to_baidu(file_name1, folder_name1, data_df)
-
-

mns_scheduler/us/k_line.py

@@ -1,91 +0,0 @@
-import akshare as ak
-import pandas as pd
-from mns_common.db.MongodbUtil import MongodbUtil
-
-mongodb_util = MongodbUtil('27017')
-
-
-def us_stock():
-    # 输入参数
-    symbol = input("请输入股票代码（all:全量(时间很长),特定代码:106.TTE）：")
-    start_date = input("请输入开始日期（格式：YYYYMMDD）：")
-    end_date = input("请输入结束日期（格式：YYYYMMDD）：")
-    fq = input("请输入复权信息（前复权:qfq,不复权:bfq,后复权:hfq）：")
-    k_line_period = input("请输入k线周期（日线:daily,周线:weekly,月线:monthly）：")
-    db_name = "us_stock_" + fq + "_" + k_line_period
-    if fq == 'bfq':
-        fq = ''
-    if symbol != 'all':
-        # 获取股票历史数据
-        stock_us_hist_df = ak.stock_us_hist(symbol=symbol,
-                                            period=k_line_period,
-                                            start_date=start_date,
-                                            end_date=end_date,
-                                            adjust=fq)
-        # 保存数据到 CSV 文件
-        stock_us_hist_df.to_csv(f"{symbol}_historical_data.csv", index=False)
-        print(f"数据已保存到 {symbol}_historical_data.csv")
-    else:
-        stock_us_spot_em_df = ak.stock_us_spot_em()
-        stock_us_spot_em_df = stock_us_spot_em_df.rename(columns={
-            "序号": "index",
-            "代码": "symbol",
-            "名称": "name",
-            "涨跌额": "change_price",
-            "涨跌幅": "chg",
-            "开盘价": "open",
-            "最高价": "high",
-            "最低价": "low",
-            "最新价": "now_price",
-            "昨收价": "last_price",
-            "总市值": "total_mv",
-            "市盈率": "pe",
-            "成交量": "volume",
-            "成交额": "amount",
-            "振幅": "pct_chg",
-            "换手率": "exchange"
-        })
-        stock_us_spot_em_df = stock_us_spot_em_df.sort_values(by=['amount'], ascending=False)
-        stock_us_spot_em_df = stock_us_spot_em_df.fillna(0)
-        stock_us_spot_em_df = stock_us_spot_em_df.loc[stock_us_spot_em_df['total_mv'] != 0]
-
-        k_line_result = pd.DataFrame()
-
-        for stock_us_one in stock_us_spot_em_df.itertuples():
-            try:
-                # 获取股票历史数据
-                stock_us_hist_df = ak.stock_us_hist(symbol=stock_us_one.symbol,
-                                                    period=k_line_period,
-                                                    start_date=start_date,
-                                                    end_date=end_date,
-                                                    adjust=fq)
-                stock_us_hist_df = stock_us_hist_df.rename(columns={
-                    "日期": "date",
-                    "涨跌额": "change_price",
-                    "涨跌幅": "chg",
-                    "开盘": "open",
-                    "最高": "high",
-                    "最低": "low",
-                    "收盘": "close",
-                    "成交量": "volume",
-                    "成交额": "amount",
-                    "振幅": "pct_chg",
-                    "换手率": "exchange"
-                })
-
-                k_line_result = pd.concat([stock_us_hist_df, k_line_result])
-                stock_us_hist_df['_id'] = stock_us_one.symbol + '_' + stock_us_hist_df['date']
-                stock_us_hist_df['symbol'] = stock_us_one.symbol
-                stock_us_hist_df['name'] = stock_us_one.name
-                mongodb_util.insert_mongo(stock_us_hist_df, db_name)
-                print(f"同步k线数据到: {stock_us_one.name}")
-            except BaseException as e:
-                print(f"同步数据发生异常: {stock_us_one.name}, {e}")
-
-        # 保存数据到 CSV 文件
-        k_line_result.to_csv(f"{symbol}_historical_data.csv", index=False)
-        print(f"数据已保存到 {symbol}_historical_data.csv")
-
-
-if __name__ == "__main__":
-    us_stock()

mns_scheduler/us/us_company_info_sync_service_api.py

@@ -1,39 +0,0 @@
-import sys
-import os
-
-file_path = os.path.abspath(__file__)
-end = file_path.index('mns') + 16
-project_path = file_path[0:end]
-sys.path.append(project_path)
-import akshare as ak
-
-
-def sync_us_company_info():
-    stock_us_spot_em_df = ak.stock_us_spot_em()
-    stock_us_spot_em_df = stock_us_spot_em_df.rename(columns={
-        "序号": "index",
-        "代码": "symbol",
-        "名称": "name",
-        "涨跌额": "change_price",
-        "涨跌幅": "chg",
-        "开盘价": "open",
-        "最高价": "high",
-        "最低价": "low",
-        "最新价": "now_price",
-        "昨收价": "last_price",
-        "总市值": "total_mv",
-        "市盈率": "pe",
-        "成交量": "volume",
-        "成交额": "amount",
-        "振幅": "pct_chg",
-        "换手率": "exchange"
-    })
-    stock_us_spot_em_df = stock_us_spot_em_df.sort_values(by=['amount'], ascending=False)
-    stock_us_spot_em_df = stock_us_spot_em_df.fillna(0)
-    stock_us_spot_em_df = stock_us_spot_em_df.loc[stock_us_spot_em_df['total_mv']!=0]
-    stock_us_spot_em_df.to_csv('us_stock.csv', index=False)
-    return stock_us_spot_em_df
-
-
-if __name__ == '__main__':
-    sync_us_company_info()