mns-scheduler 1.3.1.2__py3-none-any.whl → 1.3.1.7__py3-none-any.whl

mns_scheduler/extraIncome/a_stock/one_minute/upload/kzz_upload_to_baidu_task.py

@@ -0,0 +1,95 @@
+import sys
+import os
+
+file_path = os.path.abspath(__file__)
+end = file_path.index('mns') + 17
+project_path = file_path[0:end]
+sys.path.append(project_path)
+import sys
+import os
+
+file_path = os.path.abspath(__file__)
+end = file_path.index('mns') + 17
+project_path = file_path[0:end]
+sys.path.append(project_path)
+from mns_common.db.MongodbUtil import MongodbUtil
+from loguru import logger
+import mns_scheduler.baidu.baidu_yun_pan_handle_service as baidu_yun_pan_handle_service
+from mns_common.db.v2.MongodbUtilV2 import MongodbUtilV2
+import mns_common.utils.data_frame_util as data_frame_util
+from datetime import datetime
+import mns_common.constant.extra_income_db_name as extra_income_db_name
+import mns_common.component.em.em_stock_info_api as em_stock_info_api
+
+mongodb_util = MongodbUtil('27017')
+mongodbUtilV2_27019 = MongodbUtilV2('27019', extra_income_db_name.EXTRA_INCOME)
+
+
+def upload_kzz_to_baidu():
+    a_stock_path = '/A股/可转债/1分钟'
+
+    now_date_time = datetime.now()
+    now_year = now_date_time.year
+    month = now_date_time.month
+    a_stock_path = a_stock_path + str(now_year) + '/' + str(month)
+    # 创建路径
+    baidu_yun_pan_handle_service.mkdir_baidu_new_folder(a_stock_path)
+    em_a_kzz_info_df = em_stock_info_api.get_kzz_info()
+
+    em_a_kzz_info_df = classify_symbol(em_a_kzz_info_df)
+
+    em_a_kzz_info_df['symbol'] = em_a_kzz_info_df.apply(
+        lambda row: row['symbol'] + '.SZ' if row['classification'] in ['S', 'C']
+        else row['symbol'] + '.BJ' if row['classification'] in ['X']
+        else row['symbol'] + '.SH',
+        axis=1
+    )
+
+    file_folder_df = baidu_yun_pan_handle_service.get_file_folder(a_stock_path)
+    if data_frame_util.is_not_empty(file_folder_df):
+        # 去除文件名中的 .csv 后缀
+        file_folder_df['name'] = file_folder_df['name'].str.replace(r'\.csv$', '', regex=True)
+        em_a_kzz_info_df = em_a_kzz_info_df.loc[~(em_a_kzz_info_df['symbol'].isin(file_folder_df['name']))]
+    fail_list = []
+    for stock_one in em_a_kzz_info_df.itertuples():
+        symbol = stock_one.symbol
+        name = stock_one.name
+        try:
+
+            col_name = extra_income_db_name.ONE_MINUTE_K_LINE_BFQ_KZZ + '_' + str(now_year)
+
+            if month < 10:
+                month_str = '0' + str(month)
+            else:
+                month_str = str(month)
+            begin_time = str(now_year) + '-' + month_str + '-01 09:00:00'
+            query = {'symbol': symbol, 'time': {"$gte": begin_time}}
+            one_minute_k_line_bfq_df = mongodbUtilV2_27019.find_query_data(col_name, query)
+            if data_frame_util.is_not_empty(one_minute_k_line_bfq_df):
+                one_minute_k_line_bfq_df = one_minute_k_line_bfq_df.sort_values(by=['time'], ascending=True)
+                del one_minute_k_line_bfq_df['_id']
+                del one_minute_k_line_bfq_df['symbol']
+                result = baidu_yun_pan_handle_service.upload_to_baidu(symbol, a_stock_path, one_minute_k_line_bfq_df)
+                if result != 0:
+                    fail_list.append(symbol)
+        except BaseException as e:
+            fail_list.append(symbol)
+            logger.error("上传数据异常:{}", e)
+
+
+def classify_symbol(kzz_real_time_quotes):
+    kzz_real_time_quotes['classification'] = kzz_real_time_quotes['market'].apply(
+        lambda market: classify_symbol_one(market))
+    return kzz_real_time_quotes
+
+
+# 单个股票分类
+def classify_symbol_one(market):
+    if market == 0:
+        return 'S'
+    else:
+        return 'H'
+
+
+if __name__ == '__main__':
+    upload_kzz_to_baidu()

mns_scheduler/extraIncome/a_stock/one_minute/upload/{upload_to_baidu_task.py → stock_upload_to_baidu_task.py}

@@ -2,7 +2,7 @@ import sys
 import os
 
 file_path = os.path.abspath(__file__)
-end = file_path.index('mns') + 16
+end = file_path.index('mns') + 17
 project_path = file_path[0:end]
 sys.path.append(project_path)
 from mns_common.db.MongodbUtil import MongodbUtil
@@ -45,6 +45,8 @@ def upload_stock_to_baidu():
         em_a_stock_info_df = em_a_stock_info_df.loc[~(em_a_stock_info_df['symbol'].isin(file_folder_df['name']))]
     fail_list = []
     for stock_one in em_a_stock_info_df.itertuples():
+        symbol = stock_one.symbol
+        name = stock_one.name
         try:
             classification = stock_one.classification
             if classification == 'X':
@@ -59,7 +61,7 @@ def upload_stock_to_baidu():
             elif classification == 'C':
                 col_name = extra_income_db_name.ONE_MINUTE_K_LINE_BFQ_C
             col_name = col_name + '_' + str(now_year)
-            symbol = stock_one.symbol
+
             if month < 10:
                 month_str = '0' + str(month)
             else:

mns_scheduler/extraIncome/hk/__init__.py

@@ -2,6 +2,6 @@ import sys
 import os
 
 file_path = os.path.abspath(__file__)
-end = file_path.index('mns') + 16
+end = file_path.index('mns') + 17
 project_path = file_path[0:end]
 sys.path.append(project_path)

mns_scheduler/extraIncome/hk/hk_stock_qfq_daily_k_line.py

@@ -2,7 +2,7 @@ import os
 import sys
 
 file_path = os.path.abspath(__file__)
-end = file_path.index('mns') + 16
+end = file_path.index('mns') + 17
 project_path = file_path[0:end]
 sys.path.append(project_path)
 

mns_scheduler/extraIncome/us/__init__.py

@@ -2,6 +2,6 @@ import sys
 import os
 
 file_path = os.path.abspath(__file__)
-end = file_path.index('mns') + 16
+end = file_path.index('mns') + 17
 project_path = file_path[0:end]
 sys.path.append(project_path)

mns_scheduler/extraIncome/us/daily/__init__.py

@@ -2,6 +2,6 @@ import sys
 import os
 
 file_path = os.path.abspath(__file__)
-end = file_path.index('mns') + 16
+end = file_path.index('mns') + 17
 project_path = file_path[0:end]
 sys.path.append(project_path)

mns_scheduler/extraIncome/us/daily/us_stock_qfq_daily_k_line.py

@@ -2,7 +2,7 @@ import os
 import sys
 
 file_path = os.path.abspath(__file__)
-end = file_path.index('mns') + 16
+end = file_path.index('mns') + 17
 project_path = file_path[0:end]
 sys.path.append(project_path)
 

mns_scheduler/extraIncome/us/one_minute/__init__.py

@@ -2,6 +2,6 @@ import sys
 import os
 
 file_path = os.path.abspath(__file__)
-end = file_path.index('mns') + 16
+end = file_path.index('mns') + 17
 project_path = file_path[0:end]
 sys.path.append(project_path)

mns_scheduler/extraIncome/us/one_minute/api/__init__.py

@@ -1,7 +0,0 @@
-import sys
-import os
-
-file_path = os.path.abspath(__file__)
-end = file_path.index('mns') + 16
-project_path = file_path[0:end]
-sys.path.append(project_path)

mns_scheduler/extraIncome/us/one_minute/api/alpha_vantage_api.py

@@ -2,7 +2,7 @@ import sys
 import os
 
 file_path = os.path.abspath(__file__)
-end = file_path.index('mns') + 16
+end = file_path.index('mns') + 17
 project_path = file_path[0:end]
 sys.path.append(project_path)
 from alpha_vantage.timeseries import TimeSeries
@@ -31,4 +31,4 @@ def sync_one_minute_data(symbol, month):
 
 
 if __name__ == '__main__':
-    sync_one_minute_data('IBM', '2000-01')
+    sync_one_minute_data('TSLA', '2020-08')

mns_scheduler/extraIncome/us/one_minute/api/em_us_one_minute_api.py

@@ -0,0 +1,139 @@
+import sys
+import os
+
+file_path = os.path.abspath(__file__)
+end = file_path.index('mns') + 17
+project_path = file_path[0:end]
+sys.path.append(project_path)
+
+import pandas as pd
+import requests
+
+
+def stock_hk_hist_min_em(
+        symbol: str = "01611",
+        period: str = "1",
+        adjust: str = "",
+        start_date: str = "1979-09-01 09:32:00",
+        end_date: str = "2222-01-01 09:32:00",
+) -> pd.DataFrame:
+    """
+    东方财富网-行情-港股-每日分时行情
+    https://quote.eastmoney.com/hk/00948.html
+    :param symbol: 股票代码
+    :type symbol: str
+    :param period: choice of {'1', '5', '15', '30', '60'}
+    :type period: str
+    :param adjust: choice of {'', 'qfq', 'hfq'}
+    :type adjust: str
+    :param start_date: 开始日期
+    :type start_date: str
+    :param end_date: 结束日期
+    :type end_date: str
+    :return: 每日分时行情
+    :rtype: pandas.DataFrame
+    """
+    adjust_map = {
+        "": "0",
+        "qfq": "1",
+        "hfq": "2",
+    }
+    if period == "1":
+        url = "https://push2his.eastmoney.com/api/qt/stock/trends2/get"
+        params = {
+            "fields1": "f1,f2,f3,f4,f5,f6,f7,f8,f9,f10,f11,f12,f13",
+            "fields2": "f51,f52,f53,f54,f55,f56,f57,f58",
+            "ut": "fa5fd1943c7b386f172d6893dbfba10b",
+            "iscr": "0",
+            "ndays": "5",
+            "secid": f"116.{symbol}",
+            "_": "1623766962675",
+        }
+        r = requests.get(url, timeout=15, params=params)
+        data_json = r.json()
+        temp_df = pd.DataFrame(
+            [item.split(",") for item in data_json["data"]["trends"]]
+        )
+        temp_df.columns = [
+            "时间",
+            "开盘",
+            "收盘",
+            "最高",
+            "最低",
+            "成交量",
+            "成交额",
+            "最新价",
+        ]
+        temp_df.index = pd.to_datetime(temp_df["时间"])
+        temp_df = temp_df[start_date:end_date]
+        temp_df.reset_index(drop=True, inplace=True)
+        temp_df["开盘"] = pd.to_numeric(temp_df["开盘"], errors="coerce")
+        temp_df["收盘"] = pd.to_numeric(temp_df["收盘"], errors="coerce")
+        temp_df["最高"] = pd.to_numeric(temp_df["最高"], errors="coerce")
+        temp_df["最低"] = pd.to_numeric(temp_df["最低"], errors="coerce")
+        temp_df["成交量"] = pd.to_numeric(temp_df["成交量"], errors="coerce")
+        temp_df["成交额"] = pd.to_numeric(temp_df["成交额"], errors="coerce")
+        temp_df["最新价"] = pd.to_numeric(temp_df["最新价"], errors="coerce")
+        temp_df["时间"] = pd.to_datetime(temp_df["时间"]).astype(str)
+        return temp_df
+    else:
+        url = "https://push2his.eastmoney.com/api/qt/stock/kline/get"
+        params = {
+            "fields1": "f1,f2,f3,f4,f5,f6",
+            "fields2": "f51,f52,f53,f54,f55,f56,f57,f58,f59,f60,f61",
+            "ut": "bd1d9ddb04089700cf9c27f6f7426281",
+            "klt": period,
+            "fqt": adjust_map[adjust],
+            "secid": f"116.{symbol}",
+            "beg": "0",
+            "end": "20500000",
+            "_": "1630930917857",
+        }
+        r = requests.get(url, timeout=15, params=params)
+        data_json = r.json()
+        temp_df = pd.DataFrame(
+            [item.split(",") for item in data_json["data"]["klines"]]
+        )
+        temp_df.columns = [
+            "时间",
+            "开盘",
+            "收盘",
+            "最高",
+            "最低",
+            "成交量",
+            "成交额",
+            "振幅",
+            "涨跌幅",
+            "涨跌额",
+            "换手率",
+        ]
+        temp_df.index = pd.to_datetime(temp_df["时间"])
+        temp_df = temp_df[start_date:end_date]
+        temp_df.reset_index(drop=True, inplace=True)
+        temp_df["开盘"] = pd.to_numeric(temp_df["开盘"], errors="coerce")
+        temp_df["收盘"] = pd.to_numeric(temp_df["收盘"], errors="coerce")
+        temp_df["最高"] = pd.to_numeric(temp_df["最高"], errors="coerce")
+        temp_df["最低"] = pd.to_numeric(temp_df["最低"], errors="coerce")
+        temp_df["成交量"] = pd.to_numeric(temp_df["成交量"], errors="coerce")
+        temp_df["成交额"] = pd.to_numeric(temp_df["成交额"], errors="coerce")
+        temp_df["振幅"] = pd.to_numeric(temp_df["振幅"], errors="coerce")
+        temp_df["涨跌幅"] = pd.to_numeric(temp_df["涨跌幅"], errors="coerce")
+        temp_df["涨跌额"] = pd.to_numeric(temp_df["涨跌额"], errors="coerce")
+        temp_df["换手率"] = pd.to_numeric(temp_df["换手率"], errors="coerce")
+        temp_df["时间"] = pd.to_datetime(temp_df["时间"]).astype(str)
+        temp_df = temp_df[
+            [
+                "时间",
+                "开盘",
+                "收盘",
+                "最高",
+                "最低",
+                "涨跌幅",
+                "涨跌额",
+                "成交量",
+                "成交额",
+                "振幅",
+                "换手率",
+            ]
+        ]
+    return temp_df

mns_scheduler/extraIncome/us/one_minute/api/stock_etf_info_api.py

@@ -0,0 +1,68 @@
+import sys
+import os
+
+file_path = os.path.abspath(__file__)
+end = file_path.index('mns') + 17
+project_path = file_path[0:end]
+sys.path.append(project_path)
+
+import csv
+import requests
+import pandas as pd
+import mns_common.component.em.em_stock_info_api as em_stock_info_api
+from functools import lru_cache
+
+
+@lru_cache()
+def get_us_stock_info():
+    em_us_stock_info_df = em_stock_info_api.get_us_stock_info()
+    em_us_stock_info_df['symbol'] = em_us_stock_info_df['symbol'].str.replace('_', '-')
+    em_us_stock_info_df = em_us_stock_info_df.loc[em_us_stock_info_df['total_mv'] != 0]
+
+    alpha_us_stock_info = get_us_stock_list()
+    alpha_us_stock_info = alpha_us_stock_info.loc[alpha_us_stock_info['assetType'] == 'Stock']
+
+    em_us_stock_info_df = em_us_stock_info_df.loc[
+        em_us_stock_info_df['symbol'].isin(alpha_us_stock_info['symbol'])]
+    em_us_stock_info_df = em_us_stock_info_df.sort_values(by=['flow_mv'], ascending=False)
+    return em_us_stock_info_df
+
+
+@lru_cache()
+def get_us_etf_info():
+    em_us_stock_info_df = em_stock_info_api.get_us_stock_info()
+    em_us_stock_info_df['symbol'] = em_us_stock_info_df['symbol'].str.replace('_', '-')
+    em_us_stock_info_df = em_us_stock_info_df.loc[em_us_stock_info_df['total_mv'] != 0]
+
+    alpha_us_stock_info = get_us_stock_list()
+    alpha_us_stock_info = alpha_us_stock_info.loc[alpha_us_stock_info['assetType'] == 'ETF']
+
+    em_us_stock_info_df = em_us_stock_info_df.loc[
+        em_us_stock_info_df['symbol'].isin(alpha_us_stock_info['symbol'])]
+    em_us_stock_info_df = em_us_stock_info_df.sort_values(by=['flow_mv'], ascending=False)
+    return em_us_stock_info_df
+
+
+@lru_cache()
+def get_us_stock_list():
+    # replace the "demo" apikey below with your own key from https://www.alphavantage.co/support/#api-key
+    CSV_URL = 'https://www.alphavantage.co/query?function=LISTING_STATUS&apikey=demo'
+    with requests.Session() as s:
+        download = s.get(CSV_URL)
+        decoded_content = download.content.decode('utf-8')
+        cr = csv.reader(decoded_content.splitlines(), delimiter=',')
+        my_list = list(cr)
+        # 提取列名（第1行）
+        columns = my_list[0]
+        # 提取数据（第2行及以后）
+        values = my_list[1:]
+
+        # 转换为 DataFrame
+        df = pd.DataFrame(values, columns=columns)
+        df = df.rename(columns={'ipoDate': 'list_date'})
+        return df
+
+
+if __name__ == '__main__':
+    get_us_stock_info()
+    get_us_etf_info()

mns_scheduler/extraIncome/us/one_minute/api/y_finance_api.py

@@ -2,7 +2,7 @@ import sys
 import os
 
 file_path = os.path.abspath(__file__)
-end = file_path.index('mns') + 16
+end = file_path.index('mns') + 17
 project_path = file_path[0:end]
 sys.path.append(project_path)
 
@@ -14,10 +14,28 @@ os.environ['HTTPS_PROXY'] = proxy
 import yfinance as yf
 
 
+#
+# “history”函数包含许多参数，了解它们对于正确解释您收到的数据非常重要：
+#
+# period：如前所述，特别有用的是值“max”。以下是有效值：1d、5d、1mo、3mo、6mo、1y、2y、5y、10y、ytd、max。
+# 间隔：定义每个条形的大小。条形越小，限制越严格，只能检索 7 天的 1 分钟数据。以下是有效值：1m,2m,5m,15m,30m,60m,90m,1h,1d,5d,1wk,1mo,3mo
+# start：开始日期。服务器需要格式为 YYYY-MM-DD 的字符串。
+# end：结束日期。服务器需要格式为 YYYY-MM-DD 的字符串。
+# repost：定义是否包含与正常交易时间不对应的数据。默认值为 False
+# auto_adjust：是否根据股票分割和股息支付调整价格。默认值是true。
+# 雅虎财经获取基本数据方法
 def get_us_one_minute(symbol, start_time, end_time):
+    # start_time = '2025-05-01'
+    # end_time = '2025-05-09'
+    start_time = '2025-05-09'
+    end_time = '2025-05-11'
     yf_ticker = yf.Ticker(symbol)
-    df = yf_ticker.history(period='5d', interval='1m',
-                           start=start_time, end=end_time, prepost=True)
+    df = yf_ticker.history(period='5d',
+                           interval='1m',
+                           start=start_time,
+                           end=end_time,
+                           prepost=False,
+                           auto_adjust=False)
     df = df[[
         "Open",
         "High",
@@ -35,13 +53,11 @@ def get_us_one_minute(symbol, start_time, end_time):
         "time",
     ]
     df['time'] = df['time'].dt.strftime('%Y-%m-%d %H:%M:%S')
-    df['str_day'] = df['time'].str.slice(0, 10)
-    df['minute'] = df['time'].str.slice(11, 19)
+    # df['str_day'] = df['time'].str.slice(0, 10)
+    # df['minute'] = df['time'].str.slice(11, 19)
     return df
-    # df_test = df.loc[df['str_day'] == '2025-04-30']
-    # print(sum(df_test['volume']))
-    # sum(df.loc[(df['str_day'] == '2025-04-30') & (df['time'] > '16:00:00')]['volume'])
 
 
 if __name__ == '__main__':
-    get_us_one_minute('QQQ', '2025-05-01', '2025-05-07')
+    df=get_us_one_minute('B', '2025-05-01', '2025-05-10')
+    print(df)

mns_scheduler/extraIncome/us/one_minute/etf/__init__.py

@@ -2,6 +2,6 @@ import sys
 import os
 
 file_path = os.path.abspath(__file__)
-end = file_path.index('mns') + 16
+end = file_path.index('mns') + 17
 project_path = file_path[0:end]
 sys.path.append(project_path)

mns_scheduler/extraIncome/us/one_minute/stock/__init__.py

@@ -2,6 +2,6 @@ import sys
 import os
 
 file_path = os.path.abspath(__file__)
-end = file_path.index('mns') + 16
+end = file_path.index('mns') + 17
 project_path = file_path[0:end]
 sys.path.append(project_path)

mns_scheduler/extraIncome/us/one_minute/stock/down_load/__init__.py

@@ -2,6 +2,6 @@ import sys
 import os
 
 file_path = os.path.abspath(__file__)
-end = file_path.index('mns') + 16
+end = file_path.index('mns') + 17
 project_path = file_path[0:end]
 sys.path.append(project_path)

mns_scheduler/extraIncome/us/one_minute/stock/down_load/etf/__init__.py

@@ -2,6 +2,10 @@ import sys
 import os
 
 file_path = os.path.abspath(__file__)
-end = file_path.index('mns') + 16
+end = file_path.index('mns') + 17
 project_path = file_path[0:end]
 sys.path.append(project_path)
+import akshare as ak
+
+us_stock_current_df = ak.stock_us_spot()
+print(us_stock_current_df)

mns_scheduler/extraIncome/us/one_minute/stock/down_load/etf/down_load_ETF_his_2024.py

@@ -2,7 +2,7 @@ import sys
 import os
 
 file_path = os.path.abspath(__file__)
-end = file_path.index('mns') + 16
+end = file_path.index('mns') + 17
 project_path = file_path[0:end]
 sys.path.append(project_path)
 import mns_common.component.em.em_stock_info_api as em_stock_info_api

mns_scheduler/extraIncome/us/one_minute/stock/down_load/etf/handle_down_load_fail_ETF.py

@@ -2,7 +2,7 @@ import sys
 import os
 
 file_path = os.path.abspath(__file__)
-end = file_path.index('mns') + 16
+end = file_path.index('mns') + 17
 project_path = file_path[0:end]
 sys.path.append(project_path)