mns-common 1.3.3.5__py3-none-any.whl → 1.5.7.2__py3-none-any.whl

mns_common/api/em/{east_money_debt_api.py → real_time/east_money_debt_api.py}

@@ -1,42 +1,76 @@
 import sys
 import os
 
-import sys
-import os
-
 file_path = os.path.abspath(__file__)
 end = file_path.index('mns') + 14
 project_path = file_path[0:end]
 sys.path.append(project_path)
-from concurrent.futures import ThreadPoolExecutor
+
+import json
+import akshare as ak
 import pandas as pd
 from loguru import logger
-import json
 import requests
 import time
-import akshare as ak
 import numpy as np
+import mns_common.component.proxies.proxy_common_api as proxy_common_api
+from concurrent.futures import ThreadPoolExecutor
+from threading import Lock
+import concurrent.futures
+import mns_common.utils.data_frame_util as data_frame_util
 
-# 最大返回条数
-max_number = 600
-# 最小返回条数
-min_number = 500
 # 分页条数
 page_number = 100
 
+fields = ("f352,f2,f3,f5,f6,f8,f10,f11,f22,f12,f14,f15,f16,f17,f18,f20,f21,f26,f33,f34,f35,f62,f66,f69,f72,f184,"
+          "f211,f212,f232,f233,f234")
+
+
+def get_kzz_count(pn, proxies, page_size, time_out):
+    current_timestamp = str(int(round(time.time() * 1000, 0)))
+    url = "https://push2.eastmoney.com/api/qt/clist/get"
+
+    params = {
+        "cb": "jQuery34103608466964799838_" + current_timestamp,
+        "pn": str(pn),
+        "np": 3,
+        "ut": "8a086bfc3570bdde64a6a1c585cccb35",
+        "fltt": 1,
+        "invt": 1,
+        "fs": "m:0+e:11,m:1+e:11,m:1+e:11+s:4194304,m:0+e:11+s:8388608",
+        "dpt": "zqsc.zpg",
+        "fields": fields,
+        "wbp2u": "|0|0|0|wap",
+        "fid": "f12",
+        "po": 1,
+        "pz": str(page_size),
+        "_": current_timestamp
+    }
+    try:
+        if proxies is None:
+            r = requests.get(url, params, timeout=time_out)
+        else:
+            r = requests.get(url, params, proxies=proxies, timeout=time_out)
+        data_text = r.text
+
+        begin_index_total = data_text.index('"total":')
+
+        end_index_total = data_text.index('"diff"')
+        max_number = int(data_text[begin_index_total + 8:end_index_total - 1])
+        return max_number
+
+
+    except Exception as e:
+        logger.error("获取可转债列表,实时行情异常:{}", e)
+        return 0
 
-# fields_02 = "f1,f2,f3,f4,f5,f6,f7,f8,f9,f10,f11,f12,f13,f14,f15,f16,f17,f18,f19,f20,f21,f22,f23,f24,f25,f26,f27,f28,f29,f30,f31,f32,f33,f34,f35,f36,f37,f38,f39,f40,f41,f42,f43,f44,f45,f46,f47,f48,f49,f50,f51,f52,f53,f54,f55,f56,f57,f58,f59,f60,f61,f62,f63,f64,f65,f66,f67,f68,f69,f70,f71,f72,f73,f74,f75,f76,f77,f78,f79,f80,f81,f82,f83,f84,f85,f86,f87,f88,f89,f90,f91,f92,f93,f94,f95,f96,f97,f98,f99,f100,f101,f102,f103,f104,f105,f106,f107,f108" \
-#             ",f109,f110,f111,f112,f113,f114,f115,f116,f117,f118,f119,f120,f121,f122,f123,f124,f125,f126,f127,f128,f129,f130,f131,f132,f133,f134,f135,f136,f137,f138,f139,f140,f141,f142,f143,f144,f145,f146,f147,f148,f149,f150,f151,f152,f153,f154,f155,f156,f157,f158,f159,f160,f161,f162,f163,f164,f165,f166,f167,f168,f169,f170,f171,f172,f173,f174,f175,f176,f177,f178,f179,f180,f181,f182,f183,f184,f185,f186,f187,f188,f189,f190,f191,f192,f193,f194,f195,f196,f197,f198,f199,f200" \
-#             ",f209,f210,f211,f212,f213,f214,f215,f216,f217,f218,f219,f220,f221,f222,f223,f224,f225,f226,f227,f228,f229,f230,f231,f232,f233,f234,f235,f236,f237,f238,f239,f240,f241,f242,f243,f244,f245,f246,f247,f248,f249,f250,f251,f252,f253,f254,f255,f256,f257,f258,f259,f260,f261,f262,f263,f264,f265,f266,f267,f268,f269,f270,f271,f272,f273,f274,f275,f276,f277,f278,f279,f280,f281,f282,f283,f284,f285,f286,f287,f288,f289,f290,f291,f292,f293,f294,f295,f296,f297,f298,f299,f300" \
-#             ",f309,f310,f312,f313,f314,f315,f316,f317,f318,f319,f320,f321,f322,f323,f324,f325,f326,f327,f328,f329,f330,f331,f332,f333,f334,f335,f336,f337,f338,f339,f340,f341,f342,f343,f344,f345,f346,f347,f348,f349,f350,f351,f352,f353,f354,f355,f356,f357,f358,f359,f360,f361,f362,f363,f364,f365,f366,f367,f368,f369,f370,f371,f372,f373,f374,f375,f376,f377,f378,f379,f380,f381,f382,f383,f384,f385,f386,f387,f388,f389,f390,f391,f392,f393,f394,f395,f396,f397,f398,f399,f401"
-#
 
 #
 # url = https://push2.eastmoney.com/api/qt/clist/get?cb=jQuery34103608466964799838_1718163189869&pn=1&np=1&ut
 # =8a086bfc3570bdde64a6a1c585cccb35&fltt=1&invt=1&fs=m:0+e:11,m:1+e:11,m:1+e:11+s:4194304,
 # m:0+e:11+s:8388608&dpt=zqsc.zpg&fields=f1,f2,f3,f4,f5,f6,f8,f10,f12,f13,f14,f18,f22,f152,
 # f237&wbp2u=|0|0|0|wap&fid=f3&po=1&pz=2000&_=1718163189870
-def get_debt_page_data(fields, pn) -> pd.DataFrame:
+def get_debt_page_data(pn, proxies, page_size, time_out) -> pd.DataFrame:
     current_timestamp = str(int(round(time.time() * 1000, 0)))
     url = "https://push2.eastmoney.com/api/qt/clist/get"
 
@@ -53,12 +87,27 @@ def get_debt_page_data(fields, pn) -> pd.DataFrame:
         "wbp2u": "|0|0|0|wap",
         "fid": "f12",
         "po": 1,
-        "pz": 2000,
+        "pz": str(page_size),
         "_": current_timestamp
     }
     try:
-        r = requests.get(url, params)
+        if proxies is None:
+            r = requests.get(url, params, timeout=time_out)
+        else:
+            r = requests.get(url, params, proxies=proxies, timeout=time_out)
         data_text = r.text
+
+        if pn == 1:
+            try:
+                begin_index_total = data_text.index('"total":')
+
+                end_index_total = data_text.index('"diff"')
+                global max_number
+                max_number = int(data_text[begin_index_total + 8:end_index_total - 1])
+            except Exception as e:
+                logger.error(f"获取第{pn}页可转债列表异常: {e}")
+                return pd.DataFrame()
+
         begin_index = data_text.index('[')
         end_index = data_text.index(']')
         data_json = data_text[begin_index:end_index + 1]
@@ -72,52 +121,6 @@ def get_debt_page_data(fields, pn) -> pd.DataFrame:
         return pd.DataFrame()
 
 
-def all_debt_ticker_data(fields) -> pd.DataFrame:
-    """
-    使用多线程获取所有债券数据
-    """
-    # 计算总页数，假设总共有1000条数据，每页200条
-
-    per_page = page_number
-    total_pages = (max_number + per_page - 1) // per_page  # 向上取整
-
-    # 创建线程池
-    with ThreadPoolExecutor(max_workers=3) as executor:
-        # 提交任务，获取每页数据
-        futures = [executor.submit(get_debt_page_data, fields, pn)
-                   for pn in range(1, total_pages + 1)]
-
-        # 收集结果
-        results = []
-        for future in futures:
-            result = future.result()
-            if not result.empty:
-                results.append(result)
-
-    # 合并所有页面的数据
-    if results:
-        return pd.concat(results, ignore_index=True)
-    else:
-        return pd.DataFrame()
-
-
-def get_debt_real_time_quotes():
-    fields = ("f352,f2,f3,f5,f6,f8,f10,f11,f22,f12,f14,f15,f16,f17,f18,f20,f21,f26,f33,f34,f35,f62,f66,f69,f72,f184,"
-              "f211,f212,f232,f233,f234")
-    # 获取第一页数据
-    page_one_df = get_debt_page_data(fields, 1)
-    # 数据接口正常返回5600以上的数量
-    if page_one_df.shape[0] > min_number:
-        page_one_df = rename_real_time_quotes_df(page_one_df)
-        page_one_df.drop_duplicates('symbol', keep='last', inplace=True)
-        return page_one_df
-    else:
-        page_df = all_debt_ticker_data(fields)
-        page_df = rename_real_time_quotes_df(page_df)
-        page_df.drop_duplicates('symbol', keep='last', inplace=True)
-        return page_df
-
-
 def rename_real_time_quotes_df(temp_df):
     temp_df = temp_df.rename(columns={
         "f2": "now_price",
@@ -283,9 +286,91 @@ def get_kzz_bond_info():
         logger.error("获取可转债信息异常:{}", e)
 
 
+def repeated_acquisition_ask_etf_async(time_out, max_number, num_threads, pages_per_thread):
+    per_page = page_number
+    total_pages = (max_number + per_page - 1) // per_page  # 向上取整
+    result_df = pd.DataFrame()
+
+    # 创建线程锁以确保线程安全
+    df_lock = Lock()
+
+    # 计算每个线程处理的页数范围
+    def process_page_range(start_page, end_page, thread_id):
+        nonlocal result_df
+        local_df = pd.DataFrame()
+        current_page = start_page
+        proxy_ip = proxy_common_api.generate_proxy_ip_api(1)
+
+        while current_page <= end_page and current_page <= total_pages:
+            proxies = {"https": proxy_ip, "http": proxy_ip}
+            try:
+                page_df = get_debt_page_data(current_page, proxies, page_number, time_out)
+                if data_frame_util.is_not_empty(page_df):
+                    local_df = pd.concat([local_df, page_df])
+                    logger.info("线程{}获取页面数据成功: {}", thread_id, current_page)
+                    current_page += 1
+                else:
+                    time.sleep(0.2)
+                    proxy_ip = proxy_common_api.generate_proxy_ip_api(1)
+                    logger.info("线程{}获取页面数据失败: {}", thread_id, current_page)
+            except BaseException as e:
+                time.sleep(1)
+                proxy_ip = proxy_common_api.generate_proxy_ip_api(1)
+                logger.error("线程{}处理页面{}时发生错误: {}", thread_id, current_page, e)
+
+        with df_lock:
+            result_df = pd.concat([result_df, local_df])
+        return len(local_df)
+
+    # 计算每个线程的页面范围
+    page_ranges = []
+    for i in range(num_threads):
+        start_page = i * pages_per_thread + 1
+        end_page = (i + 1) * pages_per_thread
+        if start_page > total_pages:
+            break
+        page_ranges.append((start_page, end_page, i + 1))
+
+    # 使用线程池执行任务
+    with concurrent.futures.ThreadPoolExecutor(max_workers=num_threads) as executor:
+        # 提交所有任务
+        futures = [
+            executor.submit(process_page_range, start, end, tid)
+            for start, end, tid in page_ranges
+        ]
+
+        # 等待所有任务完成并获取结果
+        results = []
+        for future in concurrent.futures.as_completed(futures):
+            try:
+                result = future.result()
+                results.append(result)
+            except Exception as e:
+                logger.error("线程执行出错: {}", e)
+
+    return rename_real_time_quotes_df(result_df)
+
+
+def get_kzz_real_time_quotes(time_out, pages_per_thread):
+    try_numer = 3
+    while try_numer > 0:
+        proxy_ip = proxy_common_api.generate_proxy_ip_api(1)
+        proxies = {"https": proxy_ip,
+                   "http": proxy_ip}
+
+        max_number = get_kzz_count(1, proxies, 20, time_out)
+        if max_number > 0:
+            break
+        try_numer = try_numer - 1
+    if max_number == 0:
+        return pd.DataFrame()
+
+    total_pages = (max_number + page_number - 1) // page_number  # 向上取整
+
+    num_threads = int((total_pages / pages_per_thread) + 1)
+    return repeated_acquisition_ask_etf_async(time_out, max_number, num_threads, pages_per_thread)
+
+
 if __name__ == '__main__':
-    info_df = get_kzz_bond_info()
-    print(info_df)
-    while True:
-        df = get_debt_real_time_quotes()
-        logger.info(df)
+    test_df = get_kzz_real_time_quotes(30, 6)
+    print(test_df)

mns_common/api/em/{east_money_etf_api.py → real_time/east_money_etf_api.py}

@@ -6,22 +6,22 @@ end = file_path.index('mns') + 16
 project_path = file_path[0:end]
 sys.path.append(project_path)
 
-from concurrent.futures import ThreadPoolExecutor
 import pandas as pd
 from loguru import logger
 import requests
 import time
 import numpy as np
+import mns_common.component.proxies.proxy_common_api as proxy_common_api
+import concurrent.futures
+from concurrent.futures import ThreadPoolExecutor
+from threading import Lock
+import mns_common.utils.data_frame_util as data_frame_util
 
-# 最大返回条数
-max_number = 1200
-# 最小返回条数
-min_number = 1000
 # 分页条数
 page_number = 100
 
 
-def get_fund_etf_page_df(pn) -> pd.DataFrame:
+def get_etf_count(pn, proxies, page_size, time_out):
     """
     东方财富-ETF 实时行情
     https://quote.eastmoney.com/center/gridlist.html#fund_etf
@@ -32,7 +32,7 @@ def get_fund_etf_page_df(pn) -> pd.DataFrame:
     url = "https://88.push2.eastmoney.com/api/qt/clist/get"
     params = {
         "pn": str(pn),
-        "pz": "5000",
+        "pz": str(page_size),
         "po": "1",
         "np": "3",
         "ut": "bd1d9ddb04089700cf9c27f6f7426281",
@@ -52,8 +52,62 @@ def get_fund_etf_page_df(pn) -> pd.DataFrame:
         "_": str(current_timestamp),
     }
     try:
-        r = requests.get(url, timeout=15, params=params)
+        if proxies is None:
+            r = requests.get(url, params, timeout=time_out)
+        else:
+            r = requests.get(url, params, proxies=proxies, timeout=time_out)
         data_json = r.json()
+        total_number = int(data_json['data']['total'])
+        return total_number
+    except Exception as e:
+        logger.error("获取ETF列表,实时行情异常:{}", e)
+        return 0
+
+
+def get_fund_etf_page_df(pn, proxies, page_size, time_out) -> pd.DataFrame:
+    """
+    东方财富-ETF 实时行情
+    https://quote.eastmoney.com/center/gridlist.html#fund_etf
+    :return: ETF 实时行情
+    :rtype: pandas.DataFrame
+    """
+    current_timestamp = str(int(round(time.time() * 1000, 0)))
+    url = "https://88.push2.eastmoney.com/api/qt/clist/get"
+    params = {
+        "pn": str(pn),
+        "pz": str(page_size),
+        "po": "1",
+        "np": "3",
+        "ut": "bd1d9ddb04089700cf9c27f6f7426281",
+        "fltt": "2",
+        "invt": "2",
+        "wbp2u": "|0|0|0|web",
+        "fid": "f12",
+        "fs": "b:MK0021,b:MK0022,b:MK0023,b:MK0024",
+        "fields": (
+            "f1,f2,f3,f4,f5,f6,f7,f8,f9,f10,"
+            "f12,f13,f14,f15,f16,f17,f18,f20,f21,"
+            "f23,f24,f25,f26,f22,f11,f30,f31,f32,f33,"
+            "f34,f35,f38,f62,f63,f64,f65,f66,f69,"
+            "f72,f75,f78,f81,f84,f87,f115,f124,f128,"
+            "f136,f152,f184,f297,f402,f441"
+        ),
+        "_": str(current_timestamp),
+    }
+    try:
+        if proxies is None:
+            r = requests.get(url, params, timeout=time_out)
+        else:
+            r = requests.get(url, params, proxies=proxies, timeout=time_out)
+        data_json = r.json()
+        if pn == 1:
+            try:
+                global max_number
+                max_number = int(data_json['data']['total'])
+            except Exception as e:
+                logger.error("获取第{}页ETF列表异常:{}", page_size, str(e))
+                return pd.DataFrame()
+
         temp_df = pd.DataFrame(data_json["data"]["diff"])
         temp_df.rename(
             columns={
@@ -210,7 +264,7 @@ def get_fund_etf_page_df(pn) -> pd.DataFrame:
         return pd.DataFrame()
 
 
-def thread_pool_executor():
+def thread_pool_executor(proxies):
     """
        使用多线程获取所有ETF数据
        """
@@ -222,7 +276,7 @@ def thread_pool_executor():
     # 创建线程池
     with ThreadPoolExecutor(max_workers=3) as executor:
         # 提交任务，获取每页数据
-        futures = [executor.submit(get_fund_etf_page_df, pn)
+        futures = [executor.submit(get_fund_etf_page_df, pn, proxies)
                    for pn in range(1, total_pages + 1)]
 
         # 收集结果
@@ -341,23 +395,91 @@ def rename_etf(fund_etf_spot_em_df):
     return fund_etf_spot_em_df
 
 
-def get_etf_real_time_quotes():
-    # 获取第一页数据
-    page_one_df = get_fund_etf_page_df(1)
-    # 数据接口正常返回5600以上的数量
-    if page_one_df.shape[0] > min_number:
-        page_one_df = rename_etf(page_one_df)
-        page_one_df.drop_duplicates('symbol', keep='last', inplace=True)
-        return page_one_df
-    else:
-        page_df = thread_pool_executor()
-        page_df = rename_etf(page_df)
-        page_df.drop_duplicates('symbol', keep='last', inplace=True)
-        return page_df
+def repeated_acquisition_ask_etf_async(time_out, max_number, num_threads, pages_per_thread):
+    per_page = page_number
+    total_pages = (max_number + per_page - 1) // per_page  # 向上取整
+    result_df = pd.DataFrame()
+
+    # 创建线程锁以确保线程安全
+    df_lock = Lock()
+
+    # 计算每个线程处理的页数范围
+    def process_page_range(start_page, end_page, thread_id):
+        nonlocal result_df
+        local_df = pd.DataFrame()
+        current_page = start_page
+        proxy_ip = proxy_common_api.generate_proxy_ip_api(1)
+
+        while current_page <= end_page and current_page <= total_pages:
+            proxies = {"https": proxy_ip, "http": proxy_ip}
+            try:
+                page_df = get_fund_etf_page_df(current_page, proxies, page_number, time_out)
+                if data_frame_util.is_not_empty(page_df):
+                    local_df = pd.concat([local_df, page_df])
+                    logger.info("线程{}获取页面数据成功: {}", thread_id, current_page)
+                    current_page += 1
+                else:
+                    time.sleep(0.2)
+                    proxy_ip = proxy_common_api.generate_proxy_ip_api(1)
+                    logger.info("线程{}获取页面数据失败: {}", thread_id, current_page)
+            except BaseException as e:
+                time.sleep(1)
+                proxy_ip = proxy_common_api.generate_proxy_ip_api(1)
+                logger.error("线程{}处理页面{}时发生错误: {}", thread_id, current_page, e)
+
+        with df_lock:
+            result_df = pd.concat([result_df, local_df])
+        return len(local_df)
+
+    # 计算每个线程的页面范围
+    page_ranges = []
+    for i in range(num_threads):
+        start_page = i * pages_per_thread + 1
+        end_page = (i + 1) * pages_per_thread
+        if start_page > total_pages:
+            break
+        page_ranges.append((start_page, end_page, i + 1))
+
+    # 使用线程池执行任务
+    with concurrent.futures.ThreadPoolExecutor(max_workers=num_threads) as executor:
+        # 提交所有任务
+        futures = [
+            executor.submit(process_page_range, start, end, tid)
+            for start, end, tid in page_ranges
+        ]
+
+        # 等待所有任务完成并获取结果
+        results = []
+        for future in concurrent.futures.as_completed(futures):
+            try:
+                result = future.result()
+                results.append(result)
+            except Exception as e:
+                logger.error("线程执行出错: {}", e)
+
+    return rename_etf(result_df)
+
+
+def get_etf_real_time_quotes(time_out, pages_per_thread):
+    try_numer = 3
+    while try_numer > 0:
+        proxy_ip = proxy_common_api.generate_proxy_ip_api(1)
+        proxies = {"https": proxy_ip,
+                   "http": proxy_ip}
+
+        max_number = get_etf_count(1, proxies, 20, time_out)
+        if max_number > 0:
+            break
+        try_numer = try_numer - 1
+    if max_number == 0:
+        return pd.DataFrame()
+
+    total_pages = (max_number + page_number - 1) // page_number  # 向上取整
+
+    num_threads = int((total_pages / pages_per_thread) + 1)
+    return repeated_acquisition_ask_etf_async(time_out, max_number, num_threads, pages_per_thread)
 
 
 if __name__ == '__main__':
-    fund_etf_df = get_etf_real_time_quotes()
-    fund_etf_df = fund_etf_df.sort_values(by=['amount'], ascending=False)
-    fund_etf_df = fund_etf_df.fillna(0)
-    print(fund_etf_df)
+    test_df = get_etf_real_time_quotes(30, 6)
+    print(test_df)