mns-common 1.5.1.7__tar.gz → 1.5.1.9__tar.gz

{mns_common-1.5.1.7 → mns_common-1.5.1.9}/PKG-INFO

@@ -1,3 +1,3 @@
 Metadata-Version: 2.1
 Name: mns_common
-Version: 1.5.1.7
+Version: 1.5.1.9

{mns_common-1.5.1.7/mns_common/api/em → mns_common-1.5.1.9/mns_common/api/em/real_time}/real_time/east_money_debt_api.py

@@ -8,14 +8,19 @@ file_path = os.path.abspath(__file__)
 end = file_path.index('mns') + 14
 project_path = file_path[0:end]
 sys.path.append(project_path)
-from concurrent.futures import ThreadPoolExecutor
+
+import json
+import akshare as ak
 import pandas as pd
 from loguru import logger
-import json
 import requests
 import time
-import akshare as ak
 import numpy as np
+import mns_common.component.proxies.proxy_common_api as proxy_common_api
+from concurrent.futures import ThreadPoolExecutor
+from threading import Lock
+import concurrent.futures
+import mns_common.utils.data_frame_util as data_frame_util
 
 # 最大返回条数
 max_number = 600
@@ -24,19 +29,49 @@ min_number = 500
 # 分页条数
 page_number = 100
 
+fields = ("f352,f2,f3,f5,f6,f8,f10,f11,f22,f12,f14,f15,f16,f17,f18,f20,f21,f26,f33,f34,f35,f62,f66,f69,f72,f184,"
+          "f211,f212,f232,f233,f234")
+
+
+def get_kzz_count(pn, proxies, page_size, time_out):
+    current_timestamp = str(int(round(time.time() * 1000, 0)))
+    url = "https://push2.eastmoney.com/api/qt/clist/get"
+
+    params = {
+        "cb": "jQuery34103608466964799838_" + current_timestamp,
+        "pn": str(pn),
+        "np": 3,
+        "ut": "8a086bfc3570bdde64a6a1c585cccb35",
+        "fltt": 1,
+        "invt": 1,
+        "fs": "m:0+e:11,m:1+e:11,m:1+e:11+s:4194304,m:0+e:11+s:8388608",
+        "dpt": "zqsc.zpg",
+        "fields": fields,
+        "wbp2u": "|0|0|0|wap",
+        "fid": "f12",
+        "po": 1,
+        "pz": str(page_size),
+        "_": current_timestamp
+    }
+    try:
+        if proxies is None:
+            r = requests.get(url, params, timeout=time_out)
+        else:
+            r = requests.get(url, params, proxies=proxies, timeout=time_out)
+        data_text = r.text
+        total_number = int(data_json['data']['total'])
+        return total_number
+    except Exception as e:
+        logger.error("获取可转债列表,实时行情异常:{}", e)
+        return 0
 
-# fields_02 = "f1,f2,f3,f4,f5,f6,f7,f8,f9,f10,f11,f12,f13,f14,f15,f16,f17,f18,f19,f20,f21,f22,f23,f24,f25,f26,f27,f28,f29,f30,f31,f32,f33,f34,f35,f36,f37,f38,f39,f40,f41,f42,f43,f44,f45,f46,f47,f48,f49,f50,f51,f52,f53,f54,f55,f56,f57,f58,f59,f60,f61,f62,f63,f64,f65,f66,f67,f68,f69,f70,f71,f72,f73,f74,f75,f76,f77,f78,f79,f80,f81,f82,f83,f84,f85,f86,f87,f88,f89,f90,f91,f92,f93,f94,f95,f96,f97,f98,f99,f100,f101,f102,f103,f104,f105,f106,f107,f108" \
-#             ",f109,f110,f111,f112,f113,f114,f115,f116,f117,f118,f119,f120,f121,f122,f123,f124,f125,f126,f127,f128,f129,f130,f131,f132,f133,f134,f135,f136,f137,f138,f139,f140,f141,f142,f143,f144,f145,f146,f147,f148,f149,f150,f151,f152,f153,f154,f155,f156,f157,f158,f159,f160,f161,f162,f163,f164,f165,f166,f167,f168,f169,f170,f171,f172,f173,f174,f175,f176,f177,f178,f179,f180,f181,f182,f183,f184,f185,f186,f187,f188,f189,f190,f191,f192,f193,f194,f195,f196,f197,f198,f199,f200" \
-#             ",f209,f210,f211,f212,f213,f214,f215,f216,f217,f218,f219,f220,f221,f222,f223,f224,f225,f226,f227,f228,f229,f230,f231,f232,f233,f234,f235,f236,f237,f238,f239,f240,f241,f242,f243,f244,f245,f246,f247,f248,f249,f250,f251,f252,f253,f254,f255,f256,f257,f258,f259,f260,f261,f262,f263,f264,f265,f266,f267,f268,f269,f270,f271,f272,f273,f274,f275,f276,f277,f278,f279,f280,f281,f282,f283,f284,f285,f286,f287,f288,f289,f290,f291,f292,f293,f294,f295,f296,f297,f298,f299,f300" \
-#             ",f309,f310,f312,f313,f314,f315,f316,f317,f318,f319,f320,f321,f322,f323,f324,f325,f326,f327,f328,f329,f330,f331,f332,f333,f334,f335,f336,f337,f338,f339,f340,f341,f342,f343,f344,f345,f346,f347,f348,f349,f350,f351,f352,f353,f354,f355,f356,f357,f358,f359,f360,f361,f362,f363,f364,f365,f366,f367,f368,f369,f370,f371,f372,f373,f374,f375,f376,f377,f378,f379,f380,f381,f382,f383,f384,f385,f386,f387,f388,f389,f390,f391,f392,f393,f394,f395,f396,f397,f398,f399,f401"
-#
 
 #
 # url = https://push2.eastmoney.com/api/qt/clist/get?cb=jQuery34103608466964799838_1718163189869&pn=1&np=1&ut
 # =8a086bfc3570bdde64a6a1c585cccb35&fltt=1&invt=1&fs=m:0+e:11,m:1+e:11,m:1+e:11+s:4194304,
 # m:0+e:11+s:8388608&dpt=zqsc.zpg&fields=f1,f2,f3,f4,f5,f6,f8,f10,f12,f13,f14,f18,f22,f152,
 # f237&wbp2u=|0|0|0|wap&fid=f3&po=1&pz=2000&_=1718163189870
-def get_debt_page_data(fields, pn, proxies) -> pd.DataFrame:
+def get_debt_page_data(pn, proxies, page_size, time_out) -> pd.DataFrame:
     current_timestamp = str(int(round(time.time() * 1000, 0)))
     url = "https://push2.eastmoney.com/api/qt/clist/get"
 
@@ -53,14 +88,14 @@ def get_debt_page_data(fields, pn, proxies) -> pd.DataFrame:
         "wbp2u": "|0|0|0|wap",
         "fid": "f12",
         "po": 1,
-        "pz": 2000,
+        "pz": str(page_size),
         "_": current_timestamp
     }
     try:
         if proxies is None:
-            r = requests.get(url, params)
+            r = requests.get(url, params, timeout=time_out)
         else:
-            r = requests.get(url, params, proxies=proxies)
+            r = requests.get(url, params, proxies=proxies, timeout=time_out)
         data_text = r.text
 
         if pn == 1:
@@ -117,8 +152,6 @@ def all_debt_ticker_data(fields, proxies) -> pd.DataFrame:
 
 
 def get_debt_real_time_quotes(proxies):
-    fields = ("f352,f2,f3,f5,f6,f8,f10,f11,f22,f12,f14,f15,f16,f17,f18,f20,f21,f26,f33,f34,f35,f62,f66,f69,f72,f184,"
-              "f211,f212,f232,f233,f234")
     # 获取第一页数据
     page_one_df = get_debt_page_data(fields, 1, proxies)
     # 数据接口正常返回5600以上的数量
@@ -298,9 +331,92 @@ def get_kzz_bond_info():
         logger.error("获取可转债信息异常:{}", e)
 
 
+def repeated_acquisition_ask_etf_async(time_out, max_number, num_threads, pages_per_thread):
+    per_page = page_number
+    total_pages = (max_number + per_page - 1) // per_page  # 向上取整
+    result_df = pd.DataFrame()
+
+    # 创建线程锁以确保线程安全
+    df_lock = Lock()
+
+    # 计算每个线程处理的页数范围
+    def process_page_range(start_page, end_page, thread_id):
+        nonlocal result_df
+        local_df = pd.DataFrame()
+        current_page = start_page
+        proxy_ip = proxy_common_api.generate_proxy_ip_api(1)
+
+        while current_page <= end_page and current_page <= total_pages:
+            proxies = {"https": proxy_ip, "http": proxy_ip}
+            try:
+                page_df = get_debt_page_data(current_page, proxies, page_number, time_out)
+                if data_frame_util.is_not_empty(page_df):
+                    local_df = pd.concat([local_df, page_df])
+                    logger.info("线程{}获取页面数据成功: {}", thread_id, current_page)
+                    current_page += 1
+                else:
+                    time.sleep(0.2)
+                    proxy_ip = proxy_common_api.generate_proxy_ip_api(1)
+                    logger.info("线程{}获取页面数据失败: {}", thread_id, current_page)
+            except BaseException as e:
+                time.sleep(1)
+                proxy_ip = proxy_common_api.generate_proxy_ip_api(1)
+                logger.error("线程{}处理页面{}时发生错误: {}", thread_id, current_page, e)
+
+        with df_lock:
+            result_df = pd.concat([result_df, local_df])
+        return len(local_df)
+
+    # 计算每个线程的页面范围
+    page_ranges = []
+    for i in range(num_threads):
+        start_page = i * pages_per_thread + 1
+        end_page = (i + 1) * pages_per_thread
+        if start_page > total_pages:
+            break
+        page_ranges.append((start_page, end_page, i + 1))
+
+    # 使用线程池执行任务
+    with concurrent.futures.ThreadPoolExecutor(max_workers=num_threads) as executor:
+        # 提交所有任务
+        futures = [
+            executor.submit(process_page_range, start, end, tid)
+            for start, end, tid in page_ranges
+        ]
+
+        # 等待所有任务完成并获取结果
+        results = []
+        for future in concurrent.futures.as_completed(futures):
+            try:
+                result = future.result()
+                results.append(result)
+            except Exception as e:
+                logger.error("线程执行出错: {}", e)
+
+    return rename_real_time_quotes_df(result_df)
+
+
+def get_kzz_real_time_quotes(time_out, pages_per_thread):
+    try_numer = 3
+    while try_numer > 0:
+        proxy_ip = proxy_common_api.generate_proxy_ip_api(1)
+        proxies = {"https": proxy_ip,
+                   "http": proxy_ip}
+
+        max_number = get_kzz_count(1, proxies, 20, time_out)
+        if max_number > 0:
+            break
+        try_numer = try_numer - 1
+    if max_number == 0:
+        return pd.DataFrame()
+
+    total_pages = (max_number + page_number - 1) // page_number  # 向上取整
+
+    num_threads = int((total_pages / pages_per_thread) + 1)
+    return repeated_acquisition_ask_etf_async(time_out, max_number, num_threads, pages_per_thread)
+
+
 if __name__ == '__main__':
-    info_df = get_kzz_bond_info()
-    print(info_df)
-    while True:
-        df = get_debt_real_time_quotes(None)
-        logger.info(df)
+    test_df = get_kzz_real_time_quotes(30, 6)
+    print(test_df)
+

{mns_common-1.5.1.7/mns_common/api/em → mns_common-1.5.1.9/mns_common/api/em/real_time}/real_time/east_money_etf_api.py

@@ -6,12 +6,16 @@ end = file_path.index('mns') + 16
 project_path = file_path[0:end]
 sys.path.append(project_path)
 
-from concurrent.futures import ThreadPoolExecutor
 import pandas as pd
 from loguru import logger
 import requests
 import time
 import numpy as np
+import mns_common.component.proxies.proxy_common_api as proxy_common_api
+import concurrent.futures
+from concurrent.futures import ThreadPoolExecutor
+from threading import Lock
+import mns_common.utils.data_frame_util as data_frame_util
 
 # 最大返回条数
 max_number = 1200
@@ -21,7 +25,7 @@ min_number = 1000
 page_number = 100
 
 
-def get_fund_etf_page_df(pn, proxies) -> pd.DataFrame:
+def get_etf_count(pn, proxies, page_size, time_out):
     """
     东方财富-ETF 实时行情
     https://quote.eastmoney.com/center/gridlist.html#fund_etf
@@ -32,7 +36,7 @@ def get_fund_etf_page_df(pn, proxies) -> pd.DataFrame:
     url = "https://88.push2.eastmoney.com/api/qt/clist/get"
     params = {
         "pn": str(pn),
-        "pz": "5000",
+        "pz": str(page_size),
         "po": "1",
         "np": "3",
         "ut": "bd1d9ddb04089700cf9c27f6f7426281",
@@ -53,16 +57,59 @@ def get_fund_etf_page_df(pn, proxies) -> pd.DataFrame:
     }
     try:
         if proxies is None:
-            r = requests.get(url, params)
+            r = requests.get(url, params, timeout=time_out)
         else:
-            r = requests.get(url, params, proxies=proxies)
+            r = requests.get(url, params, proxies=proxies, timeout=time_out)
+        data_json = r.json()
+        total_number = int(data_json['data']['total'])
+        return total_number
+    except Exception as e:
+        logger.error("获取ETF列表,实时行情异常:{}", e)
+        return pd.DataFrame()
+
+
+def get_fund_etf_page_df(pn, proxies, page_size, time_out) -> pd.DataFrame:
+    """
+    东方财富-ETF 实时行情
+    https://quote.eastmoney.com/center/gridlist.html#fund_etf
+    :return: ETF 实时行情
+    :rtype: pandas.DataFrame
+    """
+    current_timestamp = str(int(round(time.time() * 1000, 0)))
+    url = "https://88.push2.eastmoney.com/api/qt/clist/get"
+    params = {
+        "pn": str(pn),
+        "pz": str(page_size),
+        "po": "1",
+        "np": "3",
+        "ut": "bd1d9ddb04089700cf9c27f6f7426281",
+        "fltt": "2",
+        "invt": "2",
+        "wbp2u": "|0|0|0|web",
+        "fid": "f12",
+        "fs": "b:MK0021,b:MK0022,b:MK0023,b:MK0024",
+        "fields": (
+            "f1,f2,f3,f4,f5,f6,f7,f8,f9,f10,"
+            "f12,f13,f14,f15,f16,f17,f18,f20,f21,"
+            "f23,f24,f25,f26,f22,f11,f30,f31,f32,f33,"
+            "f34,f35,f38,f62,f63,f64,f65,f66,f69,"
+            "f72,f75,f78,f81,f84,f87,f115,f124,f128,"
+            "f136,f152,f184,f297,f402,f441"
+        ),
+        "_": str(current_timestamp),
+    }
+    try:
+        if proxies is None:
+            r = requests.get(url, params, timeout=time_out)
+        else:
+            r = requests.get(url, params, proxies=proxies, timeout=time_out)
         data_json = r.json()
         if pn == 1:
             try:
                 global max_number
                 max_number = int(data_json['data']['total'])
             except Exception as e:
-                logger.error(f"获取第{pn}页ETF列表异常: {e}")
+                logger.error("获取第{}页ETF列表异常:{}", page_size, str(e))
                 return pd.DataFrame()
 
         temp_df = pd.DataFrame(data_json["data"]["diff"])
@@ -367,8 +414,91 @@ def get_etf_real_time_quotes(proxies):
         return page_df
 
 
+def repeated_acquisition_ask_etf_async(time_out, max_number, num_threads, pages_per_thread):
+    per_page = page_number
+    total_pages = (max_number + per_page - 1) // per_page  # 向上取整
+    result_df = pd.DataFrame()
+
+    # 创建线程锁以确保线程安全
+    df_lock = Lock()
+
+    # 计算每个线程处理的页数范围
+    def process_page_range(start_page, end_page, thread_id):
+        nonlocal result_df
+        local_df = pd.DataFrame()
+        current_page = start_page
+        proxy_ip = proxy_common_api.generate_proxy_ip_api(1)
+
+        while current_page <= end_page and current_page <= total_pages:
+            proxies = {"https": proxy_ip, "http": proxy_ip}
+            try:
+                page_df = get_fund_etf_page_df(current_page, proxies, page_number, time_out)
+                if data_frame_util.is_not_empty(page_df):
+                    local_df = pd.concat([local_df, page_df])
+                    logger.info("线程{}获取页面数据成功: {}", thread_id, current_page)
+                    current_page += 1
+                else:
+                    time.sleep(0.2)
+                    proxy_ip = proxy_common_api.generate_proxy_ip_api(1)
+                    logger.info("线程{}获取页面数据失败: {}", thread_id, current_page)
+            except BaseException as e:
+                time.sleep(1)
+                proxy_ip = proxy_common_api.generate_proxy_ip_api(1)
+                logger.error("线程{}处理页面{}时发生错误: {}", thread_id, current_page, e)
+
+        with df_lock:
+            result_df = pd.concat([result_df, local_df])
+        return len(local_df)
+
+    # 计算每个线程的页面范围
+    page_ranges = []
+    for i in range(num_threads):
+        start_page = i * pages_per_thread + 1
+        end_page = (i + 1) * pages_per_thread
+        if start_page > total_pages:
+            break
+        page_ranges.append((start_page, end_page, i + 1))
+
+    # 使用线程池执行任务
+    with concurrent.futures.ThreadPoolExecutor(max_workers=num_threads) as executor:
+        # 提交所有任务
+        futures = [
+            executor.submit(process_page_range, start, end, tid)
+            for start, end, tid in page_ranges
+        ]
+
+        # 等待所有任务完成并获取结果
+        results = []
+        for future in concurrent.futures.as_completed(futures):
+            try:
+                result = future.result()
+                results.append(result)
+            except Exception as e:
+                logger.error("线程执行出错: {}", e)
+
+    return rename_etf(result_df)
+
+
+def get_etf_real_time_quotes(time_out, pages_per_thread):
+    try_numer = 3
+    while try_numer > 0:
+        proxy_ip = proxy_common_api.generate_proxy_ip_api(1)
+        proxies = {"https": proxy_ip,
+                   "http": proxy_ip}
+
+        max_number = get_etf_count(1, proxies, 20, time_out)
+        if max_number>0:
+            break
+        try_numer = try_numer - 1
+    if max_number == 0:
+        return pd.DataFrame()
+
+    total_pages = (max_number + page_number - 1) // page_number  # 向上取整
+
+    num_threads = int((total_pages / pages_per_thread) + 1)
+    return repeated_acquisition_ask_etf_async(time_out, max_number, num_threads, pages_per_thread)
+
+
 if __name__ == '__main__':
-    fund_etf_df = get_etf_real_time_quotes(None)
-    fund_etf_df = fund_etf_df.sort_values(by=['amount'], ascending=False)
-    fund_etf_df = fund_etf_df.fillna(0)
-    print(fund_etf_df)
+    test_df = get_etf_real_time_quotes(30, 6)
+    print(test_df)

{mns_common-1.5.1.7/mns_common/api/em → mns_common-1.5.1.9/mns_common/api/em/real_time}/real_time/east_money_stock_a_api.py

@@ -1,6 +1,6 @@
-import os
 import sys
-from loguru import logger
+import os
+import time
 
 file_path = os.path.abspath(__file__)
 end = file_path.index('mns') + 16
@@ -12,6 +12,7 @@ import json
 import pandas as pd
 from concurrent.futures import ThreadPoolExecutor
 import datetime
+from loguru import logger
 import mns_common.utils.data_frame_util as data_frame_util
 
 mongodb_util = MongodbUtil('27017')
@@ -68,7 +69,7 @@ def get_stock_page_data(pn, proxies, page_size):
                 global max_number
                 max_number = int(data_text[begin_index_total + 8:end_index_total - 1])
             except Exception as e:
-                # logger.error(f"获取第{pn}页股票列表异常: {e}")
+                logger.error(f"获取第{pn}页股票列表异常: {e}")
                 return pd.DataFrame()
 
         begin_index = data_text.index('[')
@@ -78,11 +79,9 @@ def get_stock_page_data(pn, proxies, page_size):
         if data_json is None:
             return pd.DataFrame()
         else:
-            result_df = pd.DataFrame(data_json)
-            result_df['page_number'] = pn
-            return result_df
+            return pd.DataFrame(data_json)
     except Exception as e:
-        logger.error("获取第{}页股票列表异常:{}", pn, str(e))
+        logger.error(f"获取第{pn}页股票列表异常: {e}")
         return pd.DataFrame()
 
 
@@ -291,13 +290,16 @@ def get_sum_north_south_net_buy_amt():
     return df
 
 
-import mns_common.component.proxies.proxy_common_api as proxy_common_api
-
-if __name__ == '__main__':
-
-    proxy_ip = proxy_common_api.generate_proxy_ip_api(1)
-    proxies = {"https": proxy_ip,
-               "http": proxy_ip}
+# 示例调用
+if __name__ == "__main__":
     while True:
-        result = all_stock_ticker_data_new(proxies)
-        print(result)
+        ip_proxy_pool = mongodb_util.find_all_data('ip_proxy_pool')
+        if data_frame_util.is_not_empty(ip_proxy_pool):
+            proxy_ip = list(ip_proxy_pool['ip'])[0]
+            proxy = {
+                "https": proxy_ip}
+            df = get_real_time_quotes_all_stocks(proxy)
+            logger.info("涨停数据,{}", 1)
+        else:
+            time.sleep(1)
+            logger.error("ip为空")