mns-common 1.5.2.8__tar.gz → 1.5.8.8__tar.gz

{mns_common-1.5.2.8 → mns_common-1.5.8.8}/PKG-INFO

@@ -1,3 +1,3 @@
 Metadata-Version: 2.1
 Name: mns_common
-Version: 1.5.2.8
+Version: 1.5.8.8

{mns_common-1.5.2.8 → mns_common-1.5.8.8}/mns_common/api/akshare/stock_zb_pool.py

@@ -32,6 +32,8 @@ def stock_zb_pool_df(date):
                                       "首次封板时间": "first_closure_time",
                                       "炸板次数": "frying_plates_numbers",
                                       "炸板股统计": "statistics",
+                                      "涨停统计": "zt_statistics",
+                                      "涨停价": "zt_price",
                                       "振幅": "pct_chg",
                                       "所属行业": "industry"
                                       }, inplace=True)

{mns_common-1.5.2.8 → mns_common-1.5.8.8}/mns_common/api/em/real_time/east_money_debt_api.py

@@ -1,9 +1,6 @@
 import sys
 import os
 
-import sys
-import os
-
 file_path = os.path.abspath(__file__)
 end = file_path.index('mns') + 14
 project_path = file_path[0:end]
@@ -22,10 +19,6 @@ from threading import Lock
 import concurrent.futures
 import mns_common.utils.data_frame_util as data_frame_util
 
-# 最大返回条数
-max_number = 600
-# 最小返回条数
-min_number = 500
 # 分页条数
 page_number = 100
 
@@ -59,16 +52,13 @@ def get_kzz_count(pn, proxies, page_size, time_out):
         else:
             r = requests.get(url, params, proxies=proxies, timeout=time_out)
         data_text = r.text
-        try:
-            begin_index_total = data_text.index('"total":')
 
-            end_index_total = data_text.index('"diff"')
-            global max_number
-            max_number = int(data_text[begin_index_total + 8:end_index_total - 1])
-            return max_number
-        except Exception as e:
-            logger.error(f"获取第{pn}页可转债列表异常: {e}")
-            return  0
+        begin_index_total = data_text.index('"total":')
+
+        end_index_total = data_text.index('"diff"')
+        max_number = int(data_text[begin_index_total + 8:end_index_total - 1])
+        return max_number
+
 
     except Exception as e:
         logger.error("获取可转债列表,实时行情异常:{}", e)
@@ -131,50 +121,6 @@ def get_debt_page_data(pn, proxies, page_size, time_out) -> pd.DataFrame:
         return pd.DataFrame()
 
 
-def all_debt_ticker_data(fields, proxies) -> pd.DataFrame:
-    """
-    使用多线程获取所有债券数据
-    """
-    # 计算总页数，假设总共有1000条数据，每页200条
-
-    per_page = page_number
-    total_pages = (max_number + per_page - 1) // per_page  # 向上取整
-
-    # 创建线程池
-    with ThreadPoolExecutor(max_workers=3) as executor:
-        # 提交任务，获取每页数据
-        futures = [executor.submit(get_debt_page_data, fields, pn, proxies)
-                   for pn in range(1, total_pages + 1)]
-
-        # 收集结果
-        results = []
-        for future in futures:
-            result = future.result()
-            if not result.empty:
-                results.append(result)
-
-    # 合并所有页面的数据
-    if results:
-        return pd.concat(results, ignore_index=True)
-    else:
-        return pd.DataFrame()
-
-
-def get_debt_real_time_quotes(proxies):
-    # 获取第一页数据
-    page_one_df = get_debt_page_data(fields, 1, proxies)
-    # 数据接口正常返回5600以上的数量
-    if page_one_df.shape[0] > min_number:
-        page_one_df = rename_real_time_quotes_df(page_one_df)
-        page_one_df.drop_duplicates('symbol', keep='last', inplace=True)
-        return page_one_df
-    else:
-        page_df = all_debt_ticker_data(fields, proxies)
-        page_df = rename_real_time_quotes_df(page_df)
-        page_df.drop_duplicates('symbol', keep='last', inplace=True)
-        return page_df
-
-
 def rename_real_time_quotes_df(temp_df):
     temp_df = temp_df.rename(columns={
         "f2": "now_price",
@@ -428,4 +374,3 @@ def get_kzz_real_time_quotes(time_out, pages_per_thread):
 if __name__ == '__main__':
     test_df = get_kzz_real_time_quotes(30, 6)
     print(test_df)
-

{mns_common-1.5.2.8 → mns_common-1.5.8.8}/mns_common/api/em/real_time/east_money_etf_api.py

@@ -17,10 +17,6 @@ from concurrent.futures import ThreadPoolExecutor
 from threading import Lock
 import mns_common.utils.data_frame_util as data_frame_util
 
-# 最大返回条数
-max_number = 1200
-# 最小返回条数
-min_number = 1000
 # 分页条数
 page_number = 100
 
@@ -65,7 +61,7 @@ def get_etf_count(pn, proxies, page_size, time_out):
         return total_number
     except Exception as e:
         logger.error("获取ETF列表,实时行情异常:{}", e)
-        return pd.DataFrame()
+        return 0
 
 
 def get_fund_etf_page_df(pn, proxies, page_size, time_out) -> pd.DataFrame:
@@ -399,21 +395,6 @@ def rename_etf(fund_etf_spot_em_df):
     return fund_etf_spot_em_df
 
 
-def get_etf_real_time_quotes(proxies):
-    # 获取第一页数据
-    page_one_df = get_fund_etf_page_df(1, proxies)
-    # 数据接口正常返回5600以上的数量
-    if page_one_df.shape[0] > min_number:
-        page_one_df = rename_etf(page_one_df)
-        page_one_df.drop_duplicates('symbol', keep='last', inplace=True)
-        return page_one_df
-    else:
-        page_df = thread_pool_executor(proxies)
-        page_df = rename_etf(page_df)
-        page_df.drop_duplicates('symbol', keep='last', inplace=True)
-        return page_df
-
-
 def repeated_acquisition_ask_etf_async(time_out, max_number, num_threads, pages_per_thread):
     per_page = page_number
     total_pages = (max_number + per_page - 1) // per_page  # 向上取整
@@ -487,7 +468,7 @@ def get_etf_real_time_quotes(time_out, pages_per_thread):
                    "http": proxy_ip}
 
         max_number = get_etf_count(1, proxies, 20, time_out)
-        if max_number>0:
+        if max_number > 0:
             break
         try_numer = try_numer - 1
     if max_number == 0:

{mns_common-1.5.2.8 → mns_common-1.5.8.8}/mns_common/api/em/real_time/east_money_stock_a_v2_api.py

@@ -11,17 +11,16 @@ import requests
 import mns_common.utils.data_frame_util as data_frame_util
 import json
 import datetime
-
-import threading
 from concurrent.futures import ThreadPoolExecutor
 import mns_common.component.proxies.proxy_common_api as proxy_common_api
 from loguru import logger
 import concurrent.futures
 import pandas as pd
 import time
-from concurrent.futures import ThreadPoolExecutor, as_completed
+from concurrent.futures import ThreadPoolExecutor
 from threading import Lock
 import mns_common.api.em.real_time.real_time_quotes_repeat_api as real_time_quotes_repeat_api
+import mns_common.api.em.real_time.east_money_stock_common_api as east_money_stock_common_api
 
 #
 # fields_02 = "f1,f2,f3,f4,f5,f6,f7,f8,f9,f10,f11,f12,f13,f14,f15,f16,f17,f18,f19,f20,f21,f22,f23,f24,f25,f26,f27,f28,f29,f30,f31,f32,f33,f34,f35,f36,f37,f38,f39,f40,f41,f42,f43,f44,f45,f46,f47,f48,f49,f50,f51,f52,f53,f54,f55,f56,f57,f58,f59,f60,f61,f62,f63,f64,f65,f66,f67,f68,f69,f70,f71,f72,f73,f74,f75,f76,f77,f78,f79,f80,f81,f82,f83,f84,f85,f86,f87,f88,f89,f90,f91,f92,f93,f94,f95,f96,f97,f98,f99,f100,f101,f102,f103,f104,f105,f106,f107,f108" \
@@ -43,12 +42,8 @@ fields = ("f2,f3,f5,f6,f8,"
           "f84,f102,f184,f100,f103,"
           "f352,f191,f193,f24,f25")
 
-# 最大返回条数
-max_number = 5800
-# 最小返回条数
-min_number = 5600
 # 分页条数
-page_number = 100
+PAGE_SIZE = 100
 
 
 def get_stock_page_data(pn, proxies, page_number, time_out):
@@ -92,51 +87,72 @@ def get_stock_page_data(pn, proxies, page_number, time_out):
         else:
             return pd.DataFrame(data_json)
     except Exception as e:
-        logger.error(f"获取第{pn}页股票列表异常: {e}")
         return pd.DataFrame()
 
 
-def all_stock_ticker_data_new(fields, fs, proxies) -> pd.DataFrame:
-    """
-    使用多线程获取所有股票数据
+def all_stock_ticker_data_new(initial_proxies, time_out, max_number) -> pd.DataFrame:
     """
-
-    per_page = page_number
-    total_pages = (max_number + per_page - 1) // per_page  # 向上取整
-
-    # 创建线程池
-    with ThreadPoolExecutor(max_workers=10) as executor:
-        # 提交任务，获取每页数据
-        futures = [executor.submit(get_stock_page_data, pn, fields, fs, proxies)
-                   for pn in range(1, total_pages + 1)]
-
-        # 收集结果
-        results = []
-        for future in futures:
-            result = future.result()
-            if not result.empty:
-                results.append(result)
-
-    # 合并所有页面的数据
+            使用多线程获取所有股票数据，失败页面会使用新IP重试，最多使用10个IP
+            """
+
+    total_pages = (max_number + PAGE_SIZE - 1) // PAGE_SIZE  # 向上取整
+    all_pages = set(range(1, total_pages + 1))  # 所有需要获取的页码
+    success_pages = set()  # 成功获取的页码
+    results = []  # 存储成功获取的数据
+    used_ip_count = 1  # 已使用IP计数器（初始IP算第一个）
+    MAX_IP_LIMIT = 10  # IP使用上限
+
+    # 循环处理直到所有页面成功或达到IP上限
+    while (all_pages - success_pages) and (used_ip_count < MAX_IP_LIMIT):
+        # 获取当前需要处理的失败页码
+        current_failed_pages = all_pages - success_pages
+        if used_ip_count > 1:
+            logger.info("当前需要处理的失败页码: {}, 已使用IP数量: {}/{}", current_failed_pages, used_ip_count,
+                        MAX_IP_LIMIT)
+
+        # 首次使用初始代理，后续获取新代理
+        if len(success_pages) == 0:
+            proxies = initial_proxies
+        else:
+            # 每次重试前获取新代理并计数
+            # logger.info("获取新代理IP处理失败页面")
+            new_proxy_ip = proxy_common_api.generate_proxy_ip_api(1)
+            proxies = {"https": new_proxy_ip}
+            # logger.info("新代理IP: {}, 已使用IP数量: {}/{}", new_proxy_ip, used_ip_count + 1, MAX_IP_LIMIT)
+            used_ip_count += 1  # 增加IP计数器
+
+        # 创建线程池处理当前失败的页码
+        with ThreadPoolExecutor(max_workers=10) as executor:
+            futures = {
+                executor.submit(get_stock_page_data, pn, proxies, PAGE_SIZE, time_out): pn
+                for pn in current_failed_pages
+            }
+
+            # 收集结果并记录成功页码
+            for future, pn in futures.items():
+                try:
+                    result = future.result()
+                    if not result.empty:
+                        results.append(result)
+                        success_pages.add(pn)
+                    # else:
+                    #     logger.warning("页码 {} 未返回有效数据", pn)
+                except Exception as e:
+                    continue
+                    # logger.error("页码 {} 处理异常: {}", pn, str(e))
+
+    # 检查是否达到IP上限
+    if used_ip_count >= MAX_IP_LIMIT and (all_pages - success_pages):
+        remaining_pages = all_pages - success_pages
+        logger.warning("已达到最大IP使用限制({}个)，剩余未获取页码: {}, 返回现有数据", MAX_IP_LIMIT, remaining_pages)
+
+    # 合并所有成功获取的数据
     if results:
         return pd.concat(results, ignore_index=True)
     else:
         return pd.DataFrame()
 
 
-def get_all_real_time_quotes(proxies):
-    # 获取第一页数据
-    page_one_df = get_stock_page_data(1, fields, fs, proxies)
-    # 数据接口正常返回5600以上的数量
-    if page_one_df.shape[0] > min_number:
-        page_one_df = rename_real_time_quotes_df(page_one_df)
-        return page_one_df
-    else:
-        page_df = all_stock_ticker_data_new(fields, fs, proxies)
-        page_df = rename_real_time_quotes_df(page_df)
-        return page_df
-
-
 # 获取所有股票实时行情数据    f33,委比
 def rename_real_time_quotes_df(temp_df):
     temp_df = temp_df.rename(columns={
@@ -297,96 +313,28 @@ def rename_real_time_quotes_df(temp_df):
     return temp_df
 
 
-def repeated_acquisition_ask_async(time_out, max_number, num_threads, pages_per_thread):
-    per_page = page_number
-    total_pages = (max_number + per_page - 1) // per_page  # 向上取整
-    result_df = pd.DataFrame()
-
-    # 创建线程锁以确保线程安全
-    df_lock = Lock()
-
-    # 计算每个线程处理的页数范围
-    def process_page_range(start_page, end_page, thread_id):
-        nonlocal result_df
-        local_df = pd.DataFrame()
-        current_page = start_page
-        proxy_ip = proxy_common_api.generate_proxy_ip_api(1)
-
-        while current_page <= end_page and current_page <= total_pages:
-            proxies = {"https": proxy_ip, "http": proxy_ip}
-            try:
-                page_df = get_stock_page_data(current_page, proxies, page_number, time_out)
-                if data_frame_util.is_not_empty(page_df):
-                    local_df = pd.concat([local_df, page_df])
-                    logger.info("线程{}获取页面数据成功: {}", thread_id, current_page)
-                    current_page += 1
-                else:
-                    time.sleep(0.2)
-                    proxy_ip = proxy_common_api.generate_proxy_ip_api(1)
-                    logger.info("线程{}获取页面数据失败: {}", thread_id, current_page)
-            except BaseException as e:
-                time.sleep(1)
-                proxy_ip = proxy_common_api.generate_proxy_ip_api(1)
-                logger.error("线程{}处理页面{}时发生错误: {}", thread_id, current_page, e)
-
-        with df_lock:
-            result_df = pd.concat([result_df, local_df])
-        return len(local_df)
-
-    # 计算每个线程的页面范围
-    page_ranges = []
-    for i in range(num_threads):
-        start_page = i * pages_per_thread + 1
-        end_page = (i + 1) * pages_per_thread
-        if start_page > total_pages:
-            break
-        page_ranges.append((start_page, end_page, i + 1))
-
-    # 使用线程池执行任务
-    with concurrent.futures.ThreadPoolExecutor(max_workers=num_threads) as executor:
-        # 提交所有任务
-        futures = [
-            executor.submit(process_page_range, start, end, tid)
-            for start, end, tid in page_ranges
-        ]
-
-        # 等待所有任务完成并获取结果
-        results = []
-        for future in concurrent.futures.as_completed(futures):
-            try:
-                result = future.result()
-                results.append(result)
-            except Exception as e:
-                logger.error("线程执行出错: {}", e)
-
-    return rename_real_time_quotes_df(result_df)
-
-
-def get_stock_real_time_quotes(time_out, pages_per_thread):
+def get_stock_real_time_quotes(time_out):
     try_numer = 3
     while try_numer > 0:
         proxy_ip = proxy_common_api.generate_proxy_ip_api(1)
-        proxies = {"https": proxy_ip,
-                   "http": proxy_ip}
+        initial_proxies = {"https": proxy_ip,
+                           "http": proxy_ip}
 
-        total_number = real_time_quotes_repeat_api.get_stocks_num(1, proxies, 20, time_out)
+        total_number = east_money_stock_common_api.get_stocks_num(1, initial_proxies, 20, time_out)
         if total_number > 0:
             break
         try_numer = try_numer - 1
     if total_number == 0:
         return pd.DataFrame()
 
-    total_pages = (max_number + page_number - 1) // page_number  # 向上取整
-
-    num_threads = int((total_pages / pages_per_thread) + 1)
-    return repeated_acquisition_ask_async(time_out, max_number, num_threads, pages_per_thread)
+    page_df = all_stock_ticker_data_new(initial_proxies, time_out, total_number)
+    page_df = rename_real_time_quotes_df(page_df)
+    page_df.drop_duplicates('symbol', keep='last', inplace=True)
+    return page_df
 
 
 # 示例调用
 if __name__ == "__main__":
-    number = 1
-    while True:
-        df = get_all_real_time_quotes(None)
-        zt_df = df.loc[df['wei_bi'] == 100]
-        logger.info("同步次数,{}", number)
-        number = number + 1
+    df = get_stock_real_time_quotes(5)
+    zt_df = df.loc[df['wei_bi'] == 100]
+    logger.info("同步次数", zt_df.shape[0])

mns_common-1.5.2.8/mns_common/api/em/real_time/east_money_stock_common_rename.py → mns_common-1.5.8.8/mns_common/api/em/real_time/east_money_stock_common_api.py

@@ -1,6 +1,21 @@
-
 import pandas as pd
 import mns_common.utils.data_frame_util as data_frame_util
+import requests
+from loguru import logger
+import datetime
+
+fs = "m:0 t:6,m:0 t:80,m:1 t:2,m:1 t:23,m:0 t:81 s:2048"
+fields = ("f2,f3,f5,f6,f8,"
+          "f9,f10,f22,f12,f13,"
+          "f14,f15,f16,f17,f18,"
+          "f20,f21,f23,f26,f33,"
+          "f34,f35,f37,f38,f39,"
+          "f62,f64,f65,f67,f68,"
+          "f66,f69,f70,f71,f72,"
+          "f76,f77,f78,f82,f83,"
+          "f84,f102,f184,f100,f103,"
+          "f352,f191,f193,f24,f25")
+
 
 # 获取所有股票实时行情数据    f33,委比
 def rename_real_time_quotes_df(temp_df):
@@ -107,3 +122,53 @@ def rename_real_time_quotes_df(temp_df):
         temp_df.loc[temp_df["inner_disk"] == 0, ['disk_ratio']] = 1688
         temp_df = temp_df.sort_values(by=['chg'], ascending=False)
         return temp_df
+
+
+def get_stocks_num(pn, proxies, page_number, time_out):
+    """
+    获取单页股票数据
+    """
+    # 获取当前日期和时间
+    current_time = datetime.datetime.now()
+
+    # 将当前时间转换为时间戳（以毫秒为单位）
+    current_timestamp_ms = int(current_time.timestamp() * 1000)
+
+    url = "https://13.push2.eastmoney.com/api/qt/clist/get"
+    params = {
+        "cb": "jQuery1124046660442520420653_" + str(current_timestamp_ms),
+        "pn": str(pn),
+        "pz": str(page_number),  # 每页最大200条
+        "po": "1",
+        "np": "3",
+        "ut": "bd1d9ddb04089700cf9c27f6f7426281",
+        "fltt": "2",
+        "invt": "2",
+        "wbp2u": "|0|0|0|web",
+        "fid": "f3",
+        "fs": fs,
+        "fields": fields,
+        "_": current_timestamp_ms
+    }
+    try:
+        if proxies is None:
+            r = requests.get(url, params, timeout=time_out)
+        else:
+            r = requests.get(url, params, proxies=proxies, timeout=time_out)
+        data_text = r.text
+
+        begin_index_total = data_text.index('"total":')
+        end_index_total = data_text.index('"diff"')
+        return int(data_text[begin_index_total + 8:end_index_total - 1])
+
+    except Exception as e:
+        logger.error("获取股票数量异常:{}", str(e))
+        return 0
+
+
+import mns_common.component.proxies.proxy_common_api as proxy_common_api
+
+if __name__ == '__main__':
+    proxy_ip = proxy_common_api.generate_proxy_ip_api(1)
+    proxies = {"https": proxy_ip}
+    get_stocks_num(1, proxies, 100, 30)