mns-common 1.3.3.5__py3-none-any.whl → 1.5.7.2__py3-none-any.whl

mns_common/api/em/real_time/east_money_stock_hk_gtt_api.py

@@ -0,0 +1,260 @@
+import sys
+import os
+
+file_path = os.path.abspath(__file__)
+end = file_path.index('mns') + 16
+project_path = file_path[0:end]
+sys.path.append(project_path)
+
+import pandas as pd
+from loguru import logger
+import requests
+import time
+import mns_common.component.proxies.proxy_common_api as proxy_common_api
+import concurrent.futures
+
+from threading import Lock
+import mns_common.utils.data_frame_util as data_frame_util
+
+# 分页条数
+page_number = 100
+
+
+def rename_hg_ggt(temp_df):
+    temp_df.columns = [
+        "序号",
+        "-",
+        "最新价",
+        "涨跌幅",
+        "涨跌额",
+        "成交量",
+        "成交额",
+        "-",
+        "-",
+        "-",
+        "-",
+        "-",
+        "代码",
+        "-",
+        "名称",
+        "最高",
+        "最低",
+        "今开",
+        "昨收",
+        "-",
+        "-",
+        "-",
+        "-",
+        "-",
+        "-",
+        "-",
+        "-",
+        "-",
+        "-",
+        "-",
+        "-",
+        "-",
+        "-",
+        "-",
+        "-",
+    ]
+    temp_df = temp_df[
+        [
+            "序号",
+            "代码",
+            "名称",
+            "最新价",
+            "涨跌额",
+            "涨跌幅",
+            "今开",
+            "最高",
+            "最低",
+            "昨收",
+            "成交量",
+            "成交额",
+        ]
+    ]
+
+    temp_df = temp_df.rename(columns={
+        "序号": "index",
+        "代码": "symbol",
+        "名称": "name",
+        "最新价": "now_price",
+        "涨跌额": "range",
+        "涨跌幅": "chg",
+        "今开": "open",
+        "最高": "high",
+        "最低": "low",
+        "昨收": "yesterday_price",
+        "成交额": "amount",
+        "成交量": "volume",
+    })
+
+    return temp_df
+
+
+# 获取港股通个数
+def get_stock_hk_ggt_components_em_count(cookie, pn, proxies, page_size, time_out):
+    headers = {
+        'Cookie': cookie
+    }
+    url = "https://33.push2.eastmoney.com/api/qt/clist/get"
+    current_timestamp = str(int(round(time.time() * 1000, 0)))
+    params = {
+        "pn": str(pn),
+        "pz": str(page_size),
+        "po": "1",
+        "np": "2",
+        "ut": "bd1d9ddb04089700cf9c27f6f7426281",
+        "fltt": "2",
+        "fid": "f3",
+        "fs": "b:DLMK0146,b:DLMK0144",
+        "fields": "f1,f2,f3,f4,f5,f6,f7,f8,f9,f10,f12,f13,f14,f15,f16,f17,f18,f19,f20,f21,f23,f24,"
+                  "f25,f26,f22,f33,f11,f62,f128,f136,f115,f152",
+        "_": str(current_timestamp),
+    }
+    try:
+        if proxies is None:
+            r = requests.get(url, params, timeout=time_out, headers=headers)
+        else:
+            r = requests.get(url, params, proxies=proxies, timeout=time_out, headers=headers)
+        data_json = r.json()
+        total_number = int(data_json['data']['total'])
+        return total_number
+    except Exception as e:
+        logger.error("获取港股通列表,实时行情异常:{}", e)
+        return 0
+
+
+# 获取港股通名单 todo 被封以后替换
+def stock_hk_ggt_components_em(cookie, pn, proxies, page_size, time_out) -> pd.DataFrame:
+    """
+    东方财富网-行情中心-港股市场-港股通成份股
+    https://quote.eastmoney.com/center/gridlist.html#hk_components
+    :return: 港股通成份股
+    :rtype: pandas.DataFrame
+    """
+    headers = {
+        'Cookie': cookie
+    }
+    url = "https://33.push2.eastmoney.com/api/qt/clist/get"
+    params = {
+        "pn": str(pn),
+        "pz": str(page_size),
+        "po": "1",
+        "np": "2",
+        "ut": "bd1d9ddb04089700cf9c27f6f7426281",
+        "fltt": "2",
+        "fid": "f3",
+        "fs": "b:DLMK0146,b:DLMK0144",
+        "fields": "f1,f2,f3,f4,f5,f6,f7,f8,f9,f10,f12,f13,f14,f15,f16,f17,f18,f19,f20,f21,f23,f24,"
+                  "f25,f26,f22,f33,f11,f62,f128,f136,f115,f152",
+        "_": "1639974456250",
+    }
+    try:
+        if proxies is None:
+            r = requests.get(url, params=params, timeout=time_out, headers=headers)
+        else:
+            r = requests.get(url, params=params, proxies=proxies, timeout=time_out, headers=headers)
+
+        data_json = r.json()
+        temp_df = pd.DataFrame(data_json["data"]["diff"]).T
+        temp_df.reset_index(inplace=True)
+        temp_df["index"] = temp_df.index + 1
+        return temp_df
+    except Exception as e:
+        logger.error("获取港股通列表:{}", e)
+
+
+def repeated_acquisition_ask_hk_gtt_async(em_cookie, time_out, max_number, num_threads, pages_per_thread):
+    per_page = page_number
+    total_pages = (max_number + per_page - 1) // per_page  # 向上取整
+    result_df = pd.DataFrame()
+
+    # 创建线程锁以确保线程安全
+    df_lock = Lock()
+
+    # 计算每个线程处理的页数范围
+    def process_page_range(start_page, end_page, thread_id):
+        nonlocal result_df
+        local_df = pd.DataFrame()
+        current_page = start_page
+        proxy_ip = proxy_common_api.generate_proxy_ip_api(1)
+
+        while current_page <= end_page and current_page <= total_pages:
+            proxies = {"https": proxy_ip, "http": proxy_ip}
+            try:
+                page_df = stock_hk_ggt_components_em(em_cookie, current_page, proxies, page_number, time_out)
+                if data_frame_util.is_not_empty(page_df):
+                    local_df = pd.concat([local_df, page_df])
+                    logger.info("线程{}获取页面数据成功: {}", thread_id, current_page)
+                    current_page += 1
+                else:
+                    time.sleep(0.2)
+                    proxy_ip = proxy_common_api.generate_proxy_ip_api(1)
+                    logger.info("线程{}获取页面数据失败: {}", thread_id, current_page)
+            except BaseException as e:
+                time.sleep(1)
+                proxy_ip = proxy_common_api.generate_proxy_ip_api(1)
+                logger.error("线程{}处理页面{}时发生错误: {}", thread_id, current_page, e)
+
+        with df_lock:
+            result_df = pd.concat([result_df, local_df])
+        return len(local_df)
+
+    # 计算每个线程的页面范围
+    page_ranges = []
+    for i in range(num_threads):
+        start_page = i * pages_per_thread + 1
+        end_page = (i + 1) * pages_per_thread
+        if start_page > total_pages:
+            break
+        page_ranges.append((start_page, end_page, i + 1))
+
+    # 使用线程池执行任务
+    with concurrent.futures.ThreadPoolExecutor(max_workers=num_threads) as executor:
+        # 提交所有任务
+        futures = [
+            executor.submit(process_page_range, start, end, tid)
+            for start, end, tid in page_ranges
+        ]
+
+        # 等待所有任务完成并获取结果
+        results = []
+        for future in concurrent.futures.as_completed(futures):
+            try:
+                result = future.result()
+                results.append(result)
+            except Exception as e:
+                logger.error("线程执行出错: {}", e)
+
+    return rename_hg_ggt(result_df)
+
+
+# 港股通实时行情
+def get_ggt_real_time_quotes(em_cookie, time_out, pages_per_thread):
+    try_numer = 3
+    while try_numer > 0:
+        proxy_ip = proxy_common_api.generate_proxy_ip_api(1)
+        proxies = {"https": proxy_ip,
+                   "http": proxy_ip}
+
+        max_number = get_stock_hk_ggt_components_em_count(em_cookie, 1, proxies, 20, time_out)
+        if max_number > 0:
+            break
+        try_numer = try_numer - 1
+    if max_number == 0:
+        return pd.DataFrame()
+
+    total_pages = (max_number + page_number - 1) // page_number  # 向上取整
+
+    num_threads = int((total_pages / pages_per_thread) + 1)
+    return repeated_acquisition_ask_hk_gtt_async(em_cookie, time_out, max_number, num_threads, pages_per_thread)
+
+
+import mns_common.component.cookie.cookie_info_service as cookie_info_service
+
+if __name__ == '__main__':
+    em_cookie_test = cookie_info_service.get_em_cookie()
+    test_df = get_ggt_real_time_quotes(em_cookie_test, 30, 6)
+    print(test_df)

mns_common/api/em/real_time/east_money_stock_multi_thread_api_v3.py

@@ -0,0 +1,154 @@
+from mns_common.db.MongodbUtil import MongodbUtil
+import requests
+import json
+import pandas as pd
+from concurrent.futures import ThreadPoolExecutor
+import datetime
+from loguru import logger
+import mns_common.component.proxies.proxy_common_api as proxy_common_api
+import mns_common.api.em.real_time.east_money_stock_common_api as east_money_stock_common_api
+
+mongodb_util = MongodbUtil('27017')
+
+fields = ("f352,f2,f3,f5,f6,f8,f10,f11,f22,f12,f14,f15,f16,f17,"
+          "f18,f20,f21,f26,f33,f34,f35,f62,f66,f69,f72,f100,f184,f211,f212"),
+fs = "m:0 t:6,m:0 t:80,m:1 t:2,m:1 t:23,m:0 t:81 s:2048"
+
+# 分页条数
+PAGE_SIZE = 100
+
+
+def all_stock_ticker_data_new(initial_proxies, time_out, max_number) -> pd.DataFrame:
+    """
+        使用多线程获取所有股票数据，失败页面会使用新IP重试，最多使用10个IP
+        """
+
+    total_pages = (max_number + PAGE_SIZE - 1) // PAGE_SIZE  # 向上取整
+    all_pages = set(range(1, total_pages + 1))  # 所有需要获取的页码
+    success_pages = set()  # 成功获取的页码
+    results = []  # 存储成功获取的数据
+    used_ip_count = 1  # 已使用IP计数器（初始IP算第一个）
+    MAX_IP_LIMIT = 10  # IP使用上限
+
+    # 循环处理直到所有页面成功或达到IP上限
+    while (all_pages - success_pages) and (used_ip_count < MAX_IP_LIMIT):
+        # 获取当前需要处理的失败页码
+        current_failed_pages = all_pages - success_pages
+        if used_ip_count > 1:
+            logger.info("当前需要处理的失败页码: {}, 已使用IP数量: {}/{}", current_failed_pages, used_ip_count,
+                        MAX_IP_LIMIT)
+
+        # 首次使用初始代理，后续获取新代理
+        if len(success_pages) == 0:
+            proxies = initial_proxies
+        else:
+            # 每次重试前获取新代理并计数
+            # logger.info("获取新代理IP处理失败页面")
+            new_proxy_ip = proxy_common_api.generate_proxy_ip_api(1)
+            proxies = {"https": new_proxy_ip}
+            # logger.info("新代理IP: {}, 已使用IP数量: {}/{}", new_proxy_ip, used_ip_count + 1, MAX_IP_LIMIT)
+            used_ip_count += 1  # 增加IP计数器
+
+        # 创建线程池处理当前失败的页码
+        with ThreadPoolExecutor(max_workers=10) as executor:
+            futures = {
+                executor.submit(get_stock_page_data, pn, proxies, PAGE_SIZE, time_out): pn
+                for pn in current_failed_pages
+            }
+
+            # 收集结果并记录成功页码
+            for future, pn in futures.items():
+                try:
+                    result = future.result()
+                    if not result.empty:
+                        results.append(result)
+                        success_pages.add(pn)
+                    # else:
+                    #     logger.warning("页码 {} 未返回有效数据", pn)
+                except Exception as e:
+                    continue
+                    # logger.error("页码 {} 处理异常: {}", pn, str(e))
+
+    # 检查是否达到IP上限
+    if used_ip_count >= MAX_IP_LIMIT and (all_pages - success_pages):
+        remaining_pages = all_pages - success_pages
+        logger.warning("已达到最大IP使用限制({}个)，剩余未获取页码: {}, 返回现有数据", MAX_IP_LIMIT, remaining_pages)
+
+    # 合并所有成功获取的数据
+    if results:
+        return pd.concat(results, ignore_index=True)
+    else:
+        return pd.DataFrame()
+
+
+def get_stock_page_data(pn, proxies, page_size, time_out):
+    """获取单页股票数据"""
+    current_time = datetime.datetime.now()
+    current_timestamp_ms = int(current_time.timestamp() * 1000)
+
+    url = "https://33.push2.eastmoney.com/api/qt/clist/get"
+    params = {
+        "cb": "jQuery1124046660442520420653_" + str(current_timestamp_ms),
+        "pn": str(pn),
+        "pz": str(page_size),
+        "po": "0",
+        "np": "3",
+        "ut": "bd1d9ddb04089700cf9c27f6f7426281",
+        "fltt": "2",
+        "invt": "2",
+        "wbp2u": "|0|0|0|web",
+        "fid": "f12",
+        "fs": fs,
+        "fields": fields,
+        "_": current_timestamp_ms
+    }
+
+    try:
+        if proxies is None:
+            r = requests.get(url, params, timeout=time_out)
+        else:
+            r = requests.get(url, params, proxies=proxies, timeout=time_out)
+
+        if r.status_code != 200:
+            return pd.DataFrame()
+
+        data_text = r.text
+        begin_index = data_text.index('[')
+        end_index = data_text.index(']')
+        data_json = data_text[begin_index:end_index + 1]
+        data_json = json.loads(data_json)
+
+        if not data_json:
+            return pd.DataFrame()
+
+        result_df = pd.DataFrame(data_json)
+        result_df['page_number'] = pn
+        return result_df
+    except Exception as e:
+        return pd.DataFrame()
+
+
+def get_real_time_quotes_all_stocks(time_out):
+    try_numer = 3
+    while try_numer > 0:
+        proxy_ip = proxy_common_api.generate_proxy_ip_api(1)
+        initial_proxies = {"https": proxy_ip,
+                           "http": proxy_ip}
+
+        total_number = east_money_stock_common_api.get_stocks_num(1, initial_proxies, 20, time_out)
+        if total_number > 0:
+            break
+        try_numer = try_numer - 1
+    if total_number == 0:
+        return pd.DataFrame()
+
+    page_df = all_stock_ticker_data_new(initial_proxies, time_out, total_number)
+    page_df = east_money_stock_common_api.rename_real_time_quotes_df(page_df)
+    page_df.drop_duplicates('symbol', keep='last', inplace=True)
+    return page_df
+
+
+if __name__ == '__main__':
+    # 调用方式
+    df = get_real_time_quotes_all_stocks(5)
+    logger.info("数据条数,{}", df.shape[0])