mns-common 1.5.1.5__py3-none-any.whl → 1.5.1.6__py3-none-any.whl

mns_common/api/em/real_time/east_money_stock_a_api.py

@@ -1,6 +1,5 @@
-import sys
 import os
-import time
+import sys
 
 file_path = os.path.abspath(__file__)
 end = file_path.index('mns') + 16
@@ -12,7 +11,6 @@ import json
 import pandas as pd
 from concurrent.futures import ThreadPoolExecutor
 import datetime
-from loguru import logger
 import mns_common.utils.data_frame_util as data_frame_util
 
 mongodb_util = MongodbUtil('27017')
@@ -69,7 +67,7 @@ def get_stock_page_data(pn, proxies, page_size):
                 global max_number
                 max_number = int(data_text[begin_index_total + 8:end_index_total - 1])
             except Exception as e:
-                logger.error(f"获取第{pn}页股票列表异常: {e}")
+                # logger.error(f"获取第{pn}页股票列表异常: {e}")
                 return pd.DataFrame()
 
         begin_index = data_text.index('[')
@@ -79,9 +77,11 @@ def get_stock_page_data(pn, proxies, page_size):
         if data_json is None:
             return pd.DataFrame()
         else:
-            return pd.DataFrame(data_json)
+            result_df = pd.DataFrame(data_json)
+            result_df['page_number'] = pn
+            return result_df
     except Exception as e:
-        logger.error(f"获取第{pn}页股票列表异常: {e}")
+        # logger.error(f"获取第{pn}页股票列表异常: {e}")
         return pd.DataFrame()
 
 
@@ -290,16 +290,87 @@ def get_sum_north_south_net_buy_amt():
     return df
 
 
+def get_real_time_quotes_ip_pool(proxies_pool, timeout_time, show_msg):
+    page_df = all_stock_ticker_data_ip_pools(proxies_pool, timeout_time, show_msg)
+    page_df = rename_real_time_quotes_df(page_df)
+    page_df.drop_duplicates('symbol', keep='last', inplace=True)
+    return page_df
+
+
+def all_stock_ticker_data_ip_pools(proxies_pool, timeout_time: str, show_msg) -> pd.DataFrame:
+    """
+    使用多线程获取所有股票数据，支持代理池，并避免超时死循环
+    :param proxies_pool: 代理池列表
+    :param timeout_time: 代理过期时间，格式为 "YYYY-MM-DD HH:MM:SS"
+    :return: 股票数据的 DataFrame
+    """
+    # 将timeout_time字符串转化为 datetime 对象
+    timeout_time = datetime.datetime.strptime(timeout_time, "%Y-%m-%d %H:%M:%S")
+
+    if datetime.datetime.now() > timeout_time:
+        logger.info("当前时间已超过: {}，终止请求。", timeout_time)
+        return pd.DataFrame()  # 返回空DataFrame，表示结束
+
+    per_page = PAGE_SIZE
+    total_pages = (max_number + per_page - 1) // per_page  # 向上取整
+    results = []
+
+    def get_page_data_with_retries(pn, proxy_list):
+        """
+        使用代理池并重试，直到获取成功或超过超时时间
+        """
+        proxy_index = 0
+        while True:
+            # 检查当前时间是否超过了超时时间
+            if datetime.datetime.now() > timeout_time:
+                logger.info("当前时间已超过: {}，终止请求。", timeout_time)
+                return pd.DataFrame()  # 返回空DataFrame，表示结束
+
+            proxy_ip = proxy_list[proxy_index]
+
+            proxies = {"https": proxy_ip,
+                       "http": proxy_ip}
+
+            # 获取当前代理
+            result = get_stock_page_data(pn, proxies, PAGE_SIZE)
+            if not result.empty:
+                return result
+            else:
+                # 如果当前代理失败，切换到下一个代理
+                proxy_index = (proxy_index + 1) % len(proxy_list)
+                if show_msg:
+                    logger.warning("页面 {} 获取失败，尝试使用代理 {} 重新获取.", pn, proxy_list[proxy_index])
+
+    # 创建线程池
+    with ThreadPoolExecutor(max_workers=10) as executor:
+        # 提交任务，获取每页数据
+        futures = [executor.submit(get_page_data_with_retries, pn, proxies_pool) for pn in range(1, total_pages + 1)]
+
+        # 收集结果
+        for future in futures:
+            result = future.result()
+            if result.empty:
+                break  # 如果返回空的DataFrame（即超时），则退出循环
+            results.append(result)
+
+    # 合并所有页面的数据
+    if results:
+        return pd.concat(results, ignore_index=True)
+    else:
+        return pd.DataFrame()
+
+
+# 示例调用
+import mns_common.component.proxies.proxy_common_api as proxy_common_api
+from loguru import logger
+
 # 示例调用
 if __name__ == "__main__":
+
     while True:
-        ip_proxy_pool = mongodb_util.find_all_data('ip_proxy_pool')
-        if data_frame_util.is_not_empty(ip_proxy_pool):
-            proxy_ip = list(ip_proxy_pool['ip'])[0]
-            proxy = {
-                "https": proxy_ip}
-            df = get_real_time_quotes_all_stocks(proxy)
-            logger.info("涨停数据,{}", 1)
-        else:
-            time.sleep(1)
-            logger.error("ip为空")
+        result = proxy_common_api.get_proxy_ip_pool(3, 50, 3)
+        ip_pool_list = result['ip_pool']
+        effect_time = result['effect_time']
+
+        test_df = get_real_time_quotes_ip_pool(ip_pool_list, effect_time, False)
+        logger.info(test_df)

mns_common/api/proxies/liu_guan_proxy_api.py

@@ -57,9 +57,46 @@ def get_proxy_api(order_id, secret, unbind_time):
             return None
 
 
+# 线程池
+def get_proxy_pool_api(order_id, secret, unbind_time, ip_num):
+    num = str(ip_num)
+    pid = "-1"
+    cid = ""
+    noDuplicate = "1"
+    lineSeparator = "0"
+    singleIp = "0"
+    time_str = str(int(time.time()))  # 时间戳
+
+    # 计算sign
+    txt = "orderId=" + order_id + "&" + "secret=" + secret + "&" + "time=" + time_str
+    sign = hashlib.md5(txt.encode()).hexdigest()
+    # 访问URL获取IP
+    url = (
+            "http://api.hailiangip.com:8422/api/getIp?type=1" + "&num=" + num + "&pid=" + pid
+            + "&unbindTime=" + unbind_time + "&cid=" + cid
+            + "&orderId=" + order_id + "&time=" + time_str + "&sign=" + sign + "&dataType=0"
+            + "&lineSeparator=" + lineSeparator + "&noDuplicate=" + noDuplicate + "&singleIp=" + singleIp)
+    my_response = requests.get(url).content
+    js_res = json.loads(my_response)
+    ip_pool_list = []
+    for dic in js_res["data"]:
+        try:
+            ip = dic["ip"]
+            port = dic["port"]
+            ip_port = ip + ":" + str(port)
+            ip_pool_list.append(ip_port)
+        except BaseException as e:
+            logger.error("获取ip地址异常:{}", e)
+            return None
+    return ip_pool_list
+
+
+
+
+
 if __name__ == '__main__':
-    order_id = ''
-    secret = ''
-    unbind_time = str(60 * 10)
-    ip = get_proxy_api(order_id, secret, unbind_time)
+    order_id_test = ''
+    secret_test = ''
+    unbind_time_test = str(60 * 10)
+    ip = get_proxy_api(order_id_test, secret_test, unbind_time_test)
     print(ip)

mns_common/component/proxies/proxy_common_api.py

@@ -12,7 +12,6 @@ import mns_common.utils.data_frame_util as data_frame_util
 from mns_common.db.MongodbUtil import MongodbUtil
 import mns_common.constant.db_name_constant as db_name_constant
 import datetime
-import requests
 import time
 from loguru import logger
 from functools import lru_cache
@@ -21,23 +20,25 @@ import threading
 
 mongodb_util = MongodbUtil('27017')
 
+IP_POOL = 'ip_pool'
+ONE_IP = 'one_ip'
+query_one = {'ip_type': ONE_IP}
+query_pool = {'ip_type': IP_POOL}
+
 
 def query_liu_guan_proxy_ip():
-    ip_proxy_pool = mongodb_util.find_all_data(db_name_constant.IP_PROXY_POOL)
+    ip_proxy_pool = mongodb_util.find_one_query(db_name_constant.IP_PROXY_POOL, query_one)
     return ip_proxy_pool
 
 
 def remove_proxy_ip():
-    mongodb_util.remove_data({}, db_name_constant.IP_PROXY_POOL)
+    mongodb_util.remove_data(query_one, db_name_constant.IP_PROXY_POOL)
 
 
 def check_valid(ip_proxy_pool):
     effect_time = list(ip_proxy_pool['effect_time'])[0]
-
     now_date = datetime.datetime.now()
-
     str_now_date = now_date.strftime('%Y-%m-%d %H:%M:%S')
-
     if effect_time > str_now_date:
         return True
     else:
@@ -47,7 +48,7 @@ def check_valid(ip_proxy_pool):
 
 @lru_cache(maxsize=None)
 def get_account_cache():
-    query = {"type": "liu_guan_proxy", }
+    query = {"type": "liu_guan_proxy"}
     return mongodb_util.find_query_data(db_name_constant.STOCK_ACCOUNT_INFO, query)
 
 
@@ -77,6 +78,7 @@ def generate_proxy_ip(minutes):
             ip = generate_proxy_ip_api(minutes)
             if check_proxy(ip):
                 result_dict = {"_id": ip,
+                               'ip_type': ONE_IP,
                                'effect_time': str_now_date,
                                'ip': ip}
                 result_df = pd.DataFrame(result_dict, index=[1])
@@ -113,6 +115,73 @@ def check_proxy(proxy_ip):
         return False
 
 
+# 查询ip池子
+def query_liu_guan_proxy_ip_pool():
+    ip_proxy_pool = mongodb_util.find_query_data(db_name_constant.IP_PROXY_POOL, query_pool)
+    return ip_proxy_pool
+
+
+def remove_proxy_ip_pool():
+    mongodb_util.remove_data(query_pool, db_name_constant.IP_PROXY_POOL)
+
+
+def generate_proxy_ip_pool_api(minutes, ip_num):
+    stock_account_info = get_account_cache()
+    order_id = list(stock_account_info['password'])[0]
+    secret = list(stock_account_info['account'])[0]
+    # 获取10分钟动态ip
+    ip_pool = liu_guan_proxy_api.get_proxy_pool_api(order_id, secret, str(60 * minutes), ip_num)
+    return ip_pool
+
+
+def get_proxy_ip_pool(minutes, seconds, ip_num):
+    ip_proxy_pool = query_liu_guan_proxy_ip_pool()
+    if data_frame_util.is_empty(ip_proxy_pool):
+        return generate_proxy_ip_pool(minutes, seconds, ip_num)
+    else:
+        if check_valid(ip_proxy_pool):
+            ip_pool = list(ip_proxy_pool['ip_pool'])[0]
+            effect_time = list(ip_proxy_pool['effect_time'])[0]
+            result = {'ip_pool': ip_pool,
+                      'effect_time': effect_time}
+            return result
+        else:
+            # 已经失效 移除ip pool
+            remove_proxy_ip_pool()
+            # 重新生成
+            return generate_proxy_ip_pool(minutes, seconds, ip_num)
+
+
+# seconds 有效秒数,minutes 需要减1
+def generate_proxy_ip_pool(minutes, seconds, ip_num):
+    ip_proxy_pool = query_liu_guan_proxy_ip_pool()
+    if data_frame_util.is_not_empty(ip_proxy_pool):
+        ip_pool = list(ip_proxy_pool['ip_pool'])[0]
+        effect_time = list(ip_proxy_pool['effect_time'])[0]
+
+
+    else:
+        remove_proxy_ip_pool()
+        now_date = datetime.datetime.now()
+        # 加上分钟 少10秒
+        time_to_add = datetime.timedelta(minutes=minutes - 1, seconds=seconds)
+        new_date = now_date + time_to_add
+        effect_time = new_date.strftime('%Y-%m-%d %H:%M:%S')
+        ip_pool = generate_proxy_ip_pool_api(minutes, ip_num)
+        result_dict = {
+            "_id": [IP_POOL],
+            'ip_type': [IP_POOL],
+            'effect_time': [effect_time],
+            'ip_pool': [ip_pool]  # 每个字段都包装成列表
+        }
+        result_df = pd.DataFrame(result_dict)
+
+        mongodb_util.insert_mongo(result_df, db_name_constant.IP_PROXY_POOL)
+    result = {'ip_pool': ip_pool,
+              'effect_time': effect_time}
+    return result
+
+
 def get_em_real_time_data(proxy_ip):
     proxies = {
         "http": proxy_ip,
@@ -153,4 +222,4 @@ def call_with_timeout(func, *args, timeout=2, **kwargs):
 
 
 if __name__ == "__main__":
-    generate_proxy_ip(1)
+    get_proxy_ip_pool(1, 50, 2)

{mns_common-1.5.1.5.dist-info → mns_common-1.5.1.6.dist-info}/METADATA

@@ -1,4 +1,4 @@
 Metadata-Version: 2.1
 Name: mns-common
-Version: 1.5.1.5
+Version: 1.5.1.6
 

{mns_common-1.5.1.5.dist-info → mns_common-1.5.1.6.dist-info}/RECORD

@@ -15,7 +15,7 @@ mns_common/api/em/gd/east_money_stock_gdfx_free_top_10_api.py,sha256=I2-JjFjTjvO
 mns_common/api/em/real_time/__init__.py,sha256=wEg73KlZo-dU0yKGwpA1C2y6LZm4IBb94tNda1tqLeg,163
 mns_common/api/em/real_time/east_money_debt_api.py,sha256=jMvMZtlrDfExl_4jZ1hepHX8rUoeVLoLSOIhRBjkUGk,14753
 mns_common/api/em/real_time/east_money_etf_api.py,sha256=tCyH4fNx-KfVRFuNGkgM8d_xkvR0oAfr8T3e7_XrjTM,14414
-mns_common/api/em/real_time/east_money_stock_a_api.py,sha256=6xYcNJyMhFFL2eNxQ9c0TbQ53wgumNovVHisXgQX7bs,12737
+mns_common/api/em/real_time/east_money_stock_a_api.py,sha256=e_hPYJnNhjDOecTW5myorxEaDvuPku4pssuP9JIghG4,15621
 mns_common/api/em/real_time/east_money_stock_a_v2_api.py,sha256=mL4uuL6sVsC2Vnl09826AUnzxePGAUhlZ7I5BBFw8Ks,14530
 mns_common/api/em/real_time/east_money_stock_hk_api.py,sha256=KFIYUZ3N4ULrataeCIXwZPo775O7joKgMF466uwVDdY,15154
 mns_common/api/em/real_time/east_money_stock_us_api.py,sha256=RiTrdZDuDgTOtiMSD1Ba9aQAx4vghM66pEp_LicH3Ps,11632
@@ -43,7 +43,7 @@ mns_common/api/kpl/symbol/symbol_his_quotes_api.py,sha256=5F9L8V2UI_YUYe2dO6FbVK
 mns_common/api/msg/__init__.py,sha256=2U9DiKslxsWwLLEcZKjS8UiQPN1QgALvnK3HiJNIZE0,163
 mns_common/api/msg/push_msg_api.py,sha256=z8jDqFWygfxnCFFfQp4K-llgg27nRLv7Mx72lOddBH0,1390
 mns_common/api/proxies/__init__.py,sha256=wEg73KlZo-dU0yKGwpA1C2y6LZm4IBb94tNda1tqLeg,163
-mns_common/api/proxies/liu_guan_proxy_api.py,sha256=VASqcWnKyAEmF4UYNY39Jazh45qugVMYblRrdKfQFuE,1923
+mns_common/api/proxies/liu_guan_proxy_api.py,sha256=CheM7j_028uofZBn4wN1bNkDuyK7pNg1PzAaSf8lYAc,3204
 mns_common/api/qmt/__init__.py,sha256=wEg73KlZo-dU0yKGwpA1C2y6LZm4IBb94tNda1tqLeg,163
 mns_common/api/qmt/qmt_minunte_tick_data.py,sha256=uwSw_AkA9RaD3pXPKzxqi4TKEkpglmFUwtYl9r5E6G8,3019
 mns_common/api/ths/__init__.py,sha256=2U9DiKslxsWwLLEcZKjS8UiQPN1QgALvnK3HiJNIZE0,163
@@ -116,7 +116,7 @@ mns_common/component/k_line/patterns/pattern_Enum.py,sha256=bl8cH1H3BWdj_deVO124
 mns_common/component/price/__init__.py,sha256=wEg73KlZo-dU0yKGwpA1C2y6LZm4IBb94tNda1tqLeg,163
 mns_common/component/price/trade_price_service_api.py,sha256=0loBjbOt__o-ngc2Q4n5lF8_0x2WINRpL-cH1341Uaw,4396
 mns_common/component/proxies/__init__.py,sha256=wEg73KlZo-dU0yKGwpA1C2y6LZm4IBb94tNda1tqLeg,163
-mns_common/component/proxies/proxy_common_api.py,sha256=N2VaiEQ15KycjSNmCHAguR1xiASZzJu1y24NhFCr7BA,4663
+mns_common/component/proxies/proxy_common_api.py,sha256=GuMGK7kAOLqGv_NbRbdQlq_vhVjF3iT58pYkJoFOmCY,7302
 mns_common/component/qmt/__init__.py,sha256=wEg73KlZo-dU0yKGwpA1C2y6LZm4IBb94tNda1tqLeg,163
 mns_common/component/qmt/qmt_buy_service.py,sha256=tLTgrSxCcxuMhADRBBrW4ZWR_3MdbMZvvMdH5hbwyJU,7190
 mns_common/component/real_time/__init__.py,sha256=2U9DiKslxsWwLLEcZKjS8UiQPN1QgALvnK3HiJNIZE0,163
@@ -158,7 +158,7 @@ mns_common/utils/date_handle_util.py,sha256=XS-MyA8_7k35LOCFAYOHgVcVkMft_Kc4Wa9U
 mns_common/utils/db_util.py,sha256=hSmfNAN4vEeEaUva6_cicZEhb2jSnib-Gvk2reke1vc,2590
 mns_common/utils/file_util.py,sha256=egWu6PenGPRp_ixrNTHKarT4dAnOT6FETR82EHUZJnQ,1042
 mns_common/utils/ip_util.py,sha256=UTcYfz_uytB__6nlBf7T-izuI7hi4XdB6ET0sJgEel4,969
-mns_common-1.5.1.5.dist-info/METADATA,sha256=YLPOLO_99ZymHoEEaKi2LjFOg3DPzburAquYZHCVRLs,61
-mns_common-1.5.1.5.dist-info/WHEEL,sha256=tZoeGjtWxWRfdplE7E3d45VPlLNQnvbKiYnx7gwAy8A,92
-mns_common-1.5.1.5.dist-info/top_level.txt,sha256=ZC58kAR-8Hvc6U2xhYNBNLAh3mb6sZazbdj5nZpvEkQ,11
-mns_common-1.5.1.5.dist-info/RECORD,,
+mns_common-1.5.1.6.dist-info/METADATA,sha256=PNkBebLDEDMZuxItxFxMN8Gg-Zsl0HEphR7KNuKLf1I,61
+mns_common-1.5.1.6.dist-info/WHEEL,sha256=tZoeGjtWxWRfdplE7E3d45VPlLNQnvbKiYnx7gwAy8A,92
+mns_common-1.5.1.6.dist-info/top_level.txt,sha256=ZC58kAR-8Hvc6U2xhYNBNLAh3mb6sZazbdj5nZpvEkQ,11
+mns_common-1.5.1.6.dist-info/RECORD,,