xtn-tools-pro 1.0.0.0.6__py3-none-any.whl → 1.0.0.0.8__py3-none-any.whl

xtn_tools_pro/proxy/proxy.py

@@ -0,0 +1,189 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+# 说明：
+#    小象代理专用
+# History:
+# Date          Author    Version       Modification
+# --------------------------------------------------------------------------------------------------
+# 2024/4/27    xiatn     V00.01.000    新建
+# --------------------------------------------------------------------------------------------------
+import requests, time, random
+from xtn_tools_pro.db.RedisDB import RedisDBPro
+from xtn_tools_pro.utils.time_utils import get_time_now_timestamp, get_time_now_day59_timestamp
+
+import warnings
+from urllib3.exceptions import InsecureRequestWarning
+
+warnings.filterwarnings("ignore", category=InsecureRequestWarning)
+
+
+class ProxyPool:
+    def __init__(self, ip, port, db=0, user_pass="", redis_proxy_name="", usage_cnt=100, usage_time=100,*args,**kwargs):
+        """
+        :param ip: redis 数据库 ip
+        :param port: redis 数据库 端口
+        :param db: redis 数据库 db
+        :param user_pass: redis 数据库 密码
+        :param redis_proxy_name: redis 数据库 用于存储代理的key
+        :param usage_cnt: 每个代理最长使用次数 单位秒 维护代理时用
+        :param usage_time: 每个代理最长使用时间 单位秒 维护代理时用
+        :param is_log: 是否记录日志
+        """
+        r = RedisDBPro(ip=ip, port=port, db=db, user_pass=user_pass)
+        self.__redis_pool = r
+        self.__redisProxyName = redis_proxy_name
+
+        # 获取当天0点时间戳
+        self.__now_day59_timestamp = get_time_now_day59_timestamp()
+        self.__usage_cnt = usage_cnt
+        self.__usage_time = usage_time
+
+    def __log(self, text):
+        """
+            记录日志
+        :param text:
+        :return:
+        """
+        print(text)
+
+    def __check_proxy(self):
+        """
+            维护检查代理，删除无用代理
+            删除标准：1.代理使用超过xx次；2.使用时间超过xx秒；3.被爬虫标记使用次数为 999999 会被删除
+        :return:
+        """
+        proxy_val_list = list(self.__redis_pool.get_all_key("{}*".format(self.__redisProxyName)))
+        for proxy_val in proxy_val_list:
+            # 获取时间
+            time_out = proxy_val.split(":")[-1]
+            # 获取使用次数
+            proxy_val_count = self.__redis_pool.get(proxy_val)
+            if int(time_out) + self.__usage_time < get_time_now_timestamp(is_time_10=True):
+                del_state = self.__redis_pool.delete(proxy_val)
+                self.__log(
+                    "当前代理状态:{proxy_val},{time_out}_{py_time}当前代理已超过使用时间,删除状态为：{del_state}".
+                        format(proxy_val=proxy_val, del_state=del_state,
+                               time_out=time_out, py_time=get_time_now_timestamp(is_time_10=True)))
+            elif int(proxy_val_count) >= self.__usage_cnt:
+                del_state = self.__redis_pool.delete(proxy_val)
+                self.__log(
+                    "当前代理状态:{proxy_val},{text},删除状态为：{del_state}".format(proxy_val=proxy_val,
+                                                                         text="当前代理被爬虫标记为不可用" if proxy_val_count >= 999999 else "当前代理已超过使用时间",
+                                                                         del_state=del_state))
+
+    def __get_proxy_length(self):
+        """
+            获取代理数
+        :return:
+        """
+        proxy_val_list = list(self.__redis_pool.get_all_key("{}*".format(self.__redisProxyName)))
+        return len(proxy_val_list)
+
+    def __incr_proxy(self, proxy_val):
+        """
+            自增代理使用次数
+        :param proxy_val: 代理
+        :return:
+        """
+        proxy_val_con = self.__redis_pool.incr(proxy_val)
+        return proxy_val_con
+
+    def __get_api_proxy(self):
+        """
+            通过接口获取小象代理，并存储至数据库
+            响应：
+                {"code":1010,"success":false,"data":null,"msg":"请求过于频繁"}
+                {"code":200,"success":true,"data":[{"ip":"125.123.244.60","port":37635,"realIp":null,"startTime":"2024-04-27 14:09:42","during":2}],"msg":"操作成功"}
+        :return:
+        """
+        while True:
+            self.__check_proxy()
+            try:
+                response = requests.get(url=self.__XiaoXiangProxyAPI, verify=False, timeout=3)
+                if response.status_code == 200:
+                    if response.json().get("msg") == "请求过于频繁":
+                        self.__log("获取小象代理过于频繁，等待2s，{content}".format(content=response.text))
+                        time.sleep(2)
+                        continue
+                    # 获取data
+                    proxy_data_list = response.json().get("data", [])
+                    if not proxy_data_list:
+                        self.__log("获取小象代理失败 data 为空，等待2s，{content}".format(content=response.text))
+                        time.sleep(2)
+                        continue
+                    else:
+                        for data in proxy_data_list:
+                            ip = "http://{ip}".format(ip=data.get("ip"))
+                            port = data.get("port")
+                            time_out = get_time_now_timestamp(is_time_10=True)
+                            proxy_key = "{redis_proxy_name}:{ip}:{port}:{timeOut}".format(
+                                redis_proxy_name=self.__redisProxyName,
+                                ip=ip,
+                                port=port,
+                                timeOut=time_out,
+                            )
+                            proxy_key_con = self.__incr_proxy(proxy_key)
+                            self.__log("获取代理:{proxy_key}，插入数据库状态为{proxy_key_con}".format(proxy_key=proxy_key,
+                                                                                         proxy_key_con=proxy_key_con))
+                        return True  # 获取成功
+                else:
+                    self.__log("获取小象代理返回响应码不为200，等待2s，{content}".format(content=response.text))
+                    time.sleep(2)
+                    continue
+            except Exception as e:
+                self.__log("获取小象代理报错：{e}".format(e=e))
+
+    def run(self):
+        try:
+            while True:
+                # 检查代理
+                self.__check_proxy()
+                # 获取小象代理
+                self.__get_api_proxy()
+                time.sleep(1)
+                # 判断时间是否超过当前23:59分时间戳
+                if get_time_now_timestamp(is_time_10=True) >= self.__now_day59_timestamp:
+                    self.__log("时间23:59，结束循环，{t}".format(t=int(time.time())))
+                    break
+        except Exception as eee:
+            self.__log("程序异常报错:{eee}".format(eee=eee))
+            # self.__del__()
+
+    def get_proxy(self):
+        """
+            从代理池中获取代理
+        :return:
+        """
+        try:
+            while True:
+                proxy_val_list = list(self.__redis_pool.get_all_key("{}*".format(self.__redisProxyName)))
+                if proxy_val_list:
+                    proxy_val = random.choice(proxy_val_list)
+                    proxy_v = ":".join(str(proxy_val).split(":")[1:-1])
+                    self.__log("获取到的代理为：{proxy_v}".format(proxy_v=proxy_v))
+                    return proxy_v
+                else:
+                    self.__log("暂无代理，等待中")
+                    time.sleep(2)
+        except Exception as e:
+            self.__log("从代理池中获取代理：{e}".format(e=e))
+
+    def set_proxy_error(self, proxy_v):
+        """
+            爬虫手动传入代理，设置为 999999 不可用
+        :param proxyV:
+        :return:
+        """
+        try:
+            self.__redis_pool.set(proxy_v, "999999")
+            self.__log("设置不可用的代理 {proxy_v} 为 999999".format(proxy_v=proxy_v))
+        except Exception as e:
+            self.__log("爬虫手动传入代理：{e}".format(e=e))
+
+
+if __name__ == '__main__':
+    # p = ProxyPool(ip="127.0.0.1", port=6379, db=0, user_pass="xtn-kk", XiaoXiangProxyAppKey="1101384562594172928",
+    #               XiaoXiangProxyAppSecret="PJ0QBWML")
+    # print(p.get_proxy())
+    pass

xtn_tools_pro/utils/crypto.py

@@ -8,10 +8,11 @@
 # --------------------------------------------------------------------------------------------------
 # 2024/5/13    xiatn     V00.01.000    新建
 # --------------------------------------------------------------------------------------------------
+import base64
 import hashlib
 
 
-def get_md5_32(s, is_upper=False):
+def get_md5_32(s: str, is_upper=False):
     """
         获取文本的md5值 32位
     :param s: 文本
@@ -25,7 +26,7 @@ def get_md5_32(s, is_upper=False):
     return m.hexdigest()
 
 
-def get_md5_16(s, is_upper=False):
+def get_md5_16(s: str, is_upper=False):
     """
         获取文本的md5值 16位
     :param s: 文本
@@ -85,3 +86,36 @@ def get_file_md5_16(file_path, is_upper=False):
     """
     result = get_file_md5_32(file_path, is_upper)
     return result[8:24]
+
+
+def get_sha1(s: str, is_upper=False):
+    """
+        sha1
+    :param s: 文本
+    :param is_upper: 是否转大写 默认False
+    :return:
+    """
+    # 使用sha1算法进行哈希
+    sha1_hash = hashlib.sha1(s.encode()).hexdigest()
+    if is_upper:
+        return sha1_hash.upper()
+    return sha1_hash
+
+
+def get_base64_encode(s: str):
+    """
+        base64 编码
+    :param s: 文本
+    :return:
+    """
+    # 将字符串编码为 bytes
+    data_bytes = s.encode('utf-8')
+    # 使用 base64 进行编码
+    encoded_bytes = base64.b64encode(data_bytes)
+    # 将编码后的 bytes 转换为字符串
+    encoded_string = encoded_bytes.decode('utf-8')
+    return encoded_string
+
+
+if __name__ == '__main__':
+    print(get_base64_encode(''))

xtn_tools_pro/utils/helpers.py

@@ -38,11 +38,11 @@ def get_uuid(version=4, namespace: UUID = uuid.NAMESPACE_DNS, name=""):
     else:
         result = uuid.uuid4()
 
-    uuid_str = str(result)
+    # uuid_str = str(result)
     # uuid_hex = uuid_obj.hex
     # uuid_int = uuid_obj.int
     # uuid_bytes = uuid_obj.bytes
-    return uuid_str
+    return result
 
 
 def get_str_to_json(str_json):
@@ -120,4 +120,7 @@ def get_build_url_with_params(url, params):
     """
     encoded_params = urlencode(params)
     full_url = url + "?" + encoded_params
-    return full_url
+    return full_url
+
+if __name__ == '__main__':
+    print(get_uuid(4))

{xtn_tools_pro-1.0.0.0.6.dist-info → xtn_tools_pro-1.0.0.0.8.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: xtn-tools-pro
-Version: 1.0.0.0.6
+Version: 1.0.0.0.8
 Summary: xtn 开发工具
 Author: xtn
 Author-email: czw011122@163.com

{xtn_tools_pro-1.0.0.0.6.dist-info → xtn_tools_pro-1.0.0.0.8.dist-info}/RECORD

@@ -8,16 +8,17 @@ xtn_tools_pro/db/RedisDB.py,sha256=ep32Yj8AAkUHRshSBhKsdl06UwO7Z-gQJLaezspVRKw,6
 xtn_tools_pro/db/__init__.py,sha256=Zg91UWS02TO0Ba_0AY56s0oabRy93xLNFkpIIL_6mMM,416
 xtn_tools_pro/proxy/XiaoXiangProxy.py,sha256=6jzGgN2t2zLPIKbSgN8seixwDLY4IjoZvB26f8yTUME,9848
 xtn_tools_pro/proxy/__init__.py,sha256=WRwh6s2lruMu5buh0ejo9EK54kWT_VQhCsFGNFAmcyo,418
+xtn_tools_pro/proxy/proxy.py,sha256=No6E1pFY5yx2F4976pXPrLtq-QEVp79KupzcufjSN58,8703
 xtn_tools_pro/utils/__init__.py,sha256=I1_n_NP23F2lBqlF4EOlnOdLYxM8M4pbn63UhJN1hRE,418
-xtn_tools_pro/utils/crypto.py,sha256=1hwpD3SW0PI2fa79-_Xh37ACzeeKFH775GiVTlp5xjA,2379
+xtn_tools_pro/utils/crypto.py,sha256=RZ5AET4udlraACWMeNF-17JiZ2R6Ahb47_j4tjkV7LE,3190
 xtn_tools_pro/utils/file_utils.py,sha256=4Bmb1ISxqQC-7doPXDyY2o-H0m4upj_-hlLrO36sKJg,1509
-xtn_tools_pro/utils/helpers.py,sha256=C6w0Nc8ngpWtdwyKDAACtobnAe-qQNVCRgk5n9iM0o8,3719
+xtn_tools_pro/utils/helpers.py,sha256=VzevgDk0tZAxFmuFlMBIaJ3QyiFxp5Qg-mLPgE83jRI,3773
 xtn_tools_pro/utils/log.py,sha256=8CRMiY7Q9LWGJLhfL0YfpANBwLets9f2qWfMOe5PpNM,8139
 xtn_tools_pro/utils/retry.py,sha256=0wjHsR5DBBKpv4naMfxiky8kprrZes4WURIfFQ4H708,1657
 xtn_tools_pro/utils/sql.py,sha256=EAKzbkZP7Q09j15Gm6o0_uq0qgQmcCQT6EAawbpp4v0,6263
 xtn_tools_pro/utils/time_utils.py,sha256=TUtzG61PeVYXhaQd6pBrXAdlz7tBispNIRQRcGhE2No,4859
-xtn_tools_pro-1.0.0.0.6.dist-info/LICENSE,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-xtn_tools_pro-1.0.0.0.6.dist-info/METADATA,sha256=1vth1IfbsqTpVsmz24eaHlsUlxX2apH9eUz2gMdRjKs,431
-xtn_tools_pro-1.0.0.0.6.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
-xtn_tools_pro-1.0.0.0.6.dist-info/top_level.txt,sha256=jyB3FLDEr8zE1U7wHczTgIbvUpALhR-ULF7RVEO7O2U,14
-xtn_tools_pro-1.0.0.0.6.dist-info/RECORD,,
+xtn_tools_pro-1.0.0.0.8.dist-info/LICENSE,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+xtn_tools_pro-1.0.0.0.8.dist-info/METADATA,sha256=CN33o7GbV3bBP0HYYrlBem6vBcoHD6u64GRnlEBENKQ,431
+xtn_tools_pro-1.0.0.0.8.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
+xtn_tools_pro-1.0.0.0.8.dist-info/top_level.txt,sha256=jyB3FLDEr8zE1U7wHczTgIbvUpALhR-ULF7RVEO7O2U,14
+xtn_tools_pro-1.0.0.0.8.dist-info/RECORD,,