xtn-tools-pro 1.0.0.0.2__py3-none-any.whl → 1.0.0.0.3__py3-none-any.whl

xtn_tools_pro/db/RedisDB.py

@@ -27,6 +27,9 @@ class RedisDBPro:
         self.__redis = None
         self.get_connect()
 
+    def __del__(self):
+        self.__redis.close()
+
     @classmethod
     def from_url(cls, url):
         """
@@ -110,9 +113,53 @@ class RedisDBPro:
         else:
             return self._redis.sadd(table, values)
 
+    def incr(self, key):
+        """
+            对一个键的值进行自增操作
+        :param key: 需要自增的key
+        :return:
+        """
+        return self._redis.incr(key)
+
+    def get_all_key(self, path):
+        """
+            获取所有的key
+            常用的path：前缀为test的key test*，中间为test的key *test*
+        :param path:
+        :return:
+        """
+        return list(self._redis.scan_iter(path))
+
+    def get(self, table):
+        return self._redis.get(table)
+
+    def set(self, table, value, **kwargs):
+        """
+            字符串 set
+        :param table: 表
+        :param value: 值
+        :param kwargs: 参数解释为chatgpt提供
+        :param kwargs: ex（可选）：设置键的过期时间，以秒为单位。例如，ex=10表示键将在10秒后过期
+        :param kwargs: px（可选）：设置键的过期时间，以毫秒为单位。例如，px=10000表示键将在10秒后过期
+        :param kwargs: nx（可选）：如果设置为True，则只有在键不存在时才设置键的值
+        :param kwargs: xx（可选）：如果设置为True，则只有在键已存在时才设置键的值
+        :param kwargs: kepp_ttl（可选）：如果设置为True，则保留键的过期时间。仅当键已存在且设置了过期时间时才有效
+        :param kwargs: exat（可选）：设置键的过期时间，以UNIX时间戳表示。
+        :param kwargs: pxat（可选）：设置键的过期时间，以毫秒级的UNIX时间戳表示。
+        :param kwargs: replace（可选）：如果设置为True，则无论键是否存在，都会设置键的值。
+        :return:
+        """
+        return self._redis.set(table, value, **kwargs)
+
+    def delete(self, table):
+        return self._redis.delete(table)
+
 
 if __name__ == '__main__':
     pass
     # r = RedisDBPro(ip="127.0.0.1", port=6379, db=0, user_pass="xtn-kk")
+    r = RedisDBPro.from_url('redis://:xtn-kk@127.0.0.1:6379/0')
     # status = r.sadd("test_redis_pro", [1, 2, 3, 4, 5, "6", "7"])
     # print(status)
+    # print(r.get_all_key("*http*"))
+    print(r.delete("test_redis_pro"))

xtn_tools_pro/proxy/XiaoXiangProxy.py

@@ -0,0 +1,207 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+# 说明：
+#    小象代理专用
+# History:
+# Date          Author    Version       Modification
+# --------------------------------------------------------------------------------------------------
+# 2024/4/27    xiatn     V00.01.000    新建
+# --------------------------------------------------------------------------------------------------
+import requests, time, random
+from xtn_tools_pro.db.RedisDB import RedisDBPro
+from xtn_tools_pro.tools_time import get_time_now_timestamp, get_time_now_day59_timestamp
+
+import warnings
+from urllib3.exceptions import InsecureRequestWarning
+
+warnings.filterwarnings("ignore", category=InsecureRequestWarning)
+
+
+class ProxyPool:
+    def __init__(self, ip, port, db=0, user_pass="", redis_proxy_name="XiaoXiangProxy",
+                 XiaoXiangProxyAppKey=None, XiaoXiangProxyAppSecret=None, usage_cnt=100, usage_time=100):
+        """
+            小象代理专用
+        :param ip: redis 数据库 ip
+        :param port: redis 数据库 端口
+        :param db: redis 数据库 db
+        :param user_pass: redis 数据库 密码
+        :param redis_proxy_name: redis 数据库 用于存储代理的key
+        :param XiaoXiangProxyAppKey: 小象代理 应用id appKey
+        :param XiaoXiangProxyAppSecret: 小象代理 应用密码 appSecret
+        :param usage_cnt: 每个代理最长使用次数 单位秒 维护代理时用
+        :param usage_time: 每个代理最长使用时间 单位秒 维护代理时用
+        :param is_log: 是否记录日志
+        """
+        if not XiaoXiangProxyAppSecret or not XiaoXiangProxyAppKey:
+            raise Exception("应用密码或应用id 不能为空")
+
+        r = RedisDBPro(ip=ip, port=port, db=db, user_pass=user_pass)
+        self.__redis_pool = r
+        self.__redisProxyName = redis_proxy_name
+        self.__XiaoXiangProxyAPI = "https://api.xiaoxiangdaili.com/ip/get?appKey={appKey}&appSecret={appSecret}&cnt=&wt=json".format(
+            appKey=XiaoXiangProxyAppKey, appSecret=XiaoXiangProxyAppSecret)
+        self.__XiaoXiangAutoBinding = "https://api.xiaoxiangdaili.com/app/bindIp?appKey={appKey}&appSecret={appSecret}&i=1".format(
+            appKey=XiaoXiangProxyAppKey, appSecret=XiaoXiangProxyAppSecret)
+
+        # 获取当天0点时间戳
+        self.__now_day59_timestamp = get_time_now_day59_timestamp()
+        self.__usage_cnt = usage_cnt
+        self.__usage_time = usage_time
+
+        # if is_log:
+        #     # 日志
+        #     nowDate = str(datetime.datetime.now().strftime('%Y_%m_%d'))
+        #     logger.add(loggerPath.format(t=nowDate))
+
+    def __log(self, text):
+        """
+            记录日志
+        :param text:
+        :return:
+        """
+        print(text)
+
+    def __check_proxy(self):
+        """
+            维护检查代理，删除无用代理
+            删除标准：1.代理使用超过xx次；2.使用时间超过xx秒；3.被爬虫标记使用次数为 999999 会被删除
+        :return:
+        """
+        proxy_val_list = list(self.__redis_pool.get_all_key("{}*".format(self.__redisProxyName)))
+        for proxy_val in proxy_val_list:
+            # 获取时间
+            time_out = proxy_val.split(":")[-1]
+            # 获取使用次数
+            proxy_val_count = self.__redis_pool.get(proxy_val)
+            if int(time_out) + self.__usage_time < get_time_now_timestamp(is_time_10=True):
+                del_state = self.__redis_pool.delete(proxy_val)
+                self.__log(
+                    "当前代理状态:{proxy_val},{time_out}_{py_time}当前代理已超过使用时间,删除状态为：{del_state}".
+                        format(proxy_val=proxy_val, del_state=del_state,
+                               time_out=time_out, py_time=get_time_now_timestamp(is_time_10=True)))
+            elif int(proxy_val_count) >= self.__usage_cnt:
+                del_state = self.__redis_pool.delete(proxy_val)
+                self.__log(
+                    "当前代理状态:{proxy_val},{text},删除状态为：{del_state}".format(proxy_val=proxy_val,
+                                                                         text="当前代理被爬虫标记为不可用" if proxy_val_count >= 999999 else "当前代理已超过使用时间",
+                                                                         del_state=del_state))
+
+    def __get_proxy_length(self):
+        """
+            获取代理数
+        :return:
+        """
+        proxy_val_list = list(self.__redis_pool.get_all_key("{}*".format(self.__redisProxyName)))
+        return len(proxy_val_list)
+
+    def __incr_proxy(self, proxy_val):
+        """
+            自增代理使用次数
+        :param proxy_val: 代理
+        :return:
+        """
+        proxy_val_con = self.__redis_pool.incr(proxy_val)
+        return proxy_val_con
+
+    def __get_api_proxy(self):
+        """
+            通过接口获取小象代理，并存储至数据库
+            响应：
+                {"code":1010,"success":false,"data":null,"msg":"请求过于频繁"}
+                {"code":200,"success":true,"data":[{"ip":"125.123.244.60","port":37635,"realIp":null,"startTime":"2024-04-27 14:09:42","during":2}],"msg":"操作成功"}
+        :return:
+        """
+        while True:
+            self.__check_proxy()
+            try:
+                response = requests.get(url=self.__XiaoXiangProxyAPI, verify=False, timeout=3)
+                if response.status_code == 200:
+                    if response.json().get("msg") == "请求过于频繁":
+                        self.__log("获取小象代理过于频繁，等待2s，{content}".format(content=response.text))
+                        time.sleep(2)
+                        continue
+                    # 获取data
+                    proxy_data_list = response.json().get("data", [])
+                    if not proxy_data_list:
+                        self.__log("获取小象代理失败 data 为空，等待2s，{content}".format(content=response.text))
+                        time.sleep(2)
+                        continue
+                    else:
+                        for data in proxy_data_list:
+                            ip = "http://{ip}".format(ip=data.get("ip"))
+                            port = data.get("port")
+                            time_out = get_time_now_timestamp(is_time_10=True)
+                            proxy_key = "{redis_proxy_name}:{ip}:{port}:{timeOut}".format(
+                                redis_proxy_name=self.__redisProxyName,
+                                ip=ip,
+                                port=port,
+                                timeOut=time_out,
+                            )
+                            proxy_key_con = self.__incr_proxy(proxy_key)
+                            self.__log("获取代理:{proxy_key}，插入数据库状态为{proxy_key_con}".format(proxy_key=proxy_key,
+                                                                                         proxy_key_con=proxy_key_con))
+                        return True  # 获取成功
+                else:
+                    self.__log("获取小象代理返回响应码不为200，等待2s，{content}".format(content=response.text))
+                    time.sleep(2)
+                    continue
+            except Exception as e:
+                self.__log("获取小象代理报错：{e}".format(e=e))
+
+    def run(self):
+        try:
+            # 手动绑定终端IP
+            response = requests.get(url=self.__XiaoXiangAutoBinding, verify=False, timeout=3)
+            self.__log(response.text)
+            while True:
+                # 检查代理
+                self.__check_proxy()
+                # 获取小象代理
+                self.__get_api_proxy()
+                time.sleep(1)
+                # 判断时间是否超过当前23:59分时间戳
+                if get_time_now_timestamp(is_time_10=True) >= self.__now_day59_timestamp:
+                    self.__log("时间23:59，结束循环，{t}".format(t=int(time.time())))
+                    break
+        except Exception as eee:
+            self.__log("程序异常报错:{eee}".format(eee=eee))
+            # self.__del__()
+
+    def get_proxy(self):
+        """
+            从代理池中获取代理
+        :return:
+        """
+        try:
+            while True:
+                proxy_val_list = list(self.__redis_pool.get_all_key("{}*".format(self.__redisProxyName)))
+                if proxy_val_list:
+                    proxy_val = random.choice(proxy_val_list)
+                    proxy_v = ":".join(str(proxy_val).split(":")[1:-1])
+                    self.__log("获取到的代理为：{proxy_v}".format(proxy_v=proxy_v))
+                    return proxy_v
+                else:
+                    self.__log("暂无代理，等待中")
+                    time.sleep(2)
+        except Exception as e:
+            self.__log("从代理池中获取代理：{e}".format(e=e))
+
+    def set_proxy_error(self, proxy_v):
+        """
+            爬虫手动传入代理，设置为 999999 不可用
+        :param proxyV:
+        :return:
+        """
+        try:
+            self.__redis_pool.set(proxy_v, "999999")
+            self.__log("设置不可用的代理 {proxy_v} 为 999999".format(proxy_v=proxy_v))
+        except Exception as e:
+            self.__log("爬虫手动传入代理：{e}".format(e=e))
+
+
+if __name__ == '__main__':
+    p = ProxyPool(ip="127.0.0.1", port=6379, db=0, user_pass="xtn-kk", XiaoXiangProxyAppKey="1101384562594172928",
+                  XiaoXiangProxyAppSecret="PJ0QBWML")
+    print(p.get_proxy())

xtn_tools_pro/proxy/__init__.py

@@ -0,0 +1,10 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+# 说明：
+#    程序说明xxxxxxxxxxxxxxxxxxx
+# History:
+# Date          Author    Version       Modification
+# --------------------------------------------------------------------------------------------------
+# 2024/4/27    xiatn     V00.01.000    新建
+# --------------------------------------------------------------------------------------------------

{xtn_tools_pro-1.0.0.0.2.dist-info → xtn_tools_pro-1.0.0.0.3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: xtn-tools-pro
-Version: 1.0.0.0.2
+Version: 1.0.0.0.3
 Summary: xtn 开发工具
 Author: xtn
 Author-email: czw011122@163.com
@@ -10,5 +10,6 @@ Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: pymongo
 Requires-Dist: redis
+Requires-Dist: requests
 
 xtnkk-tools

xtn_tools_pro-1.0.0.0.3.dist-info/RECORD

@@ -0,0 +1,14 @@
+xtn_tools_pro/__init__.py,sha256=26Tf9j2wj88M1Ldg3b1DJ40KyGgN9ZmQdBLuV453388,395
+xtn_tools_pro/tools.py,sha256=e9KSPqaFBIptBGvexShCcn0nZmUQ5omlVwXgEfWZf5Y,2630
+xtn_tools_pro/tools_flie.py,sha256=B_P3J_R-nRLt_IFutnOVrBRGf6_SZ_cXoIoeaT9B7tk,1512
+xtn_tools_pro/tools_time.py,sha256=DMjsw9h4E_mrPsanPA8CEhpUE1AA6Z2FU4OJqJKZc1k,4867
+xtn_tools_pro/db/MongoDB.py,sha256=_GiX1MHNl9CtI-uLDgY_NmMSvRJei-mtKq3Hhe6ly1E,5567
+xtn_tools_pro/db/RedisDB.py,sha256=ep32Yj8AAkUHRshSBhKsdl06UwO7Z-gQJLaezspVRKw,6053
+xtn_tools_pro/db/__init__.py,sha256=Zg91UWS02TO0Ba_0AY56s0oabRy93xLNFkpIIL_6mMM,416
+xtn_tools_pro/proxy/XiaoXiangProxy.py,sha256=xrEBJZ6Cjuh6IBZVB17oXHwByaeKcWVUSSWSg17tujE,9842
+xtn_tools_pro/proxy/__init__.py,sha256=WRwh6s2lruMu5buh0ejo9EK54kWT_VQhCsFGNFAmcyo,418
+xtn_tools_pro-1.0.0.0.3.dist-info/LICENSE,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+xtn_tools_pro-1.0.0.0.3.dist-info/METADATA,sha256=VG4Bs3muLvi85SaWcP3Tw3q4TFfafKDtZhbQ-x9CpRw,358
+xtn_tools_pro-1.0.0.0.3.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
+xtn_tools_pro-1.0.0.0.3.dist-info/top_level.txt,sha256=jyB3FLDEr8zE1U7wHczTgIbvUpALhR-ULF7RVEO7O2U,14
+xtn_tools_pro-1.0.0.0.3.dist-info/RECORD,,

xtn_tools_pro-1.0.0.0.2.dist-info/RECORD

@@ -1,12 +0,0 @@
-xtn_tools_pro/__init__.py,sha256=26Tf9j2wj88M1Ldg3b1DJ40KyGgN9ZmQdBLuV453388,395
-xtn_tools_pro/tools.py,sha256=e9KSPqaFBIptBGvexShCcn0nZmUQ5omlVwXgEfWZf5Y,2630
-xtn_tools_pro/tools_flie.py,sha256=B_P3J_R-nRLt_IFutnOVrBRGf6_SZ_cXoIoeaT9B7tk,1512
-xtn_tools_pro/tools_time.py,sha256=DMjsw9h4E_mrPsanPA8CEhpUE1AA6Z2FU4OJqJKZc1k,4867
-xtn_tools_pro/db/MongoDB.py,sha256=_GiX1MHNl9CtI-uLDgY_NmMSvRJei-mtKq3Hhe6ly1E,5567
-xtn_tools_pro/db/RedisDB.py,sha256=qMffCNIHa3o7KD_yVQlsj3OupsNXMsDPRi03migwSu0,4003
-xtn_tools_pro/db/__init__.py,sha256=Zg91UWS02TO0Ba_0AY56s0oabRy93xLNFkpIIL_6mMM,416
-xtn_tools_pro-1.0.0.0.2.dist-info/LICENSE,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-xtn_tools_pro-1.0.0.0.2.dist-info/METADATA,sha256=4g3WpBcx-4rpklXL9qBj2Q2iTqlTTbCQjlHuLgPRr_8,333
-xtn_tools_pro-1.0.0.0.2.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
-xtn_tools_pro-1.0.0.0.2.dist-info/top_level.txt,sha256=jyB3FLDEr8zE1U7wHczTgIbvUpALhR-ULF7RVEO7O2U,14
-xtn_tools_pro-1.0.0.0.2.dist-info/RECORD,,