re-common 10.0.21__py3-none-any.whl → 10.0.22__py3-none-any.whl

re_common/vip/proxy/kuaidaili.py

@@ -1,87 +1,87 @@
-import json
-
-###########################################
-# 同项目调用基础包
-import os
-import sys
-import time
-import traceback
-
-filepath = os.path.abspath(__file__)
-pathlist = filepath.split(os.sep)
-pathlist = pathlist[:-4]
-TopPath = os.sep.join(pathlist)
-sys.path.insert(0, TopPath)
-print(TopPath)
-############################################
-
-from re_common.baselibrary.utils.baserequest import BaseRequest
-from re_common.facade.mysqlfacade import MysqlUtiles
-from re_common.baselibrary.utils.basedir import BaseDir
-from re_common.baselibrary.utils.basefile import BaseFile
-from re_common.facade.lazy_import import get_streamlogger
-
-
-class Kproxy(object):
-    def __init__(self):
-        self.cur_path = BaseDir.get_file_dir_absolute(__file__)
-        self.configfile = BaseFile.get_new_path(self.cur_path, "db.ini")
-        self.logger = get_streamlogger()
-        self.mysqlutils = MysqlUtiles(self.configfile, "dbkuaidaili", self.logger)
-        self.url = "https://dps.kdlapi.com/api/getdps/?orderid=990171566857288&num=15&pt=1&format=json&sep=1"
-        self.bsrequest = BaseRequest()
-        self.starttime = time.time()
-        self.starttime_val = time.time()
-
-    def get_proxy(self):
-        self.starttime = time.time()
-        BoolResult, errString, r = self.bsrequest.base_request(self.url,
-                                                               timeout=30
-                                                               )
-        if BoolResult:
-            dicts = json.loads(r.text)
-            for proxy in dicts["data"]["proxy_list"]:
-                sql = "insert into kuaidailiproxy (proxy) values ('%s') on DUPLICATE key update stat=1" % proxy
-                self.mysqlutils.ExeSqlToDB(sql)
-        else:
-            self.logger.error("获取失败")
-
-    def val(self, proxy):
-        self.starttime_val = time.time()
-        url = f"https://dps.kdlapi.com/api/getdpsvalidtime?orderid=990171566857288&signature=wm4vq53pwrat1vye458elwyxyh9awzqj&proxy={proxy}"
-        BoolResult, errString, r = self.bsrequest.base_request(url,
-                                                               timeout=30
-                                                               )
-        if BoolResult:
-            dicts = json.loads(r.text)
-            if dicts["data"][proxy] > 0:
-                sql = "update kuaidailiproxy set val_stat=0 where proxy='%s'" % proxy
-                self.mysqlutils.ExeSqlToDB(sql)
-            else:
-                sql = "update kuaidailiproxy set val_stat=0,stat=0 where proxy='%s'" % proxy
-                self.mysqlutils.ExeSqlToDB(sql)
-        else:
-            self.logger.error("获取失败")
-
-    def val_all(self):
-        self.starttime_val = time.time()
-        sql = "select proxy from kuaidailiproxy where stat=1"
-        bools, rows = self.mysqlutils.SelectFromDB(sql)
-        for row in rows:
-            try:
-                self.val(row[0])
-            except:
-                traceback.print_exc()
-
-    def run(self):
-        self.get_proxy()
-        while True:
-            if int(time.time() - self.starttime) > 20:
-                self.get_proxy()
-            if int(time.time() - self.starttime_val) > 30:
-                self.val_all()
-            time.sleep(5)
-
-
-if __name__ == "__main__":
-    Kproxy().run()
+import json
+
+###########################################
+# 同项目调用基础包
+import os
+import sys
+import time
+import traceback
+
+filepath = os.path.abspath(__file__)
+pathlist = filepath.split(os.sep)
+pathlist = pathlist[:-4]
+TopPath = os.sep.join(pathlist)
+sys.path.insert(0, TopPath)
+print(TopPath)
+############################################
+
+from re_common.baselibrary.utils.baserequest import BaseRequest
+from re_common.facade.mysqlfacade import MysqlUtiles
+from re_common.baselibrary.utils.basedir import BaseDir
+from re_common.baselibrary.utils.basefile import BaseFile
+from re_common.facade.lazy_import import get_streamlogger
+
+
+class Kproxy(object):
+    def __init__(self):
+        self.cur_path = BaseDir.get_file_dir_absolute(__file__)
+        self.configfile = BaseFile.get_new_path(self.cur_path, "db.ini")
+        self.logger = get_streamlogger()
+        self.mysqlutils = MysqlUtiles(self.configfile, "dbkuaidaili", self.logger)
+        self.url = "https://dps.kdlapi.com/api/getdps/?orderid=990171566857288&num=15&pt=1&format=json&sep=1"
+        self.bsrequest = BaseRequest()
+        self.starttime = time.time()
+        self.starttime_val = time.time()
+
+    def get_proxy(self):
+        self.starttime = time.time()
+        BoolResult, errString, r = self.bsrequest.base_request(self.url,
+                                                               timeout=30
+                                                               )
+        if BoolResult:
+            dicts = json.loads(r.text)
+            for proxy in dicts["data"]["proxy_list"]:
+                sql = "insert into kuaidailiproxy (proxy) values ('%s') on DUPLICATE key update stat=1" % proxy
+                self.mysqlutils.ExeSqlToDB(sql)
+        else:
+            self.logger.error("获取失败")
+
+    def val(self, proxy):
+        self.starttime_val = time.time()
+        url = f"https://dps.kdlapi.com/api/getdpsvalidtime?orderid=990171566857288&signature=wm4vq53pwrat1vye458elwyxyh9awzqj&proxy={proxy}"
+        BoolResult, errString, r = self.bsrequest.base_request(url,
+                                                               timeout=30
+                                                               )
+        if BoolResult:
+            dicts = json.loads(r.text)
+            if dicts["data"][proxy] > 0:
+                sql = "update kuaidailiproxy set val_stat=0 where proxy='%s'" % proxy
+                self.mysqlutils.ExeSqlToDB(sql)
+            else:
+                sql = "update kuaidailiproxy set val_stat=0,stat=0 where proxy='%s'" % proxy
+                self.mysqlutils.ExeSqlToDB(sql)
+        else:
+            self.logger.error("获取失败")
+
+    def val_all(self):
+        self.starttime_val = time.time()
+        sql = "select proxy from kuaidailiproxy where stat=1"
+        bools, rows = self.mysqlutils.SelectFromDB(sql)
+        for row in rows:
+            try:
+                self.val(row[0])
+            except:
+                traceback.print_exc()
+
+    def run(self):
+        self.get_proxy()
+        while True:
+            if int(time.time() - self.starttime) > 20:
+                self.get_proxy()
+            if int(time.time() - self.starttime_val) > 30:
+                self.val_all()
+            time.sleep(5)
+
+
+if __name__ == "__main__":
+    Kproxy().run()

re_common/vip/proxy/proxy_all.py

@@ -1,113 +1,113 @@
-import re
-
-from bs4 import BeautifulSoup
-
-from re_common.baselibrary.utils.baserequest import BaseRequest
-from re_common.baselibrary.utils.baseurl import BaseUrl
-from re_common.baselibrary.utils.myredisclient import MyRedis
-
-
-class ProxyAll(object):
-
-    def __init__(self, config="./db.ini"):
-        self.config = config
-        self.myredis = MyRedis(self.config)
-        self.myredis.set_redis_from_config()
-        self.myredis.conn_redis()
-        self.Headers = {
-            'Accept': '*/*',
-            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64; Trident/7.0; rv:11.0) like Gecko',
-        }
-        self.UserAgent = 'Mozilla/5.0 (Windows NT 10.0; WOW64; Trident/7.0; rv:11.0) like Gecko'
-        self.baserequest = BaseRequest()
-
-    def get_redis_all(self):
-        return self.myredis.getDataFromRedis()
-
-    def getProxyFromMimvp(self, num=600):
-        r""" 从 http://proxy.mimvp.com/ 获取代理地址 """
-        # http://proxy.mimvp.com/api/fetch.php?orderid=860160414114016557&num=100&http_type=1&anonymous=3&ping_time=1&transfer_time=5
-        url = 'http://proxy.mimvp.com/api/fetch.php'
-        dicts = {
-            "orderid": "860161011165812474",
-            "num": num,
-            "http_type": "1",  # 协议类型（http）
-            "anonymous": "2,3,5",  # 提取透明，匿名+欺骗，高匿
-            "ping_time": "1",  # 响应时间
-            "transfer_time": "5",  # 传输速度
-        }
-        url = BaseUrl.dicts_to_url(dicts, url=url)
-
-        proxyPool = set()
-
-        BoolResult, errString, r = self.baserequest.base_request(url, headers=self.Headers, timeout=10)
-        if not BoolResult:
-            return proxyPool
-
-        lst = r.text.split('\n')
-        for line in lst:
-            line = line.strip()
-            if line.count('.') == 3:
-                proxyPool.add(line)
-        return proxyPool
-
-    # 从 http://tpv.daxiangdaili.com 获取代理地址，<- 大象代理
-    # http://tpv.daxiangdaili.com/ip/?tid=556923006054759&num=1000
-    def getProxyFromDaxiang(self, num):
-        url = 'http://tpv.daxiangdaili.com/ip/'
-        dicts = {
-            "tid": "556923006054759",
-            "num": num,
-            # "filter":"on",
-            "foreign": "all",
-            "delay": "5"  # 延迟时间
-        }
-        url = BaseUrl.dicts_to_url(dicts, url=url)
-
-        proxyPool = set()
-
-        BoolResult, errString, r = self.baserequest.base_request(url, headers=self.Headers, timeout=10)
-        if not BoolResult:
-            return proxyPool
-
-        # print('daili666:' + repr(r.text))
-        lst = r.text.split('\n')
-        for line in lst:
-            line = line.strip()
-            if line.count('.') == 3:
-                proxyPool.add(line)
-
-        return proxyPool
-
-    # 从http://www.xici.net.co/nn/获取代理地址
-    # pageNum表示采集第几页
-    def getProxyFromXICIOnePage(self, pageNum):
-        ProxyPool = set()
-
-        url = 'http://www.xicidaili.com/nn/'
-        if pageNum > 1:
-            url += str(pageNum)
-
-        proxyPool = set()
-        BoolResult, errString, r = self.baserequest.base_request(url, headers=self.Headers, timeout=10)
-        if not BoolResult:
-            return proxyPool
-
-        html = r.content.decode('utf-8')
-
-        soup = BeautifulSoup(html, 'html.parser')
-        ipTable = soup.find('table', id='ip_list')
-        if not ipTable:
-            print('Error: not ipTable')
-            return set()
-
-        for trTag in ipTable.find_all('tr'):
-            lst = list(trTag.find_all('td'))
-            if len(lst) != 10:
-                continue
-            ip = ''.join(lst[1].stripped_strings)
-            port = ''.join(lst[2].stripped_strings)
-            item = ip + ':' + port
-            if re.match(r'^[\d\.:]+$', item):
-                ProxyPool.add(item)
-        return ProxyPool
+import re
+
+from bs4 import BeautifulSoup
+
+from re_common.baselibrary.utils.baserequest import BaseRequest
+from re_common.baselibrary.utils.baseurl import BaseUrl
+from re_common.baselibrary.utils.myredisclient import MyRedis
+
+
+class ProxyAll(object):
+
+    def __init__(self, config="./db.ini"):
+        self.config = config
+        self.myredis = MyRedis(self.config)
+        self.myredis.set_redis_from_config()
+        self.myredis.conn_redis()
+        self.Headers = {
+            'Accept': '*/*',
+            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64; Trident/7.0; rv:11.0) like Gecko',
+        }
+        self.UserAgent = 'Mozilla/5.0 (Windows NT 10.0; WOW64; Trident/7.0; rv:11.0) like Gecko'
+        self.baserequest = BaseRequest()
+
+    def get_redis_all(self):
+        return self.myredis.getDataFromRedis()
+
+    def getProxyFromMimvp(self, num=600):
+        r""" 从 http://proxy.mimvp.com/ 获取代理地址 """
+        # http://proxy.mimvp.com/api/fetch.php?orderid=860160414114016557&num=100&http_type=1&anonymous=3&ping_time=1&transfer_time=5
+        url = 'http://proxy.mimvp.com/api/fetch.php'
+        dicts = {
+            "orderid": "860161011165812474",
+            "num": num,
+            "http_type": "1",  # 协议类型（http）
+            "anonymous": "2,3,5",  # 提取透明，匿名+欺骗，高匿
+            "ping_time": "1",  # 响应时间
+            "transfer_time": "5",  # 传输速度
+        }
+        url = BaseUrl.dicts_to_url(dicts, url=url)
+
+        proxyPool = set()
+
+        BoolResult, errString, r = self.baserequest.base_request(url, headers=self.Headers, timeout=10)
+        if not BoolResult:
+            return proxyPool
+
+        lst = r.text.split('\n')
+        for line in lst:
+            line = line.strip()
+            if line.count('.') == 3:
+                proxyPool.add(line)
+        return proxyPool
+
+    # 从 http://tpv.daxiangdaili.com 获取代理地址，<- 大象代理
+    # http://tpv.daxiangdaili.com/ip/?tid=556923006054759&num=1000
+    def getProxyFromDaxiang(self, num):
+        url = 'http://tpv.daxiangdaili.com/ip/'
+        dicts = {
+            "tid": "556923006054759",
+            "num": num,
+            # "filter":"on",
+            "foreign": "all",
+            "delay": "5"  # 延迟时间
+        }
+        url = BaseUrl.dicts_to_url(dicts, url=url)
+
+        proxyPool = set()
+
+        BoolResult, errString, r = self.baserequest.base_request(url, headers=self.Headers, timeout=10)
+        if not BoolResult:
+            return proxyPool
+
+        # print('daili666:' + repr(r.text))
+        lst = r.text.split('\n')
+        for line in lst:
+            line = line.strip()
+            if line.count('.') == 3:
+                proxyPool.add(line)
+
+        return proxyPool
+
+    # 从http://www.xici.net.co/nn/获取代理地址
+    # pageNum表示采集第几页
+    def getProxyFromXICIOnePage(self, pageNum):
+        ProxyPool = set()
+
+        url = 'http://www.xicidaili.com/nn/'
+        if pageNum > 1:
+            url += str(pageNum)
+
+        proxyPool = set()
+        BoolResult, errString, r = self.baserequest.base_request(url, headers=self.Headers, timeout=10)
+        if not BoolResult:
+            return proxyPool
+
+        html = r.content.decode('utf-8')
+
+        soup = BeautifulSoup(html, 'html.parser')
+        ipTable = soup.find('table', id='ip_list')
+        if not ipTable:
+            print('Error: not ipTable')
+            return set()
+
+        for trTag in ipTable.find_all('tr'):
+            lst = list(trTag.find_all('td'))
+            if len(lst) != 10:
+                continue
+            ip = ''.join(lst[1].stripped_strings)
+            port = ''.join(lst[2].stripped_strings)
+            item = ip + ':' + port
+            if re.match(r'^[\d\.:]+$', item):
+                ProxyPool.add(item)
+        return ProxyPool

re_common/vip/proxy/update_kuaidaili_0.py

@@ -1,42 +1,42 @@
-import json
-
-###########################################
-# 同项目调用基础包
-import os
-import sys
-import time
-
-filepath = os.path.abspath(__file__)
-pathlist = filepath.split(os.sep)
-pathlist = pathlist[:-4]
-TopPath = os.sep.join(pathlist)
-sys.path.insert(0, TopPath)
-print(TopPath)
-############################################
-
-from re_common.baselibrary.utils.baserequest import BaseRequest
-from re_common.facade.mysqlfacade import MysqlUtiles
-from re_common.baselibrary.utils.basedir import BaseDir
-from re_common.baselibrary.utils.basefile import BaseFile
-from re_common.facade.lazy_import import get_streamlogger
-
-
-class Kproxy(object):
-    def __init__(self):
-        self.cur_path = BaseDir.get_file_dir_absolute(__file__)
-        self.configfile = BaseFile.get_new_path(self.cur_path, "db.ini")
-        self.logger = get_streamlogger()
-        self.mysqlutils = MysqlUtiles(self.configfile, "dbkuaidaili", self.logger)
-        self.starttime = time.time()
-        self.starttime_val = time.time()
-
-    def run(self):
-        sql = "update `kuaidailiproxy` set val_stat=1 WHERE stat=1"
-        self.mysqlutils.ExeSqlToDB(sql)
-
-
-if __name__ == "__main__":
-    k = Kproxy()
-    while True:
-        k.run()
-        time.sleep(60*2)
+import json
+
+###########################################
+# 同项目调用基础包
+import os
+import sys
+import time
+
+filepath = os.path.abspath(__file__)
+pathlist = filepath.split(os.sep)
+pathlist = pathlist[:-4]
+TopPath = os.sep.join(pathlist)
+sys.path.insert(0, TopPath)
+print(TopPath)
+############################################
+
+from re_common.baselibrary.utils.baserequest import BaseRequest
+from re_common.facade.mysqlfacade import MysqlUtiles
+from re_common.baselibrary.utils.basedir import BaseDir
+from re_common.baselibrary.utils.basefile import BaseFile
+from re_common.facade.lazy_import import get_streamlogger
+
+
+class Kproxy(object):
+    def __init__(self):
+        self.cur_path = BaseDir.get_file_dir_absolute(__file__)
+        self.configfile = BaseFile.get_new_path(self.cur_path, "db.ini")
+        self.logger = get_streamlogger()
+        self.mysqlutils = MysqlUtiles(self.configfile, "dbkuaidaili", self.logger)
+        self.starttime = time.time()
+        self.starttime_val = time.time()
+
+    def run(self):
+        sql = "update `kuaidailiproxy` set val_stat=1 WHERE stat=1"
+        self.mysqlutils.ExeSqlToDB(sql)
+
+
+if __name__ == "__main__":
+    k = Kproxy()
+    while True:
+        k.run()
+        time.sleep(60*2)