re-common 10.0.39__py3-none-any.whl → 10.0.41__py3-none-any.whl

re_common/vip/proxy/cnki_proxy.py

@@ -1,153 +1,153 @@
-import time
-
-###########################################
-# 同项目调用基础包
-import os
-import sys
-
-pathlist = os.path.abspath(__file__).split(os.sep)
-root_path = os.sep.join(pathlist[:pathlist.index("re-common") + 1])
-sys.path.insert(0, root_path)
-print(root_path)
-############################################
-from re_common.baselibrary.utils.myredisclient import MyRedis
-
-from re_common.baselibrary.mthread.MThreadingRun import MThreadingRun
-from re_common.baselibrary.mthread.mythreading import ThreadInfo, ThreadPoolManger
-from re_common.baselibrary.utils.baserequest import BaseRequest
-
-from re_common.baselibrary.utils.core.requests_core import set_proxy
-
-from re_common.facade.loggerfacade import get_streamlogger
-
-from re_common.facade.mysqlfacade import MysqlUtiles
-from proxy_all import ProxyAll
-
-
-class CnkiProxy(object):
-
-    def __init__(self, config="./db.ini"):
-        self.config = config
-        self.logger = get_streamlogger()
-        self.mysqlutils = MysqlUtiles(self.config, "dbcnki", self.logger)
-        self.Headers = {
-            'Accept': '*/*',
-            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64; Trident/7.0; rv:11.0) like Gecko',
-        }
-        self.UserAgent = 'Mozilla/5.0 (Windows NT 10.0; WOW64; Trident/7.0; rv:11.0) like Gecko'
-        self.baserequest = BaseRequest()
-
-    def checking_proxy(self, proxy):
-        url = "https://www.cnki.net/"
-        proxies = set_proxy(proxy)
-        BoolResult, errString, r = self.baserequest.base_request(url,
-                                                                 headers=self.Headers,
-                                                                 proxies=proxies,
-                                                                 marks=["product-content"],
-                                                                 timeout=5)
-        if BoolResult:
-            return proxy
-        return ""
-
-    def get_mysql_proxy(self):
-        sql = "SELECT proxy FROM `proxy_pool`"
-        bools, rows = self.mysqlutils.SelectFromDB(sql)
-        if not bools:
-            return set()
-        results = set()
-        for row in rows:
-            results.add(row[0])
-        sql = "delete from proxy_pool"
-        self.mysqlutils.ExeSqlToDB(sql)
-        return results
-
-    def get_all_proxy(self):
-        proxy_set = set()
-        proxyall = ProxyAll()
-        redisproxy = proxyall.get_redis_all()
-        mimvpproxy = proxyall.getProxyFromMimvp(1000)
-        daxiangproxy = proxyall.getProxyFromDaxiang(1000)
-        xiciproxy1 = proxyall.getProxyFromXICIOnePage(1)
-        xiciproxy2 = proxyall.getProxyFromXICIOnePage(2)
-        mysqlproxy = self.get_mysql_proxy()
-        proxy_set = proxy_set.union(mysqlproxy, redisproxy, mimvpproxy, daxiangproxy, xiciproxy1, xiciproxy2)
-        self.logger.info("all proxy size is:{}".format(len(proxy_set)))
-        self.proxy_set = proxy_set
-        return proxy_set
-
-    def get_can_use_proxy(self):
-        count = 0
-        use_proxy = set()
-        for proxy in self.proxy_set:
-            proxy_ = self.checking_proxy(proxy)
-            use_proxy.add(proxy_)
-            if len(use_proxy) > 20:
-                count = count + 1
-                if count == 1:
-                    sql = "delete from proxy_pool"
-                    self.mysqlutils.ExeSqlToDB(sql)
-                sql = "insert into proxy_pool(proxy) values ('%s')"
-                self.mysqlutils.ExeSqlMany(sql, use_proxy)
-                use_proxy.clear()
-        if len(use_proxy) > 0:
-            sql = "insert into proxy_pool(proxy) values ('%s')"
-            self.mysqlutils.ExeSqlMany(sql, use_proxy)
-            use_proxy.clear()
-
-
-class DetailThreadRun(MThreadingRun):
-    def __init__(self, num):
-        self.cnki = CnkiProxy()
-        super(DetailThreadRun, self).__init__(num)
-        self.config = "./db.ini"
-        self.myredisset = MyRedis(self.config)
-        self.myredisset.set_redis_from_config(sesc="proxysetcnkijournal")
-        self.myredisset.conn_redis()
-        self.myredisset.get_pipeline()
-
-    @ThreadPoolManger.thread_lock
-    def getTask(self, *args, **kwargs):
-        self.myredisset.delete(self.myredisset.RedisKey)
-        proxy_set = self.cnki.get_all_proxy()
-        return proxy_set
-
-    def setTask(self, results=None, *args, **kwargs):
-        for url_tasks in results:
-            # 将每一页加入任务队列
-            self.add_job(self.func, url_tasks)
-        time.sleep(10 * 60)
-
-    @ThreadPoolManger.thread_lock
-    def dealresult(self, *args, **kwargs):
-        sql = "replace into proxy_pool(`proxy`) values (%s)"
-        self.cnki.mysqlutils.ExeSqlMany(sql, self.results)
-        self.myredisset.sadd(self.myredisset.RedisKey, set(self.results))
-
-    def setProxy(self, proxysList=None):
-        pass
-
-    def is_break(self):
-        return False
-
-    def thread_pool_hook(self, threadinfo: ThreadInfo):
-        # 设置代理线程不重启，默认会重启
-        if threadinfo.get_thread_name() == self.etn.proxythreadname:
-            threadinfo.set_is_restart(False)
-        if threadinfo.get_thread_name() == self.etn.taskthreadname:
-            threadinfo.set_is_restart(False)
-        return {}
-
-    def fun(self, threadval, *args, **kwargs):
-        standardid = args[0]
-        proxys = self.cnki.checking_proxy(standardid)
-        if proxys != "":
-            threadval.result_queue.put(proxys)
-
-
-def main():
-    down = DetailThreadRun(40)
-    down.run()
-
-
-if __name__ == "__main__":
-    main()
+import time
+
+###########################################
+# 同项目调用基础包
+import os
+import sys
+
+pathlist = os.path.abspath(__file__).split(os.sep)
+root_path = os.sep.join(pathlist[:pathlist.index("re-common") + 1])
+sys.path.insert(0, root_path)
+print(root_path)
+############################################
+from re_common.baselibrary.utils.myredisclient import MyRedis
+
+from re_common.baselibrary.mthread.MThreadingRun import MThreadingRun
+from re_common.baselibrary.mthread.mythreading import ThreadInfo, ThreadPoolManger
+from re_common.baselibrary.utils.baserequest import BaseRequest
+
+from re_common.baselibrary.utils.core.requests_core import set_proxy
+
+from re_common.facade.loggerfacade import get_streamlogger
+
+from re_common.facade.mysqlfacade import MysqlUtiles
+from proxy_all import ProxyAll
+
+
+class CnkiProxy(object):
+
+    def __init__(self, config="./db.ini"):
+        self.config = config
+        self.logger = get_streamlogger()
+        self.mysqlutils = MysqlUtiles(self.config, "dbcnki", self.logger)
+        self.Headers = {
+            'Accept': '*/*',
+            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64; Trident/7.0; rv:11.0) like Gecko',
+        }
+        self.UserAgent = 'Mozilla/5.0 (Windows NT 10.0; WOW64; Trident/7.0; rv:11.0) like Gecko'
+        self.baserequest = BaseRequest()
+
+    def checking_proxy(self, proxy):
+        url = "https://www.cnki.net/"
+        proxies = set_proxy(proxy)
+        BoolResult, errString, r = self.baserequest.base_request(url,
+                                                                 headers=self.Headers,
+                                                                 proxies=proxies,
+                                                                 marks=["product-content"],
+                                                                 timeout=5)
+        if BoolResult:
+            return proxy
+        return ""
+
+    def get_mysql_proxy(self):
+        sql = "SELECT proxy FROM `proxy_pool`"
+        bools, rows = self.mysqlutils.SelectFromDB(sql)
+        if not bools:
+            return set()
+        results = set()
+        for row in rows:
+            results.add(row[0])
+        sql = "delete from proxy_pool"
+        self.mysqlutils.ExeSqlToDB(sql)
+        return results
+
+    def get_all_proxy(self):
+        proxy_set = set()
+        proxyall = ProxyAll()
+        redisproxy = proxyall.get_redis_all()
+        mimvpproxy = proxyall.getProxyFromMimvp(1000)
+        daxiangproxy = proxyall.getProxyFromDaxiang(1000)
+        xiciproxy1 = proxyall.getProxyFromXICIOnePage(1)
+        xiciproxy2 = proxyall.getProxyFromXICIOnePage(2)
+        mysqlproxy = self.get_mysql_proxy()
+        proxy_set = proxy_set.union(mysqlproxy, redisproxy, mimvpproxy, daxiangproxy, xiciproxy1, xiciproxy2)
+        self.logger.info("all proxy size is:{}".format(len(proxy_set)))
+        self.proxy_set = proxy_set
+        return proxy_set
+
+    def get_can_use_proxy(self):
+        count = 0
+        use_proxy = set()
+        for proxy in self.proxy_set:
+            proxy_ = self.checking_proxy(proxy)
+            use_proxy.add(proxy_)
+            if len(use_proxy) > 20:
+                count = count + 1
+                if count == 1:
+                    sql = "delete from proxy_pool"
+                    self.mysqlutils.ExeSqlToDB(sql)
+                sql = "insert into proxy_pool(proxy) values ('%s')"
+                self.mysqlutils.ExeSqlMany(sql, use_proxy)
+                use_proxy.clear()
+        if len(use_proxy) > 0:
+            sql = "insert into proxy_pool(proxy) values ('%s')"
+            self.mysqlutils.ExeSqlMany(sql, use_proxy)
+            use_proxy.clear()
+
+
+class DetailThreadRun(MThreadingRun):
+    def __init__(self, num):
+        self.cnki = CnkiProxy()
+        super(DetailThreadRun, self).__init__(num)
+        self.config = "./db.ini"
+        self.myredisset = MyRedis(self.config)
+        self.myredisset.set_redis_from_config(sesc="proxysetcnkijournal")
+        self.myredisset.conn_redis()
+        self.myredisset.get_pipeline()
+
+    @ThreadPoolManger.thread_lock
+    def getTask(self, *args, **kwargs):
+        self.myredisset.delete(self.myredisset.RedisKey)
+        proxy_set = self.cnki.get_all_proxy()
+        return proxy_set
+
+    def setTask(self, results=None, *args, **kwargs):
+        for url_tasks in results:
+            # 将每一页加入任务队列
+            self.add_job(self.func, url_tasks)
+        time.sleep(10 * 60)
+
+    @ThreadPoolManger.thread_lock
+    def dealresult(self, *args, **kwargs):
+        sql = "replace into proxy_pool(`proxy`) values (%s)"
+        self.cnki.mysqlutils.ExeSqlMany(sql, self.results)
+        self.myredisset.sadd(self.myredisset.RedisKey, set(self.results))
+
+    def setProxy(self, proxysList=None):
+        pass
+
+    def is_break(self):
+        return False
+
+    def thread_pool_hook(self, threadinfo: ThreadInfo):
+        # 设置代理线程不重启，默认会重启
+        if threadinfo.get_thread_name() == self.etn.proxythreadname:
+            threadinfo.set_is_restart(False)
+        if threadinfo.get_thread_name() == self.etn.taskthreadname:
+            threadinfo.set_is_restart(False)
+        return {}
+
+    def fun(self, threadval, *args, **kwargs):
+        standardid = args[0]
+        proxys = self.cnki.checking_proxy(standardid)
+        if proxys != "":
+            threadval.result_queue.put(proxys)
+
+
+def main():
+    down = DetailThreadRun(40)
+    down.run()
+
+
+if __name__ == "__main__":
+    main()

re_common/vip/proxy/kuaidaili.py

@@ -1,87 +1,87 @@
-import json
-
-###########################################
-# 同项目调用基础包
-import os
-import sys
-import time
-import traceback
-
-filepath = os.path.abspath(__file__)
-pathlist = filepath.split(os.sep)
-pathlist = pathlist[:-4]
-TopPath = os.sep.join(pathlist)
-sys.path.insert(0, TopPath)
-print(TopPath)
-############################################
-
-from re_common.baselibrary.utils.baserequest import BaseRequest
-from re_common.facade.mysqlfacade import MysqlUtiles
-from re_common.baselibrary.utils.basedir import BaseDir
-from re_common.baselibrary.utils.basefile import BaseFile
-from re_common.facade.lazy_import import get_streamlogger
-
-
-class Kproxy(object):
-    def __init__(self):
-        self.cur_path = BaseDir.get_file_dir_absolute(__file__)
-        self.configfile = BaseFile.get_new_path(self.cur_path, "db.ini")
-        self.logger = get_streamlogger()
-        self.mysqlutils = MysqlUtiles(self.configfile, "dbkuaidaili", self.logger)
-        self.url = "https://dps.kdlapi.com/api/getdps/?orderid=990171566857288&num=15&pt=1&format=json&sep=1"
-        self.bsrequest = BaseRequest()
-        self.starttime = time.time()
-        self.starttime_val = time.time()
-
-    def get_proxy(self):
-        self.starttime = time.time()
-        BoolResult, errString, r = self.bsrequest.base_request(self.url,
-                                                               timeout=30
-                                                               )
-        if BoolResult:
-            dicts = json.loads(r.text)
-            for proxy in dicts["data"]["proxy_list"]:
-                sql = "insert into kuaidailiproxy (proxy) values ('%s') on DUPLICATE key update stat=1" % proxy
-                self.mysqlutils.ExeSqlToDB(sql)
-        else:
-            self.logger.error("获取失败")
-
-    def val(self, proxy):
-        self.starttime_val = time.time()
-        url = f"https://dps.kdlapi.com/api/getdpsvalidtime?orderid=990171566857288&signature=wm4vq53pwrat1vye458elwyxyh9awzqj&proxy={proxy}"
-        BoolResult, errString, r = self.bsrequest.base_request(url,
-                                                               timeout=30
-                                                               )
-        if BoolResult:
-            dicts = json.loads(r.text)
-            if dicts["data"][proxy] > 0:
-                sql = "update kuaidailiproxy set val_stat=0 where proxy='%s'" % proxy
-                self.mysqlutils.ExeSqlToDB(sql)
-            else:
-                sql = "update kuaidailiproxy set val_stat=0,stat=0 where proxy='%s'" % proxy
-                self.mysqlutils.ExeSqlToDB(sql)
-        else:
-            self.logger.error("获取失败")
-
-    def val_all(self):
-        self.starttime_val = time.time()
-        sql = "select proxy from kuaidailiproxy where stat=1"
-        bools, rows = self.mysqlutils.SelectFromDB(sql)
-        for row in rows:
-            try:
-                self.val(row[0])
-            except:
-                traceback.print_exc()
-
-    def run(self):
-        self.get_proxy()
-        while True:
-            if int(time.time() - self.starttime) > 20:
-                self.get_proxy()
-            if int(time.time() - self.starttime_val) > 30:
-                self.val_all()
-            time.sleep(5)
-
-
-if __name__ == "__main__":
-    Kproxy().run()
+import json
+
+###########################################
+# 同项目调用基础包
+import os
+import sys
+import time
+import traceback
+
+filepath = os.path.abspath(__file__)
+pathlist = filepath.split(os.sep)
+pathlist = pathlist[:-4]
+TopPath = os.sep.join(pathlist)
+sys.path.insert(0, TopPath)
+print(TopPath)
+############################################
+
+from re_common.baselibrary.utils.baserequest import BaseRequest
+from re_common.facade.mysqlfacade import MysqlUtiles
+from re_common.baselibrary.utils.basedir import BaseDir
+from re_common.baselibrary.utils.basefile import BaseFile
+from re_common.facade.lazy_import import get_streamlogger
+
+
+class Kproxy(object):
+    def __init__(self):
+        self.cur_path = BaseDir.get_file_dir_absolute(__file__)
+        self.configfile = BaseFile.get_new_path(self.cur_path, "db.ini")
+        self.logger = get_streamlogger()
+        self.mysqlutils = MysqlUtiles(self.configfile, "dbkuaidaili", self.logger)
+        self.url = "https://dps.kdlapi.com/api/getdps/?orderid=990171566857288&num=15&pt=1&format=json&sep=1"
+        self.bsrequest = BaseRequest()
+        self.starttime = time.time()
+        self.starttime_val = time.time()
+
+    def get_proxy(self):
+        self.starttime = time.time()
+        BoolResult, errString, r = self.bsrequest.base_request(self.url,
+                                                               timeout=30
+                                                               )
+        if BoolResult:
+            dicts = json.loads(r.text)
+            for proxy in dicts["data"]["proxy_list"]:
+                sql = "insert into kuaidailiproxy (proxy) values ('%s') on DUPLICATE key update stat=1" % proxy
+                self.mysqlutils.ExeSqlToDB(sql)
+        else:
+            self.logger.error("获取失败")
+
+    def val(self, proxy):
+        self.starttime_val = time.time()
+        url = f"https://dps.kdlapi.com/api/getdpsvalidtime?orderid=990171566857288&signature=wm4vq53pwrat1vye458elwyxyh9awzqj&proxy={proxy}"
+        BoolResult, errString, r = self.bsrequest.base_request(url,
+                                                               timeout=30
+                                                               )
+        if BoolResult:
+            dicts = json.loads(r.text)
+            if dicts["data"][proxy] > 0:
+                sql = "update kuaidailiproxy set val_stat=0 where proxy='%s'" % proxy
+                self.mysqlutils.ExeSqlToDB(sql)
+            else:
+                sql = "update kuaidailiproxy set val_stat=0,stat=0 where proxy='%s'" % proxy
+                self.mysqlutils.ExeSqlToDB(sql)
+        else:
+            self.logger.error("获取失败")
+
+    def val_all(self):
+        self.starttime_val = time.time()
+        sql = "select proxy from kuaidailiproxy where stat=1"
+        bools, rows = self.mysqlutils.SelectFromDB(sql)
+        for row in rows:
+            try:
+                self.val(row[0])
+            except:
+                traceback.print_exc()
+
+    def run(self):
+        self.get_proxy()
+        while True:
+            if int(time.time() - self.starttime) > 20:
+                self.get_proxy()
+            if int(time.time() - self.starttime_val) > 30:
+                self.val_all()
+            time.sleep(5)
+
+
+if __name__ == "__main__":
+    Kproxy().run()