re-common 10.0.37__py3-none-any.whl → 10.0.39__py3-none-any.whl

re_common/vip/proxy/wp_proxy_all.py

@@ -1,182 +1,182 @@
-# -*- coding: utf-8 -*-
-# @Time    : 2020/6/23 14:38
-# @Author  : suhong
-# @File    : wp_proxy_all.py
-# @Software: PyCharm
-
-"""
-云代理 开放代理 刷新验证进入redis (db3 wp_proxy_all)
-"""
-import json
-import random
-import time
-
-from re_common.baselibrary.utils.core.requests_core import USER_AGENTS
-
-from re_common.facade.mysqlfacade import MysqlUtiles
-
-from re_common.baselibrary.mthread.mythreading import ThreadInfo, ThreadVal, ThreadPoolManger
-
-from re_common.baselibrary.mthread.MThreadingRun import MThreadingRun
-
-from re_common.baselibrary.utils.baserequest import BaseRequest
-
-from re_common.baselibrary.utils.myredisclient import MyRedis
-
-
-class ProxyAll(object):
-    def __init__(self, config="./db.ini"):
-        self.config = config
-        self.myredis = MyRedis(configpath=self.config, sesc='wp_proxy',is_conn_or_pipe=False)
-        # self.myredis.get_pipeline()
-        self.myredis.builder()
-
-        self.Headers = {
-            'Accept': '*/*',
-            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64; Trident/7.0; rv:11.0) like Gecko',
-        }
-        self.UserAgent = 'Mozilla/5.0 (Windows NT 10.0; WOW64; Trident/7.0; rv:11.0) like Gecko'
-        self.baserequest = BaseRequest()
-        self.ProxyPoolTotal = set()
-        self.starttime = time.time()
-        self.mysqlutils_proxy = MysqlUtiles('./db.ini', "db_proxy")
-
-    def get_redis_all(self):
-        return self.myredis.getDataFromRedis()
-
-    def get_proxy_ip3366(self, num=600):
-        proxyPool = set()
-        # url = "http://gea.ip3366.net/api/?key=20200622135212736&getnum={}&order=1&formats=2&proxytype=01".format(num)
-        url = "http://gea.ip3366.net/api/?key=20200622135212736&getnum={}&formats=2&proxytype=01".format(num)
-        try:
-            bools, estring, r = self.baserequest.base_request(url,
-                                                              headers=self.Headers,
-                                                              marks=['Ip'])
-            if bools:
-                json_data = json.loads(r.text)
-                for info in json_data:
-                    proxy = info['Ip'] + ":" + str(info['Port'])
-                    proxyPool.add(proxy)
-
-                return True, proxyPool
-
-            return False, proxyPool
-        except Exception as e:
-            print(e)
-            return False, proxyPool
-
-
-class YanzhengThreadRun(MThreadingRun):
-    def __init__(self, num):
-        super(YanzhengThreadRun, self).__init__(num)
-        self.pro = ProxyAll()
-        self.yz_right_set = set()
-        self.is_clean_redis = False
-
-
-    def getTask(self, *args, **kwargs):
-        return []
-
-    def setTask(self, results=None, *args, **kwargs):
-
-        if self.thread_pool.work_queue.is_empty():
-
-            redis_proxypool = self.pro.get_redis_all()
-            self.is_clean_redis = True
-            if time.time() - self.pro.starttime <= 5:
-                time.sleep(6 - (time.time() - self.pro.starttime))
-            bools, proxypool = self.pro.get_proxy_ip3366()
-            result_pro = proxypool.union(redis_proxypool)
-            if bools:
-                for raw in result_pro:
-                    self.add_job(self.func,raw)
-            self.pro.starttime = time.time()
-
-
-    def dealresult(self, *args, **kwargs):
-        if self.is_clean_redis:
-            # 清理reids
-            self.pipe.delete(self.pro.myredis.RedisKey)
-            self.is_clean_redis = False
-
-        # 处理self.yz_right_set集合
-        print('Write DataBase %s ...' % self.pro.myredis.RedisKey)
-        self.pipe.sadd(self.pro.myredis.RedisKey, *self.results)
-        curTime = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime())
-        self.pipe.hset('update_time', self.pro.myredis.RedisKey, curTime)
-        self.pipe.execute()
-        # 插入mysql
-        for raw in self.results:
-            self.deal_sql(raw)
-
-    @ThreadPoolManger.thread_lock
-    def deal_sql(self,raw):
-        # 插入mysql统计每天获取的代理个数
-        sql = "insert into gongwangproxy(proxy,cishu) Values('{}',1) on DUPLICATE key update cishu=cishu+1".format(raw)
-        self.pro.mysqlutils_proxy.ExeSqlToDB(sql)
-
-
-
-    def setProxy(self, proxysList=None):
-        pass
-
-    def is_break(self):
-        return False
-
-    def thread_pool_hook(self, threadinfo: ThreadInfo):
-        # 设置代理线程不重启，默认会重启
-        # if threadinfo.get_thread_name() == self.etn.proxythreadname:
-        #     threadinfo.set_is_restart(True)
-        # if threadinfo.get_thread_name() == self.etn.taskthreadname:
-        #     threadinfo.set_is_restart(False)
-        return {}
-
-    def fun(self, threadval: ThreadVal, *args, **kwargs):
-        """
-        验证代理有效性,百度
-        """
-        raw = args[0]
-        result_queue = threadval.get_result_queue()
-        ppp = {
-            'http': raw,
-            'https': raw
-        }
-        try:
-            url = "https://www.baidu.com/"
-            bools, e, r = self.pro.baserequest.base_request(url,
-                                                            headers=self.pro.Headers,
-                                                            proxies=ppp,
-                                                            timeout=5,
-                                                            marks=['百度一下，你就知道'])
-            if bools:
-                result_queue.put(raw)
-        except Exception as e:
-            print(e)
-
-        # 验证超星期刊
-        # yzurl = "http://qikan.chaoxing.com/mag/infos?mags=ea15bb11cfca2424ae72402ca8461604"
-        # mags = "ea15bb11cfca2424ae72402ca8461604"
-        # HEADER = {
-        #     'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9',
-        #     'User-Agent': random.choice(USER_AGENTS),
-        #     'Accept-Encoding': 'gzip, deflate',
-        #     'Referer': 'http://qikan.chaoxing.com',
-        # }
-        # HEADER['Referer'] = 'http://qikan.chaoxing.com/mag/infos?mags=' + mags
-        # try:
-        #     BoolResult, errString, r = self.pro.baserequest.base_request(url=yzurl,
-        #                                                            headers=HEADER,
-        #                                                            timeout=(5, 10),
-        #                                                            proxies=ppp,
-        #                                                            marks=['Fbookright fl'])
-        #     if BoolResult:
-        #         result_queue.put(raw)
-        # except Exception as e:
-        #     print(e)
-
-
-if __name__ == '__main__':
-    # p = ProxyAll()
-    # p.get_redis_all()
-    yz = YanzhengThreadRun(40)
+# -*- coding: utf-8 -*-
+# @Time    : 2020/6/23 14:38
+# @Author  : suhong
+# @File    : wp_proxy_all.py
+# @Software: PyCharm
+
+"""
+云代理 开放代理 刷新验证进入redis (db3 wp_proxy_all)
+"""
+import json
+import random
+import time
+
+from re_common.baselibrary.utils.core.requests_core import USER_AGENTS
+
+from re_common.facade.mysqlfacade import MysqlUtiles
+
+from re_common.baselibrary.mthread.mythreading import ThreadInfo, ThreadVal, ThreadPoolManger
+
+from re_common.baselibrary.mthread.MThreadingRun import MThreadingRun
+
+from re_common.baselibrary.utils.baserequest import BaseRequest
+
+from re_common.baselibrary.utils.myredisclient import MyRedis
+
+
+class ProxyAll(object):
+    def __init__(self, config="./db.ini"):
+        self.config = config
+        self.myredis = MyRedis(configpath=self.config, sesc='wp_proxy',is_conn_or_pipe=False)
+        # self.myredis.get_pipeline()
+        self.myredis.builder()
+
+        self.Headers = {
+            'Accept': '*/*',
+            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64; Trident/7.0; rv:11.0) like Gecko',
+        }
+        self.UserAgent = 'Mozilla/5.0 (Windows NT 10.0; WOW64; Trident/7.0; rv:11.0) like Gecko'
+        self.baserequest = BaseRequest()
+        self.ProxyPoolTotal = set()
+        self.starttime = time.time()
+        self.mysqlutils_proxy = MysqlUtiles('./db.ini', "db_proxy")
+
+    def get_redis_all(self):
+        return self.myredis.getDataFromRedis()
+
+    def get_proxy_ip3366(self, num=600):
+        proxyPool = set()
+        # url = "http://gea.ip3366.net/api/?key=20200622135212736&getnum={}&order=1&formats=2&proxytype=01".format(num)
+        url = "http://gea.ip3366.net/api/?key=20200622135212736&getnum={}&formats=2&proxytype=01".format(num)
+        try:
+            bools, estring, r = self.baserequest.base_request(url,
+                                                              headers=self.Headers,
+                                                              marks=['Ip'])
+            if bools:
+                json_data = json.loads(r.text)
+                for info in json_data:
+                    proxy = info['Ip'] + ":" + str(info['Port'])
+                    proxyPool.add(proxy)
+
+                return True, proxyPool
+
+            return False, proxyPool
+        except Exception as e:
+            print(e)
+            return False, proxyPool
+
+
+class YanzhengThreadRun(MThreadingRun):
+    def __init__(self, num):
+        super(YanzhengThreadRun, self).__init__(num)
+        self.pro = ProxyAll()
+        self.yz_right_set = set()
+        self.is_clean_redis = False
+
+
+    def getTask(self, *args, **kwargs):
+        return []
+
+    def setTask(self, results=None, *args, **kwargs):
+
+        if self.thread_pool.work_queue.is_empty():
+
+            redis_proxypool = self.pro.get_redis_all()
+            self.is_clean_redis = True
+            if time.time() - self.pro.starttime <= 5:
+                time.sleep(6 - (time.time() - self.pro.starttime))
+            bools, proxypool = self.pro.get_proxy_ip3366()
+            result_pro = proxypool.union(redis_proxypool)
+            if bools:
+                for raw in result_pro:
+                    self.add_job(self.func,raw)
+            self.pro.starttime = time.time()
+
+
+    def dealresult(self, *args, **kwargs):
+        if self.is_clean_redis:
+            # 清理reids
+            self.pipe.delete(self.pro.myredis.RedisKey)
+            self.is_clean_redis = False
+
+        # 处理self.yz_right_set集合
+        print('Write DataBase %s ...' % self.pro.myredis.RedisKey)
+        self.pipe.sadd(self.pro.myredis.RedisKey, *self.results)
+        curTime = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime())
+        self.pipe.hset('update_time', self.pro.myredis.RedisKey, curTime)
+        self.pipe.execute()
+        # 插入mysql
+        for raw in self.results:
+            self.deal_sql(raw)
+
+    @ThreadPoolManger.thread_lock
+    def deal_sql(self,raw):
+        # 插入mysql统计每天获取的代理个数
+        sql = "insert into gongwangproxy(proxy,cishu) Values('{}',1) on DUPLICATE key update cishu=cishu+1".format(raw)
+        self.pro.mysqlutils_proxy.ExeSqlToDB(sql)
+
+
+
+    def setProxy(self, proxysList=None):
+        pass
+
+    def is_break(self):
+        return False
+
+    def thread_pool_hook(self, threadinfo: ThreadInfo):
+        # 设置代理线程不重启，默认会重启
+        # if threadinfo.get_thread_name() == self.etn.proxythreadname:
+        #     threadinfo.set_is_restart(True)
+        # if threadinfo.get_thread_name() == self.etn.taskthreadname:
+        #     threadinfo.set_is_restart(False)
+        return {}
+
+    def fun(self, threadval: ThreadVal, *args, **kwargs):
+        """
+        验证代理有效性,百度
+        """
+        raw = args[0]
+        result_queue = threadval.get_result_queue()
+        ppp = {
+            'http': raw,
+            'https': raw
+        }
+        try:
+            url = "https://www.baidu.com/"
+            bools, e, r = self.pro.baserequest.base_request(url,
+                                                            headers=self.pro.Headers,
+                                                            proxies=ppp,
+                                                            timeout=5,
+                                                            marks=['百度一下，你就知道'])
+            if bools:
+                result_queue.put(raw)
+        except Exception as e:
+            print(e)
+
+        # 验证超星期刊
+        # yzurl = "http://qikan.chaoxing.com/mag/infos?mags=ea15bb11cfca2424ae72402ca8461604"
+        # mags = "ea15bb11cfca2424ae72402ca8461604"
+        # HEADER = {
+        #     'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9',
+        #     'User-Agent': random.choice(USER_AGENTS),
+        #     'Accept-Encoding': 'gzip, deflate',
+        #     'Referer': 'http://qikan.chaoxing.com',
+        # }
+        # HEADER['Referer'] = 'http://qikan.chaoxing.com/mag/infos?mags=' + mags
+        # try:
+        #     BoolResult, errString, r = self.pro.baserequest.base_request(url=yzurl,
+        #                                                            headers=HEADER,
+        #                                                            timeout=(5, 10),
+        #                                                            proxies=ppp,
+        #                                                            marks=['Fbookright fl'])
+        #     if BoolResult:
+        #         result_queue.put(raw)
+        # except Exception as e:
+        #     print(e)
+
+
+if __name__ == '__main__':
+    # p = ProxyAll()
+    # p.get_redis_all()
+    yz = YanzhengThreadRun(40)
     yz.run()

re_common/vip/read_rawid_to_txt.py

@@ -1,92 +1,92 @@
-from re_common.baselibrary.utils.basefile import BaseFile
-
-from re_common.facade.mysqlfacade import MysqlUtiles
-
-
-def get_rawid_from_mysql():
-    strings = """
-            host = 192.168.31.209
-            user = root
-            passwd = vipdatacenter
-            db = cnki_qk
-            port = 3306
-            chartset = utf8
-            """
-    mysqlrawid = r"F:\fun2\mysqlrawid.txt"
-    feilds = "filename"
-
-    dicts_change = {"key为原来的": "values为现在的"}
-
-    from re_common.baselibrary.tools.stringtodicts import StringToDicts
-
-    dicts = StringToDicts().string_to_dicts_by_equal(strings)
-    mysqlutils = MysqlUtiles("", "", builder="MysqlBuilderForDicts", dicts=dicts)
-    offset = 0
-    limit = 1000000
-    while True:
-        bools, raws = mysqlutils.SelectFromDB("select `{}` from article limit {},{}".format(feilds, offset, limit))
-        if len(raws) == 0:
-            break
-        with open(mysqlrawid, 'w', encoding="utf-8") as f:
-            for row in raws:
-                f.write(row[0] + "\n")
-        offset = offset + limit
-
-
-# get_rawid_from_mysql()
-# print(BaseFile.get_file_line_num(r'C:\Users\xuzhu\Downloads\part-r-00000'))
-
-def read_files_compose():
-    """
-    读取两个文件对比
-    :return:
-    """
-    sets1 = set()
-    sets2 = set()
-    for value in BaseFile.read_file_r_mode_yield(r'C:\Users\xuzhu\Downloads\part-r-00000'):
-        sets2.add(value)
-
-    for value in BaseFile.read_file_r_mode_yield(r'F:\fun2\mysqlrawid.txt'):
-        sets1.add(value)
-    set3 = sets1 - sets2
-    cha = r"F:\fun2\cha.txt"
-    with open(cha, 'w', encoding="utf-8") as f:
-        for row in set3:
-            f.write(row + "\n")
-
-
-#read_files_compose()
-
-def set_stat():
-    strings = """
-               host = 192.168.31.209
-               user = root
-               passwd = vipdatacenter
-               db = cnki_qk
-               port = 3306
-               chartset = utf8
-               """
-    charawid = r"F:\fun2\cha.txt"
-
-    dicts_change = {"key为原来的": "values为现在的"}
-
-    from re_common.baselibrary.tools.stringtodicts import StringToDicts
-
-    dicts = StringToDicts().string_to_dicts_by_equal(strings)
-    mysqlutils = MysqlUtiles("", "", builder="MysqlBuilderForDicts", dicts=dicts)
-    lists = []
-    for values in BaseFile.read_file_r_mode_yield(charawid):
-        lists.append(values)
-        if len(lists) > 100000:
-            sql = "update article set stat=0,ref_stat=0 where filename in {}".format(tuple(lists))
-            mysqlutils.ExeSqlToDB(sql)
-            lists.clear()
-
-    if len(lists) > 0:
-        if len(lists) == 1:
-            lists.append("test")
-        sql = "update article set stat=0,ref_stat=0 where filename in {}".format(tuple(lists))
-        mysqlutils.ExeSqlToDB(sql)
-        lists.clear()
-
+from re_common.baselibrary.utils.basefile import BaseFile
+
+from re_common.facade.mysqlfacade import MysqlUtiles
+
+
+def get_rawid_from_mysql():
+    strings = """
+            host = 192.168.31.209
+            user = root
+            passwd = vipdatacenter
+            db = cnki_qk
+            port = 3306
+            chartset = utf8
+            """
+    mysqlrawid = r"F:\fun2\mysqlrawid.txt"
+    feilds = "filename"
+
+    dicts_change = {"key为原来的": "values为现在的"}
+
+    from re_common.baselibrary.tools.stringtodicts import StringToDicts
+
+    dicts = StringToDicts().string_to_dicts_by_equal(strings)
+    mysqlutils = MysqlUtiles("", "", builder="MysqlBuilderForDicts", dicts=dicts)
+    offset = 0
+    limit = 1000000
+    while True:
+        bools, raws = mysqlutils.SelectFromDB("select `{}` from article limit {},{}".format(feilds, offset, limit))
+        if len(raws) == 0:
+            break
+        with open(mysqlrawid, 'w', encoding="utf-8") as f:
+            for row in raws:
+                f.write(row[0] + "\n")
+        offset = offset + limit
+
+
+# get_rawid_from_mysql()
+# print(BaseFile.get_file_line_num(r'C:\Users\xuzhu\Downloads\part-r-00000'))
+
+def read_files_compose():
+    """
+    读取两个文件对比
+    :return:
+    """
+    sets1 = set()
+    sets2 = set()
+    for value in BaseFile.read_file_r_mode_yield(r'C:\Users\xuzhu\Downloads\part-r-00000'):
+        sets2.add(value)
+
+    for value in BaseFile.read_file_r_mode_yield(r'F:\fun2\mysqlrawid.txt'):
+        sets1.add(value)
+    set3 = sets1 - sets2
+    cha = r"F:\fun2\cha.txt"
+    with open(cha, 'w', encoding="utf-8") as f:
+        for row in set3:
+            f.write(row + "\n")
+
+
+#read_files_compose()
+
+def set_stat():
+    strings = """
+               host = 192.168.31.209
+               user = root
+               passwd = vipdatacenter
+               db = cnki_qk
+               port = 3306
+               chartset = utf8
+               """
+    charawid = r"F:\fun2\cha.txt"
+
+    dicts_change = {"key为原来的": "values为现在的"}
+
+    from re_common.baselibrary.tools.stringtodicts import StringToDicts
+
+    dicts = StringToDicts().string_to_dicts_by_equal(strings)
+    mysqlutils = MysqlUtiles("", "", builder="MysqlBuilderForDicts", dicts=dicts)
+    lists = []
+    for values in BaseFile.read_file_r_mode_yield(charawid):
+        lists.append(values)
+        if len(lists) > 100000:
+            sql = "update article set stat=0,ref_stat=0 where filename in {}".format(tuple(lists))
+            mysqlutils.ExeSqlToDB(sql)
+            lists.clear()
+
+    if len(lists) > 0:
+        if len(lists) == 1:
+            lists.append("test")
+        sql = "update article set stat=0,ref_stat=0 where filename in {}".format(tuple(lists))
+        mysqlutils.ExeSqlToDB(sql)
+        lists.clear()
+
 set_stat()

re_common/vip/title/__init__.py

@@ -1,5 +1,5 @@
-# -*- coding:utf-8 -*-
-# @Time : 2021/12/2 9:38
-# @Author: suhong
-# @File : __init__.py.py
-# @Function :
+# -*- coding:utf-8 -*-
+# @Time : 2021/12/2 9:38
+# @Author: suhong
+# @File : __init__.py.py
+# @Function :