re-common 2.0.1__py3-none-any.whl → 10.0.0__py3-none-any.whl

re_common/vip/proxy/wp_proxy_all.py

@@ -0,0 +1,182 @@
+# -*- coding: utf-8 -*-
+# @Time    : 2020/6/23 14:38
+# @Author  : suhong
+# @File    : wp_proxy_all.py
+# @Software: PyCharm
+
+"""
+云代理 开放代理 刷新验证进入redis (db3 wp_proxy_all)
+"""
+import json
+import random
+import time
+
+from re_common.baselibrary.utils.core.requests_core import USER_AGENTS
+
+from re_common.facade.mysqlfacade import MysqlUtiles
+
+from re_common.baselibrary.mthread.mythreading import ThreadInfo, ThreadVal, ThreadPoolManger
+
+from re_common.baselibrary.mthread.MThreadingRun import MThreadingRun
+
+from re_common.baselibrary.utils.baserequest import BaseRequest
+
+from re_common.baselibrary.utils.myredisclient import MyRedis
+
+
+class ProxyAll(object):
+    def __init__(self, config="./db.ini"):
+        self.config = config
+        self.myredis = MyRedis(configpath=self.config, sesc='wp_proxy',is_conn_or_pipe=False)
+        # self.myredis.get_pipeline()
+        self.myredis.builder()
+
+        self.Headers = {
+            'Accept': '*/*',
+            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64; Trident/7.0; rv:11.0) like Gecko',
+        }
+        self.UserAgent = 'Mozilla/5.0 (Windows NT 10.0; WOW64; Trident/7.0; rv:11.0) like Gecko'
+        self.baserequest = BaseRequest()
+        self.ProxyPoolTotal = set()
+        self.starttime = time.time()
+        self.mysqlutils_proxy = MysqlUtiles('./db.ini', "db_proxy")
+
+    def get_redis_all(self):
+        return self.myredis.getDataFromRedis()
+
+    def get_proxy_ip3366(self, num=600):
+        proxyPool = set()
+        # url = "http://gea.ip3366.net/api/?key=20200622135212736&getnum={}&order=1&formats=2&proxytype=01".format(num)
+        url = "http://gea.ip3366.net/api/?key=20200622135212736&getnum={}&formats=2&proxytype=01".format(num)
+        try:
+            bools, estring, r = self.baserequest.base_request(url,
+                                                              headers=self.Headers,
+                                                              marks=['Ip'])
+            if bools:
+                json_data = json.loads(r.text)
+                for info in json_data:
+                    proxy = info['Ip'] + ":" + str(info['Port'])
+                    proxyPool.add(proxy)
+
+                return True, proxyPool
+
+            return False, proxyPool
+        except Exception as e:
+            print(e)
+            return False, proxyPool
+
+
+class YanzhengThreadRun(MThreadingRun):
+    def __init__(self, num):
+        super(YanzhengThreadRun, self).__init__(num)
+        self.pro = ProxyAll()
+        self.yz_right_set = set()
+        self.is_clean_redis = False
+
+
+    def getTask(self, *args, **kwargs):
+        return []
+
+    def setTask(self, results=None, *args, **kwargs):
+
+        if self.thread_pool.work_queue.is_empty():
+
+            redis_proxypool = self.pro.get_redis_all()
+            self.is_clean_redis = True
+            if time.time() - self.pro.starttime <= 5:
+                time.sleep(6 - (time.time() - self.pro.starttime))
+            bools, proxypool = self.pro.get_proxy_ip3366()
+            result_pro = proxypool.union(redis_proxypool)
+            if bools:
+                for raw in result_pro:
+                    self.add_job(self.func,raw)
+            self.pro.starttime = time.time()
+
+
+    def dealresult(self, *args, **kwargs):
+        if self.is_clean_redis:
+            # 清理reids
+            self.pipe.delete(self.pro.myredis.RedisKey)
+            self.is_clean_redis = False
+
+        # 处理self.yz_right_set集合
+        print('Write DataBase %s ...' % self.pro.myredis.RedisKey)
+        self.pipe.sadd(self.pro.myredis.RedisKey, *self.results)
+        curTime = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime())
+        self.pipe.hset('update_time', self.pro.myredis.RedisKey, curTime)
+        self.pipe.execute()
+        # 插入mysql
+        for raw in self.results:
+            self.deal_sql(raw)
+
+    @ThreadPoolManger.thread_lock
+    def deal_sql(self,raw):
+        # 插入mysql统计每天获取的代理个数
+        sql = "insert into gongwangproxy(proxy,cishu) Values('{}',1) on DUPLICATE key update cishu=cishu+1".format(raw)
+        self.pro.mysqlutils_proxy.ExeSqlToDB(sql)
+
+
+
+    def setProxy(self, proxysList=None):
+        pass
+
+    def is_break(self):
+        return False
+
+    def thread_pool_hook(self, threadinfo: ThreadInfo):
+        # 设置代理线程不重启，默认会重启
+        # if threadinfo.get_thread_name() == self.etn.proxythreadname:
+        #     threadinfo.set_is_restart(True)
+        # if threadinfo.get_thread_name() == self.etn.taskthreadname:
+        #     threadinfo.set_is_restart(False)
+        return {}
+
+    def fun(self, threadval: ThreadVal, *args, **kwargs):
+        """
+        验证代理有效性,百度
+        """
+        raw = args[0]
+        result_queue = threadval.get_result_queue()
+        ppp = {
+            'http': raw,
+            'https': raw
+        }
+        try:
+            url = "https://www.baidu.com/"
+            bools, e, r = self.pro.baserequest.base_request(url,
+                                                            headers=self.pro.Headers,
+                                                            proxies=ppp,
+                                                            timeout=5,
+                                                            marks=['百度一下，你就知道'])
+            if bools:
+                result_queue.put(raw)
+        except Exception as e:
+            print(e)
+
+        # 验证超星期刊
+        # yzurl = "http://qikan.chaoxing.com/mag/infos?mags=ea15bb11cfca2424ae72402ca8461604"
+        # mags = "ea15bb11cfca2424ae72402ca8461604"
+        # HEADER = {
+        #     'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9',
+        #     'User-Agent': random.choice(USER_AGENTS),
+        #     'Accept-Encoding': 'gzip, deflate',
+        #     'Referer': 'http://qikan.chaoxing.com',
+        # }
+        # HEADER['Referer'] = 'http://qikan.chaoxing.com/mag/infos?mags=' + mags
+        # try:
+        #     BoolResult, errString, r = self.pro.baserequest.base_request(url=yzurl,
+        #                                                            headers=HEADER,
+        #                                                            timeout=(5, 10),
+        #                                                            proxies=ppp,
+        #                                                            marks=['Fbookright fl'])
+        #     if BoolResult:
+        #         result_queue.put(raw)
+        # except Exception as e:
+        #     print(e)
+
+
+if __name__ == '__main__':
+    # p = ProxyAll()
+    # p.get_redis_all()
+    yz = YanzhengThreadRun(40)
+    yz.run()

re_common/vip/read_rawid_to_txt.py

@@ -0,0 +1,92 @@
+from re_common.baselibrary.utils.basefile import BaseFile
+
+from re_common.facade.mysqlfacade import MysqlUtiles
+
+
+def get_rawid_from_mysql():
+    strings = """
+            host = 192.168.31.209
+            user = root
+            passwd = vipdatacenter
+            db = cnki_qk
+            port = 3306
+            chartset = utf8
+            """
+    mysqlrawid = r"F:\fun2\mysqlrawid.txt"
+    feilds = "filename"
+
+    dicts_change = {"key为原来的": "values为现在的"}
+
+    from re_common.baselibrary.tools.stringtodicts import StringToDicts
+
+    dicts = StringToDicts().string_to_dicts_by_equal(strings)
+    mysqlutils = MysqlUtiles("", "", builder="MysqlBuilderForDicts", dicts=dicts)
+    offset = 0
+    limit = 1000000
+    while True:
+        bools, raws = mysqlutils.SelectFromDB("select `{}` from article limit {},{}".format(feilds, offset, limit))
+        if len(raws) == 0:
+            break
+        with open(mysqlrawid, 'w', encoding="utf-8") as f:
+            for row in raws:
+                f.write(row[0] + "\n")
+        offset = offset + limit
+
+
+# get_rawid_from_mysql()
+# print(BaseFile.get_file_line_num(r'C:\Users\xuzhu\Downloads\part-r-00000'))
+
+def read_files_compose():
+    """
+    读取两个文件对比
+    :return:
+    """
+    sets1 = set()
+    sets2 = set()
+    for value in BaseFile.read_file_r_mode_yield(r'C:\Users\xuzhu\Downloads\part-r-00000'):
+        sets2.add(value)
+
+    for value in BaseFile.read_file_r_mode_yield(r'F:\fun2\mysqlrawid.txt'):
+        sets1.add(value)
+    set3 = sets1 - sets2
+    cha = r"F:\fun2\cha.txt"
+    with open(cha, 'w', encoding="utf-8") as f:
+        for row in set3:
+            f.write(row + "\n")
+
+
+#read_files_compose()
+
+def set_stat():
+    strings = """
+               host = 192.168.31.209
+               user = root
+               passwd = vipdatacenter
+               db = cnki_qk
+               port = 3306
+               chartset = utf8
+               """
+    charawid = r"F:\fun2\cha.txt"
+
+    dicts_change = {"key为原来的": "values为现在的"}
+
+    from re_common.baselibrary.tools.stringtodicts import StringToDicts
+
+    dicts = StringToDicts().string_to_dicts_by_equal(strings)
+    mysqlutils = MysqlUtiles("", "", builder="MysqlBuilderForDicts", dicts=dicts)
+    lists = []
+    for values in BaseFile.read_file_r_mode_yield(charawid):
+        lists.append(values)
+        if len(lists) > 100000:
+            sql = "update article set stat=0,ref_stat=0 where filename in {}".format(tuple(lists))
+            mysqlutils.ExeSqlToDB(sql)
+            lists.clear()
+
+    if len(lists) > 0:
+        if len(lists) == 1:
+            lists.append("test")
+        sql = "update article set stat=0,ref_stat=0 where filename in {}".format(tuple(lists))
+        mysqlutils.ExeSqlToDB(sql)
+        lists.clear()
+
+set_stat()

re_common/vip/title/__init__.py

@@ -0,0 +1,5 @@
+# -*- coding:utf-8 -*-
+# @Time : 2021/12/2 9:38
+# @Author: suhong
+# @File : __init__.py.py
+# @Function :

re_common/vip/title/transform/TransformBookTitleToZt.py

@@ -0,0 +1,125 @@
+# -*- coding:utf-8 -*-
+# @Time : 2021/12/19 18:16
+# @Author: suhong
+# @File : TransformBookTitleToZt.py
+# @Function : 图书a层转智图
+from re_common.baselibrary.utils.basetime import BaseTime
+
+from re_common.facade.mysqlfacade import MysqlUtiles
+
+
+class TransformBookTitleToZt():
+    def __init__(self):
+        self.fields = [
+            "lngid",
+            "rawid",
+            "title",
+            "title_alternative",
+            "title_sub",
+            "title_edition",
+            "title_series",
+            "identifier_eisbn",
+            "identifier_pisbn",
+            "identifier_doi",
+            "creator",
+            "creator_en",
+            "creator_bio",
+            "creator_institution",
+            "publisher",
+            "date",
+            "description",
+            "description_en",
+            "description_unit",
+            "subject",
+            "subject_en",
+            "subject_clc",
+            "subject_esc",
+            "page",
+            "beginpage",
+            "endpage",
+            "pagecount",
+            "date_created",
+            "rawtype",
+            "folio_size",
+            "price",
+            "pub_place",
+            "language",
+            "country",
+            "type",
+            "provider",
+            "provider_url",
+            "provider_id",
+            "medium",
+            "batch",
+            "is_deprecated"
+        ]
+        self.zt_providermap = dict()
+        self.mysqlutils = MysqlUtiles(None, None, builder="MysqlBuilderForDicts", dicts={
+            "host": "192.168.31.24",
+            "user": "root",
+            "passwd": "vipdatacenter",
+            "db": "data_warehouse_sql",
+            "port": "3306",
+            "chartset": "utf8mb4",
+        })
+        self.get_zt_provider()
+
+    def get_zt_provider(self):
+        rows = self.mysqlutils.SelectFromDB(
+            "select sub_db_id,provider from a_transform_task where source_type = '1' and out_type = 'zt'")
+        for row in rows[1]:
+            self.zt_providermap[row[0]] = row[1]
+
+    def transform(self, titleMap):
+        transMap = dict()
+        sub_db_id = titleMap.get("sub_db_id", "")
+        transMap["lngid"] = titleMap.get("lngid", "")
+        transMap["rawid"] = titleMap.get("rawid", "")
+        transMap["title"] = titleMap.get("title", "")
+        transMap["title_alternative"] = titleMap.get("title_alt", "")
+        transMap["title_series"] = titleMap.get("title_series", "")
+        transMap["title_edition"] = titleMap.get("revision", "")
+        transMap["identifier_eisbn"] = titleMap.get("eisbn", "")
+        transMap["identifier_pisbn"] = titleMap.get("isbn", "")
+        transMap["identifier_doi"] = titleMap.get("doi", "")
+        transMap["creator"] = titleMap.get("author", "")
+        transMap["creator_en"] = titleMap.get("author_alt", "")
+        transMap["creator_institution"] = titleMap.get("organ", "")
+        transMap["creator_bio"] = titleMap.get("author_intro", "")
+        transMap["publisher"] = titleMap.get("publisher", "")
+        transMap["date"] = titleMap.get("pub_year", "")
+        transMap["description"] = titleMap.get("abstract", "")
+        transMap["description_en"] = titleMap.get("abstract_alt", "")
+        transMap["description_unit"] = titleMap.get("catalog", "")
+        transMap["subject"] = titleMap.get("keyword", "")
+        transMap["subject_en"] = titleMap.get("keyword_alt", "")
+        transMap["subject_clc"] = titleMap.get("clc_no", "")
+        transMap["subject_esc"] = titleMap.get("subject_edu", "")
+        transMap["page"] = titleMap.get("page_info", "")
+        transMap["beginpage"] = titleMap.get("begin_page", "")
+        transMap["endpage"] = titleMap.get("end_page", "")
+        transMap["pagecount"] = titleMap.get("page_cnt", "")
+        transMap["date_created"] = titleMap.get("pub_date", "")
+        transMap["rawtype"] = titleMap.get("raw_type", "")
+        transMap["folio_size"] = titleMap.get("book_size", "")
+        transMap["price"] = titleMap.get("price", "")
+        transMap["pub_place"] = titleMap.get("pub_place", "")
+        # transMap["provider"] = titleMap.get("zt_provider", "")
+        # if transMap["provider"] == "":
+        transMap["provider"] = self.zt_providermap[sub_db_id]
+        transMap["provider_url"] = transMap["provider"] + "@" + titleMap.get("provider_url")
+        transMap["provider_id"] = transMap["provider"] + "@" + titleMap.get("rawid")
+        transMap["batch"] = BaseTime().get_beijin_date_strins("%Y%m%d") + "00"
+        transMap["type"] = titleMap.get("source_type", "")
+        transMap["medium"] = "2"
+        transMap["country"] = titleMap.get("country", "")
+        transMap["language"] = titleMap.get("language", "")
+        transMap["is_deprecated"] = titleMap.get("is_deprecated", "")
+
+        for field in self.fields:
+            if field not in transMap.keys():
+                transMap[field] = ""
+        return transMap
+
+if __name__ == '__main__':
+    t = TransformBookTitleToZt()

re_common/vip/title/transform/TransformConferenceTitleToZt.py

@@ -0,0 +1,139 @@
+# -*- coding:utf-8 -*-
+# @Time : 2021/12/15 14:09
+# @Author: suhong
+# @File : TransformConferenceTitleToZt.py
+# @Function :转换会议a层到智图代码
+from re_common.baselibrary.utils.basetime import BaseTime
+
+from re_common.facade.mysqlfacade import MysqlUtiles
+
+
+class TransformConferenceTitleToZt():
+    def __init__(self):
+        # 初始化fields
+        self.fields = [
+            "lngid",
+            "rawid",
+            "title",
+            "title_alternative",
+            "title_series",
+            "title_edition",
+            "applicant",
+            "identifier_doi",
+            "creator",
+            "creator_en",
+            "creator_institution",
+            "creator_bio",
+            "creator_drafting",
+            "creator_release",
+            "source",
+            "source_en",
+            "source_institution",
+            "publisher",
+            "date",
+            "description",
+            "description_en",
+            "description_fund",
+            "subject",
+            "subject_en",
+            "page",
+            "beginpage",
+            "endpage",
+            "jumppage",
+            "pagecount",
+            "subject_clc",
+            "subject_esc",
+            "date_created",
+            "pub_place",
+            "language",
+            "country",
+            "type",
+            "provider",
+            "provider_url",
+            "provider_id",
+            "medium",
+            "batch",
+            "is_deprecated",
+            "if_pdf_fulltext",
+            "if_html_fulltext"
+        ]
+        self.zt_providermap = dict()
+        self.mysqlutils = MysqlUtiles(None, None, builder="MysqlBuilderForDicts", dicts={
+            "host": "192.168.31.24",
+            "user": "root",
+            "passwd": "vipdatacenter",
+            "db": "data_warehouse_sql",
+            "port": "3306",
+            "chartset": "utf8mb4",
+        })
+        self.get_zt_provider()
+
+    def get_zt_provider(self):
+        rows = self.mysqlutils.SelectFromDB(
+            "select sub_db_id,provider from a_transform_task where source_type = '6' and out_type = 'zt'")
+        for row in rows[1]:
+            self.zt_providermap[row[0]] = row[1]
+
+    def transform(self, titleMap):
+        transMap = dict()
+        sub_db_id = titleMap.get("sub_db_id", "")
+        transMap["lngid"] = titleMap.get("lngid", "")
+        transMap["rawid"] = titleMap.get("rawid", "")
+        transMap["title"] = titleMap.get("title", "")
+        transMap["title_alternative"] = titleMap.get("title_alt", "")
+        transMap["title_series"] = titleMap.get("meeting_record_name", "")
+        transMap["title_edition"] = titleMap.get("revision", "")
+        transMap["applicant"] = titleMap.get("applicant", "")
+        transMap["identifier_doi"] = titleMap.get("doi", "")
+        transMap["creator"] = titleMap.get("author", "")
+        transMap["creator_en"] = titleMap.get("author_alt", "")
+        transMap["creator_institution"] = titleMap.get("organ", "")
+        transMap["creator_bio"] = titleMap.get("author_intro", "")
+        transMap["creator_drafting"] = titleMap.get("society", "")
+        transMap["creator_release"] = titleMap.get("host_organ", "")
+        transMap["source"] = titleMap.get("meeting_name", "")
+        transMap["source_en"] = titleMap.get("meeting_name_alt", "")
+        transMap["source_institution"] = titleMap.get("meeting_place", "")
+        transMap["publisher"] = titleMap.get("publisher", "")
+        transMap["date"] = titleMap.get("pub_year", "")
+        transMap["description"] = titleMap.get("abstract", "")
+        transMap["description_en"] = titleMap.get("abstract_alt", "")
+        transMap["description_fund"] = titleMap.get("fund", "")
+        transMap["subject"] = titleMap.get("keyword", "")
+        transMap["subject_en"] = titleMap.get("keyword_alt", "")
+        transMap["page"] = titleMap.get("page_info", "")
+        transMap["beginpage"] = titleMap.get("begin_page", "")
+        transMap["endpage"] = titleMap.get("end_page", "")
+        transMap["jumppage"] = titleMap.get("jump_page", "")
+        transMap["pagecount"] = titleMap.get("page_cnt", "")
+        transMap["subject_clc"] = titleMap.get("clc_no", "")
+        transMap["subject_esc"] = titleMap.get("subject_edu", "")
+        transMap["date_created"] = titleMap.get("pub_date", "")
+        transMap["pub_place"] = titleMap.get("pub_place", "")
+        # transMap["provider"] = titleMap.get("zt_provider", "")
+        # if transMap["provider"] == "":
+        transMap["provider"] = self.zt_providermap[sub_db_id]
+        transMap["provider_url"] = transMap["provider"] + "@" + titleMap.get("provider_url")
+        transMap["provider_id"] = transMap["provider"] + "@" + titleMap.get("rawid")
+        transMap["batch"] = BaseTime().get_beijin_date_strins("%Y%m%d") + "00"
+        transMap["type"] = titleMap.get("source_type", "")
+        transMap["medium"] = "2"
+        transMap["country"] = titleMap.get("country", "")
+        transMap["language"] = titleMap.get("language", "")
+        fulltext_type = titleMap.get("fulltext_type", "")
+        transMap["if_pdf_fulltext"] = "0"
+        transMap["if_html_fulltext"] = "0"
+        if "pdf" in fulltext_type:
+            transMap["if_pdf_fulltext"] = "1"
+        if "html" in fulltext_type:
+            transMap["if_html_fulltext"] = "1"
+        transMap["is_deprecated"] = titleMap.get("is_deprecated", "")
+
+        for field in self.fields:
+            if field not in transMap.keys():
+                transMap[field] = ""
+        return transMap
+
+
+if __name__ == '__main__':
+    t = TransformConferenceTitleToZt()