re-common 0.2.54__py3-none-any.whl → 2.0.1__py3-none-any.whl

re_common/vip/proxy/wp_proxy_all.py

@@ -1,182 +0,0 @@
-# -*- coding: utf-8 -*-
-# @Time    : 2020/6/23 14:38
-# @Author  : suhong
-# @File    : wp_proxy_all.py
-# @Software: PyCharm
-
-"""
-云代理 开放代理 刷新验证进入redis (db3 wp_proxy_all)
-"""
-import json
-import random
-import time
-
-from re_common.baselibrary.utils.core.requests_core import USER_AGENTS
-
-from re_common.facade.mysqlfacade import MysqlUtiles
-
-from re_common.baselibrary.mthread.mythreading import ThreadInfo, ThreadVal, ThreadPoolManger
-
-from re_common.baselibrary.mthread.MThreadingRun import MThreadingRun
-
-from re_common.baselibrary.utils.baserequest import BaseRequest
-
-from re_common.baselibrary.utils.myredisclient import MyRedis
-
-
-class ProxyAll(object):
-    def __init__(self, config="./db.ini"):
-        self.config = config
-        self.myredis = MyRedis(configpath=self.config, sesc='wp_proxy',is_conn_or_pipe=False)
-        # self.myredis.get_pipeline()
-        self.myredis.builder()
-
-        self.Headers = {
-            'Accept': '*/*',
-            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64; Trident/7.0; rv:11.0) like Gecko',
-        }
-        self.UserAgent = 'Mozilla/5.0 (Windows NT 10.0; WOW64; Trident/7.0; rv:11.0) like Gecko'
-        self.baserequest = BaseRequest()
-        self.ProxyPoolTotal = set()
-        self.starttime = time.time()
-        self.mysqlutils_proxy = MysqlUtiles('./db.ini', "db_proxy")
-
-    def get_redis_all(self):
-        return self.myredis.getDataFromRedis()
-
-    def get_proxy_ip3366(self, num=600):
-        proxyPool = set()
-        # url = "http://gea.ip3366.net/api/?key=20200622135212736&getnum={}&order=1&formats=2&proxytype=01".format(num)
-        url = "http://gea.ip3366.net/api/?key=20200622135212736&getnum={}&formats=2&proxytype=01".format(num)
-        try:
-            bools, estring, r = self.baserequest.base_request(url,
-                                                              headers=self.Headers,
-                                                              marks=['Ip'])
-            if bools:
-                json_data = json.loads(r.text)
-                for info in json_data:
-                    proxy = info['Ip'] + ":" + str(info['Port'])
-                    proxyPool.add(proxy)
-
-                return True, proxyPool
-
-            return False, proxyPool
-        except Exception as e:
-            print(e)
-            return False, proxyPool
-
-
-class YanzhengThreadRun(MThreadingRun):
-    def __init__(self, num):
-        super(YanzhengThreadRun, self).__init__(num)
-        self.pro = ProxyAll()
-        self.yz_right_set = set()
-        self.is_clean_redis = False
-
-
-    def getTask(self, *args, **kwargs):
-        return []
-
-    def setTask(self, results=None, *args, **kwargs):
-
-        if self.thread_pool.work_queue.is_empty():
-
-            redis_proxypool = self.pro.get_redis_all()
-            self.is_clean_redis = True
-            if time.time() - self.pro.starttime <= 5:
-                time.sleep(6 - (time.time() - self.pro.starttime))
-            bools, proxypool = self.pro.get_proxy_ip3366()
-            result_pro = proxypool.union(redis_proxypool)
-            if bools:
-                for raw in result_pro:
-                    self.add_job(self.func,raw)
-            self.pro.starttime = time.time()
-
-
-    def dealresult(self, *args, **kwargs):
-        if self.is_clean_redis:
-            # 清理reids
-            self.pipe.delete(self.pro.myredis.RedisKey)
-            self.is_clean_redis = False
-
-        # 处理self.yz_right_set集合
-        print('Write DataBase %s ...' % self.pro.myredis.RedisKey)
-        self.pipe.sadd(self.pro.myredis.RedisKey, *self.results)
-        curTime = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime())
-        self.pipe.hset('update_time', self.pro.myredis.RedisKey, curTime)
-        self.pipe.execute()
-        # 插入mysql
-        for raw in self.results:
-            self.deal_sql(raw)
-
-    @ThreadPoolManger.thread_lock
-    def deal_sql(self,raw):
-        # 插入mysql统计每天获取的代理个数
-        sql = "insert into gongwangproxy(proxy,cishu) Values('{}',1) on DUPLICATE key update cishu=cishu+1".format(raw)
-        self.pro.mysqlutils_proxy.ExeSqlToDB(sql)
-
-
-
-    def setProxy(self, proxysList=None):
-        pass
-
-    def is_break(self):
-        return False
-
-    def thread_pool_hook(self, threadinfo: ThreadInfo):
-        # 设置代理线程不重启，默认会重启
-        # if threadinfo.get_thread_name() == self.etn.proxythreadname:
-        #     threadinfo.set_is_restart(True)
-        # if threadinfo.get_thread_name() == self.etn.taskthreadname:
-        #     threadinfo.set_is_restart(False)
-        return {}
-
-    def fun(self, threadval: ThreadVal, *args, **kwargs):
-        """
-        验证代理有效性,百度
-        """
-        raw = args[0]
-        result_queue = threadval.get_result_queue()
-        ppp = {
-            'http': raw,
-            'https': raw
-        }
-        try:
-            url = "https://www.baidu.com/"
-            bools, e, r = self.pro.baserequest.base_request(url,
-                                                            headers=self.pro.Headers,
-                                                            proxies=ppp,
-                                                            timeout=5,
-                                                            marks=['百度一下，你就知道'])
-            if bools:
-                result_queue.put(raw)
-        except Exception as e:
-            print(e)
-
-        # 验证超星期刊
-        # yzurl = "http://qikan.chaoxing.com/mag/infos?mags=ea15bb11cfca2424ae72402ca8461604"
-        # mags = "ea15bb11cfca2424ae72402ca8461604"
-        # HEADER = {
-        #     'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9',
-        #     'User-Agent': random.choice(USER_AGENTS),
-        #     'Accept-Encoding': 'gzip, deflate',
-        #     'Referer': 'http://qikan.chaoxing.com',
-        # }
-        # HEADER['Referer'] = 'http://qikan.chaoxing.com/mag/infos?mags=' + mags
-        # try:
-        #     BoolResult, errString, r = self.pro.baserequest.base_request(url=yzurl,
-        #                                                            headers=HEADER,
-        #                                                            timeout=(5, 10),
-        #                                                            proxies=ppp,
-        #                                                            marks=['Fbookright fl'])
-        #     if BoolResult:
-        #         result_queue.put(raw)
-        # except Exception as e:
-        #     print(e)
-
-
-if __name__ == '__main__':
-    # p = ProxyAll()
-    # p.get_redis_all()
-    yz = YanzhengThreadRun(40)
-    yz.run()

re_common/vip/read_rawid_to_txt.py

@@ -1,92 +0,0 @@
-from re_common.baselibrary.utils.basefile import BaseFile
-
-from re_common.facade.mysqlfacade import MysqlUtiles
-
-
-def get_rawid_from_mysql():
-    strings = """
-            host = 192.168.31.209
-            user = root
-            passwd = vipdatacenter
-            db = cnki_qk
-            port = 3306
-            chartset = utf8
-            """
-    mysqlrawid = r"F:\fun2\mysqlrawid.txt"
-    feilds = "filename"
-
-    dicts_change = {"key为原来的": "values为现在的"}
-
-    from re_common.baselibrary.tools.stringtodicts import StringToDicts
-
-    dicts = StringToDicts().string_to_dicts_by_equal(strings)
-    mysqlutils = MysqlUtiles("", "", builder="MysqlBuilderForDicts", dicts=dicts)
-    offset = 0
-    limit = 1000000
-    while True:
-        bools, raws = mysqlutils.SelectFromDB("select `{}` from article limit {},{}".format(feilds, offset, limit))
-        if len(raws) == 0:
-            break
-        with open(mysqlrawid, 'w', encoding="utf-8") as f:
-            for row in raws:
-                f.write(row[0] + "\n")
-        offset = offset + limit
-
-
-# get_rawid_from_mysql()
-# print(BaseFile.get_file_line_num(r'C:\Users\xuzhu\Downloads\part-r-00000'))
-
-def read_files_compose():
-    """
-    读取两个文件对比
-    :return:
-    """
-    sets1 = set()
-    sets2 = set()
-    for value in BaseFile.read_file_r_mode_yield(r'C:\Users\xuzhu\Downloads\part-r-00000'):
-        sets2.add(value)
-
-    for value in BaseFile.read_file_r_mode_yield(r'F:\fun2\mysqlrawid.txt'):
-        sets1.add(value)
-    set3 = sets1 - sets2
-    cha = r"F:\fun2\cha.txt"
-    with open(cha, 'w', encoding="utf-8") as f:
-        for row in set3:
-            f.write(row + "\n")
-
-
-#read_files_compose()
-
-def set_stat():
-    strings = """
-               host = 192.168.31.209
-               user = root
-               passwd = vipdatacenter
-               db = cnki_qk
-               port = 3306
-               chartset = utf8
-               """
-    charawid = r"F:\fun2\cha.txt"
-
-    dicts_change = {"key为原来的": "values为现在的"}
-
-    from re_common.baselibrary.tools.stringtodicts import StringToDicts
-
-    dicts = StringToDicts().string_to_dicts_by_equal(strings)
-    mysqlutils = MysqlUtiles("", "", builder="MysqlBuilderForDicts", dicts=dicts)
-    lists = []
-    for values in BaseFile.read_file_r_mode_yield(charawid):
-        lists.append(values)
-        if len(lists) > 100000:
-            sql = "update article set stat=0,ref_stat=0 where filename in {}".format(tuple(lists))
-            mysqlutils.ExeSqlToDB(sql)
-            lists.clear()
-
-    if len(lists) > 0:
-        if len(lists) == 1:
-            lists.append("test")
-        sql = "update article set stat=0,ref_stat=0 where filename in {}".format(tuple(lists))
-        mysqlutils.ExeSqlToDB(sql)
-        lists.clear()
-
-set_stat()

re_common/vip/title/__init__.py

@@ -1,5 +0,0 @@
-# -*- coding:utf-8 -*-
-# @Time : 2021/12/2 9:38
-# @Author: suhong
-# @File : __init__.py.py
-# @Function :

re_common/vip/title/transform/TransformBookTitleToZt.py

@@ -1,125 +0,0 @@
-# -*- coding:utf-8 -*-
-# @Time : 2021/12/19 18:16
-# @Author: suhong
-# @File : TransformBookTitleToZt.py
-# @Function : 图书a层转智图
-from re_common.baselibrary.utils.basetime import BaseTime
-
-from re_common.facade.mysqlfacade import MysqlUtiles
-
-
-class TransformBookTitleToZt():
-    def __init__(self):
-        self.fields = [
-            "lngid",
-            "rawid",
-            "title",
-            "title_alternative",
-            "title_sub",
-            "title_edition",
-            "title_series",
-            "identifier_eisbn",
-            "identifier_pisbn",
-            "identifier_doi",
-            "creator",
-            "creator_en",
-            "creator_bio",
-            "creator_institution",
-            "publisher",
-            "date",
-            "description",
-            "description_en",
-            "description_unit",
-            "subject",
-            "subject_en",
-            "subject_clc",
-            "subject_esc",
-            "page",
-            "beginpage",
-            "endpage",
-            "pagecount",
-            "date_created",
-            "rawtype",
-            "folio_size",
-            "price",
-            "pub_place",
-            "language",
-            "country",
-            "type",
-            "provider",
-            "provider_url",
-            "provider_id",
-            "medium",
-            "batch",
-            "is_deprecated"
-        ]
-        self.zt_providermap = dict()
-        self.mysqlutils = MysqlUtiles(None, None, builder="MysqlBuilderForDicts", dicts={
-            "host": "192.168.31.24",
-            "user": "root",
-            "passwd": "vipdatacenter",
-            "db": "data_warehouse_sql",
-            "port": "3306",
-            "chartset": "utf8mb4",
-        })
-        self.get_zt_provider()
-
-    def get_zt_provider(self):
-        rows = self.mysqlutils.SelectFromDB(
-            "select sub_db_id,provider from a_transform_task where source_type = '1' and out_type = 'zt'")
-        for row in rows[1]:
-            self.zt_providermap[row[0]] = row[1]
-
-    def transform(self, titleMap):
-        transMap = dict()
-        sub_db_id = titleMap.get("sub_db_id", "")
-        transMap["lngid"] = titleMap.get("lngid", "")
-        transMap["rawid"] = titleMap.get("rawid", "")
-        transMap["title"] = titleMap.get("title", "")
-        transMap["title_alternative"] = titleMap.get("title_alt", "")
-        transMap["title_series"] = titleMap.get("title_series", "")
-        transMap["title_edition"] = titleMap.get("revision", "")
-        transMap["identifier_eisbn"] = titleMap.get("eisbn", "")
-        transMap["identifier_pisbn"] = titleMap.get("isbn", "")
-        transMap["identifier_doi"] = titleMap.get("doi", "")
-        transMap["creator"] = titleMap.get("author", "")
-        transMap["creator_en"] = titleMap.get("author_alt", "")
-        transMap["creator_institution"] = titleMap.get("organ", "")
-        transMap["creator_bio"] = titleMap.get("author_intro", "")
-        transMap["publisher"] = titleMap.get("publisher", "")
-        transMap["date"] = titleMap.get("pub_year", "")
-        transMap["description"] = titleMap.get("abstract", "")
-        transMap["description_en"] = titleMap.get("abstract_alt", "")
-        transMap["description_unit"] = titleMap.get("catalog", "")
-        transMap["subject"] = titleMap.get("keyword", "")
-        transMap["subject_en"] = titleMap.get("keyword_alt", "")
-        transMap["subject_clc"] = titleMap.get("clc_no", "")
-        transMap["subject_esc"] = titleMap.get("subject_edu", "")
-        transMap["page"] = titleMap.get("page_info", "")
-        transMap["beginpage"] = titleMap.get("begin_page", "")
-        transMap["endpage"] = titleMap.get("end_page", "")
-        transMap["pagecount"] = titleMap.get("page_cnt", "")
-        transMap["date_created"] = titleMap.get("pub_date", "")
-        transMap["rawtype"] = titleMap.get("raw_type", "")
-        transMap["folio_size"] = titleMap.get("book_size", "")
-        transMap["price"] = titleMap.get("price", "")
-        transMap["pub_place"] = titleMap.get("pub_place", "")
-        # transMap["provider"] = titleMap.get("zt_provider", "")
-        # if transMap["provider"] == "":
-        transMap["provider"] = self.zt_providermap[sub_db_id]
-        transMap["provider_url"] = transMap["provider"] + "@" + titleMap.get("provider_url")
-        transMap["provider_id"] = transMap["provider"] + "@" + titleMap.get("rawid")
-        transMap["batch"] = BaseTime().get_beijin_date_strins("%Y%m%d") + "00"
-        transMap["type"] = titleMap.get("source_type", "")
-        transMap["medium"] = "2"
-        transMap["country"] = titleMap.get("country", "")
-        transMap["language"] = titleMap.get("language", "")
-        transMap["is_deprecated"] = titleMap.get("is_deprecated", "")
-
-        for field in self.fields:
-            if field not in transMap.keys():
-                transMap[field] = ""
-        return transMap
-
-if __name__ == '__main__':
-    t = TransformBookTitleToZt()

re_common/vip/title/transform/TransformConferenceTitleToZt.py

@@ -1,139 +0,0 @@
-# -*- coding:utf-8 -*-
-# @Time : 2021/12/15 14:09
-# @Author: suhong
-# @File : TransformConferenceTitleToZt.py
-# @Function :转换会议a层到智图代码
-from re_common.baselibrary.utils.basetime import BaseTime
-
-from re_common.facade.mysqlfacade import MysqlUtiles
-
-
-class TransformConferenceTitleToZt():
-    def __init__(self):
-        # 初始化fields
-        self.fields = [
-            "lngid",
-            "rawid",
-            "title",
-            "title_alternative",
-            "title_series",
-            "title_edition",
-            "applicant",
-            "identifier_doi",
-            "creator",
-            "creator_en",
-            "creator_institution",
-            "creator_bio",
-            "creator_drafting",
-            "creator_release",
-            "source",
-            "source_en",
-            "source_institution",
-            "publisher",
-            "date",
-            "description",
-            "description_en",
-            "description_fund",
-            "subject",
-            "subject_en",
-            "page",
-            "beginpage",
-            "endpage",
-            "jumppage",
-            "pagecount",
-            "subject_clc",
-            "subject_esc",
-            "date_created",
-            "pub_place",
-            "language",
-            "country",
-            "type",
-            "provider",
-            "provider_url",
-            "provider_id",
-            "medium",
-            "batch",
-            "is_deprecated",
-            "if_pdf_fulltext",
-            "if_html_fulltext"
-        ]
-        self.zt_providermap = dict()
-        self.mysqlutils = MysqlUtiles(None, None, builder="MysqlBuilderForDicts", dicts={
-            "host": "192.168.31.24",
-            "user": "root",
-            "passwd": "vipdatacenter",
-            "db": "data_warehouse_sql",
-            "port": "3306",
-            "chartset": "utf8mb4",
-        })
-        self.get_zt_provider()
-
-    def get_zt_provider(self):
-        rows = self.mysqlutils.SelectFromDB(
-            "select sub_db_id,provider from a_transform_task where source_type = '6' and out_type = 'zt'")
-        for row in rows[1]:
-            self.zt_providermap[row[0]] = row[1]
-
-    def transform(self, titleMap):
-        transMap = dict()
-        sub_db_id = titleMap.get("sub_db_id", "")
-        transMap["lngid"] = titleMap.get("lngid", "")
-        transMap["rawid"] = titleMap.get("rawid", "")
-        transMap["title"] = titleMap.get("title", "")
-        transMap["title_alternative"] = titleMap.get("title_alt", "")
-        transMap["title_series"] = titleMap.get("meeting_record_name", "")
-        transMap["title_edition"] = titleMap.get("revision", "")
-        transMap["applicant"] = titleMap.get("applicant", "")
-        transMap["identifier_doi"] = titleMap.get("doi", "")
-        transMap["creator"] = titleMap.get("author", "")
-        transMap["creator_en"] = titleMap.get("author_alt", "")
-        transMap["creator_institution"] = titleMap.get("organ", "")
-        transMap["creator_bio"] = titleMap.get("author_intro", "")
-        transMap["creator_drafting"] = titleMap.get("society", "")
-        transMap["creator_release"] = titleMap.get("host_organ", "")
-        transMap["source"] = titleMap.get("meeting_name", "")
-        transMap["source_en"] = titleMap.get("meeting_name_alt", "")
-        transMap["source_institution"] = titleMap.get("meeting_place", "")
-        transMap["publisher"] = titleMap.get("publisher", "")
-        transMap["date"] = titleMap.get("pub_year", "")
-        transMap["description"] = titleMap.get("abstract", "")
-        transMap["description_en"] = titleMap.get("abstract_alt", "")
-        transMap["description_fund"] = titleMap.get("fund", "")
-        transMap["subject"] = titleMap.get("keyword", "")
-        transMap["subject_en"] = titleMap.get("keyword_alt", "")
-        transMap["page"] = titleMap.get("page_info", "")
-        transMap["beginpage"] = titleMap.get("begin_page", "")
-        transMap["endpage"] = titleMap.get("end_page", "")
-        transMap["jumppage"] = titleMap.get("jump_page", "")
-        transMap["pagecount"] = titleMap.get("page_cnt", "")
-        transMap["subject_clc"] = titleMap.get("clc_no", "")
-        transMap["subject_esc"] = titleMap.get("subject_edu", "")
-        transMap["date_created"] = titleMap.get("pub_date", "")
-        transMap["pub_place"] = titleMap.get("pub_place", "")
-        # transMap["provider"] = titleMap.get("zt_provider", "")
-        # if transMap["provider"] == "":
-        transMap["provider"] = self.zt_providermap[sub_db_id]
-        transMap["provider_url"] = transMap["provider"] + "@" + titleMap.get("provider_url")
-        transMap["provider_id"] = transMap["provider"] + "@" + titleMap.get("rawid")
-        transMap["batch"] = BaseTime().get_beijin_date_strins("%Y%m%d") + "00"
-        transMap["type"] = titleMap.get("source_type", "")
-        transMap["medium"] = "2"
-        transMap["country"] = titleMap.get("country", "")
-        transMap["language"] = titleMap.get("language", "")
-        fulltext_type = titleMap.get("fulltext_type", "")
-        transMap["if_pdf_fulltext"] = "0"
-        transMap["if_html_fulltext"] = "0"
-        if "pdf" in fulltext_type:
-            transMap["if_pdf_fulltext"] = "1"
-        if "html" in fulltext_type:
-            transMap["if_html_fulltext"] = "1"
-        transMap["is_deprecated"] = titleMap.get("is_deprecated", "")
-
-        for field in self.fields:
-            if field not in transMap.keys():
-                transMap[field] = ""
-        return transMap
-
-
-if __name__ == '__main__':
-    t = TransformConferenceTitleToZt()