re-common 2.0.1__py3-none-any.whl → 10.0.1__py3-none-any.whl

re_common/studio/streamlitstudio/uber_pickups.py

@@ -0,0 +1,24 @@
+import streamlit as st
+import pandas as pd
+import numpy as np
+
+st.title('Uber pickups in NYC')
+
+DATE_COLUMN = 'date/time'
+DATA_URL = ('https://s3-us-west-2.amazonaws.com/streamlit-demo-data/uber-raw-data-sep14.csv.gz')
+
+@st.cache
+def load_data(nrows):
+    data = pd.read_csv(DATA_URL, nrows=nrows)
+    lowercase = lambda x: str(x).lower()
+    data.rename(lowercase, axis='columns', inplace=True)
+    data[DATE_COLUMN] = pd.to_datetime(data[DATE_COLUMN])
+    return data
+
+# Create a text element and let the reader know the data is loading.
+data_load_state.text("Done! (using st.cache)")
+
+# Load 10,000 rows of data into the dataframe.
+data = load_data(10000)
+# Notify the reader that the data was successfully loaded.
+data_load_state.text('Loading data...done!')

re_common/studio/test.py

@@ -0,0 +1,19 @@
+def test_sq():
+    from re_common.baselibrary.utils.basepymongo import BasePyMongo
+    # basemongo = BasePyMongo("mongodb://sdbadmin:sdbadmin@192.168.72.86:11817/test2?authMechanism=SCRAM-SHA-1")
+    # basemongo = BasePyMongo(
+    #     "mongodb://sdbadmin:sdbadmin@192.168.72.86:11817/dataware_house.base_obj_meta_a")
+    basemongo = BasePyMongo(
+        "mongodb://sdbadmin:sdbadmin@192.168.72.86:11817/test2.test?authMechanism=SCRAM-SHA-1")
+
+    basemongo.use_db("test2")
+    # basemongo.auth("sdbadmin", "sdbadmin", "SCRAM-SHA-1")
+    basemongo.create_col("test")
+    # for items in basemongo.find({}):
+    #     print(items["user"])
+    items = basemongo.find()
+    for item in items:
+        ids = item["id"]
+        print(ids)
+
+test_sq()

re_common/v2/baselibrary/utils/author_smi.py

@@ -1,3 +1,4 @@
+import copy
 import re
 import string
 
@@ -248,6 +249,16 @@ def AuthorRatio(
     if len(l1) == len(l2) and (is_same_or_initials_match(l1, l2) or set(l1) == set(l2)):
         return 1
 
+    # 在这里针对上面一条算法再增加一条算法，先对list 排序在对他进行上面的对比
+    # 如果长度相等 简写也是单词的首字母 那么两个名字一致 举例:Guo, Qiang @@ Q. Guo
+    sort_l1 = copy.deepcopy(l1)
+    sort_l2 = copy.deepcopy(l2)
+    sort_l1.sort()
+    sort_l2.sort()
+    if len(sort_l1) == len(sort_l2) and (is_same_or_initials_match(sort_l1, sort_l2) or set(sort_l1) == set(sort_l2)):
+        return 0.99
+
+
     ##############################################################
     # 以上为情况穷举情况，以下为其他情况的相似率计算
     ##############################################################
@@ -262,7 +273,7 @@ def AuthorRatio(
     len_ratio = len1 / len2 if len1 > len2 else len2 / len1
 
     # 计算归一化的 Indel 相似度。 对于比率<score_cutoff，返回0。
-    end_ratio = normal_end_ratio = Jaro.normalized_similarity(s1, s2)
+    end_ratio = normal_end_ratio = Jaro.normalized_similarity(s1.lower(), s2.lower())
 
     # 需要对作者的比率分布进行调研决定哪些是小比率哪些是大比率
     if len_ratio > 1.5 and len_ratio < 3:
@@ -287,7 +298,7 @@ def AuthorRatio(
 
     # 首字母相同提分
     # if is_contained(extract_initials(s1), extract_initials(s2)):
-    if is_contained_list([i[:1] for i in l1], [i[:1] for i in l2]):
+    if is_contained_list([i[:1].lower() for i in l1], [i[:1].lower() for i in l2]):
         # 应该提分
         end_ratio = end_ratio * 1.05
     else:
@@ -302,7 +313,7 @@ def AuthorRatio(
         end_ratio = end_ratio * 1.1
 
     if l1[0] != l2[0]:
-        end_ratio = end_ratio * Jaro.normalized_similarity(l1[0], l2[0])
+        end_ratio = end_ratio * Jaro.normalized_similarity(l1[0].lower(), l2[0].lower())
 
     # 如果字符串本身的相似度高 应该拉上去 否者应该拉下来
     return min(end_ratio, 1) * 0.5 + normal_end_ratio * 0.5

re_common/v2/baselibrary/utils/stringutils.py

@@ -63,6 +63,7 @@ def get_diacritic_variant(char1):
     return base_char1
 
 def get_alphabetic_ratio(text: str) -> float:
+    # 返回字母型字符所占比例
     if not text:
         return 0
 

re_common/vip/base_step_process.py

@@ -0,0 +1,11 @@
+from abc import ABC, abstractmethod
+
+
+class BaseStepProcess(ABC):
+
+    def __init__(self):
+        self.stat_dicts = {}
+
+    @abstractmethod
+    def do_task(self, *args, **kwargs):
+        pass

re_common/vip/baseencodeid.py

@@ -0,0 +1,91 @@
+import base64
+import hashlib
+
+"""
+VIP编码lngid生成
+"""
+
+class BaseLngid(object):
+    def __int__(self):
+        pass
+
+    def BaseEncodeID(self, strRaw):
+        r""" 自定义base编码 """
+
+        strEncode = base64.b32encode(strRaw.encode('utf8')).decode('utf8')
+
+        if strEncode.endswith('======'):
+            strEncode = '%s%s' % (strEncode[0:-6], '0')
+        elif strEncode.endswith('===='):
+            strEncode = '%s%s' % (strEncode[0:-4], '1')
+        elif strEncode.endswith('==='):
+            strEncode = '%s%s' % (strEncode[0:-3], '8')
+        elif strEncode.endswith('='):
+            strEncode = '%s%s' % (strEncode[0:-1], '9')
+
+        table = str.maketrans('0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ', 'ZYXWVUTSRQPONMLKJIHGFEDCBA9876543210')
+        strEncode = strEncode.translate(table)
+
+        return strEncode
+
+    def BaseDecodeID(self, strEncode):
+        r""" 自定义base解码 """
+
+        table = str.maketrans('ZYXWVUTSRQPONMLKJIHGFEDCBA9876543210', '0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ')
+        strEncode = strEncode.translate(table)
+
+        if strEncode.endswith('0'):
+            strEncode = '%s%s' % (strEncode[0:-1], '======')
+        elif strEncode.endswith('1'):
+            strEncode = '%s%s' % (strEncode[0:-1], '====')
+        elif strEncode.endswith('8'):
+            strEncode = '%s%s' % (strEncode[0:-1], '===')
+        elif strEncode.endswith('9'):
+            strEncode = '%s%s' % (strEncode[0:-1], '=')
+
+        strRaw = base64.b32decode(strEncode.encode('utf8')).decode('utf8')
+
+        return strRaw
+
+    def GetLngid(self, sub_db_id, rawid, case_insensitive=False):
+        """
+        :param sub_db_id:
+        :param rawid:
+        由 sub_db_id 和 rawid 得到 lngid。
+        :param case_insensitive: 标识源网站的 rawid 是否区分大小写
+        :return: lngid
+        """
+        uppercase_rawid = ''  # 大写版 rawid
+        if case_insensitive:  # 源网站的 rawid 区分大小写
+            for ch in rawid:
+                if ch.upper() == ch:
+                    uppercase_rawid += ch
+                else:
+                    uppercase_rawid += ch.upper() + '_'
+        else:
+            uppercase_rawid = rawid.upper()
+
+        limited_id = uppercase_rawid  # 限长ID
+        if len(uppercase_rawid) > 20:
+            limited_id = hashlib.md5(uppercase_rawid.encode('utf8')).hexdigest().upper()
+        else:
+            limited_id = self.BaseEncodeID(uppercase_rawid)
+
+        lngid = sub_db_id + limited_id
+
+        return lngid
+
+    def GetRawid(self, limited_id, case_insensitive=False):
+        try:
+            uppercase_rawid = self.BaseDecodeID(limited_id)
+            if case_insensitive:
+                str_ = "_"
+                uppercase_rawid_list = list(uppercase_rawid)
+                for num,li in enumerate(uppercase_rawid_list):
+                    if li == str_:
+                        old_str = "".join(uppercase_rawid_list[num-1:num+1])
+                        uppercase_rawid = uppercase_rawid.replace(old_str,uppercase_rawid_list[num-1].lower())
+        except Exception as e:
+            raise Exception("长度超过20，不可逆")
+
+        return uppercase_rawid

re_common/vip/changetaskname.py

@@ -0,0 +1,28 @@
+from re_common.baselibrary.tools.stringtodicts import StringToDicts
+from re_common.facade.mysqlfacade import MysqlUtiles
+
+"""
+本方法主要用于provider 变化,有时候我们可能需要更新209的provider
+用该方法更新方便快捷
+"""
+strings = """
+        host = 192.168.31.209
+        user = root
+        passwd = vipdatacenter
+        db = data_gather_record
+        port = 3306
+        chartset = utf8
+        """
+
+dicts_change = {"key为原来的": "values为现在的"}
+
+dicts = StringToDicts().string_to_dicts_by_equal(strings)
+mysqlutils = MysqlUtiles("", "", builder="MysqlBuilderForDicts", dicts=dicts)
+mysqlutils.ExeSqlToDB("SET FOREIGN_KEY_CHECKS=0;")
+for key, values in dicts_change:
+    sql1 = "update `data_gather_record`.`task` set `provider` = '{}' WHERE `provider` = '{}';".format(values, key)
+    sql2 = "update `data_gather_record`.`updating` set `provider` = '{}' WHERE `provider` = '{}';".format(values, key)
+    mysqlutils.ExeSqlToDB(sql1)
+    mysqlutils.ExeSqlToDB(sql2)
+
+mysqlutils.ExeSqlToDB("SET FOREIGN_KEY_CHECKS=1;")

re_common/vip/core_var.py

@@ -0,0 +1,24 @@
+ALL_SCHOOL_PROXY_LIST = ["192.168.31.176:8119", "192.168.31.176:8120", "192.168.31.176:8104", "192.168.31.176:8018",
+                         "192.168.31.176:8076",
+                         "192.168.31.176:8160", "192.168.31.176:8240", "192.168.31.176:8241", "192.168.31.176:8195",
+                         "192.168.31.176:8243",
+                         "192.168.31.176:8062", "192.168.31.176:8019", "192.168.31.176:8034", "192.168.31.176:8103",
+                         "192.168.31.176:8181",
+                         "192.168.31.176:8211", "192.168.31.123:8081", "192.168.31.176:8032", "192.168.31.176:8231",
+                         "192.168.31.176:8189",
+                         "192.167.31.176:8058", "192.168.31.36:8135", "192.168.31.176:8057", "192.168.31.176:8017",
+                         "192.168.31.36:8033",
+                         "192.168.31.176:8184", "192.168.31.176:8207", "192.168.31.176:8196", "192.168.31.176:8041",
+                         "192.168.31.176:8087",
+                         "192.168.31.176:8117", "192.168.31.36:8098", "192.168.31.176:8165", "192.168.31.36:8039",
+                         "192.168.31.176:8159",
+                         "192.168.31.176:8051", "192.168.31.176:8180", "192.168.31.176:8148", "192.168.31.176:8021",
+                         "192.168.31.176:8008",
+                         "192.168.31.176:8035", "192.168.31.36:8004", "192.168.31.176:8131", "192.168.31.176:8127",
+                         "192.168.31.176:8052",
+                         "192.168.31.36:8011", "192.168.31.36:8082", "192.168.31.36:8182", "192.168.31.176:8031",
+                         "192.168.31.176:8171",
+                         "192.168.31.176:8012", "192.168.31.176:8002", "192.168.31.176:8140", "192.168.31.36:8149",
+                         "192.168.31.176:8074",
+                         "192.168.31.3:8080", "192.168.31.4:8080", "192.168.31.179:8129", "192.168.31.179:8130",
+                         "192.168.31.179:8132"]

re_common/vip/mmh3Hash.py

@@ -0,0 +1,90 @@
+#!/bin/env python
+# -*- coding: utf-8-*-
+# author: ganruoxun
+# date: 2020-09-25
+
+import mmh3
+import os
+import binascii
+
+
+class Mmh3Hash(object):
+    def __init__(self, _type, resType):
+        self.typeDic = {
+            "2": "bs",
+            "3": "hy",
+            "4": "zl",
+            "5": "bz",
+            "10": "fg",
+        }
+        self.typeCode = self.typeDic[_type]
+        self.resType = resType
+        if self.typeCode == None:
+            raise RuntimeError('type 参数无法识别！')
+
+    # ##从filePath获取文件名，将文件名转大写，后缀转小写
+    # def normFileName(self, filePath):
+    #     lngid = os.path.splitext(os.path.normpath(os.path.basename(filePath)))[0].upper()
+    #     prfx = os.path.splitext(os.path.normpath(os.path.basename(filePath)))[1].lower()
+    #     return "{0}{1}".format(lngid, prfx)
+
+    ##规范文件名称，文件名转大写，文件后缀转小写
+    def normFileName(self, fileName):
+        if not "." in fileName:
+            return fileName.upper()
+        tmps = fileName.split(".")
+        filePrfx = tmps[0].upper()
+        fileSufx = tmps[1].lower()
+        return "{0}.{1}".format(filePrfx, fileSufx)
+
+    # 带有点的文件名，主要用于图片，不在标准定义当中
+    def normFileName_image(self, fileName):
+        if not "." in fileName:
+            return fileName.upper()
+        tmps = fileName.split(".")
+        fileSufx = tmps[-1].lower()
+        tmps.remove(tmps[-1])
+        filePrfx = ".".join(tmps).upper()
+
+        return "{0}.{1}".format(filePrfx, fileSufx)
+
+    ## 使用murmurhash3算法将新全文文件名进行HASH并按照规则组装成HASH目录
+    def generateHashName(self, fileName):
+        hashCode = binascii.b2a_hex(mmh3.hash_bytes(fileName)).upper()[0:3]
+        firstCode = chr(hashCode[0])
+        secondCode = chr(hashCode[1])
+        thirdCode = chr(hashCode[2])
+        if thirdCode.isdigit():
+            return firstCode + secondCode + str(int(thirdCode) % 5)
+        elif thirdCode == 'D':
+            return firstCode + secondCode + 'A'
+        elif thirdCode == 'E':
+            return firstCode + secondCode + 'B'
+        elif thirdCode == 'F':
+            return firstCode + secondCode + 'C'
+        else:
+            return firstCode + secondCode + thirdCode
+
+    # fileName：文件名称，带后缀，不能为空，专利为公开号加文件后缀名，其他为lngid加文件后缀名
+    # years：年份，不能为空
+    # country：国家，如果为空，默认为cn
+    # type：自建资源类型，不能为空，目前只有bs（博硕）,hy（会议）,bz（标准）,fg（法规）,zl（专利）
+    def generatehashPath(self, fileName, years, country, resType):
+        if years == None or len(years) != 4:
+            raise RuntimeError('years 参数错误！')
+        elif fileName == None or len(fileName) == 0:
+            raise RuntimeError('fileName 参数错误！')
+        elif country == None or len(country) == 0:
+            country = 'cn'
+        if resType in ('bs', 'hy', 'fg', 'zl', 'bz'):
+            country = "cn"
+        country = country.lower()
+        intYear = int(years)
+        if intYear < 1989:
+            years = 'befor1989'
+        fileName = self.normFileName(fileName)
+        return "\\" + resType + "\\" + years + country + self.typeCode + "\\" + self.generateHashName(fileName)+ '\\' + fileName
+        # return '\\' + years + country + _type + '\\' + generateHashName(fileName) + '\\' + fileName
+
+print(Mmh3Hash("3","").normFileName_image("aa.bb.jpg"))
+print(Mmh3Hash("4","zl").generatehashPath("cn206103553u.pdf","2016","cn","zl"))

re_common/vip/proxy/allproxys.py

@@ -0,0 +1,127 @@
+import json
+###########################################
+# 同项目调用基础包
+import os
+import sys
+import time
+
+import requests
+
+filepath = os.path.abspath(__file__)
+pathlist = filepath.split(os.sep)
+pathlist = pathlist[:-4]
+TopPath = os.sep.join(pathlist)
+sys.path.insert(0, TopPath)
+print(TopPath)
+############################################
+
+from re_common.baselibrary.utils.basedir import BaseDir
+from re_common.baselibrary.utils.basefile import BaseFile
+from re_common.baselibrary.utils.baserequest import BaseRequest
+from re_common.facade.lazy_import import get_streamlogger
+from re_common.facade.mysqlfacade import MysqlUtiles
+from re_common.baselibrary.utils.basetime import BaseTime
+
+
+class Kproxy(object):
+    def __init__(self):
+        self.cur_path = BaseDir.get_file_dir_absolute(__file__)
+        self.configfile = BaseFile.get_new_path(self.cur_path, "db.ini")
+        self.logger = get_streamlogger()
+        self.mysqlutils = MysqlUtiles(self.configfile, "allproxy", self.logger)
+        self.bsrequest = BaseRequest()
+        self.starttime = time.time()
+        self.starttime_val = time.time()
+
+    def get_taiyang_proxy(self, num=6):
+        """
+        获取太阳代理 每分钟3个
+        :param num:
+        :return:
+        """
+        self.starttime = time.time()
+        #  url = "http://http.tiqu.qingjuhe.cn/getip?num={}&type=2&pack=59105&port=1&ts=1&lb=1&pb=45&regions=".format(num)
+        url = "http://http.tiqu.alibabaapi.com/getip?num={}&type=2&pack=59105&port=1&ts=1&lb=1&pb=45&regions=".format(num)
+        BoolResult, errString, r = self.bsrequest.base_request(url,
+                                                               timeout=30
+                                                               )
+        if BoolResult:
+            dicts = json.loads(r.text)
+            for item in dicts["data"]:
+                proxy = item["ip"] + ":" + item["port"]
+                sources = "taiyang"
+                expire_time = item["expire_time"]
+                sql = "insert into proxyall_v1 (proxy,sources,expire_time) values ('%s','%s','%s') on DUPLICATE key update stat=1,expire_time='%s'" % (
+                    proxy, sources, expire_time, expire_time)
+                self.mysqlutils.ExeSqlToDB(sql)
+        else:
+            self.logger.error("获取失败")
+
+    def get_taiyang_num(self):
+        """
+        获取太阳代理 每分钟3个
+        :param num:
+        :return:
+        """
+        url = "http://ty-http-d.hamir.net/index/index/get_my_package_balance?neek=521821&appkey=1fcba6de94f71561ba3007f4c24ca0b1&ac=59105"
+        BoolResult, errString, r = self.bsrequest.base_request(url,
+                                                               timeout=30
+                                                               )
+        if BoolResult:
+            dicts = json.loads(r.text)
+            return dicts["data"]["package_balance"]
+        else:
+            self.logger.error("获取失败")
+
+    def val(self, proxy, sources):
+        # 请求地址
+        targetUrl = "https://www.baidu.com"
+        proxies = {
+            "http": "http://%s" % proxy,
+            "https": "http://%s" % proxy
+        }
+        resp = requests.get(targetUrl, proxies=proxies, timeout=5)
+        if resp.status_code == 200:
+            print(resp.status_code)
+            return True
+        else:
+            sql = "update proxyall_v1 set stat=0 where proxy='%s' and sources='%s';" % (proxy, sources)
+            self.mysqlutils.ExeSqlToDB(sql)
+            return False
+
+    def val_all(self):
+        self.starttime_val = time.time()
+        sql = "select proxy,sources from proxyall_v1 where stat=1 ORDER BY `update_time`"
+        bools, rows = self.mysqlutils.SelectFromDB(sql)
+        for row in rows:
+            try:
+                self.val(row[0], row[1])
+            except:
+                sql = "update proxyall_v1 set stat=0 where proxy='%s' and sources='%s';" % (row[0], row[1])
+                self.mysqlutils.ExeSqlToDB(sql)
+
+    def run(self):
+        sleep_time = 0
+        while True:
+            num = 10
+            if sleep_time < 0:
+                print("time sleep {}".format(str(100 + sleep_time)))
+                if 100 + sleep_time > 0:
+                    time.sleep(100 + sleep_time)
+                num = num + 10
+            nowtime = BaseTime().get_beijin_date_strins(format="%H%M%S")
+            print(nowtime)
+            if "133700" <= nowtime <= "134700":
+                num = Kproxy().get_taiyang_num()
+            start_time = time.time()
+            self.get_taiyang_proxy(num=num)
+            self.val_all()
+            use_time = int(time.time() - start_time)
+            sleep_time = 100 - use_time
+            print("time sleep {}".format(str(sleep_time)))
+            if sleep_time >= 3:
+                time.sleep(sleep_time)
+
+
+if __name__ == "__main__":
+    Kproxy().run()

re_common/vip/proxy/allproxys_thread.py

@@ -0,0 +1,159 @@
+import json
+###########################################
+# 同项目调用基础包
+import os
+import sys
+import time
+
+import requests
+
+from re_common.baselibrary.mthread.MThreadingRun import MThreadingRun
+from re_common.baselibrary.mthread.mythreading import ThreadInfo, ThreadVal, ThreadPoolManger
+
+filepath = os.path.abspath(__file__)
+pathlist = filepath.split(os.sep)
+pathlist = pathlist[:-4]
+TopPath = os.sep.join(pathlist)
+sys.path.insert(0, TopPath)
+print(TopPath)
+############################################
+
+from re_common.baselibrary.utils.basedir import BaseDir
+from re_common.baselibrary.utils.basefile import BaseFile
+from re_common.baselibrary.utils.baserequest import BaseRequest
+from re_common.facade.lazy_import import get_streamlogger
+from re_common.facade.mysqlfacade import MysqlUtiles
+
+
+class Kproxy(object):
+    def __init__(self):
+        self.cur_path = BaseDir.get_file_dir_absolute(__file__)
+        self.configfile = BaseFile.get_new_path(self.cur_path, "db.ini")
+        self.logger = get_streamlogger()
+        self.mysqlutils = MysqlUtiles(self.configfile, "allproxy", self.logger)
+        self.bsrequest = BaseRequest()
+        self.starttime = time.time()
+        self.starttime_val = time.time()
+
+    def get_taiyang_proxy(self, num=6):
+        """
+        获取太阳代理 每分钟3个
+        :param num:
+        :return:
+        """
+        self.starttime = time.time()
+        url = "http://http.tiqu.qingjuhe.cn/getip?num={}&type=2&pack=56912&port=1&ts=1&lb=1&pb=45&regions=".format(num)
+        BoolResult, errString, r = self.bsrequest.base_request(url,
+                                                               timeout=30
+                                                               )
+        if BoolResult:
+            dicts = json.loads(r.text)
+            for item in dicts["data"]:
+                proxy = item["ip"] + ":" + item["port"]
+                sources = "taiyang"
+                expire_time = item["expire_time"]
+                sql = "insert into proxyall_v1 (proxy,sources,expire_time) values ('%s','%s','%s') on DUPLICATE key update stat=1,expire_time='%s'" % (
+                    proxy, sources, expire_time, expire_time)
+                self.mysqlutils.ExeSqlToDB(sql)
+        else:
+            self.logger.error("获取失败")
+
+    def val(self, proxy, sources,threadval):
+        # 请求地址
+        targetUrl = "https://www.baidu.com"
+        proxies = {
+            "http": "http://%s" % proxy,
+            "https": "http://%s" % proxy
+        }
+        resp = requests.get(targetUrl, proxies=proxies, timeout=5)
+        if resp.status_code == 200:
+            print(resp.status_code)
+            sql = "update proxyall_v1 set stat=1 where proxy='%s' and sources='%s';" % (proxy, sources)
+            threadval.get_result_queue().put(sql)
+            return True
+        else:
+            sql = "update proxyall_v1 set stat=0 where proxy='%s' and sources='%s';" % (proxy, sources)
+            # self.mysqlutils.ExeSqlToDB(sql)
+            threadval.get_result_queue().put(sql)
+            return False
+
+    def val_all(self):
+        self.starttime_val = time.time()
+        sql = "select proxy,sources from proxyall_v1 where stat=1 ORDER BY `update_time`"
+        bools, rows = self.mysqlutils.SelectFromDB(sql)
+        for row in rows:
+            try:
+                self.val(row[0], row[1])
+            except:
+                sql = "update proxyall_v1 set stat=0 where proxy='%s' and sources='%s';" % (row[0], row[1])
+                self.mysqlutils.ExeSqlToDB(sql)
+
+    def run(self):
+        while True:
+            start_time = time.time()
+            self.get_taiyang_proxy()
+            self.val_all()
+            use_time = int(time.time() - start_time)
+            sleep_time = 100 - use_time
+            print("time sleep {}".format(str(sleep_time)))
+            if sleep_time >= 3:
+                time.sleep(sleep_time)
+
+
+class DetailThreadRun(MThreadingRun):
+    def __init__(self, num):
+        self.down = Kproxy()
+        super(DetailThreadRun, self).__init__(num)
+
+    @ThreadPoolManger.thread_lock
+    def getTask(self, *args, **kwargs):
+        sql = "select proxy,sources from proxyall_v1 where stat=0 ORDER BY `expire_time` DESC limit 1000"
+        bools, rows = self.down.mysqlutils.SelectFromDB(sql)
+        return rows
+
+    @ThreadPoolManger.thread_lock
+    def getTask2(self, *args, **kwargs):
+        sql = "select proxy,sources from proxyall_v1 where stat=1 ORDER BY `expire_time` DESC limit 1000"
+        bools, rows = self.down.mysqlutils.SelectFromDB(sql)
+        return rows
+
+    def setTask(self, results=None, *args, **kwargs):
+        if not results:
+            return self.BREAK
+        for row in results:
+            self.add_job(self.func, row[0], row[1])
+        rows = self.getTask2()
+        for row in rows:
+            self.add_job(self.func, row[0], row[1])
+        time.sleep(60*2)
+        return self.BREAK
+
+    @ThreadPoolManger.thread_lock
+    def dealresult(self, *args, **kwargs):
+        # for sql in self.results:
+        #     self.down.mysqlutils.ExeSqlToDB(sql)
+        self.down.mysqlutils.ExeSqlListToDB(self.results)
+
+    def setProxy(self, proxysList=None):
+        time.sleep(300)
+
+    def is_break(self):
+        return False
+
+    def thread_pool_hook(self, threadinfo: ThreadInfo):
+        # 设置代理线程不重启，默认会重启
+        return {}
+
+    def fun(self, threadval: ThreadVal, *args, **kwargs):
+        proxy,sources = args[0],args[1]
+        try:
+            self.down.val(proxy, sources, threadval)
+        except:
+            sql = "update proxyall_v1 set stat=0 where proxy='%s' and sources='%s';" % (proxy, sources)
+            # self.mysqlutils.ExeSqlToDB(sql)
+            threadval.get_result_queue().put(sql)
+
+
+if __name__ == '__main__':
+    down = DetailThreadRun(30)
+    down.run()