re-common 10.0.37__py3-none-any.whl → 10.0.39__py3-none-any.whl

re_common/vip/baseencodeid.py

@@ -1,91 +1,91 @@
-import base64
-import hashlib
-
-"""
-VIP编码lngid生成
-"""
-
-class BaseLngid(object):
-    def __int__(self):
-        pass
-
-    def BaseEncodeID(self, strRaw):
-        r""" 自定义base编码 """
-
-        strEncode = base64.b32encode(strRaw.encode('utf8')).decode('utf8')
-
-        if strEncode.endswith('======'):
-            strEncode = '%s%s' % (strEncode[0:-6], '0')
-        elif strEncode.endswith('===='):
-            strEncode = '%s%s' % (strEncode[0:-4], '1')
-        elif strEncode.endswith('==='):
-            strEncode = '%s%s' % (strEncode[0:-3], '8')
-        elif strEncode.endswith('='):
-            strEncode = '%s%s' % (strEncode[0:-1], '9')
-
-        table = str.maketrans('0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ', 'ZYXWVUTSRQPONMLKJIHGFEDCBA9876543210')
-        strEncode = strEncode.translate(table)
-
-        return strEncode
-
-    def BaseDecodeID(self, strEncode):
-        r""" 自定义base解码 """
-
-        table = str.maketrans('ZYXWVUTSRQPONMLKJIHGFEDCBA9876543210', '0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ')
-        strEncode = strEncode.translate(table)
-
-        if strEncode.endswith('0'):
-            strEncode = '%s%s' % (strEncode[0:-1], '======')
-        elif strEncode.endswith('1'):
-            strEncode = '%s%s' % (strEncode[0:-1], '====')
-        elif strEncode.endswith('8'):
-            strEncode = '%s%s' % (strEncode[0:-1], '===')
-        elif strEncode.endswith('9'):
-            strEncode = '%s%s' % (strEncode[0:-1], '=')
-
-        strRaw = base64.b32decode(strEncode.encode('utf8')).decode('utf8')
-
-        return strRaw
-
-    def GetLngid(self, sub_db_id, rawid, case_insensitive=False):
-        """
-        :param sub_db_id:
-        :param rawid:
-        由 sub_db_id 和 rawid 得到 lngid。
-        :param case_insensitive: 标识源网站的 rawid 是否区分大小写
-        :return: lngid
-        """
-        uppercase_rawid = ''  # 大写版 rawid
-        if case_insensitive:  # 源网站的 rawid 区分大小写
-            for ch in rawid:
-                if ch.upper() == ch:
-                    uppercase_rawid += ch
-                else:
-                    uppercase_rawid += ch.upper() + '_'
-        else:
-            uppercase_rawid = rawid.upper()
-
-        limited_id = uppercase_rawid  # 限长ID
-        if len(uppercase_rawid) > 20:
-            limited_id = hashlib.md5(uppercase_rawid.encode('utf8')).hexdigest().upper()
-        else:
-            limited_id = self.BaseEncodeID(uppercase_rawid)
-
-        lngid = sub_db_id + limited_id
-
-        return lngid
-
-    def GetRawid(self, limited_id, case_insensitive=False):
-        try:
-            uppercase_rawid = self.BaseDecodeID(limited_id)
-            if case_insensitive:
-                str_ = "_"
-                uppercase_rawid_list = list(uppercase_rawid)
-                for num,li in enumerate(uppercase_rawid_list):
-                    if li == str_:
-                        old_str = "".join(uppercase_rawid_list[num-1:num+1])
-                        uppercase_rawid = uppercase_rawid.replace(old_str,uppercase_rawid_list[num-1].lower())
-        except Exception as e:
-            raise Exception("长度超过20，不可逆")
-
+import base64
+import hashlib
+
+"""
+VIP编码lngid生成
+"""
+
+class BaseLngid(object):
+    def __int__(self):
+        pass
+
+    def BaseEncodeID(self, strRaw):
+        r""" 自定义base编码 """
+
+        strEncode = base64.b32encode(strRaw.encode('utf8')).decode('utf8')
+
+        if strEncode.endswith('======'):
+            strEncode = '%s%s' % (strEncode[0:-6], '0')
+        elif strEncode.endswith('===='):
+            strEncode = '%s%s' % (strEncode[0:-4], '1')
+        elif strEncode.endswith('==='):
+            strEncode = '%s%s' % (strEncode[0:-3], '8')
+        elif strEncode.endswith('='):
+            strEncode = '%s%s' % (strEncode[0:-1], '9')
+
+        table = str.maketrans('0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ', 'ZYXWVUTSRQPONMLKJIHGFEDCBA9876543210')
+        strEncode = strEncode.translate(table)
+
+        return strEncode
+
+    def BaseDecodeID(self, strEncode):
+        r""" 自定义base解码 """
+
+        table = str.maketrans('ZYXWVUTSRQPONMLKJIHGFEDCBA9876543210', '0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ')
+        strEncode = strEncode.translate(table)
+
+        if strEncode.endswith('0'):
+            strEncode = '%s%s' % (strEncode[0:-1], '======')
+        elif strEncode.endswith('1'):
+            strEncode = '%s%s' % (strEncode[0:-1], '====')
+        elif strEncode.endswith('8'):
+            strEncode = '%s%s' % (strEncode[0:-1], '===')
+        elif strEncode.endswith('9'):
+            strEncode = '%s%s' % (strEncode[0:-1], '=')
+
+        strRaw = base64.b32decode(strEncode.encode('utf8')).decode('utf8')
+
+        return strRaw
+
+    def GetLngid(self, sub_db_id, rawid, case_insensitive=False):
+        """
+        :param sub_db_id:
+        :param rawid:
+        由 sub_db_id 和 rawid 得到 lngid。
+        :param case_insensitive: 标识源网站的 rawid 是否区分大小写
+        :return: lngid
+        """
+        uppercase_rawid = ''  # 大写版 rawid
+        if case_insensitive:  # 源网站的 rawid 区分大小写
+            for ch in rawid:
+                if ch.upper() == ch:
+                    uppercase_rawid += ch
+                else:
+                    uppercase_rawid += ch.upper() + '_'
+        else:
+            uppercase_rawid = rawid.upper()
+
+        limited_id = uppercase_rawid  # 限长ID
+        if len(uppercase_rawid) > 20:
+            limited_id = hashlib.md5(uppercase_rawid.encode('utf8')).hexdigest().upper()
+        else:
+            limited_id = self.BaseEncodeID(uppercase_rawid)
+
+        lngid = sub_db_id + limited_id
+
+        return lngid
+
+    def GetRawid(self, limited_id, case_insensitive=False):
+        try:
+            uppercase_rawid = self.BaseDecodeID(limited_id)
+            if case_insensitive:
+                str_ = "_"
+                uppercase_rawid_list = list(uppercase_rawid)
+                for num,li in enumerate(uppercase_rawid_list):
+                    if li == str_:
+                        old_str = "".join(uppercase_rawid_list[num-1:num+1])
+                        uppercase_rawid = uppercase_rawid.replace(old_str,uppercase_rawid_list[num-1].lower())
+        except Exception as e:
+            raise Exception("长度超过20，不可逆")
+
         return uppercase_rawid

re_common/vip/changetaskname.py

@@ -1,28 +1,28 @@
-from re_common.baselibrary.tools.stringtodicts import StringToDicts
-from re_common.facade.mysqlfacade import MysqlUtiles
-
-"""
-本方法主要用于provider 变化,有时候我们可能需要更新209的provider
-用该方法更新方便快捷
-"""
-strings = """
-        host = 192.168.31.209
-        user = root
-        passwd = vipdatacenter
-        db = data_gather_record
-        port = 3306
-        chartset = utf8
-        """
-
-dicts_change = {"key为原来的": "values为现在的"}
-
-dicts = StringToDicts().string_to_dicts_by_equal(strings)
-mysqlutils = MysqlUtiles("", "", builder="MysqlBuilderForDicts", dicts=dicts)
-mysqlutils.ExeSqlToDB("SET FOREIGN_KEY_CHECKS=0;")
-for key, values in dicts_change:
-    sql1 = "update `data_gather_record`.`task` set `provider` = '{}' WHERE `provider` = '{}';".format(values, key)
-    sql2 = "update `data_gather_record`.`updating` set `provider` = '{}' WHERE `provider` = '{}';".format(values, key)
-    mysqlutils.ExeSqlToDB(sql1)
-    mysqlutils.ExeSqlToDB(sql2)
-
-mysqlutils.ExeSqlToDB("SET FOREIGN_KEY_CHECKS=1;")
+from re_common.baselibrary.tools.stringtodicts import StringToDicts
+from re_common.facade.mysqlfacade import MysqlUtiles
+
+"""
+本方法主要用于provider 变化,有时候我们可能需要更新209的provider
+用该方法更新方便快捷
+"""
+strings = """
+        host = 192.168.31.209
+        user = root
+        passwd = vipdatacenter
+        db = data_gather_record
+        port = 3306
+        chartset = utf8
+        """
+
+dicts_change = {"key为原来的": "values为现在的"}
+
+dicts = StringToDicts().string_to_dicts_by_equal(strings)
+mysqlutils = MysqlUtiles("", "", builder="MysqlBuilderForDicts", dicts=dicts)
+mysqlutils.ExeSqlToDB("SET FOREIGN_KEY_CHECKS=0;")
+for key, values in dicts_change:
+    sql1 = "update `data_gather_record`.`task` set `provider` = '{}' WHERE `provider` = '{}';".format(values, key)
+    sql2 = "update `data_gather_record`.`updating` set `provider` = '{}' WHERE `provider` = '{}';".format(values, key)
+    mysqlutils.ExeSqlToDB(sql1)
+    mysqlutils.ExeSqlToDB(sql2)
+
+mysqlutils.ExeSqlToDB("SET FOREIGN_KEY_CHECKS=1;")

re_common/vip/core_var.py

@@ -1,24 +1,24 @@
-ALL_SCHOOL_PROXY_LIST = ["192.168.31.176:8119", "192.168.31.176:8120", "192.168.31.176:8104", "192.168.31.176:8018",
-                         "192.168.31.176:8076",
-                         "192.168.31.176:8160", "192.168.31.176:8240", "192.168.31.176:8241", "192.168.31.176:8195",
-                         "192.168.31.176:8243",
-                         "192.168.31.176:8062", "192.168.31.176:8019", "192.168.31.176:8034", "192.168.31.176:8103",
-                         "192.168.31.176:8181",
-                         "192.168.31.176:8211", "192.168.31.123:8081", "192.168.31.176:8032", "192.168.31.176:8231",
-                         "192.168.31.176:8189",
-                         "192.167.31.176:8058", "192.168.31.36:8135", "192.168.31.176:8057", "192.168.31.176:8017",
-                         "192.168.31.36:8033",
-                         "192.168.31.176:8184", "192.168.31.176:8207", "192.168.31.176:8196", "192.168.31.176:8041",
-                         "192.168.31.176:8087",
-                         "192.168.31.176:8117", "192.168.31.36:8098", "192.168.31.176:8165", "192.168.31.36:8039",
-                         "192.168.31.176:8159",
-                         "192.168.31.176:8051", "192.168.31.176:8180", "192.168.31.176:8148", "192.168.31.176:8021",
-                         "192.168.31.176:8008",
-                         "192.168.31.176:8035", "192.168.31.36:8004", "192.168.31.176:8131", "192.168.31.176:8127",
-                         "192.168.31.176:8052",
-                         "192.168.31.36:8011", "192.168.31.36:8082", "192.168.31.36:8182", "192.168.31.176:8031",
-                         "192.168.31.176:8171",
-                         "192.168.31.176:8012", "192.168.31.176:8002", "192.168.31.176:8140", "192.168.31.36:8149",
-                         "192.168.31.176:8074",
-                         "192.168.31.3:8080", "192.168.31.4:8080", "192.168.31.179:8129", "192.168.31.179:8130",
-                         "192.168.31.179:8132"]
+ALL_SCHOOL_PROXY_LIST = ["192.168.31.176:8119", "192.168.31.176:8120", "192.168.31.176:8104", "192.168.31.176:8018",
+                         "192.168.31.176:8076",
+                         "192.168.31.176:8160", "192.168.31.176:8240", "192.168.31.176:8241", "192.168.31.176:8195",
+                         "192.168.31.176:8243",
+                         "192.168.31.176:8062", "192.168.31.176:8019", "192.168.31.176:8034", "192.168.31.176:8103",
+                         "192.168.31.176:8181",
+                         "192.168.31.176:8211", "192.168.31.123:8081", "192.168.31.176:8032", "192.168.31.176:8231",
+                         "192.168.31.176:8189",
+                         "192.167.31.176:8058", "192.168.31.36:8135", "192.168.31.176:8057", "192.168.31.176:8017",
+                         "192.168.31.36:8033",
+                         "192.168.31.176:8184", "192.168.31.176:8207", "192.168.31.176:8196", "192.168.31.176:8041",
+                         "192.168.31.176:8087",
+                         "192.168.31.176:8117", "192.168.31.36:8098", "192.168.31.176:8165", "192.168.31.36:8039",
+                         "192.168.31.176:8159",
+                         "192.168.31.176:8051", "192.168.31.176:8180", "192.168.31.176:8148", "192.168.31.176:8021",
+                         "192.168.31.176:8008",
+                         "192.168.31.176:8035", "192.168.31.36:8004", "192.168.31.176:8131", "192.168.31.176:8127",
+                         "192.168.31.176:8052",
+                         "192.168.31.36:8011", "192.168.31.36:8082", "192.168.31.36:8182", "192.168.31.176:8031",
+                         "192.168.31.176:8171",
+                         "192.168.31.176:8012", "192.168.31.176:8002", "192.168.31.176:8140", "192.168.31.36:8149",
+                         "192.168.31.176:8074",
+                         "192.168.31.3:8080", "192.168.31.4:8080", "192.168.31.179:8129", "192.168.31.179:8130",
+                         "192.168.31.179:8132"]

re_common/vip/mmh3Hash.py

@@ -1,90 +1,90 @@
-#!/bin/env python
-# -*- coding: utf-8-*-
-# author: ganruoxun
-# date: 2020-09-25
-
-import mmh3
-import os
-import binascii
-
-
-class Mmh3Hash(object):
-    def __init__(self, _type, resType):
-        self.typeDic = {
-            "2": "bs",
-            "3": "hy",
-            "4": "zl",
-            "5": "bz",
-            "10": "fg",
-        }
-        self.typeCode = self.typeDic[_type]
-        self.resType = resType
-        if self.typeCode == None:
-            raise RuntimeError('type 参数无法识别！')
-
-    # ##从filePath获取文件名，将文件名转大写，后缀转小写
-    # def normFileName(self, filePath):
-    #     lngid = os.path.splitext(os.path.normpath(os.path.basename(filePath)))[0].upper()
-    #     prfx = os.path.splitext(os.path.normpath(os.path.basename(filePath)))[1].lower()
-    #     return "{0}{1}".format(lngid, prfx)
-
-    ##规范文件名称，文件名转大写，文件后缀转小写
-    def normFileName(self, fileName):
-        if not "." in fileName:
-            return fileName.upper()
-        tmps = fileName.split(".")
-        filePrfx = tmps[0].upper()
-        fileSufx = tmps[1].lower()
-        return "{0}.{1}".format(filePrfx, fileSufx)
-
-    # 带有点的文件名，主要用于图片，不在标准定义当中
-    def normFileName_image(self, fileName):
-        if not "." in fileName:
-            return fileName.upper()
-        tmps = fileName.split(".")
-        fileSufx = tmps[-1].lower()
-        tmps.remove(tmps[-1])
-        filePrfx = ".".join(tmps).upper()
-
-        return "{0}.{1}".format(filePrfx, fileSufx)
-
-    ## 使用murmurhash3算法将新全文文件名进行HASH并按照规则组装成HASH目录
-    def generateHashName(self, fileName):
-        hashCode = binascii.b2a_hex(mmh3.hash_bytes(fileName)).upper()[0:3]
-        firstCode = chr(hashCode[0])
-        secondCode = chr(hashCode[1])
-        thirdCode = chr(hashCode[2])
-        if thirdCode.isdigit():
-            return firstCode + secondCode + str(int(thirdCode) % 5)
-        elif thirdCode == 'D':
-            return firstCode + secondCode + 'A'
-        elif thirdCode == 'E':
-            return firstCode + secondCode + 'B'
-        elif thirdCode == 'F':
-            return firstCode + secondCode + 'C'
-        else:
-            return firstCode + secondCode + thirdCode
-
-    # fileName：文件名称，带后缀，不能为空，专利为公开号加文件后缀名，其他为lngid加文件后缀名
-    # years：年份，不能为空
-    # country：国家，如果为空，默认为cn
-    # type：自建资源类型，不能为空，目前只有bs（博硕）,hy（会议）,bz（标准）,fg（法规）,zl（专利）
-    def generatehashPath(self, fileName, years, country, resType):
-        if years == None or len(years) != 4:
-            raise RuntimeError('years 参数错误！')
-        elif fileName == None or len(fileName) == 0:
-            raise RuntimeError('fileName 参数错误！')
-        elif country == None or len(country) == 0:
-            country = 'cn'
-        if resType in ('bs', 'hy', 'fg', 'zl', 'bz'):
-            country = "cn"
-        country = country.lower()
-        intYear = int(years)
-        if intYear < 1989:
-            years = 'befor1989'
-        fileName = self.normFileName(fileName)
-        return "\\" + resType + "\\" + years + country + self.typeCode + "\\" + self.generateHashName(fileName)+ '\\' + fileName
-        # return '\\' + years + country + _type + '\\' + generateHashName(fileName) + '\\' + fileName
-
-print(Mmh3Hash("3","").normFileName_image("aa.bb.jpg"))
+#!/bin/env python
+# -*- coding: utf-8-*-
+# author: ganruoxun
+# date: 2020-09-25
+
+import mmh3
+import os
+import binascii
+
+
+class Mmh3Hash(object):
+    def __init__(self, _type, resType):
+        self.typeDic = {
+            "2": "bs",
+            "3": "hy",
+            "4": "zl",
+            "5": "bz",
+            "10": "fg",
+        }
+        self.typeCode = self.typeDic[_type]
+        self.resType = resType
+        if self.typeCode == None:
+            raise RuntimeError('type 参数无法识别！')
+
+    # ##从filePath获取文件名，将文件名转大写，后缀转小写
+    # def normFileName(self, filePath):
+    #     lngid = os.path.splitext(os.path.normpath(os.path.basename(filePath)))[0].upper()
+    #     prfx = os.path.splitext(os.path.normpath(os.path.basename(filePath)))[1].lower()
+    #     return "{0}{1}".format(lngid, prfx)
+
+    ##规范文件名称，文件名转大写，文件后缀转小写
+    def normFileName(self, fileName):
+        if not "." in fileName:
+            return fileName.upper()
+        tmps = fileName.split(".")
+        filePrfx = tmps[0].upper()
+        fileSufx = tmps[1].lower()
+        return "{0}.{1}".format(filePrfx, fileSufx)
+
+    # 带有点的文件名，主要用于图片，不在标准定义当中
+    def normFileName_image(self, fileName):
+        if not "." in fileName:
+            return fileName.upper()
+        tmps = fileName.split(".")
+        fileSufx = tmps[-1].lower()
+        tmps.remove(tmps[-1])
+        filePrfx = ".".join(tmps).upper()
+
+        return "{0}.{1}".format(filePrfx, fileSufx)
+
+    ## 使用murmurhash3算法将新全文文件名进行HASH并按照规则组装成HASH目录
+    def generateHashName(self, fileName):
+        hashCode = binascii.b2a_hex(mmh3.hash_bytes(fileName)).upper()[0:3]
+        firstCode = chr(hashCode[0])
+        secondCode = chr(hashCode[1])
+        thirdCode = chr(hashCode[2])
+        if thirdCode.isdigit():
+            return firstCode + secondCode + str(int(thirdCode) % 5)
+        elif thirdCode == 'D':
+            return firstCode + secondCode + 'A'
+        elif thirdCode == 'E':
+            return firstCode + secondCode + 'B'
+        elif thirdCode == 'F':
+            return firstCode + secondCode + 'C'
+        else:
+            return firstCode + secondCode + thirdCode
+
+    # fileName：文件名称，带后缀，不能为空，专利为公开号加文件后缀名，其他为lngid加文件后缀名
+    # years：年份，不能为空
+    # country：国家，如果为空，默认为cn
+    # type：自建资源类型，不能为空，目前只有bs（博硕）,hy（会议）,bz（标准）,fg（法规）,zl（专利）
+    def generatehashPath(self, fileName, years, country, resType):
+        if years == None or len(years) != 4:
+            raise RuntimeError('years 参数错误！')
+        elif fileName == None or len(fileName) == 0:
+            raise RuntimeError('fileName 参数错误！')
+        elif country == None or len(country) == 0:
+            country = 'cn'
+        if resType in ('bs', 'hy', 'fg', 'zl', 'bz'):
+            country = "cn"
+        country = country.lower()
+        intYear = int(years)
+        if intYear < 1989:
+            years = 'befor1989'
+        fileName = self.normFileName(fileName)
+        return "\\" + resType + "\\" + years + country + self.typeCode + "\\" + self.generateHashName(fileName)+ '\\' + fileName
+        # return '\\' + years + country + _type + '\\' + generateHashName(fileName) + '\\' + fileName
+
+print(Mmh3Hash("3","").normFileName_image("aa.bb.jpg"))
 print(Mmh3Hash("4","zl").generatehashPath("cn206103553u.pdf","2016","cn","zl"))