re-common 10.0.39__py3-none-any.whl → 10.0.40__py3-none-any.whl

re_common/libtest/get_attr_test/settings.py

@@ -1,55 +1,55 @@
-# -*- coding: utf-8 -*-
-
-
-
-"""
-CREATE YOUR DEFAULT_CONFIG !
-
-Some configuration:
-        CONCURRENT_REQUESTS     并发数量
-        RETRIES                 重试次数
-        DOWNLOAD_DELAY          下载延时
-        RETRY_DELAY             重试延时
-        DOWNLOAD_TIMEOUT        超时限制
-        USER_AGENT              用户代理
-        LOG_FILE                日志路径
-        LOG_LEVEL               日志等级
-"""
-
-
-CONCURRENT_REQUESTS = 20
-
-
-
-
-
-MIDDLEWARE = [
-'middlewares.middleware',
-            ]
-
-
-# PIPELINES = []
-
-
-
-
-DEFAULT_REQUEST_CONFIG = {
-    "RETRIES": 0,
-    "DOWNLOAD_DELAY": 0,
-    "RETRY_DELAY": 0,
-    "DOWNLOAD_TIMEOUT": 10,
-}
-
-
-# '''生成日志文件'''
-# LOG_FILE = './asyncpy.log'
-# LOG_LEVEL = 'DEBUG'
-
-
-
-# CLOSESPIDER_TIMEOUT = 10
-
-
-
-
+# -*- coding: utf-8 -*-
+
+
+
+"""
+CREATE YOUR DEFAULT_CONFIG !
+
+Some configuration:
+        CONCURRENT_REQUESTS     并发数量
+        RETRIES                 重试次数
+        DOWNLOAD_DELAY          下载延时
+        RETRY_DELAY             重试延时
+        DOWNLOAD_TIMEOUT        超时限制
+        USER_AGENT              用户代理
+        LOG_FILE                日志路径
+        LOG_LEVEL               日志等级
+"""
+
+
+CONCURRENT_REQUESTS = 20
+
+
+
+
+
+MIDDLEWARE = [
+'middlewares.middleware',
+            ]
+
+
+# PIPELINES = []
+
+
+
+
+DEFAULT_REQUEST_CONFIG = {
+    "RETRIES": 0,
+    "DOWNLOAD_DELAY": 0,
+    "RETRY_DELAY": 0,
+    "DOWNLOAD_TIMEOUT": 10,
+}
+
+
+# '''生成日志文件'''
+# LOG_FILE = './asyncpy.log'
+# LOG_LEVEL = 'DEBUG'
+
+
+
+# CLOSESPIDER_TIMEOUT = 10
+
+
+
+
 USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3100.0 Safari/537.36"

re_common/libtest/idencode_test.py

@@ -1,54 +1,54 @@
-
-
-from re_common.vip.baseencodeid import BaseLngid
-
-
-def encode_id():
-    """"
-    由 sub_db_id 和 rawid 得到 lngid。
-    case_insensitive: 标识源网站的 rawid 是否区分大小写
-    """
-
-    sub_db_id = "1"
-    rawid = "A44cbe4375b431741"
-    b = BaseLngid()
-    print("********区分大小写************")
-    lngid = b.GetLngid(sub_db_id,
-                       rawid,
-                       case_insensitive=True)
-    print(lngid)
-    print("********不区分大小写************")
-    lngid = b.GetLngid(sub_db_id,
-                       rawid,
-                       case_insensitive=False)
-    print(lngid)
-
-def decode_id():
-    """
-    limited_id 是由 lngid去除sub_db_id后得到的字符串
-    limited_id超过20长度时，为不可逆的
-    :return:
-    """
-    limited_id_big = "F65F41FFCF049A3A21516C5CDFE40A22"
-    limited_id_small = "HLXMH9XNH5XMJC04HHXMJDG2J919Y"
-    limited_id_err = "123456" * 20
-    b = BaseLngid()
-    print("********区分大小写************")
-    rawid = b.GetRawid(limited_id_big,
-                       case_insensitive=True)
-    print(rawid)
-    print("********不区分大小写************")
-    rawid = b.GetRawid(limited_id_small,
-                       case_insensitive=False)
-    print(rawid)
-    print("*******limited_id超过20长度时*******")
-    rawid = b.GetRawid(limited_id_err,
-                       case_insensitive=False)
-
-
-
-
-
-if __name__ == '__main__':
-    # encode_id()
+
+
+from re_common.vip.baseencodeid import BaseLngid
+
+
+def encode_id():
+    """"
+    由 sub_db_id 和 rawid 得到 lngid。
+    case_insensitive: 标识源网站的 rawid 是否区分大小写
+    """
+
+    sub_db_id = "1"
+    rawid = "A44cbe4375b431741"
+    b = BaseLngid()
+    print("********区分大小写************")
+    lngid = b.GetLngid(sub_db_id,
+                       rawid,
+                       case_insensitive=True)
+    print(lngid)
+    print("********不区分大小写************")
+    lngid = b.GetLngid(sub_db_id,
+                       rawid,
+                       case_insensitive=False)
+    print(lngid)
+
+def decode_id():
+    """
+    limited_id 是由 lngid去除sub_db_id后得到的字符串
+    limited_id超过20长度时，为不可逆的
+    :return:
+    """
+    limited_id_big = "F65F41FFCF049A3A21516C5CDFE40A22"
+    limited_id_small = "HLXMH9XNH5XMJC04HHXMJDG2J919Y"
+    limited_id_err = "123456" * 20
+    b = BaseLngid()
+    print("********区分大小写************")
+    rawid = b.GetRawid(limited_id_big,
+                       case_insensitive=True)
+    print(rawid)
+    print("********不区分大小写************")
+    rawid = b.GetRawid(limited_id_small,
+                       case_insensitive=False)
+    print(rawid)
+    print("*******limited_id超过20长度时*******")
+    rawid = b.GetRawid(limited_id_err,
+                       case_insensitive=False)
+
+
+
+
+
+if __name__ == '__main__':
+    # encode_id()
     decode_id()

re_common/libtest/iniconfig_test.py

@@ -1,35 +1,35 @@
-def test_BaseAbs_config():
-    """
-    命令行下　在当前目录使用该命令测试
-    pytest iniconfig_test.py::test_BaseAbs_config -s
-    :return:
-    """
-    from re_common.baselibrary.baseabs import BaseAbs
-    # get_conf_dict　参数
-    # configfile 不传为前面配置好的路径　也可以给新的路径
-    # encoding 指定编码
-    # keep_keys_case 保持建大写，配置为Ｆａｓｌｅ会全部小写
-    orderdicts = BaseAbs.get_config_factory().get_config_factory().ini_factory().set_config_path(
-        "./config.ini").set_keep_keys_case(False).builder().get_config_dict()
-
-    print(orderdicts)
-    print(orderdicts["db"])
-    print(orderdicts["db"]["psw"])
-
-
-def test_Iniconfig_config():
-    """
-    第二种简便用法
-    命令行下　在当前目录使用该命令测试
-    pytest iniconfig_test.py::test_Iniconfig_config -s
-    :return:
-    """
-    from re_common.baselibrary import IniConfig
-    orderdicts = IniConfig("./config.ini").set_keep_keys_case(False).builder().get_config_dict()
-    print(orderdicts)
-    print(orderdicts["db"])
-    print(orderdicts["Db2"]["psw"])
-
-# 已经导如facade 可以通过　facade 直接导入
-# from facade import IniConfig
-# 建议外部使用导入时　全部通过facade 导入
+def test_BaseAbs_config():
+    """
+    命令行下　在当前目录使用该命令测试
+    pytest iniconfig_test.py::test_BaseAbs_config -s
+    :return:
+    """
+    from re_common.baselibrary.baseabs import BaseAbs
+    # get_conf_dict　参数
+    # configfile 不传为前面配置好的路径　也可以给新的路径
+    # encoding 指定编码
+    # keep_keys_case 保持建大写，配置为Ｆａｓｌｅ会全部小写
+    orderdicts = BaseAbs.get_config_factory().get_config_factory().ini_factory().set_config_path(
+        "./config.ini").set_keep_keys_case(False).builder().get_config_dict()
+
+    print(orderdicts)
+    print(orderdicts["db"])
+    print(orderdicts["db"]["psw"])
+
+
+def test_Iniconfig_config():
+    """
+    第二种简便用法
+    命令行下　在当前目录使用该命令测试
+    pytest iniconfig_test.py::test_Iniconfig_config -s
+    :return:
+    """
+    from re_common.baselibrary import IniConfig
+    orderdicts = IniConfig("./config.ini").set_keep_keys_case(False).builder().get_config_dict()
+    print(orderdicts)
+    print(orderdicts["db"])
+    print(orderdicts["Db2"]["psw"])
+
+# 已经导如facade 可以通过　facade 直接导入
+# from facade import IniConfig
+# 建议外部使用导入时　全部通过facade 导入

re_common/libtest/ip_test.py

@@ -1,35 +1,35 @@
-from re_common.baselibrary.utils.basehttpx import BaseHttpx
-from re_common.baselibrary.utils.baserequest import BaseRequest
-import re
-
-url = "https://www.ip.cn/"
-
-
-def basehttpx_test_ip(proxy):
-    patten = "<code>(.*?)</code>"
-    bshttpx = BaseHttpx()
-    BoolResult, errString, r = bshttpx.base_sn_httpx(url=url,
-                                                     proxies=proxy,
-                                                     headers="")
-    ip = re.findall(patten,r.text)
-    if len(ip) == 0:
-        print(r.text)
-    else:
-        print(ip)
-
-def baserequest_test_ip(proxy):
-    patten = "Your IP</span>: (.*?)</span>"
-    bsrequests = BaseRequest()
-    bsrequests.is_use_proxy(True)
-    bsrequests.set_proxy(proxy)
-    BoolResult, errString, r = bsrequests.base_request(url=url)
-    ip = re.findall(patten, r.text)
-    if len(ip) == 0:
-        print(r.text)
-    else:
-        print(ip)
-
-
-if __name__ == '__main__':
-    basehttpx_test_ip("192.168.31.176:8012")
+from re_common.baselibrary.utils.basehttpx import BaseHttpx
+from re_common.baselibrary.utils.baserequest import BaseRequest
+import re
+
+url = "https://www.ip.cn/"
+
+
+def basehttpx_test_ip(proxy):
+    patten = "<code>(.*?)</code>"
+    bshttpx = BaseHttpx()
+    BoolResult, errString, r = bshttpx.base_sn_httpx(url=url,
+                                                     proxies=proxy,
+                                                     headers="")
+    ip = re.findall(patten,r.text)
+    if len(ip) == 0:
+        print(r.text)
+    else:
+        print(ip)
+
+def baserequest_test_ip(proxy):
+    patten = "Your IP</span>: (.*?)</span>"
+    bsrequests = BaseRequest()
+    bsrequests.is_use_proxy(True)
+    bsrequests.set_proxy(proxy)
+    BoolResult, errString, r = bsrequests.base_request(url=url)
+    ip = re.findall(patten, r.text)
+    if len(ip) == 0:
+        print(r.text)
+    else:
+        print(ip)
+
+
+if __name__ == '__main__':
+    basehttpx_test_ip("192.168.31.176:8012")
     baserequest_test_ip("192.168.31.176:8012")

re_common/libtest/merge_file_test.py

@@ -1,20 +1,20 @@
-###########################################
-# 同项目调用基础包
-import os
-import sys
-
-filepath = os.path.abspath(__file__)
-pathlist = filepath.split(os.sep)
-pathlist = pathlist[:-3]
-TopPath = os.sep.join(pathlist)
-sys.path.insert(0, TopPath)
-print(TopPath)
-############################################
-from re_common.baselibrary.tools.merge_file import MergeFile
-
-if __name__ == '__main__':
-    old_path = r'E:\work\pnasjournal\big_json\20200116'
-    new_path = r'E:\work\pnasjournal\big_json\new'
-    a = MergeFile()
-    # a.line_all_2_one(old_path,new_path)
-    a.block_all_2_one(old_path, new_path, dst_size=1024 * 1024 * 200, block_size=200 * 1024 * 1024)
+###########################################
+# 同项目调用基础包
+import os
+import sys
+
+filepath = os.path.abspath(__file__)
+pathlist = filepath.split(os.sep)
+pathlist = pathlist[:-3]
+TopPath = os.sep.join(pathlist)
+sys.path.insert(0, TopPath)
+print(TopPath)
+############################################
+from re_common.baselibrary.tools.merge_file import MergeFile
+
+if __name__ == '__main__':
+    old_path = r'E:\work\pnasjournal\big_json\20200116'
+    new_path = r'E:\work\pnasjournal\big_json\new'
+    a = MergeFile()
+    # a.line_all_2_one(old_path,new_path)
+    a.block_all_2_one(old_path, new_path, dst_size=1024 * 1024 * 200, block_size=200 * 1024 * 1024)

re_common/libtest/mfaker_test.py

@@ -1,9 +1,9 @@
-import time
-
-from re_common.baselibrary.utils.basefile import BaseFile
-
-with open(r"\\192.168.31.171\caiji\test.txt", 'w', encoding="utf-8") as f:
-    while True:
-        f.write("test\n")
-        time.sleep(10)
+import time
+
+from re_common.baselibrary.utils.basefile import BaseFile
+
+with open(r"\\192.168.31.171\caiji\test.txt", 'w', encoding="utf-8") as f:
+    while True:
+        f.write("test\n")
+        time.sleep(10)
         print("1")

re_common/libtest/mm3_test.py

@@ -1,32 +1,32 @@
-import platform
-import traceback
-
-import win32file
-
-from re_common.baselibrary.utils.basefile import BaseFile
-
-
-# aa = r"F:\cnipa_ftp\SIPO\CN-PA-IMGS-10-A 中国发明专利申请公布标准化全文图像数据\20211109\20211109-1-002.ZIP"
-# aa = r"F:\cnipa_ftp\SIPO\CN-PA-IMGS-10-A 中国发明专利申请公布标准化全文图像数据\20211109\20211109-1-001.ZIP"
-def is_used(file_name):
-    if "Windows" == platform.system():
-        import win32file
-        try:
-            vHandle = win32file.CreateFile(file_name, win32file.GENERIC_READ, 0, None, win32file.OPEN_EXISTING,
-                                           win32file.FILE_ATTRIBUTE_NORMAL, None)
-            return int(vHandle) == win32file.INVALID_HANDLE_VALUE
-        except:
-            if "另一个程序正在使用此文件，进程无法访问。" in traceback.format_exc():
-                return True
-            else:
-                return False
-        finally:
-            try:
-                win32file.CloseHandle(vHandle)
-            except:
-                pass
-    else:
-        raise Exception("不是windows系统，请不要调用该函数判断文件是否被打开")
-
-aa = is_used(r"\\192.168.31.171\caiji\test.txt")
+import platform
+import traceback
+
+import win32file
+
+from re_common.baselibrary.utils.basefile import BaseFile
+
+
+# aa = r"F:\cnipa_ftp\SIPO\CN-PA-IMGS-10-A 中国发明专利申请公布标准化全文图像数据\20211109\20211109-1-002.ZIP"
+# aa = r"F:\cnipa_ftp\SIPO\CN-PA-IMGS-10-A 中国发明专利申请公布标准化全文图像数据\20211109\20211109-1-001.ZIP"
+def is_used(file_name):
+    if "Windows" == platform.system():
+        import win32file
+        try:
+            vHandle = win32file.CreateFile(file_name, win32file.GENERIC_READ, 0, None, win32file.OPEN_EXISTING,
+                                           win32file.FILE_ATTRIBUTE_NORMAL, None)
+            return int(vHandle) == win32file.INVALID_HANDLE_VALUE
+        except:
+            if "另一个程序正在使用此文件，进程无法访问。" in traceback.format_exc():
+                return True
+            else:
+                return False
+        finally:
+            try:
+                win32file.CloseHandle(vHandle)
+            except:
+                pass
+    else:
+        raise Exception("不是windows系统，请不要调用该函数判断文件是否被打开")
+
+aa = is_used(r"\\192.168.31.171\caiji\test.txt")
 print(aa)

re_common/libtest/mylogger_test.py

@@ -1,89 +1,89 @@
-import logging
-
-from re_common.baselibrary.utils.basedir import BaseDir
-from re_common.baselibrary import MLogger
-from re_common.baselibrary.utils.basefile import BaseFile
-
-
-def test_mlogger_streamlogger():
-    """
-    只打印数据不计入文件
-    pytest mylogger_test.py::test_mlogger_streamlogger -s
-    :return:
-    """
-    streamlogger = MLogger().streamlogger
-    streamlogger.setLevel(logging.DEBUG)
-    streamlogger.info("mesg info")
-    streamlogger.warning("msg warning")
-    streamlogger.warn("msg warn")
-    streamlogger.error("msg error")
-    streamlogger.debug("msg debug")
-    streamlogger.critical("msg critical")
-    streamlogger.exception("msg exception")
-
-
-def test_mlogger_filelogger():
-    """
-    测试将日志写入文件中　文件名以日期自动生成　如20200211.log
-    pytest mylogger_test.py::test_mlogger_filelogger -s
-    文件流的logger
-    :param filedir:
-    :return:
-    """
-    filedir = BaseDir.get_file_dir_absolute(__file__)
-    filelogger = MLogger(filedir).filelogger
-    filelogger.setLevel(logging.DEBUG)
-    filelogger.info("mesg info")
-    filelogger.warning("msg warning")
-    filelogger.warn("msg warn")
-    filelogger.error("msg error")
-    filelogger.debug("msg debug")
-    filelogger.critical("msg critical")
-    filelogger.exception("msg exception")
-
-
-def test_mlogger_allloger():
-    """
-    测试将日志写入文件中 有一个以日期的总文件　以及每个级别的分文件
-    pytest mylogger_test.py::test_mlogger_allloger -s
-    文件流的logger
-    :param filedir:
-    :return:
-    """
-    filedir = BaseDir.get_file_dir_absolute(__file__)
-    filedir = BaseFile.get_new_path(filedir, "log")
-    BaseDir.create_dir(filedir)
-    filelogger = MLogger(filedir).alllogger
-    filelogger.setLevel(logging.DEBUG)
-    filelogger.info("mesg info")
-    filelogger.warning("msg warning")
-    filelogger.warn("msg warn")
-    filelogger.error("msg error")
-    filelogger.debug("msg debug")
-    filelogger.critical("msg critical")
-    filelogger.exception("msg exception")
-
-
-def test_mlogger_rotatingfilelogger():
-    """
-    测试将日志写入文件中　文件名以日期自动生成　如20200211.log
-    pytest mylogger_test.py::test_mlogger_rotatingfilelogger -s
-    文件流的logger
-    :param filedir:
-    :return:
-    """
-    filedir = BaseDir.get_file_dir_absolute(__file__)
-    filelogger = MLogger(filedir).RotatingFileLogger(filedir, None, maxBytes=1*1024*1024,
-                                                               backupCount=10)
-    filelogger.setLevel(logging.DEBUG)
-    while True:
-        print("**************")
-        filelogger.info("mesg info")
-        filelogger.warning("msg warning")
-        filelogger.warn("msg warn")
-        filelogger.error("msg error")
-        filelogger.debug("msg debug")
-        filelogger.critical("msg critical")
-        filelogger.exception("msg exception")
-
+import logging
+
+from re_common.baselibrary.utils.basedir import BaseDir
+from re_common.baselibrary import MLogger
+from re_common.baselibrary.utils.basefile import BaseFile
+
+
+def test_mlogger_streamlogger():
+    """
+    只打印数据不计入文件
+    pytest mylogger_test.py::test_mlogger_streamlogger -s
+    :return:
+    """
+    streamlogger = MLogger().streamlogger
+    streamlogger.setLevel(logging.DEBUG)
+    streamlogger.info("mesg info")
+    streamlogger.warning("msg warning")
+    streamlogger.warn("msg warn")
+    streamlogger.error("msg error")
+    streamlogger.debug("msg debug")
+    streamlogger.critical("msg critical")
+    streamlogger.exception("msg exception")
+
+
+def test_mlogger_filelogger():
+    """
+    测试将日志写入文件中　文件名以日期自动生成　如20200211.log
+    pytest mylogger_test.py::test_mlogger_filelogger -s
+    文件流的logger
+    :param filedir:
+    :return:
+    """
+    filedir = BaseDir.get_file_dir_absolute(__file__)
+    filelogger = MLogger(filedir).filelogger
+    filelogger.setLevel(logging.DEBUG)
+    filelogger.info("mesg info")
+    filelogger.warning("msg warning")
+    filelogger.warn("msg warn")
+    filelogger.error("msg error")
+    filelogger.debug("msg debug")
+    filelogger.critical("msg critical")
+    filelogger.exception("msg exception")
+
+
+def test_mlogger_allloger():
+    """
+    测试将日志写入文件中 有一个以日期的总文件　以及每个级别的分文件
+    pytest mylogger_test.py::test_mlogger_allloger -s
+    文件流的logger
+    :param filedir:
+    :return:
+    """
+    filedir = BaseDir.get_file_dir_absolute(__file__)
+    filedir = BaseFile.get_new_path(filedir, "log")
+    BaseDir.create_dir(filedir)
+    filelogger = MLogger(filedir).alllogger
+    filelogger.setLevel(logging.DEBUG)
+    filelogger.info("mesg info")
+    filelogger.warning("msg warning")
+    filelogger.warn("msg warn")
+    filelogger.error("msg error")
+    filelogger.debug("msg debug")
+    filelogger.critical("msg critical")
+    filelogger.exception("msg exception")
+
+
+def test_mlogger_rotatingfilelogger():
+    """
+    测试将日志写入文件中　文件名以日期自动生成　如20200211.log
+    pytest mylogger_test.py::test_mlogger_rotatingfilelogger -s
+    文件流的logger
+    :param filedir:
+    :return:
+    """
+    filedir = BaseDir.get_file_dir_absolute(__file__)
+    filelogger = MLogger(filedir).RotatingFileLogger(filedir, None, maxBytes=1*1024*1024,
+                                                               backupCount=10)
+    filelogger.setLevel(logging.DEBUG)
+    while True:
+        print("**************")
+        filelogger.info("mesg info")
+        filelogger.warning("msg warning")
+        filelogger.warn("msg warn")
+        filelogger.error("msg error")
+        filelogger.debug("msg debug")
+        filelogger.critical("msg critical")
+        filelogger.exception("msg exception")
+
 test_mlogger_rotatingfilelogger()