re-common 2.0.1__py3-none-any.whl → 10.0.1__py3-none-any.whl

re_common/baselibrary/utils/baseftp.py

@@ -0,0 +1,214 @@
+import os
+import socket
+import threading
+import time
+import traceback
+from ftplib import FTP
+
+import socks
+
+
+class BaseFtp(object):
+    def __init__(self, encoding='utf-8'):
+        self.ftp = FTP()
+        self.ftp.encoding = encoding
+
+    def set_proxy_socks(self, type, ip, port):
+        """
+        type = socks.PROXY_TYPE_SOCKS5
+        :param type:
+        :param ip:
+        :param port:
+        :return:
+        """
+        socks.set_default_proxy(type, ip, port)
+        socket.socket = socks.socksocket
+        return self
+
+    def conn_ftp(self, ftp_host, port=21):
+        self.ftp.connect(ftp_host, port)
+        return self
+
+    def login(self, username, passwd):
+        self.ftp.login(username, passwd)
+        return self
+
+    def set_conn_login_pare(self, ftp_host, port, username, passwd):
+        """
+        设置参数
+        :return:
+        """
+        self.ftp_host = ftp_host
+        self.port = port
+        self.username = username
+        self.passwd = passwd
+        return self
+
+    def conn_and_login(self):
+        self.conn_ftp(self.ftp_host, self.port)
+        self.login(self.username, self.passwd)
+        return self
+
+    def cwd(self, dir):
+        """
+        更改到目录
+        :param dires:
+        :return:
+        """
+        return self.ftp.cwd(dir)  # change remote work dir
+
+    def pwd(self):
+        """
+        返回当前FTP操作的路径
+        :return:
+        """
+        return self.ftp.pwd()
+
+    def size(self, remotefile):
+        return self.ftp.size(remotefile)
+
+    def voidcmd(self, cmd):
+        """
+        和FTP.sendcmd(command)功能相似，但返回代码不在200-299之间时抛出异常
+        FTP.sendcmd(command):向服务器发送一条简单的FTP命令，返回响应结果
+        'TYPE I'
+        f"MDTM {ftp_path}" 获取时间
+        :return:
+        """
+        return self.ftp.voidcmd(cmd)
+
+    def retrlines(self, cmd, callback=None):
+        """
+        返回list的数据格式为liunx权限的格式
+        dr-x------   3 user group            0 Jan 26 11:10 CN-IC-DECO-RE 中国集成电路布图设计复审撤销案件数据
+        :param cmd: RETR、LIST或NLST命令
+        LIST以列表形式检索有关文件及其详细信息。NLST只是以列表展示文件名
+        RETR 下载
+        MLSD: 更好的信息展示
+        :param callback: 一个被调用的可选的单个参数,为每一行去掉后面的CRLF
+        比如放入列表 可以传入
+        filelist = []
+        filelist.append
+        :return:
+        """
+        return self.ftp.retrlines(cmd, callback)
+
+    def set_pasv(self, val):
+        """
+        val：为True时，启用被动模式；反之，禁用。默认情况下是被动模式
+        :param val:
+        :return:
+        """
+        return self.ftp.set_pasv(val)
+
+    def get_file_info(self, file_str):
+        """
+        将str 转换成字典
+        :param file_str:
+        :return:
+        """
+        dicts = {}
+        dicts["time"] = []
+        dicts["filename"] = []
+        lists = file_str.split()
+        count = 0
+        for item in lists:
+            item = item.strip()
+            if item != "" and count <= 8:
+                count = count + 1
+                if count == 1:
+                    # 文件属性字段
+                    """
+                    文件属性字段总共有10个字母组成；第一个字符代表文件的类型。
+                    字母“-”表示该文件是一个普通文件
+                    字母“d”表示该文件是一个目录，字母"d"，是dirtectory(目录)的缩写
+                    注意：目录或者是特殊文件，这个特殊文件存放其他文件或目录的相关信息
+                    字母“l”表示该文件是一个链接文件。字母"l"是link(链接)的缩写，类似于windows下的快捷方式
+                    字母“b”的表示块设备文件(block)，一般置于/dev目录下，设备文件是普通文件和程序访问硬件设备的入口，是很特殊的文件。没有文件大小，只有一个主设备号和一个辅设备号。一次传输数据为一整块的被称为块设备，如硬盘、光盘等。最小数据传输单位为一个数据块(通常一个数据块的大小为512字节)
+                    字母为“c”表示该文件是一个字符设备文件(character)，一般置于/dev目录下，一次传输一个字节的设备被称为字符设备，如键盘、字符终端等，传输数据的最小单位为一个字节。
+                    字母为“p”表示该文件为命令管道文件。与shell编程有关的文件。
+                    字母“s”表示该文件为sock文件。与shell编程有关的文件。
+                    """
+                    dicts["file_info"] = item
+                if count == 2:
+                    # 文件硬链接数 如果是一个目录，则第2字段表示该目录所含子目录的个数。
+                    dicts["file_hardlink_num"] = item
+                if count == 3:
+                    dicts["user"] = item
+                if count == 4:
+                    dicts["group"] = item
+
+                if count == 5:
+                    # 文件所占用的空间(以字节为单位)
+                    dicts["size"] = item
+
+                if count in (6, 7, 8):
+                    # 件（目录）最近访问（修改）时间
+                    dicts["time"].append(item)
+                    if count == 8:
+                        count = count + 1
+            else:
+                dicts["filename"].append(item)
+
+        dicts["filename"] = " ".join(dicts["filename"])
+        dicts["time"] = " ".join(dicts["time"])
+        return dicts
+
+    def get_file_info_MLSD(self, file_str):
+        """
+        MLSD 模式下转换为字典
+        :param file_str:
+        :return:
+        """
+        dicts = {}
+        lists = file_str.split(";")
+        dicts["size"] = lists[0].replace("Size=", "")
+        dicts["time"] = lists[1].replace("Modify=", "")
+        dicts["type"] = lists[2].replace("Type=", "")
+        dicts["filename"] = lists[3][1:]
+        return dicts
+
+    def down_file(self, ftp_path, local_path, blocksize=1024):
+        """
+        下载一个文件
+        :return:
+        """
+        local_file_size = 0
+        if os.path.exists(local_path):
+            local_file_size = os.path.getsize(local_path)
+        try:
+            ftp_file_size = self.size(ftp_path)
+            # TYPE I表示以二进制模式传输
+            self.voidcmd('TYPE I')
+            # 下载ftp 目标文件 local_file_size 需要跳过的size
+            conn = self.ftp.transfercmd('RETR ' + ftp_path, local_file_size)
+            with open(local_path, 'ab+') as file:
+                if local_file_size == 0 or local_file_size > ftp_file_size:
+                    file.truncate()
+                recv_num = 0
+                # 次数
+                times = 0
+                while True:
+                    data = conn.recv(blocksize)
+                    times = times + 1
+                    recv_num = recv_num + len(data)
+                    if times >= 1000:
+                        print("P/T {}:{} Time {}".format(os.getpid(), threading.get_ident(),
+                                                         time.strftime('%Y-%m-%d %H:%M:%S',
+                                                                       time.localtime(time.time()))), recv_num)
+                        times = 0
+                        recv_num = 0
+                    if not data:
+                        break
+                    file.write(data)
+            # 此命令不产生什么实际动作，它仅使服务器返回OK。
+            result = self.voidcmd('NOOP')
+            print(result)
+            # 期待以“2”开头的回复
+            result = self.ftp.voidresp()
+            print(result)
+            return True
+        except:
+            self.ftp.quit()
+            traceback.print_exc()
+            return False

re_common/baselibrary/utils/basegzip.py

@@ -0,0 +1,60 @@
+import gzip
+
+from re_common.baselibrary.utils.basefile import BaseFile
+
+
+class BaseGzip(object):
+
+    def __init__(self, bufsize, fin=None, fout=None):
+        self.bufsize = bufsize
+        self.fin = fin
+        self.fout = fout
+
+    def compress(self, src, dst):
+        """
+        压缩文件
+        :param src:
+        :param dst:
+        :return:
+        """
+        self.fin = open(src, 'rb')
+        self.fout = gzip.open(dst, 'wb')
+
+        self.__in2out()
+
+    def decompress(self, gzFile, dst):
+        """
+        解压文件
+        :param gzFile:
+        :param dst:
+        :return:
+        """
+        self.fin = gzip.open(gzFile, 'rb')
+        self.fout = open(dst, 'wb')
+
+        self.__in2out()
+
+    def __in2out(self):
+        while True:
+            buf = self.fin.read(self.bufsize)
+            if len(buf) < 1:
+                break
+            self.fout.write(buf)
+
+        self.fin.close()
+        self.fout.close()
+
+    def read_gz_file(self, file):
+        with gzip.open(file, 'r') as f:
+            for lineb in f:
+                line = lineb.decode()
+                yield line.strip()
+
+    @classmethod
+    def get_gz_line_num(self, file):
+        i = 0
+        if BaseFile.is_file_exists(file):
+            with gzip.open(file, 'rb') as f:
+                for i, l in enumerate(f):
+                    pass
+        return i

re_common/baselibrary/utils/basehdfs.py

@@ -0,0 +1,135 @@
+import os
+import sys
+import time
+
+from pyhdfs import HdfsClient
+
+
+class BaseHDFS(object):
+
+    def __init__(self):
+        self.hdfsdir = ""
+        self.localdir = ""
+        self.namenode = ""
+        self.user_name = ""
+        self.total = 0
+        self.client = None
+        self.FileSize = 0
+        self.FailedList = list()  # 失败文件列表
+        self.StartTime = time.time()
+
+    def get_client(self):
+        self.client = HdfsClient(hosts=self.namenode, user_name=self.user_name)
+
+    def mk_hdfs_dirs(self, path):
+        """
+        创建目录 目录存在创建不会报错 目录里文件不会被删除
+
+        :return:
+        """
+
+        return self.client.mkdirs(path)
+
+    def exists(self, path):
+        """
+        判断路径是否存在
+        :return:
+        """
+        return self.client.exists(path)
+
+    def get_all_files_num(self):
+        """
+        获取目录下所有文件数量
+        :return:
+        """
+        assert isinstance(self.client, HdfsClient)
+        total = 0
+        # 先遍历一遍，得到总文件个数
+        for parent, dirnames, filenames in self.client.walk(self.hdfsdir):
+            for filename in filenames:
+                total += 1
+        self.total = total
+        return total
+
+    def get_all_files(self):
+        assert isinstance(self.client, HdfsClient)
+        processed = 0
+        for parent, dirnames, filenames in self.client.walk(self.hdfsdir):
+            for filename in filenames:
+                srcFile = '%s/%s' % (parent, filename)
+                relPath = srcFile[len(self.hdfsdir) + 1:].replace('/', '\\')  # 相对于根目录的路径
+                dstFile = os.path.join(self.localdir, relPath)
+                if not self.down_proc_one(srcFile, dstFile):
+                    self.FailedList.append(srcFile)
+                processed += 1
+                print('%s:' % self.hdfsdir)
+                print('%d/%d/%d, time cost: %.2f s' % (
+                    self.total, processed, len(self.FailedList), time.time() - self.StartTime))
+                print('%d B, %.2f MB/s \n' % (
+                    self.FileSize, self.FileSize / 1024 / 1024 / (time.time() - self.StartTime)))
+
+    def down_proc_one(self, srcFile, dstFile):
+        print('ProcOne \n%s\n -> \n%s ' % (srcFile, dstFile))
+        dstDir = os.path.dirname(dstFile)
+        if not os.path.exists(dstDir):
+            os.makedirs(dstDir)
+
+        # 目标文件已经存在且大小相同
+        if os.path.exists(dstFile) and \
+                (os.path.getsize(dstFile) == self.client.list_status(srcFile)[0].length):
+            print('file exists: %s ' % dstFile)
+            return True
+
+        # 注意，如果已存在会被覆盖
+        self.client.copy_to_local(srcFile, dstFile, overwrite=True)
+
+        if os.path.getsize(dstFile) != self.client.list_status(srcFile)[0].length:  # 校验文件大小
+            return False
+
+        self.FileSize += os.path.getsize(dstFile)
+        return True
+
+    # 处理一个
+    def up_proc_one(self, client, srcFile, dstFile):
+        print('ProcOne \n%s\n -> \n%s ' % (srcFile, dstFile))
+
+        # 目标文件已经存在且大小相同
+        if client.exists(dstFile) and \
+                (os.path.getsize(srcFile) == client.list_status(dstFile)[0].length):
+            print('file exists: %s ' % dstFile)
+            return True
+
+        # 注意，如果已存在会被覆盖
+        client.copy_from_local(srcFile, dstFile, overwrite=True)
+
+        if os.path.getsize(srcFile) == client.list_status(dstFile)[0].length:  # 校验文件大小
+            self.FileSize += os.path.getsize(srcFile)
+            return True
+
+        return False
+
+    # 处理所有
+    def up_all_files(self):
+        assert isinstance(self.client, HdfsClient)
+        if not self.client.exists(self.hdfsdir):
+            print(self.hdfsdir + ' not found')
+            sys.exit(-1)
+        total = len(os.listdir(self.localdir))
+        processed = 0
+        for filename in os.listdir(self.localdir):
+            srcFile = os.path.join(self.localdir, filename)
+            dstFile = self.hdfsdir + '/' + filename
+            if not self.up_proc_one(self.client, srcFile, dstFile):
+                self.FailedList.append(srcFile)
+            processed += 1
+            print(
+                '%d/%d/%d, time cost: %.2f s' % (total, processed, len(self.FailedList), time.time() - self.StartTime))
+            print('%d B, %.2f MB/s \n' % (self.FileSize, self.FileSize / 1024 / 1024 / (time.time() - self.StartTime)))
+
+        if self.FailedList:
+            print('failedList: %s' % repr(self.FailedList))
+        else:
+            print('Good! No Error!')
+            print('%d B, %.2f MB, %.2f GB, %.2f MB/s' % \
+                  (self.FileSize, self.FileSize / 1024 / 1024, self.FileSize / 1024 / 1024 / 1024,
+                   self.FileSize / 1024 / 1024 / (time.time() - self.StartTime)))

re_common/baselibrary/utils/basehttpx.py

@@ -0,0 +1,268 @@
+"""
+basehttpx
+"""
+import traceback
+
+import httpx
+import urllib3
+from httpx import Timeout
+from httpx._config import UNSET
+
+from re_common.baselibrary.utils.core.mlamada import closeResult
+from re_common.baselibrary.utils.core.requests_core import USER_AGENT, set_proxy_httpx
+
+urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
+
+
+class BaseHttpx(object):
+    def __init__(self, logger=None):
+        if logger == None:
+            from re_common.baselibrary import MLogger
+            logger = MLogger().streamlogger
+        self.logger = logger
+        self.proxies = None
+        self.headers = {}
+        self.sn = None
+        self.as_sn = None
+
+    def httpx_get(self, url,
+                  *,
+                  params=None,
+                  headers=None,
+                  cookies=None,
+                  auth=None,
+                  allow_redirects=True,
+                  cert=None,
+                  verify=True,
+                  timeout=Timeout(timeout=5.0),
+                  trust_env=True):
+        """
+        基础httpx使用教程
+         params = {'key1': 'value1', 'key2': 'value2'}
+         r = httpx.get('https://httpbin.org/get', params=params)
+         r.url
+         params = {'key1': 'value1', 'key2': ['value2', 'value3']}
+         r.status_code
+         r.headers['content-type']
+         r.encoding
+         r.encoding = 'ISO-8859-1'
+         r.content
+         r.text
+         r.json()
+         /////////////////
+         from PIL import Image
+         from io import BytesIO
+         i = Image.open(BytesIO(r.content))
+
+        :param url:
+        :return: r
+        """
+        return httpx.get(url, params=params, headers=headers, cookies=cookies,
+                         auth=auth, follow_redirects=allow_redirects, cert=cert,
+                         verify=verify, timeout=timeout, trust_env=trust_env)
+
+    def httpx_post(self, url, *,
+                   data=None,
+                   files=None,
+                   json=None,
+                   params=None,
+                   headers=None,
+                   cookies=None,
+                   auth=None,
+                   allow_redirects=True,
+                   cert=None,
+                   verify=True,
+                   timeout=Timeout(timeout=5.0),
+                   trust_env=True):
+        """
+        基础的post请求
+        >>> r = httpx.put('https://httpbin.org/put', data={'key': 'value'})
+        >>> r = httpx.delete('https://httpbin.org/delete')
+        >>> r = httpx.head('https://httpbin.org/get')
+        >>> r = httpx.options('https://httpbin.org/get')
+        data={'key': 'value'}
+        :param url:
+        :param data:
+        :return:
+        """
+        r = httpx.post(url, data=data, json=json, files=files,
+                       params=params, headers=headers, cookies=cookies,
+                       auth=auth, follow_redirects=allow_redirects, cert=cert,
+                       verify=verify, timeout=timeout, trust_env=trust_env)
+        return r
+
+    def creat_sn(self, proxy=None, headers=None, verify=False, **kwargs):
+        """
+        创建httpx会话对象
+        :param proxy:
+        :param headers:
+        :param verify:
+        :param kwargs:
+        :return:
+        """
+        if proxy:
+            kwargs["proxies"] = set_proxy_httpx(proxy)
+            self.proxies = kwargs["proxies"]
+        if headers:
+            kwargs["headers"] = headers
+        if headers == "default":
+            kwargs["headers"]['User-Agent'] = USER_AGENT
+        kwargs["verify"] = verify
+        sn = httpx.Client(**kwargs)
+        self.sn = sn
+        return sn
+
+    def sn_close(self):
+        self.sn.close()
+
+    async def create_async_client(self, **kwargs):
+        """
+        创建异步会话对象
+        :param kwargs:
+        :return:
+        """
+        sn = httpx.AsyncClient(**kwargs)
+        self.as_sn = sn
+        return sn
+
+    async def as_sn_close(self):
+        """
+        关闭异步sn
+        """
+        await self.as_sn.close()
+
+    def base_sn_httpx(self, url, sn, endstring="", marks=[], **kwargs):
+
+        r = None
+        exMsg = None
+        try:
+            r = sn.get(url=url, **kwargs)
+        except:
+            exMsg = '* ' + traceback.format_exc()
+            self.logger.error(exMsg)
+        finally:
+            closeResult(r)
+
+        if exMsg:
+            self.logger.info("判断到except，请求出项错误{}".format(exMsg))
+            return False, "httpx", r
+
+        if r.status_code != 200:
+            self.logger.warning('r.status_code:' + str(r.status_code))
+            return False, "code", r
+
+        if endstring:
+            """
+            请求有可能是html或者json等，如果有需要判断html结束的才启动这个选项
+            """
+            html = r.text.strip()
+            if not html.endswith(endstring):
+                self.logger.info("not endswith {}".format(endstring))
+                return False, "endString", r
+
+        if marks:
+            """
+           验证请求是否成功  通过一个特征字符串或者html的标签来查找 保证下载的页面是我需要的页面
+           而不是错误页面
+           特征值有可能没有是网页出现问题  有可能是请求不完全 这个依照情况而定
+            """
+            html = r.text.strip()
+            for mark in marks:
+                if html.find(mark) == -1:
+                    self.logger.info('not found {}'.format(mark))
+                    return False, "Feature err", r
+                else:
+                    self.logger.info("found mark is {}".format(mark))
+
+        return True, "", r
+
+    def base_sn_post_httpx(self, url, sn, data=None, endstring="", marks=[], **kwargs):
+        r = None
+        exMsg = None
+        try:
+            r = sn.post(url=url, data=data, **kwargs)
+        except:
+            exMsg = '* ' + traceback.format_exc()
+            self.logger.error(exMsg)
+        finally:
+            closeResult(r)
+
+        if exMsg:
+            self.logger.info("判断到except，请求出项错误{}".format(exMsg))
+            return False, "httpx", r
+
+        if r.status_code != 200:
+            self.logger.warning('r.status_code:' + str(r.status_code))
+            return False, "code", r
+
+        if endstring:
+            """
+            请求有可能是html或者json等，如果有需要判断html结束的才启动这个选项
+            """
+            html = r.text.strip()
+            if not html.endswith(endstring):
+                self.logger.info("not endswith {}".format(endstring))
+                return False, "endString", r
+
+        if marks:
+            """
+           验证请求是否成功  通过一个特征字符串或者html的标签来查找 保证下载的页面是我需要的页面
+           而不是错误页面
+           特征值有可能没有是网页出现问题  有可能是请求不完全 这个依照情况而定
+            """
+            html = r.text.strip()
+            for mark in marks:
+                if html.find(mark) == -1:
+                    self.logger.info('not found {}'.format(mark))
+                    return False, "Feature err", r
+                else:
+                    self.logger.info("found mark is {}".format(mark))
+
+        return True, "", r
+
+    async def httpx_asyncclient(self, url, params=None,
+                                headers=None,
+                                cookies=None,
+                                auth=None,
+                                allow_redirects=True,
+                                timeout=UNSET):
+        """
+        Python 3.8+ with
+        :param url:
+        :return:
+        """
+        async with httpx.AsyncClient() as client:
+            r = await client.get(url, params=params, headers=headers,
+                                 cookies=cookies, auth=auth, follow_redirects=allow_redirects,
+                                 timeout=timeout)
+        return r
+
+    async def httpx_asyncclient_post(self, url, *,
+                                     data=None,
+                                     files=None,
+                                     json=None,
+                                     params=None,
+                                     headers=None,
+                                     cookies=None,
+                                     auth=None,
+                                     allow_redirects=True,
+                                     timeout=UNSET):
+        async with httpx.AsyncClient() as client:
+            r = await client.post(url, data=data, files=files, json=json,
+                                  params=params, headers=headers, cookies=cookies,
+                                  auth=auth, follow_redirects=allow_redirects,
+                                  timeout=timeout)
+        return r
+
+    async def check_http2(self, url, **kwargs):
+        """
+        检查http2的支持
+        :return:
+        """
+        client = httpx.AsyncClient(http2=True)
+        response = await client.get(url, **kwargs)
+        http_version = response.http_version  # "HTTP/1.0", "HTTP/1.1", or "HTTP/2".
+        if http_version == "HTTP/2":
+            return True, http_version
+        else:
+            return False, http_version