re-common 0.2.54__py3-none-any.whl → 2.0.1__py3-none-any.whl

re_common/baselibrary/utils/baseboto3.py

@@ -1,291 +0,0 @@
-import boto3
-import botocore
-from boto3.session import Session
-
-# boto3 该开发工具包由两个关键的 Python 包组成：
-# Botocore（提供在 Python 开发工具包和 AWS CLI 之间共享的低级功能的库）
-# 和 Boto3（实现 Python 开发工具包本身的包）
-
-
-"""
-aws_access_key_id = 'minioa'
-aws_secret_access_key = 'minio123'
-endpoint_url = 'http://192.168.31.164:9000'
-bbt = BaseBoto3(aws_access_key_id=aws_access_key_id,
-                aws_secret_access_key=aws_secret_access_key,
-                endpoint_url=endpoint_url)
-bbt.conn_session()
-bbt.set_is_low_level(False)
-bbt.get_client()
-print("**********************")
-print(bbt.delete_buckets("test1"))
-# bbt.set_is_low_level(False)
-# bbt.get_client()
-# print("**********************")
-# print(bbt.create_buckets("create2"))
-"""
-
-
-class BaseBoto3(object):
-
-    def __init__(self, aws_access_key_id="", aws_secret_access_key="", endpoint_url=""):
-        self.aws_access_key_id = aws_access_key_id
-        self.aws_secret_access_key = aws_secret_access_key
-        self.endpoint_url = endpoint_url
-        self.session = None
-        self.client = None
-        self.is_low_level = False
-        self.bucket = None
-
-    def set_is_low_level(self, is_low_level):
-        self.is_low_level = is_low_level
-        return self
-
-    def set_aws_access_key_id(self, aws_access_key_id):
-        self.aws_access_key_id = aws_access_key_id
-        return self
-
-    def set_aws_secret_access_key(self, aws_secret_access_key):
-        self.aws_secret_access_key = aws_secret_access_key
-        return self
-
-    def set_endpoint_url(self, endpoint_url):
-        self.endpoint_url = endpoint_url
-        return self
-
-    def conn_session(self):
-        self.session = Session(aws_access_key_id=self.aws_access_key_id,
-                               aws_secret_access_key=self.aws_secret_access_key)
-        return self.session
-
-    def get_client(self):
-        assert self.session is not None
-        if self.is_low_level:
-            # 根据名称创建低级服务客户端
-            # botocore.client.S3
-            self.client = self.session.client('s3', endpoint_url=self.endpoint_url)
-            print(type(self.client))
-        else:
-            # boto3.resources.factory.s3.ServiceResource
-            # 按名称创建资源服务客户端
-            self.client = self.session.resource('s3', endpoint_url=self.endpoint_url)
-            print(type(self.client))
-        return self
-
-    def get_all_buckets(self):
-        """
-        获取所有的桶信息
-        :return:
-        """
-        if self.is_low_level is False:
-            return self.client.buckets.all()
-        else:
-            return self.client.list_buckets()
-
-    def create_buckets(self, buckets_name):
-        """
-
-        :param buckets_name:
-        :return:
-        如果get_client 使用 client 返回
-        {'ResponseMetadata': {'RequestId': '16BC90EED4A433C4', 'HostId': '', 'HTTPStatusCode': 200, 'HTTPHeaders': {'accept-ranges': 'bytes', 'content-length': '0', 'content-security-policy': 'block-all-mixed-content', 'location': '/create1', 'server': 'MinIO', 'strict-transport-security': 'max-age=31536000; includeSubDomains', 'vary': 'Origin, Accept-Encoding', 'x-amz-request-id': '16BC90EED4A433C4', 'x-content-type-options': 'nosniff', 'x-xss-protection': '1; mode=block', 'date': 'Wed, 01 Dec 2021 07:28:39 GMT'}, 'RetryAttempts': 0}, 'Location': '/create1'}
-        如果resource 使用 client 返回
-        s3.Bucket(name='create2')
-        """
-        assert buckets_name.find("_") == -1, "新建一个bucket桶(bucket name 中不能有_下划线)"
-        # 新建一个bucket桶(bucket name 中不能有_下划线)
-        return self.client.create_bucket(Bucket=buckets_name)
-
-    def delete_buckets(self, bucket_name):
-        """
-        删除桶 删除bucket(只能删除空的bucket)
-        :return:
-        """
-        if self.is_low_level is False:
-            bucket = self.client.Bucket(bucket_name)
-            response = bucket.delete()
-        else:
-            response = self.client.delete_bucket(Bucket=bucket_name)
-        return response
-
-    def get_bucket(self, bucket_name):
-        """
-        获取 bucket 对象
-        :param bucket_name:
-        :return:
-        """
-        if self.is_low_level is False:
-            self.bucket = self.client.Bucket(bucket_name)
-            return self.bucket
-        else:
-            raise Exception("无实现方法")
-
-    def get_all_obs_filter(self, Prefix):
-        """
-        Prefix 为匹配模式
-        例：列出前缀为 haha 的文件
-        Prefix='haha'
-        :param Prefix:
-        :return: 可以调用 obj.key
-        """
-        if not self.is_low_level:
-            for obj in self.bucket.objects.filter(Prefix=Prefix):
-                yield obj
-        else:
-            raise Exception("请设置 is_low_level 为 False")
-
-    def get_object(self, bucket_name):
-        """
-        会返回包括目录在内的所有对象
-        :param bucket_name:
-        :return:
-        """
-        if self.is_low_level is False:
-            bucket = self.client.Bucket(bucket_name)
-            # boto3.resources.collection.s3.Bucket.objectsCollection
-            all_obj = bucket.objects.all()
-            return all_obj
-            # for obj in bucket.objects.all():
-            #     print('obj name:%s' % obj.key)
-        else:
-            return self.client.list_objects(Bucket=bucket_name)
-
-    def upload_file(self, local_file, bucket_name, key):
-        """
-        # key 桶中的位置 test1/test.pdf
-        :param local_file:  本地文件路径
-        :param bucket_name: 桶名
-        :param key: 远程文件路径
-        :return:
-        """
-
-        if self.is_low_level is False:
-            self.client.Bucket(bucket_name).upload_file(local_file, key)
-        else:
-            self.client.upload_file(local_file, bucket_name, key)
-
-    def upload_fileobj(self, fileobj, bucket_name, key):
-        # fileobj 字节流
-        if self.is_low_level is False:
-            self.client.Bucket(bucket_name).upload_fileobj(fileobj, key)
-        else:
-            self.client.upload_fileobj(fileobj, bucket_name, key)
-
-    def download_file(self, local_file, bucket_name, key):
-        if self.is_low_level is False:
-            self.client.Bucket(bucket_name).download_file(key, local_file)
-        else:
-            self.client.download_file(bucket_name, key, local_file)
-
-    def download_fileobj(self, fileobj, bucket_name, key):
-        if self.is_low_level is False:
-            self.client.Bucket(bucket_name).download_fileobj(key, fileobj)
-        else:
-            self.client.download_fileobj(bucket_name, key, fileobj)
-
-    def check_exist(self, bucket_name, key):
-        """
-         if self.is_low_level:
-            {'ResponseMetadata': {'RequestId': '17E6A65A2B299D3B', 'HostId': '', 'HTTPStatusCode': 200, 'HTTPHeaders': {'accept-ranges': 'bytes', 'content-length': '117', 'content-security-policy': 'block-all-mixed-content', 'content-type': 'binary/octet-stream', 'etag': '"2237a934f176003e41abf3d733291079"', 'last-modified': 'Thu, 25 Jul 2024 05:49:43 GMT', 'server': 'MinIO', 'strict-transport-security': 'max-age=31536000; includeSubDomains', 'vary': 'Origin, Accept-Encoding', 'x-amz-request-id': '17E6A65A2B299D3B', 'x-content-type-options': 'nosniff', 'x-xss-protection': '1; mode=block', 'date': 'Mon, 29 Jul 2024 09:53:33 GMT'}, 'RetryAttempts': 0}, 'AcceptRanges': 'bytes', 'LastModified': datetime.datetime(2024, 7, 25, 5, 49, 43, tzinfo=tzutc()), 'ContentLength': 117, 'ETag': '"2237a934f176003e41abf3d733291079"', 'ContentType': 'binary/octet-stream', 'Metadata': {}}
-        判定文件是否存在，
-        :param bucket_name: 桶名
-        :param key:  文件key
-        :return:
-        """
-
-        if self.is_low_level:
-            try:
-                obj_info = self.client.head_object(
-                    Bucket=bucket_name,
-                    Key=key
-                )
-                return obj_info
-            except:
-                return None
-        else:
-            # 获取指定存储桶
-            bucket = self.client.Bucket(bucket_name)
-            # 检查 key 是否存在
-            objs = list(bucket.objects.filter(Prefix=key))
-            if len(objs) > 0 and objs[0].key == key:
-                # [s3.ObjectSummary(bucket_name='crawl.dc.cqvip.com', key='foreign/organ/parsel_organ_log.txt')]
-                return objs[0]
-            else:
-                return None
-
-    def get_prefix_obj(self, bucket, prefix, delimiter):
-        """
-        Bucket="crawl.dc.cqvip.com",
-         Prefix="foreign/organ/ei/",
-         Delimiter='/'
-
-        Returns:
-
-        """
-        if self.is_low_level:
-            # for common_prefix in response.get('CommonPrefixes', []): 获取子目录
-            return self.client.list_objects_v2(Bucket=bucket,
-                                               Prefix=prefix,
-                                               Delimiter=delimiter)  # 设置 Delimiter='/' 可以确保只列出目录而不是所有对象。
-        else:
-            # 该方法只能列出文件 没办法列出目录
-            # bucket_resource = self.client.Bucket(bucket)
-            # objects = bucket_resource.objects.filter(Prefix=prefix, Delimiter=delimiter)
-            # return list(objects)
-
-            bucket_resource = self.client.Bucket(bucket)
-            return bucket_resource.meta.client.list_objects_v2(Bucket=bucket,
-                                                               Prefix=prefix,
-                                                               Delimiter=delimiter)
-
-    def get_object_value(self, bucket_name, file_key, encoding='utf-8'):
-        """
-        读取文本数据
-        Returns:
-
-        """
-        if self.is_low_level:
-            obj = self.client.get_object(Bucket=bucket_name, Key=file_key)
-            body = obj['Body'].read().decode(encoding)
-            return body
-        else:
-            bucket_resource = self.client.Bucket(bucket_name)
-            obj = bucket_resource.Object(file_key)
-            body = obj.get()['Body'].read().decode(encoding)
-            return body
-
-    def put_object(self, bucket_name, key, body):
-        """
-        直接写内容到文件
-        Args:
-            bucket_name:
-            key:
-            body: 需要 编码 .encode('utf-8')
-
-        Returns:
-
-        """
-        if self.is_low_level:
-            self.client.put_object(Bucket=bucket_name,
-                                   Key=key,
-                                   Body=body)
-        else:
-            # 获取 Bucket 对象
-            bucket_resource = self.client.Bucket(bucket_name)
-
-            # 将内容写入文件
-            bucket_resource.put_object(Key=key, Body=body)
-
-
-bb = BaseBoto3(aws_access_key_id="UM51J2G5ZG0FE5CCERB9",
-               aws_secret_access_key="u+OEmhE2fahF2L+oXB+HXe8IJs22Lo38icvlF+Yq",
-               endpoint_url="http://192.168.31.135:9000"
-               )
-bb.conn_session()
-bb.set_is_low_level(False)
-bb.get_client()
-
-result = bb.check_exist("crawl.dc.cqvip.com",
-                        "foreign/organ/parsel_organ_log.txt")
-
-print(result)

re_common/baselibrary/utils/basecsv.py

@@ -1,32 +0,0 @@
-import csv
-
-
-class BaseCsv(object):
-
-    def __init__(self):
-        pass
-
-    def read_csv(self, filepath):
-        """
-        根据文件路径逐行读取csv文件
-        :param filepath:csv文件路径
-        :return:csv每行记录
-        """
-        # mac_roman 原因
-        # https://stackoverflow.com/questions/21504319/python-3-csv-file-giving-unicodedecodeerror-utf-8-codec-cant-decode-byte-err
-        with open(filepath, "r",encoding='mac_roman') as f:
-            reader = csv.reader(f)
-            for row in reader:
-                yield row
-
-    def read_all_csv(self, filepath):
-        """
-        根据文件路径读取csv文件所有行
-        :param filepath:csv文件路径
-        :return:csv所有行记录
-        """
-        with open(filepath, "r") as f:
-            reader = csv.reader(f)
-            result = list(reader)
-            return result
-

re_common/baselibrary/utils/basedict.py

@@ -1,133 +0,0 @@
-import copy
-import sys
-
-
-class BaseDicts(object):
-
-    @classmethod
-    def removeDictsNone(self, dicts: dict) -> dict:
-        """
-        去除字典中值为None的键值
-        :param dicts:
-        :return:
-        """
-        return {key: value for key, value in dicts.items() if value is not None}
-
-    @classmethod
-    def removeDictsStringNull(self, dicts: dict) -> dict:
-        """
-        去除字典中值为''的键值
-        :param dicts:
-        :return:
-        """
-        return {key: value for key, value in dicts.items() if value != ""}
-
-    @classmethod
-    def removeDictsAllNone(self, dicts: dict) -> dict:
-        """
-        去除字典中值为'' 和 None 的键值
-        :param dicts:
-        :return:
-        """
-        return {key: value for key, value in dicts.items() if value != "" and value is not None}
-
-    @classmethod
-    def sortkeys(self, dicts, reverse=False):
-        """
-        默认升序排序，加  reverse = True 指定为降序排序
-        通过keys 对dicts 排序
-        经过测试是新的列表
-        :return:
-        """
-        return {k: dicts[k] for k in sorted(dicts.keys(), reverse=reverse)}
-
-    @classmethod
-    def sortvalues(self, dicts, reverse=False):
-        """
-        默认升序排序，加  reverse = True 指定为降序排序
-        d[1] 为值　ｄ[0]　为键
-        d 为元组　为dicts的键值
-        通过　values 对dicts 排序
-        :param dicts:
-        :return:
-        """
-        return {k: v for k, v in sorted(dicts.items(), key=lambda d: d[1], reverse=reverse)}
-
-    @classmethod
-    def is_key_have(cls, dicts, key):
-        """
-        判断key 是否存在,但只能判断一个层次
-        :param dicts:
-        :param key:
-        :return:
-        """
-        if key in dicts.keys():
-            return True
-        else:
-            return False
-
-    @classmethod
-    def is_more_key_have(cls, dicts, keys=[]):
-        """
-        判断多个key 是否存在　可以有更深的层次
-        :param dicts:
-        :param keys: ["a.b","c.d"]
-        :return:
-        """
-        for item in keys:
-            if item.find("."):
-                allstrings = ""
-                for key in item.split("."):
-                    allstrings = allstrings + '["{}"]'.format(key)
-                try:
-                    eval("dicts" + allstrings)
-                except:
-                    return False
-            else:
-                if item not in dicts.keys():
-                    return False
-        return True
-
-    @classmethod
-    def get_recursive_dict(cls, dict_a, key, call_back):
-        if isinstance(dict_a, dict):
-            for key, value in dict_a.items():
-                if isinstance(value, list) or isinstance(value, tuple) or isinstance(value, dict):
-                    dict_a[key] = cls.get_recursive_dict(value, key, call_back)
-                else:
-                    dict_a[key] = call_back(key, value)
-
-        elif isinstance(dict_a, list):
-            # 如果列表中存在一个不是字典就不需要遍历了
-            if dict_a == []:
-                return call_back(key, dict_a)
-            is_true = True
-            for i in dict_a:
-                if isinstance(i, dict):
-                    is_true = True
-                else:
-                    is_true = False
-                    continue
-            if is_true:
-                dict_temp = []
-                for value in dict_a:
-                    dict_temp.append(cls.get_recursive_dict(value, key, call_back))
-                return dict_temp
-            else:
-                return call_back(key, dict_a)
-        elif isinstance(dict_a, tuple):
-            # 如果存在元组转成列表使用
-            return cls.get_recursive_dict(list(dict_a), key, call_back)
-        else:
-            assert False, "传入类型错误：{}".format(type(dict_a))
-        return dict_a
-
-    @classmethod
-    def is_dict_exit_key(cls, dicts, key_name, default=""):
-        if dicts is not None and isinstance(dicts, dict):
-            if dicts.get(key_name, 0) != 0:
-                return dicts[key_name]
-            else:
-                return default
-        else:
-            return default

re_common/baselibrary/utils/basedir.py

@@ -1,241 +0,0 @@
-import os
-import shutil
-import warnings
-
-from re_common.baselibrary.utils.basefile import BaseFile
-
-
-class BaseDir(object):
-
-    @classmethod
-    def get_dir_all_files(cls, path):
-        """
-        返回所有的文件 不会返回目录 目的是得到一个文件列表
-        # 这里注意topdown参数。
-        # topdown决定遍历的顺序，
-        # 如果topdown为True，则先列举top下的目录，然后是目录的目录，依次类推；
-        # 反之，则先递归列举出最深层的子目录，然后是其兄弟目录，然后父目录。
-        # 我们需要先修改深层的子目录
-        :param path:  给定一个路径 如 C:/Users/XD/Pictures/所有视频/
-        :return:
-        """
-        # 遍历子目录
-        assert cls.is_dir(path), NotADirectoryError(path + " not is a dir please check")
-        for path, dirNames, fileNames in os.walk(path, topdown=True):
-            # 获取当前目录的所有文件
-            for fileName in fileNames:
-                yield os.path.join(path, fileName)
-
-    @classmethod
-    def get_dir_all_dir(cls, s_path):
-        """
-        获取目录下的最深子目录
-        # 这里注意topdown参数。
-        # topdown决定遍历的顺序，
-        # 如果topdown为True，则先列举top下的目录，然后是目录的目录，依次类推；
-        # 反之，则先递归列举出最深层的子目录，然后是其兄弟目录，然后父目录。
-        # 我们需要先修改深层的子目录
-        :param path:  给定一个路径 如 C:/Users/XD/Pictures/所有视频/
-        :return:
-        """
-        # 遍历子目录
-        assert cls.is_dir(s_path), NotADirectoryError(s_path + " not is a dir please check")
-        for path, dirNames, fileNames in os.walk(s_path, topdown=False):
-            # 获取当前目录的所有文件
-            for one_dir in dirNames:
-                f_path = os.path.join(path, one_dir)
-                for path1, dirNames1, fileNames1 in os.walk(f_path):
-                    if len(dirNames1) == 0:
-                        yield f_path
-                    # break 原因是当一个目录有子目录时 会循环子目录，所以一次判断就可以了
-                    break
-
-    @classmethod
-    def is_dir_exists(cls, pathdir):
-        """
-        目录是否存在
-        :param sPth:
-        :return:
-        """
-        return os.path.exists(pathdir)
-
-    @classmethod
-    def create_dir(cls, pathdir):
-        """
-        创建目录 该方法可以创建多级目录
-        os.mkdir(pathdir)是创建一级目录 该目录的上级目录必须存在
-        :param pathdir:
-        :return:
-        """
-        if not cls.is_dir_exists(pathdir):
-            os.makedirs(pathdir)
-
-    @classmethod
-    def get_upper_dir(cls, curPath: str, num: int) -> str:
-        """
-        获取上几层的目录，如果包含文件名 -1 代表着这个文件的目录
-        :param curPath:
-        :param num:
-        :return:
-        """
-        # 如果不是个整数
-        if not isinstance(num, int) or num > 0:
-            warnings.warn("this not int or num > 0 , The path is not processed, return curPath")
-            return curPath
-        pathlist = curPath.split(os.sep)
-        if abs(num) > (len(pathlist) - 1):
-            warnings.warn("this path length < num ,will return root dir ")
-            num = (len(pathlist) - 1) * (-1)
-        pathlist = pathlist[:num]
-        TopPath = os.sep.join(pathlist)
-        return TopPath
-
-    @classmethod
-    def get_file_dir_absolute(cls, file):
-        """
-        请传入 __file__ 作为参数
-        获取一个文件的绝对路径
-        :param __file__:
-        :return:
-        """
-        curPath = os.path.dirname(os.path.abspath(file))
-        return curPath
-
-    @classmethod
-    def is_dir(cls, spath):
-        if os.path.isdir(spath):
-            return True
-        return False
-
-    @classmethod
-    def get_dir_size(cls, dirpath):
-        """
-        1eb = 1024pb
-        1pb = 1024tb
-        1tb = 1024gb
-        1gb = 1024mb
-        1mb = 1024kb
-        1kb = 1024b
-        上面的b是Byte
-        1Byte = 8bit
-        :param dirpath:
-        :return: 这里返回的是Byte大小
-        """
-        size = 0
-        for filepath in cls.get_dir_all_files(dirpath):
-            size += sum(BaseFile.get_file_size(filepath))
-        return size
-
-    @classmethod
-    def copy_file_to_dir(cls, filePath, tarDirPath):
-        """
-
-        :param filePath:  文件路径
-        :param tarDirPath:  目录或文件路径
-        :return:
-        """
-        assert BaseFile.is_file_exists(filePath), FileNotFoundError("需要copy的文件不存在")
-        assert BaseFile.is_file(filePath), FileNotFoundError("需要copy的不是一个文件")
-        cls.create_dir(tarDirPath)
-        shutil.copy(filePath, tarDirPath)
-
-    @classmethod
-    def copy_dir_to_dir(cls, oldDir, newDir, moudle=0):
-        """
-        olddir和newdir都只能是目录，且newdir必须不存在
-        :param oldDir:
-        :param newDir:
-        :return:
-        """
-        assert cls.is_dir(oldDir), IsADirectoryError(oldDir + "目录不存在")
-        if moudle == 0:
-            # 该模式下新目录必须不存在
-            assert not cls.is_dir(newDir), IsADirectoryError(newDir + "目录存在，不能使用该模式,推荐使用模式2")
-            shutil.copytree(oldDir, newDir)
-        elif moudle == 1:
-            # 该模式下如果新目录存在先删除新目录
-            if cls.is_dir(newDir):
-                cls.delete_dir(newDir)
-            shutil.copytree(oldDir, newDir)
-        elif moudle == 3:
-            # 该模式下通过替换进行copy
-            pass
-        elif moudle == 4:
-            # 该模式会跳过已存在的文件 只将新文件放入其中
-            pass
-
-    @classmethod
-    def delete_dir(cls, dirpath):
-        """
-         os.rmdir(dirpath) 只能删除空目录
-        os.removedirs(path)  递归的删除目录
-        :param dirpath:
-        :return:
-        """
-        shutil.rmtree(dirpath)
-
-    @classmethod
-    def replace_dir_special_string(cls, strings):
-        """
-        windows 目录非法字符
-        :param strings:
-        :return:
-        """
-        strings1 = strings.replace("/", "").replace("\\", "") \
-            .replace(":", "").replace("*", "").replace("\"", "") \
-            .replace("<", "").replace(">", "").replace("|", "") \
-            .replace("?", "")
-        return strings1
-
-    @classmethod
-    def remove_file_suf(cls, dirpath, suf):
-        for file in cls.get_dir_all_files(dirpath):
-            if file.endswith(suf):
-                BaseFile.remove_file(file)
-
-    @classmethod
-    def is_null_dir(cls, dirpath):
-        """
-        是否为空目录
-        :param dirpath:
-        :return:
-        """
-        if not os.listdir(dirpath):
-            return True
-        return False
-
-    @classmethod
-    def get_dir_file_num(cls, dirpath):
-        return len([name for name in os.listdir(dirpath) if os.path.isfile(os.path.join(dirpath, name))])
-
-    @classmethod
-    def get_dir_all_file_num(cls, dirpath, ext=None):
-        """
-        获取目录下所有文件个数，包含目录下目录的个数
-        :return:
-        """
-        count = 0
-        for path in cls.get_dir_all_files(dirpath):
-            if BaseFile.is_file(path):
-                if ext is not None:
-                    if path.endswith(ext):
-                        count += 1
-            elif cls.is_dir(path):
-                count = count + cls.get_dir_all_file_num(path, ext)
-        return count
-
-    @classmethod
-    def get_dir_all_file_size(cls, dirpath, ext=None):
-        """
-        获取目录下所有文件的总大小
-        :return:
-        """
-        count = 0
-        for path in cls.get_dir_all_files(dirpath):
-            if BaseFile.is_file(path):
-                if ext is not None:
-                    if path.endswith(ext):
-                        count = count + BaseFile.get_file_size(path)
-            elif cls.is_dir(path):
-                count = count + cls.get_dir_all_file_num(path, ext)
-        return count