re-common 2.0.1__py3-none-any.whl → 10.0.1__py3-none-any.whl

re_common/baselibrary/utils/baseboto3.py

@@ -0,0 +1,291 @@
+import boto3
+import botocore
+from boto3.session import Session
+
+# boto3 该开发工具包由两个关键的 Python 包组成：
+# Botocore（提供在 Python 开发工具包和 AWS CLI 之间共享的低级功能的库）
+# 和 Boto3（实现 Python 开发工具包本身的包）
+
+
+"""
+aws_access_key_id = 'minioa'
+aws_secret_access_key = 'minio123'
+endpoint_url = 'http://192.168.31.164:9000'
+bbt = BaseBoto3(aws_access_key_id=aws_access_key_id,
+                aws_secret_access_key=aws_secret_access_key,
+                endpoint_url=endpoint_url)
+bbt.conn_session()
+bbt.set_is_low_level(False)
+bbt.get_client()
+print("**********************")
+print(bbt.delete_buckets("test1"))
+# bbt.set_is_low_level(False)
+# bbt.get_client()
+# print("**********************")
+# print(bbt.create_buckets("create2"))
+"""
+
+
+class BaseBoto3(object):
+
+    def __init__(self, aws_access_key_id="", aws_secret_access_key="", endpoint_url=""):
+        self.aws_access_key_id = aws_access_key_id
+        self.aws_secret_access_key = aws_secret_access_key
+        self.endpoint_url = endpoint_url
+        self.session = None
+        self.client = None
+        self.is_low_level = False
+        self.bucket = None
+
+    def set_is_low_level(self, is_low_level):
+        self.is_low_level = is_low_level
+        return self
+
+    def set_aws_access_key_id(self, aws_access_key_id):
+        self.aws_access_key_id = aws_access_key_id
+        return self
+
+    def set_aws_secret_access_key(self, aws_secret_access_key):
+        self.aws_secret_access_key = aws_secret_access_key
+        return self
+
+    def set_endpoint_url(self, endpoint_url):
+        self.endpoint_url = endpoint_url
+        return self
+
+    def conn_session(self):
+        self.session = Session(aws_access_key_id=self.aws_access_key_id,
+                               aws_secret_access_key=self.aws_secret_access_key)
+        return self.session
+
+    def get_client(self):
+        assert self.session is not None
+        if self.is_low_level:
+            # 根据名称创建低级服务客户端
+            # botocore.client.S3
+            self.client = self.session.client('s3', endpoint_url=self.endpoint_url)
+            print(type(self.client))
+        else:
+            # boto3.resources.factory.s3.ServiceResource
+            # 按名称创建资源服务客户端
+            self.client = self.session.resource('s3', endpoint_url=self.endpoint_url)
+            print(type(self.client))
+        return self
+
+    def get_all_buckets(self):
+        """
+        获取所有的桶信息
+        :return:
+        """
+        if self.is_low_level is False:
+            return self.client.buckets.all()
+        else:
+            return self.client.list_buckets()
+
+    def create_buckets(self, buckets_name):
+        """
+
+        :param buckets_name:
+        :return:
+        如果get_client 使用 client 返回
+        {'ResponseMetadata': {'RequestId': '16BC90EED4A433C4', 'HostId': '', 'HTTPStatusCode': 200, 'HTTPHeaders': {'accept-ranges': 'bytes', 'content-length': '0', 'content-security-policy': 'block-all-mixed-content', 'location': '/create1', 'server': 'MinIO', 'strict-transport-security': 'max-age=31536000; includeSubDomains', 'vary': 'Origin, Accept-Encoding', 'x-amz-request-id': '16BC90EED4A433C4', 'x-content-type-options': 'nosniff', 'x-xss-protection': '1; mode=block', 'date': 'Wed, 01 Dec 2021 07:28:39 GMT'}, 'RetryAttempts': 0}, 'Location': '/create1'}
+        如果resource 使用 client 返回
+        s3.Bucket(name='create2')
+        """
+        assert buckets_name.find("_") == -1, "新建一个bucket桶(bucket name 中不能有_下划线)"
+        # 新建一个bucket桶(bucket name 中不能有_下划线)
+        return self.client.create_bucket(Bucket=buckets_name)
+
+    def delete_buckets(self, bucket_name):
+        """
+        删除桶 删除bucket(只能删除空的bucket)
+        :return:
+        """
+        if self.is_low_level is False:
+            bucket = self.client.Bucket(bucket_name)
+            response = bucket.delete()
+        else:
+            response = self.client.delete_bucket(Bucket=bucket_name)
+        return response
+
+    def get_bucket(self, bucket_name):
+        """
+        获取 bucket 对象
+        :param bucket_name:
+        :return:
+        """
+        if self.is_low_level is False:
+            self.bucket = self.client.Bucket(bucket_name)
+            return self.bucket
+        else:
+            raise Exception("无实现方法")
+
+    def get_all_obs_filter(self, Prefix):
+        """
+        Prefix 为匹配模式
+        例：列出前缀为 haha 的文件
+        Prefix='haha'
+        :param Prefix:
+        :return: 可以调用 obj.key
+        """
+        if not self.is_low_level:
+            for obj in self.bucket.objects.filter(Prefix=Prefix):
+                yield obj
+        else:
+            raise Exception("请设置 is_low_level 为 False")
+
+    def get_object(self, bucket_name):
+        """
+        会返回包括目录在内的所有对象
+        :param bucket_name:
+        :return:
+        """
+        if self.is_low_level is False:
+            bucket = self.client.Bucket(bucket_name)
+            # boto3.resources.collection.s3.Bucket.objectsCollection
+            all_obj = bucket.objects.all()
+            return all_obj
+            # for obj in bucket.objects.all():
+            #     print('obj name:%s' % obj.key)
+        else:
+            return self.client.list_objects(Bucket=bucket_name)
+
+    def upload_file(self, local_file, bucket_name, key):
+        """
+        # key 桶中的位置 test1/test.pdf
+        :param local_file:  本地文件路径
+        :param bucket_name: 桶名
+        :param key: 远程文件路径
+        :return:
+        """
+
+        if self.is_low_level is False:
+            self.client.Bucket(bucket_name).upload_file(local_file, key)
+        else:
+            self.client.upload_file(local_file, bucket_name, key)
+
+    def upload_fileobj(self, fileobj, bucket_name, key):
+        # fileobj 字节流
+        if self.is_low_level is False:
+            self.client.Bucket(bucket_name).upload_fileobj(fileobj, key)
+        else:
+            self.client.upload_fileobj(fileobj, bucket_name, key)
+
+    def download_file(self, local_file, bucket_name, key):
+        if self.is_low_level is False:
+            self.client.Bucket(bucket_name).download_file(key, local_file)
+        else:
+            self.client.download_file(bucket_name, key, local_file)
+
+    def download_fileobj(self, fileobj, bucket_name, key):
+        if self.is_low_level is False:
+            self.client.Bucket(bucket_name).download_fileobj(key, fileobj)
+        else:
+            self.client.download_fileobj(bucket_name, key, fileobj)
+
+    def check_exist(self, bucket_name, key):
+        """
+         if self.is_low_level:
+            {'ResponseMetadata': {'RequestId': '17E6A65A2B299D3B', 'HostId': '', 'HTTPStatusCode': 200, 'HTTPHeaders': {'accept-ranges': 'bytes', 'content-length': '117', 'content-security-policy': 'block-all-mixed-content', 'content-type': 'binary/octet-stream', 'etag': '"2237a934f176003e41abf3d733291079"', 'last-modified': 'Thu, 25 Jul 2024 05:49:43 GMT', 'server': 'MinIO', 'strict-transport-security': 'max-age=31536000; includeSubDomains', 'vary': 'Origin, Accept-Encoding', 'x-amz-request-id': '17E6A65A2B299D3B', 'x-content-type-options': 'nosniff', 'x-xss-protection': '1; mode=block', 'date': 'Mon, 29 Jul 2024 09:53:33 GMT'}, 'RetryAttempts': 0}, 'AcceptRanges': 'bytes', 'LastModified': datetime.datetime(2024, 7, 25, 5, 49, 43, tzinfo=tzutc()), 'ContentLength': 117, 'ETag': '"2237a934f176003e41abf3d733291079"', 'ContentType': 'binary/octet-stream', 'Metadata': {}}
+        判定文件是否存在，
+        :param bucket_name: 桶名
+        :param key:  文件key
+        :return:
+        """
+
+        if self.is_low_level:
+            try:
+                obj_info = self.client.head_object(
+                    Bucket=bucket_name,
+                    Key=key
+                )
+                return obj_info
+            except:
+                return None
+        else:
+            # 获取指定存储桶
+            bucket = self.client.Bucket(bucket_name)
+            # 检查 key 是否存在
+            objs = list(bucket.objects.filter(Prefix=key))
+            if len(objs) > 0 and objs[0].key == key:
+                # [s3.ObjectSummary(bucket_name='crawl.dc.cqvip.com', key='foreign/organ/parsel_organ_log.txt')]
+                return objs[0]
+            else:
+                return None
+
+    def get_prefix_obj(self, bucket, prefix, delimiter):
+        """
+        Bucket="crawl.dc.cqvip.com",
+         Prefix="foreign/organ/ei/",
+         Delimiter='/'
+
+        Returns:
+
+        """
+        if self.is_low_level:
+            # for common_prefix in response.get('CommonPrefixes', []): 获取子目录
+            return self.client.list_objects_v2(Bucket=bucket,
+                                               Prefix=prefix,
+                                               Delimiter=delimiter)  # 设置 Delimiter='/' 可以确保只列出目录而不是所有对象。
+        else:
+            # 该方法只能列出文件 没办法列出目录
+            # bucket_resource = self.client.Bucket(bucket)
+            # objects = bucket_resource.objects.filter(Prefix=prefix, Delimiter=delimiter)
+            # return list(objects)
+
+            bucket_resource = self.client.Bucket(bucket)
+            return bucket_resource.meta.client.list_objects_v2(Bucket=bucket,
+                                                               Prefix=prefix,
+                                                               Delimiter=delimiter)
+
+    def get_object_value(self, bucket_name, file_key, encoding='utf-8'):
+        """
+        读取文本数据
+        Returns:
+
+        """
+        if self.is_low_level:
+            obj = self.client.get_object(Bucket=bucket_name, Key=file_key)
+            body = obj['Body'].read().decode(encoding)
+            return body
+        else:
+            bucket_resource = self.client.Bucket(bucket_name)
+            obj = bucket_resource.Object(file_key)
+            body = obj.get()['Body'].read().decode(encoding)
+            return body
+
+    def put_object(self, bucket_name, key, body):
+        """
+        直接写内容到文件
+        Args:
+            bucket_name:
+            key:
+            body: 需要 编码 .encode('utf-8')
+
+        Returns:
+
+        """
+        if self.is_low_level:
+            self.client.put_object(Bucket=bucket_name,
+                                   Key=key,
+                                   Body=body)
+        else:
+            # 获取 Bucket 对象
+            bucket_resource = self.client.Bucket(bucket_name)
+
+            # 将内容写入文件
+            bucket_resource.put_object(Key=key, Body=body)
+
+
+bb = BaseBoto3(aws_access_key_id="UM51J2G5ZG0FE5CCERB9",
+               aws_secret_access_key="u+OEmhE2fahF2L+oXB+HXe8IJs22Lo38icvlF+Yq",
+               endpoint_url="http://192.168.31.135:9000"
+               )
+bb.conn_session()
+bb.set_is_low_level(False)
+bb.get_client()
+
+result = bb.check_exist("crawl.dc.cqvip.com",
+                        "foreign/organ/parsel_organ_log.txt")
+
+print(result)

re_common/baselibrary/utils/basecsv.py

@@ -0,0 +1,32 @@
+import csv
+
+
+class BaseCsv(object):
+
+    def __init__(self):
+        pass
+
+    def read_csv(self, filepath):
+        """
+        根据文件路径逐行读取csv文件
+        :param filepath:csv文件路径
+        :return:csv每行记录
+        """
+        # mac_roman 原因
+        # https://stackoverflow.com/questions/21504319/python-3-csv-file-giving-unicodedecodeerror-utf-8-codec-cant-decode-byte-err
+        with open(filepath, "r",encoding='mac_roman') as f:
+            reader = csv.reader(f)
+            for row in reader:
+                yield row
+
+    def read_all_csv(self, filepath):
+        """
+        根据文件路径读取csv文件所有行
+        :param filepath:csv文件路径
+        :return:csv所有行记录
+        """
+        with open(filepath, "r") as f:
+            reader = csv.reader(f)
+            result = list(reader)
+            return result
+

re_common/baselibrary/utils/basedict.py

@@ -0,0 +1,133 @@
+import copy
+import sys
+
+
+class BaseDicts(object):
+
+    @classmethod
+    def removeDictsNone(self, dicts: dict) -> dict:
+        """
+        去除字典中值为None的键值
+        :param dicts:
+        :return:
+        """
+        return {key: value for key, value in dicts.items() if value is not None}
+
+    @classmethod
+    def removeDictsStringNull(self, dicts: dict) -> dict:
+        """
+        去除字典中值为''的键值
+        :param dicts:
+        :return:
+        """
+        return {key: value for key, value in dicts.items() if value != ""}
+
+    @classmethod
+    def removeDictsAllNone(self, dicts: dict) -> dict:
+        """
+        去除字典中值为'' 和 None 的键值
+        :param dicts:
+        :return:
+        """
+        return {key: value for key, value in dicts.items() if value != "" and value is not None}
+
+    @classmethod
+    def sortkeys(self, dicts, reverse=False):
+        """
+        默认升序排序，加  reverse = True 指定为降序排序
+        通过keys 对dicts 排序
+        经过测试是新的列表
+        :return:
+        """
+        return {k: dicts[k] for k in sorted(dicts.keys(), reverse=reverse)}
+
+    @classmethod
+    def sortvalues(self, dicts, reverse=False):
+        """
+        默认升序排序，加  reverse = True 指定为降序排序
+        d[1] 为值　ｄ[0]　为键
+        d 为元组　为dicts的键值
+        通过　values 对dicts 排序
+        :param dicts:
+        :return:
+        """
+        return {k: v for k, v in sorted(dicts.items(), key=lambda d: d[1], reverse=reverse)}
+
+    @classmethod
+    def is_key_have(cls, dicts, key):
+        """
+        判断key 是否存在,但只能判断一个层次
+        :param dicts:
+        :param key:
+        :return:
+        """
+        if key in dicts.keys():
+            return True
+        else:
+            return False
+
+    @classmethod
+    def is_more_key_have(cls, dicts, keys=[]):
+        """
+        判断多个key 是否存在　可以有更深的层次
+        :param dicts:
+        :param keys: ["a.b","c.d"]
+        :return:
+        """
+        for item in keys:
+            if item.find("."):
+                allstrings = ""
+                for key in item.split("."):
+                    allstrings = allstrings + '["{}"]'.format(key)
+                try:
+                    eval("dicts" + allstrings)
+                except:
+                    return False
+            else:
+                if item not in dicts.keys():
+                    return False
+        return True
+
+    @classmethod
+    def get_recursive_dict(cls, dict_a, key, call_back):
+        if isinstance(dict_a, dict):
+            for key, value in dict_a.items():
+                if isinstance(value, list) or isinstance(value, tuple) or isinstance(value, dict):
+                    dict_a[key] = cls.get_recursive_dict(value, key, call_back)
+                else:
+                    dict_a[key] = call_back(key, value)
+
+        elif isinstance(dict_a, list):
+            # 如果列表中存在一个不是字典就不需要遍历了
+            if dict_a == []:
+                return call_back(key, dict_a)
+            is_true = True
+            for i in dict_a:
+                if isinstance(i, dict):
+                    is_true = True
+                else:
+                    is_true = False
+                    continue
+            if is_true:
+                dict_temp = []
+                for value in dict_a:
+                    dict_temp.append(cls.get_recursive_dict(value, key, call_back))
+                return dict_temp
+            else:
+                return call_back(key, dict_a)
+        elif isinstance(dict_a, tuple):
+            # 如果存在元组转成列表使用
+            return cls.get_recursive_dict(list(dict_a), key, call_back)
+        else:
+            assert False, "传入类型错误：{}".format(type(dict_a))
+        return dict_a
+
+    @classmethod
+    def is_dict_exit_key(cls, dicts, key_name, default=""):
+        if dicts is not None and isinstance(dicts, dict):
+            if dicts.get(key_name, 0) != 0:
+                return dicts[key_name]
+            else:
+                return default
+        else:
+            return default

re_common/baselibrary/utils/basedir.py

@@ -0,0 +1,241 @@
+import os
+import shutil
+import warnings
+
+from re_common.baselibrary.utils.basefile import BaseFile
+
+
+class BaseDir(object):
+
+    @classmethod
+    def get_dir_all_files(cls, path):
+        """
+        返回所有的文件 不会返回目录 目的是得到一个文件列表
+        # 这里注意topdown参数。
+        # topdown决定遍历的顺序，
+        # 如果topdown为True，则先列举top下的目录，然后是目录的目录，依次类推；
+        # 反之，则先递归列举出最深层的子目录，然后是其兄弟目录，然后父目录。
+        # 我们需要先修改深层的子目录
+        :param path:  给定一个路径 如 C:/Users/XD/Pictures/所有视频/
+        :return:
+        """
+        # 遍历子目录
+        assert cls.is_dir(path), NotADirectoryError(path + " not is a dir please check")
+        for path, dirNames, fileNames in os.walk(path, topdown=True):
+            # 获取当前目录的所有文件
+            for fileName in fileNames:
+                yield os.path.join(path, fileName)
+
+    @classmethod
+    def get_dir_all_dir(cls, s_path):
+        """
+        获取目录下的最深子目录
+        # 这里注意topdown参数。
+        # topdown决定遍历的顺序，
+        # 如果topdown为True，则先列举top下的目录，然后是目录的目录，依次类推；
+        # 反之，则先递归列举出最深层的子目录，然后是其兄弟目录，然后父目录。
+        # 我们需要先修改深层的子目录
+        :param path:  给定一个路径 如 C:/Users/XD/Pictures/所有视频/
+        :return:
+        """
+        # 遍历子目录
+        assert cls.is_dir(s_path), NotADirectoryError(s_path + " not is a dir please check")
+        for path, dirNames, fileNames in os.walk(s_path, topdown=False):
+            # 获取当前目录的所有文件
+            for one_dir in dirNames:
+                f_path = os.path.join(path, one_dir)
+                for path1, dirNames1, fileNames1 in os.walk(f_path):
+                    if len(dirNames1) == 0:
+                        yield f_path
+                    # break 原因是当一个目录有子目录时 会循环子目录，所以一次判断就可以了
+                    break
+
+    @classmethod
+    def is_dir_exists(cls, pathdir):
+        """
+        目录是否存在
+        :param sPth:
+        :return:
+        """
+        return os.path.exists(pathdir)
+
+    @classmethod
+    def create_dir(cls, pathdir):
+        """
+        创建目录 该方法可以创建多级目录
+        os.mkdir(pathdir)是创建一级目录 该目录的上级目录必须存在
+        :param pathdir:
+        :return:
+        """
+        if not cls.is_dir_exists(pathdir):
+            os.makedirs(pathdir)
+
+    @classmethod
+    def get_upper_dir(cls, curPath: str, num: int) -> str:
+        """
+        获取上几层的目录，如果包含文件名 -1 代表着这个文件的目录
+        :param curPath:
+        :param num:
+        :return:
+        """
+        # 如果不是个整数
+        if not isinstance(num, int) or num > 0:
+            warnings.warn("this not int or num > 0 , The path is not processed, return curPath")
+            return curPath
+        pathlist = curPath.split(os.sep)
+        if abs(num) > (len(pathlist) - 1):
+            warnings.warn("this path length < num ,will return root dir ")
+            num = (len(pathlist) - 1) * (-1)
+        pathlist = pathlist[:num]
+        TopPath = os.sep.join(pathlist)
+        return TopPath
+
+    @classmethod
+    def get_file_dir_absolute(cls, file):
+        """
+        请传入 __file__ 作为参数
+        获取一个文件的绝对路径
+        :param __file__:
+        :return:
+        """
+        curPath = os.path.dirname(os.path.abspath(file))
+        return curPath
+
+    @classmethod
+    def is_dir(cls, spath):
+        if os.path.isdir(spath):
+            return True
+        return False
+
+    @classmethod
+    def get_dir_size(cls, dirpath):
+        """
+        1eb = 1024pb
+        1pb = 1024tb
+        1tb = 1024gb
+        1gb = 1024mb
+        1mb = 1024kb
+        1kb = 1024b
+        上面的b是Byte
+        1Byte = 8bit
+        :param dirpath:
+        :return: 这里返回的是Byte大小
+        """
+        size = 0
+        for filepath in cls.get_dir_all_files(dirpath):
+            size += sum(BaseFile.get_file_size(filepath))
+        return size
+
+    @classmethod
+    def copy_file_to_dir(cls, filePath, tarDirPath):
+        """
+
+        :param filePath:  文件路径
+        :param tarDirPath:  目录或文件路径
+        :return:
+        """
+        assert BaseFile.is_file_exists(filePath), FileNotFoundError("需要copy的文件不存在")
+        assert BaseFile.is_file(filePath), FileNotFoundError("需要copy的不是一个文件")
+        cls.create_dir(tarDirPath)
+        shutil.copy(filePath, tarDirPath)
+
+    @classmethod
+    def copy_dir_to_dir(cls, oldDir, newDir, moudle=0):
+        """
+        olddir和newdir都只能是目录，且newdir必须不存在
+        :param oldDir:
+        :param newDir:
+        :return:
+        """
+        assert cls.is_dir(oldDir), IsADirectoryError(oldDir + "目录不存在")
+        if moudle == 0:
+            # 该模式下新目录必须不存在
+            assert not cls.is_dir(newDir), IsADirectoryError(newDir + "目录存在，不能使用该模式,推荐使用模式2")
+            shutil.copytree(oldDir, newDir)
+        elif moudle == 1:
+            # 该模式下如果新目录存在先删除新目录
+            if cls.is_dir(newDir):
+                cls.delete_dir(newDir)
+            shutil.copytree(oldDir, newDir)
+        elif moudle == 3:
+            # 该模式下通过替换进行copy
+            pass
+        elif moudle == 4:
+            # 该模式会跳过已存在的文件 只将新文件放入其中
+            pass
+
+    @classmethod
+    def delete_dir(cls, dirpath):
+        """
+         os.rmdir(dirpath) 只能删除空目录
+        os.removedirs(path)  递归的删除目录
+        :param dirpath:
+        :return:
+        """
+        shutil.rmtree(dirpath)
+
+    @classmethod
+    def replace_dir_special_string(cls, strings):
+        """
+        windows 目录非法字符
+        :param strings:
+        :return:
+        """
+        strings1 = strings.replace("/", "").replace("\\", "") \
+            .replace(":", "").replace("*", "").replace("\"", "") \
+            .replace("<", "").replace(">", "").replace("|", "") \
+            .replace("?", "")
+        return strings1
+
+    @classmethod
+    def remove_file_suf(cls, dirpath, suf):
+        for file in cls.get_dir_all_files(dirpath):
+            if file.endswith(suf):
+                BaseFile.remove_file(file)
+
+    @classmethod
+    def is_null_dir(cls, dirpath):
+        """
+        是否为空目录
+        :param dirpath:
+        :return:
+        """
+        if not os.listdir(dirpath):
+            return True
+        return False
+
+    @classmethod
+    def get_dir_file_num(cls, dirpath):
+        return len([name for name in os.listdir(dirpath) if os.path.isfile(os.path.join(dirpath, name))])
+
+    @classmethod
+    def get_dir_all_file_num(cls, dirpath, ext=None):
+        """
+        获取目录下所有文件个数，包含目录下目录的个数
+        :return:
+        """
+        count = 0
+        for path in cls.get_dir_all_files(dirpath):
+            if BaseFile.is_file(path):
+                if ext is not None:
+                    if path.endswith(ext):
+                        count += 1
+            elif cls.is_dir(path):
+                count = count + cls.get_dir_all_file_num(path, ext)
+        return count
+
+    @classmethod
+    def get_dir_all_file_size(cls, dirpath, ext=None):
+        """
+        获取目录下所有文件的总大小
+        :return:
+        """
+        count = 0
+        for path in cls.get_dir_all_files(dirpath):
+            if BaseFile.is_file(path):
+                if ext is not None:
+                    if path.endswith(ext):
+                        count = count + BaseFile.get_file_size(path)
+            elif cls.is_dir(path):
+                count = count + cls.get_dir_all_file_num(path, ext)
+        return count