lesscode-py 0.4.18__tar.gz → 0.4.20__tar.gz

{lesscode-py-0.4.18 → lesscode-py-0.4.20}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: lesscode-py
-Version: 0.4.18
+Version: 0.4.20
 Summary: lesscode-python 是基于tornado的web开发脚手架项目，该项目初衷为简化开发过程，让研发人员更加关注业务。
 Home-page: https://gitee.com/yongchao9/lesscode-python
 Author: Chao.yy

lesscode-py-0.4.20/lesscode/utils/oss/__init__.py

@@ -0,0 +1,160 @@
+import os
+from io import BytesIO
+
+from tornado.options import options
+
+from lesscode.utils.oss.aliyun_oss import AliYunOss
+from lesscode.utils.oss.ks3_oss import Ks3Oss
+from lesscode.utils.oss.minio_oss import MinioOss
+
+
+class CommonOss:
+    def __init__(self, storage_type=None, data_type="stream", config_key="default", **kwargs):
+        """
+        初始化OSS
+        Args:
+            storage_type (str): 存储类型，目前支持ks3和file
+            data_type (str): 数据类型，目前支持stream和file_path
+            storage_config (dict): 存储配置，目前支持ks3和file,file_name,aliyun,minio
+        """
+        _storage_config = options.storage_config.get(config_key, {}) or dict()
+        self.storage_type = storage_type if storage_type else _storage_config.get("storage_type", "")
+        self.data_type = data_type
+        self.storage_config = kwargs.get("storage_config", {}) if kwargs.get("storage_config",
+                                                                             {}) else _storage_config.get(
+            "storage_config", {})
+        self.bucket_name = kwargs.get("bucket_name") or self.storage_config.pop("bucket_name", "")
+
+    def _save(self, key, io_stream: BytesIO = None, file_path: str = None, bucket_name: str = None):
+        file_url_obj = dict()
+        if self.storage_type == "ks3":
+            if self.data_type == "stream":
+                storage_config = self.storage_config or dict()
+                ks3 = Ks3Oss(bucket_name=self.bucket_name, **storage_config)
+                url = ks3.save(key=key, string_data=io_stream.getvalue(), content_type="string", policy="public-read",
+                               bucket_name=bucket_name or self.bucket_name)
+                file_url_obj = {"key": key, "url": url}
+            elif self.data_type == "file_path":
+                storage_config = self.storage_config or dict()
+                ks3 = Ks3Oss(bucket_name=self.bucket_name, **storage_config)
+                url = ks3.save(key=key, filename=file_path, content_type="filename", policy="public-read")
+                file_url_obj = {"key": key, "url": url}
+        elif self.storage_type == "aliyun":
+            storage_config = self.storage_config or dict()
+            aliyun = AliYunOss(bucket_name=self.bucket_name, **storage_config)
+            url = aliyun.save(key=key, content_type="string", data=io_stream.getvalue(),
+                              bucket_name=bucket_name or self.bucket_name)
+            file_url_obj = {"key": key, "url": url}
+        elif self.storage_type == "minio":
+            storage_config = self.storage_config or dict()
+            minio = MinioOss(**storage_config)
+            url = minio.save(key=key, content_type="string", data=io_stream.getvalue(),
+                             bucket_name=bucket_name or self.bucket_name)
+            file_url_obj = {"key": key, "url": url}
+        elif self.storage_type == "file":
+            storage_path = ""
+            storage_dir = self.storage_config.get("storage_dir", "")
+            if not storage_dir:
+                raise Exception("storage_dir is empty")
+            if self.data_type == "stream":
+                if "\\" in key:
+                    key_list = key.split("\\")
+                elif "/" in key:
+                    key_list = key.split("/")
+                else:
+                    key_list = [key]
+                storage_path = storage_dir
+                if key_list:
+                    for k in key_list:
+                        storage_path = os.path.join(storage_path, k)
+                dir_path = os.path.dirname(storage_path)
+                if not os.path.exists(dir_path):
+                    os.makedirs(dir_path)
+            elif self.data_type == "file_path":
+                with open(file_path, 'rb') as infile:
+                    io_stream = BytesIO(infile.read())
+            if storage_path:
+                with open(storage_path, 'wb') as outfile:
+                    outfile.write(io_stream.getvalue())
+                file_url_obj = {"key": key, "url": storage_path}
+        return file_url_obj
+
+    def upload(self, **kwargs):
+        """上传文件
+        Args:
+            files (list): 文件列表 可以是文件流列表，也可以是字典列表，字典格式为{"key":"文件key，可以是带路径的文件","stream":"文件流"}
+        Returns:
+            file_url_list: 文件url列表[{"key":"文件key，可以是带路径的文件","url":"本地文件存储的文件的全路径，对象存储，存放的是文件的下载地址"}]
+        """
+        file_url_list = []
+        files = kwargs.get("files", [])
+        bucket_name = kwargs.get("bucket_name", self.bucket_name)
+        if not files:
+            raise Exception("files is empty")
+        if self.data_type == "stream":
+            for f in files:
+                if not isinstance(f, dict):
+                    key = f.filename
+                    stream = f.stream.read()
+                    file_stream = BytesIO(stream)
+                else:
+                    key = f.get("key", "")
+                    _steam = f.get("stream")
+                    stream = _steam.stream.read()
+                    file_stream = BytesIO(stream)
+                file_url_obj = self._save(key=key, io_stream=file_stream, bucket_name=bucket_name)
+                file_url_list.append(file_url_obj)
+
+        elif self.data_type == "file_path":
+            for f in files:
+                if not isinstance(f, dict):
+                    if "\\" in f:
+                        file_name = f.split("\\")[-1]
+                    elif "/" in f:
+                        file_name = f.split("/")[-1]
+                    else:
+                        file_name = f
+                    key = file_name
+                    _file_path = f
+                else:
+                    key = f.get("key", "")
+                    _file_path = f.get("file_path")
+                file_url_obj = self._save(key=key, file_path=_file_path, bucket_name=bucket_name)
+                file_url_list.append(file_url_obj)
+        return file_url_list
+
+    def download(self, key, bucket_name=None):
+        """下载文件，返回文件流
+        Args:
+            key (str): 上面接口返回的文件key
+        Returns:
+            file_stream: 文件流
+            :param bucket_name:
+        """
+        if self.storage_type == "ks3":
+            storage_config = self.storage_config or dict()
+            ks3 = Ks3Oss(bucket_name=self.bucket_name, **storage_config)
+            return ks3.get_file(key=key, bucket_name=bucket_name or self.bucket_name)
+        elif self.storage_type == "aliyun":
+            storage_config = self.storage_config or dict()
+            aliyun = AliYunOss(bucket_name=self.bucket_name, **storage_config)
+            return aliyun.get_file(key=key, bucket_name=bucket_name or self.bucket_name)
+        elif self.storage_type == "minio":
+            storage_config = self.storage_config or dict()
+            minio = MinioOss(**storage_config)
+            return minio.get_file(bucket_name=bucket_name or self.bucket_name, key=key)
+        if self.storage_type == "file":
+            storage_dir = self.storage_config.get("storage_dir", "")
+            if "\\" in key:
+                key_list = key.split("\\")
+            elif "/" in key:
+                key_list = key.split("/")
+            else:
+                key_list = [key]
+            file_path = storage_dir
+            if key_list:
+                for k in key_list:
+                    file_path = os.path.join(file_path, k)
+            with open(file_path, 'rb') as f:
+                return f.read()
+        return None

{lesscode-py-0.4.18 → lesscode-py-0.4.20}/lesscode/utils/oss/aliyun_oss.py

@@ -66,7 +66,8 @@ class AliYunOss:
             self.instance = self.get_bucket_instance(bucket_name)
         return self.instance.delete_bucket()
 
-    def save(self, key, content_type, data, bucket_name="", **kwargs):
+    def save(self, key, content_type, data, bucket_name="", region="oss-cn-hangzhou", domain="aliyuncs.com",
+             protocol="https", **kwargs):
         if bucket_name:
             self.instance = self.get_bucket_instance(bucket_name)
         if content_type == "filename":
@@ -75,6 +76,7 @@ class AliYunOss:
             self.instance.put_object(key=key, data=data, **kwargs)
         else:
             self.instance.put_object(key=key, data=data, **kwargs)
+        return f'{protocol}://{self.instance.name}.{region}.{domain}/{key}'
 
     def get_url(self, key, bucket_name="", region="oss-cn-hangzhou", domain="aliyuncs.com", protocol="https"):
         if bucket_name:

lesscode-py-0.4.20/lesscode/utils/oss/minio_oss.py

@@ -0,0 +1,254 @@
+import importlib
+import logging
+import os
+
+from datetime import timedelta
+from io import BytesIO
+
+
+class MinioOss:
+    client = None
+    policy = '{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":{"AWS":["*"]},"Action":["s3:GetBucketLocation","s3:ListBucket"],"Resource":["arn:aws:s3:::%s"]},{"Effect":"Allow","Principal":{"AWS":["*"]},"Action":["s3:GetObject"],"Resource":["arn:aws:s3:::%s/*"]}]}'
+
+    def __new__(cls, *args, **kwargs):
+        if not cls.client:
+            cls.client = object.__new__(cls)
+        return cls.client
+
+    def __init__(self, service, access_key, secret_key, secure=False):
+        try:
+            minio = importlib.import_module("minio")
+        except ImportError as e:
+            raise Exception(f"minio is not exist,run:pip install minio")
+        self.service = service
+        self.client = minio.Minio(service, access_key=access_key, secret_key=secret_key, secure=secure)
+
+    def exists_bucket(self, bucket_name):
+        """
+        判断桶是否存在
+        :param bucket_name: 桶名称
+        :return:
+        """
+        return self.client.bucket_exists(bucket_name=bucket_name)
+
+    def create_bucket(self, bucket_name: str, is_policy: bool = True):
+        """
+        创建桶 + 赋予策略
+        :param bucket_name: 桶名
+        :param is_policy: 策略
+        :return:
+        """
+        if self.exists_bucket(bucket_name=bucket_name):
+            return False
+        else:
+            self.client.make_bucket(bucket_name=bucket_name)
+        if is_policy:
+            policy = self.policy % (bucket_name, bucket_name)
+            self.client.set_bucket_policy(bucket_name=bucket_name, policy=policy)
+        return True
+
+    def get_bucket_list(self):
+        """
+        列出存储桶
+        :return:
+        """
+        buckets = self.client.list_buckets()
+        bucket_list = []
+        for bucket in buckets:
+            bucket_list.append(
+                {"bucket_name": bucket.name, "create_time": bucket.creation_date}
+            )
+        return bucket_list
+
+    def remove_bucket(self, bucket_name):
+        """
+        删除桶
+        :param bucket_name:
+        :return:
+        """
+        try:
+            minio_error = importlib.import_module("minio.error")
+        except ImportError as e:
+            raise Exception(f"minio is not exist,run:pip install minio")
+        try:
+            self.client.remove_bucket(bucket_name=bucket_name)
+        except minio_error.S3Error as e:
+            logging.error(f"[error]:{e}")
+            return False
+        return True
+
+    def bucket_list_files(self, bucket_name, prefix):
+        """
+        列出存储桶中所有对象
+        :param bucket_name: 同名
+        :param prefix: 前缀
+        :return:
+        """
+        try:
+            minio_error = importlib.import_module("minio.error")
+        except ImportError as e:
+            raise Exception(f"minio is not exist,run:pip install minio")
+        try:
+            files_list = self.client.list_objects(bucket_name=bucket_name, prefix=prefix, recursive=True)
+            return files_list
+        except minio_error.S3Error as e:
+            logging.error(f"[error]:{e}")
+            return None
+
+    def bucket_policy(self, bucket_name):
+        """
+        列出桶存储策略
+        :param bucket_name:
+        :return:
+        """
+        try:
+            minio_error = importlib.import_module("minio.error")
+        except ImportError as e:
+            raise Exception(f"minio is not exist,run:pip install minio")
+        try:
+            policy = self.client.get_bucket_policy(bucket_name)
+        except minio_error.S3Error as e:
+            logging.error(f"[error]:{e}")
+            return None
+        return policy
+
+    def download_file(self, bucket_name, file, file_path, stream=1024 * 32):
+        """
+        从bucket 下载文件 + 写入指定文件
+        :return:
+        """
+        try:
+            minio_error = importlib.import_module("minio.error")
+        except ImportError as e:
+            raise Exception(f"minio is not exist,run:pip install minio")
+        try:
+            data = self.client.get_object(bucket_name, file)
+            with open(file_path, "wb") as fp:
+                for d in data.stream(stream):
+                    fp.write(d)
+        except minio_error.S3Error as e:
+            logging.error(f"[error]:{e}")
+
+    def get_file(self, bucket_name, key):
+        data = self.client.get_object(bucket_name, object_name=key)
+        return data
+
+    def fget_file(self, bucket_name, file, file_path):
+        """
+        下载保存文件保存本地
+        :param bucket_name:
+        :param file:
+        :param file_path:
+        :return:
+        """
+        self.client.fget_object(bucket_name, file, file_path)
+
+    def copy_file(self, bucket_name, file, file_path):
+        """
+        拷贝文件（最大支持5GB）
+        :param bucket_name:
+        :param file:
+        :param file_path:
+        :return:
+        """
+        self.client.copy_object(bucket_name, file, file_path)
+
+    def upload_file(self, bucket_name, file, file_path, content_type):
+        """
+        上传文件 + 写入
+        :param bucket_name: 桶名
+        :param file: 文件名
+        :param file_path: 本地文件路径
+        :param content_type: 文件类型
+        :return:
+        """
+        try:
+            minio_error = importlib.import_module("minio.error")
+        except ImportError as e:
+            raise Exception(f"minio is not exist,run:pip install minio")
+        try:
+            with open(file_path, "rb") as file_data:
+                file_stat = os.stat(file_path)
+                self.client.put_object(bucket_name, file, file_data, file_stat.st_size, content_type=content_type)
+        except minio_error.S3Error as e:
+            logging.error(f"[error]:{e}")
+
+    def save(self, key, bucket_name, data, content_type="string", **kwargs):
+        ret = None
+        if content_type == "string":
+            ret = self.client.put_object(bucket_name=bucket_name, object_name=key, data=BytesIO(data), length=len(data),
+                                         **kwargs)
+        elif content_type == "filename":
+            with open(data, "rb") as f:
+                data = f.read()
+            ret = self.client.put_object(bucket_name=bucket_name, object_name=key, data=BytesIO(data), length=len(data),
+                                         **kwargs)
+        if ret:
+            if ret.status == 200:
+                return self.client.presigned_get_object(bucket_name=bucket_name, key=key, expires=3600)
+            else:
+                return False
+        else:
+            return False
+
+    def fput_file(self, bucket_name, file, file_path):
+        """
+        上传文件
+        :param bucket_name: 桶名
+        :param file: 文件名
+        :param file_path: 本地文件路径
+        :return:
+        """
+        try:
+            minio_error = importlib.import_module("minio.error")
+        except ImportError as e:
+            raise Exception(f"minio is not exist,run:pip install minio")
+        try:
+            self.client.fput_object(bucket_name, file, file_path)
+        except minio_error.S3Error as e:
+            logging.error(f"[error]:{e}")
+
+    def stat_object(self, bucket_name, file):
+        """
+        获取文件元数据
+        :param bucket_name:
+        :param file:
+        :return:
+        """
+        try:
+            minio_error = importlib.import_module("minio.error")
+        except ImportError as e:
+            raise Exception(f"minio is not exist,run:pip install minio")
+        try:
+            data = self.client.stat_object(bucket_name, file)
+            return data
+        except minio_error.S3Error as e:
+            logging.error(f"[error]:{e}")
+            return None
+
+    def remove_file(self, bucket_name, file):
+        """
+        移除单个文件
+        :return:
+        """
+        self.client.remove_object(bucket_name, file)
+
+    def remove_files(self, bucket_name, file_list):
+        """
+        删除多个文件
+        :return:
+        """
+        try:
+            deleteobjects = importlib.import_module("minio.deleteobjects")
+        except ImportError as e:
+            raise Exception(f"minio is not exist,run:pip install minio")
+        delete_object_list = [deleteobjects.DeleteObject(file) for file in file_list]
+        for del_err in self.client.remove_objects(bucket_name, delete_object_list):
+            logging.error(del_err)
+
+    def presigned_get_file(self, bucket_name, file, days=7):
+        """
+        生成一个http GET操作 签证URL
+        :return:
+        """
+        return self.client.presigned_get_object(bucket_name, file, expires=timedelta(days=days))

{lesscode-py-0.4.18 → lesscode-py-0.4.20}/lesscode/version.py

@@ -5,4 +5,4 @@
 lesscode python version file.
 """
 
-__version__ = "0.4.18"
+__version__ = "0.4.20"

{lesscode-py-0.4.18 → lesscode-py-0.4.20}/lesscode/web/base_handler.py

@@ -271,20 +271,34 @@ class BaseHandler(RequestHandler):
                     for k, v in params_dict.items():
                         if "self" != k:
                             _params[k] = v
-                    t = Thread(target=sync_common_request_origin, kwargs={
-                        "url": options.write_operate_log_url,
-                        "method": "POST",
-                        "json": {
+                    if options.operate_log_write_method == "api":
+                        t = Thread(target=sync_common_request_origin, kwargs={
+                            "url": options.write_operate_log_url,
+                            "method": "POST",
+                            "json": {
+                                "operate_action": title,
+                                "operate_url": self.request.path,
+                                "operate_user_id": _user_id,
+                                "operate_user_username": _username,
+                                "params": _params,
+                                "operate_user_phone_no": _phone_no
+                            }
+                        })
+                        t.start()
+                        logging.info(f"添加操作日志，url:{self.request.path},title={title}")
+                    elif options.operate_log_write_method == "function":
+                        t = Thread(target=options.operate_log_write_func, kwargs={
+                            "operate_id": uuid.uuid1().hex,
                             "operate_action": title,
                             "operate_url": self.request.path,
                             "operate_user_id": _user_id,
                             "operate_user_username": _username,
                             "params": _params,
                             "operate_user_phone_no": _phone_no
-                        }
-                    })
-                    t.start()
-                    logging.info(f"添加操作日志，url:{self.request.path},title={title}")
+                        })
+                        t.start()
+                        logging.info(f"添加操作日志，url:{self.request.path},title={title}")
+
                 except Exception as e:
                     logging.error(f"添加操作日志失败，url={self.request.path},错误信息:{e}")
             # 判断是否为异步非阻塞方法，true 则直接调用

{lesscode-py-0.4.18 → lesscode-py-0.4.20}/lesscode/web/web_server.py

@@ -41,6 +41,8 @@ define("eureka_config", default={}, type=dict, help="eureka配置")
 define("rms_register_enable", default=False, type=bool, help="是否启动资源注册")
 define("rms_register_server", default="http://127.0.0.1:8918", type=str, help="启动资源注册")
 define("operate_log_enable", default=False, type=bool, help="记录操作日志开关，默认关闭")
+define("operate_log_write_method", default="api", type=str, help="记录操作日志方式，默认是api")
+define("operate_log_write_func", default=None, type=FunctionType, help="记录操作的函数")
 define("write_operate_log_url", default="http://127.0.0.1:8918", type=str, help="记录操作日志接口")
 define("running_env", default="local", type=str, help="运行环境")
 define("request_type", default="request", type=str, help="请求类型")

{lesscode-py-0.4.18 → lesscode-py-0.4.20}/lesscode_py.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: lesscode-py
-Version: 0.4.18
+Version: 0.4.20
 Summary: lesscode-python 是基于tornado的web开发脚手架项目，该项目初衷为简化开发过程，让研发人员更加关注业务。
 Home-page: https://gitee.com/yongchao9/lesscode-python
 Author: Chao.yy

{lesscode-py-0.4.18 → lesscode-py-0.4.20}/lesscode_py.egg-info/SOURCES.txt

@@ -83,6 +83,7 @@ lesscode/utils/es_log/record_log.py
 lesscode/utils/oss/__init__.py
 lesscode/utils/oss/aliyun_oss.py
 lesscode/utils/oss/ks3_oss.py
+lesscode/utils/oss/minio_oss.py
 lesscode/web/__init__.py
 lesscode/web/base_handler.py
 lesscode/web/base_server_send_event_handler.py