zyomnitools 0.0.1__py3-none-any.whl

zyomnitools/minio_client.py

@@ -0,0 +1,551 @@
+import os
+import traceback
+from typing import List, Tuple, Optional
+from minio import Minio
+from minio.error import S3Error
+
+
+class MinioClient:
+    """MinIO客户端封装类"""
+    def __init__(
+            self,
+            endpoint: str,
+            access_key: str,
+            secret_key: str,
+            secure: bool = False,
+            region: Optional[str] = None
+    ):
+        """
+        初始化MinIO客户端
+
+        :param endpoint: MinIO服务地址（如 "localhost:9000"）
+        :param access_key: 访问密钥（对应MINIO_ROOT_USER）
+        :param secret_key: 密钥（对应MINIO_ROOT_PASSWORD）
+        :param secure: 是否使用HTTPS，默认False（MinIO默认用HTTP）
+        :param region: 区域（可选，默认None）
+        """
+        self.endpoint = endpoint
+        self.access_key = access_key
+        self.secret_key = secret_key
+        self.secure = secure
+        self.region = region
+
+        self.client = Minio(
+            endpoint=self.endpoint,
+            access_key=self.access_key,
+            secret_key=self.secret_key,
+            secure=self.secure,
+            region=self.region
+        )
+
+    def _build_minio_path(self, cls_id: str, sub_path: str = "", prefix: str = "item") -> str:
+        """
+        构建MinIO中的标准路径
+
+        :param cls_id: 分类ID
+        :param sub_path: 子路径（如 "output", "input"）
+        :param prefix: 根目录前缀，默认"item"
+        :return: 完整的MinIO路径
+        """
+        if cls_id is None:
+            return sub_path
+
+        # 构建路径: /item/{cls_id}/{sub_path}/
+        path_parts = [prefix, str(cls_id)]
+        if sub_path:
+            path_parts.append(sub_path)
+
+        # 确保路径以斜杠结尾（目录语义）
+        result = '/'.join(path_parts)
+        if not result.endswith('/'):
+            result += '/'
+
+        return result
+
+    def create_bucket(self, bucket_name: str) -> bool:
+        """
+        创建存储桶（Bucket）
+
+        :param bucket_name: 存储桶名称
+        :return: 成功返回True，失败返回False
+        """
+        try:
+            if not self.client.bucket_exists(bucket_name):
+                self.client.make_bucket(bucket_name)
+                print(f"存储桶 '{bucket_name}' 创建成功")
+            else:
+                print(f"存储桶 '{bucket_name}' 已存在")
+            return True
+        except S3Error as e:
+            print(f"创建存储桶失败: {e}")
+            return False
+        except Exception as e:
+            print(f"创建存储桶时发生未知错误: {e}")
+            traceback.print_exc()
+            return False
+
+    def upload_folder(
+            self,
+            local_dir: str,
+            cls_id: str,
+            bucket_name: str = "4dsimulator",
+            content_type: str = "application/octet-stream",
+            skip_hidden: bool = True
+    ) -> bool:
+        """
+        上传本地目录到MinIO指定存储桶，保持原目录结构
+
+        :param local_dir: 本地目录路径（绝对/相对路径均可）
+        :param cls_id: 分类ID，用于构建MinIO路径为 /item/{cls_id}/output/
+        :param bucket_name: 存储桶名称, 默认4dsimulator
+        :param content_type: 默认文件MIME类型，默认二进制流
+        :param skip_hidden: 是否跳过隐藏文件/目录（以.开头），默认True
+        :return: 全部文件上传成功返回True，否则返回False
+        """
+        # 1. 校验本地目录是否存在
+        if not os.path.isdir(local_dir):
+            print(f"错误：本地目录 '{local_dir}' 不存在或不是有效目录")
+            return False
+
+        # 2. 构建MinIO基础路径
+        minio_base_path = self._build_minio_path(cls_id, "output")
+        print(f"MinIO上传路径: {minio_base_path}")
+
+        # 3. 确保存储桶存在
+        if not self.client.bucket_exists(bucket_name):
+            print(f"存储桶 '{bucket_name}' 不存在，自动创建...")
+            if not self.create_bucket(bucket_name):
+                print("创建存储桶失败，终止目录上传")
+                return False
+
+        # 4. 路径标准化处理
+        local_dir_abs = os.path.abspath(local_dir)
+
+        # 5. 遍历并上传所有文件
+        success_count = 0
+        fail_count = 0
+        failed_files = []
+
+        for root, dirs, files in os.walk(local_dir_abs):
+            # 跳过隐藏目录（如 .git、.idea）
+            if skip_hidden:
+                dirs[:] = [d for d in dirs if not d.startswith('.')]
+
+            for file_name in files:
+                # 跳过隐藏文件（如 .DS_Store、.env）
+                if skip_hidden and file_name.startswith('.'):
+                    continue
+
+                # 构建本地文件完整路径
+                local_file_path = os.path.join(root, file_name)
+
+                # 计算相对路径（相对于本地根目录）
+                relative_path = os.path.relpath(local_file_path, local_dir_abs)
+                relative_path = relative_path.replace(os.sep, '/')
+
+                # 构建MinIO中的对象名称（基础路径 + 相对路径）
+                object_name = minio_base_path + relative_path
+
+                # 上传单个文件
+                try:
+                    self.client.fput_object(
+                        bucket_name=bucket_name,
+                        object_name=object_name,
+                        file_path=local_file_path,
+                        content_type=content_type
+                    )
+                    success_count += 1
+                except (FileNotFoundError, PermissionError, S3Error) as e:
+                    print(f"✗ 上传失败: {local_file_path} -> {object_name}, 错误: {e}")
+                    fail_count += 1
+                    failed_files.append(local_file_path)
+                except Exception as e:
+                    print(f"✗ 上传失败（未知错误）: {local_file_path}, 错误: {e}")
+                    fail_count += 1
+                    failed_files.append(local_file_path)
+
+        # 6. 输出上传统计
+        total_count = success_count + fail_count
+        print(f"\n{'=' * 60}")
+        print(f"目录上传完成")
+        print(f"总计文件: {total_count} | 成功: {success_count} | 失败: {fail_count}")
+        if failed_files:
+            print(f"失败文件列表:")
+            for f in failed_files:
+                print(f"  - {f}")
+        print(f"{'=' * 60}")
+
+        return fail_count == 0
+
+    def get_mesh(
+            self,
+            cls_id: str,
+            save_path: str,
+            bucket_name: str = "4dsimulator"
+    ) -> bool:
+        """
+        从MinIO下载指定分类ID的网格文件（PLY格式）
+
+        :param cls_id: 分类ID，用于构建MinIO路径为 /item/{cls_id}/realtime/{cls_id}_mesh.ply
+        :param save_path: 本地保存路径（包含文件名），如 "./meshes/mesh.ply"
+        :param bucket_name: 存储桶名称，默认为"4dsimulator"
+        :return: 下载成功返回True，失败返回False
+        """
+        try:
+            # 1. 构建MinIO中的对象名称
+            # 路径格式: /item/{cls_id}/realtime/{cls_id}_mesh.ply
+            object_name = f"item/{cls_id}/realtime/{cls_id}_mesh.ply"
+            print(f"正在从MinIO下载网格文件: {object_name}")
+
+            # 2. 检查存储桶是否存在
+            if not self.client.bucket_exists(bucket_name):
+                print(f"错误：存储桶 '{bucket_name}' 不存在")
+                return False
+
+            # 3. 确保保存目录存在
+            save_path = os.path.join(save_path, f"{cls_id}_mesh.ply")
+            save_dir = os.path.dirname(save_path)
+            if save_dir and not os.path.exists(save_dir):
+                os.makedirs(save_dir, exist_ok=True)
+                print(f"创建本地目录: {save_dir}")
+
+            # 4. 下载文件
+            self.client.fget_object(
+                bucket_name=bucket_name,
+                object_name=object_name,
+                file_path=save_path
+            )
+
+            # 5. 验证文件是否成功下载
+            if os.path.exists(save_path):
+                file_size = os.path.getsize(save_path)
+                print(f"✓ 网格文件下载成功: {object_name} -> {save_path}")
+                print(f"  文件大小: {file_size} 字节 ({file_size / 1024:.2f} KB)")
+                return True
+            else:
+                print(f"✗ 下载的文件不存在: {save_path}")
+                return False
+
+        except S3Error as e:
+            print(f"✗ 下载网格文件失败（S3错误）: {e}")
+            if e.code == 'NoSuchKey':
+                print(f"  提示：对象 '{object_name}' 在存储桶 '{bucket_name}' 中不存在")
+            return False
+        except Exception as e:
+            print(f"✗ 下载网格文件失败（未知错误）: {e}")
+            traceback.print_exc()
+            return False
+
+    def download_output(
+            self,
+            local_dir: str,
+            cls_id: str,
+            bucket_name: str = "4dsimulator"
+    ) -> bool:
+        """
+        下载MinIO中指定分类ID下的所有文件到本地目录，保持路径结构
+
+        :param local_dir: 本地保存目录
+        :param cls_id: 分类ID，用于从 /item/{cls_id}/output/ 路径下载
+        :param bucket_name: 存储桶名称, 默认4dsimulator
+        :return: 成功返回True，失败返回False
+        """
+        try:
+            # 1. 构建MinIO路径
+            minio_base_path = self._build_minio_path(cls_id, "output")
+            print(f"MinIO下载路径: {minio_base_path}")
+
+            # 2. 检查存储桶是否存在
+            if not self.client.bucket_exists(bucket_name):
+                print(f"错误：存储桶 '{bucket_name}' 不存在")
+                return False
+
+            # 3. 递归列举路径下的所有对象
+            objects = self.client.list_objects(
+                bucket_name=bucket_name,
+                prefix=minio_base_path,
+                recursive=True
+            )
+
+            # 4. 收集对象列表
+            object_list = list(objects)
+            if not object_list:
+                print(f"路径 '{minio_base_path}' 下没有找到任何对象")
+                return True
+
+            # 5. 确保本地目录存在
+            local_dir = os.path.join(local_dir, cls_id)
+            os.makedirs(local_dir, exist_ok=True)
+
+            # 6. 遍历所有对象并下载
+            success_count = 0
+            fail_count = 0
+            failed_objects = []
+
+            for obj in object_list:
+                # 跳过目录占位符（MinIO中目录以斜杠结尾，大小为0）
+                if obj.size == 0 and obj.object_name.endswith('/'):
+                    print(f"跳过目录占位符: {obj.object_name}")
+                    continue
+
+                # 计算相对路径（去掉MinIO基础路径）
+                relative_path = obj.object_name[len(minio_base_path):]
+
+                # 构建本地保存路径
+                local_file_path = os.path.join(local_dir, relative_path)
+
+                # 创建本地目录
+                local_file_dir = os.path.dirname(local_file_path)
+                if local_file_dir:
+                    os.makedirs(local_file_dir, exist_ok=True)
+
+                # 下载单个对象
+                try:
+                    self.client.fget_object(
+                        bucket_name=bucket_name,
+                        object_name=obj.object_name,
+                        file_path=local_file_path
+                    )
+                    success_count += 1
+                except Exception as e:
+                    print(f"✗ 下载失败: {obj.object_name}, 错误: {e}")
+                    fail_count += 1
+                    failed_objects.append(obj.object_name)
+
+            # 7. 输出下载统计
+            total_count = success_count + fail_count
+            print(f"\n{'=' * 60}")
+            print(f"目录下载完成")
+            print(f"总计对象: {total_count} | 成功: {success_count} | 失败: {fail_count}")
+            if failed_objects:
+                print(f"失败对象列表:")
+                for obj in failed_objects:
+                    print(f"  - {obj}")
+            print(f"{'=' * 60}")
+
+            return fail_count == 0
+
+        except S3Error as e:
+            print(f"下载目录失败（S3错误）: {e}")
+            return False
+        except Exception as e:
+            print(f"下载目录失败（未知错误）: {e}")
+            traceback.print_exc()
+            return False
+
+    def download_realtime(
+            self,
+            local_dir: str,
+            cls_id: str,
+            bucket_name: str = "4dsimulator",
+            folders: List[str] = None  # 新增参数，允许自定义要下载的文件夹
+    ) -> bool:
+        """
+        下载MinIO中指定分类ID下的指定文件夹到本地目录，保持路径结构
+
+        :param local_dir: 本地保存目录
+        :param cls_id: 分类ID，用于从 /item/{cls_id}/realtime/ 路径下载
+        :param bucket_name: 存储桶名称, 默认4dsimulator
+        :param folders: 要下载的文件夹列表，默认为 ['masks', 'pose', 'rgb']
+        :return: 成功返回True，失败返回False
+        """
+        try:
+            # 设置默认下载的文件夹
+            if folders is None:
+                folders = ['masks', 'pose', 'rgb']
+
+            # 1. 构建MinIO基础路径
+            minio_base_path = self._build_minio_path(cls_id, "realtime")
+            print(f"MinIO下载路径: {minio_base_path}")
+            print(f"要下载的文件夹: {', '.join(folders)}")
+
+            # 2. 检查存储桶是否存在
+            if not self.client.bucket_exists(bucket_name):
+                print(f"错误：存储桶 '{bucket_name}' 不存在")
+                return False
+
+            # 3. 确保本地目录存在
+            local_save_dir = os.path.join(local_dir, str(cls_id)) if cls_id else local_dir
+            os.makedirs(local_save_dir, exist_ok=True)
+            print(f"本地保存目录: {local_save_dir}")
+
+            # 4. 遍历目标文件夹
+            success_count = 0
+            fail_count = 0
+            failed_objects = []
+
+            for folder in folders:
+                folder_path = minio_base_path + folder + '/'
+                print(f"\n正在处理文件夹: {folder_path}")
+
+                # 列举该文件夹下的所有对象
+                try:
+                    objects = self.client.list_objects(
+                        bucket_name=bucket_name,
+                        prefix=folder_path,
+                        recursive=True
+                    )
+
+                    # 收集对象列表
+                    object_list = list(objects)
+                    if not object_list:
+                        print(f"  警告：文件夹 '{folder_path}' 下没有找到任何对象")
+                        continue
+
+                    print(f"  找到 {len(object_list)} 个对象")
+
+                    # 下载该文件夹下的所有对象
+                    for obj in object_list:
+                        # 跳过目录占位符
+                        if obj.size == 0 and obj.object_name.endswith('/'):
+                            continue
+
+                        # 计算相对路径（去掉MinIO基础路径）
+                        relative_path = obj.object_name[len(minio_base_path):]
+
+                        # 构建本地保存路径
+                        local_file_path = os.path.join(local_save_dir, relative_path)
+
+                        # 创建本地目录
+                        local_file_dir = os.path.dirname(local_file_path)
+                        if local_file_dir:
+                            os.makedirs(local_file_dir, exist_ok=True)
+
+                        # 下载单个对象
+                        try:
+                            self.client.fget_object(
+                                bucket_name=bucket_name,
+                                object_name=obj.object_name,
+                                file_path=local_file_path
+                            )
+                            success_count += 1
+                        except Exception as e:
+                            print(f"  ✗ 下载失败: {obj.object_name}, 错误: {e}")
+                            fail_count += 1
+                            failed_objects.append(obj.object_name)
+
+                except S3Error as e:
+                    print(f"  错误：无法访问文件夹 '{folder_path}': {e}")
+                    continue
+                except Exception as e:
+                    print(f"  错误：处理文件夹 '{folder_path}' 时发生异常: {e}")
+                    continue
+
+            # 5. 输出下载统计
+            total_count = success_count + fail_count
+            print(f"\n{'=' * 60}")
+            print(f"Realtime数据下载完成")
+            print(f"目标文件夹: {', '.join(folders)}")
+            print(f"总计对象: {total_count} | 成功: {success_count} | 失败: {fail_count}")
+            if failed_objects:
+                print(f"失败对象列表:")
+                for obj in failed_objects:
+                    print(f"  - {obj}")
+            print(f"{'=' * 60}")
+
+            return fail_count == 0
+
+        except S3Error as e:
+            print(f"下载目录失败（S3错误）: {e}")
+            return False
+        except Exception as e:
+            print(f"下载目录失败（未知错误）: {e}")
+            traceback.print_exc()
+            return False
+
+    def delete_object(self, object_name: str, bucket_name: str = "4dsimulator") -> bool:
+        """
+        删除MinIO中的对象
+
+        :param object_name: 要删除的对象名称
+        :param bucket_name: 存储桶名称, 默认4dsimulator
+        :return: 成功返回True，失败返回False
+        """
+        try:
+            self.client.remove_object(bucket_name, object_name)
+            print(f"对象 '{object_name}' 已删除")
+            return True
+        except S3Error as e:
+            print(f"删除对象失败: {e}")
+            return False
+        except Exception as e:
+            print(f"删除对象时发生未知错误: {e}")
+            return False
+
+    def list_objects(
+            self,
+            prefix: str = "",
+            recursive: bool = False,
+            bucket_name: str = "4dsimulator"
+    ) -> List[Tuple[str, int, str]]:
+        """
+        列举存储桶中的对象
+
+        :param prefix: 前缀（用于过滤，如 "images/"）
+        :param recursive: 是否递归列举子目录，默认False
+        :param bucket_name: 存储桶名称, 默认4dsimulator
+        :return: 对象列表，每个元素为 (对象名, 大小(字节), 最后修改时间)
+        """
+        try:
+            objects = self.client.list_objects(
+                bucket_name=bucket_name,
+                prefix=prefix,
+                recursive=recursive
+            )
+            result = []
+            for obj in objects:
+                result.append((obj.object_name, obj.size, obj.last_modified))
+
+            print(f"存储桶 '{bucket_name}' 中找到 {len(result)} 个对象")
+            return result
+        except S3Error as e:
+            print(f"列举对象失败: {e}")
+            return []
+        except Exception as e:
+            print(f"列举对象时发生未知错误: {e}")
+            return []
+
+
+if __name__ == "__main__":
+    # 初始化客户端
+    minio_client = MinioClient(
+        endpoint="192.168.112.162:39000",
+        access_key="admin",
+        secret_key="cogleapai",
+        secure=False
+    )
+    cls_id = "200"
+    print("MinIO客户端测试开始")
+
+    # 1. 上传文件夹
+    minio_client.upload_folder(
+        local_dir="./test_data",
+        cls_id=cls_id
+    )
+
+    # 2. 下载output目录
+    minio_client.download_output(
+        local_dir="./downloaded_output",
+        cls_id=cls_id
+    )
+
+    # 3. 下载realtime目录（masks, pose, rgb）
+    minio_client.download_realtime(
+        local_dir="./downloaded_realtime",
+        cls_id=cls_id
+    )
+
+    # 4. 下载mesh文件
+    minio_client.get_mesh(
+        cls_id=cls_id,
+        save_path="./meshes/"
+    )
+
+    # 5. 列举文件
+    objects = minio_client.list_objects(
+        prefix=f"item/{cls_id}/output/",
+        recursive=True
+    )
+    print(f"共找到 {len(objects)} 个文件")
+
+    print("测试完成")

zyomnitools-0.0.1.dist-info/METADATA

@@ -0,0 +1,8 @@
+Metadata-Version: 2.1
+Name: zyomnitools
+Version: 0.0.1
+Summary: Need Python3.9+
+License: Apache License 2.0
+Requires-Dist: minio
+Requires-Dist: Cython
+

zyomnitools-0.0.1.dist-info/RECORD

@@ -0,0 +1,6 @@
+zyomnitools/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+zyomnitools/minio_client.py,sha256=ttHv9aFkx0o8ybhnCMebBKgY25aNXqgPN6tTYTISO28,21097
+zyomnitools-0.0.1.dist-info/METADATA,sha256=b0rx0xeUUYvJylJLoiz-5PBLUEQMl5L5V1w0JXmTSuQ,152
+zyomnitools-0.0.1.dist-info/WHEEL,sha256=tZoeGjtWxWRfdplE7E3d45VPlLNQnvbKiYnx7gwAy8A,92
+zyomnitools-0.0.1.dist-info/top_level.txt,sha256=yAzxqg8YnvRdupF8wbRGBS7FRYBtUgIg8R6rLDT6fKk,12
+zyomnitools-0.0.1.dist-info/RECORD,,

zyomnitools-0.0.1.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: bdist_wheel (0.45.1)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

zyomnitools-0.0.1.dist-info/top_level.txt

@@ -0,0 +1 @@
+zyomnitools