tamar-file-hub-client 0.1.7__py3-none-any.whl → 0.2.4__py3-none-any.whl

file_hub_client/services/file/sync_blob_service.py

@@ -9,7 +9,7 @@ from .base_file_service import BaseFileService
 from ...enums import UploadMode
 from ...errors import ValidationError
 from ...rpc import SyncGrpcClient
-from ...schemas import FileUploadResponse, UploadUrlResponse, BatchDownloadUrlResponse, DownloadUrlInfo, GcsUrlInfo, GetGcsUrlResponse, BatchGcsUrlResponse, CompressionStatusResponse, GetVariantsResponse, RecompressionResponse, VariantDownloadUrlResponse, CompressedVariant
+from ...schemas import FileUploadResponse, UploadUrlResponse, BatchDownloadUrlResponse, DownloadUrlInfo, GcsUrlInfo, GetGcsUrlResponse, BatchGcsUrlResponse, CompressionStatusResponse, GetVariantsResponse, RecompressionResponse, VariantDownloadUrlResponse, CompressedVariant, BatchFileStatusResponse, FileStatusInfo, FileStatusDetails, FileUploadStatus, FileCompressionStatus, FileSyncStatus
 from ...utils import HttpUploader, HttpDownloader, retry_with_backoff, get_file_mime_type
 
 
@@ -116,6 +116,7 @@ class SyncBlobService(BaseFileService):
             is_temporary: Optional[bool] = False,
             expire_seconds: Optional[int] = None,
             keep_original_filename: Optional[bool] = False,
+            forbid_overwrite: Optional[bool] = True,
             request_id: Optional[str] = None,
             **metadata
     ) -> FileUploadResponse:
@@ -132,6 +133,7 @@ class SyncBlobService(BaseFileService):
             is_temporary=is_temporary,
             expire_seconds=expire_seconds if is_temporary and expire_seconds else None,
             keep_original_filename=keep_original_filename,
+            forbid_overwrite=forbid_overwrite,
             request_id=request_id,
             **metadata
         )
@@ -149,12 +151,13 @@ class SyncBlobService(BaseFileService):
             "Cache-Control": "public, max-age=86400"  # 24小时公共缓存
         }
 
-        # 上传文件到对象存储
+        # 上传文件到对象存储，传递forbid_overwrite参数
         self.http_uploader.upload(
             url=upload_url_resp.upload_url,
             content=content,
             headers=headers,
             total_size=file_size,
+            forbid_overwrite=forbid_overwrite,
         )
 
         # 确认上传完成
@@ -182,6 +185,7 @@ class SyncBlobService(BaseFileService):
             is_temporary: Optional[bool] = False,
             expire_seconds: Optional[int] = None,
             keep_original_filename: Optional[bool] = False,
+            forbid_overwrite: Optional[bool] = False,
             request_id: Optional[str] = None,
             **metadata
     ) -> FileUploadResponse:
@@ -198,6 +202,7 @@ class SyncBlobService(BaseFileService):
             is_temporary=is_temporary,
             expire_seconds=expire_seconds if is_temporary and expire_seconds else None,
             keep_original_filename=keep_original_filename,
+            forbid_overwrite=forbid_overwrite,
             request_id=request_id,
             **metadata
         )
@@ -222,13 +227,14 @@ class SyncBlobService(BaseFileService):
             mime_type=mime_type,
         )
 
-        # 上传文件到对象存储
+        # 上传文件到对象存储，传递forbid_overwrite参数
         self.http_uploader.upload(
             url=upload_url,
             content=content,
             headers=headers,
             total_size=file_size,
-            is_resume=True
+            is_resume=True,
+            forbid_overwrite=forbid_overwrite,
         )
 
         # 确认上传完成
@@ -276,6 +282,7 @@ class SyncBlobService(BaseFileService):
             is_temporary: Optional[bool] = False,
             expire_seconds: Optional[int] = None,
             keep_original_filename: Optional[bool] = False,
+            forbid_overwrite: Optional[bool] = False,
             request_id: Optional[str] = None,
             **metadata
     ) -> UploadUrlResponse:
@@ -292,6 +299,7 @@ class SyncBlobService(BaseFileService):
             is_temporary: 是否为临时文件
             expire_seconds: 过期秒数
             keep_original_filename: 是否保留原始文件名（默认False）
+            forbid_overwrite: 防止覆盖同名文件（默认False）
             request_id: 请求ID（可选，如果不提供则自动生成）
             **metadata: 额外的元数据（如 x-org-id, x-user-id 等）
 
@@ -311,6 +319,7 @@ class SyncBlobService(BaseFileService):
             is_temporary=is_temporary,
             expire_seconds=expire_seconds if is_temporary and expire_seconds else None,
             keep_original_filename=keep_original_filename,
+            forbid_overwrite=forbid_overwrite,
         )
 
         if folder_id:
@@ -338,6 +347,7 @@ class SyncBlobService(BaseFileService):
             is_temporary: Optional[bool] = False,
             expire_seconds: Optional[int] = None,
             keep_original_filename: Optional[bool] = False,
+            forbid_overwrite: Optional[bool] = True,
             request_id: Optional[str] = None,
             **metadata
     ) -> UploadUrlResponse:
@@ -354,6 +364,7 @@ class SyncBlobService(BaseFileService):
             is_temporary: 是否为临时文件
             expire_seconds: 过期秒数
             keep_original_filename: 是否保留原始文件名（默认False）
+            forbid_overwrite: 防止覆盖同名文件（默认True）
             request_id: 请求ID（可选，如果不提供则自动生成）
             **metadata: 额外的元数据（如 x-org-id, x-user-id 等）
 
@@ -375,6 +386,7 @@ class SyncBlobService(BaseFileService):
             is_temporary=is_temporary,
             expire_seconds=expire_seconds if is_temporary and expire_seconds else None,
             keep_original_filename=keep_original_filename,
+            forbid_overwrite=forbid_overwrite,
         )
 
         if folder_id:
@@ -400,6 +412,7 @@ class SyncBlobService(BaseFileService):
             is_temporary: Optional[bool] = False,
             expire_seconds: Optional[int] = None,
             keep_original_filename: Optional[bool] = False,
+            forbid_overwrite: Optional[bool] = True,
             url: Optional[str] = None,
             file_name: Optional[str] = None,
             mime_type: Optional[str] = None,
@@ -416,6 +429,7 @@ class SyncBlobService(BaseFileService):
             is_temporary: 是否为临时文件
             expire_seconds: 过期秒数
             keep_original_filename: 是否保留原始文件名（默认False）
+            forbid_overwrite: 防止覆盖同名文件（默认True）
             url: 要下载并上传的URL（可选）
             file_name: 当使用url参数时指定的文件名（可选）
             mime_type: MIME类型（可选，用于推断文件扩展名，特别适用于AI生成的字节数据）
@@ -510,8 +524,16 @@ class SyncBlobService(BaseFileService):
             hundred_mb = 1024 * 1024 * 100
             if file_size >= ten_mb and file_size < hundred_mb:  # 10MB
                 mode = UploadMode.STREAM  # 大文件自动使用流式上传模式
+            # 暂时屏蔽 RESUMABLE 模式，因为OSS断点续传尚未完成开发
+            # elif file_size > hundred_mb:
+            #     mode = UploadMode.RESUMABLE  # 特大文件自动使用断点续传模式
             elif file_size > hundred_mb:
-                mode = UploadMode.RESUMABLE  # 特大文件自动使用断点续传模式
+                mode = UploadMode.STREAM  # 暂时使用流式上传代替断点续传
+
+        # OSS断点续传尚未完成，将RESUMABLE模式自动转为STREAM模式
+        if mode == UploadMode.RESUMABLE:
+            mode = UploadMode.STREAM
+            # TODO: 当OSS断点续传功能完成后，移除此转换逻辑
 
         # 根据上传模式执行不同的上传逻辑
         if mode == UploadMode.NORMAL:
@@ -542,6 +564,7 @@ class SyncBlobService(BaseFileService):
                 is_temporary=is_temporary,
                 expire_seconds=expire_seconds if is_temporary and expire_seconds else None,
                 keep_original_filename=keep_original_filename,
+                forbid_overwrite=forbid_overwrite,
                 request_id=request_id,
                 **metadata
             )
@@ -559,6 +582,7 @@ class SyncBlobService(BaseFileService):
                 is_temporary=is_temporary,
                 expire_seconds=expire_seconds if is_temporary and expire_seconds else None,
                 keep_original_filename=keep_original_filename,
+                forbid_overwrite=forbid_overwrite,
                 request_id=request_id,
                 **metadata
             )
@@ -993,3 +1017,115 @@ class SyncBlobService(BaseFileService):
             error=response.error if response.error else None,
             variant_info=variant_info
         )
+
+    def batch_get_file_status(
+        self,
+        file_ids: List[str],
+        *,
+        include_details: Optional[bool] = False,
+        request_id: Optional[str] = None,
+        **metadata
+    ) -> BatchFileStatusResponse:
+        """
+        批量获取文件状态
+        
+        Args:
+            file_ids: 文件ID列表（最多100个）
+            include_details: 是否包含详细状态信息（默认False）
+            request_id: 请求ID，用于追踪
+            **metadata: 额外的gRPC元数据
+            
+        Returns:
+            BatchFileStatusResponse: 批量文件状态响应
+        """
+        from ...rpc.gen import file_service_pb2, file_service_pb2_grpc
+        
+        stub = self.client.get_stub(file_service_pb2_grpc.FileServiceStub)
+        
+        request = file_service_pb2.BatchFileStatusRequest(
+            file_ids=file_ids,
+            include_details=include_details if include_details is not None else False
+        )
+        
+        # 构建元数据
+        grpc_metadata = self.client.build_metadata(request_id=request_id, **metadata)
+        
+        response = stub.BatchGetFileStatus(request, metadata=grpc_metadata)
+        
+        # 转换文件状态信息
+        statuses = []
+        for status_info in response.statuses:
+            # 转换状态详情（如果存在）
+            details = None
+            if status_info.HasField('details'):
+                details = FileStatusDetails(
+                    file_size=status_info.details.file_size if status_info.details.HasField('file_size') else None,
+                    storage_type=status_info.details.storage_type if status_info.details.HasField('storage_type') else None,
+                    storage_region=status_info.details.storage_region if status_info.details.HasField('storage_region') else None,
+                    compression_task_id=status_info.details.compression_task_id if status_info.details.HasField('compression_task_id') else None,
+                    compression_variants_count=status_info.details.compression_variants_count if status_info.details.HasField('compression_variants_count') else None,
+                    compression_progress=status_info.details.compression_progress if status_info.details.HasField('compression_progress') else None,
+                    sync_regions_total=status_info.details.sync_regions_total if status_info.details.HasField('sync_regions_total') else None,
+                    sync_regions_completed=status_info.details.sync_regions_completed if status_info.details.HasField('sync_regions_completed') else None,
+                    sync_pending_regions=list(status_info.details.sync_pending_regions)
+                )
+            
+            # 转换枚举值
+            upload_status = self._convert_upload_status(status_info.upload_status)
+            compression_status = self._convert_compression_status(status_info.compression_status)
+            sync_status = self._convert_sync_status(status_info.sync_status)
+            
+            statuses.append(FileStatusInfo(
+                file_id=status_info.file_id,
+                upload_status=upload_status,
+                compression_status=compression_status,
+                sync_status=sync_status,
+                details=details,
+                error_message=status_info.error_message if status_info.HasField('error_message') else None
+            ))
+        
+        return BatchFileStatusResponse(
+            statuses=statuses,
+            timestamp=response.timestamp,
+            cache_hit_count=response.cache_hit_count
+        )
+
+    def _convert_upload_status(self, proto_status: int) -> FileUploadStatus:
+        """转换上传状态枚举"""
+        status_map = {
+            0: FileUploadStatus.UPLOAD_UNKNOWN,
+            1: FileUploadStatus.UPLOAD_PENDING,
+            2: FileUploadStatus.UPLOAD_PROCESSING,
+            3: FileUploadStatus.UPLOAD_COMPLETED,
+            4: FileUploadStatus.UPLOAD_FAILED,
+            5: FileUploadStatus.UPLOAD_FILE_NOT_FOUND,
+        }
+        return status_map.get(proto_status, FileUploadStatus.UPLOAD_UNKNOWN)
+
+    def _convert_compression_status(self, proto_status: int) -> FileCompressionStatus:
+        """转换压缩状态枚举"""
+        status_map = {
+            0: FileCompressionStatus.COMPRESSION_UNKNOWN,
+            1: FileCompressionStatus.COMPRESSION_NOT_APPLICABLE,
+            2: FileCompressionStatus.COMPRESSION_PENDING,
+            3: FileCompressionStatus.COMPRESSION_PROCESSING,
+            4: FileCompressionStatus.COMPRESSION_COMPLETED,
+            5: FileCompressionStatus.COMPRESSION_FAILED,
+            6: FileCompressionStatus.COMPRESSION_SKIPPED,
+            7: FileCompressionStatus.COMPRESSION_FILE_NOT_FOUND,
+        }
+        return status_map.get(proto_status, FileCompressionStatus.COMPRESSION_UNKNOWN)
+
+    def _convert_sync_status(self, proto_status: int) -> FileSyncStatus:
+        """转换同步状态枚举"""
+        status_map = {
+            0: FileSyncStatus.SYNC_UNKNOWN,
+            1: FileSyncStatus.SYNC_NOT_REQUIRED,
+            2: FileSyncStatus.SYNC_PENDING,
+            3: FileSyncStatus.SYNC_PROCESSING,
+            4: FileSyncStatus.SYNC_PARTIAL,
+            5: FileSyncStatus.SYNC_COMPLETED,
+            6: FileSyncStatus.SYNC_FAILED,
+            7: FileSyncStatus.SYNC_FILE_NOT_FOUND,
+        }
+        return status_map.get(proto_status, FileSyncStatus.SYNC_UNKNOWN)

file_hub_client/services/file/sync_file_service.py

@@ -490,3 +490,69 @@ class SyncFileService(BaseFileService):
             error=response.error if response.error else None,
             variant_info=variant_info
         )
+
+    def import_from_gcs(
+            self,
+            gcs_uri: str,
+            operation_type: str,
+            *,
+            folder_id: Optional[str] = None,
+            file_name: Optional[str] = None,
+            keep_original_filename: bool = False,
+            created_by_role: Optional[str] = None,
+            created_by: Optional[str] = None,
+            request_id: Optional[str] = None,
+            **metadata
+    ) -> 'ImportFromGcsResponse':
+        """
+        从GCS导入文件
+        
+        Args:
+            gcs_uri: GCS URI, 例如 gs://bucket/path/to/file
+            operation_type: 操作类型，"copy" 或 "move"
+            folder_id: 目标文件夹ID（可选）
+            file_name: 自定义文件名（可选）
+            keep_original_filename: 保留原始文件名，默认False
+            created_by_role: 创建者角色（可选）
+            created_by: 创建者ID（可选）
+            request_id: 请求ID，用于追踪
+            **metadata: 额外的gRPC元数据
+            
+        Returns:
+            ImportFromGcsResponse: 导入响应，包含文件信息和上传文件信息
+        """
+        from ...rpc.gen import file_service_pb2, file_service_pb2_grpc
+        from ...schemas import ImportFromGcsResponse
+        
+        stub = self.client.get_stub(file_service_pb2_grpc.FileServiceStub)
+        
+        request = file_service_pb2.ImportFromGcsRequest(
+            gcs_uri=gcs_uri,
+            operation_type=operation_type,
+            keep_original_filename=keep_original_filename
+        )
+        
+        if folder_id:
+            request.folder_id = folder_id
+        if file_name:
+            request.file_name = file_name
+        if created_by_role:
+            request.created_by_role = created_by_role
+        if created_by:
+            request.created_by = created_by
+        
+        # 构建元数据
+        grpc_metadata = self.client.build_metadata(request_id=request_id, **metadata)
+        
+        response = stub.ImportFromGcs(request, metadata=grpc_metadata)
+        
+        # 转换文件信息
+        file_info = self._convert_file_info(response.file)
+        
+        # 转换上传文件信息
+        upload_file_info = self._convert_upload_file_info(response.upload_file)
+        
+        return ImportFromGcsResponse(
+            file=file_info,
+            upload_file=upload_file_info
+        )

file_hub_client/utils/logging.py

@@ -115,22 +115,18 @@ def setup_logging(
     
     # 初始化日志（使用JSON格式）
     if enable_grpc_logging:
-        log_record = logging.LogRecord(
-            name=logger.name,
-            level=logging.INFO,
-            pathname="",
-            lineno=0,
-            msg="📡 文件中心客户端 gRPC 日志已初始化",
-            args=(),
-            exc_info=None
+        # 使用 logger.debug() 以遵守日志级别设置
+        logger.debug(
+            "📡 文件中心客户端 gRPC 日志已初始化",
+            extra={
+                "log_type": "debug",
+                "data": {
+                    "level": level,
+                    "grpc_logging": enable_grpc_logging,
+                    "json_format": use_json_format
+                }
+            }
         )
-        log_record.log_type = "info"
-        log_record.data = {
-            "level": level,
-            "grpc_logging": enable_grpc_logging,
-            "json_format": use_json_format
-        }
-        logger.handle(log_record)
 
 
 def get_logger() -> logging.Logger:

file_hub_client/utils/upload_helper.py

@@ -9,11 +9,61 @@ import asyncio
 import aiohttp
 import requests
 from pathlib import Path
-from typing import Union, BinaryIO, Optional, Callable, Dict, Any, AsyncGenerator
+from typing import Union, BinaryIO, Optional, Callable, Dict, Any, AsyncGenerator, Tuple
 from dataclasses import dataclass
 import hashlib
 
 
+def detect_storage_type(url: str) -> str:
+    """
+    根据URL检测存储类型
+
+    Args:
+        url: 上传URL
+
+    Returns:
+        存储类型: 'gcs'、'oss' 或 'unknown'
+    """
+    if 'storage.googleapis.com' in url or 'storage.cloud.google.com' in url:
+        return 'gcs'
+    elif 'aliyuncs.com' in url:
+        return 'oss'
+    else:
+        return 'gcs'
+
+
+def get_forbid_overwrite_headers(url: str, forbid_overwrite: bool = False) -> Dict[str, str]:
+    """
+    获取防止覆盖的headers
+
+    Args:
+        url: 上传URL
+        forbid_overwrite: 是否防止覆盖
+
+    Returns:
+        包含防止覆盖header的字典
+
+    Note:
+        - GCS: 需要在HTTP header中添加 x-goog-if-generation-match: 0
+        - OSS: 需要在HTTP header中添加 x-oss-forbid-overwrite: true
+    """
+    if not forbid_overwrite:
+        return {}
+
+    storage_type = detect_storage_type(url)
+
+    if storage_type == 'gcs':
+        return {
+            'x-goog-if-generation-match': '0'
+        }
+    elif storage_type == 'oss':
+        return {
+            'x-oss-forbid-overwrite': 'true'
+        }
+    else:
+        return {}
+
+
 @dataclass
 class UploadProgress:
     """上传进度信息"""
@@ -127,10 +177,11 @@ class HttpUploader:
             progress_callback: Optional[Callable[[UploadProgress], None]] = None,
             total_size: Optional[int] = None,
             is_resume: bool = False,
+            forbid_overwrite: bool = False,
     ) -> requests.Response:
         """
         上传文件到指定URL
-        
+
         Args:
             url: 上传URL
             content: 文件内容
@@ -138,9 +189,15 @@ class HttpUploader:
             headers: 请求头
             progress_callback: 进度回调函数
             is_resume: 是否断点续传
+            forbid_overwrite: 是否防止覆盖（添加相应的header）
         """
         headers = headers or {}
 
+        # 添加防止覆盖的headers
+        if forbid_overwrite:
+            forbid_headers = get_forbid_overwrite_headers(url, forbid_overwrite)
+            headers.update(forbid_headers)
+
         # 获取文件大小（不生成 chunk，避免提前读取）
         final_total_size = self._calculate_total_size(content) if total_size is None else total_size
 
@@ -360,19 +417,26 @@ class AsyncHttpUploader:
             progress_callback: Optional[Callable[[UploadProgress], None]] = None,
             total_size: Optional[int] = None,
             is_resume: bool = False,
+            forbid_overwrite: bool = False,
     ) -> aiohttp.ClientResponse:
         """
         异步上传文件到指定URL
-        
+
         Args:
             url: 上传URL
             content: 文件内容
             headers: 请求头
             progress_callback: 进度回调函数
             is_resume: 是否断点续传
+            forbid_overwrite: 是否防止覆盖（添加相应的header）
         """
         headers = headers or {}
 
+        # 添加防止覆盖的headers
+        if forbid_overwrite:
+            forbid_headers = get_forbid_overwrite_headers(url, forbid_overwrite)
+            headers.update(forbid_headers)
+
         # 获取文件大小（避免读取内容）
         final_total_size = total_size or await self._calculate_total_size(content)