tamar-file-hub-client 0.0.1__py3-none-any.whl

file_hub_client/utils/__init__.py

@@ -0,0 +1,48 @@
+"""
+工具函数模块
+"""
+from .converter import (
+    timestamp_to_datetime
+)
+from .file_utils import (
+    get_file_mime_type,
+    get_file_extension,
+    humanize_file_size,
+    calculate_file_hash,
+    split_file_chunks,
+)
+from .retry import retry_with_backoff
+from .upload_helper import (
+    HttpUploader,
+    AsyncHttpUploader,
+    UploadProgress,
+    calculate_file_md5,
+)
+from .download_helper import (
+    HttpDownloader,
+    AsyncHttpDownloader,
+    DownloadProgress,
+)
+
+__all__ = [
+    # 文件工具
+    "get_file_mime_type",
+    "get_file_extension",
+    "humanize_file_size",
+    "calculate_file_hash",
+    "split_file_chunks",
+
+    # 重试工具
+    "retry_with_backoff",
+
+    # 上传助手
+    "HttpUploader",
+    "AsyncHttpUploader",
+    "UploadProgress",
+    "calculate_file_md5",
+
+    # 下载助手
+    "HttpDownloader",
+    "AsyncHttpDownloader",
+    "DownloadProgress",
+]

file_hub_client/utils/converter.py

@@ -0,0 +1,108 @@
+"""
+Proto和Model之间的转换工具
+"""
+from datetime import datetime
+from typing import Any
+from google.protobuf.timestamp_pb2 import Timestamp
+
+from ..schemas import File, FolderInfo
+
+
+def timestamp_to_datetime(timestamp: Timestamp) -> datetime:
+    """将protobuf timestamp转换为datetime"""
+    return datetime.fromtimestamp(timestamp.seconds + timestamp.nanos / 1e9)
+#
+#
+# def datetime_to_timestamp(dt: datetime) -> Timestamp:
+#     """将datetime转换为protobuf timestamp"""
+#     timestamp = Timestamp()
+#     timestamp.FromDatetime(dt)
+#     return timestamp
+#
+#
+# def convert_proto_to_model(proto_obj: Any) -> Any:
+#     """
+#     将Proto对象转换为Model对象
+#
+#     Args:
+#         proto_obj: Proto对象
+#
+#     Returns:
+#         Model对象
+#     """
+#     # 动态导入，避免循环导入
+#     from ..rpc.gen import file_service_pb2 as file_hub_pb2
+#
+#     if isinstance(proto_obj, file_hub_pb2.FileInfo):
+#         return File(
+#             id=proto_obj.id,
+#             #name=proto_obj.name,
+#             folder_id=proto_obj.folder_id or None,
+#             #type=ModelFileType.TRADITIONAL if proto_obj.type == file_hub_pb2.FILE_TYPE_TRADITIONAL else ModelFileType.CUSTOM,
+#             #size=proto_obj.size,
+#             mime_type=proto_obj.mime_type or None,
+#             created_at=timestamp_to_datetime(proto_obj.created_at),
+#             updated_at=timestamp_to_datetime(proto_obj.updated_at),
+#             #metadata=dict(proto_obj.metadata),
+#             #storage_path=proto_obj.storage_path or None,
+#         )
+#
+#     elif isinstance(proto_obj, file_hub_pb2.FolderInfo):
+#         return FolderInfo(
+#             id=proto_obj.id,
+#             folder_name=proto_obj.name,
+#             parent_id=proto_obj.parent_id or None,
+#             created_at=timestamp_to_datetime(proto_obj.created_at),
+#             updated_at=timestamp_to_datetime(proto_obj.updated_at),
+#             #metadata=dict(proto_obj.metadata),
+#         )
+#
+#     else:
+#         raise ValueError(f"不支持的Proto类型: {type(proto_obj)}")
+#
+#
+# def convert_model_to_proto(model_obj: Any) -> Any:
+#     """
+#     将Model对象转换为Proto对象
+#
+#     Args:
+#         model_obj: Model对象
+#
+#     Returns:
+#         Proto对象
+#     """
+#     # 动态导入，避免循环导入
+#     from ..rpc.gen import file_service_pb2 as file_hub_pb2
+#
+#     if isinstance(model_obj, File):
+#         proto_type = (
+#             file_hub_pb2.FILE_TYPE_TRADITIONAL
+#             if model_obj.type == ModelFileType.TRADITIONAL
+#             else file_hub_pb2.FILE_TYPE_CUSTOM
+#         )
+#
+#         return file_hub_pb2.FileInfo(
+#             id=model_obj.id,
+#             name=model_obj.name,
+#             folder_id=model_obj.folder_id or "",
+#             type=proto_type,
+#             size=model_obj.size,
+#             mime_type=model_obj.mime_type or "",
+#             created_at=datetime_to_timestamp(model_obj.created_at),
+#             updated_at=datetime_to_timestamp(model_obj.updated_at),
+#             metadata=model_obj.metadata,
+#             storage_path=model_obj.storage_path or "",
+#         )
+#
+#     elif isinstance(model_obj, FolderInfo):
+#         return file_hub_pb2.FolderInfo(
+#             id=model_obj.id,
+#             name=model_obj.name,
+#             parent_id=model_obj.parent_id or "",
+#             created_at=datetime_to_timestamp(model_obj.created_at),
+#             updated_at=datetime_to_timestamp(model_obj.updated_at),
+#             metadata=model_obj.metadata,
+#         )
+#
+#     else:
+#         raise ValueError(f"不支持的Model类型: {type(model_obj)}")

file_hub_client/utils/download_helper.py

@@ -0,0 +1,355 @@
+"""
+下载助手模块
+
+提供HTTP下载、流式下载、进度监控等功能
+"""
+import time
+import asyncio
+from urllib.parse import urlparse
+
+import aiohttp
+import requests
+from pathlib import Path
+from typing import Union, Optional, Callable
+from dataclasses import dataclass
+
+
+@dataclass
+class DownloadProgress:
+    """下载进度信息"""
+    total_size: int
+    downloaded_size: int
+    percentage: float
+    speed: float  # bytes per second
+    remaining_time: float  # seconds
+
+    @property
+    def is_completed(self) -> bool:
+        return self.downloaded_size >= self.total_size if self.total_size > 0 else False
+
+
+class HttpDownloader:
+    """HTTP下载器，支持同步下载"""
+
+    def __init__(self, chunk_size: int = 1024 * 1024, total_retries: int = 3, retry_delay_seconds: int = 5):  # 默认1MB分片
+        self.chunk_size = chunk_size
+        self.total_retries = total_retries
+        self.retry_delay_seconds = retry_delay_seconds
+
+    def download(
+            self,
+            url: str,
+            save_path: Optional[Union[str, Path]] = None,
+            chunk_size: Optional[int] = None,
+            headers: Optional[dict] = None,
+            progress_callback: Optional[Callable[[DownloadProgress], None]] = None,
+            timeout: Optional[int] = None
+    ) -> Union[bytes, Path]:
+        """
+        从URL下载文件
+        
+        Args:
+            url: 下载URL
+            save_path: 保存路径（如果为None，返回字节数据）
+            chunk_size: 分块大小
+            headers: 请求头
+            progress_callback: 进度回调函数
+            timeout: 超时时间（秒）
+            
+        Returns:
+            下载的内容（字节）或保存的文件路径
+        """
+        headers = headers or {}
+        chunk_size = chunk_size or self.chunk_size
+        save_path = Path(save_path) if save_path else None
+
+        # 校验扩展名一致性（若提供）
+        parsed_url = urlparse(url)
+        url_suffix = Path(parsed_url.path).suffix.lower()
+        if save_path and save_path.suffix and save_path.suffix.lower() != url_suffix:
+            raise ValueError(
+                f"File extension mismatch: download_url ends with '{url_suffix}', but save_path ends with '{save_path.suffix.lower()}'")
+
+        for attempt in range(self.total_retries):
+            try:
+                resume_from = 0
+                if save_path:
+                    temp_path = save_path.with_suffix(save_path.suffix + ".part")
+                    if temp_path.exists():
+                        resume_from = temp_path.stat().st_size
+                        headers["Range"] = f"bytes={resume_from}-"
+
+                # 发送请求
+                response = requests.get(url, headers=headers, stream=True, timeout=timeout)
+                response.raise_for_status()
+
+                content_length = int(response.headers.get('content-length', 0))
+                total_size = content_length + resume_from if save_path else content_length
+
+                if save_path is None:
+                    return self._download_to_memory(response, total_size, chunk_size, progress_callback, resume_from)
+                else:
+                    return self._download_to_file(response, save_path, total_size, chunk_size, progress_callback,
+                                                  resume_from)
+            except Exception as e:
+                if attempt == self.total_retries - 1:
+                    raise
+                time.sleep(self.retry_delay_seconds)
+
+    def _download_to_memory(
+            self,
+            response: requests.Response,
+            total_size: int,
+            chunk_size: int,
+            progress_callback: Optional[Callable[[DownloadProgress], None]],
+            resume_from: int = 0
+    ) -> bytes:
+        """下载到内存"""
+        chunks = []
+        downloaded_size = resume_from
+        start_time = time.time()
+
+        for chunk in response.iter_content(chunk_size=chunk_size):
+            if chunk:
+                chunks.append(chunk)
+                downloaded_size += len(chunk)
+
+                if progress_callback:
+                    self._report_progress(
+                        downloaded_size, total_size, start_time, progress_callback
+                    )
+
+        return b''.join(chunks)
+
+    def _download_to_file(
+            self,
+            response: requests.Response,
+            save_path: Path,
+            total_size: int,
+            chunk_size: int,
+            progress_callback: Optional[Callable[[DownloadProgress], None]],
+            resume_from: int = 0
+    ) -> Path:
+        """下载到文件"""
+        # 创建目录
+        save_path.parent.mkdir(parents=True, exist_ok=True)
+
+        # 临时保存为 .part 文件
+        temp_path = save_path.with_suffix(save_path.suffix + ".part")
+
+        downloaded_size = resume_from
+        start_time = time.time()
+
+        mode = 'ab' if resume_from > 0 else 'wb'
+
+        with open(temp_path, mode) as f:
+            for chunk in response.iter_content(chunk_size=chunk_size):
+                if chunk:
+                    f.write(chunk)
+                    downloaded_size += len(chunk)
+                    if progress_callback:
+                        self._report_progress(downloaded_size, total_size, start_time, progress_callback)
+
+        if save_path.exists():
+            save_path.unlink()
+        temp_path.rename(save_path)
+        return save_path
+
+    def _report_progress(
+            self,
+            downloaded_size: int,
+            total_size: int,
+            start_time: float,
+            callback: Callable[[DownloadProgress], None]
+    ):
+        """报告下载进度"""
+        elapsed = time.time() - start_time
+        speed = downloaded_size / elapsed if elapsed > 0 else 0
+        percentage = (downloaded_size / total_size * 100) if total_size > 0 else 0
+        remaining = (total_size - downloaded_size) / speed if speed > 0 else 0
+
+        progress = DownloadProgress(
+            total_size=total_size,
+            downloaded_size=downloaded_size,
+            percentage=percentage,
+            speed=speed,
+            remaining_time=remaining
+        )
+        callback(progress)
+
+
+class AsyncHttpDownloader:
+    """异步HTTP下载器"""
+
+    def __init__(self, chunk_size: int = 1024 * 1024, total_retries: int = 3, retry_delay_seconds: int = 5):  # 默认1MB分片
+        self.chunk_size = chunk_size
+        self.total_retries = total_retries
+        self.retry_delay_seconds = retry_delay_seconds
+
+    async def download(
+            self,
+            url: str,
+            save_path: Optional[Union[str, Path]] = None,
+            chunk_size: Optional[int] = None,
+            headers: Optional[dict] = None,
+            progress_callback: Optional[Callable[[DownloadProgress], None]] = None,
+            timeout: Optional[int] = None
+    ) -> Union[bytes, Path]:
+        """
+        异步从URL下载文件
+        
+        Args:
+            url: 下载URL
+            save_path: 保存路径（如果为None，返回字节数据）
+            chunk_size: 分块大小
+            headers: 请求头
+            progress_callback: 进度回调函数
+            timeout: 超时时间（秒）
+            
+        Returns:
+            下载的内容（字节）或保存的文件路径
+        """
+        headers = headers or {}
+        save_path = Path(save_path) if save_path else None
+        chunk_size = chunk_size or self.chunk_size
+
+        # 校验扩展名一致性（若提供）
+        parsed_url = urlparse(url)
+        url_suffix = Path(parsed_url.path).suffix.lower()
+        if save_path and save_path.suffix and save_path.suffix.lower() != url_suffix:
+            raise ValueError(
+                f"File extension mismatch: download_url ends with '{url_suffix}', but save_path ends with '{save_path.suffix.lower()}'")
+
+        for attempt in range(self.total_retries):
+            try:
+                resume_from = 0
+                if save_path:
+                    temp_path = save_path.with_suffix(save_path.suffix + ".part")
+                    if temp_path.exists():
+                        resume_from = temp_path.stat().st_size
+                        headers["Range"] = f"bytes={resume_from}-"
+
+                timeout_config = aiohttp.ClientTimeout(total=timeout) if timeout else None
+
+                async with aiohttp.ClientSession() as session:
+                    async with session.get(url, headers=headers, timeout=timeout_config) as response:
+                        if response.status not in (200, 206):
+                            raise Exception(f"Unexpected status: {response.status}")
+
+                        total_size = int(response.headers.get("content-length", 0)) + resume_from
+
+                        if save_path is None:
+                            return await self._download_to_memory(response, total_size, chunk_size, progress_callback,
+                                                                  resume_from)
+                        else:
+                            return await self._download_to_file(response, save_path, total_size, chunk_size,
+                                                                progress_callback, resume_from)
+            except Exception as e:
+                if attempt == self.total_retries - 1:
+                    raise
+                await asyncio.sleep(self.retry_delay_seconds)
+
+    async def _download_to_memory(
+            self,
+            response: aiohttp.ClientResponse,
+            total_size: int,
+            chunk_size: Optional[int],
+            progress_callback: Optional[Callable[[DownloadProgress], None]],
+            resume_from: int = 0
+    ) -> bytes:
+        """异步下载到内存"""
+        chunks = []
+        downloaded_size = resume_from
+        start_time = asyncio.get_event_loop().time()
+
+        async for chunk in response.content.iter_chunked(chunk_size):
+            chunks.append(chunk)
+            downloaded_size += len(chunk)
+
+            if progress_callback:
+                await self._report_progress(
+                    downloaded_size, total_size, start_time, progress_callback
+                )
+
+        return b''.join(chunks)
+
+    async def _download_to_file(
+            self,
+            response: aiohttp.ClientResponse,
+            save_path: Path,
+            total_size: int,
+            chunk_size: Optional[int],
+            progress_callback: Optional[Callable[[DownloadProgress], None]],
+            resume_from: int = 0
+    ) -> Path:
+        """异步下载到文件"""
+        # 创建目录
+        save_path.parent.mkdir(parents=True, exist_ok=True)
+
+        # 临时保存为 .part 文件
+        temp_path = save_path.with_suffix(save_path.suffix + ".part")
+
+        downloaded_size = 0
+        start_time = time.time()
+
+        mode = 'ab' if resume_from > 0 else 'wb'
+
+        if aiofiles:
+            # 使用异步文件IO
+            async with aiofiles.open(temp_path, mode) as f:
+                async for chunk in response.content.iter_chunked(chunk_size):
+                    await f.write(chunk)
+                    downloaded_size += len(chunk)
+
+                    if progress_callback:
+                        await self._report_progress(
+                            downloaded_size, total_size, start_time, progress_callback
+                        )
+        else:
+            # 回退到同步文件IO
+            with open(temp_path, mode) as f:
+                async for chunk in response.content.iter_chunked(chunk_size):
+                    f.write(chunk)
+                    downloaded_size += len(chunk)
+
+                    if progress_callback:
+                        await self._report_progress(
+                            downloaded_size, total_size, start_time, progress_callback
+                        )
+        if save_path.exists():
+            save_path.unlink()
+        temp_path.rename(save_path)
+        return save_path
+
+    async def _report_progress(
+            self,
+            downloaded_size: int,
+            total_size: int,
+            start_time: float,
+            callback: Callable[[DownloadProgress], None]
+    ):
+        """报告下载进度"""
+        elapsed = asyncio.get_event_loop().time() - start_time
+        speed = downloaded_size / elapsed if elapsed > 0 else 0
+        percentage = (downloaded_size / total_size * 100) if total_size > 0 else 0
+        remaining = (total_size - downloaded_size) / speed if speed > 0 else 0
+
+        progress = DownloadProgress(
+            total_size=total_size,
+            downloaded_size=downloaded_size,
+            percentage=percentage,
+            speed=speed,
+            remaining_time=remaining
+        )
+
+        if asyncio.iscoroutinefunction(callback):
+            await callback(progress)
+        else:
+            callback(progress)
+
+
+# 尝试导入aiofiles（可选依赖）
+try:
+    import aiofiles
+except ImportError:
+    aiofiles = None

file_hub_client/utils/file_utils.py

@@ -0,0 +1,105 @@
+"""
+文件工具函数
+"""
+import hashlib
+import mimetypes
+from pathlib import Path
+from typing import Generator, Optional, BinaryIO, Union
+
+
+def get_file_mime_type(file_path: Union[str, Path]) -> str:
+    """
+    获取文件的MIME类型
+    
+    Args:
+        file_path: 文件路径
+        
+    Returns:
+        MIME类型
+    """
+    file_path = Path(file_path)
+    mime_type, _ = mimetypes.guess_type(str(file_path))
+    return mime_type or "application/octet-stream"
+
+
+def get_file_extension(file_name: str) -> str:
+    """
+    获取文件扩展名
+    
+    Args:
+        file_name: 文件名
+        
+    Returns:
+        文件扩展名（包含点号）
+    """
+    return Path(file_name).suffix.lower()
+
+
+def humanize_file_size(size_bytes: int) -> str:
+    """
+    将文件大小转换为人类可读的格式
+    
+    Args:
+        size_bytes: 文件大小（字节）
+        
+    Returns:
+        人类可读的文件大小
+    """
+    for unit in ["B", "KB", "MB", "GB", "TB"]:
+        if size_bytes < 1024.0:
+            return f"{size_bytes:.2f} {unit}"
+        size_bytes /= 1024.0
+    return f"{size_bytes:.2f} PB"
+
+
+def calculate_file_hash(file_path: Union[str, Path], algorithm: str = "sha256") -> str:
+    """
+    计算文件哈希值
+    
+    Args:
+        file_path: 文件路径
+        algorithm: 哈希算法（md5, sha1, sha256等）
+        
+    Returns:
+        文件哈希值（十六进制）
+    """
+    file_path = Path(file_path)
+    hash_obj = hashlib.new(algorithm)
+    
+    with open(file_path, "rb") as f:
+        while chunk := f.read(8192):
+            hash_obj.update(chunk)
+    
+    return hash_obj.hexdigest()
+
+
+def split_file_chunks(
+    file_obj: BinaryIO,
+    chunk_size: int = 1024 * 1024,  # 默认1MB
+    start_offset: int = 0
+) -> Generator[tuple[bytes, int, bool], None, None]:
+    """
+    将文件分割成块
+    
+    Args:
+        file_obj: 文件对象
+        chunk_size: 块大小（字节）
+        start_offset: 起始偏移量
+        
+    Yields:
+        (块数据, 偏移量, 是否最后一块)
+    """
+    file_obj.seek(start_offset)
+    offset = start_offset
+    
+    while True:
+        chunk = file_obj.read(chunk_size)
+        if not chunk:
+            break
+            
+        is_last = len(chunk) < chunk_size
+        yield chunk, offset, is_last
+        
+        offset += len(chunk)
+        if is_last:
+            break 

file_hub_client/utils/retry.py

@@ -0,0 +1,69 @@
+"""
+重试工具
+"""
+import asyncio
+import functools
+import time
+from typing import TypeVar, Callable, Type, Tuple
+
+T = TypeVar("T")
+
+
+def retry_with_backoff(
+    max_retries: int = 3,
+    initial_delay: float = 1.0,
+    backoff_factor: float = 2.0,
+    max_delay: float = 60.0,
+    exceptions: Tuple[Type[Exception], ...] = (Exception,)
+):
+    """
+    带退避策略的重试装饰器
+    
+    Args:
+        max_retries: 最大重试次数
+        initial_delay: 初始延迟（秒）
+        backoff_factor: 退避因子
+        max_delay: 最大延迟（秒）
+        exceptions: 需要重试的异常类型
+    """
+    def decorator(func: Callable[..., T]) -> Callable[..., T]:
+        if asyncio.iscoroutinefunction(func):
+            @functools.wraps(func)
+            async def async_wrapper(*args, **kwargs) -> T:
+                delay = initial_delay
+                last_exception = None
+                
+                for attempt in range(max_retries + 1):
+                    try:
+                        return await func(*args, **kwargs)
+                    except exceptions as e:
+                        last_exception = e
+                        if attempt < max_retries:
+                            await asyncio.sleep(delay)
+                            delay = min(delay * backoff_factor, max_delay)
+                        else:
+                            raise
+                            
+                raise last_exception
+            return async_wrapper
+        else:
+            @functools.wraps(func)
+            def sync_wrapper(*args, **kwargs) -> T:
+                delay = initial_delay
+                last_exception = None
+                
+                for attempt in range(max_retries + 1):
+                    try:
+                        return func(*args, **kwargs)
+                    except exceptions as e:
+                        last_exception = e
+                        if attempt < max_retries:
+                            time.sleep(delay)
+                            delay = min(delay * backoff_factor, max_delay)
+                        else:
+                            raise
+                            
+                raise last_exception
+            return sync_wrapper
+            
+    return decorator