tamar-file-hub-client 0.0.1__py3-none-any.whl → 0.0.3__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (36) hide show
  1. file_hub_client/__init__.py +39 -0
  2. file_hub_client/client.py +43 -6
  3. file_hub_client/rpc/async_client.py +91 -11
  4. file_hub_client/rpc/gen/taple_service_pb2.py +225 -0
  5. file_hub_client/rpc/gen/taple_service_pb2_grpc.py +1626 -0
  6. file_hub_client/rpc/generate_grpc.py +2 -2
  7. file_hub_client/rpc/interceptors.py +550 -0
  8. file_hub_client/rpc/protos/taple_service.proto +874 -0
  9. file_hub_client/rpc/sync_client.py +91 -9
  10. file_hub_client/schemas/__init__.py +60 -0
  11. file_hub_client/schemas/taple.py +413 -0
  12. file_hub_client/services/__init__.py +5 -0
  13. file_hub_client/services/file/async_blob_service.py +558 -482
  14. file_hub_client/services/file/async_file_service.py +18 -9
  15. file_hub_client/services/file/base_file_service.py +19 -6
  16. file_hub_client/services/file/sync_blob_service.py +556 -478
  17. file_hub_client/services/file/sync_file_service.py +18 -9
  18. file_hub_client/services/folder/async_folder_service.py +20 -11
  19. file_hub_client/services/folder/sync_folder_service.py +20 -11
  20. file_hub_client/services/taple/__init__.py +10 -0
  21. file_hub_client/services/taple/async_taple_service.py +2281 -0
  22. file_hub_client/services/taple/base_taple_service.py +353 -0
  23. file_hub_client/services/taple/idempotent_taple_mixin.py +142 -0
  24. file_hub_client/services/taple/sync_taple_service.py +2256 -0
  25. file_hub_client/utils/__init__.py +43 -1
  26. file_hub_client/utils/file_utils.py +59 -11
  27. file_hub_client/utils/idempotency.py +196 -0
  28. file_hub_client/utils/logging.py +315 -0
  29. file_hub_client/utils/retry.py +241 -2
  30. file_hub_client/utils/smart_retry.py +403 -0
  31. tamar_file_hub_client-0.0.3.dist-info/METADATA +2050 -0
  32. tamar_file_hub_client-0.0.3.dist-info/RECORD +57 -0
  33. tamar_file_hub_client-0.0.1.dist-info/METADATA +0 -874
  34. tamar_file_hub_client-0.0.1.dist-info/RECORD +0 -44
  35. {tamar_file_hub_client-0.0.1.dist-info → tamar_file_hub_client-0.0.3.dist-info}/WHEEL +0 -0
  36. {tamar_file_hub_client-0.0.1.dist-info → tamar_file_hub_client-0.0.3.dist-info}/top_level.txt +0 -0
@@ -1,482 +1,558 @@
1
- """
2
- 异步二进制大对象服务
3
- """
4
- import asyncio
5
-
6
- from pathlib import Path
7
- from typing import Optional, Union, BinaryIO, AsyncIterator
8
-
9
- from .base_file_service import BaseFileService
10
- from ...enums import UploadMode
11
- from ...errors import ValidationError
12
- from ...rpc import AsyncGrpcClient
13
- from ...schemas import FileUploadResponse, UploadUrlResponse
14
- from ...utils import AsyncHttpUploader, AsyncHttpDownloader, retry_with_backoff, get_file_mime_type
15
-
16
-
17
- class AsyncBlobService(BaseFileService):
18
- """异步文件(二进制大对象)服务"""
19
-
20
- def __init__(self, client: AsyncGrpcClient):
21
- """
22
- 初始化文件(二进制大对象)服务
23
-
24
- Args:
25
- client: 异步gRPC客户端
26
- """
27
- self.client = client
28
- self.http_uploader = AsyncHttpUploader()
29
- self.http_downloader = AsyncHttpDownloader()
30
-
31
- async def _generate_resumable_upload_url(
32
- self,
33
- file_name: str,
34
- file_size: int,
35
- folder_id: Optional[str] = None,
36
- file_type: str = "file",
37
- mime_type: Optional[str] = None,
38
- file_hash: Optional[str] = None,
39
- is_temporary: Optional[bool] = False,
40
- expire_seconds: Optional[int] = None,
41
- **metadata
42
- ) -> UploadUrlResponse:
43
- """
44
- 生成断点续传URL
45
-
46
- Args:
47
- file_name: 文件名
48
- file_size: 文件大小
49
- folder_id: 文件夹ID
50
- file_type: 文件类型
51
- mime_type: MIME类型
52
- file_hash: 文件哈希
53
- is_temporary: 是否为临时文件
54
- expire_seconds: 过期秒数
55
- **metadata: 额外的元数据(如 x-org-id, x-user-id 等)
56
-
57
- Returns:
58
- 上传URL响应
59
- """
60
- from ...rpc.gen import file_service_pb2, file_service_pb2_grpc
61
-
62
- stub = await self.client.get_stub(file_service_pb2_grpc.FileServiceStub)
63
-
64
- request = file_service_pb2.UploadUrlRequest(
65
- file_name=file_name,
66
- file_size=file_size,
67
- file_type=file_type,
68
- mime_type=mime_type or "application/octet-stream",
69
- file_hash=file_hash,
70
- is_temporary=is_temporary,
71
- expire_seconds=expire_seconds if is_temporary and expire_seconds else None,
72
- )
73
-
74
- if folder_id:
75
- request.folder_id = folder_id
76
-
77
- # 构建元数据
78
- grpc_metadata = self.client.build_metadata(**metadata)
79
-
80
- response = await stub.GenerateResumableUploadUrl(request, metadata=grpc_metadata)
81
-
82
- return UploadUrlResponse(
83
- file=self._convert_file_info(response.file),
84
- upload_file=self._convert_upload_file_info(response.upload_file),
85
- upload_url=response.url
86
- )
87
-
88
- async def _confirm_upload_completed(self, file_id: str, **metadata) -> None:
89
- """
90
- 确认上传完成
91
-
92
- Args:
93
- file_id: 文件ID
94
- **metadata: 额外的元数据(如 x-org-id, x-user-id 等)
95
- """
96
- from ...rpc.gen import file_service_pb2, file_service_pb2_grpc
97
-
98
- stub = await self.client.get_stub(file_service_pb2_grpc.FileServiceStub)
99
-
100
- request = file_service_pb2.UploadCompletedRequest(file_id=file_id)
101
-
102
- # 构建元数据
103
- grpc_metadata = self.client.build_metadata(**metadata)
104
-
105
- await stub.ConfirmUploadCompleted(request, metadata=grpc_metadata)
106
-
107
- @retry_with_backoff(max_retries=3)
108
- async def _upload_file(
109
- self,
110
- file_name: str,
111
- content: Union[bytes, BinaryIO, Path],
112
- folder_id: Optional[str] = None,
113
- file_type: str = "file",
114
- mime_type: Optional[str] = None,
115
- is_temporary: Optional[bool] = False,
116
- expire_seconds: Optional[int] = None,
117
- **metadata
118
- ) -> FileUploadResponse:
119
- """
120
- 普通上传
121
-
122
- Args:
123
- file_name: 文件名
124
- content: 文件内容(字节、文件对象或路径)
125
- folder_id: 文件夹ID
126
- file_type: 文件类型
127
- mime_type: MIME类型
128
- is_temporary: 是否为临时文件
129
- expire_seconds: 过期秒数
130
- **metadata: 额外的元数据(如 x-org-id, x-user-id 等)
131
-
132
- Returns:
133
- 文件信息
134
- """
135
- from ...rpc.gen import file_service_pb2, file_service_pb2_grpc
136
-
137
- stub = await self.client.get_stub(file_service_pb2_grpc.FileServiceStub)
138
-
139
- # 处理不同类型的内容
140
- if isinstance(content, Path):
141
- if not content.exists():
142
- raise ValidationError(f"文件不存在: {content}")
143
- with open(content, "rb") as f:
144
- file_bytes = f.read()
145
- if not mime_type:
146
- mime_type = get_file_mime_type(content)
147
- elif isinstance(content, bytes):
148
- file_bytes = content
149
- elif hasattr(content, 'read'):
150
- file_bytes = content.read()
151
- else:
152
- raise ValidationError("不支持的内容类型")
153
-
154
- # 构建请求
155
- request = file_service_pb2.UploadFileRequest(
156
- file_name=file_name,
157
- content=file_bytes,
158
- file_type=file_type,
159
- mime_type=mime_type or "application/octet-stream",
160
- is_temporary=is_temporary,
161
- expire_seconds=expire_seconds if is_temporary and expire_seconds else None,
162
- )
163
-
164
- if folder_id:
165
- request.folder_id = folder_id
166
-
167
- # 构建元数据
168
- grpc_metadata = self.client.build_metadata(**metadata)
169
-
170
- # 发送请求
171
- response = await stub.UploadFile(request, metadata=grpc_metadata)
172
-
173
- # 转换响应
174
- return FileUploadResponse(
175
- file=self._convert_file_info(response.file),
176
- upload_file=self._convert_upload_file_info(response.upload_file),
177
- )
178
-
179
- async def _upload_stream(
180
- self,
181
- file_name: str,
182
- content: Union[bytes, BinaryIO, Path],
183
- file_size: int,
184
- folder_id: Optional[str],
185
- file_type: str,
186
- mime_type: str,
187
- file_hash: str,
188
- is_temporary: Optional[bool] = False,
189
- expire_seconds: Optional[int] = None,
190
- **metadata
191
- ) -> FileUploadResponse:
192
- """流式上传"""
193
-
194
- # 获取上传URL,以及对应的文件和上传文件信息
195
- upload_url_resp = await self.generate_upload_url(
196
- file_name=file_name,
197
- file_size=file_size,
198
- folder_id=folder_id,
199
- file_type=file_type,
200
- mime_type=mime_type,
201
- file_hash=file_hash,
202
- is_temporary=is_temporary,
203
- expire_seconds=expire_seconds if is_temporary and expire_seconds else None,
204
- **metadata
205
- )
206
-
207
- # 上传文件到对象存储
208
- await self.http_uploader.upload(
209
- url=upload_url_resp.upload_url,
210
- content=content,
211
- headers={"Content-Type": mime_type},
212
- total_size=file_size,
213
- )
214
-
215
- # 确认上传完成
216
- await self._confirm_upload_completed(
217
- file_id=upload_url_resp.file.id,
218
- **metadata
219
- )
220
-
221
- # 返回文件信息
222
- return FileUploadResponse(
223
- file=upload_url_resp.file,
224
- upload_file=upload_url_resp.upload_file
225
- )
226
-
227
- async def _upload_resumable(
228
- self,
229
- file_name: str,
230
- content: Union[bytes, BinaryIO, Path],
231
- file_size: int,
232
- folder_id: Optional[str],
233
- file_type: str,
234
- mime_type: str,
235
- file_hash: str,
236
- is_temporary: Optional[bool] = False,
237
- expire_seconds: Optional[int] = None,
238
- **metadata
239
- ) -> FileUploadResponse:
240
- """断点续传实现"""
241
- # 获取断点续传URL,以及对应的文件和上传文件信息
242
- upload_url_resp = await self._generate_resumable_upload_url(
243
- file_name=file_name,
244
- file_size=file_size,
245
- folder_id=folder_id,
246
- file_type=file_type,
247
- mime_type=mime_type,
248
- file_hash=file_hash,
249
- is_temporary=is_temporary,
250
- expire_seconds=expire_seconds if is_temporary and expire_seconds else None,
251
- **metadata
252
- )
253
-
254
- # 开启断点续传
255
- upload_url = await self.http_uploader.start_resumable_session(
256
- url=upload_url_resp.upload_url,
257
- total_file_size=file_size,
258
- mine_type=mime_type,
259
- )
260
-
261
- # 上传文件到对象存储
262
- await self.http_uploader.upload(
263
- url=upload_url,
264
- content=content,
265
- headers={"Content-Type": mime_type},
266
- total_size=file_size,
267
- )
268
-
269
- # 确认上传完成
270
- await self._confirm_upload_completed(
271
- file_id=upload_url_resp.file.id,
272
- **metadata
273
- )
274
-
275
- # 获取文件信息
276
- return FileUploadResponse(
277
- file=upload_url_resp.file,
278
- upload_file=upload_url_resp.upload_file
279
- )
280
-
281
- async def generate_upload_url(
282
- self,
283
- file_name: str,
284
- file_size: int,
285
- folder_id: str = None,
286
- file_type: str = "file",
287
- mime_type: str = None,
288
- file_hash: str = None,
289
- is_temporary: Optional[bool] = False,
290
- expire_seconds: Optional[int] = None,
291
- **metadata
292
- ) -> UploadUrlResponse:
293
- """
294
- 生成上传URL(用于客户端直传)
295
-
296
- Args:
297
- file_name: 文件名
298
- file_size: 文件大小
299
- folder_id: 文件夹ID
300
- file_type: 文件类型
301
- mime_type: MIME类型
302
- file_hash: 文件哈希
303
- is_temporary: 是否为临时文件
304
- expire_seconds: 过期秒数
305
- **metadata: 额外的元数据(如 x-org-id, x-user-id 等)
306
-
307
- Returns:
308
- 上传URL响应
309
- """
310
- from ...rpc.gen import file_service_pb2, file_service_pb2_grpc
311
-
312
- stub = await self.client.get_stub(file_service_pb2_grpc.FileServiceStub)
313
-
314
- request = file_service_pb2.UploadUrlRequest(
315
- file_name=file_name,
316
- file_size=file_size,
317
- file_type=file_type,
318
- mime_type=mime_type or "application/octet-stream",
319
- file_hash=file_hash,
320
- is_temporary=is_temporary,
321
- expire_seconds=expire_seconds if is_temporary and expire_seconds else None,
322
- )
323
-
324
- if folder_id:
325
- request.folder_id = folder_id
326
-
327
- # 构建元数据
328
- grpc_metadata = self.client.build_metadata(**metadata)
329
-
330
- response = await stub.GenerateUploadUrl(request, metadata=grpc_metadata)
331
-
332
- return UploadUrlResponse(
333
- file=self._convert_file_info(response.file),
334
- upload_file=self._convert_upload_file_info(response.upload_file),
335
- upload_url=response.url
336
- )
337
-
338
- async def upload(
339
- self,
340
- file: Union[str, Path, BinaryIO, bytes],
341
- *,
342
- folder_id: Optional[str] = None,
343
- mode: Optional[UploadMode] = UploadMode.NORMAL,
344
- is_temporary: Optional[bool] = False,
345
- expire_seconds: Optional[int] = None,
346
- **metadata
347
- ) -> FileUploadResponse:
348
- """
349
- 统一的文件上传接口
350
-
351
- Args:
352
- file: 文件路径、Path对象、文件对象或字节数据
353
- folder_id: 目标文件夹ID(可选)
354
- mode: 上传模式(NORMAL/DIRECT/RESUMABLE/STREAM)
355
- is_temporary: 是否为临时文件
356
- expire_seconds: 过期秒数
357
- **metadata: 额外的元数据
358
-
359
- Returns:
360
- 文件信息
361
- """
362
- # 解析文件参数,提取文件信息
363
- file_name, content, file_size, mime_type, file_type, file_hash = self._extract_file_info(file)
364
-
365
- # 根据文件大小自动选择上传模式
366
- if mode == UploadMode.NORMAL:
367
- ten_mb = 1024 * 1024 * 10
368
- hundred_mb = 1024 * 1024 * 100
369
- if file_size >= ten_mb and file_size < hundred_mb: # 10MB
370
- mode = UploadMode.STREAM # 大文件自动使用流式上传模式
371
- elif file_size > hundred_mb:
372
- mode = UploadMode.RESUMABLE # 特大文件自动使用断点续传模式
373
-
374
- # 根据上传模式执行不同的上传逻辑
375
- if mode == UploadMode.NORMAL:
376
- # 普通上传(通过gRPC)
377
- return await self._upload_file(
378
- file_name=file_name,
379
- content=content,
380
- folder_id=folder_id,
381
- file_type=file_type,
382
- mime_type=mime_type,
383
- is_temporary=is_temporary,
384
- expire_seconds=expire_seconds,
385
- **metadata
386
- )
387
-
388
- elif mode == UploadMode.STREAM:
389
- # 流式上传(目前使用直传实现)
390
- return await self._upload_stream(
391
- file_name=file_name,
392
- content=content,
393
- file_size=file_size,
394
- folder_id=folder_id,
395
- file_type=file_type,
396
- mime_type=mime_type,
397
- file_hash=file_hash,
398
- is_temporary=is_temporary,
399
- expire_seconds=expire_seconds,
400
- **metadata
401
- )
402
-
403
- elif mode == UploadMode.RESUMABLE:
404
- # 断点续传
405
- return await self._upload_resumable(
406
- file_name=file_name,
407
- content=content,
408
- file_size=file_size,
409
- folder_id=folder_id,
410
- file_type=file_type,
411
- mime_type=mime_type,
412
- file_hash=file_hash,
413
- is_temporary=is_temporary,
414
- expire_seconds=expire_seconds,
415
- **metadata
416
- )
417
-
418
- else:
419
- raise ValidationError(f"不支持的上传模式: {mode}")
420
-
421
- async def generate_download_url(
422
- self,
423
- file_id: str,
424
- *,
425
- expire_seconds: Optional[int] = None,
426
- **metadata
427
- ) -> str:
428
- """
429
- 生成下载信息
430
-
431
- Args:
432
- file_id: 文件ID
433
- expire_seconds: 过期时间(秒)
434
- **metadata: 额外的元数据(如 x-org-id, x-user-id 等)
435
-
436
- Returns:
437
- 下载信息
438
- """
439
- from ...rpc.gen import file_service_pb2, file_service_pb2_grpc
440
-
441
- stub = await self.client.get_stub(file_service_pb2_grpc.FileServiceStub)
442
-
443
- request = file_service_pb2.DownloadUrlRequest(file_id=file_id,
444
- expire_seconds=expire_seconds if expire_seconds else None)
445
-
446
- # 构建元数据
447
- grpc_metadata = self.client.build_metadata(**metadata)
448
-
449
- download_url_resp = await stub.GenerateDownloadUrl(request, metadata=grpc_metadata)
450
-
451
- return download_url_resp.url
452
-
453
- async def download(
454
- self,
455
- file_id: str,
456
- *,
457
- save_path: Optional[Union[str, Path]] = None,
458
- chunk_size: Optional[int] = None,
459
- **metadata
460
- ) -> Union[bytes, Path, AsyncIterator[bytes]]:
461
- """
462
- 统一的文件下载接口
463
-
464
- Args:
465
- file_id: 文件ID
466
- save_path: 保存路径(如果为None,返回字节数据)
467
- chunk_size: 分片大小(用于流式下载)
468
- **metadata: 额外的元数据
469
-
470
- Returns:
471
- - NORMAL模式:下载的内容(字节)或保存的文件路径
472
- - STREAM模式:返回异步迭代器,逐块返回数据
473
- """
474
-
475
- # 获取下载URL
476
- download_url = await self.generate_download_url(file_id, **metadata)
477
-
478
- return await self.http_downloader.download(
479
- url=download_url,
480
- save_path=save_path,
481
- chunk_size=chunk_size,
482
- )
1
+ """
2
+ 异步二进制大对象服务
3
+ """
4
+ import asyncio
5
+ import hashlib
6
+
7
+ from pathlib import Path
8
+ from typing import Optional, Union, BinaryIO, AsyncIterator
9
+
10
+ from .base_file_service import BaseFileService
11
+ from ...enums import UploadMode
12
+ from ...errors import ValidationError
13
+ from ...rpc import AsyncGrpcClient
14
+ from ...schemas import FileUploadResponse, UploadUrlResponse
15
+ from ...utils import AsyncHttpUploader, AsyncHttpDownloader, retry_with_backoff, get_file_mime_type
16
+
17
+
18
+ class AsyncBlobService(BaseFileService):
19
+ """异步文件(二进制大对象)服务"""
20
+
21
+ def __init__(self, client: AsyncGrpcClient):
22
+ """
23
+ 初始化文件(二进制大对象)服务
24
+
25
+ Args:
26
+ client: 异步gRPC客户端
27
+ """
28
+ self.client = client
29
+ self.http_uploader = AsyncHttpUploader()
30
+ self.http_downloader = AsyncHttpDownloader()
31
+
32
+ async def _generate_resumable_upload_url(
33
+ self,
34
+ file_name: str,
35
+ file_size: int,
36
+ folder_id: Optional[str] = None,
37
+ file_type: str = "dat",
38
+ mime_type: Optional[str] = None,
39
+ file_hash: Optional[str] = None,
40
+ is_temporary: Optional[bool] = False,
41
+ expire_seconds: Optional[int] = None,
42
+ request_id: Optional[str] = None,
43
+ **metadata
44
+ ) -> UploadUrlResponse:
45
+ """
46
+ 生成断点续传URL
47
+
48
+ Args:
49
+ file_name: 文件名
50
+ file_size: 文件大小
51
+ folder_id: 文件夹ID
52
+ file_type: 文件类型
53
+ mime_type: MIME类型
54
+ file_hash: 文件哈希
55
+ is_temporary: 是否为临时文件
56
+ expire_seconds: 过期秒数
57
+ **metadata: 额外的元数据(如 x-org-id, x-user-id 等)
58
+
59
+ Returns:
60
+ 上传URL响应
61
+ """
62
+ from ...rpc.gen import file_service_pb2, file_service_pb2_grpc
63
+
64
+ stub = await self.client.get_stub(file_service_pb2_grpc.FileServiceStub)
65
+
66
+ request = file_service_pb2.UploadUrlRequest(
67
+ file_name=file_name,
68
+ file_size=file_size,
69
+ file_type=file_type,
70
+ mime_type=mime_type or "application/octet-stream",
71
+ file_hash=file_hash,
72
+ is_temporary=is_temporary,
73
+ expire_seconds=expire_seconds if is_temporary and expire_seconds else None,
74
+ )
75
+
76
+ if folder_id:
77
+ request.folder_id = folder_id
78
+
79
+ # 构建元数据
80
+ grpc_metadata = self.client.build_metadata(request_id=request_id, **metadata)
81
+
82
+ response = await stub.GenerateResumableUploadUrl(request, metadata=grpc_metadata)
83
+
84
+ return UploadUrlResponse(
85
+ file=self._convert_file_info(response.file),
86
+ upload_file=self._convert_upload_file_info(response.upload_file),
87
+ upload_url=response.url
88
+ )
89
+
90
+ async def _confirm_upload_completed(self, file_id: str, request_id: Optional[str] = None,
91
+ **metadata) -> None:
92
+ """
93
+ 确认上传完成
94
+
95
+ Args:
96
+ file_id: 文件ID
97
+ request_id: 请求ID(可选,如果不提供则自动生成)
98
+ **metadata: 额外的元数据(如 x-org-id, x-user-id 等)
99
+ """
100
+ from ...rpc.gen import file_service_pb2, file_service_pb2_grpc
101
+
102
+ stub = await self.client.get_stub(file_service_pb2_grpc.FileServiceStub)
103
+
104
+ request = file_service_pb2.UploadCompletedRequest(file_id=file_id)
105
+
106
+ # 构建元数据
107
+ grpc_metadata = self.client.build_metadata(request_id=request_id, **metadata)
108
+
109
+ await stub.ConfirmUploadCompleted(request, metadata=grpc_metadata)
110
+
111
+ @retry_with_backoff(max_retries=3)
112
+ async def _upload_file(
113
+ self,
114
+ file_name: str,
115
+ content: Union[bytes, BinaryIO, Path],
116
+ folder_id: Optional[str] = None,
117
+ file_type: str = "dat",
118
+ mime_type: Optional[str] = None,
119
+ is_temporary: Optional[bool] = False,
120
+ expire_seconds: Optional[int] = None,
121
+ request_id: Optional[str] = None,
122
+ **metadata
123
+ ) -> FileUploadResponse:
124
+ """
125
+ 普通上传
126
+
127
+ Args:
128
+ file_name: 文件名
129
+ content: 文件内容(字节、文件对象或路径)
130
+ folder_id: 文件夹ID
131
+ file_type: 文件类型
132
+ mime_type: MIME类型
133
+ is_temporary: 是否为临时文件
134
+ expire_seconds: 过期秒数
135
+ request_id: 请求ID(可选,如果不提供则自动生成)
136
+ **metadata: 额外的元数据(如 x-org-id, x-user-id 等)
137
+
138
+ Returns:
139
+ 文件信息
140
+ """
141
+ from ...rpc.gen import file_service_pb2, file_service_pb2_grpc
142
+
143
+ stub = await self.client.get_stub(file_service_pb2_grpc.FileServiceStub)
144
+
145
+ # 处理不同类型的内容
146
+ if isinstance(content, Path):
147
+ if not content.exists():
148
+ raise ValidationError(f"文件不存在: {content}")
149
+ with open(content, "rb") as f:
150
+ file_bytes = f.read()
151
+ if not mime_type:
152
+ mime_type = get_file_mime_type(content)
153
+ elif isinstance(content, bytes):
154
+ file_bytes = content
155
+ elif hasattr(content, 'read'):
156
+ file_bytes = content.read()
157
+ else:
158
+ raise ValidationError("不支持的内容类型")
159
+
160
+ # 构建请求
161
+ request = file_service_pb2.UploadFileRequest(
162
+ file_name=file_name,
163
+ content=file_bytes,
164
+ file_type=file_type,
165
+ mime_type=mime_type or "application/octet-stream",
166
+ is_temporary=is_temporary,
167
+ expire_seconds=expire_seconds if is_temporary and expire_seconds else None,
168
+ )
169
+
170
+ if folder_id:
171
+ request.folder_id = folder_id
172
+
173
+ # 构建元数据
174
+ grpc_metadata = self.client.build_metadata(request_id=request_id, **metadata)
175
+
176
+ # 发送请求
177
+ response = await stub.UploadFile(request, metadata=grpc_metadata)
178
+
179
+ # 转换响应
180
+ return FileUploadResponse(
181
+ file=self._convert_file_info(response.file),
182
+ upload_file=self._convert_upload_file_info(response.upload_file),
183
+ )
184
+
185
+ async def _upload_stream(
186
+ self,
187
+ file_name: str,
188
+ content: Union[bytes, BinaryIO, Path],
189
+ file_size: int,
190
+ folder_id: Optional[str],
191
+ file_type: str,
192
+ mime_type: str,
193
+ file_hash: str,
194
+ is_temporary: Optional[bool] = False,
195
+ expire_seconds: Optional[int] = None,
196
+ request_id: Optional[str] = None,
197
+ **metadata
198
+ ) -> FileUploadResponse:
199
+ """流式上传"""
200
+
201
+ # 获取上传URL,以及对应的文件和上传文件信息
202
+ upload_url_resp = await self.generate_upload_url(
203
+ file_name=file_name,
204
+ file_size=file_size,
205
+ folder_id=folder_id,
206
+ file_type=file_type,
207
+ mime_type=mime_type,
208
+ file_hash=file_hash,
209
+ is_temporary=is_temporary,
210
+ expire_seconds=expire_seconds if is_temporary and expire_seconds else None,
211
+ request_id=request_id,
212
+ **metadata
213
+ )
214
+
215
+ # 如果URL为空,说明文件已存在(哈希重复),直接返回
216
+ if not upload_url_resp.upload_url:
217
+ return FileUploadResponse(
218
+ file=upload_url_resp.file,
219
+ upload_file=upload_url_resp.upload_file
220
+ )
221
+
222
+ # 上传文件到对象存储
223
+ await self.http_uploader.upload(
224
+ url=upload_url_resp.upload_url,
225
+ content=content,
226
+ headers={"Content-Type": mime_type},
227
+ total_size=file_size,
228
+ )
229
+
230
+ # 确认上传完成
231
+ await self._confirm_upload_completed(
232
+ file_id=upload_url_resp.file.id,
233
+ request_id=request_id,
234
+ **metadata
235
+ )
236
+
237
+ # 返回文件信息
238
+ return FileUploadResponse(
239
+ file=upload_url_resp.file,
240
+ upload_file=upload_url_resp.upload_file
241
+ )
242
+
243
+ async def _upload_resumable(
244
+ self,
245
+ file_name: str,
246
+ content: Union[bytes, BinaryIO, Path],
247
+ file_size: int,
248
+ folder_id: Optional[str],
249
+ file_type: str,
250
+ mime_type: str,
251
+ file_hash: str,
252
+ is_temporary: Optional[bool] = False,
253
+ expire_seconds: Optional[int] = None,
254
+ request_id: Optional[str] = None,
255
+ **metadata
256
+ ) -> FileUploadResponse:
257
+ """断点续传实现"""
258
+ # 获取断点续传URL,以及对应的文件和上传文件信息
259
+ upload_url_resp = await self._generate_resumable_upload_url(
260
+ file_name=file_name,
261
+ file_size=file_size,
262
+ folder_id=folder_id,
263
+ file_type=file_type,
264
+ mime_type=mime_type,
265
+ file_hash=file_hash,
266
+ is_temporary=is_temporary,
267
+ expire_seconds=expire_seconds if is_temporary and expire_seconds else None,
268
+ request_id=request_id,
269
+ **metadata
270
+ )
271
+
272
+ # 如果URL为空,说明文件已存在(哈希重复),直接返回
273
+ if not upload_url_resp.upload_url:
274
+ return FileUploadResponse(
275
+ file=upload_url_resp.file,
276
+ upload_file=upload_url_resp.upload_file
277
+ )
278
+
279
+ # 开启断点续传
280
+ upload_url = await self.http_uploader.start_resumable_session(
281
+ url=upload_url_resp.upload_url,
282
+ total_file_size=file_size,
283
+ mine_type=mime_type,
284
+ )
285
+
286
+ # 上传文件到对象存储
287
+ await self.http_uploader.upload(
288
+ url=upload_url,
289
+ content=content,
290
+ headers={"Content-Type": mime_type},
291
+ total_size=file_size,
292
+ )
293
+
294
+ # 确认上传完成
295
+ await self._confirm_upload_completed(
296
+ file_id=upload_url_resp.file.id,
297
+ request_id=request_id,
298
+ **metadata
299
+ )
300
+
301
+ # 获取文件信息
302
+ return FileUploadResponse(
303
+ file=upload_url_resp.file,
304
+ upload_file=upload_url_resp.upload_file
305
+ )
306
+
307
+ async def generate_upload_url(
308
+ self,
309
+ file_name: str,
310
+ file_size: int,
311
+ folder_id: str = None,
312
+ file_type: str = "dat",
313
+ mime_type: str = None,
314
+ file_hash: str = None,
315
+ is_temporary: Optional[bool] = False,
316
+ expire_seconds: Optional[int] = None,
317
+ request_id: Optional[str] = None,
318
+ **metadata
319
+ ) -> UploadUrlResponse:
320
+ """
321
+ 生成上传URL(用于客户端直传)
322
+
323
+ Args:
324
+ file_name: 文件名
325
+ file_size: 文件大小
326
+ folder_id: 文件夹ID
327
+ file_type: 文件类型
328
+ mime_type: MIME类型
329
+ file_hash: 文件哈希
330
+ is_temporary: 是否为临时文件
331
+ expire_seconds: 过期秒数
332
+ request_id: 请求ID(可选,如果不提供则自动生成)
333
+ **metadata: 额外的元数据(如 x-org-id, x-user-id 等)
334
+
335
+ Returns:
336
+ 上传URL响应
337
+ 注意:如果文件哈希已存在(重复上传),upload_url 会为空,
338
+ 此时应直接使用返回的 file 和 upload_file 信息,无需再次上传
339
+ """
340
+ from ...rpc.gen import file_service_pb2, file_service_pb2_grpc
341
+
342
+ stub = await self.client.get_stub(file_service_pb2_grpc.FileServiceStub)
343
+
344
+ request = file_service_pb2.UploadUrlRequest(
345
+ file_name=file_name,
346
+ file_size=file_size,
347
+ file_type=file_type,
348
+ mime_type=mime_type or "application/octet-stream",
349
+ file_hash=file_hash,
350
+ is_temporary=is_temporary,
351
+ expire_seconds=expire_seconds if is_temporary and expire_seconds else None,
352
+ )
353
+
354
+ if folder_id:
355
+ request.folder_id = folder_id
356
+
357
+ # 构建元数据
358
+ grpc_metadata = self.client.build_metadata(request_id=request_id, **metadata)
359
+
360
+ response = await stub.GenerateUploadUrl(request, metadata=grpc_metadata)
361
+
362
+ return UploadUrlResponse(
363
+ file=self._convert_file_info(response.file),
364
+ upload_file=self._convert_upload_file_info(response.upload_file),
365
+ upload_url=response.url
366
+ )
367
+
368
+ async def upload(
369
+ self,
370
+ file: Optional[Union[str, Path, BinaryIO, bytes]] = None,
371
+ *,
372
+ folder_id: Optional[str] = None,
373
+ mode: Optional[UploadMode] = UploadMode.NORMAL,
374
+ is_temporary: Optional[bool] = False,
375
+ expire_seconds: Optional[int] = None,
376
+ url: Optional[str] = None,
377
+ file_name: Optional[str] = None,
378
+ request_id: Optional[str] = None,
379
+ **metadata
380
+ ) -> FileUploadResponse:
381
+ """
382
+ 统一的文件上传接口
383
+
384
+ Args:
385
+ file: 文件路径、Path对象、文件对象或字节数据(当使用url参数时可省略)
386
+ folder_id: 目标文件夹ID(可选)
387
+ mode: 上传模式(NORMAL/DIRECT/RESUMABLE/STREAM)
388
+ is_temporary: 是否为临时文件
389
+ expire_seconds: 过期秒数
390
+ url: 要下载并上传的URL(可选)
391
+ file_name: 当使用url参数时指定的文件名(可选)
392
+ request_id: 请求ID(可选,如果不提供则自动生成)
393
+ **metadata: 额外的元数据
394
+
395
+ Returns:
396
+ 文件信息
397
+
398
+ Note:
399
+ 必须提供 file 或 url 参数之一
400
+ """
401
+ # 参数验证:必须提供 file 或 url 之一
402
+ if file is None and not url:
403
+ raise ValidationError("必须提供 file 或 url 参数之一")
404
+
405
+ # 如果提供了URL,先下载文件
406
+ if url:
407
+ # 下载文件到内存
408
+ downloaded_content = await self.http_downloader.download(url)
409
+
410
+ # 如果没有指定文件名,从URL中提取
411
+ if not file_name:
412
+ from urllib.parse import urlparse
413
+ from pathlib import Path as PathLib
414
+ parsed_url = urlparse(url)
415
+ url_path = PathLib(parsed_url.path)
416
+ file_name = url_path.name if url_path.name else f"download_{hashlib.md5(url.encode()).hexdigest()[:8]}"
417
+
418
+ # 使用下载的内容作为file参数
419
+ file = downloaded_content
420
+
421
+ # 提取文件信息(bytes会返回默认的MIME类型,我们稍后会基于文件名重新计算)
422
+ extracted_file_name, content, file_size, _, _, file_hash = self._extract_file_info(file)
423
+
424
+ # file_name已经在上面设置了(要么是用户指定的,要么是从URL提取的)
425
+ extracted_file_name = file_name
426
+
427
+ # 基于文件名计算文件类型和MIME类型
428
+ file_type = Path(extracted_file_name).suffix.lstrip('.').lower() if Path(extracted_file_name).suffix else 'dat'
429
+ mime_type = get_file_mime_type(Path(extracted_file_name))
430
+ else:
431
+ # 解析文件参数,提取文件信息
432
+ extracted_file_name, content, file_size, mime_type, file_type, file_hash = self._extract_file_info(file)
433
+
434
+ # 根据文件大小自动选择上传模式
435
+ if mode == UploadMode.NORMAL:
436
+ ten_mb = 1024 * 1024 * 10
437
+ hundred_mb = 1024 * 1024 * 100
438
+ if file_size >= ten_mb and file_size < hundred_mb: # 10MB
439
+ mode = UploadMode.STREAM # 大文件自动使用流式上传模式
440
+ elif file_size > hundred_mb:
441
+ mode = UploadMode.RESUMABLE # 特大文件自动使用断点续传模式
442
+
443
+ # 根据上传模式执行不同的上传逻辑
444
+ if mode == UploadMode.NORMAL:
445
+ # 普通上传(通过gRPC)
446
+ return await self._upload_file(
447
+ file_name=extracted_file_name,
448
+ content=content,
449
+ folder_id=folder_id,
450
+ file_type=file_type,
451
+ mime_type=mime_type,
452
+ is_temporary=is_temporary,
453
+ expire_seconds=expire_seconds,
454
+ request_id=request_id,
455
+ **metadata
456
+ )
457
+
458
+ elif mode == UploadMode.STREAM:
459
+ # 流式上传(目前使用直传实现)
460
+ return await self._upload_stream(
461
+ file_name=extracted_file_name,
462
+ content=content,
463
+ file_size=file_size,
464
+ folder_id=folder_id,
465
+ file_type=file_type,
466
+ mime_type=mime_type,
467
+ file_hash=file_hash,
468
+ is_temporary=is_temporary,
469
+ expire_seconds=expire_seconds,
470
+ request_id=request_id,
471
+ **metadata
472
+ )
473
+
474
+ elif mode == UploadMode.RESUMABLE:
475
+ # 断点续传
476
+ return await self._upload_resumable(
477
+ file_name=extracted_file_name,
478
+ content=content,
479
+ file_size=file_size,
480
+ folder_id=folder_id,
481
+ file_type=file_type,
482
+ mime_type=mime_type,
483
+ file_hash=file_hash,
484
+ is_temporary=is_temporary,
485
+ expire_seconds=expire_seconds,
486
+ request_id=request_id,
487
+ **metadata
488
+ )
489
+
490
+ else:
491
+ raise ValidationError(f"不支持的上传模式: {mode}")
492
+
493
+ async def generate_download_url(
494
+ self,
495
+ file_id: str,
496
+ *,
497
+ expire_seconds: Optional[int] = None,
498
+ request_id: Optional[str] = None,
499
+ **metadata
500
+ ) -> str:
501
+ """
502
+ 生成下载信息
503
+
504
+ Args:
505
+ file_id: 文件ID
506
+ expire_seconds: 过期时间(秒)
507
+ request_id: 请求ID(可选,如果不提供则自动生成)
508
+ **metadata: 额外的元数据(如 x-org-id, x-user-id 等)
509
+
510
+ Returns:
511
+ 下载信息
512
+ """
513
+ from ...rpc.gen import file_service_pb2, file_service_pb2_grpc
514
+
515
+ stub = await self.client.get_stub(file_service_pb2_grpc.FileServiceStub)
516
+
517
+ request = file_service_pb2.DownloadUrlRequest(file_id=file_id,
518
+ expire_seconds=expire_seconds if expire_seconds else None)
519
+
520
+ # 构建元数据
521
+ grpc_metadata = self.client.build_metadata(request_id=request_id, **metadata)
522
+
523
+ download_url_resp = await stub.GenerateDownloadUrl(request, metadata=grpc_metadata)
524
+
525
+ return download_url_resp.url
526
+
527
+ async def download(
528
+ self,
529
+ file_id: str,
530
+ *,
531
+ save_path: Optional[Union[str, Path]] = None,
532
+ chunk_size: Optional[int] = None,
533
+ request_id: Optional[str] = None,
534
+ **metadata
535
+ ) -> Union[bytes, Path, AsyncIterator[bytes]]:
536
+ """
537
+ 统一的文件下载接口
538
+
539
+ Args:
540
+ file_id: 文件ID
541
+ save_path: 保存路径(如果为None,返回字节数据)
542
+ chunk_size: 分片大小(用于流式下载)
543
+ request_id: 请求ID(可选,如果不提供则自动生成)
544
+ **metadata: 额外的元数据
545
+
546
+ Returns:
547
+ - NORMAL模式:下载的内容(字节)或保存的文件路径
548
+ - STREAM模式:返回异步迭代器,逐块返回数据
549
+ """
550
+
551
+ # 获取下载URL
552
+ download_url = await self.generate_download_url(file_id, request_id=request_id, **metadata)
553
+
554
+ return await self.http_downloader.download(
555
+ url=download_url,
556
+ save_path=save_path,
557
+ chunk_size=chunk_size,
558
+ )