tamar-file-hub-client 0.0.1__py3-none-any.whl → 0.0.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- file_hub_client/__init__.py +39 -0
- file_hub_client/client.py +43 -6
- file_hub_client/rpc/async_client.py +91 -11
- file_hub_client/rpc/gen/taple_service_pb2.py +225 -0
- file_hub_client/rpc/gen/taple_service_pb2_grpc.py +1626 -0
- file_hub_client/rpc/generate_grpc.py +2 -2
- file_hub_client/rpc/interceptors.py +550 -0
- file_hub_client/rpc/protos/taple_service.proto +874 -0
- file_hub_client/rpc/sync_client.py +91 -9
- file_hub_client/schemas/__init__.py +60 -0
- file_hub_client/schemas/taple.py +413 -0
- file_hub_client/services/__init__.py +5 -0
- file_hub_client/services/file/async_blob_service.py +558 -482
- file_hub_client/services/file/async_file_service.py +18 -9
- file_hub_client/services/file/base_file_service.py +19 -6
- file_hub_client/services/file/sync_blob_service.py +554 -478
- file_hub_client/services/file/sync_file_service.py +18 -9
- file_hub_client/services/folder/async_folder_service.py +20 -11
- file_hub_client/services/folder/sync_folder_service.py +20 -11
- file_hub_client/services/taple/__init__.py +10 -0
- file_hub_client/services/taple/async_taple_service.py +2281 -0
- file_hub_client/services/taple/base_taple_service.py +353 -0
- file_hub_client/services/taple/idempotent_taple_mixin.py +142 -0
- file_hub_client/services/taple/sync_taple_service.py +2256 -0
- file_hub_client/utils/__init__.py +43 -1
- file_hub_client/utils/file_utils.py +59 -11
- file_hub_client/utils/idempotency.py +196 -0
- file_hub_client/utils/logging.py +315 -0
- file_hub_client/utils/retry.py +241 -2
- file_hub_client/utils/smart_retry.py +403 -0
- tamar_file_hub_client-0.0.2.dist-info/METADATA +2050 -0
- tamar_file_hub_client-0.0.2.dist-info/RECORD +57 -0
- tamar_file_hub_client-0.0.1.dist-info/METADATA +0 -874
- tamar_file_hub_client-0.0.1.dist-info/RECORD +0 -44
- {tamar_file_hub_client-0.0.1.dist-info → tamar_file_hub_client-0.0.2.dist-info}/WHEEL +0 -0
- {tamar_file_hub_client-0.0.1.dist-info → tamar_file_hub_client-0.0.2.dist-info}/top_level.txt +0 -0
@@ -1,478 +1,554 @@
|
|
1
|
-
"""
|
2
|
-
同步二进制大对象服务
|
3
|
-
"""
|
4
|
-
|
5
|
-
from
|
6
|
-
|
7
|
-
|
8
|
-
from
|
9
|
-
from ...
|
10
|
-
from ...
|
11
|
-
from ...
|
12
|
-
from ...
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
self.
|
27
|
-
self.
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
|
52
|
-
|
53
|
-
|
54
|
-
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
-
|
60
|
-
|
61
|
-
|
62
|
-
|
63
|
-
|
64
|
-
|
65
|
-
|
66
|
-
|
67
|
-
|
68
|
-
|
69
|
-
|
70
|
-
|
71
|
-
|
72
|
-
|
73
|
-
|
74
|
-
|
75
|
-
|
76
|
-
|
77
|
-
|
78
|
-
|
79
|
-
|
80
|
-
|
81
|
-
|
82
|
-
|
83
|
-
|
84
|
-
|
85
|
-
|
86
|
-
|
87
|
-
|
88
|
-
|
89
|
-
|
90
|
-
|
91
|
-
|
92
|
-
|
93
|
-
|
94
|
-
|
95
|
-
|
96
|
-
|
97
|
-
|
98
|
-
|
99
|
-
|
100
|
-
|
101
|
-
|
102
|
-
|
103
|
-
|
104
|
-
|
105
|
-
|
106
|
-
|
107
|
-
|
108
|
-
|
109
|
-
|
110
|
-
|
111
|
-
|
112
|
-
|
113
|
-
|
114
|
-
|
115
|
-
|
116
|
-
|
117
|
-
|
118
|
-
|
119
|
-
|
120
|
-
|
121
|
-
|
122
|
-
|
123
|
-
|
124
|
-
|
125
|
-
|
126
|
-
|
127
|
-
|
128
|
-
|
129
|
-
|
130
|
-
|
131
|
-
|
132
|
-
|
133
|
-
|
134
|
-
|
135
|
-
|
136
|
-
|
137
|
-
|
138
|
-
|
139
|
-
|
140
|
-
|
141
|
-
|
142
|
-
|
143
|
-
|
144
|
-
|
145
|
-
|
146
|
-
|
147
|
-
|
148
|
-
|
149
|
-
|
150
|
-
|
151
|
-
|
152
|
-
|
153
|
-
|
154
|
-
|
155
|
-
|
156
|
-
|
157
|
-
|
158
|
-
|
159
|
-
|
160
|
-
|
161
|
-
|
162
|
-
|
163
|
-
|
164
|
-
|
165
|
-
|
166
|
-
|
167
|
-
|
168
|
-
|
169
|
-
|
170
|
-
|
171
|
-
#
|
172
|
-
|
173
|
-
|
174
|
-
|
175
|
-
)
|
176
|
-
|
177
|
-
|
178
|
-
|
179
|
-
|
180
|
-
|
181
|
-
|
182
|
-
|
183
|
-
|
184
|
-
|
185
|
-
|
186
|
-
|
187
|
-
|
188
|
-
|
189
|
-
|
190
|
-
|
191
|
-
|
192
|
-
|
193
|
-
|
194
|
-
|
195
|
-
|
196
|
-
|
197
|
-
|
198
|
-
|
199
|
-
|
200
|
-
|
201
|
-
|
202
|
-
|
203
|
-
|
204
|
-
|
205
|
-
|
206
|
-
|
207
|
-
|
208
|
-
|
209
|
-
|
210
|
-
)
|
211
|
-
|
212
|
-
#
|
213
|
-
|
214
|
-
|
215
|
-
|
216
|
-
|
217
|
-
|
218
|
-
|
219
|
-
|
220
|
-
|
221
|
-
|
222
|
-
|
223
|
-
|
224
|
-
|
225
|
-
|
226
|
-
|
227
|
-
|
228
|
-
|
229
|
-
|
230
|
-
|
231
|
-
|
232
|
-
|
233
|
-
|
234
|
-
|
235
|
-
|
236
|
-
|
237
|
-
|
238
|
-
|
239
|
-
|
240
|
-
|
241
|
-
|
242
|
-
|
243
|
-
|
244
|
-
|
245
|
-
|
246
|
-
|
247
|
-
|
248
|
-
|
249
|
-
|
250
|
-
|
251
|
-
|
252
|
-
|
253
|
-
|
254
|
-
|
255
|
-
|
256
|
-
|
257
|
-
|
258
|
-
|
259
|
-
|
260
|
-
|
261
|
-
|
262
|
-
|
263
|
-
|
264
|
-
|
265
|
-
|
266
|
-
|
267
|
-
|
268
|
-
|
269
|
-
|
270
|
-
|
271
|
-
|
272
|
-
|
273
|
-
|
274
|
-
|
275
|
-
|
276
|
-
|
277
|
-
|
278
|
-
|
279
|
-
|
280
|
-
|
281
|
-
|
282
|
-
|
283
|
-
|
284
|
-
|
285
|
-
|
286
|
-
|
287
|
-
|
288
|
-
|
289
|
-
|
290
|
-
|
291
|
-
|
292
|
-
|
293
|
-
|
294
|
-
|
295
|
-
|
296
|
-
|
297
|
-
|
298
|
-
|
299
|
-
|
300
|
-
|
301
|
-
|
302
|
-
|
303
|
-
|
304
|
-
|
305
|
-
|
306
|
-
|
307
|
-
|
308
|
-
|
309
|
-
|
310
|
-
|
311
|
-
|
312
|
-
|
313
|
-
|
314
|
-
|
315
|
-
|
316
|
-
|
317
|
-
|
318
|
-
|
319
|
-
|
320
|
-
|
321
|
-
|
322
|
-
|
323
|
-
|
324
|
-
|
325
|
-
|
326
|
-
|
327
|
-
|
328
|
-
|
329
|
-
|
330
|
-
|
331
|
-
|
332
|
-
|
333
|
-
|
334
|
-
|
335
|
-
|
336
|
-
|
337
|
-
|
338
|
-
|
339
|
-
|
340
|
-
|
341
|
-
|
342
|
-
|
343
|
-
|
344
|
-
|
345
|
-
|
346
|
-
|
347
|
-
|
348
|
-
|
349
|
-
|
350
|
-
|
351
|
-
|
352
|
-
|
353
|
-
|
354
|
-
|
355
|
-
|
356
|
-
|
357
|
-
|
358
|
-
|
359
|
-
|
360
|
-
|
361
|
-
|
362
|
-
|
363
|
-
|
364
|
-
|
365
|
-
|
366
|
-
|
367
|
-
|
368
|
-
|
369
|
-
|
370
|
-
|
371
|
-
|
372
|
-
|
373
|
-
|
374
|
-
|
375
|
-
|
376
|
-
|
377
|
-
|
378
|
-
|
379
|
-
|
380
|
-
|
381
|
-
|
382
|
-
|
383
|
-
|
384
|
-
|
385
|
-
|
386
|
-
|
387
|
-
|
388
|
-
|
389
|
-
|
390
|
-
|
391
|
-
|
392
|
-
|
393
|
-
|
394
|
-
|
395
|
-
|
396
|
-
|
397
|
-
|
398
|
-
|
399
|
-
|
400
|
-
|
401
|
-
|
402
|
-
|
403
|
-
|
404
|
-
|
405
|
-
|
406
|
-
|
407
|
-
|
408
|
-
|
409
|
-
|
410
|
-
|
411
|
-
|
412
|
-
|
413
|
-
|
414
|
-
|
415
|
-
|
416
|
-
|
417
|
-
|
418
|
-
|
419
|
-
|
420
|
-
self
|
421
|
-
|
422
|
-
|
423
|
-
|
424
|
-
|
425
|
-
|
426
|
-
|
427
|
-
|
428
|
-
|
429
|
-
|
430
|
-
|
431
|
-
|
432
|
-
|
433
|
-
|
434
|
-
|
435
|
-
|
436
|
-
|
437
|
-
|
438
|
-
|
439
|
-
|
440
|
-
|
441
|
-
|
442
|
-
|
443
|
-
|
444
|
-
|
445
|
-
|
446
|
-
|
447
|
-
|
448
|
-
|
449
|
-
|
450
|
-
|
451
|
-
|
452
|
-
|
453
|
-
|
454
|
-
|
455
|
-
|
456
|
-
|
457
|
-
|
458
|
-
|
459
|
-
|
460
|
-
|
461
|
-
|
462
|
-
|
463
|
-
|
464
|
-
|
465
|
-
|
466
|
-
|
467
|
-
|
468
|
-
|
469
|
-
|
470
|
-
|
471
|
-
|
472
|
-
|
473
|
-
|
474
|
-
|
475
|
-
|
476
|
-
|
477
|
-
|
478
|
-
|
1
|
+
"""
|
2
|
+
同步二进制大对象服务
|
3
|
+
"""
|
4
|
+
import hashlib
|
5
|
+
from pathlib import Path
|
6
|
+
from typing import Optional, Union, BinaryIO, Iterator
|
7
|
+
|
8
|
+
from .base_file_service import BaseFileService
|
9
|
+
from ...enums import UploadMode
|
10
|
+
from ...errors import ValidationError
|
11
|
+
from ...rpc import SyncGrpcClient
|
12
|
+
from ...schemas import FileUploadResponse, UploadUrlResponse
|
13
|
+
from ...utils import HttpUploader, HttpDownloader, retry_with_backoff, get_file_mime_type
|
14
|
+
|
15
|
+
|
16
|
+
class SyncBlobService(BaseFileService):
|
17
|
+
"""同步文件(二进制大对象)服务"""
|
18
|
+
|
19
|
+
def __init__(self, client: SyncGrpcClient):
|
20
|
+
"""
|
21
|
+
初始化文件(二进制大对象)服务
|
22
|
+
|
23
|
+
Args:
|
24
|
+
client: 同步gRPC客户端
|
25
|
+
"""
|
26
|
+
self.client = client
|
27
|
+
self.http_uploader = HttpUploader()
|
28
|
+
self.http_downloader = HttpDownloader()
|
29
|
+
|
30
|
+
def _generate_resumable_upload_url(
|
31
|
+
self,
|
32
|
+
file_name: str,
|
33
|
+
file_size: int,
|
34
|
+
folder_id: Optional[str] = None,
|
35
|
+
file_type: str = "dat",
|
36
|
+
mime_type: str = None,
|
37
|
+
file_hash: str = None,
|
38
|
+
is_temporary: Optional[bool] = False,
|
39
|
+
expire_seconds: Optional[int] = None,
|
40
|
+
request_id: Optional[str] = None,
|
41
|
+
**metadata
|
42
|
+
) -> UploadUrlResponse:
|
43
|
+
"""
|
44
|
+
生成断点续传URL
|
45
|
+
|
46
|
+
Args:
|
47
|
+
file_name: 文件名
|
48
|
+
file_size: 文件大小
|
49
|
+
folder_id: 文件夹ID
|
50
|
+
file_type: 文件类型
|
51
|
+
mime_type: MIME类型
|
52
|
+
file_hash: 文件哈希
|
53
|
+
is_temporary: 是否为临时文件
|
54
|
+
expire_seconds: 过期秒数
|
55
|
+
**metadata: 额外的元数据(如 x-org-id, x-user-id 等)
|
56
|
+
|
57
|
+
Returns:
|
58
|
+
上传URL响应
|
59
|
+
"""
|
60
|
+
from ...rpc.gen import file_service_pb2, file_service_pb2_grpc
|
61
|
+
|
62
|
+
stub = self.client.get_stub(file_service_pb2_grpc.FileServiceStub)
|
63
|
+
|
64
|
+
request = file_service_pb2.UploadUrlRequest(
|
65
|
+
file_name=file_name,
|
66
|
+
file_size=file_size,
|
67
|
+
file_type=file_type,
|
68
|
+
mime_type=mime_type or "application/octet-stream",
|
69
|
+
file_hash=file_hash,
|
70
|
+
is_temporary=is_temporary,
|
71
|
+
expire_seconds=expire_seconds if is_temporary and expire_seconds else None,
|
72
|
+
)
|
73
|
+
|
74
|
+
if folder_id:
|
75
|
+
request.folder_id = folder_id
|
76
|
+
|
77
|
+
# 构建元数据
|
78
|
+
grpc_metadata = self.client.build_metadata(request_id=request_id, **metadata)
|
79
|
+
|
80
|
+
response = stub.GenerateResumableUploadUrl(request, metadata=grpc_metadata)
|
81
|
+
|
82
|
+
return UploadUrlResponse(
|
83
|
+
file=self._convert_file_info(response.file),
|
84
|
+
upload_file=self._convert_upload_file_info(response.upload_file),
|
85
|
+
upload_url=response.url
|
86
|
+
)
|
87
|
+
|
88
|
+
def _confirm_upload_completed(self, file_id: str, request_id: Optional[str] = None,
|
89
|
+
**metadata) -> None:
|
90
|
+
"""
|
91
|
+
确认上传完成
|
92
|
+
|
93
|
+
Args:
|
94
|
+
file_id: 文件ID
|
95
|
+
request_id: 请求ID(可选,如果不提供则自动生成)
|
96
|
+
**metadata: 额外的元数据(如 x-org-id, x-user-id 等)
|
97
|
+
"""
|
98
|
+
from ...rpc.gen import file_service_pb2, file_service_pb2_grpc
|
99
|
+
|
100
|
+
stub = self.client.get_stub(file_service_pb2_grpc.FileServiceStub)
|
101
|
+
|
102
|
+
request = file_service_pb2.UploadCompletedRequest(file_id=file_id)
|
103
|
+
|
104
|
+
# 构建元数据
|
105
|
+
grpc_metadata = self.client.build_metadata(request_id=request_id, **metadata)
|
106
|
+
|
107
|
+
stub.ConfirmUploadCompleted(request, metadata=grpc_metadata)
|
108
|
+
|
109
|
+
@retry_with_backoff(max_retries=3)
|
110
|
+
def _upload_file(
|
111
|
+
self,
|
112
|
+
file_name: str,
|
113
|
+
content: Union[bytes, BinaryIO, Path],
|
114
|
+
folder_id: Optional[str] = None,
|
115
|
+
file_type: str = "dat",
|
116
|
+
mime_type: Optional[str] = None,
|
117
|
+
is_temporary: Optional[bool] = False,
|
118
|
+
expire_seconds: Optional[int] = None,
|
119
|
+
request_id: Optional[str] = None,
|
120
|
+
**metadata
|
121
|
+
) -> FileUploadResponse:
|
122
|
+
"""
|
123
|
+
直接上传文件
|
124
|
+
|
125
|
+
Args:
|
126
|
+
file_name: 文件名
|
127
|
+
content: 文件内容(字节、文件对象或路径)
|
128
|
+
folder_id: 文件夹ID
|
129
|
+
file_type: 文件类型
|
130
|
+
mime_type: MIME类型
|
131
|
+
is_temporary: 是否为临时文件
|
132
|
+
expire_seconds: 过期秒数
|
133
|
+
request_id: 请求ID(可选,如果不提供则自动生成)
|
134
|
+
**metadata: 额外的元数据(如 x-org-id, x-user-id 等)
|
135
|
+
|
136
|
+
Returns:
|
137
|
+
文件信息
|
138
|
+
"""
|
139
|
+
from ...rpc.gen import file_service_pb2, file_service_pb2_grpc
|
140
|
+
|
141
|
+
stub = self.client.get_stub(file_service_pb2_grpc.FileServiceStub)
|
142
|
+
|
143
|
+
# 处理不同类型的内容
|
144
|
+
if isinstance(content, Path):
|
145
|
+
if not content.exists():
|
146
|
+
raise ValidationError(f"文件不存在: {content}")
|
147
|
+
with open(content, "rb") as f:
|
148
|
+
file_bytes = f.read()
|
149
|
+
if not mime_type:
|
150
|
+
mime_type = get_file_mime_type(content)
|
151
|
+
elif isinstance(content, bytes):
|
152
|
+
file_bytes = content
|
153
|
+
elif hasattr(content, 'read'):
|
154
|
+
file_bytes = content.read()
|
155
|
+
else:
|
156
|
+
raise ValidationError("不支持的内容类型")
|
157
|
+
|
158
|
+
# 构建请求
|
159
|
+
request = file_service_pb2.UploadFileRequest(
|
160
|
+
file_name=file_name,
|
161
|
+
content=file_bytes,
|
162
|
+
file_type=file_type,
|
163
|
+
mime_type=mime_type or "application/octet-stream",
|
164
|
+
is_temporary=is_temporary,
|
165
|
+
expire_seconds=expire_seconds,
|
166
|
+
)
|
167
|
+
|
168
|
+
if folder_id:
|
169
|
+
request.folder_id = folder_id
|
170
|
+
|
171
|
+
# 构建元数据
|
172
|
+
grpc_metadata = self.client.build_metadata(request_id=request_id, **metadata)
|
173
|
+
|
174
|
+
# 发送请求
|
175
|
+
response = stub.UploadFile(request, metadata=grpc_metadata)
|
176
|
+
|
177
|
+
# 转换响应
|
178
|
+
return FileUploadResponse(
|
179
|
+
file=self._convert_file_info(response.file),
|
180
|
+
upload_file=self._convert_upload_file_info(response.upload_file),
|
181
|
+
)
|
182
|
+
|
183
|
+
def _upload_stream(
|
184
|
+
self,
|
185
|
+
file_name: str,
|
186
|
+
content: Union[bytes, BinaryIO, Path],
|
187
|
+
file_size: int,
|
188
|
+
folder_id: Optional[str],
|
189
|
+
file_type: str,
|
190
|
+
mime_type: str,
|
191
|
+
file_hash: str,
|
192
|
+
is_temporary: Optional[bool] = False,
|
193
|
+
expire_seconds: Optional[int] = None,
|
194
|
+
request_id: Optional[str] = None,
|
195
|
+
**metadata
|
196
|
+
) -> FileUploadResponse:
|
197
|
+
"""客户端直传实现"""
|
198
|
+
# 获取上传URL,以及对应的文件和上传文件信息
|
199
|
+
upload_url_resp = self.generate_upload_url(
|
200
|
+
file_name=file_name,
|
201
|
+
file_size=file_size,
|
202
|
+
folder_id=folder_id,
|
203
|
+
file_type=file_type,
|
204
|
+
mime_type=mime_type,
|
205
|
+
file_hash=file_hash,
|
206
|
+
is_temporary=is_temporary,
|
207
|
+
expire_seconds=expire_seconds if is_temporary and expire_seconds else None,
|
208
|
+
request_id=request_id,
|
209
|
+
**metadata
|
210
|
+
)
|
211
|
+
|
212
|
+
# 如果URL为空,说明文件已存在(哈希重复),直接返回
|
213
|
+
if not upload_url_resp.upload_url:
|
214
|
+
return FileUploadResponse(
|
215
|
+
file=upload_url_resp.file,
|
216
|
+
upload_file=upload_url_resp.upload_file
|
217
|
+
)
|
218
|
+
|
219
|
+
# 上传文件到对象存储
|
220
|
+
self.http_uploader.upload(
|
221
|
+
url=upload_url_resp.upload_url,
|
222
|
+
content=content,
|
223
|
+
headers={"Content-Type": mime_type},
|
224
|
+
total_size=file_size,
|
225
|
+
)
|
226
|
+
|
227
|
+
# 确认上传完成
|
228
|
+
self._confirm_upload_completed(
|
229
|
+
file_id=upload_url_resp.file.id,
|
230
|
+
request_id=request_id,
|
231
|
+
**metadata
|
232
|
+
)
|
233
|
+
|
234
|
+
# 返回文件信息
|
235
|
+
return FileUploadResponse(
|
236
|
+
file=upload_url_resp.file,
|
237
|
+
upload_file=upload_url_resp.upload_file
|
238
|
+
)
|
239
|
+
|
240
|
+
def _upload_resumable(
|
241
|
+
self,
|
242
|
+
file_name: str,
|
243
|
+
content: Union[bytes, BinaryIO, Path],
|
244
|
+
file_size: int,
|
245
|
+
folder_id: Optional[str],
|
246
|
+
file_type: str,
|
247
|
+
mime_type: str,
|
248
|
+
file_hash: str,
|
249
|
+
is_temporary: Optional[bool] = False,
|
250
|
+
expire_seconds: Optional[int] = None,
|
251
|
+
request_id: Optional[str] = None,
|
252
|
+
**metadata
|
253
|
+
) -> FileUploadResponse:
|
254
|
+
"""断点续传实现"""
|
255
|
+
# 获取断点续传URL,以及对应的文件和上传文件信息
|
256
|
+
upload_url_resp = self._generate_resumable_upload_url(
|
257
|
+
file_name=file_name,
|
258
|
+
file_size=file_size,
|
259
|
+
folder_id=folder_id,
|
260
|
+
file_type=file_type,
|
261
|
+
mime_type=mime_type,
|
262
|
+
file_hash=file_hash,
|
263
|
+
is_temporary=is_temporary,
|
264
|
+
expire_seconds=expire_seconds if is_temporary and expire_seconds else None,
|
265
|
+
request_id=request_id,
|
266
|
+
**metadata
|
267
|
+
)
|
268
|
+
|
269
|
+
# 如果URL为空,说明文件已存在(哈希重复),直接返回
|
270
|
+
if not upload_url_resp.upload_url:
|
271
|
+
return FileUploadResponse(
|
272
|
+
file=upload_url_resp.file,
|
273
|
+
upload_file=upload_url_resp.upload_file
|
274
|
+
)
|
275
|
+
|
276
|
+
# 开启断点续传
|
277
|
+
upload_url = self.http_uploader.start_resumable_session(
|
278
|
+
url=upload_url_resp.upload_url,
|
279
|
+
total_file_size=file_size,
|
280
|
+
mine_type=mime_type,
|
281
|
+
)
|
282
|
+
|
283
|
+
# 上传文件到对象存储
|
284
|
+
self.http_uploader.upload(
|
285
|
+
url=upload_url,
|
286
|
+
content=content,
|
287
|
+
headers={"Content-Type": mime_type},
|
288
|
+
total_size=file_size,
|
289
|
+
is_resume=True
|
290
|
+
)
|
291
|
+
|
292
|
+
# 确认上传完成
|
293
|
+
self._confirm_upload_completed(
|
294
|
+
file_id=upload_url_resp.file.id,
|
295
|
+
request_id=request_id,
|
296
|
+
**metadata
|
297
|
+
)
|
298
|
+
|
299
|
+
# 返回文件信息
|
300
|
+
return FileUploadResponse(
|
301
|
+
file=upload_url_resp.file,
|
302
|
+
upload_file=upload_url_resp.upload_file
|
303
|
+
)
|
304
|
+
|
305
|
+
def generate_upload_url(
|
306
|
+
self,
|
307
|
+
file_name: str,
|
308
|
+
file_size: int,
|
309
|
+
folder_id: Optional[str] = None,
|
310
|
+
file_type: str = "dat",
|
311
|
+
mime_type: str = None,
|
312
|
+
file_hash: str = None,
|
313
|
+
is_temporary: Optional[bool] = False,
|
314
|
+
expire_seconds: Optional[int] = None,
|
315
|
+
request_id: Optional[str] = None,
|
316
|
+
**metadata
|
317
|
+
) -> UploadUrlResponse:
|
318
|
+
"""
|
319
|
+
生成上传URL(用于客户端直传)
|
320
|
+
|
321
|
+
Args:
|
322
|
+
file_name: 文件名
|
323
|
+
file_size: 文件大小
|
324
|
+
folder_id: 文件夹ID
|
325
|
+
file_type: 文件类型
|
326
|
+
mime_type: MIME类型
|
327
|
+
file_hash: 文件哈希
|
328
|
+
is_temporary: 是否为临时文件
|
329
|
+
expire_seconds: 过期秒数
|
330
|
+
request_id: 请求ID(可选,如果不提供则自动生成)
|
331
|
+
**metadata: 额外的元数据(如 x-org-id, x-user-id 等)
|
332
|
+
|
333
|
+
Returns:
|
334
|
+
上传URL响应
|
335
|
+
注意:如果文件哈希已存在(重复上传),upload_url 会为空,
|
336
|
+
此时应直接使用返回的 file 和 upload_file 信息,无需再次上传
|
337
|
+
"""
|
338
|
+
from ...rpc.gen import file_service_pb2, file_service_pb2_grpc
|
339
|
+
|
340
|
+
stub = self.client.get_stub(file_service_pb2_grpc.FileServiceStub)
|
341
|
+
|
342
|
+
request = file_service_pb2.UploadUrlRequest(
|
343
|
+
file_name=file_name,
|
344
|
+
file_size=file_size,
|
345
|
+
file_type=file_type,
|
346
|
+
mime_type=mime_type or "application/octet-stream",
|
347
|
+
file_hash=file_hash,
|
348
|
+
is_temporary=is_temporary,
|
349
|
+
expire_seconds=expire_seconds if is_temporary and expire_seconds else None,
|
350
|
+
)
|
351
|
+
|
352
|
+
if folder_id:
|
353
|
+
request.folder_id = folder_id
|
354
|
+
|
355
|
+
# 构建元数据
|
356
|
+
grpc_metadata = self.client.build_metadata(request_id=request_id, **metadata)
|
357
|
+
|
358
|
+
response = stub.GenerateUploadUrl(request, metadata=grpc_metadata)
|
359
|
+
|
360
|
+
return UploadUrlResponse(
|
361
|
+
file=self._convert_file_info(response.file),
|
362
|
+
upload_file=self._convert_upload_file_info(response.upload_file),
|
363
|
+
upload_url=response.url
|
364
|
+
)
|
365
|
+
|
366
|
+
def upload(
|
367
|
+
self,
|
368
|
+
file: Optional[Union[str, Path, BinaryIO, bytes]] = None,
|
369
|
+
*,
|
370
|
+
folder_id: Optional[str] = None,
|
371
|
+
mode: Optional[UploadMode] = UploadMode.NORMAL,
|
372
|
+
is_temporary: Optional[bool] = False,
|
373
|
+
expire_seconds: Optional[int] = None,
|
374
|
+
url: Optional[str] = None,
|
375
|
+
file_name: Optional[str] = None,
|
376
|
+
request_id: Optional[str] = None,
|
377
|
+
**metadata
|
378
|
+
) -> FileUploadResponse:
|
379
|
+
"""
|
380
|
+
统一的文件上传接口
|
381
|
+
|
382
|
+
Args:
|
383
|
+
file: 文件路径、Path对象、文件对象或字节数据(当使用url参数时可省略)
|
384
|
+
folder_id: 目标文件夹ID(可选)
|
385
|
+
mode: 上传模式(NORMAL/DIRECT/RESUMABLE/STREAM)
|
386
|
+
is_temporary: 是否为临时文件
|
387
|
+
expire_seconds: 过期秒数
|
388
|
+
url: 要下载并上传的URL(可选)
|
389
|
+
file_name: 当使用url参数时指定的文件名(可选)
|
390
|
+
request_id: 请求ID(可选,如果不提供则自动生成)
|
391
|
+
**metadata: 额外的元数据
|
392
|
+
|
393
|
+
Returns:
|
394
|
+
文件信息
|
395
|
+
|
396
|
+
Note:
|
397
|
+
必须提供 file 或 url 参数之一
|
398
|
+
"""
|
399
|
+
# 参数验证:必须提供 file 或 url 之一
|
400
|
+
if file is None and not url:
|
401
|
+
raise ValidationError("必须提供 file 或 url 参数之一")
|
402
|
+
|
403
|
+
# 如果提供了URL,先下载文件
|
404
|
+
if url:
|
405
|
+
# 下载文件到内存
|
406
|
+
downloaded_content = self.http_downloader.download(url)
|
407
|
+
|
408
|
+
# 如果没有指定文件名,从URL中提取
|
409
|
+
if not file_name:
|
410
|
+
from urllib.parse import urlparse
|
411
|
+
from pathlib import Path as PathLib
|
412
|
+
parsed_url = urlparse(url)
|
413
|
+
url_path = PathLib(parsed_url.path)
|
414
|
+
file_name = url_path.name if url_path.name else f"download_{hashlib.md5(url.encode()).hexdigest()[:8]}"
|
415
|
+
|
416
|
+
# 使用下载的内容作为file参数
|
417
|
+
file = downloaded_content
|
418
|
+
|
419
|
+
# 提取文件信息(bytes会返回默认的MIME类型,我们稍后会基于文件名重新计算)
|
420
|
+
extracted_file_name, content, file_size, _, _, file_hash = self._extract_file_info(file)
|
421
|
+
|
422
|
+
# file_name已经在上面设置了(要么是用户指定的,要么是从URL提取的)
|
423
|
+
extracted_file_name = file_name
|
424
|
+
|
425
|
+
# 基于文件名计算文件类型和MIME类型
|
426
|
+
file_type = Path(extracted_file_name).suffix.lstrip('.').lower() if Path(extracted_file_name).suffix else 'dat'
|
427
|
+
mime_type = get_file_mime_type(Path(extracted_file_name))
|
428
|
+
else:
|
429
|
+
# 解析文件参数,提取文件信息
|
430
|
+
extracted_file_name, content, file_size, mime_type, file_type, file_hash = self._extract_file_info(file)
|
431
|
+
|
432
|
+
# 根据文件大小自动选择上传模式
|
433
|
+
if mode == UploadMode.NORMAL:
|
434
|
+
ten_mb = 1024 * 1024 * 10
|
435
|
+
hundred_mb = 1024 * 1024 * 100
|
436
|
+
if file_size >= ten_mb and file_size < hundred_mb: # 10MB
|
437
|
+
mode = UploadMode.STREAM # 大文件自动使用流式上传模式
|
438
|
+
elif file_size > hundred_mb:
|
439
|
+
mode = UploadMode.RESUMABLE # 特大文件自动使用断点续传模式
|
440
|
+
|
441
|
+
# 根据上传模式执行不同的上传逻辑
|
442
|
+
if mode == UploadMode.NORMAL:
|
443
|
+
# 普通上传(通过gRPC)
|
444
|
+
return self._upload_file(
|
445
|
+
file_name=extracted_file_name,
|
446
|
+
content=content,
|
447
|
+
folder_id=folder_id,
|
448
|
+
file_type=file_type,
|
449
|
+
mime_type=mime_type,
|
450
|
+
is_temporary=is_temporary,
|
451
|
+
expire_seconds=expire_seconds if is_temporary and expire_seconds else None,
|
452
|
+
request_id=request_id,
|
453
|
+
**metadata
|
454
|
+
)
|
455
|
+
|
456
|
+
elif mode == UploadMode.STREAM:
|
457
|
+
# 流式上传
|
458
|
+
return self._upload_stream(
|
459
|
+
file_name=extracted_file_name,
|
460
|
+
content=content,
|
461
|
+
file_size=file_size,
|
462
|
+
folder_id=folder_id,
|
463
|
+
file_type=file_type,
|
464
|
+
mime_type=mime_type,
|
465
|
+
file_hash=file_hash,
|
466
|
+
is_temporary=is_temporary,
|
467
|
+
expire_seconds=expire_seconds if is_temporary and expire_seconds else None,
|
468
|
+
request_id=request_id,
|
469
|
+
**metadata
|
470
|
+
)
|
471
|
+
|
472
|
+
elif mode == UploadMode.RESUMABLE:
|
473
|
+
# 断点续传
|
474
|
+
return self._upload_resumable(
|
475
|
+
file_name=extracted_file_name,
|
476
|
+
content=content,
|
477
|
+
file_size=file_size,
|
478
|
+
folder_id=folder_id,
|
479
|
+
file_type=file_type,
|
480
|
+
mime_type=mime_type,
|
481
|
+
file_hash=file_hash,
|
482
|
+
is_temporary=is_temporary,
|
483
|
+
expire_seconds=expire_seconds if is_temporary and expire_seconds else None,
|
484
|
+
request_id=request_id,
|
485
|
+
**metadata
|
486
|
+
)
|
487
|
+
|
488
|
+
else:
|
489
|
+
raise ValidationError(f"不支持的上传模式: {mode}")
|
490
|
+
|
491
|
+
def generate_download_url(
|
492
|
+
self,
|
493
|
+
file_id: str,
|
494
|
+
*,
|
495
|
+
expire_seconds: Optional[int] = None,
|
496
|
+
request_id: Optional[str] = None,
|
497
|
+
**metadata
|
498
|
+
) -> str:
|
499
|
+
"""
|
500
|
+
生成下载URL
|
501
|
+
|
502
|
+
Args:
|
503
|
+
file_id: 文件ID
|
504
|
+
request_id: 请求ID(可选,如果不提供则自动生成)
|
505
|
+
**metadata: 额外的元数据(如 x-org-id, x-user-id 等)
|
506
|
+
|
507
|
+
Returns:
|
508
|
+
下载URL
|
509
|
+
"""
|
510
|
+
from ...rpc.gen import file_service_pb2, file_service_pb2_grpc
|
511
|
+
|
512
|
+
stub = self.client.get_stub(file_service_pb2_grpc.FileServiceStub)
|
513
|
+
|
514
|
+
request = file_service_pb2.DownloadUrlRequest(file_id=file_id,
|
515
|
+
expire_seconds=expire_seconds if expire_seconds else None)
|
516
|
+
|
517
|
+
# 构建元数据
|
518
|
+
grpc_metadata = self.client.build_metadata(request_id=request_id, **metadata)
|
519
|
+
|
520
|
+
download_url_resp = stub.GenerateDownloadUrl(request, metadata=grpc_metadata)
|
521
|
+
|
522
|
+
return download_url_resp.url
|
523
|
+
|
524
|
+
def download(
|
525
|
+
self,
|
526
|
+
file_id: str,
|
527
|
+
save_path: Optional[Union[str, Path]] = None,
|
528
|
+
chunk_size: Optional[int] = None,
|
529
|
+
request_id: Optional[str] = None,
|
530
|
+
**metadata
|
531
|
+
) -> Union[bytes, Path, Iterator[bytes]]:
|
532
|
+
"""
|
533
|
+
统一的文件下载接口
|
534
|
+
|
535
|
+
Args:
|
536
|
+
file_id: 文件ID
|
537
|
+
save_path: 保存路径(如果为None,返回字节数据)
|
538
|
+
chunk_size: 分片大小
|
539
|
+
request_id: 请求ID(可选,如果不提供则自动生成)
|
540
|
+
**metadata: 额外的元数据
|
541
|
+
|
542
|
+
Returns:
|
543
|
+
- NORMAL模式:下载的内容(字节)或保存的文件路径
|
544
|
+
- STREAM模式:返回迭代器,逐块返回数据
|
545
|
+
"""
|
546
|
+
|
547
|
+
# 获取下载URL
|
548
|
+
download_url = self.generate_download_url(file_id, request_id=request_id, **metadata)
|
549
|
+
|
550
|
+
return self.http_downloader.download(
|
551
|
+
url=download_url,
|
552
|
+
save_path=save_path,
|
553
|
+
chunk_size=chunk_size,
|
554
|
+
)
|