tamar-file-hub-client 0.1.7__py3-none-any.whl → 0.2.4__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- file_hub_client/rpc/gen/file_service_pb2.py +86 -68
- file_hub_client/rpc/gen/file_service_pb2_grpc.py +923 -835
- file_hub_client/rpc/protos/file_service.proto +101 -0
- file_hub_client/schemas/__init__.py +17 -0
- file_hub_client/schemas/file.py +73 -0
- file_hub_client/services/file/async_blob_service.py +140 -4
- file_hub_client/services/file/async_file_service.py +66 -0
- file_hub_client/services/file/sync_blob_service.py +141 -5
- file_hub_client/services/file/sync_file_service.py +66 -0
- file_hub_client/utils/logging.py +11 -15
- file_hub_client/utils/upload_helper.py +67 -3
- tamar_file_hub_client-0.2.4.dist-info/METADATA +704 -0
- {tamar_file_hub_client-0.1.7.dist-info → tamar_file_hub_client-0.2.4.dist-info}/RECORD +15 -15
- tamar_file_hub_client-0.1.7.dist-info/METADATA +0 -2453
- {tamar_file_hub_client-0.1.7.dist-info → tamar_file_hub_client-0.2.4.dist-info}/WHEEL +0 -0
- {tamar_file_hub_client-0.1.7.dist-info → tamar_file_hub_client-0.2.4.dist-info}/top_level.txt +0 -0
|
@@ -28,6 +28,12 @@ service FileService {
|
|
|
28
28
|
rpc GetCompressedVariants (GetVariantsRequest) returns (GetVariantsResponse);
|
|
29
29
|
rpc TriggerRecompression (RecompressionRequest) returns (RecompressionResponse);
|
|
30
30
|
rpc GenerateVariantDownloadUrl (VariantDownloadUrlRequest) returns (VariantDownloadUrlResponse);
|
|
31
|
+
|
|
32
|
+
// 批量文件状态查询API
|
|
33
|
+
rpc BatchGetFileStatus (BatchFileStatusRequest) returns (BatchFileStatusResponse);
|
|
34
|
+
|
|
35
|
+
// 从GCS导入文件
|
|
36
|
+
rpc ImportFromGcs (ImportFromGcsRequest) returns (ImportFromGcsResponse);
|
|
31
37
|
}
|
|
32
38
|
|
|
33
39
|
// ========= 数据结构定义 =========
|
|
@@ -80,6 +86,7 @@ message UploadUrlRequest {
|
|
|
80
86
|
optional bool is_temporary = 7;
|
|
81
87
|
optional int32 expire_seconds = 8;
|
|
82
88
|
optional bool keep_original_filename = 9; // 保留原始文件名,默认false
|
|
89
|
+
optional bool forbid_overwrite = 10; // 防止覆盖同名文件,默认false
|
|
83
90
|
}
|
|
84
91
|
|
|
85
92
|
message UploadCompletedRequest {
|
|
@@ -145,6 +152,16 @@ message BatchGetGcsUrlRequest {
|
|
|
145
152
|
repeated string file_ids = 1;
|
|
146
153
|
}
|
|
147
154
|
|
|
155
|
+
message ImportFromGcsRequest {
|
|
156
|
+
string gcs_uri = 1; // GCS URI, e.g., gs://bucket/path/to/file
|
|
157
|
+
optional string folder_id = 2; // 目标文件夹ID
|
|
158
|
+
string operation_type = 3; // "copy" or "move"
|
|
159
|
+
optional string file_name = 4; // 自定义文件名(可选)
|
|
160
|
+
optional bool keep_original_filename = 5; // 保留原始文件名,默认false
|
|
161
|
+
optional string created_by_role = 6; // 创建者角色
|
|
162
|
+
optional string created_by = 7; // 创建者ID
|
|
163
|
+
}
|
|
164
|
+
|
|
148
165
|
// ========= 响应结构 =========
|
|
149
166
|
|
|
150
167
|
message UploadFileResponse {
|
|
@@ -195,6 +212,11 @@ message BatchGetGcsUrlResponse {
|
|
|
195
212
|
repeated GcsUrlInfo gcs_urls = 1;
|
|
196
213
|
}
|
|
197
214
|
|
|
215
|
+
message ImportFromGcsResponse {
|
|
216
|
+
File file = 1; // 创建的文件信息
|
|
217
|
+
UploadFile upload_file = 2; // 上传文件信息
|
|
218
|
+
}
|
|
219
|
+
|
|
198
220
|
message GcsUrlInfo {
|
|
199
221
|
string file_id = 1;
|
|
200
222
|
string gcs_url = 2;
|
|
@@ -262,4 +284,83 @@ message VariantDownloadUrlResponse {
|
|
|
262
284
|
optional CompressedVariant variant_info = 3; // 返回变体详细信息
|
|
263
285
|
}
|
|
264
286
|
|
|
287
|
+
// ========= 批量文件状态查询相关结构 =========
|
|
288
|
+
|
|
289
|
+
message BatchFileStatusRequest {
|
|
290
|
+
repeated string file_ids = 1; // 批量查询的文件ID列表(最多100个)
|
|
291
|
+
optional bool include_details = 2; // 是否返回详细信息,默认false
|
|
292
|
+
}
|
|
293
|
+
|
|
294
|
+
message BatchFileStatusResponse {
|
|
295
|
+
repeated FileStatusInfo statuses = 1; // 文件状态信息(FileStatusInfo.file_id关联请求的file_id)
|
|
296
|
+
int64 timestamp = 2; // 查询时间戳
|
|
297
|
+
int32 cache_hit_count = 3; // 缓存命中数量(用于性能监控)
|
|
298
|
+
}
|
|
299
|
+
|
|
300
|
+
message FileStatusInfo {
|
|
301
|
+
string file_id = 1;
|
|
302
|
+
|
|
303
|
+
// 三大状态
|
|
304
|
+
FileUploadStatus upload_status = 2; // 上传状态
|
|
305
|
+
FileCompressionStatus compression_status = 3; // 压缩状态
|
|
306
|
+
FileSyncStatus sync_status = 4; // 备份同步状态
|
|
307
|
+
|
|
308
|
+
// 扩展信息(当include_details=true时返回)
|
|
309
|
+
optional FileStatusDetails details = 5;
|
|
310
|
+
|
|
311
|
+
// 错误信息
|
|
312
|
+
optional string error_message = 6;
|
|
313
|
+
}
|
|
314
|
+
|
|
315
|
+
message FileStatusDetails {
|
|
316
|
+
// 上传详情
|
|
317
|
+
optional int64 file_size = 1;
|
|
318
|
+
optional string storage_type = 2; // gcs, oss
|
|
319
|
+
optional string storage_region = 3;
|
|
320
|
+
|
|
321
|
+
// 压缩详情
|
|
322
|
+
optional string compression_task_id = 4;
|
|
323
|
+
optional int32 compression_variants_count = 5;
|
|
324
|
+
optional double compression_progress = 6; // 0.0-1.0
|
|
325
|
+
|
|
326
|
+
// 同步详情
|
|
327
|
+
optional int32 sync_regions_total = 7;
|
|
328
|
+
optional int32 sync_regions_completed = 8;
|
|
329
|
+
repeated string sync_pending_regions = 9;
|
|
330
|
+
}
|
|
331
|
+
|
|
332
|
+
// 文件上传状态枚举
|
|
333
|
+
enum FileUploadStatus {
|
|
334
|
+
UPLOAD_UNKNOWN = 0;
|
|
335
|
+
UPLOAD_PENDING = 1; // 待上传
|
|
336
|
+
UPLOAD_PROCESSING = 2; // 上传中
|
|
337
|
+
UPLOAD_COMPLETED = 3; // 已完成
|
|
338
|
+
UPLOAD_FAILED = 4; // 失败
|
|
339
|
+
UPLOAD_FILE_NOT_FOUND = 5; // 文件不存在
|
|
340
|
+
}
|
|
341
|
+
|
|
342
|
+
// 文件压缩状态枚举
|
|
343
|
+
enum FileCompressionStatus {
|
|
344
|
+
COMPRESSION_UNKNOWN = 0;
|
|
345
|
+
COMPRESSION_NOT_APPLICABLE = 1; // 不需要压缩
|
|
346
|
+
COMPRESSION_PENDING = 2; // 等待压缩
|
|
347
|
+
COMPRESSION_PROCESSING = 3; // 压缩中
|
|
348
|
+
COMPRESSION_COMPLETED = 4; // 已完成
|
|
349
|
+
COMPRESSION_FAILED = 5; // 失败
|
|
350
|
+
COMPRESSION_SKIPPED = 6; // 跳过压缩
|
|
351
|
+
COMPRESSION_FILE_NOT_FOUND = 7; // 文件不存在
|
|
352
|
+
}
|
|
353
|
+
|
|
354
|
+
// 文件同步状态枚举
|
|
355
|
+
enum FileSyncStatus {
|
|
356
|
+
SYNC_UNKNOWN = 0;
|
|
357
|
+
SYNC_NOT_REQUIRED = 1; // 不需要同步
|
|
358
|
+
SYNC_PENDING = 2; // 等待同步
|
|
359
|
+
SYNC_PROCESSING = 3; // 同步中
|
|
360
|
+
SYNC_PARTIAL = 4; // 部分完成
|
|
361
|
+
SYNC_COMPLETED = 5; // 全部完成
|
|
362
|
+
SYNC_FAILED = 6; // 同步失败
|
|
363
|
+
SYNC_FILE_NOT_FOUND = 7; // 文件不存在
|
|
364
|
+
}
|
|
365
|
+
|
|
265
366
|
message Empty {}
|
|
@@ -21,6 +21,15 @@ from .file import (
|
|
|
21
21
|
GetVariantsResponse,
|
|
22
22
|
RecompressionResponse,
|
|
23
23
|
VariantDownloadUrlResponse,
|
|
24
|
+
# 文件状态相关
|
|
25
|
+
FileUploadStatus,
|
|
26
|
+
FileCompressionStatus,
|
|
27
|
+
FileSyncStatus,
|
|
28
|
+
FileStatusDetails,
|
|
29
|
+
FileStatusInfo,
|
|
30
|
+
BatchFileStatusResponse,
|
|
31
|
+
# GCS 导入
|
|
32
|
+
ImportFromGcsResponse,
|
|
24
33
|
)
|
|
25
34
|
from .folder import (
|
|
26
35
|
FolderInfo,
|
|
@@ -83,6 +92,14 @@ __all__ = [
|
|
|
83
92
|
"GetVariantsResponse",
|
|
84
93
|
"RecompressionResponse",
|
|
85
94
|
"VariantDownloadUrlResponse",
|
|
95
|
+
# 文件状态相关
|
|
96
|
+
"FileUploadStatus",
|
|
97
|
+
"FileCompressionStatus",
|
|
98
|
+
"FileSyncStatus",
|
|
99
|
+
"FileStatusDetails",
|
|
100
|
+
"FileStatusInfo",
|
|
101
|
+
"BatchFileStatusResponse",
|
|
102
|
+
"ImportFromGcsResponse",
|
|
86
103
|
|
|
87
104
|
# 文件夹相关
|
|
88
105
|
"FolderInfo",
|
file_hub_client/schemas/file.py
CHANGED
|
@@ -4,6 +4,7 @@
|
|
|
4
4
|
from datetime import datetime
|
|
5
5
|
from typing import Optional, Dict, List, Any
|
|
6
6
|
from pydantic import BaseModel, Field
|
|
7
|
+
from enum import Enum
|
|
7
8
|
|
|
8
9
|
|
|
9
10
|
class File(BaseModel):
|
|
@@ -169,3 +170,75 @@ class VariantDownloadUrlResponse(BaseModel):
|
|
|
169
170
|
url: str = Field(..., description="下载URL")
|
|
170
171
|
error: Optional[str] = Field(None, description="错误信息")
|
|
171
172
|
variant_info: Optional[CompressedVariant] = Field(None, description="变体详细信息")
|
|
173
|
+
|
|
174
|
+
|
|
175
|
+
# ========= 文件状态服务相关模型 =========
|
|
176
|
+
|
|
177
|
+
class FileUploadStatus(str, Enum):
|
|
178
|
+
"""文件上传状态枚举"""
|
|
179
|
+
UPLOAD_UNKNOWN = "UPLOAD_UNKNOWN"
|
|
180
|
+
UPLOAD_PENDING = "UPLOAD_PENDING"
|
|
181
|
+
UPLOAD_PROCESSING = "UPLOAD_PROCESSING"
|
|
182
|
+
UPLOAD_COMPLETED = "UPLOAD_COMPLETED"
|
|
183
|
+
UPLOAD_FAILED = "UPLOAD_FAILED"
|
|
184
|
+
UPLOAD_FILE_NOT_FOUND = "UPLOAD_FILE_NOT_FOUND"
|
|
185
|
+
|
|
186
|
+
|
|
187
|
+
class FileCompressionStatus(str, Enum):
|
|
188
|
+
"""文件压缩状态枚举"""
|
|
189
|
+
COMPRESSION_UNKNOWN = "COMPRESSION_UNKNOWN"
|
|
190
|
+
COMPRESSION_NOT_APPLICABLE = "COMPRESSION_NOT_APPLICABLE"
|
|
191
|
+
COMPRESSION_PENDING = "COMPRESSION_PENDING"
|
|
192
|
+
COMPRESSION_PROCESSING = "COMPRESSION_PROCESSING"
|
|
193
|
+
COMPRESSION_COMPLETED = "COMPRESSION_COMPLETED"
|
|
194
|
+
COMPRESSION_FAILED = "COMPRESSION_FAILED"
|
|
195
|
+
COMPRESSION_SKIPPED = "COMPRESSION_SKIPPED"
|
|
196
|
+
COMPRESSION_FILE_NOT_FOUND = "COMPRESSION_FILE_NOT_FOUND"
|
|
197
|
+
|
|
198
|
+
|
|
199
|
+
class FileSyncStatus(str, Enum):
|
|
200
|
+
"""文件同步状态枚举"""
|
|
201
|
+
SYNC_UNKNOWN = "SYNC_UNKNOWN"
|
|
202
|
+
SYNC_NOT_REQUIRED = "SYNC_NOT_REQUIRED"
|
|
203
|
+
SYNC_PENDING = "SYNC_PENDING"
|
|
204
|
+
SYNC_PROCESSING = "SYNC_PROCESSING"
|
|
205
|
+
SYNC_PARTIAL = "SYNC_PARTIAL"
|
|
206
|
+
SYNC_COMPLETED = "SYNC_COMPLETED"
|
|
207
|
+
SYNC_FAILED = "SYNC_FAILED"
|
|
208
|
+
SYNC_FILE_NOT_FOUND = "SYNC_FILE_NOT_FOUND"
|
|
209
|
+
|
|
210
|
+
|
|
211
|
+
class FileStatusDetails(BaseModel):
|
|
212
|
+
"""文件状态详情"""
|
|
213
|
+
file_size: Optional[int] = Field(None, description="文件大小")
|
|
214
|
+
storage_type: Optional[str] = Field(None, description="存储类型")
|
|
215
|
+
storage_region: Optional[str] = Field(None, description="存储区域")
|
|
216
|
+
compression_task_id: Optional[str] = Field(None, description="压缩任务ID")
|
|
217
|
+
compression_variants_count: Optional[int] = Field(None, description="压缩变体数量")
|
|
218
|
+
compression_progress: Optional[float] = Field(None, description="压缩进度")
|
|
219
|
+
sync_regions_total: Optional[int] = Field(None, description="同步区域总数")
|
|
220
|
+
sync_regions_completed: Optional[int] = Field(None, description="已完成同步区域数")
|
|
221
|
+
sync_pending_regions: List[str] = Field(default_factory=list, description="待同步区域列表")
|
|
222
|
+
|
|
223
|
+
|
|
224
|
+
class FileStatusInfo(BaseModel):
|
|
225
|
+
"""文件状态信息"""
|
|
226
|
+
file_id: str = Field(..., description="文件ID")
|
|
227
|
+
upload_status: FileUploadStatus = Field(..., description="上传状态")
|
|
228
|
+
compression_status: FileCompressionStatus = Field(..., description="压缩状态")
|
|
229
|
+
sync_status: FileSyncStatus = Field(..., description="同步状态")
|
|
230
|
+
details: Optional[FileStatusDetails] = Field(None, description="状态详情")
|
|
231
|
+
error_message: Optional[str] = Field(None, description="错误信息")
|
|
232
|
+
|
|
233
|
+
|
|
234
|
+
class BatchFileStatusResponse(BaseModel):
|
|
235
|
+
"""批量文件状态响应"""
|
|
236
|
+
statuses: List[FileStatusInfo] = Field(default_factory=list, description="文件状态列表")
|
|
237
|
+
timestamp: int = Field(..., description="查询时间戳")
|
|
238
|
+
cache_hit_count: int = Field(..., description="缓存命中数量(用于性能监控)")
|
|
239
|
+
|
|
240
|
+
|
|
241
|
+
class ImportFromGcsResponse(BaseModel):
|
|
242
|
+
"""从GCS导入文件响应"""
|
|
243
|
+
file: File = Field(..., description="创建的文件信息")
|
|
244
|
+
upload_file: UploadFile = Field(..., description="上传文件信息")
|
|
@@ -11,7 +11,7 @@ from .base_file_service import BaseFileService
|
|
|
11
11
|
from ...enums import UploadMode
|
|
12
12
|
from ...errors import ValidationError
|
|
13
13
|
from ...rpc import AsyncGrpcClient
|
|
14
|
-
from ...schemas import FileUploadResponse, UploadUrlResponse, BatchDownloadUrlResponse, DownloadUrlInfo, GcsUrlInfo, GetGcsUrlResponse, BatchGcsUrlResponse, CompressionStatusResponse, GetVariantsResponse, RecompressionResponse, VariantDownloadUrlResponse, CompressedVariant
|
|
14
|
+
from ...schemas import FileUploadResponse, UploadUrlResponse, BatchDownloadUrlResponse, DownloadUrlInfo, GcsUrlInfo, GetGcsUrlResponse, BatchGcsUrlResponse, CompressionStatusResponse, GetVariantsResponse, RecompressionResponse, VariantDownloadUrlResponse, CompressedVariant, BatchFileStatusResponse, FileStatusInfo, FileStatusDetails, FileUploadStatus, FileCompressionStatus, FileSyncStatus
|
|
15
15
|
from ...utils import AsyncHttpUploader, AsyncHttpDownloader, retry_with_backoff, get_file_mime_type
|
|
16
16
|
|
|
17
17
|
|
|
@@ -118,6 +118,7 @@ class AsyncBlobService(BaseFileService):
|
|
|
118
118
|
is_temporary: Optional[bool] = False,
|
|
119
119
|
expire_seconds: Optional[int] = None,
|
|
120
120
|
keep_original_filename: Optional[bool] = False,
|
|
121
|
+
forbid_overwrite: Optional[bool] = True,
|
|
121
122
|
request_id: Optional[str] = None,
|
|
122
123
|
**metadata
|
|
123
124
|
) -> FileUploadResponse:
|
|
@@ -134,6 +135,7 @@ class AsyncBlobService(BaseFileService):
|
|
|
134
135
|
is_temporary=is_temporary,
|
|
135
136
|
expire_seconds=expire_seconds if is_temporary and expire_seconds else None,
|
|
136
137
|
keep_original_filename=keep_original_filename,
|
|
138
|
+
forbid_overwrite=forbid_overwrite,
|
|
137
139
|
request_id=request_id,
|
|
138
140
|
**metadata
|
|
139
141
|
)
|
|
@@ -151,12 +153,13 @@ class AsyncBlobService(BaseFileService):
|
|
|
151
153
|
"Cache-Control": "public, max-age=86400" # 24小时公共缓存
|
|
152
154
|
}
|
|
153
155
|
|
|
154
|
-
#
|
|
156
|
+
# 上传文件到对象存储,传递forbid_overwrite参数
|
|
155
157
|
await self.http_uploader.upload(
|
|
156
158
|
url=upload_url_resp.upload_url,
|
|
157
159
|
content=content,
|
|
158
160
|
headers=headers,
|
|
159
161
|
total_size=file_size,
|
|
162
|
+
forbid_overwrite=forbid_overwrite,
|
|
160
163
|
)
|
|
161
164
|
|
|
162
165
|
# 确认上传完成
|
|
@@ -184,6 +187,7 @@ class AsyncBlobService(BaseFileService):
|
|
|
184
187
|
is_temporary: Optional[bool] = False,
|
|
185
188
|
expire_seconds: Optional[int] = None,
|
|
186
189
|
keep_original_filename: Optional[bool] = False,
|
|
190
|
+
forbid_overwrite: Optional[bool] = False,
|
|
187
191
|
request_id: Optional[str] = None,
|
|
188
192
|
**metadata
|
|
189
193
|
) -> FileUploadResponse:
|
|
@@ -200,6 +204,7 @@ class AsyncBlobService(BaseFileService):
|
|
|
200
204
|
is_temporary=is_temporary,
|
|
201
205
|
expire_seconds=expire_seconds if is_temporary and expire_seconds else None,
|
|
202
206
|
keep_original_filename=keep_original_filename,
|
|
207
|
+
forbid_overwrite=forbid_overwrite,
|
|
203
208
|
request_id=request_id,
|
|
204
209
|
**metadata
|
|
205
210
|
)
|
|
@@ -224,12 +229,13 @@ class AsyncBlobService(BaseFileService):
|
|
|
224
229
|
mime_type=mime_type,
|
|
225
230
|
)
|
|
226
231
|
|
|
227
|
-
#
|
|
232
|
+
# 上传文件到对象存储,传递forbid_overwrite参数
|
|
228
233
|
await self.http_uploader.upload(
|
|
229
234
|
url=upload_url,
|
|
230
235
|
content=content,
|
|
231
236
|
headers=headers,
|
|
232
237
|
total_size=file_size,
|
|
238
|
+
forbid_overwrite=forbid_overwrite,
|
|
233
239
|
)
|
|
234
240
|
|
|
235
241
|
# 确认上传完成
|
|
@@ -277,6 +283,7 @@ class AsyncBlobService(BaseFileService):
|
|
|
277
283
|
is_temporary: Optional[bool] = False,
|
|
278
284
|
expire_seconds: Optional[int] = None,
|
|
279
285
|
keep_original_filename: Optional[bool] = False,
|
|
286
|
+
forbid_overwrite: Optional[bool] = False,
|
|
280
287
|
request_id: Optional[str] = None,
|
|
281
288
|
**metadata
|
|
282
289
|
) -> UploadUrlResponse:
|
|
@@ -293,6 +300,7 @@ class AsyncBlobService(BaseFileService):
|
|
|
293
300
|
is_temporary: 是否为临时文件
|
|
294
301
|
expire_seconds: 过期秒数
|
|
295
302
|
keep_original_filename: 是否保留原始文件名(默认False)
|
|
303
|
+
forbid_overwrite: 防止覆盖同名文件(默认False)
|
|
296
304
|
request_id: 请求ID(可选,如果不提供则自动生成)
|
|
297
305
|
**metadata: 额外的元数据(如 x-org-id, x-user-id 等)
|
|
298
306
|
|
|
@@ -312,6 +320,7 @@ class AsyncBlobService(BaseFileService):
|
|
|
312
320
|
is_temporary=is_temporary,
|
|
313
321
|
expire_seconds=expire_seconds if is_temporary and expire_seconds else None,
|
|
314
322
|
keep_original_filename=keep_original_filename,
|
|
323
|
+
forbid_overwrite=forbid_overwrite,
|
|
315
324
|
)
|
|
316
325
|
|
|
317
326
|
if folder_id:
|
|
@@ -339,6 +348,7 @@ class AsyncBlobService(BaseFileService):
|
|
|
339
348
|
is_temporary: Optional[bool] = False,
|
|
340
349
|
expire_seconds: Optional[int] = None,
|
|
341
350
|
keep_original_filename: Optional[bool] = False,
|
|
351
|
+
forbid_overwrite: Optional[bool] = True,
|
|
342
352
|
request_id: Optional[str] = None,
|
|
343
353
|
**metadata
|
|
344
354
|
) -> UploadUrlResponse:
|
|
@@ -355,6 +365,7 @@ class AsyncBlobService(BaseFileService):
|
|
|
355
365
|
is_temporary: 是否为临时文件
|
|
356
366
|
expire_seconds: 过期秒数
|
|
357
367
|
keep_original_filename: 是否保留原始文件名(默认False)
|
|
368
|
+
forbid_overwrite: 防止覆盖同名文件(默认False)
|
|
358
369
|
request_id: 请求ID(可选,如果不提供则自动生成)
|
|
359
370
|
**metadata: 额外的元数据(如 x-org-id, x-user-id 等)
|
|
360
371
|
|
|
@@ -376,6 +387,7 @@ class AsyncBlobService(BaseFileService):
|
|
|
376
387
|
is_temporary=is_temporary,
|
|
377
388
|
expire_seconds=expire_seconds if is_temporary and expire_seconds else None,
|
|
378
389
|
keep_original_filename=keep_original_filename,
|
|
390
|
+
forbid_overwrite=forbid_overwrite,
|
|
379
391
|
)
|
|
380
392
|
|
|
381
393
|
if folder_id:
|
|
@@ -401,6 +413,7 @@ class AsyncBlobService(BaseFileService):
|
|
|
401
413
|
is_temporary: Optional[bool] = False,
|
|
402
414
|
expire_seconds: Optional[int] = None,
|
|
403
415
|
keep_original_filename: Optional[bool] = False,
|
|
416
|
+
forbid_overwrite: Optional[bool] = True,
|
|
404
417
|
url: Optional[str] = None,
|
|
405
418
|
file_name: Optional[str] = None,
|
|
406
419
|
mime_type: Optional[str] = None,
|
|
@@ -417,6 +430,7 @@ class AsyncBlobService(BaseFileService):
|
|
|
417
430
|
is_temporary: 是否为临时文件
|
|
418
431
|
expire_seconds: 过期秒数
|
|
419
432
|
keep_original_filename: 是否保留原始文件名(默认False)
|
|
433
|
+
forbid_overwrite: 防止覆盖同名文件(默认False)
|
|
420
434
|
url: 要下载并上传的URL(可选)
|
|
421
435
|
file_name: 当使用url参数时指定的文件名(可选)
|
|
422
436
|
mime_type: MIME类型(可选,用于推断文件扩展名,特别适用于AI生成的字节数据)
|
|
@@ -511,8 +525,16 @@ class AsyncBlobService(BaseFileService):
|
|
|
511
525
|
hundred_mb = 1024 * 1024 * 100
|
|
512
526
|
if file_size >= ten_mb and file_size < hundred_mb: # 10MB
|
|
513
527
|
mode = UploadMode.STREAM # 大文件自动使用流式上传模式
|
|
528
|
+
# 暂时屏蔽 RESUMABLE 模式,因为OSS断点续传尚未完成开发
|
|
529
|
+
# elif file_size > hundred_mb:
|
|
530
|
+
# mode = UploadMode.RESUMABLE # 特大文件自动使用断点续传模式
|
|
514
531
|
elif file_size > hundred_mb:
|
|
515
|
-
mode = UploadMode.
|
|
532
|
+
mode = UploadMode.STREAM # 暂时使用流式上传代替断点续传
|
|
533
|
+
|
|
534
|
+
# OSS断点续传尚未完成,将RESUMABLE模式自动转为STREAM模式
|
|
535
|
+
if mode == UploadMode.RESUMABLE:
|
|
536
|
+
mode = UploadMode.STREAM
|
|
537
|
+
# TODO: 当OSS断点续传功能完成后,移除此转换逻辑
|
|
516
538
|
|
|
517
539
|
# 根据上传模式执行不同的上传逻辑
|
|
518
540
|
if mode == UploadMode.NORMAL:
|
|
@@ -543,6 +565,7 @@ class AsyncBlobService(BaseFileService):
|
|
|
543
565
|
is_temporary=is_temporary,
|
|
544
566
|
expire_seconds=expire_seconds,
|
|
545
567
|
keep_original_filename=keep_original_filename,
|
|
568
|
+
forbid_overwrite=forbid_overwrite,
|
|
546
569
|
request_id=request_id,
|
|
547
570
|
**metadata
|
|
548
571
|
)
|
|
@@ -560,6 +583,7 @@ class AsyncBlobService(BaseFileService):
|
|
|
560
583
|
is_temporary=is_temporary,
|
|
561
584
|
expire_seconds=expire_seconds,
|
|
562
585
|
keep_original_filename=keep_original_filename,
|
|
586
|
+
forbid_overwrite=forbid_overwrite,
|
|
563
587
|
request_id=request_id,
|
|
564
588
|
**metadata
|
|
565
589
|
)
|
|
@@ -995,3 +1019,115 @@ class AsyncBlobService(BaseFileService):
|
|
|
995
1019
|
error=response.error if response.error else None,
|
|
996
1020
|
variant_info=variant_info
|
|
997
1021
|
)
|
|
1022
|
+
|
|
1023
|
+
async def batch_get_file_status(
|
|
1024
|
+
self,
|
|
1025
|
+
file_ids: List[str],
|
|
1026
|
+
*,
|
|
1027
|
+
include_details: Optional[bool] = False,
|
|
1028
|
+
request_id: Optional[str] = None,
|
|
1029
|
+
**metadata
|
|
1030
|
+
) -> BatchFileStatusResponse:
|
|
1031
|
+
"""
|
|
1032
|
+
批量获取文件状态(异步版本)
|
|
1033
|
+
|
|
1034
|
+
Args:
|
|
1035
|
+
file_ids: 文件ID列表(最多100个)
|
|
1036
|
+
include_details: 是否包含详细状态信息(默认False)
|
|
1037
|
+
request_id: 请求ID,用于追踪
|
|
1038
|
+
**metadata: 额外的gRPC元数据
|
|
1039
|
+
|
|
1040
|
+
Returns:
|
|
1041
|
+
BatchFileStatusResponse: 批量文件状态响应
|
|
1042
|
+
"""
|
|
1043
|
+
from ...rpc.gen import file_service_pb2, file_service_pb2_grpc
|
|
1044
|
+
|
|
1045
|
+
stub = await self.client.get_stub(file_service_pb2_grpc.FileServiceStub)
|
|
1046
|
+
|
|
1047
|
+
request = file_service_pb2.BatchFileStatusRequest(
|
|
1048
|
+
file_ids=file_ids,
|
|
1049
|
+
include_details=include_details if include_details is not None else False
|
|
1050
|
+
)
|
|
1051
|
+
|
|
1052
|
+
# 构建元数据
|
|
1053
|
+
grpc_metadata = self.client.build_metadata(request_id=request_id, **metadata)
|
|
1054
|
+
|
|
1055
|
+
response = await stub.BatchGetFileStatus(request, metadata=grpc_metadata)
|
|
1056
|
+
|
|
1057
|
+
# 转换文件状态信息
|
|
1058
|
+
statuses = []
|
|
1059
|
+
for status_info in response.statuses:
|
|
1060
|
+
# 转换状态详情(如果存在)
|
|
1061
|
+
details = None
|
|
1062
|
+
if status_info.HasField('details'):
|
|
1063
|
+
details = FileStatusDetails(
|
|
1064
|
+
file_size=status_info.details.file_size if status_info.details.HasField('file_size') else None,
|
|
1065
|
+
storage_type=status_info.details.storage_type if status_info.details.HasField('storage_type') else None,
|
|
1066
|
+
storage_region=status_info.details.storage_region if status_info.details.HasField('storage_region') else None,
|
|
1067
|
+
compression_task_id=status_info.details.compression_task_id if status_info.details.HasField('compression_task_id') else None,
|
|
1068
|
+
compression_variants_count=status_info.details.compression_variants_count if status_info.details.HasField('compression_variants_count') else None,
|
|
1069
|
+
compression_progress=status_info.details.compression_progress if status_info.details.HasField('compression_progress') else None,
|
|
1070
|
+
sync_regions_total=status_info.details.sync_regions_total if status_info.details.HasField('sync_regions_total') else None,
|
|
1071
|
+
sync_regions_completed=status_info.details.sync_regions_completed if status_info.details.HasField('sync_regions_completed') else None,
|
|
1072
|
+
sync_pending_regions=list(status_info.details.sync_pending_regions)
|
|
1073
|
+
)
|
|
1074
|
+
|
|
1075
|
+
# 转换枚举值
|
|
1076
|
+
upload_status = self._convert_upload_status(status_info.upload_status)
|
|
1077
|
+
compression_status = self._convert_compression_status(status_info.compression_status)
|
|
1078
|
+
sync_status = self._convert_sync_status(status_info.sync_status)
|
|
1079
|
+
|
|
1080
|
+
statuses.append(FileStatusInfo(
|
|
1081
|
+
file_id=status_info.file_id,
|
|
1082
|
+
upload_status=upload_status,
|
|
1083
|
+
compression_status=compression_status,
|
|
1084
|
+
sync_status=sync_status,
|
|
1085
|
+
details=details,
|
|
1086
|
+
error_message=status_info.error_message if status_info.HasField('error_message') else None
|
|
1087
|
+
))
|
|
1088
|
+
|
|
1089
|
+
return BatchFileStatusResponse(
|
|
1090
|
+
statuses=statuses,
|
|
1091
|
+
timestamp=response.timestamp,
|
|
1092
|
+
cache_hit_count=response.cache_hit_count
|
|
1093
|
+
)
|
|
1094
|
+
|
|
1095
|
+
def _convert_upload_status(self, proto_status: int) -> FileUploadStatus:
|
|
1096
|
+
"""转换上传状态枚举"""
|
|
1097
|
+
status_map = {
|
|
1098
|
+
0: FileUploadStatus.UPLOAD_UNKNOWN,
|
|
1099
|
+
1: FileUploadStatus.UPLOAD_PENDING,
|
|
1100
|
+
2: FileUploadStatus.UPLOAD_PROCESSING,
|
|
1101
|
+
3: FileUploadStatus.UPLOAD_COMPLETED,
|
|
1102
|
+
4: FileUploadStatus.UPLOAD_FAILED,
|
|
1103
|
+
5: FileUploadStatus.UPLOAD_FILE_NOT_FOUND,
|
|
1104
|
+
}
|
|
1105
|
+
return status_map.get(proto_status, FileUploadStatus.UPLOAD_UNKNOWN)
|
|
1106
|
+
|
|
1107
|
+
def _convert_compression_status(self, proto_status: int) -> FileCompressionStatus:
|
|
1108
|
+
"""转换压缩状态枚举"""
|
|
1109
|
+
status_map = {
|
|
1110
|
+
0: FileCompressionStatus.COMPRESSION_UNKNOWN,
|
|
1111
|
+
1: FileCompressionStatus.COMPRESSION_NOT_APPLICABLE,
|
|
1112
|
+
2: FileCompressionStatus.COMPRESSION_PENDING,
|
|
1113
|
+
3: FileCompressionStatus.COMPRESSION_PROCESSING,
|
|
1114
|
+
4: FileCompressionStatus.COMPRESSION_COMPLETED,
|
|
1115
|
+
5: FileCompressionStatus.COMPRESSION_FAILED,
|
|
1116
|
+
6: FileCompressionStatus.COMPRESSION_SKIPPED,
|
|
1117
|
+
7: FileCompressionStatus.COMPRESSION_FILE_NOT_FOUND,
|
|
1118
|
+
}
|
|
1119
|
+
return status_map.get(proto_status, FileCompressionStatus.COMPRESSION_UNKNOWN)
|
|
1120
|
+
|
|
1121
|
+
def _convert_sync_status(self, proto_status: int) -> FileSyncStatus:
|
|
1122
|
+
"""转换同步状态枚举"""
|
|
1123
|
+
status_map = {
|
|
1124
|
+
0: FileSyncStatus.SYNC_UNKNOWN,
|
|
1125
|
+
1: FileSyncStatus.SYNC_NOT_REQUIRED,
|
|
1126
|
+
2: FileSyncStatus.SYNC_PENDING,
|
|
1127
|
+
3: FileSyncStatus.SYNC_PROCESSING,
|
|
1128
|
+
4: FileSyncStatus.SYNC_PARTIAL,
|
|
1129
|
+
5: FileSyncStatus.SYNC_COMPLETED,
|
|
1130
|
+
6: FileSyncStatus.SYNC_FAILED,
|
|
1131
|
+
7: FileSyncStatus.SYNC_FILE_NOT_FOUND,
|
|
1132
|
+
}
|
|
1133
|
+
return status_map.get(proto_status, FileSyncStatus.SYNC_UNKNOWN)
|
|
@@ -492,3 +492,69 @@ class AsyncFileService(BaseFileService):
|
|
|
492
492
|
error=response.error if response.error else None,
|
|
493
493
|
variant_info=variant_info
|
|
494
494
|
)
|
|
495
|
+
|
|
496
|
+
async def import_from_gcs(
|
|
497
|
+
self,
|
|
498
|
+
gcs_uri: str,
|
|
499
|
+
operation_type: str,
|
|
500
|
+
*,
|
|
501
|
+
folder_id: Optional[str] = None,
|
|
502
|
+
file_name: Optional[str] = None,
|
|
503
|
+
keep_original_filename: bool = False,
|
|
504
|
+
created_by_role: Optional[str] = None,
|
|
505
|
+
created_by: Optional[str] = None,
|
|
506
|
+
request_id: Optional[str] = None,
|
|
507
|
+
**metadata
|
|
508
|
+
) -> 'ImportFromGcsResponse':
|
|
509
|
+
"""
|
|
510
|
+
从GCS导入文件
|
|
511
|
+
|
|
512
|
+
Args:
|
|
513
|
+
gcs_uri: GCS URI, 例如 gs://bucket/path/to/file
|
|
514
|
+
operation_type: 操作类型,"copy" 或 "move"
|
|
515
|
+
folder_id: 目标文件夹ID(可选)
|
|
516
|
+
file_name: 自定义文件名(可选)
|
|
517
|
+
keep_original_filename: 保留原始文件名,默认False
|
|
518
|
+
created_by_role: 创建者角色(可选)
|
|
519
|
+
created_by: 创建者ID(可选)
|
|
520
|
+
request_id: 请求ID,用于追踪
|
|
521
|
+
**metadata: 额外的gRPC元数据
|
|
522
|
+
|
|
523
|
+
Returns:
|
|
524
|
+
ImportFromGcsResponse: 导入响应,包含文件信息和上传文件信息
|
|
525
|
+
"""
|
|
526
|
+
from ...rpc.gen import file_service_pb2, file_service_pb2_grpc
|
|
527
|
+
from ...schemas import ImportFromGcsResponse
|
|
528
|
+
|
|
529
|
+
stub = await self.client.get_stub(file_service_pb2_grpc.FileServiceStub)
|
|
530
|
+
|
|
531
|
+
request = file_service_pb2.ImportFromGcsRequest(
|
|
532
|
+
gcs_uri=gcs_uri,
|
|
533
|
+
operation_type=operation_type,
|
|
534
|
+
keep_original_filename=keep_original_filename
|
|
535
|
+
)
|
|
536
|
+
|
|
537
|
+
if folder_id:
|
|
538
|
+
request.folder_id = folder_id
|
|
539
|
+
if file_name:
|
|
540
|
+
request.file_name = file_name
|
|
541
|
+
if created_by_role:
|
|
542
|
+
request.created_by_role = created_by_role
|
|
543
|
+
if created_by:
|
|
544
|
+
request.created_by = created_by
|
|
545
|
+
|
|
546
|
+
# 构建元数据
|
|
547
|
+
grpc_metadata = self.client.build_metadata(request_id=request_id, **metadata)
|
|
548
|
+
|
|
549
|
+
response = await stub.ImportFromGcs(request, metadata=grpc_metadata)
|
|
550
|
+
|
|
551
|
+
# 转换文件信息
|
|
552
|
+
file_info = self._convert_file_info(response.file)
|
|
553
|
+
|
|
554
|
+
# 转换上传文件信息
|
|
555
|
+
upload_file_info = self._convert_upload_file_info(response.upload_file)
|
|
556
|
+
|
|
557
|
+
return ImportFromGcsResponse(
|
|
558
|
+
file=file_info,
|
|
559
|
+
upload_file=upload_file_info
|
|
560
|
+
)
|