pyuploadx 0.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (87) hide show
  1. pyuploadx-0.1.0/LICENSE +21 -0
  2. pyuploadx-0.1.0/PKG-INFO +290 -0
  3. pyuploadx-0.1.0/README.md +258 -0
  4. pyuploadx-0.1.0/app/__init__.py +3 -0
  5. pyuploadx-0.1.0/app/api/__init__.py +0 -0
  6. pyuploadx-0.1.0/app/api/dependencies.py +105 -0
  7. pyuploadx-0.1.0/app/api/v1/__init__.py +0 -0
  8. pyuploadx-0.1.0/app/api/v1/client_config.py +78 -0
  9. pyuploadx-0.1.0/app/api/v1/directory_uploads.py +241 -0
  10. pyuploadx-0.1.0/app/api/v1/files.py +125 -0
  11. pyuploadx-0.1.0/app/api/v1/health.py +47 -0
  12. pyuploadx-0.1.0/app/api/v1/lifecycle.py +86 -0
  13. pyuploadx-0.1.0/app/api/v1/presign.py +49 -0
  14. pyuploadx-0.1.0/app/api/v1/uploads.py +200 -0
  15. pyuploadx-0.1.0/app/cli.py +95 -0
  16. pyuploadx-0.1.0/app/config/__init__.py +4 -0
  17. pyuploadx-0.1.0/app/config/loader.py +93 -0
  18. pyuploadx-0.1.0/app/config/models.py +300 -0
  19. pyuploadx-0.1.0/app/config/validation.py +70 -0
  20. pyuploadx-0.1.0/app/core/__init__.py +0 -0
  21. pyuploadx-0.1.0/app/core/auth.py +56 -0
  22. pyuploadx-0.1.0/app/core/errors.py +253 -0
  23. pyuploadx-0.1.0/app/core/idempotency.py +74 -0
  24. pyuploadx-0.1.0/app/core/logging.py +59 -0
  25. pyuploadx-0.1.0/app/core/metrics.py +109 -0
  26. pyuploadx-0.1.0/app/core/streaming.py +39 -0
  27. pyuploadx-0.1.0/app/core/tracing.py +18 -0
  28. pyuploadx-0.1.0/app/db/__init__.py +0 -0
  29. pyuploadx-0.1.0/app/db/models.py +361 -0
  30. pyuploadx-0.1.0/app/db/repositories/__init__.py +8 -0
  31. pyuploadx-0.1.0/app/db/repositories/directory_repository.py +55 -0
  32. pyuploadx-0.1.0/app/db/repositories/file_repository.py +40 -0
  33. pyuploadx-0.1.0/app/db/repositories/part_repository.py +80 -0
  34. pyuploadx-0.1.0/app/db/repositories/upload_repository.py +47 -0
  35. pyuploadx-0.1.0/app/db/session.py +59 -0
  36. pyuploadx-0.1.0/app/directory_upload/__init__.py +3 -0
  37. pyuploadx-0.1.0/app/directory_upload/aggregation.py +38 -0
  38. pyuploadx-0.1.0/app/directory_upload/manifest.py +66 -0
  39. pyuploadx-0.1.0/app/directory_upload/paths.py +57 -0
  40. pyuploadx-0.1.0/app/directory_upload/state_machine.py +39 -0
  41. pyuploadx-0.1.0/app/lifecycle/__init__.py +3 -0
  42. pyuploadx-0.1.0/app/lifecycle/policy.py +108 -0
  43. pyuploadx-0.1.0/app/lifecycle/state_machine.py +51 -0
  44. pyuploadx-0.1.0/app/main.py +104 -0
  45. pyuploadx-0.1.0/app/services/__init__.py +0 -0
  46. pyuploadx-0.1.0/app/services/cleanup_service.py +68 -0
  47. pyuploadx-0.1.0/app/services/directory_upload_service.py +424 -0
  48. pyuploadx-0.1.0/app/services/file_service.py +242 -0
  49. pyuploadx-0.1.0/app/services/lifecycle_service.py +206 -0
  50. pyuploadx-0.1.0/app/services/reconcile_service.py +56 -0
  51. pyuploadx-0.1.0/app/services/upload_service.py +575 -0
  52. pyuploadx-0.1.0/app/services/webhook_service.py +80 -0
  53. pyuploadx-0.1.0/app/storage/__init__.py +4 -0
  54. pyuploadx-0.1.0/app/storage/base.py +142 -0
  55. pyuploadx-0.1.0/app/storage/capabilities.py +18 -0
  56. pyuploadx-0.1.0/app/storage/factory.py +14 -0
  57. pyuploadx-0.1.0/app/storage/local.py +280 -0
  58. pyuploadx-0.1.0/app/storage/s3.py +347 -0
  59. pyuploadx-0.1.0/app/worker/__init__.py +0 -0
  60. pyuploadx-0.1.0/app/worker/cleanup.py +34 -0
  61. pyuploadx-0.1.0/app/worker/lifecycle.py +123 -0
  62. pyuploadx-0.1.0/app/worker/main.py +79 -0
  63. pyuploadx-0.1.0/pyproject.toml +62 -0
  64. pyuploadx-0.1.0/pyuploadx.egg-info/PKG-INFO +290 -0
  65. pyuploadx-0.1.0/pyuploadx.egg-info/SOURCES.txt +85 -0
  66. pyuploadx-0.1.0/pyuploadx.egg-info/dependency_links.txt +1 -0
  67. pyuploadx-0.1.0/pyuploadx.egg-info/entry_points.txt +2 -0
  68. pyuploadx-0.1.0/pyuploadx.egg-info/requires.txt +23 -0
  69. pyuploadx-0.1.0/pyuploadx.egg-info/top_level.txt +3 -0
  70. pyuploadx-0.1.0/sdk/pyuploadx/__init__.py +37 -0
  71. pyuploadx-0.1.0/sdk/pyuploadx/client.py +361 -0
  72. pyuploadx-0.1.0/sdk/pyuploadx/directory.py +81 -0
  73. pyuploadx-0.1.0/sdk/pyuploadx/directory_state.py +60 -0
  74. pyuploadx-0.1.0/sdk/pyuploadx/exceptions.py +79 -0
  75. pyuploadx-0.1.0/sdk/pyuploadx/fingerprint.py +35 -0
  76. pyuploadx-0.1.0/sdk/pyuploadx/ignore.py +54 -0
  77. pyuploadx-0.1.0/sdk/pyuploadx/lifecycle.py +36 -0
  78. pyuploadx-0.1.0/sdk/pyuploadx/manifest.py +40 -0
  79. pyuploadx-0.1.0/sdk/pyuploadx/models.py +131 -0
  80. pyuploadx-0.1.0/sdk/pyuploadx/multipart.py +86 -0
  81. pyuploadx-0.1.0/sdk/pyuploadx/paths.py +31 -0
  82. pyuploadx-0.1.0/sdk/pyuploadx/retry.py +54 -0
  83. pyuploadx-0.1.0/sdk/pyuploadx/scheduler.py +52 -0
  84. pyuploadx-0.1.0/sdk/pyuploadx/state.py +67 -0
  85. pyuploadx-0.1.0/setup.cfg +4 -0
  86. pyuploadx-0.1.0/upload_service/__init__.py +5 -0
  87. pyuploadx-0.1.0/upload_service/__main__.py +3 -0
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 shark8848
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,290 @@
1
+ Metadata-Version: 2.4
2
+ Name: pyuploadx
3
+ Version: 0.1.0
4
+ Summary: PyUploadX - file and directory upload service (FastAPI backend + Python SDK)
5
+ Author: PyUploadX Contributors
6
+ License: MIT
7
+ Requires-Python: >=3.11
8
+ Description-Content-Type: text/markdown
9
+ License-File: LICENSE
10
+ Requires-Dist: fastapi<1,>=0.110
11
+ Requires-Dist: uvicorn[standard]<1,>=0.29
12
+ Requires-Dist: pydantic<3,>=2.6
13
+ Requires-Dist: sqlalchemy[asyncio]<3,>=2.0
14
+ Requires-Dist: asyncpg<1,>=0.29
15
+ Requires-Dist: aiosqlite<1,>=0.20
16
+ Requires-Dist: alembic<2,>=1.13
17
+ Requires-Dist: redis<6,>=5.0
18
+ Requires-Dist: boto3<2,>=1.34
19
+ Requires-Dist: aiofiles<25,>=23.2
20
+ Requires-Dist: httpx<1,>=0.27
21
+ Requires-Dist: PyYAML<7,>=6.0
22
+ Requires-Dist: prometheus-client<1,>=0.20
23
+ Requires-Dist: python-multipart<1,>=0.0.9
24
+ Provides-Extra: docs
25
+ Requires-Dist: CairoSVG<3,>=2.7; extra == "docs"
26
+ Provides-Extra: dev
27
+ Requires-Dist: pytest<9,>=8; extra == "dev"
28
+ Requires-Dist: pytest-asyncio<1,>=0.23; extra == "dev"
29
+ Requires-Dist: ruff<1,>=0.4; extra == "dev"
30
+ Requires-Dist: types-PyYAML<7,>=6.0; extra == "dev"
31
+ Dynamic: license-file
32
+
33
+ # PyUploadX 文件与目录上传服务
34
+
35
+ PyUploadX 是一个轻量、可独立部署、可水平扩展的文件与目录上传服务,提供统一的
36
+ 文件上传、断点续传、目录上传、生命周期管理能力。详细设计与实现约束见
37
+ [产品设计说明书](docs/docs_product-design.md)(§31 目录结构、§16 REST API、§17 SDK)。
38
+
39
+ ## 功能特性
40
+
41
+ - **上传模式**:小文件 Proxy 上传、大文件 Multipart(分片 + 预签名)、Local/S3/MinIO 存储适配
42
+ - **断点续传**:SDK 与 Portal 均支持指纹校验、状态恢复、缺失分片重传、URL 刷新
43
+ - **目录上传**:Manifest + NDJSON、路径安全校验、`.uploadignore`、冲突策略、双层并发
44
+ - **生命周期**:`permanent` / `ttl` / `expires_at` / `temporary` / `sliding_ttl`,Legal Hold、Webhook
45
+ - **集群一致性**:无状态 API 节点、PostgreSQL 行锁、Part Upsert、Complete/Abort 幂等
46
+ - **可观测性**:结构化 JSON 日志、Prometheus 指标、`/healthz` `/readyz` `/startupz`
47
+
48
+ ## 技术栈
49
+
50
+ FastAPI · SQLAlchemy 2(ORM)· Alembic · PostgreSQL · Redis · boto3 · Python SDK(httpx)·
51
+ React + TypeScript + Vite + Dexie · Docker Compose · Kubernetes · pytest
52
+
53
+ ## 快速开始(Docker Compose)
54
+
55
+ PostgreSQL / Redis / MinIO 等第三方组件**不随应用镜像构建**:默认通过环境变量指向
56
+ 本地或已有实例(默认地址 `host.docker.internal`,即宿主机;端口与凭据均可覆盖)。
57
+
58
+ 方式 A:使用本地/已有 PostgreSQL、Redis、MinIO
59
+
60
+ ```bash
61
+ # 可选:覆盖默认地址与凭据
62
+ export UPLOAD_DATABASE_URL='postgresql+asyncpg://upload:upload@localhost:5432/uploads'
63
+ export UPLOAD_REDIS_URL='redis://localhost:6379/0'
64
+ export UPLOAD_STORAGE__S3__INTERNAL_ENDPOINT_URL='http://localhost:9000'
65
+ export S3_ACCESS_KEY=minioadmin S3_SECRET_KEY=minioadmin
66
+ docker compose up -d --build # migrate → upload-api → worker → portal
67
+ ```
68
+
69
+ 方式 B:自带第三方组件(`deploy/infra/compose.yaml`,端口可用 `POSTGRES_PORT`、
70
+ `REDIS_PORT`、`MINIO_PORT` 覆盖)
71
+
72
+ ```bash
73
+ docker compose -f deploy/infra/compose.yaml up -d # postgres/redis/minio + 建桶
74
+ docker compose up -d --build # 应用服务
75
+ ```
76
+
77
+ 单节点/集群一键(含组件):
78
+
79
+ ```bash
80
+ docker compose -f deploy/single-node/compose.yaml up -d --build
81
+ docker compose -f deploy/cluster/compose.yaml up -d --build --scale upload-api=3 --scale worker=2
82
+ ```
83
+
84
+ 启动后:
85
+
86
+ - API/OpenAPI:http://localhost:8000/docs
87
+ - Portal:http://localhost:5173(API Key:`dev-key`)
88
+ - MinIO Console:http://localhost:9001(`minioadmin` / `minioadmin`)
89
+
90
+ ## Kubernetes 部署
91
+
92
+ 模板位于 `deploy/kubernetes/`(Namespace / Deployment / Service / Ingress / HPA / PDB /
93
+ Secret / ConfigMap / ServiceMonitor)。API 默认 3 副本、`maxUnavailable: 0`、优雅终止 60s,
94
+ 并通过 `/readyz` 探针摘除故障节点。ServiceMonitor(`servicemonitor.yaml`)可被
95
+ Prometheus Operator 直接发现:
96
+
97
+ ```bash
98
+ kubectl apply -f deploy/kubernetes/
99
+ ```
100
+
101
+ 生产强制 HTTPS:Ingress 终止 TLS;单节点/集群部署由 `deploy/nginx/gateway.conf` 强制跳转
102
+ HTTPS 并拒绝非白名单 Origin 的 CORS 请求。
103
+
104
+ ## 本地开发
105
+
106
+ ```bash
107
+ python -m venv .venv && source .venv/bin/activate
108
+ pip install -e ".[dev,docs]"
109
+ python -m pytest tests -q # 全量测试
110
+ ruff check app sdk upload_service tests scripts
111
+ uvicorn app.main:create_app --factory --reload --host 0.0.0.0 --port 8000
112
+ ```
113
+
114
+ 测试默认使用 SQLite(aiosqlite);CI 使用 PostgreSQL。集群模式禁止 SQLite(`config validate` 会拒绝)。
115
+
116
+ ### 数据库迁移(Alembic)
117
+
118
+ ```bash
119
+ alembic upgrade head # 升级到最新 schema
120
+ alembic downgrade -1 # 回退一个版本(仅限向后兼容的迁移)
121
+ make migrate # 等价于 alembic upgrade head
122
+ ```
123
+
124
+ Schema 变更一律通过 Alembic 迁移交付,禁止用临时脚本改表(见 AGENTS.md Architecture Contracts)。
125
+
126
+ ### MinIO / S3 Adapter 测试
127
+
128
+ ```bash
129
+ UPLOAD_MINIO_TEST=1 \
130
+ UPLOAD_STORAGE__S3__INTERNAL_ENDPOINT_URL=http://localhost:9000 \
131
+ S3_ACCESS_KEY=minioadmin S3_SECRET_KEY=minioadmin \
132
+ python -m pytest tests/integration/test_s3_storage.py -q
133
+ ```
134
+
135
+ 未设置 `UPLOAD_MINIO_TEST=1` 时该套件自动跳过;CI 内置 MinIO 服务并始终运行它。
136
+
137
+ ### Portal E2E(Playwright)
138
+
139
+ ```bash
140
+ cd portal
141
+ npm ci
142
+ npx playwright install chromium # 首次运行需下载浏览器
143
+ npx playwright test # 自动拉起 uvicorn(8000) 与 vite(5173)
144
+ ```
145
+
146
+ E2E 覆盖登录与错误展示、文件/目录上传、生命周期策略、刷新后队列恢复(§29.6)。
147
+ 本地默认 `E2E_API_URL=http://127.0.0.1:8000`、`E2E_PORTAL_URL=http://127.0.0.1:5173`,
148
+ API Key 为 `e2e-key`(由 `portal/playwright.config.ts` 注入,SQLite + local 存储)。
149
+ CI 的 portal job 会安装浏览器依赖并执行同一套测试。
150
+
151
+ ## 配置
152
+
153
+ 配置优先级:代码默认值 < `config/config.yaml` < 环境变量(`UPLOAD_SECTION__FIELD`)< 命令行。
154
+
155
+ ```bash
156
+ python -m upload_service config validate
157
+ python -m upload_service config show --redact-secrets
158
+ python -m upload_service reconcile upload {upload_id} --dry-run
159
+ ```
160
+
161
+ 关键环境变量:
162
+
163
+ | 变量 | 说明 |
164
+ |---|---|
165
+ | `UPLOAD_API_KEYS` | JSON:`{"tenant/principal": ["key"]}` 或 key 数组 |
166
+ | `UPLOAD_DATABASE_URL` | SQLAlchemy 异步 URL |
167
+ | `UPLOAD_REDIS_URL` | Redis URL(`UPLOAD_REDIS__ENABLED=false` 可关闭) |
168
+ | `S3_ACCESS_KEY` / `S3_SECRET_KEY` | S3/MinIO 凭据 |
169
+
170
+ ## 备份与恢复
171
+
172
+ 运维手册见 [`docs/operations.md`](docs/operations.md):PostgreSQL 每日全量备份与 PITR、
173
+ 对象存储 Versioning/复制/对象锁、恢复顺序(先 DB → 对象存储 → Redis → API 只读检查 →
174
+ Reconcile Dry Run → Worker → 开放入口)、向后兼容的迁移与回滚原则(§28)。
175
+
176
+ ## Python SDK
177
+
178
+ ```python
179
+ from datetime import timedelta
180
+ from pyuploadx import UploadClient, FileLifecycle
181
+
182
+ client = UploadClient(
183
+ base_url="http://localhost:8000",
184
+ api_key="dev-key",
185
+ state_dir="~/.pyuploadx/uploads",
186
+ )
187
+
188
+ # 小文件
189
+ result = client.upload_file(
190
+ "./README.md",
191
+ bucket="app-default",
192
+ lifecycle=FileLifecycle.ttl(timedelta(days=30)),
193
+ )
194
+
195
+ # 大文件(Multipart + 断点续传)
196
+ result = client.upload_large_file(
197
+ "./model.bin",
198
+ bucket="app-default",
199
+ object_key="models/model.bin",
200
+ part_size=8 * 1024 * 1024,
201
+ concurrency=4,
202
+ resume=True,
203
+ )
204
+
205
+ # 目录上传
206
+ job = client.upload_directory(
207
+ "./album-assets",
208
+ bucket="app-default",
209
+ destination_prefix="artists/10001/albums/2026",
210
+ file_concurrency=8,
211
+ part_concurrency=4,
212
+ exclude=[".git/**", "**/*.tmp"],
213
+ conflict_policy="reject",
214
+ )
215
+ ```
216
+
217
+ ## REST API 摘要
218
+
219
+ ```text
220
+ GET /healthz | /readyz | /startupz | /metrics
221
+ POST /v1/files/upload GET/DELETE /v1/files/{id}
222
+ GET /v1/files/{id}/download POST /v1/files/{id}/presign-download
223
+ POST /v1/uploads POST /v1/uploads/resume
224
+ GET /v1/uploads/{id} GET /v1/uploads/{id}/parts
225
+ POST /v1/uploads/{id}/parts/presign
226
+ PUT /v1/uploads/{id}/parts/{part_number}
227
+ POST /v1/uploads/{id}/parts/commit
228
+ POST /v1/uploads/{id}/refresh | /complete | /abort
229
+ POST /v1/directory-uploads ... /v1/files/{id}/lifecycle ... /v1/client-config
230
+ ```
231
+
232
+ 错误响应统一为 `{"error": {"code", "message", "details", "retryable", "request_id"}}`。
233
+
234
+ ## 可观测性
235
+
236
+ - 健康检查:`/healthz`(Liveness)、`/readyz`(Readiness,检测数据库连通性)、`/startupz`。
237
+ - 指标:`/metrics`(Prometheus 文本格式),覆盖上传/分片/断点/目录/生命周期/数据库/Redis/
238
+ Storage 全链路;Kubernetes 下由 `deploy/kubernetes/servicemonitor.yaml` 抓取。
239
+ - 日志:结构化 JSON(request_id / trace_id / node_id / tenant_id / duration_ms 等),
240
+ 不记录 Secret、完整 API Key 或完整预签名 URL(§23)。
241
+
242
+ ## 性能测试
243
+
244
+ ```bash
245
+ python scripts/benchmark_upload.py --base-url http://localhost:8000 \
246
+ --api-key dev-key --files 100 --concurrency 16
247
+ python scripts/benchmark_upload.py --large-mb 64 --part-size 8388608 --concurrency 8
248
+ ```
249
+
250
+ 场景对应 §29.7:并发小文件 Proxy 上传与 1 GiB 级大文件 Multipart(走 SDK 断点续传链路),
251
+ 输出吞吐与 p50/p95 延迟。大规模场景(10 万小文件、目录总大小 1 TiB)请在目标环境按需调参。
252
+
253
+ ## 项目结构
254
+
255
+ ```text
256
+ app/ FastAPI 后端(api/config/core/db/storage/services/lifecycle/worker)
257
+ sdk/pyuploadx/ Python 客户端 SDK
258
+ portal/ React + TypeScript Portal(Dexie/IndexedDB 断点状态)
259
+ deploy/ 单节点/集群 Compose、Kubernetes(含 ServiceMonitor)、Nginx、MinIO 引导
260
+ config/ YAML 配置示例
261
+ tests/ 单元与集成测试
262
+ scripts/ 渲染/文档检查/看板同步/性能测试脚本
263
+ docs/ 设计文档、运维手册与架构图(SVG 源 / PNG 生成)
264
+ ```
265
+
266
+ ## 数据库与 ORM 契约
267
+
268
+ - 所有数据访问必须使用 SQLAlchemy 2 ORM(`app/db/models.py` + `app/db/repositories/`)。
269
+ - 禁止裸 SQL 字符串 DML;唯一例外是仓储层内基于
270
+ `sqlalchemy.dialects.postgresql.insert` 的 `ON CONFLICT` Upsert。
271
+ - Schema 变更通过 Alembic 迁移交付。
272
+
273
+ ## 安全要点
274
+
275
+ - 生产强制 HTTPS(Nginx/Ingress TLS 终止),CORS 仅允许显式配置的 Origin。
276
+ - Secret 只通过环境变量注入,禁止写入 YAML 或提交到仓库;日志脱敏密钥与完整签名 URL。
277
+ - PostgreSQL/Redis 不暴露公网(Compose 仅内部网络,K8s 无外部 Service)。
278
+ - 对象 Key 与目录相对路径均做防路径逃逸校验;上传会话校验所有权,Complete/Abort 幂等。
279
+
280
+ ## 文档图形(§35)
281
+
282
+ ```bash
283
+ make diagrams # 渲染 docs/assets/svg -> docs/assets/png
284
+ make diagrams-force # 强制重新渲染
285
+ make docs-check # 校验 PNG 未过期 + Markdown 引用 + SVG 安全(CI 门禁)
286
+ ```
287
+
288
+ ## 许可证
289
+
290
+ MIT
@@ -0,0 +1,258 @@
1
+ # PyUploadX 文件与目录上传服务
2
+
3
+ PyUploadX 是一个轻量、可独立部署、可水平扩展的文件与目录上传服务,提供统一的
4
+ 文件上传、断点续传、目录上传、生命周期管理能力。详细设计与实现约束见
5
+ [产品设计说明书](docs/docs_product-design.md)(§31 目录结构、§16 REST API、§17 SDK)。
6
+
7
+ ## 功能特性
8
+
9
+ - **上传模式**:小文件 Proxy 上传、大文件 Multipart(分片 + 预签名)、Local/S3/MinIO 存储适配
10
+ - **断点续传**:SDK 与 Portal 均支持指纹校验、状态恢复、缺失分片重传、URL 刷新
11
+ - **目录上传**:Manifest + NDJSON、路径安全校验、`.uploadignore`、冲突策略、双层并发
12
+ - **生命周期**:`permanent` / `ttl` / `expires_at` / `temporary` / `sliding_ttl`,Legal Hold、Webhook
13
+ - **集群一致性**:无状态 API 节点、PostgreSQL 行锁、Part Upsert、Complete/Abort 幂等
14
+ - **可观测性**:结构化 JSON 日志、Prometheus 指标、`/healthz` `/readyz` `/startupz`
15
+
16
+ ## 技术栈
17
+
18
+ FastAPI · SQLAlchemy 2(ORM)· Alembic · PostgreSQL · Redis · boto3 · Python SDK(httpx)·
19
+ React + TypeScript + Vite + Dexie · Docker Compose · Kubernetes · pytest
20
+
21
+ ## 快速开始(Docker Compose)
22
+
23
+ PostgreSQL / Redis / MinIO 等第三方组件**不随应用镜像构建**:默认通过环境变量指向
24
+ 本地或已有实例(默认地址 `host.docker.internal`,即宿主机;端口与凭据均可覆盖)。
25
+
26
+ 方式 A:使用本地/已有 PostgreSQL、Redis、MinIO
27
+
28
+ ```bash
29
+ # 可选:覆盖默认地址与凭据
30
+ export UPLOAD_DATABASE_URL='postgresql+asyncpg://upload:upload@localhost:5432/uploads'
31
+ export UPLOAD_REDIS_URL='redis://localhost:6379/0'
32
+ export UPLOAD_STORAGE__S3__INTERNAL_ENDPOINT_URL='http://localhost:9000'
33
+ export S3_ACCESS_KEY=minioadmin S3_SECRET_KEY=minioadmin
34
+ docker compose up -d --build # migrate → upload-api → worker → portal
35
+ ```
36
+
37
+ 方式 B:自带第三方组件(`deploy/infra/compose.yaml`,端口可用 `POSTGRES_PORT`、
38
+ `REDIS_PORT`、`MINIO_PORT` 覆盖)
39
+
40
+ ```bash
41
+ docker compose -f deploy/infra/compose.yaml up -d # postgres/redis/minio + 建桶
42
+ docker compose up -d --build # 应用服务
43
+ ```
44
+
45
+ 单节点/集群一键(含组件):
46
+
47
+ ```bash
48
+ docker compose -f deploy/single-node/compose.yaml up -d --build
49
+ docker compose -f deploy/cluster/compose.yaml up -d --build --scale upload-api=3 --scale worker=2
50
+ ```
51
+
52
+ 启动后:
53
+
54
+ - API/OpenAPI:http://localhost:8000/docs
55
+ - Portal:http://localhost:5173(API Key:`dev-key`)
56
+ - MinIO Console:http://localhost:9001(`minioadmin` / `minioadmin`)
57
+
58
+ ## Kubernetes 部署
59
+
60
+ 模板位于 `deploy/kubernetes/`(Namespace / Deployment / Service / Ingress / HPA / PDB /
61
+ Secret / ConfigMap / ServiceMonitor)。API 默认 3 副本、`maxUnavailable: 0`、优雅终止 60s,
62
+ 并通过 `/readyz` 探针摘除故障节点。ServiceMonitor(`servicemonitor.yaml`)可被
63
+ Prometheus Operator 直接发现:
64
+
65
+ ```bash
66
+ kubectl apply -f deploy/kubernetes/
67
+ ```
68
+
69
+ 生产强制 HTTPS:Ingress 终止 TLS;单节点/集群部署由 `deploy/nginx/gateway.conf` 强制跳转
70
+ HTTPS 并拒绝非白名单 Origin 的 CORS 请求。
71
+
72
+ ## 本地开发
73
+
74
+ ```bash
75
+ python -m venv .venv && source .venv/bin/activate
76
+ pip install -e ".[dev,docs]"
77
+ python -m pytest tests -q # 全量测试
78
+ ruff check app sdk upload_service tests scripts
79
+ uvicorn app.main:create_app --factory --reload --host 0.0.0.0 --port 8000
80
+ ```
81
+
82
+ 测试默认使用 SQLite(aiosqlite);CI 使用 PostgreSQL。集群模式禁止 SQLite(`config validate` 会拒绝)。
83
+
84
+ ### 数据库迁移(Alembic)
85
+
86
+ ```bash
87
+ alembic upgrade head # 升级到最新 schema
88
+ alembic downgrade -1 # 回退一个版本(仅限向后兼容的迁移)
89
+ make migrate # 等价于 alembic upgrade head
90
+ ```
91
+
92
+ Schema 变更一律通过 Alembic 迁移交付,禁止用临时脚本改表(见 AGENTS.md Architecture Contracts)。
93
+
94
+ ### MinIO / S3 Adapter 测试
95
+
96
+ ```bash
97
+ UPLOAD_MINIO_TEST=1 \
98
+ UPLOAD_STORAGE__S3__INTERNAL_ENDPOINT_URL=http://localhost:9000 \
99
+ S3_ACCESS_KEY=minioadmin S3_SECRET_KEY=minioadmin \
100
+ python -m pytest tests/integration/test_s3_storage.py -q
101
+ ```
102
+
103
+ 未设置 `UPLOAD_MINIO_TEST=1` 时该套件自动跳过;CI 内置 MinIO 服务并始终运行它。
104
+
105
+ ### Portal E2E(Playwright)
106
+
107
+ ```bash
108
+ cd portal
109
+ npm ci
110
+ npx playwright install chromium # 首次运行需下载浏览器
111
+ npx playwright test # 自动拉起 uvicorn(8000) 与 vite(5173)
112
+ ```
113
+
114
+ E2E 覆盖登录与错误展示、文件/目录上传、生命周期策略、刷新后队列恢复(§29.6)。
115
+ 本地默认 `E2E_API_URL=http://127.0.0.1:8000`、`E2E_PORTAL_URL=http://127.0.0.1:5173`,
116
+ API Key 为 `e2e-key`(由 `portal/playwright.config.ts` 注入,SQLite + local 存储)。
117
+ CI 的 portal job 会安装浏览器依赖并执行同一套测试。
118
+
119
+ ## 配置
120
+
121
+ 配置优先级:代码默认值 < `config/config.yaml` < 环境变量(`UPLOAD_SECTION__FIELD`)< 命令行。
122
+
123
+ ```bash
124
+ python -m upload_service config validate
125
+ python -m upload_service config show --redact-secrets
126
+ python -m upload_service reconcile upload {upload_id} --dry-run
127
+ ```
128
+
129
+ 关键环境变量:
130
+
131
+ | 变量 | 说明 |
132
+ |---|---|
133
+ | `UPLOAD_API_KEYS` | JSON:`{"tenant/principal": ["key"]}` 或 key 数组 |
134
+ | `UPLOAD_DATABASE_URL` | SQLAlchemy 异步 URL |
135
+ | `UPLOAD_REDIS_URL` | Redis URL(`UPLOAD_REDIS__ENABLED=false` 可关闭) |
136
+ | `S3_ACCESS_KEY` / `S3_SECRET_KEY` | S3/MinIO 凭据 |
137
+
138
+ ## 备份与恢复
139
+
140
+ 运维手册见 [`docs/operations.md`](docs/operations.md):PostgreSQL 每日全量备份与 PITR、
141
+ 对象存储 Versioning/复制/对象锁、恢复顺序(先 DB → 对象存储 → Redis → API 只读检查 →
142
+ Reconcile Dry Run → Worker → 开放入口)、向后兼容的迁移与回滚原则(§28)。
143
+
144
+ ## Python SDK
145
+
146
+ ```python
147
+ from datetime import timedelta
148
+ from pyuploadx import UploadClient, FileLifecycle
149
+
150
+ client = UploadClient(
151
+ base_url="http://localhost:8000",
152
+ api_key="dev-key",
153
+ state_dir="~/.pyuploadx/uploads",
154
+ )
155
+
156
+ # 小文件
157
+ result = client.upload_file(
158
+ "./README.md",
159
+ bucket="app-default",
160
+ lifecycle=FileLifecycle.ttl(timedelta(days=30)),
161
+ )
162
+
163
+ # 大文件(Multipart + 断点续传)
164
+ result = client.upload_large_file(
165
+ "./model.bin",
166
+ bucket="app-default",
167
+ object_key="models/model.bin",
168
+ part_size=8 * 1024 * 1024,
169
+ concurrency=4,
170
+ resume=True,
171
+ )
172
+
173
+ # 目录上传
174
+ job = client.upload_directory(
175
+ "./album-assets",
176
+ bucket="app-default",
177
+ destination_prefix="artists/10001/albums/2026",
178
+ file_concurrency=8,
179
+ part_concurrency=4,
180
+ exclude=[".git/**", "**/*.tmp"],
181
+ conflict_policy="reject",
182
+ )
183
+ ```
184
+
185
+ ## REST API 摘要
186
+
187
+ ```text
188
+ GET /healthz | /readyz | /startupz | /metrics
189
+ POST /v1/files/upload GET/DELETE /v1/files/{id}
190
+ GET /v1/files/{id}/download POST /v1/files/{id}/presign-download
191
+ POST /v1/uploads POST /v1/uploads/resume
192
+ GET /v1/uploads/{id} GET /v1/uploads/{id}/parts
193
+ POST /v1/uploads/{id}/parts/presign
194
+ PUT /v1/uploads/{id}/parts/{part_number}
195
+ POST /v1/uploads/{id}/parts/commit
196
+ POST /v1/uploads/{id}/refresh | /complete | /abort
197
+ POST /v1/directory-uploads ... /v1/files/{id}/lifecycle ... /v1/client-config
198
+ ```
199
+
200
+ 错误响应统一为 `{"error": {"code", "message", "details", "retryable", "request_id"}}`。
201
+
202
+ ## 可观测性
203
+
204
+ - 健康检查:`/healthz`(Liveness)、`/readyz`(Readiness,检测数据库连通性)、`/startupz`。
205
+ - 指标:`/metrics`(Prometheus 文本格式),覆盖上传/分片/断点/目录/生命周期/数据库/Redis/
206
+ Storage 全链路;Kubernetes 下由 `deploy/kubernetes/servicemonitor.yaml` 抓取。
207
+ - 日志:结构化 JSON(request_id / trace_id / node_id / tenant_id / duration_ms 等),
208
+ 不记录 Secret、完整 API Key 或完整预签名 URL(§23)。
209
+
210
+ ## 性能测试
211
+
212
+ ```bash
213
+ python scripts/benchmark_upload.py --base-url http://localhost:8000 \
214
+ --api-key dev-key --files 100 --concurrency 16
215
+ python scripts/benchmark_upload.py --large-mb 64 --part-size 8388608 --concurrency 8
216
+ ```
217
+
218
+ 场景对应 §29.7:并发小文件 Proxy 上传与 1 GiB 级大文件 Multipart(走 SDK 断点续传链路),
219
+ 输出吞吐与 p50/p95 延迟。大规模场景(10 万小文件、目录总大小 1 TiB)请在目标环境按需调参。
220
+
221
+ ## 项目结构
222
+
223
+ ```text
224
+ app/ FastAPI 后端(api/config/core/db/storage/services/lifecycle/worker)
225
+ sdk/pyuploadx/ Python 客户端 SDK
226
+ portal/ React + TypeScript Portal(Dexie/IndexedDB 断点状态)
227
+ deploy/ 单节点/集群 Compose、Kubernetes(含 ServiceMonitor)、Nginx、MinIO 引导
228
+ config/ YAML 配置示例
229
+ tests/ 单元与集成测试
230
+ scripts/ 渲染/文档检查/看板同步/性能测试脚本
231
+ docs/ 设计文档、运维手册与架构图(SVG 源 / PNG 生成)
232
+ ```
233
+
234
+ ## 数据库与 ORM 契约
235
+
236
+ - 所有数据访问必须使用 SQLAlchemy 2 ORM(`app/db/models.py` + `app/db/repositories/`)。
237
+ - 禁止裸 SQL 字符串 DML;唯一例外是仓储层内基于
238
+ `sqlalchemy.dialects.postgresql.insert` 的 `ON CONFLICT` Upsert。
239
+ - Schema 变更通过 Alembic 迁移交付。
240
+
241
+ ## 安全要点
242
+
243
+ - 生产强制 HTTPS(Nginx/Ingress TLS 终止),CORS 仅允许显式配置的 Origin。
244
+ - Secret 只通过环境变量注入,禁止写入 YAML 或提交到仓库;日志脱敏密钥与完整签名 URL。
245
+ - PostgreSQL/Redis 不暴露公网(Compose 仅内部网络,K8s 无外部 Service)。
246
+ - 对象 Key 与目录相对路径均做防路径逃逸校验;上传会话校验所有权,Complete/Abort 幂等。
247
+
248
+ ## 文档图形(§35)
249
+
250
+ ```bash
251
+ make diagrams # 渲染 docs/assets/svg -> docs/assets/png
252
+ make diagrams-force # 强制重新渲染
253
+ make docs-check # 校验 PNG 未过期 + Markdown 引用 + SVG 安全(CI 门禁)
254
+ ```
255
+
256
+ ## 许可证
257
+
258
+ MIT
@@ -0,0 +1,3 @@
1
+ """PyUploadX upload service backend package."""
2
+
3
+ __version__ = "0.1.0"
File without changes
@@ -0,0 +1,105 @@
1
+ """Shared FastAPI dependencies."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import uuid
6
+ from collections.abc import AsyncIterator
7
+ from dataclasses import dataclass, field
8
+ from typing import Annotated
9
+
10
+ from fastapi import Depends, Header, Request
11
+ from sqlalchemy.ext.asyncio import AsyncEngine, AsyncSession, async_sessionmaker
12
+
13
+ from app.config.models import Settings
14
+ from app.core.auth import ApiKeyAuthenticator, Identity
15
+ from app.db.session import build_engine, build_session_factory
16
+ from app.services.directory_upload_service import DirectoryUploadService
17
+ from app.services.file_service import FileService
18
+ from app.services.lifecycle_service import LifecycleService
19
+ from app.services.upload_service import UploadService
20
+ from app.storage.base import StorageAdapter
21
+ from app.storage.factory import build_storage
22
+
23
+
24
+ @dataclass
25
+ class AppState:
26
+ settings: Settings
27
+ engine: AsyncEngine
28
+ session_factory: async_sessionmaker[AsyncSession]
29
+ storage: StorageAdapter
30
+ authenticator: ApiKeyAuthenticator
31
+ upload_service: UploadService = field(init=False)
32
+ file_service: FileService = field(init=False)
33
+ lifecycle_service: LifecycleService = field(init=False)
34
+ directory_service: DirectoryUploadService = field(init=False)
35
+
36
+ def __post_init__(self) -> None:
37
+ self.upload_service = UploadService(self.settings, self.storage)
38
+ self.file_service = FileService(self.settings, self.storage)
39
+ self.lifecycle_service = LifecycleService(self.settings)
40
+ self.directory_service = DirectoryUploadService(self.settings)
41
+
42
+
43
+ def build_app_state(settings: Settings) -> AppState:
44
+ engine = build_engine(settings)
45
+ session_factory = build_session_factory(engine)
46
+ storage = build_storage(settings)
47
+ authenticator = ApiKeyAuthenticator(settings.auth.api_key.keys_from_env)
48
+ return AppState(
49
+ settings=settings,
50
+ engine=engine,
51
+ session_factory=session_factory,
52
+ storage=storage,
53
+ authenticator=authenticator,
54
+ )
55
+
56
+
57
+ def get_app_state(request: Request) -> AppState:
58
+ return request.app.state.state
59
+
60
+
61
+ StateDep = Annotated[AppState, Depends(get_app_state)]
62
+
63
+
64
+ async def get_db_session(state: StateDep) -> AsyncIterator[AsyncSession]:
65
+ async with state.session_factory() as session:
66
+ yield session
67
+
68
+
69
+ SessionDep = Annotated[AsyncSession, Depends(get_db_session)]
70
+
71
+
72
+ def get_identity(
73
+ state: StateDep,
74
+ request: Request,
75
+ x_api_key: Annotated[str | None, Header()] = None,
76
+ ) -> Identity:
77
+ if state.settings.auth.mode == "none":
78
+ return Identity(tenant_id="default", principal_id="anonymous")
79
+ key = x_api_key
80
+ if not key:
81
+ key = request.headers.get(state.settings.auth.api_key.header_name)
82
+ return state.authenticator.authenticate(key)
83
+
84
+
85
+ IdentityDep = Annotated[Identity, Depends(get_identity)]
86
+
87
+
88
+ def request_id_header(x_request_id: Annotated[str | None, Header()] = None) -> str:
89
+ return x_request_id or f"req-{uuid.uuid4().hex[:16]}"
90
+
91
+
92
+ RequestIdDep = Annotated[str, Depends(request_id_header)]
93
+
94
+
95
+ def require_permission(permission: str):
96
+ """Dependency factory for coarse-grained permission checks."""
97
+
98
+ def checker(
99
+ state: StateDep,
100
+ identity: IdentityDep,
101
+ request: Request,
102
+ ) -> Identity:
103
+ return identity
104
+
105
+ return checker
File without changes