pyuploadx-server 0.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (72) hide show
  1. pyuploadx_server-0.1.0/LICENSE +21 -0
  2. pyuploadx_server-0.1.0/PKG-INFO +309 -0
  3. pyuploadx_server-0.1.0/README.md +269 -0
  4. pyuploadx_server-0.1.0/app/__init__.py +3 -0
  5. pyuploadx_server-0.1.0/app/api/__init__.py +0 -0
  6. pyuploadx_server-0.1.0/app/api/dependencies.py +105 -0
  7. pyuploadx_server-0.1.0/app/api/v1/__init__.py +0 -0
  8. pyuploadx_server-0.1.0/app/api/v1/client_config.py +78 -0
  9. pyuploadx_server-0.1.0/app/api/v1/directory_uploads.py +241 -0
  10. pyuploadx_server-0.1.0/app/api/v1/files.py +125 -0
  11. pyuploadx_server-0.1.0/app/api/v1/health.py +47 -0
  12. pyuploadx_server-0.1.0/app/api/v1/lifecycle.py +86 -0
  13. pyuploadx_server-0.1.0/app/api/v1/presign.py +49 -0
  14. pyuploadx_server-0.1.0/app/api/v1/uploads.py +200 -0
  15. pyuploadx_server-0.1.0/app/cli.py +95 -0
  16. pyuploadx_server-0.1.0/app/config/__init__.py +4 -0
  17. pyuploadx_server-0.1.0/app/config/loader.py +93 -0
  18. pyuploadx_server-0.1.0/app/config/models.py +300 -0
  19. pyuploadx_server-0.1.0/app/config/validation.py +70 -0
  20. pyuploadx_server-0.1.0/app/core/__init__.py +0 -0
  21. pyuploadx_server-0.1.0/app/core/auth.py +56 -0
  22. pyuploadx_server-0.1.0/app/core/errors.py +253 -0
  23. pyuploadx_server-0.1.0/app/core/idempotency.py +74 -0
  24. pyuploadx_server-0.1.0/app/core/logging.py +59 -0
  25. pyuploadx_server-0.1.0/app/core/metrics.py +109 -0
  26. pyuploadx_server-0.1.0/app/core/streaming.py +39 -0
  27. pyuploadx_server-0.1.0/app/core/tracing.py +18 -0
  28. pyuploadx_server-0.1.0/app/db/__init__.py +0 -0
  29. pyuploadx_server-0.1.0/app/db/models.py +361 -0
  30. pyuploadx_server-0.1.0/app/db/repositories/__init__.py +8 -0
  31. pyuploadx_server-0.1.0/app/db/repositories/directory_repository.py +55 -0
  32. pyuploadx_server-0.1.0/app/db/repositories/file_repository.py +40 -0
  33. pyuploadx_server-0.1.0/app/db/repositories/part_repository.py +80 -0
  34. pyuploadx_server-0.1.0/app/db/repositories/upload_repository.py +47 -0
  35. pyuploadx_server-0.1.0/app/db/session.py +59 -0
  36. pyuploadx_server-0.1.0/app/directory_upload/__init__.py +3 -0
  37. pyuploadx_server-0.1.0/app/directory_upload/aggregation.py +38 -0
  38. pyuploadx_server-0.1.0/app/directory_upload/manifest.py +66 -0
  39. pyuploadx_server-0.1.0/app/directory_upload/paths.py +57 -0
  40. pyuploadx_server-0.1.0/app/directory_upload/state_machine.py +39 -0
  41. pyuploadx_server-0.1.0/app/lifecycle/__init__.py +3 -0
  42. pyuploadx_server-0.1.0/app/lifecycle/policy.py +108 -0
  43. pyuploadx_server-0.1.0/app/lifecycle/state_machine.py +51 -0
  44. pyuploadx_server-0.1.0/app/main.py +104 -0
  45. pyuploadx_server-0.1.0/app/services/__init__.py +0 -0
  46. pyuploadx_server-0.1.0/app/services/cleanup_service.py +68 -0
  47. pyuploadx_server-0.1.0/app/services/directory_upload_service.py +424 -0
  48. pyuploadx_server-0.1.0/app/services/file_service.py +242 -0
  49. pyuploadx_server-0.1.0/app/services/lifecycle_service.py +206 -0
  50. pyuploadx_server-0.1.0/app/services/reconcile_service.py +56 -0
  51. pyuploadx_server-0.1.0/app/services/upload_service.py +575 -0
  52. pyuploadx_server-0.1.0/app/services/webhook_service.py +80 -0
  53. pyuploadx_server-0.1.0/app/storage/__init__.py +4 -0
  54. pyuploadx_server-0.1.0/app/storage/base.py +142 -0
  55. pyuploadx_server-0.1.0/app/storage/capabilities.py +18 -0
  56. pyuploadx_server-0.1.0/app/storage/factory.py +14 -0
  57. pyuploadx_server-0.1.0/app/storage/local.py +280 -0
  58. pyuploadx_server-0.1.0/app/storage/s3.py +347 -0
  59. pyuploadx_server-0.1.0/app/worker/__init__.py +0 -0
  60. pyuploadx_server-0.1.0/app/worker/cleanup.py +34 -0
  61. pyuploadx_server-0.1.0/app/worker/lifecycle.py +123 -0
  62. pyuploadx_server-0.1.0/app/worker/main.py +79 -0
  63. pyuploadx_server-0.1.0/pyproject.toml +72 -0
  64. pyuploadx_server-0.1.0/pyuploadx_server.egg-info/PKG-INFO +309 -0
  65. pyuploadx_server-0.1.0/pyuploadx_server.egg-info/SOURCES.txt +70 -0
  66. pyuploadx_server-0.1.0/pyuploadx_server.egg-info/dependency_links.txt +1 -0
  67. pyuploadx_server-0.1.0/pyuploadx_server.egg-info/entry_points.txt +2 -0
  68. pyuploadx_server-0.1.0/pyuploadx_server.egg-info/requires.txt +23 -0
  69. pyuploadx_server-0.1.0/pyuploadx_server.egg-info/top_level.txt +2 -0
  70. pyuploadx_server-0.1.0/setup.cfg +4 -0
  71. pyuploadx_server-0.1.0/upload_service/__init__.py +5 -0
  72. pyuploadx_server-0.1.0/upload_service/__main__.py +3 -0
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 shark8848
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,309 @@
1
+ Metadata-Version: 2.4
2
+ Name: pyuploadx-server
3
+ Version: 0.1.0
4
+ Summary: PyUploadX file and directory upload service (FastAPI backend)
5
+ Author: PyUploadX Contributors
6
+ License: MIT
7
+ Keywords: upload,fastapi,multipart,resume,directory-upload,s3,minio
8
+ Classifier: Development Status :: 3 - Alpha
9
+ Classifier: Intended Audience :: System Administrators
10
+ Classifier: Programming Language :: Python :: 3
11
+ Classifier: Programming Language :: Python :: 3.11
12
+ Classifier: Programming Language :: Python :: 3.12
13
+ Classifier: Framework :: FastAPI
14
+ Classifier: Topic :: Internet :: WWW/HTTP :: HTTP Servers
15
+ Requires-Python: >=3.11
16
+ Description-Content-Type: text/markdown
17
+ License-File: LICENSE
18
+ Requires-Dist: fastapi<1,>=0.110
19
+ Requires-Dist: uvicorn[standard]<1,>=0.29
20
+ Requires-Dist: pydantic<3,>=2.6
21
+ Requires-Dist: sqlalchemy[asyncio]<3,>=2.0
22
+ Requires-Dist: asyncpg<1,>=0.29
23
+ Requires-Dist: aiosqlite<1,>=0.20
24
+ Requires-Dist: alembic<2,>=1.13
25
+ Requires-Dist: redis<6,>=5.0
26
+ Requires-Dist: boto3<2,>=1.34
27
+ Requires-Dist: aiofiles<25,>=23.2
28
+ Requires-Dist: httpx<1,>=0.27
29
+ Requires-Dist: PyYAML<7,>=6.0
30
+ Requires-Dist: prometheus-client<1,>=0.20
31
+ Requires-Dist: python-multipart<1,>=0.0.9
32
+ Provides-Extra: docs
33
+ Requires-Dist: CairoSVG<3,>=2.7; extra == "docs"
34
+ Provides-Extra: dev
35
+ Requires-Dist: pytest<9,>=8; extra == "dev"
36
+ Requires-Dist: pytest-asyncio<1,>=0.23; extra == "dev"
37
+ Requires-Dist: ruff<1,>=0.4; extra == "dev"
38
+ Requires-Dist: types-PyYAML<7,>=6.0; extra == "dev"
39
+ Dynamic: license-file
40
+
41
+ # PyUploadX 文件与目录上传服务
42
+
43
+ PyUploadX 是一个轻量、可独立部署、可水平扩展的文件与目录上传服务,提供统一的
44
+ 文件上传、断点续传、目录上传、生命周期管理能力。详细设计与实现约束见
45
+ [产品设计说明书](docs/docs_product-design.md)(§31 目录结构、§16 REST API、§17 SDK)。
46
+
47
+ ## 功能特性
48
+
49
+ - **上传模式**:小文件 Proxy 上传、大文件 Multipart(分片 + 预签名)、Local/S3/MinIO 存储适配
50
+ - **断点续传**:SDK 与 Portal 均支持指纹校验、状态恢复、缺失分片重传、URL 刷新
51
+ - **目录上传**:Manifest + NDJSON、路径安全校验、`.uploadignore`、冲突策略、双层并发
52
+ - **生命周期**:`permanent` / `ttl` / `expires_at` / `temporary` / `sliding_ttl`,Legal Hold、Webhook
53
+ - **集群一致性**:无状态 API 节点、PostgreSQL 行锁、Part Upsert、Complete/Abort 幂等
54
+ - **可观测性**:结构化 JSON 日志、Prometheus 指标、`/healthz` `/readyz` `/startupz`
55
+
56
+ ## 技术栈
57
+
58
+ FastAPI · SQLAlchemy 2(ORM)· Alembic · PostgreSQL · Redis · boto3 · Python SDK(httpx)·
59
+ React + TypeScript + Vite + Dexie · Docker Compose · Kubernetes · pytest
60
+
61
+ ## 快速开始(Docker Compose)
62
+
63
+ PostgreSQL / Redis / MinIO 等第三方组件**不随应用镜像构建**:默认通过环境变量指向
64
+ 本地或已有实例(默认地址 `host.docker.internal`,即宿主机;端口与凭据均可覆盖)。
65
+
66
+ 方式 A:使用本地/已有 PostgreSQL、Redis、MinIO
67
+
68
+ ```bash
69
+ # 可选:覆盖默认地址与凭据
70
+ export UPLOAD_DATABASE_URL='postgresql+asyncpg://upload:upload@localhost:5432/uploads'
71
+ export UPLOAD_REDIS_URL='redis://localhost:6379/0'
72
+ export UPLOAD_STORAGE__S3__INTERNAL_ENDPOINT_URL='http://localhost:9000'
73
+ export S3_ACCESS_KEY=minioadmin S3_SECRET_KEY=minioadmin
74
+ docker compose up -d --build # migrate → upload-api → worker → portal
75
+ ```
76
+
77
+ 方式 B:自带第三方组件(`deploy/infra/compose.yaml`,端口可用 `POSTGRES_PORT`、
78
+ `REDIS_PORT`、`MINIO_PORT` 覆盖)
79
+
80
+ ```bash
81
+ docker compose -f deploy/infra/compose.yaml up -d # postgres/redis/minio + 建桶
82
+ docker compose up -d --build # 应用服务
83
+ ```
84
+
85
+ 单节点/集群一键(含组件):
86
+
87
+ ```bash
88
+ docker compose -f deploy/single-node/compose.yaml up -d --build
89
+ docker compose -f deploy/cluster/compose.yaml up -d --build --scale upload-api=3 --scale worker=2
90
+ ```
91
+
92
+ 启动后:
93
+
94
+ - API/OpenAPI:http://localhost:8000/docs
95
+ - Portal:http://localhost:5173(API Key:`dev-key`)
96
+ - MinIO Console:http://localhost:9001(`minioadmin` / `minioadmin`)
97
+
98
+ ## Kubernetes 部署
99
+
100
+ 模板位于 `deploy/kubernetes/`(Namespace / Deployment / Service / Ingress / HPA / PDB /
101
+ Secret / ConfigMap / ServiceMonitor)。API 默认 3 副本、`maxUnavailable: 0`、优雅终止 60s,
102
+ 并通过 `/readyz` 探针摘除故障节点。ServiceMonitor(`servicemonitor.yaml`)可被
103
+ Prometheus Operator 直接发现:
104
+
105
+ ```bash
106
+ kubectl apply -f deploy/kubernetes/
107
+ ```
108
+
109
+ 生产强制 HTTPS:Ingress 终止 TLS;单节点/集群部署由 `deploy/nginx/gateway.conf` 强制跳转
110
+ HTTPS 并拒绝非白名单 Origin 的 CORS 请求。
111
+
112
+ ## 本地开发
113
+
114
+ ```bash
115
+ python -m venv .venv && source .venv/bin/activate
116
+ pip install -e ".[dev,docs]"
117
+ python -m pytest tests -q # 全量测试
118
+ ruff check app sdk upload_service tests scripts
119
+ uvicorn app.main:create_app --factory --reload --host 0.0.0.0 --port 8000
120
+ ```
121
+
122
+ 测试默认使用 SQLite(aiosqlite);CI 使用 PostgreSQL。集群模式禁止 SQLite(`config validate` 会拒绝)。
123
+
124
+ ### 数据库迁移(Alembic)
125
+
126
+ ```bash
127
+ alembic upgrade head # 升级到最新 schema
128
+ alembic downgrade -1 # 回退一个版本(仅限向后兼容的迁移)
129
+ make migrate # 等价于 alembic upgrade head
130
+ ```
131
+
132
+ Schema 变更一律通过 Alembic 迁移交付,禁止用临时脚本改表(见 AGENTS.md Architecture Contracts)。
133
+
134
+ ### MinIO / S3 Adapter 测试
135
+
136
+ ```bash
137
+ UPLOAD_MINIO_TEST=1 \
138
+ UPLOAD_STORAGE__S3__INTERNAL_ENDPOINT_URL=http://localhost:9000 \
139
+ S3_ACCESS_KEY=minioadmin S3_SECRET_KEY=minioadmin \
140
+ python -m pytest tests/integration/test_s3_storage.py -q
141
+ ```
142
+
143
+ 未设置 `UPLOAD_MINIO_TEST=1` 时该套件自动跳过;CI 内置 MinIO 服务并始终运行它。
144
+
145
+ ### Portal E2E(Playwright)
146
+
147
+ ```bash
148
+ cd portal
149
+ npm ci
150
+ npx playwright install chromium # 首次运行需下载浏览器
151
+ npx playwright test # 自动拉起 uvicorn(8000) 与 vite(5173)
152
+ ```
153
+
154
+ E2E 覆盖登录与错误展示、文件/目录上传、生命周期策略、刷新后队列恢复(§29.6)。
155
+ 本地默认 `E2E_API_URL=http://127.0.0.1:8000`、`E2E_PORTAL_URL=http://127.0.0.1:5173`,
156
+ API Key 为 `e2e-key`(由 `portal/playwright.config.ts` 注入,SQLite + local 存储)。
157
+ CI 的 portal job 会安装浏览器依赖并执行同一套测试。
158
+
159
+ ## 配置
160
+
161
+ 配置优先级:代码默认值 < `config/config.yaml` < 环境变量(`UPLOAD_SECTION__FIELD`)< 命令行。
162
+
163
+ ```bash
164
+ python -m upload_service config validate
165
+ python -m upload_service config show --redact-secrets
166
+ python -m upload_service reconcile upload {upload_id} --dry-run
167
+ ```
168
+
169
+ 关键环境变量:
170
+
171
+ | 变量 | 说明 |
172
+ |---|---|
173
+ | `UPLOAD_API_KEYS` | JSON:`{"tenant/principal": ["key"]}` 或 key 数组 |
174
+ | `UPLOAD_DATABASE_URL` | SQLAlchemy 异步 URL |
175
+ | `UPLOAD_REDIS_URL` | Redis URL(`UPLOAD_REDIS__ENABLED=false` 可关闭) |
176
+ | `S3_ACCESS_KEY` / `S3_SECRET_KEY` | S3/MinIO 凭据 |
177
+
178
+ ## 备份与恢复
179
+
180
+ 运维手册见 [`docs/operations.md`](docs/operations.md):PostgreSQL 每日全量备份与 PITR、
181
+ 对象存储 Versioning/复制/对象锁、恢复顺序(先 DB → 对象存储 → Redis → API 只读检查 →
182
+ Reconcile Dry Run → Worker → 开放入口)、向后兼容的迁移与回滚原则(§28)。
183
+
184
+ ## Python SDK
185
+
186
+ ```python
187
+ from datetime import timedelta
188
+ from pyuploadx import UploadClient, FileLifecycle
189
+
190
+ client = UploadClient(
191
+ base_url="http://localhost:8000",
192
+ api_key="dev-key",
193
+ state_dir="~/.pyuploadx/uploads",
194
+ )
195
+
196
+ # 小文件
197
+ result = client.upload_file(
198
+ "./README.md",
199
+ bucket="app-default",
200
+ lifecycle=FileLifecycle.ttl(timedelta(days=30)),
201
+ )
202
+
203
+ # 大文件(Multipart + 断点续传)
204
+ result = client.upload_large_file(
205
+ "./model.bin",
206
+ bucket="app-default",
207
+ object_key="models/model.bin",
208
+ part_size=8 * 1024 * 1024,
209
+ concurrency=4,
210
+ resume=True,
211
+ )
212
+
213
+ # 目录上传
214
+ job = client.upload_directory(
215
+ "./album-assets",
216
+ bucket="app-default",
217
+ destination_prefix="artists/10001/albums/2026",
218
+ file_concurrency=8,
219
+ part_concurrency=4,
220
+ exclude=[".git/**", "**/*.tmp"],
221
+ conflict_policy="reject",
222
+ )
223
+ ```
224
+
225
+ ### 安装
226
+
227
+ ```bash
228
+ pip install pyuploadx # 官方 PyPI:Python SDK(Python ≥ 3.11,第三方依赖仅 httpx)
229
+ pip install pyuploadx-server # 官方 PyPI:服务端(FastAPI,Docker 部署时通常无需 pip 安装)
230
+ pip install dist/pyuploadx-0.2.0-py3-none-any.whl # 或仓库直装(保留历史版本,见 dist/README.md)
231
+ ```
232
+
233
+ SDK 与服务端为两个独立发布包(`pyuploadx` / `pyuploadx-server`);发版与版本管理见
234
+ [docs/docs_product-design.md §37](docs/docs_product-design.md)。
235
+
236
+ ## REST API 摘要
237
+
238
+ ```text
239
+ GET /healthz | /readyz | /startupz | /metrics
240
+ POST /v1/files/upload GET/DELETE /v1/files/{id}
241
+ GET /v1/files/{id}/download POST /v1/files/{id}/presign-download
242
+ POST /v1/uploads POST /v1/uploads/resume
243
+ GET /v1/uploads/{id} GET /v1/uploads/{id}/parts
244
+ POST /v1/uploads/{id}/parts/presign
245
+ PUT /v1/uploads/{id}/parts/{part_number}
246
+ POST /v1/uploads/{id}/parts/commit
247
+ POST /v1/uploads/{id}/refresh | /complete | /abort
248
+ POST /v1/directory-uploads ... /v1/files/{id}/lifecycle ... /v1/client-config
249
+ ```
250
+
251
+ 错误响应统一为 `{"error": {"code", "message", "details", "retryable", "request_id"}}`。
252
+
253
+ ## 可观测性
254
+
255
+ - 健康检查:`/healthz`(Liveness)、`/readyz`(Readiness,检测数据库连通性)、`/startupz`。
256
+ - 指标:`/metrics`(Prometheus 文本格式),覆盖上传/分片/断点/目录/生命周期/数据库/Redis/
257
+ Storage 全链路;Kubernetes 下由 `deploy/kubernetes/servicemonitor.yaml` 抓取。
258
+ - 日志:结构化 JSON(request_id / trace_id / node_id / tenant_id / duration_ms 等),
259
+ 不记录 Secret、完整 API Key 或完整预签名 URL(§23)。
260
+
261
+ ## 性能测试
262
+
263
+ ```bash
264
+ python scripts/benchmark_upload.py --base-url http://localhost:8000 \
265
+ --api-key dev-key --files 100 --concurrency 16
266
+ python scripts/benchmark_upload.py --large-mb 64 --part-size 8388608 --concurrency 8
267
+ ```
268
+
269
+ 场景对应 §29.7:并发小文件 Proxy 上传与 1 GiB 级大文件 Multipart(走 SDK 断点续传链路),
270
+ 输出吞吐与 p50/p95 延迟。大规模场景(10 万小文件、目录总大小 1 TiB)请在目标环境按需调参。
271
+
272
+ ## 项目结构
273
+
274
+ ```text
275
+ app/ FastAPI 后端(api/config/core/db/storage/services/lifecycle/worker)
276
+ sdk/pyuploadx/ Python 客户端 SDK
277
+ portal/ React + TypeScript Portal(Dexie/IndexedDB 断点状态)
278
+ deploy/ 单节点/集群 Compose、Kubernetes(含 ServiceMonitor)、Nginx、MinIO 引导
279
+ config/ YAML 配置示例
280
+ tests/ 单元与集成测试
281
+ scripts/ 渲染/文档检查/看板同步/性能测试脚本
282
+ docs/ 设计文档、运维手册与架构图(SVG 源 / PNG 生成)
283
+ ```
284
+
285
+ ## 数据库与 ORM 契约
286
+
287
+ - 所有数据访问必须使用 SQLAlchemy 2 ORM(`app/db/models.py` + `app/db/repositories/`)。
288
+ - 禁止裸 SQL 字符串 DML;唯一例外是仓储层内基于
289
+ `sqlalchemy.dialects.postgresql.insert` 的 `ON CONFLICT` Upsert。
290
+ - Schema 变更通过 Alembic 迁移交付。
291
+
292
+ ## 安全要点
293
+
294
+ - 生产强制 HTTPS(Nginx/Ingress TLS 终止),CORS 仅允许显式配置的 Origin。
295
+ - Secret 只通过环境变量注入,禁止写入 YAML 或提交到仓库;日志脱敏密钥与完整签名 URL。
296
+ - PostgreSQL/Redis 不暴露公网(Compose 仅内部网络,K8s 无外部 Service)。
297
+ - 对象 Key 与目录相对路径均做防路径逃逸校验;上传会话校验所有权,Complete/Abort 幂等。
298
+
299
+ ## 文档图形(§35)
300
+
301
+ ```bash
302
+ make diagrams # 渲染 docs/assets/svg -> docs/assets/png
303
+ make diagrams-force # 强制重新渲染
304
+ make docs-check # 校验 PNG 未过期 + Markdown 引用 + SVG 安全(CI 门禁)
305
+ ```
306
+
307
+ ## 许可证
308
+
309
+ MIT
@@ -0,0 +1,269 @@
1
+ # PyUploadX 文件与目录上传服务
2
+
3
+ PyUploadX 是一个轻量、可独立部署、可水平扩展的文件与目录上传服务,提供统一的
4
+ 文件上传、断点续传、目录上传、生命周期管理能力。详细设计与实现约束见
5
+ [产品设计说明书](docs/docs_product-design.md)(§31 目录结构、§16 REST API、§17 SDK)。
6
+
7
+ ## 功能特性
8
+
9
+ - **上传模式**:小文件 Proxy 上传、大文件 Multipart(分片 + 预签名)、Local/S3/MinIO 存储适配
10
+ - **断点续传**:SDK 与 Portal 均支持指纹校验、状态恢复、缺失分片重传、URL 刷新
11
+ - **目录上传**:Manifest + NDJSON、路径安全校验、`.uploadignore`、冲突策略、双层并发
12
+ - **生命周期**:`permanent` / `ttl` / `expires_at` / `temporary` / `sliding_ttl`,Legal Hold、Webhook
13
+ - **集群一致性**:无状态 API 节点、PostgreSQL 行锁、Part Upsert、Complete/Abort 幂等
14
+ - **可观测性**:结构化 JSON 日志、Prometheus 指标、`/healthz` `/readyz` `/startupz`
15
+
16
+ ## 技术栈
17
+
18
+ FastAPI · SQLAlchemy 2(ORM)· Alembic · PostgreSQL · Redis · boto3 · Python SDK(httpx)·
19
+ React + TypeScript + Vite + Dexie · Docker Compose · Kubernetes · pytest
20
+
21
+ ## 快速开始(Docker Compose)
22
+
23
+ PostgreSQL / Redis / MinIO 等第三方组件**不随应用镜像构建**:默认通过环境变量指向
24
+ 本地或已有实例(默认地址 `host.docker.internal`,即宿主机;端口与凭据均可覆盖)。
25
+
26
+ 方式 A:使用本地/已有 PostgreSQL、Redis、MinIO
27
+
28
+ ```bash
29
+ # 可选:覆盖默认地址与凭据
30
+ export UPLOAD_DATABASE_URL='postgresql+asyncpg://upload:upload@localhost:5432/uploads'
31
+ export UPLOAD_REDIS_URL='redis://localhost:6379/0'
32
+ export UPLOAD_STORAGE__S3__INTERNAL_ENDPOINT_URL='http://localhost:9000'
33
+ export S3_ACCESS_KEY=minioadmin S3_SECRET_KEY=minioadmin
34
+ docker compose up -d --build # migrate → upload-api → worker → portal
35
+ ```
36
+
37
+ 方式 B:自带第三方组件(`deploy/infra/compose.yaml`,端口可用 `POSTGRES_PORT`、
38
+ `REDIS_PORT`、`MINIO_PORT` 覆盖)
39
+
40
+ ```bash
41
+ docker compose -f deploy/infra/compose.yaml up -d # postgres/redis/minio + 建桶
42
+ docker compose up -d --build # 应用服务
43
+ ```
44
+
45
+ 单节点/集群一键(含组件):
46
+
47
+ ```bash
48
+ docker compose -f deploy/single-node/compose.yaml up -d --build
49
+ docker compose -f deploy/cluster/compose.yaml up -d --build --scale upload-api=3 --scale worker=2
50
+ ```
51
+
52
+ 启动后:
53
+
54
+ - API/OpenAPI:http://localhost:8000/docs
55
+ - Portal:http://localhost:5173(API Key:`dev-key`)
56
+ - MinIO Console:http://localhost:9001(`minioadmin` / `minioadmin`)
57
+
58
+ ## Kubernetes 部署
59
+
60
+ 模板位于 `deploy/kubernetes/`(Namespace / Deployment / Service / Ingress / HPA / PDB /
61
+ Secret / ConfigMap / ServiceMonitor)。API 默认 3 副本、`maxUnavailable: 0`、优雅终止 60s,
62
+ 并通过 `/readyz` 探针摘除故障节点。ServiceMonitor(`servicemonitor.yaml`)可被
63
+ Prometheus Operator 直接发现:
64
+
65
+ ```bash
66
+ kubectl apply -f deploy/kubernetes/
67
+ ```
68
+
69
+ 生产强制 HTTPS:Ingress 终止 TLS;单节点/集群部署由 `deploy/nginx/gateway.conf` 强制跳转
70
+ HTTPS 并拒绝非白名单 Origin 的 CORS 请求。
71
+
72
+ ## 本地开发
73
+
74
+ ```bash
75
+ python -m venv .venv && source .venv/bin/activate
76
+ pip install -e ".[dev,docs]"
77
+ python -m pytest tests -q # 全量测试
78
+ ruff check app sdk upload_service tests scripts
79
+ uvicorn app.main:create_app --factory --reload --host 0.0.0.0 --port 8000
80
+ ```
81
+
82
+ 测试默认使用 SQLite(aiosqlite);CI 使用 PostgreSQL。集群模式禁止 SQLite(`config validate` 会拒绝)。
83
+
84
+ ### 数据库迁移(Alembic)
85
+
86
+ ```bash
87
+ alembic upgrade head # 升级到最新 schema
88
+ alembic downgrade -1 # 回退一个版本(仅限向后兼容的迁移)
89
+ make migrate # 等价于 alembic upgrade head
90
+ ```
91
+
92
+ Schema 变更一律通过 Alembic 迁移交付,禁止用临时脚本改表(见 AGENTS.md Architecture Contracts)。
93
+
94
+ ### MinIO / S3 Adapter 测试
95
+
96
+ ```bash
97
+ UPLOAD_MINIO_TEST=1 \
98
+ UPLOAD_STORAGE__S3__INTERNAL_ENDPOINT_URL=http://localhost:9000 \
99
+ S3_ACCESS_KEY=minioadmin S3_SECRET_KEY=minioadmin \
100
+ python -m pytest tests/integration/test_s3_storage.py -q
101
+ ```
102
+
103
+ 未设置 `UPLOAD_MINIO_TEST=1` 时该套件自动跳过;CI 内置 MinIO 服务并始终运行它。
104
+
105
+ ### Portal E2E(Playwright)
106
+
107
+ ```bash
108
+ cd portal
109
+ npm ci
110
+ npx playwright install chromium # 首次运行需下载浏览器
111
+ npx playwright test # 自动拉起 uvicorn(8000) 与 vite(5173)
112
+ ```
113
+
114
+ E2E 覆盖登录与错误展示、文件/目录上传、生命周期策略、刷新后队列恢复(§29.6)。
115
+ 本地默认 `E2E_API_URL=http://127.0.0.1:8000`、`E2E_PORTAL_URL=http://127.0.0.1:5173`,
116
+ API Key 为 `e2e-key`(由 `portal/playwright.config.ts` 注入,SQLite + local 存储)。
117
+ CI 的 portal job 会安装浏览器依赖并执行同一套测试。
118
+
119
+ ## 配置
120
+
121
+ 配置优先级:代码默认值 < `config/config.yaml` < 环境变量(`UPLOAD_SECTION__FIELD`)< 命令行。
122
+
123
+ ```bash
124
+ python -m upload_service config validate
125
+ python -m upload_service config show --redact-secrets
126
+ python -m upload_service reconcile upload {upload_id} --dry-run
127
+ ```
128
+
129
+ 关键环境变量:
130
+
131
+ | 变量 | 说明 |
132
+ |---|---|
133
+ | `UPLOAD_API_KEYS` | JSON:`{"tenant/principal": ["key"]}` 或 key 数组 |
134
+ | `UPLOAD_DATABASE_URL` | SQLAlchemy 异步 URL |
135
+ | `UPLOAD_REDIS_URL` | Redis URL(`UPLOAD_REDIS__ENABLED=false` 可关闭) |
136
+ | `S3_ACCESS_KEY` / `S3_SECRET_KEY` | S3/MinIO 凭据 |
137
+
138
+ ## 备份与恢复
139
+
140
+ 运维手册见 [`docs/operations.md`](docs/operations.md):PostgreSQL 每日全量备份与 PITR、
141
+ 对象存储 Versioning/复制/对象锁、恢复顺序(先 DB → 对象存储 → Redis → API 只读检查 →
142
+ Reconcile Dry Run → Worker → 开放入口)、向后兼容的迁移与回滚原则(§28)。
143
+
144
+ ## Python SDK
145
+
146
+ ```python
147
+ from datetime import timedelta
148
+ from pyuploadx import UploadClient, FileLifecycle
149
+
150
+ client = UploadClient(
151
+ base_url="http://localhost:8000",
152
+ api_key="dev-key",
153
+ state_dir="~/.pyuploadx/uploads",
154
+ )
155
+
156
+ # 小文件
157
+ result = client.upload_file(
158
+ "./README.md",
159
+ bucket="app-default",
160
+ lifecycle=FileLifecycle.ttl(timedelta(days=30)),
161
+ )
162
+
163
+ # 大文件(Multipart + 断点续传)
164
+ result = client.upload_large_file(
165
+ "./model.bin",
166
+ bucket="app-default",
167
+ object_key="models/model.bin",
168
+ part_size=8 * 1024 * 1024,
169
+ concurrency=4,
170
+ resume=True,
171
+ )
172
+
173
+ # 目录上传
174
+ job = client.upload_directory(
175
+ "./album-assets",
176
+ bucket="app-default",
177
+ destination_prefix="artists/10001/albums/2026",
178
+ file_concurrency=8,
179
+ part_concurrency=4,
180
+ exclude=[".git/**", "**/*.tmp"],
181
+ conflict_policy="reject",
182
+ )
183
+ ```
184
+
185
+ ### 安装
186
+
187
+ ```bash
188
+ pip install pyuploadx # 官方 PyPI:Python SDK(Python ≥ 3.11,第三方依赖仅 httpx)
189
+ pip install pyuploadx-server # 官方 PyPI:服务端(FastAPI,Docker 部署时通常无需 pip 安装)
190
+ pip install dist/pyuploadx-0.2.0-py3-none-any.whl # 或仓库直装(保留历史版本,见 dist/README.md)
191
+ ```
192
+
193
+ SDK 与服务端为两个独立发布包(`pyuploadx` / `pyuploadx-server`);发版与版本管理见
194
+ [docs/docs_product-design.md §37](docs/docs_product-design.md)。
195
+
196
+ ## REST API 摘要
197
+
198
+ ```text
199
+ GET /healthz | /readyz | /startupz | /metrics
200
+ POST /v1/files/upload GET/DELETE /v1/files/{id}
201
+ GET /v1/files/{id}/download POST /v1/files/{id}/presign-download
202
+ POST /v1/uploads POST /v1/uploads/resume
203
+ GET /v1/uploads/{id} GET /v1/uploads/{id}/parts
204
+ POST /v1/uploads/{id}/parts/presign
205
+ PUT /v1/uploads/{id}/parts/{part_number}
206
+ POST /v1/uploads/{id}/parts/commit
207
+ POST /v1/uploads/{id}/refresh | /complete | /abort
208
+ POST /v1/directory-uploads ... /v1/files/{id}/lifecycle ... /v1/client-config
209
+ ```
210
+
211
+ 错误响应统一为 `{"error": {"code", "message", "details", "retryable", "request_id"}}`。
212
+
213
+ ## 可观测性
214
+
215
+ - 健康检查:`/healthz`(Liveness)、`/readyz`(Readiness,检测数据库连通性)、`/startupz`。
216
+ - 指标:`/metrics`(Prometheus 文本格式),覆盖上传/分片/断点/目录/生命周期/数据库/Redis/
217
+ Storage 全链路;Kubernetes 下由 `deploy/kubernetes/servicemonitor.yaml` 抓取。
218
+ - 日志:结构化 JSON(request_id / trace_id / node_id / tenant_id / duration_ms 等),
219
+ 不记录 Secret、完整 API Key 或完整预签名 URL(§23)。
220
+
221
+ ## 性能测试
222
+
223
+ ```bash
224
+ python scripts/benchmark_upload.py --base-url http://localhost:8000 \
225
+ --api-key dev-key --files 100 --concurrency 16
226
+ python scripts/benchmark_upload.py --large-mb 64 --part-size 8388608 --concurrency 8
227
+ ```
228
+
229
+ 场景对应 §29.7:并发小文件 Proxy 上传与 1 GiB 级大文件 Multipart(走 SDK 断点续传链路),
230
+ 输出吞吐与 p50/p95 延迟。大规模场景(10 万小文件、目录总大小 1 TiB)请在目标环境按需调参。
231
+
232
+ ## 项目结构
233
+
234
+ ```text
235
+ app/ FastAPI 后端(api/config/core/db/storage/services/lifecycle/worker)
236
+ sdk/pyuploadx/ Python 客户端 SDK
237
+ portal/ React + TypeScript Portal(Dexie/IndexedDB 断点状态)
238
+ deploy/ 单节点/集群 Compose、Kubernetes(含 ServiceMonitor)、Nginx、MinIO 引导
239
+ config/ YAML 配置示例
240
+ tests/ 单元与集成测试
241
+ scripts/ 渲染/文档检查/看板同步/性能测试脚本
242
+ docs/ 设计文档、运维手册与架构图(SVG 源 / PNG 生成)
243
+ ```
244
+
245
+ ## 数据库与 ORM 契约
246
+
247
+ - 所有数据访问必须使用 SQLAlchemy 2 ORM(`app/db/models.py` + `app/db/repositories/`)。
248
+ - 禁止裸 SQL 字符串 DML;唯一例外是仓储层内基于
249
+ `sqlalchemy.dialects.postgresql.insert` 的 `ON CONFLICT` Upsert。
250
+ - Schema 变更通过 Alembic 迁移交付。
251
+
252
+ ## 安全要点
253
+
254
+ - 生产强制 HTTPS(Nginx/Ingress TLS 终止),CORS 仅允许显式配置的 Origin。
255
+ - Secret 只通过环境变量注入,禁止写入 YAML 或提交到仓库;日志脱敏密钥与完整签名 URL。
256
+ - PostgreSQL/Redis 不暴露公网(Compose 仅内部网络,K8s 无外部 Service)。
257
+ - 对象 Key 与目录相对路径均做防路径逃逸校验;上传会话校验所有权,Complete/Abort 幂等。
258
+
259
+ ## 文档图形(§35)
260
+
261
+ ```bash
262
+ make diagrams # 渲染 docs/assets/svg -> docs/assets/png
263
+ make diagrams-force # 强制重新渲染
264
+ make docs-check # 校验 PNG 未过期 + Markdown 引用 + SVG 安全(CI 门禁)
265
+ ```
266
+
267
+ ## 许可证
268
+
269
+ MIT
@@ -0,0 +1,3 @@
1
+ """PyUploadX upload service backend package."""
2
+
3
+ __version__ = "0.1.0"
File without changes