simple-module-file-storage 0.0.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- simple_module_file_storage-0.0.1/.gitignore +59 -0
- simple_module_file_storage-0.0.1/LICENSE +21 -0
- simple_module_file_storage-0.0.1/PKG-INFO +86 -0
- simple_module_file_storage-0.0.1/README.md +55 -0
- simple_module_file_storage-0.0.1/file_storage/__init__.py +1 -0
- simple_module_file_storage-0.0.1/file_storage/backends/__init__.py +77 -0
- simple_module_file_storage-0.0.1/file_storage/backends/filesystem.py +84 -0
- simple_module_file_storage-0.0.1/file_storage/backends/s3.py +156 -0
- simple_module_file_storage-0.0.1/file_storage/constants.py +107 -0
- simple_module_file_storage-0.0.1/file_storage/contracts/__init__.py +25 -0
- simple_module_file_storage-0.0.1/file_storage/contracts/events.py +23 -0
- simple_module_file_storage-0.0.1/file_storage/contracts/schemas.py +37 -0
- simple_module_file_storage-0.0.1/file_storage/contracts/service.py +82 -0
- simple_module_file_storage-0.0.1/file_storage/deps.py +26 -0
- simple_module_file_storage-0.0.1/file_storage/endpoints/__init__.py +0 -0
- simple_module_file_storage-0.0.1/file_storage/endpoints/api.py +179 -0
- simple_module_file_storage-0.0.1/file_storage/endpoints/views.py +41 -0
- simple_module_file_storage-0.0.1/file_storage/locales/en.json +42 -0
- simple_module_file_storage-0.0.1/file_storage/models.py +45 -0
- simple_module_file_storage-0.0.1/file_storage/module.py +129 -0
- simple_module_file_storage-0.0.1/file_storage/pages/Browse.tsx +190 -0
- simple_module_file_storage-0.0.1/file_storage/pages/components/UploadDropzone.tsx +54 -0
- simple_module_file_storage-0.0.1/file_storage/pages/constants.ts +21 -0
- simple_module_file_storage-0.0.1/file_storage/py.typed +0 -0
- simple_module_file_storage-0.0.1/file_storage/service.py +208 -0
- simple_module_file_storage-0.0.1/file_storage/services.py +28 -0
- simple_module_file_storage-0.0.1/file_storage/settings.py +87 -0
- simple_module_file_storage-0.0.1/package.json +16 -0
- simple_module_file_storage-0.0.1/pyproject.toml +57 -0
- simple_module_file_storage-0.0.1/tests/test_api.py +66 -0
- simple_module_file_storage-0.0.1/tests/test_backend_registry.py +93 -0
- simple_module_file_storage-0.0.1/tests/test_filesystem_backend.py +67 -0
- simple_module_file_storage-0.0.1/tests/test_s3_backend.py +103 -0
- simple_module_file_storage-0.0.1/tests/test_service.py +204 -0
- simple_module_file_storage-0.0.1/tsconfig.json +11 -0
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
# Python
|
|
2
|
+
__pycache__/
|
|
3
|
+
*.py[cod]
|
|
4
|
+
*.egg-info/
|
|
5
|
+
dist/
|
|
6
|
+
build/
|
|
7
|
+
.venv/
|
|
8
|
+
*.egg
|
|
9
|
+
|
|
10
|
+
# UV
|
|
11
|
+
uv.lock
|
|
12
|
+
|
|
13
|
+
# Node
|
|
14
|
+
node_modules/
|
|
15
|
+
|
|
16
|
+
# IDE
|
|
17
|
+
.idea/
|
|
18
|
+
.vscode/
|
|
19
|
+
*.swp
|
|
20
|
+
*.swo
|
|
21
|
+
|
|
22
|
+
# Environment
|
|
23
|
+
.env
|
|
24
|
+
|
|
25
|
+
# Database
|
|
26
|
+
*.db
|
|
27
|
+
*.sqlite3
|
|
28
|
+
|
|
29
|
+
# Module-managed runtime state (e.g. uploaded dataset files,
|
|
30
|
+
# default storage_dir for SM_DATASETS_STORAGE_DIR).
|
|
31
|
+
var/
|
|
32
|
+
|
|
33
|
+
# file_storage filesystem backend default root (override via SM_FILE_STORAGE_FS_ROOT_PATH).
|
|
34
|
+
uploads/
|
|
35
|
+
|
|
36
|
+
# Vite
|
|
37
|
+
host/static/dist/
|
|
38
|
+
|
|
39
|
+
# Auto-generated frontend module manifest (regenerated by the host at boot
|
|
40
|
+
# or via `make gen-pages`).
|
|
41
|
+
host/client_app/modules.manifest.json
|
|
42
|
+
host/client_app/modules.generated.ts
|
|
43
|
+
host/client_app/modules.generated.css
|
|
44
|
+
|
|
45
|
+
# Worktrees
|
|
46
|
+
.worktrees/
|
|
47
|
+
|
|
48
|
+
# Performance profiles
|
|
49
|
+
.memray/
|
|
50
|
+
.benchmarks/
|
|
51
|
+
|
|
52
|
+
# OS
|
|
53
|
+
.DS_Store
|
|
54
|
+
Thumbs.db
|
|
55
|
+
|
|
56
|
+
.playwright-cli/*
|
|
57
|
+
.playwright-mcp/*
|
|
58
|
+
host/client_app/.playwright-cli/*
|
|
59
|
+
.superpowers/
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Anto Subash
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: simple_module_file_storage
|
|
3
|
+
Version: 0.0.1
|
|
4
|
+
Summary: Pluggable file upload + storage (local or S3 via extras) module for simple_module apps
|
|
5
|
+
Project-URL: Homepage, https://github.com/antosubash/simple_module_python
|
|
6
|
+
Project-URL: Repository, https://github.com/antosubash/simple_module_python
|
|
7
|
+
Project-URL: Issues, https://github.com/antosubash/simple_module_python/issues
|
|
8
|
+
Project-URL: Changelog, https://github.com/antosubash/simple_module_python/blob/main/CHANGELOG.md
|
|
9
|
+
Author-email: Anto Subash <antosubash@live.com>
|
|
10
|
+
License-Expression: MIT
|
|
11
|
+
License-File: LICENSE
|
|
12
|
+
Keywords: file-upload,s3,simple-module,storage
|
|
13
|
+
Classifier: Development Status :: 3 - Alpha
|
|
14
|
+
Classifier: Framework :: FastAPI
|
|
15
|
+
Classifier: Intended Audience :: Developers
|
|
16
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
17
|
+
Classifier: Operating System :: OS Independent
|
|
18
|
+
Classifier: Programming Language :: Python :: 3
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
20
|
+
Classifier: Topic :: Internet :: WWW/HTTP
|
|
21
|
+
Classifier: Topic :: Software Development :: Libraries :: Application Frameworks
|
|
22
|
+
Classifier: Typing :: Typed
|
|
23
|
+
Requires-Python: >=3.12
|
|
24
|
+
Requires-Dist: aiofiles>=23
|
|
25
|
+
Requires-Dist: simple-module-core==0.0.1
|
|
26
|
+
Requires-Dist: simple-module-db==0.0.1
|
|
27
|
+
Requires-Dist: simple-module-hosting==0.0.1
|
|
28
|
+
Provides-Extra: s3
|
|
29
|
+
Requires-Dist: aioboto3>=13; extra == 's3'
|
|
30
|
+
Description-Content-Type: text/markdown
|
|
31
|
+
|
|
32
|
+
# simple_module_file_storage
|
|
33
|
+
|
|
34
|
+
Pluggable file-upload + storage module for [simple_module](https://github.com/antosubash/simple_module_python) apps. Defaults to local-disk storage for development; install the `[s3]` extra to switch to any S3-compatible backend via `aioboto3`.
|
|
35
|
+
|
|
36
|
+
## Install
|
|
37
|
+
|
|
38
|
+
```bash
|
|
39
|
+
# local-disk storage (dev default)
|
|
40
|
+
pip install simple_module_file_storage
|
|
41
|
+
|
|
42
|
+
# S3-compatible storage (production)
|
|
43
|
+
pip install "simple_module_file_storage[s3]"
|
|
44
|
+
```
|
|
45
|
+
|
|
46
|
+
## What it provides
|
|
47
|
+
|
|
48
|
+
- `POST /api/files` upload endpoint with multipart + metadata.
|
|
49
|
+
- `GET /api/files/{id}` signed-URL or stream download.
|
|
50
|
+
- Pluggable backend via `SM_FILE_STORAGE_BACKEND` (`local` | `s3`).
|
|
51
|
+
- S3 config via `SM_FILE_STORAGE_S3_BUCKET`, `SM_FILE_STORAGE_S3_ENDPOINT` (for R2/MinIO/etc.), `SM_FILE_STORAGE_S3_REGION`.
|
|
52
|
+
|
|
53
|
+
## Usage
|
|
54
|
+
|
|
55
|
+
From another module:
|
|
56
|
+
|
|
57
|
+
```python
|
|
58
|
+
from file_storage.service import FileStorageService # type: ignore[import-not-found]
|
|
59
|
+
|
|
60
|
+
async def attach_receipt(
|
|
61
|
+
svc: FileStorageService = Depends(FileStorageService),
|
|
62
|
+
upload: UploadFile = File(...),
|
|
63
|
+
):
|
|
64
|
+
record = await svc.save(upload, folder="receipts/")
|
|
65
|
+
return {"file_id": record.id, "url": record.url}
|
|
66
|
+
```
|
|
67
|
+
|
|
68
|
+
Env config (example, S3):
|
|
69
|
+
|
|
70
|
+
```
|
|
71
|
+
SM_FILE_STORAGE_BACKEND=s3
|
|
72
|
+
SM_FILE_STORAGE_S3_BUCKET=my-app-uploads
|
|
73
|
+
SM_FILE_STORAGE_S3_REGION=us-east-1
|
|
74
|
+
AWS_ACCESS_KEY_ID=...
|
|
75
|
+
AWS_SECRET_ACCESS_KEY=...
|
|
76
|
+
```
|
|
77
|
+
|
|
78
|
+
## Depends on
|
|
79
|
+
|
|
80
|
+
- `simple_module_core`, `simple_module_db`, `simple_module_hosting`
|
|
81
|
+
- `aiofiles`
|
|
82
|
+
- Optional: `aioboto3` (install the `[s3]` extra)
|
|
83
|
+
|
|
84
|
+
## License
|
|
85
|
+
|
|
86
|
+
MIT — see [LICENSE](https://github.com/antosubash/simple_module_python/blob/main/LICENSE).
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
# simple_module_file_storage
|
|
2
|
+
|
|
3
|
+
Pluggable file-upload + storage module for [simple_module](https://github.com/antosubash/simple_module_python) apps. Defaults to local-disk storage for development; install the `[s3]` extra to switch to any S3-compatible backend via `aioboto3`.
|
|
4
|
+
|
|
5
|
+
## Install
|
|
6
|
+
|
|
7
|
+
```bash
|
|
8
|
+
# local-disk storage (dev default)
|
|
9
|
+
pip install simple_module_file_storage
|
|
10
|
+
|
|
11
|
+
# S3-compatible storage (production)
|
|
12
|
+
pip install "simple_module_file_storage[s3]"
|
|
13
|
+
```
|
|
14
|
+
|
|
15
|
+
## What it provides
|
|
16
|
+
|
|
17
|
+
- `POST /api/files` upload endpoint with multipart + metadata.
|
|
18
|
+
- `GET /api/files/{id}` signed-URL or stream download.
|
|
19
|
+
- Pluggable backend via `SM_FILE_STORAGE_BACKEND` (`local` | `s3`).
|
|
20
|
+
- S3 config via `SM_FILE_STORAGE_S3_BUCKET`, `SM_FILE_STORAGE_S3_ENDPOINT` (for R2/MinIO/etc.), `SM_FILE_STORAGE_S3_REGION`.
|
|
21
|
+
|
|
22
|
+
## Usage
|
|
23
|
+
|
|
24
|
+
From another module:
|
|
25
|
+
|
|
26
|
+
```python
|
|
27
|
+
from file_storage.service import FileStorageService # type: ignore[import-not-found]
|
|
28
|
+
|
|
29
|
+
async def attach_receipt(
|
|
30
|
+
svc: FileStorageService = Depends(FileStorageService),
|
|
31
|
+
upload: UploadFile = File(...),
|
|
32
|
+
):
|
|
33
|
+
record = await svc.save(upload, folder="receipts/")
|
|
34
|
+
return {"file_id": record.id, "url": record.url}
|
|
35
|
+
```
|
|
36
|
+
|
|
37
|
+
Env config (example, S3):
|
|
38
|
+
|
|
39
|
+
```
|
|
40
|
+
SM_FILE_STORAGE_BACKEND=s3
|
|
41
|
+
SM_FILE_STORAGE_S3_BUCKET=my-app-uploads
|
|
42
|
+
SM_FILE_STORAGE_S3_REGION=us-east-1
|
|
43
|
+
AWS_ACCESS_KEY_ID=...
|
|
44
|
+
AWS_SECRET_ACCESS_KEY=...
|
|
45
|
+
```
|
|
46
|
+
|
|
47
|
+
## Depends on
|
|
48
|
+
|
|
49
|
+
- `simple_module_core`, `simple_module_db`, `simple_module_hosting`
|
|
50
|
+
- `aiofiles`
|
|
51
|
+
- Optional: `aioboto3` (install the `[s3]` extra)
|
|
52
|
+
|
|
53
|
+
## License
|
|
54
|
+
|
|
55
|
+
MIT — see [LICENSE](https://github.com/antosubash/simple_module_python/blob/main/LICENSE).
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""FileStorage module."""
|
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
"""Storage backend registry — extension point for storage providers.
|
|
2
|
+
|
|
3
|
+
Built-in providers (filesystem, S3) self-register on import. Third-party
|
|
4
|
+
packages add new providers via :func:`register_backend`::
|
|
5
|
+
|
|
6
|
+
from file_storage.backends import register_backend
|
|
7
|
+
from file_storage.contracts.service import StorageBackend
|
|
8
|
+
|
|
9
|
+
@register_backend("azure_blob")
|
|
10
|
+
def _build(settings) -> StorageBackend:
|
|
11
|
+
return AzureBlobBackend(...)
|
|
12
|
+
|
|
13
|
+
The factory receives the parsed :class:`FileStorageSettings` so providers
|
|
14
|
+
can read their own ``SM_FILE_STORAGE_*`` keys (or extend the settings
|
|
15
|
+
class with extra fields). :func:`build_backend` resolves the active
|
|
16
|
+
factory by ``settings.backend`` and raises :class:`ConfigurationError`
|
|
17
|
+
on an unknown id, listing all currently registered providers.
|
|
18
|
+
"""
|
|
19
|
+
|
|
20
|
+
from __future__ import annotations
|
|
21
|
+
|
|
22
|
+
from collections.abc import Callable
|
|
23
|
+
from typing import TYPE_CHECKING
|
|
24
|
+
|
|
25
|
+
from file_storage.contracts.service import ConfigurationError
|
|
26
|
+
|
|
27
|
+
if TYPE_CHECKING:
|
|
28
|
+
from file_storage.contracts.service import StorageBackend
|
|
29
|
+
from file_storage.settings import FileStorageSettings
|
|
30
|
+
|
|
31
|
+
BackendFactory = Callable[["FileStorageSettings"], "StorageBackend"]
|
|
32
|
+
|
|
33
|
+
_REGISTRY: dict[str, BackendFactory] = {}
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def register_backend(backend_id: str) -> Callable[[BackendFactory], BackendFactory]:
|
|
37
|
+
"""Decorator that registers a factory under ``backend_id``."""
|
|
38
|
+
|
|
39
|
+
def decorator(factory: BackendFactory) -> BackendFactory:
|
|
40
|
+
_REGISTRY[backend_id] = factory
|
|
41
|
+
return factory
|
|
42
|
+
|
|
43
|
+
return decorator
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def unregister_backend(backend_id: str) -> None:
|
|
47
|
+
"""Remove a registered backend. Primarily for tests."""
|
|
48
|
+
_REGISTRY.pop(backend_id, None)
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def registered_backends() -> list[str]:
|
|
52
|
+
"""Return all currently registered backend ids, sorted."""
|
|
53
|
+
return sorted(_REGISTRY)
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def build_backend(settings: FileStorageSettings) -> StorageBackend:
|
|
57
|
+
"""Construct the backend selected by ``settings.backend``."""
|
|
58
|
+
factory = _REGISTRY.get(settings.backend)
|
|
59
|
+
if factory is None:
|
|
60
|
+
raise ConfigurationError(
|
|
61
|
+
f"Unknown storage backend {settings.backend!r}. Registered: {registered_backends()}."
|
|
62
|
+
)
|
|
63
|
+
return factory(settings)
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
# Trigger self-registration of built-in providers. Order is irrelevant; both
|
|
67
|
+
# add themselves to the same module-global registry.
|
|
68
|
+
from file_storage.backends import filesystem as _filesystem # noqa: E402,F401
|
|
69
|
+
from file_storage.backends import s3 as _s3 # noqa: E402,F401
|
|
70
|
+
|
|
71
|
+
__all__ = [
|
|
72
|
+
"BackendFactory",
|
|
73
|
+
"build_backend",
|
|
74
|
+
"register_backend",
|
|
75
|
+
"registered_backends",
|
|
76
|
+
"unregister_backend",
|
|
77
|
+
]
|
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
"""Filesystem storage backend — writes objects under a configurable root."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from collections.abc import AsyncIterator
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
|
|
8
|
+
import aiofiles
|
|
9
|
+
|
|
10
|
+
from file_storage import constants
|
|
11
|
+
from file_storage.backends import register_backend
|
|
12
|
+
from file_storage.contracts.service import (
|
|
13
|
+
NotSupportedError,
|
|
14
|
+
StorageBackendError,
|
|
15
|
+
StorageNotFoundError,
|
|
16
|
+
)
|
|
17
|
+
from file_storage.settings import FileStorageSettings
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
class FilesystemBackend:
|
|
21
|
+
"""Stores objects on the local filesystem under ``root``.
|
|
22
|
+
|
|
23
|
+
Keys are sharded by their first two characters to keep any single
|
|
24
|
+
directory from accumulating millions of entries (which slows ``readdir``
|
|
25
|
+
on most filesystems). A key like ``2026/04/19/abc123.png`` is written to
|
|
26
|
+
``<root>/20/2026/04/19/abc123.png``.
|
|
27
|
+
"""
|
|
28
|
+
|
|
29
|
+
backend_id = constants.BackendId.FILESYSTEM
|
|
30
|
+
supports_presigned_url = False
|
|
31
|
+
|
|
32
|
+
def __init__(self, root: Path) -> None:
|
|
33
|
+
self.root = root
|
|
34
|
+
|
|
35
|
+
def _resolve(self, key: str) -> Path:
|
|
36
|
+
# Reject path traversal: ".." segments and absolute paths can escape root.
|
|
37
|
+
if key.startswith("/") or ".." in Path(key).parts:
|
|
38
|
+
raise StorageBackendError(f"Invalid storage key: {key!r}")
|
|
39
|
+
shard = key[:2] if len(key) >= 2 else "_"
|
|
40
|
+
return self.root / shard / key
|
|
41
|
+
|
|
42
|
+
async def put(
|
|
43
|
+
self,
|
|
44
|
+
key: str,
|
|
45
|
+
stream: AsyncIterator[bytes],
|
|
46
|
+
*,
|
|
47
|
+
content_type: str,
|
|
48
|
+
size: int,
|
|
49
|
+
) -> None:
|
|
50
|
+
path = self._resolve(key)
|
|
51
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
52
|
+
async with aiofiles.open(path, "wb") as fh:
|
|
53
|
+
async for chunk in stream:
|
|
54
|
+
await fh.write(chunk)
|
|
55
|
+
|
|
56
|
+
async def get(self, key: str) -> AsyncIterator[bytes]:
|
|
57
|
+
path = self._resolve(key)
|
|
58
|
+
if not path.exists():
|
|
59
|
+
raise StorageNotFoundError(key)
|
|
60
|
+
return _stream_file(path)
|
|
61
|
+
|
|
62
|
+
async def delete(self, key: str) -> None:
|
|
63
|
+
path = self._resolve(key)
|
|
64
|
+
path.unlink(missing_ok=True)
|
|
65
|
+
|
|
66
|
+
async def exists(self, key: str) -> bool:
|
|
67
|
+
return self._resolve(key).exists()
|
|
68
|
+
|
|
69
|
+
async def presigned_get_url(self, key: str, ttl_seconds: int) -> str:
|
|
70
|
+
raise NotSupportedError("Filesystem backend cannot mint presigned URLs.")
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
async def _stream_file(path: Path) -> AsyncIterator[bytes]:
|
|
74
|
+
async with aiofiles.open(path, "rb") as fh:
|
|
75
|
+
while True:
|
|
76
|
+
chunk = await fh.read(constants.DEFAULT_CHUNK_SIZE)
|
|
77
|
+
if not chunk:
|
|
78
|
+
break
|
|
79
|
+
yield chunk
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
@register_backend(constants.BackendId.FILESYSTEM)
|
|
83
|
+
def _build(settings: FileStorageSettings) -> FilesystemBackend:
|
|
84
|
+
return FilesystemBackend(root=settings.resolved_fs_root())
|
|
@@ -0,0 +1,156 @@
|
|
|
1
|
+
"""S3-compatible storage backend (AWS S3, MinIO, Cloudflare R2, ...).
|
|
2
|
+
|
|
3
|
+
``aioboto3`` is imported lazily at backend construction so the module loads
|
|
4
|
+
even when the optional S3 extra isn't installed; the failure mode is a clear
|
|
5
|
+
:class:`ConfigurationError` at boot rather than a confusing import error.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
from collections.abc import AsyncIterator
|
|
11
|
+
from tempfile import SpooledTemporaryFile
|
|
12
|
+
from typing import TYPE_CHECKING, Any
|
|
13
|
+
|
|
14
|
+
from file_storage import constants
|
|
15
|
+
from file_storage.backends import register_backend
|
|
16
|
+
from file_storage.contracts.service import (
|
|
17
|
+
ConfigurationError,
|
|
18
|
+
StorageBackendError,
|
|
19
|
+
StorageNotFoundError,
|
|
20
|
+
)
|
|
21
|
+
from file_storage.settings import FileStorageSettings
|
|
22
|
+
|
|
23
|
+
if TYPE_CHECKING:
|
|
24
|
+
pass
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
class S3Backend:
|
|
28
|
+
"""S3-compatible object storage.
|
|
29
|
+
|
|
30
|
+
Each operation opens a fresh ``aioboto3`` client context (cheap; the
|
|
31
|
+
underlying ``aiobotocore`` session is reused via ``self._session``). We
|
|
32
|
+
deliberately do not hold one long-lived client across requests because
|
|
33
|
+
aioboto3 client contexts are per-event-loop and tests run multiple loops.
|
|
34
|
+
"""
|
|
35
|
+
|
|
36
|
+
backend_id = constants.BackendId.S3
|
|
37
|
+
supports_presigned_url = True
|
|
38
|
+
|
|
39
|
+
def __init__(self, *, bucket: str, region: str, client_kwargs: dict[str, Any]) -> None:
|
|
40
|
+
try:
|
|
41
|
+
import aioboto3
|
|
42
|
+
except ImportError as exc:
|
|
43
|
+
raise ConfigurationError(
|
|
44
|
+
"S3 backend requires the 'aioboto3' package. "
|
|
45
|
+
"Install with: uv add --optional s3 aioboto3"
|
|
46
|
+
) from exc
|
|
47
|
+
self._aioboto3 = aioboto3
|
|
48
|
+
self._session = aioboto3.Session()
|
|
49
|
+
self.bucket = bucket
|
|
50
|
+
self.region = region
|
|
51
|
+
self.client_kwargs = client_kwargs
|
|
52
|
+
|
|
53
|
+
def _client(self):
|
|
54
|
+
return self._session.client("s3", **self.client_kwargs)
|
|
55
|
+
|
|
56
|
+
async def put(
|
|
57
|
+
self,
|
|
58
|
+
key: str,
|
|
59
|
+
stream: AsyncIterator[bytes],
|
|
60
|
+
*,
|
|
61
|
+
content_type: str,
|
|
62
|
+
size: int,
|
|
63
|
+
) -> None:
|
|
64
|
+
# Spool to memory up to SPOOL_MAX_SIZE_BYTES, then to a temp file.
|
|
65
|
+
# put_object needs a seekable file-like; multipart upload would be
|
|
66
|
+
# the next step for very large files (deferred to v2).
|
|
67
|
+
with SpooledTemporaryFile(max_size=constants.SPOOL_MAX_SIZE_BYTES) as buf:
|
|
68
|
+
async for chunk in stream:
|
|
69
|
+
buf.write(chunk)
|
|
70
|
+
length = buf.tell()
|
|
71
|
+
buf.seek(0)
|
|
72
|
+
try:
|
|
73
|
+
async with self._client() as client:
|
|
74
|
+
await client.put_object(
|
|
75
|
+
Bucket=self.bucket,
|
|
76
|
+
Key=key,
|
|
77
|
+
Body=buf,
|
|
78
|
+
ContentType=content_type,
|
|
79
|
+
ContentLength=length,
|
|
80
|
+
)
|
|
81
|
+
except Exception as exc:
|
|
82
|
+
raise StorageBackendError(f"S3 put_object failed: {exc}") from exc
|
|
83
|
+
|
|
84
|
+
async def get(self, key: str) -> AsyncIterator[bytes]:
|
|
85
|
+
try:
|
|
86
|
+
async with self._client() as client:
|
|
87
|
+
resp = await client.get_object(Bucket=self.bucket, Key=key)
|
|
88
|
+
# ``Body.read()`` works uniformly across aioboto3 (real) and
|
|
89
|
+
# moto's patched aiobotocore (tests); ``iter_chunks`` is sync
|
|
90
|
+
# under moto, breaking ``async for``.
|
|
91
|
+
data = await resp["Body"].read()
|
|
92
|
+
except Exception as exc:
|
|
93
|
+
if _is_not_found(exc):
|
|
94
|
+
raise StorageNotFoundError(key) from exc
|
|
95
|
+
raise StorageBackendError(f"S3 get_object failed: {exc}") from exc
|
|
96
|
+
|
|
97
|
+
for offset in range(0, len(data), constants.DEFAULT_CHUNK_SIZE):
|
|
98
|
+
yield data[offset : offset + constants.DEFAULT_CHUNK_SIZE]
|
|
99
|
+
|
|
100
|
+
async def delete(self, key: str) -> None:
|
|
101
|
+
try:
|
|
102
|
+
async with self._client() as client:
|
|
103
|
+
await client.delete_object(Bucket=self.bucket, Key=key)
|
|
104
|
+
except Exception as exc:
|
|
105
|
+
raise StorageBackendError(f"S3 delete_object failed: {exc}") from exc
|
|
106
|
+
|
|
107
|
+
async def exists(self, key: str) -> bool:
|
|
108
|
+
try:
|
|
109
|
+
async with self._client() as client:
|
|
110
|
+
await client.head_object(Bucket=self.bucket, Key=key)
|
|
111
|
+
return True
|
|
112
|
+
except Exception as exc:
|
|
113
|
+
if _is_not_found(exc):
|
|
114
|
+
return False
|
|
115
|
+
raise StorageBackendError(f"S3 head_object failed: {exc}") from exc
|
|
116
|
+
|
|
117
|
+
async def presigned_get_url(self, key: str, ttl_seconds: int) -> str:
|
|
118
|
+
try:
|
|
119
|
+
async with self._client() as client:
|
|
120
|
+
return await client.generate_presigned_url(
|
|
121
|
+
"get_object",
|
|
122
|
+
Params={"Bucket": self.bucket, "Key": key},
|
|
123
|
+
ExpiresIn=ttl_seconds,
|
|
124
|
+
)
|
|
125
|
+
except Exception as exc:
|
|
126
|
+
raise StorageBackendError(f"presign failed: {exc}") from exc
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
def _is_not_found(exc: Exception) -> bool:
|
|
130
|
+
"""Detect a 404 from botocore's heterogeneous error shapes."""
|
|
131
|
+
response = getattr(exc, "response", None)
|
|
132
|
+
if isinstance(response, dict):
|
|
133
|
+
code = response.get("Error", {}).get("Code")
|
|
134
|
+
if code in {"NoSuchKey", "404", "NotFound"}:
|
|
135
|
+
return True
|
|
136
|
+
return getattr(exc, "__class__", type(exc)).__name__ in {
|
|
137
|
+
"NoSuchKey",
|
|
138
|
+
"ClientError",
|
|
139
|
+
} and "404" in str(exc)
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
@register_backend(constants.BackendId.S3)
|
|
143
|
+
def _build(settings: FileStorageSettings) -> S3Backend:
|
|
144
|
+
if not settings.s3_bucket or not settings.s3_region:
|
|
145
|
+
raise ConfigurationError("S3 backend requires s3_bucket and s3_region.")
|
|
146
|
+
client_kwargs: dict[str, Any] = {"region_name": settings.s3_region}
|
|
147
|
+
if settings.s3_endpoint_url:
|
|
148
|
+
client_kwargs["endpoint_url"] = settings.s3_endpoint_url
|
|
149
|
+
if settings.s3_access_key_id and settings.s3_secret_access_key:
|
|
150
|
+
client_kwargs["aws_access_key_id"] = settings.s3_access_key_id
|
|
151
|
+
client_kwargs["aws_secret_access_key"] = settings.s3_secret_access_key
|
|
152
|
+
return S3Backend(
|
|
153
|
+
bucket=settings.s3_bucket,
|
|
154
|
+
region=settings.s3_region,
|
|
155
|
+
client_kwargs=client_kwargs,
|
|
156
|
+
)
|
|
@@ -0,0 +1,107 @@
|
|
|
1
|
+
"""Single source of truth for every literal in the file_storage module.
|
|
2
|
+
|
|
3
|
+
Permissions, event topics, route prefixes, env-var prefix, page names, table
|
|
4
|
+
names, error codes, default config values — all live here. Other files in the
|
|
5
|
+
module import from this one rather than embedding string literals, so renames
|
|
6
|
+
stay localised and `rg` can prove there are no magic strings outside of this
|
|
7
|
+
file.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
from typing import Final
|
|
13
|
+
|
|
14
|
+
# ── Module identity ──────────────────────────────────────────────────
|
|
15
|
+
# ``MODULE_NAME`` is the snake_case package name; ``MODULE_PASCAL`` matches the
|
|
16
|
+
# Inertia page-prefix convention (PascalCase of the module dir). ``meta.name``
|
|
17
|
+
# uses ``MODULE_PASCAL`` so the diagnostic code SM003 lines up against
|
|
18
|
+
# ``FileStorage/Browse.tsx``. ``MODULE_DISPLAY_NAME`` is the user-visible label
|
|
19
|
+
# (menu, permission group) — keep it free-form.
|
|
20
|
+
MODULE_NAME: Final = "file_storage"
|
|
21
|
+
MODULE_PASCAL: Final = "FileStorage"
|
|
22
|
+
MODULE_DISPLAY_NAME: Final = "Files"
|
|
23
|
+
|
|
24
|
+
# ── Configuration ────────────────────────────────────────────────────
|
|
25
|
+
ENV_PREFIX: Final = "SM_FILE_STORAGE_"
|
|
26
|
+
|
|
27
|
+
# ── Routing ──────────────────────────────────────────────────────────
|
|
28
|
+
ROUTE_PREFIX_API: Final = "/api/file-storage"
|
|
29
|
+
ROUTE_PREFIX_VIEW: Final = "/file-storage"
|
|
30
|
+
|
|
31
|
+
# REST sub-paths (joined to ROUTE_PREFIX_API by the router)
|
|
32
|
+
PATH_UPLOAD: Final = "/upload"
|
|
33
|
+
PATH_FILES: Final = "/files"
|
|
34
|
+
PATH_FILE_BY_ID: Final = "/files/{file_id}"
|
|
35
|
+
PATH_FILE_DOWNLOAD: Final = "/files/{file_id}/download"
|
|
36
|
+
|
|
37
|
+
# ── Inertia page names ───────────────────────────────────────────────
|
|
38
|
+
PAGE_BROWSE: Final = "FileStorage/Browse"
|
|
39
|
+
|
|
40
|
+
# ── Database ─────────────────────────────────────────────────────────
|
|
41
|
+
SCHEMA_NAME: Final = "file_storage"
|
|
42
|
+
TABLE_STORED_FILE: Final = "file_storage_stored_file"
|
|
43
|
+
|
|
44
|
+
# ── i18n ─────────────────────────────────────────────────────────────
|
|
45
|
+
LOCALE_NAMESPACE: Final = MODULE_NAME
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
class BackendId:
|
|
49
|
+
"""Stable identifiers for built-in storage providers.
|
|
50
|
+
|
|
51
|
+
Third-party providers register themselves under any string id, but the
|
|
52
|
+
built-ins are referenced from settings, the model, and tests, so they
|
|
53
|
+
deserve named constants.
|
|
54
|
+
"""
|
|
55
|
+
|
|
56
|
+
FILESYSTEM: Final = "filesystem"
|
|
57
|
+
S3: Final = "s3"
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
class Permission:
|
|
61
|
+
UPLOAD: Final = "file_storage.upload"
|
|
62
|
+
DOWNLOAD: Final = "file_storage.download"
|
|
63
|
+
DELETE: Final = "file_storage.delete"
|
|
64
|
+
MANAGE: Final = "file_storage.manage"
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
class Event:
|
|
68
|
+
FILE_UPLOADED: Final = "file_storage.file.uploaded"
|
|
69
|
+
FILE_DELETED: Final = "file_storage.file.deleted"
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
class FeatureFlag:
|
|
73
|
+
PUBLIC_UPLOADS: Final = "file_storage.public_uploads"
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
class ErrorCode:
|
|
77
|
+
"""Machine-readable codes returned in 4xx response bodies."""
|
|
78
|
+
|
|
79
|
+
TOO_LARGE: Final = "file_storage.too_large"
|
|
80
|
+
BAD_TYPE: Final = "file_storage.bad_type"
|
|
81
|
+
NOT_FOUND: Final = "file_storage.not_found"
|
|
82
|
+
BACKEND_ERROR: Final = "file_storage.backend_error"
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
class I18nKey:
|
|
86
|
+
"""Translator keys (full dotted paths). Mirror locales/en.json shape."""
|
|
87
|
+
|
|
88
|
+
ERR_NOT_FOUND: Final = "file_storage.errors.not_found"
|
|
89
|
+
ERR_TOO_LARGE: Final = "file_storage.errors.too_large"
|
|
90
|
+
ERR_BAD_TYPE: Final = "file_storage.errors.bad_type"
|
|
91
|
+
ERR_BACKEND: Final = "file_storage.errors.backend_error"
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
# ── Defaults ─────────────────────────────────────────────────────────
|
|
95
|
+
DEFAULT_BACKEND: Final = BackendId.FILESYSTEM
|
|
96
|
+
DEFAULT_FS_ROOT: Final = "./uploads"
|
|
97
|
+
DEFAULT_MAX_FILE_SIZE_BYTES: Final = 100 * 1024 * 1024 # 100 MB
|
|
98
|
+
DEFAULT_PRESIGN_TTL_SECONDS: Final = 300 # 5 minutes
|
|
99
|
+
DEFAULT_CHUNK_SIZE: Final = 64 * 1024 # 64 KB
|
|
100
|
+
SPOOL_MAX_SIZE_BYTES: Final = 10 * 1024 * 1024 # 10 MB before disk-spill
|
|
101
|
+
|
|
102
|
+
# ── Menu ─────────────────────────────────────────────────────────────
|
|
103
|
+
MENU_ICON: Final = "files"
|
|
104
|
+
MENU_ORDER: Final = 40
|
|
105
|
+
MENU_ROLES: Final = ("admin",)
|
|
106
|
+
ADMIN_ROLE: Final = "admin"
|
|
107
|
+
USER_ROLE: Final = "user"
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
"""file_storage contracts — public interface for other modules."""
|
|
2
|
+
|
|
3
|
+
from file_storage.contracts.events import FileDeleted, FileUploaded
|
|
4
|
+
from file_storage.contracts.schemas import StoredFileListOut, StoredFileOut
|
|
5
|
+
from file_storage.contracts.service import (
|
|
6
|
+
ConfigurationError,
|
|
7
|
+
NotSupportedError,
|
|
8
|
+
StorageBackend,
|
|
9
|
+
StorageBackendError,
|
|
10
|
+
StorageError,
|
|
11
|
+
StorageNotFoundError,
|
|
12
|
+
)
|
|
13
|
+
|
|
14
|
+
__all__ = [
|
|
15
|
+
"ConfigurationError",
|
|
16
|
+
"FileDeleted",
|
|
17
|
+
"FileUploaded",
|
|
18
|
+
"NotSupportedError",
|
|
19
|
+
"StorageBackend",
|
|
20
|
+
"StorageBackendError",
|
|
21
|
+
"StorageError",
|
|
22
|
+
"StorageNotFoundError",
|
|
23
|
+
"StoredFileListOut",
|
|
24
|
+
"StoredFileOut",
|
|
25
|
+
]
|