lfss 0.9.4__py3-none-any.whl → 0.10.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- Readme.md +4 -4
- docs/Changelog.md +4 -0
- docs/Enviroment_variables.md +1 -1
- docs/Permission.md +4 -4
- lfss/api/connector.py +23 -5
- lfss/eng/config.py +0 -1
- lfss/eng/connection_pool.py +11 -7
- lfss/eng/database.py +81 -35
- lfss/eng/error.py +2 -0
- lfss/eng/utils.py +3 -3
- lfss/svc/app_base.py +2 -1
- lfss/svc/app_native.py +36 -45
- lfss/svc/common_impl.py +1 -4
- {lfss-0.9.4.dist-info → lfss-0.10.0.dist-info}/METADATA +6 -5
- {lfss-0.9.4.dist-info → lfss-0.10.0.dist-info}/RECORD +17 -17
- {lfss-0.9.4.dist-info → lfss-0.10.0.dist-info}/WHEEL +0 -0
- {lfss-0.9.4.dist-info → lfss-0.10.0.dist-info}/entry_points.txt +0 -0
Readme.md
CHANGED
@@ -1,5 +1,5 @@
|
|
1
|
-
#
|
2
|
-
[](https://pypi.org/project/lfss/)
|
1
|
+
# Lite File Storage Service (LFSS)
|
2
|
+
[](https://pypi.org/project/lfss/) [](https://pypi.org/project/lfss/)
|
3
3
|
|
4
4
|
My experiment on a lightweight and high-performance file/object storage service...
|
5
5
|
|
@@ -32,8 +32,8 @@ Or, you can start a web server at `/frontend` and open `index.html` in your brow
|
|
32
32
|
|
33
33
|
The API usage is simple, just `GET`, `PUT`, `DELETE` to the `/<username>/file/url` path.
|
34
34
|
The authentication can be acheived through one of the following methods:
|
35
|
-
1. `Authorization` header with the value `Bearer sha256(<username
|
36
|
-
2. `token` query parameter with the value `sha256(<username
|
35
|
+
1. `Authorization` header with the value `Bearer sha256(<username>:<password>)`.
|
36
|
+
2. `token` query parameter with the value `sha256(<username>:<password>)`.
|
37
37
|
3. HTTP Basic Authentication with the username and password (If WebDAV is enabled).
|
38
38
|
|
39
39
|
You can refer to `frontend` as an application example, `lfss/api/connector.py` for more APIs.
|
docs/Changelog.md
CHANGED
docs/Enviroment_variables.md
CHANGED
@@ -9,4 +9,4 @@
|
|
9
9
|
|
10
10
|
**Client**
|
11
11
|
- `LFSS_ENDPOINT`: The fallback server endpoint. Default is `http://localhost:8000`.
|
12
|
-
- `LFSS_TOKEN`: The fallback token to authenticate. Should be `sha256(<username
|
12
|
+
- `LFSS_TOKEN`: The fallback token to authenticate. Should be `sha256(<username>:<password>)`.
|
docs/Permission.md
CHANGED
@@ -22,16 +22,16 @@ A file is owned by the user who created it, may not necessarily be the user unde
|
|
22
22
|
## File access with `GET` permission
|
23
23
|
|
24
24
|
### File access
|
25
|
-
For accessing file content, the user must have `GET` permission of the file, which is determined by the `permission` field of both the owner and the file.
|
25
|
+
For accessing file content, the user must have `GET` permission of the file, which is determined by the `permission` field of both the path-owner and the file.
|
26
26
|
|
27
27
|
There are four types of permissions: `unset`, `public`, `protected`, `private`.
|
28
28
|
Non-admin users can access files based on:
|
29
29
|
|
30
30
|
- If the file is `public`, then all users can access it.
|
31
31
|
- If the file is `protected`, then only the logged-in user can access it.
|
32
|
-
- If the file is `private`, then only the owner can access it.
|
33
|
-
- If the file is `unset`, then the file's permission is inherited from the owner's permission.
|
34
|
-
- If both the owner and the file have `unset` permission, then the file is `public`.
|
32
|
+
- If the file is `private`, then only the owner/path-owner can access it.
|
33
|
+
- If the file is `unset`, then the file's permission is inherited from the path-owner's permission.
|
34
|
+
- If both the path-owner and the file have `unset` permission, then the file is `public`.
|
35
35
|
|
36
36
|
## File creation with `PUT`/`POST` permission
|
37
37
|
`PUT`/`POST` permission is not allowed for non-peer users.
|
lfss/api/connector.py
CHANGED
@@ -1,5 +1,6 @@
|
|
1
1
|
from __future__ import annotations
|
2
|
-
from typing import Optional, Literal
|
2
|
+
from typing import Optional, Literal
|
3
|
+
from collections.abc import Iterator
|
3
4
|
import os
|
4
5
|
import requests
|
5
6
|
import requests.adapters
|
@@ -14,12 +15,13 @@ from lfss.eng.utils import ensure_uri_compnents
|
|
14
15
|
|
15
16
|
_default_endpoint = os.environ.get('LFSS_ENDPOINT', 'http://localhost:8000')
|
16
17
|
_default_token = os.environ.get('LFSS_TOKEN', '')
|
18
|
+
num_t = float | int
|
17
19
|
|
18
20
|
class Connector:
|
19
21
|
class Session:
|
20
22
|
def __init__(
|
21
23
|
self, connector: Connector, pool_size: int = 10,
|
22
|
-
retry: int = 1, backoff_factor:
|
24
|
+
retry: int = 1, backoff_factor: num_t = 0.5, status_forcelist: list[int] = [503]
|
23
25
|
):
|
24
26
|
self.connector = connector
|
25
27
|
self.pool_size = pool_size
|
@@ -46,13 +48,21 @@ class Connector:
|
|
46
48
|
def __exit__(self, exc_type, exc_value, traceback):
|
47
49
|
self.close()
|
48
50
|
|
49
|
-
def __init__(self, endpoint=_default_endpoint, token=_default_token):
|
51
|
+
def __init__(self, endpoint=_default_endpoint, token=_default_token, timeout: Optional[num_t | tuple[num_t, num_t]]=None, verify: Optional[bool | str] = None):
|
52
|
+
"""
|
53
|
+
- endpoint: the URL of the LFSS server. Default to $LFSS_ENDPOINT or http://localhost:8000.
|
54
|
+
- token: the access token. Default to $LFSS_TOKEN.
|
55
|
+
- timeout: the timeout for each request, can be either a single value or a tuple of two values (connect, read), refer to requests.Session.request.
|
56
|
+
- verify: either a boolean or a string, to control SSL verification. Default to True, refer to requests.Session.request.
|
57
|
+
"""
|
50
58
|
assert token, "No token provided. Please set LFSS_TOKEN environment variable."
|
51
59
|
self.config = {
|
52
60
|
"endpoint": endpoint,
|
53
61
|
"token": token
|
54
62
|
}
|
55
63
|
self._session: Optional[requests.Session] = None
|
64
|
+
self.timeout = timeout
|
65
|
+
self.verify = verify
|
56
66
|
|
57
67
|
def session( self, pool_size: int = 10, **kwargs):
|
58
68
|
""" avoid creating a new session for each request. """
|
@@ -73,11 +83,11 @@ class Connector:
|
|
73
83
|
})
|
74
84
|
headers.update(extra_headers)
|
75
85
|
if self._session is not None:
|
76
|
-
response = self._session.request(method, url, headers=headers, **kwargs)
|
86
|
+
response = self._session.request(method, url, headers=headers, timeout=self.timeout, verify=self.verify, **kwargs)
|
77
87
|
response.raise_for_status()
|
78
88
|
else:
|
79
89
|
with requests.Session() as s:
|
80
|
-
response = s.request(method, url, headers=headers, **kwargs)
|
90
|
+
response = s.request(method, url, headers=headers, timeout=self.timeout, verify=self.verify, **kwargs)
|
81
91
|
response.raise_for_status()
|
82
92
|
return response
|
83
93
|
return f
|
@@ -276,6 +286,14 @@ class Connector:
|
|
276
286
|
self._fetch_factory('POST', '_api/copy', {'src': src, 'dst': dst})(
|
277
287
|
headers = {'Content-Type': 'application/www-form-urlencoded'}
|
278
288
|
)
|
289
|
+
|
290
|
+
def bundle(self, path: str) -> Iterator[bytes]:
|
291
|
+
"""Bundle a path into a zip file."""
|
292
|
+
response = self._fetch_factory('GET', '_api/bundle', {'path': path})(
|
293
|
+
headers = {'Content-Type': 'application/www-form-urlencoded'},
|
294
|
+
stream = True
|
295
|
+
)
|
296
|
+
return response.iter_content(chunk_size=1024)
|
279
297
|
|
280
298
|
def whoami(self) -> UserRecord:
|
281
299
|
"""Gets information about the current user."""
|
lfss/eng/config.py
CHANGED
@@ -19,7 +19,6 @@ if __env_large_file is not None:
|
|
19
19
|
else:
|
20
20
|
LARGE_FILE_BYTES = 8 * 1024 * 1024 # 8MB
|
21
21
|
MAX_MEM_FILE_BYTES = 128 * 1024 * 1024 # 128MB
|
22
|
-
MAX_BUNDLE_BYTES = 512 * 1024 * 1024 # 512MB
|
23
22
|
CHUNK_SIZE = 1024 * 1024 # 1MB chunks for streaming (on large files)
|
24
23
|
DEBUG_MODE = os.environ.get('LFSS_DEBUG', '0') == '1'
|
25
24
|
|
lfss/eng/connection_pool.py
CHANGED
@@ -8,7 +8,7 @@ from functools import wraps
|
|
8
8
|
from typing import Callable, Awaitable
|
9
9
|
|
10
10
|
from .log import get_logger
|
11
|
-
from .error import DatabaseLockedError
|
11
|
+
from .error import DatabaseLockedError, DatabaseTransactionError
|
12
12
|
from .config import DATA_HOME
|
13
13
|
|
14
14
|
async def execute_sql(conn: aiosqlite.Connection | aiosqlite.Cursor, name: str):
|
@@ -147,6 +147,14 @@ def global_entrance(n_read: int = 1):
|
|
147
147
|
return wrapper
|
148
148
|
return decorator
|
149
149
|
|
150
|
+
def handle_sqlite_error(e: Exception):
|
151
|
+
if 'database is locked' in str(e):
|
152
|
+
raise DatabaseLockedError from e
|
153
|
+
if 'cannot start a transaction within a transaction' in str(e):
|
154
|
+
get_logger('database', global_instance=True).error(f"Unexpected error: {e}")
|
155
|
+
raise DatabaseTransactionError from e
|
156
|
+
raise e
|
157
|
+
|
150
158
|
@asynccontextmanager
|
151
159
|
async def unique_cursor(is_write: bool = False):
|
152
160
|
if not is_write:
|
@@ -155,9 +163,7 @@ async def unique_cursor(is_write: bool = False):
|
|
155
163
|
try:
|
156
164
|
yield await connection_obj.conn.cursor()
|
157
165
|
except Exception as e:
|
158
|
-
|
159
|
-
raise DatabaseLockedError from e
|
160
|
-
raise e
|
166
|
+
handle_sqlite_error(e)
|
161
167
|
finally:
|
162
168
|
await g_pool.release(connection_obj)
|
163
169
|
else:
|
@@ -166,9 +172,7 @@ async def unique_cursor(is_write: bool = False):
|
|
166
172
|
try:
|
167
173
|
yield await connection_obj.conn.cursor()
|
168
174
|
except Exception as e:
|
169
|
-
|
170
|
-
raise DatabaseLockedError from e
|
171
|
-
raise e
|
175
|
+
handle_sqlite_error(e)
|
172
176
|
finally:
|
173
177
|
await g_pool.release(connection_obj)
|
174
178
|
|
lfss/eng/database.py
CHANGED
@@ -1,10 +1,11 @@
|
|
1
1
|
|
2
|
-
from typing import Optional, Literal,
|
2
|
+
from typing import Optional, Literal, overload
|
3
|
+
from collections.abc import AsyncIterable
|
3
4
|
from contextlib import asynccontextmanager
|
4
5
|
from abc import ABC
|
5
6
|
|
7
|
+
import uuid, datetime
|
6
8
|
import urllib.parse
|
7
|
-
import uuid
|
8
9
|
import zipfile, io, asyncio
|
9
10
|
|
10
11
|
import aiosqlite, aiofiles
|
@@ -82,9 +83,11 @@ class UserConn(DBObjectBase):
|
|
82
83
|
self, username: str, password: str, is_admin: bool = False,
|
83
84
|
max_storage: int = 1073741824, permission: FileReadPermission = FileReadPermission.UNSET
|
84
85
|
) -> int:
|
85
|
-
|
86
|
-
|
87
|
-
|
86
|
+
def validate_username(username: str):
|
87
|
+
assert not username.startswith('_'), "Error: reserved username"
|
88
|
+
assert not ('/' in username or ':' in username or len(username) > 255), "Invalid username"
|
89
|
+
assert urllib.parse.quote(username) == username, "Invalid username, must be URL safe"
|
90
|
+
validate_username(username)
|
88
91
|
self.logger.debug(f"Creating user {username}")
|
89
92
|
credential = hash_credential(username, password)
|
90
93
|
assert await self.get_user(username) is None, "Duplicate username"
|
@@ -926,14 +929,50 @@ class Database:
|
|
926
929
|
else:
|
927
930
|
blob = await fconn.get_file_blob(f_id)
|
928
931
|
yield r, blob
|
932
|
+
|
933
|
+
async def zip_path_stream(self, top_url: str, op_user: Optional[UserRecord] = None) -> AsyncIterable[bytes]:
|
934
|
+
from stat import S_IFREG
|
935
|
+
from stream_zip import async_stream_zip, ZIP_64
|
936
|
+
if top_url.startswith('/'):
|
937
|
+
top_url = top_url[1:]
|
938
|
+
|
939
|
+
if op_user:
|
940
|
+
if await check_path_permission(top_url, op_user) < AccessLevel.READ:
|
941
|
+
raise PermissionDeniedError(f"Permission denied: {op_user.username} cannot zip path {top_url}")
|
942
|
+
|
943
|
+
# https://stream-zip.docs.trade.gov.uk/async-interface/
|
944
|
+
async def data_iter():
|
945
|
+
async for (r, blob) in self.iter_path(top_url, None):
|
946
|
+
rel_path = r.url[len(top_url):]
|
947
|
+
rel_path = decode_uri_compnents(rel_path)
|
948
|
+
b_iter: AsyncIterable[bytes]
|
949
|
+
if isinstance(blob, bytes):
|
950
|
+
async def blob_iter(): yield blob
|
951
|
+
b_iter = blob_iter() # type: ignore
|
952
|
+
else:
|
953
|
+
assert isinstance(blob, AsyncIterable)
|
954
|
+
b_iter = blob
|
955
|
+
yield (
|
956
|
+
rel_path,
|
957
|
+
datetime.datetime.now(),
|
958
|
+
S_IFREG | 0o600,
|
959
|
+
ZIP_64,
|
960
|
+
b_iter
|
961
|
+
)
|
962
|
+
return async_stream_zip(data_iter())
|
929
963
|
|
930
964
|
@concurrent_wrap()
|
931
|
-
async def zip_path(self, top_url: str,
|
965
|
+
async def zip_path(self, top_url: str, op_user: Optional[UserRecord]) -> io.BytesIO:
|
932
966
|
if top_url.startswith('/'):
|
933
967
|
top_url = top_url[1:]
|
968
|
+
|
969
|
+
if op_user:
|
970
|
+
if await check_path_permission(top_url, op_user) < AccessLevel.READ:
|
971
|
+
raise PermissionDeniedError(f"Permission denied: {op_user.username} cannot zip path {top_url}")
|
972
|
+
|
934
973
|
buffer = io.BytesIO()
|
935
974
|
with zipfile.ZipFile(buffer, 'w') as zf:
|
936
|
-
async for (r, blob) in self.iter_path(top_url,
|
975
|
+
async for (r, blob) in self.iter_path(top_url, None):
|
937
976
|
rel_path = r.url[len(top_url):]
|
938
977
|
rel_path = decode_uri_compnents(rel_path)
|
939
978
|
if r.external:
|
@@ -945,39 +984,50 @@ class Database:
|
|
945
984
|
buffer.seek(0)
|
946
985
|
return buffer
|
947
986
|
|
948
|
-
def
|
987
|
+
async def _get_path_owner(cur: aiosqlite.Cursor, path: str) -> UserRecord:
|
988
|
+
path_username = path.split('/')[0]
|
989
|
+
uconn = UserConn(cur)
|
990
|
+
path_user = await uconn.get_user(path_username)
|
991
|
+
if path_user is None:
|
992
|
+
raise PathNotFoundError(f"Invalid path: {path_username} is not a valid username")
|
993
|
+
return path_user
|
994
|
+
|
995
|
+
async def check_file_read_permission(user: UserRecord, file: FileRecord, cursor: Optional[aiosqlite.Cursor] = None) -> tuple[bool, str]:
|
949
996
|
"""
|
950
997
|
This does not consider alias level permission,
|
951
998
|
use check_path_permission for alias level permission check first:
|
952
999
|
```
|
953
1000
|
if await check_path_permission(path, user) < AccessLevel.READ:
|
954
|
-
read_allowed, reason = check_file_read_permission(user,
|
1001
|
+
read_allowed, reason = check_file_read_permission(user, file)
|
955
1002
|
```
|
1003
|
+
The implementation assumes the user is not admin and is not the owner of the file/path
|
956
1004
|
"""
|
957
|
-
|
958
|
-
|
1005
|
+
@asynccontextmanager
|
1006
|
+
async def this_cur():
|
1007
|
+
if cursor is None:
|
1008
|
+
async with unique_cursor() as _cur:
|
1009
|
+
yield _cur
|
1010
|
+
else:
|
1011
|
+
yield cursor
|
1012
|
+
|
1013
|
+
f_perm = file.permission
|
1014
|
+
|
1015
|
+
# if file permission unset, use path owner's permission as fallback
|
1016
|
+
if f_perm == FileReadPermission.UNSET:
|
1017
|
+
async with this_cur() as cur:
|
1018
|
+
path_owner = await _get_path_owner(cur, file.url)
|
1019
|
+
f_perm = path_owner.permission
|
959
1020
|
|
960
1021
|
# check permission of the file
|
961
|
-
if
|
962
|
-
|
963
|
-
|
964
|
-
elif file.permission == FileReadPermission.PROTECTED:
|
1022
|
+
if f_perm == FileReadPermission.PRIVATE:
|
1023
|
+
return False, "Permission denied, private file"
|
1024
|
+
elif f_perm == FileReadPermission.PROTECTED:
|
965
1025
|
if user.id == 0:
|
966
1026
|
return False, "Permission denied, protected file"
|
967
|
-
elif
|
1027
|
+
elif f_perm == FileReadPermission.PUBLIC:
|
968
1028
|
return True, ""
|
969
1029
|
else:
|
970
|
-
assert
|
971
|
-
|
972
|
-
# use owner's permission as fallback
|
973
|
-
if owner.permission == FileReadPermission.PRIVATE:
|
974
|
-
if user.id != owner.id:
|
975
|
-
return False, "Permission denied, private user file"
|
976
|
-
elif owner.permission == FileReadPermission.PROTECTED:
|
977
|
-
if user.id == 0:
|
978
|
-
return False, "Permission denied, protected user file"
|
979
|
-
else:
|
980
|
-
assert owner.permission == FileReadPermission.PUBLIC or owner.permission == FileReadPermission.UNSET
|
1030
|
+
assert f_perm == FileReadPermission.UNSET
|
981
1031
|
|
982
1032
|
return True, ""
|
983
1033
|
|
@@ -1000,15 +1050,11 @@ async def check_path_permission(path: str, user: UserRecord, cursor: Optional[ai
|
|
1000
1050
|
yield cursor
|
1001
1051
|
|
1002
1052
|
# check if path user exists
|
1003
|
-
path_username = path.split('/')[0]
|
1004
1053
|
async with this_cur() as cur:
|
1005
|
-
|
1006
|
-
path_user = await uconn.get_user(path_username)
|
1007
|
-
if path_user is None:
|
1008
|
-
raise PathNotFoundError(f"Invalid path: {path_username} is not a valid username")
|
1054
|
+
path_owner = await _get_path_owner(cur, path)
|
1009
1055
|
|
1010
|
-
# check if user is admin
|
1011
|
-
if user.is_admin or user.
|
1056
|
+
# check if user is admin or the owner of the path
|
1057
|
+
if user.is_admin or user.id == path_owner.id:
|
1012
1058
|
return AccessLevel.ALL
|
1013
1059
|
|
1014
1060
|
# if the path is a file, check if the user is the owner
|
@@ -1022,4 +1068,4 @@ async def check_path_permission(path: str, user: UserRecord, cursor: Optional[ai
|
|
1022
1068
|
# check alias level
|
1023
1069
|
async with this_cur() as cur:
|
1024
1070
|
uconn = UserConn(cur)
|
1025
|
-
return await uconn.query_peer_level(user.id,
|
1071
|
+
return await uconn.query_peer_level(user.id, path_owner.id)
|
lfss/eng/error.py
CHANGED
@@ -12,6 +12,8 @@ class InvalidPathError(LFSSExceptionBase, ValueError):...
|
|
12
12
|
|
13
13
|
class DatabaseLockedError(LFSSExceptionBase, sqlite3.DatabaseError):...
|
14
14
|
|
15
|
+
class DatabaseTransactionError(LFSSExceptionBase, sqlite3.DatabaseError):...
|
16
|
+
|
15
17
|
class PathNotFoundError(LFSSExceptionBase, FileNotFoundError):...
|
16
18
|
|
17
19
|
class FileDuplicateError(LFSSExceptionBase, FileExistsError):...
|
lfss/eng/utils.py
CHANGED
@@ -20,7 +20,7 @@ async def copy_file(source: str|pathlib.Path, destination: str|pathlib.Path):
|
|
20
20
|
await dest.write(chunk)
|
21
21
|
|
22
22
|
def hash_credential(username: str, password: str):
|
23
|
-
return hashlib.sha256(
|
23
|
+
return hashlib.sha256(f"{username}:{password}".encode()).hexdigest()
|
24
24
|
|
25
25
|
def encode_uri_compnents(path: str):
|
26
26
|
path_sp = path.split("/")
|
@@ -155,7 +155,7 @@ def fmt_storage_size(size: int) -> str:
|
|
155
155
|
return f"{size/1024**4:.2f}T"
|
156
156
|
|
157
157
|
_FnReturnT = TypeVar('_FnReturnT')
|
158
|
-
_AsyncReturnT = Awaitable
|
158
|
+
_AsyncReturnT = TypeVar('_AsyncReturnT', bound=Awaitable)
|
159
159
|
_g_executor = None
|
160
160
|
def get_global_executor():
|
161
161
|
global _g_executor
|
@@ -179,7 +179,7 @@ def concurrent_wrap(executor=None):
|
|
179
179
|
def sync_fn(*args, **kwargs):
|
180
180
|
loop = asyncio.new_event_loop()
|
181
181
|
return loop.run_until_complete(func(*args, **kwargs))
|
182
|
-
return sync_fn
|
182
|
+
return sync_fn # type: ignore
|
183
183
|
return _concurrent_wrap
|
184
184
|
|
185
185
|
# https://stackoverflow.com/a/279586/6775765
|
lfss/svc/app_base.py
CHANGED
@@ -27,7 +27,7 @@ req_conn = RequestDB()
|
|
27
27
|
async def lifespan(app: FastAPI):
|
28
28
|
global db
|
29
29
|
try:
|
30
|
-
await global_connection_init(n_read =
|
30
|
+
await global_connection_init(n_read = 8 if not DEBUG_MODE else 1)
|
31
31
|
await asyncio.gather(db.init(), req_conn.init())
|
32
32
|
yield
|
33
33
|
await req_conn.commit()
|
@@ -54,6 +54,7 @@ def handle_exception(fn):
|
|
54
54
|
if isinstance(e, FileExistsError): raise HTTPException(status_code=409, detail=str(e))
|
55
55
|
if isinstance(e, TooManyItemsError): raise HTTPException(status_code=400, detail=str(e))
|
56
56
|
if isinstance(e, DatabaseLockedError): raise HTTPException(status_code=503, detail=str(e))
|
57
|
+
if isinstance(e, DatabaseTransactionError): raise HTTPException(status_code=503, detail=str(e))
|
57
58
|
if isinstance(e, FileLockedError): raise HTTPException(status_code=423, detail=str(e))
|
58
59
|
logger.error(f"Uncaptured error in {fn.__name__}: {e}")
|
59
60
|
raise
|
lfss/svc/app_native.py
CHANGED
@@ -1,14 +1,15 @@
|
|
1
1
|
from typing import Optional, Literal
|
2
2
|
|
3
3
|
from fastapi import Depends, Request, Response, UploadFile
|
4
|
+
from fastapi.responses import StreamingResponse
|
4
5
|
from fastapi.exceptions import HTTPException
|
5
6
|
|
6
|
-
from ..eng.config import MAX_BUNDLE_BYTES
|
7
7
|
from ..eng.utils import ensure_uri_compnents
|
8
|
+
from ..eng.config import MAX_MEM_FILE_BYTES
|
8
9
|
from ..eng.connection_pool import unique_cursor
|
9
10
|
from ..eng.database import check_file_read_permission, check_path_permission, UserConn, FileConn
|
10
11
|
from ..eng.datatype import (
|
11
|
-
FileReadPermission,
|
12
|
+
FileReadPermission, UserRecord, AccessLevel,
|
12
13
|
FileSortKey, DirSortKey
|
13
14
|
)
|
14
15
|
|
@@ -81,48 +82,40 @@ async def delete_file(path: str, user: UserRecord = Depends(registered_user)):
|
|
81
82
|
async def bundle_files(path: str, user: UserRecord = Depends(registered_user)):
|
82
83
|
logger.info(f"GET bundle({path}), user: {user.username}")
|
83
84
|
path = ensure_uri_compnents(path)
|
84
|
-
|
85
|
-
|
86
|
-
if
|
85
|
+
if not path.endswith("/"):
|
86
|
+
raise HTTPException(status_code=400, detail="Path must end with /")
|
87
|
+
if path[0] == "/": # adapt to both /path and path
|
87
88
|
path = path[1:]
|
89
|
+
if path == "":
|
90
|
+
raise HTTPException(status_code=400, detail="Cannot bundle root")
|
88
91
|
|
89
|
-
|
90
|
-
|
91
|
-
|
92
|
-
|
93
|
-
|
94
|
-
|
95
|
-
|
96
|
-
|
97
|
-
|
98
|
-
|
99
|
-
|
100
|
-
|
101
|
-
|
102
|
-
|
103
|
-
|
104
|
-
|
105
|
-
|
106
|
-
|
107
|
-
|
108
|
-
|
109
|
-
|
110
|
-
|
111
|
-
|
112
|
-
|
113
|
-
|
114
|
-
|
115
|
-
|
116
|
-
raise HTTPException(status_code=400, detail="Too large to zip")
|
117
|
-
|
118
|
-
file_paths = [f.url for f in files]
|
119
|
-
zip_buffer = await db.zip_path(path, file_paths)
|
120
|
-
return Response(
|
121
|
-
content=zip_buffer.getvalue(), media_type="application/zip", headers={
|
122
|
-
"Content-Disposition": f"attachment; filename=bundle.zip",
|
123
|
-
"Content-Length": str(zip_buffer.getbuffer().nbytes)
|
124
|
-
}
|
125
|
-
)
|
92
|
+
async with unique_cursor() as cur:
|
93
|
+
dir_record = await FileConn(cur).get_path_record(path)
|
94
|
+
|
95
|
+
pathname = f"{path.split('/')[-2]}"
|
96
|
+
|
97
|
+
if dir_record.size < MAX_MEM_FILE_BYTES:
|
98
|
+
logger.debug(f"Bundle {path} in memory")
|
99
|
+
dir_bytes = (await db.zip_path(path, op_user=user)).getvalue()
|
100
|
+
return Response(
|
101
|
+
content = dir_bytes,
|
102
|
+
media_type = "application/zip",
|
103
|
+
headers = {
|
104
|
+
f"Content-Disposition": f"attachment; filename=bundle-{pathname}.zip",
|
105
|
+
"Content-Length": str(len(dir_bytes)),
|
106
|
+
"X-Content-Bytes": str(dir_record.size),
|
107
|
+
}
|
108
|
+
)
|
109
|
+
else:
|
110
|
+
logger.debug(f"Bundle {path} in stream")
|
111
|
+
return StreamingResponse(
|
112
|
+
content = await db.zip_path_stream(path, op_user=user),
|
113
|
+
media_type = "application/zip",
|
114
|
+
headers = {
|
115
|
+
f"Content-Disposition": f"attachment; filename=bundle-{pathname}.zip",
|
116
|
+
"X-Content-Bytes": str(dir_record.size),
|
117
|
+
}
|
118
|
+
)
|
126
119
|
|
127
120
|
@router_api.get("/meta")
|
128
121
|
@handle_exception
|
@@ -135,9 +128,7 @@ async def get_file_meta(path: str, user: UserRecord = Depends(registered_user)):
|
|
135
128
|
if is_file:
|
136
129
|
record = await fconn.get_file_record(path, throw=True)
|
137
130
|
if await check_path_permission(path, user, cursor=cur) < AccessLevel.READ:
|
138
|
-
|
139
|
-
owner = await uconn.get_user_by_id(record.owner_id, throw=True)
|
140
|
-
is_allowed, reason = check_file_read_permission(user, owner, record)
|
131
|
+
is_allowed, reason = await check_file_read_permission(user, record, cursor=cur)
|
141
132
|
if not is_allowed:
|
142
133
|
raise HTTPException(status_code=403, detail=reason)
|
143
134
|
else:
|
lfss/svc/common_impl.py
CHANGED
@@ -112,11 +112,8 @@ async def get_impl(
|
|
112
112
|
async with unique_cursor() as cur:
|
113
113
|
fconn = FileConn(cur)
|
114
114
|
file_record = await fconn.get_file_record(path, throw=True)
|
115
|
-
uconn = UserConn(cur)
|
116
|
-
owner = await uconn.get_user_by_id(file_record.owner_id, throw=True)
|
117
|
-
|
118
115
|
if not await check_path_permission(path, user, cursor=cur) >= AccessLevel.READ:
|
119
|
-
allow_access, reason = check_file_read_permission(user,
|
116
|
+
allow_access, reason = await check_file_read_permission(user, file_record, cursor=cur)
|
120
117
|
if not allow_access:
|
121
118
|
raise HTTPException(status_code=403 if user.id != 0 else 401, detail=reason)
|
122
119
|
|
@@ -1,6 +1,6 @@
|
|
1
1
|
Metadata-Version: 2.1
|
2
2
|
Name: lfss
|
3
|
-
Version: 0.
|
3
|
+
Version: 0.10.0
|
4
4
|
Summary: Lightweight file storage service
|
5
5
|
Home-page: https://github.com/MenxLi/lfss
|
6
6
|
Author: li_mengxun
|
@@ -17,12 +17,13 @@ Requires-Dist: mimesniff (==1.*)
|
|
17
17
|
Requires-Dist: pillow
|
18
18
|
Requires-Dist: python-multipart
|
19
19
|
Requires-Dist: requests (==2.*)
|
20
|
+
Requires-Dist: stream-zip (==0.*)
|
20
21
|
Requires-Dist: uvicorn (==0.*)
|
21
22
|
Project-URL: Repository, https://github.com/MenxLi/lfss
|
22
23
|
Description-Content-Type: text/markdown
|
23
24
|
|
24
|
-
#
|
25
|
-
[](https://pypi.org/project/lfss/)
|
25
|
+
# Lite File Storage Service (LFSS)
|
26
|
+
[](https://pypi.org/project/lfss/) [](https://pypi.org/project/lfss/)
|
26
27
|
|
27
28
|
My experiment on a lightweight and high-performance file/object storage service...
|
28
29
|
|
@@ -55,8 +56,8 @@ Or, you can start a web server at `/frontend` and open `index.html` in your brow
|
|
55
56
|
|
56
57
|
The API usage is simple, just `GET`, `PUT`, `DELETE` to the `/<username>/file/url` path.
|
57
58
|
The authentication can be acheived through one of the following methods:
|
58
|
-
1. `Authorization` header with the value `Bearer sha256(<username
|
59
|
-
2. `token` query parameter with the value `sha256(<username
|
59
|
+
1. `Authorization` header with the value `Bearer sha256(<username>:<password>)`.
|
60
|
+
2. `token` query parameter with the value `sha256(<username>:<password>)`.
|
60
61
|
3. HTTP Basic Authentication with the username and password (If WebDAV is enabled).
|
61
62
|
|
62
63
|
You can refer to `frontend` as an application example, `lfss/api/connector.py` for more APIs.
|
@@ -1,8 +1,8 @@
|
|
1
|
-
Readme.md,sha256=
|
2
|
-
docs/Changelog.md,sha256=
|
3
|
-
docs/Enviroment_variables.md,sha256=
|
1
|
+
Readme.md,sha256=B-foESzFWoSI5MEd89AWUzKcVRrTwipM28TK8GN0o8c,1920
|
2
|
+
docs/Changelog.md,sha256=QYej_hmGnv9t8wjFHXBvmrBOvY7aACZ82oa5SVkIyzM,882
|
3
|
+
docs/Enviroment_variables.md,sha256=xaL8qBwT8B2Qe11FaOU3xWrRCh1mJ1VyTFCeFbkd0rs,570
|
4
4
|
docs/Known_issues.md,sha256=ZqETcWP8lzTOel9b2mxEgCnADFF8IxOrEtiVO1NoMAk,251
|
5
|
-
docs/Permission.md,sha256=
|
5
|
+
docs/Permission.md,sha256=thUJx7YRoU63Pb-eqo5l5450DrZN3QYZ36GCn8r66no,3152
|
6
6
|
docs/Webdav.md,sha256=-Ja-BTWSY1BEMAyZycvEMNnkNTPZ49gSPzmf3Lbib70,1547
|
7
7
|
frontend/api.js,sha256=GlQsNoZFEcy7QUUsLbXv7aP-KxRnIxM37FQHTaakGiQ,19387
|
8
8
|
frontend/index.html,sha256=-k0bJ5FRqdl_H-O441D_H9E-iejgRCaL_z5UeYaS2qc,3384
|
@@ -19,7 +19,7 @@ frontend/thumb.css,sha256=rNsx766amYS2DajSQNabhpQ92gdTpNoQKmV69OKvtpI,295
|
|
19
19
|
frontend/thumb.js,sha256=46ViD2TlTTWy0fx6wjoAs_5CQ4ajYB90vVzM7UO2IHw,6182
|
20
20
|
frontend/utils.js,sha256=IYUZl77ugiXKcLxSNOWC4NSS0CdD5yRgUsDb665j0xM,2556
|
21
21
|
lfss/api/__init__.py,sha256=8IJqrpWK1doIyVVbntvVic82A57ncwl5b0BRHX4Ri6A,6660
|
22
|
-
lfss/api/connector.py,sha256=
|
22
|
+
lfss/api/connector.py,sha256=Duh57M3dOeG_M5UidZ4hMHK7ot1JsUC6RdXgIn6KTC8,12913
|
23
23
|
lfss/cli/__init__.py,sha256=lPwPmqpa7EXQ4zlU7E7LOe6X2kw_xATGdwoHphUEirA,827
|
24
24
|
lfss/cli/balance.py,sha256=fUbKKAUyaDn74f7mmxMfBL4Q4voyBLHu6Lg_g8GfMOQ,4121
|
25
25
|
lfss/cli/cli.py,sha256=aYjB8d4k6JUd9efxZK-XOj-mlG4JeOr_0lnj2qqCiK0,8066
|
@@ -29,23 +29,23 @@ lfss/cli/user.py,sha256=1mTroQbaKxHjFCPHT67xwd08v-zxH0RZ_OnVc-4MzL0,5364
|
|
29
29
|
lfss/cli/vacuum.py,sha256=GOG72d3NYe9bYCNc3y8JecEmM-DrKlGq3JQcisv_xBg,3702
|
30
30
|
lfss/eng/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
31
31
|
lfss/eng/bounded_pool.py,sha256=BI1dU-MBf82TMwJBYbjhEty7w1jIUKc5Bn9SnZ_-hoY,1288
|
32
|
-
lfss/eng/config.py,sha256=
|
33
|
-
lfss/eng/connection_pool.py,sha256=
|
34
|
-
lfss/eng/database.py,sha256=
|
32
|
+
lfss/eng/config.py,sha256=Vni6h52Ce0njVrHZLAWFL8g34YDBdlmGrmRhpxElxQ8,868
|
33
|
+
lfss/eng/connection_pool.py,sha256=1aq7nSgd7hB9YNV4PjD1RDRyl_moDw3ubBtSLyfgGBs,6320
|
34
|
+
lfss/eng/database.py,sha256=RYIG2506_-S84f6CsOQ6pgpr1vgPT3p1kP1FsZwTOnM,49098
|
35
35
|
lfss/eng/datatype.py,sha256=27UB7-l9SICy5lAvKjdzpTL_GohZjzstQcr9PtAq7nM,2709
|
36
|
-
lfss/eng/error.py,sha256=
|
36
|
+
lfss/eng/error.py,sha256=JGf5NV-f4rL6tNIDSAx5-l9MG8dEj7F2w_MuOjj1d1o,732
|
37
37
|
lfss/eng/log.py,sha256=u6WRZZsE7iOx6_CV2NHh1ugea26p408FI4WstZh896A,5139
|
38
38
|
lfss/eng/thumb.py,sha256=x9jIHHU1tskmp-TavPPcxGpbmEjCp9gbH6ZlsEfqUxY,3383
|
39
|
-
lfss/eng/utils.py,sha256=
|
39
|
+
lfss/eng/utils.py,sha256=WYoXFFi5308UWtFC8VP792gpzrVbHZZHhP3PaFjxIEY,6770
|
40
40
|
lfss/sql/init.sql,sha256=8LjHx0TBCkBD62xFfssSeHDqKYVQQJkZAg4rSm046f4,1496
|
41
41
|
lfss/sql/pragma.sql,sha256=uENx7xXjARmro-A3XAK8OM8v5AxDMdCCRj47f86UuXg,206
|
42
42
|
lfss/svc/app.py,sha256=ftWCpepBx-gTSG7i-TB-IdinPPstAYYQjCgnTfeMZeI,219
|
43
|
-
lfss/svc/app_base.py,sha256=
|
43
|
+
lfss/svc/app_base.py,sha256=bTQbz945xalyB3UZLlqVBvL6JKGNQ8Fm2KpIvvucPZQ,6850
|
44
44
|
lfss/svc/app_dav.py,sha256=D0KSgjtTktPjIhyIKG5eRmBdh5X8HYFYH151E6gzlbc,18245
|
45
|
-
lfss/svc/app_native.py,sha256=
|
46
|
-
lfss/svc/common_impl.py,sha256=
|
45
|
+
lfss/svc/app_native.py,sha256=JbPge-F9irl26tXKAzfA5DfyjCh0Dgttflztqqrvt0A,8890
|
46
|
+
lfss/svc/common_impl.py,sha256=5ZRM24zVZpAeipgDtZUVBMFtArkydlAkn17ic_XL7v8,13733
|
47
47
|
lfss/svc/request_log.py,sha256=v8yXEIzPjaksu76Oh5vgdbUEUrw8Kt4etLAXBWSGie8,3207
|
48
|
-
lfss-0.
|
49
|
-
lfss-0.
|
50
|
-
lfss-0.
|
51
|
-
lfss-0.
|
48
|
+
lfss-0.10.0.dist-info/METADATA,sha256=NewpmEw8OUj28rPkujuPi3ZySJG_JCvSEKY5JBxg6cw,2715
|
49
|
+
lfss-0.10.0.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
|
50
|
+
lfss-0.10.0.dist-info/entry_points.txt,sha256=VJ8svMz7RLtMCgNk99CElx7zo7M-N-z7BWDVw2HA92E,205
|
51
|
+
lfss-0.10.0.dist-info/RECORD,,
|
File without changes
|
File without changes
|