lfss 0.8.1__tar.gz → 0.8.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {lfss-0.8.1 → lfss-0.8.2}/PKG-INFO +1 -1
- {lfss-0.8.1 → lfss-0.8.2}/lfss/api/__init__.py +16 -2
- lfss-0.8.2/lfss/cli/__init__.py +27 -0
- {lfss-0.8.1 → lfss-0.8.2}/lfss/cli/cli.py +77 -8
- {lfss-0.8.1 → lfss-0.8.2}/lfss/cli/user.py +13 -4
- {lfss-0.8.1 → lfss-0.8.2}/lfss/src/database.py +44 -45
- {lfss-0.8.1 → lfss-0.8.2}/lfss/src/datatype.py +5 -6
- {lfss-0.8.1 → lfss-0.8.2}/lfss/src/server.py +14 -10
- {lfss-0.8.1 → lfss-0.8.2}/lfss/src/utils.py +20 -1
- {lfss-0.8.1 → lfss-0.8.2}/pyproject.toml +1 -1
- {lfss-0.8.1 → lfss-0.8.2}/Readme.md +0 -0
- {lfss-0.8.1 → lfss-0.8.2}/docs/Known_issues.md +0 -0
- {lfss-0.8.1 → lfss-0.8.2}/docs/Permission.md +0 -0
- {lfss-0.8.1 → lfss-0.8.2}/frontend/api.js +0 -0
- {lfss-0.8.1 → lfss-0.8.2}/frontend/index.html +0 -0
- {lfss-0.8.1 → lfss-0.8.2}/frontend/info.css +0 -0
- {lfss-0.8.1 → lfss-0.8.2}/frontend/info.js +0 -0
- {lfss-0.8.1 → lfss-0.8.2}/frontend/login.css +0 -0
- {lfss-0.8.1 → lfss-0.8.2}/frontend/login.js +0 -0
- {lfss-0.8.1 → lfss-0.8.2}/frontend/popup.css +0 -0
- {lfss-0.8.1 → lfss-0.8.2}/frontend/popup.js +0 -0
- {lfss-0.8.1 → lfss-0.8.2}/frontend/scripts.js +0 -0
- {lfss-0.8.1 → lfss-0.8.2}/frontend/state.js +0 -0
- {lfss-0.8.1 → lfss-0.8.2}/frontend/styles.css +0 -0
- {lfss-0.8.1 → lfss-0.8.2}/frontend/thumb.css +0 -0
- {lfss-0.8.1 → lfss-0.8.2}/frontend/thumb.js +0 -0
- {lfss-0.8.1 → lfss-0.8.2}/frontend/utils.js +0 -0
- {lfss-0.8.1 → lfss-0.8.2}/lfss/api/connector.py +0 -0
- {lfss-0.8.1 → lfss-0.8.2}/lfss/cli/balance.py +0 -0
- {lfss-0.8.1 → lfss-0.8.2}/lfss/cli/panel.py +0 -0
- {lfss-0.8.1 → lfss-0.8.2}/lfss/cli/serve.py +0 -0
- {lfss-0.8.1 → lfss-0.8.2}/lfss/cli/vacuum.py +0 -0
- {lfss-0.8.1 → lfss-0.8.2}/lfss/sql/init.sql +0 -0
- {lfss-0.8.1 → lfss-0.8.2}/lfss/sql/pragma.sql +0 -0
- {lfss-0.8.1 → lfss-0.8.2}/lfss/src/__init__.py +0 -0
- {lfss-0.8.1 → lfss-0.8.2}/lfss/src/bounded_pool.py +0 -0
- {lfss-0.8.1 → lfss-0.8.2}/lfss/src/config.py +0 -0
- {lfss-0.8.1 → lfss-0.8.2}/lfss/src/connection_pool.py +0 -0
- {lfss-0.8.1 → lfss-0.8.2}/lfss/src/error.py +0 -0
- {lfss-0.8.1 → lfss-0.8.2}/lfss/src/log.py +0 -0
- {lfss-0.8.1 → lfss-0.8.2}/lfss/src/stat.py +0 -0
- {lfss-0.8.1 → lfss-0.8.2}/lfss/src/thumb.py +0 -0
@@ -16,6 +16,11 @@ def upload_file(
|
|
16
16
|
) -> tuple[bool, str]:
|
17
17
|
this_try = 0
|
18
18
|
error_msg = ""
|
19
|
+
assert not file_path.endswith('/'), "File path must not end with a slash."
|
20
|
+
if dst_url.endswith('/'):
|
21
|
+
fname = file_path.split('/')[-1]
|
22
|
+
dst_url = f"{dst_url}{fname}"
|
23
|
+
|
19
24
|
while this_try <= n_retries:
|
20
25
|
try:
|
21
26
|
fsize = os.path.getsize(file_path)
|
@@ -31,7 +36,9 @@ def upload_file(
|
|
31
36
|
raise e
|
32
37
|
if verbose:
|
33
38
|
print(f"Error uploading {file_path}: {e}, retrying...")
|
34
|
-
|
39
|
+
error_msg = str(e)
|
40
|
+
if hasattr(e, 'response'):
|
41
|
+
error_msg = f"{error_msg}, {e.response.text}" # type: ignore
|
35
42
|
this_try += 1
|
36
43
|
finally:
|
37
44
|
time.sleep(interval)
|
@@ -95,7 +102,12 @@ def download_file(
|
|
95
102
|
) -> tuple[bool, str]:
|
96
103
|
this_try = 0
|
97
104
|
error_msg = ""
|
105
|
+
assert not src_url.endswith('/'), "Source URL must not end with a slash."
|
98
106
|
while this_try <= n_retries:
|
107
|
+
if os.path.isdir(file_path):
|
108
|
+
fname = src_url.split('/')[-1]
|
109
|
+
file_path = os.path.join(file_path, fname)
|
110
|
+
|
99
111
|
if not overwrite and os.path.exists(file_path):
|
100
112
|
if verbose:
|
101
113
|
print(f"File {file_path} already exists, skipping download.")
|
@@ -124,7 +136,9 @@ def download_file(
|
|
124
136
|
raise e
|
125
137
|
if verbose:
|
126
138
|
print(f"Error downloading {src_url}: {e}, retrying...")
|
127
|
-
|
139
|
+
error_msg = str(e)
|
140
|
+
if hasattr(e, 'response'):
|
141
|
+
error_msg = f"{error_msg}, {e.response.text}" # type: ignore
|
128
142
|
this_try += 1
|
129
143
|
finally:
|
130
144
|
time.sleep(interval)
|
@@ -0,0 +1,27 @@
|
|
1
|
+
from contextlib import contextmanager
|
2
|
+
from typing import Iterable, TypeVar, Generator
|
3
|
+
import requests, os
|
4
|
+
|
5
|
+
@contextmanager
|
6
|
+
def catch_request_error():
|
7
|
+
try:
|
8
|
+
yield
|
9
|
+
except requests.RequestException as e:
|
10
|
+
print(f"\033[31m[Request error]: {e}\033[0m")
|
11
|
+
if e.response is not None:
|
12
|
+
print(f"\033[91m[Error message]: {e.response.text}\033[0m")
|
13
|
+
|
14
|
+
T = TypeVar('T')
|
15
|
+
def line_sep(iter: Iterable[T], enable=True, start=True, end=True, color="\033[90m") -> Generator[T, None, None]:
|
16
|
+
screen_width = os.get_terminal_size().columns
|
17
|
+
def print_ln():
|
18
|
+
print(color + "-" * screen_width + "\033[0m")
|
19
|
+
|
20
|
+
if start and enable:
|
21
|
+
print_ln()
|
22
|
+
for i, line in enumerate(iter):
|
23
|
+
if enable and i > 0:
|
24
|
+
print_ln()
|
25
|
+
yield line
|
26
|
+
if end and enable:
|
27
|
+
print_ln()
|
@@ -1,7 +1,9 @@
|
|
1
1
|
from lfss.api import Connector, upload_directory, upload_file, download_file, download_directory
|
2
2
|
from pathlib import Path
|
3
|
-
import argparse
|
4
|
-
from lfss.src.datatype import FileReadPermission
|
3
|
+
import argparse, typing
|
4
|
+
from lfss.src.datatype import FileReadPermission, FileSortKey, DirSortKey
|
5
|
+
from lfss.src.utils import decode_uri_compnents
|
6
|
+
from . import catch_request_error, line_sep
|
5
7
|
|
6
8
|
def parse_permission(s: str) -> FileReadPermission:
|
7
9
|
if s.lower() == "public":
|
@@ -39,6 +41,29 @@ def parse_arguments():
|
|
39
41
|
sp_download.add_argument("--overwrite", action="store_true", help="Overwrite existing files")
|
40
42
|
sp_download.add_argument("--retries", type=int, default=0, help="Number of retries")
|
41
43
|
|
44
|
+
# query
|
45
|
+
sp_query = sp.add_parser("query", help="Query files or directories metadata from the server")
|
46
|
+
sp_query.add_argument("path", help="Path to query", nargs="*", type=str)
|
47
|
+
|
48
|
+
# list directories
|
49
|
+
sp_list_d = sp.add_parser("list-dirs", help="List directories of a given path")
|
50
|
+
sp_list_d.add_argument("path", help="Path to list", type=str)
|
51
|
+
sp_list_d.add_argument("--offset", type=int, default=0, help="Offset of the list")
|
52
|
+
sp_list_d.add_argument("--limit", type=int, default=100, help="Limit of the list")
|
53
|
+
sp_list_d.add_argument("-l", "--long", action="store_true", help="Detailed list, including all metadata")
|
54
|
+
sp_list_d.add_argument("--order", "--order-by", type=str, help="Order of the list", default="", choices=typing.get_args(DirSortKey))
|
55
|
+
sp_list_d.add_argument("--reverse", "--order-desc", action="store_true", help="Reverse the list order")
|
56
|
+
|
57
|
+
# list files
|
58
|
+
sp_list_f = sp.add_parser("list-files", help="List files of a given path")
|
59
|
+
sp_list_f.add_argument("path", help="Path to list", type=str)
|
60
|
+
sp_list_f.add_argument("--offset", type=int, default=0, help="Offset of the list")
|
61
|
+
sp_list_f.add_argument("--limit", type=int, default=100, help="Limit of the list")
|
62
|
+
sp_list_f.add_argument("-r", "--recursive", "--flat", action="store_true", help="List files recursively")
|
63
|
+
sp_list_f.add_argument("-l", "--long", action="store_true", help="Detailed list, including all metadata")
|
64
|
+
sp_list_f.add_argument("--order", "--order-by", type=str, help="Order of the list", default="", choices=typing.get_args(FileSortKey))
|
65
|
+
sp_list_f.add_argument("--reverse", "--order-desc", action="store_true", help="Reverse the list order")
|
66
|
+
|
42
67
|
return parser.parse_args()
|
43
68
|
|
44
69
|
def main():
|
@@ -57,11 +82,11 @@ def main():
|
|
57
82
|
permission=args.permission
|
58
83
|
)
|
59
84
|
if failed_upload:
|
60
|
-
print("
|
85
|
+
print("\033[91mFailed to upload:\033[0m")
|
61
86
|
for path in failed_upload:
|
62
87
|
print(f" {path}")
|
63
88
|
else:
|
64
|
-
success = upload_file(
|
89
|
+
success, msg = upload_file(
|
65
90
|
connector,
|
66
91
|
file_path = args.src,
|
67
92
|
dst_url = args.dst,
|
@@ -72,7 +97,7 @@ def main():
|
|
72
97
|
permission=args.permission
|
73
98
|
)
|
74
99
|
if not success:
|
75
|
-
print("
|
100
|
+
print("\033[91mFailed to upload: \033[0m", msg)
|
76
101
|
|
77
102
|
elif args.command == "download":
|
78
103
|
is_dir = args.src.endswith("/")
|
@@ -86,11 +111,11 @@ def main():
|
|
86
111
|
overwrite=args.overwrite
|
87
112
|
)
|
88
113
|
if failed_download:
|
89
|
-
print("
|
114
|
+
print("\033[91mFailed to download:\033[0m")
|
90
115
|
for path in failed_download:
|
91
116
|
print(f" {path}")
|
92
117
|
else:
|
93
|
-
success = download_file(
|
118
|
+
success, msg = download_file(
|
94
119
|
connector,
|
95
120
|
src_url = args.src,
|
96
121
|
file_path = args.dst,
|
@@ -100,7 +125,51 @@ def main():
|
|
100
125
|
overwrite=args.overwrite
|
101
126
|
)
|
102
127
|
if not success:
|
103
|
-
print("
|
128
|
+
print("\033[91mFailed to download: \033[0m", msg)
|
129
|
+
|
130
|
+
elif args.command == "query":
|
131
|
+
for path in args.path:
|
132
|
+
with catch_request_error():
|
133
|
+
res = connector.get_metadata(path)
|
134
|
+
if res is None:
|
135
|
+
print(f"\033[31mNot found\033[0m ({path})")
|
136
|
+
else:
|
137
|
+
print(res)
|
138
|
+
|
139
|
+
elif args.command == "list-files":
|
140
|
+
with catch_request_error():
|
141
|
+
res = connector.list_files(
|
142
|
+
args.path,
|
143
|
+
offset=args.offset,
|
144
|
+
limit=args.limit,
|
145
|
+
flat=args.recursive,
|
146
|
+
order_by=args.order,
|
147
|
+
order_desc=args.reverse,
|
148
|
+
)
|
149
|
+
for i, f in enumerate(line_sep(res)):
|
150
|
+
f.url = decode_uri_compnents(f.url)
|
151
|
+
print(f"[{i+1}] {f if args.long else f.url}")
|
152
|
+
|
153
|
+
if len(res) == args.limit:
|
154
|
+
print(f"\033[33m[Warning] List limit reached, use --offset and --limit to list more files.")
|
155
|
+
|
156
|
+
elif args.command == "list-dirs":
|
157
|
+
with catch_request_error():
|
158
|
+
res = connector.list_dirs(
|
159
|
+
args.path,
|
160
|
+
offset=args.offset,
|
161
|
+
limit=args.limit,
|
162
|
+
skim=not args.long,
|
163
|
+
order_by=args.order,
|
164
|
+
order_desc=args.reverse,
|
165
|
+
)
|
166
|
+
for i, d in enumerate(line_sep(res)):
|
167
|
+
d.url = decode_uri_compnents(d.url)
|
168
|
+
print(f"[{i+1}] {d if args.long else d.url}")
|
169
|
+
|
170
|
+
if len(res) == args.limit:
|
171
|
+
print(f"\033[33m[Warning] List limit reached, use --offset and --limit to list more directories.")
|
172
|
+
|
104
173
|
else:
|
105
174
|
raise NotImplementedError(f"Command {args.command} not implemented.")
|
106
175
|
|
@@ -1,8 +1,8 @@
|
|
1
|
-
import argparse, asyncio
|
1
|
+
import argparse, asyncio, os
|
2
2
|
from contextlib import asynccontextmanager
|
3
3
|
from .cli import parse_permission, FileReadPermission
|
4
|
-
from ..src.utils import parse_storage_size
|
5
|
-
from ..src.database import Database, FileReadPermission, transaction, UserConn
|
4
|
+
from ..src.utils import parse_storage_size, fmt_storage_size
|
5
|
+
from ..src.database import Database, FileReadPermission, transaction, UserConn, unique_cursor, FileConn
|
6
6
|
from ..src.connection_pool import global_entrance
|
7
7
|
|
8
8
|
@global_entrance(1)
|
@@ -33,6 +33,7 @@ async def _main():
|
|
33
33
|
sp_set.add_argument('--max-storage', type=parse_storage_size, default=None)
|
34
34
|
|
35
35
|
sp_list = sp.add_parser('list')
|
36
|
+
sp_list.add_argument("username", nargs='*', type=str, default=None)
|
36
37
|
sp_list.add_argument("-l", "--long", action="store_true")
|
37
38
|
|
38
39
|
args = parser.parse_args()
|
@@ -73,10 +74,18 @@ async def _main():
|
|
73
74
|
|
74
75
|
if args.subparser_name == 'list':
|
75
76
|
async with get_uconn() as uconn:
|
77
|
+
term_width = os.get_terminal_size().columns
|
76
78
|
async for user in uconn.all():
|
79
|
+
if args.username and not user.username in args.username:
|
80
|
+
continue
|
81
|
+
print("\033[90m-\033[0m" * term_width)
|
77
82
|
print(user)
|
78
83
|
if args.long:
|
79
|
-
|
84
|
+
async with unique_cursor() as c:
|
85
|
+
fconn = FileConn(c)
|
86
|
+
user_size_used = await fconn.user_size(user.id)
|
87
|
+
print('- Credential: ', user.credential)
|
88
|
+
print(f'- Storage: {fmt_storage_size(user_size_used)} / {fmt_storage_size(user.max_storage)}')
|
80
89
|
|
81
90
|
def main():
|
82
91
|
asyncio.run(_main())
|
@@ -21,6 +21,12 @@ from .utils import decode_uri_compnents, hash_credential, concurrent_wrap, debou
|
|
21
21
|
from .error import *
|
22
22
|
|
23
23
|
class DBObjectBase(ABC):
|
24
|
+
"""
|
25
|
+
NOTE:
|
26
|
+
The object of this class should hold a cursor to the database.
|
27
|
+
The methods calling the cursor should not be called concurrently.
|
28
|
+
"""
|
29
|
+
|
24
30
|
logger = get_logger('database', global_instance=True)
|
25
31
|
_cur: aiosqlite.Cursor
|
26
32
|
|
@@ -206,7 +212,7 @@ class FileConn(DBObjectBase):
|
|
206
212
|
return DirectoryRecord(dir_url)
|
207
213
|
else:
|
208
214
|
return await self.get_path_record(dir_url)
|
209
|
-
dirs = await
|
215
|
+
dirs = [await get_dir(url + d) for d in dirs_str]
|
210
216
|
return dirs
|
211
217
|
|
212
218
|
async def count_path_files(self, url: str, flat: bool = False):
|
@@ -308,18 +314,14 @@ class FileConn(DBObjectBase):
|
|
308
314
|
return res[0] or 0
|
309
315
|
|
310
316
|
async def update_file_record(
|
311
|
-
self, url,
|
317
|
+
self, url,
|
318
|
+
permission: Optional[FileReadPermission] = None,
|
319
|
+
mime_type: Optional[str] = None
|
312
320
|
):
|
313
|
-
|
314
|
-
|
315
|
-
if
|
316
|
-
|
317
|
-
if permission is None:
|
318
|
-
permission = old.permission
|
319
|
-
await self.cur.execute(
|
320
|
-
"UPDATE fmeta SET owner_id = ?, permission = ? WHERE url = ?",
|
321
|
-
(owner_id, int(permission), url)
|
322
|
-
)
|
321
|
+
if permission is not None:
|
322
|
+
await self.cur.execute("UPDATE fmeta SET permission = ? WHERE url = ?", (int(permission), url))
|
323
|
+
if mime_type is not None:
|
324
|
+
await self.cur.execute("UPDATE fmeta SET mime_type = ? WHERE url = ?", (mime_type, url))
|
323
325
|
self.logger.info(f"Updated file {url}")
|
324
326
|
|
325
327
|
async def set_file_record(
|
@@ -392,7 +394,7 @@ class FileConn(DBObjectBase):
|
|
392
394
|
self.logger.info(f"Deleted {len(ret)} file records for user {owner_id}") # type: ignore
|
393
395
|
return ret
|
394
396
|
|
395
|
-
async def delete_path_records(self, path: str,
|
397
|
+
async def delete_path_records(self, path: str, under_owner_id: Optional[int] = None) -> list[FileRecord]:
|
396
398
|
"""Delete all records with url starting with path"""
|
397
399
|
# update user size
|
398
400
|
cursor = await self.cur.execute("SELECT DISTINCT owner_id FROM fmeta WHERE url LIKE ?", (path + '%', ))
|
@@ -406,10 +408,10 @@ class FileConn(DBObjectBase):
|
|
406
408
|
# if any new records are created here, the size update may be inconsistent
|
407
409
|
# but it's not a big deal... we should have only one writer
|
408
410
|
|
409
|
-
if
|
411
|
+
if under_owner_id is None:
|
410
412
|
res = await self.cur.execute("DELETE FROM fmeta WHERE url LIKE ? RETURNING *", (path + '%', ))
|
411
413
|
else:
|
412
|
-
res = await self.cur.execute("DELETE FROM fmeta WHERE url LIKE ? AND owner_id = ? RETURNING *", (path + '%',
|
414
|
+
res = await self.cur.execute("DELETE FROM fmeta WHERE url LIKE ? AND owner_id = ? RETURNING *", (path + '%', under_owner_id))
|
413
415
|
all_f_rec = await res.fetchall()
|
414
416
|
self.logger.info(f"Deleted {len(all_f_rec)} file(s) for path {path}") # type: ignore
|
415
417
|
return [self.parse_record(r) for r in all_f_rec]
|
@@ -521,15 +523,16 @@ class Database:
|
|
521
523
|
await execute_sql(conn, 'init.sql')
|
522
524
|
return self
|
523
525
|
|
524
|
-
async def update_file_record(self,
|
526
|
+
async def update_file_record(self, url: str, permission: FileReadPermission, op_user: Optional[UserRecord] = None):
|
525
527
|
validate_url(url)
|
526
528
|
async with transaction() as conn:
|
527
529
|
fconn = FileConn(conn)
|
528
530
|
r = await fconn.get_file_record(url)
|
529
531
|
if r is None:
|
530
532
|
raise PathNotFoundError(f"File {url} not found")
|
531
|
-
if
|
532
|
-
|
533
|
+
if op_user is not None:
|
534
|
+
if r.owner_id != op_user.id and not op_user.is_admin:
|
535
|
+
raise PermissionDeniedError(f"Permission denied: {op_user.username} cannot update file {url}")
|
533
536
|
await fconn.update_file_record(url, permission=permission)
|
534
537
|
|
535
538
|
async def save_file(
|
@@ -561,8 +564,7 @@ class Database:
|
|
561
564
|
|
562
565
|
# check mime type
|
563
566
|
if mime_type is None:
|
564
|
-
|
565
|
-
mime_type, _ = mimetypes.guess_type(fname)
|
567
|
+
mime_type, _ = mimetypes.guess_type(url)
|
566
568
|
if mime_type is None:
|
567
569
|
await f.seek(0)
|
568
570
|
mime_type = mimesniff.what(await f.read(1024))
|
@@ -591,8 +593,6 @@ class Database:
|
|
591
593
|
await FileConn(w_cur).set_file_record(
|
592
594
|
url, owner_id=user.id, file_id=f_id, file_size=file_size,
|
593
595
|
permission=permission, external=True, mime_type=mime_type)
|
594
|
-
|
595
|
-
await delayed_log_activity(user.username)
|
596
596
|
return file_size
|
597
597
|
|
598
598
|
async def read_file_stream(self, url: str) -> AsyncIterable[bytes]:
|
@@ -605,11 +605,8 @@ class Database:
|
|
605
605
|
if not r.external:
|
606
606
|
raise ValueError(f"File {url} is not stored externally, should use read_file instead")
|
607
607
|
ret = fconn.get_file_blob_external(r.file_id)
|
608
|
-
|
609
|
-
await delayed_log_access(url)
|
610
608
|
return ret
|
611
609
|
|
612
|
-
|
613
610
|
async def read_file(self, url: str) -> bytes:
|
614
611
|
validate_url(url)
|
615
612
|
|
@@ -625,11 +622,9 @@ class Database:
|
|
625
622
|
blob = await fconn.get_file_blob(f_id)
|
626
623
|
if blob is None:
|
627
624
|
raise FileNotFoundError(f"File {url} data not found")
|
628
|
-
|
629
|
-
await delayed_log_access(url)
|
630
625
|
return blob
|
631
626
|
|
632
|
-
async def delete_file(self, url: str,
|
627
|
+
async def delete_file(self, url: str, op_user: Optional[UserRecord] = None) -> Optional[FileRecord]:
|
633
628
|
validate_url(url)
|
634
629
|
|
635
630
|
async with transaction() as cur:
|
@@ -637,10 +632,10 @@ class Database:
|
|
637
632
|
r = await fconn.delete_file_record(url)
|
638
633
|
if r is None:
|
639
634
|
return None
|
640
|
-
if
|
641
|
-
if r.owner_id !=
|
635
|
+
if op_user is not None:
|
636
|
+
if r.owner_id != op_user.id and not op_user.is_admin:
|
642
637
|
# will rollback
|
643
|
-
raise PermissionDeniedError(f"Permission denied: {
|
638
|
+
raise PermissionDeniedError(f"Permission denied: {op_user.username} cannot delete file {url}")
|
644
639
|
f_id = r.file_id
|
645
640
|
if r.external:
|
646
641
|
await fconn.delete_file_blob_external(f_id)
|
@@ -648,7 +643,7 @@ class Database:
|
|
648
643
|
await fconn.delete_file_blob(f_id)
|
649
644
|
return r
|
650
645
|
|
651
|
-
async def move_file(self, old_url: str, new_url: str,
|
646
|
+
async def move_file(self, old_url: str, new_url: str, op_user: Optional[UserRecord] = None):
|
652
647
|
validate_url(old_url)
|
653
648
|
validate_url(new_url)
|
654
649
|
|
@@ -657,12 +652,16 @@ class Database:
|
|
657
652
|
r = await fconn.get_file_record(old_url)
|
658
653
|
if r is None:
|
659
654
|
raise FileNotFoundError(f"File {old_url} not found")
|
660
|
-
if
|
661
|
-
if r.owner_id !=
|
662
|
-
raise PermissionDeniedError(f"Permission denied: {
|
655
|
+
if op_user is not None:
|
656
|
+
if r.owner_id != op_user.id:
|
657
|
+
raise PermissionDeniedError(f"Permission denied: {op_user.username} cannot move file {old_url}")
|
663
658
|
await fconn.move_file(old_url, new_url)
|
659
|
+
|
660
|
+
new_mime, _ = mimetypes.guess_type(new_url)
|
661
|
+
if not new_mime is None:
|
662
|
+
await fconn.update_file_record(new_url, mime_type=new_mime)
|
664
663
|
|
665
|
-
async def move_path(self,
|
664
|
+
async def move_path(self, old_url: str, new_url: str, op_user: UserRecord):
|
666
665
|
validate_url(old_url, is_file=False)
|
667
666
|
validate_url(new_url, is_file=False)
|
668
667
|
|
@@ -676,20 +675,20 @@ class Database:
|
|
676
675
|
|
677
676
|
async with transaction() as cur:
|
678
677
|
first_component = new_url.split('/')[0]
|
679
|
-
if not (first_component ==
|
680
|
-
raise PermissionDeniedError(f"Permission denied: path must start with {
|
681
|
-
elif
|
678
|
+
if not (first_component == op_user.username or op_user.is_admin):
|
679
|
+
raise PermissionDeniedError(f"Permission denied: path must start with {op_user.username}")
|
680
|
+
elif op_user.is_admin:
|
682
681
|
uconn = UserConn(cur)
|
683
682
|
_is_user = await uconn.get_user(first_component)
|
684
683
|
if not _is_user:
|
685
684
|
raise PermissionDeniedError(f"Invalid path: {first_component} is not a valid username")
|
686
685
|
|
687
686
|
# check if old path is under user's directory (non-admin)
|
688
|
-
if not old_url.startswith(
|
689
|
-
raise PermissionDeniedError(f"Permission denied: {
|
687
|
+
if not old_url.startswith(op_user.username + '/') and not op_user.is_admin:
|
688
|
+
raise PermissionDeniedError(f"Permission denied: {op_user.username} cannot move path {old_url}")
|
690
689
|
|
691
690
|
fconn = FileConn(cur)
|
692
|
-
await fconn.move_path(old_url, new_url, 'overwrite',
|
691
|
+
await fconn.move_path(old_url, new_url, 'overwrite', op_user.id)
|
693
692
|
|
694
693
|
async def __batch_delete_file_blobs(self, fconn: FileConn, file_records: list[FileRecord], batch_size: int = 512):
|
695
694
|
# https://github.com/langchain-ai/langchain/issues/10321
|
@@ -709,13 +708,13 @@ class Database:
|
|
709
708
|
await fconn.delete_file_blob_external(external_ids[i])
|
710
709
|
await asyncio.gather(del_internal(), del_external())
|
711
710
|
|
712
|
-
async def delete_path(self, url: str,
|
711
|
+
async def delete_path(self, url: str, op_user: Optional[UserRecord] = None) -> Optional[list[FileRecord]]:
|
713
712
|
validate_url(url, is_file=False)
|
714
|
-
|
713
|
+
from_owner_id = op_user.id if op_user is not None and not op_user.is_admin else None
|
715
714
|
|
716
715
|
async with transaction() as cur:
|
717
716
|
fconn = FileConn(cur)
|
718
|
-
records = await fconn.delete_path_records(url,
|
717
|
+
records = await fconn.delete_path_records(url, from_owner_id)
|
719
718
|
if not records:
|
720
719
|
return None
|
721
720
|
await self.__batch_delete_file_blobs(fconn, records)
|
@@ -1,6 +1,5 @@
|
|
1
1
|
from enum import IntEnum
|
2
|
-
|
3
|
-
import dataclasses
|
2
|
+
import dataclasses, typing
|
4
3
|
|
5
4
|
class FileReadPermission(IntEnum):
|
6
5
|
UNSET = 0 # not set
|
@@ -55,7 +54,7 @@ class PathContents:
|
|
55
54
|
dirs: list[DirectoryRecord] = dataclasses.field(default_factory=list)
|
56
55
|
files: list[FileRecord] = dataclasses.field(default_factory=list)
|
57
56
|
|
58
|
-
FileSortKey = Literal['', 'url', 'file_size', 'create_time', 'access_time', 'mime_type']
|
59
|
-
isValidFileSortKey = lambda x: x in
|
60
|
-
DirSortKey = Literal['', 'dirname']
|
61
|
-
isValidDirSortKey = lambda x: x in
|
57
|
+
FileSortKey = typing.Literal['', 'url', 'file_size', 'create_time', 'access_time', 'mime_type']
|
58
|
+
isValidFileSortKey = lambda x: x in typing.get_args(FileSortKey)
|
59
|
+
DirSortKey = typing.Literal['', 'dirname']
|
60
|
+
isValidDirSortKey = lambda x: x in typing.get_args(DirSortKey)
|
@@ -6,7 +6,6 @@ from fastapi.responses import StreamingResponse
|
|
6
6
|
from fastapi.exceptions import HTTPException
|
7
7
|
from fastapi.security import HTTPBearer, HTTPAuthorizationCredentials
|
8
8
|
from fastapi.middleware.cors import CORSMiddleware
|
9
|
-
import mimesniff
|
10
9
|
|
11
10
|
import asyncio, json, time
|
12
11
|
from contextlib import asynccontextmanager
|
@@ -14,10 +13,11 @@ from contextlib import asynccontextmanager
|
|
14
13
|
from .error import *
|
15
14
|
from .log import get_logger
|
16
15
|
from .stat import RequestDB
|
17
|
-
from .config import MAX_BUNDLE_BYTES,
|
16
|
+
from .config import MAX_BUNDLE_BYTES, CHUNK_SIZE
|
18
17
|
from .utils import ensure_uri_compnents, format_last_modified, now_stamp, wait_for_debounce_tasks
|
19
18
|
from .connection_pool import global_connection_init, global_connection_close, unique_cursor
|
20
|
-
from .database import Database, DECOY_USER, check_user_permission, UserConn, FileConn
|
19
|
+
from .database import Database, DECOY_USER, check_user_permission, UserConn, FileConn
|
20
|
+
from .database import delayed_log_activity, delayed_log_access
|
21
21
|
from .datatype import (
|
22
22
|
FileReadPermission, FileRecord, UserRecord, PathContents,
|
23
23
|
FileSortKey, DirSortKey
|
@@ -80,6 +80,10 @@ async def get_current_user(
|
|
80
80
|
|
81
81
|
if not user:
|
82
82
|
raise HTTPException(status_code=401, detail="Invalid token")
|
83
|
+
|
84
|
+
if not user.id == 0:
|
85
|
+
await delayed_log_activity(user.username)
|
86
|
+
|
83
87
|
return user
|
84
88
|
|
85
89
|
async def registered_user(user: UserRecord = Depends(get_current_user)):
|
@@ -165,6 +169,8 @@ async def emit_file(
|
|
165
169
|
media_type = file_record.mime_type
|
166
170
|
path = file_record.url
|
167
171
|
fname = path.split("/")[-1]
|
172
|
+
|
173
|
+
await delayed_log_access(path)
|
168
174
|
if not file_record.external:
|
169
175
|
fblob = await db.read_file(path)
|
170
176
|
return Response(
|
@@ -360,11 +366,10 @@ async def delete_file(path: str, user: UserRecord = Depends(registered_user)):
|
|
360
366
|
logger.info(f"DELETE {path}, user: {user.username}")
|
361
367
|
|
362
368
|
if path.endswith("/"):
|
363
|
-
res = await db.delete_path(path, user
|
369
|
+
res = await db.delete_path(path, user)
|
364
370
|
else:
|
365
|
-
res = await db.delete_file(path, user
|
371
|
+
res = await db.delete_file(path, user)
|
366
372
|
|
367
|
-
await delayed_log_activity(user.username)
|
368
373
|
if res:
|
369
374
|
return Response(status_code=200, content="Deleted")
|
370
375
|
else:
|
@@ -456,16 +461,15 @@ async def update_file_meta(
|
|
456
461
|
path = ensure_uri_compnents(path)
|
457
462
|
if path.startswith("/"):
|
458
463
|
path = path[1:]
|
459
|
-
await delayed_log_activity(user.username)
|
460
464
|
|
461
465
|
# file
|
462
466
|
if not path.endswith("/"):
|
463
467
|
if perm is not None:
|
464
468
|
logger.info(f"Update permission of {path} to {perm}")
|
465
469
|
await db.update_file_record(
|
466
|
-
user = user,
|
467
470
|
url = path,
|
468
|
-
permission = FileReadPermission(perm)
|
471
|
+
permission = FileReadPermission(perm),
|
472
|
+
op_user = user,
|
469
473
|
)
|
470
474
|
|
471
475
|
if new_path is not None:
|
@@ -480,7 +484,7 @@ async def update_file_meta(
|
|
480
484
|
new_path = ensure_uri_compnents(new_path)
|
481
485
|
logger.info(f"Update path of {path} to {new_path}")
|
482
486
|
# currently only move own file, with overwrite
|
483
|
-
await db.move_path(
|
487
|
+
await db.move_path(path, new_path, user)
|
484
488
|
|
485
489
|
return Response(status_code=200, content="OK")
|
486
490
|
|
@@ -109,6 +109,17 @@ def parse_storage_size(s: str) -> int:
|
|
109
109
|
case 'g': return int(s[:-1]) * 1024**3
|
110
110
|
case 't': return int(s[:-1]) * 1024**4
|
111
111
|
case _: raise ValueError(f"Invalid file size string: {s}")
|
112
|
+
def fmt_storage_size(size: int) -> str:
|
113
|
+
""" Format the file size to human-readable format """
|
114
|
+
if size < 1024:
|
115
|
+
return f"{size}B"
|
116
|
+
if size < 1024**2:
|
117
|
+
return f"{size/1024:.2f}K"
|
118
|
+
if size < 1024**3:
|
119
|
+
return f"{size/1024**2:.2f}M"
|
120
|
+
if size < 1024**4:
|
121
|
+
return f"{size/1024**3:.2f}G"
|
122
|
+
return f"{size/1024**4:.2f}T"
|
112
123
|
|
113
124
|
_FnReturnT = TypeVar('_FnReturnT')
|
114
125
|
_AsyncReturnT = Awaitable[_FnReturnT]
|
@@ -136,4 +147,12 @@ def concurrent_wrap(executor=None):
|
|
136
147
|
loop = asyncio.new_event_loop()
|
137
148
|
return loop.run_until_complete(func(*args, **kwargs))
|
138
149
|
return sync_fn
|
139
|
-
return _concurrent_wrap
|
150
|
+
return _concurrent_wrap
|
151
|
+
|
152
|
+
# https://stackoverflow.com/a/279586/6775765
|
153
|
+
def static_vars(**kwargs):
|
154
|
+
def decorate(func):
|
155
|
+
for k in kwargs:
|
156
|
+
setattr(func, k, kwargs[k])
|
157
|
+
return func
|
158
|
+
return decorate
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|