lfss 0.8.1__tar.gz → 0.8.3__tar.gz

{lfss-0.8.1 → lfss-0.8.3}/PKG-INFO

@@ -1,13 +1,12 @@
 Metadata-Version: 2.1
 Name: lfss
-Version: 0.8.1
+Version: 0.8.3
 Summary: Lightweight file storage service
 Home-page: https://github.com/MenxLi/lfss
 Author: li, mengxun
 Author-email: limengxun45@outlook.com
-Requires-Python: >=3.9
+Requires-Python: >=3.10
 Classifier: Programming Language :: Python :: 3
-Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12

{lfss-0.8.1 → lfss-0.8.3}/frontend/info.js

@@ -12,8 +12,10 @@ const ensureSlashEnd = (path) => {
 /**
  * @param {FileRecord} r
  * @param {UserRecord} u
+ * @param {Connector} c
  */
-export function showInfoPanel(r, u){
+export function showInfoPanel(r, u, c){
+    const origin = c.config.endpoint;
     const innerHTML = `
 <div class="info-container">
     <div class="info-container-left">
@@ -46,7 +48,7 @@ export function showInfoPanel(r, u){
     </div>
     <div class="info-container-right">
         <div class="info-path-copy">
-            <input type="text" value="${window.location.origin}/${r.url}" readonly>
+            <input type="text" value="${origin}/${r.url}" readonly>
             <button class="copy-button" id='copy-btn-full-path'>📋</button>
         </div>
         <div class="info-path-copy">
@@ -58,7 +60,7 @@ export function showInfoPanel(r, u){
     `
     const [win, closeWin] = createFloatingWindow(innerHTML, {title: 'File Info'});
     document.getElementById('copy-btn-full-path').onclick = () => {
-        copyToClipboard(window.location.origin + '/' + r.url);
+        copyToClipboard(origin + '/' + r.url);
         showPopup('Path copied to clipboard', {timeout: 2000, level: 'success'});
     }
     document.getElementById('copy-btn-rel-path').onclick = () => {
@@ -77,6 +79,7 @@ export function showDirInfoPanel(r, u, c){
     if (fmtPath.endsWith('/')) {
         fmtPath = fmtPath.slice(0, -1);
     }
+    const origin = c.config.endpoint;
     const innerHTML = `
 <div class="info-container">
     <div class="info-container-left">
@@ -105,7 +108,7 @@ export function showDirInfoPanel(r, u, c){
     </div>
     <div class="info-container-right">
         <div class="info-path-copy">
-            <input type="text" value="${window.location.origin}/${ensureSlashEnd(r.url)}" readonly>
+            <input type="text" value="${origin}/${ensureSlashEnd(r.url)}" readonly>
             <button class="copy-button" id='copy-btn-full-path'>📋</button>
         </div>
         <div class="info-path-copy">
@@ -117,7 +120,7 @@ export function showDirInfoPanel(r, u, c){
     `
     const [win, closeWin] = createFloatingWindow(innerHTML, {title: 'File Info'});
     document.getElementById('copy-btn-full-path').onclick = () => {
-        copyToClipboard(window.location.origin + '/' + ensureSlashEnd(r.url));
+        copyToClipboard(origin + '/' + ensureSlashEnd(r.url));
         showPopup('Path copied to clipboard', {timeout: 2000, level: 'success'});
     }
     document.getElementById('copy-btn-rel-path').onclick = () => {

{lfss-0.8.1 → lfss-0.8.3}/frontend/scripts.js

@@ -459,7 +459,7 @@ async function refreshFileList(){
                     infoButton.style.cursor = 'pointer';
                     infoButton.textContent = 'Details';
                     infoButton.addEventListener('click', () => {
-                        showInfoPanel(file, userRecord);
+                        showInfoPanel(file, userRecord, conn);
                     });
                     actContainer.appendChild(infoButton);
 

{lfss-0.8.1 → lfss-0.8.3}/frontend/thumb.js

@@ -13,6 +13,7 @@ const ICON_ZIP = '<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><t
 const ICON_CODE = '<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><title>file-code-outline</title><path d="M14 2H6C4.89 2 4 2.9 4 4V20C4 21.11 4.89 22 6 22H18C19.11 22 20 21.11 20 20V8L14 2M18 20H6V4H13V9H18V20M9.54 15.65L11.63 17.74L10.35 19L7 15.65L10.35 12.3L11.63 13.56L9.54 15.65M17 15.65L13.65 19L12.38 17.74L14.47 15.65L12.38 13.56L13.65 12.3L17 15.65Z" /></svg>'
 const ICON_VIDEO = '<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><title>television-play</title><path d="M21,3H3C1.89,3 1,3.89 1,5V17A2,2 0 0,0 3,19H8V21H16V19H21A2,2 0 0,0 23,17V5C23,3.89 22.1,3 21,3M21,17H3V5H21M16,11L9,15V7" /></svg>'
 const ICON_IMAGE = '<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><title>image-outline</title><path d="M19,19H5V5H19M19,3H5A2,2 0 0,0 3,5V19A2,2 0 0,0 5,21H19A2,2 0 0,0 21,19V5A2,2 0 0,0 19,3M13.96,12.29L11.21,15.83L9.25,13.47L6.5,17H17.5L13.96,12.29Z" /></svg>'
+const ICON_MUSIC = '<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><title>music-box-outline</title><path d="M16,9H13V14.5A2.5,2.5 0 0,1 10.5,17A2.5,2.5 0 0,1 8,14.5A2.5,2.5 0 0,1 10.5,12C11.07,12 11.58,12.19 12,12.5V7H16V9M19,3A2,2 0 0,1 21,5V19A2,2 0 0,1 19,21H5A2,2 0 0,1 3,19V5A2,2 0 0,1 5,3H19M5,5V19H19V5H5Z" /></svg>'
 
 function getIconSVGFromMimeType(mimeType){
     if (mimeType == 'directory'){
@@ -24,6 +25,9 @@ function getIconSVGFromMimeType(mimeType){
     if (mimeType.startsWith('image/')){
         return ICON_IMAGE;
     }
+    if (mimeType.startsWith('audio/')){
+        return ICON_MUSIC;
+    }
 
     if (['application/pdf', 'application/x-pdf'].includes(mimeType)){
         return ICON_PDF;

{lfss-0.8.1 → lfss-0.8.3}/lfss/api/__init__.py

@@ -16,6 +16,11 @@ def upload_file(
     ) -> tuple[bool, str]:
     this_try = 0
     error_msg = ""
+    assert not file_path.endswith('/'), "File path must not end with a slash."
+    if dst_url.endswith('/'):
+        fname = file_path.split('/')[-1]
+        dst_url = f"{dst_url}{fname}"
+
     while this_try <= n_retries:
         try:
             fsize = os.path.getsize(file_path)
@@ -31,7 +36,9 @@ def upload_file(
                 raise e
             if verbose:
                 print(f"Error uploading {file_path}: {e}, retrying...")
-                error_msg = str(e)
+            error_msg = str(e)
+            if hasattr(e, 'response'):
+                error_msg = f"{error_msg}, {e.response.text}"   # type: ignore
             this_try += 1
         finally:
             time.sleep(interval)
@@ -95,7 +102,12 @@ def download_file(
     ) -> tuple[bool, str]:
     this_try = 0
     error_msg = ""
+    assert not src_url.endswith('/'), "Source URL must not end with a slash."
     while this_try <= n_retries:
+        if os.path.isdir(file_path):
+            fname = src_url.split('/')[-1]
+            file_path = os.path.join(file_path, fname)
+
         if not overwrite and os.path.exists(file_path):
             if verbose:
                 print(f"File {file_path} already exists, skipping download.")
@@ -124,7 +136,9 @@ def download_file(
                 raise e
             if verbose:
                 print(f"Error downloading {src_url}: {e}, retrying...")
-                error_msg = str(e)
+            error_msg = str(e)
+            if hasattr(e, 'response'):
+                error_msg = f"{error_msg}, {e.response.text}"   # type: ignore
             this_try += 1
         finally:
             time.sleep(interval)

{lfss-0.8.1 → lfss-0.8.3}/lfss/api/connector.py

@@ -54,7 +54,7 @@ class Connector:
     
     def _fetch_factory(
         self, method: Literal['GET', 'POST', 'PUT', 'DELETE'], 
-        path: str, search_params: dict = {}
+        path: str, search_params: dict = {}, extra_headers: dict = {}
     ):
         if path.startswith('/'):
             path = path[1:]
@@ -65,6 +65,7 @@ class Connector:
             headers.update({
                 'Authorization': f"Bearer {self.config['token']}",
             })
+            headers.update(extra_headers)
             if self._session is not None:
                 response = self._session.request(method, url, headers=headers, **kwargs)
                 response.raise_for_status()
@@ -168,6 +169,17 @@ class Connector:
         response = self._get(path)
         if response is None: return None
         return response.content
+
+    def get_partial(self, path: str, range_start: int = -1, range_end: int = -1) -> Optional[bytes]:
+        """
+        Downloads a partial file from the specified path.
+        start and end are the byte offsets, both inclusive.
+        """
+        response = self._fetch_factory('GET', path, extra_headers={
+            'Range': f"bytes={range_start if range_start >= 0 else ''}-{range_end if range_end >= 0 else ''}"
+        })()
+        if response is None: return None
+        return response.content
     
     def get_stream(self, path: str) -> Iterator[bytes]:
         """Downloads a file from the specified path, will raise PathNotFoundError if path not found."""

lfss-0.8.3/lfss/cli/__init__.py

@@ -0,0 +1,27 @@
+from contextlib import contextmanager
+from typing import Iterable, TypeVar, Generator
+import requests, os
+
+@contextmanager
+def catch_request_error():
+    try:
+        yield
+    except requests.RequestException as e:
+        print(f"\033[31m[Request error]: {e}\033[0m")
+        if e.response is not None:
+            print(f"\033[91m[Error message]: {e.response.text}\033[0m")
+
+T = TypeVar('T')
+def line_sep(iter: Iterable[T], enable=True, start=True, end=True, color="\033[90m") -> Generator[T, None, None]:
+    screen_width = os.get_terminal_size().columns
+    def print_ln():
+        print(color + "-" * screen_width + "\033[0m")
+
+    if start and enable:
+        print_ln()
+    for i, line in enumerate(iter):
+        if enable and i > 0:
+            print_ln()
+        yield line
+    if end and enable:
+        print_ln()

{lfss-0.8.1 → lfss-0.8.3}/lfss/cli/cli.py

@@ -1,7 +1,9 @@
 from lfss.api import Connector, upload_directory, upload_file, download_file, download_directory
 from pathlib import Path
-import argparse
-from lfss.src.datatype import FileReadPermission
+import argparse, typing
+from lfss.src.datatype import FileReadPermission, FileSortKey, DirSortKey
+from lfss.src.utils import decode_uri_compnents
+from . import catch_request_error, line_sep
 
 def parse_permission(s: str) -> FileReadPermission:
     if s.lower() == "public":
@@ -39,6 +41,29 @@ def parse_arguments():
     sp_download.add_argument("--overwrite", action="store_true", help="Overwrite existing files")
     sp_download.add_argument("--retries", type=int, default=0, help="Number of retries")
 
+    # query
+    sp_query = sp.add_parser("query", help="Query files or directories metadata from the server")
+    sp_query.add_argument("path", help="Path to query", nargs="*", type=str)
+
+    # list directories
+    sp_list_d = sp.add_parser("list-dirs", help="List directories of a given path")
+    sp_list_d.add_argument("path", help="Path to list", type=str)
+    sp_list_d.add_argument("--offset", type=int, default=0, help="Offset of the list")
+    sp_list_d.add_argument("--limit", type=int, default=100, help="Limit of the list")
+    sp_list_d.add_argument("-l", "--long", action="store_true", help="Detailed list, including all metadata")
+    sp_list_d.add_argument("--order", "--order-by", type=str, help="Order of the list", default="", choices=typing.get_args(DirSortKey))
+    sp_list_d.add_argument("--reverse", "--order-desc", action="store_true", help="Reverse the list order")
+
+    # list files
+    sp_list_f = sp.add_parser("list-files", help="List files of a given path")
+    sp_list_f.add_argument("path", help="Path to list", type=str)
+    sp_list_f.add_argument("--offset", type=int, default=0, help="Offset of the list")
+    sp_list_f.add_argument("--limit", type=int, default=100, help="Limit of the list")
+    sp_list_f.add_argument("-r", "--recursive", "--flat", action="store_true", help="List files recursively")
+    sp_list_f.add_argument("-l", "--long", action="store_true", help="Detailed list, including all metadata")
+    sp_list_f.add_argument("--order", "--order-by", type=str, help="Order of the list", default="", choices=typing.get_args(FileSortKey))
+    sp_list_f.add_argument("--reverse", "--order-desc", action="store_true", help="Reverse the list order")
+
     return parser.parse_args()
 
 def main():
@@ -57,11 +82,11 @@ def main():
                 permission=args.permission
             )
             if failed_upload:
-                print("Failed to upload:")
+                print("\033[91mFailed to upload:\033[0m")
                 for path in failed_upload:
                     print(f"  {path}")
         else:
-            success = upload_file(
+            success, msg = upload_file(
                 connector, 
                 file_path = args.src, 
                 dst_url = args.dst, 
@@ -72,7 +97,7 @@ def main():
                 permission=args.permission
             )
             if not success:
-                print("Failed to upload.")
+                print("\033[91mFailed to upload: \033[0m", msg)
     
     elif args.command == "download":
         is_dir = args.src.endswith("/")
@@ -86,11 +111,11 @@ def main():
                 overwrite=args.overwrite
             )
             if failed_download:
-                print("Failed to download:")
+                print("\033[91mFailed to download:\033[0m")
                 for path in failed_download:
                     print(f"  {path}")
         else:
-            success = download_file(
+            success, msg = download_file(
                 connector, 
                 src_url = args.src, 
                 file_path = args.dst, 
@@ -100,7 +125,51 @@ def main():
                 overwrite=args.overwrite
             )
             if not success:
-                print("Failed to download.")
+                print("\033[91mFailed to download: \033[0m", msg)
+    
+    elif args.command == "query":
+        for path in args.path:
+            with catch_request_error():
+                res = connector.get_metadata(path)
+                if res is None:
+                    print(f"\033[31mNot found\033[0m ({path})")
+                else:
+                    print(res)
+    
+    elif args.command == "list-files":
+        with catch_request_error():
+            res = connector.list_files(
+                args.path, 
+                offset=args.offset, 
+                limit=args.limit, 
+                flat=args.recursive, 
+                order_by=args.order,
+                order_desc=args.reverse,
+            )
+            for i, f in enumerate(line_sep(res)):
+                f.url = decode_uri_compnents(f.url)
+                print(f"[{i+1}] {f if args.long else f.url}")
+
+            if len(res) == args.limit:
+                print(f"\033[33m[Warning] List limit reached, use --offset and --limit to list more files.")
+        
+    elif args.command == "list-dirs":
+        with catch_request_error():
+            res = connector.list_dirs(
+                args.path, 
+                offset=args.offset, 
+                limit=args.limit, 
+                skim=not args.long, 
+                order_by=args.order,
+                order_desc=args.reverse,
+            )
+            for i, d in enumerate(line_sep(res)):
+                d.url = decode_uri_compnents(d.url)
+                print(f"[{i+1}] {d if args.long else d.url}")
+
+            if len(res) == args.limit:
+                print(f"\033[33m[Warning] List limit reached, use --offset and --limit to list more directories.")
+    
     else:
         raise NotImplementedError(f"Command {args.command} not implemented.")
     

{lfss-0.8.1 → lfss-0.8.3}/lfss/cli/user.py

@@ -1,8 +1,8 @@
-import argparse, asyncio
+import argparse, asyncio, os
 from contextlib import asynccontextmanager
 from .cli import parse_permission, FileReadPermission
-from ..src.utils import parse_storage_size
-from ..src.database import Database, FileReadPermission, transaction, UserConn
+from ..src.utils import parse_storage_size, fmt_storage_size
+from ..src.database import Database, FileReadPermission, transaction, UserConn, unique_cursor, FileConn
 from ..src.connection_pool import global_entrance
 
 @global_entrance(1)
@@ -33,6 +33,7 @@ async def _main():
     sp_set.add_argument('--max-storage', type=parse_storage_size, default=None)
     
     sp_list = sp.add_parser('list')
+    sp_list.add_argument("username", nargs='*', type=str, default=None)
     sp_list.add_argument("-l", "--long", action="store_true")
     
     args = parser.parse_args()
@@ -73,10 +74,18 @@ async def _main():
     
     if args.subparser_name == 'list':
         async with get_uconn() as uconn:
+            term_width = os.get_terminal_size().columns
             async for user in uconn.all():
+                if args.username and not user.username in args.username:
+                    continue
+                print("\033[90m-\033[0m" * term_width)
                 print(user)
                 if args.long:
-                    print('  ', user.credential)
+                    async with unique_cursor() as c:
+                        fconn = FileConn(c)
+                        user_size_used = await fconn.user_size(user.id)
+                    print('- Credential: ', user.credential)
+                    print(f'- Storage: {fmt_storage_size(user_size_used)} / {fmt_storage_size(user.max_storage)}')
         
 def main():
     asyncio.run(_main())

{lfss-0.8.1 → lfss-0.8.3}/lfss/src/database.py

@@ -21,6 +21,12 @@ from .utils import decode_uri_compnents, hash_credential, concurrent_wrap, debou
 from .error import *
 
 class DBObjectBase(ABC):
+    """
+    NOTE: 
+    The object of this class should hold a cursor to the database. 
+    The methods calling the cursor should not be called concurrently. 
+    """
+
     logger = get_logger('database', global_instance=True)
     _cur: aiosqlite.Cursor
 
@@ -206,7 +212,7 @@ class FileConn(DBObjectBase):
                 return DirectoryRecord(dir_url)
             else:
                 return await self.get_path_record(dir_url)
-        dirs = await asyncio.gather(*[get_dir(url + d) for d in dirs_str])
+        dirs = [await get_dir(url + d) for d in dirs_str]
         return dirs
     
     async def count_path_files(self, url: str, flat: bool = False):
@@ -308,18 +314,14 @@ class FileConn(DBObjectBase):
         return res[0] or 0
     
     async def update_file_record(
-        self, url, owner_id: Optional[int] = None, permission: Optional[FileReadPermission] = None
+        self, url, 
+        permission: Optional[FileReadPermission] = None, 
+        mime_type: Optional[str] = None
         ):
-        old = await self.get_file_record(url)
-        assert old is not None, f"File {url} not found"
-        if owner_id is None:
-            owner_id = old.owner_id
-        if permission is None:
-            permission = old.permission
-        await self.cur.execute(
-            "UPDATE fmeta SET owner_id = ?, permission = ? WHERE url = ?", 
-            (owner_id, int(permission), url)
-            )
+        if permission is not None:
+            await self.cur.execute("UPDATE fmeta SET permission = ? WHERE url = ?", (int(permission), url))
+        if mime_type is not None:
+            await self.cur.execute("UPDATE fmeta SET mime_type = ? WHERE url = ?", (mime_type, url))
         self.logger.info(f"Updated file {url}")
     
     async def set_file_record(
@@ -392,7 +394,7 @@ class FileConn(DBObjectBase):
         self.logger.info(f"Deleted {len(ret)} file records for user {owner_id}") # type: ignore
         return ret
     
-    async def delete_path_records(self, path: str, under_user_id: Optional[int] = None) -> list[FileRecord]:
+    async def delete_path_records(self, path: str, under_owner_id: Optional[int] = None) -> list[FileRecord]:
         """Delete all records with url starting with path"""
         # update user size
         cursor = await self.cur.execute("SELECT DISTINCT owner_id FROM fmeta WHERE url LIKE ?", (path + '%', ))
@@ -406,10 +408,10 @@ class FileConn(DBObjectBase):
         # if any new records are created here, the size update may be inconsistent
         # but it's not a big deal... we should have only one writer
         
-        if under_user_id is None:
+        if under_owner_id is None:
             res = await self.cur.execute("DELETE FROM fmeta WHERE url LIKE ? RETURNING *", (path + '%', ))
         else:
-            res = await self.cur.execute("DELETE FROM fmeta WHERE url LIKE ? AND owner_id = ? RETURNING *", (path + '%', under_user_id))
+            res = await self.cur.execute("DELETE FROM fmeta WHERE url LIKE ? AND owner_id = ? RETURNING *", (path + '%', under_owner_id))
         all_f_rec = await res.fetchall()
         self.logger.info(f"Deleted {len(all_f_rec)} file(s) for path {path}") # type: ignore
         return [self.parse_record(r) for r in all_f_rec]
@@ -431,20 +433,41 @@ class FileConn(DBObjectBase):
             raise
         return size_sum
     
-    async def get_file_blob(self, file_id: str) -> Optional[bytes]:
+    async def get_file_blob(self, file_id: str, start_byte = -1, end_byte = -1) -> bytes:
         cursor = await self.cur.execute("SELECT data FROM blobs.fdata WHERE file_id = ?", (file_id, ))
         res = await cursor.fetchone()
         if res is None:
-            return None
-        return res[0]
+            raise FileNotFoundError(f"File {file_id} not found")
+        blob = res[0]
+        match (start_byte, end_byte):
+            case (-1, -1):
+                return blob
+            case (s, -1):
+                return blob[s:]
+            case (-1, e):
+                return blob[:e]
+            case (s, e):
+                return blob[s:e]
     
-    async def get_file_blob_external(self, file_id: str) -> AsyncIterable[bytes]:
+    @staticmethod
+    async def get_file_blob_external(file_id: str, start_byte = -1, end_byte = -1) -> AsyncIterable[bytes]:
         assert (LARGE_BLOB_DIR / file_id).exists(), f"File {file_id} not found"
         async with aiofiles.open(LARGE_BLOB_DIR / file_id, 'rb') as f:
-            while True:
-                chunk = await f.read(CHUNK_SIZE)
-                if not chunk: break
-                yield chunk
+            if start_byte >= 0:
+                await f.seek(start_byte)
+            if end_byte >= 0:
+                while True:
+                    head_ptr = await f.tell()
+                    if head_ptr >= end_byte:
+                        break
+                    chunk = await f.read(min(CHUNK_SIZE, end_byte - head_ptr))
+                    if not chunk: break
+                    yield chunk
+            else:
+                while True:
+                    chunk = await f.read(CHUNK_SIZE)
+                    if not chunk: break
+                    yield chunk
     
     @staticmethod
     async def delete_file_blob_external(file_id: str):
@@ -521,15 +544,16 @@ class Database:
             await execute_sql(conn, 'init.sql')
         return self
     
-    async def update_file_record(self, user: UserRecord, url: str, permission: FileReadPermission):
+    async def update_file_record(self, url: str, permission: FileReadPermission, op_user: Optional[UserRecord] = None):
         validate_url(url)
         async with transaction() as conn:
             fconn = FileConn(conn)
             r = await fconn.get_file_record(url)
             if r is None:
                 raise PathNotFoundError(f"File {url} not found")
-            if r.owner_id != user.id and not user.is_admin:
-                raise PermissionDeniedError(f"Permission denied: {user.username} cannot update file {url}")
+            if op_user is not None:
+                if r.owner_id != op_user.id and not op_user.is_admin:
+                    raise PermissionDeniedError(f"Permission denied: {op_user.username} cannot update file {url}")
             await fconn.update_file_record(url, permission=permission)
     
     async def save_file(
@@ -561,8 +585,7 @@ class Database:
                 
                 # check mime type
                 if mime_type is None:
-                    fname = url.split('/')[-1]
-                    mime_type, _ = mimetypes.guess_type(fname)
+                    mime_type, _ = mimetypes.guess_type(url)
                 if mime_type is None:
                     await f.seek(0)
                     mime_type = mimesniff.what(await f.read(1024))
@@ -591,45 +614,25 @@ class Database:
                         await FileConn(w_cur).set_file_record(
                             url, owner_id=user.id, file_id=f_id, file_size=file_size, 
                             permission=permission, external=True, mime_type=mime_type)
-
-        await delayed_log_activity(user.username)
         return file_size
 
-    async def read_file_stream(self, url: str) -> AsyncIterable[bytes]:
+    async def read_file(self, url: str, start_byte = -1, end_byte = -1) -> AsyncIterable[bytes]:
+        # end byte is exclusive: [start_byte, end_byte)
         validate_url(url)
-        async with unique_cursor() as cur:
-            fconn = FileConn(cur)
-            r = await fconn.get_file_record(url)
-            if r is None:
-                raise FileNotFoundError(f"File {url} not found")
-            if not r.external:
-                raise ValueError(f"File {url} is not stored externally, should use read_file instead")
-            ret = fconn.get_file_blob_external(r.file_id)
-
-        await delayed_log_access(url)
-        return ret
-
-
-    async def read_file(self, url: str) -> bytes:
-        validate_url(url)
-
         async with unique_cursor() as cur:
             fconn = FileConn(cur)
             r = await fconn.get_file_record(url)
             if r is None:
                 raise FileNotFoundError(f"File {url} not found")
             if r.external:
-                raise ValueError(f"File {url} is stored externally, should use read_file_stream instead")
-
-            f_id = r.file_id
-            blob = await fconn.get_file_blob(f_id)
-            if blob is None:
-                raise FileNotFoundError(f"File {url} data not found")
-
-        await delayed_log_access(url)
-        return blob
+                ret = fconn.get_file_blob_external(r.file_id, start_byte=start_byte, end_byte=end_byte)
+            else:
+                async def blob_stream():
+                    yield await fconn.get_file_blob(r.file_id, start_byte=start_byte, end_byte=end_byte)
+                ret = blob_stream()
+        return ret
 
-    async def delete_file(self, url: str, assure_user: Optional[UserRecord] = None) -> Optional[FileRecord]:
+    async def delete_file(self, url: str, op_user: Optional[UserRecord] = None) -> Optional[FileRecord]:
         validate_url(url)
 
         async with transaction() as cur:
@@ -637,10 +640,10 @@ class Database:
             r = await fconn.delete_file_record(url)
             if r is None:
                 return None
-            if assure_user is not None:
-                if r.owner_id != assure_user.id:
+            if op_user is not None:
+                if r.owner_id != op_user.id and not op_user.is_admin:
                     # will rollback
-                    raise PermissionDeniedError(f"Permission denied: {assure_user.username} cannot delete file {url}")
+                    raise PermissionDeniedError(f"Permission denied: {op_user.username} cannot delete file {url}")
             f_id = r.file_id
             if r.external:
                 await fconn.delete_file_blob_external(f_id)
@@ -648,7 +651,7 @@ class Database:
                 await fconn.delete_file_blob(f_id)
             return r
     
-    async def move_file(self, old_url: str, new_url: str, ensure_user: Optional[UserRecord] = None):
+    async def move_file(self, old_url: str, new_url: str, op_user: Optional[UserRecord] = None):
         validate_url(old_url)
         validate_url(new_url)
 
@@ -657,12 +660,16 @@ class Database:
             r = await fconn.get_file_record(old_url)
             if r is None:
                 raise FileNotFoundError(f"File {old_url} not found")
-            if ensure_user is not None:
-                if r.owner_id != ensure_user.id:
-                    raise PermissionDeniedError(f"Permission denied: {ensure_user.username} cannot move file {old_url}")
+            if op_user is not None:
+                if r.owner_id != op_user.id:
+                    raise PermissionDeniedError(f"Permission denied: {op_user.username} cannot move file {old_url}")
             await fconn.move_file(old_url, new_url)
+
+            new_mime, _ = mimetypes.guess_type(new_url)
+            if not new_mime is None:
+                await fconn.update_file_record(new_url, mime_type=new_mime)
     
-    async def move_path(self, user: UserRecord, old_url: str, new_url: str):
+    async def move_path(self, old_url: str, new_url: str, op_user: UserRecord):
         validate_url(old_url, is_file=False)
         validate_url(new_url, is_file=False)
 
@@ -676,20 +683,20 @@ class Database:
 
         async with transaction() as cur:
             first_component = new_url.split('/')[0]
-            if not (first_component == user.username or user.is_admin):
-                raise PermissionDeniedError(f"Permission denied: path must start with {user.username}")
-            elif user.is_admin:
+            if not (first_component == op_user.username or op_user.is_admin):
+                raise PermissionDeniedError(f"Permission denied: path must start with {op_user.username}")
+            elif op_user.is_admin:
                 uconn = UserConn(cur)
                 _is_user = await uconn.get_user(first_component)
                 if not _is_user:
                     raise PermissionDeniedError(f"Invalid path: {first_component} is not a valid username")
             
             # check if old path is under user's directory (non-admin)
-            if not old_url.startswith(user.username + '/') and not user.is_admin:
-                raise PermissionDeniedError(f"Permission denied: {user.username} cannot move path {old_url}")
+            if not old_url.startswith(op_user.username + '/') and not op_user.is_admin:
+                raise PermissionDeniedError(f"Permission denied: {op_user.username} cannot move path {old_url}")
 
             fconn = FileConn(cur)
-            await fconn.move_path(old_url, new_url, 'overwrite', user.id)
+            await fconn.move_path(old_url, new_url, 'overwrite', op_user.id)
 
     async def __batch_delete_file_blobs(self, fconn: FileConn, file_records: list[FileRecord], batch_size: int = 512):
         # https://github.com/langchain-ai/langchain/issues/10321
@@ -709,13 +716,13 @@ class Database:
                 await fconn.delete_file_blob_external(external_ids[i])
         await asyncio.gather(del_internal(), del_external())
 
-    async def delete_path(self, url: str, under_user: Optional[UserRecord] = None) -> Optional[list[FileRecord]]:
+    async def delete_path(self, url: str, op_user: Optional[UserRecord] = None) -> Optional[list[FileRecord]]:
         validate_url(url, is_file=False)
-        user_id = under_user.id if under_user is not None else None
+        from_owner_id = op_user.id if op_user is not None and not op_user.is_admin else None
 
         async with transaction() as cur:
             fconn = FileConn(cur)
-            records = await fconn.delete_path_records(url, user_id)
+            records = await fconn.delete_path_records(url, from_owner_id)
             if not records:
                 return None
             await self.__batch_delete_file_blobs(fconn, records)
@@ -759,9 +766,6 @@ class Database:
                     blob = fconn.get_file_blob_external(f_id)
                 else:
                     blob = await fconn.get_file_blob(f_id)
-                    if blob is None:
-                        self.logger.warning(f"Blob not found for {url}")
-                        continue
                 yield r, blob
 
     @concurrent_wrap()

{lfss-0.8.1 → lfss-0.8.3}/lfss/src/datatype.py

@@ -1,6 +1,5 @@
 from enum import IntEnum
-from typing import Literal
-import dataclasses
+import dataclasses, typing
 
 class FileReadPermission(IntEnum):
     UNSET = 0           # not set
@@ -55,7 +54,7 @@ class PathContents:
     dirs: list[DirectoryRecord] = dataclasses.field(default_factory=list)
     files: list[FileRecord] = dataclasses.field(default_factory=list)
     
-FileSortKey = Literal['', 'url', 'file_size', 'create_time', 'access_time', 'mime_type']
-isValidFileSortKey = lambda x: x in ['', 'url', 'file_size', 'create_time', 'access_time', 'mime_type']
-DirSortKey = Literal['', 'dirname']
-isValidDirSortKey = lambda x: x in ['', 'dirname']
+FileSortKey = typing.Literal['', 'url', 'file_size', 'create_time', 'access_time', 'mime_type']
+isValidFileSortKey = lambda x: x in typing.get_args(FileSortKey)
+DirSortKey = typing.Literal['', 'dirname']
+isValidDirSortKey = lambda x: x in typing.get_args(DirSortKey)

{lfss-0.8.1 → lfss-0.8.3}/lfss/src/server.py

@@ -6,7 +6,6 @@ from fastapi.responses import StreamingResponse
 from fastapi.exceptions import HTTPException 
 from fastapi.security import HTTPBearer, HTTPAuthorizationCredentials
 from fastapi.middleware.cors import CORSMiddleware
-import mimesniff
 
 import asyncio, json, time
 from contextlib import asynccontextmanager
@@ -14,10 +13,11 @@ from contextlib import asynccontextmanager
 from .error import *
 from .log import get_logger
 from .stat import RequestDB
-from .config import MAX_BUNDLE_BYTES, MAX_MEM_FILE_BYTES, LARGE_FILE_BYTES, CHUNK_SIZE
+from .config import MAX_BUNDLE_BYTES, CHUNK_SIZE
 from .utils import ensure_uri_compnents, format_last_modified, now_stamp, wait_for_debounce_tasks
 from .connection_pool import global_connection_init, global_connection_close, unique_cursor
-from .database import Database, DECOY_USER, check_user_permission, UserConn, FileConn, delayed_log_activity, get_user
+from .database import Database, DECOY_USER, check_user_permission, UserConn, FileConn
+from .database import delayed_log_activity, delayed_log_access
 from .datatype import (
     FileReadPermission, FileRecord, UserRecord, PathContents, 
     FileSortKey, DirSortKey
@@ -80,6 +80,10 @@ async def get_current_user(
 
     if not user:
         raise HTTPException(status_code=401, detail="Invalid token")
+    
+    if not user.id == 0:
+        await delayed_log_activity(user.username)
+
     return user
 
 async def registered_user(user: UserRecord = Depends(get_current_user)):
@@ -137,7 +141,8 @@ router_fs = APIRouter(prefix="")
 @skip_request_log
 async def emit_thumbnail(
     path: str, download: bool,
-    create_time: Optional[str] = None
+    create_time: Optional[str] = None, 
+    is_head = False
     ):
     if path.endswith("/"):
         fname = path.split("/")[-2]
@@ -153,42 +158,69 @@ async def emit_thumbnail(
     }
     if create_time is not None:
         headers["Last-Modified"] = format_last_modified(create_time)
+    if is_head: return Response(status_code=200, headers=headers)
     return Response(
         content=thumb_blob, media_type=mime_type, headers=headers
     )
 async def emit_file(
     file_record: FileRecord, 
     media_type: Optional[str] = None, 
-    disposition = "attachment"
+    disposition = "attachment", 
+    is_head = False, 
+    range_start = -1,
+    range_end = -1
     ):
+    if range_start < 0: assert range_start == -1
+    if range_end < 0: assert range_end == -1
+
     if media_type is None:
         media_type = file_record.mime_type
     path = file_record.url
     fname = path.split("/")[-1]
-    if not file_record.external:
-        fblob = await db.read_file(path)
-        return Response(
-            content=fblob, media_type=media_type, headers={
-                "Content-Disposition": f"{disposition}; filename={fname}", 
-                "Content-Length": str(len(fblob)), 
-                "Last-Modified": format_last_modified(file_record.create_time)
-            }
-        )
+
+    if range_start == -1:
+        arng_s = 0          # actual range start
     else:
-        return StreamingResponse(
-            await db.read_file_stream(path), media_type=media_type, headers={
-                "Content-Disposition": f"{disposition}; filename={fname}", 
-                "Content-Length": str(file_record.file_size),
-                "Last-Modified": format_last_modified(file_record.create_time)
-            }
-        )
+        arng_s = range_start
+    if range_end == -1:
+        arng_e = file_record.file_size - 1
+    else:
+        arng_e = range_end
+    
+    if arng_s >= file_record.file_size or arng_e >= file_record.file_size:
+        raise HTTPException(status_code=416, detail="Range not satisfiable")
+    if arng_s > arng_e:
+        raise HTTPException(status_code=416, detail="Invalid range")
 
-@router_fs.get("/{path:path}")
-@handle_exception
-async def get_file(
+    headers = {
+        "Content-Disposition": f"{disposition}; filename={fname}", 
+        "Content-Length": str(arng_e - arng_s + 1),
+        "Content-Range": f"bytes {arng_s}-{arng_e}/{file_record.file_size}",
+        "Last-Modified": format_last_modified(file_record.create_time), 
+        "Accept-Ranges": "bytes", 
+    }
+
+    if is_head: return Response(status_code=200 if (range_start == -1 and range_end == -1) else 206, headers=headers)
+
+    await delayed_log_access(path)
+    return StreamingResponse(
+        await db.read_file(
+            path, 
+            start_byte=arng_s if range_start != -1 else -1,
+            end_byte=arng_e + 1 if range_end != -1 else -1
+        ),
+        media_type=media_type, 
+        headers=headers, 
+        status_code=206 if range_start != -1 or range_end != -1 else 200
+    )
+
+async def get_file_impl(
+    request: Request,
+    user: UserRecord, 
     path: str, 
-    download: bool = False, thumb: bool = False,
-    user: UserRecord = Depends(get_current_user)
+    download: bool = False, 
+    thumb: bool = False,
+    is_head = False,
     ):
     path = ensure_uri_compnents(path)
 
@@ -230,13 +262,58 @@ async def get_file(
     if not allow_access:
         raise HTTPException(status_code=403, detail=reason)
     
+    req_range = request.headers.get("Range", None)
+    if not req_range is None:
+        # handle range request
+        if not req_range.startswith("bytes="):
+            raise HTTPException(status_code=400, detail="Invalid range request")
+        range_str = req_range[6:].strip()
+        if "," in range_str:
+            raise HTTPException(status_code=400, detail="Multiple ranges not supported")
+        if "-" not in range_str:
+            raise HTTPException(status_code=400, detail="Invalid range request")
+        range_start, range_end = map(lambda x: int(x) if x != "" else -1 , range_str.split("-"))
+    else:
+        range_start, range_end = -1, -1
+    
     if thumb:
-        return await emit_thumbnail(path, download, create_time=file_record.create_time)
+        if (range_start != -1 or range_end != -1): logger.warning("Range request for thumbnail")
+        return await emit_thumbnail(path, download, create_time=file_record.create_time, is_head=is_head)
     else:
         if download:
-            return await emit_file(file_record, 'application/octet-stream', "attachment")
+            return await emit_file(file_record, 'application/octet-stream', "attachment", is_head = is_head, range_start=range_start, range_end=range_end)
         else:
-            return await emit_file(file_record, None, "inline")
+            return await emit_file(file_record, None, "inline", is_head = is_head, range_start=range_start, range_end=range_end)
+
+@router_fs.get("/{path:path}")
+@handle_exception
+async def get_file(
+    request: Request,
+    path: str, 
+    download: bool = False, thumb: bool = False,
+    user: UserRecord = Depends(get_current_user)
+    ):
+    return await get_file_impl(
+        request = request,
+        user = user, path = path, download = download, thumb = thumb
+        )
+
+@router_fs.head("/{path:path}")
+@handle_exception
+async def head_file(
+    request: Request,
+    path: str, 
+    download: bool = False, thumb: bool = False,
+    user: UserRecord = Depends(get_current_user)
+    ):
+    if path.startswith("_api/"):
+        raise HTTPException(status_code=405, detail="HEAD not supported for API")
+    if path.endswith("/"):
+        raise HTTPException(status_code=405, detail="HEAD not supported for directory")
+    return await get_file_impl(
+        request = request,
+        user = user, path = path, download = download, thumb = thumb, is_head = True
+        )
 
 @router_fs.put("/{path:path}")
 @handle_exception
@@ -360,11 +437,10 @@ async def delete_file(path: str, user: UserRecord = Depends(registered_user)):
     logger.info(f"DELETE {path}, user: {user.username}")
 
     if path.endswith("/"):
-        res = await db.delete_path(path, user if not user.is_admin else None)
+        res = await db.delete_path(path, user)
     else:
-        res = await db.delete_file(path, user if not user.is_admin else None)
+        res = await db.delete_file(path, user)
 
-    await delayed_log_activity(user.username)
     if res:
         return Response(status_code=200, content="Deleted")
     else:
@@ -456,16 +532,15 @@ async def update_file_meta(
     path = ensure_uri_compnents(path)
     if path.startswith("/"):
         path = path[1:]
-    await delayed_log_activity(user.username)
 
     # file
     if not path.endswith("/"):
         if perm is not None:
             logger.info(f"Update permission of {path} to {perm}")
             await db.update_file_record(
-                user = user,
                 url = path, 
-                permission = FileReadPermission(perm)
+                permission = FileReadPermission(perm), 
+                op_user = user,
             )
     
         if new_path is not None:
@@ -480,7 +555,7 @@ async def update_file_meta(
             new_path = ensure_uri_compnents(new_path)
             logger.info(f"Update path of {path} to {new_path}")
             # currently only move own file, with overwrite
-            await db.move_path(user, path, new_path)
+            await db.move_path(path, new_path, user)
 
     return Response(status_code=200, content="OK")
 

{lfss-0.8.1 → lfss-0.8.3}/lfss/src/utils.py

@@ -109,6 +109,17 @@ def parse_storage_size(s: str) -> int:
         case 'g': return int(s[:-1]) * 1024**3
         case 't': return int(s[:-1]) * 1024**4
         case _: raise ValueError(f"Invalid file size string: {s}")
+def fmt_storage_size(size: int) -> str:
+    """ Format the file size to human-readable format """
+    if size < 1024:
+        return f"{size}B"
+    if size < 1024**2:
+        return f"{size/1024:.2f}K"
+    if size < 1024**3:
+        return f"{size/1024**2:.2f}M"
+    if size < 1024**4:
+        return f"{size/1024**3:.2f}G"
+    return f"{size/1024**4:.2f}T"
 
 _FnReturnT = TypeVar('_FnReturnT')
 _AsyncReturnT = Awaitable[_FnReturnT]
@@ -136,4 +147,12 @@ def concurrent_wrap(executor=None):
             loop = asyncio.new_event_loop()
             return loop.run_until_complete(func(*args, **kwargs))
         return sync_fn
-    return _concurrent_wrap
+    return _concurrent_wrap
+
+# https://stackoverflow.com/a/279586/6775765
+def static_vars(**kwargs):
+    def decorate(func):
+        for k in kwargs:
+            setattr(func, k, kwargs[k])
+        return func
+    return decorate

{lfss-0.8.1 → lfss-0.8.3}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "lfss"
-version = "0.8.1"
+version = "0.8.3"
 description = "Lightweight file storage service"
 authors = ["li, mengxun <limengxun45@outlook.com>"]
 readme = "Readme.md"
@@ -9,7 +9,7 @@ repository = "https://github.com/MenxLi/lfss"
 include = ["Readme.md", "docs/*", "frontend/*", "lfss/sql/*"]
 
 [tool.poetry.dependencies]
-python = ">=3.9"
+python = ">=3.10"   # PEP-622
 requests = "2.*"
 aiosqlite = "0.*"
 aiofiles = "23.*"