lfss 0.10.0__py3-none-any.whl → 0.11.0__py3-none-any.whl

lfss/api/__init__.py

@@ -170,14 +170,15 @@ def download_directory(
     _counter = 0
     _counter_lock = Lock()
     failed_items: list[tuple[str, str]] = []
+    file_count = 0
     def get_file(c, src_url):
-        nonlocal _counter, failed_items
+        nonlocal _counter, failed_items, file_count, verbose
         with _counter_lock:
             _counter += 1
             this_count = _counter
             dst_path = f"{directory}{os.path.relpath(decode_uri_compnents(src_url), decode_uri_compnents(src_path))}"
             if verbose:
-                print(f"[{this_count}] Downloading {src_url} to {dst_path}")
+                print(f"[{this_count}/{file_count}] Downloading {src_url} to {dst_path}")
 
         if not (res:=download_file(
             c, src_url, dst_path, 
@@ -185,11 +186,13 @@ def download_directory(
             ))[0]:
             failed_items.append((src_url, res[1]))
         
-    batch_size = 10000
+    batch_size = 10_000
     file_list: list[FileRecord] = []
     with connector.session(n_concurrent) as c:
         file_count = c.count_files(src_path, flat=True)
         for offset in range(0, file_count, batch_size):
+            if verbose:
+                print(f"Retrieving file list... ({offset}/{file_count})", end='\r')
             file_list.extend(c.list_files(
                 src_path, offset=offset, limit=batch_size, flat=True
             ))

lfss/cli/cli.py

@@ -12,7 +12,7 @@ def parse_permission(s: str) -> FileReadPermission:
     raise ValueError(f"Invalid permission {s}")
 
 def parse_arguments():
-    parser = argparse.ArgumentParser(description="Command line interface, please set LFSS_ENDPOINT and LFSS_TOKEN environment variables.")
+    parser = argparse.ArgumentParser(description="Client-side command line interface, set LFSS_ENDPOINT and LFSS_TOKEN environment variables for authentication.")
 
     sp = parser.add_subparsers(dest="command", required=True)
 

lfss/cli/vacuum.py

@@ -2,10 +2,11 @@
 Vacuum the database and external storage to ensure that the storage is consistent and minimal.
 """
 
-from lfss.eng.config import LARGE_BLOB_DIR
-import argparse, time
+from lfss.eng.config import LARGE_BLOB_DIR, THUMB_DB
+import argparse, time, itertools
 from functools import wraps
 from asyncio import Semaphore
+import aiosqlite
 import aiofiles, asyncio
 import aiofiles.os
 from contextlib import contextmanager
@@ -32,7 +33,7 @@ def barriered(func):
     return wrapper
 
 @global_entrance()
-async def vacuum_main(index: bool = False, blobs: bool = False):
+async def vacuum_main(index: bool = False, blobs: bool = False, thumbs: bool = False, vacuum_all: bool = False):
 
     # check if any file in the Large Blob directory is not in the database
     # the reverse operation is not necessary, because by design, the database should be the source of truth...
@@ -49,23 +50,63 @@ async def vacuum_main(index: bool = False, blobs: bool = False):
 
     # create a temporary index to speed up the process...
     with indicator("Clearing un-referenced files in external storage"):
-        async with transaction() as c:
-            await c.execute("CREATE INDEX IF NOT EXISTS fmeta_file_id ON fmeta (file_id)")
-        for i, f in enumerate(LARGE_BLOB_DIR.iterdir()):
-            f_id = f.name
-            await ensure_external_consistency(f_id)
-            if (i+1) % 1_000 == 0:
-                print(f"Checked {(i+1)//1000}k files in external storage.", end='\r')
-        async with transaction() as c:
-            await c.execute("DROP INDEX IF EXISTS fmeta_file_id")
+        try:
+            async with transaction() as c:
+                await c.execute("CREATE INDEX IF NOT EXISTS fmeta_file_id ON fmeta (file_id)")
+            for i, f in enumerate(LARGE_BLOB_DIR.iterdir()):
+                f_id = f.name
+                await ensure_external_consistency(f_id)
+                if (i+1) % 1_000 == 0:
+                    print(f"Checked {(i+1)//1000}k files in external storage.", end='\r')
+        finally:
+            async with transaction() as c:
+                await c.execute("DROP INDEX IF EXISTS fmeta_file_id")
 
-    async with unique_cursor(is_write=True) as c:
-        if index:
-            with indicator("VACUUM-index"):
+    if index or vacuum_all:
+        with indicator("VACUUM-index"):
+            async with transaction() as c:
+                await c.execute("DELETE FROM dupcount WHERE count = 0")
+            async with unique_cursor(is_write=True) as c:
                 await c.execute("VACUUM main")
-        if blobs:
-            with indicator("VACUUM-blobs"):
+    if blobs or vacuum_all:
+        with indicator("VACUUM-blobs"):
+            async with unique_cursor(is_write=True) as c:
                 await c.execute("VACUUM blobs")
+    
+    if thumbs or vacuum_all:
+        try:
+            async with transaction() as c:
+                await c.execute("CREATE INDEX IF NOT EXISTS fmeta_file_id ON fmeta (file_id)")
+            with indicator("VACUUM-thumbs"):
+                if not THUMB_DB.exists():
+                    raise FileNotFoundError("Thumbnail database not found.")
+                async with unique_cursor() as db_c:
+                    async with aiosqlite.connect(THUMB_DB) as t_conn:
+                        batch_size = 10_000
+                        for batch_count in itertools.count(start=0):
+                            exceeded_rows = list(await (await t_conn.execute(
+                                "SELECT file_id FROM thumbs LIMIT ? OFFSET ?",
+                                (batch_size, batch_size * batch_count)
+                            )).fetchall())
+                            if not exceeded_rows:
+                                break
+                            batch_ids = [row[0] for row in exceeded_rows]
+                            for f_id in batch_ids:
+                                cursor = await db_c.execute("SELECT file_id FROM fmeta WHERE file_id = ?", (f_id,))
+                                if not await cursor.fetchone():
+                                    print(f"Thumbnail {f_id} not found in database, removing from thumb cache.")
+                                    await t_conn.execute("DELETE FROM thumbs WHERE file_id = ?", (f_id,))
+                            print(f"Checked {batch_count+1} batches of {batch_size} thumbnails.")
+
+                        await t_conn.commit()
+                        await t_conn.execute("VACUUM")
+        except FileNotFoundError as e:
+            if "Thumbnail database not found." in str(e):
+                print("Thumbnail database not found, skipping.")
+            
+        finally:
+            async with transaction() as c:
+                await c.execute("DROP INDEX IF EXISTS fmeta_file_id")
 
 async def vacuum_requests():
     with indicator("VACUUM-requests"):
@@ -76,15 +117,17 @@ async def vacuum_requests():
 def main():
     global sem
     parser = argparse.ArgumentParser(description="Balance the storage by ensuring that large file thresholds are met.")
+    parser.add_argument("--all", action="store_true", help="Vacuum all")
     parser.add_argument("-j", "--jobs", type=int, default=2, help="Number of concurrent jobs")
     parser.add_argument("-m", "--metadata", action="store_true", help="Vacuum metadata")
     parser.add_argument("-d", "--data", action="store_true", help="Vacuum blobs")
+    parser.add_argument("-t", "--thumb", action="store_true", help="Vacuum thumbnails")
     parser.add_argument("-r", "--requests", action="store_true", help="Vacuum request logs to only keep at most recent 1M rows in 7 days")
     args = parser.parse_args()
     sem = Semaphore(args.jobs)
-    asyncio.run(vacuum_main(index=args.metadata, blobs=args.data))
+    asyncio.run(vacuum_main(index=args.metadata, blobs=args.data, thumbs=args.thumb, vacuum_all=args.all))
 
-    if args.requests:
+    if args.requests or args.all:
         asyncio.run(vacuum_requests())
 
 if __name__ == '__main__':

lfss/eng/config.py

@@ -22,5 +22,5 @@ MAX_MEM_FILE_BYTES = 128 * 1024 * 1024   # 128MB
 CHUNK_SIZE = 1024 * 1024   # 1MB chunks for streaming (on large files)
 DEBUG_MODE = os.environ.get('LFSS_DEBUG', '0') == '1'
 
-THUMB_DB = DATA_HOME / 'thumbs.db'
+THUMB_DB = DATA_HOME / 'thumbs.v0-11.db'
 THUMB_SIZE = (48, 48)

lfss/eng/database.py

@@ -3,6 +3,7 @@ from typing import Optional, Literal, overload
 from collections.abc import AsyncIterable
 from contextlib import asynccontextmanager
 from abc import ABC
+import re
 
 import uuid, datetime
 import urllib.parse
@@ -20,7 +21,7 @@ from .datatype import (
     )
 from .config import LARGE_BLOB_DIR, CHUNK_SIZE, LARGE_FILE_BYTES, MAX_MEM_FILE_BYTES
 from .log import get_logger
-from .utils import decode_uri_compnents, hash_credential, concurrent_wrap, debounce_async, copy_file
+from .utils import decode_uri_compnents, hash_credential, concurrent_wrap, debounce_async, static_vars
 from .error import *
 
 class DBObjectBase(ABC):
@@ -84,8 +85,9 @@ class UserConn(DBObjectBase):
         max_storage: int = 1073741824, permission: FileReadPermission = FileReadPermission.UNSET
         ) -> int:
         def validate_username(username: str):
+            assert not set(username) & {'/', ':'}, "Invalid username"
             assert not username.startswith('_'), "Error: reserved username"
-            assert not ('/' in username or ':' in username or len(username) > 255), "Invalid username"
+            assert not (len(username) > 255), "Username too long"
             assert urllib.parse.quote(username) == username, "Invalid username, must be URL safe"
         validate_username(username)
         self.logger.debug(f"Creating user {username}")
@@ -249,7 +251,7 @@ class FileConn(DBObjectBase):
 
     async def list_path_dirs(
         self, url: str, 
-        offset: int = 0, limit: int = int(1e5), 
+        offset: int = 0, limit: int = 10_000, 
         order_by: DirSortKey = '', order_desc: bool = False,
         skim: bool = True
         ) -> list[DirectoryRecord]:
@@ -293,7 +295,7 @@ class FileConn(DBObjectBase):
 
     async def list_path_files(
         self, url: str, 
-        offset: int = 0, limit: int = int(1e5), 
+        offset: int = 0, limit: int = 10_000, 
         order_by: FileSortKey = '', order_desc: bool = False,
         flat: bool = False, 
         ) -> list[FileRecord]:
@@ -324,7 +326,7 @@ class FileConn(DBObjectBase):
         - It cannot flatten directories
         - It cannot list directories with details
         """
-        MAX_ITEMS = int(1e4)
+        MAX_ITEMS = 10_000
         dir_count = await self.count_path_dirs(url)
         file_count = await self.count_path_files(url, flat=False)
         if dir_count + file_count > MAX_ITEMS:
@@ -417,16 +419,12 @@ class FileConn(DBObjectBase):
         new_exists = await self.get_file_record(new_url)
         if new_exists is not None:
             raise FileExistsError(f"File {new_url} already exists")
-        new_fid = str(uuid.uuid4())
         user_id = old.owner_id if user_id is None else user_id
         await self.cur.execute(
             "INSERT INTO fmeta (url, owner_id, file_id, file_size, permission, external, mime_type) VALUES (?, ?, ?, ?, ?, ?, ?)", 
-            (new_url, user_id, new_fid, old.file_size, old.permission, old.external, old.mime_type)
+            (new_url, user_id, old.file_id, old.file_size, old.permission, old.external, old.mime_type)
             )
-        if not old.external:
-            await self.set_file_blob(new_fid, await self.get_file_blob(old.file_id))
-        else:
-            await copy_file(LARGE_BLOB_DIR / old.file_id, LARGE_BLOB_DIR / new_fid)
+        await self.cur.execute("INSERT OR REPLACE INTO dupcount (file_id, count) VALUES (?, COALESCE((SELECT count FROM dupcount WHERE file_id = ?), 0) + 1)", (old.file_id, old.file_id))
         await self._user_size_inc(user_id, old.file_size)
         self.logger.info(f"Copied file {old_url} to {new_url}")
     
@@ -444,16 +442,12 @@ class FileConn(DBObjectBase):
             new_r = new_url + old_record.url[len(old_url):]
             if await (await self.cur.execute("SELECT url FROM fmeta WHERE url = ?", (new_r, ))).fetchone() is not None:
                 raise FileExistsError(f"File {new_r} already exists")
-            new_fid = str(uuid.uuid4())
             user_id = old_record.owner_id if user_id is None else user_id
             await self.cur.execute(
                 "INSERT INTO fmeta (url, owner_id, file_id, file_size, permission, external, mime_type) VALUES (?, ?, ?, ?, ?, ?, ?)", 
-                (new_r, user_id, new_fid, old_record.file_size, old_record.permission, old_record.external, old_record.mime_type)
+                (new_r, user_id, old_record.file_id, old_record.file_size, old_record.permission, old_record.external, old_record.mime_type)
                 )
-            if not old_record.external:
-                await self.set_file_blob(new_fid, await self.get_file_blob(old_record.file_id))
-            else:
-                await copy_file(LARGE_BLOB_DIR / old_record.file_id, LARGE_BLOB_DIR / new_fid)
+            await self.cur.execute("INSERT OR REPLACE INTO dupcount (file_id, count) VALUES (?, COALESCE((SELECT count FROM dupcount WHERE file_id = ?), 0) + 1)", (old_record.file_id, old_record.file_id))
             await self._user_size_inc(user_id, old_record.file_size)
         self.logger.info(f"Copied path {old_url} to {new_url}")
     
@@ -497,6 +491,7 @@ class FileConn(DBObjectBase):
         return file_record
     
     async def delete_user_file_records(self, owner_id: int) -> list[FileRecord]:
+        """ Delete all records with owner_id """
         cursor = await self.cur.execute("SELECT * FROM fmeta WHERE owner_id = ?", (owner_id, ))
         res = await cursor.fetchall()
         await self.cur.execute("DELETE FROM usize WHERE user_id = ?", (owner_id, ))
@@ -528,7 +523,7 @@ class FileConn(DBObjectBase):
         return [self.parse_record(r) for r in all_f_rec]
     
     async def set_file_blob(self, file_id: str, blob: bytes):
-        await self.cur.execute("INSERT OR REPLACE INTO blobs.fdata (file_id, data) VALUES (?, ?)", (file_id, blob))
+        await self.cur.execute("INSERT INTO blobs.fdata (file_id, data) VALUES (?, ?)", (file_id, blob))
     
     @staticmethod
     async def set_file_blob_external(file_id: str, stream: AsyncIterable[bytes])->int:
@@ -580,16 +575,78 @@ class FileConn(DBObjectBase):
                     if not chunk: break
                     yield chunk
     
-    @staticmethod
-    async def delete_file_blob_external(file_id: str):
+    async def unlink_file_blob_external(self, file_id: str):
+        # first check if the file has duplication
+        cursor = await self.cur.execute("SELECT count FROM dupcount WHERE file_id = ?", (file_id, ))
+        res = await cursor.fetchone()
+        if res is not None and res[0] > 0:
+            await self.cur.execute("UPDATE dupcount SET count = count - 1 WHERE file_id = ?", (file_id, ))
+            return
+
+        # finally delete the file and the duplication count
         if (LARGE_BLOB_DIR / file_id).exists():
             await aiofiles.os.remove(LARGE_BLOB_DIR / file_id)
+        await self.cur.execute("DELETE FROM dupcount WHERE file_id = ?", (file_id, ))
     
-    async def delete_file_blob(self, file_id: str):
+    async def unlink_file_blob(self, file_id: str):
+        # first check if the file has duplication
+        cursor = await self.cur.execute("SELECT count FROM dupcount WHERE file_id = ?", (file_id, ))
+        res = await cursor.fetchone()
+        if res is not None and res[0] > 0:
+            await self.cur.execute("UPDATE dupcount SET count = count - 1 WHERE file_id = ?", (file_id, ))
+            return
+
+        # finally delete the file and the duplication count
         await self.cur.execute("DELETE FROM blobs.fdata WHERE file_id = ?", (file_id, ))
+        await self.cur.execute("DELETE FROM dupcount WHERE file_id = ?", (file_id, ))
+    
+    async def _group_del(self, file_ids_all: list[str]):
+        """
+        The file_ids_all may contain duplication, 
+        yield tuples of unique (to_del_ids, to_dec_ids) for each iteration, 
+        every iteration should unlink one copy of the files, repeat until all re-occurrence in the input list are removed.
+        """
+        async def check_dup(file_ids: set[str]):
+            cursor = await self.cur.execute("SELECT file_id FROM dupcount WHERE file_id IN ({}) AND count > 0".format(','.join(['?'] * len(file_ids))), tuple(file_ids))
+            res = await cursor.fetchall()
+            to_dec_ids = [r[0] for r in res]
+            to_del_ids = list(file_ids - set(to_dec_ids))
+            return to_del_ids, to_dec_ids
+        # gather duplication from all file_ids
+        fid_occurrence = {}
+        for file_id in file_ids_all:
+            fid_occurrence[file_id] = fid_occurrence.get(file_id, 0) + 1
+        while fid_occurrence:
+            to_del_ids, to_dec_ids = await check_dup(set(fid_occurrence.keys()))
+            for file_id in to_del_ids:
+                del fid_occurrence[file_id]
+            for file_id in to_dec_ids:
+                fid_occurrence[file_id] -= 1
+                if fid_occurrence[file_id] == 0:
+                    del fid_occurrence[file_id]
+            yield (to_del_ids, to_dec_ids)
     
-    async def delete_file_blobs(self, file_ids: list[str]):
-        await self.cur.execute("DELETE FROM blobs.fdata WHERE file_id IN ({})".format(','.join(['?'] * len(file_ids))), file_ids)
+    async def unlink_file_blobs(self, file_ids: list[str]):
+        async for (to_del_ids, to_dec_ids) in self._group_del(file_ids):
+            # delete the only copy
+            await self.cur.execute("DELETE FROM blobs.fdata WHERE file_id IN ({})".format(','.join(['?'] * len(to_del_ids))), to_del_ids)
+            await self.cur.execute("DELETE FROM dupcount WHERE file_id IN ({})".format(','.join(['?'] * len(to_del_ids))), to_del_ids)
+            # decrease duplication count
+            await self.cur.execute("UPDATE dupcount SET count = count - 1 WHERE file_id IN ({})".format(','.join(['?'] * len(to_dec_ids))), to_dec_ids)
+    
+    async def unlink_file_blobs_external(self, file_ids: list[str]):
+        async def del_file(file_id: str):
+            if (LARGE_BLOB_DIR / file_id).exists():
+                await aiofiles.os.remove(LARGE_BLOB_DIR / file_id)
+        async for (to_del_ids, to_dec_ids) in self._group_del(file_ids):
+            # delete the only copy
+            await asyncio.gather(*(
+                [del_file(file_id) for file_id in to_del_ids] + 
+                [self.cur.execute("DELETE FROM dupcount WHERE file_id = ?", (file_id, )) for file_id in to_del_ids]
+                ))
+            # decrease duplication count
+            await self.cur.execute("UPDATE dupcount SET count = count - 1 WHERE file_id IN ({})".format(','.join(['?'] * len(to_dec_ids))), to_dec_ids)
+        
 
 _log_active_queue = []
 _log_active_lock = asyncio.Lock()
@@ -621,20 +678,35 @@ async def delayed_log_access(url: str):
         _log_access_queue.append(url)
     await _log_all_access()
 
+@static_vars(
+    prohibited_regex = re.compile(
+            r"^[/_.]",              # start with / or _ or .
+        ),
+    prohibited_part_regex = re.compile(
+        "|".join([
+            r"^\s*\.+\s*$",       # dot path
+            "[{}]".format("".join(re.escape(c) for c in ('/', "\\", "'", '"', "*", "%"))), # prohibited characters
+        ])
+    ),
+)
 def validate_url(url: str, is_file = True):
-    prohibited_chars = ['..', ';', "'", '"', '\\', '\0', '\n', '\r', '\t', '\x0b', '\x0c']
-    ret = not url.startswith('/') and not url.startswith('_') and not url.startswith('.')
-    ret = ret and not any([c in url for c in prohibited_chars])
+    """ Check if a path is valid. The input path is considered url safe """
+    if len(url) > 1024: 
+        raise InvalidPathError(f"URL too long: {url}")
 
-    if not ret:
+    is_valid = validate_url.prohibited_regex.search(url) is None
+    if not is_valid:    # early return, no need to check further
         raise InvalidPathError(f"Invalid URL: {url}")
-    
-    if is_file:
-        ret = ret and not url.endswith('/')
-    else:
-        ret = ret and url.endswith('/')
 
-    if not ret:
+    for part in url.split('/'):
+        if validate_url.prohibited_part_regex.search(urllib.parse.unquote(part)):
+            is_valid = False
+            break
+
+    if is_file: is_valid = is_valid and not url.endswith('/')
+    else: is_valid = is_valid and url.endswith('/')
+
+    if not is_valid: 
         raise InvalidPathError(f"Invalid URL: {url}")
 
 async def get_user(cur: aiosqlite.Cursor, user: int | str) -> Optional[UserRecord]:
@@ -771,9 +843,9 @@ class Database:
                     raise PermissionDeniedError(f"Permission denied: {op_user.username} cannot delete file {url}")
             f_id = r.file_id
             if r.external:
-                await fconn.delete_file_blob_external(f_id)
+                await fconn.unlink_file_blob_external(f_id)
             else:
-                await fconn.delete_file_blob(f_id)
+                await fconn.unlink_file_blob(f_id)
             return r
     
     async def move_file(self, old_url: str, new_url: str, op_user: Optional[UserRecord] = None):
@@ -872,11 +944,12 @@ class Database:
         
         async def del_internal():
             for i in range(0, len(internal_ids), batch_size):
-                await fconn.delete_file_blobs([r for r in internal_ids[i:i+batch_size]])
+                await fconn.unlink_file_blobs([r for r in internal_ids[i:i+batch_size]])
         async def del_external():
-            for i in range(0, len(external_ids)):
-                await fconn.delete_file_blob_external(external_ids[i])
-        await asyncio.gather(del_internal(), del_external())
+            for i in range(0, len(external_ids), batch_size):
+                await fconn.unlink_file_blobs_external([r for r in external_ids[i:i+batch_size]])
+        await del_internal()
+        await del_external()
 
     async def delete_path(self, url: str, op_user: Optional[UserRecord] = None) -> Optional[list[FileRecord]]:
         validate_url(url, is_file=False)
@@ -997,7 +1070,7 @@ async def check_file_read_permission(user: UserRecord, file: FileRecord, cursor:
     This does not consider alias level permission,
     use check_path_permission for alias level permission check first:
     ```
-    if await check_path_permission(path, user) < AccessLevel.READ:
+    if await check_path_permission(file.url, user) < AccessLevel.READ:
         read_allowed, reason = check_file_read_permission(user, file)
     ```
     The implementation assumes the user is not admin and is not the owner of the file/path
@@ -1041,6 +1114,9 @@ async def check_path_permission(path: str, user: UserRecord, cursor: Optional[ai
     if user.id == 0:
         return AccessLevel.GUEST
     
+    if user.is_admin:
+        return AccessLevel.ALL
+    
     @asynccontextmanager
     async def this_cur():
         if cursor is None:
@@ -1054,7 +1130,7 @@ async def check_path_permission(path: str, user: UserRecord, cursor: Optional[ai
         path_owner = await _get_path_owner(cur, path)
 
     # check if user is admin or the owner of the path
-    if user.is_admin or user.id == path_owner.id:
+    if user.id == path_owner.id:
         return AccessLevel.ALL
     
     # if the path is a file, check if the user is the owner

lfss/eng/thumb.py

@@ -11,47 +11,42 @@ from contextlib import asynccontextmanager
 async def _maybe_init_thumb(c: aiosqlite.Cursor):
     await c.execute('''
         CREATE TABLE IF NOT EXISTS thumbs (
-            path TEXT PRIMARY KEY,
-            ctime TEXT,
+            file_id CHAR(32) PRIMARY KEY,
             thumb BLOB
         )
     ''')
-    await c.execute('CREATE INDEX IF NOT EXISTS thumbs_path_idx ON thumbs (path)')
+    await c.execute('CREATE INDEX IF NOT EXISTS thumbs_path_idx ON thumbs (file_id)')
 
-async def _get_cache_thumb(c: aiosqlite.Cursor, path: str, ctime: str) -> Optional[bytes]:
+async def _get_cache_thumb(c: aiosqlite.Cursor, file_id: str) -> Optional[bytes]:
     res = await c.execute('''
-        SELECT ctime, thumb FROM thumbs WHERE path = ? 
-    ''', (path, ))
+        SELECT thumb FROM thumbs WHERE file_id = ? 
+    ''', (file_id, ))
     row = await res.fetchone()
     if row is None:
         return None
-    # check if ctime matches, if not delete and return None
-    if row[0] != ctime:
-        await _delete_cache_thumb(c, path)
-        return None
-    blob: bytes = row[1]
+    blob: bytes = row[0]
     return blob
     
-async def _save_cache_thumb(c: aiosqlite.Cursor, path: str, ctime: str, raw_bytes: bytes) -> bytes:
+async def _save_cache_thumb(c: aiosqlite.Cursor, file_id: str, raw_bytes: bytes) -> bytes:
     try:
         raw_img = Image.open(BytesIO(raw_bytes))
     except Exception:
-        raise InvalidDataError('Invalid image data for thumbnail: ' + path)
+        raise InvalidDataError('Invalid image data for thumbnail: ' + file_id)
     raw_img.thumbnail(THUMB_SIZE)
     img = raw_img.convert('RGB')
     bio = BytesIO()
     img.save(bio, 'JPEG')
     blob = bio.getvalue()
     await c.execute('''
-        INSERT OR REPLACE INTO thumbs (path, ctime, thumb) VALUES (?, ?, ?)
-    ''', (path, ctime, blob))
+        INSERT OR REPLACE INTO thumbs (file_id, thumb) VALUES (?, ?)
+    ''', (file_id, blob))
     await c.execute('COMMIT')  # commit immediately
     return blob
 
-async def _delete_cache_thumb(c: aiosqlite.Cursor, path: str):
+async def _delete_cache_thumb(c: aiosqlite.Cursor, file_id: str):
     await c.execute('''
-        DELETE FROM thumbs WHERE path = ?
-    ''', (path, ))
+        DELETE FROM thumbs WHERE file_id = ?
+    ''', (file_id, ))
     await c.execute('COMMIT')
 
 @asynccontextmanager
@@ -75,15 +70,13 @@ async def get_thumb(path: str) -> Optional[tuple[bytes, str]]:
         r = await fconn.get_file_record(path)
 
     if r is None:
-        async with cache_cursor() as cur:
-            await _delete_cache_thumb(cur, path)
         raise FileNotFoundError(f'File not found: {path}')
     if not r.mime_type.startswith('image/'):
         return None
 
+    file_id = r.file_id
     async with cache_cursor() as cur:
-        c_time = r.create_time
-        thumb_blob = await _get_cache_thumb(cur, path, c_time)
+        thumb_blob = await _get_cache_thumb(cur, file_id)
         if thumb_blob is not None:
             return thumb_blob, "image/jpeg"
         
@@ -98,5 +91,5 @@ async def get_thumb(path: str) -> Optional[tuple[bytes, str]]:
                 data = await fconn.get_file_blob(r.file_id)
             assert data is not None
         
-        thumb_blob = await _save_cache_thumb(cur, path, c_time, data)
+        thumb_blob = await _save_cache_thumb(cur, file_id, data)
         return thumb_blob, "image/jpeg"

lfss/sql/init.sql

@@ -1,4 +1,4 @@
-CREATE TABLE IF NOT EXISTS user (
+CREATE TABLE IF NOT EXISTS main.user (
     id INTEGER PRIMARY KEY AUTOINCREMENT,
     username VARCHAR(256) UNIQUE NOT NULL,
     credential VARCHAR(256) NOT NULL,
@@ -9,7 +9,7 @@ CREATE TABLE IF NOT EXISTS user (
     permission INTEGER DEFAULT 0
 );
 
-CREATE TABLE IF NOT EXISTS fmeta (
+CREATE TABLE IF NOT EXISTS main.fmeta (
     url VARCHAR(1024) PRIMARY KEY,
     owner_id INTEGER NOT NULL,
     file_id CHAR(32) NOT NULL,
@@ -22,12 +22,17 @@ CREATE TABLE IF NOT EXISTS fmeta (
     FOREIGN KEY(owner_id) REFERENCES user(id)
 );
 
-CREATE TABLE IF NOT EXISTS usize (
+CREATE TABLE IF NOT EXISTS main.dupcount (
+    file_id CHAR(32) PRIMARY KEY,
+    count INTEGER DEFAULT 0
+);
+
+CREATE TABLE IF NOT EXISTS main.usize (
     user_id INTEGER PRIMARY KEY,
     size INTEGER DEFAULT 0
 );
 
-CREATE TABLE IF NOT EXISTS upeer (
+CREATE TABLE IF NOT EXISTS main.upeer (
     src_user_id INTEGER NOT NULL,
     dst_user_id INTEGER NOT NULL,
     access_level INTEGER DEFAULT 0,

lfss/svc/app.py

@@ -6,4 +6,4 @@ app.include_router(router_api)
 if ENABLE_WEBDAV:
     from .app_dav import *
     app.include_router(router_dav)
-app.include_router(router_fs)
+app.include_router(router_fs)

{lfss-0.10.0.dist-info → lfss-0.11.0.dist-info}/METADATA

@@ -1,10 +1,10 @@
 Metadata-Version: 2.1
 Name: lfss
-Version: 0.10.0
+Version: 0.11.0
 Summary: Lightweight file storage service
 Home-page: https://github.com/MenxLi/lfss
-Author: li_mengxun
-Author-email: limengxun45@outlookc.com
+Author: Li, Mengxun
+Author-email: mengxunli@whu.edu.cn
 Requires-Python: >=3.10
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.10

{lfss-0.10.0.dist-info → lfss-0.11.0.dist-info}/RECORD

@@ -1,9 +1,9 @@
 Readme.md,sha256=B-foESzFWoSI5MEd89AWUzKcVRrTwipM28TK8GN0o8c,1920
-docs/Changelog.md,sha256=QYej_hmGnv9t8wjFHXBvmrBOvY7aACZ82oa5SVkIyzM,882
 docs/Enviroment_variables.md,sha256=xaL8qBwT8B2Qe11FaOU3xWrRCh1mJ1VyTFCeFbkd0rs,570
 docs/Known_issues.md,sha256=ZqETcWP8lzTOel9b2mxEgCnADFF8IxOrEtiVO1NoMAk,251
 docs/Permission.md,sha256=thUJx7YRoU63Pb-eqo5l5450DrZN3QYZ36GCn8r66no,3152
 docs/Webdav.md,sha256=-Ja-BTWSY1BEMAyZycvEMNnkNTPZ49gSPzmf3Lbib70,1547
+docs/changelog.md,sha256=QYej_hmGnv9t8wjFHXBvmrBOvY7aACZ82oa5SVkIyzM,882
 frontend/api.js,sha256=GlQsNoZFEcy7QUUsLbXv7aP-KxRnIxM37FQHTaakGiQ,19387
 frontend/index.html,sha256=-k0bJ5FRqdl_H-O441D_H9E-iejgRCaL_z5UeYaS2qc,3384
 frontend/info.css,sha256=Ny0N3GywQ3a9q1_Qph_QFEKB4fEnTe_2DJ1Y5OsLLmQ,595
@@ -18,34 +18,34 @@ frontend/styles.css,sha256=xcNLqI3KBsY5TLnku8UIP0Jfr7QLajr1_KNlZj9eheM,4935
 frontend/thumb.css,sha256=rNsx766amYS2DajSQNabhpQ92gdTpNoQKmV69OKvtpI,295
 frontend/thumb.js,sha256=46ViD2TlTTWy0fx6wjoAs_5CQ4ajYB90vVzM7UO2IHw,6182
 frontend/utils.js,sha256=IYUZl77ugiXKcLxSNOWC4NSS0CdD5yRgUsDb665j0xM,2556
-lfss/api/__init__.py,sha256=8IJqrpWK1doIyVVbntvVic82A57ncwl5b0BRHX4Ri6A,6660
+lfss/api/__init__.py,sha256=vg9xx7RwfA9ypeqIteGkjDbjMq_kZy2Uti74-XlE7vM,6822
 lfss/api/connector.py,sha256=Duh57M3dOeG_M5UidZ4hMHK7ot1JsUC6RdXgIn6KTC8,12913
 lfss/cli/__init__.py,sha256=lPwPmqpa7EXQ4zlU7E7LOe6X2kw_xATGdwoHphUEirA,827
 lfss/cli/balance.py,sha256=fUbKKAUyaDn74f7mmxMfBL4Q4voyBLHu6Lg_g8GfMOQ,4121
-lfss/cli/cli.py,sha256=aYjB8d4k6JUd9efxZK-XOj-mlG4JeOr_0lnj2qqCiK0,8066
+lfss/cli/cli.py,sha256=ZgX3M-0gdArDmOi-zo8RLnRy-4GSwJDRGV1scnE4IJs,8090
 lfss/cli/panel.py,sha256=Xq3I_n-ctveym-Gh9LaUpzHiLlvt3a_nuDiwUS-MGrg,1597
 lfss/cli/serve.py,sha256=vTo6_BiD7Dn3VLvHsC5RKRBC3lMu45JVr_0SqpgHdj0,1086
 lfss/cli/user.py,sha256=1mTroQbaKxHjFCPHT67xwd08v-zxH0RZ_OnVc-4MzL0,5364
-lfss/cli/vacuum.py,sha256=GOG72d3NYe9bYCNc3y8JecEmM-DrKlGq3JQcisv_xBg,3702
+lfss/cli/vacuum.py,sha256=SciDsIdy7cfRqrXcCKBAFb9FOLyXriZBZnXlCuy6F5I,6232
 lfss/eng/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 lfss/eng/bounded_pool.py,sha256=BI1dU-MBf82TMwJBYbjhEty7w1jIUKc5Bn9SnZ_-hoY,1288
-lfss/eng/config.py,sha256=Vni6h52Ce0njVrHZLAWFL8g34YDBdlmGrmRhpxElxQ8,868
+lfss/eng/config.py,sha256=FcTtPL7bOpg54nVL_gX-VTIjfN1cafy423ezoWGvouY,874
 lfss/eng/connection_pool.py,sha256=1aq7nSgd7hB9YNV4PjD1RDRyl_moDw3ubBtSLyfgGBs,6320
-lfss/eng/database.py,sha256=RYIG2506_-S84f6CsOQ6pgpr1vgPT3p1kP1FsZwTOnM,49098
+lfss/eng/database.py,sha256=bzby4R2CbWuRsNQoWtnN-3fBLMjwLoSL-iirp6IsA_4,53247
 lfss/eng/datatype.py,sha256=27UB7-l9SICy5lAvKjdzpTL_GohZjzstQcr9PtAq7nM,2709
 lfss/eng/error.py,sha256=JGf5NV-f4rL6tNIDSAx5-l9MG8dEj7F2w_MuOjj1d1o,732
 lfss/eng/log.py,sha256=u6WRZZsE7iOx6_CV2NHh1ugea26p408FI4WstZh896A,5139
-lfss/eng/thumb.py,sha256=x9jIHHU1tskmp-TavPPcxGpbmEjCp9gbH6ZlsEfqUxY,3383
+lfss/eng/thumb.py,sha256=AFyWEkkpuCKGWOB9bLlaDwPKzQ9JtCSSmHMhX2Gu3CI,3096
 lfss/eng/utils.py,sha256=WYoXFFi5308UWtFC8VP792gpzrVbHZZHhP3PaFjxIEY,6770
-lfss/sql/init.sql,sha256=8LjHx0TBCkBD62xFfssSeHDqKYVQQJkZAg4rSm046f4,1496
+lfss/sql/init.sql,sha256=FBmVzkNjYUnWjEELRFzf7xb50GngmzmeDVffT1Uk8u8,1625
 lfss/sql/pragma.sql,sha256=uENx7xXjARmro-A3XAK8OM8v5AxDMdCCRj47f86UuXg,206
-lfss/svc/app.py,sha256=ftWCpepBx-gTSG7i-TB-IdinPPstAYYQjCgnTfeMZeI,219
+lfss/svc/app.py,sha256=r1KUO3sPaaJWbkJF0bcVTD7arPKLs2jFlq52Ixicomo,220
 lfss/svc/app_base.py,sha256=bTQbz945xalyB3UZLlqVBvL6JKGNQ8Fm2KpIvvucPZQ,6850
 lfss/svc/app_dav.py,sha256=D0KSgjtTktPjIhyIKG5eRmBdh5X8HYFYH151E6gzlbc,18245
 lfss/svc/app_native.py,sha256=JbPge-F9irl26tXKAzfA5DfyjCh0Dgttflztqqrvt0A,8890
 lfss/svc/common_impl.py,sha256=5ZRM24zVZpAeipgDtZUVBMFtArkydlAkn17ic_XL7v8,13733
 lfss/svc/request_log.py,sha256=v8yXEIzPjaksu76Oh5vgdbUEUrw8Kt4etLAXBWSGie8,3207
-lfss-0.10.0.dist-info/METADATA,sha256=NewpmEw8OUj28rPkujuPi3ZySJG_JCvSEKY5JBxg6cw,2715
-lfss-0.10.0.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-lfss-0.10.0.dist-info/entry_points.txt,sha256=VJ8svMz7RLtMCgNk99CElx7zo7M-N-z7BWDVw2HA92E,205
-lfss-0.10.0.dist-info/RECORD,,
+lfss-0.11.0.dist-info/METADATA,sha256=Exr7PdhSmrOqhURUXUiEyP_q8cUSTQ8ZWTgX7tcmp7s,2712
+lfss-0.11.0.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+lfss-0.11.0.dist-info/entry_points.txt,sha256=VJ8svMz7RLtMCgNk99CElx7zo7M-N-z7BWDVw2HA92E,205
+lfss-0.11.0.dist-info/RECORD,,