lfss 0.7.15__py3-none-any.whl → 0.8.1__py3-none-any.whl

lfss/src/database.py

@@ -8,12 +8,16 @@ import zipfile, io, asyncio
 
 import aiosqlite, aiofiles
 import aiofiles.os
+import mimetypes, mimesniff
 
 from .connection_pool import execute_sql, unique_cursor, transaction
-from .datatype import UserRecord, FileReadPermission, FileRecord, DirectoryRecord, PathContents
-from .config import LARGE_BLOB_DIR, CHUNK_SIZE
+from .datatype import (
+    UserRecord, FileReadPermission, FileRecord, DirectoryRecord, PathContents, 
+    FileSortKey, DirSortKey, isValidFileSortKey, isValidDirSortKey
+    )
+from .config import LARGE_BLOB_DIR, CHUNK_SIZE, LARGE_FILE_BYTES, MAX_MEM_FILE_BYTES
 from .log import get_logger
-from .utils import decode_uri_compnents, hash_credential, concurrent_wrap
+from .utils import decode_uri_compnents, hash_credential, concurrent_wrap, debounce_async
 from .error import *
 
 class DBObjectBase(ABC):
@@ -156,55 +160,108 @@ class FileConn(DBObjectBase):
             dirs = [await self.get_path_record(u) for u in dirnames] if not skim else [DirectoryRecord(u) for u in dirnames]
             return dirs
     
-    async def list_path(self, url: str, flat: bool = False) -> PathContents:
-        """
-        List all files and directories under the given path
-        if flat is True, list all files under the path, with out delimiting directories
-        """
-        self.logger.debug(f"Listing path {url}, flat={flat}")
-        if not url.endswith('/'):
-            url += '/'
-        if url == '/':
-            # users cannot be queried using '/', because we store them without '/' prefix, 
-            # so we need to handle this case separately, 
-            if flat:
-                cursor = await self.cur.execute("SELECT * FROM fmeta")
-                res = await cursor.fetchall()
-                files = [self.parse_record(r) for r in res]
-                return PathContents([], files)
+    async def count_path_dirs(self, url: str):
+        if not url.endswith('/'): url += '/'
+        if url == '/': url = ''
+        cursor = await self.cur.execute("""
+        SELECT COUNT(*) FROM (
+            SELECT DISTINCT SUBSTR( 
+                url, LENGTH(?) + 1, 
+                INSTR(SUBSTR(url, LENGTH(?) + 1), '/')
+                ) AS dirname
+            FROM fmeta WHERE url LIKE ? AND dirname != ''
+        )
+        """, (url, url, url + '%'))
+        res = await cursor.fetchone()
+        assert res is not None, "Error: count_path_dirs"
+        return res[0]
 
+    async def list_path_dirs(
+        self, url: str, 
+        offset: int = 0, limit: int = int(1e5), 
+        order_by: DirSortKey = '', order_desc: bool = False,
+        skim: bool = True
+        ) -> list[DirectoryRecord]:
+        if not isValidDirSortKey(order_by):
+            raise ValueError(f"Invalid order_by ({order_by})")
+
+        if not url.endswith('/'): url += '/'
+        if url == '/': url = ''
+
+        sql_qury = """
+            SELECT DISTINCT SUBSTR(
+                url, 
+                1 + LENGTH(?),
+                INSTR(SUBSTR(url, 1 + LENGTH(?)), '/')
+            ) AS dirname 
+            FROM fmeta WHERE url LIKE ? AND dirname != ''
+        """ \
+        + (f"ORDER BY {order_by} {'DESC' if order_desc else 'ASC'}" if order_by else '') \
+        + " LIMIT ? OFFSET ?"
+        cursor = await self.cur.execute(sql_qury, (url, url, url + '%', limit, offset))
+        res = await cursor.fetchall()
+        dirs_str = [r[0] for r in res]
+        async def get_dir(dir_url):
+            if skim:
+                return DirectoryRecord(dir_url)
             else:
-                return PathContents(await self.list_root_dirs(), [])
-        
+                return await self.get_path_record(dir_url)
+        dirs = await asyncio.gather(*[get_dir(url + d) for d in dirs_str])
+        return dirs
+    
+    async def count_path_files(self, url: str, flat: bool = False):
+        if not url.endswith('/'): url += '/'
+        if url == '/': url = ''
         if flat:
-            cursor = await self.cur.execute("SELECT * FROM fmeta WHERE url LIKE ?", (url + '%', ))
-            res = await cursor.fetchall()
-            files = [self.parse_record(r) for r in res]
-            return PathContents([], files)
+            cursor = await self.cur.execute("SELECT COUNT(*) FROM fmeta WHERE url LIKE ?", (url + '%', ))
+        else:
+            cursor = await self.cur.execute("SELECT COUNT(*) FROM fmeta WHERE url LIKE ? AND url NOT LIKE ?", (url + '%', url + '%/%'))
+        res = await cursor.fetchone()
+        assert res is not None, "Error: count_path_files"
+        return res[0]
 
-        cursor = await self.cur.execute("SELECT * FROM fmeta WHERE url LIKE ? AND url NOT LIKE ?", (url + '%', url + '%/%'))
+    async def list_path_files(
+        self, url: str, 
+        offset: int = 0, limit: int = int(1e5), 
+        order_by: FileSortKey = '', order_desc: bool = False,
+        flat: bool = False, 
+        ) -> list[FileRecord]:
+        if not isValidFileSortKey(order_by):
+            raise ValueError(f"Invalid order_by {order_by}")
+
+        if not url.endswith('/'): url += '/'
+        if url == '/': url = ''
+
+        sql_query = "SELECT * FROM fmeta WHERE url LIKE ?"
+        if not flat: sql_query += " AND url NOT LIKE ?"
+        if order_by: sql_query += f" ORDER BY {order_by} {'DESC' if order_desc else 'ASC'}"
+        sql_query += " LIMIT ? OFFSET ?"
+        if flat:
+            cursor = await self.cur.execute(sql_query, (url + '%', limit, offset))
+        else:
+            cursor = await self.cur.execute(sql_query, (url + '%', url + '%/%', limit, offset))
         res = await cursor.fetchall()
         files = [self.parse_record(r) for r in res]
-
-        # substr indexing starts from 1
-        cursor = await self.cur.execute(
-            """
-            SELECT DISTINCT 
-                SUBSTR(
-                    url, 
-                    1 + LENGTH(?),
-                    INSTR(SUBSTR(url, 1 + LENGTH(?)), '/') - 1
-                ) AS subdir 
-                FROM fmeta WHERE url LIKE ?
-            """, 
-            (url, url, url + '%')
+        return files
+    
+    async def list_path(self, url: str) -> PathContents:
+        """
+        List all files and directories under the given path.  
+        This method is a handy way file browsing, but has limitaions:
+        - It does not support pagination
+        - It does not support sorting
+        - It cannot flatten directories
+        - It cannot list directories with details
+        """
+        MAX_ITEMS = int(1e4)
+        dir_count = await self.count_path_dirs(url)
+        file_count = await self.count_path_files(url, flat=False)
+        if dir_count + file_count > MAX_ITEMS:
+            raise TooManyItemsError("Too many items, please paginate")
+        return PathContents(
+            dirs = await self.list_path_dirs(url, skim=True, limit=MAX_ITEMS),
+            files = await self.list_path_files(url, flat=False, limit=MAX_ITEMS)
             )
-        res = await cursor.fetchall()
-        dirs_str = [r[0] + '/' for r in res if r[0] != '/']
-        async def get_dir(dir_url):
-            return DirectoryRecord(dir_url, -1)
-        dirs = await asyncio.gather(*[get_dir(url + d) for d in dirs_str])
-        return PathContents(dirs, files)
     
     async def get_path_record(self, url: str) -> DirectoryRecord:
         """
@@ -229,8 +286,7 @@ class FileConn(DBObjectBase):
     async def user_size(self, user_id: int) -> int:
         cursor = await self.cur.execute("SELECT size FROM usize WHERE user_id = ?", (user_id, ))
         res = await cursor.fetchone()
-        if res is None:
-            return -1
+        if res is None: return 0
         return res[0]
     async def _user_size_inc(self, user_id: int, inc: int):
         self.logger.debug(f"Increasing user {user_id} size by {inc}")
@@ -361,7 +417,8 @@ class FileConn(DBObjectBase):
     async def set_file_blob(self, file_id: str, blob: bytes):
         await self.cur.execute("INSERT OR REPLACE INTO blobs.fdata (file_id, data) VALUES (?, ?)", (file_id, blob))
     
-    async def set_file_blob_external(self, file_id: str, stream: AsyncIterable[bytes])->int:
+    @staticmethod
+    async def set_file_blob_external(file_id: str, stream: AsyncIterable[bytes])->int:
         size_sum = 0
         try:
             async with aiofiles.open(LARGE_BLOB_DIR / file_id, 'wb') as f:
@@ -389,7 +446,8 @@ class FileConn(DBObjectBase):
                 if not chunk: break
                 yield chunk
     
-    async def delete_file_blob_external(self, file_id: str):
+    @staticmethod
+    async def delete_file_blob_external(file_id: str):
         if (LARGE_BLOB_DIR / file_id).exists():
             await aiofiles.os.remove(LARGE_BLOB_DIR / file_id)
     
@@ -399,6 +457,36 @@ class FileConn(DBObjectBase):
     async def delete_file_blobs(self, file_ids: list[str]):
         await self.cur.execute("DELETE FROM blobs.fdata WHERE file_id IN ({})".format(','.join(['?'] * len(file_ids))), file_ids)
 
+_log_active_queue = []
+_log_active_lock = asyncio.Lock()
+@debounce_async()
+async def _set_all_active():
+    async with transaction() as conn:
+        uconn = UserConn(conn)
+        async with _log_active_lock:
+            for u in _log_active_queue:
+                await uconn.set_active(u)
+            _log_active_queue.clear()
+async def delayed_log_activity(username: str):
+    async with _log_active_lock:
+        _log_active_queue.append(username)
+    await _set_all_active()
+
+_log_access_queue = []
+_log_access_lock = asyncio.Lock()
+@debounce_async()
+async def _log_all_access():
+    async with transaction() as conn:
+        fconn = FileConn(conn)
+        async with _log_access_lock:
+            for r in _log_access_queue:
+                await fconn.log_access(r)
+            _log_access_queue.clear()
+async def delayed_log_access(url: str):
+    async with _log_access_lock:
+        _log_access_queue.append(url)
+    await _log_all_access()
+
 def validate_url(url: str, is_file = True):
     prohibited_chars = ['..', ';', "'", '"', '\\', '\0', '\n', '\r', '\t', '\x0b', '\x0c']
     ret = not url.startswith('/') and not url.startswith('_') and not url.startswith('.')
@@ -433,11 +521,6 @@ class Database:
             await execute_sql(conn, 'init.sql')
         return self
     
-    async def record_user_activity(self, u: str):
-        async with transaction() as conn:
-            uconn = UserConn(conn)
-            await uconn.set_active(u)
-    
     async def update_file_record(self, user: UserRecord, url: str, permission: FileReadPermission):
         validate_url(url)
         async with transaction() as conn:
@@ -451,53 +534,66 @@ class Database:
     
     async def save_file(
         self, u: int | str, url: str, 
-        blob: bytes | AsyncIterable[bytes], 
+        blob_stream: AsyncIterable[bytes], 
         permission: FileReadPermission = FileReadPermission.UNSET, 
-        mime_type: str = 'application/octet-stream'
-        ):
+        mime_type: Optional[str] = None
+        ) -> int:
         """
-        if file_size is not provided, the blob must be bytes
+        Save a file to the database. 
+        Will check file size and user storage limit, 
+        should check permission before calling this method. 
         """
         validate_url(url)
-        async with transaction() as cur:
-            uconn = UserConn(cur)
-            fconn = FileConn(cur)
+        async with unique_cursor() as cur:
             user = await get_user(cur, u)
-            if user is None:
-                return
-            
-            # check if the user is the owner of the path, or is admin
-            if url.startswith('/'):
-                url = url[1:]
-            first_component = url.split('/')[0]
-            if first_component != user.username:
-                if not user.is_admin:
-                    raise PermissionDeniedError(f"Permission denied: {user.username} cannot write to {url}")
-                else:
-                    if await get_user(cur, first_component) is None:
-                        raise PermissionDeniedError(f"Invalid path: {first_component} is not a valid username")
+            assert user is not None, f"User {u} not found"
             
-            user_size_used = await fconn.user_size(user.id)
-            if isinstance(blob, bytes):
-                file_size = len(blob)
-                if user_size_used + file_size > user.max_storage:
-                    raise StorageExceededError(f"Unable to save file, user {user.username} has storage limit of {user.max_storage}, used {user_size_used}, requested {file_size}")
-                f_id = uuid.uuid4().hex
-                await fconn.set_file_blob(f_id, blob)
-                await fconn.set_file_record(
-                    url, owner_id=user.id, file_id=f_id, file_size=file_size, 
-                    permission=permission, external=False, mime_type=mime_type)
-            else:
-                assert isinstance(blob, AsyncIterable)
-                f_id = uuid.uuid4().hex
-                file_size = await fconn.set_file_blob_external(f_id, blob)
+            fconn_r = FileConn(cur)
+            user_size_used = await fconn_r.user_size(user.id)
+
+            f_id = uuid.uuid4().hex
+            async with aiofiles.tempfile.SpooledTemporaryFile(max_size=MAX_MEM_FILE_BYTES) as f:
+                async for chunk in blob_stream:
+                    await f.write(chunk)
+                file_size = await f.tell()
                 if user_size_used + file_size > user.max_storage:
-                    await fconn.delete_file_blob_external(f_id)
                     raise StorageExceededError(f"Unable to save file, user {user.username} has storage limit of {user.max_storage}, used {user_size_used}, requested {file_size}")
-                await fconn.set_file_record(
-                    url, owner_id=user.id, file_id=f_id, file_size=file_size, 
-                    permission=permission, external=True, mime_type=mime_type)
-            await uconn.set_active(user.username)
+                
+                # check mime type
+                if mime_type is None:
+                    fname = url.split('/')[-1]
+                    mime_type, _ = mimetypes.guess_type(fname)
+                if mime_type is None:
+                    await f.seek(0)
+                    mime_type = mimesniff.what(await f.read(1024))
+                if mime_type is None:
+                    mime_type = 'application/octet-stream'
+                await f.seek(0)
+                
+                if file_size < LARGE_FILE_BYTES:
+                    blob = await f.read()
+                    async with transaction() as w_cur:
+                        fconn_w = FileConn(w_cur)
+                        await fconn_w.set_file_blob(f_id, blob)
+                        await fconn_w.set_file_record(
+                            url, owner_id=user.id, file_id=f_id, file_size=file_size, 
+                            permission=permission, external=False, mime_type=mime_type)
+                
+                else:
+                    async def blob_stream_tempfile():
+                        nonlocal f
+                        while True:
+                            chunk = await f.read(CHUNK_SIZE)
+                            if not chunk: break
+                            yield chunk
+                    await FileConn.set_file_blob_external(f_id, blob_stream_tempfile())
+                    async with transaction() as w_cur:
+                        await FileConn(w_cur).set_file_record(
+                            url, owner_id=user.id, file_id=f_id, file_size=file_size, 
+                            permission=permission, external=True, mime_type=mime_type)
+
+        await delayed_log_activity(user.username)
+        return file_size
 
     async def read_file_stream(self, url: str) -> AsyncIterable[bytes]:
         validate_url(url)
@@ -510,9 +606,7 @@ class Database:
                 raise ValueError(f"File {url} is not stored externally, should use read_file instead")
             ret = fconn.get_file_blob_external(r.file_id)
 
-        async with transaction() as w_cur:
-            await FileConn(w_cur).log_access(url)
-        
+        await delayed_log_access(url)
         return ret
 
 
@@ -532,9 +626,7 @@ class Database:
             if blob is None:
                 raise FileNotFoundError(f"File {url} data not found")
 
-        async with transaction() as w_cur:
-            await FileConn(w_cur).log_access(url)
-
+        await delayed_log_access(url)
         return blob
 
     async def delete_file(self, url: str, assure_user: Optional[UserRecord] = None) -> Optional[FileRecord]:
@@ -653,7 +745,8 @@ class Database:
         async with unique_cursor() as cur:
             fconn = FileConn(cur)
             if urls is None:
-                urls = [r.url for r in (await fconn.list_path(top_url, flat=True)).files]
+                fcount = await fconn.count_path_files(top_url, flat=True)
+                urls = [r.url for r in (await fconn.list_path_files(top_url, flat=True, limit=fcount))]
 
             for url in urls:
                 if not url.startswith(top_url):

lfss/src/datatype.py

@@ -1,4 +1,5 @@
 from enum import IntEnum
+from typing import Literal
 import dataclasses
 
 class FileReadPermission(IntEnum):
@@ -51,6 +52,10 @@ class DirectoryRecord:
 
 @dataclasses.dataclass
 class PathContents:
-    dirs: list[DirectoryRecord]
-    files: list[FileRecord]
-    
+    dirs: list[DirectoryRecord] = dataclasses.field(default_factory=list)
+    files: list[FileRecord] = dataclasses.field(default_factory=list)
+    
+FileSortKey = Literal['', 'url', 'file_size', 'create_time', 'access_time', 'mime_type']
+isValidFileSortKey = lambda x: x in ['', 'url', 'file_size', 'create_time', 'access_time', 'mime_type']
+DirSortKey = Literal['', 'dirname']
+isValidDirSortKey = lambda x: x in ['', 'dirname']

lfss/src/error.py

@@ -7,4 +7,6 @@ class PermissionDeniedError(LFSSExceptionBase, PermissionError):...
 
 class InvalidPathError(LFSSExceptionBase, ValueError):...
 
-class StorageExceededError(LFSSExceptionBase):...
+class StorageExceededError(LFSSExceptionBase):...
+
+class TooManyItemsError(LFSSExceptionBase):...