lfss 0.7.15__py3-none-any.whl → 0.8.0__py3-none-any.whl

lfss/{client/api.py → api/connector.py}

@@ -1,9 +1,13 @@
-from typing import Optional, Literal
+from __future__ import annotations
+from typing import Optional, Literal, Iterator
 import os
 import requests
+import requests.adapters
 import urllib.parse
+from lfss.src.error import PathNotFoundError
 from lfss.src.datatype import (
-    FileReadPermission, FileRecord, DirectoryRecord, UserRecord, PathContents
+    FileReadPermission, FileRecord, DirectoryRecord, UserRecord, PathContents, 
+    FileSortKey, DirSortKey
     )
 from lfss.src.utils import ensure_uri_compnents
 
@@ -11,12 +15,41 @@ _default_endpoint = os.environ.get('LFSS_ENDPOINT', 'http://localhost:8000')
 _default_token = os.environ.get('LFSS_TOKEN', '')
 
 class Connector:
+    class Session:
+        def __init__(self, connector: Connector, pool_size: int = 10):
+            self.connector = connector
+            self.pool_size = pool_size
+        def open(self):
+            self.close()
+            if self.connector._session is None:
+                s = requests.Session()
+                adapter = requests.adapters.HTTPAdapter(pool_connections=self.pool_size, pool_maxsize=self.pool_size)
+                s.mount('http://', adapter)
+                s.mount('https://', adapter)
+                self.connector._session = s
+        def close(self):
+            if self.connector._session is not None:
+                self.connector._session.close()
+            self.connector._session = None
+        def __call__(self):
+            return self.connector
+        def __enter__(self):
+            self.open()
+            return self.connector
+        def __exit__(self, exc_type, exc_value, traceback):
+            self.close()
+
     def __init__(self, endpoint=_default_endpoint, token=_default_token):
         assert token, "No token provided. Please set LFSS_TOKEN environment variable."
         self.config = {
             "endpoint": endpoint,
             "token": token
         }
+        self._session: Optional[requests.Session] = None
+    
+    def session(self, pool_size: int = 10):
+        """ avoid creating a new session for each request.  """
+        return self.Session(self, pool_size)
     
     def _fetch_factory(
         self, method: Literal['GET', 'POST', 'PUT', 'DELETE'], 
@@ -31,9 +64,13 @@ class Connector:
             headers.update({
                 'Authorization': f"Bearer {self.config['token']}",
             })
-            with requests.Session() as s:
-                response = s.request(method, url, headers=headers, **kwargs)
+            if self._session is not None:
+                response = self._session.request(method, url, headers=headers, **kwargs)
                 response.raise_for_status()
+            else:
+                with requests.Session() as s:
+                    response = s.request(method, url, headers=headers, **kwargs)
+                    response.raise_for_status()
             return response
         return f
 
@@ -80,9 +117,9 @@ class Connector:
         )
         return response.json()
     
-    def _get(self, path: str) -> Optional[requests.Response]:
+    def _get(self, path: str, stream: bool = False) -> Optional[requests.Response]:
         try:
-            response = self._fetch_factory('GET', path)()
+            response = self._fetch_factory('GET', path)(stream=stream)
         except requests.exceptions.HTTPError as e:
             if e.response.status_code == 404:
                 return None
@@ -94,6 +131,12 @@ class Connector:
         response = self._get(path)
         if response is None: return None
         return response.content
+    
+    def get_stream(self, path: str) -> Iterator[bytes]:
+        """Downloads a file from the specified path, will raise PathNotFoundError if path not found."""
+        response = self._get(path, stream=True)
+        if response is None: raise PathNotFoundError("Path not found: " + path)
+        return response.iter_content(chunk_size=1024)
 
     def get_json(self, path: str) -> Optional[dict]:
         response = self._get(path)
@@ -118,12 +161,50 @@ class Connector:
                 return None
             raise e
     
-    def list_path(self, path: str, flat: bool = False) -> PathContents:
+    def list_path(self, path: str) -> PathContents:
+        """ 
+        shorthand list with limited options, 
+        for large directories / more options, use list_files and list_dirs instead.
+        """
         assert path.endswith('/')
-        response = self._fetch_factory('GET', path, {'flat': flat})()
+        response = self._fetch_factory('GET', path)()
         dirs = [DirectoryRecord(**d) for d in response.json()['dirs']]
         files = [FileRecord(**f) for f in response.json()['files']]
         return PathContents(dirs=dirs, files=files)
+    
+    def count_files(self, path: str, flat: bool = False) -> int:
+        assert path.endswith('/')
+        response = self._fetch_factory('GET', '_api/count-files', {'path': path, 'flat': flat})()
+        return response.json()['count']
+
+    def list_files(
+        self, path: str, offset: int = 0, limit: int = 1000,
+        order_by: FileSortKey = '', order_desc: bool = False, 
+        flat: bool = False
+    ) -> list[FileRecord]:
+        assert path.endswith('/')
+        response = self._fetch_factory('GET', "_api/list-files", {
+            'path': path,
+            'offset': offset, 'limit': limit, 'order_by': order_by, 'order_desc': order_desc, 'flat': flat
+        })()
+        return [FileRecord(**f) for f in response.json()]
+    
+    def count_dirs(self, path: str) -> int:
+        assert path.endswith('/')
+        response = self._fetch_factory('GET', '_api/count-dirs', {'path': path})()
+        return response.json()['count']
+        
+    def list_dirs(
+        self, path: str, offset: int = 0, limit: int = 1000,
+        order_by: DirSortKey = '', order_desc: bool = False, 
+        skim: bool = True
+    ) -> list[DirectoryRecord]:
+        assert path.endswith('/')
+        response = self._fetch_factory('GET', "_api/list-dirs", {
+            'path': path,
+            'offset': offset, 'limit': limit, 'order_by': order_by, 'order_desc': order_desc, 'skim': skim
+        })()
+        return [DirectoryRecord(**d) for d in response.json()]
 
     def set_file_permission(self, path: str, permission: int | FileReadPermission):
         """Sets the file permission for the specified path."""

lfss/cli/cli.py

@@ -1,4 +1,4 @@
-from lfss.client import Connector, upload_directory, upload_file, download_file, download_directory
+from lfss.api import Connector, upload_directory, upload_file, download_file, download_directory
 from pathlib import Path
 import argparse
 from lfss.src.datatype import FileReadPermission

lfss/cli/user.py

@@ -29,7 +29,7 @@ async def _main():
     sp_set.add_argument('username', type=str)
     sp_set.add_argument('-p', '--password', type=str, default=None)
     sp_set.add_argument('-a', '--admin', type=parse_bool, default=None)
-    sp_set.add_argument('--permission', type=int, default=None)
+    sp_set.add_argument('--permission', type=parse_permission, default=None)
     sp_set.add_argument('--max-storage', type=parse_storage_size, default=None)
     
     sp_list = sp.add_parser('list')

lfss/src/connection_pool.py

@@ -46,7 +46,7 @@ class SqlConnection:
 
 class SqlConnectionPool:
     _r_sem: Semaphore
-    _w_sem: Semaphore
+    _w_sem: Lock | Semaphore
     def __init__(self):
         self._readers: list[SqlConnection] = []
         self._writer: None | SqlConnection = None
@@ -57,7 +57,8 @@ class SqlConnectionPool:
         self._readers = []
 
         self._writer = SqlConnection(await get_connection(read_only=False))
-        self._w_sem = Semaphore(1)
+        self._w_sem = Lock()
+        # self._w_sem = Semaphore(1)
 
         for _ in range(n_read):
             conn = await get_connection(read_only=True)

lfss/src/database.py

@@ -10,10 +10,13 @@ import aiosqlite, aiofiles
 import aiofiles.os
 
 from .connection_pool import execute_sql, unique_cursor, transaction
-from .datatype import UserRecord, FileReadPermission, FileRecord, DirectoryRecord, PathContents
+from .datatype import (
+    UserRecord, FileReadPermission, FileRecord, DirectoryRecord, PathContents, 
+    FileSortKey, DirSortKey, isValidFileSortKey, isValidDirSortKey
+    )
 from .config import LARGE_BLOB_DIR, CHUNK_SIZE
 from .log import get_logger
-from .utils import decode_uri_compnents, hash_credential, concurrent_wrap
+from .utils import decode_uri_compnents, hash_credential, concurrent_wrap, debounce_async
 from .error import *
 
 class DBObjectBase(ABC):
@@ -156,55 +159,108 @@ class FileConn(DBObjectBase):
             dirs = [await self.get_path_record(u) for u in dirnames] if not skim else [DirectoryRecord(u) for u in dirnames]
             return dirs
     
-    async def list_path(self, url: str, flat: bool = False) -> PathContents:
-        """
-        List all files and directories under the given path
-        if flat is True, list all files under the path, with out delimiting directories
-        """
-        self.logger.debug(f"Listing path {url}, flat={flat}")
-        if not url.endswith('/'):
-            url += '/'
-        if url == '/':
-            # users cannot be queried using '/', because we store them without '/' prefix, 
-            # so we need to handle this case separately, 
-            if flat:
-                cursor = await self.cur.execute("SELECT * FROM fmeta")
-                res = await cursor.fetchall()
-                files = [self.parse_record(r) for r in res]
-                return PathContents([], files)
+    async def count_path_dirs(self, url: str):
+        if not url.endswith('/'): url += '/'
+        if url == '/': url = ''
+        cursor = await self.cur.execute("""
+        SELECT COUNT(*) FROM (
+            SELECT DISTINCT SUBSTR( 
+                url, LENGTH(?) + 1, 
+                INSTR(SUBSTR(url, LENGTH(?) + 1), '/')
+                ) AS dirname
+            FROM fmeta WHERE url LIKE ? AND dirname != ''
+        )
+        """, (url, url, url + '%'))
+        res = await cursor.fetchone()
+        assert res is not None, "Error: count_path_dirs"
+        return res[0]
 
+    async def list_path_dirs(
+        self, url: str, 
+        offset: int = 0, limit: int = int(1e5), 
+        order_by: DirSortKey = '', order_desc: bool = False,
+        skim: bool = True
+        ) -> list[DirectoryRecord]:
+        if not isValidDirSortKey(order_by):
+            raise ValueError(f"Invalid order_by ({order_by})")
+
+        if not url.endswith('/'): url += '/'
+        if url == '/': url = ''
+
+        sql_qury = """
+            SELECT DISTINCT SUBSTR(
+                url, 
+                1 + LENGTH(?),
+                INSTR(SUBSTR(url, 1 + LENGTH(?)), '/')
+            ) AS dirname 
+            FROM fmeta WHERE url LIKE ? AND dirname != ''
+        """ \
+        + (f"ORDER BY {order_by} {'DESC' if order_desc else 'ASC'}" if order_by else '') \
+        + " LIMIT ? OFFSET ?"
+        cursor = await self.cur.execute(sql_qury, (url, url, url + '%', limit, offset))
+        res = await cursor.fetchall()
+        dirs_str = [r[0] for r in res]
+        async def get_dir(dir_url):
+            if skim:
+                return DirectoryRecord(dir_url)
             else:
-                return PathContents(await self.list_root_dirs(), [])
-        
+                return await self.get_path_record(dir_url)
+        dirs = await asyncio.gather(*[get_dir(url + d) for d in dirs_str])
+        return dirs
+    
+    async def count_path_files(self, url: str, flat: bool = False):
+        if not url.endswith('/'): url += '/'
+        if url == '/': url = ''
         if flat:
-            cursor = await self.cur.execute("SELECT * FROM fmeta WHERE url LIKE ?", (url + '%', ))
-            res = await cursor.fetchall()
-            files = [self.parse_record(r) for r in res]
-            return PathContents([], files)
+            cursor = await self.cur.execute("SELECT COUNT(*) FROM fmeta WHERE url LIKE ?", (url + '%', ))
+        else:
+            cursor = await self.cur.execute("SELECT COUNT(*) FROM fmeta WHERE url LIKE ? AND url NOT LIKE ?", (url + '%', url + '%/%'))
+        res = await cursor.fetchone()
+        assert res is not None, "Error: count_path_files"
+        return res[0]
 
-        cursor = await self.cur.execute("SELECT * FROM fmeta WHERE url LIKE ? AND url NOT LIKE ?", (url + '%', url + '%/%'))
+    async def list_path_files(
+        self, url: str, 
+        offset: int = 0, limit: int = int(1e5), 
+        order_by: FileSortKey = '', order_desc: bool = False,
+        flat: bool = False, 
+        ) -> list[FileRecord]:
+        if not isValidFileSortKey(order_by):
+            raise ValueError(f"Invalid order_by {order_by}")
+
+        if not url.endswith('/'): url += '/'
+        if url == '/': url = ''
+
+        sql_query = "SELECT * FROM fmeta WHERE url LIKE ?"
+        if not flat: sql_query += " AND url NOT LIKE ?"
+        if order_by: sql_query += f" ORDER BY {order_by} {'DESC' if order_desc else 'ASC'}"
+        sql_query += " LIMIT ? OFFSET ?"
+        if flat:
+            cursor = await self.cur.execute(sql_query, (url + '%', limit, offset))
+        else:
+            cursor = await self.cur.execute(sql_query, (url + '%', url + '%/%', limit, offset))
         res = await cursor.fetchall()
         files = [self.parse_record(r) for r in res]
-
-        # substr indexing starts from 1
-        cursor = await self.cur.execute(
-            """
-            SELECT DISTINCT 
-                SUBSTR(
-                    url, 
-                    1 + LENGTH(?),
-                    INSTR(SUBSTR(url, 1 + LENGTH(?)), '/') - 1
-                ) AS subdir 
-                FROM fmeta WHERE url LIKE ?
-            """, 
-            (url, url, url + '%')
+        return files
+    
+    async def list_path(self, url: str) -> PathContents:
+        """
+        List all files and directories under the given path.  
+        This method is a handy way file browsing, but has limitaions:
+        - It does not support pagination
+        - It does not support sorting
+        - It cannot flatten directories
+        - It cannot list directories with details
+        """
+        MAX_ITEMS = int(1e4)
+        dir_count = await self.count_path_dirs(url)
+        file_count = await self.count_path_files(url, flat=False)
+        if dir_count + file_count > MAX_ITEMS:
+            raise TooManyItemsError("Too many items, please paginate")
+        return PathContents(
+            dirs = await self.list_path_dirs(url, skim=True, limit=MAX_ITEMS),
+            files = await self.list_path_files(url, flat=False, limit=MAX_ITEMS)
             )
-        res = await cursor.fetchall()
-        dirs_str = [r[0] + '/' for r in res if r[0] != '/']
-        async def get_dir(dir_url):
-            return DirectoryRecord(dir_url, -1)
-        dirs = await asyncio.gather(*[get_dir(url + d) for d in dirs_str])
-        return PathContents(dirs, files)
     
     async def get_path_record(self, url: str) -> DirectoryRecord:
         """
@@ -361,7 +417,8 @@ class FileConn(DBObjectBase):
     async def set_file_blob(self, file_id: str, blob: bytes):
         await self.cur.execute("INSERT OR REPLACE INTO blobs.fdata (file_id, data) VALUES (?, ?)", (file_id, blob))
     
-    async def set_file_blob_external(self, file_id: str, stream: AsyncIterable[bytes])->int:
+    @staticmethod
+    async def set_file_blob_external(file_id: str, stream: AsyncIterable[bytes])->int:
         size_sum = 0
         try:
             async with aiofiles.open(LARGE_BLOB_DIR / file_id, 'wb') as f:
@@ -389,7 +446,8 @@ class FileConn(DBObjectBase):
                 if not chunk: break
                 yield chunk
     
-    async def delete_file_blob_external(self, file_id: str):
+    @staticmethod
+    async def delete_file_blob_external(file_id: str):
         if (LARGE_BLOB_DIR / file_id).exists():
             await aiofiles.os.remove(LARGE_BLOB_DIR / file_id)
     
@@ -399,6 +457,36 @@ class FileConn(DBObjectBase):
     async def delete_file_blobs(self, file_ids: list[str]):
         await self.cur.execute("DELETE FROM blobs.fdata WHERE file_id IN ({})".format(','.join(['?'] * len(file_ids))), file_ids)
 
+_log_active_queue = []
+_log_active_lock = asyncio.Lock()
+@debounce_async()
+async def _set_all_active():
+    async with transaction() as conn:
+        uconn = UserConn(conn)
+        async with _log_active_lock:
+            for u in _log_active_queue:
+                await uconn.set_active(u)
+            _log_active_queue.clear()
+async def delayed_log_activity(username: str):
+    async with _log_active_lock:
+        _log_active_queue.append(username)
+    await _set_all_active()
+
+_log_access_queue = []
+_log_access_lock = asyncio.Lock()
+@debounce_async()
+async def _log_all_access():
+    async with transaction() as conn:
+        fconn = FileConn(conn)
+        async with _log_access_lock:
+            for r in _log_access_queue:
+                await fconn.log_access(r)
+            _log_access_queue.clear()
+async def delayed_log_access(url: str):
+    async with _log_access_lock:
+        _log_access_queue.append(url)
+    await _log_all_access()
+
 def validate_url(url: str, is_file = True):
     prohibited_chars = ['..', ';', "'", '"', '\\', '\0', '\n', '\r', '\t', '\x0b', '\x0c']
     ret = not url.startswith('/') and not url.startswith('_') and not url.startswith('.')
@@ -433,11 +521,6 @@ class Database:
             await execute_sql(conn, 'init.sql')
         return self
     
-    async def record_user_activity(self, u: str):
-        async with transaction() as conn:
-            uconn = UserConn(conn)
-            await uconn.set_active(u)
-    
     async def update_file_record(self, user: UserRecord, url: str, permission: FileReadPermission):
         validate_url(url)
         async with transaction() as conn:
@@ -459,9 +542,7 @@ class Database:
         if file_size is not provided, the blob must be bytes
         """
         validate_url(url)
-        async with transaction() as cur:
-            uconn = UserConn(cur)
-            fconn = FileConn(cur)
+        async with unique_cursor() as cur:
             user = await get_user(cur, u)
             if user is None:
                 return
@@ -477,27 +558,35 @@ class Database:
                     if await get_user(cur, first_component) is None:
                         raise PermissionDeniedError(f"Invalid path: {first_component} is not a valid username")
             
-            user_size_used = await fconn.user_size(user.id)
+            fconn_r = FileConn(cur)
+            user_size_used = await fconn_r.user_size(user.id)
+
             if isinstance(blob, bytes):
                 file_size = len(blob)
                 if user_size_used + file_size > user.max_storage:
                     raise StorageExceededError(f"Unable to save file, user {user.username} has storage limit of {user.max_storage}, used {user_size_used}, requested {file_size}")
                 f_id = uuid.uuid4().hex
-                await fconn.set_file_blob(f_id, blob)
-                await fconn.set_file_record(
-                    url, owner_id=user.id, file_id=f_id, file_size=file_size, 
-                    permission=permission, external=False, mime_type=mime_type)
+
+                async with transaction() as w_cur:
+                    fconn_w = FileConn(w_cur)
+                    await fconn_w.set_file_blob(f_id, blob)
+                    await fconn_w.set_file_record(
+                        url, owner_id=user.id, file_id=f_id, file_size=file_size, 
+                        permission=permission, external=False, mime_type=mime_type)
             else:
                 assert isinstance(blob, AsyncIterable)
                 f_id = uuid.uuid4().hex
-                file_size = await fconn.set_file_blob_external(f_id, blob)
+                file_size = await FileConn.set_file_blob_external(f_id, blob)
                 if user_size_used + file_size > user.max_storage:
-                    await fconn.delete_file_blob_external(f_id)
+                    await FileConn.delete_file_blob_external(f_id)
                     raise StorageExceededError(f"Unable to save file, user {user.username} has storage limit of {user.max_storage}, used {user_size_used}, requested {file_size}")
-                await fconn.set_file_record(
-                    url, owner_id=user.id, file_id=f_id, file_size=file_size, 
-                    permission=permission, external=True, mime_type=mime_type)
-            await uconn.set_active(user.username)
+                
+                async with transaction() as w_cur:
+                    await FileConn(w_cur).set_file_record(
+                        url, owner_id=user.id, file_id=f_id, file_size=file_size, 
+                        permission=permission, external=True, mime_type=mime_type)
+            
+        await delayed_log_activity(user.username)
 
     async def read_file_stream(self, url: str) -> AsyncIterable[bytes]:
         validate_url(url)
@@ -510,9 +599,7 @@ class Database:
                 raise ValueError(f"File {url} is not stored externally, should use read_file instead")
             ret = fconn.get_file_blob_external(r.file_id)
 
-        async with transaction() as w_cur:
-            await FileConn(w_cur).log_access(url)
-        
+        await delayed_log_access(url)
         return ret
 
 
@@ -532,9 +619,7 @@ class Database:
             if blob is None:
                 raise FileNotFoundError(f"File {url} data not found")
 
-        async with transaction() as w_cur:
-            await FileConn(w_cur).log_access(url)
-
+        await delayed_log_access(url)
         return blob
 
     async def delete_file(self, url: str, assure_user: Optional[UserRecord] = None) -> Optional[FileRecord]:
@@ -653,7 +738,8 @@ class Database:
         async with unique_cursor() as cur:
             fconn = FileConn(cur)
             if urls is None:
-                urls = [r.url for r in (await fconn.list_path(top_url, flat=True)).files]
+                fcount = await fconn.count_path_files(top_url, flat=True)
+                urls = [r.url for r in (await fconn.list_path_files(top_url, flat=True, limit=fcount))]
 
             for url in urls:
                 if not url.startswith(top_url):

lfss/src/datatype.py

@@ -1,4 +1,5 @@
 from enum import IntEnum
+from typing import Literal
 import dataclasses
 
 class FileReadPermission(IntEnum):
@@ -51,6 +52,10 @@ class DirectoryRecord:
 
 @dataclasses.dataclass
 class PathContents:
-    dirs: list[DirectoryRecord]
-    files: list[FileRecord]
-    
+    dirs: list[DirectoryRecord] = dataclasses.field(default_factory=list)
+    files: list[FileRecord] = dataclasses.field(default_factory=list)
+    
+FileSortKey = Literal['', 'url', 'file_size', 'create_time', 'access_time', 'mime_type']
+isValidFileSortKey = lambda x: x in ['', 'url', 'file_size', 'create_time', 'access_time', 'mime_type']
+DirSortKey = Literal['', 'dirname']
+isValidDirSortKey = lambda x: x in ['', 'dirname']

lfss/src/error.py

@@ -7,4 +7,6 @@ class PermissionDeniedError(LFSSExceptionBase, PermissionError):...
 
 class InvalidPathError(LFSSExceptionBase, ValueError):...
 
-class StorageExceededError(LFSSExceptionBase):...
+class StorageExceededError(LFSSExceptionBase):...
+
+class TooManyItemsError(LFSSExceptionBase):...