lfss 0.7.4__py3-none-any.whl → 0.7.5__py3-none-any.whl

frontend/api.js

@@ -133,12 +133,17 @@ export default class Connector {
 
     /**
      * @param {string} path - the path to the file directory, should ends with '/'
+     * @param {Object} options - the options for the request
      * @returns {Promise<PathListResponse>} - the promise of the request
      */
-    async listPath(path){
+    async listPath(path, {
+        flat = false
+    } = {}){
         if (path.startsWith('/')){ path = path.slice(1); }
         if (!path.endsWith('/')){ path += '/'; }
-        const res = await fetch(this.config.endpoint + '/' + path, {
+        const dst = new URL(this.config.endpoint + '/' + path);
+        dst.searchParams.append('flat', flat);
+        const res = await fetch(dst.toString(), {
             method: 'GET',
             headers: {
                 'Authorization': 'Bearer ' + this.config.token

lfss/client/__init__.py

@@ -150,6 +150,6 @@ def download_directory(
             failed_files.append(src_url)
         
     with ThreadPoolExecutor(n_concurrent) as executor:
-        for file in connector.list_path(src_path).files:
+        for file in connector.list_path(src_path, flat=True).files:
             executor.submit(get_file, file.url)
     return failed_files

lfss/client/api.py

@@ -117,9 +117,9 @@ class Connector:
                 return None
             raise e
     
-    def list_path(self, path: str) -> PathContents:
+    def list_path(self, path: str, flat: bool = False) -> PathContents:
         assert path.endswith('/')
-        response = self._fetch_factory('GET', path)()
+        response = self._fetch_factory('GET', path, {'flat': flat})()
         dirs = [DirectoryRecord(**d) for d in response.json()['dirs']]
         files = [FileRecord(**f) for f in response.json()['files']]
         return PathContents(dirs=dirs, files=files)

lfss/src/config.py

@@ -1,5 +1,5 @@
 from pathlib import Path
-import os, hashlib
+import os
 
 __default_dir = '.storage_data'
 

lfss/src/connection_pool.py

@@ -18,13 +18,22 @@ async def execute_sql(conn: aiosqlite.Connection | aiosqlite.Cursor, name: str):
     for s in sql:
         await conn.execute(s)
 
-async def get_connection() -> aiosqlite.Connection:
+async def get_connection(read_only: bool = False) -> aiosqlite.Connection:
     if not os.environ.get('SQLITE_TEMPDIR'):
         os.environ['SQLITE_TEMPDIR'] = str(DATA_HOME)
-    # large blobs are stored in a separate database, should be more efficient
-    conn = await aiosqlite.connect(DATA_HOME / 'index.db', timeout = 60)
+
+    def get_db_uri(path: Path, read_only: bool = False):
+        return f"file:{path}?mode={ 'ro' if read_only else 'rwc' }"
+
+    conn = await aiosqlite.connect(
+        get_db_uri(DATA_HOME / 'index.db', read_only=read_only), 
+        timeout = 60, uri = True
+        )
     async with conn.cursor() as c:
-        await c.execute(f"ATTACH DATABASE ? AS blobs", (str(DATA_HOME/'blobs.db'), ))
+        await c.execute(
+            f"ATTACH DATABASE ? AS blobs", 
+            (get_db_uri(DATA_HOME/'blobs.db', read_only=read_only), )
+            )
     await execute_sql(conn, 'pragma.sql')
     return conn
 
@@ -35,47 +44,49 @@ class SqlConnection:
     is_available: bool = True
 
 class SqlConnectionPool:
-    _sem: Semaphore
+    _r_sem: Semaphore
     _w_sem: Semaphore
     def __init__(self):
-        self._connections: list[SqlConnection] = []
-        self._w_connection: None | SqlConnection = None
+        self._readers: list[SqlConnection] = []
+        self._writer: None | SqlConnection = None
         self._lock = Lock()
     
     async def init(self, n_read: int):
         await self.close()
-        self._connections = []
-        for _ in range(n_read):
-            conn = await get_connection()
-            self._connections.append(SqlConnection(conn))
-        self._w_connection = SqlConnection(await get_connection())
-        self._sem = Semaphore(n_read)
+        self._readers = []
+
+        self._writer = SqlConnection(await get_connection(read_only=False))
         self._w_sem = Semaphore(1)
+
+        for _ in range(n_read):
+            conn = await get_connection(read_only=True)
+            self._readers.append(SqlConnection(conn))
+        self._r_sem = Semaphore(n_read)
     
     @property
     def n_read(self):
-        return len(self._connections)
+        return len(self._readers)
     @property
-    def sem(self):
-        return self._sem
+    def r_sem(self):
+        return self._r_sem
     @property
     def w_sem(self):
         return self._w_sem
     
     async def get(self, w: bool = False) -> SqlConnection:
-        if len(self._connections) == 0:
+        if len(self._readers) == 0:
             raise Exception("No available connections, please init the pool first")
         
         async with self._lock:
             if w:
-                assert self._w_connection
-                if self._w_connection.is_available:
-                    self._w_connection.is_available = False
-                    return self._w_connection
+                assert self._writer
+                if self._writer.is_available:
+                    self._writer.is_available = False
+                    return self._writer
                 raise Exception("Write connection is not available")
 
             else:
-                for c in self._connections:
+                for c in self._readers:
                     if c.is_available:
                         c.is_available = False
                         return c
@@ -83,19 +94,19 @@ class SqlConnectionPool:
     
     async def release(self, conn: SqlConnection):
         async with self._lock:
-            if conn == self._w_connection:
+            if conn == self._writer:
                 conn.is_available = True
                 return
 
-            if not conn in self._connections:
+            if not conn in self._readers:
                 raise Exception("Connection not in pool")
             conn.is_available = True
     
     async def close(self):
-        for c in self._connections:
+        for c in self._readers:
             await c.conn.close()
-        if self._w_connection:
-            await self._w_connection.conn.close()
+        if self._writer:
+            await self._writer.conn.close()
 
 # these two functions shold be called before and after the event loop
 g_pool = SqlConnectionPool()
@@ -125,7 +136,7 @@ def global_entrance(n_read: int = 1):
 @asynccontextmanager
 async def unique_cursor(is_write: bool = False):
     if not is_write:
-        async with g_pool.sem:
+        async with g_pool.r_sem:
             connection_obj = await g_pool.get()
             try:
                 yield await connection_obj.conn.cursor()

lfss/src/database.py

@@ -1,5 +1,5 @@
 
-from typing import Optional, overload, Literal, AsyncIterable
+from typing import Optional, Literal, AsyncIterable
 from abc import ABC
 
 import urllib.parse
@@ -29,9 +29,6 @@ class DBObjectBase(ABC):
             raise ValueError("Connection not set")
         return self._cur
 
-    # async def commit(self):
-    #     await self.conn.commit()
-
 DECOY_USER = UserRecord(0, 'decoy', 'decoy', False, '2021-01-01 00:00:00', '2021-01-01 00:00:00', 0, FileReadPermission.PRIVATE)
 class UserConn(DBObjectBase):
 
@@ -43,10 +40,6 @@ class UserConn(DBObjectBase):
     def parse_record(record) -> UserRecord:
         return UserRecord(*record)
 
-    async def init(self, cur: aiosqlite.Cursor):
-        self.set_cursor(cur)
-        return self
-    
     async def get_user(self, username: str) -> Optional[UserRecord]:
         await self.cur.execute("SELECT * FROM user WHERE username = ?", (username, ))
         res = await self.cur.fetchone()
@@ -132,10 +125,6 @@ class FileConn(DBObjectBase):
     def parse_record(record) -> FileRecord:
         return FileRecord(*record)
     
-    def init(self, cur: aiosqlite.Cursor):
-        self.set_cursor(cur)
-        return self
-    
     async def get_file_record(self, url: str) -> Optional[FileRecord]:
         cursor = await self.cur.execute("SELECT * FROM fmeta WHERE url = ?", (url, ))
         res = await cursor.fetchone()
@@ -150,7 +139,7 @@ class FileConn(DBObjectBase):
             return []
         return [self.parse_record(r) for r in res]
     
-    async def list_root(self, *usernames: str) -> list[DirectoryRecord]:
+    async def list_root_dirs(self, *usernames: str) -> list[DirectoryRecord]:
         """
         Efficiently list users' directories, if usernames is empty, list all users' directories.
         """
@@ -167,17 +156,12 @@ class FileConn(DBObjectBase):
             dirs = [DirectoryRecord(u, await self.path_size(u, include_subpath=True)) for u in dirnames]
             return dirs
     
-    @overload
-    async def list_path(self, url: str, flat: Literal[True]) -> list[FileRecord]:...
-    @overload
-    async def list_path(self, url: str, flat: Literal[False]) -> PathContents:...
-    
-    async def list_path(self, url: str, flat: bool = False) -> list[FileRecord] | PathContents:
+    async def list_path(self, url: str, flat: bool = False) -> PathContents:
         """
-        List all files and directories under the given path, 
-        if flat is True, return a list of FileDBRecord, recursively including all subdirectories. 
-        Otherwise, return a tuple of (dirs, files), where dirs is a list of DirectoryRecord,
+        List all files and directories under the given path
+        if flat is True, list all files under the path, with out delimiting directories
         """
+        self.logger.debug(f"Listing path {url}, flat={flat}")
         if not url.endswith('/'):
             url += '/'
         if url == '/':
@@ -186,15 +170,17 @@ class FileConn(DBObjectBase):
             if flat:
                 cursor = await self.cur.execute("SELECT * FROM fmeta")
                 res = await cursor.fetchall()
-                return [self.parse_record(r) for r in res]
+                files = [self.parse_record(r) for r in res]
+                return PathContents([], files)
 
             else:
-                return PathContents(await self.list_root(), [])
+                return PathContents(await self.list_root_dirs(), [])
         
         if flat:
             cursor = await self.cur.execute("SELECT * FROM fmeta WHERE url LIKE ?", (url + '%', ))
             res = await cursor.fetchall()
-            return [self.parse_record(r) for r in res]
+            files = [self.parse_record(r) for r in res]
+            return PathContents([], files)
 
         cursor = await self.cur.execute("SELECT * FROM fmeta WHERE url LIKE ? AND url NOT LIKE ?", (url + '%', url + '%/%'))
         res = await cursor.fetchall()
@@ -358,7 +344,7 @@ class FileConn(DBObjectBase):
                 await self._user_size_dec(r[0], size[0])
         
         # if any new records are created here, the size update may be inconsistent
-        # but it's not a big deal...
+        # but it's not a big deal... we should have only one writer
         
         if under_user_id is None:
             res = await self.cur.execute("DELETE FROM fmeta WHERE url LIKE ? RETURNING *", (path + '%', ))
@@ -432,7 +418,7 @@ async def get_user(cur: aiosqlite.Cursor, user: int | str) -> Optional[UserRecor
     else:
         return None
 
-# mostly transactional operations
+# higher level database operations, mostly transactional
 class Database:
     logger = get_logger('database', global_instance=True)
 
@@ -615,11 +601,13 @@ class Database:
             else:
                 internal_ids.append(r.file_id)
         
-        for i in range(0, len(internal_ids), batch_size):
-            await fconn.delete_file_blobs([r for r in internal_ids[i:i+batch_size]])
-        for i in range(0, len(external_ids)):
-            await fconn.delete_file_blob_external(external_ids[i])
-            
+        async def del_internal():
+            for i in range(0, len(internal_ids), batch_size):
+                await fconn.delete_file_blobs([r for r in internal_ids[i:i+batch_size]])
+        async def del_external():
+            for i in range(0, len(external_ids)):
+                await fconn.delete_file_blob_external(external_ids[i])
+        await asyncio.gather(del_internal(), del_external())
 
     async def delete_path(self, url: str, under_user: Optional[UserRecord] = None) -> Optional[list[FileRecord]]:
         validate_url(url, is_file=False)
@@ -655,7 +643,7 @@ class Database:
         async with unique_cursor() as cur:
             fconn = FileConn(cur)
             if urls is None:
-                urls = [r.url for r in await fconn.list_path(top_url, flat=True)]
+                urls = [r.url for r in (await fconn.list_path(top_url, flat=True)).files]
 
             for url in urls:
                 if not url.startswith(top_url):

lfss/src/log.py

@@ -151,9 +151,6 @@ def log_access(
             return wrapper          # type: ignore
     return _log_access
 
-def get_dummy_logger() -> BaseLogger:
-    return BaseLogger('dummy')
-
 __ALL__ = [
-    'get_logger', 'clear_handlers', 'log_access', 'get_dummy_logger'
+    'get_logger', 'log_access'
 ]

lfss/src/server.py

@@ -18,7 +18,7 @@ from .stat import RequestDB
 from .config import MAX_BUNDLE_BYTES, MAX_FILE_BYTES, LARGE_FILE_BYTES
 from .utils import ensure_uri_compnents, format_last_modified, now_stamp
 from .connection_pool import global_connection_init, global_connection_close, unique_cursor
-from .database import Database, UserRecord, DECOY_USER, FileRecord, check_user_permission, FileReadPermission, UserConn, FileConn
+from .database import Database, UserRecord, DECOY_USER, FileRecord, check_user_permission, FileReadPermission, UserConn, FileConn, PathContents
 
 logger = get_logger("server", term_level="DEBUG")
 logger_failed_request = get_logger("failed_requests", term_level="INFO")
@@ -120,7 +120,7 @@ router_fs = APIRouter(prefix="")
 
 @router_fs.get("/{path:path}")
 @handle_exception
-async def get_file(path: str, download = False, user: UserRecord = Depends(get_current_user)):
+async def get_file(path: str, download: bool = False, flat: bool = False, user: UserRecord = Depends(get_current_user)):
     path = ensure_uri_compnents(path)
 
     # handle directory query
@@ -130,18 +130,20 @@ async def get_file(path: str, download = False, user: UserRecord = Depends(get_c
         async with unique_cursor() as conn:
             fconn = FileConn(conn)
             if user.id == 0:
-                raise HTTPException(status_code=403, detail="Permission denied, credential required")
+                raise HTTPException(status_code=401, detail="Permission denied, credential required")
             if path == "/":
-                return {
-                    "dirs": await fconn.list_root(user.username) \
-                        if not user.is_admin else await fconn.list_root(),
-                    "files": []
-                }
+                if flat:
+                    raise HTTPException(status_code=400, detail="Flat query not supported for root path")
+                return PathContents(
+                    dirs = await fconn.list_root_dirs(user.username) \
+                        if not user.is_admin else await fconn.list_root_dirs(),
+                    files = []
+                )
 
             if not path.startswith(f"{user.username}/") and not user.is_admin:
                 raise HTTPException(status_code=403, detail="Permission denied, path must start with username")
 
-            return await fconn.list_path(path, flat = False)
+            return await fconn.list_path(path, flat = flat)
     
     async with unique_cursor() as conn:
         fconn = FileConn(conn)
@@ -217,7 +219,6 @@ async def put_file(
             return Response(status_code=200, headers={
                 "Content-Type": "application/json",
             }, content=json.dumps({"url": path}))
-        # remove the old file
         exists_flag = True
         if not user.is_admin and not file_record.owner_id == user.id:
             raise HTTPException(status_code=403, detail="Permission denied, cannot overwrite other's file")
@@ -317,7 +318,7 @@ async def bundle_files(path: str, user: UserRecord = Depends(registered_user)):
     
     async with unique_cursor() as conn:
         fconn = FileConn(conn)
-        files = await fconn.list_path(path, flat = True)
+        files = (await fconn.list_path(path, flat = True)).files
     files = [f for f in files if await is_access_granted(f)]
     if len(files) == 0:
         raise HTTPException(status_code=404, detail="No files found")
@@ -339,14 +340,6 @@ async def bundle_files(path: str, user: UserRecord = Depends(registered_user)):
 @router_api.get("/meta")
 @handle_exception
 async def get_file_meta(path: str, user: UserRecord = Depends(registered_user)):
-    """
-    Permission:
-        for file:
-            if file is under user's path, return the meta, 
-            else, determine by the permission same as get_file
-        for path:
-            if path is under user's path, return the meta, else return 403
-    """
     logger.info(f"GET meta({path}), user: {user.username}")
     path = ensure_uri_compnents(path)
     is_file = not path.endswith("/")

lfss/src/utils.py

@@ -8,33 +8,21 @@ def hash_credential(username: str, password: str):
     return hashlib.sha256((username + password).encode()).hexdigest()
 
 def encode_uri_compnents(path: str):
-    """
-    Encode the path components to encode the special characters, 
-    also to avoid path traversal attack
-    """
     path_sp = path.split("/")
     mapped = map(lambda x: urllib.parse.quote(x), path_sp)
     return "/".join(mapped)
 
 def decode_uri_compnents(path: str):
-    """
-    Decode the path components to decode the special characters
-    """
     path_sp = path.split("/")
     mapped = map(lambda x: urllib.parse.unquote(x), path_sp)
     return "/".join(mapped)
 
 def ensure_uri_compnents(path: str):
-    """
-    Ensure the path components are safe to use
-    """
+    """ Ensure the path components are safe to use """
     return encode_uri_compnents(decode_uri_compnents(path))
 
 def debounce_async(delay: float = 0):
-    """
-    Decorator to debounce the async function (procedure)
-    The function must return None
-    """
+    """ Debounce the async procedure """
     def debounce_wrap(func):
         # https://docs.python.org/3/library/asyncio-task.html#asyncio.Task.cancel
         async def delayed_func(*args, **kwargs):
@@ -55,10 +43,9 @@ def debounce_async(delay: float = 0):
         return wrapper
     return debounce_wrap
 
-# https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Last-Modified
 def format_last_modified(last_modified_gmt: str):
     """
-    Format the last modified time to the HTTP standard format
+    Format the last modified time to the [HTTP standard format](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Last-Modified)
     - last_modified_gmt: The last modified time in SQLite ISO 8601 GMT format: e.g. '2021-09-01 12:00:00'
     """
     assert len(last_modified_gmt) == 19

{lfss-0.7.4.dist-info → lfss-0.7.5.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: lfss
-Version: 0.7.4
+Version: 0.7.5
 Summary: Lightweight file storage service
 Home-page: https://github.com/MenxLi/lfss
 Author: li, mengxun

{lfss-0.7.4.dist-info → lfss-0.7.5.dist-info}/RECORD

@@ -1,7 +1,7 @@
 Readme.md,sha256=vsPotlwPAaHI5plh4aaszpi3rr7ZGDn7-wLdEYTWQ0k,1275
 docs/Known_issues.md,sha256=rfdG3j1OJF-59S9E06VPyn0nZKbW-ybPxkoZ7MEZWp8,81
 docs/Permission.md,sha256=X0VNfBKU52f93QYqcVyiBFJ3yURiSkhIo9S_5fdSgzM,2265
-frontend/api.js,sha256=-ouhsmucEunAK3m1H__MqffQkXAjoeVEfM15BvqfIZs,7677
+frontend/api.js,sha256=lHqT7zGmsUZItE-FRR0LfTl_WYCcqlNssfa00XYo-EY,7865
 frontend/index.html,sha256=VPJDs2LG8ep9kjlsKzjWzpN9vc1VGgdvOUlNTZWyQoQ,2088
 frontend/popup.css,sha256=VzkjG1ZTLxhHMtTyobnlvqYmVsTmdbJJed2Pu1cc06c,1007
 frontend/popup.js,sha256=3PgaGZmxSdV1E-D_MWgcR7aHWkcsHA1BNKSOkmP66tA,5191
@@ -13,21 +13,21 @@ lfss/cli/cli.py,sha256=Yup3xIVEQPu10uM8dq1bvre1fK5ngweQHxXZsgQq4Hc,4187
 lfss/cli/panel.py,sha256=iGdVmdWYjA_7a78ZzWEB_3ggIOBeUKTzg6F5zLaB25c,1401
 lfss/cli/serve.py,sha256=bO3GT0kuylMGN-7bZWP4e71MlugGZ_lEMkYaYld_Ntg,985
 lfss/cli/user.py,sha256=h-USWF6lB0Ztm9vwQznqsghKJ5INq5mBmaQeX2D5F-w,3490
-lfss/client/__init__.py,sha256=R9erioMInKIPZ0rXu1J-4mezbwyGMjig18532ATFP5s,4545
-lfss/client/api.py,sha256=aun5HWVNPBeJK6x0_iaM8gVcE3wX6yaqX0TsfsfifSw,5728
+lfss/client/__init__.py,sha256=8uvcKs3PYQamDd_cjfN-fX9QUohEzJqeJlOYkBlzC3M,4556
+lfss/client/api.py,sha256=kSkB4wADTu012-1wl6v90OiZrw6aTQ42GU4jtV4KO0k,5764
 lfss/sql/init.sql,sha256=C-JtQAlaOjESI8uoF1Y_9dKukEVSw5Ll-7yA3gG-XHU,1210
 lfss/sql/pragma.sql,sha256=uENx7xXjARmro-A3XAK8OM8v5AxDMdCCRj47f86UuXg,206
 lfss/src/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-lfss/src/config.py,sha256=z0aVOW8yGgKSryhQCWTf2RY3iHmKMcKIZ-HiosTnPRs,539
-lfss/src/connection_pool.py,sha256=69QMJ4gRQ62qi39t0JKdvIaWRBrbU9S7slutIpCc30A,4959
-lfss/src/database.py,sha256=VKTLJSUFImF3pzMUaqGWEh1H06Yk3VgJa78bw5rAvG8,32147
+lfss/src/config.py,sha256=aTfjWORE9Mx7LSEjbfmHnULlrmIWEvEBSZ4fJKWZNjM,530
+lfss/src/connection_pool.py,sha256=teW_4DMiwlCN_bS7AhjkbY9cHZqUFlmHE_J2yPjHVsA,5125
+lfss/src/database.py,sha256=G9U_Iijp7euuGj3fcWdSGJPetMhn56X0vI8iWr6ZUr8,31904
 lfss/src/datatype.py,sha256=BLS7vuuKnFZQg0nrKeP9SymqUhcN6HwPgejU0yBd_Ak,1622
 lfss/src/error.py,sha256=imbhwnbhnI3HLhkbfICROe3F0gleKrOk4XnqHJDOtuI,285
-lfss/src/log.py,sha256=qNE04sHoZ2rMbQ5dR2zT7Xaz1KVAfYp5hKpWJX42S9g,5244
-lfss/src/server.py,sha256=4GYDAa9Fx4H4fYh_He1u1zlIro3zaQmaEBNKpPO2Q2E,16374
+lfss/src/log.py,sha256=xOnkuH-gB_jSVGqNnDVEW05iki6SCJ2xdEhjz5eEsMo,5136
+lfss/src/server.py,sha256=EA5fK4qc98tF8qoS9F6VaxIE65D5X8Ztkjqy8EUYIv8,16276
 lfss/src/stat.py,sha256=hTMtQyM_Ukmhc33Bb9FGCfBMIX02KrGHQg8nL7sC8sU,2082
-lfss/src/utils.py,sha256=ZE3isOS3gafEYw1z8s2ucY08eWQHIN4YUbKYH8F1hEQ,2409
-lfss-0.7.4.dist-info/METADATA,sha256=XeJ4ft1B9KDdfcxyFrFdwUMTYc1QsUAkgqI8v5rwbq4,1967
-lfss-0.7.4.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-lfss-0.7.4.dist-info/entry_points.txt,sha256=d_Ri3GXxUW-S0E6q953A8od0YMmUAnZGlJSKS46OiW8,172
-lfss-0.7.4.dist-info/RECORD,,
+lfss/src/utils.py,sha256=miGsv7udupDtSpVSd66IvUt0_QMi3JXYCp_BjdPJY-M,2134
+lfss-0.7.5.dist-info/METADATA,sha256=cGOrCSM3pjCqYJdrFNrLfO2P08zx1cmwgfmAmTT4MJg,1967
+lfss-0.7.5.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+lfss-0.7.5.dist-info/entry_points.txt,sha256=d_Ri3GXxUW-S0E6q953A8od0YMmUAnZGlJSKS46OiW8,172
+lfss-0.7.5.dist-info/RECORD,,