lfss 0.7.0__py3-none-any.whl → 0.7.2__py3-none-any.whl

Readme.md

@@ -1,7 +1,9 @@
 # Lightweight File Storage Service (LFSS)
 [![PyPI](https://img.shields.io/pypi/v/lfss)](https://pypi.org/project/lfss/)
 
-A lightweight file/object storage service!
+My experiment on a lightweight file/object storage service.  
+It stores small files and metadata in sqlite, large files in the filesystem, and serves them through a simple REST API.  
+Tested on 2 million files, and it works fine... thanks to the sqlite database!
 
 Usage: 
 ```sh
@@ -21,7 +23,7 @@ Or, you can start a web server at `/frontend` and open `index.html` in your brow
 
 The API usage is simple, just `GET`, `PUT`, `DELETE` to the `/<username>/file/url` path.  
 Authentication is done via `Authorization` header with the value `Bearer <token>`, or through the `token` query parameter.  
-You can refer to `frontend` as an application example, and `frontend/api.js` or `lfss.client.api.py` for the API usage.
+You can refer to `frontend` as an application example, and `frontend/api.js` or `lfss/client/api.py` for the API usage.
 
 By default, the service exposes all files to the public for `GET` requests, 
 but file-listing is restricted to the user's own files.  

frontend/scripts.js

@@ -132,7 +132,7 @@ uploadButton.addEventListener('click', () => {
     }
     path = path + fileName;
     showPopup('Uploading...', {level: 'info', timeout: 3000});
-    conn.put(path, file)
+    conn.put(path, file, {'conflict': 'overwrite'})
         .then(() => {
             refreshFileList();
             uploadFileNameInput.value = '';

lfss/cli/balance.py

@@ -2,14 +2,15 @@
 Balance the storage by ensuring that large file thresholds are met.
 """
 
-from lfss.src.config import DATA_HOME, LARGE_BLOB_DIR, LARGE_FILE_BYTES
+from lfss.src.config import LARGE_BLOB_DIR, LARGE_FILE_BYTES
 import argparse, time
 from functools import wraps
 from asyncio import Semaphore
-import aiosqlite, aiofiles, asyncio
+import aiofiles, asyncio
+from lfss.src.database import transaction, unique_cursor
+from lfss.src.connection_pool import global_entrance
 
 sem = Semaphore(1)
-db_file = DATA_HOME / 'lfss.db'
 
 def _get_sem():
     return sem
@@ -23,51 +24,35 @@ def barriered(func):
 
 @barriered
 async def move_to_external(f_id: str, flag: str = ''):
-    async with aiosqlite.connect(db_file, timeout = 60) as c:
-        async with c.execute( "SELECT data FROM blobs.fdata WHERE file_id = ?", (f_id,)) as cursor:
-            blob_row = await cursor.fetchone()
-            if blob_row is None:
-                print(f"{flag}File {f_id} not found in blobs.fdata")
-                return
+    async with transaction() as c:
+        cursor = await c.execute( "SELECT data FROM blobs.fdata WHERE file_id = ?", (f_id,))
+        blob_row = await cursor.fetchone()
+        if blob_row is None:
+            print(f"{flag}File {f_id} not found in blobs.fdata")
+            return
         await c.execute("BEGIN")
         blob: bytes = blob_row[0]
-        try:
-            async with aiofiles.open(LARGE_BLOB_DIR / f_id, 'wb') as f:
-                await f.write(blob)
-            await c.execute( "UPDATE fmeta SET external = 1 WHERE file_id = ?", (f_id,))
-            await c.execute( "DELETE FROM blobs.fdata WHERE file_id = ?", (f_id,))
-            await c.commit()
-            print(f"{flag}Moved {f_id} to external storage")
-        except Exception as e:
-            await c.rollback()
-            print(f"{flag}Error moving {f_id}: {e}")
-
-            if isinstance(e, KeyboardInterrupt):
-                raise e
+        async with aiofiles.open(LARGE_BLOB_DIR / f_id, 'wb') as f:
+            await f.write(blob)
+        await c.execute( "UPDATE fmeta SET external = 1 WHERE file_id = ?", (f_id,))
+        await c.execute( "DELETE FROM blobs.fdata WHERE file_id = ?", (f_id,))
+        print(f"{flag}Moved {f_id} to external storage")
 
 @barriered
 async def move_to_internal(f_id: str, flag: str = ''):
-    async with aiosqlite.connect(db_file, timeout = 60) as c:
+    async with transaction() as c:
         if not (LARGE_BLOB_DIR / f_id).exists():
             print(f"{flag}File {f_id} not found in external storage")
             return
         async with aiofiles.open(LARGE_BLOB_DIR / f_id, 'rb') as f:
             blob = await f.read()
 
-        await c.execute("BEGIN")
-        try:
-            await c.execute("INSERT INTO blobs.fdata (file_id, data) VALUES (?, ?)", (f_id, blob))
-            await c.execute("UPDATE fmeta SET external = 0 WHERE file_id = ?", (f_id,))
-            await c.commit()
-            (LARGE_BLOB_DIR / f_id).unlink(missing_ok=True)
-            print(f"{flag}Moved {f_id} to internal storage")
-        except Exception as e:
-            await c.rollback()
-            print(f"{flag}Error moving {f_id}: {e}")
-            if isinstance(e, KeyboardInterrupt):
-                raise e
-
+        await c.execute("INSERT INTO blobs.fdata (file_id, data) VALUES (?, ?)", (f_id, blob))
+        await c.execute("UPDATE fmeta SET external = 0 WHERE file_id = ?", (f_id,))
+        (LARGE_BLOB_DIR / f_id).unlink(missing_ok=True)
+        print(f"{flag}Moved {f_id} to internal storage")
 
+@global_entrance()
 async def _main(batch_size: int = 10000):
 
     tasks = []
@@ -76,7 +61,7 @@ async def _main(batch_size: int = 10000):
     e_cout = 0
     batch_count = 0
     while True:
-        async with aiosqlite.connect(db_file) as conn:
+        async with unique_cursor() as conn:
             exceeded_rows = list(await (await conn.execute( 
                 "SELECT file_id FROM fmeta WHERE file_size > ? AND external = 0 LIMIT ? OFFSET ?",
                 (LARGE_FILE_BYTES, batch_size, batch_size * batch_count)
@@ -93,7 +78,7 @@ async def _main(batch_size: int = 10000):
     i_count = 0
     batch_count = 0
     while True:
-        async with aiosqlite.connect(db_file) as conn:
+        async with unique_cursor() as conn:
             under_rows = list(await (await conn.execute(
                 "SELECT file_id, file_size, external FROM fmeta WHERE file_size <= ? AND external = 1 LIMIT ? OFFSET ?",
                 (LARGE_FILE_BYTES, batch_size, batch_size * batch_count)

lfss/cli/user.py

@@ -1,5 +1,7 @@
 import argparse, asyncio
-from ..src.database import Database, FileReadPermission
+from contextlib import asynccontextmanager
+from ..src.database import Database, FileReadPermission, transaction, UserConn
+from ..src.connection_pool import global_entrance
 
 def parse_storage_size(s: str) -> int:
     if s[-1] in 'Kk':
@@ -12,6 +14,7 @@ def parse_storage_size(s: str) -> int:
         return int(s[:-1]) * 1024 * 1024 * 1024 * 1024
     return int(s)
 
+@global_entrance(1)
 async def _main():
     parser = argparse.ArgumentParser()
     sp = parser.add_subparsers(dest='subparser_name', required=True)
@@ -42,49 +45,48 @@ async def _main():
     sp_list.add_argument("-l", "--long", action="store_true")
     
     args = parser.parse_args()
-    conn = await Database().init()
+
+    @asynccontextmanager
+    async def get_uconn():
+        async with transaction() as conn:
+            yield UserConn(conn)
     
-    try:
-        if args.subparser_name == 'add':
-            await conn.user.create_user(args.username, args.password, args.admin, max_storage=args.max_storage, permission=args.permission)
-            user = await conn.user.get_user(args.username)
+    if args.subparser_name == 'add':
+        async with get_uconn() as uconn:
+            await uconn.create_user(args.username, args.password, args.admin, max_storage=args.max_storage, permission=args.permission)
+            user = await uconn.get_user(args.username)
             assert user is not None
             print('User created, credential:', user.credential)
-        
-        if args.subparser_name == 'delete':
-            user = await conn.user.get_user(args.username)
-            if user is None:
-                print('User not found')
-                exit(1)
-            else:
-                await conn.delete_user(user.id)
-            print('User deleted')
-        
-        if args.subparser_name == 'set':
-            user = await conn.user.get_user(args.username)
+    
+    if args.subparser_name == 'delete':
+        async with get_uconn() as uconn:
+            user = await uconn.get_user(args.username)
+        if user is None:
+            print('User not found')
+            exit(1)
+        else:
+            db = await Database().init()
+            await db.delete_user(user.id)
+        print('User deleted')
+    
+    if args.subparser_name == 'set':
+        async with get_uconn() as uconn:
+            user = await uconn.get_user(args.username)
             if user is None:
                 print('User not found')
                 exit(1)
-            await conn.user.update_user(user.username, args.password, args.admin, max_storage=args.max_storage, permission=args.permission)
-            user = await conn.user.get_user(args.username)
+            await uconn.update_user(user.username, args.password, args.admin, max_storage=args.max_storage, permission=args.permission)
+            user = await uconn.get_user(args.username)
             assert user is not None
             print('User updated, credential:', user.credential)
-        
-        if args.subparser_name == 'list':
-            async for user in conn.user.all():
+    
+    if args.subparser_name == 'list':
+        async with get_uconn() as uconn:
+            async for user in uconn.all():
                 print(user)
                 if args.long:
                     print('  ', user.credential)
         
-        await conn.commit()
-    
-    except Exception as e:
-        conn.logger.error(f'Error: {e}')
-        await conn.rollback()
-
-    finally:
-        await conn.close()
-
 def main():
     asyncio.run(_main())
 

lfss/sql/init.sql

@@ -1,9 +1,3 @@
-PRAGMA journal_mode=MEMROY;
-PRAGMA temp_store=MEMORY;
-PRAGMA page_size=4096;
-PRAGMA synchronous=NORMAL;
-PRAGMA case_sensitive_like=ON;
-
 CREATE TABLE IF NOT EXISTS user (
     id INTEGER PRIMARY KEY AUTOINCREMENT,
     username VARCHAR(256) UNIQUE NOT NULL,

lfss/src/config.py

@@ -13,5 +13,5 @@ LARGE_BLOB_DIR.mkdir(exist_ok=True)
 
 # https://sqlite.org/fasterthanfs.html
 LARGE_FILE_BYTES = 8 * 1024 * 1024   # 8MB
-MAX_FILE_BYTES = 1024 * 1024 * 1024   # 1GB
-MAX_BUNDLE_BYTES = 1024 * 1024 * 1024   # 1GB
+MAX_FILE_BYTES = 512 * 1024 * 1024   # 512MB
+MAX_BUNDLE_BYTES = 512 * 1024 * 1024   # 512MB

lfss/src/connection_pool.py

@@ -0,0 +1,152 @@
+import os
+from pathlib import Path
+import aiosqlite, aiofiles
+from contextlib import asynccontextmanager
+from dataclasses import dataclass
+from asyncio import Semaphore, Lock
+from functools import wraps
+
+from .log import get_logger
+from .config import DATA_HOME
+
+async def execute_sql(conn: aiosqlite.Connection | aiosqlite.Cursor, name: str):
+    this_dir = Path(__file__).parent
+    sql_dir = this_dir.parent / 'sql'
+    async with aiofiles.open(sql_dir / name, 'r') as f:
+        sql = await f.read()
+    sql = sql.split(';')
+    for s in sql:
+        await conn.execute(s)
+
+async def get_connection() -> aiosqlite.Connection:
+    if not os.environ.get('SQLITE_TEMPDIR'):
+        os.environ['SQLITE_TEMPDIR'] = str(DATA_HOME)
+    # large blobs are stored in a separate database, should be more efficient
+    conn = await aiosqlite.connect(DATA_HOME / 'index.db', timeout = 60)
+    async with conn.cursor() as c:
+        await c.execute(f"ATTACH DATABASE ? AS blobs", (str(DATA_HOME/'blobs.db'), ))
+    await execute_sql(conn, 'pragma.sql')
+    return conn
+
+
+@dataclass
+class SqlConnection:
+    conn: aiosqlite.Connection
+    is_available: bool = True
+
+class SqlConnectionPool:
+    _sem: Semaphore
+    _w_sem: Semaphore
+    def __init__(self):
+        self._connections: list[SqlConnection] = []
+        self._w_connection: None | SqlConnection = None
+        self._lock = Lock()
+    
+    async def init(self, n_read: int):
+        await self.close()
+        self._connections = []
+        for _ in range(n_read):
+            conn = await get_connection()
+            self._connections.append(SqlConnection(conn))
+        self._w_connection = SqlConnection(await get_connection())
+        self._sem = Semaphore(n_read)
+        self._w_sem = Semaphore(1)
+    
+    @property
+    def n_read(self):
+        return len(self._connections)
+    @property
+    def sem(self):
+        return self._sem
+    @property
+    def w_sem(self):
+        return self._w_sem
+    
+    async def get(self, w: bool = False) -> SqlConnection:
+        if len(self._connections) == 0:
+            raise Exception("No available connections, please init the pool first")
+        
+        async with self._lock:
+            if w:
+                assert self._w_connection
+                if self._w_connection.is_available:
+                    self._w_connection.is_available = False
+                    return self._w_connection
+                raise Exception("Write connection is not available")
+
+            else:
+                for c in self._connections:
+                    if c.is_available:
+                        c.is_available = False
+                        return c
+        raise Exception("No available connections, impossible?")
+    
+    async def release(self, conn: SqlConnection):
+        async with self._lock:
+            if conn == self._w_connection:
+                conn.is_available = True
+                return
+
+            if not conn in self._connections:
+                raise Exception("Connection not in pool")
+            conn.is_available = True
+    
+    async def close(self):
+        for c in self._connections:
+            await c.conn.close()
+        if self._w_connection:
+            await self._w_connection.conn.close()
+
+# these two functions shold be called before and after the event loop
+g_pool = SqlConnectionPool()
+async def global_connection_init(n_read: int = 1):
+    await g_pool.init(n_read)
+
+async def global_connection_close():
+    await g_pool.close()
+
+@asynccontextmanager
+async def global_connection(n_read: int = 1):
+    await global_connection_init(n_read)
+    try:
+        yield g_pool
+    finally:
+        await global_connection_close()
+
+def global_entrance(n_read: int = 1):
+    def decorator(func):
+        @wraps(func)
+        async def wrapper(*args, **kwargs):
+            async with global_connection(n_read):
+                return await func(*args, **kwargs)
+        return wrapper
+    return decorator
+
+@asynccontextmanager
+async def unique_cursor(is_write: bool = False):
+    if not is_write:
+        async with g_pool.sem:
+            connection_obj = await g_pool.get()
+            try:
+                yield await connection_obj.conn.cursor()
+            finally:
+                await g_pool.release(connection_obj)
+    else:
+        async with g_pool.w_sem:
+            connection_obj = await g_pool.get(w=True)
+            try:
+                yield await connection_obj.conn.cursor()
+            finally:
+                await g_pool.release(connection_obj)
+
+@asynccontextmanager
+async def transaction():
+    async with unique_cursor(is_write=True) as cur:
+        try:
+            await cur.execute('BEGIN')
+            yield cur
+            await cur.execute('COMMIT')
+        except Exception as e:
+            get_logger('database', global_instance=True).error(f"Error in transaction: {e}, rollback.")
+            await cur.execute('ROLLBACK')
+            raise e