lfss 0.7.0__tar.gz → 0.7.1__tar.gz

{lfss-0.7.0 → lfss-0.7.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: lfss
-Version: 0.7.0
+Version: 0.7.1
 Summary: Lightweight file storage service
 Home-page: https://github.com/MenxLi/lfss
 Author: li, mengxun

{lfss-0.7.0 → lfss-0.7.1}/frontend/scripts.js

@@ -132,7 +132,7 @@ uploadButton.addEventListener('click', () => {
     }
     path = path + fileName;
     showPopup('Uploading...', {level: 'info', timeout: 3000});
-    conn.put(path, file)
+    conn.put(path, file, {'conflict': 'overwrite'})
         .then(() => {
             refreshFileList();
             uploadFileNameInput.value = '';

{lfss-0.7.0 → lfss-0.7.1}/lfss/cli/balance.py

@@ -2,14 +2,15 @@
 Balance the storage by ensuring that large file thresholds are met.
 """
 
-from lfss.src.config import DATA_HOME, LARGE_BLOB_DIR, LARGE_FILE_BYTES
+from lfss.src.config import LARGE_BLOB_DIR, LARGE_FILE_BYTES
 import argparse, time
 from functools import wraps
 from asyncio import Semaphore
-import aiosqlite, aiofiles, asyncio
+import aiofiles, asyncio
+from lfss.src.database import transaction, unique_cursor
+from lfss.src.connection_pool import global_entrance
 
 sem = Semaphore(1)
-db_file = DATA_HOME / 'lfss.db'
 
 def _get_sem():
     return sem
@@ -23,7 +24,8 @@ def barriered(func):
 
 @barriered
 async def move_to_external(f_id: str, flag: str = ''):
-    async with aiosqlite.connect(db_file, timeout = 60) as c:
+    # async with aiosqlite.connect(db_file, timeout = 60) as c:
+    async with transaction() as c:
         async with c.execute( "SELECT data FROM blobs.fdata WHERE file_id = ?", (f_id,)) as cursor:
             blob_row = await cursor.fetchone()
             if blob_row is None:
@@ -47,7 +49,7 @@ async def move_to_external(f_id: str, flag: str = ''):
 
 @barriered
 async def move_to_internal(f_id: str, flag: str = ''):
-    async with aiosqlite.connect(db_file, timeout = 60) as c:
+    async with transaction() as c:
         if not (LARGE_BLOB_DIR / f_id).exists():
             print(f"{flag}File {f_id} not found in external storage")
             return
@@ -68,6 +70,7 @@ async def move_to_internal(f_id: str, flag: str = ''):
                 raise e
 
 
+@global_entrance()
 async def _main(batch_size: int = 10000):
 
     tasks = []
@@ -76,7 +79,7 @@ async def _main(batch_size: int = 10000):
     e_cout = 0
     batch_count = 0
     while True:
-        async with aiosqlite.connect(db_file) as conn:
+        async with unique_cursor() as conn:
             exceeded_rows = list(await (await conn.execute( 
                 "SELECT file_id FROM fmeta WHERE file_size > ? AND external = 0 LIMIT ? OFFSET ?",
                 (LARGE_FILE_BYTES, batch_size, batch_size * batch_count)
@@ -93,7 +96,7 @@ async def _main(batch_size: int = 10000):
     i_count = 0
     batch_count = 0
     while True:
-        async with aiosqlite.connect(db_file) as conn:
+        async with unique_cursor() as conn:
             under_rows = list(await (await conn.execute(
                 "SELECT file_id, file_size, external FROM fmeta WHERE file_size <= ? AND external = 1 LIMIT ? OFFSET ?",
                 (LARGE_FILE_BYTES, batch_size, batch_size * batch_count)

{lfss-0.7.0 → lfss-0.7.1}/lfss/cli/user.py

@@ -1,5 +1,7 @@
 import argparse, asyncio
-from ..src.database import Database, FileReadPermission
+from contextlib import asynccontextmanager
+from ..src.database import Database, FileReadPermission, transaction, UserConn
+from ..src.connection_pool import global_entrance
 
 def parse_storage_size(s: str) -> int:
     if s[-1] in 'Kk':
@@ -12,6 +14,7 @@ def parse_storage_size(s: str) -> int:
         return int(s[:-1]) * 1024 * 1024 * 1024 * 1024
     return int(s)
 
+@global_entrance(1)
 async def _main():
     parser = argparse.ArgumentParser()
     sp = parser.add_subparsers(dest='subparser_name', required=True)
@@ -42,49 +45,48 @@ async def _main():
     sp_list.add_argument("-l", "--long", action="store_true")
     
     args = parser.parse_args()
-    conn = await Database().init()
+
+    @asynccontextmanager
+    async def get_uconn():
+        async with transaction() as conn:
+            yield UserConn(conn)
     
-    try:
-        if args.subparser_name == 'add':
-            await conn.user.create_user(args.username, args.password, args.admin, max_storage=args.max_storage, permission=args.permission)
-            user = await conn.user.get_user(args.username)
+    if args.subparser_name == 'add':
+        async with get_uconn() as uconn:
+            await uconn.create_user(args.username, args.password, args.admin, max_storage=args.max_storage, permission=args.permission)
+            user = await uconn.get_user(args.username)
             assert user is not None
             print('User created, credential:', user.credential)
-        
-        if args.subparser_name == 'delete':
-            user = await conn.user.get_user(args.username)
-            if user is None:
-                print('User not found')
-                exit(1)
-            else:
-                await conn.delete_user(user.id)
-            print('User deleted')
-        
-        if args.subparser_name == 'set':
-            user = await conn.user.get_user(args.username)
+    
+    if args.subparser_name == 'delete':
+        async with get_uconn() as uconn:
+            user = await uconn.get_user(args.username)
+        if user is None:
+            print('User not found')
+            exit(1)
+        else:
+            db = await Database().init()
+            await db.delete_user(user.id)
+        print('User deleted')
+    
+    if args.subparser_name == 'set':
+        async with get_uconn() as uconn:
+            user = await uconn.get_user(args.username)
             if user is None:
                 print('User not found')
                 exit(1)
-            await conn.user.update_user(user.username, args.password, args.admin, max_storage=args.max_storage, permission=args.permission)
-            user = await conn.user.get_user(args.username)
+            await uconn.update_user(user.username, args.password, args.admin, max_storage=args.max_storage, permission=args.permission)
+            user = await uconn.get_user(args.username)
             assert user is not None
             print('User updated, credential:', user.credential)
-        
-        if args.subparser_name == 'list':
-            async for user in conn.user.all():
+    
+    if args.subparser_name == 'list':
+        async with get_uconn() as uconn:
+            async for user in uconn.all():
                 print(user)
                 if args.long:
                     print('  ', user.credential)
         
-        await conn.commit()
-    
-    except Exception as e:
-        conn.logger.error(f'Error: {e}')
-        await conn.rollback()
-
-    finally:
-        await conn.close()
-
 def main():
     asyncio.run(_main())
 

{lfss-0.7.0 → lfss-0.7.1}/lfss/sql/init.sql

@@ -1,9 +1,3 @@
-PRAGMA journal_mode=MEMROY;
-PRAGMA temp_store=MEMORY;
-PRAGMA page_size=4096;
-PRAGMA synchronous=NORMAL;
-PRAGMA case_sensitive_like=ON;
-
 CREATE TABLE IF NOT EXISTS user (
     id INTEGER PRIMARY KEY AUTOINCREMENT,
     username VARCHAR(256) UNIQUE NOT NULL,

lfss-0.7.1/lfss/src/connection_pool.py

@@ -0,0 +1,151 @@
+import os
+from pathlib import Path
+import aiosqlite, aiofiles
+from contextlib import asynccontextmanager
+from dataclasses import dataclass
+from asyncio import Semaphore, Lock
+from functools import wraps
+
+from .log import get_logger
+from .config import DATA_HOME
+
+async def execute_sql(conn: aiosqlite.Connection | aiosqlite.Cursor, name: str):
+    this_dir = Path(__file__).parent
+    sql_dir = this_dir.parent / 'sql'
+    async with aiofiles.open(sql_dir / name, 'r') as f:
+        sql = await f.read()
+    sql = sql.split(';')
+    for s in sql:
+        await conn.execute(s)
+
+async def get_connection() -> aiosqlite.Connection:
+    if not os.environ.get('SQLITE_TEMPDIR'):
+        os.environ['SQLITE_TEMPDIR'] = str(DATA_HOME)
+    # large blobs are stored in a separate database, should be more efficient
+    conn = await aiosqlite.connect(DATA_HOME / 'index.db', timeout = 60)
+    async with conn.cursor() as c:
+        await c.execute(f"ATTACH DATABASE ? AS blobs", (str(DATA_HOME/'blobs.db'), ))
+    await execute_sql(conn, 'pragma.sql')
+    return conn
+
+
+@dataclass
+class SqlConnection:
+    conn: aiosqlite.Connection
+    is_available: bool = True
+
+class SqlConnectionPool:
+    _sem: Semaphore
+    _w_sem: Semaphore
+    def __init__(self):
+        self._connections: list[SqlConnection] = []
+        self._w_connection: None | SqlConnection = None
+        self._lock = Lock()
+    
+    async def init(self, n_read: int):
+        await self.close()
+        self._connections = []
+        for _ in range(n_read):
+            conn = await get_connection()
+            self._connections.append(SqlConnection(conn))
+        self._w_connection = SqlConnection(await get_connection())
+        self._sem = Semaphore(n_read)
+        self._w_sem = Semaphore(1)
+    
+    @property
+    def n_read(self):
+        return len(self._connections)
+    @property
+    def sem(self):
+        return self._sem
+    @property
+    def w_sem(self):
+        return self._w_sem
+    
+    async def get(self, w: bool = False) -> SqlConnection:
+        if len(self._connections) == 0:
+            raise Exception("No available connections, please init the pool first")
+        
+        if w:
+            assert self._w_connection
+            if self._w_connection.is_available:
+                self._w_connection.is_available = False
+                return self._w_connection
+            raise Exception("Write connection is not available")
+
+        async with self._lock:
+            for c in self._connections:
+                if c.is_available:
+                    c.is_available = False
+                    return c
+        raise Exception("No available connections, impossible?")
+    
+    async def release(self, conn: SqlConnection):
+        if conn == self._w_connection:
+            conn.is_available = True
+            return
+
+        async with self._lock:
+            if not conn in self._connections:
+                raise Exception("Connection not in pool")
+            conn.is_available = True
+    
+    async def close(self):
+        for c in self._connections:
+            await c.conn.close()
+        if self._w_connection:
+            await self._w_connection.conn.close()
+
+# these two functions shold be called before and after the event loop
+g_pool = SqlConnectionPool()
+async def global_connection_init(n_read: int = 1):
+    await g_pool.init(n_read)
+
+async def global_connection_close():
+    await g_pool.close()
+
+@asynccontextmanager
+async def global_connection(n_read: int = 1):
+    await global_connection_init(n_read)
+    try:
+        yield g_pool
+    finally:
+        await global_connection_close()
+
+def global_entrance(n_read: int = 1):
+    def decorator(func):
+        @wraps(func)
+        async def wrapper(*args, **kwargs):
+            async with global_connection(n_read):
+                return await func(*args, **kwargs)
+        return wrapper
+    return decorator
+
+@asynccontextmanager
+async def unique_cursor(is_write: bool = False):
+    if not is_write:
+        async with g_pool.sem:
+            connection_obj = await g_pool.get()
+            try:
+                yield await connection_obj.conn.cursor()
+            finally:
+                await g_pool.release(connection_obj)
+    else:
+        async with g_pool.w_sem:
+            connection_obj = await g_pool.get(w=True)
+            try:
+                yield await connection_obj.conn.cursor()
+            finally:
+                await g_pool.release(connection_obj)
+
+@asynccontextmanager
+async def transaction():
+    async with unique_cursor(is_write=True) as cur:
+        try:
+            await cur.execute('BEGIN')
+            yield cur
+            await cur.execute('COMMIT')
+        except Exception as e:
+            get_logger('database', global_instance=True).error(f"Error in transaction: {e}, rollback.")
+            await cur.execute('ROLLBACK')
+            raise e