lfss 0.5.0__tar.gz → 0.5.1__tar.gz

{lfss-0.5.0 → lfss-0.5.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: lfss
-Version: 0.5.0
+Version: 0.5.1
 Summary: Lightweight file storage service
 Home-page: https://github.com/MenxLi/lfss
 Author: li, mengxun

{lfss-0.5.0 → lfss-0.5.1}/frontend/api.js

@@ -57,13 +57,13 @@ export default class Connector {
     * @returns {Promise<string>} - the promise of the request, the url of the file
     */
     async put(path, file, {
-        overwrite = false,
+        conflict = 'abort',
         permission = 0
     } = {}){
         if (path.startsWith('/')){ path = path.slice(1); }
         const fileBytes = await file.arrayBuffer();
         const dst = new URL(this.config.endpoint + '/' + path);
-        dst.searchParams.append('overwrite', overwrite);
+        dst.searchParams.append('conflict', conflict);
         dst.searchParams.append('permission', permission);
         const res = await fetch(dst.toString(), {
             method: 'PUT',

{lfss-0.5.0 → lfss-0.5.1}/frontend/scripts.js

@@ -176,7 +176,7 @@ Are you sure you want to proceed?
             async function uploadFile(...args){
                 const [file, path] = args;
                 try{
-                    await conn.put(path, file, {overwrite: true});
+                    await conn.put(path, file, {conflict: 'overwrite'});
                 }
                 catch (err){
                     showPopup('Failed to upload file [' + file.name + ']: ' + err, {level: 'error', timeout: 5000});

lfss-0.5.1/lfss/cli/balance.py

@@ -0,0 +1,111 @@
+"""
+Balance the storage by ensuring that large file thresholds are met.
+"""
+
+from lfss.src.config import DATA_HOME, LARGE_BLOB_DIR, LARGE_FILE_BYTES
+import argparse, time
+from functools import wraps
+from asyncio import Semaphore
+import aiosqlite, aiofiles, asyncio
+
+sem = Semaphore(1)
+db_file = DATA_HOME / 'lfss.db'
+
+def _get_sem():
+    return sem
+
+def barriered(func):
+    @wraps(func)
+    async def wrapper(*args, **kwargs):
+        async with _get_sem():
+            return await func(*args, **kwargs)
+    return wrapper
+
+@barriered
+async def move_to_external(f_id: str, flag: str = ''):
+    async with aiosqlite.connect(db_file, timeout = 60) as c:
+        async with c.execute( "SELECT data FROM fdata WHERE file_id = ?", (f_id,)) as cursor:
+            blob_row = await cursor.fetchone()
+            if blob_row is None:
+                print(f"{flag}File {f_id} not found in fdata")
+                return
+        await c.execute("BEGIN")
+        blob: bytes = blob_row[0]
+        try:
+            async with aiofiles.open(LARGE_BLOB_DIR / f_id, 'wb') as f:
+                await f.write(blob)
+            await c.execute( "UPDATE fmeta SET external = 1 WHERE file_id = ?", (f_id,))
+            await c.execute( "DELETE FROM fdata WHERE file_id = ?", (f_id,))
+            await c.commit()
+            print(f"{flag}Moved {f_id} to external storage")
+        except Exception as e:
+            await c.rollback()
+            print(f"{flag}Error moving {f_id}: {e}")
+
+            if isinstance(e, KeyboardInterrupt):
+                raise e
+
+@barriered
+async def move_to_internal(f_id: str, flag: str = ''):
+    async with aiosqlite.connect(db_file, timeout = 60) as c:
+        if not (LARGE_BLOB_DIR / f_id).exists():
+            print(f"{flag}File {f_id} not found in external storage")
+            return
+        async with aiofiles.open(LARGE_BLOB_DIR / f_id, 'rb') as f:
+            blob = await f.read()
+
+        await c.execute("BEGIN")
+        try:
+            await c.execute("INSERT INTO fdata (file_id, data) VALUES (?, ?)", (f_id, blob))
+            await c.execute("UPDATE fmeta SET external = 0 WHERE file_id = ?", (f_id,))
+            await c.commit()
+            (LARGE_BLOB_DIR / f_id).unlink(missing_ok=True)
+            print(f"{flag}Moved {f_id} to internal storage")
+        except Exception as e:
+            await c.rollback()
+            print(f"{flag}Error moving {f_id}: {e}")
+            if isinstance(e, KeyboardInterrupt):
+                raise e
+
+
+async def _main():
+
+    tasks = []
+    start_time = time.time()
+    async with aiosqlite.connect(db_file) as conn:
+        exceeded_rows = await (await conn.execute( 
+            "SELECT file_id FROM fmeta WHERE file_size > ? AND external = 0",
+            (LARGE_FILE_BYTES,)
+        )).fetchall()
+        
+    for i in range(0, len(exceeded_rows)):
+        row = exceeded_rows[i]
+        f_id = row[0]
+        tasks.append(move_to_external(f_id, flag=f"[e-{i+1}/{len(exceeded_rows)}] "))
+
+    async with aiosqlite.connect(db_file) as conn:
+        under_rows = await (await conn.execute(
+            "SELECT file_id, file_size, external FROM fmeta WHERE file_size <= ? AND external = 1",
+            (LARGE_FILE_BYTES,)
+        )).fetchall()
+
+    for i in range(0, len(under_rows)):
+        row = under_rows[i]
+        f_id = row[0]
+        tasks.append(move_to_internal(f_id, flag=f"[i-{i+1}/{len(under_rows)}] "))
+        
+    await asyncio.gather(*tasks)
+    end_time = time.time()
+    print(f"Balancing complete, took {end_time - start_time:.2f} seconds. "
+          f"{len(exceeded_rows)} files moved to external storage, {len(under_rows)} files moved to internal storage.")
+            
+def main():
+    global sem
+    parser = argparse.ArgumentParser(description="Balance the storage by ensuring that large file thresholds are met.")
+    parser.add_argument("-j", "--jobs", type=int, default=2, help="Number of concurrent jobs")
+    args = parser.parse_args()
+    sem = Semaphore(args.jobs)
+    asyncio.run(_main())
+
+if __name__ == '__main__':
+    main()

{lfss-0.5.0 → lfss-0.5.1}/lfss/cli/cli.py

@@ -13,8 +13,8 @@ def parse_arguments():
     sp_upload.add_argument("src", help="Source file or directory", type=str)
     sp_upload.add_argument("dst", help="Destination path", type=str)
     sp_upload.add_argument("-j", "--jobs", type=int, default=1, help="Number of concurrent uploads")
-    sp_upload.add_argument("--interval", type=float, default=0, help="Interval between retries, only works with directory upload")
-    sp_upload.add_argument("--overwrite", action="store_true", help="Overwrite existing files")
+    sp_upload.add_argument("--interval", type=float, default=0, help="Interval between files, only works with directory upload")
+    sp_upload.add_argument("--conflict", choices=["overwrite", "abort", "skip"], default="abort", help="Conflict resolution")
     sp_upload.add_argument("--permission", type=FileReadPermission, default=FileReadPermission.UNSET, help="File permission")
     sp_upload.add_argument("--retries", type=int, default=0, help="Number of retries, only works with directory upload")
 
@@ -32,7 +32,7 @@ def main():
                 n_concurrent=args.jobs, 
                 n_reties=args.retries, 
                 interval=args.interval,
-                overwrite=args.overwrite, 
+                conflict=args.conflict,
                 permission=args.permission
             )
             if failed_upload:
@@ -44,7 +44,7 @@ def main():
                 connector.put(
                     args.dst, 
                     f.read(), 
-                    overwrite=args.overwrite, 
+                    conflict=args.conflict,
                     permission=args.permission
                     )
     else:

{lfss-0.5.0 → lfss-0.5.1}/lfss/client/api.py

@@ -34,13 +34,13 @@ class Connector:
             return response
         return f
 
-    def put(self, path: str, file_data: bytes, permission: int | FileReadPermission = 0, overwrite: bool = False):
+    def put(self, path: str, file_data: bytes, permission: int | FileReadPermission = 0, conflict: Literal['overwrite', 'abort', 'skip'] = 'abort'):
         """Uploads a file to the specified path."""
         if path.startswith('/'):
             path = path[1:]
         response = self._fetch('PUT', path, search_params={
             'permission': int(permission),
-            'overwrite': overwrite
+            'conflict': conflict
             })(
             data=file_data, 
             headers={'Content-Type': 'application/octet-stream'}

lfss-0.5.1/lfss/sql/init.sql

@@ -0,0 +1,38 @@
+CREATE TABLE IF NOT EXISTS user (
+    id INTEGER PRIMARY KEY AUTOINCREMENT,
+    username VARCHAR(255) UNIQUE NOT NULL,
+    credential VARCHAR(255) NOT NULL,
+    is_admin BOOLEAN DEFAULT FALSE,
+    create_time TIMESTAMP DEFAULT CURRENT_TIMESTAMP, 
+    last_active TIMESTAMP DEFAULT CURRENT_TIMESTAMP, 
+    max_storage INTEGER DEFAULT 1073741824,
+    permission INTEGER DEFAULT 0
+);
+
+CREATE TABLE IF NOT EXISTS fmeta (
+    url VARCHAR(512) PRIMARY KEY,
+    owner_id INTEGER NOT NULL,
+    file_id VARCHAR(256) NOT NULL,
+    file_size INTEGER,
+    create_time TIMESTAMP DEFAULT CURRENT_TIMESTAMP, 
+    access_time TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+    permission INTEGER DEFAULT 0, 
+    external BOOLEAN DEFAULT FALSE,
+    FOREIGN KEY(owner_id) REFERENCES user(id)
+);
+
+CREATE TABLE IF NOT EXISTS fdata (
+    file_id VARCHAR(256) PRIMARY KEY,
+    data BLOB
+);
+
+CREATE TABLE IF NOT EXISTS usize (
+    user_id INTEGER PRIMARY KEY,
+    size INTEGER DEFAULT 0
+);
+
+CREATE INDEX IF NOT EXISTS idx_fmeta_url ON fmeta(url);
+
+CREATE INDEX IF NOT EXISTS idx_user_username ON user(username);
+
+CREATE INDEX IF NOT EXISTS idx_user_credential ON user(credential);

lfss-0.5.1/lfss/sql/pragma.sql

@@ -0,0 +1,5 @@
+PRAGMA journal_mode=MEMROY;
+PRAGMA temp_store=MEMORY;
+PRAGMA page_size=8192;
+PRAGMA synchronous=NORMAL;
+PRAGMA case_sensitive_like=ON;

{lfss-0.5.0 → lfss-0.5.1}/lfss/src/config.py

@@ -10,6 +10,7 @@ if not DATA_HOME.exists():
 LARGE_BLOB_DIR = DATA_HOME / 'large_blobs'
 LARGE_BLOB_DIR.mkdir(exist_ok=True)
 
-LARGE_FILE_BYTES = 64 * 1024 * 1024   # 64MB
+# https://sqlite.org/fasterthanfs.html
+LARGE_FILE_BYTES = 8 * 1024 * 1024   # 8MB
 MAX_FILE_BYTES = 1024 * 1024 * 1024   # 1GB
 MAX_BUNDLE_BYTES = 1024 * 1024 * 1024   # 1GB

{lfss-0.5.0 → lfss-0.5.1}/lfss/src/database.py

@@ -3,6 +3,7 @@ from typing import Optional, overload, Literal, AsyncIterable
 from abc import ABC, abstractmethod
 
 import urllib.parse
+from pathlib import Path
 import dataclasses, hashlib, uuid
 from contextlib import asynccontextmanager
 from functools import wraps
@@ -23,6 +24,15 @@ _g_conn: Optional[aiosqlite.Connection] = None
 def hash_credential(username, password):
     return hashlib.sha256((username + password).encode()).hexdigest()
 
+async def execute_sql(conn: aiosqlite.Connection, name: str):
+    this_dir = Path(__file__).parent
+    sql_dir = this_dir.parent / 'sql'
+    async with aiofiles.open(sql_dir / name, 'r') as f:
+        sql = await f.read()
+    sql = sql.split(';')
+    for s in sql:
+        await conn.execute(s)
+
 _atomic_lock = Lock()
 def atomic(func):
     """ Ensure non-reentrancy """
@@ -48,7 +58,8 @@ class DBConnBase(ABC):
         global _g_conn
         if _g_conn is None:
             _g_conn = await aiosqlite.connect(DATA_HOME / 'lfss.db')
-            await _g_conn.execute('PRAGMA journal_mode=memory')
+            await execute_sql(_g_conn, 'pragma.sql')
+            await execute_sql(_g_conn, 'init.sql')
 
     async def commit(self):
         await self.conn.commit()
@@ -82,26 +93,6 @@ class UserConn(DBConnBase):
 
     async def init(self):
         await super().init()
-        # default to 1GB (1024x1024x1024 bytes)
-        await self.conn.execute('''
-        CREATE TABLE IF NOT EXISTS user (
-            id INTEGER PRIMARY KEY AUTOINCREMENT,
-            username VARCHAR(255) UNIQUE NOT NULL,
-            credential VARCHAR(255) NOT NULL,
-            is_admin BOOLEAN DEFAULT FALSE,
-            create_time TIMESTAMP DEFAULT CURRENT_TIMESTAMP, 
-            last_active TIMESTAMP DEFAULT CURRENT_TIMESTAMP, 
-            max_storage INTEGER DEFAULT 1073741824,
-            permission INTEGER DEFAULT 0
-        )
-        ''')
-        await self.conn.execute('''
-        CREATE INDEX IF NOT EXISTS idx_user_username ON user(username)
-        ''')
-        await self.conn.execute('''
-        CREATE INDEX IF NOT EXISTS idx_user_credential ON user(credential)
-        ''')
-
         return self
     
     async def get_user(self, username: str) -> Optional[UserRecord]:
@@ -222,37 +213,6 @@ class FileConn(DBConnBase):
     
     async def init(self):
         await super().init()
-        await self.conn.execute('''
-        CREATE TABLE IF NOT EXISTS fmeta (
-            url VARCHAR(512) PRIMARY KEY,
-            owner_id INTEGER NOT NULL,
-            file_id VARCHAR(256) NOT NULL,
-            file_size INTEGER,
-            create_time TIMESTAMP DEFAULT CURRENT_TIMESTAMP, 
-            access_time TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
-            permission INTEGER DEFAULT 0, 
-            external BOOLEAN DEFAULT FALSE,
-            FOREIGN KEY(owner_id) REFERENCES user(id)
-        )
-        ''')
-        await self.conn.execute('''
-        CREATE INDEX IF NOT EXISTS idx_fmeta_url ON fmeta(url)
-        ''')
-
-        await self.conn.execute('''
-        CREATE TABLE IF NOT EXISTS fdata (
-            file_id VARCHAR(256) PRIMARY KEY,
-            data BLOB
-        )
-        ''')
-
-        # user file size table
-        await self.conn.execute('''
-        CREATE TABLE IF NOT EXISTS usize (
-            user_id INTEGER PRIMARY KEY,
-            size INTEGER DEFAULT 0
-        )
-        ''')
         # backward compatibility, since 0.2.1
         async with self.conn.execute("SELECT * FROM user") as cursor:
             res = await cursor.fetchall()

{lfss-0.5.0 → lfss-0.5.1}/lfss/src/server.py

@@ -1,4 +1,4 @@
-from typing import Optional
+from typing import Optional, Literal
 from functools import wraps
 
 from fastapi import FastAPI, APIRouter, Depends, Request, Response
@@ -179,7 +179,7 @@ async def get_file(path: str, download = False, user: UserRecord = Depends(get_c
 async def put_file(
     request: Request, 
     path: str, 
-    overwrite: Optional[bool] = False,
+    conflict: Literal["overwrite", "skip", "abort"] = "abort",
     permission: int = 0,
     user: UserRecord = Depends(get_current_user)):
     path = ensure_uri_compnents(path)
@@ -201,8 +201,12 @@ async def put_file(
     exists_flag = False
     file_record = await conn.file.get_file_record(path)
     if file_record:
-        if not overwrite:
+        if conflict == "abort":
             raise HTTPException(status_code=409, detail="File exists")
+        if conflict == "skip":
+            return Response(status_code=200, headers={
+                "Content-Type": "application/json",
+            }, content=json.dumps({"url": path}))
         # remove the old file
         exists_flag = True
         await conn.delete_file(path)
@@ -226,8 +230,9 @@ async def put_file(
         blobs = await request.body()
     if len(blobs) > LARGE_FILE_BYTES:
         async def blob_reader():
-            for b in range(0, len(blobs), 4096):
-                yield blobs[b:b+4096]
+            chunk_size = 16 * 1024 * 1024    # 16MB
+            for b in range(0, len(blobs), chunk_size):
+                yield blobs[b:b+chunk_size]
         await conn.save_file(user.id, path, blob_reader(), permission = FileReadPermission(permission))
     else:
         await conn.save_file(user.id, path, blobs, permission = FileReadPermission(permission))

{lfss-0.5.0 → lfss-0.5.1}/pyproject.toml

@@ -1,12 +1,12 @@
 [tool.poetry]
 name = "lfss"
-version = "0.5.0"
+version = "0.5.1"
 description = "Lightweight file storage service"
 authors = ["li, mengxun <limengxun45@outlook.com>"]
 readme = "Readme.md"
 homepage = "https://github.com/MenxLi/lfss"
 repository = "https://github.com/MenxLi/lfss"
-include = ["Readme.md", "docs/*", "frontend/*"]
+include = ["Readme.md", "docs/*", "frontend/*", "lfss/sql/*"]
 
 [tool.poetry.dependencies]
 python = ">=3.9"
@@ -20,6 +20,7 @@ lfss-serve = "lfss.cli.serve:main"
 lfss-user = "lfss.cli.user:main"
 lfss-panel = "lfss.cli.panel:main"
 lfss-cli = "lfss.cli.cli:main"
+lfss-balance = "lfss.cli.balance:main"
 
 [build-system]
 requires = ["poetry-core>=1.0.0"]