lfss 0.7.7__tar.gz → 0.7.9__tar.gz

{lfss-0.7.7 → lfss-0.7.9}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: lfss
-Version: 0.7.7
+Version: 0.7.9
 Summary: Lightweight file storage service
 Home-page: https://github.com/MenxLi/lfss
 Author: li, mengxun

{lfss-0.7.7 → lfss-0.7.9}/frontend/api.js

@@ -28,6 +28,7 @@
  * @property {string} size - the size of the directory, in bytes
  * @property {string} create_time - the time the directory was created
  * @property {string} access_time - the time the directory was last accessed
+ * @property {number} n_files - the number of total files in the directory, including subdirectories
  * 
  * @typedef {Object} PathListResponse
  * @property {DirectoryRecord[]} dirs - the list of directories in the directory

{lfss-0.7.7 → lfss-0.7.9}/frontend/info.js

@@ -19,28 +19,28 @@ export function showInfoPanel(r, u){
     <div class="info-container-left">
         <table class="info-table">
             <tr>
-                <td class="info-table-key">Filename</td>
+                <td class="info-table-key">Name</td>
                 <td class="info-table-value">${decodePathURI(r.url).split('/').pop()}</td>
             </tr>
+            <tr>
+                <td class="info-table-key">Size</td>
+                <td class="info-table-value">${formatSize(r.file_size)}</td>
+            </tr>
             <tr>
                 <td class="info-table-key">File-Type</td>
                 <td class="info-table-value">${r.mime_type}</td>
             </tr>
             <tr>
-                <td class="info-table-key">Size</td>
-                <td class="info-table-value">${formatSize(r.file_size)}</td>
+                <td class="info-table-key">Owner-ID</td>
+                <td class="info-table-value">${r.owner_id}</td>
             </tr>
             <tr>
-                <td class="info-table-key">Owner</td>
-                <td class="info-table-value">${r.owner_id}</td>
+                <td class="info-table-key">Access-Time</td>
+                <td class="info-table-value">${cvtGMT2Local(r.access_time)}</td>
             </tr>
             <tr>
                 <td class="info-table-key">Create-Time</td>
                 <td class="info-table-value">${cvtGMT2Local(r.create_time)}</td>
-            </td>
-            <tr>
-                <td class="info-table-key">Access-Time</td>
-                <td class="info-table-value">${cvtGMT2Local(r.access_time)}</td>
             </tr>
         </table>
     </div>
@@ -82,13 +82,17 @@ export function showDirInfoPanel(r, u, c){
     <div class="info-container-left">
         <table class="info-table">
             <tr>
-                <td class="info-table-key">Pathname</td>
+                <td class="info-table-key">Name</td>
                 <td class="info-table-value" id="info-table-pathname">${fmtPath.split('/').pop()}</td>
             </tr>
             <tr>
                 <td class="info-table-key">Size</td>
                 <td class="info-table-value" id="info-table-pathsize">N/A</td>
             </tr>
+            <tr>
+                <td class="info-table-key">File-Count</td>
+                <td class="info-table-value" id="info-table-nfiles">N/A</td>
+            </tr>
             <tr>
                 <td class="info-table-key">Access-Time</td>
                 <td class="info-table-value" id="info-table-accesstime">1970-01-01 00:00:00</td>
@@ -124,6 +128,7 @@ export function showDirInfoPanel(r, u, c){
     const sizeValTd = document.querySelector('.info-table-value#info-table-pathsize');
     const createTimeValTd = document.querySelector('.info-table-value#info-table-createtime');
     const accessTimeValTd = document.querySelector('.info-table-value#info-table-accesstime');
+    const countValTd = document.querySelector('.info-table-value#info-table-nfiles');
     // console.log(sizeValTd, createTimeValTd, accessTimeValTd)
     c.getMetadata(ensureSlashEnd(r.url)).then((meta) => {
         if (!meta) {
@@ -133,5 +138,6 @@ export function showDirInfoPanel(r, u, c){
         sizeValTd.textContent = formatSize(meta.size);
         createTimeValTd.textContent = cvtGMT2Local(meta.create_time);
         accessTimeValTd.textContent = cvtGMT2Local(meta.access_time);
+        countValTd.textContent = meta.n_files;
     });
 }

{lfss-0.7.7 → lfss-0.7.9}/frontend/popup.css

@@ -9,6 +9,19 @@ div.floating-window.blocker{
     z-index: 100;
 }
 
+@keyframes fade-in{
+    from{
+        opacity: 0;
+        transform: translate(-50%, calc(-50% + 0.25rem));
+    }
+    to{
+        opacity: 1;
+        transform: translate(-50%, -50%);
+    }
+}
+div.floating-window.window{
+    animation: fade-in 0.1s ease-in;
+}
 div.floating-window.window{
     position: fixed;
     top: 50%;

{lfss-0.7.7 → lfss-0.7.9}/frontend/scripts.js

@@ -277,6 +277,8 @@ function refreshFileList(){
                     infoButton.style.cursor = 'pointer';
                     infoButton.textContent = 'Details';
                     infoButton.style.width = '100%';
+                    infoButton.style.display = 'block';
+                    infoButton.style.textAlign = 'center';
                     infoButton.addEventListener('click', () => {
                         showDirInfoPanel(dir, userRecord, conn);
                     });

{lfss-0.7.7 → lfss-0.7.9}/frontend/styles.css

@@ -219,4 +219,14 @@ a{
 .delete-btn:hover{
     color: white !important;
     background-color: #990511c7 !important;
+}
+
+button{
+    transition: all 0.2s;
+}
+button:hover {
+    transform: scale(1.05);
+}
+button:active {
+    transform: scale(0.95);
 }

{lfss-0.7.7 → lfss-0.7.9}/lfss/cli/balance.py

@@ -3,13 +3,22 @@ Balance the storage by ensuring that large file thresholds are met.
 """
 
 from lfss.src.config import LARGE_BLOB_DIR, LARGE_FILE_BYTES
-import argparse, time
+import argparse, time, itertools
 from functools import wraps
 from asyncio import Semaphore
 import aiofiles, asyncio
+import aiofiles.os
+from contextlib import contextmanager
 from lfss.src.database import transaction, unique_cursor
 from lfss.src.connection_pool import global_entrance
 
+@contextmanager
+def indicator(name: str):
+    print(f"\033[1;33mRunning {name}... \033[0m")
+    s = time.time()
+    yield
+    print(f"{name} took {time.time() - s:.2f} seconds.")
+
 sem = Semaphore(1)
 
 def _get_sem():
@@ -30,7 +39,6 @@ async def move_to_external(f_id: str, flag: str = ''):
         if blob_row is None:
             print(f"{flag}File {f_id} not found in blobs.fdata")
             return
-        await c.execute("BEGIN")
         blob: bytes = blob_row[0]
         async with aiofiles.open(LARGE_BLOB_DIR / f_id, 'wb') as f:
             await f.write(blob)
@@ -54,13 +62,10 @@ async def move_to_internal(f_id: str, flag: str = ''):
 
 @global_entrance()
 async def _main(batch_size: int = 10000):
-
     tasks = []
-    start_time = time.time()
 
     e_cout = 0
-    batch_count = 0
-    while True:
+    for batch_count in itertools.count(start=0):
         async with unique_cursor() as conn:
             exceeded_rows = list(await (await conn.execute( 
                 "SELECT file_id FROM fmeta WHERE file_size > ? AND external = 0 LIMIT ? OFFSET ?",
@@ -76,8 +81,7 @@ async def _main(batch_size: int = 10000):
         await asyncio.gather(*tasks)
 
     i_count = 0
-    batch_count = 0
-    while True:
+    for batch_count in itertools.count(start=0):
         async with unique_cursor() as conn:
             under_rows = list(await (await conn.execute(
                 "SELECT file_id, file_size, external FROM fmeta WHERE file_size <= ? AND external = 1 LIMIT ? OFFSET ?",
@@ -92,18 +96,58 @@ async def _main(batch_size: int = 10000):
             tasks.append(move_to_internal(f_id, flag=f"[b{batch_count+1}-i{i+1}/{len(under_rows)}] "))
         await asyncio.gather(*tasks)
 
-    end_time = time.time()
-    print(f"Balancing complete, took {end_time - start_time:.2f} seconds. "
-          f"{e_cout} files moved to external storage, {i_count} files moved to internal storage.")
+    print(f"Finished. {e_cout} files moved to external storage, {i_count} files moved to internal storage.")
+
+@global_entrance()
+async def vacuum(index: bool = False, blobs: bool = False):
+
+    # check if any file in the Large Blob directory is not in the database
+    # the reverse operation is not necessary, because by design, the database should be the source of truth...
+    # we allow un-referenced files in the Large Blob directory on failure, but not the other way around (unless manually deleted)
+    async def ensure_external_consistency(f_id: str):
+        @barriered
+        async def fn():
+            async with unique_cursor() as c:
+                cursor = await c.execute("SELECT file_id FROM fmeta WHERE file_id = ?", (f_id,))
+                if not await cursor.fetchone():
+                    print(f"File {f_id} not found in database, removing from external storage.")
+                    await aiofiles.os.remove(f)
+        await asyncio.create_task(fn())
+
+    # create a temporary index to speed up the process...
+    with indicator("Clearing un-referenced files in external storage"):
+        async with transaction() as c:
+            await c.execute("CREATE INDEX IF NOT EXISTS fmeta_file_id ON fmeta (file_id)")
+        for i, f in enumerate(LARGE_BLOB_DIR.iterdir()):
+            f_id = f.name
+            await ensure_external_consistency(f_id)
+            if (i+1) % 1_000 == 0:
+                print(f"Checked {(i+1)//1000}k files in external storage.", end='\r')
+        async with transaction() as c:
+            await c.execute("DROP INDEX IF EXISTS fmeta_file_id")
+
+    async with unique_cursor(is_write=True) as c:
+
+        if index:
+            with indicator("VACUUM-index"):
+                await c.execute("VACUUM main")
+        if blobs:
+            with indicator("VACUUM-blobs"):
+                await c.execute("VACUUM blobs")
             
 def main():
     global sem
     parser = argparse.ArgumentParser(description="Balance the storage by ensuring that large file thresholds are met.")
     parser.add_argument("-j", "--jobs", type=int, default=2, help="Number of concurrent jobs")
     parser.add_argument("-b", "--batch-size", type=int, default=10000, help="Batch size for processing files")
+    parser.add_argument("--vacuum", action="store_true", help="Run VACUUM only on index.db after balancing")
+    parser.add_argument("--vacuum-all", action="store_true", help="Run VACUUM on both index.db and blobs.db after balancing")
     args = parser.parse_args()
     sem = Semaphore(args.jobs)
-    asyncio.run(_main(args.batch_size))
+    with indicator("Balancing"):
+        asyncio.run(_main(args.batch_size))
+    if args.vacuum or args.vacuum_all:
+        asyncio.run(vacuum(index=args.vacuum or args.vacuum_all, blobs=args.vacuum_all))
 
 if __name__ == '__main__':
     main()

{lfss-0.7.7 → lfss-0.7.9}/lfss/cli/user.py

@@ -1,20 +1,10 @@
 import argparse, asyncio
 from contextlib import asynccontextmanager
 from .cli import parse_permission, FileReadPermission
+from ..src.utils import parse_storage_size
 from ..src.database import Database, FileReadPermission, transaction, UserConn
 from ..src.connection_pool import global_entrance
 
-def parse_storage_size(s: str) -> int:
-    if s[-1] in 'Kk':
-        return int(s[:-1]) * 1024
-    if s[-1] in 'Mm':
-        return int(s[:-1]) * 1024 * 1024
-    if s[-1] in 'Gg':
-        return int(s[:-1]) * 1024 * 1024 * 1024
-    if s[-1] in 'Tt':
-        return int(s[:-1]) * 1024 * 1024 * 1024 * 1024
-    return int(s)
-
 @global_entrance(1)
 async def _main():
     parser = argparse.ArgumentParser()

{lfss-0.7.7 → lfss-0.7.9}/lfss/src/config.py

@@ -1,5 +1,6 @@
-from pathlib import Path
 import os
+from pathlib import Path
+from .utils import parse_storage_size
 
 __default_dir = '.storage_data'
 
@@ -12,6 +13,10 @@ LARGE_BLOB_DIR = DATA_HOME / 'large_blobs'
 LARGE_BLOB_DIR.mkdir(exist_ok=True)
 
 # https://sqlite.org/fasterthanfs.html
-LARGE_FILE_BYTES = 8 * 1024 * 1024   # 8MB
+__env_large_file = os.environ.get('LFSS_LARGE_FILE', None)
+if __env_large_file is not None:
+    LARGE_FILE_BYTES = parse_storage_size(__env_large_file)
+else:
+    LARGE_FILE_BYTES = 8 * 1024 * 1024  # 8MB
 MAX_FILE_BYTES = 512 * 1024 * 1024   # 512MB
 MAX_BUNDLE_BYTES = 512 * 1024 * 1024   # 512MB

{lfss-0.7.7 → lfss-0.7.9}/lfss/src/connection_pool.py

@@ -5,6 +5,7 @@ from contextlib import asynccontextmanager
 from dataclasses import dataclass
 from asyncio import Semaphore, Lock
 from functools import wraps
+from typing import Callable, Awaitable
 
 from .log import get_logger
 from .config import DATA_HOME
@@ -125,7 +126,7 @@ async def global_connection(n_read: int = 1):
         await global_connection_close()
 
 def global_entrance(n_read: int = 1):
-    def decorator(func):
+    def decorator(func: Callable[..., Awaitable]):
         @wraps(func)
         async def wrapper(*args, **kwargs):
             async with global_connection(n_read):

{lfss-0.7.7 → lfss-0.7.9}/lfss/src/database.py

@@ -139,7 +139,7 @@ class FileConn(DBObjectBase):
             return []
         return [self.parse_record(r) for r in res]
     
-    async def list_root_dirs(self, *usernames: str) -> list[DirectoryRecord]:
+    async def list_root_dirs(self, *usernames: str, skim = False) -> list[DirectoryRecord]:
         """
         Efficiently list users' directories, if usernames is empty, list all users' directories.
         """
@@ -148,12 +148,12 @@ class FileConn(DBObjectBase):
             await self.cur.execute("SELECT username FROM user")
             res = await self.cur.fetchall()
             dirnames = [u[0] + '/' for u in res]
-            dirs = [DirectoryRecord(u, await self.path_size(u, include_subpath=True)) for u in dirnames]
+            dirs = [await self.get_path_record(u) for u in dirnames] if not skim else [DirectoryRecord(u) for u in dirnames]
             return dirs
         else:
             # list specific users
             dirnames = [uname + '/' for uname in usernames]
-            dirs = [DirectoryRecord(u, await self.path_size(u, include_subpath=True)) for u in dirnames]
+            dirs = [await self.get_path_record(u) for u in dirnames] if not skim else [DirectoryRecord(u) for u in dirnames]
             return dirs
     
     async def list_path(self, url: str, flat: bool = False) -> PathContents:
@@ -207,20 +207,24 @@ class FileConn(DBObjectBase):
         return PathContents(dirs, files)
     
     async def get_path_record(self, url: str) -> DirectoryRecord:
+        """
+        Get the full record of a directory, including size, create_time, update_time, access_time etc.
+        """
         assert url.endswith('/'), "Path must end with /"
         cursor = await self.cur.execute("""
             SELECT MIN(create_time) as create_time, 
                 MAX(create_time) as update_time, 
-                MAX(access_time) as access_time 
+                MAX(access_time) as access_time, 
+                COUNT(*) as n_files
             FROM fmeta 
             WHERE url LIKE ?
         """, (url + '%', ))
         result = await cursor.fetchone()
         if result is None or any(val is None for val in result):
             raise PathNotFoundError(f"Path {url} not found")
-        create_time, update_time, access_time = result
+        create_time, update_time, access_time, n_files = result
         p_size = await self.path_size(url, include_subpath=True)
-        return DirectoryRecord(url, p_size, create_time=create_time, update_time=update_time, access_time=access_time)
+        return DirectoryRecord(url, p_size, create_time=create_time, update_time=update_time, access_time=access_time, n_files=n_files)
     
     async def user_size(self, user_id: int) -> int:
         cursor = await self.cur.execute("SELECT size FROM usize WHERE user_id = ?", (user_id, ))
@@ -513,7 +517,7 @@ class Database:
     async def read_file(self, url: str) -> bytes:
         validate_url(url)
 
-        async with transaction() as cur:
+        async with unique_cursor() as cur:
             fconn = FileConn(cur)
             r = await fconn.get_file_record(url)
             if r is None:
@@ -525,7 +529,9 @@ class Database:
             blob = await fconn.get_file_blob(f_id)
             if blob is None:
                 raise FileNotFoundError(f"File {url} data not found")
-            await fconn.log_access(url)
+
+        async with transaction() as w_cur:
+            await FileConn(w_cur).log_access(url)
 
         return blob
 

{lfss-0.7.7 → lfss-0.7.9}/lfss/src/datatype.py

@@ -40,13 +40,14 @@ class FileRecord:
 @dataclasses.dataclass
 class DirectoryRecord:
     url: str
-    size: int
+    size: int = -1
     create_time: str = ""
     update_time: str = ""
     access_time: str = ""
+    n_files: int = -1
 
     def __str__(self):
-        return f"Directory {self.url} (size={self.size})"
+        return f"Directory {self.url} (size={self.size}, created at {self.create_time}, updated at {self.update_time}, accessed at {self.access_time}, n_files={self.n_files})"
 
 @dataclasses.dataclass
 class PathContents:

{lfss-0.7.7 → lfss-0.7.9}/lfss/src/server.py

@@ -135,8 +135,8 @@ async def get_file(path: str, download: bool = False, flat: bool = False, user:
                 if flat:
                     raise HTTPException(status_code=400, detail="Flat query not supported for root path")
                 return PathContents(
-                    dirs = await fconn.list_root_dirs(user.username) \
-                        if not user.is_admin else await fconn.list_root_dirs(),
+                    dirs = await fconn.list_root_dirs(user.username, skim=True) \
+                        if not user.is_admin else await fconn.list_root_dirs(skim=True),
                     files = []
                 )
 

{lfss-0.7.7 → lfss-0.7.9}/lfss/src/utils.py

@@ -62,6 +62,18 @@ def now_stamp() -> float:
 def stamp_to_str(stamp: float) -> str:
     return datetime.datetime.fromtimestamp(stamp).strftime('%Y-%m-%d %H:%M:%S')
 
+def parse_storage_size(s: str) -> int:
+    """ Parse the file size string to bytes """
+    if s[-1].isdigit():
+        return int(s)
+    unit = s[-1].lower()
+    match unit:
+        case 'b': return int(s[:-1])
+        case 'k': return int(s[:-1]) * 1024
+        case 'm': return int(s[:-1]) * 1024**2
+        case 'g': return int(s[:-1]) * 1024**3
+        case 't': return int(s[:-1]) * 1024**4
+        case _: raise ValueError(f"Invalid file size string: {s}")
 
 _FnReturnT = TypeVar('_FnReturnT')
 _AsyncReturnT = Awaitable[_FnReturnT]

{lfss-0.7.7 → lfss-0.7.9}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "lfss"
-version = "0.7.7"
+version = "0.7.9"
 description = "Lightweight file storage service"
 authors = ["li, mengxun <limengxun45@outlook.com>"]
 readme = "Readme.md"