lfss 0.7.8__tar.gz → 0.7.9__tar.gz

{lfss-0.7.8 → lfss-0.7.9}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: lfss
-Version: 0.7.8
+Version: 0.7.9
 Summary: Lightweight file storage service
 Home-page: https://github.com/MenxLi/lfss
 Author: li, mengxun

{lfss-0.7.8 → lfss-0.7.9}/frontend/api.js

@@ -28,6 +28,7 @@
  * @property {string} size - the size of the directory, in bytes
  * @property {string} create_time - the time the directory was created
  * @property {string} access_time - the time the directory was last accessed
+ * @property {number} n_files - the number of total files in the directory, including subdirectories
  * 
  * @typedef {Object} PathListResponse
  * @property {DirectoryRecord[]} dirs - the list of directories in the directory

{lfss-0.7.8 → lfss-0.7.9}/frontend/info.js

@@ -89,6 +89,10 @@ export function showDirInfoPanel(r, u, c){
                 <td class="info-table-key">Size</td>
                 <td class="info-table-value" id="info-table-pathsize">N/A</td>
             </tr>
+            <tr>
+                <td class="info-table-key">File-Count</td>
+                <td class="info-table-value" id="info-table-nfiles">N/A</td>
+            </tr>
             <tr>
                 <td class="info-table-key">Access-Time</td>
                 <td class="info-table-value" id="info-table-accesstime">1970-01-01 00:00:00</td>
@@ -124,6 +128,7 @@ export function showDirInfoPanel(r, u, c){
     const sizeValTd = document.querySelector('.info-table-value#info-table-pathsize');
     const createTimeValTd = document.querySelector('.info-table-value#info-table-createtime');
     const accessTimeValTd = document.querySelector('.info-table-value#info-table-accesstime');
+    const countValTd = document.querySelector('.info-table-value#info-table-nfiles');
     // console.log(sizeValTd, createTimeValTd, accessTimeValTd)
     c.getMetadata(ensureSlashEnd(r.url)).then((meta) => {
         if (!meta) {
@@ -133,5 +138,6 @@ export function showDirInfoPanel(r, u, c){
         sizeValTd.textContent = formatSize(meta.size);
         createTimeValTd.textContent = cvtGMT2Local(meta.create_time);
         accessTimeValTd.textContent = cvtGMT2Local(meta.access_time);
+        countValTd.textContent = meta.n_files;
     });
 }

{lfss-0.7.8 → lfss-0.7.9}/lfss/cli/balance.py

@@ -7,10 +7,18 @@ import argparse, time, itertools
 from functools import wraps
 from asyncio import Semaphore
 import aiofiles, asyncio
+import aiofiles.os
 from contextlib import contextmanager
 from lfss.src.database import transaction, unique_cursor
 from lfss.src.connection_pool import global_entrance
 
+@contextmanager
+def indicator(name: str):
+    print(f"\033[1;33mRunning {name}... \033[0m")
+    s = time.time()
+    yield
+    print(f"{name} took {time.time() - s:.2f} seconds.")
+
 sem = Semaphore(1)
 
 def _get_sem():
@@ -54,9 +62,7 @@ async def move_to_internal(f_id: str, flag: str = ''):
 
 @global_entrance()
 async def _main(batch_size: int = 10000):
-
     tasks = []
-    start_time = time.time()
 
     e_cout = 0
     for batch_count in itertools.count(start=0):
@@ -90,20 +96,38 @@ async def _main(batch_size: int = 10000):
             tasks.append(move_to_internal(f_id, flag=f"[b{batch_count+1}-i{i+1}/{len(under_rows)}] "))
         await asyncio.gather(*tasks)
 
-    end_time = time.time()
-    print(f"Balancing complete, took {end_time - start_time:.2f} seconds. "
-          f"{e_cout} files moved to external storage, {i_count} files moved to internal storage.")
+    print(f"Finished. {e_cout} files moved to external storage, {i_count} files moved to internal storage.")
 
 @global_entrance()
 async def vacuum(index: bool = False, blobs: bool = False):
-    @contextmanager
-    def indicator(name: str):
-        print(f"\033[1;33mRunning {name}... \033[0m")
-        s = time.time()
-        yield
-        print(f"{name} took {time.time() - s:.2f} seconds")
+
+    # check if any file in the Large Blob directory is not in the database
+    # the reverse operation is not necessary, because by design, the database should be the source of truth...
+    # we allow un-referenced files in the Large Blob directory on failure, but not the other way around (unless manually deleted)
+    async def ensure_external_consistency(f_id: str):
+        @barriered
+        async def fn():
+            async with unique_cursor() as c:
+                cursor = await c.execute("SELECT file_id FROM fmeta WHERE file_id = ?", (f_id,))
+                if not await cursor.fetchone():
+                    print(f"File {f_id} not found in database, removing from external storage.")
+                    await aiofiles.os.remove(f)
+        await asyncio.create_task(fn())
+
+    # create a temporary index to speed up the process...
+    with indicator("Clearing un-referenced files in external storage"):
+        async with transaction() as c:
+            await c.execute("CREATE INDEX IF NOT EXISTS fmeta_file_id ON fmeta (file_id)")
+        for i, f in enumerate(LARGE_BLOB_DIR.iterdir()):
+            f_id = f.name
+            await ensure_external_consistency(f_id)
+            if (i+1) % 1_000 == 0:
+                print(f"Checked {(i+1)//1000}k files in external storage.", end='\r')
+        async with transaction() as c:
+            await c.execute("DROP INDEX IF EXISTS fmeta_file_id")
 
     async with unique_cursor(is_write=True) as c:
+
         if index:
             with indicator("VACUUM-index"):
                 await c.execute("VACUUM main")
@@ -120,8 +144,10 @@ def main():
     parser.add_argument("--vacuum-all", action="store_true", help="Run VACUUM on both index.db and blobs.db after balancing")
     args = parser.parse_args()
     sem = Semaphore(args.jobs)
-    asyncio.run(_main(args.batch_size))
-    asyncio.run(vacuum(index=args.vacuum or args.vacuum_all, blobs=args.vacuum_all))
+    with indicator("Balancing"):
+        asyncio.run(_main(args.batch_size))
+    if args.vacuum or args.vacuum_all:
+        asyncio.run(vacuum(index=args.vacuum or args.vacuum_all, blobs=args.vacuum_all))
 
 if __name__ == '__main__':
     main()

{lfss-0.7.8 → lfss-0.7.9}/lfss/src/database.py

@@ -139,7 +139,7 @@ class FileConn(DBObjectBase):
             return []
         return [self.parse_record(r) for r in res]
     
-    async def list_root_dirs(self, *usernames: str) -> list[DirectoryRecord]:
+    async def list_root_dirs(self, *usernames: str, skim = False) -> list[DirectoryRecord]:
         """
         Efficiently list users' directories, if usernames is empty, list all users' directories.
         """
@@ -148,12 +148,12 @@ class FileConn(DBObjectBase):
             await self.cur.execute("SELECT username FROM user")
             res = await self.cur.fetchall()
             dirnames = [u[0] + '/' for u in res]
-            dirs = [DirectoryRecord(u, await self.path_size(u, include_subpath=True)) for u in dirnames]
+            dirs = [await self.get_path_record(u) for u in dirnames] if not skim else [DirectoryRecord(u) for u in dirnames]
             return dirs
         else:
             # list specific users
             dirnames = [uname + '/' for uname in usernames]
-            dirs = [DirectoryRecord(u, await self.path_size(u, include_subpath=True)) for u in dirnames]
+            dirs = [await self.get_path_record(u) for u in dirnames] if not skim else [DirectoryRecord(u) for u in dirnames]
             return dirs
     
     async def list_path(self, url: str, flat: bool = False) -> PathContents:
@@ -207,20 +207,24 @@ class FileConn(DBObjectBase):
         return PathContents(dirs, files)
     
     async def get_path_record(self, url: str) -> DirectoryRecord:
+        """
+        Get the full record of a directory, including size, create_time, update_time, access_time etc.
+        """
         assert url.endswith('/'), "Path must end with /"
         cursor = await self.cur.execute("""
             SELECT MIN(create_time) as create_time, 
                 MAX(create_time) as update_time, 
-                MAX(access_time) as access_time 
+                MAX(access_time) as access_time, 
+                COUNT(*) as n_files
             FROM fmeta 
             WHERE url LIKE ?
         """, (url + '%', ))
         result = await cursor.fetchone()
         if result is None or any(val is None for val in result):
             raise PathNotFoundError(f"Path {url} not found")
-        create_time, update_time, access_time = result
+        create_time, update_time, access_time, n_files = result
         p_size = await self.path_size(url, include_subpath=True)
-        return DirectoryRecord(url, p_size, create_time=create_time, update_time=update_time, access_time=access_time)
+        return DirectoryRecord(url, p_size, create_time=create_time, update_time=update_time, access_time=access_time, n_files=n_files)
     
     async def user_size(self, user_id: int) -> int:
         cursor = await self.cur.execute("SELECT size FROM usize WHERE user_id = ?", (user_id, ))
@@ -513,7 +517,7 @@ class Database:
     async def read_file(self, url: str) -> bytes:
         validate_url(url)
 
-        async with transaction() as cur:
+        async with unique_cursor() as cur:
             fconn = FileConn(cur)
             r = await fconn.get_file_record(url)
             if r is None:
@@ -525,7 +529,9 @@ class Database:
             blob = await fconn.get_file_blob(f_id)
             if blob is None:
                 raise FileNotFoundError(f"File {url} data not found")
-            await fconn.log_access(url)
+
+        async with transaction() as w_cur:
+            await FileConn(w_cur).log_access(url)
 
         return blob
 

{lfss-0.7.8 → lfss-0.7.9}/lfss/src/datatype.py

@@ -40,13 +40,14 @@ class FileRecord:
 @dataclasses.dataclass
 class DirectoryRecord:
     url: str
-    size: int
+    size: int = -1
     create_time: str = ""
     update_time: str = ""
     access_time: str = ""
+    n_files: int = -1
 
     def __str__(self):
-        return f"Directory {self.url} (size={self.size})"
+        return f"Directory {self.url} (size={self.size}, created at {self.create_time}, updated at {self.update_time}, accessed at {self.access_time}, n_files={self.n_files})"
 
 @dataclasses.dataclass
 class PathContents:

{lfss-0.7.8 → lfss-0.7.9}/lfss/src/server.py

@@ -135,8 +135,8 @@ async def get_file(path: str, download: bool = False, flat: bool = False, user:
                 if flat:
                     raise HTTPException(status_code=400, detail="Flat query not supported for root path")
                 return PathContents(
-                    dirs = await fconn.list_root_dirs(user.username) \
-                        if not user.is_admin else await fconn.list_root_dirs(),
+                    dirs = await fconn.list_root_dirs(user.username, skim=True) \
+                        if not user.is_admin else await fconn.list_root_dirs(skim=True),
                     files = []
                 )
 

{lfss-0.7.8 → lfss-0.7.9}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "lfss"
-version = "0.7.8"
+version = "0.7.9"
 description = "Lightweight file storage service"
 authors = ["li, mengxun <limengxun45@outlook.com>"]
 readme = "Readme.md"