lfss 0.7.8__py3-none-any.whl → 0.7.9__py3-none-any.whl

frontend/api.js

@@ -28,6 +28,7 @@
  * @property {string} size - the size of the directory, in bytes
  * @property {string} create_time - the time the directory was created
  * @property {string} access_time - the time the directory was last accessed
+ * @property {number} n_files - the number of total files in the directory, including subdirectories
  * 
  * @typedef {Object} PathListResponse
  * @property {DirectoryRecord[]} dirs - the list of directories in the directory

frontend/info.js

@@ -89,6 +89,10 @@ export function showDirInfoPanel(r, u, c){
                 <td class="info-table-key">Size</td>
                 <td class="info-table-value" id="info-table-pathsize">N/A</td>
             </tr>
+            <tr>
+                <td class="info-table-key">File-Count</td>
+                <td class="info-table-value" id="info-table-nfiles">N/A</td>
+            </tr>
             <tr>
                 <td class="info-table-key">Access-Time</td>
                 <td class="info-table-value" id="info-table-accesstime">1970-01-01 00:00:00</td>
@@ -124,6 +128,7 @@ export function showDirInfoPanel(r, u, c){
     const sizeValTd = document.querySelector('.info-table-value#info-table-pathsize');
     const createTimeValTd = document.querySelector('.info-table-value#info-table-createtime');
     const accessTimeValTd = document.querySelector('.info-table-value#info-table-accesstime');
+    const countValTd = document.querySelector('.info-table-value#info-table-nfiles');
     // console.log(sizeValTd, createTimeValTd, accessTimeValTd)
     c.getMetadata(ensureSlashEnd(r.url)).then((meta) => {
         if (!meta) {
@@ -133,5 +138,6 @@ export function showDirInfoPanel(r, u, c){
         sizeValTd.textContent = formatSize(meta.size);
         createTimeValTd.textContent = cvtGMT2Local(meta.create_time);
         accessTimeValTd.textContent = cvtGMT2Local(meta.access_time);
+        countValTd.textContent = meta.n_files;
     });
 }

lfss/cli/balance.py

@@ -7,10 +7,18 @@ import argparse, time, itertools
 from functools import wraps
 from asyncio import Semaphore
 import aiofiles, asyncio
+import aiofiles.os
 from contextlib import contextmanager
 from lfss.src.database import transaction, unique_cursor
 from lfss.src.connection_pool import global_entrance
 
+@contextmanager
+def indicator(name: str):
+    print(f"\033[1;33mRunning {name}... \033[0m")
+    s = time.time()
+    yield
+    print(f"{name} took {time.time() - s:.2f} seconds.")
+
 sem = Semaphore(1)
 
 def _get_sem():
@@ -54,9 +62,7 @@ async def move_to_internal(f_id: str, flag: str = ''):
 
 @global_entrance()
 async def _main(batch_size: int = 10000):
-
     tasks = []
-    start_time = time.time()
 
     e_cout = 0
     for batch_count in itertools.count(start=0):
@@ -90,20 +96,38 @@ async def _main(batch_size: int = 10000):
             tasks.append(move_to_internal(f_id, flag=f"[b{batch_count+1}-i{i+1}/{len(under_rows)}] "))
         await asyncio.gather(*tasks)
 
-    end_time = time.time()
-    print(f"Balancing complete, took {end_time - start_time:.2f} seconds. "
-          f"{e_cout} files moved to external storage, {i_count} files moved to internal storage.")
+    print(f"Finished. {e_cout} files moved to external storage, {i_count} files moved to internal storage.")
 
 @global_entrance()
 async def vacuum(index: bool = False, blobs: bool = False):
-    @contextmanager
-    def indicator(name: str):
-        print(f"\033[1;33mRunning {name}... \033[0m")
-        s = time.time()
-        yield
-        print(f"{name} took {time.time() - s:.2f} seconds")
+
+    # check if any file in the Large Blob directory is not in the database
+    # the reverse operation is not necessary, because by design, the database should be the source of truth...
+    # we allow un-referenced files in the Large Blob directory on failure, but not the other way around (unless manually deleted)
+    async def ensure_external_consistency(f_id: str):
+        @barriered
+        async def fn():
+            async with unique_cursor() as c:
+                cursor = await c.execute("SELECT file_id FROM fmeta WHERE file_id = ?", (f_id,))
+                if not await cursor.fetchone():
+                    print(f"File {f_id} not found in database, removing from external storage.")
+                    await aiofiles.os.remove(f)
+        await asyncio.create_task(fn())
+
+    # create a temporary index to speed up the process...
+    with indicator("Clearing un-referenced files in external storage"):
+        async with transaction() as c:
+            await c.execute("CREATE INDEX IF NOT EXISTS fmeta_file_id ON fmeta (file_id)")
+        for i, f in enumerate(LARGE_BLOB_DIR.iterdir()):
+            f_id = f.name
+            await ensure_external_consistency(f_id)
+            if (i+1) % 1_000 == 0:
+                print(f"Checked {(i+1)//1000}k files in external storage.", end='\r')
+        async with transaction() as c:
+            await c.execute("DROP INDEX IF EXISTS fmeta_file_id")
 
     async with unique_cursor(is_write=True) as c:
+
         if index:
             with indicator("VACUUM-index"):
                 await c.execute("VACUUM main")
@@ -120,8 +144,10 @@ def main():
     parser.add_argument("--vacuum-all", action="store_true", help="Run VACUUM on both index.db and blobs.db after balancing")
     args = parser.parse_args()
     sem = Semaphore(args.jobs)
-    asyncio.run(_main(args.batch_size))
-    asyncio.run(vacuum(index=args.vacuum or args.vacuum_all, blobs=args.vacuum_all))
+    with indicator("Balancing"):
+        asyncio.run(_main(args.batch_size))
+    if args.vacuum or args.vacuum_all:
+        asyncio.run(vacuum(index=args.vacuum or args.vacuum_all, blobs=args.vacuum_all))
 
 if __name__ == '__main__':
     main()

lfss/src/database.py

@@ -139,7 +139,7 @@ class FileConn(DBObjectBase):
             return []
         return [self.parse_record(r) for r in res]
     
-    async def list_root_dirs(self, *usernames: str) -> list[DirectoryRecord]:
+    async def list_root_dirs(self, *usernames: str, skim = False) -> list[DirectoryRecord]:
         """
         Efficiently list users' directories, if usernames is empty, list all users' directories.
         """
@@ -148,12 +148,12 @@ class FileConn(DBObjectBase):
             await self.cur.execute("SELECT username FROM user")
             res = await self.cur.fetchall()
             dirnames = [u[0] + '/' for u in res]
-            dirs = [DirectoryRecord(u, await self.path_size(u, include_subpath=True)) for u in dirnames]
+            dirs = [await self.get_path_record(u) for u in dirnames] if not skim else [DirectoryRecord(u) for u in dirnames]
             return dirs
         else:
             # list specific users
             dirnames = [uname + '/' for uname in usernames]
-            dirs = [DirectoryRecord(u, await self.path_size(u, include_subpath=True)) for u in dirnames]
+            dirs = [await self.get_path_record(u) for u in dirnames] if not skim else [DirectoryRecord(u) for u in dirnames]
             return dirs
     
     async def list_path(self, url: str, flat: bool = False) -> PathContents:
@@ -207,20 +207,24 @@ class FileConn(DBObjectBase):
         return PathContents(dirs, files)
     
     async def get_path_record(self, url: str) -> DirectoryRecord:
+        """
+        Get the full record of a directory, including size, create_time, update_time, access_time etc.
+        """
         assert url.endswith('/'), "Path must end with /"
         cursor = await self.cur.execute("""
             SELECT MIN(create_time) as create_time, 
                 MAX(create_time) as update_time, 
-                MAX(access_time) as access_time 
+                MAX(access_time) as access_time, 
+                COUNT(*) as n_files
             FROM fmeta 
             WHERE url LIKE ?
         """, (url + '%', ))
         result = await cursor.fetchone()
         if result is None or any(val is None for val in result):
             raise PathNotFoundError(f"Path {url} not found")
-        create_time, update_time, access_time = result
+        create_time, update_time, access_time, n_files = result
         p_size = await self.path_size(url, include_subpath=True)
-        return DirectoryRecord(url, p_size, create_time=create_time, update_time=update_time, access_time=access_time)
+        return DirectoryRecord(url, p_size, create_time=create_time, update_time=update_time, access_time=access_time, n_files=n_files)
     
     async def user_size(self, user_id: int) -> int:
         cursor = await self.cur.execute("SELECT size FROM usize WHERE user_id = ?", (user_id, ))
@@ -513,7 +517,7 @@ class Database:
     async def read_file(self, url: str) -> bytes:
         validate_url(url)
 
-        async with transaction() as cur:
+        async with unique_cursor() as cur:
             fconn = FileConn(cur)
             r = await fconn.get_file_record(url)
             if r is None:
@@ -525,7 +529,9 @@ class Database:
             blob = await fconn.get_file_blob(f_id)
             if blob is None:
                 raise FileNotFoundError(f"File {url} data not found")
-            await fconn.log_access(url)
+
+        async with transaction() as w_cur:
+            await FileConn(w_cur).log_access(url)
 
         return blob
 

lfss/src/datatype.py

@@ -40,13 +40,14 @@ class FileRecord:
 @dataclasses.dataclass
 class DirectoryRecord:
     url: str
-    size: int
+    size: int = -1
     create_time: str = ""
     update_time: str = ""
     access_time: str = ""
+    n_files: int = -1
 
     def __str__(self):
-        return f"Directory {self.url} (size={self.size})"
+        return f"Directory {self.url} (size={self.size}, created at {self.create_time}, updated at {self.update_time}, accessed at {self.access_time}, n_files={self.n_files})"
 
 @dataclasses.dataclass
 class PathContents:

lfss/src/server.py

@@ -135,8 +135,8 @@ async def get_file(path: str, download: bool = False, flat: bool = False, user:
                 if flat:
                     raise HTTPException(status_code=400, detail="Flat query not supported for root path")
                 return PathContents(
-                    dirs = await fconn.list_root_dirs(user.username) \
-                        if not user.is_admin else await fconn.list_root_dirs(),
+                    dirs = await fconn.list_root_dirs(user.username, skim=True) \
+                        if not user.is_admin else await fconn.list_root_dirs(skim=True),
                     files = []
                 )
 

{lfss-0.7.8.dist-info → lfss-0.7.9.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: lfss
-Version: 0.7.8
+Version: 0.7.9
 Summary: Lightweight file storage service
 Home-page: https://github.com/MenxLi/lfss
 Author: li, mengxun

{lfss-0.7.8.dist-info → lfss-0.7.9.dist-info}/RECORD

@@ -1,16 +1,16 @@
 Readme.md,sha256=vsPotlwPAaHI5plh4aaszpi3rr7ZGDn7-wLdEYTWQ0k,1275
 docs/Known_issues.md,sha256=rfdG3j1OJF-59S9E06VPyn0nZKbW-ybPxkoZ7MEZWp8,81
 docs/Permission.md,sha256=X0VNfBKU52f93QYqcVyiBFJ3yURiSkhIo9S_5fdSgzM,2265
-frontend/api.js,sha256=DxWmqO0AAOsWLXYtbgAzEnSmVyEJyzcxXSCH7H3STUk,7925
+frontend/api.js,sha256=o1sP4rKxxnM-rebxnlMlPkhPHzKaVW4kZC7B4ufbOK4,8026
 frontend/index.html,sha256=Mem8de9vwmZoe4x1DKqpu_aFgIBURqT3mIGdeOOTbIs,2051
 frontend/info.css,sha256=Ny0N3GywQ3a9q1_Qph_QFEKB4fEnTe_2DJ1Y5OsLLmQ,595
-frontend/info.js,sha256=mXCQnRESSx7iiR1FW63sAXHnm0NZ3REeaRmnjZNeQbU,5454
+frontend/info.js,sha256=WhOGaeqMoezEAfg4nIpK26hvejC7AZ-ZDLiJmRj0kDk,5758
 frontend/popup.css,sha256=TJZYFW1ZcdD1IVTlNPYNtMWKPbN6XDbQ4hKBOFK8uLg,1284
 frontend/popup.js,sha256=3PgaGZmxSdV1E-D_MWgcR7aHWkcsHA1BNKSOkmP66tA,5191
 frontend/scripts.js,sha256=OP99BSbnyTE1LJebGVUvV3WUnDBiZdqaC3a9SE1FF6U,20286
 frontend/styles.css,sha256=37aU9Iep_hTz3LnAAAcEhC_I7AC0A4lX6apnMuGPTlA,4214
 frontend/utils.js,sha256=Ts4nlef8pkrEgpwX-uQwAhWvwxlIzex8ijDLNCa22ps,2372
-lfss/cli/balance.py,sha256=X6Q6e7sb6LMlCXG4qSD2MUMlixV2Kc9EXPluIe2S5DA,5090
+lfss/cli/balance.py,sha256=TmK48DGU7xPMLv7kASOCS-PY8TIs6GQEsRVRK_4YtXY,6456
 lfss/cli/cli.py,sha256=LH1nx5wI1K2DZ3hvHz7oq5HcXVDoW2V6sr7q9gJ8gqo,4621
 lfss/cli/panel.py,sha256=iGdVmdWYjA_7a78ZzWEB_3ggIOBeUKTzg6F5zLaB25c,1401
 lfss/cli/serve.py,sha256=bO3GT0kuylMGN-7bZWP4e71MlugGZ_lEMkYaYld_Ntg,985
@@ -22,14 +22,14 @@ lfss/sql/pragma.sql,sha256=uENx7xXjARmro-A3XAK8OM8v5AxDMdCCRj47f86UuXg,206
 lfss/src/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 lfss/src/config.py,sha256=CIbVFWRu86dl2GVlXlCDv93W8PLwT89NtznU6TCKvtk,729
 lfss/src/connection_pool.py,sha256=r4Ho5d_Gd4S_KbT7515UJoiyfIgS6xyttqMsKqOfaIg,5190
-lfss/src/database.py,sha256=-itbpGb7cQrywZLFk4aNcuy38Krsyemtyiz8GIt4i7M,31944
-lfss/src/datatype.py,sha256=BLS7vuuKnFZQg0nrKeP9SymqUhcN6HwPgejU0yBd_Ak,1622
+lfss/src/database.py,sha256=AmyQwPFjrntFyZPTJOUB5kGvFpWuex6TPtxZKR2KK8k,32244
+lfss/src/datatype.py,sha256=WfrLALU_7wei5-i_b0TxY8xWI5mwxLUHFepHSps49zA,1767
 lfss/src/error.py,sha256=imbhwnbhnI3HLhkbfICROe3F0gleKrOk4XnqHJDOtuI,285
 lfss/src/log.py,sha256=xOnkuH-gB_jSVGqNnDVEW05iki6SCJ2xdEhjz5eEsMo,5136
-lfss/src/server.py,sha256=EA5fK4qc98tF8qoS9F6VaxIE65D5X8Ztkjqy8EUYIv8,16276
+lfss/src/server.py,sha256=rrrhFDFrglth4yCvdvvYko-4JfVJ_MTixhCr9_Hbhx0,16296
 lfss/src/stat.py,sha256=hTMtQyM_Ukmhc33Bb9FGCfBMIX02KrGHQg8nL7sC8sU,2082
 lfss/src/utils.py,sha256=S9LCJ5OkNk_zM4rZnrHg1UDjnNkDVO_ejmfsBeNJs4s,3868
-lfss-0.7.8.dist-info/METADATA,sha256=JwNtzTpXrk_e4ydOV286AkNBUANiIMlDxPiptRAoHH4,1967
-lfss-0.7.8.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-lfss-0.7.8.dist-info/entry_points.txt,sha256=d_Ri3GXxUW-S0E6q953A8od0YMmUAnZGlJSKS46OiW8,172
-lfss-0.7.8.dist-info/RECORD,,
+lfss-0.7.9.dist-info/METADATA,sha256=rPPqsNm5iZI8XyG8YzY1wWn0X1TbGa8tzrEykDkXf3o,1967
+lfss-0.7.9.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+lfss-0.7.9.dist-info/entry_points.txt,sha256=d_Ri3GXxUW-S0E6q953A8od0YMmUAnZGlJSKS46OiW8,172
+lfss-0.7.9.dist-info/RECORD,,