lfss 0.7.1__py3-none-any.whl → 0.7.2__py3-none-any.whl

Readme.md

@@ -1,7 +1,9 @@
 # Lightweight File Storage Service (LFSS)
 [![PyPI](https://img.shields.io/pypi/v/lfss)](https://pypi.org/project/lfss/)
 
-A lightweight file/object storage service!
+My experiment on a lightweight file/object storage service.  
+It stores small files and metadata in sqlite, large files in the filesystem, and serves them through a simple REST API.  
+Tested on 2 million files, and it works fine... thanks to the sqlite database!
 
 Usage: 
 ```sh
@@ -21,7 +23,7 @@ Or, you can start a web server at `/frontend` and open `index.html` in your brow
 
 The API usage is simple, just `GET`, `PUT`, `DELETE` to the `/<username>/file/url` path.  
 Authentication is done via `Authorization` header with the value `Bearer <token>`, or through the `token` query parameter.  
-You can refer to `frontend` as an application example, and `frontend/api.js` or `lfss.client.api.py` for the API usage.
+You can refer to `frontend` as an application example, and `frontend/api.js` or `lfss/client/api.py` for the API usage.
 
 By default, the service exposes all files to the public for `GET` requests, 
 but file-listing is restricted to the user's own files.  

lfss/cli/balance.py

@@ -24,28 +24,19 @@ def barriered(func):
 
 @barriered
 async def move_to_external(f_id: str, flag: str = ''):
-    # async with aiosqlite.connect(db_file, timeout = 60) as c:
     async with transaction() as c:
-        async with c.execute( "SELECT data FROM blobs.fdata WHERE file_id = ?", (f_id,)) as cursor:
-            blob_row = await cursor.fetchone()
-            if blob_row is None:
-                print(f"{flag}File {f_id} not found in blobs.fdata")
-                return
+        cursor = await c.execute( "SELECT data FROM blobs.fdata WHERE file_id = ?", (f_id,))
+        blob_row = await cursor.fetchone()
+        if blob_row is None:
+            print(f"{flag}File {f_id} not found in blobs.fdata")
+            return
         await c.execute("BEGIN")
         blob: bytes = blob_row[0]
-        try:
-            async with aiofiles.open(LARGE_BLOB_DIR / f_id, 'wb') as f:
-                await f.write(blob)
-            await c.execute( "UPDATE fmeta SET external = 1 WHERE file_id = ?", (f_id,))
-            await c.execute( "DELETE FROM blobs.fdata WHERE file_id = ?", (f_id,))
-            await c.commit()
-            print(f"{flag}Moved {f_id} to external storage")
-        except Exception as e:
-            await c.rollback()
-            print(f"{flag}Error moving {f_id}: {e}")
-
-            if isinstance(e, KeyboardInterrupt):
-                raise e
+        async with aiofiles.open(LARGE_BLOB_DIR / f_id, 'wb') as f:
+            await f.write(blob)
+        await c.execute( "UPDATE fmeta SET external = 1 WHERE file_id = ?", (f_id,))
+        await c.execute( "DELETE FROM blobs.fdata WHERE file_id = ?", (f_id,))
+        print(f"{flag}Moved {f_id} to external storage")
 
 @barriered
 async def move_to_internal(f_id: str, flag: str = ''):
@@ -56,19 +47,10 @@ async def move_to_internal(f_id: str, flag: str = ''):
         async with aiofiles.open(LARGE_BLOB_DIR / f_id, 'rb') as f:
             blob = await f.read()
 
-        await c.execute("BEGIN")
-        try:
-            await c.execute("INSERT INTO blobs.fdata (file_id, data) VALUES (?, ?)", (f_id, blob))
-            await c.execute("UPDATE fmeta SET external = 0 WHERE file_id = ?", (f_id,))
-            await c.commit()
-            (LARGE_BLOB_DIR / f_id).unlink(missing_ok=True)
-            print(f"{flag}Moved {f_id} to internal storage")
-        except Exception as e:
-            await c.rollback()
-            print(f"{flag}Error moving {f_id}: {e}")
-            if isinstance(e, KeyboardInterrupt):
-                raise e
-
+        await c.execute("INSERT INTO blobs.fdata (file_id, data) VALUES (?, ?)", (f_id, blob))
+        await c.execute("UPDATE fmeta SET external = 0 WHERE file_id = ?", (f_id,))
+        (LARGE_BLOB_DIR / f_id).unlink(missing_ok=True)
+        print(f"{flag}Moved {f_id} to internal storage")
 
 @global_entrance()
 async def _main(batch_size: int = 10000):

lfss/src/config.py

@@ -13,5 +13,5 @@ LARGE_BLOB_DIR.mkdir(exist_ok=True)
 
 # https://sqlite.org/fasterthanfs.html
 LARGE_FILE_BYTES = 8 * 1024 * 1024   # 8MB
-MAX_FILE_BYTES = 1024 * 1024 * 1024   # 1GB
-MAX_BUNDLE_BYTES = 1024 * 1024 * 1024   # 1GB
+MAX_FILE_BYTES = 512 * 1024 * 1024   # 512MB
+MAX_BUNDLE_BYTES = 512 * 1024 * 1024   # 512MB

lfss/src/connection_pool.py

@@ -66,26 +66,27 @@ class SqlConnectionPool:
         if len(self._connections) == 0:
             raise Exception("No available connections, please init the pool first")
         
-        if w:
-            assert self._w_connection
-            if self._w_connection.is_available:
-                self._w_connection.is_available = False
-                return self._w_connection
-            raise Exception("Write connection is not available")
-
         async with self._lock:
-            for c in self._connections:
-                if c.is_available:
-                    c.is_available = False
-                    return c
+            if w:
+                assert self._w_connection
+                if self._w_connection.is_available:
+                    self._w_connection.is_available = False
+                    return self._w_connection
+                raise Exception("Write connection is not available")
+
+            else:
+                for c in self._connections:
+                    if c.is_available:
+                        c.is_available = False
+                        return c
         raise Exception("No available connections, impossible?")
     
     async def release(self, conn: SqlConnection):
-        if conn == self._w_connection:
-            conn.is_available = True
-            return
-
         async with self._lock:
+            if conn == self._w_connection:
+                conn.is_available = True
+                return
+
             if not conn in self._connections:
                 raise Exception("Connection not in pool")
             conn.is_available = True

lfss/src/database.py

@@ -519,7 +519,13 @@ class Database:
                 raise FileNotFoundError(f"File {url} not found")
             if not r.external:
                 raise ValueError(f"File {url} is not stored externally, should use read_file instead")
-            return fconn.get_file_blob_external(r.file_id)
+            ret = fconn.get_file_blob_external(r.file_id)
+
+        async with transaction() as w_cur:
+            await FileConn(w_cur).log_access(url)
+        
+        return ret
+
 
     async def read_file(self, url: str) -> bytes:
         validate_url(url)

{lfss-0.7.1.dist-info → lfss-0.7.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: lfss
-Version: 0.7.1
+Version: 0.7.2
 Summary: Lightweight file storage service
 Home-page: https://github.com/MenxLi/lfss
 Author: li, mengxun
@@ -21,7 +21,9 @@ Description-Content-Type: text/markdown
 # Lightweight File Storage Service (LFSS)
 [![PyPI](https://img.shields.io/pypi/v/lfss)](https://pypi.org/project/lfss/)
 
-A lightweight file/object storage service!
+My experiment on a lightweight file/object storage service.  
+It stores small files and metadata in sqlite, large files in the filesystem, and serves them through a simple REST API.  
+Tested on 2 million files, and it works fine... thanks to the sqlite database!
 
 Usage: 
 ```sh
@@ -41,7 +43,7 @@ Or, you can start a web server at `/frontend` and open `index.html` in your brow
 
 The API usage is simple, just `GET`, `PUT`, `DELETE` to the `/<username>/file/url` path.  
 Authentication is done via `Authorization` header with the value `Bearer <token>`, or through the `token` query parameter.  
-You can refer to `frontend` as an application example, and `frontend/api.js` or `lfss.client.api.py` for the API usage.
+You can refer to `frontend` as an application example, and `frontend/api.js` or `lfss/client/api.py` for the API usage.
 
 By default, the service exposes all files to the public for `GET` requests, 
 but file-listing is restricted to the user's own files.  

{lfss-0.7.1.dist-info → lfss-0.7.2.dist-info}/RECORD

@@ -1,4 +1,4 @@
-Readme.md,sha256=8ZAADV1K20gEDhwiL-Qq_rBAePlwWQn8c5uJ_X7OCIg,1128
+Readme.md,sha256=t5dninN6gFzve_NySxqGrEkmAYhF1h1RhaMdCOeCILA,1348
 docs/Known_issues.md,sha256=rfdG3j1OJF-59S9E06VPyn0nZKbW-ybPxkoZ7MEZWp8,81
 docs/Permission.md,sha256=EY1Y4tT5PDdHDb2pXsVgMAJXxkUihTZfPT0_z7Q53FA,1485
 frontend/api.js,sha256=-ouhsmucEunAK3m1H__MqffQkXAjoeVEfM15BvqfIZs,7677
@@ -8,7 +8,7 @@ frontend/popup.js,sha256=3PgaGZmxSdV1E-D_MWgcR7aHWkcsHA1BNKSOkmP66tA,5191
 frontend/scripts.js,sha256=hQ8m3L7P-LplLqrPUWD6pBo4C_tCUl2XZKRNtkWBy8I,21155
 frontend/styles.css,sha256=wly8O-zF4EUgV12Tv1bATSfmJsLITv2u3_SiyXVaxv4,4096
 frontend/utils.js,sha256=Ts4nlef8pkrEgpwX-uQwAhWvwxlIzex8ijDLNCa22ps,2372
-lfss/cli/balance.py,sha256=vhTlZoBF97dHBf5IittHpybKswaA-xXOfiPAdvIHD5Q,4794
+lfss/cli/balance.py,sha256=heOgwH6oNnfYsKJfA4VxWKdEXPstdVbbRXWxcDqLIS0,4176
 lfss/cli/cli.py,sha256=bJOeEyri_XVWUvnjohsw_oPYKp-bELxLrg5sVWOpKQA,2259
 lfss/cli/panel.py,sha256=iGdVmdWYjA_7a78ZzWEB_3ggIOBeUKTzg6F5zLaB25c,1401
 lfss/cli/serve.py,sha256=bO3GT0kuylMGN-7bZWP4e71MlugGZ_lEMkYaYld_Ntg,985
@@ -18,16 +18,16 @@ lfss/client/api.py,sha256=ICqpcyvSf-9QmYNv9EQ5fA_MViSuLxSNn-CIBNqWkW8,5414
 lfss/sql/init.sql,sha256=C-JtQAlaOjESI8uoF1Y_9dKukEVSw5Ll-7yA3gG-XHU,1210
 lfss/sql/pragma.sql,sha256=uENx7xXjARmro-A3XAK8OM8v5AxDMdCCRj47f86UuXg,206
 lfss/src/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-lfss/src/config.py,sha256=0TjCCrqkoiLw45vOT0PbDciWbzpD7ztWtubwIwXCsPk,527
-lfss/src/connection_pool.py,sha256=Y47BJ1gsyusmmVA1PHoFBhytup8s3HbU9D_Aw0y4-TM,4848
-lfss/src/database.py,sha256=l0Z-ixfwAkYHbSgk9cVceOJ2vzkYUJeVw25Ny7HQxAA,31104
+lfss/src/config.py,sha256=G-tOdk6RrE62_d-QoD6WXD4l70YriBazUFdumTz3P20,529
+lfss/src/connection_pool.py,sha256=qBtXr2j_O7vGI2Mv0HVsza35T8nljIx85jLlyEv1SgE,4918
+lfss/src/database.py,sha256=tGcHfxThmRTmtLIlEWdJtAe1tzP64wHSC1JcbQdRU-8,31226
 lfss/src/datatype.py,sha256=BLS7vuuKnFZQg0nrKeP9SymqUhcN6HwPgejU0yBd_Ak,1622
 lfss/src/error.py,sha256=imbhwnbhnI3HLhkbfICROe3F0gleKrOk4XnqHJDOtuI,285
 lfss/src/log.py,sha256=qNE04sHoZ2rMbQ5dR2zT7Xaz1KVAfYp5hKpWJX42S9g,5244
 lfss/src/server.py,sha256=9mnBlcHJruwOV-aoHa1P146F4khpVDmif7rq58yZX3U,15306
 lfss/src/stat.py,sha256=hTMtQyM_Ukmhc33Bb9FGCfBMIX02KrGHQg8nL7sC8sU,2082
 lfss/src/utils.py,sha256=8VkrtpSmurbMiX7GyK-n7Grvzy3uwSJXHdONEsuLCDI,2272
-lfss-0.7.1.dist-info/METADATA,sha256=nNjDG5BqDBRpeYP3NNPEdM02wWeIIJNO1VDdKNrgqQ0,1820
-lfss-0.7.1.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-lfss-0.7.1.dist-info/entry_points.txt,sha256=d_Ri3GXxUW-S0E6q953A8od0YMmUAnZGlJSKS46OiW8,172
-lfss-0.7.1.dist-info/RECORD,,
+lfss-0.7.2.dist-info/METADATA,sha256=pJcPBmMpKwNujl4CWZKRchT3uXyCPaEvi1mek7SUJUE,2040
+lfss-0.7.2.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+lfss-0.7.2.dist-info/entry_points.txt,sha256=d_Ri3GXxUW-S0E6q953A8od0YMmUAnZGlJSKS46OiW8,172
+lfss-0.7.2.dist-info/RECORD,,