lfss 0.6.0__py3-none-any.whl → 0.7.0__py3-none-any.whl

lfss/cli/balance.py

@@ -24,10 +24,10 @@ def barriered(func):
 @barriered
 async def move_to_external(f_id: str, flag: str = ''):
     async with aiosqlite.connect(db_file, timeout = 60) as c:
-        async with c.execute( "SELECT data FROM fdata WHERE file_id = ?", (f_id,)) as cursor:
+        async with c.execute( "SELECT data FROM blobs.fdata WHERE file_id = ?", (f_id,)) as cursor:
             blob_row = await cursor.fetchone()
             if blob_row is None:
-                print(f"{flag}File {f_id} not found in fdata")
+                print(f"{flag}File {f_id} not found in blobs.fdata")
                 return
         await c.execute("BEGIN")
         blob: bytes = blob_row[0]
@@ -35,7 +35,7 @@ async def move_to_external(f_id: str, flag: str = ''):
             async with aiofiles.open(LARGE_BLOB_DIR / f_id, 'wb') as f:
                 await f.write(blob)
             await c.execute( "UPDATE fmeta SET external = 1 WHERE file_id = ?", (f_id,))
-            await c.execute( "DELETE FROM fdata WHERE file_id = ?", (f_id,))
+            await c.execute( "DELETE FROM blobs.fdata WHERE file_id = ?", (f_id,))
             await c.commit()
             print(f"{flag}Moved {f_id} to external storage")
         except Exception as e:
@@ -56,7 +56,7 @@ async def move_to_internal(f_id: str, flag: str = ''):
 
         await c.execute("BEGIN")
         try:
-            await c.execute("INSERT INTO fdata (file_id, data) VALUES (?, ?)", (f_id, blob))
+            await c.execute("INSERT INTO blobs.fdata (file_id, data) VALUES (?, ?)", (f_id, blob))
             await c.execute("UPDATE fmeta SET external = 0 WHERE file_id = ?", (f_id,))
             await c.commit()
             (LARGE_BLOB_DIR / f_id).unlink(missing_ok=True)

lfss/cli/cli.py

@@ -14,7 +14,7 @@ def parse_arguments():
     sp_upload.add_argument("dst", help="Destination path", type=str)
     sp_upload.add_argument("-j", "--jobs", type=int, default=1, help="Number of concurrent uploads")
     sp_upload.add_argument("--interval", type=float, default=0, help="Interval between files, only works with directory upload")
-    sp_upload.add_argument("--conflict", choices=["overwrite", "abort", "skip"], default="abort", help="Conflict resolution")
+    sp_upload.add_argument("--conflict", choices=["overwrite", "abort", "skip", "skip-ahead"], default="abort", help="Conflict resolution")
     sp_upload.add_argument("--permission", type=FileReadPermission, default=FileReadPermission.UNSET, help="File permission")
     sp_upload.add_argument("--retries", type=int, default=0, help="Number of retries, only works with directory upload")
 

lfss/client/api.py

@@ -34,8 +34,18 @@ class Connector:
             return response
         return f
 
-    def put(self, path: str, file_data: bytes, permission: int | FileReadPermission = 0, conflict: Literal['overwrite', 'abort', 'skip'] = 'abort'):
+    def put(self, path: str, file_data: bytes, permission: int | FileReadPermission = 0, conflict: Literal['overwrite', 'abort', 'skip', 'skip-ahead'] = 'abort'):
         """Uploads a file to the specified path."""
+        assert isinstance(file_data, bytes), "file_data must be bytes"
+
+        # Skip ahead by checking if the file already exists
+        if conflict == 'skip-ahead':
+            exists = self.get_metadata(path)
+            if exists is None:
+                conflict = 'skip'
+            else:
+                return {'status': 'skipped', 'path': path}
+
         response = self._fetch('PUT', path, search_params={
             'permission': int(permission),
             'conflict': conflict
@@ -45,9 +55,19 @@ class Connector:
         )
         return response.json()
     
-    def put_json(self, path: str, data: dict, permission: int | FileReadPermission = 0, conflict: Literal['overwrite', 'abort', 'skip'] = 'abort'):
+    def put_json(self, path: str, data: dict, permission: int | FileReadPermission = 0, conflict: Literal['overwrite', 'abort', 'skip', 'skip-ahead'] = 'abort'):
         """Uploads a JSON file to the specified path."""
         assert path.endswith('.json'), "Path must end with .json"
+        assert isinstance(data, dict), "data must be a dict"
+
+        # Skip ahead by checking if the file already exists
+        if conflict == 'skip-ahead':
+            exists = self.get_metadata(path)
+            if exists is None:
+                conflict = 'skip'
+            else:
+                return {'status': 'skipped', 'path': path}
+
         response = self._fetch('PUT', path, search_params={
             'permission': int(permission),
             'conflict': conflict

lfss/sql/init.sql

@@ -1,3 +1,9 @@
+PRAGMA journal_mode=MEMROY;
+PRAGMA temp_store=MEMORY;
+PRAGMA page_size=4096;
+PRAGMA synchronous=NORMAL;
+PRAGMA case_sensitive_like=ON;
+
 CREATE TABLE IF NOT EXISTS user (
     id INTEGER PRIMARY KEY AUTOINCREMENT,
     username VARCHAR(256) UNIQUE NOT NULL,
@@ -22,11 +28,6 @@ CREATE TABLE IF NOT EXISTS fmeta (
     FOREIGN KEY(owner_id) REFERENCES user(id)
 );
 
-CREATE TABLE IF NOT EXISTS fdata (
-    file_id CHAR(32) PRIMARY KEY,
-    data BLOB
-);
-
 CREATE TABLE IF NOT EXISTS usize (
     user_id INTEGER PRIMARY KEY,
     size INTEGER DEFAULT 0
@@ -37,3 +38,8 @@ CREATE INDEX IF NOT EXISTS idx_fmeta_url ON fmeta(url);
 CREATE INDEX IF NOT EXISTS idx_user_username ON user(username);
 
 CREATE INDEX IF NOT EXISTS idx_user_credential ON user(credential);
+
+CREATE TABLE IF NOT EXISTS blobs.fdata (
+    file_id CHAR(32) PRIMARY KEY,
+    data BLOB
+);

lfss/sql/pragma.sql

@@ -1,5 +1,8 @@
-PRAGMA journal_mode=MEMROY;
-PRAGMA temp_store=MEMORY;
-PRAGMA page_size=8192;
+PRAGMA journal_mode=DELETE;
 PRAGMA synchronous=NORMAL;
-PRAGMA case_sensitive_like=ON;
+PRAGMA case_sensitive_like=ON;
+PRAGMA page_size=4096;
+
+PRAGMA blobs.journal_mode=DELETE;
+PRAGMA blobs.synchronous=NORMAL;
+PRAGMA blobs.page_size=16384;

lfss/src/config.py

@@ -7,6 +7,7 @@ DATA_HOME = Path(os.environ.get('LFSS_DATA', __default_dir))
 if not DATA_HOME.exists():
     DATA_HOME.mkdir()
     print(f"[init] Created data home at {DATA_HOME}")
+DATA_HOME = DATA_HOME.resolve().absolute()
 LARGE_BLOB_DIR = DATA_HOME / 'large_blobs'
 LARGE_BLOB_DIR.mkdir(exist_ok=True)
 

lfss/src/database.py

@@ -1,6 +1,7 @@
 
 from typing import Optional, overload, Literal, AsyncIterable
 from abc import ABC, abstractmethod
+import os
 
 import urllib.parse
 from pathlib import Path
@@ -57,7 +58,12 @@ class DBConnBase(ABC):
         """Should return self"""
         global _g_conn
         if _g_conn is None:
-            _g_conn = await aiosqlite.connect(DATA_HOME / 'lfss.db')
+            if not os.environ.get('SQLITE_TEMPDIR'):
+                os.environ['SQLITE_TEMPDIR'] = str(DATA_HOME)
+            # large blobs are stored in a separate database, should be more efficient
+            _g_conn = await aiosqlite.connect(DATA_HOME / 'index.db')
+            async with _g_conn.cursor() as c:
+                await c.execute(f"ATTACH DATABASE ? AS blobs", (str(DATA_HOME/'blobs.db'), ))
             await execute_sql(_g_conn, 'pragma.sql')
             await execute_sql(_g_conn, 'init.sql')
 
@@ -162,60 +168,6 @@ class FileConn(DBConnBase):
     
     async def init(self):
         await super().init()
-        # backward compatibility, since 0.2.1
-        async with self.conn.execute("SELECT * FROM user") as cursor:
-            res = await cursor.fetchall()
-        for r in res:
-            async with self.conn.execute("SELECT user_id FROM usize WHERE user_id = ?", (r[0], )) as cursor:
-                size = await cursor.fetchone()
-            if size is None:
-                async with self.conn.execute("SELECT SUM(file_size) FROM fmeta WHERE owner_id = ?", (r[0], )) as cursor:
-                    size = await cursor.fetchone()
-                if size is not None and size[0] is not None:
-                    await self._user_size_inc(r[0], size[0])
-        
-        # backward compatibility, since 0.5.0
-        # 'external' means the file is not stored in the database, but in the external storage
-        async with self.conn.execute("SELECT * FROM fmeta") as cursor:
-            res = await cursor.fetchone()
-        if res and len(res) < 8:
-            self.logger.info("Updating fmeta table")
-            await self.conn.execute('''
-            ALTER TABLE fmeta ADD COLUMN external BOOLEAN DEFAULT FALSE
-            ''')
-
-        # backward compatibility, since 0.6.0
-        async with self.conn.execute("SELECT * FROM fmeta") as cursor:
-            res = await cursor.fetchone()
-        if res and len(res) < 9:
-            self.logger.info("Updating fmeta table")
-            await self.conn.execute('''
-            ALTER TABLE fmeta ADD COLUMN mime_type TEXT DEFAULT 'application/octet-stream'
-            ''')
-            # check all mime types
-            import mimetypes, mimesniff
-            async with self.conn.execute("SELECT url, file_id, external FROM fmeta") as cursor:
-                res = await cursor.fetchall()
-            async with self.conn.execute("SELECT count(*) FROM fmeta") as cursor:
-                count = await cursor.fetchone()
-                assert count is not None
-            for counter, r in enumerate(res, start=1):
-                print(f"Checking mimetype for {counter}/{count[0]}")
-                url, f_id, external = r
-                fname = url.split('/')[-1]
-                mime_type, _ = mimetypes.guess_type(fname)
-                if mime_type is None:
-                    # try to sniff the file
-                    if not external:
-                        async with self.conn.execute("SELECT data FROM fdata WHERE file_id = ?", (f_id, )) as cursor:
-                            blob = await cursor.fetchone()
-                        assert blob is not None
-                        blob = blob[0]
-                        mime_type = mimesniff.what(blob)
-                    else:
-                        mime_type = mimesniff.what(LARGE_BLOB_DIR / f_id)
-                await self.conn.execute("UPDATE fmeta SET mime_type = ? WHERE url = ?", (mime_type, url))
-
         return self
     
     async def get_file_record(self, url: str) -> Optional[FileRecord]:
@@ -459,7 +411,7 @@ class FileConn(DBConnBase):
     
     @atomic
     async def set_file_blob(self, file_id: str, blob: bytes):
-        await self.conn.execute("INSERT OR REPLACE INTO fdata (file_id, data) VALUES (?, ?)", (file_id, blob))
+        await self.conn.execute("INSERT OR REPLACE INTO blobs.fdata (file_id, data) VALUES (?, ?)", (file_id, blob))
     
     @atomic
     async def set_file_blob_external(self, file_id: str, stream: AsyncIterable[bytes])->int:
@@ -476,7 +428,7 @@ class FileConn(DBConnBase):
         return size_sum
     
     async def get_file_blob(self, file_id: str) -> Optional[bytes]:
-        async with self.conn.execute("SELECT data FROM fdata WHERE file_id = ?", (file_id, )) as cursor:
+        async with self.conn.execute("SELECT data FROM blobs.fdata WHERE file_id = ?", (file_id, )) as cursor:
             res = await cursor.fetchone()
         if res is None:
             return None
@@ -495,11 +447,11 @@ class FileConn(DBConnBase):
     
     @atomic
     async def delete_file_blob(self, file_id: str):
-        await self.conn.execute("DELETE FROM fdata WHERE file_id = ?", (file_id, ))
+        await self.conn.execute("DELETE FROM blobs.fdata WHERE file_id = ?", (file_id, ))
     
     @atomic
     async def delete_file_blobs(self, file_ids: list[str]):
-        await self.conn.execute("DELETE FROM fdata WHERE file_id IN ({})".format(','.join(['?'] * len(file_ids))), file_ids)
+        await self.conn.execute("DELETE FROM blobs.fdata WHERE file_id IN ({})".format(','.join(['?'] * len(file_ids))), file_ids)
 
 def validate_url(url: str, is_file = True):
     prohibited_chars = ['..', ';', "'", '"', '\\', '\0', '\n', '\r', '\t', '\x0b', '\x0c']

lfss/src/datatype.py

@@ -25,7 +25,7 @@ class UserRecord:
 class FileRecord:
     url: str
     owner_id: int
-    file_id: str      # defines mapping from fmata to fdata
+    file_id: str      # defines mapping from fmata to blobs.fdata
     file_size: int
     create_time: str
     access_time: str

{lfss-0.6.0.dist-info → lfss-0.7.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: lfss
-Version: 0.6.0
+Version: 0.7.0
 Summary: Lightweight file storage service
 Home-page: https://github.com/MenxLi/lfss
 Author: li, mengxun

{lfss-0.6.0.dist-info → lfss-0.7.0.dist-info}/RECORD

@@ -8,25 +8,25 @@ frontend/popup.js,sha256=3PgaGZmxSdV1E-D_MWgcR7aHWkcsHA1BNKSOkmP66tA,5191
 frontend/scripts.js,sha256=JkjcyT-IpzSypwI4oWwgY9UDdKkR1ZSYdSc4c6MlukE,21128
 frontend/styles.css,sha256=wly8O-zF4EUgV12Tv1bATSfmJsLITv2u3_SiyXVaxv4,4096
 frontend/utils.js,sha256=Ts4nlef8pkrEgpwX-uQwAhWvwxlIzex8ijDLNCa22ps,2372
-lfss/cli/balance.py,sha256=5kyjfknN7oNiyEs2DcDYzTMKzoOTDm0caF2nZyxJcHw,4707
-lfss/cli/cli.py,sha256=-BcLIH-6jCJxZfg48nA3gulnitO59Hnba3HoKqyKeUA,2245
+lfss/cli/balance.py,sha256=P9YidInC8QH-jwvMyL9C37SnCLOI9806628zxJW9ChY,4731
+lfss/cli/cli.py,sha256=bJOeEyri_XVWUvnjohsw_oPYKp-bELxLrg5sVWOpKQA,2259
 lfss/cli/panel.py,sha256=iGdVmdWYjA_7a78ZzWEB_3ggIOBeUKTzg6F5zLaB25c,1401
 lfss/cli/serve.py,sha256=bO3GT0kuylMGN-7bZWP4e71MlugGZ_lEMkYaYld_Ntg,985
 lfss/cli/user.py,sha256=uSNgF8wGYpdOowN8Mah3V_ii6NlxRMNecrrVj3HaemU,3328
 lfss/client/__init__.py,sha256=YttaGTBup7mxnW0CtxdZPF0HWga2cGM4twz9MXJIrts,1827
-lfss/client/api.py,sha256=KKDESF1rwjcPr_D87685tCfePxy9IUeG9KkMRPBCuds,4682
-lfss/sql/init.sql,sha256=oYcH114CpmAlIW_CF2wokSdk2yxtfAkSBybMqOpHlpA,1205
-lfss/sql/pragma.sql,sha256=krTf0ALmU_4s2hGM4PSYwmSCqjRXcaOxChmgMEhtJqI,134
+lfss/client/api.py,sha256=ICqpcyvSf-9QmYNv9EQ5fA_MViSuLxSNn-CIBNqWkW8,5414
+lfss/sql/init.sql,sha256=uHgcV03KehKOxhXWI7LfnxJL4PQCVmbwn3KHaCurs80,1346
+lfss/sql/pragma.sql,sha256=uENx7xXjARmro-A3XAK8OM8v5AxDMdCCRj47f86UuXg,206
 lfss/src/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-lfss/src/config.py,sha256=SM6WVmpkf7e-YkV3RELTJltOnQGRtXkVtA1rVHwsg0g,484
-lfss/src/database.py,sha256=vLpMUPjoY9aysqxy8IBXcvPdmfo5_xvYoiTlIa_HdKY,33527
-lfss/src/datatype.py,sha256=cqUC8sN-XWIzlsD1m9YLj6ERu-z0jzxTZV4GSE_3DKA,1616
+lfss/src/config.py,sha256=0TjCCrqkoiLw45vOT0PbDciWbzpD7ztWtubwIwXCsPk,527
+lfss/src/database.py,sha256=BJJcORkIzSccdT7yNegAozhpUlq1em7BQ3p8wG4h0fI,31071
+lfss/src/datatype.py,sha256=BLS7vuuKnFZQg0nrKeP9SymqUhcN6HwPgejU0yBd_Ak,1622
 lfss/src/error.py,sha256=imbhwnbhnI3HLhkbfICROe3F0gleKrOk4XnqHJDOtuI,285
 lfss/src/log.py,sha256=qNE04sHoZ2rMbQ5dR2zT7Xaz1KVAfYp5hKpWJX42S9g,5244
 lfss/src/server.py,sha256=J8A9hUnqf-AzCF-Q2Ve9iMPkJYKZ81OP26dFLRm89BM,16219
 lfss/src/stat.py,sha256=hTMtQyM_Ukmhc33Bb9FGCfBMIX02KrGHQg8nL7sC8sU,2082
 lfss/src/utils.py,sha256=8VkrtpSmurbMiX7GyK-n7Grvzy3uwSJXHdONEsuLCDI,2272
-lfss-0.6.0.dist-info/METADATA,sha256=dJoh4nlz5gCY-7hVkSIGiiadoqPJLUSKjHA2fD3C8b4,1820
-lfss-0.6.0.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-lfss-0.6.0.dist-info/entry_points.txt,sha256=d_Ri3GXxUW-S0E6q953A8od0YMmUAnZGlJSKS46OiW8,172
-lfss-0.6.0.dist-info/RECORD,,
+lfss-0.7.0.dist-info/METADATA,sha256=sv5tlm_odl1rk_a9sG_vz2eztFh2Vx5g-NUvtQK1XGY,1820
+lfss-0.7.0.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+lfss-0.7.0.dist-info/entry_points.txt,sha256=d_Ri3GXxUW-S0E6q953A8od0YMmUAnZGlJSKS46OiW8,172
+lfss-0.7.0.dist-info/RECORD,,