lfss 0.6.0__tar.gz → 0.7.0__tar.gz

{lfss-0.6.0 → lfss-0.7.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: lfss
-Version: 0.6.0
+Version: 0.7.0
 Summary: Lightweight file storage service
 Home-page: https://github.com/MenxLi/lfss
 Author: li, mengxun

{lfss-0.6.0 → lfss-0.7.0}/lfss/cli/balance.py

@@ -24,10 +24,10 @@ def barriered(func):
 @barriered
 async def move_to_external(f_id: str, flag: str = ''):
     async with aiosqlite.connect(db_file, timeout = 60) as c:
-        async with c.execute( "SELECT data FROM fdata WHERE file_id = ?", (f_id,)) as cursor:
+        async with c.execute( "SELECT data FROM blobs.fdata WHERE file_id = ?", (f_id,)) as cursor:
             blob_row = await cursor.fetchone()
             if blob_row is None:
-                print(f"{flag}File {f_id} not found in fdata")
+                print(f"{flag}File {f_id} not found in blobs.fdata")
                 return
         await c.execute("BEGIN")
         blob: bytes = blob_row[0]
@@ -35,7 +35,7 @@ async def move_to_external(f_id: str, flag: str = ''):
             async with aiofiles.open(LARGE_BLOB_DIR / f_id, 'wb') as f:
                 await f.write(blob)
             await c.execute( "UPDATE fmeta SET external = 1 WHERE file_id = ?", (f_id,))
-            await c.execute( "DELETE FROM fdata WHERE file_id = ?", (f_id,))
+            await c.execute( "DELETE FROM blobs.fdata WHERE file_id = ?", (f_id,))
             await c.commit()
             print(f"{flag}Moved {f_id} to external storage")
         except Exception as e:
@@ -56,7 +56,7 @@ async def move_to_internal(f_id: str, flag: str = ''):
 
         await c.execute("BEGIN")
         try:
-            await c.execute("INSERT INTO fdata (file_id, data) VALUES (?, ?)", (f_id, blob))
+            await c.execute("INSERT INTO blobs.fdata (file_id, data) VALUES (?, ?)", (f_id, blob))
             await c.execute("UPDATE fmeta SET external = 0 WHERE file_id = ?", (f_id,))
             await c.commit()
             (LARGE_BLOB_DIR / f_id).unlink(missing_ok=True)

{lfss-0.6.0 → lfss-0.7.0}/lfss/cli/cli.py

@@ -14,7 +14,7 @@ def parse_arguments():
     sp_upload.add_argument("dst", help="Destination path", type=str)
     sp_upload.add_argument("-j", "--jobs", type=int, default=1, help="Number of concurrent uploads")
     sp_upload.add_argument("--interval", type=float, default=0, help="Interval between files, only works with directory upload")
-    sp_upload.add_argument("--conflict", choices=["overwrite", "abort", "skip"], default="abort", help="Conflict resolution")
+    sp_upload.add_argument("--conflict", choices=["overwrite", "abort", "skip", "skip-ahead"], default="abort", help="Conflict resolution")
     sp_upload.add_argument("--permission", type=FileReadPermission, default=FileReadPermission.UNSET, help="File permission")
     sp_upload.add_argument("--retries", type=int, default=0, help="Number of retries, only works with directory upload")
 

{lfss-0.6.0 → lfss-0.7.0}/lfss/client/api.py

@@ -34,8 +34,18 @@ class Connector:
             return response
         return f
 
-    def put(self, path: str, file_data: bytes, permission: int | FileReadPermission = 0, conflict: Literal['overwrite', 'abort', 'skip'] = 'abort'):
+    def put(self, path: str, file_data: bytes, permission: int | FileReadPermission = 0, conflict: Literal['overwrite', 'abort', 'skip', 'skip-ahead'] = 'abort'):
         """Uploads a file to the specified path."""
+        assert isinstance(file_data, bytes), "file_data must be bytes"
+
+        # Skip ahead by checking if the file already exists
+        if conflict == 'skip-ahead':
+            exists = self.get_metadata(path)
+            if exists is None:
+                conflict = 'skip'
+            else:
+                return {'status': 'skipped', 'path': path}
+
         response = self._fetch('PUT', path, search_params={
             'permission': int(permission),
             'conflict': conflict
@@ -45,9 +55,19 @@ class Connector:
         )
         return response.json()
     
-    def put_json(self, path: str, data: dict, permission: int | FileReadPermission = 0, conflict: Literal['overwrite', 'abort', 'skip'] = 'abort'):
+    def put_json(self, path: str, data: dict, permission: int | FileReadPermission = 0, conflict: Literal['overwrite', 'abort', 'skip', 'skip-ahead'] = 'abort'):
         """Uploads a JSON file to the specified path."""
         assert path.endswith('.json'), "Path must end with .json"
+        assert isinstance(data, dict), "data must be a dict"
+
+        # Skip ahead by checking if the file already exists
+        if conflict == 'skip-ahead':
+            exists = self.get_metadata(path)
+            if exists is None:
+                conflict = 'skip'
+            else:
+                return {'status': 'skipped', 'path': path}
+
         response = self._fetch('PUT', path, search_params={
             'permission': int(permission),
             'conflict': conflict

{lfss-0.6.0 → lfss-0.7.0}/lfss/sql/init.sql

@@ -1,3 +1,9 @@
+PRAGMA journal_mode=MEMROY;
+PRAGMA temp_store=MEMORY;
+PRAGMA page_size=4096;
+PRAGMA synchronous=NORMAL;
+PRAGMA case_sensitive_like=ON;
+
 CREATE TABLE IF NOT EXISTS user (
     id INTEGER PRIMARY KEY AUTOINCREMENT,
     username VARCHAR(256) UNIQUE NOT NULL,
@@ -22,11 +28,6 @@ CREATE TABLE IF NOT EXISTS fmeta (
     FOREIGN KEY(owner_id) REFERENCES user(id)
 );
 
-CREATE TABLE IF NOT EXISTS fdata (
-    file_id CHAR(32) PRIMARY KEY,
-    data BLOB
-);
-
 CREATE TABLE IF NOT EXISTS usize (
     user_id INTEGER PRIMARY KEY,
     size INTEGER DEFAULT 0
@@ -37,3 +38,8 @@ CREATE INDEX IF NOT EXISTS idx_fmeta_url ON fmeta(url);
 CREATE INDEX IF NOT EXISTS idx_user_username ON user(username);
 
 CREATE INDEX IF NOT EXISTS idx_user_credential ON user(credential);
+
+CREATE TABLE IF NOT EXISTS blobs.fdata (
+    file_id CHAR(32) PRIMARY KEY,
+    data BLOB
+);

lfss-0.7.0/lfss/sql/pragma.sql

@@ -0,0 +1,8 @@
+PRAGMA journal_mode=DELETE;
+PRAGMA synchronous=NORMAL;
+PRAGMA case_sensitive_like=ON;
+PRAGMA page_size=4096;
+
+PRAGMA blobs.journal_mode=DELETE;
+PRAGMA blobs.synchronous=NORMAL;
+PRAGMA blobs.page_size=16384;

{lfss-0.6.0 → lfss-0.7.0}/lfss/src/config.py

@@ -7,6 +7,7 @@ DATA_HOME = Path(os.environ.get('LFSS_DATA', __default_dir))
 if not DATA_HOME.exists():
     DATA_HOME.mkdir()
     print(f"[init] Created data home at {DATA_HOME}")
+DATA_HOME = DATA_HOME.resolve().absolute()
 LARGE_BLOB_DIR = DATA_HOME / 'large_blobs'
 LARGE_BLOB_DIR.mkdir(exist_ok=True)
 

{lfss-0.6.0 → lfss-0.7.0}/lfss/src/database.py

@@ -1,6 +1,7 @@
 
 from typing import Optional, overload, Literal, AsyncIterable
 from abc import ABC, abstractmethod
+import os
 
 import urllib.parse
 from pathlib import Path
@@ -57,7 +58,12 @@ class DBConnBase(ABC):
         """Should return self"""
         global _g_conn
         if _g_conn is None:
-            _g_conn = await aiosqlite.connect(DATA_HOME / 'lfss.db')
+            if not os.environ.get('SQLITE_TEMPDIR'):
+                os.environ['SQLITE_TEMPDIR'] = str(DATA_HOME)
+            # large blobs are stored in a separate database, should be more efficient
+            _g_conn = await aiosqlite.connect(DATA_HOME / 'index.db')
+            async with _g_conn.cursor() as c:
+                await c.execute(f"ATTACH DATABASE ? AS blobs", (str(DATA_HOME/'blobs.db'), ))
             await execute_sql(_g_conn, 'pragma.sql')
             await execute_sql(_g_conn, 'init.sql')
 
@@ -162,60 +168,6 @@ class FileConn(DBConnBase):
     
     async def init(self):
         await super().init()
-        # backward compatibility, since 0.2.1
-        async with self.conn.execute("SELECT * FROM user") as cursor:
-            res = await cursor.fetchall()
-        for r in res:
-            async with self.conn.execute("SELECT user_id FROM usize WHERE user_id = ?", (r[0], )) as cursor:
-                size = await cursor.fetchone()
-            if size is None:
-                async with self.conn.execute("SELECT SUM(file_size) FROM fmeta WHERE owner_id = ?", (r[0], )) as cursor:
-                    size = await cursor.fetchone()
-                if size is not None and size[0] is not None:
-                    await self._user_size_inc(r[0], size[0])
-        
-        # backward compatibility, since 0.5.0
-        # 'external' means the file is not stored in the database, but in the external storage
-        async with self.conn.execute("SELECT * FROM fmeta") as cursor:
-            res = await cursor.fetchone()
-        if res and len(res) < 8:
-            self.logger.info("Updating fmeta table")
-            await self.conn.execute('''
-            ALTER TABLE fmeta ADD COLUMN external BOOLEAN DEFAULT FALSE
-            ''')
-
-        # backward compatibility, since 0.6.0
-        async with self.conn.execute("SELECT * FROM fmeta") as cursor:
-            res = await cursor.fetchone()
-        if res and len(res) < 9:
-            self.logger.info("Updating fmeta table")
-            await self.conn.execute('''
-            ALTER TABLE fmeta ADD COLUMN mime_type TEXT DEFAULT 'application/octet-stream'
-            ''')
-            # check all mime types
-            import mimetypes, mimesniff
-            async with self.conn.execute("SELECT url, file_id, external FROM fmeta") as cursor:
-                res = await cursor.fetchall()
-            async with self.conn.execute("SELECT count(*) FROM fmeta") as cursor:
-                count = await cursor.fetchone()
-                assert count is not None
-            for counter, r in enumerate(res, start=1):
-                print(f"Checking mimetype for {counter}/{count[0]}")
-                url, f_id, external = r
-                fname = url.split('/')[-1]
-                mime_type, _ = mimetypes.guess_type(fname)
-                if mime_type is None:
-                    # try to sniff the file
-                    if not external:
-                        async with self.conn.execute("SELECT data FROM fdata WHERE file_id = ?", (f_id, )) as cursor:
-                            blob = await cursor.fetchone()
-                        assert blob is not None
-                        blob = blob[0]
-                        mime_type = mimesniff.what(blob)
-                    else:
-                        mime_type = mimesniff.what(LARGE_BLOB_DIR / f_id)
-                await self.conn.execute("UPDATE fmeta SET mime_type = ? WHERE url = ?", (mime_type, url))
-
         return self
     
     async def get_file_record(self, url: str) -> Optional[FileRecord]:
@@ -459,7 +411,7 @@ class FileConn(DBConnBase):
     
     @atomic
     async def set_file_blob(self, file_id: str, blob: bytes):
-        await self.conn.execute("INSERT OR REPLACE INTO fdata (file_id, data) VALUES (?, ?)", (file_id, blob))
+        await self.conn.execute("INSERT OR REPLACE INTO blobs.fdata (file_id, data) VALUES (?, ?)", (file_id, blob))
     
     @atomic
     async def set_file_blob_external(self, file_id: str, stream: AsyncIterable[bytes])->int:
@@ -476,7 +428,7 @@ class FileConn(DBConnBase):
         return size_sum
     
     async def get_file_blob(self, file_id: str) -> Optional[bytes]:
-        async with self.conn.execute("SELECT data FROM fdata WHERE file_id = ?", (file_id, )) as cursor:
+        async with self.conn.execute("SELECT data FROM blobs.fdata WHERE file_id = ?", (file_id, )) as cursor:
             res = await cursor.fetchone()
         if res is None:
             return None
@@ -495,11 +447,11 @@ class FileConn(DBConnBase):
     
     @atomic
     async def delete_file_blob(self, file_id: str):
-        await self.conn.execute("DELETE FROM fdata WHERE file_id = ?", (file_id, ))
+        await self.conn.execute("DELETE FROM blobs.fdata WHERE file_id = ?", (file_id, ))
     
     @atomic
     async def delete_file_blobs(self, file_ids: list[str]):
-        await self.conn.execute("DELETE FROM fdata WHERE file_id IN ({})".format(','.join(['?'] * len(file_ids))), file_ids)
+        await self.conn.execute("DELETE FROM blobs.fdata WHERE file_id IN ({})".format(','.join(['?'] * len(file_ids))), file_ids)
 
 def validate_url(url: str, is_file = True):
     prohibited_chars = ['..', ';', "'", '"', '\\', '\0', '\n', '\r', '\t', '\x0b', '\x0c']

{lfss-0.6.0 → lfss-0.7.0}/lfss/src/datatype.py

@@ -25,7 +25,7 @@ class UserRecord:
 class FileRecord:
     url: str
     owner_id: int
-    file_id: str      # defines mapping from fmata to fdata
+    file_id: str      # defines mapping from fmata to blobs.fdata
     file_size: int
     create_time: str
     access_time: str

{lfss-0.6.0 → lfss-0.7.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "lfss"
-version = "0.6.0"
+version = "0.7.0"
 description = "Lightweight file storage service"
 authors = ["li, mengxun <limengxun45@outlook.com>"]
 readme = "Readme.md"

lfss-0.6.0/lfss/sql/pragma.sql

@@ -1,5 +0,0 @@
-PRAGMA journal_mode=MEMROY;
-PRAGMA temp_store=MEMORY;
-PRAGMA page_size=8192;
-PRAGMA synchronous=NORMAL;
-PRAGMA case_sensitive_like=ON;