lfss 0.5.1__tar.gz → 0.6.0__tar.gz

{lfss-0.5.1 → lfss-0.6.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: lfss
-Version: 0.5.1
+Version: 0.6.0
 Summary: Lightweight file storage service
 Home-page: https://github.com/MenxLi/lfss
 Author: li, mengxun

{lfss-0.5.1 → lfss-0.6.0}/frontend/api.js

@@ -191,7 +191,7 @@ export default class Connector {
      * @param {string} path - file path(url)
      * @param {string} newPath - new file path(url)
      */
-    async moveFile(path, newPath){
+    async move(path, newPath){
         if (path.startsWith('/')){ path = path.slice(1); }
         if (newPath.startsWith('/')){ newPath = newPath.slice(1); }
         const dst = new URL(this.config.endpoint + '/_api/meta');

{lfss-0.5.1 → lfss-0.6.0}/frontend/popup.js

@@ -40,7 +40,7 @@ export function createFloatingWindow(innerHTML = '',  {
     return [floatingWindow, closeWindow];
 }
 
-/* select can be "last-filename" */
+/* select can be "last-filename" or "last-pathname" */
 export function showFloatingWindowLineInput(onSubmit = (v) => {}, {
     text = "",
     placeholder = "Enter text",
@@ -72,6 +72,7 @@ export function showFloatingWindowLineInput(onSubmit = (v) => {}, {
     };
 
     if (select === "last-filename") {
+        // select the last filename, e.g. "file" in "/path/to/file.txt"
         const inputVal = input.value;
         let lastSlash = inputVal.lastIndexOf("/");
         if (lastSlash === -1) {
@@ -84,6 +85,17 @@ export function showFloatingWindowLineInput(onSubmit = (v) => {}, {
         }
         input.setSelectionRange(lastSlash + 1, lastSlash + lastDot + 1);
     }
+    else if (select === "last-pathname") {
+        // select the last pathname, e.g. "to" in "/path/to/<filename>"
+        const lastSlash = input.value.lastIndexOf("/");
+        const secondLastSlash = input.value.lastIndexOf("/", input.value.lastIndexOf("/") - 1);
+        if (secondLastSlash !== -1) {
+            input.setSelectionRange(secondLastSlash + 1, lastSlash);
+        }
+        else {
+            input.setSelectionRange(0, lastSlash);
+        }
+    }
 
     return [floatingWindow, closeWindow];
 }

{lfss-0.5.1 → lfss-0.6.0}/frontend/scripts.js

@@ -5,6 +5,9 @@ import { formatSize, decodePathURI, ensurePathURI, copyToClipboard, getRandomStr
 
 const conn = new Connector();
 let userRecord = null;
+const ensureSlashEnd = (path) => {
+    return path.endsWith('/') ? path : path + '/';
+}
 
 const endpointInput = document.querySelector('input#endpoint');
 const tokenInput = document.querySelector('input#token');
@@ -128,11 +131,13 @@ uploadButton.addEventListener('click', () => {
         throw new Error('File name cannot end with /');
     }
     path = path + fileName;
+    showPopup('Uploading...', {level: 'info', timeout: 3000});
     conn.put(path, file)
         .then(() => {
             refreshFileList();
             uploadFileNameInput.value = '';
             onFileNameInpuChange();
+            showPopup('Upload success.', {level: 'success', timeout: 3000});
         }, 
         (err) => {
             showPopup('Failed to upload file: ' + err, {level: 'error', timeout: 5000});
@@ -191,9 +196,14 @@ Are you sure you want to proceed?
                 const path = dstPath + file.name;
                 promises.push(uploadFile(file, path));
             }
+            showPopup('Uploading multiple files...', {level: 'info', timeout: 3000});
             Promise.all(promises).then(
                 () => {
+                    showPopup('Upload success.', {level: 'success', timeout: 3000});
                     refreshFileList();
+                }, 
+                (err) => {
+                    showPopup('Failed to upload some files: ' + err, {level: 'error', timeout: 5000});
                 }
             );
         }
@@ -260,15 +270,16 @@ function refreshFileList(){
                     tr.appendChild(accessTd);
                 }
                 {
+                    const dirurl = ensureSlashEnd(dir.url);
                     const actTd = document.createElement('td');
                     const actContainer = document.createElement('div');
                     actContainer.classList.add('action-container');
 
                     const showMetaButton = document.createElement('a');
-                    showMetaButton.textContent = 'Details';
+                    showMetaButton.textContent = 'Reveal';
                     showMetaButton.style.cursor = 'pointer';
                     showMetaButton.addEventListener('click', () => {
-                        const dirUrlEncap = dir.url + (dir.url.endsWith('/') ? '' : '/');
+                        const dirUrlEncap = dirurl;
                         conn.getMetadata(dirUrlEncap).then(
                             (meta) => {
                                 sizeTd.textContent = formatSize(meta.size);
@@ -280,6 +291,30 @@ function refreshFileList(){
                     });
                     actContainer.appendChild(showMetaButton);
 
+                    const moveButton = document.createElement('a');
+                    moveButton.textContent = 'Move';
+                    moveButton.style.cursor = 'pointer';
+                    moveButton.addEventListener('click', () => {
+                        showFloatingWindowLineInput((dstPath) => {
+                            dstPath = encodePathURI(dstPath);
+                            console.log("Moving", dirurl, "to", dstPath);
+                            conn.move(dirurl, dstPath)
+                                .then(() => {
+                                    refreshFileList();
+                                }, 
+                                (err) => {
+                                    showPopup('Failed to move path: ' + err, {level: 'error'});
+                                }
+                            );
+                        }, {
+                            text: 'Enter the destination path: ',
+                            placeholder: 'Destination path',
+                            value: decodePathURI(dirurl), 
+                            select: "last-pathname"
+                        });
+                    });
+                    actContainer.appendChild(moveButton);
+
                     const downloadButton = document.createElement('a');
                     downloadButton.textContent = 'Download';
                     downloadButton.href = conn.config.endpoint + '/_api/bundle?' + 
@@ -402,10 +437,7 @@ function refreshFileList(){
                     moveButton.addEventListener('click', () => {
                         showFloatingWindowLineInput((dstPath) => {
                             dstPath = encodePathURI(dstPath);
-                            if (dstPath.endsWith('/')){
-                                dstPath = dstPath.slice(0, -1);
-                            }
-                            conn.moveFile(file.url, dstPath)
+                            conn.move(file.url, dstPath)
                                 .then(() => {
                                     refreshFileList();
                                 }, 

{lfss-0.5.1 → lfss-0.6.0}/frontend/styles.css

@@ -192,7 +192,7 @@ input#file-name.duplicate{
     display: flex;
     flex-direction: row;
     gap: 10px;
-    /* justify-content: flex-end; */
+    justify-content: flex-end;
 }
 a{
     color: #195f8b;

{lfss-0.5.1 → lfss-0.6.0}/lfss/cli/balance.py

@@ -68,44 +68,57 @@ async def move_to_internal(f_id: str, flag: str = ''):
                 raise e
 
 
-async def _main():
+async def _main(batch_size: int = 10000):
 
     tasks = []
     start_time = time.time()
-    async with aiosqlite.connect(db_file) as conn:
-        exceeded_rows = await (await conn.execute( 
-            "SELECT file_id FROM fmeta WHERE file_size > ? AND external = 0",
-            (LARGE_FILE_BYTES,)
-        )).fetchall()
-        
-    for i in range(0, len(exceeded_rows)):
-        row = exceeded_rows[i]
-        f_id = row[0]
-        tasks.append(move_to_external(f_id, flag=f"[e-{i+1}/{len(exceeded_rows)}] "))
 
-    async with aiosqlite.connect(db_file) as conn:
-        under_rows = await (await conn.execute(
-            "SELECT file_id, file_size, external FROM fmeta WHERE file_size <= ? AND external = 1",
-            (LARGE_FILE_BYTES,)
-        )).fetchall()
+    e_cout = 0
+    batch_count = 0
+    while True:
+        async with aiosqlite.connect(db_file) as conn:
+            exceeded_rows = list(await (await conn.execute( 
+                "SELECT file_id FROM fmeta WHERE file_size > ? AND external = 0 LIMIT ? OFFSET ?",
+                (LARGE_FILE_BYTES, batch_size, batch_size * batch_count)
+            )).fetchall())
+            if not exceeded_rows:
+                break
+        e_cout += len(exceeded_rows)
+        for i in range(0, len(exceeded_rows)):
+            row = exceeded_rows[i]
+            f_id = row[0]
+            tasks.append(move_to_external(f_id, flag=f"[b{batch_count+1}-e{i+1}/{len(exceeded_rows)}] "))
+        await asyncio.gather(*tasks)
+
+    i_count = 0
+    batch_count = 0
+    while True:
+        async with aiosqlite.connect(db_file) as conn:
+            under_rows = list(await (await conn.execute(
+                "SELECT file_id, file_size, external FROM fmeta WHERE file_size <= ? AND external = 1 LIMIT ? OFFSET ?",
+                (LARGE_FILE_BYTES, batch_size, batch_size * batch_count)
+            )).fetchall())
+            if not under_rows:
+                break
+        i_count += len(under_rows)
+        for i in range(0, len(under_rows)):
+            row = under_rows[i]
+            f_id = row[0]
+            tasks.append(move_to_internal(f_id, flag=f"[b{batch_count+1}-i{i+1}/{len(under_rows)}] "))
+        await asyncio.gather(*tasks)
 
-    for i in range(0, len(under_rows)):
-        row = under_rows[i]
-        f_id = row[0]
-        tasks.append(move_to_internal(f_id, flag=f"[i-{i+1}/{len(under_rows)}] "))
-        
-    await asyncio.gather(*tasks)
     end_time = time.time()
     print(f"Balancing complete, took {end_time - start_time:.2f} seconds. "
-          f"{len(exceeded_rows)} files moved to external storage, {len(under_rows)} files moved to internal storage.")
+          f"{e_cout} files moved to external storage, {i_count} files moved to internal storage.")
             
 def main():
     global sem
     parser = argparse.ArgumentParser(description="Balance the storage by ensuring that large file thresholds are met.")
     parser.add_argument("-j", "--jobs", type=int, default=2, help="Number of concurrent jobs")
+    parser.add_argument("-b", "--batch-size", type=int, default=10000, help="Batch size for processing files")
     args = parser.parse_args()
     sem = Semaphore(args.jobs)
-    asyncio.run(_main())
+    asyncio.run(_main(args.batch_size))
 
 if __name__ == '__main__':
     main()

{lfss-0.5.1 → lfss-0.6.0}/lfss/client/api.py

@@ -2,7 +2,7 @@ from typing import Optional, Literal
 import os
 import requests
 import urllib.parse
-from lfss.src.database import (
+from lfss.src.datatype import (
     FileReadPermission, FileRecord, DirectoryRecord, UserRecord, PathContents
     )
 
@@ -36,8 +36,6 @@ class Connector:
 
     def put(self, path: str, file_data: bytes, permission: int | FileReadPermission = 0, conflict: Literal['overwrite', 'abort', 'skip'] = 'abort'):
         """Uploads a file to the specified path."""
-        if path.startswith('/'):
-            path = path[1:]
         response = self._fetch('PUT', path, search_params={
             'permission': int(permission),
             'conflict': conflict
@@ -47,20 +45,41 @@ class Connector:
         )
         return response.json()
     
-    def get(self, path: str) -> Optional[bytes]:
-        """Downloads a file from the specified path."""
+    def put_json(self, path: str, data: dict, permission: int | FileReadPermission = 0, conflict: Literal['overwrite', 'abort', 'skip'] = 'abort'):
+        """Uploads a JSON file to the specified path."""
+        assert path.endswith('.json'), "Path must end with .json"
+        response = self._fetch('PUT', path, search_params={
+            'permission': int(permission),
+            'conflict': conflict
+            })(
+            json=data, 
+            headers={'Content-Type': 'application/json'}
+        )
+        return response.json()
+    
+    def _get(self, path: str) -> Optional[requests.Response]:
         try:
             response = self._fetch('GET', path)()
         except requests.exceptions.HTTPError as e:
             if e.response.status_code == 404:
                 return None
             raise e
+        return response
+
+    def get(self, path: str) -> Optional[bytes]:
+        """Downloads a file from the specified path."""
+        response = self._get(path)
+        if response is None: return None
         return response.content
+
+    def get_json(self, path: str) -> Optional[dict]:
+        response = self._get(path)
+        if response is None: return None
+        assert response.headers['Content-Type'] == 'application/json'
+        return response.json()
     
     def delete(self, path: str):
         """Deletes the file at the specified path."""
-        if path.startswith('/'):
-            path = path[1:]
         self._fetch('DELETE', path)()
     
     def get_metadata(self, path: str) -> Optional[FileRecord | DirectoryRecord]:
@@ -87,8 +106,8 @@ class Connector:
             headers={'Content-Type': 'application/www-form-urlencoded'}
         )
         
-    def move_file(self, path: str, new_path: str):
-        """Moves a file to a new location."""
+    def move(self, path: str, new_path: str):
+        """Move file or directory to a new path."""
         self._fetch('POST', '_api/meta', {'path': path, 'new_path': new_path})(
             headers = {'Content-Type': 'application/www-form-urlencoded'}
         )

{lfss-0.5.1 → lfss-0.6.0}/lfss/sql/init.sql

@@ -1,7 +1,7 @@
 CREATE TABLE IF NOT EXISTS user (
     id INTEGER PRIMARY KEY AUTOINCREMENT,
-    username VARCHAR(255) UNIQUE NOT NULL,
-    credential VARCHAR(255) NOT NULL,
+    username VARCHAR(256) UNIQUE NOT NULL,
+    credential VARCHAR(256) NOT NULL,
     is_admin BOOLEAN DEFAULT FALSE,
     create_time TIMESTAMP DEFAULT CURRENT_TIMESTAMP, 
     last_active TIMESTAMP DEFAULT CURRENT_TIMESTAMP, 
@@ -10,19 +10,20 @@ CREATE TABLE IF NOT EXISTS user (
 );
 
 CREATE TABLE IF NOT EXISTS fmeta (
-    url VARCHAR(512) PRIMARY KEY,
+    url VARCHAR(1024) PRIMARY KEY,
     owner_id INTEGER NOT NULL,
-    file_id VARCHAR(256) NOT NULL,
+    file_id CHAR(32) NOT NULL,
     file_size INTEGER,
     create_time TIMESTAMP DEFAULT CURRENT_TIMESTAMP, 
     access_time TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
     permission INTEGER DEFAULT 0, 
-    external BOOLEAN DEFAULT FALSE,
+    external BOOLEAN DEFAULT FALSE, 
+    mime_type VARCHAR(256) DEFAULT 'application/octet-stream',
     FOREIGN KEY(owner_id) REFERENCES user(id)
 );
 
 CREATE TABLE IF NOT EXISTS fdata (
-    file_id VARCHAR(256) PRIMARY KEY,
+    file_id CHAR(32) PRIMARY KEY,
     data BLOB
 );
 

{lfss-0.5.1 → lfss-0.6.0}/lfss/src/database.py

@@ -4,16 +4,16 @@ from abc import ABC, abstractmethod
 
 import urllib.parse
 from pathlib import Path
-import dataclasses, hashlib, uuid
+import hashlib, uuid
 from contextlib import asynccontextmanager
 from functools import wraps
-from enum import IntEnum
 import zipfile, io, asyncio
 
 import aiosqlite, aiofiles
 import aiofiles.os
 from asyncio import Lock
 
+from .datatype import UserRecord, FileReadPermission, FileRecord, DirectoryRecord, PathContents
 from .config import DATA_HOME, LARGE_BLOB_DIR
 from .log import get_logger
 from .utils import decode_uri_compnents
@@ -64,26 +64,6 @@ class DBConnBase(ABC):
     async def commit(self):
         await self.conn.commit()
 
-class FileReadPermission(IntEnum):
-    UNSET = 0           # not set
-    PUBLIC = 1          # accessible by anyone
-    PROTECTED = 2       # accessible by any user
-    PRIVATE = 3         # accessible by owner only (including admin)
-
-@dataclasses.dataclass
-class UserRecord:
-    id: int
-    username: str
-    credential: str
-    is_admin: bool
-    create_time: str
-    last_active: str
-    max_storage: int
-    permission: 'FileReadPermission'
-
-    def __str__(self):
-        return f"User {self.username} (id={self.id}, admin={self.is_admin}, created at {self.create_time}, last active at {self.last_active}, storage={self.max_storage}, permission={self.permission})"
-
 DECOY_USER = UserRecord(0, 'decoy', 'decoy', False, '2021-01-01 00:00:00', '2021-01-01 00:00:00', 0, FileReadPermission.PRIVATE)
 class UserConn(DBConnBase):
 
@@ -174,37 +154,6 @@ class UserConn(DBConnBase):
         await self.conn.execute("DELETE FROM user WHERE username = ?", (username, ))
         self.logger.info(f"Delete user {username}")
 
-@dataclasses.dataclass
-class FileRecord:
-    url: str
-    owner_id: int
-    file_id: str      # defines mapping from fmata to fdata
-    file_size: int
-    create_time: str
-    access_time: str
-    permission: FileReadPermission
-    external: bool
-
-    def __str__(self):
-        return  f"File {self.url} (owner={self.owner_id}, created at {self.create_time}, accessed at {self.access_time}, " + \
-                f"file_id={self.file_id}, permission={self.permission}, size={self.file_size}, external={self.external})"
-
-@dataclasses.dataclass
-class DirectoryRecord:
-    url: str
-    size: int
-    create_time: str = ""
-    update_time: str = ""
-    access_time: str = ""
-
-    def __str__(self):
-        return f"Directory {self.url} (size={self.size})"
-
-@dataclasses.dataclass
-class PathContents:
-    dirs: list[DirectoryRecord]
-    files: list[FileRecord]
-    
 class FileConn(DBConnBase):
 
     @staticmethod
@@ -235,6 +184,38 @@ class FileConn(DBConnBase):
             ALTER TABLE fmeta ADD COLUMN external BOOLEAN DEFAULT FALSE
             ''')
 
+        # backward compatibility, since 0.6.0
+        async with self.conn.execute("SELECT * FROM fmeta") as cursor:
+            res = await cursor.fetchone()
+        if res and len(res) < 9:
+            self.logger.info("Updating fmeta table")
+            await self.conn.execute('''
+            ALTER TABLE fmeta ADD COLUMN mime_type TEXT DEFAULT 'application/octet-stream'
+            ''')
+            # check all mime types
+            import mimetypes, mimesniff
+            async with self.conn.execute("SELECT url, file_id, external FROM fmeta") as cursor:
+                res = await cursor.fetchall()
+            async with self.conn.execute("SELECT count(*) FROM fmeta") as cursor:
+                count = await cursor.fetchone()
+                assert count is not None
+            for counter, r in enumerate(res, start=1):
+                print(f"Checking mimetype for {counter}/{count[0]}")
+                url, f_id, external = r
+                fname = url.split('/')[-1]
+                mime_type, _ = mimetypes.guess_type(fname)
+                if mime_type is None:
+                    # try to sniff the file
+                    if not external:
+                        async with self.conn.execute("SELECT data FROM fdata WHERE file_id = ?", (f_id, )) as cursor:
+                            blob = await cursor.fetchone()
+                        assert blob is not None
+                        blob = blob[0]
+                        mime_type = mimesniff.what(blob)
+                    else:
+                        mime_type = mimesniff.what(LARGE_BLOB_DIR / f_id)
+                await self.conn.execute("UPDATE fmeta SET mime_type = ? WHERE url = ?", (mime_type, url))
+
         return self
     
     async def get_file_record(self, url: str) -> Optional[FileRecord]:
@@ -331,7 +312,7 @@ class FileConn(DBConnBase):
         dirs = await asyncio.gather(*[get_dir(url + d) for d in dirs_str])
         return PathContents(dirs, files)
     
-    async def get_path_record(self, url: str) -> Optional[DirectoryRecord]:
+    async def get_path_record(self, url: str) -> DirectoryRecord:
         assert url.endswith('/'), "Path must end with /"
         async with self.conn.execute("""
             SELECT MIN(create_time) as create_time, 
@@ -372,43 +353,42 @@ class FileConn(DBConnBase):
         assert res is not None
         return res[0] or 0
     
+    @atomic
+    async def update_file_record(
+        self, url, owner_id: Optional[int] = None, permission: Optional[FileReadPermission] = None
+        ):
+        old = await self.get_file_record(url)
+        assert old is not None, f"File {url} not found"
+        if owner_id is None:
+            owner_id = old.owner_id
+        if permission is None:
+            permission = old.permission
+        await self.conn.execute(
+            "UPDATE fmeta SET owner_id = ?, permission = ? WHERE url = ?", 
+            (owner_id, int(permission), url)
+            )
+        self.logger.info(f"Updated file {url}")
+    
     @atomic
     async def set_file_record(
         self, url: str, 
-        owner_id: Optional[int] = None, 
-        file_id: Optional[str] = None, 
-        file_size: Optional[int] = None, 
-        permission: Optional[ FileReadPermission ] = None, 
-        external: Optional[bool] = None
+        owner_id: int, 
+        file_id:str, 
+        file_size: int, 
+        permission: FileReadPermission, 
+        external: bool, 
+        mime_type: str
         ):
-
-        old = await self.get_file_record(url)
-        if old is not None:
-            self.logger.debug(f"Updating fmeta {url}: permission={permission}, owner_id={owner_id}")
-            # should delete the old blob if file_id is changed
-            assert file_id is None, "Cannot update file id"
-            assert file_size is None, "Cannot update file size"
-            assert external is None, "Cannot update external"
-
-            if owner_id is None: owner_id = old.owner_id
-            if permission is None: permission = old.permission
-            await self.conn.execute(
-                """
-                UPDATE fmeta SET owner_id = ?, permission = ?, 
-                access_time = CURRENT_TIMESTAMP WHERE url = ?
-                """, (owner_id, int(permission), url))
-            self.logger.info(f"File {url} updated")
-        else:
-            self.logger.debug(f"Creating fmeta {url}: permission={permission}, owner_id={owner_id}, file_id={file_id}, file_size={file_size}, external={external}")
-            if permission is None:
-                permission = FileReadPermission.UNSET
-            assert owner_id is not None and file_id is not None and file_size is not None and external is not None
-            await self.conn.execute(
-                "INSERT INTO fmeta (url, owner_id, file_id, file_size, permission, external) VALUES (?, ?, ?, ?, ?, ?)", 
-                (url, owner_id, file_id, file_size, int(permission), external)
-                )
-            await self._user_size_inc(owner_id, file_size)
-            self.logger.info(f"File {url} created")
+        self.logger.debug(f"Creating fmeta {url}: permission={permission}, owner_id={owner_id}, file_id={file_id}, file_size={file_size}, external={external}, mime_type={mime_type}")
+        if permission is None:
+            permission = FileReadPermission.UNSET
+        assert owner_id is not None and file_id is not None and file_size is not None and external is not None
+        await self.conn.execute(
+            "INSERT INTO fmeta (url, owner_id, file_id, file_size, permission, external, mime_type) VALUES (?, ?, ?, ?, ?, ?, ?)", 
+            (url, owner_id, file_id, file_size, int(permission), external, mime_type)
+            )
+        await self._user_size_inc(owner_id, file_size)
+        self.logger.info(f"File {url} created")
     
     @atomic
     async def move_file(self, old_url: str, new_url: str):
@@ -421,6 +401,25 @@ class FileConn(DBConnBase):
         async with self.conn.execute("UPDATE fmeta SET url = ?, create_time = CURRENT_TIMESTAMP WHERE url = ?", (new_url, old_url)):
             self.logger.info(f"Moved file {old_url} to {new_url}")
     
+    @atomic
+    async def move_path(self, old_url: str, new_url: str, conflict_handler: Literal['skip', 'overwrite'] = 'overwrite', user_id: Optional[int] = None):
+        assert old_url.endswith('/'), "Old path must end with /"
+        assert new_url.endswith('/'), "New path must end with /"
+        if user_id is None:
+            async with self.conn.execute("SELECT * FROM fmeta WHERE url LIKE ?", (old_url + '%', )) as cursor:
+                res = await cursor.fetchall()
+        else:
+            async with self.conn.execute("SELECT * FROM fmeta WHERE url LIKE ? AND owner_id = ?", (old_url + '%', user_id)) as cursor:
+                res = await cursor.fetchall()
+        for r in res:
+            new_r = new_url + r[0][len(old_url):]
+            if conflict_handler == 'overwrite':
+                await self.conn.execute("DELETE FROM fmeta WHERE url = ?", (new_r, ))
+            elif conflict_handler == 'skip':
+                if (await self.conn.execute("SELECT url FROM fmeta WHERE url = ?", (new_r, ))) is not None:
+                    continue
+            await self.conn.execute("UPDATE fmeta SET url = ?, create_time = CURRENT_TIMESTAMP WHERE url = ?", (new_r, r[0]))
+    
     async def log_access(self, url: str):
         await self.conn.execute("UPDATE fmeta SET access_time = CURRENT_TIMESTAMP WHERE url = ?", (url, ))
     
@@ -568,7 +567,8 @@ class Database:
     async def save_file(
         self, u: int | str, url: str, 
         blob: bytes | AsyncIterable[bytes], 
-        permission: FileReadPermission = FileReadPermission.UNSET
+        permission: FileReadPermission = FileReadPermission.UNSET, 
+        mime_type: str = 'application/octet-stream'
         ):
         """
         if file_size is not provided, the blob must be bytes
@@ -600,7 +600,7 @@ class Database:
                 await self.file.set_file_blob(f_id, blob)
                 await self.file.set_file_record(
                     url, owner_id=user.id, file_id=f_id, file_size=file_size, 
-                    permission=permission, external=False)
+                    permission=permission, external=False, mime_type=mime_type)
                 await self.user.set_active(user.username)
         else:
             assert isinstance(blob, AsyncIterable)
@@ -612,7 +612,7 @@ class Database:
                     raise StorageExceededError(f"Unable to save file, user {user.username} has storage limit of {user.max_storage}, used {user_size_used}, requested {file_size}")
                 await self.file.set_file_record(
                     url, owner_id=user.id, file_id=f_id, file_size=file_size, 
-                    permission=permission, external=True)
+                    permission=permission, external=True, mime_type=mime_type)
                 await self.user.set_active(user.username)
 
     async def read_file_stream(self, url: str) -> AsyncIterable[bytes]:
@@ -664,6 +664,13 @@ class Database:
 
         async with transaction(self):
             await self.file.move_file(old_url, new_url)
+    
+    async def move_path(self, old_url: str, new_url: str, user_id: Optional[int] = None):
+        validate_url(old_url, is_file=False)
+        validate_url(new_url, is_file=False)
+
+        async with transaction(self):
+            await self.file.move_path(old_url, new_url, 'overwrite', user_id)
 
     async def __batch_delete_file_blobs(self, file_records: list[FileRecord], batch_size: int = 512):
         # https://github.com/langchain-ai/langchain/issues/10321

lfss-0.6.0/lfss/src/datatype.py

@@ -0,0 +1,55 @@
+from enum import IntEnum
+import dataclasses
+
+class FileReadPermission(IntEnum):
+    UNSET = 0           # not set
+    PUBLIC = 1          # accessible by anyone
+    PROTECTED = 2       # accessible by any user
+    PRIVATE = 3         # accessible by owner only (including admin)
+
+@dataclasses.dataclass
+class UserRecord:
+    id: int
+    username: str
+    credential: str
+    is_admin: bool
+    create_time: str
+    last_active: str
+    max_storage: int
+    permission: 'FileReadPermission'
+
+    def __str__(self):
+        return f"User {self.username} (id={self.id}, admin={self.is_admin}, created at {self.create_time}, last active at {self.last_active}, storage={self.max_storage}, permission={self.permission})"
+
+@dataclasses.dataclass
+class FileRecord:
+    url: str
+    owner_id: int
+    file_id: str      # defines mapping from fmata to fdata
+    file_size: int
+    create_time: str
+    access_time: str
+    permission: FileReadPermission
+    external: bool
+    mime_type: str
+
+    def __str__(self):
+        return  f"File {self.url} [{self.mime_type}] (owner={self.owner_id}, created at {self.create_time}, accessed at {self.access_time}, " + \
+                f"file_id={self.file_id}, permission={self.permission}, size={self.file_size}, external={self.external})"
+
+@dataclasses.dataclass
+class DirectoryRecord:
+    url: str
+    size: int
+    create_time: str = ""
+    update_time: str = ""
+    access_time: str = ""
+
+    def __str__(self):
+        return f"Directory {self.url} (size={self.size})"
+
+@dataclasses.dataclass
+class PathContents:
+    dirs: list[DirectoryRecord]
+    files: list[FileRecord]
+    

{lfss-0.5.1 → lfss-0.6.0}/lfss/src/server.py

@@ -15,7 +15,7 @@ from contextlib import asynccontextmanager
 from .error import *
 from .log import get_logger
 from .stat import RequestDB
-from .config import MAX_BUNDLE_BYTES, MAX_FILE_BYTES, LARGE_BLOB_DIR, LARGE_FILE_BYTES
+from .config import MAX_BUNDLE_BYTES, MAX_FILE_BYTES, LARGE_FILE_BYTES
 from .utils import ensure_uri_compnents, format_last_modified, now_stamp
 from .database import Database, UserRecord, DECOY_USER, FileRecord, check_user_permission, FileReadPermission
 
@@ -142,12 +142,10 @@ async def get_file(path: str, download = False, user: UserRecord = Depends(get_c
     
     fname = path.split("/")[-1]
     async def send(media_type: Optional[str] = None, disposition = "attachment"):
+        if media_type is None:
+            media_type = file_record.mime_type
         if not file_record.external:
             fblob = await conn.read_file(path)
-            if media_type is None:
-                media_type, _ = mimetypes.guess_type(fname)
-            if media_type is None:
-                media_type = mimesniff.what(fblob)
             return Response(
                 content=fblob, media_type=media_type, headers={
                     "Content-Disposition": f"{disposition}; filename={fname}", 
@@ -155,12 +153,7 @@ async def get_file(path: str, download = False, user: UserRecord = Depends(get_c
                     "Last-Modified": format_last_modified(file_record.create_time)
                 }
             )
-        
         else:
-            if media_type is None:
-                media_type, _ = mimetypes.guess_type(fname)
-            if media_type is None:
-                media_type = mimesniff.what(str((LARGE_BLOB_DIR / file_record.file_id).absolute()))
             return StreamingResponse(
                 await conn.read_file_stream(path), media_type=media_type, headers={
                     "Content-Disposition": f"{disposition}; filename={fname}", 
@@ -228,14 +221,24 @@ async def put_file(
         blobs = await request.body()
     else:
         blobs = await request.body()
+    
+    # check file type
+    assert not path.endswith("/"), "Path must be a file"
+    fname = path.split("/")[-1]
+    mime_t, _ = mimetypes.guess_type(fname)
+    if mime_t is None:
+        mime_t = mimesniff.what(blobs)
+    if mime_t is None:
+        mime_t = "application/octet-stream"
+
     if len(blobs) > LARGE_FILE_BYTES:
         async def blob_reader():
             chunk_size = 16 * 1024 * 1024    # 16MB
             for b in range(0, len(blobs), chunk_size):
                 yield blobs[b:b+chunk_size]
-        await conn.save_file(user.id, path, blob_reader(), permission = FileReadPermission(permission))
+        await conn.save_file(user.id, path, blob_reader(), permission = FileReadPermission(permission), mime_type = mime_t)
     else:
-        await conn.save_file(user.id, path, blobs, permission = FileReadPermission(permission))
+        await conn.save_file(user.id, path, blobs, permission = FileReadPermission(permission), mime_type=mime_t)
 
     # https://developer.mozilla.org/zh-CN/docs/Web/HTTP/Methods/PUT
     if exists_flag:
@@ -263,6 +266,7 @@ async def delete_file(path: str, user: UserRecord = Depends(get_current_user)):
     else:
         res = await conn.delete_file(path)
 
+    await conn.user.set_active(user.username)
     if res:
         return Response(status_code=200, content="Deleted")
     else:
@@ -335,26 +339,57 @@ async def update_file_meta(
     if user.id == 0:
         raise HTTPException(status_code=401, detail="Permission denied")
     path = ensure_uri_compnents(path)
-    file_record = await conn.file.get_file_record(path)
-    if not file_record:
-        logger.debug(f"Reject update meta request from {user.username} to {path}")
-        raise HTTPException(status_code=404, detail="File not found")
-    
-    if not (user.is_admin or user.id == file_record.owner_id):
-        logger.debug(f"Reject update meta request from {user.username} to {path}")
-        raise HTTPException(status_code=403, detail="Permission denied")
+    if path.startswith("/"):
+        path = path[1:]
+    await conn.user.set_active(user.username)
+
+    # file
+    if not path.endswith("/"):
+        file_record = await conn.file.get_file_record(path)
+        if not file_record:
+            logger.debug(f"Reject update meta request from {user.username} to {path}")
+            raise HTTPException(status_code=404, detail="File not found")
+        
+        if not (user.is_admin or user.id == file_record.owner_id):
+            logger.debug(f"Reject update meta request from {user.username} to {path}")
+            raise HTTPException(status_code=403, detail="Permission denied")
+        
+        if perm is not None:
+            logger.info(f"Update permission of {path} to {perm}")
+            await conn.file.update_file_record(
+                url = file_record.url, 
+                permission = FileReadPermission(perm)
+            )
     
-    if perm is not None:
-        logger.info(f"Update permission of {path} to {perm}")
-        await conn.file.set_file_record(
-            url = file_record.url, 
-            permission = FileReadPermission(perm)
-        )
+        if new_path is not None:
+            new_path = ensure_uri_compnents(new_path)
+            logger.info(f"Update path of {path} to {new_path}")
+            await conn.move_file(path, new_path)
     
-    if new_path is not None:
-        new_path = ensure_uri_compnents(new_path)
-        logger.info(f"Update path of {path} to {new_path}")
-        await conn.move_file(path, new_path)
+    # directory
+    else:
+        assert perm is None, "Permission is not supported for directory"
+        if new_path is not None:
+            new_path = ensure_uri_compnents(new_path)
+            logger.info(f"Update path of {path} to {new_path}")
+            assert new_path.endswith("/"), "New path must end with /"
+            if new_path.startswith("/"):
+                new_path = new_path[1:]
+
+            # check if new path is under the user's directory
+            first_component = new_path.split("/")[0]
+            if not (first_component == user.username or user.is_admin):
+                raise HTTPException(status_code=403, detail="Permission denied, path must start with username")
+            elif user.is_admin:
+                _is_user = await conn.user.get_user(first_component)
+                if not _is_user:
+                    raise HTTPException(status_code=404, detail="User not found, path must start with username")
+
+            # check if old path is under the user's directory (non-admin)
+            if not path.startswith(f"{user.username}/") and not user.is_admin:
+                raise HTTPException(status_code=403, detail="Permission denied, path must start with username")
+            # currently only move own file, with overwrite
+            await conn.move_path(path, new_path, user_id = user.id)
 
     return Response(status_code=200, content="OK")
     

{lfss-0.5.1 → lfss-0.6.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "lfss"
-version = "0.5.1"
+version = "0.6.0"
 description = "Lightweight file storage service"
 authors = ["li, mengxun <limengxun45@outlook.com>"]
 readme = "Readme.md"