lfss 0.4.0__py3-none-any.whl → 0.5.0__py3-none-any.whl

frontend/api.js

@@ -25,6 +25,8 @@
  * @typedef {Object} DirectoryRecord
  * @property {string} url - the url of the directory
  * @property {string} size - the size of the directory, in bytes
+ * @property {string} create_time - the time the directory was created
+ * @property {string} access_time - the time the directory was last accessed
  * 
  * @typedef {Object} PathListResponse
  * @property {DirectoryRecord[]} dirs - the list of directories in the directory
@@ -54,10 +56,16 @@ export default class Connector {
     * @param {File} file - the file to upload
     * @returns {Promise<string>} - the promise of the request, the url of the file
     */
-    async put(path, file){
+    async put(path, file, {
+        overwrite = false,
+        permission = 0
+    } = {}){
         if (path.startsWith('/')){ path = path.slice(1); }
         const fileBytes = await file.arrayBuffer();
-        const res = await fetch(this.config.endpoint + '/' + path, {
+        const dst = new URL(this.config.endpoint + '/' + path);
+        dst.searchParams.append('overwrite', overwrite);
+        dst.searchParams.append('permission', permission);
+        const res = await fetch(dst.toString(), {
             method: 'PUT',
             headers: {
                 'Authorization': 'Bearer ' + this.config.token, 
@@ -106,12 +114,12 @@ export default class Connector {
     }
 
     /**
-     * @param {string} path - the path to the file
-     * @returns {Promise<FileRecord | null>} - the promise of the request
+     * @param {string} path - the path to the file or directory
+     * @returns {Promise<FileRecord | DirectoryRecord | null>} - the promise of the request
      */
     async getMetadata(path){
         if (path.startsWith('/')){ path = path.slice(1); }
-        const res = await fetch(this.config.endpoint + '/_api/fmeta?path=' + path, {
+        const res = await fetch(this.config.endpoint + '/_api/meta?path=' + path, {
             method: 'GET',
             headers: {
                 'Authorization': 'Bearer ' + this.config.token
@@ -165,7 +173,7 @@ export default class Connector {
      */
     async setFilePermission(path, permission){
         if (path.startsWith('/')){ path = path.slice(1); }
-        const dst = new URL(this.config.endpoint + '/_api/fmeta');
+        const dst = new URL(this.config.endpoint + '/_api/meta');
         dst.searchParams.append('path', path);
         dst.searchParams.append('perm', permission);
         const res = await fetch(dst.toString(), {
@@ -186,7 +194,7 @@ export default class Connector {
     async moveFile(path, newPath){
         if (path.startsWith('/')){ path = path.slice(1); }
         if (newPath.startsWith('/')){ newPath = newPath.slice(1); }
-        const dst = new URL(this.config.endpoint + '/_api/fmeta');
+        const dst = new URL(this.config.endpoint + '/_api/meta');
         dst.searchParams.append('path', path);
         dst.searchParams.append('new_path', newPath);
         const res = await fetch(dst.toString(), {

frontend/scripts.js

@@ -79,6 +79,10 @@ pathBackButton.addEventListener('click', () => {
 
 function onFileNameInpuChange(){
     const fileName = uploadFileNameInput.value;
+    if (fileName.endsWith('/')){
+        uploadFileNameInput.classList.add('duplicate');
+        return;
+    }
     if (fileName.length === 0){
         uploadFileNameInput.classList.remove('duplicate');
     }
@@ -172,7 +176,7 @@ Are you sure you want to proceed?
             async function uploadFile(...args){
                 const [file, path] = args;
                 try{
-                    await conn.put(path, file);
+                    await conn.put(path, file, {overwrite: true});
                 }
                 catch (err){
                     showPopup('Failed to upload file [' + file.name + ']: ' + err, {level: 'error', timeout: 5000});
@@ -219,6 +223,9 @@ function refreshFileList(){
 
             data.dirs.forEach(dir => {
                 const tr = document.createElement('tr');
+                const sizeTd = document.createElement('td');
+                const accessTimeTd = document.createElement('td');
+                const createTimeTd = document.createElement('td');
                 {
                     const nameTd = document.createElement('td');
                     if (dir.url.endsWith('/')){
@@ -239,19 +246,14 @@ function refreshFileList(){
                     tr.appendChild(nameTd);
                     tbody.appendChild(tr);
                 }
-
                 {
-                    const sizeTd = document.createElement('td');
+                    // these are initialized meta
                     sizeTd.textContent = formatSize(dir.size);
                     tr.appendChild(sizeTd);
-                }
-                {
-                    const dateTd = document.createElement('td');
-                    tr.appendChild(dateTd);
-                }
-                {
-                    const dateTd = document.createElement('td');
-                    tr.appendChild(dateTd);
+                    accessTimeTd.textContent = cvtGMT2Local(dir.access_time);
+                    tr.appendChild(accessTimeTd);
+                    createTimeTd.textContent = cvtGMT2Local(dir.create_time);
+                    tr.appendChild(createTimeTd);
                 }
                 {
                     const accessTd = document.createElement('td');
@@ -262,6 +264,22 @@ function refreshFileList(){
                     const actContainer = document.createElement('div');
                     actContainer.classList.add('action-container');
 
+                    const showMetaButton = document.createElement('a');
+                    showMetaButton.textContent = 'Details';
+                    showMetaButton.style.cursor = 'pointer';
+                    showMetaButton.addEventListener('click', () => {
+                        const dirUrlEncap = dir.url + (dir.url.endsWith('/') ? '' : '/');
+                        conn.getMetadata(dirUrlEncap).then(
+                            (meta) => {
+                                sizeTd.textContent = formatSize(meta.size);
+                                accessTimeTd.textContent = cvtGMT2Local(meta.access_time);
+                                createTimeTd.textContent = cvtGMT2Local(meta.create_time);
+                            }
+                        );
+                        showPopup('Fetching metadata...', {level: 'info', timeout: 3000});
+                    });
+                    actContainer.appendChild(showMetaButton);
+
                     const downloadButton = document.createElement('a');
                     downloadButton.textContent = 'Download';
                     downloadButton.href = conn.config.endpoint + '/_api/bundle?' + 

frontend/styles.css

@@ -192,6 +192,7 @@ input#file-name.duplicate{
     display: flex;
     flex-direction: row;
     gap: 10px;
+    /* justify-content: flex-end; */
 }
 a{
     color: #195f8b;

frontend/utils.js

@@ -1,5 +1,8 @@
 
 export function formatSize(size){
+    if (size < 0){
+        return '';
+    }
     const sizeInKb = size / 1024;
     const sizeInMb = sizeInKb / 1024;
     const sizeInGb = sizeInMb / 1024;
@@ -68,6 +71,9 @@ export function getRandomString(n, additionalCharset='0123456789_-(=)[]{}'){
  * @returns {string}
  */
 export function cvtGMT2Local(dateStr){
+    if (!dateStr || dateStr === 'N/A'){
+        return '';
+    }
     const gmtdate = new Date(dateStr);
     const localdate = new Date(gmtdate.getTime() + gmtdate.getTimezoneOffset() * 60000);
     return localdate.toISOString().slice(0, 19).replace('T', ' ');

lfss/cli/cli.py

@@ -0,0 +1,52 @@
+from lfss.client import Connector, upload_directory
+from lfss.src.database import FileReadPermission
+from pathlib import Path
+import argparse
+
+def parse_arguments():
+    parser = argparse.ArgumentParser(description="Command line interface, please set LFSS_ENDPOINT and LFSS_TOKEN environment variables.")
+
+    sp = parser.add_subparsers(dest="command", required=True)
+
+    # upload
+    sp_upload = sp.add_parser("upload", help="Upload files")
+    sp_upload.add_argument("src", help="Source file or directory", type=str)
+    sp_upload.add_argument("dst", help="Destination path", type=str)
+    sp_upload.add_argument("-j", "--jobs", type=int, default=1, help="Number of concurrent uploads")
+    sp_upload.add_argument("--interval", type=float, default=0, help="Interval between retries, only works with directory upload")
+    sp_upload.add_argument("--overwrite", action="store_true", help="Overwrite existing files")
+    sp_upload.add_argument("--permission", type=FileReadPermission, default=FileReadPermission.UNSET, help="File permission")
+    sp_upload.add_argument("--retries", type=int, default=0, help="Number of retries, only works with directory upload")
+
+    return parser.parse_args()
+
+def main():
+    args = parse_arguments()
+    connector = Connector()
+    if args.command == "upload":
+        src_path = Path(args.src)
+        if src_path.is_dir():
+            failed_upload = upload_directory(
+                connector, args.src, args.dst, 
+                verbose=True, 
+                n_concurrent=args.jobs, 
+                n_reties=args.retries, 
+                interval=args.interval,
+                overwrite=args.overwrite, 
+                permission=args.permission
+            )
+            if failed_upload:
+                print("Failed to upload:")
+                for path in failed_upload:
+                    print(f"  {path}")
+        else:
+            with open(args.src, 'rb') as f:
+                connector.put(
+                    args.dst, 
+                    f.read(), 
+                    overwrite=args.overwrite, 
+                    permission=args.permission
+                    )
+    else:
+        raise NotImplementedError(f"Command {args.command} not implemented.")
+    

lfss/cli/user.py

@@ -8,6 +8,8 @@ def parse_storage_size(s: str) -> int:
         return int(s[:-1]) * 1024 * 1024
     if s[-1] in 'Gg':
         return int(s[:-1]) * 1024 * 1024 * 1024
+    if s[-1] in 'Tt':
+        return int(s[:-1]) * 1024 * 1024 * 1024 * 1024
     return int(s)
 
 async def _main():

lfss/client/__init__.py

@@ -0,0 +1,60 @@
+import os, time
+from threading import Lock
+from concurrent.futures import ThreadPoolExecutor
+from .api import Connector
+
+def upload_directory(
+    connector: Connector, 
+    directory: str, 
+    path: str, 
+    n_concurrent: int = 1,
+    n_reties: int = 0,
+    interval: float = 0,
+    verbose: bool = False,
+    **put_kwargs
+    ) -> list[str]:
+    assert path.endswith('/'), "Path must end with a slash."
+    if path.startswith('/'):
+        path = path[1:]
+    
+    _counter = 0
+    _counter_lock = Lock()
+
+    faild_files = []
+    def put_file(file_path):
+        with _counter_lock:
+            nonlocal _counter
+            _counter += 1
+            this_count = _counter
+            dst_path = f"{path}{os.path.relpath(file_path, directory)}"
+            if verbose:
+                print(f"[{this_count}] Uploading {file_path} to {dst_path}")
+
+        this_try = 0
+        with open(file_path, 'rb') as f:
+            blob = f.read()
+
+        while this_try <= n_reties:
+            try:
+                connector.put(dst_path, blob, **put_kwargs)
+                break
+            except Exception as e:
+                if isinstance(e, KeyboardInterrupt):
+                    raise e
+                if verbose:
+                    print(f"[{this_count}] Error uploading {file_path}: {e}, retrying...")
+                this_try += 1
+            finally:
+                time.sleep(interval)
+
+        if this_try > n_reties:
+            faild_files.append(file_path)
+            if verbose:
+                print(f"[{this_count}] Failed to upload {file_path} after {n_reties} retries.")
+
+    with ThreadPoolExecutor(n_concurrent) as executor:
+        for root, dirs, files in os.walk(directory):
+            for file in files:
+                executor.submit(put_file, os.path.join(root, file))
+
+    return faild_files

lfss/client/api.py

@@ -3,7 +3,7 @@ import os
 import requests
 import urllib.parse
 from lfss.src.database import (
-    FileReadPermission, FileRecord, UserRecord, PathContents
+    FileReadPermission, FileRecord, DirectoryRecord, UserRecord, PathContents
     )
 
 _default_endpoint = os.environ.get('LFSS_ENDPOINT', 'http://localhost:8000')
@@ -34,9 +34,14 @@ class Connector:
             return response
         return f
 
-    def put(self, path: str, file_data: bytes):
+    def put(self, path: str, file_data: bytes, permission: int | FileReadPermission = 0, overwrite: bool = False):
         """Uploads a file to the specified path."""
-        response = self._fetch('PUT', path)(
+        if path.startswith('/'):
+            path = path[1:]
+        response = self._fetch('PUT', path, search_params={
+            'permission': int(permission),
+            'overwrite': overwrite
+            })(
             data=file_data, 
             headers={'Content-Type': 'application/octet-stream'}
         )
@@ -58,11 +63,14 @@ class Connector:
             path = path[1:]
         self._fetch('DELETE', path)()
     
-    def get_metadata(self, path: str) -> Optional[FileRecord]:
+    def get_metadata(self, path: str) -> Optional[FileRecord | DirectoryRecord]:
         """Gets the metadata for the file at the specified path."""
         try:
-            response = self._fetch('GET', '_api/fmeta', {'path': path})()
-            return FileRecord(**response.json())
+            response = self._fetch('GET', '_api/meta', {'path': path})()
+            if path.endswith('/'):
+                return DirectoryRecord(**response.json())
+            else:
+                return FileRecord(**response.json())
         except requests.exceptions.HTTPError as e:
             if e.response.status_code == 404:
                 return None
@@ -75,13 +83,13 @@ class Connector:
 
     def set_file_permission(self, path: str, permission: int | FileReadPermission):
         """Sets the file permission for the specified path."""
-        self._fetch('POST', '_api/fmeta', {'path': path, 'perm': int(permission)})(
+        self._fetch('POST', '_api/meta', {'path': path, 'perm': int(permission)})(
             headers={'Content-Type': 'application/www-form-urlencoded'}
         )
         
     def move_file(self, path: str, new_path: str):
         """Moves a file to a new location."""
-        self._fetch('POST', '_api/fmeta', {'path': path, 'new_path': new_path})(
+        self._fetch('POST', '_api/meta', {'path': path, 'new_path': new_path})(
             headers = {'Content-Type': 'application/www-form-urlencoded'}
         )
         

lfss/src/config.py

@@ -7,6 +7,9 @@ DATA_HOME = Path(os.environ.get('LFSS_DATA', __default_dir))
 if not DATA_HOME.exists():
     DATA_HOME.mkdir()
     print(f"[init] Created data home at {DATA_HOME}")
+LARGE_BLOB_DIR = DATA_HOME / 'large_blobs'
+LARGE_BLOB_DIR.mkdir(exist_ok=True)
 
-MAX_FILE_BYTES = 256 * 1024 * 1024   # 256MB
-MAX_BUNDLE_BYTES = 256 * 1024 * 1024   # 256MB
+LARGE_FILE_BYTES = 64 * 1024 * 1024   # 64MB
+MAX_FILE_BYTES = 1024 * 1024 * 1024   # 1GB
+MAX_BUNDLE_BYTES = 1024 * 1024 * 1024   # 1GB

lfss/src/database.py

@@ -7,12 +7,13 @@ import dataclasses, hashlib, uuid
 from contextlib import asynccontextmanager
 from functools import wraps
 from enum import IntEnum
-import zipfile, io
+import zipfile, io, asyncio
 
-import aiosqlite
+import aiosqlite, aiofiles
+import aiofiles.os
 from asyncio import Lock
 
-from .config import DATA_HOME
+from .config import DATA_HOME, LARGE_BLOB_DIR
 from .log import get_logger
 from .utils import decode_uri_compnents
 from .error import *
@@ -47,6 +48,7 @@ class DBConnBase(ABC):
         global _g_conn
         if _g_conn is None:
             _g_conn = await aiosqlite.connect(DATA_HOME / 'lfss.db')
+            await _g_conn.execute('PRAGMA journal_mode=memory')
 
     async def commit(self):
         await self.conn.commit()
@@ -190,15 +192,19 @@ class FileRecord:
     create_time: str
     access_time: str
     permission: FileReadPermission
+    external: bool
 
     def __str__(self):
         return  f"File {self.url} (owner={self.owner_id}, created at {self.create_time}, accessed at {self.access_time}, " + \
-                f"file_id={self.file_id}, permission={self.permission}, size={self.file_size})"
+                f"file_id={self.file_id}, permission={self.permission}, size={self.file_size}, external={self.external})"
 
 @dataclasses.dataclass
 class DirectoryRecord:
     url: str
     size: int
+    create_time: str = ""
+    update_time: str = ""
+    access_time: str = ""
 
     def __str__(self):
         return f"Directory {self.url} (size={self.size})"
@@ -224,7 +230,9 @@ class FileConn(DBConnBase):
             file_size INTEGER,
             create_time TIMESTAMP DEFAULT CURRENT_TIMESTAMP, 
             access_time TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
-            permission INTEGER DEFAULT 0
+            permission INTEGER DEFAULT 0, 
+            external BOOLEAN DEFAULT FALSE,
+            FOREIGN KEY(owner_id) REFERENCES user(id)
         )
         ''')
         await self.conn.execute('''
@@ -256,6 +264,16 @@ class FileConn(DBConnBase):
                     size = await cursor.fetchone()
                 if size is not None and size[0] is not None:
                     await self._user_size_inc(r[0], size[0])
+        
+        # backward compatibility, since 0.5.0
+        # 'external' means the file is not stored in the database, but in the external storage
+        async with self.conn.execute("SELECT * FROM fmeta") as cursor:
+            res = await cursor.fetchone()
+        if res and len(res) < 8:
+            self.logger.info("Updating fmeta table")
+            await self.conn.execute('''
+            ALTER TABLE fmeta ADD COLUMN external BOOLEAN DEFAULT FALSE
+            ''')
 
         return self
     
@@ -278,7 +296,7 @@ class FileConn(DBConnBase):
             res = await cursor.fetchall()
         return [self.parse_record(r) for r in res]
     
-    async def get_path_records(self, url: str) -> list[FileRecord]:
+    async def get_path_file_records(self, url: str) -> list[FileRecord]:
         async with self.conn.execute("SELECT * FROM fmeta WHERE url LIKE ?", (url + '%', )) as cursor:
             res = await cursor.fetchall()
         return [self.parse_record(r) for r in res]
@@ -348,10 +366,27 @@ class FileConn(DBConnBase):
             ) as cursor:
             res = await cursor.fetchall()
         dirs_str = [r[0] + '/' for r in res if r[0] != '/']
-        dirs = [DirectoryRecord(url + d, await self.path_size(url + d, include_subpath=True)) for d in dirs_str]
-            
+        async def get_dir(dir_url):
+            return DirectoryRecord(dir_url, -1)
+        dirs = await asyncio.gather(*[get_dir(url + d) for d in dirs_str])
         return PathContents(dirs, files)
     
+    async def get_path_record(self, url: str) -> Optional[DirectoryRecord]:
+        assert url.endswith('/'), "Path must end with /"
+        async with self.conn.execute("""
+            SELECT MIN(create_time) as create_time, 
+                MAX(create_time) as update_time, 
+                MAX(access_time) as access_time 
+            FROM fmeta 
+            WHERE url LIKE ?
+        """, (url + '%', )) as cursor:
+            result = await cursor.fetchone()
+            if result is None or any(val is None for val in result):
+                raise PathNotFoundError(f"Path {url} not found")
+            create_time, update_time, access_time = result
+        p_size = await self.path_size(url, include_subpath=True)
+        return DirectoryRecord(url, p_size, create_time=create_time, update_time=update_time, access_time=access_time)
+    
     async def user_size(self, user_id: int) -> int:
         async with self.conn.execute("SELECT size FROM usize WHERE user_id = ?", (user_id, )) as cursor:
             res = await cursor.fetchone()
@@ -383,7 +418,8 @@ class FileConn(DBConnBase):
         owner_id: Optional[int] = None, 
         file_id: Optional[str] = None, 
         file_size: Optional[int] = None, 
-        permission: Optional[ FileReadPermission ] = None
+        permission: Optional[ FileReadPermission ] = None, 
+        external: Optional[bool] = None
         ):
 
         old = await self.get_file_record(url)
@@ -392,6 +428,7 @@ class FileConn(DBConnBase):
             # should delete the old blob if file_id is changed
             assert file_id is None, "Cannot update file id"
             assert file_size is None, "Cannot update file size"
+            assert external is None, "Cannot update external"
 
             if owner_id is None: owner_id = old.owner_id
             if permission is None: permission = old.permission
@@ -402,13 +439,13 @@ class FileConn(DBConnBase):
                 """, (owner_id, int(permission), url))
             self.logger.info(f"File {url} updated")
         else:
-            self.logger.debug(f"Creating fmeta {url}: permission={permission}, owner_id={owner_id}, file_id={file_id}, file_size={file_size}")
+            self.logger.debug(f"Creating fmeta {url}: permission={permission}, owner_id={owner_id}, file_id={file_id}, file_size={file_size}, external={external}")
             if permission is None:
                 permission = FileReadPermission.UNSET
-            assert owner_id is not None and file_id is not None and file_size is not None, "Missing required fields"
+            assert owner_id is not None and file_id is not None and file_size is not None and external is not None
             await self.conn.execute(
-                "INSERT INTO fmeta (url, owner_id, file_id, file_size, permission) VALUES (?, ?, ?, ?, ?)", 
-                (url, owner_id, file_id, file_size, int(permission))
+                "INSERT INTO fmeta (url, owner_id, file_id, file_size, permission, external) VALUES (?, ?, ?, ?, ?, ?)", 
+                (url, owner_id, file_id, file_size, int(permission), external)
                 )
             await self._user_size_inc(owner_id, file_size)
             self.logger.info(f"File {url} created")
@@ -465,6 +502,20 @@ class FileConn(DBConnBase):
     async def set_file_blob(self, file_id: str, blob: bytes):
         await self.conn.execute("INSERT OR REPLACE INTO fdata (file_id, data) VALUES (?, ?)", (file_id, blob))
     
+    @atomic
+    async def set_file_blob_external(self, file_id: str, stream: AsyncIterable[bytes])->int:
+        size_sum = 0
+        try:
+            async with aiofiles.open(LARGE_BLOB_DIR / file_id, 'wb') as f:
+                async for chunk in stream:
+                    size_sum += len(chunk)
+                    await f.write(chunk)
+        except Exception as e:
+            if (LARGE_BLOB_DIR / file_id).exists():
+                await aiofiles.os.remove(LARGE_BLOB_DIR / file_id)
+            raise
+        return size_sum
+    
     async def get_file_blob(self, file_id: str) -> Optional[bytes]:
         async with self.conn.execute("SELECT data FROM fdata WHERE file_id = ?", (file_id, )) as cursor:
             res = await cursor.fetchone()
@@ -472,6 +523,17 @@ class FileConn(DBConnBase):
             return None
         return res[0]
     
+    async def get_file_blob_external(self, file_id: str) -> AsyncIterable[bytes]:
+        assert (LARGE_BLOB_DIR / file_id).exists(), f"File {file_id} not found"
+        async with aiofiles.open(LARGE_BLOB_DIR / file_id, 'rb') as f:
+            async for chunk in f:
+                yield chunk
+    
+    @atomic
+    async def delete_file_blob_external(self, file_id: str):
+        if (LARGE_BLOB_DIR / file_id).exists():
+            await aiofiles.os.remove(LARGE_BLOB_DIR / file_id)
+    
     @atomic
     async def delete_file_blob(self, file_id: str):
         await self.conn.execute("DELETE FROM fdata WHERE file_id = ?", (file_id, ))
@@ -543,9 +605,15 @@ class Database:
         if _g_conn is not None:
             await _g_conn.rollback()
 
-    async def save_file(self, u: int | str, url: str, blob: bytes):
+    async def save_file(
+        self, u: int | str, url: str, 
+        blob: bytes | AsyncIterable[bytes], 
+        permission: FileReadPermission = FileReadPermission.UNSET
+        ):
+        """
+        if file_size is not provided, the blob must be bytes
+        """
         validate_url(url)
-        assert isinstance(blob, bytes), "blob must be bytes"
 
         user = await get_user(self, u)
         if user is None:
@@ -562,25 +630,48 @@ class Database:
                 if await get_user(self, first_component) is None:
                     raise PermissionDeniedError(f"Invalid path: {first_component} is not a valid username")
         
-        # check if fize_size is within limit
-        file_size = len(blob)
         user_size_used = await self.file.user_size(user.id)
-        if user_size_used + file_size > user.max_storage:
-            raise StorageExceededError(f"Unable to save file, user {user.username} has storage limit of {user.max_storage}, used {user_size_used}, requested {file_size}")
-
-        f_id = uuid.uuid4().hex
-        async with transaction(self):
-            await self.file.set_file_blob(f_id, blob)
-            await self.file.set_file_record(url, owner_id=user.id, file_id=f_id, file_size=file_size)
-            await self.user.set_active(user.username)
+        if isinstance(blob, bytes):
+            file_size = len(blob)
+            if user_size_used + file_size > user.max_storage:
+                raise StorageExceededError(f"Unable to save file, user {user.username} has storage limit of {user.max_storage}, used {user_size_used}, requested {file_size}")
+            f_id = uuid.uuid4().hex
+            async with transaction(self):
+                await self.file.set_file_blob(f_id, blob)
+                await self.file.set_file_record(
+                    url, owner_id=user.id, file_id=f_id, file_size=file_size, 
+                    permission=permission, external=False)
+                await self.user.set_active(user.username)
+        else:
+            assert isinstance(blob, AsyncIterable)
+            async with transaction(self):
+                f_id = uuid.uuid4().hex
+                file_size = await self.file.set_file_blob_external(f_id, blob)
+                if user_size_used + file_size > user.max_storage:
+                    await self.file.delete_file_blob_external(f_id)
+                    raise StorageExceededError(f"Unable to save file, user {user.username} has storage limit of {user.max_storage}, used {user_size_used}, requested {file_size}")
+                await self.file.set_file_record(
+                    url, owner_id=user.id, file_id=f_id, file_size=file_size, 
+                    permission=permission, external=True)
+                await self.user.set_active(user.username)
+
+    async def read_file_stream(self, url: str) -> AsyncIterable[bytes]:
+        validate_url(url)
+        r = await self.file.get_file_record(url)
+        if r is None:
+            raise FileNotFoundError(f"File {url} not found")
+        if not r.external:
+            raise ValueError(f"File {url} is not stored externally, should use read_file instead")
+        return self.file.get_file_blob_external(r.file_id)
 
-    # async def read_file_stream(self, url: str): ...
     async def read_file(self, url: str) -> bytes:
         validate_url(url)
 
         r = await self.file.get_file_record(url)
         if r is None:
             raise FileNotFoundError(f"File {url} not found")
+        if r.external:
+            raise ValueError(f"File {url} is stored externally, should use read_file_stream instead")
 
         f_id = r.file_id
         blob = await self.file.get_file_blob(f_id)
@@ -600,8 +691,11 @@ class Database:
             if r is None:
                 return None
             f_id = r.file_id
-            await self.file.delete_file_blob(f_id)
             await self.file.delete_file_record(url)
+            if r.external:
+                await self.file.delete_file_blob_external(f_id)
+            else:
+                await self.file.delete_file_blob(f_id)
             return r
     
     async def move_file(self, old_url: str, new_url: str):
@@ -611,17 +705,33 @@ class Database:
         async with transaction(self):
             await self.file.move_file(old_url, new_url)
 
+    async def __batch_delete_file_blobs(self, file_records: list[FileRecord], batch_size: int = 512):
+        # https://github.com/langchain-ai/langchain/issues/10321
+        internal_ids = []
+        external_ids = []
+        for r in file_records:
+            if r.external:
+                external_ids.append(r.file_id)
+            else:
+                internal_ids.append(r.file_id)
+        
+        for i in range(0, len(internal_ids), batch_size):
+            await self.file.delete_file_blobs([r for r in internal_ids[i:i+batch_size]])
+        for i in range(0, len(external_ids)):
+            await self.file.delete_file_blob_external(external_ids[i])
+            
+
     async def delete_path(self, url: str):
         validate_url(url, is_file=False)
 
         async with transaction(self):
-            records = await self.file.get_path_records(url)
+            records = await self.file.get_path_file_records(url)
             if not records:
                 return None
-            await self.file.delete_file_blobs([r.file_id for r in records])
+            await self.__batch_delete_file_blobs(records)
             await self.file.delete_path_records(url)
             return records
-
+    
     async def delete_user(self, u: str | int):
         user = await get_user(self, u)
         if user is None:
@@ -629,11 +739,11 @@ class Database:
         
         async with transaction(self):
             records = await self.file.get_user_file_records(user.id)
-            await self.file.delete_file_blobs([r.file_id for r in records])
+            await self.__batch_delete_file_blobs(records)
             await self.file.delete_user_file_records(user.id)
             await self.user.delete_user(user.username)
     
-    async def iter_path(self, top_url: str, urls: Optional[list[str]]) -> AsyncIterable[tuple[FileRecord, bytes]]:
+    async def iter_path(self, top_url: str, urls: Optional[list[str]]) -> AsyncIterable[tuple[FileRecord, bytes | AsyncIterable[bytes]]]:
         if urls is None:
             urls = [r.url for r in await self.file.list_path(top_url, flat=True)]
 
@@ -644,10 +754,13 @@ class Database:
             if r is None:
                 continue
             f_id = r.file_id
-            blob = await self.file.get_file_blob(f_id)
-            if blob is None:
-                self.logger.warning(f"Blob not found for {url}")
-                continue
+            if r.external:
+                blob = self.file.get_file_blob_external(f_id)
+            else:
+                blob = await self.file.get_file_blob(f_id)
+                if blob is None:
+                    self.logger.warning(f"Blob not found for {url}")
+                    continue
             yield r, blob
 
     async def zip_path(self, top_url: str, urls: Optional[list[str]]) -> io.BytesIO:
@@ -658,7 +771,12 @@ class Database:
             async for (r, blob) in self.iter_path(top_url, urls):
                 rel_path = r.url[len(top_url):]
                 rel_path = decode_uri_compnents(rel_path)
-                zf.writestr(rel_path, blob)
+                if r.external:
+                    assert isinstance(blob, AsyncIterable)
+                    zf.writestr(rel_path, b''.join([chunk async for chunk in blob]))
+                else:
+                    assert isinstance(blob, bytes)
+                    zf.writestr(rel_path, blob)
         buffer.seek(0)
         return buffer
 

lfss/src/error.py

@@ -1,6 +1,8 @@
 
 class LFSSExceptionBase(Exception):...
 
+class PathNotFoundError(LFSSExceptionBase, FileNotFoundError):...
+
 class PermissionDeniedError(LFSSExceptionBase, PermissionError):...
 
 class InvalidPathError(LFSSExceptionBase, ValueError):...

lfss/src/server.py

@@ -2,6 +2,7 @@ from typing import Optional
 from functools import wraps
 
 from fastapi import FastAPI, APIRouter, Depends, Request, Response
+from fastapi.responses import StreamingResponse
 from fastapi.exceptions import HTTPException 
 from fastapi.security import HTTPBearer, HTTPAuthorizationCredentials
 from fastapi.middleware.cors import CORSMiddleware
@@ -14,7 +15,7 @@ from contextlib import asynccontextmanager
 from .error import *
 from .log import get_logger
 from .stat import RequestDB
-from .config import MAX_BUNDLE_BYTES, MAX_FILE_BYTES
+from .config import MAX_BUNDLE_BYTES, MAX_FILE_BYTES, LARGE_BLOB_DIR, LARGE_FILE_BYTES
 from .utils import ensure_uri_compnents, format_last_modified, now_stamp
 from .database import Database, UserRecord, DECOY_USER, FileRecord, check_user_permission, FileReadPermission
 
@@ -141,19 +142,32 @@ async def get_file(path: str, download = False, user: UserRecord = Depends(get_c
     
     fname = path.split("/")[-1]
     async def send(media_type: Optional[str] = None, disposition = "attachment"):
-        fblob = await conn.read_file(path)
-        if media_type is None:
-            media_type, _ = mimetypes.guess_type(fname)
-        if media_type is None:
-            media_type = mimesniff.what(fblob)
-
-        return Response(
-            content=fblob, media_type=media_type, headers={
-                "Content-Disposition": f"{disposition}; filename={fname}", 
-                "Content-Length": str(len(fblob)), 
-                "Last-Modified": format_last_modified(file_record.create_time)
-            }
-        )
+        if not file_record.external:
+            fblob = await conn.read_file(path)
+            if media_type is None:
+                media_type, _ = mimetypes.guess_type(fname)
+            if media_type is None:
+                media_type = mimesniff.what(fblob)
+            return Response(
+                content=fblob, media_type=media_type, headers={
+                    "Content-Disposition": f"{disposition}; filename={fname}", 
+                    "Content-Length": str(len(fblob)), 
+                    "Last-Modified": format_last_modified(file_record.create_time)
+                }
+            )
+        
+        else:
+            if media_type is None:
+                media_type, _ = mimetypes.guess_type(fname)
+            if media_type is None:
+                media_type = mimesniff.what(str((LARGE_BLOB_DIR / file_record.file_id).absolute()))
+            return StreamingResponse(
+                await conn.read_file_stream(path), media_type=media_type, headers={
+                    "Content-Disposition": f"{disposition}; filename={fname}", 
+                    "Content-Length": str(file_record.file_size),
+                    "Last-Modified": format_last_modified(file_record.create_time)
+                }
+            )
     
     if download:
         return await send('application/octet-stream', "attachment")
@@ -162,7 +176,12 @@ async def get_file(path: str, download = False, user: UserRecord = Depends(get_c
 
 @router_fs.put("/{path:path}")
 @handle_exception
-async def put_file(request: Request, path: str, user: UserRecord = Depends(get_current_user)):
+async def put_file(
+    request: Request, 
+    path: str, 
+    overwrite: Optional[bool] = False,
+    permission: int = 0,
+    user: UserRecord = Depends(get_current_user)):
     path = ensure_uri_compnents(path)
     if user.id == 0:
         logger.debug("Reject put request from DECOY_USER")
@@ -182,8 +201,10 @@ async def put_file(request: Request, path: str, user: UserRecord = Depends(get_c
     exists_flag = False
     file_record = await conn.file.get_file_record(path)
     if file_record:
-        exists_flag = True
+        if not overwrite:
+            raise HTTPException(status_code=409, detail="File exists")
         # remove the old file
+        exists_flag = True
         await conn.delete_file(path)
     
     # check content-type
@@ -191,20 +212,25 @@ async def put_file(request: Request, path: str, user: UserRecord = Depends(get_c
     logger.debug(f"Content-Type: {content_type}")
     if content_type == "application/json":
         body = await request.json()
-        await conn.save_file(user.id, path, json.dumps(body).encode('utf-8'))
+        blobs = json.dumps(body).encode('utf-8')
     elif content_type == "application/x-www-form-urlencoded":
         # may not work...
         body = await request.form()
         file = body.get("file")
         if isinstance(file, str) or file is None:
             raise HTTPException(status_code=400, detail="Invalid form data, file required")
-        await conn.save_file(user.id, path, await file.read())
+        blobs = await file.read()
     elif content_type == "application/octet-stream":
-        body = await request.body()
-        await conn.save_file(user.id, path, body)
+        blobs = await request.body()
+    else:
+        blobs = await request.body()
+    if len(blobs) > LARGE_FILE_BYTES:
+        async def blob_reader():
+            for b in range(0, len(blobs), 4096):
+                yield blobs[b:b+4096]
+        await conn.save_file(user.id, path, blob_reader(), permission = FileReadPermission(permission))
     else:
-        body = await request.body()
-        await conn.save_file(user.id, path, body)
+        await conn.save_file(user.id, path, blobs, permission = FileReadPermission(permission))
 
     # https://developer.mozilla.org/zh-CN/docs/Web/HTTP/Methods/PUT
     if exists_flag:
@@ -282,19 +308,18 @@ async def bundle_files(path: str, user: UserRecord = Depends(get_current_user)):
         }
     )
 
-@router_api.get("/fmeta")
+@router_api.get("/meta")
 @handle_exception
 async def get_file_meta(path: str, user: UserRecord = Depends(get_current_user)):
     logger.info(f"GET meta({path}), user: {user.username}")
-    if path.endswith("/"):
-        raise HTTPException(status_code=400, detail="Invalid path")
     path = ensure_uri_compnents(path)
-    file_record = await conn.file.get_file_record(path)
-    if not file_record:
-        raise HTTPException(status_code=404, detail="File not found")
-    return file_record
+    get_fn = conn.file.get_file_record if not path.endswith("/") else conn.file.get_path_record
+    record = await get_fn(path)
+    if not record:
+        raise HTTPException(status_code=404, detail="Path not found")
+    return record
 
-@router_api.post("/fmeta")
+@router_api.post("/meta")
 @handle_exception
 async def update_file_meta(
     path: str, 

lfss/src/stat.py

@@ -32,7 +32,7 @@ class RequestDB:
     async def commit(self):
         await self.conn.commit()
     
-    @debounce_async(0.1)
+    @debounce_async(0.05)
     async def ensure_commit_once(self):
         await self.commit()
     

{lfss-0.4.0.dist-info → lfss-0.5.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: lfss
-Version: 0.4.0
+Version: 0.5.0
 Summary: Lightweight file storage service
 Home-page: https://github.com/MenxLi/lfss
 Author: li, mengxun
@@ -11,6 +11,7 @@ Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
+Requires-Dist: aiofiles (==23.*)
 Requires-Dist: aiosqlite (==0.*)
 Requires-Dist: fastapi (==0.*)
 Requires-Dist: mimesniff (==1.*)

lfss-0.5.0.dist-info/RECORD

@@ -0,0 +1,28 @@
+Readme.md,sha256=8ZAADV1K20gEDhwiL-Qq_rBAePlwWQn8c5uJ_X7OCIg,1128
+docs/Known_issues.md,sha256=rfdG3j1OJF-59S9E06VPyn0nZKbW-ybPxkoZ7MEZWp8,81
+docs/Permission.md,sha256=EY1Y4tT5PDdHDb2pXsVgMAJXxkUihTZfPT0_z7Q53FA,1485
+frontend/api.js,sha256=aIeeaHlkxKUAHLI-BxYxqYqyfJIRxmeFPdXLYyMSdbs,7682
+frontend/index.html,sha256=VPJDs2LG8ep9kjlsKzjWzpN9vc1VGgdvOUlNTZWyQoQ,2088
+frontend/popup.css,sha256=VzkjG1ZTLxhHMtTyobnlvqYmVsTmdbJJed2Pu1cc06c,1007
+frontend/popup.js,sha256=dH5n7C2Vo9gCsMfQ4ajL4D1ETl0Wk9rIldxUb7x0f_c,4634
+frontend/scripts.js,sha256=znnV43tRupV7-GuZUlYIkNpi49J9E0DzYdlodSlTO9s,19486
+frontend/styles.css,sha256=kN2XmEeRXLMyZQowSXYPhD4_1XVXgKwTKdTdtAKuX0E,4102
+frontend/utils.js,sha256=Ts4nlef8pkrEgpwX-uQwAhWvwxlIzex8ijDLNCa22ps,2372
+lfss/cli/cli.py,sha256=2sL0RcJWjnMUjYwSHxO5m_oJP-61WUFJZC9eH3m-Jq8,2223
+lfss/cli/panel.py,sha256=iGdVmdWYjA_7a78ZzWEB_3ggIOBeUKTzg6F5zLaB25c,1401
+lfss/cli/serve.py,sha256=bO3GT0kuylMGN-7bZWP4e71MlugGZ_lEMkYaYld_Ntg,985
+lfss/cli/user.py,sha256=uSNgF8wGYpdOowN8Mah3V_ii6NlxRMNecrrVj3HaemU,3328
+lfss/client/__init__.py,sha256=YttaGTBup7mxnW0CtxdZPF0HWga2cGM4twz9MXJIrts,1827
+lfss/client/api.py,sha256=L2LrnUO7Ef4mUlvZ40_GLu0pqf9wlFgieHgaQXP0Dk0,3827
+lfss/src/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+lfss/src/config.py,sha256=i96PgONQnsmQ34hcqRXUcmo_Ullq2aRQvHbD54pIguM,447
+lfss/src/database.py,sha256=XproulWille_i_aRr3IFE0isRfL98WgY6y6MRcOlS1o,33473
+lfss/src/error.py,sha256=imbhwnbhnI3HLhkbfICROe3F0gleKrOk4XnqHJDOtuI,285
+lfss/src/log.py,sha256=qNE04sHoZ2rMbQ5dR2zT7Xaz1KVAfYp5hKpWJX42S9g,5244
+lfss/src/server.py,sha256=Zl5O3KY16ASB674xaY3YmqnyQ_8d4MLjqZwtcz4Phgs,14345
+lfss/src/stat.py,sha256=hTMtQyM_Ukmhc33Bb9FGCfBMIX02KrGHQg8nL7sC8sU,2082
+lfss/src/utils.py,sha256=8VkrtpSmurbMiX7GyK-n7Grvzy3uwSJXHdONEsuLCDI,2272
+lfss-0.5.0.dist-info/METADATA,sha256=0z_QH6HMzuhFHQwYUapOTACU1tqqrEL_D86ekyYJABc,1820
+lfss-0.5.0.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+lfss-0.5.0.dist-info/entry_points.txt,sha256=_FXOyxodFtVBaxtBhdHcupqW_IolIIx-S6y6p7CkDFk,137
+lfss-0.5.0.dist-info/RECORD,,

{lfss-0.4.0.dist-info → lfss-0.5.0.dist-info}/entry_points.txt

@@ -1,4 +1,5 @@
 [console_scripts]
+lfss-cli=lfss.cli.cli:main
 lfss-panel=lfss.cli.panel:main
 lfss-serve=lfss.cli.serve:main
 lfss-user=lfss.cli.user:main

lfss-0.4.0.dist-info/RECORD

@@ -1,27 +0,0 @@
-Readme.md,sha256=8ZAADV1K20gEDhwiL-Qq_rBAePlwWQn8c5uJ_X7OCIg,1128
-docs/Known_issues.md,sha256=rfdG3j1OJF-59S9E06VPyn0nZKbW-ybPxkoZ7MEZWp8,81
-docs/Permission.md,sha256=EY1Y4tT5PDdHDb2pXsVgMAJXxkUihTZfPT0_z7Q53FA,1485
-frontend/api.js,sha256=D_MaQlmBGzzSR30a23Kh3DxPeF8MpKYe5uXSb9srRTU,7281
-frontend/index.html,sha256=VPJDs2LG8ep9kjlsKzjWzpN9vc1VGgdvOUlNTZWyQoQ,2088
-frontend/popup.css,sha256=VzkjG1ZTLxhHMtTyobnlvqYmVsTmdbJJed2Pu1cc06c,1007
-frontend/popup.js,sha256=dH5n7C2Vo9gCsMfQ4ajL4D1ETl0Wk9rIldxUb7x0f_c,4634
-frontend/scripts.js,sha256=1Cd5cb84SWolozA0Q1MpWaXiAqeQ5R6qZyk7gdoRPgU,18266
-frontend/styles.css,sha256=Ql_-W5dbh6LlJL2Kez8qsqPSF2fckFgmOYP4SMfKJVs,4065
-frontend/utils.js,sha256=biE2te5ezswZyuwlDTYbHEt7VWKfCcUrusNt1lHjkLw,2263
-lfss/cli/panel.py,sha256=iGdVmdWYjA_7a78ZzWEB_3ggIOBeUKTzg6F5zLaB25c,1401
-lfss/cli/serve.py,sha256=bO3GT0kuylMGN-7bZWP4e71MlugGZ_lEMkYaYld_Ntg,985
-lfss/cli/user.py,sha256=906MIQO1mr4XXzPpT8Kfri1G61bWlgjDzIglxypQ7z0,3251
-lfss/client/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-lfss/client/api.py,sha256=VBiU-JxGaN6S-fXYD_KvzG7FsGBIUzZ5IisJJSMooDk,3443
-lfss/src/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-lfss/src/config.py,sha256=mc8QoiWoPgiZ5JnSvKBH8jWbp5uLSyG3l5boDiQPXS0,325
-lfss/src/database.py,sha256=wNDsvjvr7Jq1gDeUd_26tETcfeiplAmywIwXLK1TD80,27923
-lfss/src/error.py,sha256=S5ui3tJ0uKX4EZnt2Db-KbDmJXlECI_OY95cNMkuegc,218
-lfss/src/log.py,sha256=qNE04sHoZ2rMbQ5dR2zT7Xaz1KVAfYp5hKpWJX42S9g,5244
-lfss/src/server.py,sha256=P4GpsIJ869fi6lzHk2IVMyJooGWIvqh0cykkDerJ74k,13190
-lfss/src/stat.py,sha256=_4OaSvBm7D6mPgifwxnhGIEk1_q3SxfJr3lizaEoV_w,2081
-lfss/src/utils.py,sha256=8VkrtpSmurbMiX7GyK-n7Grvzy3uwSJXHdONEsuLCDI,2272
-lfss-0.4.0.dist-info/METADATA,sha256=X0qXQ4gkl_sJ5VNEGLJ0XG0JSqmNnOPixHVn6ZQ7pj0,1787
-lfss-0.4.0.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-lfss-0.4.0.dist-info/entry_points.txt,sha256=nUIJhenyZbcymvPVhrzV7SAtRhd7O52DflbRrpQUC04,110
-lfss-0.4.0.dist-info/RECORD,,