lfss 0.7.3__tar.gz → 0.7.5__tar.gz

{lfss-0.7.3 → lfss-0.7.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: lfss
-Version: 0.7.3
+Version: 0.7.5
 Summary: Lightweight file storage service
 Home-page: https://github.com/MenxLi/lfss
 Author: li, mengxun

{lfss-0.7.3 → lfss-0.7.5}/docs/Permission.md

@@ -18,6 +18,13 @@ Non-admin users can access files based on:
 - If the file is `unset`, then the file's permission is inherited from the owner's permission.
 - If both the owner and the file have `unset` permission, then the file is `public`.
 
+### Meta-data access
+- Non-login users can't access any file-meta.
+- All users can access the file-meta of files under their own path.
+- For files under other users' path, the file-meta is determined in a way same as file access.
+- Admins can access the path-meta of all users.
+- All users can access the path-meta of their own path.
+
 ### Path-listing
 - Non-login users cannot list any files.
 - All users can list the files under their own path 

{lfss-0.7.3 → lfss-0.7.5}/frontend/api.js

@@ -133,12 +133,17 @@ export default class Connector {
 
     /**
      * @param {string} path - the path to the file directory, should ends with '/'
+     * @param {Object} options - the options for the request
      * @returns {Promise<PathListResponse>} - the promise of the request
      */
-    async listPath(path){
+    async listPath(path, {
+        flat = false
+    } = {}){
         if (path.startsWith('/')){ path = path.slice(1); }
         if (!path.endsWith('/')){ path += '/'; }
-        const res = await fetch(this.config.endpoint + '/' + path, {
+        const dst = new URL(this.config.endpoint + '/' + path);
+        dst.searchParams.append('flat', flat);
+        const res = await fetch(dst.toString(), {
             method: 'GET',
             headers: {
                 'Authorization': 'Bearer ' + this.config.token

lfss-0.7.5/lfss/cli/cli.py

@@ -0,0 +1,95 @@
+from lfss.client import Connector, upload_directory, upload_file, download_file, download_directory
+from lfss.src.datatype import FileReadPermission
+from pathlib import Path
+import argparse
+
+def parse_arguments():
+    parser = argparse.ArgumentParser(description="Command line interface, please set LFSS_ENDPOINT and LFSS_TOKEN environment variables.")
+    parser.add_argument("-v", "--verbose", action="store_true", help="Verbose output")
+
+    sp = parser.add_subparsers(dest="command", required=True)
+
+    # upload
+    sp_upload = sp.add_parser("upload", help="Upload files")
+    sp_upload.add_argument("src", help="Source file or directory", type=str)
+    sp_upload.add_argument("dst", help="Destination url path", type=str)
+    sp_upload.add_argument("-j", "--jobs", type=int, default=1, help="Number of concurrent uploads")
+    sp_upload.add_argument("--interval", type=float, default=0, help="Interval between files, only works with directory upload")
+    sp_upload.add_argument("--conflict", choices=["overwrite", "abort", "skip", "skip-ahead"], default="abort", help="Conflict resolution")
+    sp_upload.add_argument("--permission", type=FileReadPermission, default=FileReadPermission.UNSET, help="File permission")
+    sp_upload.add_argument("--retries", type=int, default=0, help="Number of retries")
+
+    # download
+    sp_download = sp.add_parser("download", help="Download files")
+    sp_download.add_argument("src", help="Source url path", type=str)
+    sp_download.add_argument("dst", help="Destination file or directory", type=str)
+    sp_download.add_argument("-j", "--jobs", type=int, default=1, help="Number of concurrent downloads")
+    sp_download.add_argument("--interval", type=float, default=0, help="Interval between files, only works with directory download")
+    sp_download.add_argument("--overwrite", action="store_true", help="Overwrite existing files")
+    sp_download.add_argument("--retries", type=int, default=0, help="Number of retries")
+
+    return parser.parse_args()
+
+def main():
+    args = parse_arguments()
+    connector = Connector()
+    if args.command == "upload":
+        src_path = Path(args.src)
+        if src_path.is_dir():
+            failed_upload = upload_directory(
+                connector, args.src, args.dst, 
+                verbose=args.verbose,
+                n_concurrent=args.jobs, 
+                n_retries=args.retries, 
+                interval=args.interval,
+                conflict=args.conflict,
+                permission=args.permission
+            )
+            if failed_upload:
+                print("Failed to upload:")
+                for path in failed_upload:
+                    print(f"  {path}")
+        else:
+            success = upload_file(
+                connector, 
+                file_path = args.src, 
+                dst_url = args.dst, 
+                verbose=args.verbose,
+                n_retries=args.retries, 
+                interval=args.interval,
+                conflict=args.conflict,
+                permission=args.permission
+            )
+            if not success:
+                print("Failed to upload.")
+    
+    elif args.command == "download":
+        is_dir = args.src.endswith("/")
+        if is_dir:
+            failed_download = download_directory(
+                connector, args.src, args.dst, 
+                verbose=args.verbose,
+                n_concurrent=args.jobs, 
+                n_retries=args.retries, 
+                interval=args.interval,
+                overwrite=args.overwrite
+            )
+            if failed_download:
+                print("Failed to download:")
+                for path in failed_download:
+                    print(f"  {path}")
+        else:
+            success = download_file(
+                connector, 
+                src_url = args.src, 
+                file_path = args.dst, 
+                verbose=args.verbose,
+                n_retries=args.retries, 
+                interval=args.interval,
+                overwrite=args.overwrite
+            )
+            if not success:
+                print("Failed to download.")
+    else:
+        raise NotImplementedError(f"Command {args.command} not implemented.")
+    

lfss-0.7.5/lfss/client/__init__.py

@@ -0,0 +1,155 @@
+import os, time, pathlib
+from threading import Lock
+from concurrent.futures import ThreadPoolExecutor
+from .api import Connector
+
+def upload_file(
+    connector: Connector, 
+    file_path: str, 
+    dst_url: str, 
+    n_retries: int = 0, 
+    interval: float = 0, 
+    verbose: bool = False, 
+    **put_kwargs
+    ):
+    this_try = 0
+    while this_try <= n_retries:
+        try:
+            with open(file_path, 'rb') as f:
+                blob = f.read()
+            connector.put(dst_url, blob, **put_kwargs)
+            break
+        except Exception as e:
+            if isinstance(e, KeyboardInterrupt):
+                raise e
+            if verbose:
+                print(f"Error uploading {file_path}: {e}, retrying...")
+            this_try += 1
+        finally:
+            time.sleep(interval)
+
+    if this_try > n_retries:
+        if verbose:
+            print(f"Failed to upload {file_path} after {n_retries} retries.")
+        return False
+    return True
+
+def upload_directory(
+    connector: Connector, 
+    directory: str, 
+    path: str, 
+    n_concurrent: int = 1,
+    n_retries: int = 0,
+    interval: float = 0,
+    verbose: bool = False,
+    **put_kwargs
+    ) -> list[str]:
+    assert path.endswith('/'), "Path must end with a slash."
+    if path.startswith('/'):
+        path = path[1:]
+    directory = str(directory)
+    
+    _counter = 0
+    _counter_lock = Lock()
+
+    faild_files = []
+    def put_file(file_path):
+        with _counter_lock:
+            nonlocal _counter
+            _counter += 1
+            this_count = _counter
+            dst_path = f"{path}{os.path.relpath(file_path, directory)}"
+            if verbose:
+                print(f"[{this_count}] Uploading {file_path} to {dst_path}")
+
+        if not upload_file(
+            connector, file_path, dst_path, 
+            n_retries=n_retries, interval=interval, verbose=verbose, **put_kwargs
+            ):
+            faild_files.append(file_path)
+
+    with ThreadPoolExecutor(n_concurrent) as executor:
+        for root, dirs, files in os.walk(directory):
+            for file in files:
+                executor.submit(put_file, os.path.join(root, file))
+
+    return faild_files
+
+def download_file(
+    connector: Connector, 
+    src_url: str, 
+    file_path: str, 
+    n_retries: int = 0, 
+    interval: float = 0, 
+    verbose: bool = False, 
+    overwrite: bool = False
+    ):
+    this_try = 0
+    while this_try <= n_retries:
+        if not overwrite and os.path.exists(file_path):
+            if verbose:
+                print(f"File {file_path} already exists, skipping download.")
+            return True
+        try:
+            blob = connector.get(src_url)
+            if not blob:
+                return False
+            pathlib.Path(file_path).parent.mkdir(parents=True, exist_ok=True)
+            with open(file_path, 'wb') as f:
+                f.write(blob)
+            break
+        except Exception as e:
+            if isinstance(e, KeyboardInterrupt):
+                raise e
+            if verbose:
+                print(f"Error downloading {src_url}: {e}, retrying...")
+            this_try += 1
+        finally:
+            time.sleep(interval)
+
+    if this_try > n_retries:
+        if verbose:
+            print(f"Failed to download {src_url} after {n_retries} retries.")
+        return False
+    return True
+
+def download_directory(
+    connector: Connector,
+    src_path: str,
+    directory: str,
+    n_concurrent: int = 1,
+    n_retries: int = 0,
+    interval: float = 0,
+    verbose: bool = False,
+    overwrite: bool = False
+    ) -> list[str]:
+
+    directory = str(directory)
+
+    if not src_path.endswith('/'):
+        src_path += '/'
+    if not directory.endswith(os.sep):
+        directory += os.sep
+    
+    _counter = 0
+    _counter_lock = Lock()
+    failed_files = []
+    def get_file(src_url):
+        nonlocal _counter, failed_files
+        with _counter_lock:
+            _counter += 1
+            this_count = _counter
+            dst_path = f"{directory}{os.path.relpath(src_url, src_path)}"
+            if verbose:
+                print(f"[{this_count}] Downloading {src_url} to {dst_path}")
+
+        if not download_file(
+            connector, src_url, dst_path, 
+            n_retries=n_retries, interval=interval, verbose=verbose, overwrite=overwrite
+            ):
+            failed_files.append(src_url)
+        
+    with ThreadPoolExecutor(n_concurrent) as executor:
+        for file in connector.list_path(src_path, flat=True).files:
+            executor.submit(get_file, file.url)
+    return failed_files

{lfss-0.7.3 → lfss-0.7.5}/lfss/client/api.py

@@ -5,6 +5,7 @@ import urllib.parse
 from lfss.src.datatype import (
     FileReadPermission, FileRecord, DirectoryRecord, UserRecord, PathContents
     )
+from lfss.src.utils import ensure_uri_compnents
 
 _default_endpoint = os.environ.get('LFSS_ENDPOINT', 'http://localhost:8000')
 _default_token = os.environ.get('LFSS_TOKEN', '')
@@ -23,6 +24,7 @@ class Connector:
     ):
         if path.startswith('/'):
             path = path[1:]
+        path = ensure_uri_compnents(path)
         def f(**kwargs):
             url = f"{self.config['endpoint']}/{path}" + "?" + urllib.parse.urlencode(search_params)
             headers: dict = kwargs.pop('headers', {})
@@ -115,9 +117,9 @@ class Connector:
                 return None
             raise e
     
-    def list_path(self, path: str) -> PathContents:
+    def list_path(self, path: str, flat: bool = False) -> PathContents:
         assert path.endswith('/')
-        response = self._fetch_factory('GET', path)()
+        response = self._fetch_factory('GET', path, {'flat': flat})()
         dirs = [DirectoryRecord(**d) for d in response.json()['dirs']]
         files = [FileRecord(**f) for f in response.json()['files']]
         return PathContents(dirs=dirs, files=files)

{lfss-0.7.3 → lfss-0.7.5}/lfss/src/config.py

@@ -1,5 +1,5 @@
 from pathlib import Path
-import os, hashlib
+import os
 
 __default_dir = '.storage_data'
 
@@ -15,6 +15,3 @@ LARGE_BLOB_DIR.mkdir(exist_ok=True)
 LARGE_FILE_BYTES = 8 * 1024 * 1024   # 8MB
 MAX_FILE_BYTES = 512 * 1024 * 1024   # 512MB
 MAX_BUNDLE_BYTES = 512 * 1024 * 1024   # 512MB
-
-def hash_credential(username, password):
-    return hashlib.sha256((username + password).encode()).hexdigest()

{lfss-0.7.3 → lfss-0.7.5}/lfss/src/connection_pool.py

@@ -18,13 +18,22 @@ async def execute_sql(conn: aiosqlite.Connection | aiosqlite.Cursor, name: str):
     for s in sql:
         await conn.execute(s)
 
-async def get_connection() -> aiosqlite.Connection:
+async def get_connection(read_only: bool = False) -> aiosqlite.Connection:
     if not os.environ.get('SQLITE_TEMPDIR'):
         os.environ['SQLITE_TEMPDIR'] = str(DATA_HOME)
-    # large blobs are stored in a separate database, should be more efficient
-    conn = await aiosqlite.connect(DATA_HOME / 'index.db', timeout = 60)
+
+    def get_db_uri(path: Path, read_only: bool = False):
+        return f"file:{path}?mode={ 'ro' if read_only else 'rwc' }"
+
+    conn = await aiosqlite.connect(
+        get_db_uri(DATA_HOME / 'index.db', read_only=read_only), 
+        timeout = 60, uri = True
+        )
     async with conn.cursor() as c:
-        await c.execute(f"ATTACH DATABASE ? AS blobs", (str(DATA_HOME/'blobs.db'), ))
+        await c.execute(
+            f"ATTACH DATABASE ? AS blobs", 
+            (get_db_uri(DATA_HOME/'blobs.db', read_only=read_only), )
+            )
     await execute_sql(conn, 'pragma.sql')
     return conn
 
@@ -35,47 +44,49 @@ class SqlConnection:
     is_available: bool = True
 
 class SqlConnectionPool:
-    _sem: Semaphore
+    _r_sem: Semaphore
     _w_sem: Semaphore
     def __init__(self):
-        self._connections: list[SqlConnection] = []
-        self._w_connection: None | SqlConnection = None
+        self._readers: list[SqlConnection] = []
+        self._writer: None | SqlConnection = None
         self._lock = Lock()
     
     async def init(self, n_read: int):
         await self.close()
-        self._connections = []
-        for _ in range(n_read):
-            conn = await get_connection()
-            self._connections.append(SqlConnection(conn))
-        self._w_connection = SqlConnection(await get_connection())
-        self._sem = Semaphore(n_read)
+        self._readers = []
+
+        self._writer = SqlConnection(await get_connection(read_only=False))
         self._w_sem = Semaphore(1)
+
+        for _ in range(n_read):
+            conn = await get_connection(read_only=True)
+            self._readers.append(SqlConnection(conn))
+        self._r_sem = Semaphore(n_read)
     
     @property
     def n_read(self):
-        return len(self._connections)
+        return len(self._readers)
     @property
-    def sem(self):
-        return self._sem
+    def r_sem(self):
+        return self._r_sem
     @property
     def w_sem(self):
         return self._w_sem
     
     async def get(self, w: bool = False) -> SqlConnection:
-        if len(self._connections) == 0:
+        if len(self._readers) == 0:
             raise Exception("No available connections, please init the pool first")
         
         async with self._lock:
             if w:
-                assert self._w_connection
-                if self._w_connection.is_available:
-                    self._w_connection.is_available = False
-                    return self._w_connection
+                assert self._writer
+                if self._writer.is_available:
+                    self._writer.is_available = False
+                    return self._writer
                 raise Exception("Write connection is not available")
 
             else:
-                for c in self._connections:
+                for c in self._readers:
                     if c.is_available:
                         c.is_available = False
                         return c
@@ -83,19 +94,19 @@ class SqlConnectionPool:
     
     async def release(self, conn: SqlConnection):
         async with self._lock:
-            if conn == self._w_connection:
+            if conn == self._writer:
                 conn.is_available = True
                 return
 
-            if not conn in self._connections:
+            if not conn in self._readers:
                 raise Exception("Connection not in pool")
             conn.is_available = True
     
     async def close(self):
-        for c in self._connections:
+        for c in self._readers:
             await c.conn.close()
-        if self._w_connection:
-            await self._w_connection.conn.close()
+        if self._writer:
+            await self._writer.conn.close()
 
 # these two functions shold be called before and after the event loop
 g_pool = SqlConnectionPool()
@@ -125,7 +136,7 @@ def global_entrance(n_read: int = 1):
 @asynccontextmanager
 async def unique_cursor(is_write: bool = False):
     if not is_write:
-        async with g_pool.sem:
+        async with g_pool.r_sem:
             connection_obj = await g_pool.get()
             try:
                 yield await connection_obj.conn.cursor()

{lfss-0.7.3 → lfss-0.7.5}/lfss/src/database.py

@@ -1,5 +1,5 @@
 
-from typing import Optional, overload, Literal, AsyncIterable
+from typing import Optional, Literal, AsyncIterable
 from abc import ABC
 
 import urllib.parse
@@ -11,9 +11,9 @@ import aiofiles.os
 
 from .connection_pool import execute_sql, unique_cursor, transaction
 from .datatype import UserRecord, FileReadPermission, FileRecord, DirectoryRecord, PathContents
-from .config import LARGE_BLOB_DIR, hash_credential
+from .config import LARGE_BLOB_DIR
 from .log import get_logger
-from .utils import decode_uri_compnents
+from .utils import decode_uri_compnents, hash_credential
 from .error import *
 
 class DBObjectBase(ABC):
@@ -29,9 +29,6 @@ class DBObjectBase(ABC):
             raise ValueError("Connection not set")
         return self._cur
 
-    # async def commit(self):
-    #     await self.conn.commit()
-
 DECOY_USER = UserRecord(0, 'decoy', 'decoy', False, '2021-01-01 00:00:00', '2021-01-01 00:00:00', 0, FileReadPermission.PRIVATE)
 class UserConn(DBObjectBase):
 
@@ -43,10 +40,6 @@ class UserConn(DBObjectBase):
     def parse_record(record) -> UserRecord:
         return UserRecord(*record)
 
-    async def init(self, cur: aiosqlite.Cursor):
-        self.set_cursor(cur)
-        return self
-    
     async def get_user(self, username: str) -> Optional[UserRecord]:
         await self.cur.execute("SELECT * FROM user WHERE username = ?", (username, ))
         res = await self.cur.fetchone()
@@ -132,10 +125,6 @@ class FileConn(DBObjectBase):
     def parse_record(record) -> FileRecord:
         return FileRecord(*record)
     
-    def init(self, cur: aiosqlite.Cursor):
-        self.set_cursor(cur)
-        return self
-    
     async def get_file_record(self, url: str) -> Optional[FileRecord]:
         cursor = await self.cur.execute("SELECT * FROM fmeta WHERE url = ?", (url, ))
         res = await cursor.fetchone()
@@ -150,7 +139,7 @@ class FileConn(DBObjectBase):
             return []
         return [self.parse_record(r) for r in res]
     
-    async def list_root(self, *usernames: str) -> list[DirectoryRecord]:
+    async def list_root_dirs(self, *usernames: str) -> list[DirectoryRecord]:
         """
         Efficiently list users' directories, if usernames is empty, list all users' directories.
         """
@@ -167,17 +156,12 @@ class FileConn(DBObjectBase):
             dirs = [DirectoryRecord(u, await self.path_size(u, include_subpath=True)) for u in dirnames]
             return dirs
     
-    @overload
-    async def list_path(self, url: str, flat: Literal[True]) -> list[FileRecord]:...
-    @overload
-    async def list_path(self, url: str, flat: Literal[False]) -> PathContents:...
-    
-    async def list_path(self, url: str, flat: bool = False) -> list[FileRecord] | PathContents:
+    async def list_path(self, url: str, flat: bool = False) -> PathContents:
         """
-        List all files and directories under the given path, 
-        if flat is True, return a list of FileDBRecord, recursively including all subdirectories. 
-        Otherwise, return a tuple of (dirs, files), where dirs is a list of DirectoryRecord,
+        List all files and directories under the given path
+        if flat is True, list all files under the path, with out delimiting directories
         """
+        self.logger.debug(f"Listing path {url}, flat={flat}")
         if not url.endswith('/'):
             url += '/'
         if url == '/':
@@ -186,15 +170,17 @@ class FileConn(DBObjectBase):
             if flat:
                 cursor = await self.cur.execute("SELECT * FROM fmeta")
                 res = await cursor.fetchall()
-                return [self.parse_record(r) for r in res]
+                files = [self.parse_record(r) for r in res]
+                return PathContents([], files)
 
             else:
-                return PathContents(await self.list_root(), [])
+                return PathContents(await self.list_root_dirs(), [])
         
         if flat:
             cursor = await self.cur.execute("SELECT * FROM fmeta WHERE url LIKE ?", (url + '%', ))
             res = await cursor.fetchall()
-            return [self.parse_record(r) for r in res]
+            files = [self.parse_record(r) for r in res]
+            return PathContents([], files)
 
         cursor = await self.cur.execute("SELECT * FROM fmeta WHERE url LIKE ? AND url NOT LIKE ?", (url + '%', url + '%/%'))
         res = await cursor.fetchall()
@@ -358,7 +344,7 @@ class FileConn(DBObjectBase):
                 await self._user_size_dec(r[0], size[0])
         
         # if any new records are created here, the size update may be inconsistent
-        # but it's not a big deal...
+        # but it's not a big deal... we should have only one writer
         
         if under_user_id is None:
             res = await self.cur.execute("DELETE FROM fmeta WHERE url LIKE ? RETURNING *", (path + '%', ))
@@ -432,7 +418,7 @@ async def get_user(cur: aiosqlite.Cursor, user: int | str) -> Optional[UserRecor
     else:
         return None
 
-# mostly transactional operations
+# higher level database operations, mostly transactional
 class Database:
     logger = get_logger('database', global_instance=True)
 
@@ -615,11 +601,13 @@ class Database:
             else:
                 internal_ids.append(r.file_id)
         
-        for i in range(0, len(internal_ids), batch_size):
-            await fconn.delete_file_blobs([r for r in internal_ids[i:i+batch_size]])
-        for i in range(0, len(external_ids)):
-            await fconn.delete_file_blob_external(external_ids[i])
-            
+        async def del_internal():
+            for i in range(0, len(internal_ids), batch_size):
+                await fconn.delete_file_blobs([r for r in internal_ids[i:i+batch_size]])
+        async def del_external():
+            for i in range(0, len(external_ids)):
+                await fconn.delete_file_blob_external(external_ids[i])
+        await asyncio.gather(del_internal(), del_external())
 
     async def delete_path(self, url: str, under_user: Optional[UserRecord] = None) -> Optional[list[FileRecord]]:
         validate_url(url, is_file=False)
@@ -655,7 +643,7 @@ class Database:
         async with unique_cursor() as cur:
             fconn = FileConn(cur)
             if urls is None:
-                urls = [r.url for r in await fconn.list_path(top_url, flat=True)]
+                urls = [r.url for r in (await fconn.list_path(top_url, flat=True)).files]
 
             for url in urls:
                 if not url.startswith(top_url):

{lfss-0.7.3 → lfss-0.7.5}/lfss/src/log.py

@@ -151,9 +151,6 @@ def log_access(
             return wrapper          # type: ignore
     return _log_access
 
-def get_dummy_logger() -> BaseLogger:
-    return BaseLogger('dummy')
-
 __ALL__ = [
-    'get_logger', 'clear_handlers', 'log_access', 'get_dummy_logger'
+    'get_logger', 'log_access'
 ]

{lfss-0.7.3 → lfss-0.7.5}/lfss/src/server.py

@@ -18,7 +18,7 @@ from .stat import RequestDB
 from .config import MAX_BUNDLE_BYTES, MAX_FILE_BYTES, LARGE_FILE_BYTES
 from .utils import ensure_uri_compnents, format_last_modified, now_stamp
 from .connection_pool import global_connection_init, global_connection_close, unique_cursor
-from .database import Database, UserRecord, DECOY_USER, FileRecord, check_user_permission, FileReadPermission, UserConn, FileConn
+from .database import Database, UserRecord, DECOY_USER, FileRecord, check_user_permission, FileReadPermission, UserConn, FileConn, PathContents
 
 logger = get_logger("server", term_level="DEBUG")
 logger_failed_request = get_logger("failed_requests", term_level="INFO")
@@ -120,7 +120,7 @@ router_fs = APIRouter(prefix="")
 
 @router_fs.get("/{path:path}")
 @handle_exception
-async def get_file(path: str, download = False, user: UserRecord = Depends(get_current_user)):
+async def get_file(path: str, download: bool = False, flat: bool = False, user: UserRecord = Depends(get_current_user)):
     path = ensure_uri_compnents(path)
 
     # handle directory query
@@ -130,18 +130,20 @@ async def get_file(path: str, download = False, user: UserRecord = Depends(get_c
         async with unique_cursor() as conn:
             fconn = FileConn(conn)
             if user.id == 0:
-                raise HTTPException(status_code=403, detail="Permission denied, credential required")
+                raise HTTPException(status_code=401, detail="Permission denied, credential required")
             if path == "/":
-                return {
-                    "dirs": await fconn.list_root(user.username) \
-                        if not user.is_admin else await fconn.list_root(),
-                    "files": []
-                }
+                if flat:
+                    raise HTTPException(status_code=400, detail="Flat query not supported for root path")
+                return PathContents(
+                    dirs = await fconn.list_root_dirs(user.username) \
+                        if not user.is_admin else await fconn.list_root_dirs(),
+                    files = []
+                )
 
             if not path.startswith(f"{user.username}/") and not user.is_admin:
                 raise HTTPException(status_code=403, detail="Permission denied, path must start with username")
 
-            return await fconn.list_path(path, flat = False)
+            return await fconn.list_path(path, flat = flat)
     
     async with unique_cursor() as conn:
         fconn = FileConn(conn)
@@ -217,7 +219,6 @@ async def put_file(
             return Response(status_code=200, headers={
                 "Content-Type": "application/json",
             }, content=json.dumps({"url": path}))
-        # remove the old file
         exists_flag = True
         if not user.is_admin and not file_record.owner_id == user.id:
             raise HTTPException(status_code=403, detail="Permission denied, cannot overwrite other's file")
@@ -317,7 +318,7 @@ async def bundle_files(path: str, user: UserRecord = Depends(registered_user)):
     
     async with unique_cursor() as conn:
         fconn = FileConn(conn)
-        files = await fconn.list_path(path, flat = True)
+        files = (await fconn.list_path(path, flat = True)).files
     files = [f for f in files if await is_access_granted(f)]
     if len(files) == 0:
         raise HTTPException(status_code=404, detail="No files found")
@@ -341,13 +342,24 @@ async def bundle_files(path: str, user: UserRecord = Depends(registered_user)):
 async def get_file_meta(path: str, user: UserRecord = Depends(registered_user)):
     logger.info(f"GET meta({path}), user: {user.username}")
     path = ensure_uri_compnents(path)
+    is_file = not path.endswith("/")
     async with unique_cursor() as conn:
         fconn = FileConn(conn)
-        get_fn = fconn.get_file_record if not path.endswith("/") else fconn.get_path_record
-        record = await get_fn(path)
-
-    if not record:
-        raise HTTPException(status_code=404, detail="Path not found")
+        if is_file:
+            record = await fconn.get_file_record(path)
+            if not record:
+                raise HTTPException(status_code=404, detail="File not found")
+            if not path.startswith(f"{user.username}/") and not user.is_admin:
+                uconn = UserConn(conn)
+                owner = await uconn.get_user_by_id(record.owner_id)
+                assert owner is not None, "Owner not found"
+                is_allowed, reason = check_user_permission(user, owner, record)
+                if not is_allowed:
+                    raise HTTPException(status_code=403, detail=reason)
+        else:
+            record = await fconn.get_path_record(path)
+            if not path.startswith(f"{user.username}/") and not user.is_admin:
+                raise HTTPException(status_code=403, detail="Permission denied")
     return record
 
 @router_api.post("/meta")

{lfss-0.7.3 → lfss-0.7.5}/lfss/src/utils.py

@@ -2,35 +2,27 @@ import datetime
 import urllib.parse
 import asyncio
 import functools
+import hashlib
+
+def hash_credential(username: str, password: str):
+    return hashlib.sha256((username + password).encode()).hexdigest()
 
 def encode_uri_compnents(path: str):
-    """
-    Encode the path components to encode the special characters, 
-    also to avoid path traversal attack
-    """
     path_sp = path.split("/")
     mapped = map(lambda x: urllib.parse.quote(x), path_sp)
     return "/".join(mapped)
 
 def decode_uri_compnents(path: str):
-    """
-    Decode the path components to decode the special characters
-    """
     path_sp = path.split("/")
     mapped = map(lambda x: urllib.parse.unquote(x), path_sp)
     return "/".join(mapped)
 
 def ensure_uri_compnents(path: str):
-    """
-    Ensure the path components are safe to use
-    """
+    """ Ensure the path components are safe to use """
     return encode_uri_compnents(decode_uri_compnents(path))
 
 def debounce_async(delay: float = 0):
-    """
-    Decorator to debounce the async function (procedure)
-    The function must return None
-    """
+    """ Debounce the async procedure """
     def debounce_wrap(func):
         # https://docs.python.org/3/library/asyncio-task.html#asyncio.Task.cancel
         async def delayed_func(*args, **kwargs):
@@ -51,10 +43,9 @@ def debounce_async(delay: float = 0):
         return wrapper
     return debounce_wrap
 
-# https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Last-Modified
 def format_last_modified(last_modified_gmt: str):
     """
-    Format the last modified time to the HTTP standard format
+    Format the last modified time to the [HTTP standard format](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Last-Modified)
     - last_modified_gmt: The last modified time in SQLite ISO 8601 GMT format: e.g. '2021-09-01 12:00:00'
     """
     assert len(last_modified_gmt) == 19

{lfss-0.7.3 → lfss-0.7.5}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "lfss"
-version = "0.7.3"
+version = "0.7.5"
 description = "Lightweight file storage service"
 authors = ["li, mengxun <limengxun45@outlook.com>"]
 readme = "Readme.md"

lfss-0.7.3/lfss/cli/cli.py

@@ -1,52 +0,0 @@
-from lfss.client import Connector, upload_directory
-from lfss.src.datatype import FileReadPermission
-from pathlib import Path
-import argparse
-
-def parse_arguments():
-    parser = argparse.ArgumentParser(description="Command line interface, please set LFSS_ENDPOINT and LFSS_TOKEN environment variables.")
-
-    sp = parser.add_subparsers(dest="command", required=True)
-
-    # upload
-    sp_upload = sp.add_parser("upload", help="Upload files")
-    sp_upload.add_argument("src", help="Source file or directory", type=str)
-    sp_upload.add_argument("dst", help="Destination path", type=str)
-    sp_upload.add_argument("-j", "--jobs", type=int, default=1, help="Number of concurrent uploads")
-    sp_upload.add_argument("--interval", type=float, default=0, help="Interval between files, only works with directory upload")
-    sp_upload.add_argument("--conflict", choices=["overwrite", "abort", "skip", "skip-ahead"], default="abort", help="Conflict resolution")
-    sp_upload.add_argument("--permission", type=FileReadPermission, default=FileReadPermission.UNSET, help="File permission")
-    sp_upload.add_argument("--retries", type=int, default=0, help="Number of retries, only works with directory upload")
-
-    return parser.parse_args()
-
-def main():
-    args = parse_arguments()
-    connector = Connector()
-    if args.command == "upload":
-        src_path = Path(args.src)
-        if src_path.is_dir():
-            failed_upload = upload_directory(
-                connector, args.src, args.dst, 
-                verbose=True, 
-                n_concurrent=args.jobs, 
-                n_reties=args.retries, 
-                interval=args.interval,
-                conflict=args.conflict,
-                permission=args.permission
-            )
-            if failed_upload:
-                print("Failed to upload:")
-                for path in failed_upload:
-                    print(f"  {path}")
-        else:
-            with open(args.src, 'rb') as f:
-                connector.put(
-                    args.dst, 
-                    f.read(), 
-                    conflict=args.conflict,
-                    permission=args.permission
-                    )
-    else:
-        raise NotImplementedError(f"Command {args.command} not implemented.")
-    

lfss-0.7.3/lfss/client/__init__.py

@@ -1,60 +0,0 @@
-import os, time
-from threading import Lock
-from concurrent.futures import ThreadPoolExecutor
-from .api import Connector
-
-def upload_directory(
-    connector: Connector, 
-    directory: str, 
-    path: str, 
-    n_concurrent: int = 1,
-    n_reties: int = 0,
-    interval: float = 0,
-    verbose: bool = False,
-    **put_kwargs
-    ) -> list[str]:
-    assert path.endswith('/'), "Path must end with a slash."
-    if path.startswith('/'):
-        path = path[1:]
-    
-    _counter = 0
-    _counter_lock = Lock()
-
-    faild_files = []
-    def put_file(file_path):
-        with _counter_lock:
-            nonlocal _counter
-            _counter += 1
-            this_count = _counter
-            dst_path = f"{path}{os.path.relpath(file_path, directory)}"
-            if verbose:
-                print(f"[{this_count}] Uploading {file_path} to {dst_path}")
-
-        this_try = 0
-        with open(file_path, 'rb') as f:
-            blob = f.read()
-
-        while this_try <= n_reties:
-            try:
-                connector.put(dst_path, blob, **put_kwargs)
-                break
-            except Exception as e:
-                if isinstance(e, KeyboardInterrupt):
-                    raise e
-                if verbose:
-                    print(f"[{this_count}] Error uploading {file_path}: {e}, retrying...")
-                this_try += 1
-            finally:
-                time.sleep(interval)
-
-        if this_try > n_reties:
-            faild_files.append(file_path)
-            if verbose:
-                print(f"[{this_count}] Failed to upload {file_path} after {n_reties} retries.")
-
-    with ThreadPoolExecutor(n_concurrent) as executor:
-        for root, dirs, files in os.walk(directory):
-            for file in files:
-                executor.submit(put_file, os.path.join(root, file))
-
-    return faild_files