lfss 0.11.1__py3-none-any.whl → 0.11.3__py3-none-any.whl

lfss/eng/database.py

@@ -210,6 +210,10 @@ class FileConn(DBObjectBase):
         return self.parse_record(res)
     
     async def get_file_records(self, urls: list[str]) -> list[FileRecord]:
+        """
+        Get all file records with the given urls, only urls in the database will be returned. 
+        If the urls are not in the database, they will be ignored.
+        """
         await self.cur.execute("SELECT * FROM fmeta WHERE url IN ({})".format(','.join(['?'] * len(urls))), urls)
         res = await self.cur.fetchall()
         if res is None:
@@ -225,12 +229,12 @@ class FileConn(DBObjectBase):
             await self.cur.execute("SELECT username FROM user")
             res = await self.cur.fetchall()
             dirnames = [u[0] + '/' for u in res]
-            dirs = [await self.get_path_record(u) for u in dirnames] if not skim else [DirectoryRecord(u) for u in dirnames]
+            dirs = [await self.get_dir_record(u) for u in dirnames] if not skim else [DirectoryRecord(u) for u in dirnames]
             return dirs
         else:
             # list specific users
             dirnames = [uname + '/' for uname in usernames]
-            dirs = [await self.get_path_record(u) for u in dirnames] if not skim else [DirectoryRecord(u) for u in dirnames]
+            dirs = [await self.get_dir_record(u) for u in dirnames] if not skim else [DirectoryRecord(u) for u in dirnames]
             return dirs
     
     async def count_path_dirs(self, url: str):
@@ -278,11 +282,11 @@ class FileConn(DBObjectBase):
             if skim:
                 return DirectoryRecord(dir_url)
             else:
-                return await self.get_path_record(dir_url)
+                return await self.get_dir_record(dir_url)
         dirs = [await get_dir(url + d) for d in dirs_str]
         return dirs
     
-    async def count_path_files(self, url: str, flat: bool = False):
+    async def count_dir_files(self, url: str, flat: bool = False):
         if not url.endswith('/'): url += '/'
         if url == '/': url = ''
         if flat:
@@ -293,7 +297,7 @@ class FileConn(DBObjectBase):
         assert res is not None, "Error: count_path_files"
         return res[0]
 
-    async def list_path_files(
+    async def list_dir_files(
         self, url: str, 
         offset: int = 0, limit: int = 10_000, 
         order_by: FileSortKey = '', order_desc: bool = False,
@@ -328,15 +332,15 @@ class FileConn(DBObjectBase):
         """
         MAX_ITEMS = 10_000
         dir_count = await self.count_path_dirs(url)
-        file_count = await self.count_path_files(url, flat=False)
+        file_count = await self.count_dir_files(url, flat=False)
         if dir_count + file_count > MAX_ITEMS:
             raise TooManyItemsError("Too many items, please paginate")
         return PathContents(
             dirs = await self.list_path_dirs(url, skim=True, limit=MAX_ITEMS),
-            files = await self.list_path_files(url, flat=False, limit=MAX_ITEMS)
+            files = await self.list_dir_files(url, flat=False, limit=MAX_ITEMS)
             )
     
-    async def get_path_record(self, url: str) -> DirectoryRecord:
+    async def get_dir_record(self, url: str) -> DirectoryRecord:
         """
         Get the full record of a directory, including size, create_time, update_time, access_time etc.
         """
@@ -411,8 +415,11 @@ class FileConn(DBObjectBase):
         await self._user_size_inc(owner_id, file_size)
         self.logger.info(f"File {url} created")
 
-    # not tested
     async def copy_file(self, old_url: str, new_url: str, user_id: Optional[int] = None):
+        """
+        Copy file from old_url to new_url, 
+        if user_id is None, will not change the owner_id of the file. Otherwise, will change the owner_id to user_id.
+        """
         old = await self.get_file_record(old_url)
         if old is None:
             raise FileNotFoundError(f"File {old_url} not found")
@@ -428,15 +435,15 @@ class FileConn(DBObjectBase):
         await self._user_size_inc(user_id, old.file_size)
         self.logger.info(f"Copied file {old_url} to {new_url}")
     
-    async def copy_path(self, old_url: str, new_url: str, user_id: Optional[int] = None):
+    async def copy_dir(self, old_url: str, new_url: str, user_id: Optional[int] = None):
+        """
+        Copy all files under old_url to new_url, 
+        if user_id is None, will not change the owner_id of the files. Otherwise, will change the owner_id to user_id.
+        """
         assert old_url.endswith('/'), "Old path must end with /"
         assert new_url.endswith('/'), "New path must end with /"
-        if user_id is None:
-            cursor = await self.cur.execute("SELECT * FROM fmeta WHERE url LIKE ?", (old_url + '%', ))
-            res = await cursor.fetchall()
-        else:
-            cursor = await self.cur.execute("SELECT * FROM fmeta WHERE url LIKE ? AND owner_id = ?", (old_url + '%', user_id))
-            res = await cursor.fetchall()
+        cursor = await self.cur.execute("SELECT * FROM fmeta WHERE url LIKE ?", (old_url + '%', ))
+        res = await cursor.fetchall()
         for r in res:
             old_record = FileRecord(*r)
             new_r = new_url + old_record.url[len(old_url):]
@@ -461,7 +468,7 @@ class FileConn(DBObjectBase):
         await self.cur.execute("UPDATE fmeta SET url = ?, create_time = CURRENT_TIMESTAMP WHERE url = ?", (new_url, old_url))
         self.logger.info(f"Moved file {old_url} to {new_url}")
     
-    async def move_path(self, old_url: str, new_url: str, user_id: Optional[int] = None):
+    async def move_dir(self, old_url: str, new_url: str, user_id: Optional[int] = None):
         assert old_url.endswith('/'), "Old path must end with /"
         assert new_url.endswith('/'), "New path must end with /"
         if user_id is None:
@@ -500,7 +507,7 @@ class FileConn(DBObjectBase):
         self.logger.info(f"Deleted {len(ret)} file records for user {owner_id}") # type: ignore
         return ret
     
-    async def delete_path_records(self, path: str, under_owner_id: Optional[int] = None) -> list[FileRecord]:
+    async def delete_records_by_prefix(self, path: str, under_owner_id: Optional[int] = None) -> list[FileRecord]:
         """Delete all records with url starting with path"""
         # update user size
         cursor = await self.cur.execute("SELECT DISTINCT owner_id FROM fmeta WHERE url LIKE ?", (path + '%', ))
@@ -689,7 +696,7 @@ async def delayed_log_access(url: str):
         ])
     ),
 )
-def validate_url(url: str, is_file = True):
+def validate_url(url: str, utype: Literal['file', 'dir'] = 'file'):
     """ Check if a path is valid. The input path is considered url safe """
     if len(url) > 1024: 
         raise InvalidPathError(f"URL too long: {url}")
@@ -703,7 +710,7 @@ def validate_url(url: str, is_file = True):
             is_valid = False
             break
 
-    if is_file: is_valid = is_valid and not url.endswith('/')
+    if utype == 'file': is_valid = is_valid and not url.endswith('/')
     else: is_valid = is_valid and url.endswith('/')
 
     if not is_valid: 
@@ -827,6 +834,58 @@ class Database:
                     yield blob
         ret = blob_stream()
         return ret
+    
+    async def read_files_bulk(
+        self, urls: list[str], 
+        skip_content = False, 
+        op_user: Optional[UserRecord] = None, 
+        ) -> dict[str, Optional[bytes]]:
+        """
+        A frequent use case is to read multiple files at once, 
+        this method will read all files in the list and return a dict of url -> blob.
+        if the file is not found, the value will be None.
+        - skip_content: if True, will not read the content of the file, resulting in a dict of url -> b''
+
+        may raise StorageExceededError if the total size of the files exceeds MAX_MEM_FILE_BYTES
+        """
+        for url in urls:
+            validate_url(url)
+        
+        async with unique_cursor() as cur:
+            fconn = FileConn(cur)
+            file_records = await fconn.get_file_records(urls)
+
+            if op_user is not None:
+                for r in file_records:
+                    if await check_path_permission(r.url, op_user, cursor=cur) >= AccessLevel.READ:
+                        continue
+                    is_allowed, reason = await check_file_read_permission(op_user, r, cursor=cur)
+                    if not is_allowed:
+                        raise PermissionDeniedError(f"Permission denied: {op_user.username} cannot read file {r.url}: {reason}")
+
+        # first check if the files are too big
+        sum_size = sum([r.file_size for r in file_records])
+        if not skip_content and sum_size > MAX_MEM_FILE_BYTES:
+            raise StorageExceededError(f"Unable to read files at once, total size {sum_size} exceeds {MAX_MEM_FILE_BYTES}")
+        
+        self.logger.debug(f"Reading {len(file_records)} files{' (skip content)' if skip_content else ''}, getting {sum_size} bytes, from {urls}")
+        # read the file content
+        async with unique_cursor() as cur:
+            fconn = FileConn(cur)
+            blobs: dict[str, bytes] = {}
+            for r in file_records:
+                if skip_content:
+                    blobs[r.url] = b''
+                    continue
+
+                if r.external:
+                    blob_iter = fconn.get_file_blob_external(r.file_id)
+                    blob = b''.join([chunk async for chunk in blob_iter])
+                else:
+                    blob = await fconn.get_file_blob(r.file_id)
+                blobs[r.url] = blob
+            
+            return {url: blobs.get(url, None) for url in urls}
 
     async def delete_file(self, url: str, op_user: Optional[UserRecord] = None) -> Optional[FileRecord]:
         validate_url(url)
@@ -885,9 +944,9 @@ class Database:
                     raise PermissionDeniedError(f"Permission denied: {op_user.username} cannot copy file to {new_url}")
             await fconn.copy_file(old_url, new_url, user_id=op_user.id if op_user is not None else None)
     
-    async def move_path(self, old_url: str, new_url: str, op_user: UserRecord):
-        validate_url(old_url, is_file=False)
-        validate_url(new_url, is_file=False)
+    async def move_dir(self, old_url: str, new_url: str, op_user: UserRecord):
+        validate_url(old_url, 'dir')
+        validate_url(new_url, 'dir')
 
         if new_url.startswith('/'):
             new_url = new_url[1:]
@@ -906,12 +965,11 @@ class Database:
 
         async with transaction() as cur:
             fconn = FileConn(cur)
-            await fconn.move_path(old_url, new_url, op_user.id)
+            await fconn.move_dir(old_url, new_url, op_user.id)
     
-    # not tested
-    async def copy_path(self, old_url: str, new_url: str, op_user: UserRecord):
-        validate_url(old_url, is_file=False)
-        validate_url(new_url, is_file=False)
+    async def copy_dir(self, old_url: str, new_url: str, op_user: UserRecord):
+        validate_url(old_url, 'dir')
+        validate_url(new_url, 'dir')
         
         if new_url.startswith('/'):
             new_url = new_url[1:]
@@ -930,7 +988,7 @@ class Database:
         
         async with transaction() as cur:
             fconn = FileConn(cur)
-            await fconn.copy_path(old_url, new_url, op_user.id)
+            await fconn.copy_dir(old_url, new_url, op_user.id)
 
     async def __batch_delete_file_blobs(self, fconn: FileConn, file_records: list[FileRecord], batch_size: int = 512):
         # https://github.com/langchain-ai/langchain/issues/10321
@@ -951,13 +1009,13 @@ class Database:
         await del_internal()
         await del_external()
 
-    async def delete_path(self, url: str, op_user: Optional[UserRecord] = None) -> Optional[list[FileRecord]]:
-        validate_url(url, is_file=False)
+    async def delete_dir(self, url: str, op_user: Optional[UserRecord] = None) -> Optional[list[FileRecord]]:
+        validate_url(url, 'dir')
         from_owner_id = op_user.id if op_user is not None and not (op_user.is_admin or await check_path_permission(url, op_user) >= AccessLevel.WRITE) else None
 
         async with transaction() as cur:
             fconn = FileConn(cur)
-            records = await fconn.delete_path_records(url, from_owner_id)
+            records = await fconn.delete_records_by_prefix(url, from_owner_id)
             if not records:
                 return None
             await self.__batch_delete_file_blobs(fconn, records)
@@ -981,14 +1039,15 @@ class Database:
 
             # make sure the user's directory is deleted, 
             # may contain admin's files, but delete them all
-            await fconn.delete_path_records(user.username + '/')
+            await fconn.delete_records_by_prefix(user.username + '/')
     
-    async def iter_path(self, top_url: str, urls: Optional[list[str]]) -> AsyncIterable[tuple[FileRecord, bytes | AsyncIterable[bytes]]]:
+    async def iter_dir(self, top_url: str, urls: Optional[list[str]]) -> AsyncIterable[tuple[FileRecord, bytes | AsyncIterable[bytes]]]:
+        validate_url(top_url, 'dir')
         async with unique_cursor() as cur:
             fconn = FileConn(cur)
             if urls is None:
-                fcount = await fconn.count_path_files(top_url, flat=True)
-                urls = [r.url for r in (await fconn.list_path_files(top_url, flat=True, limit=fcount))]
+                fcount = await fconn.count_dir_files(top_url, flat=True)
+                urls = [r.url for r in (await fconn.list_dir_files(top_url, flat=True, limit=fcount))]
 
             for url in urls:
                 if not url.startswith(top_url):
@@ -1003,7 +1062,7 @@ class Database:
                     blob = await fconn.get_file_blob(f_id)
                 yield r, blob
     
-    async def zip_path_stream(self, top_url: str, op_user: Optional[UserRecord] = None) -> AsyncIterable[bytes]:
+    async def zip_dir_stream(self, top_url: str, op_user: Optional[UserRecord] = None) -> AsyncIterable[bytes]:
         from stat import S_IFREG
         from stream_zip import async_stream_zip, ZIP_64
         if top_url.startswith('/'):
@@ -1015,7 +1074,7 @@ class Database:
         
         # https://stream-zip.docs.trade.gov.uk/async-interface/
         async def data_iter():
-            async for (r, blob) in self.iter_path(top_url, None):
+            async for (r, blob) in self.iter_dir(top_url, None):
                 rel_path = r.url[len(top_url):]
                 rel_path = decode_uri_compnents(rel_path)
                 b_iter: AsyncIterable[bytes]
@@ -1035,7 +1094,7 @@ class Database:
         return async_stream_zip(data_iter())
 
     @concurrent_wrap()
-    async def zip_path(self, top_url: str, op_user: Optional[UserRecord]) -> io.BytesIO:
+    async def zip_dir(self, top_url: str, op_user: Optional[UserRecord]) -> io.BytesIO:
         if top_url.startswith('/'):
             top_url = top_url[1:]
         
@@ -1045,7 +1104,7 @@ class Database:
         
         buffer = io.BytesIO()
         with zipfile.ZipFile(buffer, 'w') as zf:
-            async for (r, blob) in self.iter_path(top_url, None):
+            async for (r, blob) in self.iter_dir(top_url, None):
                 rel_path = r.url[len(top_url):]
                 rel_path = decode_uri_compnents(rel_path)
                 if r.external:

lfss/eng/log.py

@@ -1,8 +1,9 @@
-from .config import DATA_HOME
+from .config import LOG_DIR, DISABLE_LOGGING
+import time, sqlite3, dataclasses
 from typing import TypeVar, Callable, Literal, Optional
 from concurrent.futures import ThreadPoolExecutor
 from functools import wraps
-import logging, pathlib, asyncio
+import logging, asyncio
 from logging import handlers
 
 class BCOLORS:
@@ -57,15 +58,81 @@ class BaseLogger(logging.Logger):
     @thread_wrap
     def error(self, *args, **kwargs): super().error(*args, **kwargs)
 
-_fh_T = Literal['rotate', 'simple', 'daily']
+class SQLiteFileHandler(logging.FileHandler):
+    def __init__(self, filename, *args, **kwargs):
+        super().__init__(filename, *args, **kwargs)
+        self._db_file = filename
+        self._buffer: list[logging.LogRecord] = []
+        self._buffer_size = 100
+        self._flush_interval = 10
+        self._last_flush = time.time()
+        conn = sqlite3.connect(self._db_file, check_same_thread=False)
+        conn.execute('PRAGMA journal_mode=WAL')
+        conn.execute('''
+            CREATE TABLE IF NOT EXISTS log (
+                id INTEGER PRIMARY KEY AUTOINCREMENT,
+                created TIMESTAMP,
+                created_epoch FLOAT,
+                name TEXT,
+                levelname VARCHAR(16),
+                level INTEGER,
+                message TEXT
+            )
+        ''')
+        conn.commit()
+        conn.close()
+    
+    def flush(self): 
+        def format_time(self, record: logging.LogRecord):
+            """ Create a time stamp """
+            return time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(record.created))
+        self.acquire()
+        try:
+            conn = sqlite3.connect(self._db_file, check_same_thread=False)
+            conn.executemany('''
+                INSERT INTO log (created, created_epoch, name, levelname, level, message)
+                VALUES (?, ?, ?, ?, ?, ?)
+            ''', [
+                (format_time(self, record), record.created, record.name, record.levelname, record.levelno, record.getMessage())
+                for record in self._buffer
+            ])
+            conn.commit()
+            conn.close()
+            self._buffer.clear()
+            self._last_flush = time.time()
+        finally:
+            self.release()
+
+    def emit(self, record: logging.LogRecord):
+        self._buffer.append(record)
+        if len(self._buffer) > self._buffer_size or time.time() - self._last_flush > self._flush_interval:
+            self.flush()
+    
+    def close(self):
+        self.flush()
+        return super().close()
+
+def eval_logline(row: sqlite3.Row):
+    @dataclasses.dataclass
+    class DBLogRecord:
+        id: int
+        created: str
+        created_epoch: float
+        name: str
+        levelname: str
+        level: int
+        message: str
+    return DBLogRecord(*row)
+
+_fh_T = Literal['rotate', 'simple', 'daily', 'sqlite']
 
 __g_logger_dict: dict[str, BaseLogger] = {}
 def get_logger(
     name = 'default', 
-    log_home = pathlib.Path(DATA_HOME) / 'logs', 
+    log_home = LOG_DIR, 
     level = 'DEBUG',
     term_level = 'INFO',
-    file_handler_type: _fh_T = 'rotate', 
+    file_handler_type: _fh_T = 'sqlite', 
     global_instance = True
     )->BaseLogger:
     if global_instance and name in __g_logger_dict:
@@ -87,22 +154,25 @@ def get_logger(
             if isinstance(color, str) and color.startswith('\033'):
                 format_str_plain = format_str_plain.replace(color, '')
 
-        formatter_plain = logging.Formatter(format_str_plain)
-        log_home.mkdir(exist_ok=True)
-        log_file = log_home / f'{name}.log'
-        if file_handler_type == 'simple':
-            file_handler = logging.FileHandler(log_file)
-        elif file_handler_type == 'daily':
-            file_handler = handlers.TimedRotatingFileHandler(
-                log_file, when='midnight', interval=1, backupCount=30
-            )
-        elif file_handler_type == 'rotate':
-            file_handler = handlers.RotatingFileHandler(
-                log_file, maxBytes=1024*1024, backupCount=5
-            )
-
-        file_handler.setFormatter(formatter_plain)
-        logger.addHandler(file_handler)
+        if not DISABLE_LOGGING:
+            formatter_plain = logging.Formatter(format_str_plain)
+            log_home.mkdir(exist_ok=True)
+            log_file = log_home / f'{name}.log'
+            if file_handler_type == 'simple':
+                file_handler = logging.FileHandler(log_file)
+            elif file_handler_type == 'daily':
+                file_handler = handlers.TimedRotatingFileHandler(
+                    log_file, when='midnight', interval=1, backupCount=30
+                )
+            elif file_handler_type == 'rotate':
+                file_handler = handlers.RotatingFileHandler(
+                    log_file, maxBytes=1024*1024, backupCount=5
+                )
+            elif file_handler_type == 'sqlite':
+                file_handler = SQLiteFileHandler(log_file if log_file.suffix == '.db' else log_file.with_suffix('.log.db'))
+
+            file_handler.setFormatter(formatter_plain)
+            logger.addHandler(file_handler)
     
     logger = BaseLogger(name)
     setupLogger(logger)

lfss/eng/utils.py

@@ -11,7 +11,6 @@ from concurrent.futures import ThreadPoolExecutor
 from typing import TypeVar, Callable, Awaitable
 from functools import wraps, partial
 from uuid import uuid4
-import os
 
 async def copy_file(source: str|pathlib.Path, destination: str|pathlib.Path):
     async with aiofiles.open(source, mode='rb') as src:
@@ -160,7 +159,7 @@ _g_executor = None
 def get_global_executor():
     global _g_executor
     if _g_executor is None:
-        _g_executor = ThreadPoolExecutor(max_workers=4 if (cpu_count:=os.cpu_count()) and cpu_count > 4 else cpu_count)
+        _g_executor = ThreadPoolExecutor()
     return _g_executor
 def async_wrap(executor=None):
     if executor is None:

lfss/svc/app_base.py

@@ -60,10 +60,13 @@ def handle_exception(fn):
             raise 
     return wrapper
 
+env_origins = os.environ.get("LFSS_ORIGINS", "*")
+logger.debug(f"LFSS_ORIGINS: {env_origins}")
+origins = [x.strip() for x in env_origins.split(",") if x.strip()]
 app = FastAPI(docs_url=None, redoc_url=None, lifespan=lifespan)
 app.add_middleware(
     CORSMiddleware,
-    allow_origins=["*"],
+    allow_origins=origins,
     allow_credentials=True,
     allow_methods=["*"],
     allow_headers=["*"],

lfss/svc/app_dav.py

@@ -57,9 +57,9 @@ async def eval_path(path: str) -> tuple[ptype, str, Optional[FileRecord | Direct
         if len(dir_path_sp) > 2:
             async with unique_cursor() as c:
                 fconn = FileConn(c)
-                if await fconn.count_path_files(path, flat=True) == 0:
+                if await fconn.count_dir_files(path, flat=True) == 0:
                     return None, lfss_path, None
-                return "dir", lfss_path, await fconn.get_path_record(path)
+                return "dir", lfss_path, await fconn.get_dir_record(path)
         else:
             # test if its a user's root directory
             assert len(dir_path_sp) == 2
@@ -85,8 +85,8 @@ async def eval_path(path: str) -> tuple[ptype, str, Optional[FileRecord | Direct
     async with unique_cursor() as c:
         lfss_path = path + "/"
         fconn = FileConn(c)
-        if await fconn.count_path_files(lfss_path) > 0:
-            return "dir", lfss_path, await fconn.get_path_record(lfss_path)
+        if await fconn.count_dir_files(lfss_path) > 0:
+            return "dir", lfss_path, await fconn.get_dir_record(lfss_path)
     
     return None, path, None
 
@@ -235,7 +235,7 @@ async def dav_propfind(request: Request, path: str, user: UserRecord = Depends(r
         # query root directory content
         async def user_path_record(user_name: str, cur) -> DirectoryRecord:
             try:
-                return await FileConn(cur).get_path_record(user_name + "/")
+                return await FileConn(cur).get_dir_record(user_name + "/")
             except PathNotFoundError:
                 return DirectoryRecord(user_name + "/", size=0, n_files=0, create_time="1970-01-01 00:00:00", update_time="1970-01-01 00:00:00", access_time="1970-01-01 00:00:00")
 
@@ -253,7 +253,7 @@ async def dav_propfind(request: Request, path: str, user: UserRecord = Depends(r
     elif path_type == "dir":
         # query directory content
         async with unique_cursor() as c:
-            flist = await FileConn(c).list_path_files(lfss_path, flat = True if depth == "infinity" else False)
+            flist = await FileConn(c).list_dir_files(lfss_path, flat = True if depth == "infinity" else False)
         for frecord in flist:
             if frecord.url.endswith(f"/{MKDIR_PLACEHOLDER}"): continue
             file_el = await create_file_xml_element(frecord)
@@ -315,7 +315,7 @@ async def dav_move(request: Request, path: str, user: UserRecord = Depends(regis
         assert ptype == "dir", "Directory path should end with /"
         assert lfss_path.endswith("/"), "Directory path should end with /"
         if not dlfss_path.endswith("/"): dlfss_path += "/"  # the header destination may not end with /
-        await db.move_path(lfss_path, dlfss_path, user)
+        await db.move_dir(lfss_path, dlfss_path, user)
     return Response(status_code=201)
 
 @router_dav.api_route("/{path:path}", methods=["COPY"])