lfss 0.7.15__py3-none-any.whl → 0.8.1__py3-none-any.whl

Readme.md

@@ -1,7 +1,7 @@
 # Lightweight File Storage Service (LFSS)
 [![PyPI](https://img.shields.io/pypi/v/lfss)](https://pypi.org/project/lfss/)
 
-My experiment on a lightweight file/object storage service.  
+My experiment on a lightweight and high-performance file/object storage service.  
 It stores small files and metadata in sqlite, large files in the filesystem.  
 Tested on 2 million files, and it works fine... 
 
@@ -23,7 +23,7 @@ Or, you can start a web server at `/frontend` and open `index.html` in your brow
 
 The API usage is simple, just `GET`, `PUT`, `DELETE` to the `/<username>/file/url` path.  
 Authentication is done via `Authorization` header with the value `Bearer <token>`, or through the `token` query parameter.  
-You can refer to `frontend` as an application example, and `frontend/api.js` or `lfss/client/api.py` for the API usage.
+You can refer to `frontend` as an application example, and `frontend/api.js` or `lfss/api/connector.py` for the API usage.
 
 By default, the service exposes all files to the public for `GET` requests, 
 but file-listing is restricted to the user's own files.  

docs/Permission.md

@@ -1,13 +1,15 @@
 
 # Permission System
-There are two roles in the system: Admin and User ('user' are like 'bucket' to some extent).
+There are two user roles in the system: Admin and Normal User ("users" are like "buckets" to some extent).
+
+## Ownership
+A file is owned by the user who created it, may not necessarily be the user under whose path the file is stored (admin can create files under any user's path).
 
 ## File access with `GET` permission
 The `GET` is used to access the file (if path is not ending with `/`), or to list the files under a path (if path is ending with `/`).  
 
 ### File access
 For accessing file content, the user must have `GET` permission of the file, which is determined by the `permission` field of both the owner and the file.   
-(Note: The owner of the file is the user who created the file, may not necessarily be the user under whose path the file is stored.)   
 
 There are four types of permissions: `unset`, `public`, `protected`, `private`.
 Non-admin users can access files based on:   

frontend/api.js

@@ -34,6 +34,8 @@
  * @property {DirectoryRecord[]} dirs - the list of directories in the directory
  * @property {FileRecord[]} files - the list of files in the directory
  * 
+ * @typedef {"" | "url" | "file_size" | "create_time" | "access_time" | "mime_type"} FileSortKey
+ * @typedef {"" | "dirname" } DirectorySortKey
  */
 
 export const permMap = {
@@ -71,7 +73,8 @@ export default class Connector {
             method: 'PUT',
             headers: {
                 'Authorization': 'Bearer ' + this.config.token, 
-                'Content-Type': 'application/octet-stream'
+                'Content-Type': 'application/octet-stream', 
+                'Content-Length': fileBytes.byteLength
             },
             body: fileBytes
         });
@@ -81,6 +84,38 @@ export default class Connector {
         return (await res.json()).url;
     }
 
+    /**
+    * @param {string} path - the path to the file (url)
+    * @param {File} file - the file to upload
+    * @returns {Promise<string>} - the promise of the request, the url of the file
+    */
+    async post(path, file, {
+        conflict = 'abort',
+        permission = 0
+    } = {}){
+        if (path.startsWith('/')){ path = path.slice(1); }
+        const dst = new URL(this.config.endpoint + '/' + path);
+        dst.searchParams.append('conflict', conflict);
+        dst.searchParams.append('permission', permission);
+        // post as multipart form data
+        const formData = new FormData();
+        formData.append('file', file);
+        const res = await fetch(dst.toString(), {
+            method: 'POST',
+            // don't include the content type, let the browser handle it
+            // https://muffinman.io/blog/uploading-files-using-fetch-multipart-form-data/
+            headers: {
+                'Authorization': 'Bearer ' + this.config.token, 
+            },
+            body: formData
+        });
+                
+        if (res.status != 200 && res.status != 201){
+            throw new Error(`Failed to upload file, status code: ${res.status}, message: ${await res.json()}`);
+        }
+        return (await res.json()).url;
+    }
+
     /**
     * @param {string} path - the path to the file (url), should end with .json
     * @param {Objec} data - the data to upload
@@ -133,18 +168,18 @@ export default class Connector {
         return await res.json();
     }
 
+    _sanitizeDirPath(path){
+        if (path.startsWith('/')){ path = path.slice(1); }
+        if (!path.endsWith('/')){ path += '/'; }
+        return path;
+    }
     /**
      * @param {string} path - the path to the file directory, should ends with '/'
-     * @param {Object} options - the options for the request
      * @returns {Promise<PathListResponse>} - the promise of the request
      */
-    async listPath(path, {
-        flat = false
-    } = {}){
-        if (path.startsWith('/')){ path = path.slice(1); }
-        if (!path.endsWith('/')){ path += '/'; }
+    async listPath(path){
+        path = this._sanitizeDirPath(path);
         const dst = new URL(this.config.endpoint + '/' + path);
-        dst.searchParams.append('flat', flat);
         const res = await fetch(dst.toString(), {
             method: 'GET',
             headers: {
@@ -157,6 +192,128 @@ export default class Connector {
         return await res.json();
     }
 
+    /**
+     * @param {string} path - the path to the file directory, should ends with '/'
+     * @param {boolean} flat - whether to list the files in subdirectories
+     * @returns {Promise<number>} - the promise of the request
+     * */
+    async countFiles(path, {
+        flat = false
+    } = {}){
+        path = this._sanitizeDirPath(path);
+        const dst = new URL(this.config.endpoint + '/_api/count-files');
+        dst.searchParams.append('path', path);
+        dst.searchParams.append('flat', flat);
+        const res = await fetch(dst.toString(), {
+            method: 'GET',
+            headers: {
+                'Authorization': 'Bearer ' + this.config.token
+            },
+        });
+        if (res.status != 200){
+            throw new Error(`Failed to count files, status code: ${res.status}, message: ${await res.json()}`);
+        }
+        return (await res.json()).count;
+    }
+
+    /**
+     * @typedef {Object} ListFilesOptions
+     * @property {number} offset - the offset of the list
+     * @property {number} limit - the limit of the list
+     * @property {FileSortKey} orderBy - the key to order the files
+     * @property {boolean} orderDesc - whether to order the files in descending order
+     * @property {boolean} flat - whether to list the files in subdirectories
+     * 
+     * @param {string} path - the path to the file directory, should ends with '/'
+     * @param {ListFilesOptions} options - the options for the request
+     * @returns {Promise<FileRecord[]>} - the promise of the request
+     */
+    async listFiles(path, {
+        offset = 0,
+        limit = 1000,
+        orderBy = 'create_time',
+        orderDesc = false,
+        flat = false
+    } = {}){
+        path = this._sanitizeDirPath(path);
+        const dst = new URL(this.config.endpoint + '/_api/list-files');
+        dst.searchParams.append('path', path);
+        dst.searchParams.append('offset', offset);
+        dst.searchParams.append('limit', limit);
+        dst.searchParams.append('order_by', orderBy);
+        dst.searchParams.append('order_desc', orderDesc);
+        dst.searchParams.append('flat', flat);
+        const res = await fetch(dst.toString(), {
+            method: 'GET',
+            headers: {
+                'Authorization': 'Bearer ' + this.config.token
+            },
+        });
+        if (res.status != 200){
+            throw new Error(`Failed to list files, status code: ${res.status}, message: ${await res.json()}`);
+        }
+        return await res.json();
+    }
+
+    /**
+     * @param {string} path - the path to the file directory, should ends with '/'
+     * @returns {Promise<number>} - the promise of the request
+     **/
+    async countDirs(path){
+        path = this._sanitizeDirPath(path);
+        const dst = new URL(this.config.endpoint + '/_api/count-dirs');
+        dst.searchParams.append('path', path);
+        const res = await fetch(dst.toString(), {
+            method: 'GET',
+            headers: {
+                'Authorization': 'Bearer ' + this.config.token
+            },
+        });
+        if (res.status != 200){
+            throw new Error(`Failed to count directories, status code: ${res.status}, message: ${await res.json()}`);
+        }
+        return (await res.json()).count;
+    }
+
+    /**
+     * @typedef {Object} ListDirsOptions
+     * @property {number} offset - the offset of the list
+     * @property {number} limit - the limit of the list
+     * @property {DirectorySortKey} orderBy - the key to order the directories
+     * @property {boolean} orderDesc - whether to order the directories in descending order
+     * @property {boolean} skim - whether to skim the directories
+     * 
+     * @param {string} path - the path to the file directory, should ends with '/'
+     * @param {ListDirsOptions} options - the options for the request
+     * @returns {Promise<DirectoryRecord[]>} - the promise of the request
+     **/
+    async listDirs(path, {
+        offset = 0,
+        limit = 1000,
+        orderBy = 'dirname',
+        orderDesc = false, 
+        skim = true
+    } = {}){
+        path = this._sanitizeDirPath(path);
+        const dst = new URL(this.config.endpoint + '/_api/list-dirs');
+        dst.searchParams.append('path', path);
+        dst.searchParams.append('offset', offset);
+        dst.searchParams.append('limit', limit);
+        dst.searchParams.append('order_by', orderBy);
+        dst.searchParams.append('order_desc', orderDesc);
+        dst.searchParams.append('skim', skim);
+        const res = await fetch(dst.toString(), {
+            method: 'GET',
+            headers: {
+                'Authorization': 'Bearer ' + this.config.token
+            },
+        });
+        if (res.status != 200){
+            throw new Error(`Failed to list directories, status code: ${res.status}, message: ${await res.json()}`);
+        }
+        return await res.json();
+    }
+
     /**
      * Check the user information by the token
      * @returns {Promise<UserRecord>} - the promise of the request
@@ -216,4 +373,110 @@ export default class Connector {
         }
     }
 
+}
+
+/**
+ * a function to wrap the listDirs and listFiles function into one
+ * it will return the list of directories and files in the directory, 
+ * making directories first (if the offset is less than the number of directories), 
+ * and files after that.
+ * 
+ * 
+ * @typedef {Object} ListPathOptions
+ * @property {number} offset - the offset of the list
+ * @property {number} limit - the limit of the list
+ * @property {ListFilesOptions} orderBy - the key to order the files (if set to url, will list the directories using dirname)
+ * @property {boolean} orderDesc - whether to order the files in descending order
+ * 
+ * @param {Connector} conn - the connector to the API
+ * @param {string} path - the path to the file directory
+ * @param {Object} options - the options for the request
+ * @returns {Promise<[PathListResponse, {dirs: number, files: number}]>} - the promise of the request
+ */
+export async function listPath(conn, path, {
+    offset = 0,
+    limit = 1000,
+    orderBy = '',
+    orderDesc = false,
+} = {}){
+
+    if (path === '/' || path === ''){
+        // this handles separate case for the root directory... please refer to the backend implementation
+        return [await conn.listPath(''), {dirs: 0, files: 0}];
+    }
+
+    orderBy = orderBy == 'none' ? '' : orderBy;
+    console.debug('listPath', path, offset, limit, orderBy, orderDesc);
+
+    const [dirCount, fileCount] = await Promise.all([
+        conn.countDirs(path),
+        conn.countFiles(path)
+    ]);
+
+    const dirOffset = offset;
+    const fileOffset = Math.max(offset - dirCount, 0);
+    const dirThispage = Math.max(Math.min(dirCount - dirOffset, limit), 0);
+    const fileLimit = limit - dirThispage;
+
+    console.debug('dirCount', dirCount, 'dirOffset', dirOffset, 'fileOffset', fileOffset, 'dirThispage', dirThispage, 'fileLimit', fileLimit);
+
+    const dirOrderBy = orderBy == 'url' ? 'dirname' : '';
+    const fileOrderBy = orderBy;
+
+    const [dirList, fileList] = await Promise.all([
+        (async () => {
+            if (offset < dirCount) {
+                return await conn.listDirs(path, {
+                    offset: dirOffset,
+                    limit: dirThispage,
+                    orderBy: dirOrderBy,
+                    orderDesc: orderDesc
+                });
+            }
+            return [];
+        })(), 
+        (async () => {
+            if (fileLimit >= 0 && fileCount > fileOffset) {
+                return await conn.listFiles(path, {
+                    offset: fileOffset,
+                    limit: fileLimit,
+                    orderBy: fileOrderBy,
+                    orderDesc: orderDesc
+                });
+            }
+            return [];
+        })()
+    ]);
+
+    return [{
+        dirs: dirList,
+        files: fileList
+    }, {
+        dirs: dirCount,
+        files: fileCount
+    }];
+};
+
+/**
+ * a function to wrap the upload function into one
+ * it will return the url of the file
+ * 
+ * @typedef {Object} UploadOptions
+ * @property {string} conflict - the conflict resolution strategy, can be 'abort', 'replace', 'rename'
+ * @property {number} permission - the permission of the file, can be 0, 1, 2, 3
+ * 
+ * @param {Connector} conn - the connector to the API
+ * @param {string} path - the path to the file (url)
+ * @param {File} file - the file to upload
+ * @param {UploadOptions} options - the options for the request
+ * @returns {Promise<string>} - the promise of the request, the url of the file
+ */
+export async function uploadFile(conn, path, file, {
+    conflict = 'abort',
+    permission = 0
+} = {}){
+    if (file.size < 1024 * 1024 * 10){
+        return await conn.put(path, file, {conflict, permission});
+    }
+    return await conn.post(path, file, {conflict, permission});
 }

frontend/index.html

@@ -8,44 +8,56 @@
 </head>
 <body>
 
-    <div class="container header" id="settings">
-        <div class="input-group" style="min-width: 300px;">
-            <label for="endpoint">Endpoint</label>
-            <input type="text" id="endpoint" placeholder="http://localhost:8000" autocomplete="off">
-        </div>
-        <div class="input-group" style="min-width: 300px;">
-            <label for="token">Token</label>
-            <input type="text" id="token" placeholder="" autocomplete="off">
-        </div>
+    <div class="container header">
         <div class="input-group">
             <span id='back-btn'>⬅️</span>
             <input type="text" id="path" placeholder="path/to/files/" autocomplete="off">
         </div>
-    </div>
-
-    <div class="container content" id="view">
-        <div id="top-bar">
+        <div id="settings-bar">
             <div id="position-hint" class='disconnected'>
                 <span></span>
                 <label></label>
             </div>
-            <div>
-                <span style="margin-right: 0.25rem; color: #999; font-size: small;">
-                Sort by</span>
-                <select id="sort-by-sel">
-                    <option value="none">None</option>
-                    <option value="name">Name</option>
-                    <option value="size">Size</option>
-                    <option value="access">Accessed</option>
-                    <option value="create">Created</option>
-                    <option value="mime">Type</option>
-                </select>
-                <select id="sort-order-sel">
-                    <option value="asc">Ascending</option>
-                    <option value="desc">Descending</option>
-                </select>
+            <div style="color: #999; font-size: small; display: flex; align-items: center; gap: .75rem;">
+                <div class="page-config">
+                    <span>Limit</span>
+                    <select id="page-limit-sel">
+                        <option value="10">10</option>
+                        <option value="50">50</option>
+                        <option value="100">100</option>
+                        <option value="500">500</option>
+                        <option value="1000">1000</option>
+                        <option value="5000">5000</option>
+                    </select>
+                </div>
+
+                <div class="page-config">
+                    <span>Page</span>
+                    <input type="number" id="page-num-input" value="1" min="1" style="width: 50px;">
+                    /
+                    <label id="page-count-lbl">1</label>
+                </div>
+
+                <div class="page-config">
+                    <span>Sort by</span>
+                    <select id="sort-by-sel">
+                        <option value="none">None</option>
+                        <option value="url">Name</option>
+                        <option value="file_size">Size</option>
+                        <option value="access_time">Accessed</option>
+                        <option value="create_time">Created</option>
+                        <option value="mime_type">Type</option>
+                    </select>
+                    <select id="sort-order-sel">
+                        <option value="asc">Ascending</option>
+                        <option value="desc">Descending</option>
+                    </select>
+                </div>
             </div>
         </div>
+    </div>
+
+    <div class="container content" id="view">
         <table id="files">
             <thead>
                 <tr>

frontend/login.css

@@ -0,0 +1,21 @@
+
+div#login-container{
+    display: flex;
+    flex-direction: column;
+    justify-content: center;
+    align-items: center;
+    gap: 0.5rem;
+    width: calc(max(70vw, 420px));
+    min-width: 420px;
+}
+
+label.login-lbl{
+    text-align: left;
+    font-weight: bold;
+}
+
+button#login-btn{
+    padding: 0.8rem;
+    padding-inline: 3rem;
+    margin-top: 0.5rem;
+}

frontend/login.js

@@ -0,0 +1,83 @@
+
+import { createFloatingWindow, showPopup } from "./popup.js";
+
+/**
+ * @import { store } from "./state.js";
+ * @import { UserRecord } from "./api.js";
+ * 
+ * Shows the login panel if necessary.
+ * @param {store} store - The store object.
+ * @returns {Promise<UserRecord>} - The user record.
+ */
+export async function maybeShowLoginPanel(
+    store, 
+    forceShowup = false
+){
+    if (!forceShowup){
+        try {
+            const user = await store.conn.whoami();
+            if (user.id !== 0){
+                return user;
+            }
+        }
+        catch (e) {
+            console.error(e);
+        }
+    }
+
+    const innerHTML = `
+    <div id="login-container">
+        <div class="input-group" style="min-width: 300px;">
+            <label for="endpoint-input" class="login-lbl">Endpoint</label>
+            <input type="text" id="endpoint-input" placeholder="http://localhost:8000" autocomplete="off">
+        </div>
+        <div class="input-group" style="min-width: 300px;">
+            <label for="token-input" class="login-lbl">Token</label>
+            <input type="text" id="token-input" placeholder="" autocomplete="off">
+        </div>
+
+        <button id="login-btn">Login</button>
+    </div>
+    `
+
+    const [win, closeWin] = createFloatingWindow(innerHTML, {
+        padding: '2rem',
+    });
+
+    const endpointInput = document.getElementById("endpoint-input");
+    const tokenInput = document.getElementById("token-input");
+    const loginBtn = document.getElementById("login-btn");
+
+    endpointInput.value = store.endpoint;
+    tokenInput.value = store.token;
+
+    endpointInput.addEventListener('keydown', (e) => {
+        if (e.key === 'Enter'){ loginBtn.click(); }
+    });
+    tokenInput.addEventListener('keydown', (e) => {
+        if (e.key === 'Enter'){ loginBtn.click(); }
+    });
+    endpointInput.addEventListener('input', () => {
+        store.endpoint = endpointInput.value;
+    });
+    tokenInput.addEventListener('input', () => {
+        store.token = tokenInput.value;
+    });
+
+    loginBtn.focus();
+
+    return new Promise((resolve, reject) => {
+        loginBtn.addEventListener('click', async () => {
+            try {
+                const user = await store.conn.whoami();
+                closeWin();
+                resolve(user);
+            }
+            catch (e) {
+                showPopup('Login failed: ' + e.message, {
+                    level: 'error',
+                });
+            }
+        });
+    });
+}