lfss 0.8.2__py3-none-any.whl → 0.8.4__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Readme.md CHANGED
@@ -1,9 +1,17 @@
1
1
  # Lightweight File Storage Service (LFSS)
2
2
  [![PyPI](https://img.shields.io/pypi/v/lfss)](https://pypi.org/project/lfss/)
3
3
 
4
- My experiment on a lightweight and high-performance file/object storage service.
4
+ My experiment on a lightweight and high-performance file/object storage service...
5
+
6
+ **Highlights:**
7
+
8
+ - User storage limit and access control.
9
+ - Pagination and sorted file listing for vast number of files.
10
+ - High performance: high concurrency, near-native speed on stress tests.
11
+ - Support range requests, so you can stream large files / resume download.
12
+
5
13
  It stores small files and metadata in sqlite, large files in the filesystem.
6
- Tested on 2 million files, and it works fine...
14
+ Tested on 2 million files, and it is still fast.
7
15
 
8
16
  Usage:
9
17
  ```sh
@@ -22,8 +30,8 @@ lfss-panel --open
22
30
  Or, you can start a web server at `/frontend` and open `index.html` in your browser.
23
31
 
24
32
  The API usage is simple, just `GET`, `PUT`, `DELETE` to the `/<username>/file/url` path.
25
- Authentication is done via `Authorization` header with the value `Bearer <token>`, or through the `token` query parameter.
26
- You can refer to `frontend` as an application example, and `frontend/api.js` or `lfss/api/connector.py` for the API usage.
33
+ Authentication via `Authorization` header with the value `Bearer <token>`, or through the `token` query parameter.
34
+ You can refer to `frontend` as an application example, `lfss/api/connector.py` for more APIs.
27
35
 
28
36
  By default, the service exposes all files to the public for `GET` requests,
29
37
  but file-listing is restricted to the user's own files.
frontend/info.js CHANGED
@@ -12,8 +12,10 @@ const ensureSlashEnd = (path) => {
12
12
  /**
13
13
  * @param {FileRecord} r
14
14
  * @param {UserRecord} u
15
+ * @param {Connector} c
15
16
  */
16
- export function showInfoPanel(r, u){
17
+ export function showInfoPanel(r, u, c){
18
+ const origin = c.config.endpoint;
17
19
  const innerHTML = `
18
20
  <div class="info-container">
19
21
  <div class="info-container-left">
@@ -46,7 +48,7 @@ export function showInfoPanel(r, u){
46
48
  </div>
47
49
  <div class="info-container-right">
48
50
  <div class="info-path-copy">
49
- <input type="text" value="${window.location.origin}/${r.url}" readonly>
51
+ <input type="text" value="${origin}/${r.url}" readonly>
50
52
  <button class="copy-button" id='copy-btn-full-path'>📋</button>
51
53
  </div>
52
54
  <div class="info-path-copy">
@@ -58,7 +60,7 @@ export function showInfoPanel(r, u){
58
60
  `
59
61
  const [win, closeWin] = createFloatingWindow(innerHTML, {title: 'File Info'});
60
62
  document.getElementById('copy-btn-full-path').onclick = () => {
61
- copyToClipboard(window.location.origin + '/' + r.url);
63
+ copyToClipboard(origin + '/' + r.url);
62
64
  showPopup('Path copied to clipboard', {timeout: 2000, level: 'success'});
63
65
  }
64
66
  document.getElementById('copy-btn-rel-path').onclick = () => {
@@ -77,6 +79,7 @@ export function showDirInfoPanel(r, u, c){
77
79
  if (fmtPath.endsWith('/')) {
78
80
  fmtPath = fmtPath.slice(0, -1);
79
81
  }
82
+ const origin = c.config.endpoint;
80
83
  const innerHTML = `
81
84
  <div class="info-container">
82
85
  <div class="info-container-left">
@@ -105,7 +108,7 @@ export function showDirInfoPanel(r, u, c){
105
108
  </div>
106
109
  <div class="info-container-right">
107
110
  <div class="info-path-copy">
108
- <input type="text" value="${window.location.origin}/${ensureSlashEnd(r.url)}" readonly>
111
+ <input type="text" value="${origin}/${ensureSlashEnd(r.url)}" readonly>
109
112
  <button class="copy-button" id='copy-btn-full-path'>📋</button>
110
113
  </div>
111
114
  <div class="info-path-copy">
@@ -117,7 +120,7 @@ export function showDirInfoPanel(r, u, c){
117
120
  `
118
121
  const [win, closeWin] = createFloatingWindow(innerHTML, {title: 'File Info'});
119
122
  document.getElementById('copy-btn-full-path').onclick = () => {
120
- copyToClipboard(window.location.origin + '/' + ensureSlashEnd(r.url));
123
+ copyToClipboard(origin + '/' + ensureSlashEnd(r.url));
121
124
  showPopup('Path copied to clipboard', {timeout: 2000, level: 'success'});
122
125
  }
123
126
  document.getElementById('copy-btn-rel-path').onclick = () => {
frontend/scripts.js CHANGED
@@ -459,7 +459,7 @@ async function refreshFileList(){
459
459
  infoButton.style.cursor = 'pointer';
460
460
  infoButton.textContent = 'Details';
461
461
  infoButton.addEventListener('click', () => {
462
- showInfoPanel(file, userRecord);
462
+ showInfoPanel(file, userRecord, conn);
463
463
  });
464
464
  actContainer.appendChild(infoButton);
465
465
 
frontend/thumb.js CHANGED
@@ -13,6 +13,7 @@ const ICON_ZIP = '<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><t
13
13
  const ICON_CODE = '<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><title>file-code-outline</title><path d="M14 2H6C4.89 2 4 2.9 4 4V20C4 21.11 4.89 22 6 22H18C19.11 22 20 21.11 20 20V8L14 2M18 20H6V4H13V9H18V20M9.54 15.65L11.63 17.74L10.35 19L7 15.65L10.35 12.3L11.63 13.56L9.54 15.65M17 15.65L13.65 19L12.38 17.74L14.47 15.65L12.38 13.56L13.65 12.3L17 15.65Z" /></svg>'
14
14
  const ICON_VIDEO = '<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><title>television-play</title><path d="M21,3H3C1.89,3 1,3.89 1,5V17A2,2 0 0,0 3,19H8V21H16V19H21A2,2 0 0,0 23,17V5C23,3.89 22.1,3 21,3M21,17H3V5H21M16,11L9,15V7" /></svg>'
15
15
  const ICON_IMAGE = '<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><title>image-outline</title><path d="M19,19H5V5H19M19,3H5A2,2 0 0,0 3,5V19A2,2 0 0,0 5,21H19A2,2 0 0,0 21,19V5A2,2 0 0,0 19,3M13.96,12.29L11.21,15.83L9.25,13.47L6.5,17H17.5L13.96,12.29Z" /></svg>'
16
+ const ICON_MUSIC = '<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><title>music-box-outline</title><path d="M16,9H13V14.5A2.5,2.5 0 0,1 10.5,17A2.5,2.5 0 0,1 8,14.5A2.5,2.5 0 0,1 10.5,12C11.07,12 11.58,12.19 12,12.5V7H16V9M19,3A2,2 0 0,1 21,5V19A2,2 0 0,1 19,21H5A2,2 0 0,1 3,19V5A2,2 0 0,1 5,3H19M5,5V19H19V5H5Z" /></svg>'
16
17
 
17
18
  function getIconSVGFromMimeType(mimeType){
18
19
  if (mimeType == 'directory'){
@@ -24,6 +25,9 @@ function getIconSVGFromMimeType(mimeType){
24
25
  if (mimeType.startsWith('image/')){
25
26
  return ICON_IMAGE;
26
27
  }
28
+ if (mimeType.startsWith('audio/')){
29
+ return ICON_MUSIC;
30
+ }
27
31
 
28
32
  if (['application/pdf', 'application/x-pdf'].includes(mimeType)){
29
33
  return ICON_PDF;
@@ -35,10 +39,14 @@ function getIconSVGFromMimeType(mimeType){
35
39
  return ICON_ZIP;
36
40
  }
37
41
  if ([
38
- "text/html", "application/xhtml+xml", "application/xml", "text/css", "application/javascript", "text/javascript", "application/json", "text/x-python", "text/x-java-source",
39
- "application/x-httpd-php", "text/x-ruby", "text/x-perl", "application/x-sh", "application/sql", "text/x-c", "text/x-c++", "text/x-csharp", "text/x-go", "text/x-haskell",
40
- "text/x-lua", "text/x-markdown", "application/wasm", "application/x-tcl", "text/x-yaml", "application/x-latex", "application/x-tex", "text/x-scss", "application/x-lisp",
41
- "application/x-rust", "application/x-ruby", "text/x-asm"
42
+ "text/html", "application/xhtml+xml", "application/xml", "text/css", "text/x-scss", "application/javascript", "text/javascript",
43
+ "application/json", "text/x-yaml", "text/x-markdown", "application/wasm",
44
+ "text/x-ruby", "application/x-ruby", "text/x-perl", "application/x-lisp",
45
+ "text/x-haskell", "text/x-lua", "application/x-tcl",
46
+ "text/x-python", "text/x-java-source", "text/x-go", "application/x-rust", "text/x-asm",
47
+ "application/sql", "text/x-c", "text/x-c++", "text/x-csharp",
48
+ "application/x-httpd-php", "application/x-sh", "application/x-shellscript",
49
+ "application/x-latex", "application/x-tex",
42
50
  ].includes(mimeType)){
43
51
  return ICON_CODE;
44
52
  }
lfss/api/connector.py CHANGED
@@ -54,7 +54,7 @@ class Connector:
54
54
 
55
55
  def _fetch_factory(
56
56
  self, method: Literal['GET', 'POST', 'PUT', 'DELETE'],
57
- path: str, search_params: dict = {}
57
+ path: str, search_params: dict = {}, extra_headers: dict = {}
58
58
  ):
59
59
  if path.startswith('/'):
60
60
  path = path[1:]
@@ -65,6 +65,7 @@ class Connector:
65
65
  headers.update({
66
66
  'Authorization': f"Bearer {self.config['token']}",
67
67
  })
68
+ headers.update(extra_headers)
68
69
  if self._session is not None:
69
70
  response = self._session.request(method, url, headers=headers, **kwargs)
70
71
  response.raise_for_status()
@@ -113,7 +114,6 @@ class Connector:
113
114
 
114
115
  if isinstance(file, str):
115
116
  assert os.path.exists(file), "File does not exist on disk"
116
- fsize = os.path.getsize(file)
117
117
 
118
118
  with open(file, 'rb') if isinstance(file, str) else SpooledTemporaryFile(max_size=1024*1024*32) as fp:
119
119
 
@@ -123,7 +123,6 @@ class Connector:
123
123
  fp.seek(0)
124
124
 
125
125
  # https://stackoverflow.com/questions/12385179/
126
- print(f"Uploading {fsize} bytes")
127
126
  response = self._fetch_factory('POST', path, search_params={
128
127
  'permission': int(permission),
129
128
  'conflict': conflict
@@ -168,6 +167,17 @@ class Connector:
168
167
  response = self._get(path)
169
168
  if response is None: return None
170
169
  return response.content
170
+
171
+ def get_partial(self, path: str, range_start: int = -1, range_end: int = -1) -> Optional[bytes]:
172
+ """
173
+ Downloads a partial file from the specified path.
174
+ start and end are the byte offsets, both inclusive.
175
+ """
176
+ response = self._fetch_factory('GET', path, extra_headers={
177
+ 'Range': f"bytes={range_start if range_start >= 0 else ''}-{range_end if range_end >= 0 else ''}"
178
+ })()
179
+ if response is None: return None
180
+ return response.content
171
181
 
172
182
  def get_stream(self, path: str) -> Iterator[bytes]:
173
183
  """Downloads a file from the specified path, will raise PathNotFoundError if path not found."""
lfss/cli/cli.py CHANGED
@@ -1,6 +1,6 @@
1
- from lfss.api import Connector, upload_directory, upload_file, download_file, download_directory
2
1
  from pathlib import Path
3
2
  import argparse, typing
3
+ from lfss.api import Connector, upload_directory, upload_file, download_file, download_directory
4
4
  from lfss.src.datatype import FileReadPermission, FileSortKey, DirSortKey
5
5
  from lfss.src.utils import decode_uri_compnents
6
6
  from . import catch_request_error, line_sep
@@ -18,7 +18,6 @@ def parse_permission(s: str) -> FileReadPermission:
18
18
 
19
19
  def parse_arguments():
20
20
  parser = argparse.ArgumentParser(description="Command line interface, please set LFSS_ENDPOINT and LFSS_TOKEN environment variables.")
21
- parser.add_argument("-v", "--verbose", action="store_true", help="Verbose output")
22
21
 
23
22
  sp = parser.add_subparsers(dest="command", required=True)
24
23
 
@@ -26,6 +25,7 @@ def parse_arguments():
26
25
  sp_upload = sp.add_parser("upload", help="Upload files")
27
26
  sp_upload.add_argument("src", help="Source file or directory", type=str)
28
27
  sp_upload.add_argument("dst", help="Destination url path", type=str)
28
+ sp_upload.add_argument("-v", "--verbose", action="store_true", help="Verbose output")
29
29
  sp_upload.add_argument("-j", "--jobs", type=int, default=1, help="Number of concurrent uploads")
30
30
  sp_upload.add_argument("--interval", type=float, default=0, help="Interval between files, only works with directory upload")
31
31
  sp_upload.add_argument("--conflict", choices=["overwrite", "abort", "skip", "skip-ahead"], default="abort", help="Conflict resolution")
@@ -36,6 +36,7 @@ def parse_arguments():
36
36
  sp_download = sp.add_parser("download", help="Download files")
37
37
  sp_download.add_argument("src", help="Source url path", type=str)
38
38
  sp_download.add_argument("dst", help="Destination file or directory", type=str)
39
+ sp_download.add_argument("-v", "--verbose", action="store_true", help="Verbose output")
39
40
  sp_download.add_argument("-j", "--jobs", type=int, default=1, help="Number of concurrent downloads")
40
41
  sp_download.add_argument("--interval", type=float, default=0, help="Interval between files, only works with directory download")
41
42
  sp_download.add_argument("--overwrite", action="store_true", help="Overwrite existing files")
lfss/src/database.py CHANGED
@@ -433,20 +433,41 @@ class FileConn(DBObjectBase):
433
433
  raise
434
434
  return size_sum
435
435
 
436
- async def get_file_blob(self, file_id: str) -> Optional[bytes]:
436
+ async def get_file_blob(self, file_id: str, start_byte = -1, end_byte = -1) -> bytes:
437
437
  cursor = await self.cur.execute("SELECT data FROM blobs.fdata WHERE file_id = ?", (file_id, ))
438
438
  res = await cursor.fetchone()
439
439
  if res is None:
440
- return None
441
- return res[0]
440
+ raise FileNotFoundError(f"File {file_id} not found")
441
+ blob = res[0]
442
+ match (start_byte, end_byte):
443
+ case (-1, -1):
444
+ return blob
445
+ case (s, -1):
446
+ return blob[s:]
447
+ case (-1, e):
448
+ return blob[:e]
449
+ case (s, e):
450
+ return blob[s:e]
442
451
 
443
- async def get_file_blob_external(self, file_id: str) -> AsyncIterable[bytes]:
452
+ @staticmethod
453
+ async def get_file_blob_external(file_id: str, start_byte = -1, end_byte = -1) -> AsyncIterable[bytes]:
444
454
  assert (LARGE_BLOB_DIR / file_id).exists(), f"File {file_id} not found"
445
455
  async with aiofiles.open(LARGE_BLOB_DIR / file_id, 'rb') as f:
446
- while True:
447
- chunk = await f.read(CHUNK_SIZE)
448
- if not chunk: break
449
- yield chunk
456
+ if start_byte >= 0:
457
+ await f.seek(start_byte)
458
+ if end_byte >= 0:
459
+ while True:
460
+ head_ptr = await f.tell()
461
+ if head_ptr >= end_byte:
462
+ break
463
+ chunk = await f.read(min(CHUNK_SIZE, end_byte - head_ptr))
464
+ if not chunk: break
465
+ yield chunk
466
+ else:
467
+ while True:
468
+ chunk = await f.read(CHUNK_SIZE)
469
+ if not chunk: break
470
+ yield chunk
450
471
 
451
472
  @staticmethod
452
473
  async def delete_file_blob_external(file_id: str):
@@ -595,34 +616,21 @@ class Database:
595
616
  permission=permission, external=True, mime_type=mime_type)
596
617
  return file_size
597
618
 
598
- async def read_file_stream(self, url: str) -> AsyncIterable[bytes]:
619
+ async def read_file(self, url: str, start_byte = -1, end_byte = -1) -> AsyncIterable[bytes]:
620
+ # end byte is exclusive: [start_byte, end_byte)
599
621
  validate_url(url)
600
- async with unique_cursor() as cur:
601
- fconn = FileConn(cur)
602
- r = await fconn.get_file_record(url)
603
- if r is None:
604
- raise FileNotFoundError(f"File {url} not found")
605
- if not r.external:
606
- raise ValueError(f"File {url} is not stored externally, should use read_file instead")
607
- ret = fconn.get_file_blob_external(r.file_id)
608
- return ret
609
-
610
- async def read_file(self, url: str) -> bytes:
611
- validate_url(url)
612
-
613
622
  async with unique_cursor() as cur:
614
623
  fconn = FileConn(cur)
615
624
  r = await fconn.get_file_record(url)
616
625
  if r is None:
617
626
  raise FileNotFoundError(f"File {url} not found")
618
627
  if r.external:
619
- raise ValueError(f"File {url} is stored externally, should use read_file_stream instead")
620
-
621
- f_id = r.file_id
622
- blob = await fconn.get_file_blob(f_id)
623
- if blob is None:
624
- raise FileNotFoundError(f"File {url} data not found")
625
- return blob
628
+ ret = fconn.get_file_blob_external(r.file_id, start_byte=start_byte, end_byte=end_byte)
629
+ else:
630
+ async def blob_stream():
631
+ yield await fconn.get_file_blob(r.file_id, start_byte=start_byte, end_byte=end_byte)
632
+ ret = blob_stream()
633
+ return ret
626
634
 
627
635
  async def delete_file(self, url: str, op_user: Optional[UserRecord] = None) -> Optional[FileRecord]:
628
636
  validate_url(url)
@@ -758,9 +766,6 @@ class Database:
758
766
  blob = fconn.get_file_blob_external(f_id)
759
767
  else:
760
768
  blob = await fconn.get_file_blob(f_id)
761
- if blob is None:
762
- self.logger.warning(f"Blob not found for {url}")
763
- continue
764
769
  yield r, blob
765
770
 
766
771
  @concurrent_wrap()
lfss/src/datatype.py CHANGED
@@ -1,5 +1,6 @@
1
1
  from enum import IntEnum
2
2
  import dataclasses, typing
3
+ from .utils import fmt_storage_size
3
4
 
4
5
  class FileReadPermission(IntEnum):
5
6
  UNSET = 0 # not set
@@ -18,8 +19,12 @@ class UserRecord:
18
19
  max_storage: int
19
20
  permission: 'FileReadPermission'
20
21
 
22
+ def __post_init__(self):
23
+ self.permission = FileReadPermission(self.permission)
24
+
21
25
  def __str__(self):
22
- return f"User {self.username} (id={self.id}, admin={self.is_admin}, created at {self.create_time}, last active at {self.last_active}, storage={self.max_storage}, permission={self.permission})"
26
+ return f"User {self.username} (id={self.id}, admin={self.is_admin}, created at {self.create_time}, last active at {self.last_active}, " + \
27
+ f"storage={fmt_storage_size(self.max_storage)}, permission={self.permission.name})"
23
28
 
24
29
  @dataclasses.dataclass
25
30
  class FileRecord:
@@ -33,9 +38,12 @@ class FileRecord:
33
38
  external: bool
34
39
  mime_type: str
35
40
 
41
+ def __post_init__(self):
42
+ self.permission = FileReadPermission(self.permission)
43
+
36
44
  def __str__(self):
37
45
  return f"File {self.url} [{self.mime_type}] (owner={self.owner_id}, created at {self.create_time}, accessed at {self.access_time}, " + \
38
- f"file_id={self.file_id}, permission={self.permission}, size={self.file_size}, external={self.external})"
46
+ f"file_id={self.file_id}, permission={self.permission.name}, size={fmt_storage_size(self.file_size)}, external={self.external})"
39
47
 
40
48
  @dataclasses.dataclass
41
49
  class DirectoryRecord:
@@ -47,7 +55,7 @@ class DirectoryRecord:
47
55
  n_files: int = -1
48
56
 
49
57
  def __str__(self):
50
- return f"Directory {self.url} (size={self.size}, created at {self.create_time}, updated at {self.update_time}, accessed at {self.access_time}, n_files={self.n_files})"
58
+ return f"Directory {self.url} (size={fmt_storage_size(self.size)}, created at {self.create_time}, updated at {self.update_time}, accessed at {self.access_time}, n_files={self.n_files})"
51
59
 
52
60
  @dataclasses.dataclass
53
61
  class PathContents:
lfss/src/server.py CHANGED
@@ -141,7 +141,8 @@ router_fs = APIRouter(prefix="")
141
141
  @skip_request_log
142
142
  async def emit_thumbnail(
143
143
  path: str, download: bool,
144
- create_time: Optional[str] = None
144
+ create_time: Optional[str] = None,
145
+ is_head = False
145
146
  ):
146
147
  if path.endswith("/"):
147
148
  fname = path.split("/")[-2]
@@ -157,44 +158,69 @@ async def emit_thumbnail(
157
158
  }
158
159
  if create_time is not None:
159
160
  headers["Last-Modified"] = format_last_modified(create_time)
161
+ if is_head: return Response(status_code=200, headers=headers)
160
162
  return Response(
161
163
  content=thumb_blob, media_type=mime_type, headers=headers
162
164
  )
163
165
  async def emit_file(
164
166
  file_record: FileRecord,
165
167
  media_type: Optional[str] = None,
166
- disposition = "attachment"
168
+ disposition = "attachment",
169
+ is_head = False,
170
+ range_start = -1,
171
+ range_end = -1
167
172
  ):
173
+ if range_start < 0: assert range_start == -1
174
+ if range_end < 0: assert range_end == -1
175
+
168
176
  if media_type is None:
169
177
  media_type = file_record.mime_type
170
178
  path = file_record.url
171
179
  fname = path.split("/")[-1]
172
180
 
173
- await delayed_log_access(path)
174
- if not file_record.external:
175
- fblob = await db.read_file(path)
176
- return Response(
177
- content=fblob, media_type=media_type, headers={
178
- "Content-Disposition": f"{disposition}; filename={fname}",
179
- "Content-Length": str(len(fblob)),
180
- "Last-Modified": format_last_modified(file_record.create_time)
181
- }
182
- )
181
+ if range_start == -1:
182
+ arng_s = 0 # actual range start
183
183
  else:
184
- return StreamingResponse(
185
- await db.read_file_stream(path), media_type=media_type, headers={
186
- "Content-Disposition": f"{disposition}; filename={fname}",
187
- "Content-Length": str(file_record.file_size),
188
- "Last-Modified": format_last_modified(file_record.create_time)
189
- }
190
- )
184
+ arng_s = range_start
185
+ if range_end == -1:
186
+ arng_e = file_record.file_size - 1
187
+ else:
188
+ arng_e = range_end
189
+
190
+ if arng_s >= file_record.file_size or arng_e >= file_record.file_size:
191
+ raise HTTPException(status_code=416, detail="Range not satisfiable")
192
+ if arng_s > arng_e:
193
+ raise HTTPException(status_code=416, detail="Invalid range")
191
194
 
192
- @router_fs.get("/{path:path}")
193
- @handle_exception
194
- async def get_file(
195
+ headers = {
196
+ "Content-Disposition": f"{disposition}; filename={fname}",
197
+ "Content-Length": str(arng_e - arng_s + 1),
198
+ "Content-Range": f"bytes {arng_s}-{arng_e}/{file_record.file_size}",
199
+ "Last-Modified": format_last_modified(file_record.create_time),
200
+ "Accept-Ranges": "bytes",
201
+ }
202
+
203
+ if is_head: return Response(status_code=200 if (range_start == -1 and range_end == -1) else 206, headers=headers)
204
+
205
+ await delayed_log_access(path)
206
+ return StreamingResponse(
207
+ await db.read_file(
208
+ path,
209
+ start_byte=arng_s if range_start != -1 else -1,
210
+ end_byte=arng_e + 1 if range_end != -1 else -1
211
+ ),
212
+ media_type=media_type,
213
+ headers=headers,
214
+ status_code=206 if range_start != -1 or range_end != -1 else 200
215
+ )
216
+
217
+ async def get_file_impl(
218
+ request: Request,
219
+ user: UserRecord,
195
220
  path: str,
196
- download: bool = False, thumb: bool = False,
197
- user: UserRecord = Depends(get_current_user)
221
+ download: bool = False,
222
+ thumb: bool = False,
223
+ is_head = False,
198
224
  ):
199
225
  path = ensure_uri_compnents(path)
200
226
 
@@ -236,13 +262,58 @@ async def get_file(
236
262
  if not allow_access:
237
263
  raise HTTPException(status_code=403, detail=reason)
238
264
 
265
+ req_range = request.headers.get("Range", None)
266
+ if not req_range is None:
267
+ # handle range request
268
+ if not req_range.startswith("bytes="):
269
+ raise HTTPException(status_code=400, detail="Invalid range request")
270
+ range_str = req_range[6:].strip()
271
+ if "," in range_str:
272
+ raise HTTPException(status_code=400, detail="Multiple ranges not supported")
273
+ if "-" not in range_str:
274
+ raise HTTPException(status_code=400, detail="Invalid range request")
275
+ range_start, range_end = map(lambda x: int(x) if x != "" else -1 , range_str.split("-"))
276
+ else:
277
+ range_start, range_end = -1, -1
278
+
239
279
  if thumb:
240
- return await emit_thumbnail(path, download, create_time=file_record.create_time)
280
+ if (range_start != -1 or range_end != -1): logger.warning("Range request for thumbnail")
281
+ return await emit_thumbnail(path, download, create_time=file_record.create_time, is_head=is_head)
241
282
  else:
242
283
  if download:
243
- return await emit_file(file_record, 'application/octet-stream', "attachment")
284
+ return await emit_file(file_record, 'application/octet-stream', "attachment", is_head = is_head, range_start=range_start, range_end=range_end)
244
285
  else:
245
- return await emit_file(file_record, None, "inline")
286
+ return await emit_file(file_record, None, "inline", is_head = is_head, range_start=range_start, range_end=range_end)
287
+
288
+ @router_fs.get("/{path:path}")
289
+ @handle_exception
290
+ async def get_file(
291
+ request: Request,
292
+ path: str,
293
+ download: bool = False, thumb: bool = False,
294
+ user: UserRecord = Depends(get_current_user)
295
+ ):
296
+ return await get_file_impl(
297
+ request = request,
298
+ user = user, path = path, download = download, thumb = thumb
299
+ )
300
+
301
+ @router_fs.head("/{path:path}")
302
+ @handle_exception
303
+ async def head_file(
304
+ request: Request,
305
+ path: str,
306
+ download: bool = False, thumb: bool = False,
307
+ user: UserRecord = Depends(get_current_user)
308
+ ):
309
+ if path.startswith("_api/"):
310
+ raise HTTPException(status_code=405, detail="HEAD not supported for API")
311
+ if path.endswith("/"):
312
+ raise HTTPException(status_code=405, detail="HEAD not supported for directory")
313
+ return await get_file_impl(
314
+ request = request,
315
+ user = user, path = path, download = download, thumb = thumb, is_head = True
316
+ )
246
317
 
247
318
  @router_fs.put("/{path:path}")
248
319
  @handle_exception
@@ -1,13 +1,12 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: lfss
3
- Version: 0.8.2
3
+ Version: 0.8.4
4
4
  Summary: Lightweight file storage service
5
5
  Home-page: https://github.com/MenxLi/lfss
6
6
  Author: li, mengxun
7
7
  Author-email: limengxun45@outlook.com
8
- Requires-Python: >=3.9
8
+ Requires-Python: >=3.10
9
9
  Classifier: Programming Language :: Python :: 3
10
- Classifier: Programming Language :: Python :: 3.9
11
10
  Classifier: Programming Language :: Python :: 3.10
12
11
  Classifier: Programming Language :: Python :: 3.11
13
12
  Classifier: Programming Language :: Python :: 3.12
@@ -25,9 +24,17 @@ Description-Content-Type: text/markdown
25
24
  # Lightweight File Storage Service (LFSS)
26
25
  [![PyPI](https://img.shields.io/pypi/v/lfss)](https://pypi.org/project/lfss/)
27
26
 
28
- My experiment on a lightweight and high-performance file/object storage service.
27
+ My experiment on a lightweight and high-performance file/object storage service...
28
+
29
+ **Highlights:**
30
+
31
+ - User storage limit and access control.
32
+ - Pagination and sorted file listing for vast number of files.
33
+ - High performance: high concurrency, near-native speed on stress tests.
34
+ - Support range requests, so you can stream large files / resume download.
35
+
29
36
  It stores small files and metadata in sqlite, large files in the filesystem.
30
- Tested on 2 million files, and it works fine...
37
+ Tested on 2 million files, and it is still fast.
31
38
 
32
39
  Usage:
33
40
  ```sh
@@ -46,8 +53,8 @@ lfss-panel --open
46
53
  Or, you can start a web server at `/frontend` and open `index.html` in your browser.
47
54
 
48
55
  The API usage is simple, just `GET`, `PUT`, `DELETE` to the `/<username>/file/url` path.
49
- Authentication is done via `Authorization` header with the value `Bearer <token>`, or through the `token` query parameter.
50
- You can refer to `frontend` as an application example, and `frontend/api.js` or `lfss/api/connector.py` for the API usage.
56
+ Authentication via `Authorization` header with the value `Bearer <token>`, or through the `token` query parameter.
57
+ You can refer to `frontend` as an application example, `lfss/api/connector.py` for more APIs.
51
58
 
52
59
  By default, the service exposes all files to the public for `GET` requests,
53
60
  but file-listing is restricted to the user's own files.
@@ -1,25 +1,25 @@
1
- Readme.md,sha256=LpbTvUWjCOv4keMNDrZvEnNAmCQnvaxvlq2srWixXn0,1299
1
+ Readme.md,sha256=J1tGk7B9EyIXT-RN7VGz_229UeKvZHVLpn1FvzNDxL4,1538
2
2
  docs/Known_issues.md,sha256=rfdG3j1OJF-59S9E06VPyn0nZKbW-ybPxkoZ7MEZWp8,81
3
3
  docs/Permission.md,sha256=9r9nEmhqfz18RTS8FI0fZ9F0a31r86OoAyx3EQxxpk0,2317
4
4
  frontend/api.js,sha256=wUJNAkL8QigAiwR_jaMPUhCQEsL-lp0wZ6XeueYgunE,18049
5
5
  frontend/index.html,sha256=-k0bJ5FRqdl_H-O441D_H9E-iejgRCaL_z5UeYaS2qc,3384
6
6
  frontend/info.css,sha256=Ny0N3GywQ3a9q1_Qph_QFEKB4fEnTe_2DJ1Y5OsLLmQ,595
7
- frontend/info.js,sha256=WhOGaeqMoezEAfg4nIpK26hvejC7AZ-ZDLiJmRj0kDk,5758
7
+ frontend/info.js,sha256=xGUJPCSrtDhuSu0ELLQZ77PmVWldg-prU1mwQGbdEoA,5797
8
8
  frontend/login.css,sha256=VMM0QfbDFYerxKWKSGhMI1yg5IRBXg0TTdLJEEhQZNk,355
9
9
  frontend/login.js,sha256=QoO8yKmBHDVP-ZomCMOaV7xVUVIhpl7esJrb6T5aHQE,2466
10
10
  frontend/popup.css,sha256=TJZYFW1ZcdD1IVTlNPYNtMWKPbN6XDbQ4hKBOFK8uLg,1284
11
11
  frontend/popup.js,sha256=3PgaGZmxSdV1E-D_MWgcR7aHWkcsHA1BNKSOkmP66tA,5191
12
- frontend/scripts.js,sha256=2YoMhrcAkI1bHihD_2EK6uCHZ1s0DiIR3FzZsh79x9A,21729
12
+ frontend/scripts.js,sha256=nWH6NgavZTVmjK44i2DeRi6mJzGSe4qeQPUbDaEVt58,21735
13
13
  frontend/state.js,sha256=vbNL5DProRKmSEY7xu9mZH6IY0PBenF8WGxPtGgDnLI,1680
14
14
  frontend/styles.css,sha256=xcNLqI3KBsY5TLnku8UIP0Jfr7QLajr1_KNlZj9eheM,4935
15
15
  frontend/thumb.css,sha256=rNsx766amYS2DajSQNabhpQ92gdTpNoQKmV69OKvtpI,295
16
- frontend/thumb.js,sha256=RQ_whXNwmkdG4SEbNQGeh488YYzqwoNYDc210hPeuhQ,5703
16
+ frontend/thumb.js,sha256=46ViD2TlTTWy0fx6wjoAs_5CQ4ajYB90vVzM7UO2IHw,6182
17
17
  frontend/utils.js,sha256=IYUZl77ugiXKcLxSNOWC4NSS0CdD5yRgUsDb665j0xM,2556
18
18
  lfss/api/__init__.py,sha256=c_-GokKJ_3REgve16AwgXRfFyl9mwy7__FxCIIVlk1Q,6660
19
- lfss/api/connector.py,sha256=tmcgKswE0sktBhanEDEc6mpuJA0de7C-DS2YqlpxHX4,10743
19
+ lfss/api/connector.py,sha256=e2nhqrRGWixSJXRVDBxadq9oeiL0sGWLaL7FFzvLFJ8,11231
20
20
  lfss/cli/__init__.py,sha256=lPwPmqpa7EXQ4zlU7E7LOe6X2kw_xATGdwoHphUEirA,827
21
21
  lfss/cli/balance.py,sha256=R2rbO2tg9TVnnQIVeU0GJVeMS-5LDhEdk4mbOE9qGq0,4121
22
- lfss/cli/cli.py,sha256=LxUrviHtsqi-vs_GWZw2qRs9dBNvx9PSQHLW6SwUmhA,8167
22
+ lfss/cli/cli.py,sha256=WVxDtIYCgFkEp9HoVLGi7AAhZJi5BCML7uT5D4yVcuE,8262
23
23
  lfss/cli/panel.py,sha256=iGdVmdWYjA_7a78ZzWEB_3ggIOBeUKTzg6F5zLaB25c,1401
24
24
  lfss/cli/serve.py,sha256=T-jz_PJcaY9FJRRfyWV9MbjDI73YdOOCn4Nr2PO-s0c,993
25
25
  lfss/cli/user.py,sha256=uqHQ7onddTjJAYg3B1DIc8hDl0aCkIMZolLKhQrBd0k,4046
@@ -30,15 +30,15 @@ lfss/src/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
30
30
  lfss/src/bounded_pool.py,sha256=BI1dU-MBf82TMwJBYbjhEty7w1jIUKc5Bn9SnZ_-hoY,1288
31
31
  lfss/src/config.py,sha256=7k_6L_aG7vD-lVcmgpwMIg-ggpMMDNU8IbaB4y1effE,861
32
32
  lfss/src/connection_pool.py,sha256=4YULPcmndy33FyBSo9w1fZjAlBX2q-xPP27xJOTA06I,5228
33
- lfss/src/database.py,sha256=psNgY3QxHh5mmhqdpRwy3pucgZCi2d9kjUtdtjnB8P4,36042
34
- lfss/src/datatype.py,sha256=yyOcxhGwz-EJi003f8hGl82EJuY4F92y6fSX6cK60Bc,2126
33
+ lfss/src/database.py,sha256=zoiBm7CVHHV4TqwmK6lPnZvK9mzDNtrNvAJCRaIYMU8,36302
34
+ lfss/src/datatype.py,sha256=1xdxSKhpJXoBKumUokL3zQ2VyZ0Wwp8q6PaJf1idVw0,2435
35
35
  lfss/src/error.py,sha256=Bh_GUtuNsMSMIKbFreU4CfZzL96TZdNAIcgmDxvGbQ0,333
36
36
  lfss/src/log.py,sha256=u6WRZZsE7iOx6_CV2NHh1ugea26p408FI4WstZh896A,5139
37
- lfss/src/server.py,sha256=IoqKpznDt28MIGUf79MIRFvlV_6VIaqGPmc_hkYcXuY,21150
37
+ lfss/src/server.py,sha256=YLsp6bab7q0I2hI4uUYIiWc2S0k6d6bbMaweg6VbVV4,23743
38
38
  lfss/src/stat.py,sha256=WlRiiAl0rHX9oPi3thi6K4GKn70WHpClSDCt7iZUow4,3197
39
39
  lfss/src/thumb.py,sha256=qjCNMpnCozMuzkhm-2uAYy1eAuYTeWG6xqs-13HX-7k,3266
40
40
  lfss/src/utils.py,sha256=DxjHabdiISMkrm1WQlpsZFKL3by6YrzBNQaDt_uZlRk,5744
41
- lfss-0.8.2.dist-info/METADATA,sha256=gaqq7M4te2IAQk8OntQNXMaW3jbderKLu1C5pEA6ieU,2108
42
- lfss-0.8.2.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
43
- lfss-0.8.2.dist-info/entry_points.txt,sha256=VJ8svMz7RLtMCgNk99CElx7zo7M-N-z7BWDVw2HA92E,205
44
- lfss-0.8.2.dist-info/RECORD,,
41
+ lfss-0.8.4.dist-info/METADATA,sha256=OUvtod8R5Z7DnNLczVG6FpsHEQ2J1ct8QlbdZhU2fIk,2298
42
+ lfss-0.8.4.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
43
+ lfss-0.8.4.dist-info/entry_points.txt,sha256=VJ8svMz7RLtMCgNk99CElx7zo7M-N-z7BWDVw2HA92E,205
44
+ lfss-0.8.4.dist-info/RECORD,,
File without changes