ebuffer 0.1.5__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ebuffer-0.1.5/LICENSE +21 -0
- ebuffer-0.1.5/PKG-INFO +82 -0
- ebuffer-0.1.5/Readme.md +51 -0
- ebuffer-0.1.5/ebuffer/__init__.py +1 -0
- ebuffer-0.1.5/ebuffer/apiebuffer.py +80 -0
- ebuffer-0.1.5/ebuffer/apiuserobj.py +127 -0
- ebuffer-0.1.5/ebuffer/application.py +45 -0
- ebuffer-0.1.5/ebuffer/auth/__init__.py +15 -0
- ebuffer-0.1.5/ebuffer/auth/auth.py +51 -0
- ebuffer-0.1.5/ebuffer/auth/openid.py +101 -0
- ebuffer-0.1.5/ebuffer/auth/passfile.py +140 -0
- ebuffer-0.1.5/ebuffer/backends/__init__.py +15 -0
- ebuffer-0.1.5/ebuffer/backends/acl.py +35 -0
- ebuffer-0.1.5/ebuffer/backends/backends.py +187 -0
- ebuffer-0.1.5/ebuffer/backends/memory.py +137 -0
- ebuffer-0.1.5/ebuffer/backends/mmap.py +230 -0
- ebuffer-0.1.5/ebuffer/config.py +48 -0
- ebuffer-0.1.5/ebuffer/database/__init__.py +17 -0
- ebuffer-0.1.5/ebuffer/database/cruduser.py +59 -0
- ebuffer-0.1.5/ebuffer/database/cruduserobj.py +162 -0
- ebuffer-0.1.5/ebuffer/database/database.py +54 -0
- ebuffer-0.1.5/ebuffer/database/privmodel_buffer.py +37 -0
- ebuffer-0.1.5/ebuffer/database/privmodel_user.py +39 -0
- ebuffer-0.1.5/ebuffer/database/privmodel_userobj.py +35 -0
- ebuffer-0.1.5/ebuffer/errors.py +88 -0
- ebuffer-0.1.5/ebuffer/models_buffer.py +30 -0
- ebuffer-0.1.5/ebuffer/models_common.py +123 -0
- ebuffer-0.1.5/ebuffer/routers/__init__.py +2 -0
- ebuffer-0.1.5/ebuffer/routers/routes_auth.py +47 -0
- ebuffer-0.1.5/ebuffer/routers/routes_buffer.py +206 -0
- ebuffer-0.1.5/ebuffer/server/__init__.py +1 -0
- ebuffer-0.1.5/ebuffer/server/main.py +207 -0
- ebuffer-0.1.5/ebuffer.egg-info/PKG-INFO +82 -0
- ebuffer-0.1.5/ebuffer.egg-info/SOURCES.txt +38 -0
- ebuffer-0.1.5/ebuffer.egg-info/dependency_links.txt +1 -0
- ebuffer-0.1.5/ebuffer.egg-info/requires.txt +11 -0
- ebuffer-0.1.5/ebuffer.egg-info/top_level.txt +1 -0
- ebuffer-0.1.5/pyproject.toml +74 -0
- ebuffer-0.1.5/setup.cfg +4 -0
- ebuffer-0.1.5/setup.py +3 -0
ebuffer-0.1.5/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2025 Laurent Morin
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
ebuffer-0.1.5/PKG-INFO
ADDED
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: ebuffer
|
|
3
|
+
Version: 0.1.5
|
|
4
|
+
Summary: Ephemeral buffer service
|
|
5
|
+
Author-email: Laurent Morin <Laurent.Morin@irisa.fr>, Yawa Germaine Nyatsikor <yawa-germaine.nyatsikor@irisa.fr>
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
Project-URL: Homepage, https://ebuffer.aqmo.org/
|
|
8
|
+
Project-URL: Documentation, https://ebuffer.aqmo.org/
|
|
9
|
+
Project-URL: Repository, https://ebuffer.aqmo.org/
|
|
10
|
+
Project-URL: Issues, https://ebuffer.aqmo.org/
|
|
11
|
+
Project-URL: Changelog, https://ebuffer.aqmo.org/
|
|
12
|
+
Keywords: HPC,buffers,workflow,distributed
|
|
13
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
14
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
15
|
+
Classifier: Operating System :: OS Independent
|
|
16
|
+
Classifier: Topic :: Scientific/Engineering
|
|
17
|
+
Classifier: Topic :: System :: Benchmark
|
|
18
|
+
Classifier: Topic :: System :: Distributed Computing
|
|
19
|
+
Requires-Python: >=3.10
|
|
20
|
+
Description-Content-Type: text/markdown
|
|
21
|
+
License-File: LICENSE
|
|
22
|
+
Requires-Dist: fastapi>=0.115.8
|
|
23
|
+
Requires-Dist: pydantic>=2.10.6
|
|
24
|
+
Requires-Dist: Requests>=2.32.3
|
|
25
|
+
Requires-Dist: SQLAlchemy>=2.0.38
|
|
26
|
+
Requires-Dist: sqlmodel>=0.0.22
|
|
27
|
+
Provides-Extra: all
|
|
28
|
+
Provides-Extra: docs
|
|
29
|
+
Provides-Extra: dev
|
|
30
|
+
Dynamic: license-file
|
|
31
|
+
|
|
32
|
+
# Ephemeral Buffer API
|
|
33
|
+
|
|
34
|
+
## Overview
|
|
35
|
+
This project represent the reference implementation of **Ephemeral buffer** concept.
|
|
36
|
+
A full description of the concept is available in *[the ephemeral buffer proposal](docs/2025.12.03-EphemeralBuffers-v2.md)*
|
|
37
|
+
|
|
38
|
+
## Installation
|
|
39
|
+
|
|
40
|
+
The installation is the standard installation procedure of any WSGI python library. See details below.
|
|
41
|
+
|
|
42
|
+
### From source code
|
|
43
|
+
|
|
44
|
+
Clone the repository [ebuffer](https://gitlab.aqmo.org/numpex/ebuffer.git):
|
|
45
|
+
|
|
46
|
+
```sh
|
|
47
|
+
git clone https://public:uhpzy8bFXwcYszCtkkcU@gitlab.aqmo.org/numpex/ebuffer.git
|
|
48
|
+
# Navigate into the project directory
|
|
49
|
+
cd ebuffer
|
|
50
|
+
|
|
51
|
+
# Create and load a virtual environment
|
|
52
|
+
python3 -mvenv venv
|
|
53
|
+
source venv/bin/activate
|
|
54
|
+
```
|
|
55
|
+
|
|
56
|
+
#### Local Deployment
|
|
57
|
+
For a local deployment install the server dependencies :
|
|
58
|
+
```sh
|
|
59
|
+
# Install dependencies for local deployment
|
|
60
|
+
pip install -r requirements-server.txt
|
|
61
|
+
```
|
|
62
|
+
|
|
63
|
+
#### For an WSGI Server
|
|
64
|
+
The core dependencies are in *requirements.txt*:
|
|
65
|
+
```sh
|
|
66
|
+
# Install core dependencies for an WSGI web server
|
|
67
|
+
pip install -r requirements.txt
|
|
68
|
+
```
|
|
69
|
+
|
|
70
|
+
## Authentication
|
|
71
|
+
|
|
72
|
+
The authentication mechanism depends on backends, and each depends on
|
|
73
|
+
a particular configuration. All actions that a user can operate on a
|
|
74
|
+
ephemeral buffer required a **Bearer Token**. This token is received
|
|
75
|
+
after the authentication process from keycloak (IDP used). The token
|
|
76
|
+
has a limited lifetime and can be refresh.
|
|
77
|
+
|
|
78
|
+
## Usage
|
|
79
|
+
To Run the project, start the server : `fastapi dev server/src/main.py`
|
|
80
|
+
|
|
81
|
+
A swagger documentation is available on `http://127.0.0.1:8000/docs/ui`
|
|
82
|
+
|
ebuffer-0.1.5/Readme.md
ADDED
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
# Ephemeral Buffer API
|
|
2
|
+
|
|
3
|
+
## Overview
|
|
4
|
+
This project represent the reference implementation of **Ephemeral buffer** concept.
|
|
5
|
+
A full description of the concept is available in *[the ephemeral buffer proposal](docs/2025.12.03-EphemeralBuffers-v2.md)*
|
|
6
|
+
|
|
7
|
+
## Installation
|
|
8
|
+
|
|
9
|
+
The installation is the standard installation procedure of any WSGI python library. See details below.
|
|
10
|
+
|
|
11
|
+
### From source code
|
|
12
|
+
|
|
13
|
+
Clone the repository [ebuffer](https://gitlab.aqmo.org/numpex/ebuffer.git):
|
|
14
|
+
|
|
15
|
+
```sh
|
|
16
|
+
git clone https://public:uhpzy8bFXwcYszCtkkcU@gitlab.aqmo.org/numpex/ebuffer.git
|
|
17
|
+
# Navigate into the project directory
|
|
18
|
+
cd ebuffer
|
|
19
|
+
|
|
20
|
+
# Create and load a virtual environment
|
|
21
|
+
python3 -mvenv venv
|
|
22
|
+
source venv/bin/activate
|
|
23
|
+
```
|
|
24
|
+
|
|
25
|
+
#### Local Deployment
|
|
26
|
+
For a local deployment install the server dependencies :
|
|
27
|
+
```sh
|
|
28
|
+
# Install dependencies for local deployment
|
|
29
|
+
pip install -r requirements-server.txt
|
|
30
|
+
```
|
|
31
|
+
|
|
32
|
+
#### For an WSGI Server
|
|
33
|
+
The core dependencies are in *requirements.txt*:
|
|
34
|
+
```sh
|
|
35
|
+
# Install core dependencies for an WSGI web server
|
|
36
|
+
pip install -r requirements.txt
|
|
37
|
+
```
|
|
38
|
+
|
|
39
|
+
## Authentication
|
|
40
|
+
|
|
41
|
+
The authentication mechanism depends on backends, and each depends on
|
|
42
|
+
a particular configuration. All actions that a user can operate on a
|
|
43
|
+
ephemeral buffer required a **Bearer Token**. This token is received
|
|
44
|
+
after the authentication process from keycloak (IDP used). The token
|
|
45
|
+
has a limited lifetime and can be refresh.
|
|
46
|
+
|
|
47
|
+
## Usage
|
|
48
|
+
To Run the project, start the server : `fastapi dev server/src/main.py`
|
|
49
|
+
|
|
50
|
+
A swagger documentation is available on `http://127.0.0.1:8000/docs/ui`
|
|
51
|
+
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
__version__ = "0.1.5"
|
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
from io import BytesIO
|
|
2
|
+
from asyncio import create_task
|
|
3
|
+
from sqlmodel import Session
|
|
4
|
+
from typing import AsyncGenerator
|
|
5
|
+
from ebuffer.database import get_user_db
|
|
6
|
+
from ebuffer.apiuserobj import Eb_UserObjAPI, Eb_ObjectInvalid
|
|
7
|
+
|
|
8
|
+
from ebuffer.config import EbConfig, logger
|
|
9
|
+
from ebuffer.models_common import UserId
|
|
10
|
+
from ebuffer.models_buffer import BufferStateEnum
|
|
11
|
+
from ebuffer.database import BufferEntry, Eb_BufferCRUD
|
|
12
|
+
from ebuffer.backends import Eb_DataBackend, Eb_BackendReadError
|
|
13
|
+
|
|
14
|
+
Eb_BufferInvalid = Eb_ObjectInvalid
|
|
15
|
+
|
|
16
|
+
class Eb_BufferAPI(Eb_UserObjAPI):
|
|
17
|
+
|
|
18
|
+
def __init__(self, config: EbConfig):
|
|
19
|
+
self.config = config
|
|
20
|
+
self.bufferCRUD = Eb_BufferCRUD()
|
|
21
|
+
self.backend = Eb_DataBackend.getBackend(self.config, self.bufferCRUD)
|
|
22
|
+
super().__init__(BufferEntry, config.base, self.bufferCRUD, self.backend)
|
|
23
|
+
|
|
24
|
+
def destroy(self):
|
|
25
|
+
self.backend.deep_clean()
|
|
26
|
+
del self.backend
|
|
27
|
+
|
|
28
|
+
#
|
|
29
|
+
# Housekeeping
|
|
30
|
+
#
|
|
31
|
+
async def housekeeping(self, session: Session, interval: int, start: bool = False) -> None:
|
|
32
|
+
grace_time = interval - 1
|
|
33
|
+
|
|
34
|
+
for ebuffer in self.bufferCRUD.get_expired(session, interval):
|
|
35
|
+
if ebuffer.state == BufferStateEnum.error_deleted or ebuffer.state == BufferStateEnum.deleted:
|
|
36
|
+
if not start: continue
|
|
37
|
+
|
|
38
|
+
if ebuffer.state == BufferStateEnum.initialized:
|
|
39
|
+
ebuffer.state = BufferStateEnum.error
|
|
40
|
+
ebuffer.state_desc = r'Never reached backend initialization.' if not ebuffer.state_desc else ebuffer.state_desc
|
|
41
|
+
elif ebuffer.state == BufferStateEnum.error:
|
|
42
|
+
grace_time = self.config.base.error_grace_time
|
|
43
|
+
if ebuffer.lifetime < grace_time:
|
|
44
|
+
ebuffer.state = BufferStateEnum.error_deleted
|
|
45
|
+
else:
|
|
46
|
+
if ebuffer.lifetime < grace_time:
|
|
47
|
+
ebuffer.state = BufferStateEnum.deleted
|
|
48
|
+
|
|
49
|
+
if ebuffer.state == BufferStateEnum.error_deleted or ebuffer.state == BufferStateEnum.deleted:
|
|
50
|
+
logger.debug("[housekeeping]: Destroy (%ds < %ds) %s", ebuffer.lifetime, grace_time, ebuffer)
|
|
51
|
+
await self.bufferCRUD.update(session, ebuffer, commit=False)
|
|
52
|
+
await self.destroy_after(session, grace_time, ebuffer, commit=False)
|
|
53
|
+
self.bufferCRUD.commit(session)
|
|
54
|
+
|
|
55
|
+
#
|
|
56
|
+
# I/O operations
|
|
57
|
+
#
|
|
58
|
+
async def write(self, stream: AsyncGenerator[bytes, None], uid: str, user: UserId, session: Session) -> object:
|
|
59
|
+
#userd = get_user_db(session, user, create=False)
|
|
60
|
+
ebuffer = self.bufferCRUD.get(session, uid, user) # Warning: the for now, data has to belong to the user.
|
|
61
|
+
try:
|
|
62
|
+
await self.backend.write(ebuffer, stream)
|
|
63
|
+
finally:
|
|
64
|
+
create_task(self.bufferCRUD.update(session, ebuffer))
|
|
65
|
+
return ebuffer
|
|
66
|
+
|
|
67
|
+
def _generate_bytesio_stream(self, data: BytesIO, chunk_size: int = 4 * 1024 * 1024) -> bytes:
|
|
68
|
+
while chunk := data.read(chunk_size):
|
|
69
|
+
yield chunk # Envoie un morceau au client
|
|
70
|
+
data.close()
|
|
71
|
+
|
|
72
|
+
async def read(self, uid: str, sid: str, seek: int, limit: int, user: UserId, session: Session) -> object:
|
|
73
|
+
# Warning : 'user' is not used, but shall be in order to check if the data has read policy.
|
|
74
|
+
# userd : UserIdEntry = get_user_db(session, user, create=False)
|
|
75
|
+
ebuffer = self.bufferCRUD.get(session, uid)
|
|
76
|
+
data: BytesIO; clen: int
|
|
77
|
+
clen, data = self.backend.fetch(ebuffer, sid, seek=seek, limit=limit) if sid else self.backend.fopen(ebuffer, seek=seek, limit=limit)
|
|
78
|
+
if not data:
|
|
79
|
+
raise Eb_BackendReadError(ebuffer.uuid, r'Empty data stream')
|
|
80
|
+
return clen, self._generate_bytesio_stream(data)
|
|
@@ -0,0 +1,127 @@
|
|
|
1
|
+
from sqlmodel import Session
|
|
2
|
+
from asyncio import create_task, sleep
|
|
3
|
+
|
|
4
|
+
from ebuffer.config import logger
|
|
5
|
+
from ebuffer.models_common import UserId
|
|
6
|
+
from ebuffer.database import UserIdEntry, UserObjEntry, get_user_db, get_user_by_email_db
|
|
7
|
+
from ebuffer.database import Eb_UserObjCRUD, Eb_ObjectNotFound
|
|
8
|
+
from ebuffer.errors import Eb_Exception, Eb_TagSyntaxError
|
|
9
|
+
|
|
10
|
+
#
|
|
11
|
+
# CRUD Management of User's objects
|
|
12
|
+
|
|
13
|
+
class Eb_ObjectInvalid(Eb_Exception):
|
|
14
|
+
def __init__(self, uuid: str = r''):
|
|
15
|
+
super().__init__(430, "ObjectInvalid", "Object is invalid or was destroyed.", r'Id: %s' % uuid)
|
|
16
|
+
|
|
17
|
+
class Eb_UserObjAPI():
|
|
18
|
+
|
|
19
|
+
def __init__(self, Objtype: type, config: object, uobjcrud: Eb_UserObjCRUD, backend: object = None):
|
|
20
|
+
self.ObjEntry: type = Objtype
|
|
21
|
+
self.conf = config
|
|
22
|
+
self.uobjcrud = uobjcrud
|
|
23
|
+
self.backend = backend
|
|
24
|
+
|
|
25
|
+
#
|
|
26
|
+
# Life cycle management
|
|
27
|
+
|
|
28
|
+
def assert_correct_tags(self, tags: list[str]) -> bool:
|
|
29
|
+
tag_min_size = self.conf.tag_min_size
|
|
30
|
+
tag_max_size = self.conf.tag_max_size
|
|
31
|
+
for t in tags:
|
|
32
|
+
if len(t) < tag_min_size or len(t) > tag_max_size:
|
|
33
|
+
raise Eb_TagSyntaxError(r'Invalid tag size %d ([%d:%d])' % (len(t), tag_min_size, tag_max_size))
|
|
34
|
+
return True
|
|
35
|
+
|
|
36
|
+
async def _destroy(self, session: Session, uobj: UserObjEntry, commit) -> None:
|
|
37
|
+
if self.backend: await self.backend.destroy(session, uobj, commit=commit)
|
|
38
|
+
else: await self.uobjcrud.destroy(session, uobj, commit=commit)
|
|
39
|
+
|
|
40
|
+
async def _destroy_after(self, session: Session, time_s: int, uobj: UserObjEntry, maxcount: int, commit) -> None:
|
|
41
|
+
try:
|
|
42
|
+
await sleep(time_s)
|
|
43
|
+
# session: Session = next(app.g_db.get_session())
|
|
44
|
+
await self._destroy(session, uobj, commit)
|
|
45
|
+
except Exception as e:
|
|
46
|
+
logger.debug(r"Could not finish a destroy operation: %s", str(e))
|
|
47
|
+
if maxcount > 0:
|
|
48
|
+
create_task(self._destroy_after(session, time_s, uobj, maxcount-1, commit))
|
|
49
|
+
|
|
50
|
+
async def destroy_after(self, session: Session, time_s: int, uobj: UserObjEntry, maxcount: int = 5, commit=True) -> None:
|
|
51
|
+
#create_task(self._destroy_after(session, time_s, uobj, maxcount, commit=True))
|
|
52
|
+
if time_s: create_task(self._destroy_after(session, time_s, uobj, maxcount, commit=True))
|
|
53
|
+
else: self._destroy(session, uobj, commit)
|
|
54
|
+
|
|
55
|
+
#
|
|
56
|
+
# Main API
|
|
57
|
+
|
|
58
|
+
async def create(self, uobj: UserObjEntry, blocking: bool, user: UserId, session: Session) -> UserObjEntry:
|
|
59
|
+
self.assert_correct_tags(uobj.tags)
|
|
60
|
+
# Only in order to register the used in the DB.
|
|
61
|
+
userId: UserIdEntry = get_user_db(session, user, create=True)
|
|
62
|
+
uobj.user_email = userId.email
|
|
63
|
+
|
|
64
|
+
if blocking and self.backend:
|
|
65
|
+
await self.backend.build(session, uobj)
|
|
66
|
+
else:
|
|
67
|
+
await self.uobjcrud.update(session, uobj)
|
|
68
|
+
if self.backend:
|
|
69
|
+
create_task(self.backend.build(session, uobj))
|
|
70
|
+
logger.debug(r'[api] Register %s: %s', self.ObjEntry.__name__, uobj)
|
|
71
|
+
return uobj
|
|
72
|
+
|
|
73
|
+
async def search(self, session: Session = None, user: UserId = None,
|
|
74
|
+
limit: int = 0, skip: int = 0,
|
|
75
|
+
owner: str = None, tags: list[str] = [], all: bool = False, count: bool = False,
|
|
76
|
+
ofilter: callable = None) -> list[UserObjEntry] | int:
|
|
77
|
+
self.assert_correct_tags(tags)
|
|
78
|
+
if owner:
|
|
79
|
+
owner = get_user_by_email_db(session, owner, create=False)
|
|
80
|
+
if not owner:
|
|
81
|
+
return 0 if count else []
|
|
82
|
+
results = []
|
|
83
|
+
|
|
84
|
+
# Warning : 'user' is not used, but shall be in order to check if the data has read policy.
|
|
85
|
+
if count:
|
|
86
|
+
#logger.debug("[searchObject]------ %s: %s [%s] --------", str(owner), str(tags), str(all))
|
|
87
|
+
return self.uobjcrud.count(session, limit, skip, owner, tags, all, ofilter=ofilter)
|
|
88
|
+
else:
|
|
89
|
+
#logger.debug("[searchObject]------ %s: %s [%s] --------", str(owner), str(tags), str(all))
|
|
90
|
+
for uobj in self.uobjcrud.search(session, limit, skip, owner, tags, all, ofilter=ofilter):
|
|
91
|
+
# logger.debug("[searchObject]: found %s" % uobj)
|
|
92
|
+
results.append(uobj)
|
|
93
|
+
# logger.debug("[searchObject]+++++++++++++++")
|
|
94
|
+
return results
|
|
95
|
+
|
|
96
|
+
async def get(self, uid: str, owner: str, user: UserId, session: Session) -> UserObjEntry:
|
|
97
|
+
if owner:
|
|
98
|
+
owner = get_user_by_email_db(session, owner, create=False)
|
|
99
|
+
if not owner: raise Eb_ObjectNotFound(uid)
|
|
100
|
+
# Warning : 'user' is not used, but shall be in order to check if the data has read policy.
|
|
101
|
+
return self.uobjcrud.get(session, uid, owner)
|
|
102
|
+
|
|
103
|
+
async def delete(self, uid: str, user: UserId, session: Session) -> UserObjEntry:
|
|
104
|
+
#userd = get_user_db(session, user, create=False)
|
|
105
|
+
uobj = self.uobjcrud.get(session, uid, user) # Warning: the for now, data has to belong to the user.
|
|
106
|
+
if uobj.is_deleted(): return uobj
|
|
107
|
+
uobj.set_deleted()
|
|
108
|
+
await self.uobjcrud.update(session, uobj)
|
|
109
|
+
await self.destroy_after(session, self.conf.grace_time, uobj)
|
|
110
|
+
return uobj
|
|
111
|
+
|
|
112
|
+
#
|
|
113
|
+
# Tag API
|
|
114
|
+
|
|
115
|
+
async def getTags(self, uid: str, owner: str, user: UserId, session: Session) -> UserObjEntry:
|
|
116
|
+
uobj = await self.get(uid, owner, user, session)
|
|
117
|
+
return uobj.tags if uobj else []
|
|
118
|
+
|
|
119
|
+
async def addTag(self, uid: str, tag: str, user: UserId, session: Session) -> UserObjEntry:
|
|
120
|
+
#userd = get_user_db(session, user, create=False)
|
|
121
|
+
self.assert_correct_tags((tag,))
|
|
122
|
+
uobj = self.uobjcrud.get(session, uid, user) # Warning: the for now, data has to belong to the user.
|
|
123
|
+
if uobj.is_deleted(): raise Eb_ObjectInvalid(uid)
|
|
124
|
+
uobj.tags = list(uobj.tags)
|
|
125
|
+
uobj.tags.append(tag)
|
|
126
|
+
create_task(self.uobjcrud.update(session, uobj))
|
|
127
|
+
return uobj
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
from os import getenv
|
|
2
|
+
from asyncio import create_task, sleep
|
|
3
|
+
from traceback import format_exc
|
|
4
|
+
from ebuffer.config import EbConfig, logger
|
|
5
|
+
from ebuffer.database import Eb_Database
|
|
6
|
+
from ebuffer.auth import Eb_Auth
|
|
7
|
+
from ebuffer.apiebuffer import Eb_BufferAPI
|
|
8
|
+
from fastapi.security import HTTPBearer, HTTPBasic
|
|
9
|
+
|
|
10
|
+
class Eb_Application():
|
|
11
|
+
|
|
12
|
+
def __init__(self):
|
|
13
|
+
self.regexp_email = r'^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}$'
|
|
14
|
+
self.config = EbConfig(inifile=getenv(r'EBUFFER_INI', r'eb.ini'), secretfile=getenv(r'EBUFFER_SECRET_INI', r'.eb-secret.ini'))
|
|
15
|
+
self.db = Eb_Database(self.config)
|
|
16
|
+
self.bufferAPI = Eb_BufferAPI(self.config)
|
|
17
|
+
self.auth = Eb_Auth.getAuth(self.config)
|
|
18
|
+
self.security_token = HTTPBearer(auto_error=False)
|
|
19
|
+
self.security_cred = HTTPBasic(auto_error=False)
|
|
20
|
+
|
|
21
|
+
#
|
|
22
|
+
# Housekeeping
|
|
23
|
+
#
|
|
24
|
+
async def housekeeping(self, start: bool = False) -> None:
|
|
25
|
+
interval = self.config.base.housekeeping_interval
|
|
26
|
+
await sleep(interval)
|
|
27
|
+
#logger.debug(r"Start housekeeping (%ds) %s", interval, r'Initial' if start else r'')
|
|
28
|
+
try:
|
|
29
|
+
session = next(self.db.get_session())
|
|
30
|
+
await self.bufferAPI.housekeeping(session, interval, start)
|
|
31
|
+
except Exception as e:
|
|
32
|
+
logger.error(r"Could not finish a housekeeping session %s: %s", str(e), format_exc())
|
|
33
|
+
# raise Eb_HouseKeepingError(e)
|
|
34
|
+
finally:
|
|
35
|
+
create_task(self.housekeeping())
|
|
36
|
+
|
|
37
|
+
def start(self):
|
|
38
|
+
create_task(self.housekeeping(start=True))
|
|
39
|
+
|
|
40
|
+
def destroy(self):
|
|
41
|
+
self.bufferAPI.destroy()
|
|
42
|
+
del self.db
|
|
43
|
+
self.db = None
|
|
44
|
+
|
|
45
|
+
app_g = Eb_Application()
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
from ebuffer.auth.auth import Eb_Auth
|
|
2
|
+
|
|
3
|
+
from ebuffer.auth.auth import Eb_AuthDuplicatedError, Eb_AuthMissingError
|
|
4
|
+
from ebuffer.auth.auth import Eb_AuthError, Eb_AuthExpired
|
|
5
|
+
from ebuffer.auth.openid import Eb_OI_ConnectError, Eb_OI_InternalError, Eb_OI_LoginError
|
|
6
|
+
from ebuffer.auth.passfile import Eb_AuthFile_LoginError
|
|
7
|
+
|
|
8
|
+
from ebuffer.auth.openid import OIDLoginExceptionList
|
|
9
|
+
from ebuffer.auth.openid import OIDAuthExceptionList
|
|
10
|
+
|
|
11
|
+
from ebuffer.auth.passfile import PassFileLoginExceptionList
|
|
12
|
+
from ebuffer.auth.passfile import PassFileAuthExceptionList
|
|
13
|
+
|
|
14
|
+
LoginExceptionList = {} | OIDLoginExceptionList | PassFileLoginExceptionList
|
|
15
|
+
AuthExceptionList = {} | OIDAuthExceptionList | PassFileAuthExceptionList
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
from ebuffer.config import EbConfig
|
|
2
|
+
from ebuffer.models_common import UserCred, UserId, MessageLogin
|
|
3
|
+
from ebuffer.errors import Eb_Exception
|
|
4
|
+
|
|
5
|
+
class Eb_AuthError(Eb_Exception):
|
|
6
|
+
def __init__(self, code: int = -1):
|
|
7
|
+
super().__init__(401, "AuthError", "Invalid credentials.", r'Code: %d' % code)
|
|
8
|
+
|
|
9
|
+
class Eb_AuthExpired(Eb_Exception):
|
|
10
|
+
def __init__(self, status: str = r''):
|
|
11
|
+
super().__init__(401, "AuthExpired", "Invalid or expired token.", r'Status: %s' % status)
|
|
12
|
+
|
|
13
|
+
class Eb_AuthDuplicatedError(Eb_Exception):
|
|
14
|
+
def __init__(self, scheme: str):
|
|
15
|
+
super().__init__(440, "AuthDuplicatedError", "Multiple authentication defined with the same scheme (%s)." % scheme, "Check authentication installation.")
|
|
16
|
+
|
|
17
|
+
class Eb_AuthMissingError(Eb_Exception):
|
|
18
|
+
def __init__(self, scheme: str):
|
|
19
|
+
super().__init__(440, "AuthMissingError", "Authentication '%s' not found." % scheme, "Check authentication installation.")
|
|
20
|
+
|
|
21
|
+
class Eb_Auth():
|
|
22
|
+
g_auths: dict = {}
|
|
23
|
+
|
|
24
|
+
def __init__(self, config: EbConfig, name : str = r'none'):
|
|
25
|
+
self.config = config
|
|
26
|
+
self.name : str = name
|
|
27
|
+
|
|
28
|
+
@staticmethod
|
|
29
|
+
def addAuth(name, auth_class):
|
|
30
|
+
if name in Eb_Auth.g_auths: raise Eb_AuthDuplicatedError(name)
|
|
31
|
+
Eb_Auth.g_auths[name] = auth_class
|
|
32
|
+
|
|
33
|
+
@staticmethod
|
|
34
|
+
def getAuth(config: EbConfig):
|
|
35
|
+
name = config.base.auth_backend
|
|
36
|
+
if name not in Eb_Auth.g_auths: raise Eb_AuthMissingError(name)
|
|
37
|
+
return Eb_Auth.g_auths[name](config, name)
|
|
38
|
+
|
|
39
|
+
def login(self, user: UserCred | None, headers: dict) -> MessageLogin:
|
|
40
|
+
return MessageLogin(access_token=r'b3BlbmJhcg==')
|
|
41
|
+
|
|
42
|
+
def verify_token(self, scheme: str, token: str) -> UserId:
|
|
43
|
+
return UserId(name=self.config.base.default_user, email=self.config.base.default_email)
|
|
44
|
+
|
|
45
|
+
def verify_basic(self, user: str, password: str) -> UserId:
|
|
46
|
+
return UserId(name=self.config.base.default_user, email=self.config.base.default_email)
|
|
47
|
+
|
|
48
|
+
def getRealmHeader(self) -> dict:
|
|
49
|
+
return {}
|
|
50
|
+
|
|
51
|
+
Eb_Auth.addAuth(r'none', Eb_Auth)
|
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
import requests
|
|
2
|
+
from pydantic import BaseModel
|
|
3
|
+
from ebuffer.models_common import UserCred, UserId, MessageLogin
|
|
4
|
+
from ebuffer.config import EbConfig, logger
|
|
5
|
+
from ebuffer.errors import Eb_Exception
|
|
6
|
+
from ebuffer.auth import Eb_Auth, Eb_AuthError, Eb_AuthExpired
|
|
7
|
+
|
|
8
|
+
class Eb_OI_ConnectError(Eb_Exception):
|
|
9
|
+
def __init__(self, srv: str = r'', exc: Exception = None):
|
|
10
|
+
super().__init__(501, "OIDConnErr", "Could not connect to the OpenID server.", r'Server: %s' % srv, exc)
|
|
11
|
+
|
|
12
|
+
class Eb_OI_InternalError(Eb_Exception):
|
|
13
|
+
def __init__(self, exc: Exception = None):
|
|
14
|
+
super().__init__(500, "OIDInternalError", "OIDC Internal error.", "Check server configuration.", exc)
|
|
15
|
+
|
|
16
|
+
class Eb_OI_LoginError(Eb_Exception):
|
|
17
|
+
def __init__(self, code: int = -1):
|
|
18
|
+
super().__init__(401, "OIDLoginErr", "Could not login with OIDC credentials.", r'Code: %d' % code)
|
|
19
|
+
|
|
20
|
+
OIDLoginExceptionList = \
|
|
21
|
+
Eb_OI_InternalError().model() | \
|
|
22
|
+
Eb_OI_ConnectError().model() | \
|
|
23
|
+
Eb_OI_LoginError().model()
|
|
24
|
+
|
|
25
|
+
OIDAuthExceptionList = \
|
|
26
|
+
Eb_OI_InternalError().model() | \
|
|
27
|
+
Eb_AuthError().model() | \
|
|
28
|
+
Eb_AuthExpired().model()
|
|
29
|
+
|
|
30
|
+
class EbConfigOpenID(BaseModel):
|
|
31
|
+
openid_base_url: str = r''
|
|
32
|
+
openid_client_id: str = r''
|
|
33
|
+
openid_client_secret: str = r''
|
|
34
|
+
access_token_url: str
|
|
35
|
+
introspect_url: str
|
|
36
|
+
user_info_url: str
|
|
37
|
+
logout_url: str
|
|
38
|
+
|
|
39
|
+
def __init__(self, **kw):
|
|
40
|
+
url = kw[r'openid_base_url'] if r'openid_base_url' in kw else r'https://localhost/realms'
|
|
41
|
+
kw[r'access_token_url'] = f"{url}/protocol/openid-connect/token"
|
|
42
|
+
kw[r'introspect_url'] = f"{url}/protocol/openid-connect/token/introspect"
|
|
43
|
+
kw[r'user_info_url'] = f"{url}/protocol/openid-connect/userinfo"
|
|
44
|
+
kw[r'logout_url'] = f"{url}/protocol/openid-connect/logout"
|
|
45
|
+
super().__init__(**kw)
|
|
46
|
+
|
|
47
|
+
class Eb_AuthOpenID(Eb_Auth):
|
|
48
|
+
|
|
49
|
+
def __init__(self, config: EbConfig, name : str):
|
|
50
|
+
config = config.open_section(name, EbConfigOpenID)
|
|
51
|
+
super().__init__(config, name)
|
|
52
|
+
|
|
53
|
+
def login(self, user: UserCred | None, headers: dict) -> MessageLogin:
|
|
54
|
+
logger.debug(r"----- OpenID %s ------", str(user))
|
|
55
|
+
payload = {
|
|
56
|
+
"client_id": self.config.openid_client_id,
|
|
57
|
+
"client_secret": self.config.openid_client_secret,
|
|
58
|
+
"grant_type": "password",
|
|
59
|
+
"username": user.username,
|
|
60
|
+
"password": user.password,
|
|
61
|
+
"scope": "openid",
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
headers = {"Content-Type": "application/x-www-form-urlencoded"}
|
|
65
|
+
try:
|
|
66
|
+
response = requests.post(self.config.access_token_url, data=payload, headers=headers)
|
|
67
|
+
except requests.exceptions.ConnectionError as e:
|
|
68
|
+
raise Eb_OI_ConnectError(self.config.access_token_url, e)
|
|
69
|
+
|
|
70
|
+
if response.status_code != 200:
|
|
71
|
+
raise Eb_OI_LoginError(response.status_code)
|
|
72
|
+
|
|
73
|
+
result = response.json()
|
|
74
|
+
return MessageLogin(result["access_token"])
|
|
75
|
+
|
|
76
|
+
def verify_token(self, scheme: str, token: str) -> UserId:
|
|
77
|
+
headers = {"Content-Type": "application/x-www-form-urlencoded"}
|
|
78
|
+
payloads = {
|
|
79
|
+
"token": token,
|
|
80
|
+
"client_id": self.config.openid_client_id,
|
|
81
|
+
"client_secret": self.config.openid_client_secret,
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
try:
|
|
85
|
+
response = requests.post(self.config.introspect_url, data=payloads, headers=headers)
|
|
86
|
+
if response.status_code != 200:
|
|
87
|
+
raise Eb_AuthError(response.status_code)
|
|
88
|
+
|
|
89
|
+
token_info = response.json()
|
|
90
|
+
if not token_info.get("active"):
|
|
91
|
+
raise Eb_AuthExpired(str(token_info))
|
|
92
|
+
|
|
93
|
+
return UserId(**token_info)
|
|
94
|
+
|
|
95
|
+
except requests.RequestException as e:
|
|
96
|
+
raise Eb_OI_InternalError(e)
|
|
97
|
+
|
|
98
|
+
def verify_basic(self, user: str, password: str) -> UserId:
|
|
99
|
+
raise Eb_AuthError(-1)
|
|
100
|
+
|
|
101
|
+
Eb_Auth.addAuth(r'auth::openid', Eb_AuthOpenID)
|