keepup-admin 0.2.0__tar.gz → 0.3.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {keepup_admin-0.2.0/keepup_admin.egg-info → keepup_admin-0.3.0}/PKG-INFO +7 -4
- keepup_admin-0.3.0/admin_trail.py +60 -0
- keepup_admin-0.3.0/api_docs.py +76 -0
- {keepup_admin-0.2.0 → keepup_admin-0.3.0}/audit.py +39 -2
- {keepup_admin-0.2.0 → keepup_admin-0.3.0}/auth/dependencies.py +175 -29
- keepup_admin-0.3.0/auth/external_accounts.py +144 -0
- keepup_admin-0.3.0/auth/identity/__init__.py +33 -0
- keepup_admin-0.3.0/auth/identity/access.py +119 -0
- keepup_admin-0.3.0/auth/identity/accounts.py +86 -0
- keepup_admin-0.3.0/auth/identity/config.py +220 -0
- keepup_admin-0.3.0/auth/identity/contract.py +174 -0
- keepup_admin-0.3.0/auth/identity/loader.py +119 -0
- keepup_admin-0.3.0/auth/identity/runtime.py +265 -0
- {keepup_admin-0.2.0 → keepup_admin-0.3.0}/auth/login_throttle.py +30 -1
- {keepup_admin-0.2.0 → keepup_admin-0.3.0}/auth/oidc_policy.py +13 -2
- {keepup_admin-0.2.0 → keepup_admin-0.3.0}/auth/oidc_routes.py +60 -83
- {keepup_admin-0.2.0 → keepup_admin-0.3.0}/auth/panel_session.py +186 -20
- {keepup_admin-0.2.0 → keepup_admin-0.3.0}/auth/permissions.py +7 -7
- {keepup_admin-0.2.0 → keepup_admin-0.3.0}/auth/providers/base.py +14 -0
- {keepup_admin-0.2.0 → keepup_admin-0.3.0}/auth/providers/local.py +31 -1
- {keepup_admin-0.2.0 → keepup_admin-0.3.0}/auth/routes.py +101 -211
- {keepup_admin-0.2.0 → keepup_admin-0.3.0}/auth/seed_accounts.py +47 -17
- keepup_admin-0.3.0/auth/socket_sessions.py +176 -0
- keepup_admin-0.3.0/auth/user_roles.py +376 -0
- keepup_admin-0.3.0/auth/user_routes.py +313 -0
- keepup_admin-0.3.0/auth/usernames.py +36 -0
- {keepup_admin-0.2.0 → keepup_admin-0.3.0}/auth/websocket.py +3 -1
- {keepup_admin-0.2.0 → keepup_admin-0.3.0}/body_limit.py +8 -4
- {keepup_admin-0.2.0 → keepup_admin-0.3.0}/db.py +154 -12
- {keepup_admin-0.2.0 → keepup_admin-0.3.0}/events.py +23 -1
- {keepup_admin-0.2.0 → keepup_admin-0.3.0}/events_api.py +44 -17
- {keepup_admin-0.2.0 → keepup_admin-0.3.0}/factory.py +107 -36
- {keepup_admin-0.2.0 → keepup_admin-0.3.0/keepup_admin.egg-info}/PKG-INFO +7 -4
- {keepup_admin-0.2.0 → keepup_admin-0.3.0}/keepup_admin.egg-info/SOURCES.txt +34 -0
- {keepup_admin-0.2.0 → keepup_admin-0.3.0}/keepup_admin.egg-info/requires.txt +6 -3
- {keepup_admin-0.2.0 → keepup_admin-0.3.0}/locks.py +12 -5
- {keepup_admin-0.2.0 → keepup_admin-0.3.0}/log_shipping.py +96 -51
- {keepup_admin-0.2.0 → keepup_admin-0.3.0}/logging_setup.py +22 -0
- {keepup_admin-0.2.0 → keepup_admin-0.3.0}/metrics_api.py +9 -13
- {keepup_admin-0.2.0 → keepup_admin-0.3.0}/modules.py +135 -27
- {keepup_admin-0.2.0 → keepup_admin-0.3.0}/notification_bus.py +63 -3
- {keepup_admin-0.2.0 → keepup_admin-0.3.0}/plugins/admin.py +34 -11
- {keepup_admin-0.2.0 → keepup_admin-0.3.0}/plugins/route_mask.py +8 -1
- {keepup_admin-0.2.0 → keepup_admin-0.3.0}/plugins/routes.py +34 -3
- keepup_admin-0.3.0/positional_sql.py +120 -0
- {keepup_admin-0.2.0 → keepup_admin-0.3.0}/pyproject.toml +22 -4
- {keepup_admin-0.2.0 → keepup_admin-0.3.0}/schema.py +28 -4
- keepup_admin-0.3.0/security.py +54 -0
- {keepup_admin-0.2.0 → keepup_admin-0.3.0}/settings.py +44 -6
- {keepup_admin-0.2.0 → keepup_admin-0.3.0}/static/index_nebula.html +11 -22
- {keepup_admin-0.2.0 → keepup_admin-0.3.0}/static/index_new.html +11 -22
- keepup_admin-0.3.0/static/js/layout_bootstrap.js +29 -0
- {keepup_admin-0.2.0 → keepup_admin-0.3.0}/static/js/main_new.js +187 -18
- {keepup_admin-0.2.0 → keepup_admin-0.3.0}/static/modules/js/background_tasks.js +38 -16
- {keepup_admin-0.2.0 → keepup_admin-0.3.0}/static/modules/js/cluster.js +14 -3
- {keepup_admin-0.2.0 → keepup_admin-0.3.0}/static/modules/js/event_manager.js +44 -16
- {keepup_admin-0.2.0 → keepup_admin-0.3.0}/static/modules/js/integration_logs.js +54 -26
- {keepup_admin-0.2.0 → keepup_admin-0.3.0}/static/modules/js/metrics.js +34 -13
- {keepup_admin-0.2.0 → keepup_admin-0.3.0}/static/modules/js/modules.js +68 -26
- {keepup_admin-0.2.0 → keepup_admin-0.3.0}/static/modules/js/themes.js +45 -14
- {keepup_admin-0.2.0 → keepup_admin-0.3.0}/static/modules/js/users.js +188 -74
- {keepup_admin-0.2.0 → keepup_admin-0.3.0}/tables.py +2 -0
- {keepup_admin-0.2.0 → keepup_admin-0.3.0}/themes.py +47 -12
- {keepup_admin-0.2.0 → keepup_admin-0.3.0}/web.py +16 -3
- {keepup_admin-0.2.0 → keepup_admin-0.3.0}/LICENSE +0 -0
- {keepup_admin-0.2.0 → keepup_admin-0.3.0}/NOTICE +0 -0
- {keepup_admin-0.2.0 → keepup_admin-0.3.0}/README.md +0 -0
- {keepup_admin-0.2.0 → keepup_admin-0.3.0}/THIRD-PARTY.md +0 -0
- {keepup_admin-0.2.0 → keepup_admin-0.3.0}/__init__.py +0 -0
- {keepup_admin-0.2.0 → keepup_admin-0.3.0}/api_versions.py +0 -0
- {keepup_admin-0.2.0 → keepup_admin-0.3.0}/auth/__init__.py +0 -0
- {keepup_admin-0.2.0 → keepup_admin-0.3.0}/auth/config.py +0 -0
- {keepup_admin-0.2.0 → keepup_admin-0.3.0}/auth/dto/__init__.py +0 -0
- {keepup_admin-0.2.0 → keepup_admin-0.3.0}/auth/dto/token.py +0 -0
- {keepup_admin-0.2.0 → keepup_admin-0.3.0}/auth/factory.py +0 -0
- {keepup_admin-0.2.0 → keepup_admin-0.3.0}/auth/oidc.py +0 -0
- {keepup_admin-0.2.0 → keepup_admin-0.3.0}/auth/providers/__init__.py +0 -0
- {keepup_admin-0.2.0 → keepup_admin-0.3.0}/auth/session_lifetime.py +0 -0
- {keepup_admin-0.2.0 → keepup_admin-0.3.0}/auth/signing_key.py +0 -0
- {keepup_admin-0.2.0 → keepup_admin-0.3.0}/cache.py +0 -0
- {keepup_admin-0.2.0 → keepup_admin-0.3.0}/cluster.py +0 -0
- {keepup_admin-0.2.0 → keepup_admin-0.3.0}/instance.py +0 -0
- {keepup_admin-0.2.0 → keepup_admin-0.3.0}/integrations.py +0 -0
- {keepup_admin-0.2.0 → keepup_admin-0.3.0}/keepup_admin.egg-info/dependency_links.txt +0 -0
- {keepup_admin-0.2.0 → keepup_admin-0.3.0}/keepup_admin.egg-info/top_level.txt +0 -0
- {keepup_admin-0.2.0 → keepup_admin-0.3.0}/metrics.py +0 -0
- {keepup_admin-0.2.0 → keepup_admin-0.3.0}/metrics_retention.py +0 -0
- {keepup_admin-0.2.0 → keepup_admin-0.3.0}/plugins/__init__.py +0 -0
- {keepup_admin-0.2.0 → keepup_admin-0.3.0}/plugins/base.py +0 -0
- {keepup_admin-0.2.0 → keepup_admin-0.3.0}/plugins/enablement.py +0 -0
- {keepup_admin-0.2.0 → keepup_admin-0.3.0}/plugins/registry.py +0 -0
- {keepup_admin-0.2.0 → keepup_admin-0.3.0}/retention.py +0 -0
- {keepup_admin-0.2.0 → keepup_admin-0.3.0}/roles.py +0 -0
- {keepup_admin-0.2.0 → keepup_admin-0.3.0}/scheduler.py +0 -0
- {keepup_admin-0.2.0 → keepup_admin-0.3.0}/sections.json +0 -0
- {keepup_admin-0.2.0 → keepup_admin-0.3.0}/setup.cfg +0 -0
- {keepup_admin-0.2.0 → keepup_admin-0.3.0}/static/css/aos.css +0 -0
- {keepup_admin-0.2.0 → keepup_admin-0.3.0}/static/css/main_nebula.css +0 -0
- {keepup_admin-0.2.0 → keepup_admin-0.3.0}/static/css/main_new.css +0 -0
- {keepup_admin-0.2.0 → keepup_admin-0.3.0}/static/css/tailwind.css +0 -0
- {keepup_admin-0.2.0 → keepup_admin-0.3.0}/static/js/aos.js +0 -0
- {keepup_admin-0.2.0 → keepup_admin-0.3.0}/static/js/feather-icons.js +0 -0
- {keepup_admin-0.2.0 → keepup_admin-0.3.0}/static/js/tailwind.js +0 -0
- {keepup_admin-0.2.0 → keepup_admin-0.3.0}/static/modules/css/background_tasks.css +0 -0
- {keepup_admin-0.2.0 → keepup_admin-0.3.0}/static/modules/css/cluster.css +0 -0
- {keepup_admin-0.2.0 → keepup_admin-0.3.0}/static/modules/css/event_manager.css +0 -0
- {keepup_admin-0.2.0 → keepup_admin-0.3.0}/static/modules/css/integration_logs.css +0 -0
- {keepup_admin-0.2.0 → keepup_admin-0.3.0}/static/modules/css/metrics.css +0 -0
- {keepup_admin-0.2.0 → keepup_admin-0.3.0}/static/modules/css/modules.css +0 -0
- {keepup_admin-0.2.0 → keepup_admin-0.3.0}/static/modules/css/themes.css +0 -0
- {keepup_admin-0.2.0 → keepup_admin-0.3.0}/static/modules/css/users.css +0 -0
- {keepup_admin-0.2.0 → keepup_admin-0.3.0}/static/modules/js/chart.js +0 -0
- {keepup_admin-0.2.0 → keepup_admin-0.3.0}/static/modules/js/chartjs-adapter-date-fns.bundle.min.js +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: keepup-admin
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.3.0
|
|
4
4
|
Summary: An admin framework over FastAPI: plugins, panel, users, replicas
|
|
5
5
|
Author-email: Fedor Sergeev <fksergeev@gmail.com>
|
|
6
6
|
License-Expression: Apache-2.0
|
|
@@ -25,17 +25,20 @@ Requires-Dist: sqlalchemy<3,>=2.0
|
|
|
25
25
|
Requires-Dist: psycopg2-binary>=2.9
|
|
26
26
|
Requires-Dist: apscheduler<4,>=3.10
|
|
27
27
|
Requires-Dist: bcrypt<5,>=4.0
|
|
28
|
-
Requires-Dist: pyjwt[crypto]<3,>=2.
|
|
29
|
-
Requires-Dist: python-multipart>=0.0.
|
|
28
|
+
Requires-Dist: pyjwt[crypto]<3,>=2.15
|
|
29
|
+
Requires-Dist: python-multipart>=0.0.31
|
|
30
30
|
Requires-Dist: PyYAML<7,>=6.0
|
|
31
31
|
Requires-Dist: psutil>=7.0
|
|
32
32
|
Requires-Dist: prometheus-client>=0.24
|
|
33
|
-
Requires-Dist: requests>=2.
|
|
33
|
+
Requires-Dist: requests>=2.33
|
|
34
34
|
Requires-Dist: httpx>=0.28
|
|
35
|
+
Requires-Dist: cryptography>=50.0
|
|
36
|
+
Requires-Dist: urllib3>=2.8
|
|
35
37
|
Provides-Extra: test
|
|
36
38
|
Requires-Dist: pytest>=8; extra == "test"
|
|
37
39
|
Requires-Dist: pytest-asyncio>=0.24; extra == "test"
|
|
38
40
|
Requires-Dist: pytest-cov>=5; extra == "test"
|
|
41
|
+
Requires-Dist: uvicorn>=0.30; extra == "test"
|
|
39
42
|
Provides-Extra: postgres
|
|
40
43
|
Requires-Dist: asyncpg>=0.31; extra == "postgres"
|
|
41
44
|
Dynamic: license-file
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
"""What an administrator changed, written into the event log (keepup-67).
|
|
2
|
+
|
|
3
|
+
The event log is where a deployment looks for who did what, and the framework's
|
|
4
|
+
own administrative surface left nothing there: purging the log, changing the
|
|
5
|
+
panel's sections, their grants and the themes, and switching a plugin's
|
|
6
|
+
decision were visible only in the application log -- which rotates, is shipped
|
|
7
|
+
elsewhere or nowhere, and is not what an operator reads. Each of those now
|
|
8
|
+
writes one event naming the administrator.
|
|
9
|
+
|
|
10
|
+
The types below are the framework's. Neither they nor the types an application
|
|
11
|
+
declares can be created by hand through `POST /api/events`: a journal the
|
|
12
|
+
watched party can fill with lookalikes is not evidence (keepup/events_api.py).
|
|
13
|
+
|
|
14
|
+
Writing the trail never fails the action it records: the action has already
|
|
15
|
+
happened, and answering an error for it would only invite a repeat. A trail
|
|
16
|
+
that could not be written is logged as an error.
|
|
17
|
+
"""
|
|
18
|
+
|
|
19
|
+
import logging
|
|
20
|
+
from typing import Any, Dict, Optional
|
|
21
|
+
|
|
22
|
+
from keepup import events
|
|
23
|
+
|
|
24
|
+
logger = logging.getLogger(__name__)
|
|
25
|
+
|
|
26
|
+
#: What an application may import from this module. Everything else is
|
|
27
|
+
#: internal and may change without notice -- see doc/keepup.md.
|
|
28
|
+
__all__ = [
|
|
29
|
+
"ADMIN_EVENT_TYPES",
|
|
30
|
+
"EVENTS_PURGED",
|
|
31
|
+
"PLUGIN_DECISION_CHANGED",
|
|
32
|
+
"SECTIONS_CHANGED",
|
|
33
|
+
"THEMES_CHANGED",
|
|
34
|
+
"record",
|
|
35
|
+
]
|
|
36
|
+
|
|
37
|
+
EVENTS_PURGED = "admin_events_purged"
|
|
38
|
+
SECTIONS_CHANGED = "admin_sections_changed"
|
|
39
|
+
THEMES_CHANGED = "admin_themes_changed"
|
|
40
|
+
PLUGIN_DECISION_CHANGED = "admin_plugin_decision_changed"
|
|
41
|
+
|
|
42
|
+
ADMIN_EVENT_TYPES = (EVENTS_PURGED, SECTIONS_CHANGED, THEMES_CHANGED, PLUGIN_DECISION_CHANGED)
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def _author(admin: Optional[Dict[str, Any]]) -> Dict[str, Any]:
|
|
46
|
+
admin = admin or {}
|
|
47
|
+
return {"user_id": admin.get("id"), "username": admin.get("username")}
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
async def record(event_type: str, admin: Optional[Dict[str, Any]], text: str,
|
|
51
|
+
**details: Any) -> Optional[int]:
|
|
52
|
+
"""Write one administrative event; the id, or None when it could not be written."""
|
|
53
|
+
data = {"by": _author(admin), **details}
|
|
54
|
+
try:
|
|
55
|
+
return await events.event_manager.create_event(
|
|
56
|
+
event_type=event_type, event_text=text, event_data=data)
|
|
57
|
+
except Exception as error:
|
|
58
|
+
logger.error(f"Could not record {event_type} by {data['by'].get('username')}: {error}")
|
|
59
|
+
return None
|
|
60
|
+
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
"""The API schema and the documentation pages built from it (keepup-96).
|
|
2
|
+
|
|
3
|
+
FastAPI serves its OpenAPI schema at /openapi.json to anybody, and the schema
|
|
4
|
+
describes every route of the application -- the administrative ones with their
|
|
5
|
+
parameters and bodies. For somebody outside that is a map of where to knock.
|
|
6
|
+
FastAPI has no notion of who is asking, so the framework takes both routes over:
|
|
7
|
+
|
|
8
|
+
- the schema answers an administrator only, like every other administrative
|
|
9
|
+
route of the framework, unless the application opens it on purpose
|
|
10
|
+
(``KeepupSettings.openapi_public``) or removes it (``openapi_url=None``);
|
|
11
|
+
- the documentation pages stay public shells: they fetch the schema from the
|
|
12
|
+
browser, with the panel's cookie, so an administrator sees the API and anybody
|
|
13
|
+
else sees the refusal.
|
|
14
|
+
|
|
15
|
+
Building the schema in code -- ``app.openapi()``, which ci/generate_openapi.py
|
|
16
|
+
calls -- is untouched: it never went through the route.
|
|
17
|
+
"""
|
|
18
|
+
|
|
19
|
+
from fastapi import Depends
|
|
20
|
+
from fastapi.openapi.docs import (
|
|
21
|
+
get_redoc_html,
|
|
22
|
+
get_swagger_ui_html,
|
|
23
|
+
get_swagger_ui_oauth2_redirect_html,
|
|
24
|
+
)
|
|
25
|
+
from fastapi.responses import JSONResponse
|
|
26
|
+
|
|
27
|
+
from keepup.auth.dependencies import get_current_admin
|
|
28
|
+
|
|
29
|
+
#: What an application may import from this module. Everything else is
|
|
30
|
+
#: internal and may change without notice -- see doc/keepup.md.
|
|
31
|
+
__all__ = ["OAUTH2_REDIRECT_URL", "register_api_documentation"]
|
|
32
|
+
|
|
33
|
+
#: Where Swagger UI returns from an OAuth2 sign-in; FastAPI's own default.
|
|
34
|
+
OAUTH2_REDIRECT_URL = "/docs/oauth2-redirect"
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def register_api_documentation(app, settings) -> None:
|
|
38
|
+
"""The schema route and the documentation pages, as the settings ask.
|
|
39
|
+
|
|
40
|
+
Called right after the application is created: FastAPI used to register
|
|
41
|
+
these in its constructor, ahead of any route of the application, and a page
|
|
42
|
+
registered later could be shadowed by a route that happens to match it.
|
|
43
|
+
"""
|
|
44
|
+
schema_url = settings.openapi_url
|
|
45
|
+
if not schema_url:
|
|
46
|
+
# No schema, no pages: they would have nothing to show.
|
|
47
|
+
return
|
|
48
|
+
|
|
49
|
+
if settings.openapi_public:
|
|
50
|
+
async def api_schema():
|
|
51
|
+
return JSONResponse(app.openapi())
|
|
52
|
+
else:
|
|
53
|
+
async def api_schema(admin: dict = Depends(get_current_admin)):
|
|
54
|
+
return JSONResponse(app.openapi())
|
|
55
|
+
|
|
56
|
+
app.add_api_route(schema_url, api_schema, methods=["GET"], include_in_schema=False)
|
|
57
|
+
|
|
58
|
+
if settings.docs_url:
|
|
59
|
+
async def swagger_page():
|
|
60
|
+
return get_swagger_ui_html(openapi_url=schema_url, title=f"{app.title} - API",
|
|
61
|
+
oauth2_redirect_url=OAUTH2_REDIRECT_URL)
|
|
62
|
+
|
|
63
|
+
async def swagger_redirect():
|
|
64
|
+
return get_swagger_ui_oauth2_redirect_html()
|
|
65
|
+
|
|
66
|
+
app.add_api_route(settings.docs_url, swagger_page, methods=["GET"],
|
|
67
|
+
include_in_schema=False)
|
|
68
|
+
app.add_api_route(OAUTH2_REDIRECT_URL, swagger_redirect, methods=["GET"],
|
|
69
|
+
include_in_schema=False)
|
|
70
|
+
|
|
71
|
+
if settings.redoc_url:
|
|
72
|
+
async def redoc_page():
|
|
73
|
+
return get_redoc_html(openapi_url=schema_url, title=f"{app.title} - API")
|
|
74
|
+
|
|
75
|
+
app.add_api_route(settings.redoc_url, redoc_page, methods=["GET"],
|
|
76
|
+
include_in_schema=False)
|
|
@@ -131,6 +131,17 @@ incoming_requests_lock = asyncio.Lock()
|
|
|
131
131
|
|
|
132
132
|
BUFFER_FLUSH_INTERVAL = 125
|
|
133
133
|
BUFFER_MAX_SIZE = 100
|
|
134
|
+
#: The most requests the buffer holds, whatever happens to the database
|
|
135
|
+
#: (keepup-68). A failed flush puts its rows back, so with the database away the
|
|
136
|
+
#: buffer used to grow with every request; past this the oldest go.
|
|
137
|
+
BUFFER_HARD_LIMIT = 10_000
|
|
138
|
+
#: After a failed flush, requests do not start another one for this long: the
|
|
139
|
+
#: timed flusher still retries, but not every request on its own.
|
|
140
|
+
FLUSH_RETRY_PAUSE_SECONDS = 5.0
|
|
141
|
+
|
|
142
|
+
_flush_lock = asyncio.Lock()
|
|
143
|
+
_flush_paused_until = 0.0
|
|
144
|
+
dropped_requests = 0
|
|
134
145
|
#: A request running this long is written without its outcome (keepup-41): the
|
|
135
146
|
#: process that would have finished it has most likely gone.
|
|
136
147
|
IN_FLIGHT_STALE_SECONDS = 3600
|
|
@@ -170,7 +181,13 @@ class IncomingRequestLogger:
|
|
|
170
181
|
'request_start_at': request_start,
|
|
171
182
|
'created_at': datetime.utcnow()
|
|
172
183
|
}
|
|
173
|
-
|
|
184
|
+
global dropped_requests
|
|
185
|
+
while len(incoming_requests_buffer) > BUFFER_HARD_LIMIT:
|
|
186
|
+
# Insertion order: the first key is the oldest request.
|
|
187
|
+
incoming_requests_buffer.pop(next(iter(incoming_requests_buffer)))
|
|
188
|
+
dropped_requests += 1
|
|
189
|
+
if (len(incoming_requests_buffer) >= BUFFER_MAX_SIZE and not _flush_lock.locked()
|
|
190
|
+
and time.monotonic() >= _flush_paused_until):
|
|
174
191
|
asyncio.create_task(IncomingRequestLogger.flush_buffer())
|
|
175
192
|
|
|
176
193
|
return request_id
|
|
@@ -231,6 +248,13 @@ class IncomingRequestLogger:
|
|
|
231
248
|
not grow forever; and at shutdown (``include_in_flight``) everything is
|
|
232
249
|
written -- nothing will finish afterwards.
|
|
233
250
|
"""
|
|
251
|
+
# One flush at a time: a full buffer asked for one on every request.
|
|
252
|
+
async with _flush_lock:
|
|
253
|
+
return await IncomingRequestLogger._flush_buffer(include_in_flight, now)
|
|
254
|
+
|
|
255
|
+
@staticmethod
|
|
256
|
+
async def _flush_buffer(include_in_flight: bool, now: Optional[datetime]) -> int:
|
|
257
|
+
global _flush_paused_until, dropped_requests
|
|
234
258
|
now = now or datetime.utcnow()
|
|
235
259
|
# The buffer lock is taken by the start and the end of every request, so
|
|
236
260
|
# it is held only to pick the rows and take them out -- never across the
|
|
@@ -279,6 +303,7 @@ class IncomingRequestLogger:
|
|
|
279
303
|
# Back into the buffer: the next flush tries again. A request that
|
|
280
304
|
# somehow reappeared meanwhile keeps its newer entry.
|
|
281
305
|
logger.error(f"Error flushing incoming requests buffer: {str(e)}")
|
|
306
|
+
_flush_paused_until = time.monotonic() + FLUSH_RETRY_PAUSE_SECONDS
|
|
282
307
|
async with incoming_requests_lock:
|
|
283
308
|
for request_id, req in chosen.items():
|
|
284
309
|
incoming_requests_buffer.setdefault(request_id, req)
|
|
@@ -288,6 +313,10 @@ class IncomingRequestLogger:
|
|
|
288
313
|
# duplicate alike: an ignored one is already in the table.
|
|
289
314
|
if inserted_count:
|
|
290
315
|
logger.info(f"Flushed {inserted_count} incoming requests to database")
|
|
316
|
+
if dropped_requests:
|
|
317
|
+
logger.warning(f"{dropped_requests} incoming requests were dropped from the "
|
|
318
|
+
f"audit while it could not be written")
|
|
319
|
+
dropped_requests = 0
|
|
291
320
|
return inserted_count
|
|
292
321
|
|
|
293
322
|
|
|
@@ -408,6 +437,12 @@ async def background_buffer_flusher():
|
|
|
408
437
|
await asyncio.sleep(10)
|
|
409
438
|
|
|
410
439
|
|
|
440
|
+
def _is_refusal(error: BaseException) -> bool:
|
|
441
|
+
"""An HTTP refusal a route raised on purpose, whose text is its answer."""
|
|
442
|
+
from starlette.exceptions import HTTPException
|
|
443
|
+
return isinstance(error, HTTPException)
|
|
444
|
+
|
|
445
|
+
|
|
411
446
|
@asynccontextmanager
|
|
412
447
|
async def log_api_request(
|
|
413
448
|
method: str,
|
|
@@ -434,10 +469,12 @@ async def log_api_request(
|
|
|
434
469
|
|
|
435
470
|
except Exception as e:
|
|
436
471
|
if request_id:
|
|
472
|
+
# A refusal keeps its reason: it is the answer the client was given.
|
|
473
|
+
# Anything else keeps its kind, not its text (keepup-76).
|
|
437
474
|
await IncomingRequestLogger.end_request(
|
|
438
475
|
request_id=request_id,
|
|
439
476
|
duration_ms=int((time.time() - start_time) * 1000),
|
|
440
|
-
error_message=str(e)
|
|
477
|
+
error_message=str(e) if _is_refusal(e) else type(e).__name__
|
|
441
478
|
)
|
|
442
479
|
raise
|
|
443
480
|
|
|
@@ -21,11 +21,17 @@ import jwt
|
|
|
21
21
|
from jwt import PyJWTError as JWTError
|
|
22
22
|
from starlette import status
|
|
23
23
|
|
|
24
|
+
from keepup.auth.usernames import is_valid_username
|
|
24
25
|
from keepup.auth import panel_session
|
|
25
26
|
from keepup.auth.dto.token import TokenData
|
|
27
|
+
from keepup.auth.external_accounts import AccountUnavailable
|
|
26
28
|
from keepup.auth.factory import AuthProviderFactory
|
|
29
|
+
from keepup.auth.identity import access
|
|
30
|
+
from keepup.auth.identity import runtime as identity_runtime
|
|
31
|
+
from keepup.auth.identity.contract import IdentityRejected, ProviderUnavailable
|
|
27
32
|
from keepup.auth.providers.base import AuthProvider, ALGORITHM, oauth2_scheme
|
|
28
33
|
from keepup.auth.signing_key import resolve_signing_key
|
|
34
|
+
from keepup.auth import user_roles
|
|
29
35
|
from keepup.roles import ROLE_ADMIN, ROLE_CLIENT
|
|
30
36
|
from keepup.db import DatabaseManagerV2
|
|
31
37
|
|
|
@@ -63,14 +69,20 @@ def get_user_by_username(username: str):
|
|
|
63
69
|
loop of its own in a thread that the caller waited for, holding the loop.
|
|
64
70
|
The provider answers synchronously itself now.
|
|
65
71
|
"""
|
|
66
|
-
return auth_provider.lookup_user(username)
|
|
72
|
+
return user_roles.attach_roles(auth_provider.lookup_user(username))
|
|
67
73
|
|
|
68
74
|
|
|
69
75
|
def get_user_by_id(user_id: int):
|
|
70
|
-
"""Return a user by id,
|
|
76
|
+
"""Return a user by id, with the roles it holds.
|
|
77
|
+
|
|
78
|
+
The set travels with the account wherever the framework hands one out: code
|
|
79
|
+
that asks whether this person holds a role must not have to know which of
|
|
80
|
+
the two reads it came from.
|
|
81
|
+
"""
|
|
71
82
|
# TODO: auth_provider needs a lookup by id as well; without it this only
|
|
72
83
|
# works against the local database.
|
|
73
|
-
return
|
|
84
|
+
return user_roles.attach_roles(
|
|
85
|
+
DatabaseManagerV2.execute_one("SELECT * FROM users WHERE id = :id", {"id": user_id}))
|
|
74
86
|
|
|
75
87
|
|
|
76
88
|
def get_system_user_id():
|
|
@@ -97,14 +109,28 @@ def get_system_user_id():
|
|
|
97
109
|
|
|
98
110
|
|
|
99
111
|
def get_all_users():
|
|
100
|
-
"""Return every user,
|
|
101
|
-
|
|
112
|
+
"""Return every user, with the roles each of them holds."""
|
|
113
|
+
users = DatabaseManagerV2.execute(
|
|
102
114
|
"SELECT id, username, status, role, created_at FROM users ORDER BY created_at DESC"
|
|
103
|
-
)
|
|
115
|
+
) or []
|
|
116
|
+
held = {}
|
|
117
|
+
for row in DatabaseManagerV2.execute(
|
|
118
|
+
"SELECT user_id, role_name FROM user_roles") or []:
|
|
119
|
+
held.setdefault(row["user_id"], []).append(row["role_name"])
|
|
120
|
+
# One read for the whole list rather than one per user: this is the panel's
|
|
121
|
+
# first screen, and a query per row is what makes it slow on a real stand.
|
|
122
|
+
# An account whose set has never been filled in reads as its mirror says,
|
|
123
|
+
# which is known from the row already read.
|
|
124
|
+
return [dict(user, roles=(sorted(held.get(user["id"], []))
|
|
125
|
+
or user_roles.held_by(user) or [ROLE_CLIENT]))
|
|
126
|
+
for user in users]
|
|
104
127
|
|
|
105
128
|
|
|
106
129
|
def save_user_to_db(username: str, password: str):
|
|
107
130
|
"""Create a user, for any configured auth provider."""
|
|
131
|
+
if not is_valid_username(username):
|
|
132
|
+
raise HTTPException(status_code=400, detail="Username may contain letters, digits "
|
|
133
|
+
"and . _ @ + -, at most 64 characters")
|
|
108
134
|
password_hash = bcrypt.hashpw(password.encode('utf-8'), bcrypt.gensalt()).decode('utf-8')
|
|
109
135
|
|
|
110
136
|
query = """
|
|
@@ -117,7 +143,13 @@ def save_user_to_db(username: str, password: str):
|
|
|
117
143
|
query,
|
|
118
144
|
{"username": username, "password_hash": password_hash, "status": "blocked",
|
|
119
145
|
"role": ROLE_CLIENT})
|
|
120
|
-
|
|
146
|
+
user_id = row["id"] if row else None
|
|
147
|
+
if user_id is not None:
|
|
148
|
+
# The set is the truth about who this is, so a new account gets its
|
|
149
|
+
# row now rather than relying on the read's fallback until the next
|
|
150
|
+
# start (keepup/auth/user_roles.py).
|
|
151
|
+
user_roles.set_roles(user_id, [ROLE_CLIENT])
|
|
152
|
+
return user_id
|
|
121
153
|
|
|
122
154
|
except Exception as e:
|
|
123
155
|
if "unique constraint" in str(e).lower() or "duplicate" in str(e).lower():
|
|
@@ -135,10 +167,11 @@ async def create_user(
|
|
|
135
167
|
):
|
|
136
168
|
user_id = auth_provider.create_user_in_db(username, password, email, phone, full_name, agree_terms)
|
|
137
169
|
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
170
|
+
# The account is created once, above. This used to hand the provider the
|
|
171
|
+
# same account again -- with the name and password as one-element tuples --
|
|
172
|
+
# and the provider's create_user calls back into this function: a second
|
|
173
|
+
# insert that failed on every creation and was logged and swallowed
|
|
174
|
+
# (keepup-79).
|
|
142
175
|
|
|
143
176
|
# The id is returned to the caller: it was computed and then thrown away,
|
|
144
177
|
# while callers assigned the result to a variable and got None. The nearest
|
|
@@ -162,6 +195,14 @@ def update_user(user_id: int, status: Optional[str] = None, role: Optional[str]
|
|
|
162
195
|
f"UPDATE users SET {assignments}, updated_at = CURRENT_TIMESTAMP WHERE id = :id",
|
|
163
196
|
{**changes, "id": user_id})
|
|
164
197
|
|
|
198
|
+
# A caller setting the single role means "this person is a client now", so
|
|
199
|
+
# the whole set becomes that role. Leaving ADMIN in it would quietly keep a
|
|
200
|
+
# right the administrator had just taken away (keepup-51). Unchecked, because
|
|
201
|
+
# this column never checked: turning a call that worked into a refusal is not
|
|
202
|
+
# what a deprecated field should do on its way out.
|
|
203
|
+
if role is not None:
|
|
204
|
+
user_roles.set_roles(user_id, [role], checked=False)
|
|
205
|
+
|
|
165
206
|
|
|
166
207
|
async def authenticate(username: str, password: str):
|
|
167
208
|
return await auth_provider.authenticate(username=username, password=password)
|
|
@@ -211,6 +252,15 @@ def issue_session_token(user_id: int, username: str, sid: Optional[str] = None,
|
|
|
211
252
|
if sid is None:
|
|
212
253
|
sid = panel_session.open_session(user_id, lifetime)
|
|
213
254
|
else:
|
|
255
|
+
# Every renewal is held to the window, not only /api/auth/refresh: the
|
|
256
|
+
# exchange of a token for the cookie carried a session on without the
|
|
257
|
+
# check, so a stolen token could be renewed for ever (keepup-64).
|
|
258
|
+
from keepup.auth import session_lifetime
|
|
259
|
+
if not session_lifetime.is_renewable(session_started_at):
|
|
260
|
+
raise HTTPException(
|
|
261
|
+
status_code=status.HTTP_401_UNAUTHORIZED,
|
|
262
|
+
detail="This session has run its course. Sign in again.",
|
|
263
|
+
)
|
|
214
264
|
panel_session.extend(sid, lifetime)
|
|
215
265
|
token = create_access_token({"sub": username, panel_session.SESSION_CLAIM: sid},
|
|
216
266
|
expires_delta=lifetime, session_started_at=session_started_at)
|
|
@@ -234,29 +284,49 @@ async def request_token(request: Request, bearer: Optional[str] = Depends(oauth2
|
|
|
234
284
|
return panel_session.token_from_request(request, bearer)
|
|
235
285
|
|
|
236
286
|
|
|
237
|
-
def
|
|
238
|
-
"""
|
|
239
|
-
|
|
240
|
-
A token without one was issued before sessions were recorded and is accepted
|
|
241
|
-
until it expires -- at most a day after the deployment.
|
|
242
|
-
"""
|
|
243
|
-
sid = payload.get(panel_session.SESSION_CLAIM)
|
|
244
|
-
if sid is None:
|
|
245
|
-
return True
|
|
287
|
+
def _session_owner(sid: str) -> Optional[int]:
|
|
288
|
+
"""The account the live session ``sid`` belongs to, or None."""
|
|
246
289
|
try:
|
|
247
|
-
return panel_session.
|
|
290
|
+
return panel_session.session_owner(sid)
|
|
248
291
|
except Exception as error:
|
|
249
292
|
# Failing open here would make revocation depend on the database being
|
|
250
293
|
# up; failing closed only costs a retry.
|
|
251
294
|
logger.error(f"Could not check session {str(sid)[:8]}...: {error}")
|
|
252
|
-
return
|
|
295
|
+
return None
|
|
253
296
|
|
|
254
297
|
|
|
255
298
|
def _session_and_user(payload: dict, username: str):
|
|
256
|
-
"""Whether the token's session is live and, only if it is, whose account it is.
|
|
257
|
-
|
|
299
|
+
"""Whether the token's session is live and, only if it is, whose account it is.
|
|
300
|
+
|
|
301
|
+
A token is good only while its session is live and belongs to the account
|
|
302
|
+
the token names: otherwise it was not issued by this server, or names an
|
|
303
|
+
account recreated under the same name (keepup-64). A token without a
|
|
304
|
+
session is refused: tokens from before sessions were recorded were accepted
|
|
305
|
+
until they expired, and that transition is over (keepup-81).
|
|
306
|
+
|
|
307
|
+
The role set is read here too, in the same hop out of the event loop: one
|
|
308
|
+
more statement on the thread that read the account, not a second thread and
|
|
309
|
+
a second loop (keepup-43). It is not cached -- see keepup/auth/user_roles.py.
|
|
310
|
+
"""
|
|
311
|
+
sid = payload.get(panel_session.SESSION_CLAIM)
|
|
312
|
+
if not sid:
|
|
313
|
+
return False, None
|
|
314
|
+
# One connection and one commit for the three reads, not one each: this
|
|
315
|
+
# runs on every request, and it was half of what a replica spent (keepup-85).
|
|
316
|
+
try:
|
|
317
|
+
with DatabaseManagerV2.shared_session():
|
|
318
|
+
owner = _session_owner(sid)
|
|
319
|
+
if owner is None:
|
|
320
|
+
return False, None
|
|
321
|
+
user = auth_provider.lookup_user(username)
|
|
322
|
+
if user is not None and int(user["id"]) != owner:
|
|
323
|
+
return False, None
|
|
324
|
+
return True, user_roles.attach_roles(user)
|
|
325
|
+
except Exception as error:
|
|
326
|
+
# A failed read leaves the shared session unable to commit; either way
|
|
327
|
+
# the answer is the one a failed check always gave -- refuse.
|
|
328
|
+
logger.error(f"Could not check the session of {username}: {error}")
|
|
258
329
|
return False, None
|
|
259
|
-
return True, auth_provider.lookup_user(username)
|
|
260
330
|
|
|
261
331
|
|
|
262
332
|
async def get_current_user(token: str = Depends(request_token)):
|
|
@@ -300,17 +370,29 @@ async def _get_local_user(token: Optional[str], ignore_empty_user: bool = False)
|
|
|
300
370
|
if ignore_empty_user:
|
|
301
371
|
return None
|
|
302
372
|
raise credentials_exception
|
|
303
|
-
|
|
373
|
+
# Every token this server issues has an expiry, a subject and a session;
|
|
374
|
+
# one without them was not issued here, or was issued before sessions
|
|
375
|
+
# were recorded -- long enough ago to have expired (keepup-64, keepup-81).
|
|
376
|
+
payload = jwt.decode(token, resolve_signing_key(), algorithms=[ALGORITHM],
|
|
377
|
+
options={"require": ["exp", "sub", panel_session.SESSION_CLAIM]})
|
|
304
378
|
username: str = payload.get("sub")
|
|
305
379
|
if username is None:
|
|
306
380
|
if ignore_empty_user:
|
|
307
381
|
return None
|
|
308
382
|
raise credentials_exception
|
|
309
383
|
token_data = TokenData(username=username)
|
|
310
|
-
except
|
|
384
|
+
except _OUR_TOKEN_REFUSED:
|
|
385
|
+
# The signature is ours and the rest is not right: expired, not yet
|
|
386
|
+
# valid, or missing what every token of ours carries. It is refused
|
|
387
|
+
# here and never shown to the provider, which has no business seeing
|
|
388
|
+
# this framework's tokens.
|
|
311
389
|
if ignore_empty_user:
|
|
312
390
|
return None
|
|
313
391
|
raise credentials_exception
|
|
392
|
+
except JWTError:
|
|
393
|
+
# Not a token of this framework at all: another signature, another
|
|
394
|
+
# algorithm, not a JWT. The configured provider may know it.
|
|
395
|
+
return await _foreign_user(token, credentials_exception, ignore_empty_user)
|
|
314
396
|
|
|
315
397
|
# Both reads are blocking queries, so they leave the loop together: one hop
|
|
316
398
|
# to a worker thread per request, and the account is read once -- it used
|
|
@@ -338,6 +420,68 @@ async def _get_local_user(token: Optional[str], ignore_empty_user: bool = False)
|
|
|
338
420
|
return user
|
|
339
421
|
|
|
340
422
|
|
|
423
|
+
#: Refusals that mean the token is ours: the signature matched and something
|
|
424
|
+
#: else did not. Everything else PyJWT raises means somebody else signed it.
|
|
425
|
+
_OUR_TOKEN_REFUSED = (jwt.ExpiredSignatureError, jwt.ImmatureSignatureError,
|
|
426
|
+
jwt.MissingRequiredClaimError)
|
|
427
|
+
|
|
428
|
+
|
|
429
|
+
async def _foreign_user(token: str, credentials_exception: HTTPException,
|
|
430
|
+
ignore_empty_user: bool = False):
|
|
431
|
+
"""The user of a token some other system issued, through the configured provider.
|
|
432
|
+
|
|
433
|
+
Only when the deployment switched the provider's token mode on; otherwise a
|
|
434
|
+
token that is not ours is simply not a token. Every refusal answers the same
|
|
435
|
+
as a bad token of ours. An unavailable provider is a 503: a client told its
|
|
436
|
+
token is bad throws it away, and it must not do that because somebody else's
|
|
437
|
+
server is down.
|
|
438
|
+
"""
|
|
439
|
+
runtime = identity_runtime.current()
|
|
440
|
+
if runtime is None or not runtime.config.accept_tokens:
|
|
441
|
+
if ignore_empty_user:
|
|
442
|
+
return None
|
|
443
|
+
raise credentials_exception
|
|
444
|
+
try:
|
|
445
|
+
identity = await runtime.identity_for_token(token)
|
|
446
|
+
account = await runtime.user_for_identity(identity)
|
|
447
|
+
except IdentityRejected as refusal:
|
|
448
|
+
logger.info("Token refused by identity provider %s: %s", runtime.config.name, refusal)
|
|
449
|
+
if ignore_empty_user:
|
|
450
|
+
return None
|
|
451
|
+
raise credentials_exception
|
|
452
|
+
except (ProviderUnavailable, AccountUnavailable) as error:
|
|
453
|
+
if isinstance(error, AccountUnavailable):
|
|
454
|
+
logger.error("Account for identity provider %s unavailable: %s",
|
|
455
|
+
runtime.config.name, error)
|
|
456
|
+
if ignore_empty_user:
|
|
457
|
+
return None
|
|
458
|
+
raise HTTPException(status_code=status.HTTP_503_SERVICE_UNAVAILABLE,
|
|
459
|
+
detail=access.UNAVAILABLE)
|
|
460
|
+
|
|
461
|
+
if account.get("status") != "active":
|
|
462
|
+
# Vouched for there is not the same as allowed in here.
|
|
463
|
+
raise HTTPException(
|
|
464
|
+
status_code=status.HTTP_403_FORBIDDEN,
|
|
465
|
+
detail="User account is blocked",
|
|
466
|
+
)
|
|
467
|
+
return runtime.signed_in(account, identity)
|
|
468
|
+
|
|
469
|
+
|
|
470
|
+
def refuse_foreign_session(current_user: dict) -> None:
|
|
471
|
+
"""Refuse to turn a token of another system into a session of this one.
|
|
472
|
+
|
|
473
|
+
Both renewal and the move into the panel cookie issue a session here. For a
|
|
474
|
+
caller who came with the provider's token that would be a session the
|
|
475
|
+
other system cannot revoke, outliving the token it was traded for.
|
|
476
|
+
"""
|
|
477
|
+
source = (current_user or {}).get("authenticated_by")
|
|
478
|
+
if source:
|
|
479
|
+
raise HTTPException(
|
|
480
|
+
status_code=status.HTTP_400_BAD_REQUEST,
|
|
481
|
+
detail=f"This token was issued by {source}; renew it there.",
|
|
482
|
+
)
|
|
483
|
+
|
|
484
|
+
|
|
341
485
|
async def get_optional_user(token: Optional[str]):
|
|
342
486
|
"""The user of this token, or None when there is none.
|
|
343
487
|
|
|
@@ -356,7 +500,9 @@ async def get_optional_user(token: Optional[str]):
|
|
|
356
500
|
|
|
357
501
|
|
|
358
502
|
async def get_current_admin(current_user: dict = Depends(get_current_user)):
|
|
359
|
-
|
|
503
|
+
# The right is ADMIN being among the roles, not one role being equal to it:
|
|
504
|
+
# somebody who is both an administrator and a client holds two (keepup-51).
|
|
505
|
+
if not user_roles.has_role(current_user, ROLE_ADMIN):
|
|
360
506
|
raise HTTPException(
|
|
361
507
|
status_code=status.HTTP_403_FORBIDDEN,
|
|
362
508
|
detail="Admin access required",
|
|
@@ -384,7 +530,7 @@ def configure_panel_gate(pending=None):
|
|
|
384
530
|
|
|
385
531
|
def _authenticated_by_cookie(request: Request) -> bool:
|
|
386
532
|
return panel_session.real_bearer(_bearer_of(request)) is None \
|
|
387
|
-
and bool(request.cookies
|
|
533
|
+
and bool(panel_session.session_token_in(request.cookies))
|
|
388
534
|
|
|
389
535
|
|
|
390
536
|
def _bearer_of(request: Request) -> Optional[str]:
|