keepup-admin 0.1.1__tar.gz → 0.3.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (114) hide show
  1. {keepup_admin-0.1.1/keepup_admin.egg-info → keepup_admin-0.3.0}/PKG-INFO +22 -5
  2. {keepup_admin-0.1.1 → keepup_admin-0.3.0}/README.md +15 -1
  3. keepup_admin-0.3.0/admin_trail.py +60 -0
  4. keepup_admin-0.3.0/api_docs.py +76 -0
  5. {keepup_admin-0.1.1 → keepup_admin-0.3.0}/audit.py +129 -141
  6. {keepup_admin-0.1.1 → keepup_admin-0.3.0}/auth/dependencies.py +211 -140
  7. keepup_admin-0.3.0/auth/external_accounts.py +144 -0
  8. keepup_admin-0.3.0/auth/identity/__init__.py +33 -0
  9. keepup_admin-0.3.0/auth/identity/access.py +119 -0
  10. keepup_admin-0.3.0/auth/identity/accounts.py +86 -0
  11. keepup_admin-0.3.0/auth/identity/config.py +220 -0
  12. keepup_admin-0.3.0/auth/identity/contract.py +174 -0
  13. keepup_admin-0.3.0/auth/identity/loader.py +119 -0
  14. keepup_admin-0.3.0/auth/identity/runtime.py +265 -0
  15. {keepup_admin-0.1.1 → keepup_admin-0.3.0}/auth/login_throttle.py +52 -22
  16. {keepup_admin-0.1.1 → keepup_admin-0.3.0}/auth/oidc_policy.py +13 -2
  17. {keepup_admin-0.1.1 → keepup_admin-0.3.0}/auth/oidc_routes.py +60 -83
  18. {keepup_admin-0.1.1 → keepup_admin-0.3.0}/auth/panel_session.py +186 -20
  19. {keepup_admin-0.1.1 → keepup_admin-0.3.0}/auth/permissions.py +7 -7
  20. {keepup_admin-0.1.1 → keepup_admin-0.3.0}/auth/providers/base.py +57 -9
  21. {keepup_admin-0.1.1 → keepup_admin-0.3.0}/auth/providers/local.py +60 -24
  22. {keepup_admin-0.1.1 → keepup_admin-0.3.0}/auth/routes.py +113 -254
  23. {keepup_admin-0.1.1 → keepup_admin-0.3.0}/auth/seed_accounts.py +47 -17
  24. keepup_admin-0.3.0/auth/socket_sessions.py +176 -0
  25. keepup_admin-0.3.0/auth/user_roles.py +376 -0
  26. keepup_admin-0.3.0/auth/user_routes.py +313 -0
  27. keepup_admin-0.3.0/auth/usernames.py +36 -0
  28. {keepup_admin-0.1.1 → keepup_admin-0.3.0}/auth/websocket.py +3 -1
  29. keepup_admin-0.3.0/body_limit.py +156 -0
  30. keepup_admin-0.3.0/cache.py +169 -0
  31. {keepup_admin-0.1.1 → keepup_admin-0.3.0}/cluster.py +21 -12
  32. {keepup_admin-0.1.1 → keepup_admin-0.3.0}/db.py +236 -296
  33. keepup_admin-0.3.0/events.py +506 -0
  34. keepup_admin-0.3.0/events_api.py +236 -0
  35. {keepup_admin-0.1.1 → keepup_admin-0.3.0}/factory.py +166 -41
  36. {keepup_admin-0.1.1 → keepup_admin-0.3.0}/integrations.py +46 -82
  37. {keepup_admin-0.1.1 → keepup_admin-0.3.0/keepup_admin.egg-info}/PKG-INFO +22 -5
  38. {keepup_admin-0.1.1 → keepup_admin-0.3.0}/keepup_admin.egg-info/SOURCES.txt +46 -0
  39. {keepup_admin-0.1.1 → keepup_admin-0.3.0}/keepup_admin.egg-info/requires.txt +6 -3
  40. {keepup_admin-0.1.1 → keepup_admin-0.3.0}/locks.py +92 -68
  41. keepup_admin-0.1.1/logging_setup.py → keepup_admin-0.3.0/log_shipping.py +117 -112
  42. keepup_admin-0.3.0/logging_setup.py +144 -0
  43. {keepup_admin-0.1.1 → keepup_admin-0.3.0}/metrics.py +31 -261
  44. keepup_admin-0.3.0/metrics_api.py +277 -0
  45. {keepup_admin-0.1.1 → keepup_admin-0.3.0}/metrics_retention.py +2 -1
  46. {keepup_admin-0.1.1 → keepup_admin-0.3.0}/modules.py +240 -123
  47. {keepup_admin-0.1.1 → keepup_admin-0.3.0}/notification_bus.py +96 -10
  48. {keepup_admin-0.1.1 → keepup_admin-0.3.0}/plugins/admin.py +39 -14
  49. {keepup_admin-0.1.1 → keepup_admin-0.3.0}/plugins/base.py +13 -1
  50. {keepup_admin-0.1.1 → keepup_admin-0.3.0}/plugins/registry.py +20 -1
  51. {keepup_admin-0.1.1 → keepup_admin-0.3.0}/plugins/route_mask.py +8 -1
  52. {keepup_admin-0.1.1 → keepup_admin-0.3.0}/plugins/routes.py +126 -8
  53. keepup_admin-0.3.0/positional_sql.py +120 -0
  54. {keepup_admin-0.1.1 → keepup_admin-0.3.0}/pyproject.toml +22 -4
  55. keepup_admin-0.3.0/retention.py +84 -0
  56. {keepup_admin-0.1.1 → keepup_admin-0.3.0}/scheduler.py +38 -7
  57. {keepup_admin-0.1.1 → keepup_admin-0.3.0}/schema.py +47 -22
  58. keepup_admin-0.3.0/security.py +54 -0
  59. {keepup_admin-0.1.1 → keepup_admin-0.3.0}/settings.py +81 -3
  60. {keepup_admin-0.1.1 → keepup_admin-0.3.0}/static/css/main_new.css +17 -0
  61. {keepup_admin-0.1.1 → keepup_admin-0.3.0}/static/index_nebula.html +11 -22
  62. {keepup_admin-0.1.1 → keepup_admin-0.3.0}/static/index_new.html +11 -22
  63. keepup_admin-0.3.0/static/js/layout_bootstrap.js +29 -0
  64. {keepup_admin-0.1.1 → keepup_admin-0.3.0}/static/js/main_new.js +234 -19
  65. {keepup_admin-0.1.1 → keepup_admin-0.3.0}/static/modules/js/background_tasks.js +38 -16
  66. {keepup_admin-0.1.1 → keepup_admin-0.3.0}/static/modules/js/cluster.js +14 -3
  67. {keepup_admin-0.1.1 → keepup_admin-0.3.0}/static/modules/js/event_manager.js +44 -16
  68. {keepup_admin-0.1.1 → keepup_admin-0.3.0}/static/modules/js/integration_logs.js +54 -26
  69. {keepup_admin-0.1.1 → keepup_admin-0.3.0}/static/modules/js/metrics.js +34 -13
  70. {keepup_admin-0.1.1 → keepup_admin-0.3.0}/static/modules/js/modules.js +68 -26
  71. {keepup_admin-0.1.1 → keepup_admin-0.3.0}/static/modules/js/themes.js +45 -14
  72. {keepup_admin-0.1.1 → keepup_admin-0.3.0}/static/modules/js/users.js +188 -74
  73. {keepup_admin-0.1.1 → keepup_admin-0.3.0}/tables.py +2 -0
  74. {keepup_admin-0.1.1 → keepup_admin-0.3.0}/themes.py +74 -23
  75. {keepup_admin-0.1.1 → keepup_admin-0.3.0}/web.py +21 -8
  76. keepup_admin-0.1.1/events.py +0 -764
  77. {keepup_admin-0.1.1 → keepup_admin-0.3.0}/LICENSE +0 -0
  78. {keepup_admin-0.1.1 → keepup_admin-0.3.0}/NOTICE +0 -0
  79. {keepup_admin-0.1.1 → keepup_admin-0.3.0}/THIRD-PARTY.md +0 -0
  80. {keepup_admin-0.1.1 → keepup_admin-0.3.0}/__init__.py +0 -0
  81. {keepup_admin-0.1.1 → keepup_admin-0.3.0}/api_versions.py +0 -0
  82. {keepup_admin-0.1.1 → keepup_admin-0.3.0}/auth/__init__.py +0 -0
  83. {keepup_admin-0.1.1 → keepup_admin-0.3.0}/auth/config.py +0 -0
  84. {keepup_admin-0.1.1 → keepup_admin-0.3.0}/auth/dto/__init__.py +0 -0
  85. {keepup_admin-0.1.1 → keepup_admin-0.3.0}/auth/dto/token.py +0 -0
  86. {keepup_admin-0.1.1 → keepup_admin-0.3.0}/auth/factory.py +0 -0
  87. {keepup_admin-0.1.1 → keepup_admin-0.3.0}/auth/oidc.py +0 -0
  88. {keepup_admin-0.1.1 → keepup_admin-0.3.0}/auth/providers/__init__.py +0 -0
  89. {keepup_admin-0.1.1 → keepup_admin-0.3.0}/auth/session_lifetime.py +0 -0
  90. {keepup_admin-0.1.1 → keepup_admin-0.3.0}/auth/signing_key.py +0 -0
  91. {keepup_admin-0.1.1 → keepup_admin-0.3.0}/instance.py +0 -0
  92. {keepup_admin-0.1.1 → keepup_admin-0.3.0}/keepup_admin.egg-info/dependency_links.txt +0 -0
  93. {keepup_admin-0.1.1 → keepup_admin-0.3.0}/keepup_admin.egg-info/top_level.txt +0 -0
  94. {keepup_admin-0.1.1 → keepup_admin-0.3.0}/plugins/__init__.py +0 -0
  95. {keepup_admin-0.1.1 → keepup_admin-0.3.0}/plugins/enablement.py +0 -0
  96. {keepup_admin-0.1.1 → keepup_admin-0.3.0}/roles.py +0 -0
  97. {keepup_admin-0.1.1 → keepup_admin-0.3.0}/sections.json +0 -0
  98. {keepup_admin-0.1.1 → keepup_admin-0.3.0}/setup.cfg +0 -0
  99. {keepup_admin-0.1.1 → keepup_admin-0.3.0}/static/css/aos.css +0 -0
  100. {keepup_admin-0.1.1 → keepup_admin-0.3.0}/static/css/main_nebula.css +0 -0
  101. {keepup_admin-0.1.1 → keepup_admin-0.3.0}/static/css/tailwind.css +0 -0
  102. {keepup_admin-0.1.1 → keepup_admin-0.3.0}/static/js/aos.js +0 -0
  103. {keepup_admin-0.1.1 → keepup_admin-0.3.0}/static/js/feather-icons.js +0 -0
  104. {keepup_admin-0.1.1 → keepup_admin-0.3.0}/static/js/tailwind.js +0 -0
  105. {keepup_admin-0.1.1 → keepup_admin-0.3.0}/static/modules/css/background_tasks.css +0 -0
  106. {keepup_admin-0.1.1 → keepup_admin-0.3.0}/static/modules/css/cluster.css +0 -0
  107. {keepup_admin-0.1.1 → keepup_admin-0.3.0}/static/modules/css/event_manager.css +0 -0
  108. {keepup_admin-0.1.1 → keepup_admin-0.3.0}/static/modules/css/integration_logs.css +0 -0
  109. {keepup_admin-0.1.1 → keepup_admin-0.3.0}/static/modules/css/metrics.css +0 -0
  110. {keepup_admin-0.1.1 → keepup_admin-0.3.0}/static/modules/css/modules.css +0 -0
  111. {keepup_admin-0.1.1 → keepup_admin-0.3.0}/static/modules/css/themes.css +0 -0
  112. {keepup_admin-0.1.1 → keepup_admin-0.3.0}/static/modules/css/users.css +0 -0
  113. {keepup_admin-0.1.1 → keepup_admin-0.3.0}/static/modules/js/chart.js +0 -0
  114. {keepup_admin-0.1.1 → keepup_admin-0.3.0}/static/modules/js/chartjs-adapter-date-fns.bundle.min.js +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: keepup-admin
3
- Version: 0.1.1
3
+ Version: 0.3.0
4
4
  Summary: An admin framework over FastAPI: plugins, panel, users, replicas
5
5
  Author-email: Fedor Sergeev <fksergeev@gmail.com>
6
6
  License-Expression: Apache-2.0
@@ -25,27 +25,44 @@ Requires-Dist: sqlalchemy<3,>=2.0
25
25
  Requires-Dist: psycopg2-binary>=2.9
26
26
  Requires-Dist: apscheduler<4,>=3.10
27
27
  Requires-Dist: bcrypt<5,>=4.0
28
- Requires-Dist: pyjwt[crypto]<3,>=2.10
29
- Requires-Dist: python-multipart>=0.0.20
28
+ Requires-Dist: pyjwt[crypto]<3,>=2.15
29
+ Requires-Dist: python-multipart>=0.0.31
30
30
  Requires-Dist: PyYAML<7,>=6.0
31
31
  Requires-Dist: psutil>=7.0
32
32
  Requires-Dist: prometheus-client>=0.24
33
- Requires-Dist: requests>=2.31
33
+ Requires-Dist: requests>=2.33
34
34
  Requires-Dist: httpx>=0.28
35
+ Requires-Dist: cryptography>=50.0
36
+ Requires-Dist: urllib3>=2.8
35
37
  Provides-Extra: test
36
38
  Requires-Dist: pytest>=8; extra == "test"
37
39
  Requires-Dist: pytest-asyncio>=0.24; extra == "test"
38
40
  Requires-Dist: pytest-cov>=5; extra == "test"
41
+ Requires-Dist: uvicorn>=0.30; extra == "test"
39
42
  Provides-Extra: postgres
40
43
  Requires-Dist: asyncpg>=0.31; extra == "postgres"
41
44
  Dynamic: license-file
42
45
 
43
46
  # keepup
44
47
 
48
+ [![tests](https://github.com/FedorSergeev/keepup-admin/actions/workflows/tests.yml/badge.svg?branch=main)](https://github.com/FedorSergeev/keepup-admin/actions/workflows/tests.yml)
49
+ [![security](https://github.com/FedorSergeev/keepup-admin/actions/workflows/security.yml/badge.svg?branch=main)](https://github.com/FedorSergeev/keepup-admin/actions/workflows/security.yml)
50
+ [![PyPI](https://img.shields.io/pypi/v/keepup-admin)](https://pypi.org/project/keepup-admin/)
51
+ [![licence](https://img.shields.io/pypi/l/keepup-admin)](LICENSE)
52
+
45
53
  [Project page](https://fedorsergeev.github.io/keepup-admin/) ·
46
54
  [Package](https://pypi.org/project/keepup-admin/) ·
47
55
  [Changes](CHANGELOG.md)
48
56
 
57
+ Four badges, not six checks. These answer what somebody deciding whether to
58
+ install asks: does it run on my Python, is there a known vulnerability in what
59
+ it pulls in, what will I get, and may I use it. `quality`, `build` and `codeql`
60
+ are housekeeping and live on the Actions page -- six badges in a row stop being
61
+ read at all.
62
+
63
+ A badge shows; it does not require. What keeps a red check from being merged is
64
+ a branch rule, not a picture -- see `doc/keepup.md`.
65
+
49
66
  An admin framework over FastAPI. You give it settings; it gives you an
50
67
  application with a panel, a plugin runtime, users, a database layer, an audit,
51
68
  metrics and coordination between replicas already wired.
@@ -73,7 +90,7 @@ so the import name is free.
73
90
  | **Panel** | the shell, visual themes, and the catalogue of sections with role grants |
74
91
  | **Users** | local accounts, roles, panel sessions, sign-in through an external provider |
75
92
  | **Database** | a SQLAlchemy layer with a pool, table declarations for two dialects |
76
- | **Replicas** | distributed locks, leader election, a message bus, a cluster registry |
93
+ | **Replicas** | distributed locks, a message bus, a cluster registry, performance settings in one place |
77
94
  | **Observability** | audit of incoming requests, an event log, metrics, log shipping |
78
95
 
79
96
  ## What it does not decide for you
@@ -1,9 +1,23 @@
1
1
  # keepup
2
2
 
3
+ [![tests](https://github.com/FedorSergeev/keepup-admin/actions/workflows/tests.yml/badge.svg?branch=main)](https://github.com/FedorSergeev/keepup-admin/actions/workflows/tests.yml)
4
+ [![security](https://github.com/FedorSergeev/keepup-admin/actions/workflows/security.yml/badge.svg?branch=main)](https://github.com/FedorSergeev/keepup-admin/actions/workflows/security.yml)
5
+ [![PyPI](https://img.shields.io/pypi/v/keepup-admin)](https://pypi.org/project/keepup-admin/)
6
+ [![licence](https://img.shields.io/pypi/l/keepup-admin)](LICENSE)
7
+
3
8
  [Project page](https://fedorsergeev.github.io/keepup-admin/) ·
4
9
  [Package](https://pypi.org/project/keepup-admin/) ·
5
10
  [Changes](CHANGELOG.md)
6
11
 
12
+ Four badges, not six checks. These answer what somebody deciding whether to
13
+ install asks: does it run on my Python, is there a known vulnerability in what
14
+ it pulls in, what will I get, and may I use it. `quality`, `build` and `codeql`
15
+ are housekeeping and live on the Actions page -- six badges in a row stop being
16
+ read at all.
17
+
18
+ A badge shows; it does not require. What keeps a red check from being merged is
19
+ a branch rule, not a picture -- see `doc/keepup.md`.
20
+
7
21
  An admin framework over FastAPI. You give it settings; it gives you an
8
22
  application with a panel, a plugin runtime, users, a database layer, an audit,
9
23
  metrics and coordination between replicas already wired.
@@ -31,7 +45,7 @@ so the import name is free.
31
45
  | **Panel** | the shell, visual themes, and the catalogue of sections with role grants |
32
46
  | **Users** | local accounts, roles, panel sessions, sign-in through an external provider |
33
47
  | **Database** | a SQLAlchemy layer with a pool, table declarations for two dialects |
34
- | **Replicas** | distributed locks, leader election, a message bus, a cluster registry |
48
+ | **Replicas** | distributed locks, a message bus, a cluster registry, performance settings in one place |
35
49
  | **Observability** | audit of incoming requests, an event log, metrics, log shipping |
36
50
 
37
51
  ## What it does not decide for you
@@ -0,0 +1,60 @@
1
+ """What an administrator changed, written into the event log (keepup-67).
2
+
3
+ The event log is where a deployment looks for who did what, and the framework's
4
+ own administrative surface left nothing there: purging the log, changing the
5
+ panel's sections, their grants and the themes, and switching a plugin's
6
+ decision were visible only in the application log -- which rotates, is shipped
7
+ elsewhere or nowhere, and is not what an operator reads. Each of those now
8
+ writes one event naming the administrator.
9
+
10
+ The types below are the framework's. Neither they nor the types an application
11
+ declares can be created by hand through `POST /api/events`: a journal the
12
+ watched party can fill with lookalikes is not evidence (keepup/events_api.py).
13
+
14
+ Writing the trail never fails the action it records: the action has already
15
+ happened, and answering an error for it would only invite a repeat. A trail
16
+ that could not be written is logged as an error.
17
+ """
18
+
19
+ import logging
20
+ from typing import Any, Dict, Optional
21
+
22
+ from keepup import events
23
+
24
+ logger = logging.getLogger(__name__)
25
+
26
+ #: What an application may import from this module. Everything else is
27
+ #: internal and may change without notice -- see doc/keepup.md.
28
+ __all__ = [
29
+ "ADMIN_EVENT_TYPES",
30
+ "EVENTS_PURGED",
31
+ "PLUGIN_DECISION_CHANGED",
32
+ "SECTIONS_CHANGED",
33
+ "THEMES_CHANGED",
34
+ "record",
35
+ ]
36
+
37
+ EVENTS_PURGED = "admin_events_purged"
38
+ SECTIONS_CHANGED = "admin_sections_changed"
39
+ THEMES_CHANGED = "admin_themes_changed"
40
+ PLUGIN_DECISION_CHANGED = "admin_plugin_decision_changed"
41
+
42
+ ADMIN_EVENT_TYPES = (EVENTS_PURGED, SECTIONS_CHANGED, THEMES_CHANGED, PLUGIN_DECISION_CHANGED)
43
+
44
+
45
+ def _author(admin: Optional[Dict[str, Any]]) -> Dict[str, Any]:
46
+ admin = admin or {}
47
+ return {"user_id": admin.get("id"), "username": admin.get("username")}
48
+
49
+
50
+ async def record(event_type: str, admin: Optional[Dict[str, Any]], text: str,
51
+ **details: Any) -> Optional[int]:
52
+ """Write one administrative event; the id, or None when it could not be written."""
53
+ data = {"by": _author(admin), **details}
54
+ try:
55
+ return await events.event_manager.create_event(
56
+ event_type=event_type, event_text=text, event_data=data)
57
+ except Exception as error:
58
+ logger.error(f"Could not record {event_type} by {data['by'].get('username')}: {error}")
59
+ return None
60
+
@@ -0,0 +1,76 @@
1
+ """The API schema and the documentation pages built from it (keepup-96).
2
+
3
+ FastAPI serves its OpenAPI schema at /openapi.json to anybody, and the schema
4
+ describes every route of the application -- the administrative ones with their
5
+ parameters and bodies. For somebody outside that is a map of where to knock.
6
+ FastAPI has no notion of who is asking, so the framework takes both routes over:
7
+
8
+ - the schema answers an administrator only, like every other administrative
9
+ route of the framework, unless the application opens it on purpose
10
+ (``KeepupSettings.openapi_public``) or removes it (``openapi_url=None``);
11
+ - the documentation pages stay public shells: they fetch the schema from the
12
+ browser, with the panel's cookie, so an administrator sees the API and anybody
13
+ else sees the refusal.
14
+
15
+ Building the schema in code -- ``app.openapi()``, which ci/generate_openapi.py
16
+ calls -- is untouched: it never went through the route.
17
+ """
18
+
19
+ from fastapi import Depends
20
+ from fastapi.openapi.docs import (
21
+ get_redoc_html,
22
+ get_swagger_ui_html,
23
+ get_swagger_ui_oauth2_redirect_html,
24
+ )
25
+ from fastapi.responses import JSONResponse
26
+
27
+ from keepup.auth.dependencies import get_current_admin
28
+
29
+ #: What an application may import from this module. Everything else is
30
+ #: internal and may change without notice -- see doc/keepup.md.
31
+ __all__ = ["OAUTH2_REDIRECT_URL", "register_api_documentation"]
32
+
33
+ #: Where Swagger UI returns from an OAuth2 sign-in; FastAPI's own default.
34
+ OAUTH2_REDIRECT_URL = "/docs/oauth2-redirect"
35
+
36
+
37
+ def register_api_documentation(app, settings) -> None:
38
+ """The schema route and the documentation pages, as the settings ask.
39
+
40
+ Called right after the application is created: FastAPI used to register
41
+ these in its constructor, ahead of any route of the application, and a page
42
+ registered later could be shadowed by a route that happens to match it.
43
+ """
44
+ schema_url = settings.openapi_url
45
+ if not schema_url:
46
+ # No schema, no pages: they would have nothing to show.
47
+ return
48
+
49
+ if settings.openapi_public:
50
+ async def api_schema():
51
+ return JSONResponse(app.openapi())
52
+ else:
53
+ async def api_schema(admin: dict = Depends(get_current_admin)):
54
+ return JSONResponse(app.openapi())
55
+
56
+ app.add_api_route(schema_url, api_schema, methods=["GET"], include_in_schema=False)
57
+
58
+ if settings.docs_url:
59
+ async def swagger_page():
60
+ return get_swagger_ui_html(openapi_url=schema_url, title=f"{app.title} - API",
61
+ oauth2_redirect_url=OAUTH2_REDIRECT_URL)
62
+
63
+ async def swagger_redirect():
64
+ return get_swagger_ui_oauth2_redirect_html()
65
+
66
+ app.add_api_route(settings.docs_url, swagger_page, methods=["GET"],
67
+ include_in_schema=False)
68
+ app.add_api_route(OAUTH2_REDIRECT_URL, swagger_redirect, methods=["GET"],
69
+ include_in_schema=False)
70
+
71
+ if settings.redoc_url:
72
+ async def redoc_page():
73
+ return get_redoc_html(openapi_url=schema_url, title=f"{app.title} - API")
74
+
75
+ app.add_api_route(settings.redoc_url, redoc_page, methods=["GET"],
76
+ include_in_schema=False)
@@ -17,7 +17,6 @@ recorded says so by name -- ``configure(redaction=keep_as_is)``.
17
17
  import asyncio
18
18
  import json
19
19
  import logging
20
- import os
21
20
  import time
22
21
  from contextlib import asynccontextmanager
23
22
  from datetime import datetime
@@ -26,8 +25,8 @@ from typing import Any, Dict, Optional
26
25
  from sqlalchemy import CheckConstraint, Column, DateTime, Index, Integer, Text, UniqueConstraint
27
26
  from sqlalchemy.dialects import postgresql
28
27
 
29
- from keepup import tables
30
- from keepup.db import DatabaseManager, DatabaseManagerV2, db_config
28
+ from keepup import retention, tables
29
+ from keepup.db import DatabaseManagerV2, db_config
31
30
  from keepup.instance import get_instance_id
32
31
 
33
32
  #: What an application may import from this module. Everything else is
@@ -132,6 +131,26 @@ incoming_requests_lock = asyncio.Lock()
132
131
 
133
132
  BUFFER_FLUSH_INTERVAL = 125
134
133
  BUFFER_MAX_SIZE = 100
134
+ #: The most requests the buffer holds, whatever happens to the database
135
+ #: (keepup-68). A failed flush puts its rows back, so with the database away the
136
+ #: buffer used to grow with every request; past this the oldest go.
137
+ BUFFER_HARD_LIMIT = 10_000
138
+ #: After a failed flush, requests do not start another one for this long: the
139
+ #: timed flusher still retries, but not every request on its own.
140
+ FLUSH_RETRY_PAUSE_SECONDS = 5.0
141
+
142
+ _flush_lock = asyncio.Lock()
143
+ _flush_paused_until = 0.0
144
+ dropped_requests = 0
145
+ #: A request running this long is written without its outcome (keepup-41): the
146
+ #: process that would have finished it has most likely gone.
147
+ IN_FLIGHT_STALE_SECONDS = 3600
148
+
149
+
150
+ def _as_datetime(value) -> datetime:
151
+ if isinstance(value, str):
152
+ return datetime.fromisoformat(value.replace('Z', ''))
153
+ return value
135
154
 
136
155
 
137
156
  class IncomingRequestLogger:
@@ -162,7 +181,13 @@ class IncomingRequestLogger:
162
181
  'request_start_at': request_start,
163
182
  'created_at': datetime.utcnow()
164
183
  }
165
- if len(incoming_requests_buffer) >= BUFFER_MAX_SIZE:
184
+ global dropped_requests
185
+ while len(incoming_requests_buffer) > BUFFER_HARD_LIMIT:
186
+ # Insertion order: the first key is the oldest request.
187
+ incoming_requests_buffer.pop(next(iter(incoming_requests_buffer)))
188
+ dropped_requests += 1
189
+ if (len(incoming_requests_buffer) >= BUFFER_MAX_SIZE and not _flush_lock.locked()
190
+ and time.monotonic() >= _flush_paused_until):
166
191
  asyncio.create_task(IncomingRequestLogger.flush_buffer())
167
192
 
168
193
  return request_id
@@ -211,103 +236,98 @@ class IncomingRequestLogger:
211
236
  return True
212
237
 
213
238
  @staticmethod
214
- async def flush_buffer() -> int:
215
- """Write the buffered requests to the database."""
239
+ async def flush_buffer(include_in_flight: bool = False, now: Optional[datetime] = None) -> int:
240
+ """Write the finished requests of the buffer to the database.
241
+
242
+ A request still running stays in the buffer (keepup-41): written by a
243
+ timed flush and dropped from the buffer, it could no longer receive its
244
+ outcome -- end_request found nothing, logged "not found", and the table
245
+ kept a request with no status. Two exceptions: a request running longer
246
+ than IN_FLIGHT_STALE_SECONDS is written without an outcome, since the
247
+ process that would finish it has most likely gone and the buffer must
248
+ not grow forever; and at shutdown (``include_in_flight``) everything is
249
+ written -- nothing will finish afterwards.
250
+ """
251
+ # One flush at a time: a full buffer asked for one on every request.
252
+ async with _flush_lock:
253
+ return await IncomingRequestLogger._flush_buffer(include_in_flight, now)
216
254
 
255
+ @staticmethod
256
+ async def _flush_buffer(include_in_flight: bool, now: Optional[datetime]) -> int:
257
+ global _flush_paused_until, dropped_requests
258
+ now = now or datetime.utcnow()
259
+ # The buffer lock is taken by the start and the end of every request, so
260
+ # it is held only to pick the rows and take them out -- never across the
261
+ # write, which runs in a worker thread and may take as long as the
262
+ # database does (keepup-44).
217
263
  async with incoming_requests_lock:
218
- if not incoming_requests_buffer:
219
- return 0
220
-
221
- requests_to_insert = list(incoming_requests_buffer.values())
222
- inserted_count = 0
223
-
224
- try:
225
- values = []
226
- for req in requests_to_insert:
227
- values.append((
228
- req['instance_id'],
229
- req['method'],
230
- req['endpoint'],
231
- req['host'],
232
- json.dumps(req['request_data']) if req['request_data'] else None,
233
- req['request_start_at'],
234
- req.get('request_end_at'),
235
- req.get('duration_ms'),
236
- req.get('http_status'),
237
- json.dumps(req.get('response_data')) if req.get('response_data') else None,
238
- req.get('error_message'),
239
- req['created_at']
240
- ))
241
-
242
- if values:
243
- if db_config.is_postgres():
244
- query = '''
245
- INSERT INTO incoming_requests
246
- (instance_id, method, endpoint, host, request_data, request_start_at,
247
- request_end_at, duration_ms, http_status, response_data, error_message, created_at)
248
- VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
249
- ON CONFLICT (instance_id, method, endpoint, request_start_at)
250
- DO NOTHING
251
- '''
252
- else:
253
- query = '''
254
- INSERT OR IGNORE INTO incoming_requests
255
- (instance_id, method, endpoint, host, request_data, request_start_at,
256
- request_end_at, duration_ms, http_status, response_data, error_message, created_at)
257
- VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
258
- '''
259
-
260
- if db_config.is_postgres():
261
- conn = DatabaseManager.get_connection()
262
- cursor = conn.cursor()
263
- try:
264
- cursor.executemany(query, values)
265
- inserted_count = cursor.rowcount
266
- conn.commit()
267
- finally:
268
- cursor.close()
269
- conn.close()
270
- else:
271
- conn = DatabaseManager.get_connection()
272
- cursor = conn.cursor()
273
- try:
274
- cursor.executemany(query, values)
275
- inserted_count = cursor.rowcount
276
- conn.commit()
277
- finally:
278
- cursor.close()
279
- conn.close()
280
-
281
- if inserted_count > 0:
282
- inserted_ids = []
283
- for req_id, req_data in incoming_requests_buffer.items():
284
- for v in values:
285
- if (req_data['instance_id'] == v[0] and
286
- req_data['method'] == v[1] and
287
- req_data['endpoint'] == v[2] and
288
- req_data['request_start_at'] == v[5]):
289
- inserted_ids.append(req_id)
290
- break
291
-
292
- for req_id in inserted_ids:
293
- incoming_requests_buffer.pop(req_id, None)
294
-
295
- logger.info(f"Flushed {inserted_count} incoming requests to database")
296
-
297
- except Exception as e:
298
- logger.error(f"Error flushing incoming requests buffer: {str(e)}")
299
-
300
- return inserted_count
264
+ chosen = {
265
+ request_id: req for request_id, req in incoming_requests_buffer.items()
266
+ if include_in_flight or 'request_end_at' in req
267
+ or (now - _as_datetime(req['request_start_at'])).total_seconds()
268
+ >= IN_FLIGHT_STALE_SECONDS
269
+ }
270
+ for request_id in chosen:
271
+ incoming_requests_buffer.pop(request_id, None)
272
+ if not chosen:
273
+ return 0
274
+
275
+ rows = [{
276
+ 'instance_id': req['instance_id'],
277
+ 'method': req['method'],
278
+ 'endpoint': req['endpoint'],
279
+ 'host': req['host'],
280
+ 'request_data': json.dumps(req['request_data']) if req['request_data'] else None,
281
+ 'request_start_at': req['request_start_at'],
282
+ 'request_end_at': req.get('request_end_at'),
283
+ 'duration_ms': req.get('duration_ms'),
284
+ 'http_status': req.get('http_status'),
285
+ 'response_data': (json.dumps(req.get('response_data'))
286
+ if req.get('response_data') else None),
287
+ 'error_message': req.get('error_message'),
288
+ 'created_at': req['created_at'],
289
+ } for req in chosen.values()]
290
+ columns = ("instance_id, method, endpoint, host, request_data, request_start_at, "
291
+ "request_end_at, duration_ms, http_status, response_data, error_message, "
292
+ "created_at")
293
+ placeholders = ", ".join(":" + c.strip() for c in columns.split(","))
294
+ if db_config.is_postgres():
295
+ query = (f"INSERT INTO incoming_requests ({columns}) VALUES ({placeholders}) "
296
+ f"ON CONFLICT (instance_id, method, endpoint, request_start_at) DO NOTHING")
297
+ else:
298
+ query = f"INSERT OR IGNORE INTO incoming_requests ({columns}) VALUES ({placeholders})"
299
+
300
+ try:
301
+ inserted_count = await DatabaseManagerV2.execute_many_async(query, rows)
302
+ except Exception as e:
303
+ # Back into the buffer: the next flush tries again. A request that
304
+ # somehow reappeared meanwhile keeps its newer entry.
305
+ logger.error(f"Error flushing incoming requests buffer: {str(e)}")
306
+ _flush_paused_until = time.monotonic() + FLUSH_RETRY_PAUSE_SECONDS
307
+ async with incoming_requests_lock:
308
+ for request_id, req in chosen.items():
309
+ incoming_requests_buffer.setdefault(request_id, req)
310
+ return 0
311
+
312
+ # Every chosen request is gone from the buffer, written or ignored as a
313
+ # duplicate alike: an ignored one is already in the table.
314
+ if inserted_count:
315
+ logger.info(f"Flushed {inserted_count} incoming requests to database")
316
+ if dropped_requests:
317
+ logger.warning(f"{dropped_requests} incoming requests were dropped from the "
318
+ f"audit while it could not be written")
319
+ dropped_requests = 0
320
+ return inserted_count
301
321
 
302
322
 
303
323
  #: What PostgreSQL enforces with named constraints, SQLite held as a unique
304
324
  #: index of the same name and no checks at all; each dialect keeps what it had.
305
325
  # --- how long a row lives ----------------------------------------------------
306
326
 
307
- #: How many days an audit row is kept. The table used to have no sweep at all,
308
- #: while metric snapshots and application events both had one: it grew for as
309
- #: long as the deployment ran, and every row in it was a second place a value
310
- #: from a query string lived.
327
+ #: How many days an audit row is kept. The table used to have no sweep at all:
328
+ #: it grew for as long as the deployment ran, and every row in it was a second
329
+ #: place a value from a query string lived. The event log is swept the same way
330
+ #: (keepup/retention.py).
311
331
  DEFAULT_AUDIT_RETENTION_DAYS = 30
312
332
  #: How often the sweep runs.
313
333
  AUDIT_SWEEP_INTERVAL_SECONDS = 3600
@@ -324,35 +344,12 @@ AUDIT_RETENTION_LOCK = "incoming_requests_retention"
324
344
 
325
345
 
326
346
  def retention_days():
327
- """How many days rows are kept, from the environment or the default.
328
-
329
- Returns:
330
- A positive number of days; rubbish in the variable means the default
331
- and a line in the log, because a sweep that switched itself off
332
- silently would be worse than one that swept too much.
333
- """
334
- raw = os.getenv("AUDIT_RETENTION_DAYS")
335
- if raw in (None, ""):
336
- return DEFAULT_AUDIT_RETENTION_DAYS
337
- try:
338
- value = int(raw)
339
- except (TypeError, ValueError):
340
- logger.warning("AUDIT_RETENTION_DAYS=%r is not a number, using %s",
341
- raw, DEFAULT_AUDIT_RETENTION_DAYS)
342
- return DEFAULT_AUDIT_RETENTION_DAYS
343
- if value <= 0:
344
- logger.warning("AUDIT_RETENTION_DAYS=%s must be positive, using %s",
345
- value, DEFAULT_AUDIT_RETENTION_DAYS)
346
- return DEFAULT_AUDIT_RETENTION_DAYS
347
- return value
347
+ """How many days audit rows are kept (``AUDIT_RETENTION_DAYS``, 30 by default)."""
348
+ return retention.retention_days("AUDIT_RETENTION_DAYS", DEFAULT_AUDIT_RETENTION_DAYS)
348
349
 
349
350
 
350
351
  def purge_old_requests(days=None, now=None):
351
- """Delete audit rows older than the retention period.
352
-
353
- The boundary is computed here rather than in the statement: the interval
354
- syntax differs between the engines, and a query written for one does not
355
- run on the other.
352
+ """Delete audit rows older than the retention period, in chunks.
356
353
 
357
354
  Args:
358
355
  days: how many days to keep; read from the environment when omitted.
@@ -361,27 +358,10 @@ def purge_old_requests(days=None, now=None):
361
358
  Returns:
362
359
  How many rows were deleted.
363
360
  """
364
- from datetime import timedelta
365
-
366
- keep_days = days if days is not None else retention_days()
367
- cutoff = (now or datetime.utcnow()) - timedelta(days=keep_days)
368
-
369
- removed = 0
370
- for _ in range(AUDIT_MAX_CHUNKS_PER_PASS):
371
- rows = DatabaseManagerV2.execute(
372
- "SELECT id FROM incoming_requests WHERE created_at < :cutoff "
373
- "ORDER BY created_at ASC LIMIT :limit",
374
- {"cutoff": cutoff, "limit": AUDIT_DELETE_CHUNK_ROWS})
375
- if not rows:
376
- break
377
- ids = {f"i{index}": row["id"] for index, row in enumerate(rows)}
378
- placeholders = ", ".join(f":{key}" for key in ids)
379
- DatabaseManagerV2.execute_commit(
380
- f"DELETE FROM incoming_requests WHERE id IN ({placeholders})", ids)
381
- removed += len(rows)
382
- if len(rows) < AUDIT_DELETE_CHUNK_ROWS:
383
- break
384
- return removed
361
+ return retention.purge_older_than(
362
+ "incoming_requests", "created_at",
363
+ days if days is not None else retention_days(),
364
+ AUDIT_DELETE_CHUNK_ROWS, AUDIT_MAX_CHUNKS_PER_PASS, now=now)
385
365
 
386
366
 
387
367
  async def audit_retention_background():
@@ -457,6 +437,12 @@ async def background_buffer_flusher():
457
437
  await asyncio.sleep(10)
458
438
 
459
439
 
440
+ def _is_refusal(error: BaseException) -> bool:
441
+ """An HTTP refusal a route raised on purpose, whose text is its answer."""
442
+ from starlette.exceptions import HTTPException
443
+ return isinstance(error, HTTPException)
444
+
445
+
460
446
  @asynccontextmanager
461
447
  async def log_api_request(
462
448
  method: str,
@@ -483,10 +469,12 @@ async def log_api_request(
483
469
 
484
470
  except Exception as e:
485
471
  if request_id:
472
+ # A refusal keeps its reason: it is the answer the client was given.
473
+ # Anything else keeps its kind, not its text (keepup-76).
486
474
  await IncomingRequestLogger.end_request(
487
475
  request_id=request_id,
488
476
  duration_ms=int((time.time() - start_time) * 1000),
489
- error_message=str(e)
477
+ error_message=str(e) if _is_refusal(e) else type(e).__name__
490
478
  )
491
479
  raise
492
480