keepup-admin 0.2.0__tar.gz → 0.4.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (124) hide show
  1. {keepup_admin-0.2.0/keepup_admin.egg-info → keepup_admin-0.4.0}/PKG-INFO +36 -11
  2. {keepup_admin-0.2.0 → keepup_admin-0.4.0}/README.md +5 -4
  3. {keepup_admin-0.2.0 → keepup_admin-0.4.0}/__init__.py +6 -0
  4. keepup_admin-0.4.0/admin_trail.py +60 -0
  5. keepup_admin-0.4.0/api_docs.py +76 -0
  6. {keepup_admin-0.2.0 → keepup_admin-0.4.0}/audit.py +43 -31
  7. {keepup_admin-0.2.0 → keepup_admin-0.4.0}/body_limit.py +8 -4
  8. keepup_admin-0.4.0/builtin/audit.py +107 -0
  9. keepup_admin-0.4.0/builtin/auth.py +100 -0
  10. keepup_admin-0.4.0/builtin/db.py +172 -0
  11. keepup_admin-0.4.0/builtin/integration_logs.py +297 -0
  12. keepup_admin-0.4.0/builtin/metrics.py +226 -0
  13. keepup_admin-0.4.0/builtin/postgres.py +70 -0
  14. keepup_admin-0.4.0/builtin/sqlite.py +68 -0
  15. keepup_admin-0.4.0/builtin/ui.py +146 -0
  16. keepup_admin-0.4.0/builtin/users.py +137 -0
  17. {keepup_admin-0.2.0 → keepup_admin-0.4.0}/cluster.py +13 -1
  18. keepup_admin-0.4.0/compat.py +270 -0
  19. {keepup_admin-0.2.0 → keepup_admin-0.4.0}/events_api.py +44 -17
  20. {keepup_admin-0.2.0 → keepup_admin-0.4.0}/factory.py +166 -36
  21. {keepup_admin-0.2.0 → keepup_admin-0.4.0}/integrations.py +4 -1
  22. {keepup_admin-0.2.0 → keepup_admin-0.4.0/keepup_admin.egg-info}/PKG-INFO +36 -11
  23. {keepup_admin-0.2.0 → keepup_admin-0.4.0}/keepup_admin.egg-info/SOURCES.txt +54 -46
  24. keepup_admin-0.4.0/keepup_admin.egg-info/requires.txt +55 -0
  25. keepup_admin-0.4.0/kernel/__init__.py +120 -0
  26. keepup_admin-0.4.0/kernel/call.py +304 -0
  27. keepup_admin-0.4.0/kernel/catalogue.py +243 -0
  28. keepup_admin-0.4.0/kernel/contributions.py +317 -0
  29. keepup_admin-0.4.0/kernel/datasource.py +95 -0
  30. keepup_admin-0.4.0/kernel/descriptor.py +209 -0
  31. keepup_admin-0.4.0/kernel/lifecycle.py +814 -0
  32. keepup_admin-0.4.0/kernel/loader.py +240 -0
  33. keepup_admin-0.4.0/kernel/observability.py +127 -0
  34. keepup_admin-0.4.0/kernel/security.py +113 -0
  35. keepup_admin-0.4.0/kernel/services.py +267 -0
  36. keepup_admin-0.4.0/kernel/transports.py +119 -0
  37. {keepup_admin-0.2.0 → keepup_admin-0.4.0}/locks.py +20 -6
  38. {keepup_admin-0.2.0 → keepup_admin-0.4.0}/log_shipping.py +96 -51
  39. {keepup_admin-0.2.0 → keepup_admin-0.4.0}/logging_setup.py +22 -0
  40. {keepup_admin-0.2.0 → keepup_admin-0.4.0}/metrics.py +1 -1
  41. {keepup_admin-0.2.0 → keepup_admin-0.4.0}/metrics_api.py +13 -14
  42. {keepup_admin-0.2.0 → keepup_admin-0.4.0}/metrics_retention.py +3 -1
  43. {keepup_admin-0.2.0 → keepup_admin-0.4.0}/modules.py +145 -28
  44. {keepup_admin-0.2.0 → keepup_admin-0.4.0}/notification_bus.py +66 -4
  45. {keepup_admin-0.2.0 → keepup_admin-0.4.0}/plugins/admin.py +37 -12
  46. keepup_admin-0.4.0/plugins/builtin.json +283 -0
  47. {keepup_admin-0.2.0 → keepup_admin-0.4.0}/plugins/registry.py +81 -44
  48. {keepup_admin-0.2.0 → keepup_admin-0.4.0}/plugins/route_mask.py +8 -1
  49. {keepup_admin-0.2.0 → keepup_admin-0.4.0}/plugins/routes.py +174 -112
  50. {keepup_admin-0.2.0 → keepup_admin-0.4.0}/pyproject.toml +91 -22
  51. {keepup_admin-0.2.0 → keepup_admin-0.4.0}/retention.py +4 -1
  52. {keepup_admin-0.2.0 → keepup_admin-0.4.0}/schema.py +34 -115
  53. keepup_admin-0.4.0/security.py +54 -0
  54. {keepup_admin-0.2.0 → keepup_admin-0.4.0}/settings.py +55 -6
  55. {keepup_admin-0.2.0 → keepup_admin-0.4.0}/static/index_nebula.html +11 -22
  56. {keepup_admin-0.2.0 → keepup_admin-0.4.0}/static/index_new.html +11 -22
  57. keepup_admin-0.4.0/static/js/layout_bootstrap.js +29 -0
  58. {keepup_admin-0.2.0 → keepup_admin-0.4.0}/static/js/main_new.js +187 -18
  59. {keepup_admin-0.2.0 → keepup_admin-0.4.0}/static/modules/js/background_tasks.js +38 -16
  60. {keepup_admin-0.2.0 → keepup_admin-0.4.0}/static/modules/js/cluster.js +14 -3
  61. {keepup_admin-0.2.0 → keepup_admin-0.4.0}/static/modules/js/event_manager.js +44 -16
  62. {keepup_admin-0.2.0 → keepup_admin-0.4.0}/static/modules/js/integration_logs.js +54 -26
  63. {keepup_admin-0.2.0 → keepup_admin-0.4.0}/static/modules/js/metrics.js +34 -13
  64. {keepup_admin-0.2.0 → keepup_admin-0.4.0}/static/modules/js/modules.js +68 -26
  65. {keepup_admin-0.2.0 → keepup_admin-0.4.0}/static/modules/js/themes.js +45 -14
  66. {keepup_admin-0.2.0 → keepup_admin-0.4.0}/static/modules/js/users.js +188 -74
  67. {keepup_admin-0.2.0 → keepup_admin-0.4.0}/themes.py +51 -28
  68. {keepup_admin-0.2.0 → keepup_admin-0.4.0}/web.py +24 -4
  69. keepup_admin-0.2.0/auth/__init__.py +0 -7
  70. keepup_admin-0.2.0/auth/config.py +0 -128
  71. keepup_admin-0.2.0/auth/dependencies.py +0 -410
  72. keepup_admin-0.2.0/auth/dto/__init__.py +0 -0
  73. keepup_admin-0.2.0/auth/dto/token.py +0 -31
  74. keepup_admin-0.2.0/auth/factory.py +0 -28
  75. keepup_admin-0.2.0/auth/login_throttle.py +0 -152
  76. keepup_admin-0.2.0/auth/oidc.py +0 -255
  77. keepup_admin-0.2.0/auth/oidc_policy.py +0 -111
  78. keepup_admin-0.2.0/auth/oidc_routes.py +0 -288
  79. keepup_admin-0.2.0/auth/panel_session.py +0 -220
  80. keepup_admin-0.2.0/auth/permissions.py +0 -59
  81. keepup_admin-0.2.0/auth/providers/__init__.py +0 -0
  82. keepup_admin-0.2.0/auth/providers/base.py +0 -202
  83. keepup_admin-0.2.0/auth/providers/local.py +0 -182
  84. keepup_admin-0.2.0/auth/routes.py +0 -624
  85. keepup_admin-0.2.0/auth/seed_accounts.py +0 -322
  86. keepup_admin-0.2.0/auth/session_lifetime.py +0 -104
  87. keepup_admin-0.2.0/auth/signing_key.py +0 -138
  88. keepup_admin-0.2.0/auth/websocket.py +0 -86
  89. keepup_admin-0.2.0/db.py +0 -461
  90. keepup_admin-0.2.0/events.py +0 -484
  91. keepup_admin-0.2.0/keepup_admin.egg-info/requires.txt +0 -22
  92. keepup_admin-0.2.0/tables.py +0 -302
  93. {keepup_admin-0.2.0 → keepup_admin-0.4.0}/LICENSE +0 -0
  94. {keepup_admin-0.2.0 → keepup_admin-0.4.0}/NOTICE +0 -0
  95. {keepup_admin-0.2.0 → keepup_admin-0.4.0}/THIRD-PARTY.md +0 -0
  96. {keepup_admin-0.2.0 → keepup_admin-0.4.0}/api_versions.py +0 -0
  97. {keepup_admin-0.2.0 → keepup_admin-0.4.0}/cache.py +0 -0
  98. {keepup_admin-0.2.0 → keepup_admin-0.4.0}/instance.py +0 -0
  99. {keepup_admin-0.2.0 → keepup_admin-0.4.0}/keepup_admin.egg-info/dependency_links.txt +0 -0
  100. {keepup_admin-0.2.0 → keepup_admin-0.4.0}/keepup_admin.egg-info/top_level.txt +0 -0
  101. {keepup_admin-0.2.0 → keepup_admin-0.4.0}/plugins/__init__.py +0 -0
  102. {keepup_admin-0.2.0 → keepup_admin-0.4.0}/plugins/base.py +0 -0
  103. {keepup_admin-0.2.0 → keepup_admin-0.4.0}/plugins/enablement.py +0 -0
  104. {keepup_admin-0.2.0 → keepup_admin-0.4.0}/roles.py +0 -0
  105. {keepup_admin-0.2.0 → keepup_admin-0.4.0}/scheduler.py +0 -0
  106. {keepup_admin-0.2.0 → keepup_admin-0.4.0}/sections.json +0 -0
  107. {keepup_admin-0.2.0 → keepup_admin-0.4.0}/setup.cfg +0 -0
  108. {keepup_admin-0.2.0 → keepup_admin-0.4.0}/static/css/aos.css +0 -0
  109. {keepup_admin-0.2.0 → keepup_admin-0.4.0}/static/css/main_nebula.css +0 -0
  110. {keepup_admin-0.2.0 → keepup_admin-0.4.0}/static/css/main_new.css +0 -0
  111. {keepup_admin-0.2.0 → keepup_admin-0.4.0}/static/css/tailwind.css +0 -0
  112. {keepup_admin-0.2.0 → keepup_admin-0.4.0}/static/js/aos.js +0 -0
  113. {keepup_admin-0.2.0 → keepup_admin-0.4.0}/static/js/feather-icons.js +0 -0
  114. {keepup_admin-0.2.0 → keepup_admin-0.4.0}/static/js/tailwind.js +0 -0
  115. {keepup_admin-0.2.0 → keepup_admin-0.4.0}/static/modules/css/background_tasks.css +0 -0
  116. {keepup_admin-0.2.0 → keepup_admin-0.4.0}/static/modules/css/cluster.css +0 -0
  117. {keepup_admin-0.2.0 → keepup_admin-0.4.0}/static/modules/css/event_manager.css +0 -0
  118. {keepup_admin-0.2.0 → keepup_admin-0.4.0}/static/modules/css/integration_logs.css +0 -0
  119. {keepup_admin-0.2.0 → keepup_admin-0.4.0}/static/modules/css/metrics.css +0 -0
  120. {keepup_admin-0.2.0 → keepup_admin-0.4.0}/static/modules/css/modules.css +0 -0
  121. {keepup_admin-0.2.0 → keepup_admin-0.4.0}/static/modules/css/themes.css +0 -0
  122. {keepup_admin-0.2.0 → keepup_admin-0.4.0}/static/modules/css/users.css +0 -0
  123. {keepup_admin-0.2.0 → keepup_admin-0.4.0}/static/modules/js/chart.js +0 -0
  124. {keepup_admin-0.2.0 → keepup_admin-0.4.0}/static/modules/js/chartjs-adapter-date-fns.bundle.min.js +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: keepup-admin
3
- Version: 0.2.0
3
+ Version: 0.4.0
4
4
  Summary: An admin framework over FastAPI: plugins, panel, users, replicas
5
5
  Author-email: Fedor Sergeev <fksergeev@gmail.com>
6
6
  License-Expression: Apache-2.0
@@ -22,22 +22,46 @@ Requires-Dist: fastapi<1,>=0.128
22
22
  Requires-Dist: starlette<2,>=1.3.1
23
23
  Requires-Dist: pydantic<3,>=2.11
24
24
  Requires-Dist: sqlalchemy<3,>=2.0
25
- Requires-Dist: psycopg2-binary>=2.9
26
25
  Requires-Dist: apscheduler<4,>=3.10
27
- Requires-Dist: bcrypt<5,>=4.0
28
- Requires-Dist: pyjwt[crypto]<3,>=2.10
29
- Requires-Dist: python-multipart>=0.0.20
26
+ Requires-Dist: python-multipart>=0.0.31
30
27
  Requires-Dist: PyYAML<7,>=6.0
31
28
  Requires-Dist: psutil>=7.0
32
29
  Requires-Dist: prometheus-client>=0.24
33
- Requires-Dist: requests>=2.31
30
+ Requires-Dist: requests>=2.33
34
31
  Requires-Dist: httpx>=0.28
32
+ Requires-Dist: cryptography>=50.0
33
+ Requires-Dist: urllib3>=2.8
35
34
  Provides-Extra: test
36
35
  Requires-Dist: pytest>=8; extra == "test"
37
36
  Requires-Dist: pytest-asyncio>=0.24; extra == "test"
38
37
  Requires-Dist: pytest-cov>=5; extra == "test"
38
+ Requires-Dist: uvicorn>=0.30; extra == "test"
39
+ Provides-Extra: db
40
+ Requires-Dist: keepup-db>=0.4.0; extra == "db"
41
+ Provides-Extra: auth
42
+ Requires-Dist: keepup-auth>=0.4.0; extra == "auth"
43
+ Provides-Extra: users
44
+ Requires-Dist: keepup-users>=0.4.0; extra == "users"
45
+ Provides-Extra: ui
46
+ Requires-Dist: keepup-ui>=0.4.0; extra == "ui"
47
+ Provides-Extra: audit
48
+ Requires-Dist: keepup-audit>=0.4.0; extra == "audit"
49
+ Provides-Extra: metrics
50
+ Requires-Dist: keepup-metrics>=0.4.0; extra == "metrics"
51
+ Provides-Extra: integration-log
52
+ Requires-Dist: keepup-integration-log>=0.4.0; extra == "integration-log"
53
+ Provides-Extra: panel
54
+ Requires-Dist: keepup-db>=0.4.0; extra == "panel"
55
+ Requires-Dist: keepup-auth>=0.4.0; extra == "panel"
56
+ Requires-Dist: keepup-users>=0.4.0; extra == "panel"
57
+ Requires-Dist: keepup-ui>=0.4.0; extra == "panel"
58
+ Requires-Dist: keepup-audit>=0.4.0; extra == "panel"
59
+ Requires-Dist: keepup-metrics>=0.4.0; extra == "panel"
60
+ Requires-Dist: keepup-integration-log>=0.4.0; extra == "panel"
39
61
  Provides-Extra: postgres
40
- Requires-Dist: asyncpg>=0.31; extra == "postgres"
62
+ Requires-Dist: keepup-postgres>=0.4.0; extra == "postgres"
63
+ Provides-Extra: sqlite
64
+ Requires-Dist: keepup-sqlite>=0.4.0; extra == "sqlite"
41
65
  Dynamic: license-file
42
66
 
43
67
  # keepup
@@ -122,16 +146,17 @@ between replicas needs.
122
146
  ## Tests
123
147
 
124
148
  ```bash
125
- pip install -e ".[test]"
149
+ pip install -e ".[test]" -e packages/keepup-*
126
150
  pytest tests/
127
151
  ```
128
152
 
129
153
  **Install first, then test.** Pointing pytest at the checkout does not work on
130
154
  its own: `import keepup` has to resolve, and the directory a clone lands in is
131
155
  called `keepup-admin`, so nothing is named `keepup` until the package is
132
- installed. The `test` extra carries what the suite needs beyond the package —
133
- notably `cryptography`, which the framework itself never imports and the OIDC
134
- tests sign tokens with.
156
+ installed. The capabilities are distributions of their own in `packages/` —
157
+ each carries the libraries it needs, and the suite imports them — so a clone
158
+ installs them from the tree; the extras name the same ones for an installation
159
+ from the index.
135
160
 
136
161
  The suite runs against SQLite with no network, needs no application beside the
137
162
  package, and holds a coverage floor declared in `pyproject.toml`. Checks that
@@ -80,16 +80,17 @@ between replicas needs.
80
80
  ## Tests
81
81
 
82
82
  ```bash
83
- pip install -e ".[test]"
83
+ pip install -e ".[test]" -e packages/keepup-*
84
84
  pytest tests/
85
85
  ```
86
86
 
87
87
  **Install first, then test.** Pointing pytest at the checkout does not work on
88
88
  its own: `import keepup` has to resolve, and the directory a clone lands in is
89
89
  called `keepup-admin`, so nothing is named `keepup` until the package is
90
- installed. The `test` extra carries what the suite needs beyond the package —
91
- notably `cryptography`, which the framework itself never imports and the OIDC
92
- tests sign tokens with.
90
+ installed. The capabilities are distributions of their own in `packages/` —
91
+ each carries the libraries it needs, and the suite imports them — so a clone
92
+ installs them from the tree; the extras name the same ones for an installation
93
+ from the index.
93
94
 
94
95
  The suite runs against SQLite with no network, needs no application beside the
95
96
  package, and holds a coverage floor declared in `pyproject.toml`. Checks that
@@ -26,6 +26,12 @@ except Exception:
26
26
  # itself, until the applications switch to installing by version.
27
27
  __version__ = "0.0.0+source"
28
28
 
29
+ # Names that moved into a distribution of their own keep working for one
30
+ # release, and say where they went (keepup-114, keepup-124).
31
+ from keepup import compat as _compat
32
+
33
+ _compat.install()
34
+
29
35
  __all__ = ["KeepupSettings", "StaticMount", "create_app", "__version__"]
30
36
 
31
37
 
@@ -0,0 +1,60 @@
1
+ """What an administrator changed, written into the event log (keepup-67).
2
+
3
+ The event log is where a deployment looks for who did what, and the framework's
4
+ own administrative surface left nothing there: purging the log, changing the
5
+ panel's sections, their grants and the themes, and switching a plugin's
6
+ decision were visible only in the application log -- which rotates, is shipped
7
+ elsewhere or nowhere, and is not what an operator reads. Each of those now
8
+ writes one event naming the administrator.
9
+
10
+ The types below are the framework's. Neither they nor the types an application
11
+ declares can be created by hand through `POST /api/events`: a journal the
12
+ watched party can fill with lookalikes is not evidence (keepup/events_api.py).
13
+
14
+ Writing the trail never fails the action it records: the action has already
15
+ happened, and answering an error for it would only invite a repeat. A trail
16
+ that could not be written is logged as an error.
17
+ """
18
+
19
+ import logging
20
+ from typing import Any, Dict, Optional
21
+
22
+ from keepup import events
23
+
24
+ logger = logging.getLogger(__name__)
25
+
26
+ #: What an application may import from this module. Everything else is
27
+ #: internal and may change without notice -- see doc/keepup.md.
28
+ __all__ = [
29
+ "ADMIN_EVENT_TYPES",
30
+ "EVENTS_PURGED",
31
+ "PLUGIN_DECISION_CHANGED",
32
+ "SECTIONS_CHANGED",
33
+ "THEMES_CHANGED",
34
+ "record",
35
+ ]
36
+
37
+ EVENTS_PURGED = "admin_events_purged"
38
+ SECTIONS_CHANGED = "admin_sections_changed"
39
+ THEMES_CHANGED = "admin_themes_changed"
40
+ PLUGIN_DECISION_CHANGED = "admin_plugin_decision_changed"
41
+
42
+ ADMIN_EVENT_TYPES = (EVENTS_PURGED, SECTIONS_CHANGED, THEMES_CHANGED, PLUGIN_DECISION_CHANGED)
43
+
44
+
45
+ def _author(admin: Optional[Dict[str, Any]]) -> Dict[str, Any]:
46
+ admin = admin or {}
47
+ return {"user_id": admin.get("id"), "username": admin.get("username")}
48
+
49
+
50
+ async def record(event_type: str, admin: Optional[Dict[str, Any]], text: str,
51
+ **details: Any) -> Optional[int]:
52
+ """Write one administrative event; the id, or None when it could not be written."""
53
+ data = {"by": _author(admin), **details}
54
+ try:
55
+ return await events.event_manager.create_event(
56
+ event_type=event_type, event_text=text, event_data=data)
57
+ except Exception as error:
58
+ logger.error(f"Could not record {event_type} by {data['by'].get('username')}: {error}")
59
+ return None
60
+
@@ -0,0 +1,76 @@
1
+ """The API schema and the documentation pages built from it (keepup-96).
2
+
3
+ FastAPI serves its OpenAPI schema at /openapi.json to anybody, and the schema
4
+ describes every route of the application -- the administrative ones with their
5
+ parameters and bodies. For somebody outside that is a map of where to knock.
6
+ FastAPI has no notion of who is asking, so the framework takes both routes over:
7
+
8
+ - the schema answers an administrator only, like every other administrative
9
+ route of the framework, unless the application opens it on purpose
10
+ (``KeepupSettings.openapi_public``) or removes it (``openapi_url=None``);
11
+ - the documentation pages stay public shells: they fetch the schema from the
12
+ browser, with the panel's cookie, so an administrator sees the API and anybody
13
+ else sees the refusal.
14
+
15
+ Building the schema in code -- ``app.openapi()``, which ci/generate_openapi.py
16
+ calls -- is untouched: it never went through the route.
17
+ """
18
+
19
+ from fastapi import Depends
20
+ from fastapi.openapi.docs import (
21
+ get_redoc_html,
22
+ get_swagger_ui_html,
23
+ get_swagger_ui_oauth2_redirect_html,
24
+ )
25
+ from fastapi.responses import JSONResponse
26
+
27
+ from keepup.auth.dependencies import get_current_admin
28
+
29
+ #: What an application may import from this module. Everything else is
30
+ #: internal and may change without notice -- see doc/keepup.md.
31
+ __all__ = ["OAUTH2_REDIRECT_URL", "register_api_documentation"]
32
+
33
+ #: Where Swagger UI returns from an OAuth2 sign-in; FastAPI's own default.
34
+ OAUTH2_REDIRECT_URL = "/docs/oauth2-redirect"
35
+
36
+
37
+ def register_api_documentation(app, settings) -> None:
38
+ """The schema route and the documentation pages, as the settings ask.
39
+
40
+ Called right after the application is created: FastAPI used to register
41
+ these in its constructor, ahead of any route of the application, and a page
42
+ registered later could be shadowed by a route that happens to match it.
43
+ """
44
+ schema_url = settings.openapi_url
45
+ if not schema_url:
46
+ # No schema, no pages: they would have nothing to show.
47
+ return
48
+
49
+ if settings.openapi_public:
50
+ async def api_schema():
51
+ return JSONResponse(app.openapi())
52
+ else:
53
+ async def api_schema(admin: dict = Depends(get_current_admin)):
54
+ return JSONResponse(app.openapi())
55
+
56
+ app.add_api_route(schema_url, api_schema, methods=["GET"], include_in_schema=False)
57
+
58
+ if settings.docs_url:
59
+ async def swagger_page():
60
+ return get_swagger_ui_html(openapi_url=schema_url, title=f"{app.title} - API",
61
+ oauth2_redirect_url=OAUTH2_REDIRECT_URL)
62
+
63
+ async def swagger_redirect():
64
+ return get_swagger_ui_oauth2_redirect_html()
65
+
66
+ app.add_api_route(settings.docs_url, swagger_page, methods=["GET"],
67
+ include_in_schema=False)
68
+ app.add_api_route(OAUTH2_REDIRECT_URL, swagger_redirect, methods=["GET"],
69
+ include_in_schema=False)
70
+
71
+ if settings.redoc_url:
72
+ async def redoc_page():
73
+ return get_redoc_html(openapi_url=schema_url, title=f"{app.title} - API")
74
+
75
+ app.add_api_route(settings.redoc_url, redoc_page, methods=["GET"],
76
+ include_in_schema=False)
@@ -22,8 +22,6 @@ from contextlib import asynccontextmanager
22
22
  from datetime import datetime
23
23
  from typing import Any, Dict, Optional
24
24
 
25
- from sqlalchemy import CheckConstraint, Column, DateTime, Index, Integer, Text, UniqueConstraint
26
- from sqlalchemy.dialects import postgresql
27
25
 
28
26
  from keepup import retention, tables
29
27
  from keepup.db import DatabaseManagerV2, db_config
@@ -48,6 +46,10 @@ __all__ = [
48
46
  "retention_days",
49
47
  ]
50
48
 
49
+ # The audit capability declares its own tables (keepup-124); re-exported here so
50
+ # the path that predates the catalogue still creates them.
51
+ from keepup_audit.tables import INCOMING_REQUESTS # noqa: E402
52
+
51
53
  logger = logging.getLogger(__name__)
52
54
 
53
55
  incoming_requests_buffer: Dict[str, Dict[str, Any]] = {}
@@ -131,6 +133,17 @@ incoming_requests_lock = asyncio.Lock()
131
133
 
132
134
  BUFFER_FLUSH_INTERVAL = 125
133
135
  BUFFER_MAX_SIZE = 100
136
+ #: The most requests the buffer holds, whatever happens to the database
137
+ #: (keepup-68). A failed flush puts its rows back, so with the database away the
138
+ #: buffer used to grow with every request; past this the oldest go.
139
+ BUFFER_HARD_LIMIT = 10_000
140
+ #: After a failed flush, requests do not start another one for this long: the
141
+ #: timed flusher still retries, but not every request on its own.
142
+ FLUSH_RETRY_PAUSE_SECONDS = 5.0
143
+
144
+ _flush_lock = asyncio.Lock()
145
+ _flush_paused_until = 0.0
146
+ dropped_requests = 0
134
147
  #: A request running this long is written without its outcome (keepup-41): the
135
148
  #: process that would have finished it has most likely gone.
136
149
  IN_FLIGHT_STALE_SECONDS = 3600
@@ -170,7 +183,13 @@ class IncomingRequestLogger:
170
183
  'request_start_at': request_start,
171
184
  'created_at': datetime.utcnow()
172
185
  }
173
- if len(incoming_requests_buffer) >= BUFFER_MAX_SIZE:
186
+ global dropped_requests
187
+ while len(incoming_requests_buffer) > BUFFER_HARD_LIMIT:
188
+ # Insertion order: the first key is the oldest request.
189
+ incoming_requests_buffer.pop(next(iter(incoming_requests_buffer)))
190
+ dropped_requests += 1
191
+ if (len(incoming_requests_buffer) >= BUFFER_MAX_SIZE and not _flush_lock.locked()
192
+ and time.monotonic() >= _flush_paused_until):
174
193
  asyncio.create_task(IncomingRequestLogger.flush_buffer())
175
194
 
176
195
  return request_id
@@ -231,6 +250,13 @@ class IncomingRequestLogger:
231
250
  not grow forever; and at shutdown (``include_in_flight``) everything is
232
251
  written -- nothing will finish afterwards.
233
252
  """
253
+ # One flush at a time: a full buffer asked for one on every request.
254
+ async with _flush_lock:
255
+ return await IncomingRequestLogger._flush_buffer(include_in_flight, now)
256
+
257
+ @staticmethod
258
+ async def _flush_buffer(include_in_flight: bool, now: Optional[datetime]) -> int:
259
+ global _flush_paused_until, dropped_requests
234
260
  now = now or datetime.utcnow()
235
261
  # The buffer lock is taken by the start and the end of every request, so
236
262
  # it is held only to pick the rows and take them out -- never across the
@@ -279,6 +305,7 @@ class IncomingRequestLogger:
279
305
  # Back into the buffer: the next flush tries again. A request that
280
306
  # somehow reappeared meanwhile keeps its newer entry.
281
307
  logger.error(f"Error flushing incoming requests buffer: {str(e)}")
308
+ _flush_paused_until = time.monotonic() + FLUSH_RETRY_PAUSE_SECONDS
282
309
  async with incoming_requests_lock:
283
310
  for request_id, req in chosen.items():
284
311
  incoming_requests_buffer.setdefault(request_id, req)
@@ -288,6 +315,10 @@ class IncomingRequestLogger:
288
315
  # duplicate alike: an ignored one is already in the table.
289
316
  if inserted_count:
290
317
  logger.info(f"Flushed {inserted_count} incoming requests to database")
318
+ if dropped_requests:
319
+ logger.warning(f"{dropped_requests} incoming requests were dropped from the "
320
+ f"audit while it could not be written")
321
+ dropped_requests = 0
291
322
  return inserted_count
292
323
 
293
324
 
@@ -360,33 +391,6 @@ async def audit_retention_background():
360
391
  await asyncio.sleep(AUDIT_ERROR_BACKOFF_SECONDS)
361
392
 
362
393
 
363
- INCOMING_REQUESTS = tables.table(
364
- "incoming_requests",
365
- tables.auto_id(),
366
- Column("instance_id", Text, nullable=False),
367
- Column("method", Text, nullable=False),
368
- Column("endpoint", Text, nullable=False),
369
- Column("host", Text, nullable=False),
370
- Column("request_data", postgresql.JSONB().with_variant(Text(), "sqlite")),
371
- Column("request_start_at", DateTime, nullable=False),
372
- Column("request_end_at", DateTime),
373
- Column("duration_ms", Integer),
374
- Column("http_status", Integer),
375
- Column("response_data", postgresql.JSONB().with_variant(Text(), "sqlite")),
376
- Column("error_message", Text),
377
- Column("created_at", DateTime, server_default=tables.NOW),
378
- UniqueConstraint("instance_id", "method", "endpoint", "request_start_at",
379
- name="incoming_requests_instance_method_endpoint_unique").ddl_if(dialect="postgresql"),
380
- CheckConstraint("duration_ms >= 0",
381
- name="incoming_requests_check_duration").ddl_if(dialect="postgresql"),
382
- CheckConstraint("(http_status IS NULL) OR (http_status >= 100 AND http_status <= 599)",
383
- name="incoming_requests_check_http_status").ddl_if(dialect="postgresql"),
384
- Index("incoming_requests_instance_method_endpoint_unique",
385
- "instance_id", "method", "endpoint", "request_start_at", unique=True).ddl_if(dialect="sqlite"),
386
- Index("idx_incoming_requests_instance_id", "instance_id"),
387
- Index("idx_incoming_requests_endpoint", "endpoint"),
388
- Index("idx_incoming_requests_created_at", "created_at"),
389
- )
390
394
 
391
395
 
392
396
  def init_incoming_requests_table():
@@ -408,6 +412,12 @@ async def background_buffer_flusher():
408
412
  await asyncio.sleep(10)
409
413
 
410
414
 
415
+ def _is_refusal(error: BaseException) -> bool:
416
+ """An HTTP refusal a route raised on purpose, whose text is its answer."""
417
+ from starlette.exceptions import HTTPException
418
+ return isinstance(error, HTTPException)
419
+
420
+
411
421
  @asynccontextmanager
412
422
  async def log_api_request(
413
423
  method: str,
@@ -434,10 +444,12 @@ async def log_api_request(
434
444
 
435
445
  except Exception as e:
436
446
  if request_id:
447
+ # A refusal keeps its reason: it is the answer the client was given.
448
+ # Anything else keeps its kind, not its text (keepup-76).
437
449
  await IncomingRequestLogger.end_request(
438
450
  request_id=request_id,
439
451
  duration_ms=int((time.time() - start_time) * 1000),
440
- error_message=str(e)
452
+ error_message=str(e) if _is_refusal(e) else type(e).__name__
441
453
  )
442
454
  raise
443
455
 
@@ -31,6 +31,8 @@ from typing import Optional
31
31
 
32
32
  from starlette.routing import Match
33
33
 
34
+ from keepup.logging_setup import for_log
35
+
34
36
  #: What an application may import from this module. Everything else is
35
37
  #: internal and may change without notice -- see doc/keepup.md.
36
38
  __all__ = [
@@ -47,7 +49,9 @@ ROUTE_LIMIT_ATTRIBUTE = "keepup_max_body_bytes"
47
49
  READS_OWN_BODY_ATTRIBUTE = "keepup_reads_own_body"
48
50
 
49
51
  _NOT_DECLARED = object()
50
- _BODY_METHODS = {"POST", "PUT", "PATCH", "DELETE"}
52
+ #: Methods whose requests are not checked. Everything else is: a route declared
53
+ #: with OPTIONS or a custom method beside a write reads its body as well.
54
+ _BODYLESS_METHODS = {"GET", "HEAD"}
51
55
 
52
56
 
53
57
  def declare_limit(endpoint, route: dict) -> None:
@@ -87,7 +91,7 @@ class BodyLimitMiddleware:
87
91
  return self.default_limit
88
92
 
89
93
  async def __call__(self, scope, receive, send):
90
- if scope["type"] != "http" or scope.get("method") not in _BODY_METHODS:
94
+ if scope["type"] != "http" or scope.get("method") in _BODYLESS_METHODS:
91
95
  return await self.app(scope, receive, send)
92
96
  limit = self.limit_for(scope)
93
97
  if limit is None:
@@ -96,7 +100,7 @@ class BodyLimitMiddleware:
96
100
  declared = _content_length(scope)
97
101
  if declared is not None and declared > limit:
98
102
  logger.warning("Request body refused: %s %s declares %s bytes, the route "
99
- "accepts %s", scope.get("method"), scope.get("path"),
103
+ "accepts %s", scope.get("method"), for_log(scope.get("path")),
100
104
  declared, limit)
101
105
  return await _answer_too_large(send, limit)
102
106
 
@@ -112,7 +116,7 @@ class BodyLimitMiddleware:
112
116
  state["over"] = True
113
117
  logger.warning("Request body refused: %s %s passed %s bytes without "
114
118
  "declaring its length", scope.get("method"),
115
- scope.get("path"), limit)
119
+ for_log(scope.get("path")), limit)
116
120
  return {"type": "http.disconnect"}
117
121
  return message
118
122
 
@@ -0,0 +1,107 @@
1
+ """The audit: incoming calls, the application's events and the administrator's trail.
2
+
3
+ Task keepup-111. These live in three modules of the kernel -- ``audit.py``
4
+ (incoming requests, redaction, retention), ``events.py`` (the application's own
5
+ event log) and ``admin_trail.py`` (what an administrator decided) -- and all
6
+ three import SQLAlchemy, so the base bundle cannot be assembled without a
7
+ database library while they stay. The plugin here owns the two tables and
8
+ publishes the two services; the kernel's own trail and the route wrapper reach
9
+ them through the names (``keepup/kernel/observability.py``), not by importing
10
+ these modules.
11
+
12
+ The routes of the event log and the section that shows it travel with the panel
13
+ (keepup-104); what is here is the recording itself, its retention and the two
14
+ names.
15
+ """
16
+
17
+ import logging
18
+ from typing import Any, Dict, List
19
+
20
+ from keepup.kernel.datasource import SERVICE_DATASOURCE
21
+ from keepup.kernel.descriptor import KIND_OPTIONAL, PluginDescriptor
22
+ from keepup.kernel.observability import SERVICE_AUDIT, SERVICE_EVENTS, Recording
23
+ from keepup.audit import INCOMING_REQUESTS
24
+ from keepup.events import APP_EVENTS
25
+ from keepup.plugins.base import BasePlugin
26
+
27
+ logger = logging.getLogger(__name__)
28
+
29
+ __all__ = ["AuditPlugin", "AuditRecording"]
30
+
31
+
32
+ class AuditRecording(Recording):
33
+ """The framework's own recording, behind the kernel's two names.
34
+
35
+ It delegates to the modules that already do the work (``keepup.audit``,
36
+ ``keepup.events``) so that nothing changes for a deployment today; those
37
+ modules travel into this capability's distribution in keepup-124.
38
+ """
39
+
40
+ def __init__(self, datasource: Any = None):
41
+ from keepup.audit import IncomingRequestLogger, log_api_request
42
+
43
+ super().__init__(start=log_api_request, end=IncomingRequestLogger.end_request,
44
+ name="keepup audit")
45
+
46
+ def watch(self, emitter: Any) -> None:
47
+ """Add another emitter, for a capability that keeps events of its own."""
48
+ self.emitters.append(emitter)
49
+
50
+
51
+ class AuditPlugin(BasePlugin):
52
+ """The capability: two tables, two services and the retention they need."""
53
+
54
+ descriptor = PluginDescriptor(
55
+ id="audit",
56
+ name="Event audit",
57
+ version="0.4.0",
58
+ distribution="keepup-audit",
59
+ kind=KIND_OPTIONAL,
60
+ priority=60,
61
+ provides=("audit>=1", "events>=1"),
62
+ requires=("datasource>=1",),
63
+ wants=("locks>=1", "scheduler>=1"),
64
+ contributions=("routes", "sections", "tables", "events"),
65
+ )
66
+
67
+ def __init__(self, config=None):
68
+ super().__init__("audit", "Event audit", config)
69
+
70
+ def register(self, services):
71
+ """Publish the recording where there is a data source to write it to.
72
+
73
+ Args:
74
+ services: the runtime's registry.
75
+ """
76
+ if services.has(SERVICE_DATASOURCE):
77
+ self.recording = AuditRecording(services.require(SERVICE_DATASOURCE))
78
+ services.provide(SERVICE_AUDIT, self.recording, version=1, plugin_id="audit")
79
+ services.provide(SERVICE_EVENTS, self.recording, version=1, plugin_id="audit")
80
+
81
+ async def initialize(self):
82
+ """Keep the recording published at registration; build one if there is none."""
83
+ if getattr(self, "recording", None) is None:
84
+ self.recording = AuditRecording(self.services.require(SERVICE_DATASOURCE))
85
+ return True
86
+
87
+ def get_declared_tables(self):
88
+ """The two tables this capability owns, declared once, until keepup-124."""
89
+ return [INCOMING_REQUESTS, APP_EVENTS]
90
+
91
+ def get_event_sinks(self) -> List[Any]:
92
+ """What writes an event down, for the kernel's own trail."""
93
+ return [self.emit]
94
+
95
+ async def emit(self, event_type: str, payload: Dict[str, Any] = None) -> None:
96
+ """Write one event through the module that still owns the table."""
97
+ from keepup.events import emit_event
98
+
99
+ await emit_event(event_type, payload or {})
100
+
101
+ def get_api_routes(self):
102
+ """No routes yet: the event log's own travel with the panel (keepup-104)."""
103
+ return []
104
+
105
+ def get_handlers(self):
106
+ """What another plugin may reach instead of importing the audit."""
107
+ return {"recording": self.recording, "emit": self.emit}
@@ -0,0 +1,100 @@
1
+ """Sign-in as a capability: who is calling, and what they may do.
2
+
3
+ Task keepup-116, first half. The kernel owns the shapes and the two names
4
+ (``keepup/kernel/security.py``, keepup-119); this plugin is a deployment putting
5
+ an answer behind them, and the tables that answer needs -- the panel's sessions
6
+ and the login attempts that throttle a guesser -- belong to it.
7
+
8
+ The sign-in machinery itself (``auth/routes.py``, ``panel_session.py``,
9
+ ``login_throttle.py``, the providers, OIDC and somebody else's identity system)
10
+ travels into this capability's distribution in keepup-124, together with the
11
+ routes, the sockets and the CSRF middleware it contributes. What is here is the
12
+ claim: two services, two tables, and an identity the runtime adopts.
13
+ """
14
+
15
+ import logging
16
+ from typing import Any, Dict, List
17
+
18
+ from keepup.kernel.datasource import SERVICE_DATASOURCE
19
+ from keepup.kernel.descriptor import KIND_OPTIONAL, PluginDescriptor
20
+ from keepup.kernel.security import SERVICE_AUTH, SERVICE_PERMISSIONS, Identity
21
+ from keepup.plugins.base import BasePlugin
22
+
23
+ logger = logging.getLogger(__name__)
24
+
25
+ __all__ = ["AuthPlugin", "build_identity"]
26
+
27
+
28
+ def build_identity(name: str = "keepup-auth") -> Identity:
29
+ """The framework's own sign-in, behind the shape the kernel owns.
30
+
31
+ Args:
32
+ name: what to call it in a log line.
33
+
34
+ Returns:
35
+ The identity: how a caller is found, and what decides a right.
36
+ """
37
+ from keepup.auth.dependencies import get_panel_user
38
+ from keepup.auth.identity import access
39
+
40
+ return Identity(subject_dependency=get_panel_user, checker=access.check, name=name)
41
+
42
+
43
+ class AuthPlugin(BasePlugin):
44
+ """The capability: sessions, attempts, and the two services that answer."""
45
+
46
+ descriptor = PluginDescriptor(
47
+ id="auth",
48
+ name="Sign-in and sessions",
49
+ version="0.4.0",
50
+ distribution="keepup-auth",
51
+ kind=KIND_OPTIONAL,
52
+ priority=30,
53
+ provides=("auth>=1", "permissions>=1"),
54
+ requires=("datasource>=1",),
55
+ contributions=("routes", "sockets", "middleware", "tables", "permissions"),
56
+ )
57
+
58
+ def __init__(self, config=None):
59
+ super().__init__("auth", "Sign-in and sessions", config)
60
+
61
+ def register(self, services):
62
+ """Publish the identity under both names, once there is storage.
63
+
64
+ Args:
65
+ services: the runtime's registry.
66
+ """
67
+ if services.has(SERVICE_DATASOURCE):
68
+ self.identity = build_identity()
69
+ services.provide(SERVICE_AUTH, self.identity, version=1, plugin_id="auth")
70
+ services.provide(SERVICE_PERMISSIONS, self.identity, version=1, plugin_id="auth")
71
+
72
+ async def initialize(self):
73
+ """Keep the identity published at registration; build one if there is none."""
74
+ if getattr(self, "identity", None) is None:
75
+ self.identity = build_identity()
76
+ return True
77
+
78
+ def get_declared_tables(self) -> List[Any]:
79
+ """The two tables this capability owns, declared once, until keepup-124."""
80
+ from keepup.auth.login_throttle import LOGIN_ATTEMPTS
81
+ from keepup.auth.panel_session import AUTH_SESSION
82
+
83
+ return [AUTH_SESSION, LOGIN_ATTEMPTS]
84
+
85
+ def get_route_permissions(self) -> Dict[str, Any]:
86
+ """The rights this capability's routes ask for, by name.
87
+
88
+ Empty here, and declared all the same: the kind of contribution is what
89
+ another plugin reads to know that a right it asks for exists.
90
+ """
91
+ return {}
92
+
93
+ def get_api_routes(self) -> List[Any]:
94
+ """No routes yet: the sign-in's own (sign in, out, refresh, WebSocket, CSRF)
95
+ travel into this capability's distribution in keepup-124."""
96
+ return []
97
+
98
+ def get_handlers(self) -> Dict[str, Any]:
99
+ """What another plugin may reach instead of importing the sign-in."""
100
+ return {"identity": self.identity}