keepup-admin 0.1.1__tar.gz → 0.2.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (97) hide show
  1. {keepup_admin-0.1.1/keepup_admin.egg-info → keepup_admin-0.2.0}/PKG-INFO +16 -2
  2. {keepup_admin-0.1.1 → keepup_admin-0.2.0}/README.md +15 -1
  3. {keepup_admin-0.1.1 → keepup_admin-0.2.0}/audit.py +91 -140
  4. {keepup_admin-0.1.1 → keepup_admin-0.2.0}/auth/dependencies.py +43 -118
  5. {keepup_admin-0.1.1 → keepup_admin-0.2.0}/auth/login_throttle.py +22 -21
  6. {keepup_admin-0.1.1 → keepup_admin-0.2.0}/auth/providers/base.py +43 -9
  7. {keepup_admin-0.1.1 → keepup_admin-0.2.0}/auth/providers/local.py +29 -23
  8. {keepup_admin-0.1.1 → keepup_admin-0.2.0}/auth/routes.py +53 -84
  9. keepup_admin-0.2.0/body_limit.py +152 -0
  10. keepup_admin-0.2.0/cache.py +169 -0
  11. {keepup_admin-0.1.1 → keepup_admin-0.2.0}/cluster.py +21 -12
  12. {keepup_admin-0.1.1 → keepup_admin-0.2.0}/db.py +84 -286
  13. keepup_admin-0.2.0/events.py +484 -0
  14. keepup_admin-0.2.0/events_api.py +209 -0
  15. {keepup_admin-0.1.1 → keepup_admin-0.2.0}/factory.py +62 -8
  16. {keepup_admin-0.1.1 → keepup_admin-0.2.0}/integrations.py +46 -82
  17. {keepup_admin-0.1.1 → keepup_admin-0.2.0/keepup_admin.egg-info}/PKG-INFO +16 -2
  18. {keepup_admin-0.1.1 → keepup_admin-0.2.0}/keepup_admin.egg-info/SOURCES.txt +12 -0
  19. {keepup_admin-0.1.1 → keepup_admin-0.2.0}/locks.py +82 -65
  20. keepup_admin-0.1.1/logging_setup.py → keepup_admin-0.2.0/log_shipping.py +23 -63
  21. keepup_admin-0.2.0/logging_setup.py +122 -0
  22. {keepup_admin-0.1.1 → keepup_admin-0.2.0}/metrics.py +31 -261
  23. keepup_admin-0.2.0/metrics_api.py +281 -0
  24. {keepup_admin-0.1.1 → keepup_admin-0.2.0}/metrics_retention.py +2 -1
  25. {keepup_admin-0.1.1 → keepup_admin-0.2.0}/modules.py +117 -108
  26. {keepup_admin-0.1.1 → keepup_admin-0.2.0}/notification_bus.py +33 -7
  27. {keepup_admin-0.1.1 → keepup_admin-0.2.0}/plugins/admin.py +5 -3
  28. {keepup_admin-0.1.1 → keepup_admin-0.2.0}/plugins/base.py +13 -1
  29. {keepup_admin-0.1.1 → keepup_admin-0.2.0}/plugins/registry.py +20 -1
  30. {keepup_admin-0.1.1 → keepup_admin-0.2.0}/plugins/routes.py +92 -5
  31. {keepup_admin-0.1.1 → keepup_admin-0.2.0}/pyproject.toml +1 -1
  32. keepup_admin-0.2.0/retention.py +84 -0
  33. {keepup_admin-0.1.1 → keepup_admin-0.2.0}/scheduler.py +38 -7
  34. {keepup_admin-0.1.1 → keepup_admin-0.2.0}/schema.py +21 -20
  35. {keepup_admin-0.1.1 → keepup_admin-0.2.0}/settings.py +41 -1
  36. {keepup_admin-0.1.1 → keepup_admin-0.2.0}/static/css/main_new.css +17 -0
  37. {keepup_admin-0.1.1 → keepup_admin-0.2.0}/static/js/main_new.js +49 -3
  38. {keepup_admin-0.1.1 → keepup_admin-0.2.0}/themes.py +28 -12
  39. {keepup_admin-0.1.1 → keepup_admin-0.2.0}/web.py +5 -5
  40. keepup_admin-0.1.1/events.py +0 -764
  41. {keepup_admin-0.1.1 → keepup_admin-0.2.0}/LICENSE +0 -0
  42. {keepup_admin-0.1.1 → keepup_admin-0.2.0}/NOTICE +0 -0
  43. {keepup_admin-0.1.1 → keepup_admin-0.2.0}/THIRD-PARTY.md +0 -0
  44. {keepup_admin-0.1.1 → keepup_admin-0.2.0}/__init__.py +0 -0
  45. {keepup_admin-0.1.1 → keepup_admin-0.2.0}/api_versions.py +0 -0
  46. {keepup_admin-0.1.1 → keepup_admin-0.2.0}/auth/__init__.py +0 -0
  47. {keepup_admin-0.1.1 → keepup_admin-0.2.0}/auth/config.py +0 -0
  48. {keepup_admin-0.1.1 → keepup_admin-0.2.0}/auth/dto/__init__.py +0 -0
  49. {keepup_admin-0.1.1 → keepup_admin-0.2.0}/auth/dto/token.py +0 -0
  50. {keepup_admin-0.1.1 → keepup_admin-0.2.0}/auth/factory.py +0 -0
  51. {keepup_admin-0.1.1 → keepup_admin-0.2.0}/auth/oidc.py +0 -0
  52. {keepup_admin-0.1.1 → keepup_admin-0.2.0}/auth/oidc_policy.py +0 -0
  53. {keepup_admin-0.1.1 → keepup_admin-0.2.0}/auth/oidc_routes.py +0 -0
  54. {keepup_admin-0.1.1 → keepup_admin-0.2.0}/auth/panel_session.py +0 -0
  55. {keepup_admin-0.1.1 → keepup_admin-0.2.0}/auth/permissions.py +0 -0
  56. {keepup_admin-0.1.1 → keepup_admin-0.2.0}/auth/providers/__init__.py +0 -0
  57. {keepup_admin-0.1.1 → keepup_admin-0.2.0}/auth/seed_accounts.py +0 -0
  58. {keepup_admin-0.1.1 → keepup_admin-0.2.0}/auth/session_lifetime.py +0 -0
  59. {keepup_admin-0.1.1 → keepup_admin-0.2.0}/auth/signing_key.py +0 -0
  60. {keepup_admin-0.1.1 → keepup_admin-0.2.0}/auth/websocket.py +0 -0
  61. {keepup_admin-0.1.1 → keepup_admin-0.2.0}/instance.py +0 -0
  62. {keepup_admin-0.1.1 → keepup_admin-0.2.0}/keepup_admin.egg-info/dependency_links.txt +0 -0
  63. {keepup_admin-0.1.1 → keepup_admin-0.2.0}/keepup_admin.egg-info/requires.txt +0 -0
  64. {keepup_admin-0.1.1 → keepup_admin-0.2.0}/keepup_admin.egg-info/top_level.txt +0 -0
  65. {keepup_admin-0.1.1 → keepup_admin-0.2.0}/plugins/__init__.py +0 -0
  66. {keepup_admin-0.1.1 → keepup_admin-0.2.0}/plugins/enablement.py +0 -0
  67. {keepup_admin-0.1.1 → keepup_admin-0.2.0}/plugins/route_mask.py +0 -0
  68. {keepup_admin-0.1.1 → keepup_admin-0.2.0}/roles.py +0 -0
  69. {keepup_admin-0.1.1 → keepup_admin-0.2.0}/sections.json +0 -0
  70. {keepup_admin-0.1.1 → keepup_admin-0.2.0}/setup.cfg +0 -0
  71. {keepup_admin-0.1.1 → keepup_admin-0.2.0}/static/css/aos.css +0 -0
  72. {keepup_admin-0.1.1 → keepup_admin-0.2.0}/static/css/main_nebula.css +0 -0
  73. {keepup_admin-0.1.1 → keepup_admin-0.2.0}/static/css/tailwind.css +0 -0
  74. {keepup_admin-0.1.1 → keepup_admin-0.2.0}/static/index_nebula.html +0 -0
  75. {keepup_admin-0.1.1 → keepup_admin-0.2.0}/static/index_new.html +0 -0
  76. {keepup_admin-0.1.1 → keepup_admin-0.2.0}/static/js/aos.js +0 -0
  77. {keepup_admin-0.1.1 → keepup_admin-0.2.0}/static/js/feather-icons.js +0 -0
  78. {keepup_admin-0.1.1 → keepup_admin-0.2.0}/static/js/tailwind.js +0 -0
  79. {keepup_admin-0.1.1 → keepup_admin-0.2.0}/static/modules/css/background_tasks.css +0 -0
  80. {keepup_admin-0.1.1 → keepup_admin-0.2.0}/static/modules/css/cluster.css +0 -0
  81. {keepup_admin-0.1.1 → keepup_admin-0.2.0}/static/modules/css/event_manager.css +0 -0
  82. {keepup_admin-0.1.1 → keepup_admin-0.2.0}/static/modules/css/integration_logs.css +0 -0
  83. {keepup_admin-0.1.1 → keepup_admin-0.2.0}/static/modules/css/metrics.css +0 -0
  84. {keepup_admin-0.1.1 → keepup_admin-0.2.0}/static/modules/css/modules.css +0 -0
  85. {keepup_admin-0.1.1 → keepup_admin-0.2.0}/static/modules/css/themes.css +0 -0
  86. {keepup_admin-0.1.1 → keepup_admin-0.2.0}/static/modules/css/users.css +0 -0
  87. {keepup_admin-0.1.1 → keepup_admin-0.2.0}/static/modules/js/background_tasks.js +0 -0
  88. {keepup_admin-0.1.1 → keepup_admin-0.2.0}/static/modules/js/chart.js +0 -0
  89. {keepup_admin-0.1.1 → keepup_admin-0.2.0}/static/modules/js/chartjs-adapter-date-fns.bundle.min.js +0 -0
  90. {keepup_admin-0.1.1 → keepup_admin-0.2.0}/static/modules/js/cluster.js +0 -0
  91. {keepup_admin-0.1.1 → keepup_admin-0.2.0}/static/modules/js/event_manager.js +0 -0
  92. {keepup_admin-0.1.1 → keepup_admin-0.2.0}/static/modules/js/integration_logs.js +0 -0
  93. {keepup_admin-0.1.1 → keepup_admin-0.2.0}/static/modules/js/metrics.js +0 -0
  94. {keepup_admin-0.1.1 → keepup_admin-0.2.0}/static/modules/js/modules.js +0 -0
  95. {keepup_admin-0.1.1 → keepup_admin-0.2.0}/static/modules/js/themes.js +0 -0
  96. {keepup_admin-0.1.1 → keepup_admin-0.2.0}/static/modules/js/users.js +0 -0
  97. {keepup_admin-0.1.1 → keepup_admin-0.2.0}/tables.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: keepup-admin
3
- Version: 0.1.1
3
+ Version: 0.2.0
4
4
  Summary: An admin framework over FastAPI: plugins, panel, users, replicas
5
5
  Author-email: Fedor Sergeev <fksergeev@gmail.com>
6
6
  License-Expression: Apache-2.0
@@ -42,10 +42,24 @@ Dynamic: license-file
42
42
 
43
43
  # keepup
44
44
 
45
+ [![tests](https://github.com/FedorSergeev/keepup-admin/actions/workflows/tests.yml/badge.svg?branch=main)](https://github.com/FedorSergeev/keepup-admin/actions/workflows/tests.yml)
46
+ [![security](https://github.com/FedorSergeev/keepup-admin/actions/workflows/security.yml/badge.svg?branch=main)](https://github.com/FedorSergeev/keepup-admin/actions/workflows/security.yml)
47
+ [![PyPI](https://img.shields.io/pypi/v/keepup-admin)](https://pypi.org/project/keepup-admin/)
48
+ [![licence](https://img.shields.io/pypi/l/keepup-admin)](LICENSE)
49
+
45
50
  [Project page](https://fedorsergeev.github.io/keepup-admin/) ·
46
51
  [Package](https://pypi.org/project/keepup-admin/) ·
47
52
  [Changes](CHANGELOG.md)
48
53
 
54
+ Four badges, not six checks. These answer what somebody deciding whether to
55
+ install asks: does it run on my Python, is there a known vulnerability in what
56
+ it pulls in, what will I get, and may I use it. `quality`, `build` and `codeql`
57
+ are housekeeping and live on the Actions page -- six badges in a row stop being
58
+ read at all.
59
+
60
+ A badge shows; it does not require. What keeps a red check from being merged is
61
+ a branch rule, not a picture -- see `doc/keepup.md`.
62
+
49
63
  An admin framework over FastAPI. You give it settings; it gives you an
50
64
  application with a panel, a plugin runtime, users, a database layer, an audit,
51
65
  metrics and coordination between replicas already wired.
@@ -73,7 +87,7 @@ so the import name is free.
73
87
  | **Panel** | the shell, visual themes, and the catalogue of sections with role grants |
74
88
  | **Users** | local accounts, roles, panel sessions, sign-in through an external provider |
75
89
  | **Database** | a SQLAlchemy layer with a pool, table declarations for two dialects |
76
- | **Replicas** | distributed locks, leader election, a message bus, a cluster registry |
90
+ | **Replicas** | distributed locks, a message bus, a cluster registry, performance settings in one place |
77
91
  | **Observability** | audit of incoming requests, an event log, metrics, log shipping |
78
92
 
79
93
  ## What it does not decide for you
@@ -1,9 +1,23 @@
1
1
  # keepup
2
2
 
3
+ [![tests](https://github.com/FedorSergeev/keepup-admin/actions/workflows/tests.yml/badge.svg?branch=main)](https://github.com/FedorSergeev/keepup-admin/actions/workflows/tests.yml)
4
+ [![security](https://github.com/FedorSergeev/keepup-admin/actions/workflows/security.yml/badge.svg?branch=main)](https://github.com/FedorSergeev/keepup-admin/actions/workflows/security.yml)
5
+ [![PyPI](https://img.shields.io/pypi/v/keepup-admin)](https://pypi.org/project/keepup-admin/)
6
+ [![licence](https://img.shields.io/pypi/l/keepup-admin)](LICENSE)
7
+
3
8
  [Project page](https://fedorsergeev.github.io/keepup-admin/) ·
4
9
  [Package](https://pypi.org/project/keepup-admin/) ·
5
10
  [Changes](CHANGELOG.md)
6
11
 
12
+ Four badges, not six checks. These answer what somebody deciding whether to
13
+ install asks: does it run on my Python, is there a known vulnerability in what
14
+ it pulls in, what will I get, and may I use it. `quality`, `build` and `codeql`
15
+ are housekeeping and live on the Actions page -- six badges in a row stop being
16
+ read at all.
17
+
18
+ A badge shows; it does not require. What keeps a red check from being merged is
19
+ a branch rule, not a picture -- see `doc/keepup.md`.
20
+
7
21
  An admin framework over FastAPI. You give it settings; it gives you an
8
22
  application with a panel, a plugin runtime, users, a database layer, an audit,
9
23
  metrics and coordination between replicas already wired.
@@ -31,7 +45,7 @@ so the import name is free.
31
45
  | **Panel** | the shell, visual themes, and the catalogue of sections with role grants |
32
46
  | **Users** | local accounts, roles, panel sessions, sign-in through an external provider |
33
47
  | **Database** | a SQLAlchemy layer with a pool, table declarations for two dialects |
34
- | **Replicas** | distributed locks, leader election, a message bus, a cluster registry |
48
+ | **Replicas** | distributed locks, a message bus, a cluster registry, performance settings in one place |
35
49
  | **Observability** | audit of incoming requests, an event log, metrics, log shipping |
36
50
 
37
51
  ## What it does not decide for you
@@ -17,7 +17,6 @@ recorded says so by name -- ``configure(redaction=keep_as_is)``.
17
17
  import asyncio
18
18
  import json
19
19
  import logging
20
- import os
21
20
  import time
22
21
  from contextlib import asynccontextmanager
23
22
  from datetime import datetime
@@ -26,8 +25,8 @@ from typing import Any, Dict, Optional
26
25
  from sqlalchemy import CheckConstraint, Column, DateTime, Index, Integer, Text, UniqueConstraint
27
26
  from sqlalchemy.dialects import postgresql
28
27
 
29
- from keepup import tables
30
- from keepup.db import DatabaseManager, DatabaseManagerV2, db_config
28
+ from keepup import retention, tables
29
+ from keepup.db import DatabaseManagerV2, db_config
31
30
  from keepup.instance import get_instance_id
32
31
 
33
32
  #: What an application may import from this module. Everything else is
@@ -132,6 +131,15 @@ incoming_requests_lock = asyncio.Lock()
132
131
 
133
132
  BUFFER_FLUSH_INTERVAL = 125
134
133
  BUFFER_MAX_SIZE = 100
134
+ #: A request running this long is written without its outcome (keepup-41): the
135
+ #: process that would have finished it has most likely gone.
136
+ IN_FLIGHT_STALE_SECONDS = 3600
137
+
138
+
139
+ def _as_datetime(value) -> datetime:
140
+ if isinstance(value, str):
141
+ return datetime.fromisoformat(value.replace('Z', ''))
142
+ return value
135
143
 
136
144
 
137
145
  class IncomingRequestLogger:
@@ -211,103 +219,86 @@ class IncomingRequestLogger:
211
219
  return True
212
220
 
213
221
  @staticmethod
214
- async def flush_buffer() -> int:
215
- """Write the buffered requests to the database."""
216
-
222
+ async def flush_buffer(include_in_flight: bool = False, now: Optional[datetime] = None) -> int:
223
+ """Write the finished requests of the buffer to the database.
224
+
225
+ A request still running stays in the buffer (keepup-41): written by a
226
+ timed flush and dropped from the buffer, it could no longer receive its
227
+ outcome -- end_request found nothing, logged "not found", and the table
228
+ kept a request with no status. Two exceptions: a request running longer
229
+ than IN_FLIGHT_STALE_SECONDS is written without an outcome, since the
230
+ process that would finish it has most likely gone and the buffer must
231
+ not grow forever; and at shutdown (``include_in_flight``) everything is
232
+ written -- nothing will finish afterwards.
233
+ """
234
+ now = now or datetime.utcnow()
235
+ # The buffer lock is taken by the start and the end of every request, so
236
+ # it is held only to pick the rows and take them out -- never across the
237
+ # write, which runs in a worker thread and may take as long as the
238
+ # database does (keepup-44).
217
239
  async with incoming_requests_lock:
218
- if not incoming_requests_buffer:
219
- return 0
220
-
221
- requests_to_insert = list(incoming_requests_buffer.values())
222
- inserted_count = 0
223
-
224
- try:
225
- values = []
226
- for req in requests_to_insert:
227
- values.append((
228
- req['instance_id'],
229
- req['method'],
230
- req['endpoint'],
231
- req['host'],
232
- json.dumps(req['request_data']) if req['request_data'] else None,
233
- req['request_start_at'],
234
- req.get('request_end_at'),
235
- req.get('duration_ms'),
236
- req.get('http_status'),
237
- json.dumps(req.get('response_data')) if req.get('response_data') else None,
238
- req.get('error_message'),
239
- req['created_at']
240
- ))
241
-
242
- if values:
243
- if db_config.is_postgres():
244
- query = '''
245
- INSERT INTO incoming_requests
246
- (instance_id, method, endpoint, host, request_data, request_start_at,
247
- request_end_at, duration_ms, http_status, response_data, error_message, created_at)
248
- VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
249
- ON CONFLICT (instance_id, method, endpoint, request_start_at)
250
- DO NOTHING
251
- '''
252
- else:
253
- query = '''
254
- INSERT OR IGNORE INTO incoming_requests
255
- (instance_id, method, endpoint, host, request_data, request_start_at,
256
- request_end_at, duration_ms, http_status, response_data, error_message, created_at)
257
- VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
258
- '''
259
-
260
- if db_config.is_postgres():
261
- conn = DatabaseManager.get_connection()
262
- cursor = conn.cursor()
263
- try:
264
- cursor.executemany(query, values)
265
- inserted_count = cursor.rowcount
266
- conn.commit()
267
- finally:
268
- cursor.close()
269
- conn.close()
270
- else:
271
- conn = DatabaseManager.get_connection()
272
- cursor = conn.cursor()
273
- try:
274
- cursor.executemany(query, values)
275
- inserted_count = cursor.rowcount
276
- conn.commit()
277
- finally:
278
- cursor.close()
279
- conn.close()
280
-
281
- if inserted_count > 0:
282
- inserted_ids = []
283
- for req_id, req_data in incoming_requests_buffer.items():
284
- for v in values:
285
- if (req_data['instance_id'] == v[0] and
286
- req_data['method'] == v[1] and
287
- req_data['endpoint'] == v[2] and
288
- req_data['request_start_at'] == v[5]):
289
- inserted_ids.append(req_id)
290
- break
291
-
292
- for req_id in inserted_ids:
293
- incoming_requests_buffer.pop(req_id, None)
294
-
295
- logger.info(f"Flushed {inserted_count} incoming requests to database")
296
-
297
- except Exception as e:
298
- logger.error(f"Error flushing incoming requests buffer: {str(e)}")
299
-
300
- return inserted_count
240
+ chosen = {
241
+ request_id: req for request_id, req in incoming_requests_buffer.items()
242
+ if include_in_flight or 'request_end_at' in req
243
+ or (now - _as_datetime(req['request_start_at'])).total_seconds()
244
+ >= IN_FLIGHT_STALE_SECONDS
245
+ }
246
+ for request_id in chosen:
247
+ incoming_requests_buffer.pop(request_id, None)
248
+ if not chosen:
249
+ return 0
250
+
251
+ rows = [{
252
+ 'instance_id': req['instance_id'],
253
+ 'method': req['method'],
254
+ 'endpoint': req['endpoint'],
255
+ 'host': req['host'],
256
+ 'request_data': json.dumps(req['request_data']) if req['request_data'] else None,
257
+ 'request_start_at': req['request_start_at'],
258
+ 'request_end_at': req.get('request_end_at'),
259
+ 'duration_ms': req.get('duration_ms'),
260
+ 'http_status': req.get('http_status'),
261
+ 'response_data': (json.dumps(req.get('response_data'))
262
+ if req.get('response_data') else None),
263
+ 'error_message': req.get('error_message'),
264
+ 'created_at': req['created_at'],
265
+ } for req in chosen.values()]
266
+ columns = ("instance_id, method, endpoint, host, request_data, request_start_at, "
267
+ "request_end_at, duration_ms, http_status, response_data, error_message, "
268
+ "created_at")
269
+ placeholders = ", ".join(":" + c.strip() for c in columns.split(","))
270
+ if db_config.is_postgres():
271
+ query = (f"INSERT INTO incoming_requests ({columns}) VALUES ({placeholders}) "
272
+ f"ON CONFLICT (instance_id, method, endpoint, request_start_at) DO NOTHING")
273
+ else:
274
+ query = f"INSERT OR IGNORE INTO incoming_requests ({columns}) VALUES ({placeholders})"
275
+
276
+ try:
277
+ inserted_count = await DatabaseManagerV2.execute_many_async(query, rows)
278
+ except Exception as e:
279
+ # Back into the buffer: the next flush tries again. A request that
280
+ # somehow reappeared meanwhile keeps its newer entry.
281
+ logger.error(f"Error flushing incoming requests buffer: {str(e)}")
282
+ async with incoming_requests_lock:
283
+ for request_id, req in chosen.items():
284
+ incoming_requests_buffer.setdefault(request_id, req)
285
+ return 0
286
+
287
+ # Every chosen request is gone from the buffer, written or ignored as a
288
+ # duplicate alike: an ignored one is already in the table.
289
+ if inserted_count:
290
+ logger.info(f"Flushed {inserted_count} incoming requests to database")
291
+ return inserted_count
301
292
 
302
293
 
303
294
  #: What PostgreSQL enforces with named constraints, SQLite held as a unique
304
295
  #: index of the same name and no checks at all; each dialect keeps what it had.
305
296
  # --- how long a row lives ----------------------------------------------------
306
297
 
307
- #: How many days an audit row is kept. The table used to have no sweep at all,
308
- #: while metric snapshots and application events both had one: it grew for as
309
- #: long as the deployment ran, and every row in it was a second place a value
310
- #: from a query string lived.
298
+ #: How many days an audit row is kept. The table used to have no sweep at all:
299
+ #: it grew for as long as the deployment ran, and every row in it was a second
300
+ #: place a value from a query string lived. The event log is swept the same way
301
+ #: (keepup/retention.py).
311
302
  DEFAULT_AUDIT_RETENTION_DAYS = 30
312
303
  #: How often the sweep runs.
313
304
  AUDIT_SWEEP_INTERVAL_SECONDS = 3600
@@ -324,35 +315,12 @@ AUDIT_RETENTION_LOCK = "incoming_requests_retention"
324
315
 
325
316
 
326
317
  def retention_days():
327
- """How many days rows are kept, from the environment or the default.
328
-
329
- Returns:
330
- A positive number of days; rubbish in the variable means the default
331
- and a line in the log, because a sweep that switched itself off
332
- silently would be worse than one that swept too much.
333
- """
334
- raw = os.getenv("AUDIT_RETENTION_DAYS")
335
- if raw in (None, ""):
336
- return DEFAULT_AUDIT_RETENTION_DAYS
337
- try:
338
- value = int(raw)
339
- except (TypeError, ValueError):
340
- logger.warning("AUDIT_RETENTION_DAYS=%r is not a number, using %s",
341
- raw, DEFAULT_AUDIT_RETENTION_DAYS)
342
- return DEFAULT_AUDIT_RETENTION_DAYS
343
- if value <= 0:
344
- logger.warning("AUDIT_RETENTION_DAYS=%s must be positive, using %s",
345
- value, DEFAULT_AUDIT_RETENTION_DAYS)
346
- return DEFAULT_AUDIT_RETENTION_DAYS
347
- return value
318
+ """How many days audit rows are kept (``AUDIT_RETENTION_DAYS``, 30 by default)."""
319
+ return retention.retention_days("AUDIT_RETENTION_DAYS", DEFAULT_AUDIT_RETENTION_DAYS)
348
320
 
349
321
 
350
322
  def purge_old_requests(days=None, now=None):
351
- """Delete audit rows older than the retention period.
352
-
353
- The boundary is computed here rather than in the statement: the interval
354
- syntax differs between the engines, and a query written for one does not
355
- run on the other.
323
+ """Delete audit rows older than the retention period, in chunks.
356
324
 
357
325
  Args:
358
326
  days: how many days to keep; read from the environment when omitted.
@@ -361,27 +329,10 @@ def purge_old_requests(days=None, now=None):
361
329
  Returns:
362
330
  How many rows were deleted.
363
331
  """
364
- from datetime import timedelta
365
-
366
- keep_days = days if days is not None else retention_days()
367
- cutoff = (now or datetime.utcnow()) - timedelta(days=keep_days)
368
-
369
- removed = 0
370
- for _ in range(AUDIT_MAX_CHUNKS_PER_PASS):
371
- rows = DatabaseManagerV2.execute(
372
- "SELECT id FROM incoming_requests WHERE created_at < :cutoff "
373
- "ORDER BY created_at ASC LIMIT :limit",
374
- {"cutoff": cutoff, "limit": AUDIT_DELETE_CHUNK_ROWS})
375
- if not rows:
376
- break
377
- ids = {f"i{index}": row["id"] for index, row in enumerate(rows)}
378
- placeholders = ", ".join(f":{key}" for key in ids)
379
- DatabaseManagerV2.execute_commit(
380
- f"DELETE FROM incoming_requests WHERE id IN ({placeholders})", ids)
381
- removed += len(rows)
382
- if len(rows) < AUDIT_DELETE_CHUNK_ROWS:
383
- break
384
- return removed
332
+ return retention.purge_older_than(
333
+ "incoming_requests", "created_at",
334
+ days if days is not None else retention_days(),
335
+ AUDIT_DELETE_CHUNK_ROWS, AUDIT_MAX_CHUNKS_PER_PASS, now=now)
385
336
 
386
337
 
387
338
  async def audit_retention_background():
@@ -27,7 +27,7 @@ from keepup.auth.factory import AuthProviderFactory
27
27
  from keepup.auth.providers.base import AuthProvider, ALGORITHM, oauth2_scheme
28
28
  from keepup.auth.signing_key import resolve_signing_key
29
29
  from keepup.roles import ROLE_ADMIN, ROLE_CLIENT
30
- from keepup.db import DatabaseManager, db_config
30
+ from keepup.db import DatabaseManagerV2
31
31
 
32
32
  #: What an application may import from this module. Everything else is
33
33
  #: internal and may change without notice -- see doc/keepup.md.
@@ -56,70 +56,21 @@ async def get_user_by_username_async(username: str):
56
56
 
57
57
 
58
58
  def get_user_by_username(username: str):
59
- """Return a user by name, including the extra fields."""
60
- try:
61
- loop = asyncio.get_event_loop()
62
- except RuntimeError:
63
- loop = asyncio.new_event_loop()
64
- asyncio.set_event_loop(loop)
65
-
66
- if loop.is_running():
67
- import asyncio as async_module
68
- import threading
69
-
70
- result = None
71
- event = threading.Event()
72
-
73
- def run_in_thread():
74
- nonlocal result
75
- new_loop = async_module.new_event_loop()
76
- try:
77
- async_module.set_event_loop(new_loop)
78
- result = new_loop.run_until_complete(
79
- auth_provider.get_user_info(username)
80
- )
81
- except Exception as e:
82
- logger.error(f"Error in thread: {e}")
83
- finally:
84
- # Closed on every path, including the failing one: a loop holds
85
- # an epoll descriptor and a self-pipe, and this runs once per
86
- # authenticated call. Left open, a long-lived process reaches its
87
- # descriptor limit and then cannot open anything at all -- which
88
- # surfaces as "database connection failed", not as a leak.
89
- try:
90
- new_loop.close()
91
- except Exception as e:
92
- logger.warning(f"Could not close the worker event loop: {e}")
93
- async_module.set_event_loop(None)
94
- event.set()
95
-
96
- thread = threading.Thread(target=run_in_thread)
97
- thread.start()
98
- thread.join(timeout=10)
99
-
100
- if event.is_set() and result is not None:
101
- return result
102
- else:
103
- logger.warning(f"Async call failed, falling back to DB for user {username}")
104
- return None
59
+ """Return a user by name, including the extra fields.
105
60
 
106
- user_info = loop.run_until_complete(
107
- auth_provider.get_user_info(username)
108
- )
109
-
110
- return user_info
61
+ Synchronous, for plugins and background jobs that cannot await. It used to
62
+ wrap the provider's coroutine -- and, called from a running loop, ran it on a
63
+ loop of its own in a thread that the caller waited for, holding the loop.
64
+ The provider answers synchronously itself now.
65
+ """
66
+ return auth_provider.lookup_user(username)
111
67
 
112
68
 
113
69
  def get_user_by_id(user_id: int):
114
70
  """Return a user by id, for any configured auth provider."""
115
71
  # TODO: auth_provider needs a lookup by id as well; without it this only
116
72
  # works against the local database.
117
- result = DatabaseManager.execute_query(
118
- "SELECT * FROM users WHERE id = ?",
119
- (user_id,),
120
- fetch_one=True
121
- )
122
- return result
73
+ return DatabaseManagerV2.execute_one("SELECT * FROM users WHERE id = :id", {"id": user_id})
123
74
 
124
75
 
125
76
  def get_system_user_id():
@@ -147,7 +98,7 @@ def get_system_user_id():
147
98
 
148
99
  def get_all_users():
149
100
  """Return every user, for any configured auth provider."""
150
- return DatabaseManager.execute_query(
101
+ return DatabaseManagerV2.execute(
151
102
  "SELECT id, username, status, role, created_at FROM users ORDER BY created_at DESC"
152
103
  )
153
104
 
@@ -158,18 +109,15 @@ def save_user_to_db(username: str, password: str):
158
109
 
159
110
  query = """
160
111
  INSERT INTO users (username, password_hash, status, role)
161
- VALUES (?, ?, ?, ?)
112
+ VALUES (:username, :password_hash, :status, :role)
162
113
  """
163
114
 
164
- if db_config.is_postgres():
165
- query += " RETURNING id"
166
-
167
115
  try:
168
- user_id = DatabaseManager.execute_commit(
116
+ row = DatabaseManagerV2.execute_commit_returning(
169
117
  query,
170
- (username, password_hash, "blocked", ROLE_CLIENT)
171
- )
172
- return user_id
118
+ {"username": username, "password_hash": password_hash, "status": "blocked",
119
+ "role": ROLE_CLIENT})
120
+ return row["id"] if row else None
173
121
 
174
122
  except Exception as e:
175
123
  if "unique constraint" in str(e).lower() or "duplicate" in str(e).lower():
@@ -202,47 +150,17 @@ async def create_user(
202
150
  def update_user(user_id: int, status: Optional[str] = None, role: Optional[str] = None,
203
151
  email: Optional[str] = None, phone: Optional[str] = None, full_name: Optional[str] = None):
204
152
  """Update a user, including the extra fields."""
205
- conn = DatabaseManager.get_connection()
206
- cursor = conn.cursor()
207
-
208
- update_fields = []
209
- params = []
210
-
211
- if status is not None:
212
- update_fields.append("status = ?")
213
- params.append(status)
214
-
215
- if role is not None:
216
- update_fields.append("role = ?")
217
- params.append(role)
218
-
219
- if email is not None:
220
- update_fields.append("email = ?")
221
- params.append(email)
222
-
223
- if phone is not None:
224
- update_fields.append("phone = ?")
225
- params.append(phone)
226
-
227
- if full_name is not None:
228
- update_fields.append("full_name = ?")
229
- params.append(full_name)
230
-
231
- if not update_fields:
153
+ changes = {"status": status, "role": role, "email": email, "phone": phone,
154
+ "full_name": full_name}
155
+ changes = {column: value for column, value in changes.items() if value is not None}
156
+ if not changes:
232
157
  return
233
158
 
234
- update_fields.append("updated_at = CURRENT_TIMESTAMP")
235
- params.append(user_id)
236
-
237
- query = f"UPDATE users SET {', '.join(update_fields)} WHERE id = ?"
238
-
239
- if db_config.is_postgres():
240
- query = query.replace('?', '%s')
241
-
242
- cursor.execute(query, params)
243
- conn.commit()
244
- cursor.close()
245
- conn.close()
159
+ # The column names come from the fixed set above, never from the caller.
160
+ assignments = ", ".join(f"{column} = :{column}" for column in changes)
161
+ DatabaseManagerV2.execute_commit(
162
+ f"UPDATE users SET {assignments}, updated_at = CURRENT_TIMESTAMP WHERE id = :id",
163
+ {**changes, "id": user_id})
246
164
 
247
165
 
248
166
  async def authenticate(username: str, password: str):
@@ -334,6 +252,13 @@ def _session_is_live(payload: dict) -> bool:
334
252
  return False
335
253
 
336
254
 
255
+ def _session_and_user(payload: dict, username: str):
256
+ """Whether the token's session is live and, only if it is, whose account it is."""
257
+ if not _session_is_live(payload):
258
+ return False, None
259
+ return True, auth_provider.lookup_user(username)
260
+
261
+
337
262
  async def get_current_user(token: str = Depends(request_token)):
338
263
  """Return the current user of this token.
339
264
 
@@ -387,15 +312,15 @@ async def _get_local_user(token: Optional[str], ignore_empty_user: bool = False)
387
312
  return None
388
313
  raise credentials_exception
389
314
 
390
- if not _session_is_live(payload):
391
- raise credentials_exception
392
-
393
- user = get_user_by_username(token_data.username)
394
- if user is None:
315
+ # Both reads are blocking queries, so they leave the loop together: one hop
316
+ # to a worker thread per request, and the account is read once -- it used
317
+ # to be read twice, each time on a new event loop in a thread the request
318
+ # waited for while holding the loop (keepup-43).
319
+ live, user = await asyncio.to_thread(_session_and_user, payload, token_data.username)
320
+ if not live or user is None:
395
321
  raise credentials_exception
396
322
 
397
- fresh_user = get_user_by_username(token_data.username)
398
- if fresh_user["status"] != "active":
323
+ if user["status"] != "active":
399
324
  raise HTTPException(
400
325
  status_code=status.HTTP_403_FORBIDDEN,
401
326
  detail="User account is blocked",
@@ -405,12 +330,12 @@ async def _get_local_user(token: Optional[str], ignore_empty_user: bool = False)
405
330
  # the token is decoded is the only place that knows -- and a handler that
406
331
  # needs to warn about the end (or record it alongside a websocket key)
407
332
  # cannot decode the token itself: it never sees it.
408
- fresh_user = dict(fresh_user)
409
- fresh_user["token_expires_at"] = _token_expiry(payload)
410
- fresh_user["session_started_at"] = _session_start(payload)
411
- fresh_user["session_id"] = payload.get(panel_session.SESSION_CLAIM)
333
+ user = dict(user)
334
+ user["token_expires_at"] = _token_expiry(payload)
335
+ user["session_started_at"] = _session_start(payload)
336
+ user["session_id"] = payload.get(panel_session.SESSION_CLAIM)
412
337
 
413
- return fresh_user
338
+ return user
414
339
 
415
340
 
416
341
  async def get_optional_user(token: Optional[str]):
@@ -106,34 +106,35 @@ def locked_until(username: str, now: Optional[datetime] = None) -> Optional[date
106
106
  return until if until > now else None
107
107
 
108
108
 
109
+ #: One statement, the same on both dialects (PostgreSQL 9.5+, SQLite 3.24+), so
110
+ #: the count is kept by the database rather than read, incremented here and
111
+ #: written back. The read-modify-write lost attempts: failures arriving at once
112
+ #: on several replicas read the same number and each wrote it plus one, and two
113
+ #: first failures both tried to insert, the loser's error swallowed -- so the
114
+ #: lockout came later than the setting said, which is the one thing a throttle
115
+ #: against guessing must not do (keepup-48). A name whose window has passed
116
+ #: starts its count again, so old failures do not add up to a lockout weeks later.
117
+ _RECORD_FAILURE = (
118
+ "INSERT INTO login_attempts (username, failures, last_failure_at) "
119
+ "VALUES (:username, 1, :now) "
120
+ "ON CONFLICT (username) DO UPDATE SET "
121
+ "failures = CASE WHEN login_attempts.last_failure_at IS NOT NULL "
122
+ "AND login_attempts.last_failure_at <= :window_start "
123
+ "THEN 1 ELSE login_attempts.failures + 1 END, "
124
+ "last_failure_at = :now"
125
+ )
126
+
127
+
109
128
  def record_failure(username: str, now: Optional[datetime] = None) -> None:
110
- """Count one failed attempt against this name."""
129
+ """Count one failed attempt against this name, atomically."""
111
130
  if not username:
112
131
  return
113
132
 
114
133
  now = now or datetime.utcnow()
115
134
  try:
116
- row = _row(username)
117
- if row is None:
118
- DatabaseManagerV2.execute_commit(
119
- "INSERT INTO login_attempts (username, failures, last_failure_at) "
120
- "VALUES (:username, 1, :now)",
121
- {"username": username, "now": now}
122
- )
123
- return
124
-
125
- # A name whose window has already passed starts its count again, so old
126
- # failures do not add up to a lockout weeks later.
127
- failures = (row.get("failures") or 0)
128
- last_failure = _as_datetime(row.get("last_failure_at"))
129
- if last_failure is not None and last_failure + lockout_window() <= now:
130
- failures = 0
131
-
132
135
  DatabaseManagerV2.execute_commit(
133
- "UPDATE login_attempts SET failures = :failures, last_failure_at = :now "
134
- "WHERE username = :username",
135
- {"failures": failures + 1, "now": now, "username": username}
136
- )
136
+ _RECORD_FAILURE,
137
+ {"username": username, "now": now, "window_start": now - lockout_window()})
137
138
  except Exception as error:
138
139
  logger.warning(f"Could not record a failed login of {username}: {error}")
139
140