velocity-python 0.7.0__py3-none-any.whl → 0.8.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- velocity/__init__.py +1 -1
- velocity/aws/capabilities/base.py +21 -2
- velocity/aws/capabilities/dispatch.py +7 -2
- velocity/aws/capabilities/jobs.py +32 -6
- velocity/aws/capabilities/provisioning.py +68 -15
- velocity/aws/capabilities/security.py +19 -3
- velocity/aws/capabilities/single_session.py +27 -8
- velocity/aws/job_results.py +66 -3
- {velocity_python-0.7.0.dist-info → velocity_python-0.8.0.dist-info}/METADATA +1 -1
- {velocity_python-0.7.0.dist-info → velocity_python-0.8.0.dist-info}/RECORD +14 -14
- {velocity_python-0.7.0.dist-info → velocity_python-0.8.0.dist-info}/WHEEL +0 -0
- {velocity_python-0.7.0.dist-info → velocity_python-0.8.0.dist-info}/entry_points.txt +0 -0
- {velocity_python-0.7.0.dist-info → velocity_python-0.8.0.dist-info}/licenses/LICENSE +0 -0
- {velocity_python-0.7.0.dist-info → velocity_python-0.8.0.dist-info}/top_level.txt +0 -0
velocity/__init__.py
CHANGED
|
@@ -93,6 +93,12 @@ class Capability:
|
|
|
93
93
|
tier: str = "runtime" # "runtime" (a Lambda runs it) | "infra" (stacks)
|
|
94
94
|
schema: dict = {} # {table: {column: sql type}} -> ensure_schema
|
|
95
95
|
defaults: dict = {} # flag name -> default; seeded as TEXT feature flags
|
|
96
|
+
# flag name -> the default it had before a release changed it. An
|
|
97
|
+
# installation that had the capability enabled before then keeps these
|
|
98
|
+
# (seeded for its unset flags, and its runtime floor), so a changed
|
|
99
|
+
# default never silently turns a switch off where it was on; see
|
|
100
|
+
# provisioning.defaults_baseline.
|
|
101
|
+
legacy_defaults: dict = {}
|
|
96
102
|
schedules: tuple = () # ((action, cron), ...)
|
|
97
103
|
mixins: tuple = () # handler mixins the site attaches at runtime
|
|
98
104
|
applies_to: tuple = ("demo", "production") # default stages; registry can force
|
|
@@ -180,9 +186,22 @@ def missing_dependencies(enabled) -> dict:
|
|
|
180
186
|
return missing
|
|
181
187
|
|
|
182
188
|
|
|
183
|
-
def
|
|
184
|
-
"""
|
|
189
|
+
def floor_defaults(cap, *, legacy: bool = False) -> dict:
|
|
190
|
+
"""The capability's defaults, with its ``legacy_defaults`` laid over them
|
|
191
|
+
for an installation provisioned before those defaults changed."""
|
|
185
192
|
config = dict(cap.defaults)
|
|
193
|
+
if legacy:
|
|
194
|
+
config.update(getattr(cap, "legacy_defaults", None) or {})
|
|
195
|
+
return config
|
|
196
|
+
|
|
197
|
+
|
|
198
|
+
def resolve_config(cap, flags=None, row_config=None, *, legacy: bool = False) -> dict:
|
|
199
|
+
"""defaults <- feature flags (declared knobs only) <- site registry config.
|
|
200
|
+
|
|
201
|
+
``legacy`` (an installation whose site row predates a defaults change —
|
|
202
|
+
:func:`provisioning.uses_legacy_defaults`) makes the floor the defaults it
|
|
203
|
+
was provisioned under (:func:`floor_defaults`)."""
|
|
204
|
+
config = floor_defaults(cap, legacy=legacy)
|
|
186
205
|
flags = flags or {}
|
|
187
206
|
for key in cap.defaults:
|
|
188
207
|
value = flags.get(key)
|
|
@@ -23,7 +23,7 @@ from .base import (
|
|
|
23
23
|
dependency_order,
|
|
24
24
|
resolve_config,
|
|
25
25
|
)
|
|
26
|
-
from .provisioning import SITE_TABLE, row_enabled
|
|
26
|
+
from .provisioning import SITE_TABLE, row_enabled, uses_legacy_defaults
|
|
27
27
|
|
|
28
28
|
|
|
29
29
|
def _skip(tx, name, action, reason, *, report=True):
|
|
@@ -91,7 +91,12 @@ def run_capability(tx, context, name, action):
|
|
|
91
91
|
|
|
92
92
|
raw = row.get("config")
|
|
93
93
|
row_config = json.loads(raw) if isinstance(raw, str) and raw else (raw or {})
|
|
94
|
-
config = resolve_config(
|
|
94
|
+
config = resolve_config(
|
|
95
|
+
cap,
|
|
96
|
+
flags=feature_flags.get(tx),
|
|
97
|
+
row_config=row_config,
|
|
98
|
+
legacy=uses_legacy_defaults(row),
|
|
99
|
+
)
|
|
95
100
|
|
|
96
101
|
result = cap.handle(action, tx, context, config)
|
|
97
102
|
return {"capability": name, "action": action, "result": result}
|
|
@@ -6,15 +6,19 @@ forever, and ``aws_job_activity`` is never pruned. This capability sweeps on
|
|
|
6
6
|
a schedule with the same rule the web tier uses (``Running``/``Initialized``
|
|
7
7
|
with no progress for the window), judged by the database clock
|
|
8
8
|
(:func:`stale_jobs`): stale jobs are persisted as Failed with the standard
|
|
9
|
-
message plus the status they expired from (``expired_from``),
|
|
10
|
-
|
|
9
|
+
message plus the status they expired from (``expired_from``), old activity
|
|
10
|
+
rows are purged, and stored job results past their ``expires_at``
|
|
11
|
+
(``aws_job_results``, :func:`velocity.aws.job_results.purge_expired_results`)
|
|
12
|
+
are deleted — bounded per run, so a backlog drains over several runs.
|
|
11
13
|
"""
|
|
12
14
|
|
|
13
15
|
from __future__ import annotations
|
|
14
16
|
|
|
15
17
|
import datetime
|
|
16
18
|
import json
|
|
19
|
+
import logging
|
|
17
20
|
|
|
21
|
+
from velocity.aws import job_results
|
|
18
22
|
from velocity.aws.job_lifecycle import (
|
|
19
23
|
JOB_ACTIVITY_TABLE,
|
|
20
24
|
STALE_FAILURE,
|
|
@@ -23,6 +27,10 @@ from velocity.aws.job_lifecycle import (
|
|
|
23
27
|
|
|
24
28
|
from .base import EVERY_15_MINUTES, Capability, capability
|
|
25
29
|
|
|
30
|
+
log = logging.getLogger(__name__)
|
|
31
|
+
|
|
32
|
+
MIN_STALE_MINUTES = 15 # the Lambda run-time ceiling
|
|
33
|
+
|
|
26
34
|
|
|
27
35
|
@capability
|
|
28
36
|
class Jobs(Capability):
|
|
@@ -30,7 +38,8 @@ class Jobs(Capability):
|
|
|
30
38
|
label = "Job hygiene"
|
|
31
39
|
description = (
|
|
32
40
|
"Marks asynchronous jobs whose worker died as Failed (instead of "
|
|
33
|
-
"Running forever)
|
|
41
|
+
"Running forever), prunes old job activity and deletes expired job "
|
|
42
|
+
"results. Windows are per-"
|
|
34
43
|
"environment flags; no per-site policy."
|
|
35
44
|
)
|
|
36
45
|
defaults = {"job_stale_minutes": "16", "job_retention_days": "30"}
|
|
@@ -40,7 +49,13 @@ class Jobs(Capability):
|
|
|
40
49
|
if action != "expire_stale":
|
|
41
50
|
raise ValueError(f"jobs: unknown action {action!r}")
|
|
42
51
|
now = datetime.datetime.now(datetime.timezone.utc)
|
|
43
|
-
minutes
|
|
52
|
+
# A Lambda worker runs at most 15 minutes: a shorter window would
|
|
53
|
+
# fail jobs that are still running, so it never goes below that.
|
|
54
|
+
minutes = max(int(config.get("job_stale_minutes") or 16), MIN_STALE_MINUTES)
|
|
55
|
+
days = int(config.get("job_retention_days") or 30)
|
|
56
|
+
if days < 1:
|
|
57
|
+
# "0" would purge every job row, in-flight ones included
|
|
58
|
+
raise ValueError(f"jobs: job_retention_days must be >= 1, not {days}")
|
|
44
59
|
jobs = tx.table(JOB_ACTIVITY_TABLE)
|
|
45
60
|
|
|
46
61
|
expired = 0
|
|
@@ -59,9 +74,20 @@ class Jobs(Capability):
|
|
|
59
74
|
expired += 1
|
|
60
75
|
_report_after_commit(tx, job, status)
|
|
61
76
|
|
|
62
|
-
days = int(config.get("job_retention_days") or 30)
|
|
63
77
|
purged = jobs.purge({"<sys_created": now - datetime.timedelta(days=days)})
|
|
64
|
-
|
|
78
|
+
# the expiry (and its failure reports) is durable before the result
|
|
79
|
+
# purge runs: a result store that fails must not undo it
|
|
80
|
+
tx.commit()
|
|
81
|
+
try:
|
|
82
|
+
results = job_results.purge_expired_results(tx, now=now)
|
|
83
|
+
except Exception as exc: # noqa: BLE001 - reported, the sweep stands
|
|
84
|
+
tx.rollback()
|
|
85
|
+
log.warning("jobs: expired job results not purged", exc_info=True)
|
|
86
|
+
from velocity.aws import failures
|
|
87
|
+
|
|
88
|
+
failures.report(None, exc, source="job", action="expire_stale")
|
|
89
|
+
results = {"error": type(exc).__name__}
|
|
90
|
+
return {"expired": expired, "purged": purged, "results_purged": results}
|
|
65
91
|
|
|
66
92
|
|
|
67
93
|
def stale_jobs(tx, minutes):
|
|
@@ -35,6 +35,7 @@ from .base import (
|
|
|
35
35
|
REGISTRY,
|
|
36
36
|
CapabilityDependencyError,
|
|
37
37
|
dependency_order,
|
|
38
|
+
floor_defaults,
|
|
38
39
|
get,
|
|
39
40
|
requirements,
|
|
40
41
|
)
|
|
@@ -64,7 +65,14 @@ SCHEMA = {
|
|
|
64
65
|
SITE_TABLE: {
|
|
65
66
|
# Declared SQL types. Sites provisioned before velocity honoured them
|
|
66
67
|
# hold TEXT here; upgrade_column_types() casts those in place.
|
|
67
|
-
"columns": {
|
|
68
|
+
"columns": {
|
|
69
|
+
"name": "TEXT",
|
|
70
|
+
"enabled": "BOOLEAN",
|
|
71
|
+
"config": "JSONB",
|
|
72
|
+
# which defaults the installation's flags were provisioned under
|
|
73
|
+
# (BASELINE_LEGACY | BASELINE_CURRENT); see defaults_baseline()
|
|
74
|
+
"defaults_baseline": "TEXT",
|
|
75
|
+
},
|
|
68
76
|
# provision() merges on name: ON CONFLICT needs this unique index.
|
|
69
77
|
# Named as velocity's merge self-heal names it (idx__<table>__<cols>),
|
|
70
78
|
# which is also the name app schema lanes declare with
|
|
@@ -74,6 +82,16 @@ SCHEMA = {
|
|
|
74
82
|
},
|
|
75
83
|
}
|
|
76
84
|
|
|
85
|
+
# sys_capabilities.defaults_baseline: an installation that had a capability
|
|
86
|
+
# enabled before its defaults changed is LEGACY and keeps the old values
|
|
87
|
+
# (Capability.legacy_defaults) — seeded for unset flags and used as its
|
|
88
|
+
# runtime floor — so a new default never switches a guard off where it ran;
|
|
89
|
+
# one enabled since is CURRENT. NULL (a row from before the column) reads
|
|
90
|
+
# as legacy.
|
|
91
|
+
BASELINE_LEGACY = "legacy"
|
|
92
|
+
BASELINE_CURRENT = "current"
|
|
93
|
+
BASELINES = (BASELINE_LEGACY, BASELINE_CURRENT)
|
|
94
|
+
|
|
77
95
|
_FALSE_TEXT = frozenset({"false", "f", "0", "", "no", "n", "off"})
|
|
78
96
|
|
|
79
97
|
|
|
@@ -99,6 +117,31 @@ def row_enabled(row) -> bool:
|
|
|
99
117
|
return as_bool((row or {}).get("enabled"), default=True)
|
|
100
118
|
|
|
101
119
|
|
|
120
|
+
def uses_legacy_defaults(row) -> bool:
|
|
121
|
+
"""True when the site row's flags were provisioned under the capability's
|
|
122
|
+
``legacy_defaults`` — anything but an explicit ``current`` baseline, so a
|
|
123
|
+
row written before the column existed (or before this release's
|
|
124
|
+
provisioning ran) keeps what it had."""
|
|
125
|
+
value = str((row or {}).get("defaults_baseline") or "").strip().lower()
|
|
126
|
+
return value != BASELINE_CURRENT
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
def defaults_baseline(prior, active: bool):
|
|
130
|
+
"""The baseline to record for a capability row about to be written.
|
|
131
|
+
|
|
132
|
+
A recorded baseline is kept. Otherwise a row that already exists and is
|
|
133
|
+
enabled was turned on before baselines were recorded: legacy. A
|
|
134
|
+
capability being turned on now (no row, or a disabled one) is current.
|
|
135
|
+
An inactive entry with no recorded baseline records none (``None``), so
|
|
136
|
+
turning it on later still counts as new."""
|
|
137
|
+
recorded = str((prior or {}).get("defaults_baseline") or "").strip().lower()
|
|
138
|
+
if recorded in BASELINES:
|
|
139
|
+
return recorded
|
|
140
|
+
if prior and row_enabled(prior):
|
|
141
|
+
return BASELINE_LEGACY
|
|
142
|
+
return BASELINE_CURRENT if active else None
|
|
143
|
+
|
|
144
|
+
|
|
102
145
|
def _enabled_using(column: str) -> str:
|
|
103
146
|
"""``USING`` expression casting a TEXT ``enabled`` exactly as
|
|
104
147
|
:func:`as_bool` reads it: NULL stays NULL (on), the false spellings are
|
|
@@ -281,15 +324,17 @@ def provision(tx, stage: str, entries, *, now=None) -> list:
|
|
|
281
324
|
active = _entry_active(entry, stage)
|
|
282
325
|
enabled = as_bool(entry.get("enabled", True), default=False)
|
|
283
326
|
|
|
284
|
-
tx.table(SITE_TABLE)
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
327
|
+
site = tx.table(SITE_TABLE)
|
|
328
|
+
baseline = defaults_baseline(site.find({"name": name}), active)
|
|
329
|
+
row = {
|
|
330
|
+
"name": name,
|
|
331
|
+
"enabled": active,
|
|
332
|
+
"config": json.dumps(entry.get("config") or {}),
|
|
333
|
+
"sys_modified_by": SEEDED_BY,
|
|
334
|
+
}
|
|
335
|
+
if baseline:
|
|
336
|
+
row["defaults_baseline"] = baseline
|
|
337
|
+
site.merge(row, pk={"name": name})
|
|
293
338
|
if not active:
|
|
294
339
|
_set_schedules_enabled(tx, cap, False)
|
|
295
340
|
report.append((name, "disabled" if not enabled else f"not for {stage}"))
|
|
@@ -299,7 +344,7 @@ def provision(tx, stage: str, entries, *, now=None) -> list:
|
|
|
299
344
|
tx.table(table).ensure_schema(columns)
|
|
300
345
|
|
|
301
346
|
if cap.defaults:
|
|
302
|
-
_seed_flag_defaults(tx, stage, cap)
|
|
347
|
+
_seed_flag_defaults(tx, stage, cap, legacy=baseline == BASELINE_LEGACY)
|
|
303
348
|
|
|
304
349
|
for action, cron in cap.schedules:
|
|
305
350
|
_upsert_schedule(tx, cap, action, cron, now)
|
|
@@ -308,8 +353,13 @@ def provision(tx, stage: str, entries, *, now=None) -> list:
|
|
|
308
353
|
return report
|
|
309
354
|
|
|
310
355
|
|
|
311
|
-
def _seed_flag_defaults(tx, stage, cap):
|
|
312
|
-
"""Ensure the knob columns exist and fill only the ones that are unset
|
|
356
|
+
def _seed_flag_defaults(tx, stage, cap, *, legacy=False):
|
|
357
|
+
"""Ensure the knob columns exist and fill only the ones that are unset —
|
|
358
|
+
a stored value, whatever it is, is never changed. ``legacy`` (an
|
|
359
|
+
installation enabled before the defaults changed) fills from
|
|
360
|
+
``legacy_defaults`` where declared, so it keeps what it ran with. An
|
|
361
|
+
empty default declares the knob (its column) without seeding a value,
|
|
362
|
+
leaving the runtime's own fallback in charge.
|
|
313
363
|
|
|
314
364
|
The key column rides along: on a fresh site database this call is what
|
|
315
365
|
CREATES sys_feature_flags, and without ``applied_system`` the get() below
|
|
@@ -321,10 +371,13 @@ def _seed_flag_defaults(tx, stage, cap):
|
|
|
321
371
|
row = feature_flags.get(tx, stage)
|
|
322
372
|
if not row:
|
|
323
373
|
return # the app's own seed creates the stage row; nothing to fill yet
|
|
374
|
+
values = floor_defaults(cap, legacy=legacy)
|
|
324
375
|
missing = {
|
|
325
376
|
key: str(value)
|
|
326
|
-
for key, value in
|
|
327
|
-
if
|
|
377
|
+
for key, value in values.items()
|
|
378
|
+
if key in cap.defaults
|
|
379
|
+
and value not in (None, "")
|
|
380
|
+
and row.get(key) in (None, "")
|
|
328
381
|
}
|
|
329
382
|
if missing:
|
|
330
383
|
tx.table(FLAGS_TABLE).update(
|
|
@@ -51,7 +51,7 @@ from .base import (
|
|
|
51
51
|
policy_field,
|
|
52
52
|
resolve_config,
|
|
53
53
|
)
|
|
54
|
-
from .provisioning import SITE_TABLE, row_enabled
|
|
54
|
+
from .provisioning import SITE_TABLE, row_enabled, uses_legacy_defaults
|
|
55
55
|
|
|
56
56
|
log = logging.getLogger(__name__)
|
|
57
57
|
|
|
@@ -101,14 +101,21 @@ SCHEMA = {
|
|
|
101
101
|
}
|
|
102
102
|
CACHE_TTL_SECONDS = 60 # per-container cache of the switches + site row
|
|
103
103
|
|
|
104
|
+
# Every guard is OFF until switched on, so enabling the capability (for its
|
|
105
|
+
# pruning, say) changes no request by itself. Block logging is not a guard —
|
|
106
|
+
# it refuses nothing, it only audits what an enabled guard refused — so it
|
|
107
|
+
# stays on: a guard switched on later is audited without a second switch.
|
|
104
108
|
SWITCHES = {
|
|
105
109
|
"security_ip_denylist_enabled": "false",
|
|
106
110
|
"security_ip_allowlist_enabled": "false",
|
|
107
111
|
"security_geo_enabled": "false",
|
|
108
112
|
"security_rate_limit_enabled": "false",
|
|
109
|
-
"security_payload_limit_enabled": "
|
|
113
|
+
"security_payload_limit_enabled": "false",
|
|
110
114
|
"security_block_logging_enabled": "true",
|
|
111
115
|
}
|
|
116
|
+
# Earlier releases defaulted the payload cap ON: an installation that had the
|
|
117
|
+
# capability enabled then keeps it on (seeded where unset, and its floor).
|
|
118
|
+
LEGACY_SWITCHES = {"security_payload_limit_enabled": "true"}
|
|
112
119
|
KNOBS = {
|
|
113
120
|
"security_rate_limit_per_minute": "120",
|
|
114
121
|
"security_payload_max_bytes": "1000000",
|
|
@@ -282,7 +289,12 @@ def enforce(tx, aws_event, context, *, now=None):
|
|
|
282
289
|
if not row or not row_enabled(row):
|
|
283
290
|
return None
|
|
284
291
|
cap = Security_instance()
|
|
285
|
-
config = resolve_config(
|
|
292
|
+
config = resolve_config(
|
|
293
|
+
cap,
|
|
294
|
+
flags=_flags(tx),
|
|
295
|
+
row_config=_row_config(row),
|
|
296
|
+
legacy=uses_legacy_defaults(row),
|
|
297
|
+
)
|
|
286
298
|
facts = request_facts(aws_event)
|
|
287
299
|
now = now or datetime.datetime.now(datetime.timezone.utc)
|
|
288
300
|
|
|
@@ -503,6 +515,7 @@ class Security(Capability):
|
|
|
503
515
|
"this policy holds the lists."
|
|
504
516
|
)
|
|
505
517
|
defaults = {**SWITCHES, **KNOBS}
|
|
518
|
+
legacy_defaults = dict(LEGACY_SWITCHES)
|
|
506
519
|
policy = (
|
|
507
520
|
policy_field(
|
|
508
521
|
"ip_denylist",
|
|
@@ -559,6 +572,9 @@ class Security(Capability):
|
|
|
559
572
|
raise ValueError(f"security: unknown action {action!r}")
|
|
560
573
|
now = datetime.datetime.now(datetime.timezone.utc)
|
|
561
574
|
days = int(config.get("security_events_days") or 365)
|
|
575
|
+
if days < 1:
|
|
576
|
+
# "0" would delete the whole audit of refusals
|
|
577
|
+
raise ValueError(f"security: security_events_days must be >= 1, not {days}")
|
|
562
578
|
return {
|
|
563
579
|
"rate_buckets": tx.table(RATE_TABLE).purge(
|
|
564
580
|
{"<sys_created": now - datetime.timedelta(hours=1)}
|
|
@@ -9,6 +9,13 @@ switch and staleness window as per-stage feature flags on the Security page
|
|
|
9
9
|
key + class attribute, and a **reaper** — sessions were only ever marked
|
|
10
10
|
``ended_at``, never deleted, so ``app_session`` grew forever.
|
|
11
11
|
|
|
12
|
+
Enabling the capability reaps; it does NOT enforce. Enforcement is the
|
|
13
|
+
explicit ``single_session_enforcement`` flag: declared (its column exists, so
|
|
14
|
+
the Security page can switch it) but never seeded, so until someone sets it
|
|
15
|
+
the mixin's own fallback decides — the ``SingleSessionEnforcement`` SSM key,
|
|
16
|
+
then the handler's ``single_session_enforcement`` attribute (False by
|
|
17
|
+
default). A stored value, "true" included, is never changed by provisioning.
|
|
18
|
+
|
|
12
19
|
``mixins`` names the handler mixin by dotted path: attaching it is the app's
|
|
13
20
|
one-line job, and importing the handlers package here would initialise a
|
|
14
21
|
database engine at import time.
|
|
@@ -28,13 +35,16 @@ class SingleSession(Capability):
|
|
|
28
35
|
name = "single_session"
|
|
29
36
|
label = "Single session per user"
|
|
30
37
|
description = (
|
|
31
|
-
"
|
|
32
|
-
"
|
|
33
|
-
"
|
|
38
|
+
"Deletes sign-in sessions not seen for the retention window, and "
|
|
39
|
+
"offers one active login per user (a new sign-in ends the older "
|
|
40
|
+
"session) as an opt-in: enforcement is off until its flag is switched "
|
|
41
|
+
"on. Enforcement and the windows are per-environment flags; no "
|
|
34
42
|
"per-site policy."
|
|
35
43
|
)
|
|
36
44
|
defaults = {
|
|
37
|
-
|
|
45
|
+
# declared, never seeded: enforcement is an explicit opt-in (see the
|
|
46
|
+
# module docstring); "" leaves the mixin's SSM/attribute fallback
|
|
47
|
+
"single_session_enforcement": "",
|
|
38
48
|
"session_stale_minutes": "10",
|
|
39
49
|
"session_retention_days": "30",
|
|
40
50
|
}
|
|
@@ -56,12 +66,21 @@ class SingleSession(Capability):
|
|
|
56
66
|
|
|
57
67
|
def handle(self, action, tx, context, config):
|
|
58
68
|
"""`reap`: delete sessions not seen for ``session_retention_days`` —
|
|
59
|
-
ended ones and abandoned ones alike
|
|
60
|
-
|
|
69
|
+
ended ones and abandoned ones alike. Never a live one: the window is
|
|
70
|
+
at least a day and never shorter than ``session_stale_minutes`` (a
|
|
71
|
+
session quieter than that is already expired by the mixin), and a
|
|
72
|
+
row with no ``last_seen_at`` is never matched. Batched, committed per
|
|
73
|
+
batch (Table.purge)."""
|
|
61
74
|
if action != "reap":
|
|
62
75
|
raise ValueError(f"single_session: unknown action {action!r}")
|
|
63
76
|
days = int(config.get("session_retention_days") or 30)
|
|
64
|
-
|
|
65
|
-
|
|
77
|
+
if days < 1:
|
|
78
|
+
raise ValueError(
|
|
79
|
+
f"single_session: session_retention_days must be >= 1, not {days}"
|
|
80
|
+
)
|
|
81
|
+
stale = int(config.get("session_stale_minutes") or 10)
|
|
82
|
+
window = max(
|
|
83
|
+
datetime.timedelta(days=days), datetime.timedelta(minutes=max(stale, 0))
|
|
66
84
|
)
|
|
85
|
+
cutoff = datetime.datetime.now(datetime.timezone.utc) - window
|
|
67
86
|
return {"sessions": tx.table(SESSION_TABLE).purge({"<last_seen_at": cutoff})}
|
velocity/aws/job_results.py
CHANGED
|
@@ -38,6 +38,7 @@ one JSON message shape — ``{...extra, "percent", "detail", "result"?,
|
|
|
38
38
|
from __future__ import annotations
|
|
39
39
|
|
|
40
40
|
import base64
|
|
41
|
+
import inspect
|
|
41
42
|
import json
|
|
42
43
|
import re
|
|
43
44
|
from abc import ABC, abstractmethod
|
|
@@ -69,6 +70,7 @@ __all__ = [
|
|
|
69
70
|
"get_job_result_store",
|
|
70
71
|
"default_job_result_store",
|
|
71
72
|
"registered_job_result_stores",
|
|
73
|
+
"purge_expired_results",
|
|
72
74
|
"store_job_result",
|
|
73
75
|
"record_job_result",
|
|
74
76
|
"deliver_job_result",
|
|
@@ -111,6 +113,10 @@ JOB_RESULTS_INDEXES = (
|
|
|
111
113
|
),
|
|
112
114
|
)
|
|
113
115
|
|
|
116
|
+
# purge_expired's per-call bound (rows are large: a payload each)
|
|
117
|
+
PURGE_BATCH = 500
|
|
118
|
+
PURGE_MAX_BATCHES = 20
|
|
119
|
+
|
|
114
120
|
_SAFE_FILENAME = re.compile(r"[^A-Za-z0-9._ -]+")
|
|
115
121
|
|
|
116
122
|
|
|
@@ -270,10 +276,28 @@ class DbJobResultStore(JobResultStore):
|
|
|
270
276
|
}
|
|
271
277
|
)
|
|
272
278
|
|
|
273
|
-
def purge_expired(
|
|
274
|
-
|
|
279
|
+
def purge_expired(
|
|
280
|
+
self,
|
|
281
|
+
tx,
|
|
282
|
+
now: Optional[datetime] = None,
|
|
283
|
+
*,
|
|
284
|
+
batch: int = PURGE_BATCH,
|
|
285
|
+
max_batches: int = PURGE_MAX_BATCHES,
|
|
286
|
+
) -> int:
|
|
287
|
+
"""Delete rows past ``expires_at``; returns the number removed.
|
|
288
|
+
|
|
289
|
+
Bounded: at most ``batch * max_batches`` rows per call, in batches
|
|
290
|
+
committed one at a time (:meth:`Table.purge`), so a backlog drains
|
|
291
|
+
over several runs instead of one long transaction. A row without
|
|
292
|
+
``expires_at`` (a store with no retention) never expires. A site that
|
|
293
|
+
never stored a result has no table: nothing to do."""
|
|
275
294
|
now = now or datetime.now(timezone.utc)
|
|
276
|
-
|
|
295
|
+
table = tx.table(self.table)
|
|
296
|
+
if not table.exists():
|
|
297
|
+
return 0
|
|
298
|
+
return (
|
|
299
|
+
table.purge({"<expires_at": now}, batch=batch, max_batches=max_batches) or 0
|
|
300
|
+
)
|
|
277
301
|
|
|
278
302
|
|
|
279
303
|
class S3JobResultStore(JobResultStore):
|
|
@@ -408,6 +432,45 @@ def default_job_result_store() -> JobResultStore:
|
|
|
408
432
|
return get_job_result_store(None)
|
|
409
433
|
|
|
410
434
|
|
|
435
|
+
def purge_expired_results(
|
|
436
|
+
tx,
|
|
437
|
+
*,
|
|
438
|
+
now: Optional[datetime] = None,
|
|
439
|
+
batch: int = PURGE_BATCH,
|
|
440
|
+
max_batches: int = PURGE_MAX_BATCHES,
|
|
441
|
+
) -> int:
|
|
442
|
+
"""Run ``purge_expired`` on every registered store that has one (the
|
|
443
|
+
database store; an S3 store leaves expiry to the bucket's lifecycle),
|
|
444
|
+
once per table, each bounded by ``batch * max_batches`` rows. Returns the
|
|
445
|
+
number of rows removed. The ``jobs`` capability calls this on its
|
|
446
|
+
schedule."""
|
|
447
|
+
removed = 0
|
|
448
|
+
seen = set()
|
|
449
|
+
for store in registered_job_result_stores().values():
|
|
450
|
+
purge = getattr(store, "purge_expired", None)
|
|
451
|
+
if not callable(purge):
|
|
452
|
+
continue
|
|
453
|
+
key = getattr(store, "table", None) or id(store)
|
|
454
|
+
if key in seen:
|
|
455
|
+
continue
|
|
456
|
+
seen.add(key)
|
|
457
|
+
if _takes_bounds(purge):
|
|
458
|
+
removed += purge(tx, now, batch=batch, max_batches=max_batches) or 0
|
|
459
|
+
else: # an app store with the older purge_expired(tx, now) signature
|
|
460
|
+
removed += purge(tx, now) or 0
|
|
461
|
+
return removed
|
|
462
|
+
|
|
463
|
+
|
|
464
|
+
def _takes_bounds(fn) -> bool:
|
|
465
|
+
try:
|
|
466
|
+
params = inspect.signature(fn).parameters
|
|
467
|
+
except (TypeError, ValueError):
|
|
468
|
+
return False
|
|
469
|
+
if any(p.kind is inspect.Parameter.VAR_KEYWORD for p in params.values()):
|
|
470
|
+
return True
|
|
471
|
+
return "batch" in params and "max_batches" in params
|
|
472
|
+
|
|
473
|
+
|
|
411
474
|
# The database store needs nothing configured, so it is the out-of-the-box
|
|
412
475
|
# default; apps register an S3 store when they outgrow inline delivery.
|
|
413
476
|
register_job_result_store(DbJobResultStore(), default=True)
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
velocity/__init__.py,sha256=
|
|
1
|
+
velocity/__init__.py,sha256=YLVpBYtuey7iwf6G85d1ArvP0iviZwdpkATk6NqlMW4,362
|
|
2
2
|
velocity/cache.py,sha256=pthvw7gDVEXRzkTcQtA5YVf4Q1IxAamWsba4UReT0RE,8933
|
|
3
3
|
velocity/hooks.py,sha256=jY_qtAapNHvptmryFSD-Zc7RvLhV6e5nJEH3x2xTgCw,20575
|
|
4
4
|
velocity/logging.py,sha256=MAsZap1cDSdOBNRc_9ezQxup5dHOS0ARxU9NI_-UK3E,4037
|
|
@@ -13,7 +13,7 @@ velocity/aws/failures.py,sha256=8rXWAikJZTVv7bybmcXI5geR6F_dfMw6ns-icuj9EN0,3137
|
|
|
13
13
|
velocity/aws/feature_flags.py,sha256=G2S1Ifjjex1QEmaAnAwRISyawxfGTdzaKUh3150M5aw,9604
|
|
14
14
|
velocity/aws/identity.py,sha256=u6f_oaVhDa3tNatsQkeXZPJDhMfXCzeDo1zGoTqW_Yo,5547
|
|
15
15
|
velocity/aws/job_lifecycle.py,sha256=taxXEbq1WIg3Vbn9B8Ru0orgbk_J6KWBtAUTIptChWk,19567
|
|
16
|
-
velocity/aws/job_results.py,sha256=
|
|
16
|
+
velocity/aws/job_results.py,sha256=FXQlMazFvkLFiT1BJlHiQj8RiQG5p3bf_X1Zek9StIg,22456
|
|
17
17
|
velocity/aws/mirror.py,sha256=8Ik1_GVCcD7E2nCe1ci6sp56PEVBMYZF-alkMFlP4O4,8858
|
|
18
18
|
velocity/aws/ops.py,sha256=3gUpOAjuC4sgqoU3KIqGJf7cAvdEqonP7z7lPO25_wc,19061
|
|
19
19
|
velocity/aws/permissions.py,sha256=peAJHQEWpP9wsm0i-Wq15E55Ze5w9CwAGVA5mmrZPtA,74152
|
|
@@ -33,14 +33,14 @@ velocity/aws/assets/references.py,sha256=QcsA-sshcMRc9yZQYrtWJ_LAbp5s5g-nLCusFbE
|
|
|
33
33
|
velocity/aws/assets/service.py,sha256=Td2WnL0bL2CcoXDukhlMAWlXw027TXJ6EWgh7Jos1DU,8987
|
|
34
34
|
velocity/aws/assets/usage_index.py,sha256=Gx2nnr_e7jhlnfzevG7JhXG0tPPmqFLS1FBAEXO_5Qw,4066
|
|
35
35
|
velocity/aws/capabilities/__init__.py,sha256=esWyDyNsXyuATRFWtSOjXNfSkiozGFxWp79WhUzq84g,1883
|
|
36
|
-
velocity/aws/capabilities/base.py,sha256=
|
|
37
|
-
velocity/aws/capabilities/dispatch.py,sha256=
|
|
36
|
+
velocity/aws/capabilities/base.py,sha256=GcLVSx3H1jHic8DT9WrgKirhGplgUBTN2uLHrEarlaI,14691
|
|
37
|
+
velocity/aws/capabilities/dispatch.py,sha256=dqOSPw5IRgJR6yM93DasBN423sHhZcT6PScGj4GT6vc,4907
|
|
38
38
|
velocity/aws/capabilities/infra.py,sha256=nUorjg7KpGhLW9ShUGdyh6yAwSKVy7xCAKKXwaZ7j00,4139
|
|
39
|
-
velocity/aws/capabilities/jobs.py,sha256=
|
|
40
|
-
velocity/aws/capabilities/provisioning.py,sha256=
|
|
39
|
+
velocity/aws/capabilities/jobs.py,sha256=mf6sU7NG_j4Vzth6RARpazbUD7hx79FmTowXD0xunfc,5192
|
|
40
|
+
velocity/aws/capabilities/provisioning.py,sha256=4PF5QzQ_Pa51v5XN1OOn1juDxWqWPO3WI5fupqA9Agw,17844
|
|
41
41
|
velocity/aws/capabilities/retention.py,sha256=V-vSbfPdpm7w67kaXi3OkmqtqEYFDeDn6P8kCs_rEzs,3198
|
|
42
|
-
velocity/aws/capabilities/security.py,sha256=
|
|
43
|
-
velocity/aws/capabilities/single_session.py,sha256=
|
|
42
|
+
velocity/aws/capabilities/security.py,sha256=tuq1DhZ_q_kTUmpfsSyzXIAGcAzARmnuT3C0SiKmqSU,22721
|
|
43
|
+
velocity/aws/capabilities/single_session.py,sha256=UQhBc4hR-qSZfpBM_o8434Xpg6Bt3EbzI7ugZIxJh8Y,3798
|
|
44
44
|
velocity/aws/capabilities/workflow.py,sha256=P7k6n2oZVovToAcHJUQwLdkD7duS4wKwBQKXaAMAnnU,1875
|
|
45
45
|
velocity/aws/deploy/__init__.py,sha256=nOLW_NqxEv7i2xRSoc4wmawfAEo9z7oPrjxJagKZzaM,4832
|
|
46
46
|
velocity/aws/deploy/ci_gate.py,sha256=ZTwwQRLx_MU3eBLlwTlhmCcubJucj5-IqNSK6Zlq1vg,24771
|
|
@@ -251,9 +251,9 @@ velocity/workflow/outbox.py,sha256=R-DmdeVETYZc0u__OXxhKyeLgM1Cwq555-RqAFTW7h0,9
|
|
|
251
251
|
velocity/workflow/points.py,sha256=g_vQ3cdHdx_JloVCS5dVjo_jJoSolO92K5p3suDzxlE,10654
|
|
252
252
|
velocity/workflow/schema.py,sha256=1ZAkGZ_4YrTAfY6TIPCviWQJGLhhmAJmkxisfLyy2og,21597
|
|
253
253
|
velocity/workflow/views.py,sha256=oTPjpuheND8y4VbcTY8c-4fMdSHKb0jgGzXMuT_G7_s,32175
|
|
254
|
-
velocity_python-0.
|
|
255
|
-
velocity_python-0.
|
|
256
|
-
velocity_python-0.
|
|
257
|
-
velocity_python-0.
|
|
258
|
-
velocity_python-0.
|
|
259
|
-
velocity_python-0.
|
|
254
|
+
velocity_python-0.8.0.dist-info/licenses/LICENSE,sha256=aoN245GG8s9oRUU89KNiGTU4_4OtnNmVi4hQeChg6rM,1076
|
|
255
|
+
velocity_python-0.8.0.dist-info/METADATA,sha256=9OlHhpzs0BOzi8efbGzjiB4iom00wX2KxXpTnasBQ-8,17820
|
|
256
|
+
velocity_python-0.8.0.dist-info/WHEEL,sha256=mOdXR9C3wMVov7cSY8PY-XCzo3SmmWCHvGQr5csJoao,91
|
|
257
|
+
velocity_python-0.8.0.dist-info/entry_points.txt,sha256=nXUKJHkOGQCSPoagdmMNd_qycrle1LVU-GhYknLGM1M,56
|
|
258
|
+
velocity_python-0.8.0.dist-info/top_level.txt,sha256=JW2vJPmodgdgSz7H6yoZvnxF8S3fTMIv-YJWCT1sNW0,9
|
|
259
|
+
velocity_python-0.8.0.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|