velocity-python 0.2.9__py3-none-any.whl → 0.3.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- velocity/__init__.py +1 -1
- velocity/aws/capabilities/__init__.py +8 -0
- velocity/aws/capabilities/base.py +62 -0
- velocity/aws/capabilities/dispatch.py +75 -12
- velocity/aws/capabilities/jobs.py +41 -16
- velocity/aws/capabilities/provisioning.py +72 -5
- velocity/aws/capabilities/security.py +118 -43
- velocity/aws/dirty_pipeline.py +507 -4
- velocity/aws/handlers/context.py +315 -16
- velocity/aws/handlers/lambda_handler.py +12 -4
- velocity/aws/handlers/mixins/__init__.py +8 -0
- velocity/aws/handlers/mixins/audit_trail.py +288 -83
- velocity/aws/handlers/mixins/job_handler.py +2 -16
- velocity/aws/job_lifecycle.py +415 -0
- velocity/aws/scheduler.py +567 -0
- velocity/aws/schema.py +7 -0
- velocity/db/_pgddl.py +162 -0
- velocity/db/audit.py +793 -0
- velocity/db/core/engine.py +35 -9
- velocity/db/core/transaction.py +16 -2
- velocity/db/effective.py +408 -0
- velocity/db/exceptions.py +28 -0
- velocity/db/integrity.py +782 -0
- velocity/db/tests/postgres/test_integrity_triggers.py +514 -0
- velocity/hooks.py +526 -0
- velocity/rules/__init__.py +111 -0
- velocity/rules/engine.py +857 -0
- velocity/rules/expressions.py +451 -0
- velocity/rules/facts.py +331 -0
- velocity/rules/storage.py +391 -0
- {velocity_python-0.2.9.dist-info → velocity_python-0.3.1.dist-info}/METADATA +5 -2
- {velocity_python-0.2.9.dist-info → velocity_python-0.3.1.dist-info}/RECORD +36 -24
- {velocity_python-0.2.9.dist-info → velocity_python-0.3.1.dist-info}/WHEEL +0 -0
- {velocity_python-0.2.9.dist-info → velocity_python-0.3.1.dist-info}/entry_points.txt +0 -0
- {velocity_python-0.2.9.dist-info → velocity_python-0.3.1.dist-info}/licenses/LICENSE +0 -0
- {velocity_python-0.2.9.dist-info → velocity_python-0.3.1.dist-info}/top_level.txt +0 -0
velocity/__init__.py
CHANGED
|
@@ -13,10 +13,13 @@ from .base import ( # noqa: F401
|
|
|
13
13
|
POLICY_KINDS,
|
|
14
14
|
REGISTRY,
|
|
15
15
|
Capability,
|
|
16
|
+
CapabilityDependencyError,
|
|
16
17
|
capability,
|
|
17
18
|
catalog,
|
|
19
|
+
dependency_order,
|
|
18
20
|
describe,
|
|
19
21
|
get,
|
|
22
|
+
missing_dependencies,
|
|
20
23
|
load_config,
|
|
21
24
|
policy_field,
|
|
22
25
|
resolve_config,
|
|
@@ -30,6 +33,7 @@ from .dispatch import ( # noqa: F401
|
|
|
30
33
|
)
|
|
31
34
|
from .provisioning import ( # noqa: F401
|
|
32
35
|
as_bool,
|
|
36
|
+
check_dependencies,
|
|
33
37
|
provision,
|
|
34
38
|
row_enabled,
|
|
35
39
|
trigger_key,
|
|
@@ -47,15 +51,19 @@ from .security import SecurityMixin # noqa: E402,F401
|
|
|
47
51
|
__all__ = [
|
|
48
52
|
"Capability",
|
|
49
53
|
"CapabilityCatalogMixin",
|
|
54
|
+
"CapabilityDependencyError",
|
|
50
55
|
"CapabilityDispatcher",
|
|
51
56
|
"POLICY_KINDS",
|
|
52
57
|
"REGISTRY",
|
|
53
58
|
"as_bool",
|
|
54
59
|
"capability",
|
|
55
60
|
"catalog",
|
|
61
|
+
"check_dependencies",
|
|
62
|
+
"dependency_order",
|
|
56
63
|
"describe",
|
|
57
64
|
"get",
|
|
58
65
|
"load_config",
|
|
66
|
+
"missing_dependencies",
|
|
59
67
|
"policy_field",
|
|
60
68
|
"provision",
|
|
61
69
|
"resolve_config",
|
|
@@ -72,6 +72,11 @@ COUNTRY_CODE = re.compile(r"^[A-Z]{2}$")
|
|
|
72
72
|
REGISTRY: dict = {}
|
|
73
73
|
|
|
74
74
|
|
|
75
|
+
class CapabilityDependencyError(ValueError):
|
|
76
|
+
"""A capability is enabled without one it requires, requires an unknown
|
|
77
|
+
one, or the requirements form a cycle."""
|
|
78
|
+
|
|
79
|
+
|
|
75
80
|
def policy_field(name, kind, label, help="", **extra) -> dict:
|
|
76
81
|
"""One entry of a capability's ``policy`` schema. ``extra`` carries
|
|
77
82
|
kind-specific hints an editor may use (``options`` for a choice,
|
|
@@ -92,6 +97,7 @@ class Capability:
|
|
|
92
97
|
mixins: tuple = () # handler mixins the site attaches at runtime
|
|
93
98
|
applies_to: tuple = ("demo", "production") # default stages; registry can force
|
|
94
99
|
policy: tuple = () # (policy_field(...), ...) — the per-site config shape
|
|
100
|
+
requires_capabilities: tuple = () # names that must be enabled alongside
|
|
95
101
|
|
|
96
102
|
def handle(self, action, tx, context, config):
|
|
97
103
|
"""Do the work for one scheduled/queued ``action``. Return a JSON-able
|
|
@@ -119,6 +125,61 @@ def get(name):
|
|
|
119
125
|
raise KeyError(f"unknown capability {name!r}; registered: {known}") from None
|
|
120
126
|
|
|
121
127
|
|
|
128
|
+
def requirements(name) -> tuple:
|
|
129
|
+
"""The capabilities ``name`` declares it requires (unknown name: KeyError)."""
|
|
130
|
+
return tuple(getattr(get(name), "requires_capabilities", ()) or ())
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
def dependency_order(names) -> list:
|
|
134
|
+
"""``names`` plus everything they require (transitively), each after the
|
|
135
|
+
capabilities it requires; otherwise in the order given.
|
|
136
|
+
|
|
137
|
+
Raises :class:`CapabilityDependencyError` for a cycle (naming it) or a
|
|
138
|
+
requirement that is not a registered capability.
|
|
139
|
+
"""
|
|
140
|
+
ordered: list = []
|
|
141
|
+
done: set = set()
|
|
142
|
+
visiting: list = []
|
|
143
|
+
|
|
144
|
+
def visit(name, needed_by=None):
|
|
145
|
+
if name in done:
|
|
146
|
+
return
|
|
147
|
+
if name in visiting:
|
|
148
|
+
cycle = visiting[visiting.index(name) :] + [name]
|
|
149
|
+
raise CapabilityDependencyError(
|
|
150
|
+
"capability requirements form a cycle: " + " -> ".join(cycle)
|
|
151
|
+
)
|
|
152
|
+
if name not in REGISTRY:
|
|
153
|
+
if needed_by is None:
|
|
154
|
+
get(name) # the usual "unknown capability" KeyError
|
|
155
|
+
raise CapabilityDependencyError(
|
|
156
|
+
f"capability {needed_by!r} requires {name!r}, which is not registered"
|
|
157
|
+
)
|
|
158
|
+
visiting.append(name)
|
|
159
|
+
for required in requirements(name):
|
|
160
|
+
visit(required, name)
|
|
161
|
+
visiting.pop()
|
|
162
|
+
done.add(name)
|
|
163
|
+
ordered.append(name)
|
|
164
|
+
|
|
165
|
+
for name in names:
|
|
166
|
+
visit(name)
|
|
167
|
+
return ordered
|
|
168
|
+
|
|
169
|
+
|
|
170
|
+
def missing_dependencies(enabled) -> dict:
|
|
171
|
+
"""``{name: [required names not in enabled]}`` for each name in
|
|
172
|
+
``enabled`` that lacks one; empty when every requirement is met."""
|
|
173
|
+
enabled = list(enabled)
|
|
174
|
+
present = set(enabled)
|
|
175
|
+
missing = {}
|
|
176
|
+
for name in enabled:
|
|
177
|
+
absent = [req for req in requirements(name) if req not in present]
|
|
178
|
+
if absent:
|
|
179
|
+
missing[name] = absent
|
|
180
|
+
return missing
|
|
181
|
+
|
|
182
|
+
|
|
122
183
|
def resolve_config(cap, flags=None, row_config=None) -> dict:
|
|
123
184
|
"""defaults <- feature flags (declared knobs only) <- site registry config."""
|
|
124
185
|
config = dict(cap.defaults)
|
|
@@ -148,6 +209,7 @@ def describe(cap) -> dict:
|
|
|
148
209
|
],
|
|
149
210
|
"flags": list(cap.defaults),
|
|
150
211
|
"policy": [dict(field) for field in cap.policy],
|
|
212
|
+
"requires": list(cap.requires_capabilities),
|
|
151
213
|
}
|
|
152
214
|
|
|
153
215
|
|
|
@@ -4,34 +4,92 @@ A site's queue handler mixes in :class:`CapabilityDispatcher`; the scheduler
|
|
|
4
4
|
stubs seeded by :mod:`provision` enqueue ``run-capability`` with
|
|
5
5
|
``{"capability": name, "action": action}`` and this routes it. Config is
|
|
6
6
|
resolved locally — the site's own ``sys_capabilities`` row plus its feature
|
|
7
|
-
flags — so the runtime never reaches into the control plane.
|
|
7
|
+
flags — so the runtime never reaches into the control plane. A capability
|
|
8
|
+
runs only with an enabled row of its own and enabled rows for every
|
|
9
|
+
capability it requires (``requires_capabilities``, transitively); otherwise
|
|
10
|
+
it is skipped and the skip is reported (:mod:`velocity.aws.failures`).
|
|
8
11
|
"""
|
|
9
12
|
|
|
10
13
|
from __future__ import annotations
|
|
11
14
|
|
|
12
15
|
import json
|
|
13
16
|
|
|
14
|
-
from velocity.aws import feature_flags
|
|
17
|
+
from velocity.aws import failures, feature_flags
|
|
15
18
|
|
|
16
|
-
from .base import
|
|
19
|
+
from .base import (
|
|
20
|
+
REGISTRY,
|
|
21
|
+
CapabilityDependencyError,
|
|
22
|
+
catalog,
|
|
23
|
+
dependency_order,
|
|
24
|
+
resolve_config,
|
|
25
|
+
)
|
|
17
26
|
from .provisioning import SITE_TABLE, row_enabled
|
|
18
27
|
|
|
19
28
|
|
|
29
|
+
def _skip(tx, name, action, reason, *, report=True):
|
|
30
|
+
"""The skipped response; with ``report``, a failure event (scrubbed,
|
|
31
|
+
never raising) once the transaction commits — a skip that should not
|
|
32
|
+
happen must reach an operator, not just a queue response."""
|
|
33
|
+
if report:
|
|
34
|
+
|
|
35
|
+
def send():
|
|
36
|
+
failures.report(
|
|
37
|
+
None,
|
|
38
|
+
None,
|
|
39
|
+
source="queue",
|
|
40
|
+
action="run-capability",
|
|
41
|
+
extra={
|
|
42
|
+
"capability": name,
|
|
43
|
+
"capability_action": action,
|
|
44
|
+
"skipped": reason,
|
|
45
|
+
},
|
|
46
|
+
)
|
|
47
|
+
|
|
48
|
+
on_commit = getattr(tx, "on_commit", None)
|
|
49
|
+
if callable(on_commit):
|
|
50
|
+
on_commit(send)
|
|
51
|
+
else:
|
|
52
|
+
send()
|
|
53
|
+
return {"capability": name, "action": action, "skipped": reason}
|
|
54
|
+
|
|
55
|
+
|
|
20
56
|
def run_capability(tx, context, name, action):
|
|
21
|
-
"""Resolve config and run one capability action. Returns the response body.
|
|
57
|
+
"""Resolve config and run one capability action. Returns the response body.
|
|
58
|
+
|
|
59
|
+
Runs only when the site has an enabled ``sys_capabilities`` row for the
|
|
60
|
+
capability (provisioning wrote it) and an enabled row for every
|
|
61
|
+
capability it requires, directly or through another requirement
|
|
62
|
+
(:func:`dependency_order`). Otherwise the action is skipped, and — except
|
|
63
|
+
for a plain "disabled for this site" — reported through
|
|
64
|
+
:mod:`velocity.aws.failures`: a scheduled run for a capability that was
|
|
65
|
+
never provisioned, or whose requirement is off, is a misconfiguration.
|
|
66
|
+
"""
|
|
22
67
|
cap = REGISTRY.get(name)
|
|
23
68
|
if cap is None:
|
|
24
69
|
return {"error": f"unknown capability {name!r}", "capability": name}
|
|
25
70
|
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
71
|
+
site = tx.table(SITE_TABLE)
|
|
72
|
+
row = site.find({"name": name})
|
|
73
|
+
if not row:
|
|
74
|
+
return _skip(tx, name, action, "not provisioned for this site")
|
|
75
|
+
if not row_enabled(row):
|
|
76
|
+
return _skip(tx, name, action, "disabled for this site", report=False)
|
|
77
|
+
try:
|
|
78
|
+
# every requirement, transitively (a -> b -> c needs c on too)
|
|
79
|
+
chain = [n for n in dependency_order([name]) if n != name]
|
|
80
|
+
except (CapabilityDependencyError, KeyError) as exc:
|
|
81
|
+
return _skip(tx, name, action, f"dependency error: {exc}")
|
|
82
|
+
missing = []
|
|
83
|
+
for required in chain:
|
|
84
|
+
required_row = site.find({"name": required})
|
|
85
|
+
if not required_row or not row_enabled(required_row):
|
|
86
|
+
missing.append(required)
|
|
87
|
+
if missing:
|
|
88
|
+
return _skip(
|
|
89
|
+
tx, name, action, f"requires {', '.join(missing)}, not enabled here"
|
|
90
|
+
)
|
|
33
91
|
|
|
34
|
-
raw = row.get("config")
|
|
92
|
+
raw = row.get("config")
|
|
35
93
|
row_config = json.loads(raw) if isinstance(raw, str) and raw else (raw or {})
|
|
36
94
|
config = resolve_config(cap, flags=feature_flags.get(tx), row_config=row_config)
|
|
37
95
|
|
|
@@ -43,6 +101,11 @@ class CapabilityDispatcher:
|
|
|
43
101
|
"""Queue-handler mixin: ``run-capability`` -> the registered capability."""
|
|
44
102
|
|
|
45
103
|
def OnActionRunCapability(self, tx, context):
|
|
104
|
+
# Queue records and scheduled ticks only: a handler that mixes this in
|
|
105
|
+
# must not let an HTTP caller run a capability's actions.
|
|
106
|
+
from velocity.aws.dirty_pipeline import refuse_http
|
|
107
|
+
|
|
108
|
+
refuse_http(self, "run-capability")
|
|
46
109
|
payload = context.payload() or {}
|
|
47
110
|
body = run_capability(
|
|
48
111
|
tx, context, payload.get("capability"), payload.get("action")
|
|
@@ -3,9 +3,11 @@
|
|
|
3
3
|
``JobHandler`` only discovers a stale job (worker died: Lambda timeout / OOM)
|
|
4
4
|
when someone polls its status — an abandoned job nobody polls stays "Running"
|
|
5
5
|
forever, and ``aws_job_activity`` is never pruned. This capability sweeps on
|
|
6
|
-
a schedule with the
|
|
7
|
-
|
|
8
|
-
|
|
6
|
+
a schedule with the same rule the web tier uses (``Running``/``Initialized``
|
|
7
|
+
with no progress for the window), judged by the database clock
|
|
8
|
+
(:func:`stale_jobs`): stale jobs are persisted as Failed with the standard
|
|
9
|
+
message plus the status they expired from (``expired_from``), and old
|
|
10
|
+
activity rows are purged.
|
|
9
11
|
"""
|
|
10
12
|
|
|
11
13
|
from __future__ import annotations
|
|
@@ -17,7 +19,6 @@ from velocity.aws.job_lifecycle import (
|
|
|
17
19
|
JOB_ACTIVITY_TABLE,
|
|
18
20
|
STALE_FAILURE,
|
|
19
21
|
STALE_JOB_STATUSES,
|
|
20
|
-
job_is_stale,
|
|
21
22
|
)
|
|
22
23
|
|
|
23
24
|
from .base import EVERY_15_MINUTES, Capability, capability
|
|
@@ -39,27 +40,51 @@ class Jobs(Capability):
|
|
|
39
40
|
if action != "expire_stale":
|
|
40
41
|
raise ValueError(f"jobs: unknown action {action!r}")
|
|
41
42
|
now = datetime.datetime.now(datetime.timezone.utc)
|
|
42
|
-
|
|
43
|
-
minutes=int(config.get("job_stale_minutes") or 16)
|
|
44
|
-
)
|
|
43
|
+
minutes = int(config.get("job_stale_minutes") or 16)
|
|
45
44
|
jobs = tx.table(JOB_ACTIVITY_TABLE)
|
|
46
45
|
|
|
47
46
|
expired = 0
|
|
48
|
-
for
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
47
|
+
for job in stale_jobs(tx, minutes):
|
|
48
|
+
status = job.get("status")
|
|
49
|
+
# the status in the where: a job that progressed since the select
|
|
50
|
+
# (or that another sweep expired) is left alone
|
|
51
|
+
changed = jobs.update(
|
|
52
|
+
{
|
|
53
|
+
"status": "Failed",
|
|
54
|
+
"message": json.dumps({**STALE_FAILURE, "expired_from": status}),
|
|
55
|
+
},
|
|
56
|
+
where={"sys_id": job["sys_id"], "status": status},
|
|
57
|
+
)
|
|
58
|
+
if changed:
|
|
59
|
+
expired += 1
|
|
60
|
+
_report_after_commit(tx, job, status)
|
|
57
61
|
|
|
58
62
|
days = int(config.get("job_retention_days") or 30)
|
|
59
63
|
purged = jobs.purge({"<sys_created": now - datetime.timedelta(days=days)})
|
|
60
64
|
return {"expired": expired, "purged": purged}
|
|
61
65
|
|
|
62
66
|
|
|
67
|
+
def stale_jobs(tx, minutes):
|
|
68
|
+
"""``Running``/``Initialized`` jobs with no progress for ``minutes``,
|
|
69
|
+
judged by the database clock: ``sys_modified`` is written by the row
|
|
70
|
+
trigger in the session's time zone, so it is compared with
|
|
71
|
+
``LOCALTIMESTAMP`` there, never with a Python clock (which disagrees with
|
|
72
|
+
it whenever the database does not run in UTC)."""
|
|
73
|
+
statuses = sorted(STALE_JOB_STATUSES)
|
|
74
|
+
placeholders = ", ".join(["%s"] * len(statuses))
|
|
75
|
+
return (
|
|
76
|
+
tx.execute(
|
|
77
|
+
f"SELECT sys_id, job_id, action, status FROM {JOB_ACTIVITY_TABLE} "
|
|
78
|
+
f"WHERE status IN ({placeholders}) "
|
|
79
|
+
"AND sys_modified < LOCALTIMESTAMP - %s::interval "
|
|
80
|
+
"ORDER BY sys_id",
|
|
81
|
+
[*statuses, f"{int(minutes)} minutes"],
|
|
82
|
+
)
|
|
83
|
+
.as_dict()
|
|
84
|
+
.all()
|
|
85
|
+
)
|
|
86
|
+
|
|
87
|
+
|
|
63
88
|
def _report_after_commit(tx, job, status):
|
|
64
89
|
"""A job whose worker died never reached onError, so nothing reported it:
|
|
65
90
|
publish a ``source="job"`` failure event once the sweep commits."""
|
|
@@ -12,6 +12,12 @@ table — the runtime reads config from there, never from the control plane —
|
|
|
12
12
|
only what is unset, never clobbering an edit), and (4) upserts one
|
|
13
13
|
``sys_task_triggers`` row + one ``program_objects`` stub per schedule.
|
|
14
14
|
|
|
15
|
+
Before anything is written, every capability this run leaves enabled must
|
|
16
|
+
have the capabilities it requires (``requires_capabilities``) enabled too —
|
|
17
|
+
in ``entries``, or (when ``entries`` does not name them) already enabled in
|
|
18
|
+
the site's ``sys_capabilities`` — else :class:`CapabilityDependencyError`.
|
|
19
|
+
Entries are then provisioned in dependency order (required ones first).
|
|
20
|
+
|
|
15
21
|
Everything is keyed deterministically (``cap:<name>:<action>``), so running it
|
|
16
22
|
on every deploy is idempotent — and that is the point: a demo refresh wipes
|
|
17
23
|
the triggers, the next deploy puts them back. A disabled entry disables its
|
|
@@ -25,7 +31,13 @@ import json
|
|
|
25
31
|
|
|
26
32
|
from velocity.aws import feature_flags
|
|
27
33
|
|
|
28
|
-
from .base import
|
|
34
|
+
from .base import (
|
|
35
|
+
REGISTRY,
|
|
36
|
+
CapabilityDependencyError,
|
|
37
|
+
dependency_order,
|
|
38
|
+
get,
|
|
39
|
+
requirements,
|
|
40
|
+
)
|
|
29
41
|
|
|
30
42
|
SITE_TABLE = "sys_capabilities" # per-site materialised registry
|
|
31
43
|
TRIGGERS_TABLE = "sys_task_triggers"
|
|
@@ -202,17 +214,72 @@ def program_source(cap_name: str, action: str) -> str:
|
|
|
202
214
|
)
|
|
203
215
|
|
|
204
216
|
|
|
217
|
+
def _entry_active(entry, stage) -> bool:
|
|
218
|
+
cap = get(entry["name"])
|
|
219
|
+
enabled = as_bool(entry.get("enabled", True), default=False)
|
|
220
|
+
return enabled and (stage in cap.applies_to or bool(entry.get("force")))
|
|
221
|
+
|
|
222
|
+
|
|
223
|
+
def check_dependencies(tx, stage: str, entries) -> list:
|
|
224
|
+
"""The entries in dependency order, after checking that every entry this
|
|
225
|
+
run leaves active has what it requires active too. A requirement
|
|
226
|
+
``entries`` does not mention is looked up in the site's own
|
|
227
|
+
``sys_capabilities`` (an earlier provision); one ``entries`` disables or
|
|
228
|
+
keeps out of ``stage`` is missing. Raises
|
|
229
|
+
:class:`CapabilityDependencyError` listing every gap, before any write."""
|
|
230
|
+
entries = list(entries or [])
|
|
231
|
+
by_name = {entry["name"]: entry for entry in entries}
|
|
232
|
+
active = {name for name, entry in by_name.items() if _entry_active(entry, stage)}
|
|
233
|
+
order = dependency_order(list(by_name)) # unknown names / cycles raise here
|
|
234
|
+
site = tx.table(SITE_TABLE)
|
|
235
|
+
problems = []
|
|
236
|
+
for name in order:
|
|
237
|
+
if name not in active:
|
|
238
|
+
continue
|
|
239
|
+
for required in requirements(name):
|
|
240
|
+
if required in active:
|
|
241
|
+
continue
|
|
242
|
+
if required in by_name:
|
|
243
|
+
entry = by_name[required]
|
|
244
|
+
on = as_bool(entry.get("enabled", True), default=False)
|
|
245
|
+
why = f"not for {stage}" if on else "disabled"
|
|
246
|
+
else:
|
|
247
|
+
row = site.find({"name": required})
|
|
248
|
+
if row and row_enabled(row):
|
|
249
|
+
continue
|
|
250
|
+
why = "not enabled for this site"
|
|
251
|
+
problems.append(f"{name} requires {required} ({why})")
|
|
252
|
+
# turning one off must not strand a capability left on from before
|
|
253
|
+
for name in by_name:
|
|
254
|
+
if name in active:
|
|
255
|
+
continue
|
|
256
|
+
for other, cap in sorted(REGISTRY.items()):
|
|
257
|
+
if other in by_name or name not in (cap.requires_capabilities or ()):
|
|
258
|
+
continue
|
|
259
|
+
row = site.find({"name": other})
|
|
260
|
+
if row and row_enabled(row):
|
|
261
|
+
problems.append(
|
|
262
|
+
f"{other} requires {name}, which this run turns off "
|
|
263
|
+
f"(disable {other} too, or keep {name})"
|
|
264
|
+
)
|
|
265
|
+
if problems:
|
|
266
|
+
raise CapabilityDependencyError(
|
|
267
|
+
"capability dependencies not met: " + "; ".join(problems)
|
|
268
|
+
)
|
|
269
|
+
return [by_name[name] for name in order if name in by_name]
|
|
270
|
+
|
|
271
|
+
|
|
205
272
|
def provision(tx, stage: str, entries, *, now=None) -> list:
|
|
206
|
-
"""Materialise ``entries`` for ``stage``. Returns [(name, outcome), ...]
|
|
273
|
+
"""Materialise ``entries`` for ``stage``. Returns [(name, outcome), ...]
|
|
274
|
+
in dependency order (required capabilities first)."""
|
|
207
275
|
now = now or datetime.datetime.utcnow()
|
|
208
276
|
tx.table(SITE_TABLE).ensure_schema(**SCHEMA[SITE_TABLE])
|
|
209
277
|
report = []
|
|
210
|
-
for entry in
|
|
278
|
+
for entry in check_dependencies(tx, stage, entries):
|
|
211
279
|
name = entry["name"]
|
|
212
280
|
cap = get(name)
|
|
281
|
+
active = _entry_active(entry, stage)
|
|
213
282
|
enabled = as_bool(entry.get("enabled", True), default=False)
|
|
214
|
-
applies = stage in cap.applies_to or bool(entry.get("force"))
|
|
215
|
-
active = enabled and applies
|
|
216
283
|
|
|
217
284
|
tx.table(SITE_TABLE).merge(
|
|
218
285
|
{
|
|
@@ -19,7 +19,8 @@ rate limits, payload caps, and an audit trail of what was refused and why.
|
|
|
19
19
|
Attach :class:`SecurityMixin` FIRST in a handler's bases; it runs in
|
|
20
20
|
``beforeAction`` ahead of auth and every action, and is a no-op unless the
|
|
21
21
|
site's ``sys_capabilities`` row for ``security`` is enabled. A refusal raises
|
|
22
|
-
:class:`AccessError` (a 403) after
|
|
22
|
+
:class:`AccessError` (a 403) after recording a ``sys_security_events`` row
|
|
23
|
+
(committed on its own transaction, so the 403's rollback keeps it).
|
|
23
24
|
|
|
24
25
|
``sys_security_events`` itself is a CORE table (``SCHEMA``, in
|
|
25
26
|
``velocity.aws.schema``'s ``core`` group), not the capability's: it is
|
|
@@ -39,6 +40,7 @@ import logging
|
|
|
39
40
|
|
|
40
41
|
from velocity.aws import feature_flags
|
|
41
42
|
from velocity.cache import memo, read_cache
|
|
43
|
+
from velocity.db import _pgddl, audit
|
|
42
44
|
|
|
43
45
|
from .base import (
|
|
44
46
|
CIDR_LIST,
|
|
@@ -314,7 +316,13 @@ def enforce(tx, aws_event, context, *, now=None):
|
|
|
314
316
|
guard, reason = refusal
|
|
315
317
|
if truthy(config.get("security_block_logging_enabled")):
|
|
316
318
|
record_event(
|
|
317
|
-
tx,
|
|
319
|
+
tx,
|
|
320
|
+
guard,
|
|
321
|
+
reason,
|
|
322
|
+
aws_event=aws_event,
|
|
323
|
+
context=context,
|
|
324
|
+
user_email=user,
|
|
325
|
+
actor=GUARD_AUDIT_ACTOR,
|
|
318
326
|
)
|
|
319
327
|
log.warning("security guard %s refused %s: %s", guard, facts["ip"], reason)
|
|
320
328
|
# imported here, not at module top: the capabilities package must not
|
|
@@ -326,6 +334,15 @@ def enforce(tx, aws_event, context, *, now=None):
|
|
|
326
334
|
return config
|
|
327
335
|
|
|
328
336
|
|
|
337
|
+
# A second connection waiting on a lock the caller's own transaction holds
|
|
338
|
+
# would wait forever (the database can't see it as a deadlock): bound it, so
|
|
339
|
+
# the event fails (and an audited decision is refused) instead of hanging.
|
|
340
|
+
EVENT_LOCK_TIMEOUT = "5s"
|
|
341
|
+
# The audit actor of a guard refusal: it is recorded in beforeAction, before
|
|
342
|
+
# any request actor is set.
|
|
343
|
+
GUARD_AUDIT_ACTOR = "system:security_guard"
|
|
344
|
+
|
|
345
|
+
|
|
329
346
|
def record_event(
|
|
330
347
|
tx,
|
|
331
348
|
guard,
|
|
@@ -337,58 +354,116 @@ def record_event(
|
|
|
337
354
|
user_sub=None,
|
|
338
355
|
detail=None,
|
|
339
356
|
occurred_at=None,
|
|
357
|
+
actor=None,
|
|
340
358
|
):
|
|
341
|
-
"""Append one ``sys_security_events`` row
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
359
|
+
"""Append one ``sys_security_events`` row on its own short transaction.
|
|
360
|
+
The one audit sink for the framework: the request guards, masquerade
|
|
361
|
+
grants, sign-ins, privileged decisions (``authz.require``), and anything
|
|
362
|
+
else security-relevant write here. ``user_sub`` is the identity provider's
|
|
363
|
+
subject; ``detail`` a JSON-able dict of facts that have no column;
|
|
364
|
+
``occurred_at`` when it happened, if not now (an event relayed through a
|
|
365
|
+
queue). Silent when the table is not provisioned in this site, and an
|
|
366
|
+
audit failure never breaks the request. Returns True only once the row is
|
|
367
|
+
committed.
|
|
368
|
+
|
|
369
|
+
``tx`` is the caller's transaction and is only used for its engine: the
|
|
370
|
+
row is written and committed through ``tx.engine.side_transaction()``
|
|
371
|
+
(another connection, which never waits for the pool), so it survives whatever the caller does next (raise,
|
|
372
|
+
roll back) and the caller's transaction is left exactly as it was: its
|
|
373
|
+
writes stay uncommitted, and its ``SET LOCAL`` settings (the audit actor),
|
|
374
|
+
row locks and transaction-level advisory locks are kept. A ``tx`` with no
|
|
375
|
+
engine (a test double) gets the row written on itself, uncommitted. On
|
|
376
|
+
PostgreSQL the event transaction's audit actor (``velocity.actor``, for a
|
|
377
|
+
``sys_security_events`` under the audit trigger's ``require_actor``) is
|
|
378
|
+
``actor`` when given, else the one the caller set with
|
|
379
|
+
:func:`velocity.db.audit.set_audit_actor`. Callers that record before any
|
|
380
|
+
actor is set (the request guards, the masquerade grant) pass their own.
|
|
350
381
|
|
|
351
382
|
A table created before ``user_sub``/``detail`` existed (a site whose schema
|
|
352
383
|
lane has not run since) still records the row without them: on a locked
|
|
353
384
|
schema an unknown column would otherwise lose the whole event."""
|
|
385
|
+
facts = dict(
|
|
386
|
+
aws_event=aws_event,
|
|
387
|
+
context=context,
|
|
388
|
+
user_email=user_email,
|
|
389
|
+
user_sub=user_sub,
|
|
390
|
+
detail=detail,
|
|
391
|
+
occurred_at=occurred_at,
|
|
392
|
+
)
|
|
354
393
|
try:
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
try:
|
|
377
|
-
have = {c.lower() for c in table.sys_columns()}
|
|
378
|
-
except Exception:
|
|
379
|
-
# can't tell which newer columns exist: the event still lands
|
|
380
|
-
# with the columns every version of the table has
|
|
381
|
-
log.warning("security event %s: columns unknown", guard, exc_info=True)
|
|
382
|
-
have = set()
|
|
383
|
-
row.update({k: v for k, v in extra.items() if k in have})
|
|
384
|
-
table.insert(row)
|
|
385
|
-
tx.commit()
|
|
386
|
-
return True
|
|
394
|
+
open_transaction = getattr(
|
|
395
|
+
getattr(tx, "engine", None), "side_transaction", None
|
|
396
|
+
)
|
|
397
|
+
if not callable(open_transaction):
|
|
398
|
+
return _write_event(tx, guard, reason, **facts)
|
|
399
|
+
postgres = "postgres" in str(_pgddl.server_name(tx) or "").lower()
|
|
400
|
+
if postgres and not actor:
|
|
401
|
+
actor = _caller_actor(tx)
|
|
402
|
+
# the row is committed when the block exits; a commit failure raises
|
|
403
|
+
# out of it, so True is never returned for a row that isn't durable
|
|
404
|
+
with open_transaction() as event_tx:
|
|
405
|
+
if postgres:
|
|
406
|
+
# one round trip: the bounded lock wait, and the caller's audit
|
|
407
|
+
# actor (a sys_security_events under the audit trigger's
|
|
408
|
+
# require_actor still records)
|
|
409
|
+
event_tx.execute(
|
|
410
|
+
"SELECT set_config('lock_timeout', %s, true), "
|
|
411
|
+
"set_config(%s, %s, true)",
|
|
412
|
+
(EVENT_LOCK_TIMEOUT, audit.ACTOR_SETTING, actor or ""),
|
|
413
|
+
)
|
|
414
|
+
return _write_event(event_tx, guard, reason, **facts)
|
|
387
415
|
except Exception:
|
|
388
416
|
log.warning("security event %s not recorded", guard, exc_info=True)
|
|
389
417
|
return False
|
|
390
418
|
|
|
391
419
|
|
|
420
|
+
def _caller_actor(tx):
|
|
421
|
+
"""The audit actor :func:`velocity.db.audit.set_audit_actor` set on the
|
|
422
|
+
caller's transaction, or None. Read from the transaction object only:
|
|
423
|
+
running a statement on the caller's connection here could hit a dead
|
|
424
|
+
connection, and swallowing that error would lose the caller's
|
|
425
|
+
mid-transaction failure (R31) — its next write would land on a fresh
|
|
426
|
+
connection."""
|
|
427
|
+
value = getattr(tx, "_audit_actor", None)
|
|
428
|
+
return value if isinstance(value, str) and value else None
|
|
429
|
+
|
|
430
|
+
|
|
431
|
+
def _write_event(
|
|
432
|
+
tx, guard, reason, *, aws_event, context, user_email, user_sub, detail, occurred_at
|
|
433
|
+
):
|
|
434
|
+
table = tx.table(EVENTS_TABLE)
|
|
435
|
+
if not table.exists():
|
|
436
|
+
return False
|
|
437
|
+
facts = request_facts(aws_event)
|
|
438
|
+
row = {
|
|
439
|
+
"guard": guard,
|
|
440
|
+
"reason": reason,
|
|
441
|
+
"ip": facts["ip"],
|
|
442
|
+
"country": facts["country"],
|
|
443
|
+
"path": facts["path"],
|
|
444
|
+
"action": _action(context) if context is not None else "",
|
|
445
|
+
"user_email": user_email,
|
|
446
|
+
"user_agent": facts["user_agent"][:512],
|
|
447
|
+
}
|
|
448
|
+
extra = {
|
|
449
|
+
"user_sub": user_sub,
|
|
450
|
+
"detail": detail,
|
|
451
|
+
"occurred_at": occurred_at or datetime.datetime.now(datetime.timezone.utc),
|
|
452
|
+
}
|
|
453
|
+
extra = {k: v for k, v in extra.items() if v is not None}
|
|
454
|
+
if extra:
|
|
455
|
+
try:
|
|
456
|
+
have = {c.lower() for c in table.sys_columns()}
|
|
457
|
+
except Exception:
|
|
458
|
+
# can't tell which newer columns exist: the event still lands
|
|
459
|
+
# with the columns every version of the table has
|
|
460
|
+
log.warning("security event %s: columns unknown", guard, exc_info=True)
|
|
461
|
+
have = set()
|
|
462
|
+
row.update({k: v for k, v in extra.items() if k in have})
|
|
463
|
+
table.insert(row)
|
|
464
|
+
return True
|
|
465
|
+
|
|
466
|
+
|
|
392
467
|
def _user_email(context):
|
|
393
468
|
try:
|
|
394
469
|
session = context.session() or {}
|