keelrun 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- keel/__init__.py +26 -0
- keel/__main__.py +7 -0
- keel/_backend.py +149 -0
- keel/_defaults.py +116 -0
- keel/_discovery.py +368 -0
- keel/_errors.py +31 -0
- keel/_flow.py +327 -0
- keel/_hook.py +156 -0
- keel/_policy.py +136 -0
- keel/_record.py +266 -0
- keel/_run.py +131 -0
- keel/_runtime.py +62 -0
- keel/_sim.py +268 -0
- keel/_targets.py +214 -0
- keel/_wrap.py +157 -0
- keel/adapters/__init__.py +178 -0
- keel/adapters/_http.py +527 -0
- keel/adapters/_llm_policy.py +290 -0
- keel/adapters/_pack.py +53 -0
- keel/adapters/aiohttp_pack.py +437 -0
- keel/adapters/boto3_pack.py +360 -0
- keel/adapters/httpx_pack.py +653 -0
- keel/adapters/psycopg_pack.py +317 -0
- keel/adapters/requests_pack.py +390 -0
- keel/adapters/urllib3_pack.py +342 -0
- keel/bootstrap.py +210 -0
- keel/packs/__init__.py +111 -0
- keel/packs/_framework.py +66 -0
- keel/packs/_provider.py +95 -0
- keel/packs/adk_pack.py +356 -0
- keel/packs/anthropic_pack.py +41 -0
- keel/packs/crewai_pack.py +243 -0
- keel/packs/google_genai_pack.py +76 -0
- keel/packs/langgraph_pack.py +590 -0
- keel/packs/llm.py +145 -0
- keel/packs/mcp_pack.py +485 -0
- keel/packs/openai_agents_pack.py +208 -0
- keel/packs/openai_pack.py +41 -0
- keel/packs/pydantic_ai_pack.py +192 -0
- keel/packs/tool.py +307 -0
- keel/testing.py +181 -0
- keel_core_stub/__init__.py +744 -0
- keelrun-0.1.0.dist-info/METADATA +10 -0
- keelrun-0.1.0.dist-info/RECORD +47 -0
- keelrun-0.1.0.dist-info/WHEEL +5 -0
- keelrun-0.1.0.dist-info/entry_points.txt +2 -0
- keelrun-0.1.0.dist-info/top_level.txt +2 -0
keel/__init__.py
ADDED
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
"""Keel — the Python front end (Tier 1: resilience, zero code changes).
|
|
2
|
+
|
|
3
|
+
`keel run app.py` installs, before user code loads: (1) a `sys.meta_path`
|
|
4
|
+
import hook that wraps functions matching `py:` policy targets, and (2)
|
|
5
|
+
per-call discovery recording to `.keel/discovery.db`. The resilience
|
|
6
|
+
semantics live in the backend (`keel_core` native module when importable,
|
|
7
|
+
else the in-repo pure-Python `keel_core_stub`); this package is the thin,
|
|
8
|
+
stdlib-only front end that drives it.
|
|
9
|
+
|
|
10
|
+
DX invariants (docs/dx-spec.md §3) this package must uphold:
|
|
11
|
+
1. zero code changes in user code;
|
|
12
|
+
2. uninstall = remove the package (no lock-in): see `uninstall_keel`;
|
|
13
|
+
4. silence on success (one stderr banner at startup, then quiet);
|
|
14
|
+
5. never swallow errors — the original exception propagates unchanged,
|
|
15
|
+
with a `keel_outcome` attachment for those who look;
|
|
16
|
+
and KEEL_DISABLE=1 makes a run byte-identical to one without Keel.
|
|
17
|
+
|
|
18
|
+
The public entry points are `keel.bootstrap.install_keel` /
|
|
19
|
+
`uninstall_keel` and the `keel._run` runner used by `python -m keel run`.
|
|
20
|
+
"""
|
|
21
|
+
|
|
22
|
+
from __future__ import annotations
|
|
23
|
+
|
|
24
|
+
__all__ = ["__version__"]
|
|
25
|
+
|
|
26
|
+
__version__ = "0.1.0"
|
keel/__main__.py
ADDED
keel/_backend.py
ADDED
|
@@ -0,0 +1,149 @@
|
|
|
1
|
+
"""Backend selection, isolated behind one seam (architecture §5.1, "the swap").
|
|
2
|
+
|
|
3
|
+
A backend exposes `configure(policy)`, `execute(request, effect)`, and
|
|
4
|
+
`report()` — the four-operation core surface the stub defines (advance_clock
|
|
5
|
+
is test-only and unused here). Selection:
|
|
6
|
+
|
|
7
|
+
* native (`keel_core`) — the eventual PyO3 module (Task 6/14); probed by
|
|
8
|
+
import and required to expose `configure`/`execute`. It may not exist
|
|
9
|
+
yet, so a failed import is normal.
|
|
10
|
+
* stub (`keel_core_stub.KeelCoreStub`) — the in-repo pure-Python core.
|
|
11
|
+
|
|
12
|
+
`KEEL_BACKEND` overrides selection:
|
|
13
|
+
* `stub` → force the stub, never probe native
|
|
14
|
+
* `native` → require `keel_core`; raise KEEL-E040 if not loadable
|
|
15
|
+
* `auto` / unset → native if loadable, else the stub
|
|
16
|
+
"""
|
|
17
|
+
|
|
18
|
+
from __future__ import annotations
|
|
19
|
+
|
|
20
|
+
import importlib
|
|
21
|
+
import os
|
|
22
|
+
from pathlib import Path
|
|
23
|
+
from typing import Any, Mapping, Protocol, runtime_checkable
|
|
24
|
+
|
|
25
|
+
from ._errors import KeelError
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
@runtime_checkable
|
|
29
|
+
class Backend(Protocol):
|
|
30
|
+
def configure(self, policy: dict[str, Any]) -> None: ...
|
|
31
|
+
|
|
32
|
+
def execute(self, request: dict[str, Any], effect: Any) -> dict[str, Any]: ...
|
|
33
|
+
|
|
34
|
+
def report(self) -> dict[str, Any]: ...
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
class _NativeBackend:
|
|
38
|
+
"""Front-end adapter over the native ``keel_core`` core, adding one thing the
|
|
39
|
+
PyO3 surface does not expose: ``layer(target, key)``, resolved from the
|
|
40
|
+
configured policy exactly as the stub's ``_layer`` / Node's
|
|
41
|
+
``NativeBackend.layer`` do. The adapter packs read ``backend.layer`` to honor
|
|
42
|
+
``idempotency.header`` and gate cache-body buffering; without this the knob is
|
|
43
|
+
dead under native (a Python/Node parity break). Every other attribute
|
|
44
|
+
(``execute``/``execute_async``/``report``/``persistent``/the flow surface)
|
|
45
|
+
delegates straight through, so the swap is transparent."""
|
|
46
|
+
|
|
47
|
+
def __init__(self, core: Any) -> None:
|
|
48
|
+
self._core = core
|
|
49
|
+
self._policy: dict[str, Any] = {}
|
|
50
|
+
|
|
51
|
+
def configure(self, policy: dict[str, Any]) -> None:
|
|
52
|
+
self._policy = policy if isinstance(policy, dict) else {}
|
|
53
|
+
self._core.configure(policy)
|
|
54
|
+
|
|
55
|
+
def layer(self, target: str, key: str) -> Any:
|
|
56
|
+
t = self._policy.get("target")
|
|
57
|
+
if isinstance(t, dict) and isinstance(t.get(target), dict) and key in t[target]:
|
|
58
|
+
return t[target][key]
|
|
59
|
+
defaults = self._policy.get("defaults")
|
|
60
|
+
if not isinstance(defaults, dict):
|
|
61
|
+
return None
|
|
62
|
+
if target.startswith("llm:"):
|
|
63
|
+
llm = defaults.get("llm")
|
|
64
|
+
if isinstance(llm, dict) and key in llm:
|
|
65
|
+
return llm[key]
|
|
66
|
+
outbound = defaults.get("outbound")
|
|
67
|
+
return outbound.get(key) if isinstance(outbound, dict) else None
|
|
68
|
+
|
|
69
|
+
def __getattr__(self, name: str) -> Any:
|
|
70
|
+
# Delegate everything else to the native core (execute, execute_async,
|
|
71
|
+
# report, persistent, enter_flow/exit_flow/journal_*, advance_clock, …).
|
|
72
|
+
return getattr(self._core, name)
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def _journal_path(cwd: str | Path | None, env: Mapping[str, str]) -> str | None:
|
|
76
|
+
"""Where the native core attaches its journal (persistent dev cache + Tier 2).
|
|
77
|
+
`KEEL_JOURNAL` overrides the path; an explicit empty value disables it. This
|
|
78
|
+
is the *construction-time* default: keel.toml's `journal` key replaces it at
|
|
79
|
+
configure time unless KEEL_JOURNAL is set, in which case the env wins
|
|
80
|
+
(see ``bootstrap.apply_journal_env_override``)."""
|
|
81
|
+
override = env.get("KEEL_JOURNAL")
|
|
82
|
+
if override is not None:
|
|
83
|
+
return override or None # empty string ⇒ no journal
|
|
84
|
+
base = Path(cwd) if cwd is not None else Path.cwd()
|
|
85
|
+
return str(base / ".keel" / "journal.db")
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def _try_load_native(journal_path: str | None) -> Backend | None:
|
|
89
|
+
try:
|
|
90
|
+
mod = importlib.import_module("keel_core")
|
|
91
|
+
except ImportError:
|
|
92
|
+
return None
|
|
93
|
+
ctor = getattr(mod, "KeelCore", None) or getattr(mod, "Core", None)
|
|
94
|
+
if ctor is None:
|
|
95
|
+
return None
|
|
96
|
+
inst = None
|
|
97
|
+
if journal_path:
|
|
98
|
+
try:
|
|
99
|
+
inst = ctor(journal_path=journal_path) # attaches the persistent journal
|
|
100
|
+
except Exception:
|
|
101
|
+
inst = None # journal open failed — fall back to an in-memory native core
|
|
102
|
+
if inst is None:
|
|
103
|
+
inst = ctor()
|
|
104
|
+
if callable(getattr(inst, "configure", None)) and callable(getattr(inst, "execute", None)):
|
|
105
|
+
# The native KeelCore exposes configure/execute/execute_async/report/
|
|
106
|
+
# persistent; wrap it only to add `layer(target, key)` (idempotency.header
|
|
107
|
+
# + cache-ttl gate parity), delegating everything else.
|
|
108
|
+
return _NativeBackend(inst) # type: ignore[return-value]
|
|
109
|
+
return None
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
def load_backend(
|
|
113
|
+
preferred: str | None = None,
|
|
114
|
+
*,
|
|
115
|
+
cwd: str | Path | None = None,
|
|
116
|
+
env: Mapping[str, str] | None = None,
|
|
117
|
+
) -> Backend:
|
|
118
|
+
"""Resolve the runtime backend per `KEEL_BACKEND` (or `preferred`). Under a
|
|
119
|
+
native backend, a journal is attached at `<cwd>/.keel/journal.db` (created on
|
|
120
|
+
demand) so the dev cache's `scope=persistent` replays across runs."""
|
|
121
|
+
environ = env if env is not None else os.environ
|
|
122
|
+
choice = (preferred if preferred is not None else environ.get("KEEL_BACKEND", "auto")) or "auto"
|
|
123
|
+
if choice not in ("auto", "native", "stub"):
|
|
124
|
+
# A user env-var mistake, not a Keel bug — E001 (config-level), not E040.
|
|
125
|
+
raise KeelError(
|
|
126
|
+
"KEEL-E001",
|
|
127
|
+
f"KEEL_BACKEND must be one of auto, native, or stub (got {choice!r}); "
|
|
128
|
+
"unset it or correct the value",
|
|
129
|
+
)
|
|
130
|
+
|
|
131
|
+
if choice != "stub":
|
|
132
|
+
native = _try_load_native(_journal_path(cwd, environ))
|
|
133
|
+
if native is not None:
|
|
134
|
+
return native
|
|
135
|
+
if choice == "native":
|
|
136
|
+
# A missing build/install is a user-environment problem, not a Keel
|
|
137
|
+
# bug, and not a malformed policy: the configuration is valid, the
|
|
138
|
+
# capability is absent — E005 (unsupported-configuration) with a
|
|
139
|
+
# concrete next step, never E040 ("file an issue").
|
|
140
|
+
raise KeelError(
|
|
141
|
+
"KEEL-E005",
|
|
142
|
+
"KEEL_BACKEND=native requires the keel_core native module, which is not "
|
|
143
|
+
"installed; build it (`maturin develop -m crates/keel-py/Cargo.toml`) or "
|
|
144
|
+
"unset KEEL_BACKEND to use the pure-Python stub",
|
|
145
|
+
)
|
|
146
|
+
|
|
147
|
+
from keel_core_stub import KeelCoreStub
|
|
148
|
+
|
|
149
|
+
return KeelCoreStub()
|
keel/_defaults.py
ADDED
|
@@ -0,0 +1,116 @@
|
|
|
1
|
+
"""Level 0 embedded smart-defaults pack (DX spec §1) + the policy-merge helper.
|
|
2
|
+
|
|
3
|
+
The policy applied when no keel.toml is present. It MIRRORS
|
|
4
|
+
contracts/defaults.toml verbatim (that file is the frozen source of truth);
|
|
5
|
+
we embed rather than read it so the installed package is self-contained
|
|
6
|
+
(stdlib-only, works offline). Drift against the contract is caught by the
|
|
7
|
+
parity test, which parses contracts/defaults.toml with `tomllib` and asserts
|
|
8
|
+
deep equality when the repo file is present.
|
|
9
|
+
|
|
10
|
+
`apply_pack_defaults` layers these defaults (and any provider-pack fragments)
|
|
11
|
+
UNDER the user's keel.toml — the Python twin of `applyPackDefaults` in
|
|
12
|
+
node/keel/src/defaults.mjs; with no pack fragments it is identical to the Node
|
|
13
|
+
twin, and both front ends must agree (merge parity is a cross-language
|
|
14
|
+
contract).
|
|
15
|
+
|
|
16
|
+
The Level 0 hard rules — never change success-path semantics; never retry
|
|
17
|
+
non-idempotent calls; do nothing if a call can't be wrapped safely — are
|
|
18
|
+
BEHAVIOR, not config, and are enforced in the front end / backend, not here.
|
|
19
|
+
"""
|
|
20
|
+
|
|
21
|
+
from __future__ import annotations
|
|
22
|
+
|
|
23
|
+
from copy import deepcopy
|
|
24
|
+
from typing import Any, Iterable
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def outbound_defaults() -> dict[str, Any]:
|
|
28
|
+
"""`[defaults.outbound]` — any intercepted network call."""
|
|
29
|
+
return {
|
|
30
|
+
"timeout": "30s",
|
|
31
|
+
"retry": {
|
|
32
|
+
"attempts": 3,
|
|
33
|
+
"schedule": "exp(200ms, x2, max 30s, jitter)",
|
|
34
|
+
"on": ["conn", "timeout", "429", "5xx"],
|
|
35
|
+
},
|
|
36
|
+
"breaker": {"failures": 5, "cooldown": "15s"},
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def llm_defaults() -> dict[str, Any]:
|
|
41
|
+
"""`[defaults.llm]` — any `llm:*` target; the generic LLM pack layer.
|
|
42
|
+
|
|
43
|
+
Retry-After-aware retry (retry on conn/timeout/429/5xx; the core waits
|
|
44
|
+
`max(schedule_wait, retry_after)`) plus the dev-loop response cache
|
|
45
|
+
(`cache = { mode = "dev" }`, resolved to a concrete ttl off-prod and
|
|
46
|
+
dropped in prod by `keel.packs.llm.resolve_dev_cache`).
|
|
47
|
+
"""
|
|
48
|
+
return {
|
|
49
|
+
"timeout": "120s",
|
|
50
|
+
"retry": {
|
|
51
|
+
"attempts": 6,
|
|
52
|
+
"schedule": "exp(500ms, x2, max 60s, jitter)",
|
|
53
|
+
"on": ["conn", "timeout", "429", "5xx"],
|
|
54
|
+
},
|
|
55
|
+
"breaker": {"failures": 5, "cooldown": "30s"},
|
|
56
|
+
"cache": {"mode": "dev"},
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def level0_defaults() -> dict[str, Any]:
|
|
61
|
+
"""The embedded Level 0 policy, as the dict the backend's `configure`
|
|
62
|
+
expects (identical shape to keel.toml parsed to JSON)."""
|
|
63
|
+
return {"defaults": {"outbound": outbound_defaults(), "llm": llm_defaults()}}
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def _table(v: Any) -> dict[str, Any]:
|
|
67
|
+
return v if isinstance(v, dict) else {}
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def apply_pack_defaults(
|
|
71
|
+
policy: dict[str, Any],
|
|
72
|
+
pack_fragments: Iterable[dict[str, Any]] = (),
|
|
73
|
+
) -> dict[str, Any]:
|
|
74
|
+
"""Merge Level 0 defaults, then provider-pack fragments, then the user
|
|
75
|
+
policy — precedence ``defaults < packs < user`` — filling in any
|
|
76
|
+
``defaults.outbound`` / ``defaults.llm`` key a higher layer did not set,
|
|
77
|
+
while a higher layer replaces a key it DOES set wholesale (the
|
|
78
|
+
per-layer-wholesale semantics of the defaults.toml header, matching the
|
|
79
|
+
engine's own per-key layer resolution).
|
|
80
|
+
|
|
81
|
+
Target tables are left untouched — the engine resolves target →
|
|
82
|
+
defaults.llm → defaults.outbound precedence per key at execute time.
|
|
83
|
+
|
|
84
|
+
Provider packs (``keel.packs.openai_pack`` / ``anthropic_pack``) currently
|
|
85
|
+
emit the generic ``[defaults.llm]`` layer, so folding their fragments is the
|
|
86
|
+
identity over the embedded defaults; the parameter is the seam by which a
|
|
87
|
+
future per-provider refinement would take effect.
|
|
88
|
+
|
|
89
|
+
Returns a NEW policy; the input is never mutated. Idempotent on
|
|
90
|
+
``level0_defaults()``. Mirrors Node's ``applyPackDefaults`` (empty
|
|
91
|
+
``pack_fragments`` → identical result).
|
|
92
|
+
"""
|
|
93
|
+
out = deepcopy(policy) if isinstance(policy, dict) else {}
|
|
94
|
+
user_defaults = _table(out.get("defaults"))
|
|
95
|
+
|
|
96
|
+
pack_outbound: dict[str, Any] = {}
|
|
97
|
+
pack_llm: dict[str, Any] = {}
|
|
98
|
+
for frag in pack_fragments:
|
|
99
|
+
frag_defaults = _table(frag.get("defaults")) if isinstance(frag, dict) else {}
|
|
100
|
+
pack_outbound.update(_table(frag_defaults.get("outbound")))
|
|
101
|
+
pack_llm.update(_table(frag_defaults.get("llm")))
|
|
102
|
+
|
|
103
|
+
out["defaults"] = {
|
|
104
|
+
**user_defaults,
|
|
105
|
+
"outbound": {
|
|
106
|
+
**outbound_defaults(),
|
|
107
|
+
**pack_outbound,
|
|
108
|
+
**_table(user_defaults.get("outbound")),
|
|
109
|
+
},
|
|
110
|
+
"llm": {
|
|
111
|
+
**llm_defaults(),
|
|
112
|
+
**pack_llm,
|
|
113
|
+
**_table(user_defaults.get("llm")),
|
|
114
|
+
},
|
|
115
|
+
}
|
|
116
|
+
return out
|
keel/_discovery.py
ADDED
|
@@ -0,0 +1,368 @@
|
|
|
1
|
+
"""The discovery store: per-target traffic aggregates in `.keel/discovery.db`.
|
|
2
|
+
|
|
3
|
+
This is the third evidence source behind `keel init`/`status`/`doctor` (DX
|
|
4
|
+
spec §2). Its schema matches the canonical one owned by the keel-journal
|
|
5
|
+
crate (Task 1) column-for-column, so a `.keel/discovery.db` written by the
|
|
6
|
+
Python front end is readable by the same tools as one written by the core:
|
|
7
|
+
|
|
8
|
+
CREATE TABLE discovery (
|
|
9
|
+
target TEXT PRIMARY KEY,
|
|
10
|
+
calls INTEGER NOT NULL DEFAULT 0, -- intercepted calls
|
|
11
|
+
attempts INTEGER NOT NULL DEFAULT 0, -- upstream attempts (Σ)
|
|
12
|
+
retries INTEGER NOT NULL DEFAULT 0, -- attempts beyond the 1st
|
|
13
|
+
successes INTEGER NOT NULL DEFAULT 0,
|
|
14
|
+
failures INTEGER NOT NULL DEFAULT 0,
|
|
15
|
+
cache_hits INTEGER NOT NULL DEFAULT 0,
|
|
16
|
+
throttled INTEGER NOT NULL DEFAULT 0,
|
|
17
|
+
breaker_opens INTEGER NOT NULL DEFAULT 0,
|
|
18
|
+
total_latency_ms INTEGER NOT NULL DEFAULT 0,
|
|
19
|
+
max_latency_ms INTEGER NOT NULL DEFAULT 0,
|
|
20
|
+
first_seen_ms INTEGER NOT NULL,
|
|
21
|
+
last_seen_ms INTEGER NOT NULL,
|
|
22
|
+
last_error_class TEXT,
|
|
23
|
+
last_error_status INTEGER,
|
|
24
|
+
not_retried INTEGER NOT NULL DEFAULT 0, -- KEEL-E014: observed, not retried
|
|
25
|
+
unwrapped_calls INTEGER NOT NULL DEFAULT 0 -- calls with no [target] policy entry
|
|
26
|
+
) WITHOUT ROWID;
|
|
27
|
+
|
|
28
|
+
CREATE TABLE discovery_daily ( -- rolling daily buckets
|
|
29
|
+
target TEXT NOT NULL, -- (kept RETENTION_DAYS days)
|
|
30
|
+
day INTEGER NOT NULL, -- UTC day index: ms / 86_400_000
|
|
31
|
+
calls INTEGER NOT NULL DEFAULT 0,
|
|
32
|
+
attempts INTEGER NOT NULL DEFAULT 0,
|
|
33
|
+
retries INTEGER NOT NULL DEFAULT 0,
|
|
34
|
+
successes INTEGER NOT NULL DEFAULT 0,
|
|
35
|
+
failures INTEGER NOT NULL DEFAULT 0,
|
|
36
|
+
cache_hits INTEGER NOT NULL DEFAULT 0,
|
|
37
|
+
throttled INTEGER NOT NULL DEFAULT 0,
|
|
38
|
+
breaker_opens INTEGER NOT NULL DEFAULT 0,
|
|
39
|
+
not_retried INTEGER NOT NULL DEFAULT 0,
|
|
40
|
+
unwrapped_calls INTEGER NOT NULL DEFAULT 0,
|
|
41
|
+
PRIMARY KEY (target, day)
|
|
42
|
+
) WITHOUT ROWID;
|
|
43
|
+
|
|
44
|
+
Accounting mirrors the crate: a cache hit is a `call` and a `cache_hit` only
|
|
45
|
+
(no upstream attempt), so `calls == successes + failures + cache_hits`.
|
|
46
|
+
`breaker_opens` counts calls that FAILED FAST on an open breaker — outcomes with
|
|
47
|
+
error code `KEEL-E012` — matching the Rust core (`error.code == BreakerOpen`,
|
|
48
|
+
crates/keel-journal/src/discovery.rs) and the Node twin, NOT every outcome whose
|
|
49
|
+
`breaker` field reads "open" (that field is also stamped on the terminal failure
|
|
50
|
+
that trips the breaker and on a cache hit served while open). `not_retried`
|
|
51
|
+
counts calls that resolved KEEL-E014 (failed, and Keel refused to retry because
|
|
52
|
+
the call is not idempotent — the DX Level 0 hard rule's "observed, not
|
|
53
|
+
retried"). `unwrapped_calls` counts calls whose target had no explicit
|
|
54
|
+
`[target."…"]` entry in the EFFECTIVE policy handed to `backend.configure()`
|
|
55
|
+
(the same policy the core layers no pack underneath, per the CCR) — the honest
|
|
56
|
+
coverage gap `keel status` reports; `Discovery` is told the set of explicit
|
|
57
|
+
target keys once, at construction, by `bootstrap.install_keel`. Every mutation
|
|
58
|
+
is a single UPSERT per table, so two processes recording into one file
|
|
59
|
+
accumulate correctly without a transaction (WAL, `busy_timeout`).
|
|
60
|
+
|
|
61
|
+
Migration: a file written by the previous (v1, `user_version = 0`) schema is
|
|
62
|
+
upgraded in place on first open — the two counter columns are appended
|
|
63
|
+
(`ALTER TABLE … ADD COLUMN`, so column order matches a fresh v2 file), the
|
|
64
|
+
daily table is created, and `user_version` is stamped to 2. Mirrors
|
|
65
|
+
`crates/keel-journal/src/discovery.rs::migrate` exactly, so either writer can
|
|
66
|
+
open a file the other created.
|
|
67
|
+
|
|
68
|
+
Discovery is best-effort: it must never throw into, slow, or add output to
|
|
69
|
+
the user's program (DX invariant 4). Every public method swallows its own
|
|
70
|
+
errors and returns quietly.
|
|
71
|
+
"""
|
|
72
|
+
|
|
73
|
+
from __future__ import annotations
|
|
74
|
+
|
|
75
|
+
import sqlite3
|
|
76
|
+
import threading
|
|
77
|
+
from pathlib import Path
|
|
78
|
+
from time import time as _wall_clock # captured at import: immune to in-flow
|
|
79
|
+
from typing import Any # time virtualization (keel's own clock is never journaled)
|
|
80
|
+
|
|
81
|
+
#: Current discovery schema version, stamped in `PRAGMA user_version`.
|
|
82
|
+
#: Mirrors `keel_journal::discovery::DISCOVERY_SCHEMA_VERSION`.
|
|
83
|
+
SCHEMA_VERSION = 2
|
|
84
|
+
|
|
85
|
+
#: How many trailing UTC days of `discovery_daily` buckets are kept. Mirrors
|
|
86
|
+
#: `keel_journal::discovery::RETENTION_DAYS`.
|
|
87
|
+
RETENTION_DAYS = 30
|
|
88
|
+
|
|
89
|
+
#: Milliseconds per UTC day; `day = now_ms // MS_PER_DAY` is the bucket key.
|
|
90
|
+
MS_PER_DAY = 86_400_000
|
|
91
|
+
|
|
92
|
+
_DISCOVERY_SCHEMA = """\
|
|
93
|
+
CREATE TABLE IF NOT EXISTS discovery (
|
|
94
|
+
target TEXT PRIMARY KEY,
|
|
95
|
+
calls INTEGER NOT NULL DEFAULT 0,
|
|
96
|
+
attempts INTEGER NOT NULL DEFAULT 0,
|
|
97
|
+
retries INTEGER NOT NULL DEFAULT 0,
|
|
98
|
+
successes INTEGER NOT NULL DEFAULT 0,
|
|
99
|
+
failures INTEGER NOT NULL DEFAULT 0,
|
|
100
|
+
cache_hits INTEGER NOT NULL DEFAULT 0,
|
|
101
|
+
throttled INTEGER NOT NULL DEFAULT 0,
|
|
102
|
+
breaker_opens INTEGER NOT NULL DEFAULT 0,
|
|
103
|
+
total_latency_ms INTEGER NOT NULL DEFAULT 0,
|
|
104
|
+
max_latency_ms INTEGER NOT NULL DEFAULT 0,
|
|
105
|
+
first_seen_ms INTEGER NOT NULL,
|
|
106
|
+
last_seen_ms INTEGER NOT NULL,
|
|
107
|
+
last_error_class TEXT,
|
|
108
|
+
last_error_status INTEGER,
|
|
109
|
+
not_retried INTEGER NOT NULL DEFAULT 0,
|
|
110
|
+
unwrapped_calls INTEGER NOT NULL DEFAULT 0
|
|
111
|
+
) WITHOUT ROWID;"""
|
|
112
|
+
|
|
113
|
+
_DAILY_SCHEMA = """\
|
|
114
|
+
CREATE TABLE IF NOT EXISTS discovery_daily (
|
|
115
|
+
target TEXT NOT NULL,
|
|
116
|
+
day INTEGER NOT NULL,
|
|
117
|
+
calls INTEGER NOT NULL DEFAULT 0,
|
|
118
|
+
attempts INTEGER NOT NULL DEFAULT 0,
|
|
119
|
+
retries INTEGER NOT NULL DEFAULT 0,
|
|
120
|
+
successes INTEGER NOT NULL DEFAULT 0,
|
|
121
|
+
failures INTEGER NOT NULL DEFAULT 0,
|
|
122
|
+
cache_hits INTEGER NOT NULL DEFAULT 0,
|
|
123
|
+
throttled INTEGER NOT NULL DEFAULT 0,
|
|
124
|
+
breaker_opens INTEGER NOT NULL DEFAULT 0,
|
|
125
|
+
not_retried INTEGER NOT NULL DEFAULT 0,
|
|
126
|
+
unwrapped_calls INTEGER NOT NULL DEFAULT 0,
|
|
127
|
+
PRIMARY KEY (target, day)
|
|
128
|
+
) WITHOUT ROWID;"""
|
|
129
|
+
|
|
130
|
+
# `?` (qmark) placeholders in the same column order as the crate's numbered
|
|
131
|
+
# statement; the ON CONFLICT body is identical (counters add, extremes keep,
|
|
132
|
+
# first_seen shrinks / last_seen grows, error columns move together).
|
|
133
|
+
_UPSERT = """\
|
|
134
|
+
INSERT INTO discovery
|
|
135
|
+
(target, calls, attempts, retries, successes, failures, cache_hits,
|
|
136
|
+
throttled, breaker_opens, total_latency_ms, max_latency_ms,
|
|
137
|
+
first_seen_ms, last_seen_ms, last_error_class, last_error_status,
|
|
138
|
+
not_retried, unwrapped_calls)
|
|
139
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
140
|
+
ON CONFLICT(target) DO UPDATE SET
|
|
141
|
+
calls = calls + excluded.calls,
|
|
142
|
+
attempts = attempts + excluded.attempts,
|
|
143
|
+
retries = retries + excluded.retries,
|
|
144
|
+
successes = successes + excluded.successes,
|
|
145
|
+
failures = failures + excluded.failures,
|
|
146
|
+
cache_hits = cache_hits + excluded.cache_hits,
|
|
147
|
+
throttled = throttled + excluded.throttled,
|
|
148
|
+
breaker_opens = breaker_opens + excluded.breaker_opens,
|
|
149
|
+
total_latency_ms = total_latency_ms + excluded.total_latency_ms,
|
|
150
|
+
max_latency_ms = max(max_latency_ms, excluded.max_latency_ms),
|
|
151
|
+
first_seen_ms = min(first_seen_ms, excluded.first_seen_ms),
|
|
152
|
+
last_seen_ms = max(last_seen_ms, excluded.last_seen_ms),
|
|
153
|
+
last_error_class = coalesce(excluded.last_error_class, last_error_class),
|
|
154
|
+
last_error_status = CASE
|
|
155
|
+
WHEN excluded.last_error_class IS NOT NULL THEN excluded.last_error_status
|
|
156
|
+
ELSE last_error_status END,
|
|
157
|
+
not_retried = not_retried + excluded.not_retried,
|
|
158
|
+
unwrapped_calls = unwrapped_calls + excluded.unwrapped_calls"""
|
|
159
|
+
|
|
160
|
+
_DAILY_UPSERT = """\
|
|
161
|
+
INSERT INTO discovery_daily
|
|
162
|
+
(target, day, calls, attempts, retries, successes, failures, cache_hits,
|
|
163
|
+
throttled, breaker_opens, not_retried, unwrapped_calls)
|
|
164
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
165
|
+
ON CONFLICT(target, day) DO UPDATE SET
|
|
166
|
+
calls = calls + excluded.calls,
|
|
167
|
+
attempts = attempts + excluded.attempts,
|
|
168
|
+
retries = retries + excluded.retries,
|
|
169
|
+
successes = successes + excluded.successes,
|
|
170
|
+
failures = failures + excluded.failures,
|
|
171
|
+
cache_hits = cache_hits + excluded.cache_hits,
|
|
172
|
+
throttled = throttled + excluded.throttled,
|
|
173
|
+
breaker_opens = breaker_opens + excluded.breaker_opens,
|
|
174
|
+
not_retried = not_retried + excluded.not_retried,
|
|
175
|
+
unwrapped_calls = unwrapped_calls + excluded.unwrapped_calls"""
|
|
176
|
+
|
|
177
|
+
|
|
178
|
+
class Discovery:
|
|
179
|
+
"""A per-target traffic ledger over its own WAL-mode SQLite file. One
|
|
180
|
+
connection per process (shared across threads under a lock, matching the
|
|
181
|
+
crate's `Mutex<Connection>`); one UPSERT per table per recorded call.
|
|
182
|
+
|
|
183
|
+
`known_targets` is the set of EXPLICIT `[target."…"]` keys in the
|
|
184
|
+
effective policy (defaults < packs < user, before `backend.configure()`)
|
|
185
|
+
— used to classify each recorded call as wrapped (an explicit entry
|
|
186
|
+
applied) or not (the coverage gap `keel status` reports). Passing none
|
|
187
|
+
(the default) means every call is counted unwrapped, which is correct for
|
|
188
|
+
a store opened outside `bootstrap.install_keel` (e.g. read-only tooling
|
|
189
|
+
that never records)."""
|
|
190
|
+
|
|
191
|
+
def __init__(
|
|
192
|
+
self,
|
|
193
|
+
cwd: str | Path | None = None,
|
|
194
|
+
known_targets: frozenset[str] | None = None,
|
|
195
|
+
) -> None:
|
|
196
|
+
self.db_path = Path(cwd or Path.cwd()) / ".keel" / "discovery.db"
|
|
197
|
+
self._known_targets = known_targets or frozenset()
|
|
198
|
+
self._lock = threading.Lock()
|
|
199
|
+
self._conn: sqlite3.Connection | None = None
|
|
200
|
+
self._last_prune_day: int | None = None
|
|
201
|
+
|
|
202
|
+
def _connect(self) -> sqlite3.Connection | None:
|
|
203
|
+
"""Open (once) the connection, lazily so a disabled/never-recording
|
|
204
|
+
run never touches the filesystem. Returns None if the store can't be
|
|
205
|
+
opened (permissions, fs) — recording then no-ops."""
|
|
206
|
+
if self._conn is not None:
|
|
207
|
+
return self._conn
|
|
208
|
+
try:
|
|
209
|
+
self.db_path.parent.mkdir(parents=True, exist_ok=True)
|
|
210
|
+
conn = sqlite3.connect(self.db_path, check_same_thread=False)
|
|
211
|
+
conn.execute("PRAGMA journal_mode = WAL;")
|
|
212
|
+
conn.execute("PRAGMA busy_timeout = 5000;")
|
|
213
|
+
conn.execute("PRAGMA synchronous = NORMAL;")
|
|
214
|
+
_migrate(conn)
|
|
215
|
+
except sqlite3.Error:
|
|
216
|
+
return None
|
|
217
|
+
self._conn = conn
|
|
218
|
+
return conn
|
|
219
|
+
|
|
220
|
+
def record(self, target: str, outcome: dict[str, Any], latency_ms: int) -> None:
|
|
221
|
+
"""Fold one intercepted call's outcome envelope into its target's
|
|
222
|
+
aggregates (lifetime row plus the clock-day bucket). Best-effort:
|
|
223
|
+
never raises."""
|
|
224
|
+
wrapped = target in self._known_targets
|
|
225
|
+
row = _row_from_outcome(target, outcome, latency_ms, wrapped)
|
|
226
|
+
now_ms = row[12] # last_seen_ms, per _row_from_outcome's column order
|
|
227
|
+
day = now_ms // MS_PER_DAY
|
|
228
|
+
try:
|
|
229
|
+
with self._lock:
|
|
230
|
+
conn = self._connect()
|
|
231
|
+
if conn is None:
|
|
232
|
+
return
|
|
233
|
+
conn.execute(_UPSERT, row)
|
|
234
|
+
conn.execute(_DAILY_UPSERT, _daily_row(row, day))
|
|
235
|
+
self._prune(conn, day)
|
|
236
|
+
conn.commit()
|
|
237
|
+
except sqlite3.Error:
|
|
238
|
+
pass # best-effort: discovery never breaks the user's program
|
|
239
|
+
|
|
240
|
+
def _prune(self, conn: sqlite3.Connection, day: int) -> None:
|
|
241
|
+
"""Drop daily buckets older than the retention window, at most once
|
|
242
|
+
per advanced day (mirrors the crate's `DiscoveryStore::prune`)."""
|
|
243
|
+
if self._last_prune_day == day:
|
|
244
|
+
return
|
|
245
|
+
self._last_prune_day = day
|
|
246
|
+
conn.execute(
|
|
247
|
+
"DELETE FROM discovery_daily WHERE day < ?",
|
|
248
|
+
(day - (RETENTION_DAYS - 1),),
|
|
249
|
+
)
|
|
250
|
+
|
|
251
|
+
def close(self) -> None:
|
|
252
|
+
with self._lock:
|
|
253
|
+
if self._conn is not None:
|
|
254
|
+
try:
|
|
255
|
+
self._conn.close()
|
|
256
|
+
finally:
|
|
257
|
+
self._conn = None
|
|
258
|
+
|
|
259
|
+
|
|
260
|
+
def _migrate(conn: sqlite3.Connection) -> None:
|
|
261
|
+
"""Bring a connection to [`SCHEMA_VERSION`]: create the current schema on
|
|
262
|
+
a fresh file, or append the v2 counter columns and the daily table to a
|
|
263
|
+
legacy (v1) one. Mirrors `keel_journal::discovery::migrate` — idempotent,
|
|
264
|
+
so re-opening an already-migrated file is a no-op."""
|
|
265
|
+
(version,) = conn.execute("PRAGMA user_version").fetchone()
|
|
266
|
+
if version >= SCHEMA_VERSION:
|
|
267
|
+
return
|
|
268
|
+
has_table = conn.execute(
|
|
269
|
+
"SELECT EXISTS(SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'discovery')"
|
|
270
|
+
).fetchone()[0]
|
|
271
|
+
if has_table:
|
|
272
|
+
has_column = conn.execute(
|
|
273
|
+
"SELECT EXISTS(SELECT 1 FROM pragma_table_info('discovery') "
|
|
274
|
+
"WHERE name = 'not_retried')"
|
|
275
|
+
).fetchone()[0]
|
|
276
|
+
if not has_column:
|
|
277
|
+
# Appended, so a migrated file's column order matches a fresh v2 one.
|
|
278
|
+
conn.execute(
|
|
279
|
+
"ALTER TABLE discovery ADD COLUMN not_retried INTEGER NOT NULL DEFAULT 0"
|
|
280
|
+
)
|
|
281
|
+
conn.execute(
|
|
282
|
+
"ALTER TABLE discovery ADD COLUMN unwrapped_calls INTEGER NOT NULL DEFAULT 0"
|
|
283
|
+
)
|
|
284
|
+
else:
|
|
285
|
+
conn.executescript(_DISCOVERY_SCHEMA)
|
|
286
|
+
conn.executescript(_DAILY_SCHEMA)
|
|
287
|
+
conn.execute(f"PRAGMA user_version = {SCHEMA_VERSION}")
|
|
288
|
+
conn.commit()
|
|
289
|
+
|
|
290
|
+
|
|
291
|
+
def _row_from_outcome(
|
|
292
|
+
target: str, outcome: dict[str, Any], latency_ms: int, wrapped: bool = True
|
|
293
|
+
) -> tuple[Any, ...]:
|
|
294
|
+
"""Project a core outcome envelope onto one `discovery` row's 17 values."""
|
|
295
|
+
result = outcome.get("result")
|
|
296
|
+
from_cache = bool(outcome.get("from_cache"))
|
|
297
|
+
attempts = int(outcome.get("attempts", 0) or 0)
|
|
298
|
+
|
|
299
|
+
cache_hit = result == "ok" and from_cache
|
|
300
|
+
success = result == "ok" and not from_cache
|
|
301
|
+
failure = result != "ok"
|
|
302
|
+
|
|
303
|
+
err = outcome.get("error") or {}
|
|
304
|
+
last_error_class = err.get("class") if failure else None
|
|
305
|
+
last_error_status = err.get("http_status") if failure else None
|
|
306
|
+
|
|
307
|
+
now = int(_wall_clock() * 1000)
|
|
308
|
+
return (
|
|
309
|
+
target,
|
|
310
|
+
1, # calls
|
|
311
|
+
attempts,
|
|
312
|
+
attempts - 1 if attempts > 0 else 0, # retries
|
|
313
|
+
1 if success else 0,
|
|
314
|
+
1 if failure else 0,
|
|
315
|
+
1 if cache_hit else 0,
|
|
316
|
+
1 if outcome.get("throttled") else 0,
|
|
317
|
+
# breaker_opens = fail-fast rejections only (KEEL-E012), the canonical
|
|
318
|
+
# rule shared with the Rust core and Node — NOT any breaker=="open" stamp.
|
|
319
|
+
1 if err.get("code") == "KEEL-E012" else 0,
|
|
320
|
+
latency_ms, # total_latency_ms
|
|
321
|
+
latency_ms, # max_latency_ms
|
|
322
|
+
now, # first_seen_ms
|
|
323
|
+
now, # last_seen_ms
|
|
324
|
+
last_error_class,
|
|
325
|
+
last_error_status,
|
|
326
|
+
# not_retried = KEEL-E014: observed, not retried (Level 0 hard rule).
|
|
327
|
+
1 if err.get("code") == "KEEL-E014" else 0,
|
|
328
|
+
0 if wrapped else 1, # unwrapped_calls
|
|
329
|
+
)
|
|
330
|
+
|
|
331
|
+
|
|
332
|
+
def _daily_row(row: tuple[Any, ...], day: int) -> tuple[Any, ...]:
|
|
333
|
+
"""Project a `discovery` row (see [`_row_from_outcome`]) onto its
|
|
334
|
+
`discovery_daily` twin: same target and counters, keyed by `day` instead
|
|
335
|
+
of the seen-timestamps and error columns the daily table omits."""
|
|
336
|
+
(
|
|
337
|
+
target,
|
|
338
|
+
calls,
|
|
339
|
+
attempts,
|
|
340
|
+
retries,
|
|
341
|
+
successes,
|
|
342
|
+
failures,
|
|
343
|
+
cache_hits,
|
|
344
|
+
throttled,
|
|
345
|
+
breaker_opens,
|
|
346
|
+
_total_latency_ms,
|
|
347
|
+
_max_latency_ms,
|
|
348
|
+
_first_seen_ms,
|
|
349
|
+
_last_seen_ms,
|
|
350
|
+
_last_error_class,
|
|
351
|
+
_last_error_status,
|
|
352
|
+
not_retried,
|
|
353
|
+
unwrapped_calls,
|
|
354
|
+
) = row
|
|
355
|
+
return (
|
|
356
|
+
target,
|
|
357
|
+
day,
|
|
358
|
+
calls,
|
|
359
|
+
attempts,
|
|
360
|
+
retries,
|
|
361
|
+
successes,
|
|
362
|
+
failures,
|
|
363
|
+
cache_hits,
|
|
364
|
+
throttled,
|
|
365
|
+
breaker_opens,
|
|
366
|
+
not_retried,
|
|
367
|
+
unwrapped_calls,
|
|
368
|
+
)
|