kaiten-cli 0.3.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- kaiten_cli/__init__.py +5 -0
- kaiten_cli/__main__.py +9 -0
- kaiten_cli/agent_gateway.py +577 -0
- kaiten_cli/app.py +1636 -0
- kaiten_cli/completion.py +570 -0
- kaiten_cli/discovery.py +39 -0
- kaiten_cli/errors.py +79 -0
- kaiten_cli/i18n.py +92 -0
- kaiten_cli/locales/ru.json +2967 -0
- kaiten_cli/localized_click.py +160 -0
- kaiten_cli/models.py +202 -0
- kaiten_cli/profiles.py +316 -0
- kaiten_cli/registry/__init__.py +260 -0
- kaiten_cli/registry/addons.py +392 -0
- kaiten_cli/registry/audit_and_analytics.py +451 -0
- kaiten_cli/registry/automation_schema.py +44 -0
- kaiten_cli/registry/automation_schemas.json +7417 -0
- kaiten_cli/registry/automations.py +342 -0
- kaiten_cli/registry/base.py +62 -0
- kaiten_cli/registry/blockers.py +320 -0
- kaiten_cli/registry/boards.py +520 -0
- kaiten_cli/registry/card_relations.py +376 -0
- kaiten_cli/registry/card_types.py +363 -0
- kaiten_cli/registry/cards.py +1329 -0
- kaiten_cli/registry/charts.py +539 -0
- kaiten_cli/registry/checklists.py +600 -0
- kaiten_cli/registry/columns.py +558 -0
- kaiten_cli/registry/comments.py +224 -0
- kaiten_cli/registry/custom_directories.py +980 -0
- kaiten_cli/registry/custom_properties.py +1460 -0
- kaiten_cli/registry/dashboards.py +564 -0
- kaiten_cli/registry/documents.py +761 -0
- kaiten_cli/registry/external_links.py +128 -0
- kaiten_cli/registry/files.py +738 -0
- kaiten_cli/registry/github_addon.py +613 -0
- kaiten_cli/registry/iterations.py +397 -0
- kaiten_cli/registry/lanes.py +205 -0
- kaiten_cli/registry/live_contracts.py +361 -0
- kaiten_cli/registry/members.py +336 -0
- kaiten_cli/registry/module_docs.py +108 -0
- kaiten_cli/registry/projects.py +369 -0
- kaiten_cli/registry/query.py +129 -0
- kaiten_cli/registry/restrictions.py +249 -0
- kaiten_cli/registry/roles_and_groups.py +1102 -0
- kaiten_cli/registry/scim.py +263 -0
- kaiten_cli/registry/service_desk.py +1270 -0
- kaiten_cli/registry/snapshot.py +187 -0
- kaiten_cli/registry/spaces.py +272 -0
- kaiten_cli/registry/subscribers.py +168 -0
- kaiten_cli/registry/tags.py +193 -0
- kaiten_cli/registry/time_logs.py +353 -0
- kaiten_cli/registry/tree.py +352 -0
- kaiten_cli/registry/utilities.py +287 -0
- kaiten_cli/registry/webhooks.py +265 -0
- kaiten_cli/runtime/__init__.py +1 -0
- kaiten_cli/runtime/automation_validation.py +115 -0
- kaiten_cli/runtime/behaviors.py +787 -0
- kaiten_cli/runtime/cache.py +733 -0
- kaiten_cli/runtime/client.py +387 -0
- kaiten_cli/runtime/endpoints.py +59 -0
- kaiten_cli/runtime/executor.py +269 -0
- kaiten_cli/runtime/fs_security.py +168 -0
- kaiten_cli/runtime/input.py +310 -0
- kaiten_cli/runtime/output.py +59 -0
- kaiten_cli/runtime/public_validation.py +35 -0
- kaiten_cli/runtime/snapshot_build.py +527 -0
- kaiten_cli/runtime/snapshot_common.py +363 -0
- kaiten_cli/runtime/snapshot_query.py +306 -0
- kaiten_cli/runtime/snapshot_store.py +1192 -0
- kaiten_cli/runtime/snapshots.py +62 -0
- kaiten_cli/runtime/sqlite_errors.py +23 -0
- kaiten_cli/runtime/support/__init__.py +1 -0
- kaiten_cli/runtime/support/addons.py +953 -0
- kaiten_cli/runtime/support/audit.py +70 -0
- kaiten_cli/runtime/support/batch.py +234 -0
- kaiten_cli/runtime/support/card_move_url.py +213 -0
- kaiten_cli/runtime/support/cards.py +174 -0
- kaiten_cli/runtime/support/checklists.py +21 -0
- kaiten_cli/runtime/support/company_users.py +117 -0
- kaiten_cli/runtime/support/dashboards.py +35 -0
- kaiten_cli/runtime/support/documents.py +175 -0
- kaiten_cli/runtime/support/files.py +561 -0
- kaiten_cli/runtime/support/markdown_export.py +525 -0
- kaiten_cli/runtime/support/pagination.py +167 -0
- kaiten_cli/runtime/support/projects.py +66 -0
- kaiten_cli/runtime/support/relations.py +86 -0
- kaiten_cli/runtime/support/restrictions.py +62 -0
- kaiten_cli/runtime/support/spaces.py +20 -0
- kaiten_cli/runtime/support/time_logs.py +61 -0
- kaiten_cli/runtime/support/tree.py +140 -0
- kaiten_cli/runtime/support/tree_sharing.py +447 -0
- kaiten_cli/runtime/trace.py +500 -0
- kaiten_cli/runtime/transforms.py +128 -0
- kaiten_cli/schema_docs.py +22 -0
- kaiten_cli/update_check.py +519 -0
- kaiten_cli-0.3.3.dist-info/METADATA +1110 -0
- kaiten_cli-0.3.3.dist-info/RECORD +99 -0
- kaiten_cli-0.3.3.dist-info/WHEEL +4 -0
- kaiten_cli-0.3.3.dist-info/entry_points.txt +3 -0
|
@@ -0,0 +1,733 @@
|
|
|
1
|
+
"""Execution-scoped and persistent cache helpers."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import asyncio
|
|
6
|
+
import copy
|
|
7
|
+
import hashlib
|
|
8
|
+
import json
|
|
9
|
+
import sqlite3
|
|
10
|
+
import time
|
|
11
|
+
from contextlib import closing
|
|
12
|
+
from dataclasses import dataclass, field
|
|
13
|
+
from datetime import datetime, timezone
|
|
14
|
+
from pathlib import Path
|
|
15
|
+
from typing import Any
|
|
16
|
+
|
|
17
|
+
from platformdirs import user_cache_path
|
|
18
|
+
|
|
19
|
+
from kaiten_cli.i18n import tr
|
|
20
|
+
from kaiten_cli.models import (
|
|
21
|
+
CACHE_MODE_AUTO,
|
|
22
|
+
CACHE_MODE_READWRITE,
|
|
23
|
+
CACHE_MODE_REFRESH,
|
|
24
|
+
CACHE_POLICY_NONE,
|
|
25
|
+
CACHE_POLICY_PERSISTENT_HEAVY,
|
|
26
|
+
PERSISTENT_CACHE_POLICIES,
|
|
27
|
+
DebugReporter,
|
|
28
|
+
ResolvedProfile,
|
|
29
|
+
)
|
|
30
|
+
from kaiten_cli.runtime.fs_security import ensure_private_file
|
|
31
|
+
from kaiten_cli.runtime.sqlite_errors import is_corrupt_database_error
|
|
32
|
+
from kaiten_cli.runtime.trace import ExecutionStats, path_family_for
|
|
33
|
+
|
|
34
|
+
HTTP_CACHE_DB_SCHEMA_VERSION = 4
|
|
35
|
+
AUTO_ENTITY_TTL_SECONDS = 30 * 60
|
|
36
|
+
AUTO_MEDIUM_TTL_SECONDS = 6 * 60 * 60
|
|
37
|
+
AUTO_HEAVY_TTL_SECONDS = 24 * 60 * 60
|
|
38
|
+
AUTO_HISTORICAL_TTL_SECONDS = 7 * 24 * 60 * 60
|
|
39
|
+
AUTO_HEAVY_ROW_THRESHOLD = 500
|
|
40
|
+
AUTO_MEDIUM_ROW_THRESHOLD = 100
|
|
41
|
+
AUTO_HEAVY_PAYLOAD_BYTES = 1_000_000
|
|
42
|
+
AUTO_MEDIUM_PAYLOAD_BYTES = 100_000
|
|
43
|
+
AUTO_DENSE_FAMILY_WINDOW_SECONDS = 30 * 60
|
|
44
|
+
AUTO_DENSE_FAMILY_MEDIUM_THRESHOLD = 25
|
|
45
|
+
AUTO_DENSE_FAMILY_HEAVY_THRESHOLD = 100
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def persistent_cache_path() -> Path:
|
|
49
|
+
return user_cache_path("kaiten-cli") / "http-cache.sqlite3"
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def _normalize_params(params: dict[str, Any] | None) -> dict[str, Any]:
|
|
53
|
+
if not params:
|
|
54
|
+
return {}
|
|
55
|
+
return {key: value for key, value in sorted(params.items()) if value is not None}
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def _payload_rows_count(payload: Any) -> int:
|
|
59
|
+
if isinstance(payload, list):
|
|
60
|
+
return len(payload)
|
|
61
|
+
if isinstance(payload, dict):
|
|
62
|
+
for key in ("items", "data", "results"):
|
|
63
|
+
value = payload.get(key)
|
|
64
|
+
if isinstance(value, list):
|
|
65
|
+
return len(value)
|
|
66
|
+
return 1 if payload is not None else 0
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def _parse_datetime(value: Any) -> datetime | None:
|
|
70
|
+
if value is None:
|
|
71
|
+
return None
|
|
72
|
+
text = str(value).strip()
|
|
73
|
+
if not text:
|
|
74
|
+
return None
|
|
75
|
+
try:
|
|
76
|
+
parsed = datetime.fromisoformat(text.replace("Z", "+00:00"))
|
|
77
|
+
except ValueError:
|
|
78
|
+
return None
|
|
79
|
+
if parsed.tzinfo is None:
|
|
80
|
+
parsed = parsed.replace(tzinfo=timezone.utc)
|
|
81
|
+
return parsed.astimezone(timezone.utc)
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def _looks_historical(params_json: str) -> bool:
|
|
85
|
+
try:
|
|
86
|
+
params = json.loads(params_json)
|
|
87
|
+
except json.JSONDecodeError:
|
|
88
|
+
return False
|
|
89
|
+
if not isinstance(params, dict):
|
|
90
|
+
return False
|
|
91
|
+
for key in ("created_before", "updated_before", "to", "window_end", "date_to", "date-to"):
|
|
92
|
+
value = _parse_datetime(params.get(key))
|
|
93
|
+
if value is None:
|
|
94
|
+
continue
|
|
95
|
+
today_start = datetime.now(timezone.utc).replace(hour=0, minute=0, second=0, microsecond=0)
|
|
96
|
+
if value < today_start:
|
|
97
|
+
return True
|
|
98
|
+
return False
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
@dataclass(frozen=True, slots=True)
|
|
102
|
+
class RequestCacheKey:
|
|
103
|
+
scope: str
|
|
104
|
+
method: str
|
|
105
|
+
path: str
|
|
106
|
+
params_json: str
|
|
107
|
+
path_family: str
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
class PersistentCache:
|
|
111
|
+
"""Small sqlite-backed cache for cross-process safe GET reuse."""
|
|
112
|
+
|
|
113
|
+
def __init__(self, path: Path, reporter: DebugReporter | None = None):
|
|
114
|
+
self.path = path
|
|
115
|
+
self.reporter = reporter
|
|
116
|
+
|
|
117
|
+
def _debug(self, message: str) -> None:
|
|
118
|
+
if self.reporter is not None:
|
|
119
|
+
self.reporter(message)
|
|
120
|
+
|
|
121
|
+
def _open_connection(self) -> sqlite3.Connection:
|
|
122
|
+
ensure_private_file(self.path)
|
|
123
|
+
return sqlite3.connect(self.path)
|
|
124
|
+
|
|
125
|
+
def _initialize_schema(self, conn: sqlite3.Connection) -> None:
|
|
126
|
+
conn.execute(
|
|
127
|
+
"""
|
|
128
|
+
CREATE TABLE IF NOT EXISTS responses (
|
|
129
|
+
scope TEXT NOT NULL,
|
|
130
|
+
method TEXT NOT NULL,
|
|
131
|
+
path TEXT NOT NULL,
|
|
132
|
+
path_family TEXT NOT NULL DEFAULT '',
|
|
133
|
+
params_json TEXT NOT NULL,
|
|
134
|
+
payload_json TEXT NOT NULL,
|
|
135
|
+
expires_at REAL NOT NULL,
|
|
136
|
+
updated_at REAL NOT NULL,
|
|
137
|
+
created_at REAL NOT NULL DEFAULT 0,
|
|
138
|
+
cache_policy TEXT NOT NULL DEFAULT '',
|
|
139
|
+
ttl_seconds INTEGER NOT NULL DEFAULT 0,
|
|
140
|
+
payload_bytes INTEGER NOT NULL DEFAULT 0,
|
|
141
|
+
rows_count INTEGER NOT NULL DEFAULT 0,
|
|
142
|
+
PRIMARY KEY (scope, method, path, params_json)
|
|
143
|
+
)
|
|
144
|
+
"""
|
|
145
|
+
)
|
|
146
|
+
conn.execute(
|
|
147
|
+
"""
|
|
148
|
+
CREATE INDEX IF NOT EXISTS idx_responses_family_recent
|
|
149
|
+
ON responses (scope, method, path_family, updated_at)
|
|
150
|
+
"""
|
|
151
|
+
)
|
|
152
|
+
conn.execute(f"PRAGMA user_version = {HTTP_CACHE_DB_SCHEMA_VERSION}")
|
|
153
|
+
conn.commit()
|
|
154
|
+
|
|
155
|
+
def _close_quietly(self, conn: sqlite3.Connection | None) -> None:
|
|
156
|
+
if conn is None:
|
|
157
|
+
return
|
|
158
|
+
try:
|
|
159
|
+
conn.close()
|
|
160
|
+
except sqlite3.Error:
|
|
161
|
+
return
|
|
162
|
+
|
|
163
|
+
def _reset_store(self, reason: str) -> sqlite3.Connection | None:
|
|
164
|
+
self._debug(
|
|
165
|
+
tr("cache: local store dropped store=http-cache reason={value_0}", value_0=reason)
|
|
166
|
+
)
|
|
167
|
+
try:
|
|
168
|
+
self.path.unlink(missing_ok=True)
|
|
169
|
+
except OSError as exc:
|
|
170
|
+
self._debug(
|
|
171
|
+
tr(
|
|
172
|
+
"cache: reset bypass store=http-cache reason={value_0}",
|
|
173
|
+
value_0=type(exc).__name__,
|
|
174
|
+
)
|
|
175
|
+
)
|
|
176
|
+
return None
|
|
177
|
+
conn: sqlite3.Connection | None = None
|
|
178
|
+
try:
|
|
179
|
+
conn = self._open_connection()
|
|
180
|
+
self._initialize_schema(conn)
|
|
181
|
+
self._debug(tr("cache: local store recreated store=http-cache"))
|
|
182
|
+
return conn
|
|
183
|
+
except (OSError, sqlite3.Error) as exc:
|
|
184
|
+
self._close_quietly(conn)
|
|
185
|
+
self._debug(
|
|
186
|
+
tr(
|
|
187
|
+
"cache: reset bypass store=http-cache reason={value_0}",
|
|
188
|
+
value_0=type(exc).__name__,
|
|
189
|
+
)
|
|
190
|
+
)
|
|
191
|
+
return None
|
|
192
|
+
|
|
193
|
+
def _connect(self) -> sqlite3.Connection | None:
|
|
194
|
+
self.path.parent.mkdir(parents=True, exist_ok=True)
|
|
195
|
+
existing = self.path.exists()
|
|
196
|
+
conn: sqlite3.Connection | None = None
|
|
197
|
+
try:
|
|
198
|
+
conn = self._open_connection()
|
|
199
|
+
version = int(conn.execute("PRAGMA user_version").fetchone()[0])
|
|
200
|
+
if existing and version != HTTP_CACHE_DB_SCHEMA_VERSION:
|
|
201
|
+
self._close_quietly(conn)
|
|
202
|
+
return self._reset_store(f"incompatible-schema:{version}")
|
|
203
|
+
self._initialize_schema(conn)
|
|
204
|
+
return conn
|
|
205
|
+
except (OSError, sqlite3.Error) as exc:
|
|
206
|
+
self._close_quietly(conn)
|
|
207
|
+
if isinstance(exc, sqlite3.Error) and is_corrupt_database_error(exc):
|
|
208
|
+
return self._reset_store(type(exc).__name__)
|
|
209
|
+
self._debug(
|
|
210
|
+
tr(
|
|
211
|
+
"cache: local store bypass store=http-cache reason={value_0}",
|
|
212
|
+
value_0=type(exc).__name__,
|
|
213
|
+
)
|
|
214
|
+
)
|
|
215
|
+
return None
|
|
216
|
+
|
|
217
|
+
def get(self, key: RequestCacheKey) -> tuple[str, Any | None]:
|
|
218
|
+
now = time.time()
|
|
219
|
+
conn = self._connect()
|
|
220
|
+
if conn is None:
|
|
221
|
+
return "miss", None
|
|
222
|
+
with closing(conn), conn:
|
|
223
|
+
row = conn.execute(
|
|
224
|
+
"""
|
|
225
|
+
SELECT payload_json, expires_at
|
|
226
|
+
FROM responses
|
|
227
|
+
WHERE scope = ? AND method = ? AND path = ? AND params_json = ?
|
|
228
|
+
""",
|
|
229
|
+
(key.scope, key.method, key.path, key.params_json),
|
|
230
|
+
).fetchone()
|
|
231
|
+
if row is None:
|
|
232
|
+
return "miss", None
|
|
233
|
+
payload_json, expires_at = row
|
|
234
|
+
if expires_at <= now:
|
|
235
|
+
conn.execute(
|
|
236
|
+
"""
|
|
237
|
+
DELETE FROM responses
|
|
238
|
+
WHERE scope = ? AND method = ? AND path = ? AND params_json = ?
|
|
239
|
+
""",
|
|
240
|
+
(key.scope, key.method, key.path, key.params_json),
|
|
241
|
+
)
|
|
242
|
+
conn.commit()
|
|
243
|
+
return "expired", None
|
|
244
|
+
return "hit", json.loads(payload_json)
|
|
245
|
+
|
|
246
|
+
def set(
|
|
247
|
+
self,
|
|
248
|
+
key: RequestCacheKey,
|
|
249
|
+
payload: Any,
|
|
250
|
+
*,
|
|
251
|
+
ttl_seconds: int,
|
|
252
|
+
cache_policy: str,
|
|
253
|
+
payload_bytes: int,
|
|
254
|
+
rows_count: int,
|
|
255
|
+
) -> None:
|
|
256
|
+
if payload is None:
|
|
257
|
+
return
|
|
258
|
+
payload_json = json.dumps(payload, ensure_ascii=False, separators=(",", ":"))
|
|
259
|
+
now = time.time()
|
|
260
|
+
expires_at = now + ttl_seconds
|
|
261
|
+
conn = self._connect()
|
|
262
|
+
if conn is None:
|
|
263
|
+
return
|
|
264
|
+
with closing(conn), conn:
|
|
265
|
+
conn.execute(
|
|
266
|
+
"""
|
|
267
|
+
INSERT INTO responses (
|
|
268
|
+
scope, method, path, path_family, params_json, payload_json, expires_at, updated_at,
|
|
269
|
+
created_at, cache_policy, ttl_seconds, payload_bytes, rows_count
|
|
270
|
+
)
|
|
271
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
272
|
+
ON CONFLICT(scope, method, path, params_json)
|
|
273
|
+
DO UPDATE SET
|
|
274
|
+
path_family = excluded.path_family,
|
|
275
|
+
payload_json = excluded.payload_json,
|
|
276
|
+
expires_at = excluded.expires_at,
|
|
277
|
+
updated_at = excluded.updated_at,
|
|
278
|
+
cache_policy = excluded.cache_policy,
|
|
279
|
+
ttl_seconds = excluded.ttl_seconds,
|
|
280
|
+
payload_bytes = excluded.payload_bytes,
|
|
281
|
+
rows_count = excluded.rows_count
|
|
282
|
+
""",
|
|
283
|
+
(
|
|
284
|
+
key.scope,
|
|
285
|
+
key.method,
|
|
286
|
+
key.path,
|
|
287
|
+
key.path_family,
|
|
288
|
+
key.params_json,
|
|
289
|
+
payload_json,
|
|
290
|
+
expires_at,
|
|
291
|
+
now,
|
|
292
|
+
now,
|
|
293
|
+
cache_policy,
|
|
294
|
+
ttl_seconds,
|
|
295
|
+
payload_bytes,
|
|
296
|
+
rows_count,
|
|
297
|
+
),
|
|
298
|
+
)
|
|
299
|
+
conn.commit()
|
|
300
|
+
|
|
301
|
+
def count_recent_family(
|
|
302
|
+
self, *, scope: str, method: str, path_family: str, since: float
|
|
303
|
+
) -> int:
|
|
304
|
+
conn = self._connect()
|
|
305
|
+
if conn is None:
|
|
306
|
+
return 0
|
|
307
|
+
with closing(conn), conn:
|
|
308
|
+
row = conn.execute(
|
|
309
|
+
"""
|
|
310
|
+
SELECT COUNT(*)
|
|
311
|
+
FROM responses
|
|
312
|
+
WHERE scope = ? AND method = ? AND path_family = ? AND updated_at >= ?
|
|
313
|
+
""",
|
|
314
|
+
(scope, method, path_family, since),
|
|
315
|
+
).fetchone()
|
|
316
|
+
return int(row[0]) if row is not None else 0
|
|
317
|
+
|
|
318
|
+
def extend_recent_family(
|
|
319
|
+
self,
|
|
320
|
+
*,
|
|
321
|
+
scope: str,
|
|
322
|
+
method: str,
|
|
323
|
+
path_family: str,
|
|
324
|
+
since: float,
|
|
325
|
+
ttl_seconds: int,
|
|
326
|
+
) -> int:
|
|
327
|
+
now = time.time()
|
|
328
|
+
expires_at = now + ttl_seconds
|
|
329
|
+
conn = self._connect()
|
|
330
|
+
if conn is None:
|
|
331
|
+
return 0
|
|
332
|
+
with closing(conn), conn:
|
|
333
|
+
cursor = conn.execute(
|
|
334
|
+
"""
|
|
335
|
+
UPDATE responses
|
|
336
|
+
SET
|
|
337
|
+
expires_at = CASE WHEN expires_at < ? THEN ? ELSE expires_at END,
|
|
338
|
+
ttl_seconds = CASE WHEN ttl_seconds < ? THEN ? ELSE ttl_seconds END
|
|
339
|
+
WHERE scope = ? AND method = ? AND path_family = ? AND updated_at >= ?
|
|
340
|
+
""",
|
|
341
|
+
(
|
|
342
|
+
expires_at,
|
|
343
|
+
expires_at,
|
|
344
|
+
ttl_seconds,
|
|
345
|
+
ttl_seconds,
|
|
346
|
+
scope,
|
|
347
|
+
method,
|
|
348
|
+
path_family,
|
|
349
|
+
since,
|
|
350
|
+
),
|
|
351
|
+
)
|
|
352
|
+
conn.commit()
|
|
353
|
+
return int(cursor.rowcount)
|
|
354
|
+
|
|
355
|
+
def clear_scope(self, scope: str) -> None:
|
|
356
|
+
conn = self._connect()
|
|
357
|
+
if conn is None:
|
|
358
|
+
return
|
|
359
|
+
with closing(conn), conn:
|
|
360
|
+
conn.execute("DELETE FROM responses WHERE scope = ?", (scope,))
|
|
361
|
+
|
|
362
|
+
|
|
363
|
+
@dataclass(slots=True)
|
|
364
|
+
class ExecutionContext:
|
|
365
|
+
profile: ResolvedProfile
|
|
366
|
+
reporter: DebugReporter | None = None
|
|
367
|
+
persistent_cache: PersistentCache | None = None
|
|
368
|
+
stats: ExecutionStats = field(default_factory=ExecutionStats)
|
|
369
|
+
_request_cache: dict[RequestCacheKey, Any] = field(default_factory=dict, init=False)
|
|
370
|
+
_inflight: dict[RequestCacheKey, asyncio.Task[Any]] = field(default_factory=dict, init=False)
|
|
371
|
+
_lock: asyncio.Lock = field(default_factory=asyncio.Lock, init=False)
|
|
372
|
+
_rate_lock: asyncio.Lock = field(default_factory=asyncio.Lock, init=False)
|
|
373
|
+
_last_request_time: float = field(default=0.0, init=False)
|
|
374
|
+
|
|
375
|
+
@classmethod
|
|
376
|
+
def for_profile(
|
|
377
|
+
cls, profile: ResolvedProfile, reporter: DebugReporter | None = None
|
|
378
|
+
) -> ExecutionContext:
|
|
379
|
+
persistent = None
|
|
380
|
+
if profile.cache_mode in {CACHE_MODE_AUTO, CACHE_MODE_READWRITE, CACHE_MODE_REFRESH}:
|
|
381
|
+
persistent = PersistentCache(persistent_cache_path(), reporter=reporter)
|
|
382
|
+
return cls(
|
|
383
|
+
profile=profile,
|
|
384
|
+
reporter=reporter,
|
|
385
|
+
persistent_cache=persistent,
|
|
386
|
+
stats=ExecutionStats(
|
|
387
|
+
cache_mode=profile.cache_mode,
|
|
388
|
+
cache_ttl_seconds=(
|
|
389
|
+
profile.cache_ttl_seconds
|
|
390
|
+
if profile.cache_mode in {CACHE_MODE_READWRITE, CACHE_MODE_REFRESH}
|
|
391
|
+
else None
|
|
392
|
+
),
|
|
393
|
+
),
|
|
394
|
+
)
|
|
395
|
+
|
|
396
|
+
@property
|
|
397
|
+
def scope(self) -> str:
|
|
398
|
+
credential_fingerprint = hashlib.sha256(self.profile.token.encode("utf-8")).hexdigest()[:16]
|
|
399
|
+
scope_components = json.dumps(
|
|
400
|
+
[
|
|
401
|
+
self.profile.domain,
|
|
402
|
+
self.profile.name or "environment",
|
|
403
|
+
credential_fingerprint,
|
|
404
|
+
],
|
|
405
|
+
ensure_ascii=False,
|
|
406
|
+
separators=(",", ":"),
|
|
407
|
+
)
|
|
408
|
+
return f"scope-{hashlib.sha256(scope_components.encode('utf-8')).hexdigest()}"
|
|
409
|
+
|
|
410
|
+
async def wait_for_rate_slot(self, delay_seconds: float) -> None:
|
|
411
|
+
"""Share one low-load request budget across all clients in this execution."""
|
|
412
|
+
|
|
413
|
+
async with self._rate_lock:
|
|
414
|
+
now = asyncio.get_running_loop().time()
|
|
415
|
+
elapsed = now - self._last_request_time
|
|
416
|
+
if elapsed < delay_seconds:
|
|
417
|
+
await asyncio.sleep(delay_seconds - elapsed)
|
|
418
|
+
self._last_request_time = asyncio.get_running_loop().time()
|
|
419
|
+
|
|
420
|
+
def _debug(self, message: str) -> None:
|
|
421
|
+
if self.reporter is not None:
|
|
422
|
+
self.reporter(message)
|
|
423
|
+
|
|
424
|
+
def _make_key(self, method: str, path: str, params: dict[str, Any] | None) -> RequestCacheKey:
|
|
425
|
+
normalized_params = _normalize_params(params)
|
|
426
|
+
params_json = json.dumps(
|
|
427
|
+
normalized_params, ensure_ascii=False, sort_keys=True, separators=(",", ":")
|
|
428
|
+
)
|
|
429
|
+
return RequestCacheKey(
|
|
430
|
+
scope=self.scope,
|
|
431
|
+
method=method.upper(),
|
|
432
|
+
path=path,
|
|
433
|
+
params_json=params_json,
|
|
434
|
+
path_family=path_family_for(path),
|
|
435
|
+
)
|
|
436
|
+
|
|
437
|
+
def _persistent_allowed(self, cache_policy: str) -> bool:
|
|
438
|
+
return (
|
|
439
|
+
cache_policy in PERSISTENT_CACHE_POLICIES
|
|
440
|
+
and self.persistent_cache is not None
|
|
441
|
+
and self.profile.cache_mode
|
|
442
|
+
in {CACHE_MODE_AUTO, CACHE_MODE_READWRITE, CACHE_MODE_REFRESH}
|
|
443
|
+
)
|
|
444
|
+
|
|
445
|
+
def _read_from_disk(self, key: RequestCacheKey, *, cache_policy: str) -> Any | None:
|
|
446
|
+
if cache_policy == CACHE_POLICY_NONE:
|
|
447
|
+
return None
|
|
448
|
+
if self.persistent_cache is None or cache_policy not in PERSISTENT_CACHE_POLICIES:
|
|
449
|
+
self.stats.record_cache_bypass(
|
|
450
|
+
cache="disk", method=key.method, path_family=key.path_family
|
|
451
|
+
)
|
|
452
|
+
self._debug(
|
|
453
|
+
tr(
|
|
454
|
+
"cache: disk bypass method={value_0} path={value_1}",
|
|
455
|
+
value_0=key.method,
|
|
456
|
+
value_1=key.path,
|
|
457
|
+
)
|
|
458
|
+
)
|
|
459
|
+
return None
|
|
460
|
+
if self.profile.cache_mode == CACHE_MODE_REFRESH:
|
|
461
|
+
self.stats.record_cache_bypass(
|
|
462
|
+
cache="disk", method=key.method, path_family=key.path_family
|
|
463
|
+
)
|
|
464
|
+
self._debug(
|
|
465
|
+
tr(
|
|
466
|
+
"cache: disk bypass refresh method={value_0} path={value_1}",
|
|
467
|
+
value_0=key.method,
|
|
468
|
+
value_1=key.path,
|
|
469
|
+
)
|
|
470
|
+
)
|
|
471
|
+
return None
|
|
472
|
+
try:
|
|
473
|
+
status, payload = self.persistent_cache.get(key)
|
|
474
|
+
except (OSError, sqlite3.Error, ValueError, json.JSONDecodeError) as exc:
|
|
475
|
+
self.stats.record_cache_bypass(
|
|
476
|
+
cache="disk", method=key.method, path_family=key.path_family
|
|
477
|
+
)
|
|
478
|
+
self._debug(
|
|
479
|
+
tr(
|
|
480
|
+
"cache: disk bypass method={value_0} path={value_1} reason={value_2}",
|
|
481
|
+
value_0=key.method,
|
|
482
|
+
value_1=key.path,
|
|
483
|
+
value_2=type(exc).__name__,
|
|
484
|
+
)
|
|
485
|
+
)
|
|
486
|
+
return None
|
|
487
|
+
if status == "hit":
|
|
488
|
+
self.stats.record_cache_hit(
|
|
489
|
+
cache="disk", method=key.method, path_family=key.path_family
|
|
490
|
+
)
|
|
491
|
+
elif status == "miss":
|
|
492
|
+
self.stats.record_cache_miss(
|
|
493
|
+
cache="disk", method=key.method, path_family=key.path_family
|
|
494
|
+
)
|
|
495
|
+
elif status == "expired":
|
|
496
|
+
self.stats.record_cache_miss(
|
|
497
|
+
cache="disk_expired", method=key.method, path_family=key.path_family
|
|
498
|
+
)
|
|
499
|
+
self._debug(
|
|
500
|
+
tr(
|
|
501
|
+
"cache: disk {value_0} method={value_1} path={value_2}",
|
|
502
|
+
value_0=status,
|
|
503
|
+
value_1=key.method,
|
|
504
|
+
value_2=key.path,
|
|
505
|
+
)
|
|
506
|
+
)
|
|
507
|
+
return payload
|
|
508
|
+
|
|
509
|
+
def _write_to_disk(self, key: RequestCacheKey, payload: Any, *, cache_policy: str) -> None:
|
|
510
|
+
if not self._persistent_allowed(cache_policy):
|
|
511
|
+
return
|
|
512
|
+
payload_json = json.dumps(payload, ensure_ascii=False, separators=(",", ":"))
|
|
513
|
+
payload_bytes = len(payload_json.encode("utf-8"))
|
|
514
|
+
rows_count = _payload_rows_count(payload)
|
|
515
|
+
ttl_seconds = self._ttl_seconds_for_payload(
|
|
516
|
+
key,
|
|
517
|
+
payload_bytes=payload_bytes,
|
|
518
|
+
rows_count=rows_count,
|
|
519
|
+
cache_policy=cache_policy,
|
|
520
|
+
)
|
|
521
|
+
try:
|
|
522
|
+
self.persistent_cache.set(
|
|
523
|
+
key,
|
|
524
|
+
payload,
|
|
525
|
+
ttl_seconds=ttl_seconds,
|
|
526
|
+
cache_policy=cache_policy,
|
|
527
|
+
payload_bytes=payload_bytes,
|
|
528
|
+
rows_count=rows_count,
|
|
529
|
+
)
|
|
530
|
+
self._debug(
|
|
531
|
+
tr(
|
|
532
|
+
"cache: disk write method={value_0} path={value_1} ttl_seconds={value_2} rows={value_3} bytes={value_4} family={value_5}",
|
|
533
|
+
value_0=key.method,
|
|
534
|
+
value_1=key.path,
|
|
535
|
+
value_2=ttl_seconds,
|
|
536
|
+
value_3=rows_count,
|
|
537
|
+
value_4=payload_bytes,
|
|
538
|
+
value_5=key.path_family,
|
|
539
|
+
)
|
|
540
|
+
)
|
|
541
|
+
self._extend_dense_family(key, ttl_seconds=ttl_seconds)
|
|
542
|
+
except (OSError, TypeError, sqlite3.Error, ValueError) as exc:
|
|
543
|
+
self._debug(
|
|
544
|
+
tr(
|
|
545
|
+
"cache: disk bypass method={value_0} path={value_1} reason={value_2}",
|
|
546
|
+
value_0=key.method,
|
|
547
|
+
value_1=key.path,
|
|
548
|
+
value_2=type(exc).__name__,
|
|
549
|
+
)
|
|
550
|
+
)
|
|
551
|
+
|
|
552
|
+
def _ttl_seconds_for_payload(
|
|
553
|
+
self,
|
|
554
|
+
key: RequestCacheKey,
|
|
555
|
+
*,
|
|
556
|
+
payload_bytes: int,
|
|
557
|
+
rows_count: int,
|
|
558
|
+
cache_policy: str,
|
|
559
|
+
) -> int:
|
|
560
|
+
if self.profile.cache_mode != CACHE_MODE_AUTO:
|
|
561
|
+
return self.profile.cache_ttl_seconds
|
|
562
|
+
if _looks_historical(key.params_json):
|
|
563
|
+
return AUTO_HISTORICAL_TTL_SECONDS
|
|
564
|
+
if cache_policy == CACHE_POLICY_PERSISTENT_HEAVY:
|
|
565
|
+
return AUTO_HEAVY_TTL_SECONDS
|
|
566
|
+
if rows_count >= AUTO_HEAVY_ROW_THRESHOLD or payload_bytes >= AUTO_HEAVY_PAYLOAD_BYTES:
|
|
567
|
+
return AUTO_HEAVY_TTL_SECONDS
|
|
568
|
+
if rows_count >= AUTO_MEDIUM_ROW_THRESHOLD or payload_bytes >= AUTO_MEDIUM_PAYLOAD_BYTES:
|
|
569
|
+
return AUTO_MEDIUM_TTL_SECONDS
|
|
570
|
+
recent_family_count = self._recent_family_count(key) + 1
|
|
571
|
+
if recent_family_count >= AUTO_DENSE_FAMILY_HEAVY_THRESHOLD:
|
|
572
|
+
return AUTO_HEAVY_TTL_SECONDS
|
|
573
|
+
if recent_family_count >= AUTO_DENSE_FAMILY_MEDIUM_THRESHOLD:
|
|
574
|
+
return AUTO_MEDIUM_TTL_SECONDS
|
|
575
|
+
return AUTO_ENTITY_TTL_SECONDS
|
|
576
|
+
|
|
577
|
+
def _recent_family_count(self, key: RequestCacheKey) -> int:
|
|
578
|
+
if self.persistent_cache is None:
|
|
579
|
+
return 0
|
|
580
|
+
since = time.time() - AUTO_DENSE_FAMILY_WINDOW_SECONDS
|
|
581
|
+
try:
|
|
582
|
+
return self.persistent_cache.count_recent_family(
|
|
583
|
+
scope=key.scope,
|
|
584
|
+
method=key.method,
|
|
585
|
+
path_family=key.path_family,
|
|
586
|
+
since=since,
|
|
587
|
+
)
|
|
588
|
+
except (OSError, sqlite3.Error):
|
|
589
|
+
return 0
|
|
590
|
+
|
|
591
|
+
def _extend_dense_family(self, key: RequestCacheKey, *, ttl_seconds: int) -> None:
|
|
592
|
+
if (
|
|
593
|
+
self.persistent_cache is None
|
|
594
|
+
or self.profile.cache_mode != CACHE_MODE_AUTO
|
|
595
|
+
or ttl_seconds < AUTO_MEDIUM_TTL_SECONDS
|
|
596
|
+
):
|
|
597
|
+
return
|
|
598
|
+
since = time.time() - AUTO_DENSE_FAMILY_WINDOW_SECONDS
|
|
599
|
+
try:
|
|
600
|
+
updated = self.persistent_cache.extend_recent_family(
|
|
601
|
+
scope=key.scope,
|
|
602
|
+
method=key.method,
|
|
603
|
+
path_family=key.path_family,
|
|
604
|
+
since=since,
|
|
605
|
+
ttl_seconds=ttl_seconds,
|
|
606
|
+
)
|
|
607
|
+
except (OSError, sqlite3.Error) as exc:
|
|
608
|
+
self._debug(
|
|
609
|
+
tr(
|
|
610
|
+
"cache: dense family extend bypass family={value_0} reason={value_1}",
|
|
611
|
+
value_0=key.path_family,
|
|
612
|
+
value_1=type(exc).__name__,
|
|
613
|
+
)
|
|
614
|
+
)
|
|
615
|
+
return
|
|
616
|
+
if updated:
|
|
617
|
+
self._debug(
|
|
618
|
+
tr(
|
|
619
|
+
"cache: dense family extended method={value_0} family={value_1} ttl_seconds={value_2} rows={value_3}",
|
|
620
|
+
value_0=key.method,
|
|
621
|
+
value_1=key.path_family,
|
|
622
|
+
value_2=ttl_seconds,
|
|
623
|
+
value_3=updated,
|
|
624
|
+
)
|
|
625
|
+
)
|
|
626
|
+
|
|
627
|
+
async def _load_or_fetch(
|
|
628
|
+
self,
|
|
629
|
+
key: RequestCacheKey,
|
|
630
|
+
*,
|
|
631
|
+
cache_policy: str,
|
|
632
|
+
fetch,
|
|
633
|
+
) -> Any:
|
|
634
|
+
cached = self._read_from_disk(key, cache_policy=cache_policy)
|
|
635
|
+
if cached is not None:
|
|
636
|
+
async with self._lock:
|
|
637
|
+
self._request_cache[key] = copy.deepcopy(cached)
|
|
638
|
+
return copy.deepcopy(cached)
|
|
639
|
+
|
|
640
|
+
payload = await fetch()
|
|
641
|
+
if payload is not None:
|
|
642
|
+
async with self._lock:
|
|
643
|
+
self._request_cache[key] = copy.deepcopy(payload)
|
|
644
|
+
self._write_to_disk(key, payload, cache_policy=cache_policy)
|
|
645
|
+
return copy.deepcopy(payload)
|
|
646
|
+
|
|
647
|
+
async def get_json(
|
|
648
|
+
self,
|
|
649
|
+
*,
|
|
650
|
+
method: str,
|
|
651
|
+
path: str,
|
|
652
|
+
params: dict[str, Any] | None,
|
|
653
|
+
cache_policy: str,
|
|
654
|
+
fetch,
|
|
655
|
+
) -> Any:
|
|
656
|
+
if method.upper() != "GET" or cache_policy == CACHE_POLICY_NONE:
|
|
657
|
+
return await fetch()
|
|
658
|
+
|
|
659
|
+
key = self._make_key(method, path, params)
|
|
660
|
+
async with self._lock:
|
|
661
|
+
if key in self._request_cache:
|
|
662
|
+
self.stats.record_cache_hit(
|
|
663
|
+
cache="request", method=key.method, path_family=key.path_family
|
|
664
|
+
)
|
|
665
|
+
self._debug(
|
|
666
|
+
tr(
|
|
667
|
+
"cache: request hit method={value_0} path={value_1}",
|
|
668
|
+
value_0=key.method,
|
|
669
|
+
value_1=key.path,
|
|
670
|
+
)
|
|
671
|
+
)
|
|
672
|
+
return copy.deepcopy(self._request_cache[key])
|
|
673
|
+
task = self._inflight.get(key)
|
|
674
|
+
if task is None:
|
|
675
|
+
self.stats.record_cache_miss(
|
|
676
|
+
cache="request", method=key.method, path_family=key.path_family
|
|
677
|
+
)
|
|
678
|
+
self._debug(
|
|
679
|
+
tr(
|
|
680
|
+
"cache: request miss method={value_0} path={value_1}",
|
|
681
|
+
value_0=key.method,
|
|
682
|
+
value_1=key.path,
|
|
683
|
+
)
|
|
684
|
+
)
|
|
685
|
+
task = asyncio.create_task(
|
|
686
|
+
self._load_or_fetch(key, cache_policy=cache_policy, fetch=fetch)
|
|
687
|
+
)
|
|
688
|
+
self._inflight[key] = task
|
|
689
|
+
else:
|
|
690
|
+
self.stats.record_cache_hit(
|
|
691
|
+
cache="inflight_dedup", method=key.method, path_family=key.path_family
|
|
692
|
+
)
|
|
693
|
+
self._debug(
|
|
694
|
+
tr(
|
|
695
|
+
"cache: inflight dedup hit method={value_0} path={value_1}",
|
|
696
|
+
value_0=key.method,
|
|
697
|
+
value_1=key.path,
|
|
698
|
+
)
|
|
699
|
+
)
|
|
700
|
+
|
|
701
|
+
try:
|
|
702
|
+
return copy.deepcopy(await task)
|
|
703
|
+
finally:
|
|
704
|
+
if task.done():
|
|
705
|
+
async with self._lock:
|
|
706
|
+
if self._inflight.get(key) is task:
|
|
707
|
+
self._inflight.pop(key, None)
|
|
708
|
+
|
|
709
|
+
async def clear_cache_scope(self, *, reason: str) -> None:
|
|
710
|
+
async with self._lock:
|
|
711
|
+
self._request_cache.clear()
|
|
712
|
+
if self.persistent_cache is not None:
|
|
713
|
+
try:
|
|
714
|
+
self.persistent_cache.clear_scope(self.scope)
|
|
715
|
+
self._debug(
|
|
716
|
+
tr(
|
|
717
|
+
"cache: profile cleared scope={value_0} reason={value_1}",
|
|
718
|
+
value_0=self.scope,
|
|
719
|
+
value_1=reason,
|
|
720
|
+
)
|
|
721
|
+
)
|
|
722
|
+
except (OSError, sqlite3.Error) as exc:
|
|
723
|
+
self._debug(
|
|
724
|
+
tr(
|
|
725
|
+
"cache: clear bypass scope={value_0} reason={value_1} error={value_2}",
|
|
726
|
+
value_0=self.scope,
|
|
727
|
+
value_1=reason,
|
|
728
|
+
value_2=type(exc).__name__,
|
|
729
|
+
)
|
|
730
|
+
)
|
|
731
|
+
|
|
732
|
+
async def invalidate_after_mutation(self) -> None:
|
|
733
|
+
await self.clear_cache_scope(reason="mutation")
|