agent-foundry-runtime 0.4.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- agent_foundry/__init__.py +2 -0
- agent_foundry/__main__.py +5 -0
- agent_foundry/cli.py +613 -0
- agent_foundry/config.py +190 -0
- agent_foundry/daemon.py +220 -0
- agent_foundry/eval.py +93 -0
- agent_foundry/executor.py +159 -0
- agent_foundry/indexer.py +272 -0
- agent_foundry/judge.py +88 -0
- agent_foundry/logging_db.py +437 -0
- agent_foundry/loop.py +226 -0
- agent_foundry/mcp_server.py +191 -0
- agent_foundry/models.py +166 -0
- agent_foundry/planner.py +288 -0
- agent_foundry_runtime-0.4.0.dist-info/METADATA +382 -0
- agent_foundry_runtime-0.4.0.dist-info/RECORD +21 -0
- agent_foundry_runtime-0.4.0.dist-info/WHEEL +5 -0
- agent_foundry_runtime-0.4.0.dist-info/entry_points.txt +2 -0
- agent_foundry_runtime-0.4.0.dist-info/licenses/AUTHORSHIP.md +7 -0
- agent_foundry_runtime-0.4.0.dist-info/licenses/LICENSE +21 -0
- agent_foundry_runtime-0.4.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,437 @@
|
|
|
1
|
+
"""SQLite logging with feedback + instincts schema.
|
|
2
|
+
|
|
3
|
+
P1: feedback column, instincts table, learn queries.
|
|
4
|
+
"""
|
|
5
|
+
from __future__ import annotations
|
|
6
|
+
|
|
7
|
+
import sqlite3
|
|
8
|
+
import re
|
|
9
|
+
from collections import Counter, defaultdict
|
|
10
|
+
from datetime import datetime, timezone
|
|
11
|
+
from pathlib import Path
|
|
12
|
+
from typing import Any
|
|
13
|
+
import hashlib
|
|
14
|
+
|
|
15
|
+
# Engram-style N-gram constants
|
|
16
|
+
ENGRAM_STOPWORDS = frozenset({
|
|
17
|
+
"the", "and", "for", "that", "this", "with", "from", "how",
|
|
18
|
+
"what", "when", "why", "where", "which", "your", "have",
|
|
19
|
+
"will", "can", "are", "not", "but", "all", "was", "out", "use",
|
|
20
|
+
})
|
|
21
|
+
ENGRAM_N = 3 # trigrams
|
|
22
|
+
|
|
23
|
+
def _ngram_hash(prompt: str) -> str:
|
|
24
|
+
"""Engram-style N-gram fingerprint (DeepSeek Engram paper).
|
|
25
|
+
|
|
26
|
+
Lowercase, strip punctuation, tokenize, generate N-grams, SHA-1 the
|
|
27
|
+
joined canonical form, return the first 16 hex chars.
|
|
28
|
+
|
|
29
|
+
Returns "" if no usable tokens after stopword filtering.
|
|
30
|
+
"""
|
|
31
|
+
tokens = [t for t in re.findall(r"[a-z]{3,}", prompt.lower())
|
|
32
|
+
if t not in ENGRAM_STOPWORDS]
|
|
33
|
+
if len(tokens) < ENGRAM_N:
|
|
34
|
+
return ""
|
|
35
|
+
ngrams = [" ".join(tokens[i:i + ENGRAM_N]) for i in range(len(tokens) - ENGRAM_N + 1)]
|
|
36
|
+
canonical = "\n".join(sorted(set(ngrams))).encode("utf-8")
|
|
37
|
+
return hashlib.sha1(canonical).hexdigest()[:16]
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def engram_lookup(db_path: Path, prompt: str, min_confidence: float = 0.25) -> dict | None:
|
|
41
|
+
"""O(1) Engram-style routing lookup.
|
|
42
|
+
|
|
43
|
+
Returns the highest-confidence instinct matching this prompt's
|
|
44
|
+
N-gram fingerprint, or None if no match above min_confidence.
|
|
45
|
+
|
|
46
|
+
Default min_confidence=0.25: matches freshly-learned instincts (initial
|
|
47
|
+
confidence 0.3). Bump to 0.5+ for high-stakes routing.
|
|
48
|
+
|
|
49
|
+
Caller should fall back to the full plan flow when this returns None.
|
|
50
|
+
"""
|
|
51
|
+
init_db(db_path)
|
|
52
|
+
h = _ngram_hash(prompt)
|
|
53
|
+
if not h:
|
|
54
|
+
return None
|
|
55
|
+
conn = sqlite3.connect(str(db_path), isolation_level=None)
|
|
56
|
+
try:
|
|
57
|
+
row = conn.execute(
|
|
58
|
+
"""SELECT id, source_skill_id, trigger_pattern, confidence, samples
|
|
59
|
+
FROM instincts
|
|
60
|
+
WHERE ngram_hash = ? AND approved = 1 AND confidence >= ?
|
|
61
|
+
ORDER BY confidence DESC, samples DESC
|
|
62
|
+
LIMIT 1""",
|
|
63
|
+
(h, min_confidence),
|
|
64
|
+
).fetchone()
|
|
65
|
+
finally:
|
|
66
|
+
conn.close()
|
|
67
|
+
if not row:
|
|
68
|
+
return None
|
|
69
|
+
return {
|
|
70
|
+
"id": row[0],
|
|
71
|
+
"skill_id": row[1],
|
|
72
|
+
"trigger_pattern": row[2],
|
|
73
|
+
"confidence": row[3],
|
|
74
|
+
"samples": row[4],
|
|
75
|
+
"matched_via": "engram",
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
SCHEMA = """
|
|
79
|
+
CREATE TABLE IF NOT EXISTS executions (
|
|
80
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
81
|
+
timestamp DATETIME NOT NULL,
|
|
82
|
+
skill_id TEXT NOT NULL,
|
|
83
|
+
prompt TEXT NOT NULL,
|
|
84
|
+
output TEXT,
|
|
85
|
+
tokens_used INTEGER NOT NULL DEFAULT 0,
|
|
86
|
+
duration_seconds REAL NOT NULL DEFAULT 0,
|
|
87
|
+
success INTEGER NOT NULL DEFAULT 0,
|
|
88
|
+
error TEXT,
|
|
89
|
+
planner_score REAL,
|
|
90
|
+
was_fallback INTEGER NOT NULL DEFAULT 0,
|
|
91
|
+
judge_corr REAL,
|
|
92
|
+
judge_slop REAL,
|
|
93
|
+
judge_scope REAL,
|
|
94
|
+
judge_verdict TEXT,
|
|
95
|
+
feedback INTEGER,
|
|
96
|
+
cost_usd REAL NOT NULL DEFAULT 0
|
|
97
|
+
);
|
|
98
|
+
CREATE INDEX IF NOT EXISTS idx_executions_ts ON executions(timestamp);
|
|
99
|
+
CREATE INDEX IF NOT EXISTS idx_executions_skill ON executions(skill_id);
|
|
100
|
+
|
|
101
|
+
CREATE TABLE IF NOT EXISTS skill_weights (
|
|
102
|
+
skill_id TEXT PRIMARY KEY,
|
|
103
|
+
weight REAL NOT NULL DEFAULT 1.0,
|
|
104
|
+
samples INTEGER NOT NULL DEFAULT 0,
|
|
105
|
+
updated DATETIME NOT NULL
|
|
106
|
+
);
|
|
107
|
+
|
|
108
|
+
CREATE TABLE IF NOT EXISTS loop_sessions (
|
|
109
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
110
|
+
prompt TEXT NOT NULL,
|
|
111
|
+
skill_id TEXT NOT NULL,
|
|
112
|
+
status TEXT NOT NULL DEFAULT 'running',
|
|
113
|
+
turns INTEGER NOT NULL DEFAULT 0,
|
|
114
|
+
output TEXT,
|
|
115
|
+
created DATETIME NOT NULL,
|
|
116
|
+
updated DATETIME NOT NULL
|
|
117
|
+
);
|
|
118
|
+
|
|
119
|
+
CREATE TABLE IF NOT EXISTS instincts (
|
|
120
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
121
|
+
source_skill_id TEXT NOT NULL,
|
|
122
|
+
trigger_pattern TEXT NOT NULL,
|
|
123
|
+
ngram_hash TEXT, -- Engram-style: SHA-1 hex of normalized prompt N-grams
|
|
124
|
+
confidence REAL NOT NULL DEFAULT 0.0,
|
|
125
|
+
samples INTEGER NOT NULL DEFAULT 1,
|
|
126
|
+
created DATETIME NOT NULL,
|
|
127
|
+
last_seen DATETIME NOT NULL,
|
|
128
|
+
approved INTEGER NOT NULL DEFAULT 0
|
|
129
|
+
);
|
|
130
|
+
CREATE INDEX IF NOT EXISTS idx_instincts_skill ON instincts(source_skill_id);
|
|
131
|
+
CREATE INDEX IF NOT EXISTS idx_instincts_ngram ON instincts(ngram_hash);
|
|
132
|
+
"""
|
|
133
|
+
|
|
134
|
+
MIGRATIONS = [
|
|
135
|
+
"ALTER TABLE executions ADD COLUMN feedback INTEGER",
|
|
136
|
+
"CREATE TABLE IF NOT EXISTS instincts ("
|
|
137
|
+
"id INTEGER PRIMARY KEY AUTOINCREMENT,"
|
|
138
|
+
"source_skill_id TEXT NOT NULL,"
|
|
139
|
+
"trigger_pattern TEXT NOT NULL,"
|
|
140
|
+
"confidence REAL NOT NULL DEFAULT 0.0,"
|
|
141
|
+
"samples INTEGER NOT NULL DEFAULT 1,"
|
|
142
|
+
"created DATETIME NOT NULL,"
|
|
143
|
+
"last_seen DATETIME NOT NULL,"
|
|
144
|
+
"approved INTEGER NOT NULL DEFAULT 0"
|
|
145
|
+
")",
|
|
146
|
+
"CREATE INDEX IF NOT EXISTS idx_instincts_skill ON instincts(source_skill_id)",
|
|
147
|
+
# v3 -> v4: Engram-style N-gram hash column on instincts
|
|
148
|
+
"ALTER TABLE instincts ADD COLUMN ngram_hash TEXT",
|
|
149
|
+
"CREATE INDEX IF NOT EXISTS idx_instincts_ngram ON instincts(ngram_hash)",
|
|
150
|
+
# v5: cost tracking
|
|
151
|
+
"ALTER TABLE executions ADD COLUMN cost_usd REAL NOT NULL DEFAULT 0",
|
|
152
|
+
]
|
|
153
|
+
|
|
154
|
+
def init_db(db_path: Path) -> None:
|
|
155
|
+
db_path.parent.mkdir(parents=True, exist_ok=True)
|
|
156
|
+
conn = sqlite3.connect(str(db_path), isolation_level=None) # autocommit
|
|
157
|
+
try:
|
|
158
|
+
conn.executescript(SCHEMA)
|
|
159
|
+
# Apply migrations (safe to re-run)
|
|
160
|
+
for mig in MIGRATIONS:
|
|
161
|
+
try:
|
|
162
|
+
conn.execute(mig)
|
|
163
|
+
except sqlite3.OperationalError:
|
|
164
|
+
pass # column already exists / table already exists
|
|
165
|
+
finally:
|
|
166
|
+
conn.close()
|
|
167
|
+
|
|
168
|
+
def log_execution(db_path: Path, *, skill_id: str, prompt: str, output: str,
|
|
169
|
+
tokens_used: int, duration_seconds: float, success: bool,
|
|
170
|
+
error: str | None = None, planner_score: float | None = None,
|
|
171
|
+
was_fallback: bool = False,
|
|
172
|
+
judge_corr: float | None = None,
|
|
173
|
+
judge_slop: float | None = None,
|
|
174
|
+
judge_scope: float | None = None,
|
|
175
|
+
judge_verdict: str | None = None,
|
|
176
|
+
cost_usd: float = 0.0) -> int:
|
|
177
|
+
init_db(db_path)
|
|
178
|
+
conn = sqlite3.connect(str(db_path), isolation_level=None)
|
|
179
|
+
try:
|
|
180
|
+
cur = conn.execute(
|
|
181
|
+
"""INSERT INTO executions
|
|
182
|
+
(timestamp, skill_id, prompt, output, tokens_used, duration_seconds,
|
|
183
|
+
success, error, planner_score, was_fallback,
|
|
184
|
+
judge_corr, judge_slop, judge_scope, judge_verdict, cost_usd)
|
|
185
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""",
|
|
186
|
+
(
|
|
187
|
+
datetime.now(timezone.utc).isoformat(),
|
|
188
|
+
skill_id, prompt, output, int(tokens_used), float(duration_seconds),
|
|
189
|
+
1 if success else 0, error, planner_score,
|
|
190
|
+
1 if was_fallback else 0,
|
|
191
|
+
judge_corr, judge_slop, judge_scope, judge_verdict,
|
|
192
|
+
float(cost_usd),
|
|
193
|
+
),
|
|
194
|
+
)
|
|
195
|
+
return int(cur.lastrowid)
|
|
196
|
+
finally:
|
|
197
|
+
conn.close()
|
|
198
|
+
|
|
199
|
+
|
|
200
|
+
def get_skill_weights(db_path: Path) -> dict[str, float]:
|
|
201
|
+
"""Learned per-skill multipliers from judge scores and feedback."""
|
|
202
|
+
if not db_path.exists():
|
|
203
|
+
return {}
|
|
204
|
+
init_db(db_path)
|
|
205
|
+
conn = sqlite3.connect(str(db_path))
|
|
206
|
+
try:
|
|
207
|
+
rows = conn.execute("SELECT skill_id, weight FROM skill_weights").fetchall()
|
|
208
|
+
finally:
|
|
209
|
+
conn.close()
|
|
210
|
+
return {r[0]: float(r[1]) for r in rows}
|
|
211
|
+
|
|
212
|
+
|
|
213
|
+
def create_loop_session(db_path: Path, prompt: str, skill_id: str) -> int:
|
|
214
|
+
init_db(db_path)
|
|
215
|
+
now = datetime.now(timezone.utc).isoformat()
|
|
216
|
+
conn = sqlite3.connect(str(db_path), isolation_level=None)
|
|
217
|
+
try:
|
|
218
|
+
cur = conn.execute(
|
|
219
|
+
"""INSERT INTO loop_sessions (prompt, skill_id, status, turns, created, updated)
|
|
220
|
+
VALUES (?, ?, 'running', 0, ?, ?)""",
|
|
221
|
+
(prompt, skill_id, now, now),
|
|
222
|
+
)
|
|
223
|
+
return int(cur.lastrowid)
|
|
224
|
+
finally:
|
|
225
|
+
conn.close()
|
|
226
|
+
|
|
227
|
+
|
|
228
|
+
def update_loop_session(db_path: Path, session_id: int, *, status: str | None = None,
|
|
229
|
+
output: str | None = None, increment_turn: bool = False) -> None:
|
|
230
|
+
init_db(db_path)
|
|
231
|
+
now = datetime.now(timezone.utc).isoformat()
|
|
232
|
+
conn = sqlite3.connect(str(db_path), isolation_level=None)
|
|
233
|
+
try:
|
|
234
|
+
if status is not None:
|
|
235
|
+
conn.execute("UPDATE loop_sessions SET status = ?, updated = ? WHERE id = ?",
|
|
236
|
+
(status, now, session_id))
|
|
237
|
+
if output is not None:
|
|
238
|
+
conn.execute("UPDATE loop_sessions SET output = ?, updated = ? WHERE id = ?",
|
|
239
|
+
(output, now, session_id))
|
|
240
|
+
if increment_turn:
|
|
241
|
+
conn.execute("UPDATE loop_sessions SET turns = turns + 1, updated = ? WHERE id = ?",
|
|
242
|
+
(now, session_id))
|
|
243
|
+
finally:
|
|
244
|
+
conn.close()
|
|
245
|
+
|
|
246
|
+
|
|
247
|
+
def get_loop_session(db_path: Path, session_id: int) -> dict | None:
|
|
248
|
+
if not db_path.exists():
|
|
249
|
+
return None
|
|
250
|
+
init_db(db_path)
|
|
251
|
+
conn = sqlite3.connect(str(db_path))
|
|
252
|
+
try:
|
|
253
|
+
row = conn.execute(
|
|
254
|
+
"""SELECT id, prompt, skill_id, status, turns, output, created, updated
|
|
255
|
+
FROM loop_sessions WHERE id = ?""",
|
|
256
|
+
(session_id,),
|
|
257
|
+
).fetchone()
|
|
258
|
+
finally:
|
|
259
|
+
conn.close()
|
|
260
|
+
if not row:
|
|
261
|
+
return None
|
|
262
|
+
return {"id": row[0], "prompt": row[1], "skill_id": row[2], "status": row[3],
|
|
263
|
+
"turns": row[4], "output": row[5], "created": row[6], "updated": row[7]}
|
|
264
|
+
|
|
265
|
+
|
|
266
|
+
def update_skill_weights(db_path: Path) -> int:
|
|
267
|
+
"""Recompute weights: judge mean plus feedback, clipped to [0.6, 1.4]."""
|
|
268
|
+
init_db(db_path)
|
|
269
|
+
now = datetime.now(timezone.utc).isoformat()
|
|
270
|
+
updated = 0
|
|
271
|
+
conn = sqlite3.connect(str(db_path), isolation_level=None)
|
|
272
|
+
try:
|
|
273
|
+
rows = conn.execute(
|
|
274
|
+
"""SELECT skill_id,
|
|
275
|
+
AVG((judge_corr + judge_slop + judge_scope) / 3.0),
|
|
276
|
+
SUM(CASE feedback WHEN 1 THEN 1 WHEN -1 THEN -1 ELSE 0 END),
|
|
277
|
+
COUNT(*)
|
|
278
|
+
FROM executions
|
|
279
|
+
WHERE judge_corr IS NOT NULL OR feedback IS NOT NULL
|
|
280
|
+
GROUP BY skill_id"""
|
|
281
|
+
).fetchall()
|
|
282
|
+
for skill_id, judge_mean, feedback_sum, samples in rows:
|
|
283
|
+
base = (judge_mean / 10.0) if judge_mean is not None else 0.5
|
|
284
|
+
weight = 0.8 + 0.4 * base + 0.05 * float(feedback_sum or 0) / max(1, samples)
|
|
285
|
+
weight = max(0.6, min(1.4, weight))
|
|
286
|
+
conn.execute(
|
|
287
|
+
"""INSERT INTO skill_weights (skill_id, weight, samples, updated)
|
|
288
|
+
VALUES (?, ?, ?, ?)
|
|
289
|
+
ON CONFLICT(skill_id) DO UPDATE SET
|
|
290
|
+
weight=excluded.weight, samples=excluded.samples, updated=excluded.updated""",
|
|
291
|
+
(skill_id, weight, samples, now),
|
|
292
|
+
)
|
|
293
|
+
updated += 1
|
|
294
|
+
finally:
|
|
295
|
+
conn.close()
|
|
296
|
+
return updated
|
|
297
|
+
|
|
298
|
+
def set_feedback(db_path: Path, execution_id: int, feedback: int) -> bool:
|
|
299
|
+
"""feedback: 1 = thumbs_up, -1 = thumbs_down."""
|
|
300
|
+
if feedback not in (1, -1):
|
|
301
|
+
return False
|
|
302
|
+
init_db(db_path)
|
|
303
|
+
conn = sqlite3.connect(str(db_path), isolation_level=None)
|
|
304
|
+
try:
|
|
305
|
+
cur = conn.execute("UPDATE executions SET feedback = ? WHERE id = ?", (feedback, execution_id))
|
|
306
|
+
return cur.rowcount > 0
|
|
307
|
+
finally:
|
|
308
|
+
conn.close()
|
|
309
|
+
|
|
310
|
+
def count_executions(db_path: Path) -> int:
|
|
311
|
+
if not db_path.exists():
|
|
312
|
+
return 0
|
|
313
|
+
with sqlite3.connect(str(db_path)) as conn:
|
|
314
|
+
cur = conn.execute("SELECT COUNT(*) FROM executions")
|
|
315
|
+
return int(cur.fetchone()[0])
|
|
316
|
+
|
|
317
|
+
def get_stats(db_path: Path) -> dict:
|
|
318
|
+
init_db(db_path)
|
|
319
|
+
stats = {"total_executions": 0, "successful": 0, "failed": 0,
|
|
320
|
+
"fallback_count": 0, "avg_tokens": 0, "avg_duration": 0,
|
|
321
|
+
"thumbs_up": 0, "thumbs_down": 0, "instincts_count": 0}
|
|
322
|
+
try:
|
|
323
|
+
with sqlite3.connect(str(db_path)) as conn:
|
|
324
|
+
row = conn.execute("SELECT COUNT(*) FROM executions").fetchone()
|
|
325
|
+
stats["total_executions"] = row[0] if row else 0
|
|
326
|
+
row = conn.execute("SELECT COUNT(*) FROM executions WHERE success = 1").fetchone()
|
|
327
|
+
stats["successful"] = row[0] if row else 0
|
|
328
|
+
stats["failed"] = stats["total_executions"] - stats["successful"]
|
|
329
|
+
row = conn.execute("SELECT COUNT(*) FROM executions WHERE was_fallback = 1").fetchone()
|
|
330
|
+
stats["fallback_count"] = row[0] if row else 0
|
|
331
|
+
row = conn.execute("SELECT AVG(tokens_used) FROM executions WHERE success = 1").fetchone()
|
|
332
|
+
stats["avg_tokens"] = round(row[0]) if row and row[0] else 0
|
|
333
|
+
row = conn.execute("SELECT AVG(duration_seconds) FROM executions WHERE success = 1").fetchone()
|
|
334
|
+
stats["avg_duration"] = round(row[0], 2) if row and row[0] else 0
|
|
335
|
+
row = conn.execute("SELECT COUNT(*) FROM executions WHERE feedback = 1").fetchone()
|
|
336
|
+
stats["thumbs_up"] = row[0] if row else 0
|
|
337
|
+
row = conn.execute("SELECT COUNT(*) FROM executions WHERE feedback = -1").fetchone()
|
|
338
|
+
stats["thumbs_down"] = row[0] if row else 0
|
|
339
|
+
row = conn.execute("SELECT COUNT(*) FROM instincts").fetchone()
|
|
340
|
+
stats["instincts_count"] = row[0] if row else 0
|
|
341
|
+
except Exception:
|
|
342
|
+
pass
|
|
343
|
+
return stats
|
|
344
|
+
|
|
345
|
+
def learn(db_path: Path) -> dict:
|
|
346
|
+
"""Analyze execution log, extract trigger word patterns, write instincts.
|
|
347
|
+
|
|
348
|
+
Each instinct also gets an Engram-style N-gram hash so future lookups can
|
|
349
|
+
be O(1) via `engram_lookup()`.
|
|
350
|
+
|
|
351
|
+
Returns summary of what was learned.
|
|
352
|
+
"""
|
|
353
|
+
init_db(db_path)
|
|
354
|
+
results = {"new_instincts": 0, "updated_instincts": 0, "patterns_found": [],
|
|
355
|
+
"weights_updated": update_skill_weights(db_path)}
|
|
356
|
+
|
|
357
|
+
with sqlite3.connect(str(db_path)) as conn:
|
|
358
|
+
rows = conn.execute(
|
|
359
|
+
"""SELECT skill_id, prompt, feedback, id FROM executions
|
|
360
|
+
WHERE feedback IS NOT NULL AND success = 1
|
|
361
|
+
ORDER BY id DESC LIMIT 200"""
|
|
362
|
+
).fetchall()
|
|
363
|
+
|
|
364
|
+
if not rows:
|
|
365
|
+
return results
|
|
366
|
+
|
|
367
|
+
# Group by skill: collect keywords from successful + positive-feedback prompts
|
|
368
|
+
skill_keywords: dict[str, Counter] = defaultdict(Counter)
|
|
369
|
+
skill_prompts: dict[str, list[str]] = defaultdict(list)
|
|
370
|
+
for skill_id, prompt, feedback, _eid in rows:
|
|
371
|
+
if feedback != 1:
|
|
372
|
+
continue
|
|
373
|
+
words = re.findall(r'[a-z]{3,}', prompt.lower())
|
|
374
|
+
skill_keywords[skill_id].update(w for w in words if w not in
|
|
375
|
+
{'the', 'and', 'for', 'that', 'this', 'with', 'from', 'how',
|
|
376
|
+
'what', 'when', 'why', 'where', 'which', 'your', 'have',
|
|
377
|
+
'will', 'can', 'are', 'not', 'but', 'all', 'was', 'out', 'use'})
|
|
378
|
+
skill_prompts[skill_id].append(prompt)
|
|
379
|
+
|
|
380
|
+
now = datetime.now(timezone.utc).isoformat()
|
|
381
|
+
for skill_id, counter in skill_keywords.items():
|
|
382
|
+
top = counter.most_common(5)
|
|
383
|
+
tokens = [w for w, _c in top]
|
|
384
|
+
pattern = " or ".join(tokens) if tokens else ""
|
|
385
|
+
if not pattern:
|
|
386
|
+
continue
|
|
387
|
+
# Engram hash from the union of positive prompts for this skill
|
|
388
|
+
ngram_hash = ""
|
|
389
|
+
for prompt in skill_prompts[skill_id]:
|
|
390
|
+
h = _ngram_hash(prompt)
|
|
391
|
+
if h:
|
|
392
|
+
ngram_hash = h
|
|
393
|
+
break
|
|
394
|
+
|
|
395
|
+
with sqlite3.connect(str(db_path)) as conn:
|
|
396
|
+
existing = conn.execute(
|
|
397
|
+
"SELECT id, confidence, samples FROM instincts WHERE source_skill_id = ? AND trigger_pattern = ?",
|
|
398
|
+
(skill_id, pattern)
|
|
399
|
+
).fetchone()
|
|
400
|
+
if existing:
|
|
401
|
+
conn.execute(
|
|
402
|
+
"UPDATE instincts SET confidence = ?, samples = samples + 1, last_seen = ? WHERE id = ?",
|
|
403
|
+
(min(1.0, existing[1] + 0.1), now, existing[0])
|
|
404
|
+
)
|
|
405
|
+
results["updated_instincts"] += 1
|
|
406
|
+
else:
|
|
407
|
+
conn.execute(
|
|
408
|
+
"INSERT INTO instincts (source_skill_id, trigger_pattern, ngram_hash, confidence, samples, created, last_seen, approved) VALUES (?, ?, ?, ?, ?, ?, ?, 0)",
|
|
409
|
+
(skill_id, pattern, ngram_hash, 0.3, 1, now, now)
|
|
410
|
+
)
|
|
411
|
+
results["new_instincts"] += 1
|
|
412
|
+
results["patterns_found"].append({"skill": skill_id, "pattern": pattern, "ngram_hash": ngram_hash})
|
|
413
|
+
conn.commit()
|
|
414
|
+
|
|
415
|
+
return results
|
|
416
|
+
|
|
417
|
+
def list_instincts(db_path: Path, approved_only: bool = False) -> list[dict]:
|
|
418
|
+
"""Return instincts as dicts."""
|
|
419
|
+
init_db(db_path)
|
|
420
|
+
conn = sqlite3.connect(str(db_path), isolation_level=None)
|
|
421
|
+
try:
|
|
422
|
+
if approved_only:
|
|
423
|
+
rows = conn.execute("SELECT id, source_skill_id, trigger_pattern, confidence, samples, approved FROM instincts WHERE approved = 1 ORDER BY confidence DESC").fetchall()
|
|
424
|
+
else:
|
|
425
|
+
rows = conn.execute("SELECT id, source_skill_id, trigger_pattern, confidence, samples, approved FROM instincts ORDER BY confidence DESC").fetchall()
|
|
426
|
+
finally:
|
|
427
|
+
conn.close()
|
|
428
|
+
return [{"id": r[0], "skill": r[1], "pattern": r[2], "confidence": r[3], "samples": r[4], "approved": bool(r[5])} for r in rows]
|
|
429
|
+
|
|
430
|
+
def approve_instinct(db_path: Path, instinct_id: int) -> bool:
|
|
431
|
+
init_db(db_path)
|
|
432
|
+
conn = sqlite3.connect(str(db_path), isolation_level=None)
|
|
433
|
+
try:
|
|
434
|
+
cur = conn.execute("UPDATE instincts SET approved = 1 WHERE id = ?", (instinct_id,))
|
|
435
|
+
return cur.rowcount > 0
|
|
436
|
+
finally:
|
|
437
|
+
conn.close()
|
agent_foundry/loop.py
ADDED
|
@@ -0,0 +1,226 @@
|
|
|
1
|
+
"""Loop: plan -> budget guard -> execute -> return."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
import time
|
|
5
|
+
from pathlib import Path
|
|
6
|
+
|
|
7
|
+
from .config import Config
|
|
8
|
+
|
|
9
|
+
from .executor import build_messages, complete, estimate_prompt_tokens
|
|
10
|
+
from .indexer import GENERIC_REASONING_ID, get_index_cached
|
|
11
|
+
from .models import (
|
|
12
|
+
LoopRequest,
|
|
13
|
+
LoopResumeRequest,
|
|
14
|
+
LoopResponse,
|
|
15
|
+
PlanRequest,
|
|
16
|
+
PlanResult,
|
|
17
|
+
TokenEstimate,
|
|
18
|
+
)
|
|
19
|
+
from .planner import rank_skills
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def _estimate_total(req: LoopRequest, idx, chosen_skill_id: str) -> TokenEstimate:
|
|
23
|
+
"""Estimate total tokens for chosen skill + prompt."""
|
|
24
|
+
# Pull chosen skill cost
|
|
25
|
+
skill = next((s for s in idx.skills if s.id == chosen_skill_id), None)
|
|
26
|
+
if skill is None:
|
|
27
|
+
# Fallback default
|
|
28
|
+
return TokenEstimate(input=500, output=1500)
|
|
29
|
+
in_tok = int(skill.estimated_token_cost.get("input", 0)) + estimate_prompt_tokens(req.prompt)
|
|
30
|
+
out_tok = int(skill.estimated_token_cost.get("output", 0))
|
|
31
|
+
return TokenEstimate(input=in_tok, output=out_tok)
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def _pick_skill(prompt: str, idx, cfg: Config,
|
|
35
|
+
weights: dict[str, float] | None = None) -> tuple[str | None, list[PlanResult], float | None]:
|
|
36
|
+
"""Pick the best skill. Return (skill_id, plan, planner_score).
|
|
37
|
+
If nothing matches, use the generic-reasoning fallback."""
|
|
38
|
+
rerank = cfg.core.default_llm if cfg.planner.rerank_enabled else None
|
|
39
|
+
ranked = rank_skills(prompt, idx, cfg.planner, weights=weights, rerank_model=rerank)
|
|
40
|
+
if ranked:
|
|
41
|
+
top = ranked[0]
|
|
42
|
+
return (top.skill_id, ranked, top.score)
|
|
43
|
+
return (GENERIC_REASONING_ID, [], None)
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def run_loop(req: LoopRequest, *, cfg: Config, db_path, model_override: str | None = None) -> LoopResponse:
|
|
47
|
+
idx = get_index_cached(Path(cfg.core.index_path).expanduser(),
|
|
48
|
+
ttl_seconds=cfg.core.index_cache_ttl_seconds)
|
|
49
|
+
if idx is None:
|
|
50
|
+
return LoopResponse(success=False,
|
|
51
|
+
error="No skill index. Run `agent-foundry init && agent-foundry index`.",
|
|
52
|
+
was_fallback=False)
|
|
53
|
+
|
|
54
|
+
from .logging_db import get_skill_weights
|
|
55
|
+
|
|
56
|
+
skill_id, plan_list, planner_score = _pick_skill(
|
|
57
|
+
req.prompt, idx, cfg, weights=get_skill_weights(db_path))
|
|
58
|
+
if skill_id is None:
|
|
59
|
+
return LoopResponse(plan=plan_list, success=False,
|
|
60
|
+
error="No skill could be selected.",
|
|
61
|
+
was_fallback=False)
|
|
62
|
+
|
|
63
|
+
was_fallback = (skill_id == GENERIC_REASONING_ID)
|
|
64
|
+
# If fallback was chosen, surface it in the plan list
|
|
65
|
+
if was_fallback:
|
|
66
|
+
skill = next(s for s in idx.skills if s.id == skill_id)
|
|
67
|
+
plan_list = [PlanResult(
|
|
68
|
+
skill_id=skill_id,
|
|
69
|
+
name=skill.name,
|
|
70
|
+
description=skill.description,
|
|
71
|
+
score=0.0,
|
|
72
|
+
matched_patterns=[],
|
|
73
|
+
estimated_cost=TokenEstimate(
|
|
74
|
+
input=int(skill.estimated_token_cost.get("input", 0)),
|
|
75
|
+
output=int(skill.estimated_token_cost.get("output", 0)),
|
|
76
|
+
),
|
|
77
|
+
location=skill.location,
|
|
78
|
+
)]
|
|
79
|
+
|
|
80
|
+
# Budget guard
|
|
81
|
+
estimate = _estimate_total(req, idx, skill_id)
|
|
82
|
+
budget = req.budget if req.budget is not None else cfg.core.token_budget
|
|
83
|
+
if estimate.total > budget and not req.force:
|
|
84
|
+
return LoopResponse(
|
|
85
|
+
plan=plan_list,
|
|
86
|
+
skill_id=skill_id,
|
|
87
|
+
requires_confirmation=True,
|
|
88
|
+
estimated_cost=estimate,
|
|
89
|
+
was_fallback=was_fallback,
|
|
90
|
+
)
|
|
91
|
+
|
|
92
|
+
# Execute via the shared completion path so timeout/retries/limits apply.
|
|
93
|
+
from .logging_db import log_execution
|
|
94
|
+
|
|
95
|
+
model = model_override or cfg.core.default_llm
|
|
96
|
+
fallback_prompt = cfg.fallback_system_prompt
|
|
97
|
+
skill_hash = next((s.sha256 for s in idx.skills if s.id == skill_id), "")
|
|
98
|
+
from .logging_db import create_loop_session, update_loop_session
|
|
99
|
+
session_id = create_loop_session(db_path, req.prompt, skill_id)
|
|
100
|
+
|
|
101
|
+
t0 = time.time()
|
|
102
|
+
try:
|
|
103
|
+
messages, _ = build_messages(skill_id, req.prompt, idx.skill_paths,
|
|
104
|
+
fallback_prompt=fallback_prompt,
|
|
105
|
+
expected_sha256=skill_hash)
|
|
106
|
+
text, tokens_used, cost_usd = complete(
|
|
107
|
+
messages, model=model,
|
|
108
|
+
timeout_seconds=cfg.core.executor_timeout_seconds,
|
|
109
|
+
max_tokens=cfg.core.executor_max_tokens,
|
|
110
|
+
retries=cfg.core.executor_retries,
|
|
111
|
+
)
|
|
112
|
+
except Exception as exc:
|
|
113
|
+
dur = time.time() - t0
|
|
114
|
+
log_execution(db_path, skill_id=skill_id, prompt=req.prompt, output="",
|
|
115
|
+
tokens_used=0, duration_seconds=dur, success=False,
|
|
116
|
+
error=str(exc)[:1000], planner_score=planner_score,
|
|
117
|
+
was_fallback=was_fallback)
|
|
118
|
+
update_loop_session(db_path, session_id, status="failed", output="")
|
|
119
|
+
return LoopResponse(plan=plan_list, skill_id=skill_id, success=False,
|
|
120
|
+
error=str(exc)[:1000], duration_seconds=dur,
|
|
121
|
+
was_fallback=was_fallback, session_id=session_id)
|
|
122
|
+
|
|
123
|
+
dur = time.time() - t0
|
|
124
|
+
truncated = text[:8000]
|
|
125
|
+
update_loop_session(db_path, session_id, status="completed", output=text,
|
|
126
|
+
increment_turn=True)
|
|
127
|
+
|
|
128
|
+
judge_payload = None
|
|
129
|
+
judged_flag = False
|
|
130
|
+
if getattr(req, "judge", False):
|
|
131
|
+
from .judge import judge_output
|
|
132
|
+
judge_payload = judge_output(
|
|
133
|
+
task=req.prompt,
|
|
134
|
+
output=truncated,
|
|
135
|
+
model=model,
|
|
136
|
+
skill_paths=idx.skill_paths,
|
|
137
|
+
fallback_prompt=fallback_prompt,
|
|
138
|
+
)
|
|
139
|
+
judged_flag = judge_payload is not None
|
|
140
|
+
|
|
141
|
+
log_execution(
|
|
142
|
+
db_path,
|
|
143
|
+
skill_id=skill_id,
|
|
144
|
+
prompt=req.prompt,
|
|
145
|
+
output=truncated,
|
|
146
|
+
tokens_used=tokens_used,
|
|
147
|
+
duration_seconds=dur,
|
|
148
|
+
success=True,
|
|
149
|
+
planner_score=planner_score,
|
|
150
|
+
was_fallback=was_fallback,
|
|
151
|
+
judge_corr=judge_payload.get("correctness") if judge_payload else None,
|
|
152
|
+
judge_slop=judge_payload.get("slop") if judge_payload else None,
|
|
153
|
+
judge_scope=judge_payload.get("scope") if judge_payload else None,
|
|
154
|
+
judge_verdict=judge_payload.get("verdict") if judge_payload else None,
|
|
155
|
+
)
|
|
156
|
+
|
|
157
|
+
return LoopResponse(
|
|
158
|
+
plan=plan_list,
|
|
159
|
+
skill_id=skill_id,
|
|
160
|
+
output=truncated,
|
|
161
|
+
tokens_used=tokens_used,
|
|
162
|
+
duration_seconds=dur,
|
|
163
|
+
success=True,
|
|
164
|
+
was_fallback=was_fallback,
|
|
165
|
+
judge_score=judge_payload,
|
|
166
|
+
judged=judged_flag,
|
|
167
|
+
cost_usd=cost_usd,
|
|
168
|
+
session_id=session_id,
|
|
169
|
+
)
|
|
170
|
+
|
|
171
|
+
|
|
172
|
+
def resume_loop(req: LoopResumeRequest, *, cfg: Config, db_path,
|
|
173
|
+
model_override: str | None = None) -> LoopResponse:
|
|
174
|
+
"""Continue an existing loop session with a follow-up prompt."""
|
|
175
|
+
from .logging_db import get_loop_session, log_execution, update_loop_session
|
|
176
|
+
|
|
177
|
+
session = get_loop_session(db_path, req.session_id)
|
|
178
|
+
if session is None:
|
|
179
|
+
return LoopResponse(success=False,
|
|
180
|
+
error=f"loop session {req.session_id} not found")
|
|
181
|
+
|
|
182
|
+
idx = get_index_cached(Path(cfg.core.index_path).expanduser(),
|
|
183
|
+
ttl_seconds=cfg.core.index_cache_ttl_seconds)
|
|
184
|
+
if idx is None:
|
|
185
|
+
return LoopResponse(success=False,
|
|
186
|
+
error="No skill index. Run `agent-foundry init && agent-foundry index`.")
|
|
187
|
+
|
|
188
|
+
skill_id = session["skill_id"]
|
|
189
|
+
skill_hash = next((s.sha256 for s in idx.skills if s.id == skill_id), "")
|
|
190
|
+
model = model_override or cfg.core.default_llm
|
|
191
|
+
was_fallback = skill_id == GENERIC_REASONING_ID
|
|
192
|
+
|
|
193
|
+
t0 = time.time()
|
|
194
|
+
try:
|
|
195
|
+
messages, _ = build_messages(skill_id, session["prompt"], idx.skill_paths,
|
|
196
|
+
fallback_prompt=cfg.fallback_system_prompt,
|
|
197
|
+
expected_sha256=skill_hash)
|
|
198
|
+
if session.get("output"):
|
|
199
|
+
messages.append({"role": "assistant", "content": session["output"]})
|
|
200
|
+
messages.append({"role": "user", "content": req.prompt})
|
|
201
|
+
text, tokens_used, cost_usd = complete(
|
|
202
|
+
messages, model=model,
|
|
203
|
+
timeout_seconds=cfg.core.executor_timeout_seconds,
|
|
204
|
+
max_tokens=cfg.core.executor_max_tokens,
|
|
205
|
+
retries=cfg.core.executor_retries,
|
|
206
|
+
)
|
|
207
|
+
except Exception as exc:
|
|
208
|
+
dur = time.time() - t0
|
|
209
|
+
log_execution(db_path, skill_id=skill_id, prompt=req.prompt, output="",
|
|
210
|
+
tokens_used=0, duration_seconds=dur, success=False,
|
|
211
|
+
error=str(exc)[:1000], was_fallback=was_fallback)
|
|
212
|
+
return LoopResponse(skill_id=skill_id, success=False, error=str(exc)[:1000],
|
|
213
|
+
duration_seconds=dur, was_fallback=was_fallback,
|
|
214
|
+
session_id=req.session_id)
|
|
215
|
+
|
|
216
|
+
dur = time.time() - t0
|
|
217
|
+
truncated = text[:8000]
|
|
218
|
+
update_loop_session(db_path, req.session_id, status="completed", output=text,
|
|
219
|
+
increment_turn=True)
|
|
220
|
+
log_execution(db_path, skill_id=skill_id, prompt=req.prompt, output=truncated,
|
|
221
|
+
tokens_used=tokens_used, duration_seconds=dur, success=True,
|
|
222
|
+
was_fallback=was_fallback)
|
|
223
|
+
return LoopResponse(skill_id=skill_id, output=truncated,
|
|
224
|
+
tokens_used=tokens_used, duration_seconds=dur,
|
|
225
|
+
success=True, was_fallback=was_fallback,
|
|
226
|
+
cost_usd=cost_usd, session_id=req.session_id)
|