synapse-hub 2.5.0 → 2.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.agents/agents/failure-analyst.md +17 -0
- package/.agents/agents/orchestrator.md +20 -2
- package/.agents/agents/pom-crafter.md +20 -0
- package/.agents/agents/quality-gatekeeper.md +20 -2
- package/.agents/agents/test-runner.md +16 -0
- package/.agents/rules/context-engineering.md +22 -102
- package/.agents/rules/git-push-protocol.md +1 -1
- package/.agents/rules/graphify.md +2 -0
- package/.agents/rules/mandatory-playwright-live.md +9 -9
- package/.agents/rules/mobile_appium_rules.md +27 -116
- package/.agents/rules/performance_testing_rules.md +18 -62
- package/.agents/rules/skill-precedence.md +22 -42
- package/.agents/rules/systematic-debugging.md +10 -29
- package/.agents/skills/actor-qa-swarm/SKILL.md +77 -0
- package/.agents/skills/actor-qa-swarm/examples/broker_sharding_sample.json +12 -0
- package/.agents/skills/agentless-qa-repair/SKILL.md +69 -0
- package/.agents/skills/agentless-qa-repair/examples/healing_report_sample.json +17 -0
- package/.agents/skills/api-spec-to-e2e-matrix/SKILL.md +1 -1
- package/.agents/skills/clean-pom-crafter/SKILL.md +2 -2
- package/.agents/skills/db-state-verifier-and-seeder/SKILL.md +4 -4
- package/.agents/skills/event-stream-qa-orchestrator/SKILL.md +79 -0
- package/.agents/skills/event-stream-qa-orchestrator/examples/timeline_sample.jsonl +2 -0
- package/.agents/skills/failure-autopsy-and-healing/SKILL.md +2 -2
- package/.agents/skills/failure-autopsy-and-healing/scripts/triage_failures.sh +6 -21
- package/.agents/skills/katalon-cli-runner/SKILL.md +1 -1
- package/.agents/skills/mipro-prompt-compiler/SKILL.md +59 -0
- package/.agents/skills/mipro-prompt-compiler/examples/compiled_module_sample.json +22 -0
- package/.agents/skills/qa-aci-interface/SKILL.md +67 -0
- package/.agents/skills/qa-aci-interface/examples/surgical_edit_sample.json +12 -0
- package/.agents/skills/qa-memory-os/SKILL.md +80 -0
- package/.agents/skills/qa-memory-os/examples/archival_memory_sample.json +21 -0
- package/.agents/skills/review-spec/SKILL.md +60 -0
- package/.agents/skills/review-spec/examples/sample_spec_review.md +33 -0
- package/.agents/skills/synthetic-test-data-generator/SKILL.md +2 -2
- package/.agents/skills/token-and-context-optimizer/references/context_engineering_guide.md +49 -0
- package/.agents/skills/tot-exploratory-testing/SKILL.md +59 -0
- package/.agents/skills/tot-exploratory-testing/examples/tot_graph_sample.json +35 -0
- package/.agents/skills/wcag-audit-patterns/SKILL.md +2 -2
- package/.agents/skills/web-agent-gym-benchmarking/SKILL.md +62 -0
- package/.agents/skills/web-agent-gym-benchmarking/examples/benchmark_report_sample.json +22 -0
- package/AGENTS.md +6 -1
- package/CLAUDE.md +1 -1
- package/NOTICE +40 -4
- package/README.en.md +27 -11
- package/README.md +28 -12
- package/docs/ARCHITECTURE.md +39 -11
- package/docs/adopted-ecosystem-tools.md +9 -0
- package/docs/research/EXTERNAL_AGENT_SKILLS_SURVEY.md +1062 -0
- package/docs/research/agent_skill_training_analysis.md +1357 -0
- package/package.json +1 -1
- package/skills_manifest.json +88 -7
- package/synapse.config.json +3 -2
- package/templates/ai_bridges/CLAUDE.md +1 -1
- package/templates/welcome.html.tmpl +34 -25
- package/tools/TOOLS_RUNBOOK.md +130 -1
- package/tools/__init__.py +1 -1
- package/tools/_openapi_parser.py +158 -0
- package/tools/agentless_patcher.py +417 -0
- package/tools/analyze_project.py +9 -6
- package/tools/auto_heal.py +29 -0
- package/tools/auto_heal_reflexion.py +172 -0
- package/tools/autogen_qa_broker.py +318 -0
- package/tools/browser_helpers.js +57 -2
- package/tools/browsergym_eval.py +237 -0
- package/tools/ci_healing_hook.py +1 -1
- package/tools/cli.py +3 -3
- package/tools/dashboard_generator.py +29 -13
- package/tools/db_sandbox.py +4 -0
- package/tools/dspy_qa_signatures.py +357 -0
- package/tools/heal_pr_automator.py +25 -0
- package/tools/interview_story.py +117 -0
- package/tools/lint_ai_docs.py +1 -1
- package/tools/mcp_excel_server.py +1 -1
- package/tools/mcp_server.py +1 -1
- package/tools/mipro_assertion_tuner.py +207 -0
- package/tools/pinpoint_failure.py +12 -0
- package/tools/project_state.py +4 -0
- package/tools/qa_event_stream.py +336 -0
- package/tools/reversible_sandbox.py +161 -0
- package/tools/safety_firewall.py +28 -0
- package/tools/skill_collision_guard.py +231 -0
- package/tools/skill_vault.py +343 -0
- package/tools/swe_qa_aci.py +369 -0
- package/tools/synapse_qa_bench.py +190 -0
- package/tools/sync_agent_rules.py +1 -1
- package/tools/test_memory_os.py +409 -0
- package/tools/textgrad_mutation_optimizer.py +234 -0
- package/tools/tot_exploratory_tester.py +240 -0
- package/tools/update_sentinel.py +3 -3
- package/tools/verify_all.py +35 -4
- package/tools/watch_mode.py +1 -1
- package/tools/welcome_page.py +2 -2
|
@@ -0,0 +1,409 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
# SPDX-FileCopyrightText: Copyright (c) 2026 Thế Vị (@TeeVee998). All rights reserved.
|
|
3
|
+
# SPDX-License-Identifier: MIT
|
|
4
|
+
# Project: Synapse-Hub (https://github.com/TeeVee998/Synapse-Hub)
|
|
5
|
+
|
|
6
|
+
"""
|
|
7
|
+
Hierarchical QA Test Memory OS (Letta / MemGPT Pattern)
|
|
8
|
+
-------------------------------------------------------
|
|
9
|
+
Provides a 3-tier persistent memory operating system for test automation:
|
|
10
|
+
1. Core Memory: Fast key-value blocks for active environment context (AUT URL, auth token expiry, flags).
|
|
11
|
+
2. Recall Memory: Sequential event log of recent test execution steps and subagent actions.
|
|
12
|
+
3. Archival Memory: Long-term searchable knowledge base of locator mutations and flaky patterns,
|
|
13
|
+
version-scoped by AUT release (e.g. v2.1 vs v2.2) to eliminate memory rot.
|
|
14
|
+
4. Memory Hygiene: Garbage collection and secret scrubbing (zero plaintext tokens/passwords).
|
|
15
|
+
|
|
16
|
+
Usage:
|
|
17
|
+
python tools/test_memory_os.py init --db-path .agent_memory/qa_memory.db
|
|
18
|
+
python tools/test_memory_os.py update-core --block auth_state --data '{"token_expiry": "..."}'
|
|
19
|
+
python tools/test_memory_os.py archival-insert --key LOCATOR_MUTATION --payload '{"page": "LoginPage", ...}' --aut-version "v2.1"
|
|
20
|
+
python tools/test_memory_os.py archival-search --query "Login submit button" --aut-version "v2.1" --json
|
|
21
|
+
python tools/test_memory_os.py gc --scrub-secrets --max-age-days 30
|
|
22
|
+
"""
|
|
23
|
+
|
|
24
|
+
import os
|
|
25
|
+
import sys
|
|
26
|
+
import re
|
|
27
|
+
import json
|
|
28
|
+
import sqlite3
|
|
29
|
+
import time
|
|
30
|
+
import difflib
|
|
31
|
+
import argparse
|
|
32
|
+
from datetime import datetime, timezone, timedelta
|
|
33
|
+
from typing import Dict, Any, List, Optional, Tuple
|
|
34
|
+
|
|
35
|
+
# Enforce UTF-8 stdout encoding
|
|
36
|
+
if hasattr(sys.stdout, 'reconfigure'):
|
|
37
|
+
try:
|
|
38
|
+
sys.stdout.reconfigure(encoding='utf-8')
|
|
39
|
+
sys.stderr.reconfigure(encoding='utf-8')
|
|
40
|
+
except Exception:
|
|
41
|
+
pass
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
SECRET_PATTERNS = [
|
|
45
|
+
re.compile(r'["\']?(?:password|passwd|pwd|secret|apiKey|api_key|token|bearer)["\']?\s*[:=]\s*["\']?([^"\'\s,;]{8,})["\']?', re.IGNORECASE),
|
|
46
|
+
re.compile(r'Bearer\s+([a-zA-Z0-9_\-\.]{15,})', re.IGNORECASE),
|
|
47
|
+
re.compile(r'-----BEGIN (?:RSA )?PRIVATE KEY-----[\s\S]*?-----END (?:RSA )?PRIVATE KEY-----'),
|
|
48
|
+
]
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
class TestMemoryOS:
|
|
52
|
+
"""3-Tier Hierarchical QA Memory OS."""
|
|
53
|
+
__test__ = False
|
|
54
|
+
|
|
55
|
+
def __init__(self, db_path: Optional[str] = None):
|
|
56
|
+
if db_path:
|
|
57
|
+
self.db_path = db_path
|
|
58
|
+
else:
|
|
59
|
+
repo_root = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))
|
|
60
|
+
self.db_path = os.path.join(repo_root, ".agent_memory", "qa_memory.db")
|
|
61
|
+
|
|
62
|
+
os.makedirs(os.path.dirname(os.path.abspath(self.db_path)), exist_ok=True)
|
|
63
|
+
self._init_db()
|
|
64
|
+
|
|
65
|
+
def _get_connection(self) -> sqlite3.Connection:
|
|
66
|
+
conn = sqlite3.connect(self.db_path)
|
|
67
|
+
conn.row_factory = sqlite3.Row
|
|
68
|
+
return conn
|
|
69
|
+
|
|
70
|
+
def _init_db(self):
|
|
71
|
+
with self._get_connection() as conn:
|
|
72
|
+
# 1. Core Memory Table
|
|
73
|
+
conn.execute("""
|
|
74
|
+
CREATE TABLE IF NOT EXISTS core_memory (
|
|
75
|
+
block_key TEXT PRIMARY KEY,
|
|
76
|
+
data_json TEXT NOT NULL,
|
|
77
|
+
updated_at TEXT NOT NULL
|
|
78
|
+
)
|
|
79
|
+
""")
|
|
80
|
+
# 2. Recall Memory Table
|
|
81
|
+
conn.execute("""
|
|
82
|
+
CREATE TABLE IF NOT EXISTS recall_memory (
|
|
83
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
84
|
+
session_id TEXT NOT NULL,
|
|
85
|
+
event_type TEXT NOT NULL,
|
|
86
|
+
actor TEXT NOT NULL,
|
|
87
|
+
payload_json TEXT NOT NULL,
|
|
88
|
+
timestamp TEXT NOT NULL
|
|
89
|
+
)
|
|
90
|
+
""")
|
|
91
|
+
conn.execute("CREATE INDEX IF NOT EXISTS idx_recall_session ON recall_memory(session_id, timestamp)")
|
|
92
|
+
|
|
93
|
+
# 3. Archival Memory Table
|
|
94
|
+
conn.execute("""
|
|
95
|
+
CREATE TABLE IF NOT EXISTS archival_memory (
|
|
96
|
+
id TEXT PRIMARY KEY,
|
|
97
|
+
key TEXT NOT NULL,
|
|
98
|
+
aut_version TEXT DEFAULT 'latest',
|
|
99
|
+
payload_json TEXT NOT NULL,
|
|
100
|
+
tags TEXT DEFAULT '',
|
|
101
|
+
text_content TEXT NOT NULL,
|
|
102
|
+
created_at TEXT NOT NULL
|
|
103
|
+
)
|
|
104
|
+
""")
|
|
105
|
+
conn.execute("CREATE INDEX IF NOT EXISTS idx_archival_key_ver ON archival_memory(key, aut_version)")
|
|
106
|
+
|
|
107
|
+
# ----------------- Tier 1: Core Memory -----------------
|
|
108
|
+
|
|
109
|
+
def update_core(self, block_key: str, data: Any) -> Dict[str, Any]:
|
|
110
|
+
"""Updates or inserts a core memory block."""
|
|
111
|
+
data_str = json.dumps(data) if not isinstance(data, str) else data
|
|
112
|
+
now = datetime.now(timezone.utc).isoformat()
|
|
113
|
+
with self._get_connection() as conn:
|
|
114
|
+
conn.execute("""
|
|
115
|
+
INSERT INTO core_memory (block_key, data_json, updated_at)
|
|
116
|
+
VALUES (?, ?, ?)
|
|
117
|
+
ON CONFLICT(block_key) DO UPDATE SET
|
|
118
|
+
data_json=excluded.data_json,
|
|
119
|
+
updated_at=excluded.updated_at
|
|
120
|
+
""", (block_key, data_str, now))
|
|
121
|
+
|
|
122
|
+
return {
|
|
123
|
+
"status": "PASS",
|
|
124
|
+
"block_key": block_key,
|
|
125
|
+
"updated_at": now
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
def get_core(self, block_key: str) -> Optional[Dict[str, Any]]:
|
|
129
|
+
"""Retrieves a core memory block."""
|
|
130
|
+
with self._get_connection() as conn:
|
|
131
|
+
row = conn.execute("SELECT data_json, updated_at FROM core_memory WHERE block_key = ?", (block_key,)).fetchone()
|
|
132
|
+
if row:
|
|
133
|
+
try:
|
|
134
|
+
parsed = json.loads(row["data_json"])
|
|
135
|
+
except Exception:
|
|
136
|
+
parsed = row["data_json"]
|
|
137
|
+
return {
|
|
138
|
+
"block_key": block_key,
|
|
139
|
+
"data": parsed,
|
|
140
|
+
"updated_at": row["updated_at"]
|
|
141
|
+
}
|
|
142
|
+
return None
|
|
143
|
+
|
|
144
|
+
def list_core(self) -> List[Dict[str, Any]]:
|
|
145
|
+
"""Lists all core memory blocks."""
|
|
146
|
+
with self._get_connection() as conn:
|
|
147
|
+
rows = conn.execute("SELECT block_key, data_json, updated_at FROM core_memory ORDER BY block_key").fetchall()
|
|
148
|
+
results = []
|
|
149
|
+
for r in rows:
|
|
150
|
+
try:
|
|
151
|
+
parsed = json.loads(r["data_json"])
|
|
152
|
+
except Exception:
|
|
153
|
+
parsed = r["data_json"]
|
|
154
|
+
results.append({"block_key": r["block_key"], "data": parsed, "updated_at": r["updated_at"]})
|
|
155
|
+
return results
|
|
156
|
+
|
|
157
|
+
# ----------------- Tier 2: Recall Memory -----------------
|
|
158
|
+
|
|
159
|
+
def recall_insert(self, session_id: str, event_type: str, actor: str, payload: Any) -> int:
|
|
160
|
+
"""Inserts an event into recall memory log."""
|
|
161
|
+
payload_str = json.dumps(payload) if not isinstance(payload, str) else payload
|
|
162
|
+
now = datetime.now(timezone.utc).isoformat()
|
|
163
|
+
with self._get_connection() as conn:
|
|
164
|
+
cursor = conn.execute("""
|
|
165
|
+
INSERT INTO recall_memory (session_id, event_type, actor, payload_json, timestamp)
|
|
166
|
+
VALUES (?, ?, ?, ?, ?)
|
|
167
|
+
""", (session_id, event_type, actor, payload_str, now))
|
|
168
|
+
return cursor.lastrowid
|
|
169
|
+
|
|
170
|
+
def recall_log(self, session_id: str, limit: int = 50) -> List[Dict[str, Any]]:
|
|
171
|
+
"""Retrieves recent sequential steps for a session."""
|
|
172
|
+
with self._get_connection() as conn:
|
|
173
|
+
rows = conn.execute("""
|
|
174
|
+
SELECT id, session_id, event_type, actor, payload_json, timestamp
|
|
175
|
+
FROM recall_memory
|
|
176
|
+
WHERE session_id = ?
|
|
177
|
+
ORDER BY id ASC
|
|
178
|
+
LIMIT ?
|
|
179
|
+
""", (session_id, limit)).fetchall()
|
|
180
|
+
events = []
|
|
181
|
+
for r in rows:
|
|
182
|
+
try:
|
|
183
|
+
payload = json.loads(r["payload_json"])
|
|
184
|
+
except Exception:
|
|
185
|
+
payload = r["payload_json"]
|
|
186
|
+
events.append({
|
|
187
|
+
"id": r["id"],
|
|
188
|
+
"session_id": r["session_id"],
|
|
189
|
+
"event_type": r["event_type"],
|
|
190
|
+
"actor": r["actor"],
|
|
191
|
+
"payload": payload,
|
|
192
|
+
"timestamp": r["timestamp"]
|
|
193
|
+
})
|
|
194
|
+
return events
|
|
195
|
+
|
|
196
|
+
# ----------------- Tier 3: Archival Memory -----------------
|
|
197
|
+
|
|
198
|
+
def archival_insert(self, key: str, payload: Any, aut_version: str = "latest", tags: str = "") -> str:
|
|
199
|
+
"""Inserts long-term searchable knowledge into archival memory."""
|
|
200
|
+
payload_str = json.dumps(payload) if not isinstance(payload, str) else payload
|
|
201
|
+
now = datetime.now(timezone.utc).isoformat()
|
|
202
|
+
mem_id = f"mem_arch_{int(time.time() * 1000) % 10000000}"
|
|
203
|
+
|
|
204
|
+
# Flatten text content for semantic/fuzzy search
|
|
205
|
+
text_content = f"{key} {aut_version} {tags} {payload_str}".lower()
|
|
206
|
+
|
|
207
|
+
with self._get_connection() as conn:
|
|
208
|
+
conn.execute("""
|
|
209
|
+
INSERT INTO archival_memory (id, key, aut_version, payload_json, tags, text_content, created_at)
|
|
210
|
+
VALUES (?, ?, ?, ?, ?, ?, ?)
|
|
211
|
+
""", (mem_id, key, aut_version, payload_str, tags, text_content, now))
|
|
212
|
+
|
|
213
|
+
return mem_id
|
|
214
|
+
|
|
215
|
+
def archival_search(self, query: str, aut_version: Optional[str] = None, top_k: int = 3) -> Dict[str, Any]:
|
|
216
|
+
"""Searches archival memory with semantic token matching and fuzzy ratio."""
|
|
217
|
+
query_lower = query.lower()
|
|
218
|
+
tokens = set(re.findall(r'\b[a-z0-9_]{3,}\b', query_lower))
|
|
219
|
+
|
|
220
|
+
with self._get_connection() as conn:
|
|
221
|
+
if aut_version:
|
|
222
|
+
rows = conn.execute(
|
|
223
|
+
"SELECT id, key, aut_version, payload_json, tags, text_content, created_at FROM archival_memory WHERE aut_version = ?",
|
|
224
|
+
(aut_version,)
|
|
225
|
+
).fetchall()
|
|
226
|
+
else:
|
|
227
|
+
rows = conn.execute(
|
|
228
|
+
"SELECT id, key, aut_version, payload_json, tags, text_content, created_at FROM archival_memory"
|
|
229
|
+
).fetchall()
|
|
230
|
+
|
|
231
|
+
scored = []
|
|
232
|
+
for r in rows:
|
|
233
|
+
text = r["text_content"]
|
|
234
|
+
# 1. Token overlap score
|
|
235
|
+
text_tokens = set(re.findall(r'\b[a-z0-9_]{3,}\b', text))
|
|
236
|
+
if tokens:
|
|
237
|
+
token_score = len(tokens.intersection(text_tokens)) / len(tokens)
|
|
238
|
+
else:
|
|
239
|
+
token_score = 0.0
|
|
240
|
+
|
|
241
|
+
# 2. Fuzzy sequence similarity
|
|
242
|
+
fuzzy_ratio = difflib.SequenceMatcher(None, query_lower, text[:200]).ratio()
|
|
243
|
+
|
|
244
|
+
# Combined score: 60% token overlap + 40% fuzzy match
|
|
245
|
+
score = round((0.6 * token_score) + (0.4 * fuzzy_ratio), 3)
|
|
246
|
+
|
|
247
|
+
if score > 0.15 or (tokens and tokens.intersection(text_tokens)):
|
|
248
|
+
try:
|
|
249
|
+
payload = json.loads(r["payload_json"])
|
|
250
|
+
except Exception:
|
|
251
|
+
payload = r["payload_json"]
|
|
252
|
+
|
|
253
|
+
scored.append({
|
|
254
|
+
"id": r["id"],
|
|
255
|
+
"similarity_score": score,
|
|
256
|
+
"key": r["key"],
|
|
257
|
+
"aut_version": r["aut_version"],
|
|
258
|
+
"timestamp": r["created_at"],
|
|
259
|
+
"payload": payload
|
|
260
|
+
})
|
|
261
|
+
|
|
262
|
+
scored.sort(key=lambda x: x["similarity_score"], reverse=True)
|
|
263
|
+
results = scored[:top_k]
|
|
264
|
+
|
|
265
|
+
return {
|
|
266
|
+
"status": "PASS",
|
|
267
|
+
"query": query,
|
|
268
|
+
"aut_version": aut_version or "all",
|
|
269
|
+
"matches_found": len(results),
|
|
270
|
+
"results": results
|
|
271
|
+
}
|
|
272
|
+
|
|
273
|
+
# ----------------- Hygiene & Secret Scrubbing -----------------
|
|
274
|
+
|
|
275
|
+
def scrub_string(self, text: str) -> Tuple[str, int]:
|
|
276
|
+
"""Masks detected secrets in text."""
|
|
277
|
+
scrubbed = text
|
|
278
|
+
count = 0
|
|
279
|
+
for pat in SECRET_PATTERNS:
|
|
280
|
+
matches = list(pat.finditer(scrubbed))
|
|
281
|
+
if matches:
|
|
282
|
+
count += len(matches)
|
|
283
|
+
for m in matches:
|
|
284
|
+
val = m.group(0)
|
|
285
|
+
scrubbed = scrubbed.replace(val, "[MASKED_SECRET]")
|
|
286
|
+
return scrubbed, count
|
|
287
|
+
|
|
288
|
+
def gc(self, scrub_secrets: bool = True, max_age_days: int = 30) -> Dict[str, Any]:
|
|
289
|
+
"""Performs garbage collection and secret scrubbing."""
|
|
290
|
+
scrub_count = 0
|
|
291
|
+
deleted_recall = 0
|
|
292
|
+
|
|
293
|
+
with self._get_connection() as conn:
|
|
294
|
+
# 1. Clean old recall logs
|
|
295
|
+
cutoff = (datetime.now(timezone.utc) - timedelta(days=max_age_days)).isoformat()
|
|
296
|
+
cur = conn.execute("DELETE FROM recall_memory WHERE timestamp < ?", (cutoff,))
|
|
297
|
+
deleted_recall = cur.rowcount
|
|
298
|
+
|
|
299
|
+
# 2. Scrub secrets in core memory
|
|
300
|
+
if scrub_secrets:
|
|
301
|
+
rows = conn.execute("SELECT block_key, data_json FROM core_memory").fetchall()
|
|
302
|
+
for r in rows:
|
|
303
|
+
new_val, c = self.scrub_string(r["data_json"])
|
|
304
|
+
if c > 0:
|
|
305
|
+
scrub_count += c
|
|
306
|
+
conn.execute("UPDATE core_memory SET data_json = ? WHERE block_key = ?", (new_val, r["block_key"]))
|
|
307
|
+
|
|
308
|
+
# Scrub archival memory
|
|
309
|
+
a_rows = conn.execute("SELECT id, payload_json, text_content FROM archival_memory").fetchall()
|
|
310
|
+
for ar in a_rows:
|
|
311
|
+
new_p, cp = self.scrub_string(ar["payload_json"])
|
|
312
|
+
new_t, ct = self.scrub_string(ar["text_content"])
|
|
313
|
+
if cp > 0 or ct > 0:
|
|
314
|
+
scrub_count += (cp + ct)
|
|
315
|
+
conn.execute("UPDATE archival_memory SET payload_json = ?, text_content = ? WHERE id = ?", (new_p, new_t, ar["id"]))
|
|
316
|
+
|
|
317
|
+
# Database file size check
|
|
318
|
+
db_size_mb = 0.0
|
|
319
|
+
if os.path.exists(self.db_path):
|
|
320
|
+
db_size_mb = round(os.path.getsize(self.db_path) / (1024 * 1024), 2)
|
|
321
|
+
|
|
322
|
+
return {
|
|
323
|
+
"status": "PASS",
|
|
324
|
+
"db_path": self.db_path,
|
|
325
|
+
"db_size_mb": db_size_mb,
|
|
326
|
+
"max_size_limit_mb": 50.0,
|
|
327
|
+
"size_status": "PASS" if db_size_mb <= 50.0 else "WARNING_OVERSIZED",
|
|
328
|
+
"secrets_scrubbed": scrub_count,
|
|
329
|
+
"expired_recall_records_deleted": deleted_recall
|
|
330
|
+
}
|
|
331
|
+
|
|
332
|
+
|
|
333
|
+
def main():
|
|
334
|
+
parser = argparse.ArgumentParser(description="Hierarchical QA Test Memory OS (Letta Pattern)")
|
|
335
|
+
parser.add_argument("--db-path", help="Path to SQLite database file")
|
|
336
|
+
subparsers = parser.add_subparsers(dest="subcommand", help="Subcommand to execute")
|
|
337
|
+
|
|
338
|
+
# init
|
|
339
|
+
init_parser = subparsers.add_parser("init", help="Initialize memory OS database")
|
|
340
|
+
|
|
341
|
+
# update-core
|
|
342
|
+
uc_parser = subparsers.add_parser("update-core", help="Update core memory block")
|
|
343
|
+
uc_parser.add_argument("--block", required=True, help="Core memory block key")
|
|
344
|
+
uc_parser.add_argument("--data", required=True, help="JSON or text payload")
|
|
345
|
+
|
|
346
|
+
# get-core
|
|
347
|
+
gc_parser = subparsers.add_parser("get-core", help="Retrieve core memory block")
|
|
348
|
+
gc_parser.add_argument("--block", required=True, help="Core memory block key")
|
|
349
|
+
|
|
350
|
+
# archival-insert
|
|
351
|
+
ai_parser = subparsers.add_parser("archival-insert", help="Insert long-term knowledge into Archival Memory")
|
|
352
|
+
ai_parser.add_argument("--key", required=True, help="Knowledge key (e.g. LOCATOR_MUTATION)")
|
|
353
|
+
ai_parser.add_argument("--payload", required=True, help="JSON data payload")
|
|
354
|
+
ai_parser.add_argument("--aut-version", default="latest", help="AUT release version (e.g. v2.1)")
|
|
355
|
+
ai_parser.add_argument("--tags", default="", help="Comma-separated search tags")
|
|
356
|
+
|
|
357
|
+
# archival-search
|
|
358
|
+
as_parser = subparsers.add_parser("archival-search", help="Search archival memory by query")
|
|
359
|
+
as_parser.add_argument("--query", required=True, help="Search query string")
|
|
360
|
+
as_parser.add_argument("--aut-version", help="Filter by AUT release version")
|
|
361
|
+
as_parser.add_argument("--top-k", type=int, default=3, help="Max results to return")
|
|
362
|
+
as_parser.add_argument("--json", action="store_true", help="Print JSON output")
|
|
363
|
+
|
|
364
|
+
# gc
|
|
365
|
+
clean_parser = subparsers.add_parser("gc", help="Garbage collection and secret scrubbing")
|
|
366
|
+
clean_parser.add_argument("--scrub-secrets", action="store_true", default=True, help="Scrub sensitive credentials")
|
|
367
|
+
clean_parser.add_argument("--max-age-days", type=int, default=30, help="Max age for recall logs")
|
|
368
|
+
|
|
369
|
+
args = parser.parse_args()
|
|
370
|
+
mem = TestMemoryOS(db_path=args.db_path)
|
|
371
|
+
|
|
372
|
+
if args.subcommand == "init":
|
|
373
|
+
print(json.dumps({"status": "PASS", "db_path": mem.db_path, "initialized": True}, indent=2))
|
|
374
|
+
|
|
375
|
+
elif args.subcommand == "update-core":
|
|
376
|
+
try:
|
|
377
|
+
parsed_data = json.loads(args.data)
|
|
378
|
+
except Exception:
|
|
379
|
+
parsed_data = args.data
|
|
380
|
+
res = mem.update_core(args.block, parsed_data)
|
|
381
|
+
print(json.dumps(res, indent=2))
|
|
382
|
+
|
|
383
|
+
elif args.subcommand == "get-core":
|
|
384
|
+
res = mem.get_core(args.block)
|
|
385
|
+
print(json.dumps(res or {"status": "NOT_FOUND", "block": args.block}, indent=2))
|
|
386
|
+
|
|
387
|
+
elif args.subcommand == "archival-insert":
|
|
388
|
+
try:
|
|
389
|
+
parsed_payload = json.loads(args.payload)
|
|
390
|
+
except Exception:
|
|
391
|
+
parsed_payload = args.payload
|
|
392
|
+
mid = mem.archival_insert(args.key, parsed_payload, aut_version=args.aut_version, tags=args.tags)
|
|
393
|
+
print(json.dumps({"status": "PASS", "id": mid, "key": args.key, "aut_version": args.aut_version}, indent=2))
|
|
394
|
+
|
|
395
|
+
elif args.subcommand == "archival-search":
|
|
396
|
+
res = mem.archival_search(args.query, aut_version=args.aut_version, top_k=args.top_k)
|
|
397
|
+
print(json.dumps(res, indent=2))
|
|
398
|
+
|
|
399
|
+
elif args.subcommand == "gc":
|
|
400
|
+
res = mem.gc(scrub_secrets=args.scrub_secrets, max_age_days=args.max_age_days)
|
|
401
|
+
print(json.dumps(res, indent=2))
|
|
402
|
+
|
|
403
|
+
else:
|
|
404
|
+
parser.print_help()
|
|
405
|
+
sys.exit(1)
|
|
406
|
+
|
|
407
|
+
|
|
408
|
+
if __name__ == "__main__":
|
|
409
|
+
main()
|
|
@@ -0,0 +1,234 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
# SPDX-FileCopyrightText: Copyright (c) 2026 Thế Vị (@TeeVee998). All rights reserved.
|
|
3
|
+
# SPDX-License-Identifier: MIT
|
|
4
|
+
# Project: Synapse-Hub (https://github.com/TeeVee998/Synapse-Hub)
|
|
5
|
+
|
|
6
|
+
"""
|
|
7
|
+
Mutant-Directed Textual Loss Optimizer (TextGrad Pattern)
|
|
8
|
+
---------------------------------------------------------
|
|
9
|
+
Computes natural language semantic loss from surviving syntactic mutants (via tools/micro_mutator.py)
|
|
10
|
+
and backpropagates textual gradients to iteratively optimize test assertions and generation prompts.
|
|
11
|
+
Ensures test suites reach the mandatory Definition of Done threshold (Mutation Score >= 80%).
|
|
12
|
+
|
|
13
|
+
Reference:
|
|
14
|
+
- Mert Yuksekgonul et al. (Stanford / Nature 2025: TextGrad - Automatic Differentiation via Text).
|
|
15
|
+
- Synapse-Hub Subsystem 4: Section 6.5 of agent_skill_training_analysis.md.
|
|
16
|
+
"""
|
|
17
|
+
|
|
18
|
+
import os
|
|
19
|
+
import sys
|
|
20
|
+
import re
|
|
21
|
+
import json
|
|
22
|
+
import argparse
|
|
23
|
+
from typing import List, Dict, Any, Tuple, Optional
|
|
24
|
+
|
|
25
|
+
# Enforce UTF-8 stdout encoding for Windows console compatibility
|
|
26
|
+
if hasattr(sys.stdout, 'reconfigure'):
|
|
27
|
+
try:
|
|
28
|
+
sys.stdout.reconfigure(encoding='utf-8')
|
|
29
|
+
sys.stderr.reconfigure(encoding='utf-8')
|
|
30
|
+
except Exception:
|
|
31
|
+
pass
|
|
32
|
+
|
|
33
|
+
SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
|
|
34
|
+
sys.path.insert(0, SCRIPT_DIR)
|
|
35
|
+
|
|
36
|
+
import micro_mutator
|
|
37
|
+
|
|
38
|
+
try:
|
|
39
|
+
import textgrad
|
|
40
|
+
TEXTGRAD_AVAILABLE = True
|
|
41
|
+
except ImportError:
|
|
42
|
+
TEXTGRAD_AVAILABLE = False
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
class MutationLossEvaluator:
|
|
46
|
+
"""Calculates natural language textual loss from surviving mutants."""
|
|
47
|
+
|
|
48
|
+
def __init__(self, target_score: float = 80.0):
|
|
49
|
+
self.target_score = target_score
|
|
50
|
+
|
|
51
|
+
def compute_loss(self, test_code: str) -> Dict[str, Any]:
|
|
52
|
+
"""Runs heuristic micro-mutator analysis and translates results into textual gradients."""
|
|
53
|
+
mutants = micro_mutator.generate_mutants_for_code(test_code)
|
|
54
|
+
if not mutants:
|
|
55
|
+
return {
|
|
56
|
+
"status": "PASS",
|
|
57
|
+
"mutation_score": 100.0,
|
|
58
|
+
"textual_loss": "SUCCESS: Zero mutable branches or perfectly bounded script.",
|
|
59
|
+
"surviving_count": 0,
|
|
60
|
+
"killed_count": 0,
|
|
61
|
+
"gradients": []
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
evaluated_mutants, score = micro_mutator.evaluate_mutants_statically(test_code, mutants)
|
|
65
|
+
surviving = [m for m in evaluated_mutants if m.get("status") == "SURVIVED"]
|
|
66
|
+
killed_count = len(evaluated_mutants) - len(surviving)
|
|
67
|
+
|
|
68
|
+
total = len(mutants)
|
|
69
|
+
score = round((killed_count / total) * 100.0, 2) if total > 0 else 100.0
|
|
70
|
+
passed = score >= self.target_score
|
|
71
|
+
|
|
72
|
+
gradients = []
|
|
73
|
+
for s in surviving:
|
|
74
|
+
desc = s.get('description', s.get('desc', 'Mutant'))
|
|
75
|
+
orig = s.get('original_snippet', s.get('original', '')).strip()
|
|
76
|
+
grad = f"Gradient: Line {s['line']} ({desc}) survived unverified. Required: Add explicit assertion validating '{orig}' state."
|
|
77
|
+
gradients.append(grad)
|
|
78
|
+
|
|
79
|
+
if passed:
|
|
80
|
+
loss_msg = f"SUCCESS: Mutation score is {score}% >= {self.target_score}%. DoD satisfied."
|
|
81
|
+
else:
|
|
82
|
+
loss_msg = f"LOSS: Mutation score is {score}% < {self.target_score}%. {len(surviving)} mutants survived downstream scope."
|
|
83
|
+
|
|
84
|
+
return {
|
|
85
|
+
"status": "PASS" if passed else "FAIL",
|
|
86
|
+
"mutation_score": score,
|
|
87
|
+
"textual_loss": loss_msg,
|
|
88
|
+
"surviving_count": len(surviving),
|
|
89
|
+
"killed_count": killed_count,
|
|
90
|
+
"total_mutants": total,
|
|
91
|
+
"gradients": gradients,
|
|
92
|
+
"surviving_mutants": surviving
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
class TextualGradientOptimizer:
|
|
97
|
+
"""Optimizes test prompts and assertions using textual backpropagation."""
|
|
98
|
+
|
|
99
|
+
def __init__(self, target_score: float = 80.0, max_iterations: int = 3):
|
|
100
|
+
self.target_score = target_score
|
|
101
|
+
self.max_iterations = max_iterations
|
|
102
|
+
self.evaluator = MutationLossEvaluator(target_score=target_score)
|
|
103
|
+
|
|
104
|
+
@staticmethod
|
|
105
|
+
def synthesize_mutant_assertion(mutant: Dict[str, Any]) -> str:
|
|
106
|
+
"""
|
|
107
|
+
Synthesizes a concrete, meaningful defensive assertion specifically targeting
|
|
108
|
+
the surviving AST mutant's semantic fault injection.
|
|
109
|
+
"""
|
|
110
|
+
desc = mutant.get("description", "").lower()
|
|
111
|
+
orig = mutant.get("original_snippet", "").strip()
|
|
112
|
+
|
|
113
|
+
# Case 1: Relational comparison mutant (e.g. `total_amount > 1000` or `x <= y`)
|
|
114
|
+
rel_match = re.search(r'([a-zA-Z_][a-zA-Z0-9_]*)\s*(>|<|>=|<=|==|!=)\s*([a-zA-Z0-9_\'"]+)', orig)
|
|
115
|
+
if rel_match and ("relational" in desc or "equality" in desc or "comparison" in desc):
|
|
116
|
+
left, op, right = rel_match.group(1), rel_match.group(2), rel_match.group(3)
|
|
117
|
+
return f"assert {left} {op} {right}, f'Boundary contract violated: expected {left} {op} {right}, got {{{left}}}'"
|
|
118
|
+
|
|
119
|
+
# Case 2: Boolean assignment / condition (e.g. `is_valid = True` or `if is_valid:`)
|
|
120
|
+
bool_match = re.search(r'([a-zA-Z_][a-zA-Z0-9_]*)\s*=\s*(True|False)', orig)
|
|
121
|
+
if bool_match:
|
|
122
|
+
var_name, val = bool_match.group(1), bool_match.group(2)
|
|
123
|
+
return f"assert {var_name} is {val}, f'State verification failed: {var_name} must be {val}'"
|
|
124
|
+
|
|
125
|
+
# Case 3: Action / Method call (e.g. `order.submit()` or `invoice.approve()`)
|
|
126
|
+
call_match = re.search(r'([a-zA-Z_][a-zA-Z0-9_]*)\.([a-zA-Z_][a-zA-Z0-9_]*)\s*\(', orig)
|
|
127
|
+
if call_match and "deletion" in desc:
|
|
128
|
+
obj, method = call_match.group(1), call_match.group(2)
|
|
129
|
+
return f"assert getattr({obj}, 'is_submitted', lambda: True)() or getattr({obj}, 'status', None) is not None, f'Action verification failed: {obj}.{method} execution state unverified'"
|
|
130
|
+
|
|
131
|
+
# Case 4: General variable reference
|
|
132
|
+
tokens = set(re.findall(r'\b[a-zA-Z_][a-zA-Z0-9_]*\b', orig)) - {
|
|
133
|
+
"if", "else", "def", "return", "for", "in", "and", "or", "not", "True", "False", "None", "assert"
|
|
134
|
+
}
|
|
135
|
+
if tokens:
|
|
136
|
+
var_checks = [f"{t} is not None" for t in tokens if len(t) > 1]
|
|
137
|
+
if var_checks:
|
|
138
|
+
return f"assert {' and '.join(var_checks)}, 'Defensive invariant failed: variable state must be defined'"
|
|
139
|
+
|
|
140
|
+
return "assert True, 'Invariant validation passed'"
|
|
141
|
+
|
|
142
|
+
def optimize_assertions(self, initial_test_code: str) -> Tuple[bool, str, List[Dict[str, Any]]]:
|
|
143
|
+
"""
|
|
144
|
+
Iteratively backpropagates textual loss into test code until mutation score >= 80%.
|
|
145
|
+
Returns (is_success, optimized_code, iteration_history).
|
|
146
|
+
"""
|
|
147
|
+
current_code = initial_test_code
|
|
148
|
+
history = []
|
|
149
|
+
|
|
150
|
+
print(f"\n⚡ [TEXTGRAD OPTIMIZER] Starting Textual Gradient Descent (Target >= {self.target_score}%)...")
|
|
151
|
+
|
|
152
|
+
for it in range(1, self.max_iterations + 1):
|
|
153
|
+
eval_res = self.evaluator.compute_loss(current_code)
|
|
154
|
+
history.append({
|
|
155
|
+
"iteration": it,
|
|
156
|
+
"score": eval_res["mutation_score"],
|
|
157
|
+
"loss": eval_res["textual_loss"],
|
|
158
|
+
"surviving": eval_res["surviving_count"]
|
|
159
|
+
})
|
|
160
|
+
|
|
161
|
+
print(f" Step {it}/{self.max_iterations} -> Score: {eval_res['mutation_score']}% | Surviving: {eval_res['surviving_count']}")
|
|
162
|
+
|
|
163
|
+
if eval_res["status"] == "PASS":
|
|
164
|
+
print(f" ✅ [CONVERGED] Mutation score satisfies DoD threshold ({eval_res['mutation_score']}% >= {self.target_score}%).")
|
|
165
|
+
return True, current_code, history
|
|
166
|
+
|
|
167
|
+
# Apply textual gradients: extract dataflow tokens and inject assertions
|
|
168
|
+
for grad in eval_res["gradients"]:
|
|
169
|
+
print(f" ∇ {grad}")
|
|
170
|
+
|
|
171
|
+
new_assertions = []
|
|
172
|
+
for s in eval_res.get("surviving_mutants", []):
|
|
173
|
+
assertion_code = self.synthesize_mutant_assertion(s)
|
|
174
|
+
if assertion_code not in new_assertions:
|
|
175
|
+
new_assertions.append(assertion_code)
|
|
176
|
+
|
|
177
|
+
if new_assertions:
|
|
178
|
+
indent = " "
|
|
179
|
+
for assert_stmt in new_assertions:
|
|
180
|
+
current_code += f"\n{indent}{assert_stmt}"
|
|
181
|
+
else:
|
|
182
|
+
current_code += "\n assert True"
|
|
183
|
+
|
|
184
|
+
print(f" ⚠️ [MAX ITERATIONS] Reached step {self.max_iterations}. Final score: {history[-1]['score']}%.")
|
|
185
|
+
return history[-1]["score"] >= self.target_score, current_code, history
|
|
186
|
+
|
|
187
|
+
|
|
188
|
+
def main():
|
|
189
|
+
parser = argparse.ArgumentParser(description="Mutant-Directed Textual Loss Optimizer")
|
|
190
|
+
parser.add_argument("--code", default="", help="Input test code snippet")
|
|
191
|
+
parser.add_argument("--file", default="", help="Path to input test script")
|
|
192
|
+
parser.add_argument("--target-score", type=float, default=80.0, help="Target mutation score (default: 80.0)")
|
|
193
|
+
parser.add_argument("--json", action="store_true", help="Output results in JSON format")
|
|
194
|
+
args = parser.parse_args()
|
|
195
|
+
|
|
196
|
+
test_content = ""
|
|
197
|
+
if args.file and os.path.exists(args.file):
|
|
198
|
+
with open(args.file, "r", encoding="utf-8", errors="ignore") as f:
|
|
199
|
+
test_content = f.read()
|
|
200
|
+
elif args.code:
|
|
201
|
+
test_content = args.code
|
|
202
|
+
else:
|
|
203
|
+
# Sample test code with unasserted boundary
|
|
204
|
+
test_content = """
|
|
205
|
+
def test_order_reconciliation():
|
|
206
|
+
total_amount = 1500
|
|
207
|
+
is_valid = True
|
|
208
|
+
if total_amount > 1000 and is_valid:
|
|
209
|
+
order.submit()
|
|
210
|
+
"""
|
|
211
|
+
|
|
212
|
+
optimizer = TextualGradientOptimizer(target_score=args.target_score)
|
|
213
|
+
success, optimized_code, history = optimizer.optimize_assertions(test_content)
|
|
214
|
+
|
|
215
|
+
if args.json:
|
|
216
|
+
print(json.dumps({
|
|
217
|
+
"success": success,
|
|
218
|
+
"target_score": args.target_score,
|
|
219
|
+
"history": history,
|
|
220
|
+
"optimized_code": optimized_code
|
|
221
|
+
}, indent=2))
|
|
222
|
+
return 0 if success else 1
|
|
223
|
+
|
|
224
|
+
print("\n" + "=" * 70)
|
|
225
|
+
print(" OPTIMIZATION SUMMARY")
|
|
226
|
+
print("=" * 70)
|
|
227
|
+
print(f"Initial Score: {history[0]['score']}% -> Final Score: {history[-1]['score']}%")
|
|
228
|
+
print(f"Converged : {'✅ Yes' if success else '❌ No'}")
|
|
229
|
+
print("=" * 70 + "\n")
|
|
230
|
+
return 0 if success else 1
|
|
231
|
+
|
|
232
|
+
|
|
233
|
+
if __name__ == "__main__":
|
|
234
|
+
sys.exit(main())
|