mimic-agent 0.3.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (62) hide show
  1. mimic/__init__.py +6 -0
  2. mimic/__main__.py +3 -0
  3. mimic/agent.py +459 -0
  4. mimic/api/__init__.py +0 -0
  5. mimic/api/main.py +37 -0
  6. mimic/approval.py +291 -0
  7. mimic/cli.py +179 -0
  8. mimic/conditions.py +180 -0
  9. mimic/config.py +25 -0
  10. mimic/db.py +13 -0
  11. mimic/doctor.py +183 -0
  12. mimic/dom.py +140 -0
  13. mimic/events.py +71 -0
  14. mimic/extract.py +441 -0
  15. mimic/healing.py +302 -0
  16. mimic/learn.py +86 -0
  17. mimic/live.py +52 -0
  18. mimic/llm.py +47 -0
  19. mimic/mailbox.py +68 -0
  20. mimic/mapping.py +122 -0
  21. mimic/matcher.py +371 -0
  22. mimic/matcher_data.py +236 -0
  23. mimic/models.py +59 -0
  24. mimic/outcome.py +132 -0
  25. mimic/recorder.py +290 -0
  26. mimic/risk.py +180 -0
  27. mimic/riskml.py +317 -0
  28. mimic/run.py +677 -0
  29. mimic/sandbox/__init__.py +0 -0
  30. mimic/sandbox/app.py +359 -0
  31. mimic/sandbox/generate_documents.py +168 -0
  32. mimic/sandbox/generate_invoices.py +45 -0
  33. mimic/sandbox/generate_mail.py +73 -0
  34. mimic/sandbox/templates/approval_done.html +7 -0
  35. mimic/sandbox/templates/approval_new.html +17 -0
  36. mimic/sandbox/templates/base.html +72 -0
  37. mimic/sandbox/templates/expense_done.html +7 -0
  38. mimic/sandbox/templates/expense_list.html +18 -0
  39. mimic/sandbox/templates/expense_new.html +22 -0
  40. mimic/sandbox/templates/expense_review.html +16 -0
  41. mimic/sandbox/templates/expenses_base.html +41 -0
  42. mimic/sandbox/templates/invoice_v1.html +23 -0
  43. mimic/sandbox/templates/invoice_v2.html +29 -0
  44. mimic/sandbox/templates/login.html +12 -0
  45. mimic/sandbox/templates/mail_inbox.html +12 -0
  46. mimic/sandbox/templates/mail_message.html +16 -0
  47. mimic/sandbox/templates/mail_reply.html +13 -0
  48. mimic/sandbox/templates/mail_sent.html +7 -0
  49. mimic/sandbox/templates/suite_base.html +52 -0
  50. mimic/server.py +222 -0
  51. mimic/skills.py +263 -0
  52. mimic/static/index.html +544 -0
  53. mimic/storage.py +503 -0
  54. mimic/store.py +33 -0
  55. mimic/tables.py +136 -0
  56. mimic/vision.py +150 -0
  57. mimic/workflow.py +526 -0
  58. mimic_agent-0.3.0.dist-info/METADATA +422 -0
  59. mimic_agent-0.3.0.dist-info/RECORD +62 -0
  60. mimic_agent-0.3.0.dist-info/WHEEL +4 -0
  61. mimic_agent-0.3.0.dist-info/entry_points.txt +2 -0
  62. mimic_agent-0.3.0.dist-info/licenses/LICENSE +21 -0
mimic/__init__.py ADDED
@@ -0,0 +1,6 @@
1
+ from importlib.metadata import PackageNotFoundError, version
2
+
3
+ try:
4
+ __version__ = version("mimic-agent")
5
+ except PackageNotFoundError: # running from a source tree that is not installed
6
+ __version__ = "0.0.0+source"
mimic/__main__.py ADDED
@@ -0,0 +1,3 @@
1
+ from mimic.cli import main
2
+
3
+ raise SystemExit(main())
mimic/agent.py ADDED
@@ -0,0 +1,459 @@
1
+ """Replay agent: executes a skill for one task with a LangGraph loop.
2
+
3
+ locate ──► act ──► verify ──► next step / done
4
+ ▲ │
5
+ └──── retry ◄──────┘ (next locator strategy, max MAX_ATTEMPTS)
6
+
7
+ Locator strategies, from most to least precise. Each one that the recorder captured is tried:
8
+ selector #vendor (exact, breaks when ids change)
9
+ label <label>Vendor</label> (survives id changes)
10
+ name name="vendor" (survives relabelling)
11
+ text button "Save invoice" (buttons and links)
12
+ heal when all of the above fail: memory → semantic → vision (see healing.py)
13
+ """
14
+
15
+ from __future__ import annotations
16
+
17
+ import time
18
+ from dataclasses import dataclass, field
19
+ from pathlib import Path
20
+ from typing import Literal, TypedDict
21
+ from urllib.parse import urlsplit
22
+
23
+ from langgraph.graph import END, START, StateGraph
24
+ from playwright.sync_api import Locator, Page
25
+
26
+ from mimic.models import ElementInfo
27
+
28
+ MAX_ATTEMPTS = 3
29
+ LOCATE_TIMEOUT_MS = 1500
30
+ GRACE_MS = 150
31
+ NAV_TIMEOUT_MS = 5000
32
+ ACT_TIMEOUT_MS = 1000 # fail fast, like a production agent; mitigations wait longer
33
+
34
+ STRATEGY_CONFIDENCE = {"selector": 1.0, "option": 1.0, "label": 0.8, "name": 0.7, "text": 0.6}
35
+ TRUE_WORDS = {"true", "yes", "y", "1", "x", "on", "oui", "checked", "vrai"}
36
+
37
+
38
+ def as_bool(value) -> bool:
39
+ return str(value).strip().lower() in TRUE_WORDS
40
+
41
+
42
+ def _radio_value(el: ElementInfo, value: str) -> str:
43
+ """Match the task's value to one of the options seen in the demo, ignoring case."""
44
+ wanted = str(value).strip().lower()
45
+ return next((o for o in el.options if o.lower() == wanted), str(value).strip())
46
+
47
+
48
+ @dataclass
49
+ class StepResult:
50
+ index: int
51
+ action: str
52
+ strategy: str | None
53
+ confidence: float
54
+ attempts: int
55
+ latency_ms: int
56
+ success: bool
57
+ error: str | None = None
58
+ fallbacks: int = 0 # strategies tried before the one that worked
59
+ healed: bool = False
60
+ features: dict | None = None # observed right before acting (risk model input)
61
+ risk: float | None = None # predicted failure probability
62
+ mitigations: list[str] = field(default_factory=list)
63
+
64
+
65
+ @dataclass
66
+ class TaskResult:
67
+ success: bool
68
+ skipped: bool = False # the user declined the submit
69
+ steps: list[StepResult] = field(default_factory=list)
70
+ duration_ms: int = 0
71
+ error: str | None = None
72
+
73
+
74
+ class AgentState(TypedDict):
75
+ actions: list[dict]
76
+ idx: int
77
+ attempt: int
78
+ tried: list[str]
79
+ locator_strategy: str | None
80
+ confidence: float
81
+ error: str | None
82
+ step_started: float
83
+ results: list[StepResult]
84
+ status: Literal["running", "done", "failed", "skipped"]
85
+
86
+
87
+ def _candidates(page: Page, step: dict) -> list[tuple[str, Locator]]:
88
+ el: ElementInfo = step["element"]
89
+ action = step["action"]
90
+ if action == "choose":
91
+ # The demo clicked one radio; this task may need another option of the same group.
92
+ value = _radio_value(el, step["value"])
93
+ found = []
94
+ if el.name:
95
+ css = f'input[type=radio][name="{el.name}"][value="{value}"]'
96
+ found.append(("option", page.locator(css)))
97
+ found.append(("label", page.get_by_role("radio", name=value, exact=True)))
98
+ return found
99
+ found = []
100
+ if el.selector:
101
+ found.append(("selector", page.locator(el.selector)))
102
+ if el.label and action != "click":
103
+ found.append(("label", page.get_by_label(el.label, exact=True)))
104
+ if el.name:
105
+ found.append(("name", page.locator(f'[name="{el.name}"]')))
106
+ if el.text and action == "click":
107
+ found.append(("text", page.get_by_role("button", name=el.text, exact=True)))
108
+ return found
109
+
110
+
111
+ class ReplayAgent:
112
+ """Runs one rendered skill (list of concrete actions) on an open page."""
113
+
114
+ def __init__(
115
+ self,
116
+ page: Page,
117
+ on_step=None,
118
+ healer=None,
119
+ skill_name: str = "skill",
120
+ before_submit=None, # callable(action) -> bool: human confirmation before submitting
121
+ risk_policy: str = "never", # never | always | predicted
122
+ risk_model=None, # RiskModel, required for the "predicted" policy
123
+ ):
124
+ if risk_policy == "predicted" and risk_model is None:
125
+ raise ValueError("risk_policy='predicted' needs a trained risk model")
126
+ self.before_submit = before_submit
127
+ self.risk_policy = risk_policy
128
+ self.risk_model = risk_model
129
+ self._observation: dict = {}
130
+ self.page = page
131
+ self.healer = healer
132
+ self.skill_name = skill_name
133
+ self.on_step = on_step # callback(StepResult) for telemetry / live progress
134
+ self._locator: Locator | None = None
135
+ self._pending_heal = None
136
+ self.graph = self._build()
137
+
138
+ # ---------- nodes ----------
139
+
140
+ def _skip_ahead(self, state: AgentState) -> int | None:
141
+ """The app skipped a page of the demo (already signed in, no consent screen today...):
142
+ jump to the first later step recorded on the page we are actually on."""
143
+ actions, idx = state["actions"], state["idx"]
144
+ recorded = actions[idx].get("page")
145
+ here = urlsplit(self.page.url).path or "/"
146
+ if not recorded or recorded == here or state["tried"]:
147
+ return None
148
+ return next(
149
+ (j for j in range(idx + 1, len(actions)) if actions[j].get("page") == here), None
150
+ )
151
+
152
+ def locate(self, state: AgentState) -> dict:
153
+ skipped: dict = {}
154
+ jump = self._skip_ahead(state)
155
+ if jump is not None:
156
+ done = [
157
+ StepResult(
158
+ index=k,
159
+ action=state["actions"][k]["action"],
160
+ strategy="not needed",
161
+ confidence=1.0,
162
+ attempts=0,
163
+ latency_ms=0,
164
+ success=True,
165
+ )
166
+ for k in range(state["idx"], jump)
167
+ ]
168
+ for r in done:
169
+ if self.on_step:
170
+ self.on_step(r)
171
+ skipped = {"idx": jump, "results": [*state["results"], *done]}
172
+ state = {**state, **skipped}
173
+ update = self._locate(state)
174
+ return {**skipped, **update}
175
+
176
+ def _locate(self, state: AgentState) -> dict:
177
+ action = state["actions"][state["idx"]]
178
+ candidates = [
179
+ (name, loc)
180
+ for name, loc in _candidates(self.page, action)
181
+ if name not in state["tried"]
182
+ ]
183
+
184
+ def first_match():
185
+ # Instant check of every strategy: no waiting on the ones that cannot match.
186
+ for name, loc in candidates:
187
+ try:
188
+ if loc.count() == 1 and loc.first.is_visible():
189
+ return name, loc.first
190
+ except Exception: # noqa: BLE001
191
+ continue
192
+ return None
193
+
194
+ match = first_match()
195
+ if match is None and candidates:
196
+ # Nothing yet: the page may still be rendering. Wait (long only if it is still
197
+ # loading, a short grace period otherwise), then check again.
198
+ try:
199
+ loaded = self.page.evaluate("document.readyState") == "complete"
200
+ self.page.wait_for_load_state()
201
+ timeout = GRACE_MS if loaded else LOCATE_TIMEOUT_MS
202
+ candidates[0][1].first.wait_for(state="visible", timeout=timeout)
203
+ except Exception: # noqa: BLE001
204
+ pass
205
+ match = first_match()
206
+
207
+ if match is not None:
208
+ name, self._locator = match
209
+ return {
210
+ "locator_strategy": name,
211
+ "confidence": STRATEGY_CONFIDENCE[name],
212
+ "tried": [*state["tried"], name],
213
+ }
214
+
215
+ tried = [*state["tried"], *(n for n, _ in candidates)]
216
+ # Radio options are found by their value; healing would guess an option, so it stays off.
217
+ if self.healer is not None and "heal" not in tried and action["action"] != "choose":
218
+ tried.append("heal")
219
+ healed = self.healer.heal(self.page, action, self.skill_name)
220
+ self._pending_heal = healed
221
+ if healed is not None:
222
+ self._locator = self.page.locator(healed.selector).first
223
+ return {
224
+ "locator_strategy": healed.strategy,
225
+ "confidence": healed.confidence,
226
+ "tried": tried,
227
+ }
228
+ self._locator = None
229
+ return {
230
+ "locator_strategy": None,
231
+ "confidence": 0.0,
232
+ "tried": tried,
233
+ "error": "element not found",
234
+ }
235
+
236
+ def act(self, state: AgentState) -> dict:
237
+ if state["locator_strategy"] is None:
238
+ return {}
239
+ action = state["actions"][state["idx"]]
240
+ self._observe_and_mitigate(state, action)
241
+ try:
242
+ kind = action["action"]
243
+ if kind == "fill":
244
+ self._locator.fill(str(action["value"]), timeout=ACT_TIMEOUT_MS)
245
+ elif kind == "select":
246
+ self._locator.select_option(str(action["value"]), timeout=ACT_TIMEOUT_MS)
247
+ elif kind == "check":
248
+ self._locator.set_checked(as_bool(action["value"]), timeout=ACT_TIMEOUT_MS)
249
+ elif kind == "choose":
250
+ self._locator.check(timeout=ACT_TIMEOUT_MS)
251
+ elif kind == "upload":
252
+ path = Path(str(action["value"])).expanduser()
253
+ if not path.is_file():
254
+ return {"error": f"file not found: {path}"}
255
+ self._locator.set_input_files(str(path), timeout=ACT_TIMEOUT_MS)
256
+ elif kind == "press":
257
+ if action.get("navigates"):
258
+ if self.before_submit is not None and not self.before_submit(action):
259
+ return {"error": "declined by user", "status": "skipped"}
260
+ return self._then_new_page(
261
+ lambda: self._locator.press(str(action["value"]), timeout=ACT_TIMEOUT_MS)
262
+ )
263
+ self._locator.press(str(action["value"]), timeout=ACT_TIMEOUT_MS)
264
+ elif action["element"].type == "submit":
265
+ if self.before_submit is not None and not self.before_submit(action):
266
+ return {"error": "declined by user", "status": "skipped"}
267
+ # A submit in the demo navigated: if nothing happens, it was the wrong button.
268
+ with self.page.expect_navigation(timeout=NAV_TIMEOUT_MS) as nav:
269
+ self._locator.click(timeout=ACT_TIMEOUT_MS)
270
+ response = nav.value
271
+ if response is not None and response.status >= 400:
272
+ return {"error": f"server error {response.status}"}
273
+ elif action.get("navigates"):
274
+ # A link or a "Next" button: the demo moved to another page (or route).
275
+ return self._then_new_page(lambda: self._locator.click(timeout=ACT_TIMEOUT_MS))
276
+ else:
277
+ self._locator.click(timeout=ACT_TIMEOUT_MS)
278
+ self.page.wait_for_load_state()
279
+ return {"error": None}
280
+ except Exception as exc: # noqa: BLE001
281
+ return {"error": f"{type(exc).__name__}: {exc}".splitlines()[0][:200]}
282
+
283
+ def _then_new_page(self, do) -> dict:
284
+ """Act, then wait until the URL changes: real navigations and single-page apps alike."""
285
+ before = self.page.url
286
+ do()
287
+ try:
288
+ self.page.wait_for_url(lambda url: url != before, timeout=NAV_TIMEOUT_MS)
289
+ except Exception: # noqa: BLE001
290
+ return {"error": "the page did not change as in the demo"}
291
+ self.page.wait_for_load_state()
292
+ return {"error": None}
293
+
294
+ def _observe_and_mitigate(self, state: AgentState, action: dict) -> None:
295
+ """Probe the element, score the risk, and fix the page first if it is worth it."""
296
+ from mimic.risk import mitigate, probe
297
+
298
+ features = probe(
299
+ self._locator,
300
+ action,
301
+ state["locator_strategy"],
302
+ state["confidence"],
303
+ max(len(state["tried"]) - 1, 0),
304
+ )
305
+ if self._observation:
306
+ # Retry of the same step: keep what was observed at the first attempt (the decision
307
+ # point the model learns), just add the new mitigations.
308
+ first = self._observation
309
+ features = first["features"]
310
+ risk = self.risk_model.score(features) if self.risk_model is not None else None
311
+ should = self.risk_policy == "always" or (
312
+ self.risk_policy == "predicted"
313
+ and risk is not None
314
+ and risk >= self.risk_model.threshold
315
+ )
316
+ done = mitigate(self.page, self._locator) if should else []
317
+ if self._observation:
318
+ self._observation["mitigations"] += done
319
+ else:
320
+ self._observation = {"features": features, "risk": risk, "mitigations": done}
321
+
322
+ def verify(self, state: AgentState) -> dict:
323
+ if state["status"] == "skipped":
324
+ self._pending_heal = None # not verified either way: neither remember nor reject
325
+ return {}
326
+ action = state["actions"][state["idx"]]
327
+ ok = state["locator_strategy"] is not None and state["error"] is None
328
+ if ok and action["action"] in ("fill", "select", "check", "choose", "upload"):
329
+ try:
330
+ problem = self._check_value(action)
331
+ if problem:
332
+ ok, state = False, {**state, "error": problem}
333
+ except Exception as exc: # noqa: BLE001
334
+ ok, state = False, {**state, "error": type(exc).__name__}
335
+
336
+ if self._pending_heal is not None:
337
+ (self.healer.confirm if ok else self.healer.reject)(self._pending_heal)
338
+ self._pending_heal = None
339
+
340
+ attempts = state["attempt"] + 1
341
+ if ok or attempts >= MAX_ATTEMPTS or not self._strategies_left(state):
342
+ result = StepResult(
343
+ index=state["idx"],
344
+ action=action["action"],
345
+ strategy=state["locator_strategy"] if ok else None,
346
+ confidence=state["confidence"] if ok else 0.0,
347
+ attempts=attempts,
348
+ latency_ms=int((time.monotonic() - state["step_started"]) * 1000),
349
+ success=ok,
350
+ error=None if ok else state["error"],
351
+ fallbacks=max(len(state["tried"]) - 1, 0),
352
+ healed=ok
353
+ and state["locator_strategy"] in ("memory", "matcher", "semantic", "vision"),
354
+ features=self._observation.get("features"),
355
+ risk=self._observation.get("risk"),
356
+ mitigations=self._observation.get("mitigations", []),
357
+ )
358
+ self._observation = {}
359
+ if self.on_step:
360
+ self.on_step(result)
361
+ last = state["idx"] + 1 >= len(state["actions"])
362
+ status = "failed" if not ok else ("done" if last else "running")
363
+ return {
364
+ "results": [*state["results"], result],
365
+ "idx": state["idx"] + (1 if ok else 0),
366
+ "attempt": 0,
367
+ "tried": [],
368
+ "error": None if ok else state["error"],
369
+ "step_started": time.monotonic(),
370
+ "status": status,
371
+ }
372
+ return {"attempt": attempts, "error": state["error"]} # retry with next strategy
373
+
374
+ def _check_value(self, action: dict) -> str | None:
375
+ """Read the element back: the action only counts if the page shows the right value."""
376
+ kind = action["action"]
377
+ if kind == "check":
378
+ want = as_bool(action["value"])
379
+ return None if self._locator.is_checked() == want else "checkbox did not change"
380
+ if kind == "choose":
381
+ return None if self._locator.is_checked() else "option not selected"
382
+ if kind == "upload":
383
+ name = self._locator.evaluate(
384
+ "el => el.files && el.files.length ? el.files[0].name : ''"
385
+ )
386
+ return None if name == Path(str(action["value"])).name else "file not attached"
387
+ if kind == "select":
388
+ # A dropdown shows a label ("Travel") but stores a value ("travel"): either is fine.
389
+ value, label = self._locator.evaluate(
390
+ "el => [el.value, el.selectedOptions[0] ? el.selectedOptions[0].text : '']"
391
+ )
392
+ want = " ".join(str(action["value"]).lower().split())
393
+ if want in (" ".join(value.lower().split()), " ".join(label.lower().split())):
394
+ return None
395
+ return f"value mismatch: {label or value!r}"
396
+ actual = self._locator.input_value()
397
+ if _same_value(actual, str(action["value"])):
398
+ return None
399
+ return f"value mismatch: {actual!r}"
400
+
401
+ def _strategies_left(self, state: AgentState) -> bool:
402
+ action = state["actions"][state["idx"]]
403
+ names = [n for n, _ in _candidates(self.page, action)]
404
+ if self.healer is not None and action["action"] != "choose":
405
+ names.append("heal")
406
+ return any(n not in state["tried"] for n in names)
407
+
408
+ @staticmethod
409
+ def route(state: AgentState) -> str:
410
+ if state["status"] in ("done", "failed", "skipped"):
411
+ return END
412
+ return "locate"
413
+
414
+ # ---------- graph ----------
415
+
416
+ def _build(self):
417
+ g = StateGraph(AgentState)
418
+ g.add_node("locate", self.locate)
419
+ g.add_node("act", self.act)
420
+ g.add_node("verify", self.verify)
421
+ g.add_edge(START, "locate")
422
+ g.add_edge("locate", "act")
423
+ g.add_edge("act", "verify")
424
+ g.add_conditional_edges("verify", self.route, ["locate", END])
425
+ return g.compile()
426
+
427
+ def run(self, actions: list[dict]) -> TaskResult:
428
+ t0 = time.monotonic()
429
+ final = self.graph.invoke(
430
+ AgentState(
431
+ actions=actions,
432
+ idx=0,
433
+ attempt=0,
434
+ tried=[],
435
+ locator_strategy=None,
436
+ confidence=0.0,
437
+ error=None,
438
+ step_started=t0,
439
+ results=[],
440
+ status="running",
441
+ ),
442
+ config={"recursion_limit": 20 * len(actions) + 10},
443
+ )
444
+ return TaskResult(
445
+ success=final["status"] == "done",
446
+ skipped=final["status"] == "skipped",
447
+ steps=final["results"],
448
+ duration_ms=int((time.monotonic() - t0) * 1000),
449
+ error=final["error"],
450
+ )
451
+
452
+
453
+ def _same_value(actual: str, expected: str) -> bool:
454
+ if actual == expected:
455
+ return True
456
+ try:
457
+ return abs(float(actual) - float(expected)) < 1e-9
458
+ except ValueError:
459
+ return False
mimic/api/__init__.py ADDED
File without changes
mimic/api/main.py ADDED
@@ -0,0 +1,37 @@
1
+ """Mimic API: control plane for skills and agent runs."""
2
+
3
+ import httpx
4
+ from fastapi import FastAPI
5
+
6
+ from mimic import __version__
7
+ from mimic.config import get_settings
8
+
9
+ app = FastAPI(title="Mimic", version=__version__)
10
+
11
+
12
+ def _check_db() -> str:
13
+ try:
14
+ from mimic.db import connect
15
+
16
+ with connect() as conn:
17
+ conn.execute("SELECT 1")
18
+ return "ok"
19
+ except Exception as exc: # noqa: BLE001
20
+ return f"down ({type(exc).__name__})"
21
+
22
+
23
+ def _check_ollama() -> str:
24
+ try:
25
+ r = httpx.get(f"{get_settings().ollama_host}/api/tags", timeout=2)
26
+ r.raise_for_status()
27
+ models = [m["name"] for m in r.json().get("models", [])]
28
+ return f"ok ({len(models)} models)"
29
+ except Exception as exc: # noqa: BLE001
30
+ return f"down ({type(exc).__name__})"
31
+
32
+
33
+ @app.get("/health")
34
+ def health() -> dict:
35
+ checks = {"db": _check_db(), "ollama": _check_ollama()}
36
+ status = "ok" if all(v.startswith("ok") for v in checks.values()) else "degraded"
37
+ return {"status": status, "version": __version__, "checks": checks}