agentdynamics 0.5.0__tar.gz → 0.6.0__tar.gz

{agentdynamics-0.5.0 → agentdynamics-0.6.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: agentdynamics
-Version: 0.5.0
+Version: 0.6.0
 Summary: Application performance monitoring for AI agents: flows, cost, reliability and process quality
 License-Expression: Apache-2.0
 Project-URL: Homepage, https://github.com/Aditya31398/agentdynamics

{agentdynamics-0.5.0 → agentdynamics-0.6.0}/agentdynamics/__init__.py

@@ -5,7 +5,7 @@
 
 See agentdynamics.autotrace for @trace / @tool / span.
 """
-__version__ = "0.5.0"
+__version__ = "0.6.0"
 
 from .autotrace import flush, init, llm_call, outcome, record_llm, span, tool, trace  # noqa: E402,F401
 

{agentdynamics-0.5.0 → agentdynamics-0.6.0}/agentdynamics/analysis.py

@@ -9,6 +9,7 @@ AppDynamics -> AgentDynamics mapping implemented here:
   Code-level hotspots   -> waste findings (redundant reads, loops, error streaks)
 """
 import math
+import operator
 import re
 import statistics
 import time
@@ -623,8 +624,69 @@ def _apply_grades(tasks, grades):
             t["outcome_source"], t["outcome_reason"] = "inferred", None
 
 
-def finalize(runs, tasks_by_run, rules=None, now=None, grades=None):
-    """Cross-run analysis: outcomes, subagent roll-up, baselines, scores, events."""
+# Everything finalize writes onto a task *before* scoring, for tasks whose run did not change. If one of
+# these moves, the task's row must be rewritten and re-scored; if none did and its baseline didn't either,
+# the previous score and events still hold. A field added to the settle phase below must be added here,
+# or incremental refreshes will serve it stale -- test_incremental.py compares every column against a
+# full rebuild to catch exactly that.
+SETTLED_FIELDS = ("outcome", "outcome_source", "outcome_reason", "next_prompt", "rework", "root_error",
+                  "task_type", "task_type_source", "task_type_match", "workflow",
+                  "subagent_cost", "subagents", "parent_task_id")
+
+
+_settled = operator.itemgetter(*SETTLED_FIELDS)     # built in C: this runs for every task on every refresh
+
+
+def _settled_values(t):
+    try:
+        return _settled(t)
+    except KeyError:          # a field some sources never set (Claude Code tasks carry no root_error)
+        return tuple(t.get(f) for f in SETTLED_FIELDS)
+
+
+class ScoreCache:
+    """What finalize needs to skip unchanged tasks on the next refresh.
+
+    sig[task_id]     the settled fields plus the two baseline numbers scoring reads
+    events[task_id]  the health events that signature produced
+    changed          task ids scored this time (their rows must be written)
+    """
+
+    def __init__(self):
+        self.sig, self.events, self.changed = {}, {}, set()
+
+
+BASELINE_EXACT_UP_TO = 20      # below this many tasks, a type's baseline uses all of them
+BASELINE_GROWTH = 1.05          # above it, only when the count has grown 5% since the last step
+
+
+def baseline_sample_size(n):
+    """How many of a type's n tasks its baseline is computed from.
+
+    Every task is scored against its type's baseline, so a baseline that moved with each arrival would
+    make every refresh re-score the whole type -- no cheaper than rebuilding. Instead the baseline is
+    computed from the type's *earliest* M tasks (by start, then id), and M only advances in 5% steps: a
+    type re-scores once per 5% of growth, about 20 re-scores per new task however large the store, and
+    never oscillates. It depends on the data alone, so a full rebuild computes the same baseline. It
+    leaves out at most the newest 5% of a type's history, which is noise for a "what is normal" figure.
+    """
+    if n <= BASELINE_EXACT_UP_TO:
+        return n
+    m = BASELINE_EXACT_UP_TO
+    while True:
+        nxt = max(m + 1, int(m * BASELINE_GROWTH))
+        if nxt > n:
+            return m
+        m = nxt
+
+
+def finalize(runs, tasks_by_run, rules=None, now=None, grades=None, cache=None, dirty=()):
+    """Cross-run analysis: outcomes, subagent roll-up, baselines, scores, events.
+
+    With a `cache` (a ScoreCache kept between calls), tasks from runs not in `dirty` whose settled
+    fields and baseline are unchanged keep their previous score and events instead of being re-scored;
+    `cache.changed` then lists the task ids that were. The result is the same as without a cache.
+    """
     rules = rules or DEFAULT_RULES
     now = now or time.time()
     all_tasks = [t for r in runs for t in tasks_by_run[r["id"]]]
@@ -637,11 +699,19 @@ def finalize(runs, tasks_by_run, rules=None, now=None, grades=None):
         if run["source"] == "claude-code" or not run.get("workflow"):
             _outcomes_claude(ts, now)
         else:
+            # Derived afresh every pass, never carried over. The engine keeps an unchanged run's task
+            # objects between refreshes, and linking below only *sets* these on tasks that have a
+            # successor: a task whose follow-up was deleted, edited or moved to another thread kept the
+            # old values -- and so its "rework" outcome -- until a full rebuild. test_incremental found it.
+            for t in ts:
+                t["next_prompt"], t["rework"] = None, 0
             if run.get("thread_id"):
                 threads[(run["source"], run["thread_id"])].extend(ts)
     # traced conversations: the next trace in the same thread plays the role of "your next message"
     for group in threads.values():
-        group.sort(key=lambda x: x["started"] or 0)
+        # ties on start time break by id, not by whatever order the runs happen to be held in -- which
+        # differs between a long-running engine and a fresh rebuild
+        group.sort(key=lambda x: (x["started"] or 0, x["id"]))
         for a, b in zip(group, group[1:]):
             a["next_prompt"] = b["prompt"][:300]
             a["rework"] = 1 if CORRECTION.search(b["prompt"][:200]) else 0
@@ -694,10 +764,12 @@ def finalize(runs, tasks_by_run, rules=None, now=None, grades=None):
     for k, g in groups.items():
         if len(g) < 3 and k != "__all__":
             continue
+        n = len(g)
+        g = sorted(g, key=lambda t: (t["started"] or 0, t["id"]))[:baseline_sample_size(n)]
         costs = [t["cost"] + t["subagent_cost"] for t in g]
         durs = [t["duration_s"] for t in g]
         baselines[k] = {
-            "n": len(g),
+            "n": n, "sample": len(g),
             "cost_p50": pct(costs, 0.5), "cost_p90": pct(costs, 0.9),
             "duration_p50": pct(durs, 0.5), "duration_p90": pct(durs, 0.9),
             "tokens_p50": pct([t["total_tokens"] for t in g], 0.5),
@@ -706,13 +778,31 @@ def finalize(runs, tasks_by_run, rules=None, now=None, grades=None):
         }
 
     events = []
+    dirty = set(dirty)
+    if cache is not None:
+        cache.changed = set()
+        live = {t["id"] for t in all_tasks}
+        for tid in [k for k in cache.sig if k not in live]:     # tasks that no longer exist
+            cache.sig.pop(tid, None)
+            cache.events.pop(tid, None)
     for t in all_tasks:
+        if cache is not None:
+            b = baselines.get(t["task_type"]) or baselines.get("__all__") or {}
+            sig = (_settled_values(t), b.get("cost_p50"), b.get("duration_p50"))
+            if t["run_id"] not in dirty and cache.sig.get(t["id"]) == sig:
+                events.extend(cache.events[t["id"]])            # nothing it depends on moved
+                continue
         t["failed"] = 1 if t.get("outcome") == "failed" else 0
         if t["llm_calls"] == 0 and t["tool_calls"] == 0:
             t.update({"scores": {}, "score": None, "apdex": None, "cost_vs_baseline": None, "duration_vs_baseline": None})
-            continue
-        score_task(t, baselines)
-        events.extend(evaluate_rules(t, rules))
+            ev = []
+        else:
+            score_task(t, baselines)
+            ev = evaluate_rules(t, rules)
+        events.extend(ev)
+        if cache is not None:
+            cache.sig[t["id"]], cache.events[t["id"]] = sig, ev
+            cache.changed.add(t["id"])
     return all_tasks, baselines, events
 
 
@@ -725,7 +815,11 @@ def analyze(runs, rules=None, now=None):
 
 def process_insights(tasks):
     """Aggregate findings meant to help a human assess how the agent works."""
-    main = [t for t in tasks if not t["is_subagent"] and t["llm_calls"] > 0]
+    # One canonical order. The engine holds tasks in whatever order runs arrived, which differs between
+    # a long-running process and a fresh rebuild; float sums round differently in a different order,
+    # and example lists are cut to their first few. Sorting makes the result depend on the data alone.
+    main = sorted((t for t in tasks if not t["is_subagent"] and t["llm_calls"] > 0),
+                  key=lambda t: (t["started"] or 0, t["id"]))
     if not main:
         return []
     out = []

{agentdynamics-0.5.0 → agentdynamics-0.6.0}/agentdynamics/engine.py

@@ -64,6 +64,11 @@ class Engine:
         # traffic happened to dirty something. force=True would also work, but it throws away every
         # cache and re-parses every source to re-settle outcomes that only need re-finalizing.
         self._regrade = False
+        # Scores carried between refreshes, so a refresh re-scores and rewrites only the tasks whose
+        # inputs changed instead of the whole store (analysis.ScoreCache). The clock is injectable
+        # because outcomes depend on it: a recent task is "in progress", an old one is not.
+        self._cache = analysis.ScoreCache()
+        self._clock = time.time
         self.stats = {"spans_ingested": 0, "refreshes": 0, "alerts_sent": 0}
         self.sources = {}
         if claude_root:
@@ -319,6 +324,7 @@ class Engine:
                 self._span_since = 0
                 self._runs.clear()
                 self._tasks.clear()
+                self._cache = analysis.ScoreCache()
             changed, removed = self._scan_files()
             dirty = {}
             for op, x in changed:
@@ -341,6 +347,10 @@ class Engine:
             if not dirty and not removed and not force and not self._first and not self._regrade:
                 return False
             self._regrade = False
+            full = force or self._first
+            # every task id that may no longer exist: those of removed runs, and the previous tasks of
+            # runs being re-analysed (a PATCH can re-segment a run into fewer tasks)
+            gone = {t["id"] for rid in list(removed) + list(dirty) for t in self._tasks.get(rid, [])}
             for rid in removed:
                 self._runs.pop(rid, None)
                 self._tasks.pop(rid, None)
@@ -351,12 +361,21 @@ class Engine:
                 self._runs[rid] = run
                 self._tasks[rid] = analysis.run_tasks(run)
             runs = list(self._runs.values())
-            tasks, baselines, events = analysis.finalize(runs, self._tasks, self.rules(),
-                                                         grades=store.get_grades(self.con))
+            tasks, baselines, events = analysis.finalize(runs, self._tasks, self.rules(), now=self._clock(),
+                                                         grades=store.get_grades(self.con),
+                                                         cache=self._cache, dirty=dirty.keys())
             insights = analysis.process_insights(tasks)
             meta = {"refreshed": time.time(), "runs": len(runs), "tasks": len(tasks), "insights": insights}
             store.write_runs(self.con, list(dirty.values()), removed, self.redactor)
-            store.write_analysis(self.con, tasks, baselines, events, meta, self.redactor)
+            if full:
+                # the first refresh of a process must replace whatever the last process left behind
+                store.write_analysis(self.con, tasks, baselines, events, meta, self.redactor)
+            else:
+                changed = self._cache.changed
+                events = [e for e in events if e["task_id"] in changed]
+                store.write_analysis_delta(self.con, [t for t in tasks if t["id"] in changed],
+                                           gone - changed, events, baselines, meta, self.redactor)
+            self.stats["tasks_rescored"] = len(self._cache.changed)
             self._dispatch_alerts(events)
             self._first = False
             self.last_refresh = time.time()

{agentdynamics-0.5.0 → agentdynamics-0.6.0}/agentdynamics/store.py

@@ -61,6 +61,7 @@ CREATE INDEX IF NOT EXISTS steps_run ON steps(run_id);
 CREATE INDEX IF NOT EXISTS steps_node ON steps(node);
 CREATE INDEX IF NOT EXISTS tasks_run ON tasks(run_id);
 CREATE INDEX IF NOT EXISTS tasks_started ON tasks(started);
+CREATE INDEX IF NOT EXISTS events_task ON events(task_id);
 
 CREATE TABLE IF NOT EXISTS spans_raw (source, trace_id, span_id, fmt, doc, updated REAL, PRIMARY KEY(source, span_id)) WITHOUT ROWID;
 CREATE INDEX IF NOT EXISTS spans_trace ON spans_raw(source, trace_id);
@@ -145,30 +146,55 @@ def update_step_analysis(con, runs):
         con.executemany("UPDATE steps SET task_id=?, flags=?, attributed_cost=? WHERE run_id=? AND seq=?", rows_)
 
 
+def _task_row(t, red):
+    row = [t.get(c) for c in TASK_COLS] + [json.dumps(t.get(c) if t.get(c) is not None else ([] if c == "path" else {}))
+                                           for c in TASK_JSON]
+    if red:
+        for f in ("prompt", "final_text", "next_prompt", "root_error"):
+            i = TASK_COLS.index(f)
+            row[i] = red.text(row[i])
+    return row
+
+
+def _event_row(e, red):
+    row = [e.get(c) for c in EVENT_COLS]
+    if red and red.rx is not None:  # messages can quote user text (e.g. the correcting follow-up)
+        i = EVENT_COLS.index("message")
+        row[i] = red.rx.sub("[REDACTED]", row[i] or "")
+    return row
+
+
+def _write_small(con, baselines, meta):
+    for t in ("baselines", "meta"):
+        con.execute(f"DELETE FROM {t}")
+    _ins(con, "baselines", ["task_type", "data"], [[k, json.dumps(v)] for k, v in baselines.items()])
+    _ins(con, "meta", ["k", "v"], [[k, json.dumps(v)] for k, v in meta.items()])
+
+
 def write_analysis(con, tasks, baselines, events, meta, red=None):
+    """Replace every derived analysis row: after a full rebuild."""
     with con:
-        for t in ("tasks", "events", "baselines", "meta"):
+        for t in ("tasks", "events"):
             con.execute(f"DELETE FROM {t}")
-        trows = []
-        for t in tasks:
-            row = [t.get(c) for c in TASK_COLS] + [json.dumps(t.get(c) if t.get(c) is not None else ([] if c == "path" else {}))
-                                                   for c in TASK_JSON]
-            if red:
-                for f in ("prompt", "final_text", "next_prompt", "root_error"):
-                    i = TASK_COLS.index(f)
-                    row[i] = red.text(row[i])
-            trows.append(row)
-        _ins(con, "tasks", TASK_COLS + TASK_JSON, trows)
-        erows = []
-        for e in events:
-            row = [e.get(c) for c in EVENT_COLS]
-            if red and red.rx is not None:  # messages can quote user text (e.g. the correcting follow-up)
-                i = EVENT_COLS.index("message")
-                row[i] = red.rx.sub("[REDACTED]", row[i] or "")
-            erows.append(row)
-        _ins(con, "events", EVENT_COLS, erows)
-        _ins(con, "baselines", ["task_type", "data"], [[k, json.dumps(v)] for k, v in baselines.items()])
-        _ins(con, "meta", ["k", "v"], [[k, json.dumps(v)] for k, v in meta.items()])
+        _ins(con, "tasks", TASK_COLS + TASK_JSON, [_task_row(t, red) for t in tasks])
+        _ins(con, "events", EVENT_COLS, [_event_row(e, red) for e in events])
+        _write_small(con, baselines, meta)
+
+
+def write_analysis_delta(con, changed, removed_ids, events, baselines, meta, red=None):
+    """Write only what an incremental refresh changed: the rows of `changed` tasks and their events,
+    the removal of `removed_ids`, and the small baselines/meta tables. `events` must be exactly the
+    events of the changed tasks. Rewriting every row cost 2.1 s of a 5.6 s refresh at 20k tasks."""
+    ids = [t["id"] for t in changed] + list(removed_ids)
+    with con:
+        for i in range(0, len(ids), 500):
+            chunk = ids[i:i + 500]
+            marks = ",".join("?" * len(chunk))
+            con.execute(f"DELETE FROM tasks WHERE id IN ({marks})", chunk)
+            con.execute(f"DELETE FROM events INDEXED BY events_task WHERE task_id IN ({marks})", chunk)
+        _ins(con, "tasks", TASK_COLS + TASK_JSON, [_task_row(t, red) for t in changed])
+        _ins(con, "events", EVENT_COLS, [_event_row(e, red) for e in events])
+        _write_small(con, baselines, meta)
 
 
 # ---------------------------------------------------------------- durable span store

{agentdynamics-0.5.0 → agentdynamics-0.6.0}/agentdynamics.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: agentdynamics
-Version: 0.5.0
+Version: 0.6.0
 Summary: Application performance monitoring for AI agents: flows, cost, reliability and process quality
 License-Expression: Apache-2.0
 Project-URL: Homepage, https://github.com/Aditya31398/agentdynamics

{agentdynamics-0.5.0 → agentdynamics-0.6.0}/agentdynamics.egg-info/SOURCES.txt

@@ -43,6 +43,7 @@ tests/test_autotrace.py
 tests/test_console_ui.py
 tests/test_core.py
 tests/test_ecosystem.py
+tests/test_incremental.py
 tests/test_integrations.py
 tests/test_live_anthropic.py
 tests/test_outcomes.py

{agentdynamics-0.5.0 → agentdynamics-0.6.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "agentdynamics"
-version = "0.5.0"
+version = "0.6.0"
 description = "Application performance monitoring for AI agents: flows, cost, reliability and process quality"
 readme = "README.md"
 license = "Apache-2.0"

agentdynamics-0.6.0/tests/test_incremental.py

@@ -0,0 +1,240 @@
+"""Incremental refresh (#5) must give exactly what a full rebuild gives.
+
+An incremental refresh re-scores and rewrites only the tasks whose inputs changed. That is only safe if
+"inputs" is complete, and several of them cross task boundaries: a new trace in a conversation changes
+the *previous* task's rework flag; a subagent run changes its parent's cost; a grade changes a task no
+new data touched; the clock turns "in progress" into "unknown"; and every task is scored against its
+type's baseline, which new traffic moves.
+
+So each scenario here is a seeded random sequence of ingests, updates, deletions, grades and clock
+advances, refreshed incrementally after every step. At checkpoints the store is compared, every column
+of every task, event and baseline row, against a full rebuild from the same sources.
+"""
+import json
+import os
+import random
+import shutil
+import sys
+import tempfile
+import time
+import unittest
+
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), ".."))
+
+from agentdynamics import analysis  # noqa: E402
+from agentdynamics.engine import Engine  # noqa: E402
+
+WORKFLOWS = ["support", "billing", "research"]
+PROMPTS = ["Where is my refund?", "Update my address", "still not working", "Cancel the order",
+           "no, that's wrong", "Thanks, that worked"]
+
+
+class Scenario:
+    def __init__(self, seed, tmp):
+        self.rng = random.Random(seed)
+        self.clock = 1_900_000_000.0            # fixed and injectable: outcomes depend on "now"
+        self.eng = Engine(os.path.join(tmp, "data"), None)
+        self.eng._clock = lambda: self.clock
+        self.runs = {}                          # run id -> payload, for updates and subagents
+        self.n = 0
+
+    def run(self, rid=None, **over):
+        rng = self.rng
+        self.n += 1
+        rid = rid or f"r{self.n}"
+        ts = self.clock - rng.choice([30, 200, 900, 5000])       # some inside the 600 s "in progress" window
+        steps = [{"kind": "prompt", "ts": ts, "text": rng.choice(PROMPTS)}]
+        t = ts
+        for _ in range(rng.randint(1, 3)):
+            steps.append({"kind": "llm", "ts": t, "end_ts": t + 1, "model": "claude-sonnet-5",
+                          "input_tokens": rng.randint(100, 5000), "output_tokens": rng.randint(10, 800),
+                          "stop_reason": "end_turn"})
+            t += 1.5
+            if rng.random() < 0.7:
+                steps.append({"kind": "tool", "ts": t, "end_ts": t + 0.2, "name": rng.choice(["kb", "db", "http"]),
+                              "is_error": rng.random() < 0.15, "input": {"q": rng.randint(1, 9)}})
+                t += 0.3
+        if rng.random() < 0.15:
+            steps.append({"kind": "notice", "name": "outcome", "ts": t,
+                          "outcome": rng.choice(["completed", "failed", "rework"]), "text": "graded in run"})
+        p = {"id": rid, "project": "inc", "workflow": rng.choice(WORKFLOWS), "steps": steps,
+             "thread_id": rng.choice([None, "th-a", "th-a", "th-b"]),
+             "status": "error" if rng.random() < 0.1 else "ok", "complete": rng.random() > 0.2}
+        if rng.random() < 0.2:
+            p["feedback"] = [{"key": "user", "score": rng.choice([0.1, 0.9])}]
+        p.update(over)
+        self.runs[rid] = p
+        self.eng.ingest(p)
+        return rid
+
+    def step(self):
+        rng, e = self.rng, self.eng
+        op = rng.random()
+        if op < 0.45 or not self.runs:
+            self.run()
+        elif op < 0.55:                                          # a subagent: changes its (clean) parent
+            parent = rng.choice(sorted(self.runs))
+            self.run(parent_id=parent)
+        elif op < 0.65:                                          # an existing run is updated
+            self.run(rid=rng.choice(sorted(self.runs)))
+        elif op < 0.72:                                          # a run is deleted at its source
+            rid = rng.choice(sorted(self.runs))
+            path = os.path.join(e.runs_dir, f"{rid}.json")
+            if os.path.exists(path):
+                os.remove(path)
+            self.runs.pop(rid)
+        elif op < 0.82:                                          # a grade on a task no data touched
+            rid = rng.choice(sorted(self.runs))
+            e.grade(f"{rid}#0", rng.choice(["completed", "failed", "rework"]), "graded later")
+        elif op < 0.86:
+            rid = rng.choice(sorted(self.runs))
+            e.ungrade(f"{rid}#0")
+        elif op < 0.93:                                          # a late span on an existing OTLP trace
+            self.otlp(rng.randint(1, 4))
+        else:                                                    # time passes: "in progress" expires
+            self.clock += rng.choice([300, 700, 4000])
+        e.refresh()
+
+    def otlp(self, trace_no):
+        rng = self.rng
+        tid = f"{trace_no:032x}"
+        ts = self.clock - 800
+        sid = f"{rng.randint(1, 10**12):016x}"
+        body = {"resourceSpans": [{"resource": {"attributes": [
+            {"key": "service.name", "value": {"stringValue": "inc-otlp"}}]},
+            "scopeSpans": [{"spans": [{
+                "traceId": tid, "spanId": sid, "name": "chat", "status": {},
+                "startTimeUnixNano": str(int(ts * 1e9)), "endTimeUnixNano": str(int((ts + 1) * 1e9)),
+                "attributes": [
+                    {"key": "gen_ai.operation.name", "value": {"stringValue": "chat"}},
+                    {"key": "gen_ai.request.model", "value": {"stringValue": "claude-haiku-4-5"}},
+                    {"key": "gen_ai.usage.input_tokens", "value": {"intValue": str(rng.randint(100, 3000))}},
+                    {"key": "gen_ai.usage.output_tokens", "value": {"intValue": str(rng.randint(10, 300))}}]}]}]}]}
+        self.eng.ingest_otlp(json.dumps(body).encode(), "application/json")
+
+    def snapshot(self):
+        con = self.eng.con
+
+        def table(q):
+            return [dict(r) for r in con.execute(q)]
+        meta = {r["k"]: json.loads(r["v"]) for r in con.execute("SELECT k, v FROM meta") if r["k"] != "refreshed"}
+        return {"tasks": table("SELECT * FROM tasks ORDER BY id"),
+                "events": table("SELECT * FROM events ORDER BY id"),
+                "baselines": table("SELECT * FROM baselines ORDER BY task_type"),
+                "meta": meta}
+
+
+class IncrementalEqualsFullTest(unittest.TestCase):
+    def setUp(self):
+        self.tmp = tempfile.mkdtemp()
+
+    def tearDown(self):
+        shutil.rmtree(self.tmp, ignore_errors=True)
+
+    def assert_same(self, inc, full, where):
+        self.assertEqual(len(inc["tasks"]), len(full["tasks"]), f"{where}: task count")
+        for a, b in zip(inc["tasks"], full["tasks"]):
+            if a != b:
+                diff = {k: (a[k], b[k]) for k in a if a[k] != b.get(k)}
+                self.fail(f"{where}: task {a['id']} differs from a full rebuild (incremental, full): {diff}")
+        self.assertEqual(inc["events"], full["events"], f"{where}: events")
+        self.assertEqual(inc["baselines"], full["baselines"], f"{where}: baselines")
+        self.assertEqual(inc["meta"], full["meta"], f"{where}: insights")
+
+    def test_random_histories(self):
+        rescored = total = 0
+        for seed in range(12):
+            with self.subTest(seed=seed):
+                sc = Scenario(seed, os.path.join(self.tmp, f"s{seed}"))
+                try:
+                    sc.eng.refresh(force=True)
+                    for i in range(1, 121):
+                        sc.step()
+                        rescored += sc.eng.stats.get("tasks_rescored", 0)
+                        total += sc.eng.con.execute("SELECT COUNT(*) FROM tasks").fetchone()[0]
+                        if i % 30 == 0:
+                            inc = sc.snapshot()
+                            sc.eng.refresh(force=True)                      # a full rebuild from sources
+                            self.assert_same(inc, sc.snapshot(), f"seed {seed}, step {i}")
+                finally:
+                    sc.eng.con.close()
+        # and it has to actually be incremental: most refreshes re-score a small part of the store
+        self.assertLess(rescored / total, 0.5, f"re-scored {rescored} of {total} task-refreshes")
+
+
+class OnlyNewTrafficIsScoredTest(unittest.TestCase):
+    """The point of #5, asserted as a count rather than a timing, so it cannot flake: into a store of
+    several hundred tasks, 10 new ones are re-scored and nothing else is -- unless the type has grown
+    past its next baseline step, when the whole type is re-scored once."""
+
+    def test_new_tasks_only_until_the_baseline_steps(self):
+        from agentdynamics.analysis import baseline_sample_size as m
+        n0 = next(n for n in range(500, 2000) if m(n) == m(n + 10) and m(n + 10) < m(n + 60))
+        tmp = tempfile.mkdtemp()
+        eng = Engine(os.path.join(tmp, "data"), None)
+        try:
+            eng.ingest_otlp(otlp_traces(0, n0), "application/json")
+            time.sleep(2.1)                  # outside the 2 s span-reassembly window (see bench/bench.py)
+            eng.refresh(force=True)
+            eng.ingest_otlp(otlp_traces(n0, 10), "application/json")
+            eng.refresh()
+            self.assertEqual(eng.stats["tasks_rescored"], 10, "only the new tasks")
+            # grow past the next step: the type's baseline moves, so every task of it is re-scored once
+            time.sleep(2.1)
+            eng.ingest_otlp(otlp_traces(n0 + 10, 50), "application/json")
+            eng.refresh()
+            self.assertGreater(eng.stats["tasks_rescored"], 50)
+            self.assertEqual(eng.stats["tasks_rescored"],
+                             eng.con.execute("SELECT COUNT(*) FROM tasks").fetchone()[0])
+        finally:
+            eng.con.close()
+            shutil.rmtree(tmp, ignore_errors=True)
+
+
+def otlp_traces(first, n):
+    """n OTLP traces of one workflow, each a single model call with a varying cost."""
+    now = time.time() - 3600
+    spans = []
+    for i in range(first, first + n):
+        tid = f"{i + 1:032x}"
+        ts = now + i
+        spans.append({"traceId": tid, "spanId": f"{i + 1:016x}", "name": "support", "status": {},
+                      "startTimeUnixNano": str(int(ts * 1e9)), "endTimeUnixNano": str(int((ts + 2) * 1e9)),
+                      "attributes": [
+                          {"key": "gen_ai.operation.name", "value": {"stringValue": "chat"}},
+                          {"key": "gen_ai.agent.name", "value": {"stringValue": "support"}},
+                          {"key": "gen_ai.request.model", "value": {"stringValue": "claude-sonnet-5"}},
+                          {"key": "gen_ai.usage.input_tokens", "value": {"intValue": str(500 + (i * 37) % 3000)}},
+                          {"key": "gen_ai.usage.output_tokens", "value": {"intValue": str(50 + (i * 11) % 400)}}]})
+    return json.dumps({"resourceSpans": [{"resource": {"attributes": [
+        {"key": "service.name", "value": {"stringValue": "inc"}}]}, "scopeSpans": [{"spans": spans}]}]}).encode()
+
+
+class SettledFieldsTest(unittest.TestCase):
+    def test_every_field_the_settle_phase_writes_is_in_the_signature(self):
+        """A field finalize writes before scoring but leaves out of SETTLED_FIELDS is served stale by
+        incremental refreshes. Diff a task's fields before and after the settle phase on a history that
+        exercises threads, subagents, grades and feedback, and require every changed one be listed."""
+        tmp = tempfile.mkdtemp()
+        try:
+            sc = Scenario(3, tmp)
+            for _ in range(60):
+                sc.step()
+            runs = list(sc.eng._runs.values())
+            fresh = {r["id"]: analysis.run_tasks(r) for r in runs}
+            before = {t["id"]: dict(t) for ts in fresh.values() for t in ts}
+            scored = {"scores", "score", "apdex", "cost_vs_baseline", "duration_vs_baseline", "failed", "_sdk_grade"}
+            analysis.finalize(runs, fresh, now=sc.clock, grades={"r1#0": {"outcome": "failed"}})
+            written = set()
+            for ts in fresh.values():
+                for t in ts:
+                    written |= {k for k, v in t.items() if before[t["id"]].get(k, object()) != v}
+            missing = written - scored - set(analysis.SETTLED_FIELDS)
+            self.assertEqual(missing, set(), "finalize writes these but SETTLED_FIELDS leaves them out")
+            sc.eng.con.close()
+        finally:
+            shutil.rmtree(tmp, ignore_errors=True)
+
+
+if __name__ == "__main__":
+    unittest.main()