agentdebugx 0.4.0__tar.gz → 0.5.0__tar.gz

{agentdebugx-0.4.0 → agentdebugx-0.5.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: agentdebugx
-Version: 0.4.0
+Version: 0.5.0
 Summary: Portable error analysis, tracing, and recovery framework for agentic AI systems. Import as `agentdebug`.
 License: MIT
 License-File: LICENSE

agentdebugx-0.5.0/docs/taxonomy_manifest.md

@@ -0,0 +1,36 @@
+# Taxonomy manifest and classification contract
+
+`agentdebug.taxonomy_manifest` freezes `SEED_FAILURE_MODES` into a JSON-serialisable
+document with a content fingerprint and the package revision it came from, and defines a
+side-effect-free classification contract on top of it. A label stored in a corpus is only
+interpretable if the reader can recover the exact definitions the labeller saw; the
+fingerprint is how a corpus notices that its labels and its taxonomy drifted apart.
+
+```python
+from agentdebug.taxonomy_manifest import (
+    ClassificationRequest, cohens_kappa, compile_classification_prompt,
+    parse_classification_response, taxonomy_manifest,
+)
+
+manifest = taxonomy_manifest()             # 23 modes in 9 families, fingerprint, source_revision
+system, user = compile_classification_prompt(manifest, ClassificationRequest(
+    trace_uid="tr-1", task_statement="...", outcome_text="failed", window=rendered_tail))
+reply = my_client.complete(system=system, user=user)   # the consumer owns the call and its budget
+result = parse_classification_response(reply.text, manifest)
+```
+
+`ClassificationResult` carries `mode_id`, `family`, optional `submode`, `decisive_step`,
+`evidence_quote`, `rationale`, `confidence` in [0, 1], and an explicit `abstain` with one of
+`ABSTAIN_REASONS` (`infrastructure_fault`, `decisive_step_not_in_window`, `ambiguous`,
+`unknown_mode`, `malformed_response`, `unparsed`). Infrastructure faults and ambiguous cases
+are never forced into an agent-error family. `parse_classification_response` never raises:
+fenced JSON, trailing prose, an unknown mode id, duplicate keys and malformed JSON all map to a
+result or an abstention with the raw object preserved.
+
+The grouping from leaf modes to families keeps every family, including `verification`,
+`multiagent`, `multimodal` and `observation`. `cohens_kappa(labels_a, labels_b)` measures
+two-judge agreement over the items both labelled and returns `None` where kappa is undefined.
+
+Consumers keep provider calls, budgets, metering, trace binding and append-only storage; a
+row should record the manifest `fingerprint`, `source_revision` and the SHA-256 of the
+compiled prompt so the classification is reproducible from the stored reply.

{agentdebugx-0.4.0 → agentdebugx-0.5.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "agentdebugx"
-version = "0.4.0"
+version = "0.5.0"
 description = "Portable error analysis, tracing, and recovery framework for agentic AI systems. Import as `agentdebug`."
 authors = ["ULab @ UIUC <ulab@illinois.edu>"]
 license = "MIT"

{agentdebugx-0.4.0 → agentdebugx-0.5.0}/src/agentdebug/__init__.py

@@ -291,7 +291,7 @@ __all__ = [
     'write_converted_trajectory',
 ]
 
-__version__ = '0.4.0'
+__version__ = '0.5.0'
 
 _COMPAT_MODULE_ALIASES = {
     'agentdebug.models': _core_models_mod,

{agentdebugx-0.4.0 → agentdebugx-0.5.0}/src/agentdebug/diagnose/attribute/__init__.py

@@ -18,7 +18,10 @@ from agentdebug.diagnose.attribute.attribution import (
     CorrectedAction,
     CounterfactualAttributor,
     EnsembleAttributor,
+    AttributionUnavailable,
     HeuristicAttributor,
+    NO_FALLBACK,
+    NoFallback,
     SBFLAttributor,
     StepByStepAttributor,
 )
@@ -42,7 +45,10 @@ __all__ = [
     'DeepDebugResult',
     'DeepDebugRound',
     'EnsembleAttributor',
+    'AttributionUnavailable',
     'HeuristicAttributor',
+    'NO_FALLBACK',
+    'NoFallback',
     'SBFLAttributor',
     'StepByStepAttributor',
     'attribute_async',

{agentdebugx-0.4.0 → agentdebugx-0.5.0}/src/agentdebug/diagnose/attribute/attribution.py

@@ -162,6 +162,47 @@ class Attributor(Protocol):
         ...
 
 
+class AttributionUnavailable(RuntimeError):
+    """The model-based attributor produced nothing and the caller refused a substitute.
+
+    Raised by :data:`NO_FALLBACK` where an attributor would otherwise hand the
+    trajectory to its ``fallback``. The message names the attributor and the
+    reason (an LLM error, or a reply with no JSON block), so a corpus can
+    record the attempt as a dropout instead of a heuristic label that looks
+    like a model one.
+    """
+
+
+class NoFallback:
+    """An ``Attributor`` that refuses, for callers who want fail-closed attribution.
+
+    Every model-based attributor defaults to ``fallback or HeuristicAttributor()``:
+    when the model call fails or returns no JSON, a model-free ranking of the
+    detector findings is returned under the *same* result type, and nothing
+    downstream can tell the two apart. Pass ``fallback=NO_FALLBACK`` to make that
+    path raise :class:`AttributionUnavailable` instead. Measured by a consumer
+    on 887 rows: the silent substitute fired on 16.7% of one model's attempts
+    against 0.5% of another's, all labelled as model attributions.
+    """
+
+    id = 'no_fallback'
+    requires_findings = False
+
+    def attribute(
+        self,
+        trajectory: AgentTrajectory,
+        findings: Optional[List[FailureFinding]] = None,
+    ) -> AttributionResult:
+        raise AttributionUnavailable(
+            f'attribution unavailable for trace {trajectory.trace_id!r}: the model-based '
+            'attributor produced no usable result and fallback is disabled (NO_FALLBACK)'
+        )
+
+
+#: Pass as ``fallback=`` to any attributor to disable the heuristic substitute.
+NO_FALLBACK = NoFallback()
+
+
 class HeuristicAttributor:
     """Cheap, model-free fallback that picks the earliest highest-confidence finding.
 

{agentdebugx-0.4.0 → agentdebugx-0.5.0}/src/agentdebug/diagnose/detect/README.md

@@ -64,6 +64,64 @@ still fails.
 
 Adapted from TrajDebug Stage B (THU-KEG/TrajDebug, MIT).
 
+### Where a quote came from
+
+`quote_verified` is a boolean, which is what a detector needs to keep or drop
+a finding. A consumer that stores findings needs to know *where* a quote was
+found, because position is evidence too: a verbatim quote of the grader's
+verdict at the last event does not support a diagnosis of step 4.
+
+`locate_quote(trajectory, quote, anchor=None, shown=None)` returns a
+`QuoteLocation`:
+
+- `rung` -- `exact` (verbatim substring of the stored field), `normalized`
+  (after the same normalisation `verify_finding_quotes` applies, plus removal
+  of the renderer's `[step 3] event_id=... output=` framing and one pair of
+  wrapping quotation marks), `anchored` (the text was found nowhere but the
+  quote or the caller named an event id that exists -- a pointer, not a
+  transcription), or `unresolvable`
+- `region` -- `event`, `shown` (the text a detector rendered for an event),
+  `goal`, or `none`; with `event_id`, `event_index`, `step_index`,
+  `event_type` and `field` (`input` / `output` / `error`) for an event
+- `span` and `text` -- character offsets into the stored field (as `str`, or
+  `repr` for a non-string value), the `shown` text, or the goal, and the slice
+  they cover. Offsets are into the raw field even when the match was made after
+  unescaping, so `event.output[start:end]` is the cited text
+- `anchor` and `anchor_status` -- `not_declared`, `resolved`, `unknown_event`
+  (an id the trajectory does not contain), or `elsewhere` (a real id whose
+  event does not contain the text: a mislabelling, not a fabrication)
+- `grounded` -- `rung` is `exact` or `normalized` and `region` is `event` or
+  `shown`. A goal quote is faithful and grounds nothing about the trajectory;
+  a real pointer locates an event but cannot be checked by string search.
+
+Rungs are tried strictest first; within a rung the anchor's event is searched
+before the others, then the remaining events in order, then the goal.
+
+`resolve_anchor(trajectory, event_id)` returns the event an id names, by exact
+match, or `None`.
+
+`grounds_trajectory(findings, trajectory, shown=None)` locates the
+`wrong_content_quote` (anchored on the blamed event) and `reference_quote` of
+each finding and places each relative to the blamed event: `position` is
+`before` / `at` / `after`, and `at_or_before_blame` is the question a consumer
+asks -- was this text in front of the agent when it acted? `FindingGrounding.
+grounded` is `None` for a finding with no quotes (as `quote_verified` is),
+and otherwise requires every quote to be located under a grounding rung, none
+to come from after the blame, and the wrong-content quote to sit at it.
+`annotate_evidence_regions(findings, trajectory)` writes that as JSON into
+`finding.metadata['evidence_regions']`, next to `quote_verified`.
+
+```python
+from agentdebug.diagnose.detect.evidence import locate_quote, grounds_trajectory
+
+loc = locate_quote(trajectory, 'you see a cd 3')
+loc.rung, loc.event_id, loc.field, loc.span   # 'exact', 'evt_...', 'output', (17, 31)
+
+for g in grounds_trajectory(report.findings, trajectory):
+    for q in g.quotes:
+        print(g.finding_id, q.role, q.location.rung, q.position, q.at_or_before_blame)
+```
+
 ## Context budget
 
 Both LLM detectors render the trajectory by clipping each event to a fixed