dpaa 0.0.5__py3-none-any.whl

dpaa/__init__.py

@@ -0,0 +1,10 @@
+from dpaa.analyze import analyze as analyze
+from dpaa.core.metric_spec import MetricAnchors as MetricAnchors
+from dpaa.core.metric_spec import MetricId as MetricId
+from dpaa.core.metric_spec import MetricSpec as MetricSpec
+from dpaa.core.metric_spec import ProvidesMetricValues as ProvidesMetricValues
+from dpaa.core.metric_spec import ScorerSpec as ScorerSpec
+from dpaa.core.traces import AgentStep as AgentStep
+from dpaa.core.traces import AgentStepKind as AgentStepKind
+from dpaa.core.traces import AgentTrace as AgentTrace
+from dpaa.core.traces import AgentTraces as AgentTraces

dpaa/analyze.py

@@ -0,0 +1,27 @@
+from __future__ import annotations
+
+from concurrent.futures import ThreadPoolExecutor
+from typing import TYPE_CHECKING, overload
+
+if TYPE_CHECKING:
+    from dpaa.core.traces import AgentTraces
+    from dpaa.core.traces_scored import ScoredAgentTraces
+    from dpaa.scorer import Scorer
+
+
+@overload
+def analyze(runs: AgentTraces, scorer: Scorer) -> ScoredAgentTraces: ...
+
+
+@overload
+def analyze(runs: AgentTraces, scorer: dict[str, Scorer]) -> dict[str, ScoredAgentTraces]: ...
+
+
+def analyze(
+    runs: AgentTraces, scorer: Scorer | dict[str, Scorer]
+) -> ScoredAgentTraces | dict[str, ScoredAgentTraces]:
+    if isinstance(scorer, dict):
+        with ThreadPoolExecutor(max_workers=len(scorer)) as executor:
+            futures = {name: executor.submit(s.score, runs) for name, s in scorer.items()}
+            return {name: future.result() for name, future in futures.items()}
+    return scorer.score(runs)

dpaa/core/__init__.py

@@ -0,0 +1 @@
+

dpaa/core/llm_usage.py

@@ -0,0 +1,196 @@
+from __future__ import annotations
+
+import math
+from collections.abc import Mapping
+from typing import Any, cast
+
+from dpaa.core.metadata import (
+    CACHE_WRITE_TOKENS,
+    CACHED_INPUT_TOKENS,
+    COST_USD,
+    EST_INPUT_TOKENS,
+    EST_OUTPUT_TOKENS,
+    EST_TOTAL_TOKENS,
+    INPUT_TOKENS,
+    OUTPUT_TOKENS,
+    REASONING_TOKENS,
+    RUN_STEPS_N,
+    RUN_USAGE,
+    STEP_USAGE,
+    TOTAL_TOKENS,
+)
+from dpaa.core.traces import AgentStep, AgentStepKind, AgentTrace
+from dpaa.utils.token_counter import TokenCounter
+
+
+def build_step_usage_meta(
+    payload: object, *, attributes: Mapping[str, Any] | None = None
+) -> dict[str, Any]:
+    raw: Mapping[str, Any] = (
+        cast(Mapping[str, Any], payload) if isinstance(payload, Mapping) else {}
+    )
+    usage = {**_reported_usage(attributes or {}), **_normalize_usage(raw)}
+    return {STEP_USAGE: usage} if any(usage.values()) else {}
+
+
+def get_or_compute_step_usage(step: AgentStep | None) -> Mapping[str, float]:
+    if step is None:
+        return {}
+    payload = step.metadata.get(STEP_USAGE)
+    if isinstance(payload, Mapping):
+        return _normalize_usage(cast(Mapping[str, Any], payload))
+    return _reported_usage(step.metadata)
+
+
+def enrich_metadata_with_usage(
+    trace: AgentTrace, counter: TokenCounter | None = None
+) -> AgentTrace:
+    counter = counter or TokenCounter()
+    usage = _compute_steps_usage(trace.steps)
+    if TOTAL_TOKENS not in usage:
+        usage |= estimate_steps_usage(trace.steps, counter)
+    trace.metadata[RUN_USAGE] = usage
+    return trace
+
+
+def get_or_compute_trace_usage(trace: AgentTrace) -> Mapping[str, float]:
+    recorded = trace.metadata.get(RUN_USAGE)
+    if isinstance(recorded, Mapping):
+        return cast(Mapping[str, Any], recorded)
+    return _compute_steps_usage(trace.steps)
+
+
+def step_total_tokens(step: AgentStep | None) -> float | None:
+    return usage_total_tokens(get_or_compute_step_usage(step))
+
+
+_ALIASES: dict[str, tuple[str, ...]] = {
+    INPUT_TOKENS: (INPUT_TOKENS, "prompt_tokens", "promptTokens", "gen_ai.usage.input_tokens"),
+    OUTPUT_TOKENS: (
+        OUTPUT_TOKENS,
+        "completion_tokens",
+        "completionTokens",
+        "gen_ai.usage.output_tokens",
+    ),
+    CACHED_INPUT_TOKENS: (
+        CACHED_INPUT_TOKENS,
+        "cached_tokens",
+        "cache_read_input_tokens",
+        "input_tokens_cache_read",
+        "gen_ai.usage.cache_read_input_tokens",
+    ),
+    CACHE_WRITE_TOKENS: (
+        CACHE_WRITE_TOKENS,
+        "cache_creation_input_tokens",
+        "input_tokens_cache_write",
+        "gen_ai.usage.cache_creation_input_tokens",
+    ),
+    REASONING_TOKENS: (REASONING_TOKENS, "gen_ai.usage.reasoning_tokens"),
+    TOTAL_TOKENS: (TOTAL_TOKENS, "totalTokens", "gen_ai.usage.total_tokens"),
+    COST_USD: (COST_USD, "gen_ai.usage.cost", "gen_ai.usage.cost_usd", "totalCost"),
+}
+_PAYLOAD_ALIASES: dict[str, tuple[str, ...]] = {
+    INPUT_TOKENS: ("input",),
+    OUTPUT_TOKENS: ("output",),
+    TOTAL_TOKENS: ("total",),
+    COST_USD: ("cost",),
+}
+_ANY_VALUE_FIELDS = ("intValue", "doubleValue", "stringValue")
+_RUN_SUMMED = (
+    INPUT_TOKENS,
+    OUTPUT_TOKENS,
+    CACHED_INPUT_TOKENS,
+    CACHE_WRITE_TOKENS,
+    REASONING_TOKENS,
+    COST_USD,
+)
+
+
+def _compute_steps_usage(steps: list[AgentStep]) -> dict[str, float]:
+    totals: dict[str, float] = {RUN_STEPS_N: len(steps)}
+    for step in steps:
+        if step.kind is not AgentStepKind.LLM:
+            continue
+        usage = get_or_compute_step_usage(step)
+        reported = {name: usage[name] for name in _RUN_SUMMED if name in usage}
+        total = usage_total_tokens(usage)
+        if total is not None:
+            reported[TOTAL_TOKENS] = total
+        for name, value in reported.items():
+            totals[name] = totals.get(name, 0.0) + value
+    return totals
+
+
+def estimate_steps_usage(
+    steps: list[AgentStep], counter: TokenCounter | None = None
+) -> dict[str, float]:
+    counter = counter or TokenCounter()
+    inputs = float(sum(counter(step.input) for step in steps))
+    outputs = float(sum(counter(step.output) for step in steps))
+    estimated = {
+        EST_INPUT_TOKENS: inputs,
+        EST_OUTPUT_TOKENS: outputs,
+        EST_TOTAL_TOKENS: inputs + outputs,
+    }
+    return {name: value for name, value in estimated.items() if value}
+
+
+def usage_total_tokens(usage: Mapping[str, float]) -> float | None:
+    total = usage.get(TOTAL_TOKENS)
+    if total is not None:
+        return total
+    parts = [usage[name] for name in (INPUT_TOKENS, OUTPUT_TOKENS) if name in usage]
+    return sum(parts) if parts else None
+
+
+def _normalize_usage(payload: Mapping[str, Any]) -> dict[str, float]:
+    return _read(_flatten(payload), _ALIASES, _PAYLOAD_ALIASES)
+
+
+def _reported_usage(metadata: Mapping[str, Any]) -> dict[str, float]:
+    return _read(metadata, _ALIASES)
+
+
+def _read(raw: Mapping[str, Any], *tables: Mapping[str, tuple[str, ...]]) -> dict[str, float]:
+    usage: dict[str, float] = {}
+    for table in tables:
+        for name, aliases in table.items():
+            if name in usage:
+                continue
+            for alias in aliases:
+                value = _as_number(raw.get(alias))
+                if value is not None:
+                    usage[name] = value
+                    break
+    return usage
+
+
+def _flatten(payload: Mapping[str, Any]) -> dict[str, Any]:
+    top: dict[str, Any] = {}
+    nested: dict[str, Any] = {}
+    for key, value in payload.items():
+        details = cast(Mapping[str, Any], value) if isinstance(value, Mapping) else None
+        if details is not None and _as_number(details) is None:
+            nested.update(details)
+        else:
+            top[key] = value
+    return {**nested, **top}
+
+
+def _as_number(value: object) -> float | None:
+    if value is None or isinstance(value, bool):
+        return None
+    if isinstance(value, Mapping):
+        wrapper = cast(Mapping[str, Any], value)
+        for field in _ANY_VALUE_FIELDS:
+            if field in wrapper:
+                return _as_number(wrapper[field])
+        return None
+    if isinstance(value, (int, float)):
+        number = float(value)
+    else:
+        try:
+            number = float(str(value))
+        except ValueError:
+            return None
+    return number if math.isfinite(number) else None

dpaa/core/metadata.py

@@ -0,0 +1,97 @@
+from __future__ import annotations
+
+from collections.abc import Mapping
+from dataclasses import dataclass
+from typing import Any
+
+import pandas as pd
+
+DATASET = "dataset"
+MODEL = "model"
+TASK_ID = "task_id"
+RUN_NAME = "run_name"
+RESOLVED = "resolved"
+RESOLVED_N = "resolved_n"
+RESOLVED_RATE = "resolved_rate"
+NUM_TOOL_CALLS = "num_tool_calls"
+NUM_STEPS = "num_steps"
+NUM_TOKENS = "num_tokens"
+DURATION_S = "duration_s"
+AVG_TOOL_CALLS = "avg_tool_calls"
+AVG_TOOL_CALLS_RESOLVED = "avg_tool_calls_resolved"
+AVG_TOOL_CALLS_UNRESOLVED = "avg_tool_calls_unresolved"
+AVG_STEPS = "avg_steps"
+AVG_TOKENS = "avg_tokens"
+AVG_TOKENS_RESOLVED = "avg_tokens_resolved"
+AVG_TOKENS_UNRESOLVED = "avg_tokens_unresolved"
+UNIQUE_TOOLS_N = "unique_tools"
+STEP_USAGE = "step_usage"
+RUN_USAGE = "run_usage"
+RUN_STEPS_N = "run_steps_n"
+INPUT_TOKENS = "input_tokens"
+OUTPUT_TOKENS = "output_tokens"
+CACHED_INPUT_TOKENS = "cached_input_tokens"
+CACHE_WRITE_TOKENS = "cache_write_tokens"
+REASONING_TOKENS = "reasoning_tokens"
+TOTAL_TOKENS = "total_tokens"
+COST_USD = "cost_usd"
+EST_INPUT_TOKENS = "est_input_tokens"
+EST_OUTPUT_TOKENS = "est_output_tokens"
+EST_TOTAL_TOKENS = "est_total_tokens"
+
+
+def resolved_str(resolved: bool | None) -> str:
+    if resolved is None:
+        return ""
+    return str(int(resolved))
+
+
+def resolved_series(series: pd.Series) -> pd.Series:
+    lowered = series.astype("string").str.lower().str.strip()
+    result = pd.Series(pd.NA, index=series.index, dtype="boolean")
+    result[lowered.isin(["true", "1", "1.0"])] = True
+    result[lowered.isin(["false", "0", "0.0"])] = False
+    return result
+
+
+@dataclass(frozen=True)
+class TaskKey:
+    dataset: str
+    model: str
+    task_id: str
+
+    @classmethod
+    def of(cls, metadata: Mapping[str, Any], task_id: str = "") -> TaskKey:
+        return cls(
+            dataset=str(metadata.get(DATASET, "")),
+            model=str(metadata.get(MODEL, "")),
+            task_id=str(metadata.get(TASK_ID) or task_id),
+        )
+
+    def run(self, run_name: str) -> RunKey:
+        return RunKey(
+            dataset=self.dataset, model=self.model, task_id=self.task_id, run_name=run_name
+        )
+
+
+@dataclass(frozen=True)
+class RunKey:
+    dataset: str
+    model: str
+    task_id: str
+    run_name: str
+
+    @property
+    def cell(self) -> TaskKey:
+        return TaskKey(self.dataset, self.model, self.task_id)
+
+
+CELL_KEY: tuple[str, str, str] = (DATASET, MODEL, TASK_ID)
+RUN_KEY: tuple[str, str, str, str] = (DATASET, MODEL, TASK_ID, RUN_NAME)
+
+
+def as_metadata(subject: RunKey | TaskKey) -> dict[str, str]:
+    keys = {DATASET: subject.dataset, MODEL: subject.model, TASK_ID: subject.task_id}
+    if isinstance(subject, RunKey):
+        keys[RUN_NAME] = subject.run_name
+    return keys

dpaa/core/metric_spec.py

@@ -0,0 +1,137 @@
+from __future__ import annotations
+
+import math
+from collections.abc import Mapping
+from dataclasses import dataclass, replace
+from enum import StrEnum
+from typing import NewType, Protocol, runtime_checkable
+
+
+class MetricDirection(StrEnum):
+    HIGHER_BETTER = "higher_better"
+    LOWER_BETTER = "lower_better"
+    NEUTRAL = "neutral"
+
+
+class MetricGrain(StrEnum):
+    TRACE = "trace"
+    GROUP = "group"
+
+
+class MetricPriority(StrEnum):
+    PRIMARY = "primary"
+    SECONDARY = "secondary"
+    TERTIARY = "tertiary"
+    DIAGNOSTIC = "diagnostic"
+    SPLIT = "split"
+
+    @property
+    def is_score(self) -> bool:
+        return self not in (MetricPriority.DIAGNOSTIC, MetricPriority.SPLIT)
+
+
+@dataclass(frozen=True)
+class MetricAnchors:
+    upper: float | None
+    lower: float | None
+    lower_label: str = "weak"
+    mid_label: str = "mixed"
+    upper_label: str = "strong"
+    inner_label: str | None = None
+
+    def __str__(self):
+        return self.anchor_line()
+
+    def anchor_line(self) -> str:
+        if self.inner_label and self.lower is not None and (self.upper is not None):
+            return f"`{self.lower:g}-{self.upper:g}` {self.inner_label}"
+        if self.lower is not None:
+            if self.upper is not None:
+                return f"`>= {self.upper}` {self.upper_label}, `{self.lower:g}-<{self.upper:g}` {self.mid_label}, `< {self.lower:g}` {self.lower_label}"
+            else:
+                return f"`< {self.lower:g}` {self.lower_label}"
+        elif self.upper is not None:
+            return f"`>= {self.upper:g}` {self.upper_label}"
+        return ""
+
+
+Family = NewType("Family", str)
+Variant = NewType("Variant", str)
+MetricIdStr = NewType("MetricIdStr", str)
+EMPTY_VARIANT = Variant("")
+EMPTY_FAMILY = Family("")
+
+
+@dataclass(frozen=True)
+class MetricId:
+    family: Family
+    variant: Variant
+    key: str
+
+    @property
+    def id_str(self) -> MetricIdStr:
+        if not self.family and (not self.variant):
+            return MetricIdStr(self.key)
+        return MetricIdStr("/".join(dict.fromkeys((self.family, self.variant, self.key))))
+
+
+@dataclass(frozen=True)
+class MetricSpec:
+    key: str
+    label: str
+    description: str
+    priority: MetricPriority
+    value_range: tuple[float, float] | None
+    direction: MetricDirection = MetricDirection.HIGHER_BETTER
+    grain: MetricGrain = MetricGrain.TRACE
+    anchors: MetricAnchors | None = None
+    family: Family = EMPTY_FAMILY
+    variant: Variant = EMPTY_VARIANT
+
+    @property
+    def id(self) -> MetricId:
+        return MetricId(family=self.family, variant=self.variant, key=self.key)
+
+    @property
+    def metric_id(self) -> MetricIdStr:
+        return self.id.id_str
+
+    def bind(self, family: str, *, variant: str) -> MetricSpec:
+        if self.family and self.family != family:
+            raise ValueError(f"metric {self.key} family {self.family} != {family} conflict")
+        if self.variant and self.variant != variant:
+            raise ValueError(f"metric {self.key} variant {self.variant} != {variant} conflict")
+        return replace(self, family=Family(family), variant=variant)
+
+    def value_range_finite(self) -> float | None:
+        v = self.value_range
+        if v and math.isfinite(v[0]) and math.isfinite(v[1]):
+            return math.fabs(v[1] - v[0])
+        return None
+
+    @property
+    def is_score(self) -> bool:
+        return self.priority.is_score
+
+
+@dataclass(frozen=True, kw_only=True)
+class ScorerSpec:
+    family: Family
+    metrics: dict[str, MetricSpec]
+    description: str = ""
+    variant: Variant = EMPTY_VARIANT
+
+    def __post_init__(self):
+        variant = self.variant or self.family
+        bound = {k: m.bind(self.family, variant=variant) for k, m in self.metrics.items()}
+        object.__setattr__(self, "variant", variant)
+        object.__setattr__(self, "metrics", bound)
+
+    @property
+    def id(self) -> str:
+        return "/".join(dict.fromkeys((self.family, self.variant)))
+
+
+@runtime_checkable
+class ProvidesMetricValues(Protocol):
+    def metric_values(self, variant: str) -> Mapping[str, float | None]: ...

dpaa/core/tool_representation/__init__.py

@@ -0,0 +1,88 @@
+from __future__ import annotations
+
+from dpaa.core.tool_representation.canonicalizer import (
+    ActionCanonicalizer,
+    Canonicalizer,
+    ContentHashCanonicalizer,
+    ShellTargetCanonicalizer,
+)
+from dpaa.core.tool_representation.context_tracker import (
+    ContextTracker,
+    EditSetContextTracker,
+    FileLifecycle,
+    LifecycleContextTracker,
+    StepCategory,
+    TrajectoryContext,
+    record_context,
+)
+from dpaa.core.tool_representation.equivalence import (
+    JACCARD_THRESHOLD,
+    LINE_OVERLAP_MIN,
+    AdaptiveJaccardThreshold,
+    EquivalenceEngine,
+    EquivalenceMargin,
+    EquivalenceResult,
+    equivalence_margin,
+)
+from dpaa.core.tool_representation.extractor import ToolCallFeatureExtractor
+from dpaa.core.tool_representation.features import StepFeatures
+from dpaa.core.tool_representation.functional_groups import (
+    DEFAULT_TERMINAL_FUNCTIONAL_GROUPS,
+    canonical_verb,
+)
+from dpaa.core.tool_representation.intent_labeler import (
+    DEFAULT_SEARCH_GROUPS,
+    IntentLabeler,
+    SplitReadSearchIntentLabeler,
+    StateDrivenIntentLabeler,
+    ToolIdentityIntentLabeler,
+    label_intent,
+)
+from dpaa.core.tool_representation.representation import ToolCallRepresenter, compose_representation
+from dpaa.core.tool_representation.state_assembler import assemble_state
+from dpaa.core.tool_representation.terminal_command_classifier import classify_terminal_command
+from dpaa.core.tool_representation.tools_config import (
+    CommandExtractor,
+    TargetFilesExtractor,
+    TerminalCommandClassifier,
+    ToolsConfig,
+)
+
+__all__ = [
+    "DEFAULT_SEARCH_GROUPS",
+    "DEFAULT_TERMINAL_FUNCTIONAL_GROUPS",
+    "JACCARD_THRESHOLD",
+    "LINE_OVERLAP_MIN",
+    "AdaptiveJaccardThreshold",
+    "ActionCanonicalizer",
+    "Canonicalizer",
+    "EquivalenceEngine",
+    "CommandExtractor",
+    "ToolCallFeatureExtractor",
+    "ContentHashCanonicalizer",
+    "ContextTracker",
+    "EditSetContextTracker",
+    "EquivalenceResult",
+    "FileLifecycle",
+    "IntentLabeler",
+    "LifecycleContextTracker",
+    "EquivalenceMargin",
+    "ShellTargetCanonicalizer",
+    "SplitReadSearchIntentLabeler",
+    "StateDrivenIntentLabeler",
+    "StepCategory",
+    "StepFeatures",
+    "TargetFilesExtractor",
+    "TerminalCommandClassifier",
+    "ToolCallRepresenter",
+    "ToolIdentityIntentLabeler",
+    "ToolsConfig",
+    "TrajectoryContext",
+    "assemble_state",
+    "canonical_verb",
+    "classify_terminal_command",
+    "compose_representation",
+    "label_intent",
+    "equivalence_margin",
+    "record_context",
+]

dpaa/core/tool_representation/canonicalizer.py

@@ -0,0 +1,96 @@
+from __future__ import annotations
+
+import os
+import re
+from dataclasses import dataclass, replace
+from typing import Protocol
+
+from dpaa.core.tool_representation.features import StepFeatures
+from dpaa.core.tool_representation.functional_groups import canonical_verb
+from dpaa.core.tool_representation.terminal_command_classifier import (
+    _first_exe,
+    _strip_command_prefix,
+)
+from dpaa.core.tool_representation.tools_config import ToolsConfig
+
+_FILE_OPERAND_RE = re.compile(
+    "\\.(?:py|kt|java|ts|tsx|js|jsx|rb|go|rs|cpp|cc|c|h|hpp|cs|sh|md|rst|txt|json|ya?ml|toml|cfg|ini|xml|html|css|sql)$",
+    re.IGNORECASE,
+)
+
+
+def _normalize_operand(token: str) -> str:
+    token = token.rstrip("/")
+    if not token:
+        return ""
+    return os.path.abspath(os.path.normpath(token))
+
+
+def _shell_target_files(command: str) -> list[str] | None:
+    tokens = _strip_command_prefix(command).split()
+    targets: list[str] = []
+    seen: set[str] = set()
+    for tok in tokens[1:]:
+        if tok.startswith("-") or not _FILE_OPERAND_RE.search(tok):
+            continue
+        name = _normalize_operand(tok)
+        if name and name not in seen:
+            seen.add(name)
+            targets.append(name)
+    return targets or None
+
+
+def _canonical_action(
+    tool_name: str, command: str, target_files: list[str] | None, tool_aliases: dict[str, str]
+) -> str:
+    if command:
+        verb = canonical_verb(_first_exe(command))
+        operands = _shell_target_files(command) or target_files or []
+    else:
+        verb = tool_aliases.get(tool_name, tool_name)
+        operands = target_files or []
+    normalized = sorted({_normalize_operand(o) for o in operands})
+    return verb + "|" + " ".join(normalized)
+
+
+class Canonicalizer(Protocol):
+    def canonicalize(self, features: StepFeatures) -> StepFeatures: ...
+
+
+@dataclass(frozen=True)
+class ContentHashCanonicalizer(Canonicalizer):
+    tool_registry: ToolsConfig
+
+    def canonicalize(self, features: StepFeatures) -> StepFeatures:
+        stage_token = str(self.tool_registry.tool_to_stage.get(features.tool_name, ""))
+        return features.to_canonical(stage_token + features.raw_input + features.raw_output)
+
+
+@dataclass(frozen=True)
+class ActionCanonicalizer(Canonicalizer):
+    tool_aliases: dict[str, str]
+
+    def canonicalize(self, features: StepFeatures) -> StepFeatures:
+        return features.to_canonical(
+            _canonical_action(
+                features.tool_name, features.command, features.target_files, self.tool_aliases
+            )
+        )
+
+
+@dataclass(frozen=True)
+class ShellTargetCanonicalizer(Canonicalizer):
+    base: Canonicalizer
+    execute_tools: frozenset[str]
+
+    def canonicalize(self, features: StepFeatures) -> StepFeatures:
+        canonical = self.base.canonicalize(features)
+        if (
+            features.tool_name in self.execute_tools
+            and (not canonical.target_files)
+            and features.command
+        ):
+            targets = _shell_target_files(features.command)
+            if targets is not None:
+                return replace(canonical, target_files=targets)
+        return canonical