agentloop-profiler 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- agentloop/__init__.py +136 -0
- agentloop/ablation_protocol.py +229 -0
- agentloop/ablations.py +564 -0
- agentloop/aggregate_cli.py +88 -0
- agentloop/aggregate_stats.py +175 -0
- agentloop/aggregates.py +532 -0
- agentloop/audit.py +39 -0
- agentloop/autoinstrument.py +145 -0
- agentloop/batch_experiments.py +295 -0
- agentloop/batching.py +95 -0
- agentloop/budget_types.py +335 -0
- agentloop/budgets.py +539 -0
- agentloop/calibration.py +872 -0
- agentloop/checkpoint_state.py +306 -0
- agentloop/checkpoint_store.py +86 -0
- agentloop/ci.py +202 -0
- agentloop/cli.py +1314 -0
- agentloop/client.py +228 -0
- agentloop/completion.py +345 -0
- agentloop/completion_types.py +199 -0
- agentloop/config.py +50 -0
- agentloop/context_controls.py +443 -0
- agentloop/context_experiments.py +337 -0
- agentloop/context_types.py +216 -0
- agentloop/costs.py +667 -0
- agentloop/cursor_validation.py +48 -0
- agentloop/decorators.py +445 -0
- agentloop/demo.py +300 -0
- agentloop/doctor.py +341 -0
- agentloop/drift.py +361 -0
- agentloop/drift_cli.py +147 -0
- agentloop/drift_types.py +110 -0
- agentloop/drift_windows.py +340 -0
- agentloop/entrypoint.py +201 -0
- agentloop/estimates.py +133 -0
- agentloop/events.py +75 -0
- agentloop/experiment_artifacts.py +80 -0
- agentloop/experiment_observations.py +169 -0
- agentloop/experiment_reports.py +545 -0
- agentloop/experiment_types.py +355 -0
- agentloop/experiments.py +392 -0
- agentloop/exporters.py +117 -0
- agentloop/finding_benchmark_types.py +131 -0
- agentloop/finding_benchmarks.py +424 -0
- agentloop/findings.py +419 -0
- agentloop/graph.py +464 -0
- agentloop/harness.py +825 -0
- agentloop/harness_bench/__init__.py +1 -0
- agentloop/harness_bench/cli.py +77 -0
- agentloop/harness_bench/drivers.py +96 -0
- agentloop/harness_bench/offline.py +507 -0
- agentloop/harness_evidence.py +585 -0
- agentloop/harness_usage.py +91 -0
- agentloop/html_report.py +860 -0
- agentloop/integrations/__init__.py +1 -0
- agentloop/integrations/crewai.py +144 -0
- agentloop/integrations/harbor/__init__.py +1 -0
- agentloop/integrations/harbor/atif.py +982 -0
- agentloop/integrations/harbor/atif_validation.py +329 -0
- agentloop/integrations/harbor/cli.py +207 -0
- agentloop/integrations/harbor/manifest.py +86 -0
- agentloop/integrations/harbor/trial_evidence.py +196 -0
- agentloop/integrations/harbor/trials.py +1109 -0
- agentloop/integrations/harbor/verifier.py +118 -0
- agentloop/integrations/langgraph.py +150 -0
- agentloop/integrations/langgraph_harness.py +466 -0
- agentloop/integrations/langgraph_recovery.py +366 -0
- agentloop/integrations/omnigent/__init__.py +1 -0
- agentloop/integrations/omnigent/cli.py +78 -0
- agentloop/integrations/omnigent/evidence.py +93 -0
- agentloop/integrations/omnigent/telemetry.py +422 -0
- agentloop/integrations/openai.py +415 -0
- agentloop/integrations/openai_agents.py +291 -0
- agentloop/integrations/sdk.py +82 -0
- agentloop/integrations/vercel_ai.py +83 -0
- agentloop/interoperability/__init__.py +5 -0
- agentloop/interoperability/artifacts.py +112 -0
- agentloop/interoperability/cli.py +59 -0
- agentloop/interoperability/cohort_evidence.py +175 -0
- agentloop/interoperability/cohorts.py +681 -0
- agentloop/interoperability/contracts.py +427 -0
- agentloop/interoperability/coordination.py +554 -0
- agentloop/interoperability/evidence.py +215 -0
- agentloop/interoperability/otlp_jsonl.py +775 -0
- agentloop/interoperability/otlp_validation.py +496 -0
- agentloop/interoperability/registry.py +561 -0
- agentloop/interoperability/validation.py +333 -0
- agentloop/intervention_service.py +110 -0
- agentloop/intervention_store.py +118 -0
- agentloop/interventions.py +190 -0
- agentloop/issues.py +90 -0
- agentloop/judgment_adapters.py +125 -0
- agentloop/judgment_benchmark_types.py +98 -0
- agentloop/judgment_benchmarks.py +606 -0
- agentloop/judgment_types.py +241 -0
- agentloop/judgment_views.py +38 -0
- agentloop/judgments.py +342 -0
- agentloop/loop_guards.py +274 -0
- agentloop/loop_types.py +66 -0
- agentloop/markdown.py +102 -0
- agentloop/metrics.py +342 -0
- agentloop/migrations.py +303 -0
- agentloop/model_routing.py +470 -0
- agentloop/onboarding.py +315 -0
- agentloop/operations.py +44 -0
- agentloop/optimizer.py +199 -0
- agentloop/otel.py +715 -0
- agentloop/otel_ids.py +81 -0
- agentloop/otel_semantics.py +134 -0
- agentloop/parallelism.py +141 -0
- agentloop/patches.py +603 -0
- agentloop/plan_export.py +126 -0
- agentloop/policy_promotion.py +325 -0
- agentloop/promotion_evidence.py +304 -0
- agentloop/promotion_types.py +209 -0
- agentloop/quality.py +446 -0
- agentloop/ranking.py +529 -0
- agentloop/recovery_evidence.py +96 -0
- agentloop/replay.py +753 -0
- agentloop/retention.py +375 -0
- agentloop/retention_types.py +113 -0
- agentloop/reuse_experiments.py +247 -0
- agentloop/routing_checks.py +116 -0
- agentloop/routing_types.py +241 -0
- agentloop/rules.py +396 -0
- agentloop/runtime.py +286 -0
- agentloop/savings.py +158 -0
- agentloop/scheduling_plan.py +99 -0
- agentloop/scheduling_types.py +169 -0
- agentloop/schema.py +257 -0
- agentloop/semantic_rules.py +200 -0
- agentloop/semantic_waste.py +345 -0
- agentloop/semantic_waste_types.py +88 -0
- agentloop/server.py +494 -0
- agentloop/store.py +1429 -0
- agentloop/structural_experiments.py +431 -0
- agentloop/structural_types.py +113 -0
- agentloop/structured_quality.py +1020 -0
- agentloop/studies.py +629 -0
- agentloop/study_statistics.py +58 -0
- agentloop/substitution_evidence.py +199 -0
- agentloop/substitution_export.py +190 -0
- agentloop/substitution_reports.py +407 -0
- agentloop/substitution_types.py +284 -0
- agentloop/substitutions.py +267 -0
- agentloop/timing.py +142 -0
- agentloop/tokens.py +188 -0
- agentloop/tool_scheduling.py +410 -0
- agentloop/trace_sources.py +31 -0
- agentloop/tracer.py +494 -0
- agentloop/value.py +306 -0
- agentloop/version.py +3 -0
- agentloop/workflow_types.py +264 -0
- agentloop/workflows.py +284 -0
- agentloop_profiler-0.1.0.dist-info/METADATA +513 -0
- agentloop_profiler-0.1.0.dist-info/RECORD +166 -0
- agentloop_profiler-0.1.0.dist-info/WHEEL +5 -0
- agentloop_profiler-0.1.0.dist-info/entry_points.txt +2 -0
- agentloop_profiler-0.1.0.dist-info/licenses/LICENSE +201 -0
- agentloop_profiler-0.1.0.dist-info/licenses/THIRD_PARTY_LICENSES.md +101 -0
- agentloop_profiler-0.1.0.dist-info/top_level.txt +2 -0
- dashboard/__init__.py +5 -0
- dashboard/app.py +922 -0
- dashboard/streamlit_compat.py +63 -0
- dashboard/trace_upload.py +57 -0
- dashboard/value_view.py +105 -0
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from functools import wraps
|
|
4
|
+
from importlib.metadata import PackageNotFoundError, version
|
|
5
|
+
from typing import Any
|
|
6
|
+
|
|
7
|
+
_STRETCH_WIDTH_MIN_VERSION = (1, 49)
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
def _streamlit_version() -> tuple[int, int]:
|
|
11
|
+
try:
|
|
12
|
+
raw = version("streamlit")
|
|
13
|
+
except PackageNotFoundError:
|
|
14
|
+
return (0, 0)
|
|
15
|
+
parts = raw.split(".")
|
|
16
|
+
try:
|
|
17
|
+
return (int(parts[0]), int(parts[1]))
|
|
18
|
+
except (IndexError, ValueError):
|
|
19
|
+
return (0, 0)
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def _normalize_dataframe_kwargs(kwargs: dict[str, Any]) -> dict[str, Any]:
|
|
23
|
+
"""Translate the modern stretch width to the older supported API."""
|
|
24
|
+
normalized = dict(kwargs)
|
|
25
|
+
if normalized.get("width") == "stretch":
|
|
26
|
+
normalized["width"] = None
|
|
27
|
+
normalized["use_container_width"] = True
|
|
28
|
+
return normalized
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def install_dataframe_width_compat() -> None:
|
|
32
|
+
"""Keep the dashboard compatible with the declared Streamlit 1.34+ floor.
|
|
33
|
+
|
|
34
|
+
Streamlit 1.49 added string dataframe widths such as ``"stretch"``. Older
|
|
35
|
+
supported releases use ``use_container_width=True`` instead. The dashboard
|
|
36
|
+
package installs this adapter only for Streamlit versions below 1.49 so the
|
|
37
|
+
modern API remains untouched on current releases.
|
|
38
|
+
"""
|
|
39
|
+
if _streamlit_version() >= _STRETCH_WIDTH_MIN_VERSION:
|
|
40
|
+
return
|
|
41
|
+
|
|
42
|
+
import streamlit as st
|
|
43
|
+
from streamlit.delta_generator import DeltaGenerator
|
|
44
|
+
|
|
45
|
+
original_method = DeltaGenerator.dataframe
|
|
46
|
+
if getattr(original_method, "__agentloop_width_compat__", False):
|
|
47
|
+
return
|
|
48
|
+
|
|
49
|
+
@wraps(original_method)
|
|
50
|
+
def dataframe_method(self, *args, **kwargs):
|
|
51
|
+
return original_method(self, *args, **_normalize_dataframe_kwargs(kwargs))
|
|
52
|
+
|
|
53
|
+
dataframe_method.__agentloop_width_compat__ = True
|
|
54
|
+
DeltaGenerator.dataframe = dataframe_method
|
|
55
|
+
|
|
56
|
+
original_module_function = st.dataframe
|
|
57
|
+
|
|
58
|
+
@wraps(original_module_function)
|
|
59
|
+
def dataframe(*args, **kwargs):
|
|
60
|
+
return original_module_function(*args, **_normalize_dataframe_kwargs(kwargs))
|
|
61
|
+
|
|
62
|
+
dataframe.__agentloop_width_compat__ = True
|
|
63
|
+
st.dataframe = dataframe
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
"""Decode an uploaded trace file into an :class:`AgentTrace`, or say what to fix.
|
|
2
|
+
|
|
3
|
+
The Ingest page reads an uploaded file on every Streamlit rerun, so the three ways a
|
|
4
|
+
file can be unusable -- not UTF-8, not JSON, not a trace -- arrive as three exceptions
|
|
5
|
+
from three layers, each with its own idea of what a useful message looks like. Doing
|
|
6
|
+
that translation inline in ``app.py`` put the only copy of it inside a page that can
|
|
7
|
+
only be exercised through ``AppTest``. It lives here so it can be unit tested directly.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
import json
|
|
13
|
+
from typing import Any
|
|
14
|
+
|
|
15
|
+
from agentloop.schema import TraceValidationError
|
|
16
|
+
from agentloop.tracer import AgentTrace
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
class TraceUploadError(ValueError):
|
|
20
|
+
"""An uploaded trace the operator can fix, carrying the reason it was rejected.
|
|
21
|
+
|
|
22
|
+
``str(exc)`` is the operator-facing clause and nothing else -- no prefix, no
|
|
23
|
+
trailing period -- so a caller can place it in whatever sentence its surface
|
|
24
|
+
needs. It subclasses :class:`ValueError`, matching
|
|
25
|
+
:class:`~agentloop.schema.TraceValidationError`.
|
|
26
|
+
"""
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def parse_uploaded_trace(payload: bytes) -> AgentTrace:
|
|
30
|
+
"""Return the trace in ``payload``, or raise :class:`TraceUploadError` saying why not.
|
|
31
|
+
|
|
32
|
+
A parse position is reported only for malformed JSON, which is the only failure
|
|
33
|
+
that has one: a decode error is located by byte offset, and a schema violation is
|
|
34
|
+
located by field path.
|
|
35
|
+
"""
|
|
36
|
+
|
|
37
|
+
try:
|
|
38
|
+
text = payload.decode("utf-8")
|
|
39
|
+
except UnicodeDecodeError as exc:
|
|
40
|
+
raise TraceUploadError(
|
|
41
|
+
f"the file is not UTF-8 text ({exc.reason} at byte {exc.start})"
|
|
42
|
+
) from exc
|
|
43
|
+
|
|
44
|
+
try:
|
|
45
|
+
document: Any = json.loads(text)
|
|
46
|
+
except json.JSONDecodeError as exc:
|
|
47
|
+
raise TraceUploadError(f"{exc.msg} at line {exc.lineno}, column {exc.colno}") from exc
|
|
48
|
+
|
|
49
|
+
if not isinstance(document, dict):
|
|
50
|
+
# json.loads happily returns a list, string or number, and from_dict would
|
|
51
|
+
# then fail on subscripting rather than on the schema.
|
|
52
|
+
raise TraceUploadError(f"a trace must be a JSON object, not {type(document).__name__}")
|
|
53
|
+
|
|
54
|
+
try:
|
|
55
|
+
return AgentTrace.from_dict(document)
|
|
56
|
+
except TraceValidationError as exc:
|
|
57
|
+
raise TraceUploadError(str(exc)) from exc
|
dashboard/value_view.py
ADDED
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import json
|
|
4
|
+
from typing import Any
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
def money(value: float | None, decimals: int = 2) -> str:
|
|
8
|
+
return "unavailable" if value is None else f"${float(value):,.{decimals}f}"
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def seconds(ms: float) -> str:
|
|
12
|
+
return f"{float(ms) / 1000:,.2f}s"
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def percentage(value: float | None, decimals: int = 1) -> str:
|
|
16
|
+
return "unavailable" if value is None else f"{float(value):.{decimals}f}%"
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def _streamlit() -> Any:
|
|
20
|
+
try:
|
|
21
|
+
import streamlit as st
|
|
22
|
+
except ModuleNotFoundError as exc:
|
|
23
|
+
raise ModuleNotFoundError(
|
|
24
|
+
"Install dashboard support with: uv sync --locked --extra dashboard"
|
|
25
|
+
) from exc
|
|
26
|
+
return st
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def assumption_inputs(prefix: str = "") -> tuple[int, float, float]:
|
|
30
|
+
st = _streamlit()
|
|
31
|
+
c1, c2, c3 = st.columns(3)
|
|
32
|
+
runs_per_month = c1.number_input(
|
|
33
|
+
"Runs per month",
|
|
34
|
+
min_value=0,
|
|
35
|
+
value=5000,
|
|
36
|
+
step=500,
|
|
37
|
+
key=f"{prefix}runs_per_month",
|
|
38
|
+
)
|
|
39
|
+
engineer_hourly_rate_usd = c2.number_input(
|
|
40
|
+
"Engineer hourly rate",
|
|
41
|
+
min_value=0.0,
|
|
42
|
+
value=150.0,
|
|
43
|
+
step=25.0,
|
|
44
|
+
key=f"{prefix}engineer_hourly_rate_usd",
|
|
45
|
+
)
|
|
46
|
+
incident_cost_usd = c3.number_input(
|
|
47
|
+
"Reliability event cost",
|
|
48
|
+
min_value=0.0,
|
|
49
|
+
value=500.0,
|
|
50
|
+
step=100.0,
|
|
51
|
+
key=f"{prefix}incident_cost_usd",
|
|
52
|
+
)
|
|
53
|
+
return int(runs_per_month), float(engineer_hourly_rate_usd), float(incident_cost_usd)
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def render_value_report(value: dict, *, show_download: bool = True) -> None:
|
|
57
|
+
st = _streamlit()
|
|
58
|
+
monthly = value["monthly_value"]
|
|
59
|
+
per_run = value["per_run"]
|
|
60
|
+
pricing = value.get("pricing", {})
|
|
61
|
+
reliability = value["reliability"]
|
|
62
|
+
|
|
63
|
+
st.subheader("Operational value")
|
|
64
|
+
c1, c2, c3, c4 = st.columns(4)
|
|
65
|
+
c1.metric("Monthly value", money(monthly["total_value_usd"]))
|
|
66
|
+
c2.metric("Cost savings / month", money(monthly["direct_model_cost_savings_usd"]))
|
|
67
|
+
c3.metric("Engineering hours saved", monthly["engineering_hours_saved"])
|
|
68
|
+
c4.metric("Latency hours saved", monthly["latency_hours_saved"])
|
|
69
|
+
|
|
70
|
+
c5, c6, c7, c8 = st.columns(4)
|
|
71
|
+
c5.metric("Latency saved / run", seconds(per_run["latency_savings_ms"]))
|
|
72
|
+
c6.metric("Cost saved / run", money(per_run["cost_savings_usd"], decimals=6))
|
|
73
|
+
c7.metric("Reliability risk", f"{reliability['risk_score']}/100")
|
|
74
|
+
c8.metric("High confidence fixes", reliability["high_confidence_fixes"])
|
|
75
|
+
|
|
76
|
+
st.subheader("Pricing scenario")
|
|
77
|
+
p1, p2, p3, p4 = st.columns(4)
|
|
78
|
+
p1.metric("Modeled tier", pricing.get("suggested_plan") or "unavailable")
|
|
79
|
+
p2.metric("Modeled price", money(pricing.get("suggested_monthly_price_usd", 0), decimals=0))
|
|
80
|
+
ratio = pricing.get("value_to_price_ratio")
|
|
81
|
+
p3.metric("Value / price", "n/a" if ratio is None else f"{ratio}x")
|
|
82
|
+
p4.metric("Monthly value", money(pricing.get("estimated_monthly_value_usd", 0)))
|
|
83
|
+
|
|
84
|
+
st.info(pricing.get("rationale", "No pricing rationale available."))
|
|
85
|
+
|
|
86
|
+
notes = pricing.get("scenario_notes", [])
|
|
87
|
+
if notes:
|
|
88
|
+
st.markdown("#### Scenario includes")
|
|
89
|
+
st.write("\n".join(f"- {note}" for note in notes))
|
|
90
|
+
|
|
91
|
+
st.subheader("Value summary")
|
|
92
|
+
st.write(value["value_summary"])
|
|
93
|
+
|
|
94
|
+
risks = reliability.get("top_risks", [])
|
|
95
|
+
if risks:
|
|
96
|
+
st.subheader("Top reliability risks")
|
|
97
|
+
st.write("\n".join(f"- {risk}" for risk in risks))
|
|
98
|
+
|
|
99
|
+
if show_download:
|
|
100
|
+
st.download_button(
|
|
101
|
+
"Download value report JSON",
|
|
102
|
+
data=json.dumps(value, indent=2),
|
|
103
|
+
file_name="agentloop_value_report.json",
|
|
104
|
+
mime="application/json",
|
|
105
|
+
)
|