agentloop-profiler 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (166) hide show
  1. agentloop/__init__.py +136 -0
  2. agentloop/ablation_protocol.py +229 -0
  3. agentloop/ablations.py +564 -0
  4. agentloop/aggregate_cli.py +88 -0
  5. agentloop/aggregate_stats.py +175 -0
  6. agentloop/aggregates.py +532 -0
  7. agentloop/audit.py +39 -0
  8. agentloop/autoinstrument.py +145 -0
  9. agentloop/batch_experiments.py +295 -0
  10. agentloop/batching.py +95 -0
  11. agentloop/budget_types.py +335 -0
  12. agentloop/budgets.py +539 -0
  13. agentloop/calibration.py +872 -0
  14. agentloop/checkpoint_state.py +306 -0
  15. agentloop/checkpoint_store.py +86 -0
  16. agentloop/ci.py +202 -0
  17. agentloop/cli.py +1314 -0
  18. agentloop/client.py +228 -0
  19. agentloop/completion.py +345 -0
  20. agentloop/completion_types.py +199 -0
  21. agentloop/config.py +50 -0
  22. agentloop/context_controls.py +443 -0
  23. agentloop/context_experiments.py +337 -0
  24. agentloop/context_types.py +216 -0
  25. agentloop/costs.py +667 -0
  26. agentloop/cursor_validation.py +48 -0
  27. agentloop/decorators.py +445 -0
  28. agentloop/demo.py +300 -0
  29. agentloop/doctor.py +341 -0
  30. agentloop/drift.py +361 -0
  31. agentloop/drift_cli.py +147 -0
  32. agentloop/drift_types.py +110 -0
  33. agentloop/drift_windows.py +340 -0
  34. agentloop/entrypoint.py +201 -0
  35. agentloop/estimates.py +133 -0
  36. agentloop/events.py +75 -0
  37. agentloop/experiment_artifacts.py +80 -0
  38. agentloop/experiment_observations.py +169 -0
  39. agentloop/experiment_reports.py +545 -0
  40. agentloop/experiment_types.py +355 -0
  41. agentloop/experiments.py +392 -0
  42. agentloop/exporters.py +117 -0
  43. agentloop/finding_benchmark_types.py +131 -0
  44. agentloop/finding_benchmarks.py +424 -0
  45. agentloop/findings.py +419 -0
  46. agentloop/graph.py +464 -0
  47. agentloop/harness.py +825 -0
  48. agentloop/harness_bench/__init__.py +1 -0
  49. agentloop/harness_bench/cli.py +77 -0
  50. agentloop/harness_bench/drivers.py +96 -0
  51. agentloop/harness_bench/offline.py +507 -0
  52. agentloop/harness_evidence.py +585 -0
  53. agentloop/harness_usage.py +91 -0
  54. agentloop/html_report.py +860 -0
  55. agentloop/integrations/__init__.py +1 -0
  56. agentloop/integrations/crewai.py +144 -0
  57. agentloop/integrations/harbor/__init__.py +1 -0
  58. agentloop/integrations/harbor/atif.py +982 -0
  59. agentloop/integrations/harbor/atif_validation.py +329 -0
  60. agentloop/integrations/harbor/cli.py +207 -0
  61. agentloop/integrations/harbor/manifest.py +86 -0
  62. agentloop/integrations/harbor/trial_evidence.py +196 -0
  63. agentloop/integrations/harbor/trials.py +1109 -0
  64. agentloop/integrations/harbor/verifier.py +118 -0
  65. agentloop/integrations/langgraph.py +150 -0
  66. agentloop/integrations/langgraph_harness.py +466 -0
  67. agentloop/integrations/langgraph_recovery.py +366 -0
  68. agentloop/integrations/omnigent/__init__.py +1 -0
  69. agentloop/integrations/omnigent/cli.py +78 -0
  70. agentloop/integrations/omnigent/evidence.py +93 -0
  71. agentloop/integrations/omnigent/telemetry.py +422 -0
  72. agentloop/integrations/openai.py +415 -0
  73. agentloop/integrations/openai_agents.py +291 -0
  74. agentloop/integrations/sdk.py +82 -0
  75. agentloop/integrations/vercel_ai.py +83 -0
  76. agentloop/interoperability/__init__.py +5 -0
  77. agentloop/interoperability/artifacts.py +112 -0
  78. agentloop/interoperability/cli.py +59 -0
  79. agentloop/interoperability/cohort_evidence.py +175 -0
  80. agentloop/interoperability/cohorts.py +681 -0
  81. agentloop/interoperability/contracts.py +427 -0
  82. agentloop/interoperability/coordination.py +554 -0
  83. agentloop/interoperability/evidence.py +215 -0
  84. agentloop/interoperability/otlp_jsonl.py +775 -0
  85. agentloop/interoperability/otlp_validation.py +496 -0
  86. agentloop/interoperability/registry.py +561 -0
  87. agentloop/interoperability/validation.py +333 -0
  88. agentloop/intervention_service.py +110 -0
  89. agentloop/intervention_store.py +118 -0
  90. agentloop/interventions.py +190 -0
  91. agentloop/issues.py +90 -0
  92. agentloop/judgment_adapters.py +125 -0
  93. agentloop/judgment_benchmark_types.py +98 -0
  94. agentloop/judgment_benchmarks.py +606 -0
  95. agentloop/judgment_types.py +241 -0
  96. agentloop/judgment_views.py +38 -0
  97. agentloop/judgments.py +342 -0
  98. agentloop/loop_guards.py +274 -0
  99. agentloop/loop_types.py +66 -0
  100. agentloop/markdown.py +102 -0
  101. agentloop/metrics.py +342 -0
  102. agentloop/migrations.py +303 -0
  103. agentloop/model_routing.py +470 -0
  104. agentloop/onboarding.py +315 -0
  105. agentloop/operations.py +44 -0
  106. agentloop/optimizer.py +199 -0
  107. agentloop/otel.py +715 -0
  108. agentloop/otel_ids.py +81 -0
  109. agentloop/otel_semantics.py +134 -0
  110. agentloop/parallelism.py +141 -0
  111. agentloop/patches.py +603 -0
  112. agentloop/plan_export.py +126 -0
  113. agentloop/policy_promotion.py +325 -0
  114. agentloop/promotion_evidence.py +304 -0
  115. agentloop/promotion_types.py +209 -0
  116. agentloop/quality.py +446 -0
  117. agentloop/ranking.py +529 -0
  118. agentloop/recovery_evidence.py +96 -0
  119. agentloop/replay.py +753 -0
  120. agentloop/retention.py +375 -0
  121. agentloop/retention_types.py +113 -0
  122. agentloop/reuse_experiments.py +247 -0
  123. agentloop/routing_checks.py +116 -0
  124. agentloop/routing_types.py +241 -0
  125. agentloop/rules.py +396 -0
  126. agentloop/runtime.py +286 -0
  127. agentloop/savings.py +158 -0
  128. agentloop/scheduling_plan.py +99 -0
  129. agentloop/scheduling_types.py +169 -0
  130. agentloop/schema.py +257 -0
  131. agentloop/semantic_rules.py +200 -0
  132. agentloop/semantic_waste.py +345 -0
  133. agentloop/semantic_waste_types.py +88 -0
  134. agentloop/server.py +494 -0
  135. agentloop/store.py +1429 -0
  136. agentloop/structural_experiments.py +431 -0
  137. agentloop/structural_types.py +113 -0
  138. agentloop/structured_quality.py +1020 -0
  139. agentloop/studies.py +629 -0
  140. agentloop/study_statistics.py +58 -0
  141. agentloop/substitution_evidence.py +199 -0
  142. agentloop/substitution_export.py +190 -0
  143. agentloop/substitution_reports.py +407 -0
  144. agentloop/substitution_types.py +284 -0
  145. agentloop/substitutions.py +267 -0
  146. agentloop/timing.py +142 -0
  147. agentloop/tokens.py +188 -0
  148. agentloop/tool_scheduling.py +410 -0
  149. agentloop/trace_sources.py +31 -0
  150. agentloop/tracer.py +494 -0
  151. agentloop/value.py +306 -0
  152. agentloop/version.py +3 -0
  153. agentloop/workflow_types.py +264 -0
  154. agentloop/workflows.py +284 -0
  155. agentloop_profiler-0.1.0.dist-info/METADATA +513 -0
  156. agentloop_profiler-0.1.0.dist-info/RECORD +166 -0
  157. agentloop_profiler-0.1.0.dist-info/WHEEL +5 -0
  158. agentloop_profiler-0.1.0.dist-info/entry_points.txt +2 -0
  159. agentloop_profiler-0.1.0.dist-info/licenses/LICENSE +201 -0
  160. agentloop_profiler-0.1.0.dist-info/licenses/THIRD_PARTY_LICENSES.md +101 -0
  161. agentloop_profiler-0.1.0.dist-info/top_level.txt +2 -0
  162. dashboard/__init__.py +5 -0
  163. dashboard/app.py +922 -0
  164. dashboard/streamlit_compat.py +63 -0
  165. dashboard/trace_upload.py +57 -0
  166. dashboard/value_view.py +105 -0
@@ -0,0 +1,63 @@
1
+ from __future__ import annotations
2
+
3
+ from functools import wraps
4
+ from importlib.metadata import PackageNotFoundError, version
5
+ from typing import Any
6
+
7
+ _STRETCH_WIDTH_MIN_VERSION = (1, 49)
8
+
9
+
10
+ def _streamlit_version() -> tuple[int, int]:
11
+ try:
12
+ raw = version("streamlit")
13
+ except PackageNotFoundError:
14
+ return (0, 0)
15
+ parts = raw.split(".")
16
+ try:
17
+ return (int(parts[0]), int(parts[1]))
18
+ except (IndexError, ValueError):
19
+ return (0, 0)
20
+
21
+
22
+ def _normalize_dataframe_kwargs(kwargs: dict[str, Any]) -> dict[str, Any]:
23
+ """Translate the modern stretch width to the older supported API."""
24
+ normalized = dict(kwargs)
25
+ if normalized.get("width") == "stretch":
26
+ normalized["width"] = None
27
+ normalized["use_container_width"] = True
28
+ return normalized
29
+
30
+
31
+ def install_dataframe_width_compat() -> None:
32
+ """Keep the dashboard compatible with the declared Streamlit 1.34+ floor.
33
+
34
+ Streamlit 1.49 added string dataframe widths such as ``"stretch"``. Older
35
+ supported releases use ``use_container_width=True`` instead. The dashboard
36
+ package installs this adapter only for Streamlit versions below 1.49 so the
37
+ modern API remains untouched on current releases.
38
+ """
39
+ if _streamlit_version() >= _STRETCH_WIDTH_MIN_VERSION:
40
+ return
41
+
42
+ import streamlit as st
43
+ from streamlit.delta_generator import DeltaGenerator
44
+
45
+ original_method = DeltaGenerator.dataframe
46
+ if getattr(original_method, "__agentloop_width_compat__", False):
47
+ return
48
+
49
+ @wraps(original_method)
50
+ def dataframe_method(self, *args, **kwargs):
51
+ return original_method(self, *args, **_normalize_dataframe_kwargs(kwargs))
52
+
53
+ dataframe_method.__agentloop_width_compat__ = True
54
+ DeltaGenerator.dataframe = dataframe_method
55
+
56
+ original_module_function = st.dataframe
57
+
58
+ @wraps(original_module_function)
59
+ def dataframe(*args, **kwargs):
60
+ return original_module_function(*args, **_normalize_dataframe_kwargs(kwargs))
61
+
62
+ dataframe.__agentloop_width_compat__ = True
63
+ st.dataframe = dataframe
@@ -0,0 +1,57 @@
1
+ """Decode an uploaded trace file into an :class:`AgentTrace`, or say what to fix.
2
+
3
+ The Ingest page reads an uploaded file on every Streamlit rerun, so the three ways a
4
+ file can be unusable -- not UTF-8, not JSON, not a trace -- arrive as three exceptions
5
+ from three layers, each with its own idea of what a useful message looks like. Doing
6
+ that translation inline in ``app.py`` put the only copy of it inside a page that can
7
+ only be exercised through ``AppTest``. It lives here so it can be unit tested directly.
8
+ """
9
+
10
+ from __future__ import annotations
11
+
12
+ import json
13
+ from typing import Any
14
+
15
+ from agentloop.schema import TraceValidationError
16
+ from agentloop.tracer import AgentTrace
17
+
18
+
19
+ class TraceUploadError(ValueError):
20
+ """An uploaded trace the operator can fix, carrying the reason it was rejected.
21
+
22
+ ``str(exc)`` is the operator-facing clause and nothing else -- no prefix, no
23
+ trailing period -- so a caller can place it in whatever sentence its surface
24
+ needs. It subclasses :class:`ValueError`, matching
25
+ :class:`~agentloop.schema.TraceValidationError`.
26
+ """
27
+
28
+
29
+ def parse_uploaded_trace(payload: bytes) -> AgentTrace:
30
+ """Return the trace in ``payload``, or raise :class:`TraceUploadError` saying why not.
31
+
32
+ A parse position is reported only for malformed JSON, which is the only failure
33
+ that has one: a decode error is located by byte offset, and a schema violation is
34
+ located by field path.
35
+ """
36
+
37
+ try:
38
+ text = payload.decode("utf-8")
39
+ except UnicodeDecodeError as exc:
40
+ raise TraceUploadError(
41
+ f"the file is not UTF-8 text ({exc.reason} at byte {exc.start})"
42
+ ) from exc
43
+
44
+ try:
45
+ document: Any = json.loads(text)
46
+ except json.JSONDecodeError as exc:
47
+ raise TraceUploadError(f"{exc.msg} at line {exc.lineno}, column {exc.colno}") from exc
48
+
49
+ if not isinstance(document, dict):
50
+ # json.loads happily returns a list, string or number, and from_dict would
51
+ # then fail on subscripting rather than on the schema.
52
+ raise TraceUploadError(f"a trace must be a JSON object, not {type(document).__name__}")
53
+
54
+ try:
55
+ return AgentTrace.from_dict(document)
56
+ except TraceValidationError as exc:
57
+ raise TraceUploadError(str(exc)) from exc
@@ -0,0 +1,105 @@
1
+ from __future__ import annotations
2
+
3
+ import json
4
+ from typing import Any
5
+
6
+
7
+ def money(value: float | None, decimals: int = 2) -> str:
8
+ return "unavailable" if value is None else f"${float(value):,.{decimals}f}"
9
+
10
+
11
+ def seconds(ms: float) -> str:
12
+ return f"{float(ms) / 1000:,.2f}s"
13
+
14
+
15
+ def percentage(value: float | None, decimals: int = 1) -> str:
16
+ return "unavailable" if value is None else f"{float(value):.{decimals}f}%"
17
+
18
+
19
+ def _streamlit() -> Any:
20
+ try:
21
+ import streamlit as st
22
+ except ModuleNotFoundError as exc:
23
+ raise ModuleNotFoundError(
24
+ "Install dashboard support with: uv sync --locked --extra dashboard"
25
+ ) from exc
26
+ return st
27
+
28
+
29
+ def assumption_inputs(prefix: str = "") -> tuple[int, float, float]:
30
+ st = _streamlit()
31
+ c1, c2, c3 = st.columns(3)
32
+ runs_per_month = c1.number_input(
33
+ "Runs per month",
34
+ min_value=0,
35
+ value=5000,
36
+ step=500,
37
+ key=f"{prefix}runs_per_month",
38
+ )
39
+ engineer_hourly_rate_usd = c2.number_input(
40
+ "Engineer hourly rate",
41
+ min_value=0.0,
42
+ value=150.0,
43
+ step=25.0,
44
+ key=f"{prefix}engineer_hourly_rate_usd",
45
+ )
46
+ incident_cost_usd = c3.number_input(
47
+ "Reliability event cost",
48
+ min_value=0.0,
49
+ value=500.0,
50
+ step=100.0,
51
+ key=f"{prefix}incident_cost_usd",
52
+ )
53
+ return int(runs_per_month), float(engineer_hourly_rate_usd), float(incident_cost_usd)
54
+
55
+
56
+ def render_value_report(value: dict, *, show_download: bool = True) -> None:
57
+ st = _streamlit()
58
+ monthly = value["monthly_value"]
59
+ per_run = value["per_run"]
60
+ pricing = value.get("pricing", {})
61
+ reliability = value["reliability"]
62
+
63
+ st.subheader("Operational value")
64
+ c1, c2, c3, c4 = st.columns(4)
65
+ c1.metric("Monthly value", money(monthly["total_value_usd"]))
66
+ c2.metric("Cost savings / month", money(monthly["direct_model_cost_savings_usd"]))
67
+ c3.metric("Engineering hours saved", monthly["engineering_hours_saved"])
68
+ c4.metric("Latency hours saved", monthly["latency_hours_saved"])
69
+
70
+ c5, c6, c7, c8 = st.columns(4)
71
+ c5.metric("Latency saved / run", seconds(per_run["latency_savings_ms"]))
72
+ c6.metric("Cost saved / run", money(per_run["cost_savings_usd"], decimals=6))
73
+ c7.metric("Reliability risk", f"{reliability['risk_score']}/100")
74
+ c8.metric("High confidence fixes", reliability["high_confidence_fixes"])
75
+
76
+ st.subheader("Pricing scenario")
77
+ p1, p2, p3, p4 = st.columns(4)
78
+ p1.metric("Modeled tier", pricing.get("suggested_plan") or "unavailable")
79
+ p2.metric("Modeled price", money(pricing.get("suggested_monthly_price_usd", 0), decimals=0))
80
+ ratio = pricing.get("value_to_price_ratio")
81
+ p3.metric("Value / price", "n/a" if ratio is None else f"{ratio}x")
82
+ p4.metric("Monthly value", money(pricing.get("estimated_monthly_value_usd", 0)))
83
+
84
+ st.info(pricing.get("rationale", "No pricing rationale available."))
85
+
86
+ notes = pricing.get("scenario_notes", [])
87
+ if notes:
88
+ st.markdown("#### Scenario includes")
89
+ st.write("\n".join(f"- {note}" for note in notes))
90
+
91
+ st.subheader("Value summary")
92
+ st.write(value["value_summary"])
93
+
94
+ risks = reliability.get("top_risks", [])
95
+ if risks:
96
+ st.subheader("Top reliability risks")
97
+ st.write("\n".join(f"- {risk}" for risk in risks))
98
+
99
+ if show_download:
100
+ st.download_button(
101
+ "Download value report JSON",
102
+ data=json.dumps(value, indent=2),
103
+ file_name="agentloop_value_report.json",
104
+ mime="application/json",
105
+ )