slmeval-cli 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (94) hide show
  1. slmeval/__init__.py +71 -0
  2. slmeval/__main__.py +14 -0
  3. slmeval/adapters/__init__.py +24 -0
  4. slmeval/adapters/instrument.py +135 -0
  5. slmeval/adapters/langgraph.py +237 -0
  6. slmeval/cli/__init__.py +15 -0
  7. slmeval/cli/commands/__init__.py +35 -0
  8. slmeval/cli/commands/baseline_cmd.py +201 -0
  9. slmeval/cli/commands/cache_cmd.py +249 -0
  10. slmeval/cli/commands/dataset_cmd.py +52 -0
  11. slmeval/cli/commands/doctor_cmd.py +68 -0
  12. slmeval/cli/commands/gate_cmd.py +602 -0
  13. slmeval/cli/commands/init_cmd.py +144 -0
  14. slmeval/cli/commands/judge_cmd.py +284 -0
  15. slmeval/cli/commands/record_cmd.py +154 -0
  16. slmeval/cli/commands/report_cmd.py +98 -0
  17. slmeval/cli/commands/run_cmd.py +247 -0
  18. slmeval/cli/commands/show_cmd.py +482 -0
  19. slmeval/cli/commands/stubs.py +57 -0
  20. slmeval/cli/commands/validate_cmd.py +296 -0
  21. slmeval/cli/loading.py +149 -0
  22. slmeval/cli/main.py +91 -0
  23. slmeval/cli/render.py +251 -0
  24. slmeval/cli/wrap.py +31 -0
  25. slmeval/config/__init__.py +47 -0
  26. slmeval/config/loader.py +363 -0
  27. slmeval/config/locate.py +97 -0
  28. slmeval/config/models.py +430 -0
  29. slmeval/config/templates.py +246 -0
  30. slmeval/core/__init__.py +78 -0
  31. slmeval/core/adapter.py +253 -0
  32. slmeval/core/context.py +418 -0
  33. slmeval/core/outcome.py +260 -0
  34. slmeval/core/router.py +319 -0
  35. slmeval/core/runner.py +1209 -0
  36. slmeval/core/trace.py +188 -0
  37. slmeval/core/verdict.py +127 -0
  38. slmeval/dataset/__init__.py +8 -0
  39. slmeval/dataset/loader.py +130 -0
  40. slmeval/dataset/models.py +108 -0
  41. slmeval/doctor/__init__.py +7 -0
  42. slmeval/doctor/checks.py +380 -0
  43. slmeval/errors.py +76 -0
  44. slmeval/eval/__init__.py +64 -0
  45. slmeval/eval/assertions.py +356 -0
  46. slmeval/eval/calibration.py +364 -0
  47. slmeval/eval/cost.py +291 -0
  48. slmeval/eval/pipeline.py +273 -0
  49. slmeval/eval/stats.py +376 -0
  50. slmeval/eval/tier1.py +510 -0
  51. slmeval/eval/tier2.py +582 -0
  52. slmeval/eval/tier3.py +415 -0
  53. slmeval/exit_codes.py +28 -0
  54. slmeval/gate.py +722 -0
  55. slmeval/providers/__init__.py +69 -0
  56. slmeval/providers/base.py +426 -0
  57. slmeval/providers/fake.py +760 -0
  58. slmeval/providers/jsonx.py +239 -0
  59. slmeval/providers/ladder.py +339 -0
  60. slmeval/providers/openai_compat.py +392 -0
  61. slmeval/providers/registry.py +185 -0
  62. slmeval/providers/retry.py +81 -0
  63. slmeval/providers/schema.py +136 -0
  64. slmeval/providers/tokens.py +150 -0
  65. slmeval/py.typed +0 -0
  66. slmeval/report/__init__.py +115 -0
  67. slmeval/report/attribution.py +197 -0
  68. slmeval/report/baseline.py +353 -0
  69. slmeval/report/drilldown.py +1395 -0
  70. slmeval/report/export.py +450 -0
  71. slmeval/report/model.py +562 -0
  72. slmeval/report/progress.py +257 -0
  73. slmeval/report/scorecard.py +367 -0
  74. slmeval/sandbox/__init__.py +145 -0
  75. slmeval/sandbox/diagnosis.py +690 -0
  76. slmeval/sandbox/envelope.py +486 -0
  77. slmeval/sandbox/faults.py +794 -0
  78. slmeval/sandbox/keys.py +458 -0
  79. slmeval/sandbox/normalizers.py +333 -0
  80. slmeval/sandbox/store.py +525 -0
  81. slmeval/sandbox/tool.py +685 -0
  82. slmeval/storage/__init__.py +73 -0
  83. slmeval/storage/blobs.py +308 -0
  84. slmeval/storage/codec.py +93 -0
  85. slmeval/storage/db.py +1334 -0
  86. slmeval/storage/ingest.py +346 -0
  87. slmeval/storage/migrations.py +358 -0
  88. slmeval/storage/models.py +554 -0
  89. slmeval/storage/provenance.py +201 -0
  90. slmeval_cli-0.1.0.dist-info/METADATA +484 -0
  91. slmeval_cli-0.1.0.dist-info/RECORD +94 -0
  92. slmeval_cli-0.1.0.dist-info/WHEEL +4 -0
  93. slmeval_cli-0.1.0.dist-info/entry_points.txt +2 -0
  94. slmeval_cli-0.1.0.dist-info/licenses/LICENSE +201 -0
@@ -0,0 +1,145 @@
1
+ """The VCR tool sandbox.
2
+
3
+ Recording a tool call once and replaying it is what makes a benchmark possible
4
+ at all: you cannot run 500 trajectories against a customer's live Stripe, ERP or
5
+ production database, and you cannot compare two models against an API that
6
+ returns different data every call.
7
+
8
+ Three pieces, and they are separable on purpose:
9
+
10
+ :mod:`slmeval.sandbox.keys`
11
+ What identifies a recorded call, and the declared per-tool policy that
12
+ decides when two calls are the same call.
13
+ :mod:`slmeval.sandbox.normalizers`
14
+ The named normalizers that policy refers to.
15
+ :mod:`slmeval.sandbox.envelope`
16
+ How a return value is stored so the agent sees the same *types* on replay
17
+ that it saw on record - and how recording refuses, loudly, when it cannot
18
+ promise that.
19
+
20
+ :mod:`slmeval.sandbox.store`
21
+ Where recordings live: SQLite with an LRU in front, so the second replay of
22
+ a case never touches the disk.
23
+ :mod:`slmeval.sandbox.tool`
24
+ ``@sandbox_tool`` and the four modes - ``record``, ``replay``, ``auto``,
25
+ ``live``.
26
+ :mod:`slmeval.sandbox.diagnosis`
27
+ What a miss *means*: the nearest recorded call, a per-argument diff, and a
28
+ verdict that separates "the model got it wrong" from "we never recorded
29
+ this path".
30
+ :mod:`slmeval.sandbox.faults`
31
+ Deterministic fault injection, scheduled per tool per call index so one
32
+ extra call cannot shift every later fault onto a different step.
33
+
34
+ The one invariant across all of it: no path falls back to the real tool by
35
+ accident. A replay miss is a finding and surfaces as one, because silently
36
+ calling a customer's live API instead would spend their money and invalidate
37
+ the run without telling anyone.
38
+ """
39
+
40
+ from __future__ import annotations
41
+
42
+ from slmeval.sandbox.diagnosis import (
43
+ ArgDiff,
44
+ MissDiagnosis,
45
+ Verdict,
46
+ argument_distance,
47
+ diagnose,
48
+ diff_arguments,
49
+ )
50
+ from slmeval.sandbox.envelope import (
51
+ EnvelopeDecodeError,
52
+ UnserializableResponse,
53
+ decode,
54
+ decode_json,
55
+ encode,
56
+ encode_json,
57
+ )
58
+ from slmeval.sandbox.faults import (
59
+ Delivery,
60
+ Fault,
61
+ FaultError,
62
+ FaultKind,
63
+ FaultSchedule,
64
+ FaultVector,
65
+ InjectedFault,
66
+ ParityReport,
67
+ RateLimited,
68
+ ToolTimeout,
69
+ apply_fault,
70
+ compare_vectors,
71
+ fault_schedule,
72
+ )
73
+ from slmeval.sandbox.keys import (
74
+ CASCADE,
75
+ CassetteKeys,
76
+ KeyEntry,
77
+ MatchPolicy,
78
+ MatchTier,
79
+ PolicyError,
80
+ canonicalize,
81
+ declare,
82
+ )
83
+ from slmeval.sandbox.normalizers import (
84
+ Normalizer,
85
+ UnknownNormalizer,
86
+ resolve,
87
+ sql_backend,
88
+ )
89
+ from slmeval.sandbox.store import Cassette, CassetteStore, Hit, StoreStats
90
+ from slmeval.sandbox.tool import (
91
+ CassetteMiss,
92
+ SandboxedTool,
93
+ SandboxNotReady,
94
+ registered_tools,
95
+ sandbox_tool,
96
+ )
97
+
98
+ __all__ = [
99
+ "CASCADE",
100
+ "ArgDiff",
101
+ "Cassette",
102
+ "CassetteKeys",
103
+ "CassetteMiss",
104
+ "CassetteStore",
105
+ "Delivery",
106
+ "EnvelopeDecodeError",
107
+ "Fault",
108
+ "FaultError",
109
+ "FaultKind",
110
+ "FaultSchedule",
111
+ "FaultVector",
112
+ "Hit",
113
+ "InjectedFault",
114
+ "KeyEntry",
115
+ "MatchPolicy",
116
+ "MatchTier",
117
+ "MissDiagnosis",
118
+ "Normalizer",
119
+ "ParityReport",
120
+ "PolicyError",
121
+ "RateLimited",
122
+ "SandboxNotReady",
123
+ "SandboxedTool",
124
+ "StoreStats",
125
+ "ToolTimeout",
126
+ "UnknownNormalizer",
127
+ "UnserializableResponse",
128
+ "Verdict",
129
+ "apply_fault",
130
+ "argument_distance",
131
+ "canonicalize",
132
+ "compare_vectors",
133
+ "declare",
134
+ "decode",
135
+ "decode_json",
136
+ "diagnose",
137
+ "diff_arguments",
138
+ "encode",
139
+ "encode_json",
140
+ "fault_schedule",
141
+ "registered_tools",
142
+ "resolve",
143
+ "sandbox_tool",
144
+ "sql_backend",
145
+ ]